diff --git a/PoliNetwork.Graduatorie.Parser/Objects/Json/DateFound.cs b/PoliNetwork.Graduatorie.Parser/Objects/Json/DateFound.cs index faed551f..5801b95b 100644 --- a/PoliNetwork.Graduatorie.Parser/Objects/Json/DateFound.cs +++ b/PoliNetwork.Graduatorie.Parser/Objects/Json/DateFound.cs @@ -55,7 +55,7 @@ private void SetDate(string path, DateTime? minDateTime) private DateTime? GetMinTime(Ranking variable, string path) { FirstDate ??= new SortedDictionary(); - var dateTime = new DateTime(variable.Year ?? DateTime.UtcNow.Year, DateTime.UtcNow.Month, DateTime.UtcNow.Day); + var dateTime = new DateTime(variable.Year, DateTime.UtcNow.Month, DateTime.UtcNow.Day); var tryGetValue = TryGetValue(FirstDate, path); var minDateTime = tryGetValue.Item2 switch { diff --git a/PoliNetwork.Graduatorie.Parser/Objects/Json/Indexes/Specific/BySchoolYearCourseJson.cs b/PoliNetwork.Graduatorie.Parser/Objects/Json/Indexes/Specific/BySchoolYearCourseJson.cs index d41629b2..d705807a 100644 --- a/PoliNetwork.Graduatorie.Parser/Objects/Json/Indexes/Specific/BySchoolYearCourseJson.cs +++ b/PoliNetwork.Graduatorie.Parser/Objects/Json/Indexes/Specific/BySchoolYearCourseJson.cs @@ -40,13 +40,13 @@ public static BySchoolYearCourseJson From(RankingsSet set) mainJson.All = list; // group rankings by school - var bySchool = set.Rankings.Where(r => r.School != null).GroupBy(r => r.School!.Value); + var bySchool = set.Rankings.GroupBy(r => r.School); foreach (var schoolGroup in bySchool) { var school = schoolGroup.Key; - var byYears = schoolGroup.Where(r => r.Year != null).GroupBy(r => r.Year!.Value); + var byYears = schoolGroup.GroupBy(r => r.Year); var yearsDict = GetYearsDict(byYears); mainJson.Schools.Add(school, yearsDict); @@ -75,9 +75,6 @@ private static CoursesDict GetCoursesDict(IEnumerable yearGroup) foreach (var ranking in yearGroup) { - if (ranking.ByCourse == null) - continue; - var byTitle = ranking.ByCourse.Where(c => c.Title != null).GroupBy(c => c.Title!); // e.g. INGEGNERIA AEROSPAZIALE diff --git a/PoliNetwork.Graduatorie.Parser/Objects/Json/Indexes/Specific/BySchoolYearJson.cs b/PoliNetwork.Graduatorie.Parser/Objects/Json/Indexes/Specific/BySchoolYearJson.cs index fd5a1c5f..5c21d7ac 100644 --- a/PoliNetwork.Graduatorie.Parser/Objects/Json/Indexes/Specific/BySchoolYearJson.cs +++ b/PoliNetwork.Graduatorie.Parser/Objects/Json/Indexes/Specific/BySchoolYearJson.cs @@ -35,11 +35,11 @@ public static BySchoolYearJson From(RankingsSet set) mainJson.All = list; // group rankings by school - var bySchool = set.Rankings.Where(r => r.School != null).GroupBy(r => r.School!.Value); + var bySchool = set.Rankings.GroupBy(r => r.School); foreach (var schoolGroup in bySchool) { var school = schoolGroup.Key; - var byYears = schoolGroup.Where(r => r.Year != null).GroupBy(r => r.Year!.Value); + var byYears = schoolGroup.GroupBy(r => r.Year); var yearsDict = GetYearsDict(byYears); mainJson.Schools.Add(school, yearsDict); diff --git a/PoliNetwork.Graduatorie.Parser/Objects/Json/Indexes/Specific/ByYearSchoolJson.cs b/PoliNetwork.Graduatorie.Parser/Objects/Json/Indexes/Specific/ByYearSchoolJson.cs index d1e973b9..72edc981 100644 --- a/PoliNetwork.Graduatorie.Parser/Objects/Json/Indexes/Specific/ByYearSchoolJson.cs +++ b/PoliNetwork.Graduatorie.Parser/Objects/Json/Indexes/Specific/ByYearSchoolJson.cs @@ -34,11 +34,11 @@ public static ByYearSchoolJson From(RankingsSet set) mainJson.All = list; // group rankings by year - var byYear = set.Rankings.Where(r => r.Year != null).GroupBy(r => r.Year!.Value); + var byYear = set.Rankings.GroupBy(r => r.Year); foreach (var yearGroup in byYear) { var year = yearGroup.Key; - var bySchools = yearGroup.Where(r => r.School != null).GroupBy(r => r.School!.Value); + var bySchools = yearGroup.GroupBy(r => r.School); var schoolsDict = GetSchoolsDict(bySchools); mainJson.Years.Add(year, schoolsDict); diff --git a/PoliNetwork.Graduatorie.Parser/Objects/Json/Stats/StatsJson.cs b/PoliNetwork.Graduatorie.Parser/Objects/Json/Stats/StatsJson.cs index 1befb2b1..34012187 100644 --- a/PoliNetwork.Graduatorie.Parser/Objects/Json/Stats/StatsJson.cs +++ b/PoliNetwork.Graduatorie.Parser/Objects/Json/Stats/StatsJson.cs @@ -25,7 +25,7 @@ public static StatsJson From(RankingsSet rankingsSet) { var statsJson = new StatsJson(); - var byYears = rankingsSet.Rankings.Where(r => r.Year != null).GroupBy(r => r.Year!.Value); + var byYears = rankingsSet.Rankings.GroupBy(r => r.Year); foreach (var yearGroup in byYears) { var statsYear = StatsYear.From(yearGroup.ToList()); diff --git a/PoliNetwork.Graduatorie.Parser/Objects/Json/Stats/StatsSchool.cs b/PoliNetwork.Graduatorie.Parser/Objects/Json/Stats/StatsSchool.cs index 0a80f736..4df9e83f 100644 --- a/PoliNetwork.Graduatorie.Parser/Objects/Json/Stats/StatsSchool.cs +++ b/PoliNetwork.Graduatorie.Parser/Objects/Json/Stats/StatsSchool.cs @@ -17,15 +17,13 @@ public class StatsSchool public List List = new(); public int NumStudents; - public static StatsSchool From(IEnumerable pRankings) + public static StatsSchool From(IEnumerable rankings) { var statsSchool = new StatsSchool(); - var rankings = pRankings.Where(r => r is { Year: not null, School: not null }).ToList(); - statsSchool.NumStudents = - rankings.Select(x => (x.RankingSummary ?? x.CreateSummary()).HowManyStudents ?? 0).Sum(); - - statsSchool.List = rankings + var rankingsEnumerable = rankings.ToList(); + statsSchool.NumStudents = rankingsEnumerable.Select(x => x.RankingSummary.HowManyStudents ?? 0).Sum(); + statsSchool.List = rankingsEnumerable .SelectMany(r => r.ToStats()) .DistinctBy(x => new { x.SingleCourseJson.Id, x.SingleCourseJson.Location }) .OrderBy(x => x.SingleCourseJson.Id) diff --git a/PoliNetwork.Graduatorie.Parser/Objects/Json/Stats/StatsSingleJson.cs b/PoliNetwork.Graduatorie.Parser/Objects/Json/Stats/StatsSingleJson.cs index 07ca9f28..c3ac88e8 100644 --- a/PoliNetwork.Graduatorie.Parser/Objects/Json/Stats/StatsSingleJson.cs +++ b/PoliNetwork.Graduatorie.Parser/Objects/Json/Stats/StatsSingleJson.cs @@ -24,7 +24,6 @@ public StatsSingleCourseJson(SingleCourseJson singleCourseJson, RankingSummary s public static List From(Ranking ranking) { var singleCourseJsons = ranking.ToSingleCourseJson(); - ranking.RankingSummary ??= ranking.CreateSummary(); return singleCourseJsons.Select(scj => new StatsSingleCourseJson(scj, ranking.RankingSummary)).ToList(); } diff --git a/PoliNetwork.Graduatorie.Parser/Objects/Json/Stats/StatsYear.cs b/PoliNetwork.Graduatorie.Parser/Objects/Json/Stats/StatsYear.cs index 53d4805a..c4b2ceac 100644 --- a/PoliNetwork.Graduatorie.Parser/Objects/Json/Stats/StatsYear.cs +++ b/PoliNetwork.Graduatorie.Parser/Objects/Json/Stats/StatsYear.cs @@ -26,14 +26,14 @@ public class StatsYear public static StatsYear From(List rankings) { + if (rankings.Count == 0) return new StatsYear(); var statsYear = new StatsYear { - Year = rankings.First(r => r.Year != null).Year!.Value, // just hilarious - NumStudents = rankings.Select(r => (r.RankingSummary ?? r.CreateSummary()).HowManyStudents ?? 0) - .Sum() // this ?? is crazy + Year = rankings.First().Year, // just hilarious + NumStudents = rankings.Select(r => r.RankingSummary.HowManyStudents ?? 0).Sum() // this ?? is crazy }; - var bySchool = rankings.Where(r => r.School != null).GroupBy(r => r.School!.Value); + var bySchool = rankings.GroupBy(r => r.School); foreach (var schoolGroup in bySchool) { var statsSchool = StatsSchool.From(schoolGroup.ToList()); diff --git a/PoliNetwork.Graduatorie.Parser/Objects/RankingNS/Ranking.cs b/PoliNetwork.Graduatorie.Parser/Objects/RankingNS/Ranking.cs index 2eac3d2f..a4812f52 100644 --- a/PoliNetwork.Graduatorie.Parser/Objects/RankingNS/Ranking.cs +++ b/PoliNetwork.Graduatorie.Parser/Objects/RankingNS/Ranking.cs @@ -1,6 +1,5 @@ #region -using System.Globalization; using Newtonsoft.Json; using Newtonsoft.Json.Serialization; using PoliNetwork.Graduatorie.Common.Data; @@ -20,15 +19,15 @@ namespace PoliNetwork.Graduatorie.Parser.Objects.RankingNS; [JsonObject(MemberSerialization.Fields, NamingStrategyType = typeof(CamelCaseNamingStrategy))] public class Ranking : IComparable, IEquatable { - public List? ByCourse; - public MeritTable? ByMerit; - public string? Extra; - public DateTime LastUpdate; - public RankingOrder? RankingOrder; - public RankingSummary? RankingSummary; - public SchoolEnum? School; - public RankingUrl? Url; - public int? Year; + public required DateTime LastUpdate; + public required SchoolEnum School; + public required int Year; + public required RankingUrl Url; + public required string Extra; + public required RankingOrder RankingOrder; + public required RankingSummary RankingSummary; + public required MeritTable ByMerit; + public required List ByCourse; public int CompareTo(Ranking? other) { @@ -47,7 +46,7 @@ public bool Equals(Ranking? other) public RankingSummaryStudent GetRankingSummaryStudent() { - return new RankingSummaryStudent(RankingOrder?.Phase, School, Year, Url); + return new RankingSummaryStudent(RankingOrder.Phase, School, Year, Url); } public static Ranking? FromJson(string fullPath) @@ -67,9 +66,9 @@ public bool IsSimilarTo(Ranking ranking) { return Year == ranking.Year && School == ranking.School && - RankingOrder?.Phase == ranking.RankingOrder?.Phase && + RankingOrder.Phase == ranking.RankingOrder.Phase && Extra == ranking.Extra && - Url?.Url == ranking.Url?.Url; + Url.Url == ranking.Url.Url; } @@ -83,17 +82,14 @@ public string GetId() { var idList = new List(); - var schoolShort = School?.ToShortName(); - if (schoolShort != null) idList.Add(schoolShort); + var schoolShort = School.ToShortName(); + idList.Add(schoolShort); var yearStr = Year.ToString(); - if (yearStr != null) idList.Add(yearStr); + idList.Add(yearStr); - var orderId = RankingOrder?.GetId(); - if (orderId != null) idList.Add(orderId); - - var fallback = DateTime.UtcNow.ToString("yyyyMMddTHHmmss", CultureInfo.InvariantCulture) + "Z"; - if (idList.Count == 0) idList.Add(fallback); + var orderId = RankingOrder.GetId(); + idList.Add(orderId); return string.Join("_", idList); } @@ -101,9 +97,8 @@ public string GetId() public List ToSingleCourseJson() { var result = new List(); - var schoolString = School == null ? null : Enum.GetName(typeof(SchoolEnum), School); + var schoolString = Enum.GetName(typeof(SchoolEnum), School); var courseTables = ByCourse; - if (courseTables == null) return result; result.AddRange(courseTables.Select(variable => new SingleCourseJson { Link = GetFilename(), @@ -160,24 +155,18 @@ public void WriteAsJson(string outFolder, bool forceReparse = false) public int GetHashWithoutLastUpdate() { var i = "Ranking".GetHashCode(); - i ^= Extra?.GetHashCode() ?? "Extra".GetHashCode(); - i ^= RankingOrder?.GetHashWithoutLastUpdate() ?? "RankingOrder".GetHashCode(); - i ^= RankingSummary?.GetHashWithoutLastUpdate() ?? "RankingSummary".GetHashCode(); - i ^= School?.GetHashCode() ?? "School".GetHashCode(); - i ^= Url?.GetHashWithoutLastUpdate() ?? "Url".GetHashCode(); - i ^= Year?.GetHashCode() ?? "Year".GetHashCode(); - var iMerit = ByMerit?.GetHashWithoutLastUpdate(); - i ^= iMerit ?? "ByMerit".GetHashCode(); - - - if (ByCourse == null) - i ^= "ByCourse".GetHashCode(); - else - i = ByCourse.Aggregate(i, (current, variable) => - { - var hashWithoutLastUpdate = variable.GetHashWithoutLastUpdate(); - return current ^ hashWithoutLastUpdate; - }); + i ^= Extra.GetHashCode(); + i ^= RankingOrder.GetHashWithoutLastUpdate(); + i ^= RankingSummary.GetHashWithoutLastUpdate(); + i ^= School.GetHashCode(); + i ^= Url.GetHashWithoutLastUpdate(); + i ^= Year.GetHashCode(); + i ^= ByMerit.GetHashWithoutLastUpdate(); + i = ByCourse.Aggregate(i, (current, variable) => + { + var hashWithoutLastUpdate = variable.GetHashWithoutLastUpdate(); + return current ^ hashWithoutLastUpdate; + }); return i; } diff --git a/PoliNetwork.Graduatorie.Parser/Objects/RankingNS/RankingSummary.cs b/PoliNetwork.Graduatorie.Parser/Objects/RankingNS/RankingSummary.cs index fb4cc3cc..01289048 100644 --- a/PoliNetwork.Graduatorie.Parser/Objects/RankingNS/RankingSummary.cs +++ b/PoliNetwork.Graduatorie.Parser/Objects/RankingNS/RankingSummary.cs @@ -30,19 +30,19 @@ public int GetHashWithoutLastUpdate() public static RankingSummary From(Ranking ranking) { - var byMeritRows = ranking.ByMerit?.Rows; + var byMeritRows = ranking.ByMerit.Rows; var results = CalculateResultsScores(byMeritRows); var keyValuePairs = results?.OrderBy(x => x.Key) .ToDictionary(obj => obj.Key, obj => obj.Value); - var courseTableStatsList = ranking.ByCourse?.Select(x => x.GetStats()) + var courseTableStatsList = ranking.ByCourse.Select(x => x.GetStats()) .OrderBy(x => x.Title).ThenBy(x => x.Location).ToList(); var howManyCanEnroll = byMeritRows?.Count(x => x.EnrollType?.CanEnroll ?? false); - var groupBy = courseTableStatsList?.GroupBy(x => + var groupBy = courseTableStatsList.GroupBy(x => { TitleLocation titleLocation; titleLocation.Title = x.Title; @@ -50,14 +50,14 @@ public static RankingSummary From(Ranking ranking) return titleLocation; }); var distinctBy = groupBy - ?.DistinctBy(x => + .DistinctBy(x => { TitleLocation titleLocation; titleLocation.Title = x.Key.Title; titleLocation.Location = x.Key.Location; return titleLocation; }); - var tableStatsList = distinctBy?.ToList(); + var tableStatsList = distinctBy.ToList(); var tableStatsList2 = Get(tableStatsList); var resultsSummarized = new SortedDictionary(keyValuePairs ?? new Dictionary()); return new RankingSummary diff --git a/PoliNetwork.Graduatorie.Parser/Objects/Tables/Course/CourseTable.cs b/PoliNetwork.Graduatorie.Parser/Objects/Tables/Course/CourseTable.cs index 4e6360fa..5030036b 100644 --- a/PoliNetwork.Graduatorie.Parser/Objects/Tables/Course/CourseTable.cs +++ b/PoliNetwork.Graduatorie.Parser/Objects/Tables/Course/CourseTable.cs @@ -26,7 +26,7 @@ public CourseTableStats GetStats() public RankingSummaryStudent GetRankingSummaryStudent(Ranking ranking) { - return new RankingSummaryStudent(Title, ranking.RankingOrder?.Phase, ranking.School, + return new RankingSummaryStudent(Title, ranking.RankingOrder.Phase, ranking.School, ranking.Url, ranking.Year); } diff --git a/PoliNetwork.Graduatorie.Parser/Utils/Output/HashMatricoleWrite.cs b/PoliNetwork.Graduatorie.Parser/Utils/Output/HashMatricoleWrite.cs index 80396565..2831a60a 100644 --- a/PoliNetwork.Graduatorie.Parser/Utils/Output/HashMatricoleWrite.cs +++ b/PoliNetwork.Graduatorie.Parser/Utils/Output/HashMatricoleWrite.cs @@ -46,7 +46,7 @@ private static IdsDict GetIdsDict(RankingsSet rankingsSet) var dictionary = new IdsDict(); foreach (var ranking in rankingsSet.Rankings) { - var byMeritRows = ranking.ByMerit?.Rows; + var byMeritRows = ranking.ByMerit.Rows; if (byMeritRows != null) foreach (var student in byMeritRows.Where(student => !string.IsNullOrEmpty(student.Id))) { @@ -56,7 +56,6 @@ private static IdsDict GetIdsDict(RankingsSet rankingsSet) } var rankingByCourse = ranking.ByCourse; - if (rankingByCourse == null) continue; foreach (var courseTable in rankingByCourse.Where(c => c.Rows != null)) { var row = courseTable.Rows!; diff --git a/PoliNetwork.Graduatorie.Parser/Utils/Transformer/ParserNS/Parser.cs b/PoliNetwork.Graduatorie.Parser/Utils/Transformer/ParserNS/Parser.cs index b986614d..b3e8430d 100644 --- a/PoliNetwork.Graduatorie.Parser/Utils/Transformer/ParserNS/Parser.cs +++ b/PoliNetwork.Graduatorie.Parser/Utils/Transformer/ParserNS/Parser.cs @@ -78,9 +78,6 @@ private RankingsSet ParseSavedRankings(ICollection htmls) // removing urls of rankings found foreach (var ranking in savedSet.Rankings) { - if (ranking.Url == null) - continue; - ranking.Url.FixSlashes(); var relatedHtmls = htmls.Where(h => h.Url.IsSameRanking(ranking.Url)).ToList(); foreach (var related in relatedHtmls) @@ -183,7 +180,6 @@ private RankingsSet ParseNewRankings(IReadOnlyCollection htmls) private static Ranking? InitRanking(RankingUrl indexUrl, HtmlNode doc) { - var ranking = new Ranking(); // get ranking info var intestazioni = doc.GetElementsByClassName("intestazione") .Select(i => i.Descendants("#text").ToList()[0].InnerText) @@ -200,27 +196,27 @@ private RankingsSet ParseNewRankings(IReadOnlyCollection htmls) return null; } - ranking.Url = indexUrl; - ranking.School = school; - ranking.Year = Convert.ToInt16(intestazioni[1].Split("Year ")[1].Split("/")[0]); + var year = Convert.ToInt16(intestazioni[1].Split("Year ")[1].Split("/")[0]); - var extraEuStr = intestazioni[4].Split("\n")[0].ToLower(); + var extra = intestazioni[4]; + var extraEuStr = extra.Split("\n")[0].ToLower(); var isExtraEu = extraEuStr.Contains("extra-ue"); - if (ranking.Year < 2024) + RankingOrder rankingOrder; + if (year < 2024) { // layout valid until 2023 var phase = string.Join(" ", intestazioni[3].Split(" - ")[1..]); - ranking.RankingOrder = new RankingOrder(phase, isExtraEu); - if (ranking.School == SchoolEnum.Architettura && ranking.RankingOrder.Primary == null && - ranking.RankingOrder.Secondary == null && ranking.RankingOrder.IsExtraEu) + rankingOrder = new RankingOrder(phase, isExtraEu); + if (school == SchoolEnum.Architettura && rankingOrder.Primary == null && + rankingOrder.Secondary == null && rankingOrder.IsExtraEu) // this is a fallback for 2020-2023: // POLIMI was used to add the ranking number (Secondary, e.g. "Prima Graduatoria") for ExtraEU starting // from the second ranking. // e.g. Extra-EU first ranking => phase = "Extra-ue", // Extra-EU second ranking => phase = "Extra-ue - Seconda Graduatoria" // so this is a fallback to add the equivalent of "Prima Graduatoria" to the first ExtraEU ranking. - ranking.RankingOrder.Secondary = 1; + rankingOrder.Secondary = 1; } else { @@ -228,14 +224,22 @@ private RankingsSet ParseNewRankings(IReadOnlyCollection htmls) var phase = intestazioni[3]; var isEnglish = intestazioni[2].Contains("taught in english") || intestazioni[2].Contains("erogati in inglese"); - ranking.RankingOrder = new RankingOrder(phase, isExtraEu, isEnglish); + rankingOrder = new RankingOrder(phase, isExtraEu, isEnglish); } - ranking.Extra = intestazioni[4]; - ranking.LastUpdate = DateTime.UtcNow; - ranking.ByCourse = new List(); - return ranking; + return new Ranking + { + LastUpdate = DateTime.UtcNow, + School = school, + Year = year, + Url = indexUrl, + Extra = extra, + RankingOrder = rankingOrder, + RankingSummary = new RankingSummary(), + ByMerit = new MeritTable(), + ByCourse = new List(), + }; } private (IEnumerable, IEnumerable) ParseIndexBy( @@ -757,9 +761,6 @@ private IEnumerable ParseLocalHtmlFiles() if (obj1 == null) return null; - if (obj1.RankingOrder != null) - return obj1; - var jValue = (JValue)jToken; var phase = jValue.Value?.ToString(); if (!string.IsNullOrEmpty(phase))