From f557adadf899bf2166fc9c02eb9f675221d5f1fb Mon Sep 17 00:00:00 2001 From: Guillaume PIERRE Date: Tue, 11 Jun 2024 11:51:30 +0200 Subject: [PATCH 01/28] Remove TS links from solver : extract link names from study. --- src/libs/antares/study/area/links.cpp | 10 +-- src/tools/ts-generator/main.cpp | 90 ++++++++++++++++++++++++++- 2 files changed, 94 insertions(+), 6 deletions(-) diff --git a/src/libs/antares/study/area/links.cpp b/src/libs/antares/study/area/links.cpp index 40da9df149..3464017299 100644 --- a/src/libs/antares/study/area/links.cpp +++ b/src/libs/antares/study/area/links.cpp @@ -572,7 +572,7 @@ bool AreaLinksInternalLoadFromProperty(AreaLink& link, const String& key, const } } // anonymous namespace -bool AreaLinksLoadFromFolder(Study& study, AreaList* l, Area* area, const fs::path& folder, bool loadTSGen) +bool AreaLinksLoadFromFolder(Study& study, AreaList* areaList, Area* area, const fs::path& folder, bool loadTSGen) { // Assert assert(area); @@ -594,16 +594,16 @@ bool AreaLinksLoadFromFolder(Study& study, AreaList* l, Area* area, const fs::pa for (auto* s = ini.firstSection; s; s = s->next) { // Getting the name of the area - std::string buffer = transformNameIntoID(s->name); + std::string targetAreaName = transformNameIntoID(s->name); // Trying to find it - Area* linkedWith = AreaListLFind(l, buffer.c_str()); - if (!linkedWith) + Area* targetArea = AreaListLFind(areaList, targetAreaName.c_str()); + if (!targetArea) { logs.error() << '`' << s->name << "`: Impossible to find the area"; continue; } - AreaLink* lnk = AreaAddLinkBetweenAreas(area, linkedWith); + AreaLink* lnk = AreaAddLinkBetweenAreas(area, targetArea); if (!lnk) { logs.error() << "Impossible to create a link between two areas"; diff --git a/src/tools/ts-generator/main.cpp b/src/tools/ts-generator/main.cpp index 8f23f60619..ea796564c5 100644 --- a/src/tools/ts-generator/main.cpp +++ b/src/tools/ts-generator/main.cpp @@ -39,6 +39,9 @@ using namespace Antares; namespace fs = std::filesystem; +using LinkPair = std::pair; +using LinkPairs = std::vector; + struct Settings { std::string studyFolder; @@ -133,6 +136,84 @@ TSGenerator::listOfLinks getLinksToGen(Data::AreaList& areas, const std::string& return links; } +std::vector extractTargetAreas(fs::path sourceLinkDir) +{ + std::vector to_return; + fs::path pathToIni = sourceLinkDir / "properties.ini"; + IniFile ini; + ini.open(pathToIni); + for (auto* s = ini.firstSection; s; s = s->next) + { + std::string targetAreaName = transformNameIntoID(s->name); + to_return.push_back(targetAreaName); + } + return to_return; +} + +LinkPairs extractLinksFromStudy(fs::path studyDir) +{ + LinkPairs to_return; + fs::path linksDir = studyDir / "input" / "links"; + for (auto const& item : fs::directory_iterator{linksDir}) + { + if (item.is_directory()) + { + std::string sourceAreaName = item.path().filename().generic_string(); + auto targetAreas = extractTargetAreas(item); + for (auto& targetAreaName : targetAreas) + { + auto linkPair = std::make_pair(sourceAreaName, targetAreaName); + to_return.push_back(linkPair); + } + } + } + return to_return; +} + +bool pairs_match(const LinkPair& p1, const LinkPair& p2) +{ + return (p1.first == p2.first && p1.second == p2.second) + || (p1.first == p2.second && p1.second == p2.first); +} + +const LinkPair* getMatchingPairInCollection(const LinkPair& pair, const LinkPairs& collection) +{ + for(const auto& p : collection) + { + if (pairs_match(pair, p)) + return &p; + } + return nullptr; +} + +LinkPairs extractLinksFromCmdLine(const LinkPairs& allLinks, + const std::string linksFromCmdLine) +{ + LinkPairs to_return; + LinkPairs pairsFromCmdLine = splitStringIntoPairs(linksFromCmdLine, ';', '.'); + for (auto& p : pairsFromCmdLine) + { + if (const auto* found_pair = getMatchingPairInCollection(p, allLinks); found_pair) + { + to_return.push_back(*found_pair); + } + else + { + logs.error() << "Link '" << p.first << "." << p.second << "' not found"; + } + } + return to_return; +} + +void logLinks(std::string title, LinkPairs& links) +{ + std::cout << title << " : " << std::endl; + for (auto& link : links) + { + std::cout << "+ " << link.first << "." << link.second << std::endl; + } +} + int main(int argc, char* argv[]) { logs.applicationName("ts-generator"); @@ -216,6 +297,13 @@ int main(int argc, char* argv[]) } // LINKS + + auto allLinksPairs = extractLinksFromStudy(settings.studyFolder); + logLinks("All links", allLinksPairs); + + auto linksFromCmdLine = extractLinksFromCmdLine(allLinksPairs, settings.linksListToGen); + logLinks("Links from cmd line", linksFromCmdLine); + TSGenerator::listOfLinks links; if (settings.allLinks) { @@ -239,4 +327,4 @@ int main(int argc, char* argv[]) && ret; return !ret; // return 0 for success -} +} \ No newline at end of file From 0c9ff69267c8876772cd7e56eff673936ddcc672 Mon Sep 17 00:00:00 2001 From: Guillaume PIERRE Date: Tue, 11 Jun 2024 15:45:07 +0200 Subject: [PATCH 02/28] Remove TS links from solver : extract study general parameters related to link TS generation --- src/tools/ts-generator/main.cpp | 58 ++++++++++++++++++++++++++++++++- 1 file changed, 57 insertions(+), 1 deletion(-) diff --git a/src/tools/ts-generator/main.cpp b/src/tools/ts-generator/main.cpp index ea796564c5..34adf615e8 100644 --- a/src/tools/ts-generator/main.cpp +++ b/src/tools/ts-generator/main.cpp @@ -136,12 +136,14 @@ TSGenerator::listOfLinks getLinksToGen(Data::AreaList& areas, const std::string& return links; } +// ===== New code for TS generation links ==================================== + std::vector extractTargetAreas(fs::path sourceLinkDir) { std::vector to_return; fs::path pathToIni = sourceLinkDir / "properties.ini"; IniFile ini; - ini.open(pathToIni); + ini.open(pathToIni); // gp : we should handle reading issues for (auto* s = ini.firstSection; s; s = s->next) { std::string targetAreaName = transformNameIntoID(s->name); @@ -214,6 +216,58 @@ void logLinks(std::string title, LinkPairs& links) } } +struct StudyParamsForLinkTS +{ + bool derated = false; + unsigned int nbLinkTStoGenerate = 1; + unsigned int seed = Data::antaresSeedDefaultValue; +}; + +bool readProperty(StudyParamsForLinkTS& params, + const Yuni::String& key, + const Yuni::String& value) +{ + if (key == "derated") + { + return value.to(params.derated); + } + if (key == "nbtimeserieslinks") + { + return value.to(params.nbLinkTStoGenerate); + } + if (key == "seed-tsgen-links") + { + return value.to(params.seed); + } + return true; // gp : should we return true here ? +} + +StudyParamsForLinkTS readParamsForLinksTS(fs::path studyDir) +{ + StudyParamsForLinkTS to_return; + fs::path pathToGeneraldata = studyDir / "settings" / "generaldata.ini"; + IniFile ini; + ini.open(pathToGeneraldata); // gp : we should handle reading issues + for (auto* section = ini.firstSection; section; section = section->next) + { + // Skipping sections useless in the current context + Yuni::String sectionName = section->name; + if (sectionName != "general" && sectionName != "seeds - Mersenne Twister") + continue; + + for (const IniFile::Property* p = section->firstProperty; p; p = p->next) + { + if (! readProperty(to_return, p->key, p->value)) + { + logs.warning() << ini.filename() << ": '" << p->key << "': Unknown property"; + } + } + } + return to_return; +} + +// ============================================================================ + int main(int argc, char* argv[]) { logs.applicationName("ts-generator"); @@ -304,6 +358,8 @@ int main(int argc, char* argv[]) auto linksFromCmdLine = extractLinksFromCmdLine(allLinksPairs, settings.linksListToGen); logLinks("Links from cmd line", linksFromCmdLine); + StudyParamsForLinkTS params = readParamsForLinksTS(settings.studyFolder); + TSGenerator::listOfLinks links; if (settings.allLinks) { From 75476f2dea7a1061e19564024cb27351f3ee39a0 Mon Sep 17 00:00:00 2001 From: Guillaume PIERRE Date: Wed, 12 Jun 2024 09:28:02 +0200 Subject: [PATCH 03/28] Remove TS links from solver : extract properties from files "properties.ini" for links chosen by user --- src/libs/antares/study/area/links.cpp | 1 - src/tools/ts-generator/main.cpp | 191 +++++++++++++++++++++++--- 2 files changed, 175 insertions(+), 17 deletions(-) diff --git a/src/libs/antares/study/area/links.cpp b/src/libs/antares/study/area/links.cpp index 3464017299..64f1657034 100644 --- a/src/libs/antares/study/area/links.cpp +++ b/src/libs/antares/study/area/links.cpp @@ -510,7 +510,6 @@ bool handleTSGenKey(Data::LinkTsGeneration& out, const std::string& key, const String& value) { - if (key == "unitcount") { return value.to(out.unitCount); diff --git a/src/tools/ts-generator/main.cpp b/src/tools/ts-generator/main.cpp index 34adf615e8..da450d4142 100644 --- a/src/tools/ts-generator/main.cpp +++ b/src/tools/ts-generator/main.cpp @@ -34,6 +34,7 @@ #include #include #include +#include using namespace Antares; @@ -189,7 +190,7 @@ const LinkPair* getMatchingPairInCollection(const LinkPair& pair, const LinkPair } LinkPairs extractLinksFromCmdLine(const LinkPairs& allLinks, - const std::string linksFromCmdLine) + const std::string linksFromCmdLine) { LinkPairs to_return; LinkPairs pairsFromCmdLine = splitStringIntoPairs(linksFromCmdLine, ';', '.'); @@ -218,19 +219,16 @@ void logLinks(std::string title, LinkPairs& links) struct StudyParamsForLinkTS { - bool derated = false; unsigned int nbLinkTStoGenerate = 1; + // gp : we will have a problem with that if seed-tsgen-links not set in + // gp : generaldata.ini. In that case, our default value is wrong. unsigned int seed = Data::antaresSeedDefaultValue; }; -bool readProperty(StudyParamsForLinkTS& params, - const Yuni::String& key, - const Yuni::String& value) +bool readLinkGeneralProperty(StudyParamsForLinkTS& params, + const Yuni::String& key, + const Yuni::String& value) { - if (key == "derated") - { - return value.to(params.derated); - } if (key == "nbtimeserieslinks") { return value.to(params.nbLinkTStoGenerate); @@ -242,7 +240,7 @@ bool readProperty(StudyParamsForLinkTS& params, return true; // gp : should we return true here ? } -StudyParamsForLinkTS readParamsForLinksTS(fs::path studyDir) +StudyParamsForLinkTS readGeneralParamsForLinksTS(fs::path studyDir) { StudyParamsForLinkTS to_return; fs::path pathToGeneraldata = studyDir / "settings" / "generaldata.ini"; @@ -257,15 +255,164 @@ StudyParamsForLinkTS readParamsForLinksTS(fs::path studyDir) for (const IniFile::Property* p = section->firstProperty; p; p = p->next) { - if (! readProperty(to_return, p->key, p->value)) + if (! readLinkGeneralProperty(to_return, p->key, p->value)) { - logs.warning() << ini.filename() << ": '" << p->key << "': Unknown property"; + logs.warning() << ini.filename() << ": reading value of '" << p->key << "' went wrong"; } } } return to_return; } +struct LinkTSgenerationParams +{ + LinkPair namesPair; + StudyParamsForLinkTS generalParams; + + unsigned unitCount = 0; + double nominalCapacity = 0; + + double forcedVolatility = 0.; + double plannedVolatility = 0.; + + Data::StatisticalLaw forcedLaw = Data::LawUniform; + Data::StatisticalLaw plannedLaw = Data::LawUniform; + + std::unique_ptr prepro; + + Matrix<> modulationCapacityDirect; + Matrix<> modulationCapacityIndirect; + + bool forceNoGeneration = false; +}; + +std::vector CreateLinkList(const LinkPairs& linksFromCmdLine, + const StudyParamsForLinkTS& params) +{ + std::vector to_return(linksFromCmdLine.size()); + unsigned int index = 0; + for (const auto& link_pair : linksFromCmdLine) + { + to_return[index].namesPair = link_pair; + to_return[index].generalParams = params; + index++; + } + return to_return; +} + +LinkTSgenerationParams* findLinkInList(const LinkPair& link_to_find, + std::vector& linkList) +{ + for(auto& link : linkList) + { + if (link.namesPair == link_to_find) + return &link; + } + return nullptr; +} + +bool readLinkIniProperty(LinkTSgenerationParams* link, + const Yuni::String& key, + const Yuni::String& value) +{ + if (key == "unitcount") + { + return value.to(link->unitCount); + } + + if (key == "nominalcapacity") + { + return value.to(link->nominalCapacity); + } + + if (key == "law.planned") + { + return value.to(link->plannedLaw); + } + + if (key == "law.forced") + { + return value.to(link->forcedLaw); + } + + if (key == "volatility.planned") + { + return value.to(link->plannedVolatility); + } + + if (key == "volatility.forced") + { + return value.to(link->forcedVolatility); + } + + if (key == "force-no-generation") + { + return value.to(link->forceNoGeneration); + } + + return true; +} + +void readLinkIniProperties(LinkTSgenerationParams* link, + IniFile::Section* section) +{ + for (const IniFile::Property* p = section->firstProperty; p; p = p->next) + { + if (! readLinkIniProperty(link, p->key, p->value)) + { + std::string linkName = link->namesPair.first + "." + link->namesPair.second; + logs.warning() << "Link '" << linkName << "' : reading value of '" << p->key << "' went wrong"; + } + } +} + +void readSourceAreaIniFile(fs::path sourceLinkDir, + std::string sourceAreaName, + std::vector& linkList) +{ + fs::path pathToIni = sourceLinkDir / "properties.ini"; + IniFile ini; + ini.open(pathToIni); // gp : we should handle reading issues + for (auto* section = ini.firstSection; section; section = section->next) + { + std::string targetAreaName = transformNameIntoID(section->name); + const LinkPair processedLink = std::make_pair(sourceAreaName, targetAreaName); + if (auto* foundLink = findLinkInList(processedLink, linkList); foundLink) + { + readLinkIniProperties(foundLink, section); + } + } +} + +void readIniProperties(std::vector& linkList, fs::path toLinksDir) +{ + for (auto const& item : fs::directory_iterator{toLinksDir}) + { + if (item.is_directory()) + { + std::string sourceAreaName = item.path().filename().generic_string(); + readSourceAreaIniFile(item, sourceAreaName, linkList); + } + } +} + +bool readPreproTimeSeries(std::vector& linkList, fs::path toLinksDir) +{ + return true; +} + +bool readLinksSpecificTSparameters(std::vector& linkList, + fs::path studyFolder) +{ + fs::path toLinksDir = studyFolder / "input" / "links"; + readIniProperties(linkList, toLinksDir); + if (! readPreproTimeSeries(linkList, toLinksDir)) + { + // gp : Log a problem here + return false; + } + return true; +} // ============================================================================ int main(int argc, char* argv[]) @@ -351,14 +498,26 @@ int main(int argc, char* argv[]) } // LINKS - + // ===== New code for TS generation links ==================================== auto allLinksPairs = extractLinksFromStudy(settings.studyFolder); - logLinks("All links", allLinksPairs); + // logLinks("All links", allLinksPairs); auto linksFromCmdLine = extractLinksFromCmdLine(allLinksPairs, settings.linksListToGen); - logLinks("Links from cmd line", linksFromCmdLine); + // logLinks("Links from cmd line", linksFromCmdLine); + if (settings.allLinks) + linksFromCmdLine = allLinksPairs; + + StudyParamsForLinkTS params = readGeneralParamsForLinksTS(settings.studyFolder); + + std::vector linkList = CreateLinkList(linksFromCmdLine, params); + if (! readLinksSpecificTSparameters(linkList, settings.studyFolder)) + { + // Something wrong while reading particular data associated to + // link TS generation + } + - StudyParamsForLinkTS params = readParamsForLinksTS(settings.studyFolder); + // ============================================================================ TSGenerator::listOfLinks links; if (settings.allLinks) From efa2b35130bb99a18ce44fb153618a40db82ff99 Mon Sep 17 00:00:00 2001 From: Guillaume PIERRE Date: Wed, 12 Jun 2024 15:21:50 +0200 Subject: [PATCH 04/28] Remove TS links from solver : adding the reading of prepro TS --- src/tools/ts-generator/main.cpp | 100 ++++++++++++++++++++++++++------ 1 file changed, 82 insertions(+), 18 deletions(-) diff --git a/src/tools/ts-generator/main.cpp b/src/tools/ts-generator/main.cpp index da450d4142..8f04b50290 100644 --- a/src/tools/ts-generator/main.cpp +++ b/src/tools/ts-generator/main.cpp @@ -153,7 +153,7 @@ std::vector extractTargetAreas(fs::path sourceLinkDir) return to_return; } -LinkPairs extractLinksFromStudy(fs::path studyDir) +LinkPairs extractLinkNamesFromStudy(fs::path studyDir) { LinkPairs to_return; fs::path linksDir = studyDir / "input" / "links"; @@ -189,7 +189,7 @@ const LinkPair* getMatchingPairInCollection(const LinkPair& pair, const LinkPair return nullptr; } -LinkPairs extractLinksFromCmdLine(const LinkPairs& allLinks, +LinkPairs extractLinkNamesFromCmdLine(const LinkPairs& allLinks, const std::string linksFromCmdLine) { LinkPairs to_return; @@ -257,7 +257,8 @@ StudyParamsForLinkTS readGeneralParamsForLinksTS(fs::path studyDir) { if (! readLinkGeneralProperty(to_return, p->key, p->value)) { - logs.warning() << ini.filename() << ": reading value of '" << p->key << "' went wrong"; + logs.warning() << ini.filename() << ": reading value of '" + << p->key << "' went wrong"; } } } @@ -284,6 +285,7 @@ struct LinkTSgenerationParams Matrix<> modulationCapacityIndirect; bool forceNoGeneration = false; + bool hasValidData = true; }; std::vector CreateLinkList(const LinkPairs& linksFromCmdLine, @@ -361,16 +363,16 @@ void readLinkIniProperties(LinkTSgenerationParams* link, if (! readLinkIniProperty(link, p->key, p->value)) { std::string linkName = link->namesPair.first + "." + link->namesPair.second; - logs.warning() << "Link '" << linkName << "' : reading value of '" << p->key << "' went wrong"; + logs.warning() << "Link '" << linkName << "' : reading value of '" + << p->key << "' went wrong"; } } } -void readSourceAreaIniFile(fs::path sourceLinkDir, +void readSourceAreaIniFile(fs::path pathToIni, std::string sourceAreaName, std::vector& linkList) { - fs::path pathToIni = sourceLinkDir / "properties.ini"; IniFile ini; ini.open(pathToIni); // gp : we should handle reading issues for (auto* section = ini.firstSection; section; section = section->next) @@ -386,18 +388,76 @@ void readSourceAreaIniFile(fs::path sourceLinkDir, void readIniProperties(std::vector& linkList, fs::path toLinksDir) { - for (auto const& item : fs::directory_iterator{toLinksDir}) + for(auto& link : linkList) { - if (item.is_directory()) - { - std::string sourceAreaName = item.path().filename().generic_string(); - readSourceAreaIniFile(item, sourceAreaName, linkList); - } + std::string sourceAreaName = link.namesPair.first; + fs::path pathToIni = toLinksDir / sourceAreaName / "properties.ini"; + readSourceAreaIniFile(pathToIni, sourceAreaName, linkList); } } -bool readPreproTimeSeries(std::vector& linkList, fs::path toLinksDir) +bool readLinkPreproTimeSeries(LinkTSgenerationParams& link, + fs::path sourceAreaDir) { + bool to_return = true; + const auto preproId = link.namesPair.first + "/" + link.namesPair.second; + link.prepro = std::make_unique(preproId, link.unitCount); + + auto preproFileRoot = sourceAreaDir / "prepro" / link.namesPair.second; + + auto preproFile = preproFileRoot; + preproFile += ".txt"; + if (fs::exists(preproFile)) + { + to_return = link.prepro->data.loadFromCSVFile( + preproFile.string(), + Data::PreproAvailability::preproAvailabilityMax, + DAYS_PER_YEAR) + && link.prepro->validate() + && to_return; + } + + auto modulationFileDirect = preproFileRoot; + modulationFileDirect += "_mod_direct.txt"; + if (fs::exists(modulationFileDirect)) + { + to_return = link.modulationCapacityDirect.loadFromCSVFile( + modulationFileDirect.string(), + 1, + HOURS_PER_YEAR) + && to_return; + } + + auto modulationFileIndirect = preproFileRoot; + modulationFileIndirect += "_mod_indirect.txt"; + if (fs::exists(modulationFileIndirect)) + { + to_return = link.modulationCapacityIndirect.loadFromCSVFile( + modulationFileIndirect.string(), + 1, + HOURS_PER_YEAR) + && to_return; + } + // Makes possible a skip of TS generation when time comes + link.hasValidData = link.hasValidData && to_return; + + return to_return; +} + +bool readPreproTimeSeries(std::vector& linkList, + fs::path toLinksDir) +{ + for(auto& link : linkList) + { + std::string sourceAreaName = link.namesPair.first; + fs::path sourceAreaDir = toLinksDir / sourceAreaName; + if (! readLinkPreproTimeSeries(link, sourceAreaDir)) + { + logs.error() << "Could not load all prepro data for links '" + << link.namesPair.first << "." << link.namesPair.second << "'"; + return false; + } + } return true; } @@ -499,10 +559,11 @@ int main(int argc, char* argv[]) // LINKS // ===== New code for TS generation links ==================================== - auto allLinksPairs = extractLinksFromStudy(settings.studyFolder); + auto allLinksPairs = extractLinkNamesFromStudy(settings.studyFolder); // logLinks("All links", allLinksPairs); - auto linksFromCmdLine = extractLinksFromCmdLine(allLinksPairs, settings.linksListToGen); + auto linksFromCmdLine = extractLinkNamesFromCmdLine(allLinksPairs, + settings.linksListToGen); // logLinks("Links from cmd line", linksFromCmdLine); if (settings.allLinks) linksFromCmdLine = allLinksPairs; @@ -512,11 +573,14 @@ int main(int argc, char* argv[]) std::vector linkList = CreateLinkList(linksFromCmdLine, params); if (! readLinksSpecificTSparameters(linkList, settings.studyFolder)) { - // Something wrong while reading particular data associated to - // link TS generation + logs.warning() << "All data could not be loaded for links TS generation"; + // gp : let's be more careful about how we handle problems when loading data. + // gp : - examining to throw / catch exceptions and marking a link as invalid + // gp : when catching the exception at the link level + // gp : - if a link does have all of its data loaded, then we don't generate TS + // gp ! for this link. } - // ============================================================================ TSGenerator::listOfLinks links; From 825ca9b7532c663d2485fd5504a9bc59d814ccdc Mon Sep 17 00:00:00 2001 From: Guillaume PIERRE Date: Thu, 13 Jun 2024 11:45:15 +0200 Subject: [PATCH 05/28] Remove TS links from solver : actually generating TS for links --- src/solver/ts-generator/availability.cpp | 102 ++++++++++++++++-- .../antares/solver/ts-generator/generator.h | 44 ++++++++ src/tools/ts-generator/main.cpp | 82 +++++++------- 3 files changed, 179 insertions(+), 49 deletions(-) diff --git a/src/solver/ts-generator/availability.cpp b/src/solver/ts-generator/availability.cpp index 69f62a10a1..9b6df2d04c 100644 --- a/src/solver/ts-generator/availability.cpp +++ b/src/solver/ts-generator/availability.cpp @@ -28,6 +28,7 @@ #include #include #include +#include // For Antares::IO::fileSetContent #include "antares/study/simulation.h" #define SEP Yuni::IO::Separator @@ -54,6 +55,23 @@ AvailabilityTSGeneratorData::AvailabilityTSGeneratorData(Data::LinkTsGeneration& Data::TimeSeries& capacity, Matrix<>& modulation, const std::string& areaDestName): + unitCount(source.unitCount), + nominalCapacity(source.nominalCapacity), + forcedVolatility(source.forcedVolatility), + plannedVolatility(source.plannedVolatility), + forcedLaw(source.forcedLaw), + plannedLaw(source.plannedLaw), + prepro(source.prepro.get()), + series(capacity.timeSeries), + modulationCapacity(modulation[0]), + name(areaDestName) +{ +} + +AvailabilityTSGeneratorData::AvailabilityTSGeneratorData(LinkTSgenerationParams& source, + Data::TimeSeries& capacity, + Matrix<>& modulation, + const std::string& areaDestName): unitCount(source.unitCount), nominalCapacity(source.nominalCapacity), forcedVolatility(source.forcedVolatility), @@ -73,8 +91,9 @@ class GeneratorTempData final { public: explicit GeneratorTempData(Data::Study&, unsigned, MersenneTwister&); + explicit GeneratorTempData(bool, unsigned, MersenneTwister&); - void generateTS(const Data::Area& area, AvailabilityTSGeneratorData& cluster) const; + void generateTS(std::string, AvailabilityTSGeneratorData&) const; private: bool derated; @@ -106,6 +125,13 @@ GeneratorTempData::GeneratorTempData(Data::Study& study, unsigned nbOfSeriesToGe { } +GeneratorTempData::GeneratorTempData(bool derated, unsigned int nbOfSeriesToGen, MersenneTwister& rndGenerator): + derated(derated), + nbOfSeriesToGen_(nbOfSeriesToGen), + rndgenerator(rndGenerator) +{ +} + template void GeneratorTempData::prepareIndispoFromLaw(Data::StatisticalLaw law, double volatility, @@ -184,13 +210,13 @@ int GeneratorTempData::durationGenerator(Data::StatisticalLaw law, return 0; } -void GeneratorTempData::generateTS(const Data::Area& area, +void GeneratorTempData::generateTS(std::string sourceAreaName, AvailabilityTSGeneratorData& cluster) const { - if (!cluster.prepro) + if (!cluster.prepro) // gp : loaded data already checked way before this point (see : link.valid for links) { logs.error() - << "Cluster: " << area.name << '/' << cluster.name + << "Cluster: " << sourceAreaName << '/' << cluster.name << ": The timeseries will not be regenerated. All data related to the ts-generator for " << "'thermal' have been released."; return; @@ -641,6 +667,20 @@ void writeResultsToDisk(const Data::Study& study, writer.addEntryFromBuffer(savePath, buffer); } +void writeResultsToDisk(const Matrix<>& series, + const std::filesystem::path savePath) +{ + std::string buffer; + series.saveToBuffer(buffer, 0); + + std::filesystem::path parentDir = savePath.parent_path(); + if (! std::filesystem::exists(parentDir)) + { + std::filesystem::create_directories(parentDir); + } + Antares::IO::fileSetContent(savePath.string(), buffer); +} + bool generateThermalTimeSeries(Data::Study& study, const std::vector& clusters, Solver::IResultWriter& writer, @@ -659,7 +699,7 @@ bool generateThermalTimeSeries(Data::Study& study, for (auto* cluster: clusters) { AvailabilityTSGeneratorData tsConfigData(cluster); - generator.generateTS(*cluster->parentArea, tsConfigData); + generator.generateTS(cluster->parentArea->name, tsConfigData); if (archive) // compatibilty with in memory { @@ -700,7 +740,7 @@ bool generateLinkTimeSeries(Data::Study& study, // DIRECT AvailabilityTSGeneratorData tsConfigDataDirect(tsGenStruct, ts, tsGenStruct.modulationCapacityDirect, link->with->name); - generator.generateTS(*link->from, tsConfigDataDirect); + generator.generateTS(link->from->id.c_str(), tsConfigDataDirect); std::string filePath = savePath + SEP + link->from->id + SEP + link->with->id.c_str() + "_direct.txt"; @@ -709,13 +749,59 @@ bool generateLinkTimeSeries(Data::Study& study, // INDIRECT AvailabilityTSGeneratorData tsConfigDataIndirect(tsGenStruct, ts, tsGenStruct.modulationCapacityIndirect, link->with->name); - generator.generateTS(*link->from, tsConfigDataIndirect); + generator.generateTS(link->from->id.c_str(), tsConfigDataIndirect); filePath = savePath + SEP + link->from->id + SEP + link->with->id.c_str() - + "_indirect.txt"; + + "_indirect.txt"; writeResultsToDisk(study, writer, ts.timeSeries, filePath); } return true; } + +// gp : we should try to add const identifiers before args here +bool generateLinkTimeSeries(std::vector& links, + StudyParamsForLinkTS& generalParams, + const std::string& savePath) +{ + logs.info(); + logs.info() << "New Generation of links time-series"; + + auto generator = GeneratorTempData(generalParams.derated, + generalParams.nbLinkTStoGenerate, + generalParams.random); + for (auto& link: links) + { + + Data::TimeSeriesNumbers fakeTSnumbers; // gp : to quickly get rid of + Data::TimeSeries ts(fakeTSnumbers); + ts.resize(generalParams.nbLinkTStoGenerate, HOURS_PER_YEAR); + + if (! link.hasValidData) + { + logs.error() << "Missing data for link " << link.namesPair.first << "/" << link.namesPair.second; + return false; + } + + // DIRECT + AvailabilityTSGeneratorData tsConfigDataDirect(link, ts, link.modulationCapacityDirect, link.namesPair.second); + + generator.generateTS(link.namesPair.first, tsConfigDataDirect); + + std::string filePath = savePath + SEP + link.namesPair.first + SEP + link.namesPair.second + + "_direct.txt"; + writeResultsToDisk(ts.timeSeries, filePath); + + // INDIRECT + AvailabilityTSGeneratorData tsConfigDataIndirect(link, ts, link.modulationCapacityIndirect, link.namesPair.second); + + generator.generateTS(link.namesPair.first, tsConfigDataIndirect); + + filePath = savePath + SEP + link.namesPair.first + SEP + link.namesPair.second + + "_indirect.txt"; + writeResultsToDisk(ts.timeSeries, filePath); + } + + return true; +} } // namespace Antares::TSGenerator diff --git a/src/solver/ts-generator/include/antares/solver/ts-generator/generator.h b/src/solver/ts-generator/include/antares/solver/ts-generator/generator.h index 1c32e67bdd..cf976ff640 100644 --- a/src/solver/ts-generator/include/antares/solver/ts-generator/generator.h +++ b/src/solver/ts-generator/include/antares/solver/ts-generator/generator.h @@ -33,8 +33,44 @@ #include "xcast/xcast.h" +using LinkPair = std::pair; +using LinkPairs = std::vector; + namespace Antares::TSGenerator { + +struct StudyParamsForLinkTS +{ + unsigned int nbLinkTStoGenerate = 1; + bool derated = false; + // gp : we will have a problem with that if seed-tsgen-links not set in + // gp : generaldata.ini. In that case, our default value is wrong. + MersenneTwister random; +}; + +struct LinkTSgenerationParams +{ + LinkPair namesPair; + + unsigned unitCount = 0; + double nominalCapacity = 0; + + double forcedVolatility = 0.; + double plannedVolatility = 0.; + + Data::StatisticalLaw forcedLaw = Data::LawUniform; + Data::StatisticalLaw plannedLaw = Data::LawUniform; + + std::unique_ptr prepro; + + Matrix<> modulationCapacityDirect; + Matrix<> modulationCapacityIndirect; + + bool forceNoGeneration = false; + bool hasValidData = true; +}; + + class AvailabilityTSGeneratorData { public: @@ -43,6 +79,10 @@ class AvailabilityTSGeneratorData Data::TimeSeries&, Matrix<>& modulation, const std::string& name); + AvailabilityTSGeneratorData(LinkTSgenerationParams&, + Data::TimeSeries&, + Matrix<>& modulation, + const std::string& name); const unsigned& unitCount; const double& nominalCapacity; @@ -82,6 +122,10 @@ bool generateLinkTimeSeries(Data::Study& study, Solver::IResultWriter& writer, const std::string& savePath); +bool generateLinkTimeSeries(std::vector& links, + StudyParamsForLinkTS&, + const std::string& savePath); + std::vector getAllClustersToGen(const Data::AreaList& areas, bool globalThermalTSgeneration); diff --git a/src/tools/ts-generator/main.cpp b/src/tools/ts-generator/main.cpp index 8f04b50290..c0e583aced 100644 --- a/src/tools/ts-generator/main.cpp +++ b/src/tools/ts-generator/main.cpp @@ -23,6 +23,7 @@ #include #include +#include #include #include @@ -37,12 +38,10 @@ #include using namespace Antares; +using namespace Antares::TSGenerator; namespace fs = std::filesystem; -using LinkPair = std::pair; -using LinkPairs = std::vector; - struct Settings { std::string studyFolder; @@ -217,25 +216,26 @@ void logLinks(std::string title, LinkPairs& links) } } -struct StudyParamsForLinkTS -{ - unsigned int nbLinkTStoGenerate = 1; - // gp : we will have a problem with that if seed-tsgen-links not set in - // gp : generaldata.ini. In that case, our default value is wrong. - unsigned int seed = Data::antaresSeedDefaultValue; -}; - bool readLinkGeneralProperty(StudyParamsForLinkTS& params, const Yuni::String& key, const Yuni::String& value) { + if (key == "derated") + { + return value.to(params.derated); + } if (key == "nbtimeserieslinks") { return value.to(params.nbLinkTStoGenerate); } if (key == "seed-tsgen-links") { - return value.to(params.seed); + unsigned int seed {0}; + if (value.to(seed)) + { + params.random.reset(seed); + return true; + } } return true; // gp : should we return true here ? } @@ -265,38 +265,14 @@ StudyParamsForLinkTS readGeneralParamsForLinksTS(fs::path studyDir) return to_return; } -struct LinkTSgenerationParams -{ - LinkPair namesPair; - StudyParamsForLinkTS generalParams; - - unsigned unitCount = 0; - double nominalCapacity = 0; - - double forcedVolatility = 0.; - double plannedVolatility = 0.; - - Data::StatisticalLaw forcedLaw = Data::LawUniform; - Data::StatisticalLaw plannedLaw = Data::LawUniform; - - std::unique_ptr prepro; - - Matrix<> modulationCapacityDirect; - Matrix<> modulationCapacityIndirect; - - bool forceNoGeneration = false; - bool hasValidData = true; -}; - -std::vector CreateLinkList(const LinkPairs& linksFromCmdLine, - const StudyParamsForLinkTS& params) +std::vector CreateLinkList(const LinkPairs& linksFromCmdLine) { std::vector to_return(linksFromCmdLine.size()); + // gp : following loop should be improved : we shouldn't need an index unsigned int index = 0; for (const auto& link_pair : linksFromCmdLine) { to_return[index].namesPair = link_pair; - to_return[index].generalParams = params; index++; } return to_return; @@ -473,6 +449,22 @@ bool readLinksSpecificTSparameters(std::vector& linkList } return true; } + +std::string DateAndTime() +{ + YString to_return; + unsigned int now = Yuni::DateTime::Now(); + Yuni::DateTime::TimestampToString(to_return, "%Y%m%d-%H%M", now); + return to_return.to(); +} + +bool GenerateLinkTS(std::vector& linkList, + fs::path outputPath) +{ + + return true; +} + // ============================================================================ int main(int argc, char* argv[]) @@ -568,9 +560,9 @@ int main(int argc, char* argv[]) if (settings.allLinks) linksFromCmdLine = allLinksPairs; - StudyParamsForLinkTS params = readGeneralParamsForLinksTS(settings.studyFolder); + StudyParamsForLinkTS generalParams = readGeneralParamsForLinksTS(settings.studyFolder); - std::vector linkList = CreateLinkList(linksFromCmdLine, params); + std::vector linkList = CreateLinkList(linksFromCmdLine); if (! readLinksSpecificTSparameters(linkList, settings.studyFolder)) { logs.warning() << "All data could not be loaded for links TS generation"; @@ -578,9 +570,14 @@ int main(int argc, char* argv[]) // gp : - examining to throw / catch exceptions and marking a link as invalid // gp : when catching the exception at the link level // gp : - if a link does have all of its data loaded, then we don't generate TS - // gp ! for this link. + // gp : for this link. } + // Now generate TS for links + auto saveLinksTSpath = fs::path(settings.studyFolder) / "output" / DateAndTime(); + saveLinksTSpath /= "ts-generator"; + saveLinksTSpath /= "links"; + // ============================================================================ TSGenerator::listOfLinks links; @@ -605,5 +602,8 @@ int main(int argc, char* argv[]) ret = TSGenerator::generateLinkTimeSeries(*study, links, *resultWriter, linksSavePath.string()) && ret; + ret = TSGenerator::generateLinkTimeSeries(linkList, generalParams, saveLinksTSpath.string()) + && ret; + return !ret; // return 0 for success } \ No newline at end of file From a2b3bed78e3418082de8d9245b16ec7b3b6b6593 Mon Sep 17 00:00:00 2001 From: Guillaume PIERRE Date: Thu, 13 Jun 2024 14:20:25 +0200 Subject: [PATCH 06/28] Remove TS links from solver : remove now useless code --- src/libs/antares/study/area/links.cpp | 109 +----------------- src/libs/antares/study/area/list.cpp | 2 +- .../antares/study/cleaner/cleaner-v20.cpp | 2 +- .../study/include/antares/study/area/area.h | 3 +- .../study/include/antares/study/area/links.h | 2 - .../include/antares/study/load-options.h | 3 - src/solver/ts-generator/availability.cpp | 83 ++----------- .../antares/solver/ts-generator/generator.h | 11 +- src/tools/ts-generator/main.cpp | 39 +------ 9 files changed, 22 insertions(+), 232 deletions(-) diff --git a/src/libs/antares/study/area/links.cpp b/src/libs/antares/study/area/links.cpp index 64f1657034..c3df521908 100644 --- a/src/libs/antares/study/area/links.cpp +++ b/src/libs/antares/study/area/links.cpp @@ -137,58 +137,6 @@ bool AreaLink::linkLoadTimeSeries_for_version_820_and_later(const AnyString& fol return success; } -// This function is "lazy", it only loads files if they exist -// and set a `valid` flag -bool AreaLink::loadTSGenTimeSeries(const fs::path& folder) -{ - const std::string idprepro = std::string(from->id) + "/" + std::string(with->id); - tsGeneration.prepro = - std::make_unique(idprepro, tsGeneration.unitCount); - - bool anyFileWasLoaded = false; - - // file name without suffix, .txt for general infos and mod_direct/indirect.txt - fs::path preproFile = folder / "prepro" / with->id.c_str(); - - // Prepro - fs::path filepath = preproFile; - filepath += ".txt"; - if (fs::exists(filepath)) - { - anyFileWasLoaded = true; - tsGeneration.valid = tsGeneration.prepro->data.loadFromCSVFile( - filepath.string(), - Antares::Data::PreproAvailability::preproAvailabilityMax, - DAYS_PER_YEAR) - && tsGeneration.prepro->validate(); - } - - // Modulation - filepath = preproFile; - filepath += "_mod_direct.txt"; - if (fs::exists(filepath)) - { - anyFileWasLoaded = true; - tsGeneration.valid &= tsGeneration.modulationCapacityDirect - .loadFromCSVFile(filepath.string(), 1, HOURS_PER_YEAR); - } - - filepath = preproFile; - filepath += "_mod_indirect.txt"; - if (fs::exists(filepath)) - { - anyFileWasLoaded = true; - tsGeneration.valid &= tsGeneration.modulationCapacityIndirect - .loadFromCSVFile(filepath.string(), 1, HOURS_PER_YEAR); - } - - if (anyFileWasLoaded) - { - return tsGeneration.valid; - } - return true; -} - bool AreaLink::isLinkPhysical() const { // All link types are physical, except arVirt @@ -360,7 +308,8 @@ AreaLink* AreaAddLinkBetweenAreas(Area* area, Area* with, bool warning) namespace // anonymous { -bool handleKey(Data::AreaLink& link, const String& key, const String& value) + +bool AreaLinksInternalLoadFromProperty(AreaLink& link, const String& key, const String& value) { if (key == "hurdles-cost") { @@ -506,53 +455,6 @@ bool handleKey(Data::AreaLink& link, const String& key, const String& value) return false; } -bool handleTSGenKey(Data::LinkTsGeneration& out, - const std::string& key, - const String& value) -{ - if (key == "unitcount") - { - return value.to(out.unitCount); - } - - if (key == "nominalcapacity") - { - return value.to(out.nominalCapacity); - } - - if (key == "law.planned") - { - return value.to(out.plannedLaw); - } - - if (key == "law.forced") - { - return value.to(out.forcedLaw); - } - - if (key == "volatility.planned") - { - return value.to(out.plannedVolatility); - } - - if (key == "volatility.forced") - { - return value.to(out.forcedVolatility); - } - - if (key == "force-no-generation") - { - return value.to(out.forceNoGeneration); - } - - return false; -} - -bool AreaLinksInternalLoadFromProperty(AreaLink& link, const String& key, const String& value) -{ - return handleKey(link, key, value) || handleTSGenKey(link.tsGeneration, key, value); -} - [[noreturn]] void logLinkDataCheckError(const AreaLink& link, const String& msg, int hour) { logs.error() << "Link (" << link.from->name << "/" << link.with->name << "): Invalid values (" @@ -571,7 +473,7 @@ bool AreaLinksInternalLoadFromProperty(AreaLink& link, const String& key, const } } // anonymous namespace -bool AreaLinksLoadFromFolder(Study& study, AreaList* areaList, Area* area, const fs::path& folder, bool loadTSGen) +bool AreaLinksLoadFromFolder(Study& study, AreaList* areaList, Area* area, const fs::path& folder) { // Assert assert(area); @@ -697,11 +599,6 @@ bool AreaLinksLoadFromFolder(Study& study, AreaList* areaList, Area* area, const } } - if (loadTSGen) - { - ret = link.loadTSGenTimeSeries(folder) && ret; - } - // From the solver only if (study.usedByTheSolver) { diff --git a/src/libs/antares/study/area/list.cpp b/src/libs/antares/study/area/list.cpp index f76afb5686..fd2b3bb5b7 100644 --- a/src/libs/antares/study/area/list.cpp +++ b/src/libs/antares/study/area/list.cpp @@ -881,7 +881,7 @@ static bool AreaListLoadFromFolderSingleArea(Study& study, // Links { fs::path folder = fs::path(study.folderInput.c_str()) / "links" / area.id.c_str(); - ret = AreaLinksLoadFromFolder(study, list, &area, folder, options.linksLoadTSGen) && ret; + ret = AreaLinksLoadFromFolder(study, list, &area, folder) && ret; } // UI diff --git a/src/libs/antares/study/cleaner/cleaner-v20.cpp b/src/libs/antares/study/cleaner/cleaner-v20.cpp index 7d1aadf58e..3c8f183be4 100644 --- a/src/libs/antares/study/cleaner/cleaner-v20.cpp +++ b/src/libs/antares/study/cleaner/cleaner-v20.cpp @@ -362,7 +362,7 @@ bool listOfFilesAnDirectoriesToKeep(StudyCleaningInfos* infos) logs.verbosityLevel = Logs::Verbosity::Warning::level; // load all links buffer.clear() << infos->folder << "/input/links/" << area->id; - if (not AreaLinksLoadFromFolder(*study, arealist, area, buffer.c_str(), false)) + if (not AreaLinksLoadFromFolder(*study, arealist, area, buffer.c_str())) { delete arealist; delete study; diff --git a/src/libs/antares/study/include/antares/study/area/area.h b/src/libs/antares/study/include/antares/study/area/area.h index 6fbb69dde4..6661ab8556 100644 --- a/src/libs/antares/study/include/antares/study/area/area.h +++ b/src/libs/antares/study/include/antares/study/area/area.h @@ -727,8 +727,7 @@ AreaLink* AreaAddLinkBetweenAreas(Area* area, Area* with, bool warning = true); bool AreaLinksLoadFromFolder(Study& s, AreaList* l, Area* area, - const std::filesystem::path& folder, - bool loadTSGen); + const std::filesystem::path& folder); /*! ** \brief Save interconnections of a given area into a folder (`input/areas/[area]/ntc`) diff --git a/src/libs/antares/study/include/antares/study/area/links.h b/src/libs/antares/study/include/antares/study/area/links.h index 5b01ba3efd..c03c6c566e 100644 --- a/src/libs/antares/study/include/antares/study/area/links.h +++ b/src/libs/antares/study/include/antares/study/area/links.h @@ -71,8 +71,6 @@ class AreaLink final: public Yuni::NonCopyable bool loadTimeSeries(const StudyVersion& version, const AnyString& folder); - bool loadTSGenTimeSeries(const std::filesystem::path& folder); - void storeTimeseriesNumbers(Solver::IResultWriter& writer) const; //! \name Area diff --git a/src/libs/antares/study/include/antares/study/load-options.h b/src/libs/antares/study/include/antares/study/load-options.h index 1ecd34b968..41eee86cbb 100644 --- a/src/libs/antares/study/include/antares/study/load-options.h +++ b/src/libs/antares/study/include/antares/study/load-options.h @@ -53,9 +53,6 @@ class StudyLoadOptions //! Force the year-by-year flag bool forceYearByYear; - //! Load data associated to link TS generation - bool linksLoadTSGen = false; - //! Force the derated mode bool forceDerated; diff --git a/src/solver/ts-generator/availability.cpp b/src/solver/ts-generator/availability.cpp index 9b6df2d04c..c1ce9ecc79 100644 --- a/src/solver/ts-generator/availability.cpp +++ b/src/solver/ts-generator/availability.cpp @@ -51,23 +51,6 @@ AvailabilityTSGeneratorData::AvailabilityTSGeneratorData(Data::ThermalCluster* s { } -AvailabilityTSGeneratorData::AvailabilityTSGeneratorData(Data::LinkTsGeneration& source, - Data::TimeSeries& capacity, - Matrix<>& modulation, - const std::string& areaDestName): - unitCount(source.unitCount), - nominalCapacity(source.nominalCapacity), - forcedVolatility(source.forcedVolatility), - plannedVolatility(source.plannedVolatility), - forcedLaw(source.forcedLaw), - plannedLaw(source.plannedLaw), - prepro(source.prepro.get()), - series(capacity.timeSeries), - modulationCapacity(modulation[0]), - name(areaDestName) -{ -} - AvailabilityTSGeneratorData::AvailabilityTSGeneratorData(LinkTSgenerationParams& source, Data::TimeSeries& capacity, Matrix<>& modulation, @@ -212,8 +195,11 @@ int GeneratorTempData::durationGenerator(Data::StatisticalLaw law, void GeneratorTempData::generateTS(std::string sourceAreaName, AvailabilityTSGeneratorData& cluster) const + // gp : cluster ?? Used for links as well ! { - if (!cluster.prepro) // gp : loaded data already checked way before this point (see : link.valid for links) + // gp : loaded data is supposed to be already checked way before + // gp : this point (see : hasValidData for links) + if (!cluster.prepro) { logs.error() << "Cluster: " << sourceAreaName << '/' << cluster.name @@ -712,77 +698,32 @@ bool generateThermalTimeSeries(Data::Study& study, return true; } -bool generateLinkTimeSeries(Data::Study& study, - const listOfLinks& links, - Solver::IResultWriter& writer, - const std::string& savePath) -{ - logs.info(); - logs.info() << "Generating the links time-series"; - - auto generator = GeneratorTempData(study, - study.parameters.nbLinkTStoGenerate, - study.runtime->random[Data::seedTsGenLinks]); - - for (const auto& link: links) - { - Data::TimeSeries ts(link->timeseriesNumbers); - ts.resize(study.parameters.nbLinkTStoGenerate, HOURS_PER_YEAR); - - auto& tsGenStruct = link->tsGeneration; - - if (!tsGenStruct.valid) - { - logs.error() << "Missing data for link " << link->from->id << "/" << link->with->id; - return false; - } - - // DIRECT - AvailabilityTSGeneratorData tsConfigDataDirect(tsGenStruct, ts, tsGenStruct.modulationCapacityDirect, link->with->name); - - generator.generateTS(link->from->id.c_str(), tsConfigDataDirect); - - std::string filePath = savePath + SEP + link->from->id + SEP + link->with->id.c_str() - + "_direct.txt"; - writeResultsToDisk(study, writer, ts.timeSeries, filePath); - - // INDIRECT - AvailabilityTSGeneratorData tsConfigDataIndirect(tsGenStruct, ts, tsGenStruct.modulationCapacityIndirect, link->with->name); - - generator.generateTS(link->from->id.c_str(), tsConfigDataIndirect); - - filePath = savePath + SEP + link->from->id + SEP + link->with->id.c_str() - + "_indirect.txt"; - writeResultsToDisk(study, writer, ts.timeSeries, filePath); - } - - return true; -} - // gp : we should try to add const identifiers before args here bool generateLinkTimeSeries(std::vector& links, StudyParamsForLinkTS& generalParams, const std::string& savePath) { logs.info(); - logs.info() << "New Generation of links time-series"; + logs.info() << "Generation of links time-series"; auto generator = GeneratorTempData(generalParams.derated, generalParams.nbLinkTStoGenerate, generalParams.random); for (auto& link: links) { - - Data::TimeSeriesNumbers fakeTSnumbers; // gp : to quickly get rid of - Data::TimeSeries ts(fakeTSnumbers); - ts.resize(generalParams.nbLinkTStoGenerate, HOURS_PER_YEAR); - if (! link.hasValidData) { logs.error() << "Missing data for link " << link.namesPair.first << "/" << link.namesPair.second; return false; } + if (link.forceNoGeneration) + continue; // Skipping the link + + Data::TimeSeriesNumbers fakeTSnumbers; // gp : to quickly get rid of + Data::TimeSeries ts(fakeTSnumbers); + ts.resize(generalParams.nbLinkTStoGenerate, HOURS_PER_YEAR); + // DIRECT AvailabilityTSGeneratorData tsConfigDataDirect(link, ts, link.modulationCapacityDirect, link.namesPair.second); diff --git a/src/solver/ts-generator/include/antares/solver/ts-generator/generator.h b/src/solver/ts-generator/include/antares/solver/ts-generator/generator.h index cf976ff640..8cddc50679 100644 --- a/src/solver/ts-generator/include/antares/solver/ts-generator/generator.h +++ b/src/solver/ts-generator/include/antares/solver/ts-generator/generator.h @@ -75,10 +75,7 @@ class AvailabilityTSGeneratorData { public: explicit AvailabilityTSGeneratorData(Data::ThermalCluster*); - AvailabilityTSGeneratorData(Data::LinkTsGeneration&, - Data::TimeSeries&, - Matrix<>& modulation, - const std::string& name); + AvailabilityTSGeneratorData(LinkTSgenerationParams&, Data::TimeSeries&, Matrix<>& modulation, @@ -117,11 +114,6 @@ bool generateThermalTimeSeries(Data::Study& study, Solver::IResultWriter& writer, const std::string& savePath); -bool generateLinkTimeSeries(Data::Study& study, - const listOfLinks& links, - Solver::IResultWriter& writer, - const std::string& savePath); - bool generateLinkTimeSeries(std::vector& links, StudyParamsForLinkTS&, const std::string& savePath); @@ -129,7 +121,6 @@ bool generateLinkTimeSeries(std::vector& links, std::vector getAllClustersToGen(const Data::AreaList& areas, bool globalThermalTSgeneration); -listOfLinks getAllLinksToGen(Data::AreaList& areas); /*! ** \brief Destroy all TS Generators diff --git a/src/tools/ts-generator/main.cpp b/src/tools/ts-generator/main.cpp index c0e583aced..28beb11e23 100644 --- a/src/tools/ts-generator/main.cpp +++ b/src/tools/ts-generator/main.cpp @@ -207,15 +207,6 @@ LinkPairs extractLinkNamesFromCmdLine(const LinkPairs& allLinks, return to_return; } -void logLinks(std::string title, LinkPairs& links) -{ - std::cout << title << " : " << std::endl; - for (auto& link : links) - { - std::cout << "+ " << link.first << "." << link.second << std::endl; - } -} - bool readLinkGeneralProperty(StudyParamsForLinkTS& params, const Yuni::String& key, const Yuni::String& value) @@ -502,7 +493,6 @@ int main(int argc, char* argv[]) auto study = std::make_shared(true); Data::StudyLoadOptions studyOptions; studyOptions.prepareOutput = true; - studyOptions.linksLoadTSGen = true; if (!study->loadFromFolder(settings.studyFolder, studyOptions)) { @@ -531,9 +521,8 @@ int main(int argc, char* argv[]) durationCollector); const auto thermalSavePath = fs::path("ts-generator") / "thermal"; - const auto linksSavePath = fs::path("ts-generator") / "links"; - // THERMAL + // ============ THERMAL : Getting data for generating time-series ========= std::vector clusters; if (settings.allThermal) { @@ -549,14 +538,10 @@ int main(int argc, char* argv[]) logs.debug() << c->id(); } - // LINKS - // ===== New code for TS generation links ==================================== + // ============ LINKS : Getting data for generating LINKS time-series ===== auto allLinksPairs = extractLinkNamesFromStudy(settings.studyFolder); - // logLinks("All links", allLinksPairs); - auto linksFromCmdLine = extractLinkNamesFromCmdLine(allLinksPairs, settings.linksListToGen); - // logLinks("Links from cmd line", linksFromCmdLine); if (settings.allLinks) linksFromCmdLine = allLinksPairs; @@ -573,34 +558,16 @@ int main(int argc, char* argv[]) // gp : for this link. } - // Now generate TS for links auto saveLinksTSpath = fs::path(settings.studyFolder) / "output" / DateAndTime(); saveLinksTSpath /= "ts-generator"; saveLinksTSpath /= "links"; - // ============================================================================ - - TSGenerator::listOfLinks links; - if (settings.allLinks) - { - links = TSGenerator::getAllLinksToGen(study->areas); - } - else if (!settings.linksListToGen.empty()) - { - links = getLinksToGen(study->areas, settings.linksListToGen); - } - - for (auto& l: links) - { - logs.debug() << l->getName(); - } + // ============ TS Generation ============================================= bool ret = TSGenerator::generateThermalTimeSeries(*study, clusters, *resultWriter, thermalSavePath.string()); - ret = TSGenerator::generateLinkTimeSeries(*study, links, *resultWriter, linksSavePath.string()) - && ret; ret = TSGenerator::generateLinkTimeSeries(linkList, generalParams, saveLinksTSpath.string()) && ret; From 040e83c16603e4ab6b286546ddf0c77e06b4c635 Mon Sep 17 00:00:00 2001 From: Guillaume PIERRE Date: Thu, 13 Jun 2024 15:53:10 +0200 Subject: [PATCH 07/28] Remove TS links from solver : remove more code + correcting reading errors handling --- src/libs/antares/study/area/links.cpp | 21 ++++++++++++++---- src/libs/antares/study/fwd.cpp | 4 ---- .../study/include/antares/study/area/links.h | 3 --- .../antares/study/include/antares/study/fwd.h | 2 -- .../study/include/antares/study/parameters.h | 2 -- src/libs/antares/study/parameters.cpp | 13 ++++++----- src/solver/ts-generator/availability.cpp | 17 -------------- .../antares/solver/ts-generator/prepro.h | 22 ------------------- src/tools/ts-generator/main.cpp | 9 ++++---- 9 files changed, 28 insertions(+), 65 deletions(-) diff --git a/src/libs/antares/study/area/links.cpp b/src/libs/antares/study/area/links.cpp index c3df521908..82003ff0d7 100644 --- a/src/libs/antares/study/area/links.cpp +++ b/src/libs/antares/study/area/links.cpp @@ -23,10 +23,7 @@ #include #include - -#include - -#include +#include #include #include @@ -309,6 +306,16 @@ AreaLink* AreaAddLinkBetweenAreas(Area* area, Area* with, bool warning) namespace // anonymous { +bool isPropertyUsedForLinkTSgeneration(const std::string key) +{ + std::array listKeys + = {"unitcount", "nominalcapacity", "law.planned", "law.forced", + "volatility.planned", "volatility.forced", "force-no-generation"}; + if (std::find(listKeys.begin(), listKeys.end(), key) != listKeys.end()) + return true; + return false; +} + bool AreaLinksInternalLoadFromProperty(AreaLink& link, const String& key, const String& value) { if (key == "hurdles-cost") @@ -452,6 +459,12 @@ bool AreaLinksInternalLoadFromProperty(AreaLink& link, const String& key, const link.filterYearByYear = stringIntoDatePrecision(value); return true; } + if (isPropertyUsedForLinkTSgeneration(key.to())) + { + // Properties used by TS generator only. + // We just skip them (otherwise : reading error) + return true; + } return false; } diff --git a/src/libs/antares/study/fwd.cpp b/src/libs/antares/study/fwd.cpp index f45f093aaf..100a4b127e 100644 --- a/src/libs/antares/study/fwd.cpp +++ b/src/libs/antares/study/fwd.cpp @@ -53,8 +53,6 @@ const char* SeedToCString(SeedIndex seed) return "Noise on virtual Hydro costs"; case seedHydroManagement: return "Initial reservoir levels"; - case seedTsGenLinks: - return "Links time-series generation"; case seedMax: return ""; } @@ -87,8 +85,6 @@ const char* SeedToID(SeedIndex seed) return "seed-hydro-costs"; case seedHydroManagement: return "seed-initial-reservoir-levels"; - case seedTsGenLinks: - return "seed-tsgen-links"; case seedMax: return ""; } diff --git a/src/libs/antares/study/include/antares/study/area/links.h b/src/libs/antares/study/include/antares/study/area/links.h index c03c6c566e..fd4b8e69d8 100644 --- a/src/libs/antares/study/include/antares/study/area/links.h +++ b/src/libs/antares/study/include/antares/study/area/links.h @@ -206,9 +206,6 @@ class AreaLink final: public Yuni::NonCopyable int linkWidth; friend struct CompareLinkName; - - LinkTsGeneration tsGeneration; - }; // class AreaLink struct CompareLinkName final diff --git a/src/libs/antares/study/include/antares/study/fwd.h b/src/libs/antares/study/include/antares/study/fwd.h index 7256eda4a2..6fa2819aca 100644 --- a/src/libs/antares/study/include/antares/study/fwd.h +++ b/src/libs/antares/study/include/antares/study/fwd.h @@ -361,8 +361,6 @@ enum SeedIndex seedHydroCosts, //! Seed - Hydro management seedHydroManagement, - //! The seed for links - seedTsGenLinks, //! The number of seeds seedMax, }; diff --git a/src/libs/antares/study/include/antares/study/parameters.h b/src/libs/antares/study/include/antares/study/parameters.h index 791e20f865..3222548e87 100644 --- a/src/libs/antares/study/include/antares/study/parameters.h +++ b/src/libs/antares/study/include/antares/study/parameters.h @@ -258,8 +258,6 @@ class Parameters final uint nbTimeSeriesThermal; //! Nb of timeSeries : Solar uint nbTimeSeriesSolar; - //! Nb of timeSeries : Links - uint nbLinkTStoGenerate = 1; //@} //! \name Time-series refresh diff --git a/src/libs/antares/study/parameters.cpp b/src/libs/antares/study/parameters.cpp index 7337b9c8b0..e01e4750c4 100644 --- a/src/libs/antares/study/parameters.cpp +++ b/src/libs/antares/study/parameters.cpp @@ -530,7 +530,9 @@ static bool SGDIntLoadFamily_General(Parameters& d, } if (key == "nbtimeserieslinks") { - return value.to(d.nbLinkTStoGenerate); + // This data is among solver data, but is useless while running a simulation + // Only by TS generator. We skip it here (otherwise, we get a reading error). + return true; } // Interval values if (key == "refreshintervalload") @@ -1026,10 +1028,6 @@ static bool SGDIntLoadFamily_SeedsMersenneTwister(Parameters& d, { return value.to(d.seed[seedTsGenSolar]); } - if (key == "seed_links") - { - return value.to(d.seed[seedTsGenLinks]); - } if (key == "seed_timeseriesnumbers") { return value.to(d.seed[seedTimeseriesNumbers]); @@ -1046,6 +1044,10 @@ static bool SGDIntLoadFamily_SeedsMersenneTwister(Parameters& d, return value.to(d.seed[sd]); } } + if (key == "seed-tsgen-links") + { + return true; // Useless for solver, belongs to TS generator + } } } return false; @@ -1766,7 +1768,6 @@ void Parameters::saveToINI(IniFile& ini) const section->add("nbTimeSeriesWind", nbTimeSeriesWind); section->add("nbTimeSeriesThermal", nbTimeSeriesThermal); section->add("nbTimeSeriesSolar", nbTimeSeriesSolar); - section->add("nbtimeserieslinks", nbLinkTStoGenerate); // Refresh ParametersSaveTimeSeries(section, "refreshTimeSeries", timeSeriesToRefresh); diff --git a/src/solver/ts-generator/availability.cpp b/src/solver/ts-generator/availability.cpp index c1ce9ecc79..9220fb6162 100644 --- a/src/solver/ts-generator/availability.cpp +++ b/src/solver/ts-generator/availability.cpp @@ -621,23 +621,6 @@ std::vector getAllClustersToGen(const Data::AreaList& are return clusters; } -listOfLinks getAllLinksToGen(Data::AreaList& areas) -{ - listOfLinks links; - - areas.each( - [&links](const Data::Area& area) - { - std::ranges::for_each(area.links, [&links](auto& l) - { - if (!l.second->tsGeneration.forceNoGeneration) - links.push_back(l.second); - }); - }); - - return links; -} - void writeResultsToDisk(const Data::Study& study, Solver::IResultWriter& writer, const Matrix<>& series, diff --git a/src/solver/ts-generator/include/antares/solver/ts-generator/prepro.h b/src/solver/ts-generator/include/antares/solver/ts-generator/prepro.h index ac36a826ea..4d93f56f8a 100644 --- a/src/solver/ts-generator/include/antares/solver/ts-generator/prepro.h +++ b/src/solver/ts-generator/include/antares/solver/ts-generator/prepro.h @@ -114,28 +114,6 @@ class PreproAvailability YString id; unsigned int unitCount; }; // class PreproAvailability - -struct LinkTsGeneration -{ - unsigned unitCount = 0; - double nominalCapacity = 0; - - double forcedVolatility = 0.; - double plannedVolatility = 0.; - - Data::StatisticalLaw forcedLaw = LawUniform; - Data::StatisticalLaw plannedLaw = LawUniform; - - std::unique_ptr prepro; - - Matrix<> modulationCapacityDirect; - Matrix<> modulationCapacityIndirect; - - bool valid = false; - - bool forceNoGeneration = false; -}; - } // namespace Antares::Data #endif // __ANTARES_LIBS_STUDY_PARTS_THERMAL_PREPRO_HXX__ diff --git a/src/tools/ts-generator/main.cpp b/src/tools/ts-generator/main.cpp index 28beb11e23..a53f7e0085 100644 --- a/src/tools/ts-generator/main.cpp +++ b/src/tools/ts-generator/main.cpp @@ -222,11 +222,10 @@ bool readLinkGeneralProperty(StudyParamsForLinkTS& params, if (key == "seed-tsgen-links") { unsigned int seed {0}; - if (value.to(seed)) - { - params.random.reset(seed); - return true; - } + if (! value.to(seed)) + return false; + params.random.reset(seed); + return true; } return true; // gp : should we return true here ? } From 3ea7537ada29339a84a95b4915b3ea53d105f517 Mon Sep 17 00:00:00 2001 From: Guillaume PIERRE Date: Thu, 13 Jun 2024 16:14:04 +0200 Subject: [PATCH 08/28] Remove TS links from solver : useless code removal --- src/tools/ts-generator/main.cpp | 22 ---------------------- 1 file changed, 22 deletions(-) diff --git a/src/tools/ts-generator/main.cpp b/src/tools/ts-generator/main.cpp index a53f7e0085..abec02ae84 100644 --- a/src/tools/ts-generator/main.cpp +++ b/src/tools/ts-generator/main.cpp @@ -114,28 +114,6 @@ std::vector getClustersToGen(Data::AreaList& areas, return clusters; } -TSGenerator::listOfLinks getLinksToGen(Data::AreaList& areas, const std::string& linksToGen) -{ - TSGenerator::listOfLinks links; - const auto ids = splitStringIntoPairs(linksToGen, ';', '.'); - - for (const auto& [areaFromID, areaWithID]: ids) - { - logs.info() << "Searching for link: " << areaFromID << "/" << areaWithID; - - auto* link = areas.findLink(areaFromID, areaWithID); - if (!link) - { - logs.warning() << "Link not found: " << areaFromID << "/" << areaWithID; - continue; - } - - links.emplace_back(link); - } - - return links; -} - // ===== New code for TS generation links ==================================== std::vector extractTargetAreas(fs::path sourceLinkDir) From 1999d0c8f434534c70cf6ca1ceaed841d8b57883 Mon Sep 17 00:00:00 2001 From: Guillaume PIERRE Date: Thu, 13 Jun 2024 16:18:30 +0200 Subject: [PATCH 09/28] [skip ci] Remove TS links from solver : code removal --- src/tools/ts-generator/main.cpp | 8 -------- 1 file changed, 8 deletions(-) diff --git a/src/tools/ts-generator/main.cpp b/src/tools/ts-generator/main.cpp index abec02ae84..3afc12a522 100644 --- a/src/tools/ts-generator/main.cpp +++ b/src/tools/ts-generator/main.cpp @@ -425,14 +425,6 @@ std::string DateAndTime() Yuni::DateTime::TimestampToString(to_return, "%Y%m%d-%H%M", now); return to_return.to(); } - -bool GenerateLinkTS(std::vector& linkList, - fs::path outputPath) -{ - - return true; -} - // ============================================================================ int main(int argc, char* argv[]) From 1862e31fd2aa778b3803ff98a3d24ce986d35dde Mon Sep 17 00:00:00 2001 From: Guillaume PIERRE Date: Fri, 14 Jun 2024 11:37:01 +0200 Subject: [PATCH 10/28] Remove TS links from solver : review corrections + more simplifications --- src/libs/antares/study/area/links.cpp | 18 ++---- src/solver/ts-generator/availability.cpp | 82 ++++++++++-------------- 2 files changed, 40 insertions(+), 60 deletions(-) diff --git a/src/libs/antares/study/area/links.cpp b/src/libs/antares/study/area/links.cpp index 82003ff0d7..96a304d28e 100644 --- a/src/libs/antares/study/area/links.cpp +++ b/src/libs/antares/study/area/links.cpp @@ -306,14 +306,12 @@ AreaLink* AreaAddLinkBetweenAreas(Area* area, Area* with, bool warning) namespace // anonymous { -bool isPropertyUsedForLinkTSgeneration(const std::string key) +bool isPropertyUsedForLinkTSgeneration(const std::string& key) { std::array listKeys = {"unitcount", "nominalcapacity", "law.planned", "law.forced", "volatility.planned", "volatility.forced", "force-no-generation"}; - if (std::find(listKeys.begin(), listKeys.end(), key) != listKeys.end()) - return true; - return false; + return std::find(listKeys.begin(), listKeys.end(), key) != listKeys.end(); } bool AreaLinksInternalLoadFromProperty(AreaLink& link, const String& key, const String& value) @@ -459,13 +457,9 @@ bool AreaLinksInternalLoadFromProperty(AreaLink& link, const String& key, const link.filterYearByYear = stringIntoDatePrecision(value); return true; } - if (isPropertyUsedForLinkTSgeneration(key.to())) - { - // Properties used by TS generator only. - // We just skip them (otherwise : reading error) - return true; - } - return false; + // Properties used by TS generator only. + // We just skip them (otherwise : reading error) + return isPropertyUsedForLinkTSgeneration(key.to()); } [[noreturn]] void logLinkDataCheckError(const AreaLink& link, const String& msg, int hour) @@ -508,7 +502,7 @@ bool AreaLinksLoadFromFolder(Study& study, AreaList* areaList, Area* area, const for (auto* s = ini.firstSection; s; s = s->next) { // Getting the name of the area - std::string targetAreaName = transformNameIntoID(s->name); + const std::string targetAreaName = transformNameIntoID(s->name); // Trying to find it Area* targetArea = AreaListLFind(areaList, targetAreaName.c_str()); diff --git a/src/solver/ts-generator/availability.cpp b/src/solver/ts-generator/availability.cpp index 9220fb6162..696725f846 100644 --- a/src/solver/ts-generator/availability.cpp +++ b/src/solver/ts-generator/availability.cpp @@ -37,17 +37,17 @@ constexpr double FAILURE_RATE_EQ_1 = 0.999; namespace Antares::TSGenerator { -AvailabilityTSGeneratorData::AvailabilityTSGeneratorData(Data::ThermalCluster* source): - unitCount(source->unitCount), - nominalCapacity(source->nominalCapacity), - forcedVolatility(source->forcedVolatility), - plannedVolatility(source->plannedVolatility), - forcedLaw(source->forcedLaw), - plannedLaw(source->plannedLaw), - prepro(source->prepro), - series(source->series.timeSeries), - modulationCapacity(source->modulation[Data::thermalModulationCapacity]), - name(source->name()) +AvailabilityTSGeneratorData::AvailabilityTSGeneratorData(Data::ThermalCluster* cluster): + unitCount(cluster->unitCount), + nominalCapacity(cluster->nominalCapacity), + forcedVolatility(cluster->forcedVolatility), + plannedVolatility(cluster->plannedVolatility), + forcedLaw(cluster->forcedLaw), + plannedLaw(cluster->plannedLaw), + prepro(cluster->prepro), + series(cluster->series.timeSeries), + modulationCapacity(cluster->modulation[Data::thermalModulationCapacity]), + name(cluster->name()) { } @@ -76,7 +76,7 @@ class GeneratorTempData final explicit GeneratorTempData(Data::Study&, unsigned, MersenneTwister&); explicit GeneratorTempData(bool, unsigned, MersenneTwister&); - void generateTS(std::string, AvailabilityTSGeneratorData&) const; + void generateTS(AvailabilityTSGeneratorData&) const; private: bool derated; @@ -193,31 +193,18 @@ int GeneratorTempData::durationGenerator(Data::StatisticalLaw law, return 0; } -void GeneratorTempData::generateTS(std::string sourceAreaName, - AvailabilityTSGeneratorData& cluster) const - // gp : cluster ?? Used for links as well ! +void GeneratorTempData::generateTS(AvailabilityTSGeneratorData& tsGenerationData) const { - // gp : loaded data is supposed to be already checked way before - // gp : this point (see : hasValidData for links) - if (!cluster.prepro) - { - logs.error() - << "Cluster: " << sourceAreaName << '/' << cluster.name - << ": The timeseries will not be regenerated. All data related to the ts-generator for " - << "'thermal' have been released."; - return; - } - - assert(cluster.prepro); + assert(tsGenerationData.prepro); - if (0 == cluster.unitCount || 0 == cluster.nominalCapacity) + if (0 == tsGenerationData.unitCount || 0 == tsGenerationData.nominalCapacity) { return; } - const auto& preproData = *(cluster.prepro); + const auto& preproData = *(tsGenerationData.prepro); - int AUN = cluster.unitCount; + int AUN = tsGenerationData.unitCount; auto& FOD = preproData.data[Data::PreproAvailability::foDuration]; @@ -231,13 +218,13 @@ void GeneratorTempData::generateTS(std::string sourceAreaName, auto& NPOmax = preproData.data[Data::PreproAvailability::npoMax]; - double f_volatility = cluster.forcedVolatility; + double f_volatility = tsGenerationData.forcedVolatility; - double p_volatility = cluster.plannedVolatility; + double p_volatility = tsGenerationData.plannedVolatility; - auto f_law = cluster.forcedLaw; + auto f_law = tsGenerationData.forcedLaw; - auto p_law = cluster.plannedLaw; + auto p_law = tsGenerationData.plannedLaw; std::vector> FPOW(DAYS_PER_YEAR); std::vector> PPOW(DAYS_PER_YEAR); @@ -259,8 +246,8 @@ void GeneratorTempData::generateTS(std::string sourceAreaName, for (uint d = 0; d < DAYS_PER_YEAR; ++d) { - FPOW[d].resize(cluster.unitCount + 1); - PPOW[d].resize(cluster.unitCount + 1); + FPOW[d].resize(tsGenerationData.unitCount + 1); + PPOW[d].resize(tsGenerationData.unitCount + 1); PODOfTheDay = (int)POD[d]; FODOfTheDay = (int)FOD[d]; @@ -293,7 +280,7 @@ void GeneratorTempData::generateTS(std::string sourceAreaName, pp[d] = lp[d] / b; } - for (uint k = 0; k != cluster.unitCount + 1; ++k) + for (uint k = 0; k != tsGenerationData.unitCount + 1; ++k) { FPOW[d][k] = pow(a, (double)k); PPOW[d][k] = pow(b, (double)k); @@ -325,7 +312,7 @@ void GeneratorTempData::generateTS(std::string sourceAreaName, double cumul = 0; double last = 0; - auto& modulation = cluster.modulationCapacity; + auto& modulation = tsGenerationData.modulationCapacity; double* dstSeries = nullptr; const uint tsCount = nbOfSeriesToGen_ + 2; @@ -335,7 +322,7 @@ void GeneratorTempData::generateTS(std::string sourceAreaName, if (tsIndex > 1) { - dstSeries = cluster.series[tsIndex - 2]; + dstSeries = tsGenerationData.series[tsIndex - 2]; } for (uint dayInTheYear = 0; dayInTheYear < DAYS_PER_YEAR; ++dayInTheYear) @@ -510,7 +497,7 @@ void GeneratorTempData::generateTS(std::string sourceAreaName, } } - if (cluster.unitCount == 1) + if (tsGenerationData.unitCount == 1) { if (POC == 1 && FOC == 1) { @@ -580,7 +567,7 @@ void GeneratorTempData::generateTS(std::string sourceAreaName, } NOW = (NOW + 1) % Log_size; - AVP[dayInTheYear] = AUN * cluster.nominalCapacity; + AVP[dayInTheYear] = AUN * tsGenerationData.nominalCapacity; if (tsIndex > 1) { @@ -596,7 +583,7 @@ void GeneratorTempData::generateTS(std::string sourceAreaName, if (derated) { - cluster.series.averageTimeseries(); + tsGenerationData.series.averageTimeseries(); } } } // namespace @@ -664,13 +651,12 @@ bool generateThermalTimeSeries(Data::Study& study, study.parameters.nbTimeSeriesThermal, study.runtime->random[Data::seedTsGenThermal]); - // TODO VP: parallel for (auto* cluster: clusters) { - AvailabilityTSGeneratorData tsConfigData(cluster); - generator.generateTS(cluster->parentArea->name, tsConfigData); + AvailabilityTSGeneratorData tsGenerationData(cluster); + generator.generateTS(tsGenerationData); - if (archive) // compatibilty with in memory + if (archive) // For compatibilty with in memory thermal TS generation { std::string filePath = savePath + SEP + cluster->parentArea->id + SEP + cluster->id() + ".txt"; @@ -710,7 +696,7 @@ bool generateLinkTimeSeries(std::vector& links, // DIRECT AvailabilityTSGeneratorData tsConfigDataDirect(link, ts, link.modulationCapacityDirect, link.namesPair.second); - generator.generateTS(link.namesPair.first, tsConfigDataDirect); + generator.generateTS(tsConfigDataDirect); std::string filePath = savePath + SEP + link.namesPair.first + SEP + link.namesPair.second + "_direct.txt"; @@ -719,7 +705,7 @@ bool generateLinkTimeSeries(std::vector& links, // INDIRECT AvailabilityTSGeneratorData tsConfigDataIndirect(link, ts, link.modulationCapacityIndirect, link.namesPair.second); - generator.generateTS(link.namesPair.first, tsConfigDataIndirect); + generator.generateTS(tsConfigDataIndirect); filePath = savePath + SEP + link.namesPair.first + SEP + link.namesPair.second + "_indirect.txt"; From e28410201974d99a5d2ae4febe107244952476b2 Mon Sep 17 00:00:00 2001 From: Guillaume PIERRE Date: Fri, 14 Jun 2024 14:07:11 +0200 Subject: [PATCH 11/28] Remove TS links from solver : simplifying the data load error handling --- src/tools/ts-generator/main.cpp | 29 ++++++----------------------- 1 file changed, 6 insertions(+), 23 deletions(-) diff --git a/src/tools/ts-generator/main.cpp b/src/tools/ts-generator/main.cpp index 3afc12a522..e131cfa40e 100644 --- a/src/tools/ts-generator/main.cpp +++ b/src/tools/ts-generator/main.cpp @@ -35,7 +35,6 @@ #include #include #include -#include using namespace Antares; using namespace Antares::TSGenerator; @@ -295,7 +294,6 @@ bool readLinkIniProperty(LinkTSgenerationParams* link, { return value.to(link->forceNoGeneration); } - return true; } @@ -309,6 +307,7 @@ void readLinkIniProperties(LinkTSgenerationParams* link, std::string linkName = link->namesPair.first + "." + link->namesPair.second; logs.warning() << "Link '" << linkName << "' : reading value of '" << p->key << "' went wrong"; + link->hasValidData = false; } } } @@ -384,11 +383,10 @@ bool readLinkPreproTimeSeries(LinkTSgenerationParams& link, } // Makes possible a skip of TS generation when time comes link.hasValidData = link.hasValidData && to_return; - return to_return; } -bool readPreproTimeSeries(std::vector& linkList, +void readPreproTimeSeries(std::vector& linkList, fs::path toLinksDir) { for(auto& link : linkList) @@ -397,25 +395,18 @@ bool readPreproTimeSeries(std::vector& linkList, fs::path sourceAreaDir = toLinksDir / sourceAreaName; if (! readLinkPreproTimeSeries(link, sourceAreaDir)) { - logs.error() << "Could not load all prepro data for links '" + logs.warning() << "Could not load all prepro data for link '" << link.namesPair.first << "." << link.namesPair.second << "'"; - return false; } } - return true; } -bool readLinksSpecificTSparameters(std::vector& linkList, +void readLinksSpecificTSparameters(std::vector& linkList, fs::path studyFolder) { fs::path toLinksDir = studyFolder / "input" / "links"; readIniProperties(linkList, toLinksDir); - if (! readPreproTimeSeries(linkList, toLinksDir)) - { - // gp : Log a problem here - return false; - } - return true; + readPreproTimeSeries(linkList, toLinksDir); } std::string DateAndTime() @@ -517,15 +508,7 @@ int main(int argc, char* argv[]) StudyParamsForLinkTS generalParams = readGeneralParamsForLinksTS(settings.studyFolder); std::vector linkList = CreateLinkList(linksFromCmdLine); - if (! readLinksSpecificTSparameters(linkList, settings.studyFolder)) - { - logs.warning() << "All data could not be loaded for links TS generation"; - // gp : let's be more careful about how we handle problems when loading data. - // gp : - examining to throw / catch exceptions and marking a link as invalid - // gp : when catching the exception at the link level - // gp : - if a link does have all of its data loaded, then we don't generate TS - // gp : for this link. - } + readLinksSpecificTSparameters(linkList, settings.studyFolder); auto saveLinksTSpath = fs::path(settings.studyFolder) / "output" / DateAndTime(); saveLinksTSpath /= "ts-generator"; From b671e2296884cbaf54f41e8ed4beed755eb2f4b4 Mon Sep 17 00:00:00 2001 From: Florian OMNES <26088210+flomnes@users.noreply.github.com> Date: Fri, 14 Jun 2024 17:17:07 +0200 Subject: [PATCH 12/28] Eliminate need for index in loop --- src/tools/ts-generator/main.cpp | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/tools/ts-generator/main.cpp b/src/tools/ts-generator/main.cpp index e131cfa40e..60ad9dd720 100644 --- a/src/tools/ts-generator/main.cpp +++ b/src/tools/ts-generator/main.cpp @@ -234,13 +234,13 @@ StudyParamsForLinkTS readGeneralParamsForLinksTS(fs::path studyDir) std::vector CreateLinkList(const LinkPairs& linksFromCmdLine) { - std::vector to_return(linksFromCmdLine.size()); - // gp : following loop should be improved : we shouldn't need an index - unsigned int index = 0; + std::vector to_return; + to_return.reserve(linksFromCmdLine.size()); for (const auto& link_pair : linksFromCmdLine) { - to_return[index].namesPair = link_pair; - index++; + LinkTSgenerationParams params; + params.namesPair = link_pair; + to_return.push_back(std::move(params)); } return to_return; } From c844e0305b8fb4a51e64f06cd491f06857aeac3a Mon Sep 17 00:00:00 2001 From: Guillaume PIERRE Date: Fri, 14 Jun 2024 15:37:39 +0200 Subject: [PATCH 13/28] Split TS generation into files : move options handling to separate source files --- src/tools/ts-generator/CMakeLists.txt | 3 + src/tools/ts-generator/linksTSgenerator.h | 11 +++ src/tools/ts-generator/main.cpp | 76 ++----------------- .../ts-generator/tsGenerationOptions.cpp | 66 ++++++++++++++++ src/tools/ts-generator/tsGenerationOptions.h | 29 +++++++ 5 files changed, 116 insertions(+), 69 deletions(-) create mode 100644 src/tools/ts-generator/linksTSgenerator.h create mode 100644 src/tools/ts-generator/tsGenerationOptions.cpp create mode 100644 src/tools/ts-generator/tsGenerationOptions.h diff --git a/src/tools/ts-generator/CMakeLists.txt b/src/tools/ts-generator/CMakeLists.txt index cea38ae0cd..9695be97e6 100644 --- a/src/tools/ts-generator/CMakeLists.txt +++ b/src/tools/ts-generator/CMakeLists.txt @@ -1,5 +1,8 @@ set(SRCS main.cpp + linksTSgenerator.h + tsGenerationOptions.h + tsGenerationOptions.cpp ) set(execname "antares-ts-generator") diff --git a/src/tools/ts-generator/linksTSgenerator.h b/src/tools/ts-generator/linksTSgenerator.h new file mode 100644 index 0000000000..9af3f368fc --- /dev/null +++ b/src/tools/ts-generator/linksTSgenerator.h @@ -0,0 +1,11 @@ + +#pragma once + +class LinksTSgenerator +{ + void loadData(); + void generate(); + + +}; + diff --git a/src/tools/ts-generator/main.cpp b/src/tools/ts-generator/main.cpp index 60ad9dd720..002435006d 100644 --- a/src/tools/ts-generator/main.cpp +++ b/src/tools/ts-generator/main.cpp @@ -22,7 +22,6 @@ #include #include -#include #include #include @@ -36,53 +35,13 @@ #include #include +#include "tsGenerationOptions.h" + using namespace Antares; using namespace Antares::TSGenerator; namespace fs = std::filesystem; -struct Settings -{ - std::string studyFolder; - - /// generate TS for all clusters if activated - bool allThermal = false; - /// generate TS for a list "area.cluster;area2.cluster2;" - std::string thermalListToGen = ""; - - /// generate TS for all links if activated - bool allLinks = false; - /// generate TS for a list "area.link;area2.link2;" - std::string linksListToGen = ""; -}; - -std::unique_ptr createTsGeneratorParser(Settings& settings) -{ - auto parser = std::make_unique(); - parser->addParagraph("Antares Time Series generator\n"); - - parser->addFlag(settings.allThermal, - ' ', - "all-thermal", - "Generate TS for all thermal clusters"); - parser->addFlag(settings.thermalListToGen, - ' ', - "thermal", - "Generate TS for a list of area IDs and thermal clusters IDs, " - "\nusage: --thermal=\"areaID.clusterID;area2ID.clusterID\""); - - parser->addFlag(settings.allLinks, ' ', "all-links", "Generate TS capacities for all links"); - parser->addFlag(settings.linksListToGen, - ' ', - "links", - "Generate TS capacities for a list of 2 area IDs, " - "usage: --links=\"areaID.area2ID;area3ID.area1ID\""); - - parser->remainingArguments(settings.studyFolder); - - return parser; -} - std::vector getClustersToGen(Data::AreaList& areas, const std::string& clustersToGen) { @@ -317,7 +276,7 @@ void readSourceAreaIniFile(fs::path pathToIni, std::vector& linkList) { IniFile ini; - ini.open(pathToIni); // gp : we should handle reading issues + ini.open(pathToIni); // gp : we should handle reading problems here for (auto* section = ini.firstSection; section; section = section->next) { std::string targetAreaName = transformNameIntoID(section->name); @@ -381,7 +340,7 @@ bool readLinkPreproTimeSeries(LinkTSgenerationParams& link, HOURS_PER_YEAR) && to_return; } - // Makes possible a skip of TS generation when time comes + // Makes it possible to skip a link's TS generation when time comes link.hasValidData = link.hasValidData && to_return; return to_return; } @@ -423,32 +382,11 @@ int main(int argc, char* argv[]) logs.applicationName("ts-generator"); Settings settings; - - auto parser = createTsGeneratorParser(settings); - switch (auto ret = parser->operator()(argc, argv); ret) - { - using namespace Yuni::GetOpt; - case ReturnCode::error: - logs.error() << "Unknown arguments, aborting"; - return parser->errors(); - case ReturnCode::help: - // End the program - return 0; - default: - break; - } - - if (settings.allThermal && !settings.thermalListToGen.empty()) - { - logs.error() << "Conflicting options, either choose all thermal clusters or a list"; + if (! parseOptions(argc, argv, settings)) return 1; - } - if (settings.allLinks && !settings.linksListToGen.empty()) - { - logs.error() << "Conflicting options, either choose all links or a list"; + if (! checkOptions(settings)) return 1; - } auto study = std::make_shared(true); Data::StudyLoadOptions studyOptions; @@ -486,7 +424,7 @@ int main(int argc, char* argv[]) std::vector clusters; if (settings.allThermal) { - clusters = TSGenerator::getAllClustersToGen(study->areas, true); + clusters = getAllClustersToGen(study->areas, true); } else if (!settings.thermalListToGen.empty()) { diff --git a/src/tools/ts-generator/tsGenerationOptions.cpp b/src/tools/ts-generator/tsGenerationOptions.cpp new file mode 100644 index 0000000000..d80d73448c --- /dev/null +++ b/src/tools/ts-generator/tsGenerationOptions.cpp @@ -0,0 +1,66 @@ +#include "tsGenerationOptions.h" +#include + +namespace Antares::TSGenerator { + +std::unique_ptr createTsGeneratorParser(Settings &settings) { + auto parser = std::make_unique(); + parser->addParagraph("Antares Time Series generator\n"); + + parser->addFlag(settings.allThermal, + ' ', + "all-thermal", + "Generate TS for all thermal clusters"); + parser->addFlag(settings.thermalListToGen, + ' ', + "thermal", + "Generate TS for a list of area IDs and thermal clusters IDs, " + "\nusage: --thermal=\"areaID.clusterID;area2ID.clusterID\""); + + parser->addFlag(settings.allLinks, ' ', "all-links", "Generate TS capacities for all links"); + parser->addFlag(settings.linksListToGen, + ' ', + "links", + "Generate TS capacities for a list of 2 area IDs, " + "usage: --links=\"areaID.area2ID;area3ID.area1ID\""); + + parser->remainingArguments(settings.studyFolder); + + return parser; +} + + +bool parseOptions(int argc, char* argv[], Settings& options) +{ + auto parser = createTsGeneratorParser(options); + switch (auto ret = parser->operator()(argc, argv); ret) + { + using namespace Yuni::GetOpt; + case ReturnCode::error: + logs.error() << "Unknown arguments, aborting"; + return false; + case ReturnCode::help: + return false; + default: + break; + } + return true; +} + +bool checkOptions(Settings& options) +{ + if (options.allThermal && !options.thermalListToGen.empty()) + { + logs.error() << "Conflicting options, either choose all thermal clusters or a list"; + return false; + } + + if (options.allLinks && !options.linksListToGen.empty()) + { + logs.error() << "Conflicting options, either choose all links or a list"; + return false; + } + return true; +} + +} \ No newline at end of file diff --git a/src/tools/ts-generator/tsGenerationOptions.h b/src/tools/ts-generator/tsGenerationOptions.h new file mode 100644 index 0000000000..4a565d5c52 --- /dev/null +++ b/src/tools/ts-generator/tsGenerationOptions.h @@ -0,0 +1,29 @@ + +#pragma once + +#include + +#include + +namespace Antares::TSGenerator +{ +struct Settings +{ + std::string studyFolder; + + /// generate TS for all clusters if activated + bool allThermal = false; + /// generate TS for a list "area.cluster;area2.cluster2;" + std::string thermalListToGen = ""; + + /// generate TS for all links if activated + bool allLinks = false; + /// generate TS for a list "area.link;area2.link2;" + std::string linksListToGen = ""; +}; + +bool parseOptions(int, char*[], Settings&); +std::unique_ptr createTsGeneratorParser(Settings&); + +bool checkOptions(Settings& options); +} From fcca4837cb15bc484696b4ca0646cfe63ee8bda8 Mon Sep 17 00:00:00 2001 From: Guillaume PIERRE Date: Fri, 14 Jun 2024 16:01:11 +0200 Subject: [PATCH 14/28] Split TS generation into files : start moving code for links TS to new source files --- src/tools/ts-generator/CMakeLists.txt | 1 + src/tools/ts-generator/linksTSgenerator.cpp | 19 +++++++++++++++++++ src/tools/ts-generator/linksTSgenerator.h | 13 ++++++++++--- src/tools/ts-generator/main.cpp | 7 +++++++ 4 files changed, 37 insertions(+), 3 deletions(-) create mode 100644 src/tools/ts-generator/linksTSgenerator.cpp diff --git a/src/tools/ts-generator/CMakeLists.txt b/src/tools/ts-generator/CMakeLists.txt index 9695be97e6..77549c3c82 100644 --- a/src/tools/ts-generator/CMakeLists.txt +++ b/src/tools/ts-generator/CMakeLists.txt @@ -3,6 +3,7 @@ set(SRCS linksTSgenerator.h tsGenerationOptions.h tsGenerationOptions.cpp + linksTSgenerator.cpp ) set(execname "antares-ts-generator") diff --git a/src/tools/ts-generator/linksTSgenerator.cpp b/src/tools/ts-generator/linksTSgenerator.cpp new file mode 100644 index 0000000000..07511b6906 --- /dev/null +++ b/src/tools/ts-generator/linksTSgenerator.cpp @@ -0,0 +1,19 @@ + +#include "linksTSgenerator.h" + +namespace Antares::TSGenerator { + +LinksTSgenerator::LinksTSgenerator(Settings & settings) : settings_(settings) +{ +} + +void LinksTSgenerator::extractData() +{ +} + +void LinksTSgenerator::generate() +{ +} + + +} \ No newline at end of file diff --git a/src/tools/ts-generator/linksTSgenerator.h b/src/tools/ts-generator/linksTSgenerator.h index 9af3f368fc..8b32f42622 100644 --- a/src/tools/ts-generator/linksTSgenerator.h +++ b/src/tools/ts-generator/linksTSgenerator.h @@ -1,11 +1,18 @@ #pragma once +#include "tsGenerationOptions.h" + +namespace Antares::TSGenerator { + class LinksTSgenerator { - void loadData(); +public: + LinksTSgenerator(Settings&); + void extractData(); void generate(); - - +private: + Settings settings_; }; +} // End Antares::TSGenerator diff --git a/src/tools/ts-generator/main.cpp b/src/tools/ts-generator/main.cpp index 002435006d..28d96543b7 100644 --- a/src/tools/ts-generator/main.cpp +++ b/src/tools/ts-generator/main.cpp @@ -36,6 +36,7 @@ #include #include "tsGenerationOptions.h" +#include "linksTSgenerator.h" using namespace Antares; using namespace Antares::TSGenerator; @@ -437,6 +438,12 @@ int main(int argc, char* argv[]) } // ============ LINKS : Getting data for generating LINKS time-series ===== + + LinksTSgenerator linksTSgenerator(settings); + linksTSgenerator.extractData(); + linksTSgenerator.generate(); + + auto allLinksPairs = extractLinkNamesFromStudy(settings.studyFolder); auto linksFromCmdLine = extractLinkNamesFromCmdLine(allLinksPairs, settings.linksListToGen); From 2a1fa53c37d029f91823a83fd4f83d7f38993c3b Mon Sep 17 00:00:00 2001 From: Guillaume PIERRE Date: Fri, 14 Jun 2024 18:20:36 +0200 Subject: [PATCH 15/28] Split TS generation into files : move code related to TS for links from main to other source files --- src/tools/ts-generator/linksTSgenerator.cpp | 333 ++++++++++++++++++- src/tools/ts-generator/linksTSgenerator.h | 17 +- src/tools/ts-generator/main.cpp | 327 +----------------- src/tools/ts-generator/tsGenerationOptions.h | 2 +- 4 files changed, 348 insertions(+), 331 deletions(-) diff --git a/src/tools/ts-generator/linksTSgenerator.cpp b/src/tools/ts-generator/linksTSgenerator.cpp index 07511b6906..0005ac51ea 100644 --- a/src/tools/ts-generator/linksTSgenerator.cpp +++ b/src/tools/ts-generator/linksTSgenerator.cpp @@ -1,18 +1,345 @@ #include "linksTSgenerator.h" +#include -namespace Antares::TSGenerator { +namespace Antares::TSGenerator +{ +// ================== +// Free functions +// ================== +std::vector extractTargetAreas(fs::path sourceLinkDir) +{ + std::vector to_return; + fs::path pathToIni = sourceLinkDir / "properties.ini"; + IniFile ini; + ini.open(pathToIni); // gp : we should handle reading issues + for (auto* s = ini.firstSection; s; s = s->next) + { + std::string targetAreaName = transformNameIntoID(s->name); + to_return.push_back(targetAreaName); + } + return to_return; +} + +bool pairs_match(const LinkPair& p1, const LinkPair& p2) +{ + return (p1.first == p2.first && p1.second == p2.second) + || (p1.first == p2.second && p1.second == p2.first); +} + +const LinkPair* getMatchingPairInCollection(const LinkPair& pair, const LinkPairs& collection) +{ + for(const auto& p : collection) + { + if (pairs_match(pair, p)) + return &p; + } + return nullptr; +} + +bool readLinkGeneralProperty(StudyParamsForLinkTS& params, + const Yuni::String& key, + const Yuni::String& value) +{ + if (key == "derated") + { + return value.to(params.derated); + } + if (key == "nbtimeserieslinks") + { + return value.to(params.nbLinkTStoGenerate); + } + if (key == "seed-tsgen-links") + { + unsigned int seed {0}; + if (! value.to(seed)) + return false; + params.random.reset(seed); + return true; + } + return true; // gp : should we return true here ? +} + +std::vector CreateLinkList(const LinkPairs& linksFromCmdLine) +{ + std::vector to_return(linksFromCmdLine.size()); + // gp : following loop should be improved : we shouldn't need an index + unsigned int index = 0; + for (const auto& link_pair : linksFromCmdLine) + { + to_return[index].namesPair = link_pair; + index++; + } + return to_return; +} + +LinkTSgenerationParams* findLinkInList(const LinkPair& link_to_find, + std::vector& linkList) +{ + for(auto& link : linkList) + { + if (link.namesPair == link_to_find) + return &link; + } + return nullptr; +} + +bool readLinkIniProperty(LinkTSgenerationParams* link, + const Yuni::String& key, + const Yuni::String& value) +{ + if (key == "unitcount") + { + return value.to(link->unitCount); + } + + if (key == "nominalcapacity") + { + return value.to(link->nominalCapacity); + } + + if (key == "law.planned") + { + return value.to(link->plannedLaw); + } + + if (key == "law.forced") + { + return value.to(link->forcedLaw); + } + + if (key == "volatility.planned") + { + return value.to(link->plannedVolatility); + } + + if (key == "volatility.forced") + { + return value.to(link->forcedVolatility); + } + + if (key == "force-no-generation") + { + return value.to(link->forceNoGeneration); + } + return true; +} + +void readLinkIniProperties(LinkTSgenerationParams* link, + IniFile::Section* section) +{ + for (const IniFile::Property* p = section->firstProperty; p; p = p->next) + { + if (! readLinkIniProperty(link, p->key, p->value)) + { + std::string linkName = link->namesPair.first + "." + link->namesPair.second; + logs.warning() << "Link '" << linkName << "' : reading value of '" + << p->key << "' went wrong"; + link->hasValidData = false; + } + } +} + +void readSourceAreaIniFile(fs::path pathToIni, + std::string sourceAreaName, + std::vector& linkList) +{ + IniFile ini; + ini.open(pathToIni); // gp : we should handle reading problems here + for (auto* section = ini.firstSection; section; section = section->next) + { + std::string targetAreaName = transformNameIntoID(section->name); + const LinkPair processedLink = std::make_pair(sourceAreaName, targetAreaName); + if (auto* foundLink = findLinkInList(processedLink, linkList); foundLink) + { + readLinkIniProperties(foundLink, section); + } + } +} + +void readIniProperties(std::vector& linkList, fs::path toLinksDir) +{ + for(auto& link : linkList) + { + std::string sourceAreaName = link.namesPair.first; + fs::path pathToIni = toLinksDir / sourceAreaName / "properties.ini"; + readSourceAreaIniFile(pathToIni, sourceAreaName, linkList); + } +} + + bool readLinkPreproTimeSeries(LinkTSgenerationParams& link, + fs::path sourceAreaDir) + { + bool to_return = true; + const auto preproId = link.namesPair.first + "/" + link.namesPair.second; + link.prepro = std::make_unique(preproId, link.unitCount); + + auto preproFileRoot = sourceAreaDir / "prepro" / link.namesPair.second; + + auto preproFile = preproFileRoot; + preproFile += ".txt"; + if (fs::exists(preproFile)) + { + to_return = link.prepro->data.loadFromCSVFile( + preproFile.string(), + Data::PreproAvailability::preproAvailabilityMax, + DAYS_PER_YEAR) + && link.prepro->validate() + && to_return; + } + + auto modulationFileDirect = preproFileRoot; + modulationFileDirect += "_mod_direct.txt"; + if (fs::exists(modulationFileDirect)) + { + to_return = link.modulationCapacityDirect.loadFromCSVFile( + modulationFileDirect.string(), + 1, + HOURS_PER_YEAR) + && to_return; + } -LinksTSgenerator::LinksTSgenerator(Settings & settings) : settings_(settings) + auto modulationFileIndirect = preproFileRoot; + modulationFileIndirect += "_mod_indirect.txt"; + if (fs::exists(modulationFileIndirect)) + { + to_return = link.modulationCapacityIndirect.loadFromCSVFile( + modulationFileIndirect.string(), + 1, + HOURS_PER_YEAR) + && to_return; + } + // Makes it possible to skip a link's TS generation when time comes + link.hasValidData = link.hasValidData && to_return; + return to_return; + } + +void readPreproTimeSeries(std::vector& linkList, + fs::path toLinksDir) +{ + for(auto& link : linkList) + { + std::string sourceAreaName = link.namesPair.first; + fs::path sourceAreaDir = toLinksDir / sourceAreaName; + if (! readLinkPreproTimeSeries(link, sourceAreaDir)) + { + logs.warning() << "Could not load all prepro data for link '" + << link.namesPair.first << "." << link.namesPair.second << "'"; + } + } +} + +std::string DateAndTime() +{ + YString to_return; + unsigned int now = Yuni::DateTime::Now(); + Yuni::DateTime::TimestampToString(to_return, "%Y%m%d-%H%M", now); + return to_return.to(); +} + +// ================== +// Class methods +// ================== +LinksTSgenerator::LinksTSgenerator(Settings & settings) : + studyFolder_(settings.studyFolder), + linksFromCmdLineOptions_(settings.linksListToGen), + generateTSforAllLinks_(settings.allLinks) { } void LinksTSgenerator::extractData() { + auto allLinksPairs = extractLinkNamesFromStudy(); + + LinkPairs namesLinksToGenerate; + if (generateTSforAllLinks_) + namesLinksToGenerate = allLinksPairs; + else + namesLinksToGenerate = extractLinkNamesFromCmdLine(allLinksPairs); + + linkList_ = CreateLinkList(namesLinksToGenerate); + extractLinksSpecificTSparameters(); + + generalParams_ = readGeneralParamsForLinksTS(); +} + +LinkPairs LinksTSgenerator::extractLinkNamesFromStudy() +{ + LinkPairs to_return; + fs::path linksDir = studyFolder_ / "input" / "links"; + for (auto const& item : fs::directory_iterator{linksDir}) + { + if (item.is_directory()) + { + std::string sourceAreaName = item.path().filename().generic_string(); + auto targetAreas = extractTargetAreas(item); + for (auto& targetAreaName : targetAreas) + { + auto linkPair = std::make_pair(sourceAreaName, targetAreaName); + to_return.push_back(linkPair); + } + } + } + return to_return; +} + +LinkPairs LinksTSgenerator::extractLinkNamesFromCmdLine(const LinkPairs& allLinks) +{ + LinkPairs to_return; + LinkPairs pairsFromCmdLine = splitStringIntoPairs(linksFromCmdLineOptions_, ';', '.'); + for (auto& p : pairsFromCmdLine) + { + if (const auto* found_pair = getMatchingPairInCollection(p, allLinks); found_pair) + { + to_return.push_back(*found_pair); + } + else + { + logs.error() << "Link '" << p.first << "." << p.second << "' not found"; + } + } + return to_return; } -void LinksTSgenerator::generate() +StudyParamsForLinkTS LinksTSgenerator::readGeneralParamsForLinksTS() { + StudyParamsForLinkTS to_return; + fs::path pathToGeneraldata = studyFolder_ / "settings" / "generaldata.ini"; + IniFile ini; + ini.open(pathToGeneraldata); // gp : we should handle reading issues + for (auto* section = ini.firstSection; section; section = section->next) + { + // Skipping sections useless in the current context + Yuni::String sectionName = section->name; + if (sectionName != "general" && sectionName != "seeds - Mersenne Twister") + continue; + + for (const IniFile::Property* p = section->firstProperty; p; p = p->next) + { + if (! readLinkGeneralProperty(to_return, p->key, p->value)) + { + logs.warning() << ini.filename() << ": reading value of '" + << p->key << "' went wrong"; + } + } + } + return to_return; +} + +void LinksTSgenerator::extractLinksSpecificTSparameters() +{ + fs::path toLinksDir = studyFolder_ / "input" / "links"; + readIniProperties(linkList_, toLinksDir); + readPreproTimeSeries(linkList_, toLinksDir); +} + +bool LinksTSgenerator::generate() +{ + auto saveTSpath = fs::path(studyFolder_) / "output" / DateAndTime(); + saveTSpath /= "ts-generator"; + saveTSpath /= "links"; + + return generateLinkTimeSeries(linkList_, generalParams_, saveTSpath.string()); } diff --git a/src/tools/ts-generator/linksTSgenerator.h b/src/tools/ts-generator/linksTSgenerator.h index 8b32f42622..54d33e165a 100644 --- a/src/tools/ts-generator/linksTSgenerator.h +++ b/src/tools/ts-generator/linksTSgenerator.h @@ -2,6 +2,9 @@ #pragma once #include "tsGenerationOptions.h" +#include + +namespace fs = std::filesystem; namespace Antares::TSGenerator { @@ -10,9 +13,19 @@ class LinksTSgenerator public: LinksTSgenerator(Settings&); void extractData(); - void generate(); + bool generate(); + private: - Settings settings_; + LinkPairs extractLinkNamesFromStudy(); + LinkPairs extractLinkNamesFromCmdLine(const LinkPairs&); + StudyParamsForLinkTS readGeneralParamsForLinksTS(); + void extractLinksSpecificTSparameters(); + + std::string linksFromCmdLineOptions_; + fs::path studyFolder_; + bool generateTSforAllLinks_ = false; + std::vector linkList_; + StudyParamsForLinkTS generalParams_; }; } // End Antares::TSGenerator diff --git a/src/tools/ts-generator/main.cpp b/src/tools/ts-generator/main.cpp index 28d96543b7..bfa74f381d 100644 --- a/src/tools/ts-generator/main.cpp +++ b/src/tools/ts-generator/main.cpp @@ -73,311 +73,6 @@ std::vector getClustersToGen(Data::AreaList& areas, return clusters; } -// ===== New code for TS generation links ==================================== - -std::vector extractTargetAreas(fs::path sourceLinkDir) -{ - std::vector to_return; - fs::path pathToIni = sourceLinkDir / "properties.ini"; - IniFile ini; - ini.open(pathToIni); // gp : we should handle reading issues - for (auto* s = ini.firstSection; s; s = s->next) - { - std::string targetAreaName = transformNameIntoID(s->name); - to_return.push_back(targetAreaName); - } - return to_return; -} - -LinkPairs extractLinkNamesFromStudy(fs::path studyDir) -{ - LinkPairs to_return; - fs::path linksDir = studyDir / "input" / "links"; - for (auto const& item : fs::directory_iterator{linksDir}) - { - if (item.is_directory()) - { - std::string sourceAreaName = item.path().filename().generic_string(); - auto targetAreas = extractTargetAreas(item); - for (auto& targetAreaName : targetAreas) - { - auto linkPair = std::make_pair(sourceAreaName, targetAreaName); - to_return.push_back(linkPair); - } - } - } - return to_return; -} - -bool pairs_match(const LinkPair& p1, const LinkPair& p2) -{ - return (p1.first == p2.first && p1.second == p2.second) - || (p1.first == p2.second && p1.second == p2.first); -} - -const LinkPair* getMatchingPairInCollection(const LinkPair& pair, const LinkPairs& collection) -{ - for(const auto& p : collection) - { - if (pairs_match(pair, p)) - return &p; - } - return nullptr; -} - -LinkPairs extractLinkNamesFromCmdLine(const LinkPairs& allLinks, - const std::string linksFromCmdLine) -{ - LinkPairs to_return; - LinkPairs pairsFromCmdLine = splitStringIntoPairs(linksFromCmdLine, ';', '.'); - for (auto& p : pairsFromCmdLine) - { - if (const auto* found_pair = getMatchingPairInCollection(p, allLinks); found_pair) - { - to_return.push_back(*found_pair); - } - else - { - logs.error() << "Link '" << p.first << "." << p.second << "' not found"; - } - } - return to_return; -} - -bool readLinkGeneralProperty(StudyParamsForLinkTS& params, - const Yuni::String& key, - const Yuni::String& value) -{ - if (key == "derated") - { - return value.to(params.derated); - } - if (key == "nbtimeserieslinks") - { - return value.to(params.nbLinkTStoGenerate); - } - if (key == "seed-tsgen-links") - { - unsigned int seed {0}; - if (! value.to(seed)) - return false; - params.random.reset(seed); - return true; - } - return true; // gp : should we return true here ? -} - -StudyParamsForLinkTS readGeneralParamsForLinksTS(fs::path studyDir) -{ - StudyParamsForLinkTS to_return; - fs::path pathToGeneraldata = studyDir / "settings" / "generaldata.ini"; - IniFile ini; - ini.open(pathToGeneraldata); // gp : we should handle reading issues - for (auto* section = ini.firstSection; section; section = section->next) - { - // Skipping sections useless in the current context - Yuni::String sectionName = section->name; - if (sectionName != "general" && sectionName != "seeds - Mersenne Twister") - continue; - - for (const IniFile::Property* p = section->firstProperty; p; p = p->next) - { - if (! readLinkGeneralProperty(to_return, p->key, p->value)) - { - logs.warning() << ini.filename() << ": reading value of '" - << p->key << "' went wrong"; - } - } - } - return to_return; -} - -std::vector CreateLinkList(const LinkPairs& linksFromCmdLine) -{ - std::vector to_return; - to_return.reserve(linksFromCmdLine.size()); - for (const auto& link_pair : linksFromCmdLine) - { - LinkTSgenerationParams params; - params.namesPair = link_pair; - to_return.push_back(std::move(params)); - } - return to_return; -} - -LinkTSgenerationParams* findLinkInList(const LinkPair& link_to_find, - std::vector& linkList) -{ - for(auto& link : linkList) - { - if (link.namesPair == link_to_find) - return &link; - } - return nullptr; -} - -bool readLinkIniProperty(LinkTSgenerationParams* link, - const Yuni::String& key, - const Yuni::String& value) -{ - if (key == "unitcount") - { - return value.to(link->unitCount); - } - - if (key == "nominalcapacity") - { - return value.to(link->nominalCapacity); - } - - if (key == "law.planned") - { - return value.to(link->plannedLaw); - } - - if (key == "law.forced") - { - return value.to(link->forcedLaw); - } - - if (key == "volatility.planned") - { - return value.to(link->plannedVolatility); - } - - if (key == "volatility.forced") - { - return value.to(link->forcedVolatility); - } - - if (key == "force-no-generation") - { - return value.to(link->forceNoGeneration); - } - return true; -} - -void readLinkIniProperties(LinkTSgenerationParams* link, - IniFile::Section* section) -{ - for (const IniFile::Property* p = section->firstProperty; p; p = p->next) - { - if (! readLinkIniProperty(link, p->key, p->value)) - { - std::string linkName = link->namesPair.first + "." + link->namesPair.second; - logs.warning() << "Link '" << linkName << "' : reading value of '" - << p->key << "' went wrong"; - link->hasValidData = false; - } - } -} - -void readSourceAreaIniFile(fs::path pathToIni, - std::string sourceAreaName, - std::vector& linkList) -{ - IniFile ini; - ini.open(pathToIni); // gp : we should handle reading problems here - for (auto* section = ini.firstSection; section; section = section->next) - { - std::string targetAreaName = transformNameIntoID(section->name); - const LinkPair processedLink = std::make_pair(sourceAreaName, targetAreaName); - if (auto* foundLink = findLinkInList(processedLink, linkList); foundLink) - { - readLinkIniProperties(foundLink, section); - } - } -} - -void readIniProperties(std::vector& linkList, fs::path toLinksDir) -{ - for(auto& link : linkList) - { - std::string sourceAreaName = link.namesPair.first; - fs::path pathToIni = toLinksDir / sourceAreaName / "properties.ini"; - readSourceAreaIniFile(pathToIni, sourceAreaName, linkList); - } -} - -bool readLinkPreproTimeSeries(LinkTSgenerationParams& link, - fs::path sourceAreaDir) -{ - bool to_return = true; - const auto preproId = link.namesPair.first + "/" + link.namesPair.second; - link.prepro = std::make_unique(preproId, link.unitCount); - - auto preproFileRoot = sourceAreaDir / "prepro" / link.namesPair.second; - - auto preproFile = preproFileRoot; - preproFile += ".txt"; - if (fs::exists(preproFile)) - { - to_return = link.prepro->data.loadFromCSVFile( - preproFile.string(), - Data::PreproAvailability::preproAvailabilityMax, - DAYS_PER_YEAR) - && link.prepro->validate() - && to_return; - } - - auto modulationFileDirect = preproFileRoot; - modulationFileDirect += "_mod_direct.txt"; - if (fs::exists(modulationFileDirect)) - { - to_return = link.modulationCapacityDirect.loadFromCSVFile( - modulationFileDirect.string(), - 1, - HOURS_PER_YEAR) - && to_return; - } - - auto modulationFileIndirect = preproFileRoot; - modulationFileIndirect += "_mod_indirect.txt"; - if (fs::exists(modulationFileIndirect)) - { - to_return = link.modulationCapacityIndirect.loadFromCSVFile( - modulationFileIndirect.string(), - 1, - HOURS_PER_YEAR) - && to_return; - } - // Makes it possible to skip a link's TS generation when time comes - link.hasValidData = link.hasValidData && to_return; - return to_return; -} - -void readPreproTimeSeries(std::vector& linkList, - fs::path toLinksDir) -{ - for(auto& link : linkList) - { - std::string sourceAreaName = link.namesPair.first; - fs::path sourceAreaDir = toLinksDir / sourceAreaName; - if (! readLinkPreproTimeSeries(link, sourceAreaDir)) - { - logs.warning() << "Could not load all prepro data for link '" - << link.namesPair.first << "." << link.namesPair.second << "'"; - } - } -} - -void readLinksSpecificTSparameters(std::vector& linkList, - fs::path studyFolder) -{ - fs::path toLinksDir = studyFolder / "input" / "links"; - readIniProperties(linkList, toLinksDir); - readPreproTimeSeries(linkList, toLinksDir); -} - -std::string DateAndTime() -{ - YString to_return; - unsigned int now = Yuni::DateTime::Now(); - Yuni::DateTime::TimestampToString(to_return, "%Y%m%d-%H%M", now); - return to_return.to(); -} -// ============================================================================ - int main(int argc, char* argv[]) { logs.applicationName("ts-generator"); @@ -437,27 +132,10 @@ int main(int argc, char* argv[]) logs.debug() << c->id(); } - // ============ LINKS : Getting data for generating LINKS time-series ===== + LinksTSgenerator linksTSgenerator(settings); linksTSgenerator.extractData(); - linksTSgenerator.generate(); - - - auto allLinksPairs = extractLinkNamesFromStudy(settings.studyFolder); - auto linksFromCmdLine = extractLinkNamesFromCmdLine(allLinksPairs, - settings.linksListToGen); - if (settings.allLinks) - linksFromCmdLine = allLinksPairs; - - StudyParamsForLinkTS generalParams = readGeneralParamsForLinksTS(settings.studyFolder); - - std::vector linkList = CreateLinkList(linksFromCmdLine); - readLinksSpecificTSparameters(linkList, settings.studyFolder); - - auto saveLinksTSpath = fs::path(settings.studyFolder) / "output" / DateAndTime(); - saveLinksTSpath /= "ts-generator"; - saveLinksTSpath /= "links"; // ============ TS Generation ============================================= @@ -466,8 +144,7 @@ int main(int argc, char* argv[]) *resultWriter, thermalSavePath.string()); - ret = TSGenerator::generateLinkTimeSeries(linkList, generalParams, saveLinksTSpath.string()) - && ret; + ret = linksTSgenerator.generate() && ret; return !ret; // return 0 for success } \ No newline at end of file diff --git a/src/tools/ts-generator/tsGenerationOptions.h b/src/tools/ts-generator/tsGenerationOptions.h index 4a565d5c52..dfc58257f5 100644 --- a/src/tools/ts-generator/tsGenerationOptions.h +++ b/src/tools/ts-generator/tsGenerationOptions.h @@ -19,7 +19,7 @@ struct Settings /// generate TS for all links if activated bool allLinks = false; /// generate TS for a list "area.link;area2.link2;" - std::string linksListToGen = ""; + std::string linksListToGen; }; bool parseOptions(int, char*[], Settings&); From 46cf3799382be436def4fddeb4e69cdb6040d0dc Mon Sep 17 00:00:00 2001 From: Guillaume PIERRE Date: Mon, 17 Jun 2024 15:21:49 +0200 Subject: [PATCH 16/28] Split TS generation into files : removing useless header include and namespace declaration --- src/tools/ts-generator/main.cpp | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/tools/ts-generator/main.cpp b/src/tools/ts-generator/main.cpp index bfa74f381d..23881d5caf 100644 --- a/src/tools/ts-generator/main.cpp +++ b/src/tools/ts-generator/main.cpp @@ -22,14 +22,11 @@ #include #include -#include - #include #include #include #include #include -#include #include #include #include @@ -38,7 +35,6 @@ #include "tsGenerationOptions.h" #include "linksTSgenerator.h" -using namespace Antares; using namespace Antares::TSGenerator; namespace fs = std::filesystem; From 8d03e1eeead0f238ed7070114e7619b27ab529b9 Mon Sep 17 00:00:00 2001 From: Guillaume PIERRE Date: Mon, 24 Jun 2024 15:20:26 +0200 Subject: [PATCH 17/28] Split TS generation into files : simplifying a loop --- src/tools/ts-generator/linksTSgenerator.cpp | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/src/tools/ts-generator/linksTSgenerator.cpp b/src/tools/ts-generator/linksTSgenerator.cpp index 0005ac51ea..fcacaab409 100644 --- a/src/tools/ts-generator/linksTSgenerator.cpp +++ b/src/tools/ts-generator/linksTSgenerator.cpp @@ -63,12 +63,10 @@ bool readLinkGeneralProperty(StudyParamsForLinkTS& params, std::vector CreateLinkList(const LinkPairs& linksFromCmdLine) { std::vector to_return(linksFromCmdLine.size()); - // gp : following loop should be improved : we shouldn't need an index - unsigned int index = 0; - for (const auto& link_pair : linksFromCmdLine) + for (auto link = to_return.begin(); const auto& link_pair : linksFromCmdLine) { - to_return[index].namesPair = link_pair; - index++; + link->namesPair = link_pair; + link++; } return to_return; } From c60b905296e9817ebf9cf0bb12734b49bf33914c Mon Sep 17 00:00:00 2001 From: Guillaume PIERRE Date: Tue, 25 Jun 2024 11:00:56 +0200 Subject: [PATCH 18/28] Split TS generation into files : move header files to the right place --- src/tools/ts-generator/CMakeLists.txt | 14 +++++++------- .../antares/tools/ts-generator}/linksTSgenerator.h | 2 +- .../tools/ts-generator}/tsGenerationOptions.h | 0 src/tools/ts-generator/linksTSgenerator.cpp | 2 +- src/tools/ts-generator/main.cpp | 4 ++-- src/tools/ts-generator/tsGenerationOptions.cpp | 2 +- 6 files changed, 12 insertions(+), 12 deletions(-) rename src/tools/ts-generator/{ => include/antares/tools/ts-generator}/linksTSgenerator.h (92%) rename src/tools/ts-generator/{ => include/antares/tools/ts-generator}/tsGenerationOptions.h (100%) diff --git a/src/tools/ts-generator/CMakeLists.txt b/src/tools/ts-generator/CMakeLists.txt index 77549c3c82..f037e6f1dd 100644 --- a/src/tools/ts-generator/CMakeLists.txt +++ b/src/tools/ts-generator/CMakeLists.txt @@ -1,7 +1,7 @@ set(SRCS main.cpp - linksTSgenerator.h - tsGenerationOptions.h + include/antares/tools/ts-generator/linksTSgenerator.h + include/antares/tools/ts-generator/tsGenerationOptions.h tsGenerationOptions.cpp linksTSgenerator.cpp ) @@ -17,15 +17,15 @@ INSTALL(EXPORT ${execname} target_link_libraries(${execname} PRIVATE - Antares::utils - antares-solver-ts-generator - Antares::study - Antares::checks + Antares::utils + antares-solver-ts-generator + Antares::study + Antares::checks ) target_include_directories(${execname} PRIVATE - "${CMAKE_CURRENT_SOURCE_DIR}/include" + "${CMAKE_CURRENT_SOURCE_DIR}/include" ) import_std_libs(${execname}) diff --git a/src/tools/ts-generator/linksTSgenerator.h b/src/tools/ts-generator/include/antares/tools/ts-generator/linksTSgenerator.h similarity index 92% rename from src/tools/ts-generator/linksTSgenerator.h rename to src/tools/ts-generator/include/antares/tools/ts-generator/linksTSgenerator.h index 54d33e165a..517d99a633 100644 --- a/src/tools/ts-generator/linksTSgenerator.h +++ b/src/tools/ts-generator/include/antares/tools/ts-generator/linksTSgenerator.h @@ -1,7 +1,7 @@ #pragma once -#include "tsGenerationOptions.h" +#include "antares/tools/ts-generator/tsGenerationOptions.h" #include namespace fs = std::filesystem; diff --git a/src/tools/ts-generator/tsGenerationOptions.h b/src/tools/ts-generator/include/antares/tools/ts-generator/tsGenerationOptions.h similarity index 100% rename from src/tools/ts-generator/tsGenerationOptions.h rename to src/tools/ts-generator/include/antares/tools/ts-generator/tsGenerationOptions.h diff --git a/src/tools/ts-generator/linksTSgenerator.cpp b/src/tools/ts-generator/linksTSgenerator.cpp index fcacaab409..1fa6ea49b7 100644 --- a/src/tools/ts-generator/linksTSgenerator.cpp +++ b/src/tools/ts-generator/linksTSgenerator.cpp @@ -1,5 +1,5 @@ -#include "linksTSgenerator.h" +#include "antares/tools/ts-generator/linksTSgenerator.h" #include namespace Antares::TSGenerator diff --git a/src/tools/ts-generator/main.cpp b/src/tools/ts-generator/main.cpp index 23881d5caf..fcb3b29061 100644 --- a/src/tools/ts-generator/main.cpp +++ b/src/tools/ts-generator/main.cpp @@ -32,8 +32,8 @@ #include #include -#include "tsGenerationOptions.h" -#include "linksTSgenerator.h" +#include "antares/tools/ts-generator/tsGenerationOptions.h" +#include "antares/tools/ts-generator/linksTSgenerator.h" using namespace Antares::TSGenerator; diff --git a/src/tools/ts-generator/tsGenerationOptions.cpp b/src/tools/ts-generator/tsGenerationOptions.cpp index d80d73448c..16750a2173 100644 --- a/src/tools/ts-generator/tsGenerationOptions.cpp +++ b/src/tools/ts-generator/tsGenerationOptions.cpp @@ -1,4 +1,4 @@ -#include "tsGenerationOptions.h" +#include "antares/tools/ts-generator/tsGenerationOptions.h" #include namespace Antares::TSGenerator { From ebdef473463ae855e08e34a9ccc8887f9f5b088f Mon Sep 17 00:00:00 2001 From: Guillaume PIERRE Date: Tue, 25 Jun 2024 15:24:32 +0200 Subject: [PATCH 19/28] Split TS generation into files : taking code review into account --- src/tools/ts-generator/linksTSgenerator.cpp | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/src/tools/ts-generator/linksTSgenerator.cpp b/src/tools/ts-generator/linksTSgenerator.cpp index 1fa6ea49b7..8385e104c7 100644 --- a/src/tools/ts-generator/linksTSgenerator.cpp +++ b/src/tools/ts-generator/linksTSgenerator.cpp @@ -7,16 +7,15 @@ namespace Antares::TSGenerator // ================== // Free functions // ================== -std::vector extractTargetAreas(fs::path sourceLinkDir) +std::vector extractTargetAreas(const fs::path& sourceLinkDir) { std::vector to_return; fs::path pathToIni = sourceLinkDir / "properties.ini"; IniFile ini; ini.open(pathToIni); // gp : we should handle reading issues - for (auto* s = ini.firstSection; s; s = s->next) + for (auto s = ini.firstSection; s; s = s->next) { - std::string targetAreaName = transformNameIntoID(s->name); - to_return.push_back(targetAreaName); + to_return.push_back(transformNameIntoID(s->name)); } return to_return; } @@ -62,12 +61,14 @@ bool readLinkGeneralProperty(StudyParamsForLinkTS& params, std::vector CreateLinkList(const LinkPairs& linksFromCmdLine) { - std::vector to_return(linksFromCmdLine.size()); - for (auto link = to_return.begin(); const auto& link_pair : linksFromCmdLine) - { - link->namesPair = link_pair; - link++; - } + std::vector to_return; + std::for_each(linksFromCmdLine.begin(), linksFromCmdLine.end(), + [&to_return](const auto& link_pair) + { + LinkTSgenerationParams link; + link.namesPair = link_pair; + to_return.push_back(std::move(link)); + }); return to_return; } From b2cc58eda9da3fc360ac1c37ff8643c8df42136b Mon Sep 17 00:00:00 2001 From: Guillaume PIERRE Date: Tue, 25 Jun 2024 16:20:28 +0200 Subject: [PATCH 20/28] Split TS generation into files : remove code duplication about time --- .../antares/utils/include/antares/utils/utils.h | 6 ++---- src/libs/antares/utils/utils.cpp | 14 ++++++++++++++ src/solver/application/application.cpp | 16 +--------------- src/tools/ts-generator/linksTSgenerator.cpp | 13 ++----------- 4 files changed, 19 insertions(+), 30 deletions(-) diff --git a/src/libs/antares/utils/include/antares/utils/utils.h b/src/libs/antares/utils/include/antares/utils/utils.h index 1043e7ca4e..a2f50a2e54 100644 --- a/src/libs/antares/utils/include/antares/utils/utils.h +++ b/src/libs/antares/utils/include/antares/utils/utils.h @@ -36,9 +36,10 @@ namespace Antares */ template void TransformNameIntoID(const AnyString& name, StringT& out); - std::string transformNameIntoID(const std::string& name); +std::string FormattedTime(const std::string& format); + /*! ** \brief Beautify a name, for renaming an area for example */ @@ -51,11 +52,8 @@ std::vector> splitStringIntoPairs(const std: namespace Utils { - bool isZero(double d); - double round(double d, unsigned precision); - } // namespace Utils } // namespace Antares diff --git a/src/libs/antares/utils/utils.cpp b/src/libs/antares/utils/utils.cpp index 047881b0e6..de69fddf09 100644 --- a/src/libs/antares/utils/utils.cpp +++ b/src/libs/antares/utils/utils.cpp @@ -96,6 +96,20 @@ void BeautifyName(std::string& out, const std::string& oldname) out = yuniOut.c_str(); } +std::string FormattedTime(const std::string& format) +{ + using namespace std::chrono; + auto time = system_clock::to_time_t(system_clock::now()); + std::tm local_time = *std::localtime(&time); + + char time_buffer[256]; + std::strftime(time_buffer, sizeof(time_buffer), format.c_str(), &local_time); + + std::string currentTime = time_buffer; + + return currentTime; +} + std::vector> splitStringIntoPairs(const std::string& s, char delimiter1, char delimiter2) diff --git a/src/solver/application/application.cpp b/src/solver/application/application.cpp index d222ad88c0..dc7aca49cd 100644 --- a/src/solver/application/application.cpp +++ b/src/solver/application/application.cpp @@ -426,20 +426,6 @@ void Application::runSimulationInAdequacyMode() pOptimizationInfo); } -static std::string timeToString() -{ - using namespace std::chrono; - auto time = system_clock::to_time_t(system_clock::now()); - std::tm local_time = *std::localtime(&time); - - char time_buffer[256]; - std::strftime(time_buffer, sizeof(time_buffer), "%Y%m%d-%H%M%S", &local_time); - - std::string currentTime = time_buffer; - - return currentTime; -} - void Application::resetLogFilename() const { fs::path logfile = fs::path(pSettings.studyFolder.c_str()) / "logs"; @@ -451,7 +437,7 @@ void Application::resetLogFilename() const } logfile /= "solver-"; // append the filename - logfile += timeToString() + ".log"; // complete filename with timestamp and extension + logfile += FormattedTime("%Y%m%d-%H%M%S") + ".log"; // complete filename with timestamp and extension // Assigning the log filename logs.logfile(logfile.string()); diff --git a/src/tools/ts-generator/linksTSgenerator.cpp b/src/tools/ts-generator/linksTSgenerator.cpp index 8385e104c7..b87351f4b4 100644 --- a/src/tools/ts-generator/linksTSgenerator.cpp +++ b/src/tools/ts-generator/linksTSgenerator.cpp @@ -1,6 +1,6 @@ #include "antares/tools/ts-generator/linksTSgenerator.h" -#include +#include "antares/utils/utils.h" namespace Antares::TSGenerator { @@ -228,14 +228,6 @@ void readPreproTimeSeries(std::vector& linkList, } } -std::string DateAndTime() -{ - YString to_return; - unsigned int now = Yuni::DateTime::Now(); - Yuni::DateTime::TimestampToString(to_return, "%Y%m%d-%H%M", now); - return to_return.to(); -} - // ================== // Class methods // ================== @@ -334,12 +326,11 @@ void LinksTSgenerator::extractLinksSpecificTSparameters() bool LinksTSgenerator::generate() { - auto saveTSpath = fs::path(studyFolder_) / "output" / DateAndTime(); + auto saveTSpath = fs::path(studyFolder_) / "output" / FormattedTime("%Y%m%d-%H%M"); saveTSpath /= "ts-generator"; saveTSpath /= "links"; return generateLinkTimeSeries(linkList_, generalParams_, saveTSpath.string()); } - } \ No newline at end of file From 2710aa47b6c0ac14d601dd1ea57e463cf95d1380 Mon Sep 17 00:00:00 2001 From: Guillaume PIERRE Date: Tue, 25 Jun 2024 16:54:40 +0200 Subject: [PATCH 21/28] Split TS generation into files : renaming --- src/solver/ts-generator/availability.cpp | 56 +++++++++++---------- src/tools/ts-generator/linksTSgenerator.cpp | 2 +- 2 files changed, 31 insertions(+), 27 deletions(-) diff --git a/src/solver/ts-generator/availability.cpp b/src/solver/ts-generator/availability.cpp index 696725f846..0c4c83fd32 100644 --- a/src/solver/ts-generator/availability.cpp +++ b/src/solver/ts-generator/availability.cpp @@ -70,13 +70,13 @@ AvailabilityTSGeneratorData::AvailabilityTSGeneratorData(LinkTSgenerationParams& namespace { -class GeneratorTempData final +class AvailabilityTSgenerator final { public: - explicit GeneratorTempData(Data::Study&, unsigned, MersenneTwister&); - explicit GeneratorTempData(bool, unsigned, MersenneTwister&); + explicit AvailabilityTSgenerator(Data::Study&, unsigned, MersenneTwister&); + explicit AvailabilityTSgenerator(bool, unsigned, MersenneTwister&); - void generateTS(AvailabilityTSGeneratorData&) const; + void run(AvailabilityTSGeneratorData&) const; private: bool derated; @@ -101,14 +101,18 @@ class GeneratorTempData final const T& duration) const; }; -GeneratorTempData::GeneratorTempData(Data::Study& study, unsigned nbOfSeriesToGen, MersenneTwister& rndGenerator): +AvailabilityTSgenerator::AvailabilityTSgenerator(Data::Study& study, + unsigned nbOfSeriesToGen, + MersenneTwister& rndGenerator): derated(study.parameters.derated), nbOfSeriesToGen_(nbOfSeriesToGen), rndgenerator(rndGenerator) { } -GeneratorTempData::GeneratorTempData(bool derated, unsigned int nbOfSeriesToGen, MersenneTwister& rndGenerator): +AvailabilityTSgenerator::AvailabilityTSgenerator(bool derated, + unsigned int nbOfSeriesToGen, + MersenneTwister& rndGenerator): derated(derated), nbOfSeriesToGen_(nbOfSeriesToGen), rndgenerator(rndGenerator) @@ -116,11 +120,11 @@ GeneratorTempData::GeneratorTempData(bool derated, unsigned int nbOfSeriesToGen, } template -void GeneratorTempData::prepareIndispoFromLaw(Data::StatisticalLaw law, - double volatility, - std::array& A, - std::array& B, - const T& duration) const +void AvailabilityTSgenerator::prepareIndispoFromLaw(Data::StatisticalLaw law, + double volatility, + std::array& A, + std::array& B, + const T& duration) const { switch (law) { @@ -161,11 +165,11 @@ void GeneratorTempData::prepareIndispoFromLaw(Data::StatisticalLaw law, } } -int GeneratorTempData::durationGenerator(Data::StatisticalLaw law, - int expec, - double volat, - double a, - double b) const +int AvailabilityTSgenerator::durationGenerator(Data::StatisticalLaw law, + int expec, + double volat, + double a, + double b) const { if (volat == 0 || expec == 1) { @@ -193,7 +197,7 @@ int GeneratorTempData::durationGenerator(Data::StatisticalLaw law, return 0; } -void GeneratorTempData::generateTS(AvailabilityTSGeneratorData& tsGenerationData) const +void AvailabilityTSgenerator::run(AvailabilityTSGeneratorData& tsGenerationData) const { assert(tsGenerationData.prepro); @@ -647,14 +651,14 @@ bool generateThermalTimeSeries(Data::Study& study, bool archive = study.parameters.timeSeriesToArchive & Data::timeSeriesThermal; - auto generator = GeneratorTempData(study, - study.parameters.nbTimeSeriesThermal, - study.runtime->random[Data::seedTsGenThermal]); + auto generator = AvailabilityTSgenerator(study, + study.parameters.nbTimeSeriesThermal, + study.runtime->random[Data::seedTsGenThermal]); for (auto* cluster: clusters) { AvailabilityTSGeneratorData tsGenerationData(cluster); - generator.generateTS(tsGenerationData); + generator.run(tsGenerationData); if (archive) // For compatibilty with in memory thermal TS generation { @@ -675,9 +679,9 @@ bool generateLinkTimeSeries(std::vector& links, logs.info(); logs.info() << "Generation of links time-series"; - auto generator = GeneratorTempData(generalParams.derated, - generalParams.nbLinkTStoGenerate, - generalParams.random); + auto generator = AvailabilityTSgenerator(generalParams.derated, + generalParams.nbLinkTStoGenerate, + generalParams.random); for (auto& link: links) { if (! link.hasValidData) @@ -696,7 +700,7 @@ bool generateLinkTimeSeries(std::vector& links, // DIRECT AvailabilityTSGeneratorData tsConfigDataDirect(link, ts, link.modulationCapacityDirect, link.namesPair.second); - generator.generateTS(tsConfigDataDirect); + generator.run(tsConfigDataDirect); std::string filePath = savePath + SEP + link.namesPair.first + SEP + link.namesPair.second + "_direct.txt"; @@ -705,7 +709,7 @@ bool generateLinkTimeSeries(std::vector& links, // INDIRECT AvailabilityTSGeneratorData tsConfigDataIndirect(link, ts, link.modulationCapacityIndirect, link.namesPair.second); - generator.generateTS(tsConfigDataIndirect); + generator.run(tsConfigDataIndirect); filePath = savePath + SEP + link.namesPair.first + SEP + link.namesPair.second + "_indirect.txt"; diff --git a/src/tools/ts-generator/linksTSgenerator.cpp b/src/tools/ts-generator/linksTSgenerator.cpp index b87351f4b4..87d1e94aad 100644 --- a/src/tools/ts-generator/linksTSgenerator.cpp +++ b/src/tools/ts-generator/linksTSgenerator.cpp @@ -56,7 +56,7 @@ bool readLinkGeneralProperty(StudyParamsForLinkTS& params, params.random.reset(seed); return true; } - return true; // gp : should we return true here ? + return true; } std::vector CreateLinkList(const LinkPairs& linksFromCmdLine) From f842001c39e44666a1b106e7c88c4f9533cde9c9 Mon Sep 17 00:00:00 2001 From: Guillaume PIERRE Date: Tue, 25 Jun 2024 17:32:33 +0200 Subject: [PATCH 22/28] Split TS generation into files : simplifying and renaming function writeResultsToDisk which now does exactly what its name says it does --- src/solver/ts-generator/availability.cpp | 35 +++++++++++------------- 1 file changed, 16 insertions(+), 19 deletions(-) diff --git a/src/solver/ts-generator/availability.cpp b/src/solver/ts-generator/availability.cpp index 0c4c83fd32..9e9e5000aa 100644 --- a/src/solver/ts-generator/availability.cpp +++ b/src/solver/ts-generator/availability.cpp @@ -612,23 +612,17 @@ std::vector getAllClustersToGen(const Data::AreaList& are return clusters; } -void writeResultsToDisk(const Data::Study& study, - Solver::IResultWriter& writer, - const Matrix<>& series, - const std::string& savePath) +void writeTStoDisk(Solver::IResultWriter& writer, + const Matrix<>& series, + const std::string& savePath) { - if (study.parameters.noOutput) - { - return; - } - std::string buffer; series.saveToBuffer(buffer, 0); writer.addEntryFromBuffer(savePath, buffer); } -void writeResultsToDisk(const Matrix<>& series, - const std::filesystem::path savePath) +void writeTStoDisk(const Matrix<>& series, + const std::filesystem::path savePath) { std::string buffer; series.saveToBuffer(buffer, 0); @@ -650,6 +644,12 @@ bool generateThermalTimeSeries(Data::Study& study, logs.info() << "Generating the thermal time-series"; bool archive = study.parameters.timeSeriesToArchive & Data::timeSeriesThermal; + bool doWeWrite = archive && !study.parameters.noOutput; + if (! doWeWrite) + { + logs.info() << "Study parammeters such that we don't write thermal TS."; + return true; + } auto generator = AvailabilityTSgenerator(study, study.parameters.nbTimeSeriesThermal, @@ -660,12 +660,9 @@ bool generateThermalTimeSeries(Data::Study& study, AvailabilityTSGeneratorData tsGenerationData(cluster); generator.run(tsGenerationData); - if (archive) // For compatibilty with in memory thermal TS generation - { - std::string filePath = savePath + SEP + cluster->parentArea->id + SEP + cluster->id() - + ".txt"; - writeResultsToDisk(study, writer, cluster->series.timeSeries, filePath); - } + std::string filePath = savePath + SEP + cluster->parentArea->id + SEP + cluster->id() + + ".txt"; + writeTStoDisk(writer, cluster->series.timeSeries, filePath); } return true; @@ -704,7 +701,7 @@ bool generateLinkTimeSeries(std::vector& links, std::string filePath = savePath + SEP + link.namesPair.first + SEP + link.namesPair.second + "_direct.txt"; - writeResultsToDisk(ts.timeSeries, filePath); + writeTStoDisk(ts.timeSeries, filePath); // INDIRECT AvailabilityTSGeneratorData tsConfigDataIndirect(link, ts, link.modulationCapacityIndirect, link.namesPair.second); @@ -713,7 +710,7 @@ bool generateLinkTimeSeries(std::vector& links, filePath = savePath + SEP + link.namesPair.first + SEP + link.namesPair.second + "_indirect.txt"; - writeResultsToDisk(ts.timeSeries, filePath); + writeTStoDisk(ts.timeSeries, filePath); } return true; From 82d9bb0b6c39cbbda592275b4233627c84e40945 Mon Sep 17 00:00:00 2001 From: Guillaume PIERRE Date: Wed, 26 Jun 2024 10:23:10 +0200 Subject: [PATCH 23/28] Split TS generation into files : correction for a regression on CI --- src/solver/ts-generator/availability.cpp | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/src/solver/ts-generator/availability.cpp b/src/solver/ts-generator/availability.cpp index 9e9e5000aa..a1761c402a 100644 --- a/src/solver/ts-generator/availability.cpp +++ b/src/solver/ts-generator/availability.cpp @@ -643,6 +643,16 @@ bool generateThermalTimeSeries(Data::Study& study, logs.info(); logs.info() << "Generating the thermal time-series"; + auto generator = AvailabilityTSgenerator(study, + study.parameters.nbTimeSeriesThermal, + study.runtime->random[Data::seedTsGenThermal]); + + for (auto* cluster: clusters) + { + AvailabilityTSGeneratorData tsGenerationData(cluster); + generator.run(tsGenerationData); + } + bool archive = study.parameters.timeSeriesToArchive & Data::timeSeriesThermal; bool doWeWrite = archive && !study.parameters.noOutput; if (! doWeWrite) @@ -651,15 +661,8 @@ bool generateThermalTimeSeries(Data::Study& study, return true; } - auto generator = AvailabilityTSgenerator(study, - study.parameters.nbTimeSeriesThermal, - study.runtime->random[Data::seedTsGenThermal]); - for (auto* cluster: clusters) { - AvailabilityTSGeneratorData tsGenerationData(cluster); - generator.run(tsGenerationData); - std::string filePath = savePath + SEP + cluster->parentArea->id + SEP + cluster->id() + ".txt"; writeTStoDisk(writer, cluster->series.timeSeries, filePath); From 4961cebec1e144ccee06c74bac9f7cb68874625c Mon Sep 17 00:00:00 2001 From: guilpier-code <62292552+guilpier-code@users.noreply.github.com> Date: Fri, 28 Jun 2024 10:46:09 +0200 Subject: [PATCH 24/28] TS : generator : make it work for thermal (#2200) This PR addresses [this go-pro ticket](https://gopro-tickets.rte-france.com/browse/ANT-1852) What was done : - **Cause of the crash** : allocate (give a size to) the storage containers supposed to receive the thermal TS generation results - Remove the use of **result writer**, too complicated (for example, it forces us to use a duration collector we don't need). - Use of **std::filesystem** whenever possible See helping **gitHub** comments inserted in sources. --- .../antares/solver/simulation/solver.hxx | 9 ++++---- src/solver/ts-generator/availability.cpp | 22 +++++++------------ .../antares/solver/ts-generator/generator.h | 1 - src/tools/ts-generator/main.cpp | 12 +++------- 4 files changed, 15 insertions(+), 29 deletions(-) diff --git a/src/solver/simulation/include/antares/solver/simulation/solver.hxx b/src/solver/simulation/include/antares/solver/simulation/solver.hxx index 232fa520fc..dc7ff6ecad 100644 --- a/src/solver/simulation/include/antares/solver/simulation/solver.hxx +++ b/src/solver/simulation/include/antares/solver/simulation/solver.hxx @@ -41,6 +41,7 @@ #include "hydro-final-reservoir-level-functions.h" +namespace fs = std::filesystem; namespace Antares::Solver::Simulation { @@ -488,11 +489,9 @@ void ISimulation::regenerateTimeSeries(uint year) if (refreshTSonCurrentYear) { auto clusters = getAllClustersToGen(study.areas, pData.haveToRefreshTSThermal); -#define SEP Yuni::IO::Separator - const std::string savePath = std::string("ts-generator") + SEP + "thermal" + SEP + "mc-" - + std::to_string(year); -#undef SEP - generateThermalTimeSeries(study, clusters, pResultWriter, savePath); + fs::path savePath = fs::path(study.folderOutput.to()) / "ts-generator" + / "thermal" / "mc-" / std::to_string(year); + generateThermalTimeSeries(study, clusters, savePath.string()); // apply the spinning if we generated some in memory clusters for (auto* cluster: clusters) diff --git a/src/solver/ts-generator/availability.cpp b/src/solver/ts-generator/availability.cpp index a1761c402a..c51b006dde 100644 --- a/src/solver/ts-generator/availability.cpp +++ b/src/solver/ts-generator/availability.cpp @@ -32,6 +32,7 @@ #include "antares/study/simulation.h" #define SEP Yuni::IO::Separator +namespace fs = std::filesystem; constexpr double FAILURE_RATE_EQ_1 = 0.999; @@ -612,15 +613,6 @@ std::vector getAllClustersToGen(const Data::AreaList& are return clusters; } -void writeTStoDisk(Solver::IResultWriter& writer, - const Matrix<>& series, - const std::string& savePath) -{ - std::string buffer; - series.saveToBuffer(buffer, 0); - writer.addEntryFromBuffer(savePath, buffer); -} - void writeTStoDisk(const Matrix<>& series, const std::filesystem::path savePath) { @@ -637,7 +629,6 @@ void writeTStoDisk(const Matrix<>& series, bool generateThermalTimeSeries(Data::Study& study, const std::vector& clusters, - Solver::IResultWriter& writer, const std::string& savePath) { logs.info(); @@ -649,6 +640,7 @@ bool generateThermalTimeSeries(Data::Study& study, for (auto* cluster: clusters) { + cluster->series.timeSeries.reset(study.parameters.nbTimeSeriesThermal, HOURS_PER_YEAR); AvailabilityTSGeneratorData tsGenerationData(cluster); generator.run(tsGenerationData); } @@ -657,15 +649,17 @@ bool generateThermalTimeSeries(Data::Study& study, bool doWeWrite = archive && !study.parameters.noOutput; if (! doWeWrite) { - logs.info() << "Study parammeters such that we don't write thermal TS."; + logs.info() << "Study parameters forbid writing thermal TS."; return true; } for (auto* cluster: clusters) { - std::string filePath = savePath + SEP + cluster->parentArea->id + SEP + cluster->id() - + ".txt"; - writeTStoDisk(writer, cluster->series.timeSeries, filePath); + auto areaName = cluster->parentArea->id.to(); + auto clusterName = cluster->id(); + auto filePath = fs::path(savePath) / areaName / clusterName += ".txt"; + + writeTStoDisk(cluster->series.timeSeries, filePath.string()); } return true; diff --git a/src/solver/ts-generator/include/antares/solver/ts-generator/generator.h b/src/solver/ts-generator/include/antares/solver/ts-generator/generator.h index 8cddc50679..75255206f0 100644 --- a/src/solver/ts-generator/include/antares/solver/ts-generator/generator.h +++ b/src/solver/ts-generator/include/antares/solver/ts-generator/generator.h @@ -111,7 +111,6 @@ bool GenerateTimeSeries(Data::Study& study, uint year, IResultWriter& writer); bool generateThermalTimeSeries(Data::Study& study, const std::vector& clusters, - Solver::IResultWriter& writer, const std::string& savePath); bool generateLinkTimeSeries(std::vector& links, diff --git a/src/tools/ts-generator/main.cpp b/src/tools/ts-generator/main.cpp index fcb3b29061..27caf21556 100644 --- a/src/tools/ts-generator/main.cpp +++ b/src/tools/ts-generator/main.cpp @@ -103,14 +103,9 @@ int main(int argc, char* argv[]) Antares::logs.error() << ex.what(); } - Benchmarking::DurationCollector durationCollector; - - auto resultWriter = Solver::resultWriterFactory(Data::ResultFormat::legacyFilesDirectories, - study->folderOutput, - nullptr, - durationCollector); - - const auto thermalSavePath = fs::path("ts-generator") / "thermal"; + auto thermalSavePath = fs::path(settings.studyFolder) / "output" / FormattedTime("%Y%m%d-%H%M"); + thermalSavePath /= "ts-generator"; + thermalSavePath /= "thermal"; // ============ THERMAL : Getting data for generating time-series ========= std::vector clusters; @@ -137,7 +132,6 @@ int main(int argc, char* argv[]) bool ret = TSGenerator::generateThermalTimeSeries(*study, clusters, - *resultWriter, thermalSavePath.string()); ret = linksTSgenerator.generate() && ret; From 2abab724ef182452b056db109f71823695e190cd Mon Sep 17 00:00:00 2001 From: Guillaume PIERRE Date: Fri, 28 Jun 2024 13:12:12 +0200 Subject: [PATCH 25/28] Split TS generation into files : correction after merge --- src/solver/ts-generator/availability.cpp | 15 --- src/tools/ts-generator/main.cpp | 162 ----------------------- 2 files changed, 177 deletions(-) diff --git a/src/solver/ts-generator/availability.cpp b/src/solver/ts-generator/availability.cpp index 938d395089..c51b006dde 100644 --- a/src/solver/ts-generator/availability.cpp +++ b/src/solver/ts-generator/availability.cpp @@ -76,7 +76,6 @@ class AvailabilityTSgenerator final public: explicit AvailabilityTSgenerator(Data::Study&, unsigned, MersenneTwister&); explicit AvailabilityTSgenerator(bool, unsigned, MersenneTwister&); - explicit GeneratorTempData(bool, unsigned, MersenneTwister&); void run(AvailabilityTSGeneratorData&) const; @@ -628,20 +627,6 @@ void writeTStoDisk(const Matrix<>& series, Antares::IO::fileSetContent(savePath.string(), buffer); } -void writeResultsToDisk(const Matrix<>& series, - const std::filesystem::path savePath) -{ - std::string buffer; - series.saveToBuffer(buffer, 0); - - std::filesystem::path parentDir = savePath.parent_path(); - if (! std::filesystem::exists(parentDir)) - { - std::filesystem::create_directories(parentDir); - } - Antares::IO::fileSetContent(savePath.string(), buffer); -} - bool generateThermalTimeSeries(Data::Study& study, const std::vector& clusters, const std::string& savePath) diff --git a/src/tools/ts-generator/main.cpp b/src/tools/ts-generator/main.cpp index 3910ebfa5b..74040e3f69 100644 --- a/src/tools/ts-generator/main.cpp +++ b/src/tools/ts-generator/main.cpp @@ -70,168 +70,6 @@ std::vector getClustersToGen(Data::AreaList& areas, return clusters; } -bool readLinkIniProperty(LinkTSgenerationParams* link, - const Yuni::String& key, - const Yuni::String& value) -{ - if (key == "unitcount") - { - return value.to(link->unitCount); - } - - if (key == "nominalcapacity") - { - return value.to(link->nominalCapacity); - } - - if (key == "law.planned") - { - return value.to(link->plannedLaw); - } - - if (key == "law.forced") - { - return value.to(link->forcedLaw); - } - - if (key == "volatility.planned") - { - return value.to(link->plannedVolatility); - } - - if (key == "volatility.forced") - { - return value.to(link->forcedVolatility); - } - - if (key == "force-no-generation") - { - return value.to(link->forceNoGeneration); - } - return true; -} - -void readLinkIniProperties(LinkTSgenerationParams* link, - IniFile::Section* section) -{ - for (const IniFile::Property* p = section->firstProperty; p; p = p->next) - { - if (! readLinkIniProperty(link, p->key, p->value)) - { - std::string linkName = link->namesPair.first + "." + link->namesPair.second; - logs.warning() << "Link '" << linkName << "' : reading value of '" - << p->key << "' went wrong"; - link->hasValidData = false; - } - } -} - -void readSourceAreaIniFile(fs::path pathToIni, - std::string sourceAreaName, - std::vector& linkList) -{ - IniFile ini; - ini.open(pathToIni); // gp : we should handle reading issues - for (auto* section = ini.firstSection; section; section = section->next) - { - std::string targetAreaName = transformNameIntoID(section->name); - const LinkPair processedLink = std::make_pair(sourceAreaName, targetAreaName); - if (auto* foundLink = findLinkInList(processedLink, linkList); foundLink) - { - readLinkIniProperties(foundLink, section); - } - } -} - -void readIniProperties(std::vector& linkList, fs::path toLinksDir) -{ - for(auto& link : linkList) - { - std::string sourceAreaName = link.namesPair.first; - fs::path pathToIni = toLinksDir / sourceAreaName / "properties.ini"; - readSourceAreaIniFile(pathToIni, sourceAreaName, linkList); - } -} - -bool readLinkPreproTimeSeries(LinkTSgenerationParams& link, - fs::path sourceAreaDir) -{ - bool to_return = true; - const auto preproId = link.namesPair.first + "/" + link.namesPair.second; - link.prepro = std::make_unique(preproId, link.unitCount); - - auto preproFileRoot = sourceAreaDir / "prepro" / link.namesPair.second; - - auto preproFile = preproFileRoot; - preproFile += ".txt"; - if (fs::exists(preproFile)) - { - to_return = link.prepro->data.loadFromCSVFile( - preproFile.string(), - Data::PreproAvailability::preproAvailabilityMax, - DAYS_PER_YEAR) - && link.prepro->validate() - && to_return; - } - - auto modulationFileDirect = preproFileRoot; - modulationFileDirect += "_mod_direct.txt"; - if (fs::exists(modulationFileDirect)) - { - to_return = link.modulationCapacityDirect.loadFromCSVFile( - modulationFileDirect.string(), - 1, - HOURS_PER_YEAR) - && to_return; - } - - auto modulationFileIndirect = preproFileRoot; - modulationFileIndirect += "_mod_indirect.txt"; - if (fs::exists(modulationFileIndirect)) - { - to_return = link.modulationCapacityIndirect.loadFromCSVFile( - modulationFileIndirect.string(), - 1, - HOURS_PER_YEAR) - && to_return; - } - // Makes possible a skip of TS generation when time comes - link.hasValidData = link.hasValidData && to_return; - return to_return; -} - -void readPreproTimeSeries(std::vector& linkList, - fs::path toLinksDir) -{ - for(auto& link : linkList) - { - std::string sourceAreaName = link.namesPair.first; - fs::path sourceAreaDir = toLinksDir / sourceAreaName; - if (! readLinkPreproTimeSeries(link, sourceAreaDir)) - { - logs.warning() << "Could not load all prepro data for link '" - << link.namesPair.first << "." << link.namesPair.second << "'"; - } - } -} - -void readLinksSpecificTSparameters(std::vector& linkList, - fs::path studyFolder) -{ - fs::path toLinksDir = studyFolder / "input" / "links"; - readIniProperties(linkList, toLinksDir); - readPreproTimeSeries(linkList, toLinksDir); -} - -std::string DateAndTime() -{ - YString to_return; - unsigned int now = Yuni::DateTime::Now(); - Yuni::DateTime::TimestampToString(to_return, "%Y%m%d-%H%M", now); - return to_return.to(); -} -// ============================================================================ - int main(int argc, char* argv[]) { logs.applicationName("ts-generator"); From c133e8866f325098e629c90ace711af77b4c1a0b Mon Sep 17 00:00:00 2001 From: guilpier-code <62292552+guilpier-code@users.noreply.github.com> Date: Fri, 5 Jul 2024 16:29:31 +0200 Subject: [PATCH 26/28] TS generator : refactor thermal part (#2202) Aim of this PR : see the title. What was done (see helping **gitHub** comments in this PR) : - simplifications : - [x] Standardizing TS generation with link TS : remove a constructor in a class containing data for TS generation. - [x] Remove use of **RuntimeInfos** in **main.cpp** (many code lines used to be called there, just to get a random generator). - [x] Remove check on min stable power, useless. - [x] Function that actually generates the TS no longer takes the TS to be generated as argument, but returns it. - [x] **TS generator** code no longer contains code related to whether we should skip the TS writing on disk. It writes, that'all. This kind of code belongs to **solver**, not to **TS generator**. - [x] Use **std::filesystem** whenever possible instead of **Yuni** - Unwanted behavior correction : - [x] Execute links or thermal TS generation code only depending on user requirements : indeed, thermal TS generation still needs the whole study to be loaded, but link TS generation does not (it retrieves from study only the required data). Being forced to load the study if only link TS generation is required is a clear performance issue. We avoid that here. - [x] When data files required to generate TS (modulation files & prepro files) don't exist, we used to ignore it (this can lead to a crash). We now raise an error before TS generation time comes. - Moving code : _to be completed_ --------- Co-authored-by: Florian OMNES --- .../antares/solver/simulation/solver.hxx | 14 ++- src/solver/ts-generator/availability.cpp | 110 ++++++------------ .../antares/solver/ts-generator/generator.h | 16 ++- .../tools/ts-generator/tsGenerationOptions.h | 2 + src/tools/ts-generator/linksTSgenerator.cpp | 78 +++++++------ src/tools/ts-generator/main.cpp | 93 ++++++--------- .../ts-generator/tsGenerationOptions.cpp | 8 ++ 7 files changed, 143 insertions(+), 178 deletions(-) diff --git a/src/solver/simulation/include/antares/solver/simulation/solver.hxx b/src/solver/simulation/include/antares/solver/simulation/solver.hxx index be3c566f59..c9acc921b9 100644 --- a/src/solver/simulation/include/antares/solver/simulation/solver.hxx +++ b/src/solver/simulation/include/antares/solver/simulation/solver.hxx @@ -461,10 +461,16 @@ void ISimulation::regenerateTimeSeries(uint year) if (refreshTSonCurrentYear) { auto clusters = getAllClustersToGen(study.areas, pData.haveToRefreshTSThermal); - namespace fs = std::filesystem; - fs::path savePath = fs::path(study.folderOutput.to()) / "ts-generator" - / "thermal" / "mc-" / std::to_string(year); - generateThermalTimeSeries(study, clusters, savePath.string()); + generateThermalTimeSeries(study, clusters, study.runtime->random[Data::seedTsGenThermal]); + + bool archive = study.parameters.timeSeriesToArchive & Data::timeSeriesThermal; + bool doWeWrite = archive && !study.parameters.noOutput; + if (doWeWrite) + { + fs::path savePath = fs::path(study.folderOutput.to()) / "ts-generator" + / "thermal" / "mc-" / std::to_string(year); + writeThermalTimeSeries(clusters, savePath); + } // apply the spinning if we generated some in memory clusters for (auto* cluster: clusters) diff --git a/src/solver/ts-generator/availability.cpp b/src/solver/ts-generator/availability.cpp index c51b006dde..ba13f46691 100644 --- a/src/solver/ts-generator/availability.cpp +++ b/src/solver/ts-generator/availability.cpp @@ -19,20 +19,13 @@ ** along with Antares_Simulator. If not, see . */ -#include -#include #include #include #include #include #include -#include #include // For Antares::IO::fileSetContent -#include "antares/study/simulation.h" - -#define SEP Yuni::IO::Separator -namespace fs = std::filesystem; constexpr double FAILURE_RATE_EQ_1 = 0.999; @@ -46,14 +39,12 @@ AvailabilityTSGeneratorData::AvailabilityTSGeneratorData(Data::ThermalCluster* c forcedLaw(cluster->forcedLaw), plannedLaw(cluster->plannedLaw), prepro(cluster->prepro), - series(cluster->series.timeSeries), modulationCapacity(cluster->modulation[Data::thermalModulationCapacity]), name(cluster->name()) { } AvailabilityTSGeneratorData::AvailabilityTSGeneratorData(LinkTSgenerationParams& source, - Data::TimeSeries& capacity, Matrix<>& modulation, const std::string& areaDestName): unitCount(source.unitCount), @@ -63,7 +54,6 @@ AvailabilityTSGeneratorData::AvailabilityTSGeneratorData(LinkTSgenerationParams& forcedLaw(source.forcedLaw), plannedLaw(source.plannedLaw), prepro(source.prepro.get()), - series(capacity.timeSeries), modulationCapacity(modulation[0]), name(areaDestName) { @@ -74,17 +64,16 @@ namespace class AvailabilityTSgenerator final { public: - explicit AvailabilityTSgenerator(Data::Study&, unsigned, MersenneTwister&); explicit AvailabilityTSgenerator(bool, unsigned, MersenneTwister&); - void run(AvailabilityTSGeneratorData&) const; + Matrix run(AvailabilityTSGeneratorData&) const; private: bool derated; uint nbOfSeriesToGen_; - MersenneTwister& rndgenerator; + MersenneTwister& randomGenerator_; static constexpr int Log_size = 4000; @@ -102,21 +91,12 @@ class AvailabilityTSgenerator final const T& duration) const; }; -AvailabilityTSgenerator::AvailabilityTSgenerator(Data::Study& study, - unsigned nbOfSeriesToGen, - MersenneTwister& rndGenerator): - derated(study.parameters.derated), - nbOfSeriesToGen_(nbOfSeriesToGen), - rndgenerator(rndGenerator) -{ -} - AvailabilityTSgenerator::AvailabilityTSgenerator(bool derated, unsigned int nbOfSeriesToGen, - MersenneTwister& rndGenerator): + MersenneTwister& randomGenerator): derated(derated), nbOfSeriesToGen_(nbOfSeriesToGen), - rndgenerator(rndGenerator) + randomGenerator_(randomGenerator) { } @@ -177,7 +157,7 @@ int AvailabilityTSgenerator::durationGenerator(Data::StatisticalLaw law, return expec; } - double rndnumber = rndgenerator.next(); + double rndnumber = randomGenerator_.next(); switch (law) { @@ -198,13 +178,16 @@ int AvailabilityTSgenerator::durationGenerator(Data::StatisticalLaw law, return 0; } -void AvailabilityTSgenerator::run(AvailabilityTSGeneratorData& tsGenerationData) const +Matrix AvailabilityTSgenerator::run(AvailabilityTSGeneratorData& tsGenerationData) const { assert(tsGenerationData.prepro); + Matrix to_return; + to_return.reset(nbOfSeriesToGen_, 24 * DAYS_PER_YEAR); + if (0 == tsGenerationData.unitCount || 0 == tsGenerationData.nominalCapacity) { - return; + return to_return; } const auto& preproData = *(tsGenerationData.prepro); @@ -318,18 +301,11 @@ void AvailabilityTSgenerator::run(AvailabilityTSGeneratorData& tsGenerationData) double last = 0; auto& modulation = tsGenerationData.modulationCapacity; - double* dstSeries = nullptr; const uint tsCount = nbOfSeriesToGen_ + 2; for (uint tsIndex = 0; tsIndex != tsCount; ++tsIndex) { uint hour = 0; - - if (tsIndex > 1) - { - dstSeries = tsGenerationData.series[tsIndex - 2]; - } - for (uint dayInTheYear = 0; dayInTheYear < DAYS_PER_YEAR; ++dayInTheYear) { assert(dayInTheYear < 366); @@ -389,7 +365,7 @@ void AvailabilityTSgenerator::run(AvailabilityTSGeneratorData& tsGenerationData) if (lf[dayInTheYear] > 0. && lf[dayInTheYear] <= FAILURE_RATE_EQ_1) { - A = rndgenerator.next(); + A = randomGenerator_.next(); last = FPOW[dayInTheYear][AUN]; if (A > last) @@ -425,7 +401,7 @@ void AvailabilityTSgenerator::run(AvailabilityTSGeneratorData& tsGenerationData) } last = PPOW[dayInTheYear][AUN_app]; - A = rndgenerator.next(); + A = randomGenerator_.next(); if (A > last) { @@ -579,7 +555,7 @@ void AvailabilityTSgenerator::run(AvailabilityTSGeneratorData& tsGenerationData) double AVPDayInTheYear = AVP[dayInTheYear]; for (uint h = 0; h != 24; ++h) { - dstSeries[hour] = std::round(AVPDayInTheYear * modulation[hour]); + to_return[tsIndex - 2][hour] = std::round(AVPDayInTheYear * modulation[hour]); ++hour; } } @@ -588,8 +564,9 @@ void AvailabilityTSgenerator::run(AvailabilityTSGeneratorData& tsGenerationData) if (derated) { - tsGenerationData.series.averageTimeseries(); + to_return.averageTimeseries(); } + return to_return; } } // namespace @@ -629,46 +606,41 @@ void writeTStoDisk(const Matrix<>& series, bool generateThermalTimeSeries(Data::Study& study, const std::vector& clusters, - const std::string& savePath) + MersenneTwister& thermalRandom) { logs.info(); logs.info() << "Generating the thermal time-series"; - auto generator = AvailabilityTSgenerator(study, + auto generator = AvailabilityTSgenerator(study.parameters.derated, study.parameters.nbTimeSeriesThermal, - study.runtime->random[Data::seedTsGenThermal]); + thermalRandom); for (auto* cluster: clusters) { - cluster->series.timeSeries.reset(study.parameters.nbTimeSeriesThermal, HOURS_PER_YEAR); AvailabilityTSGeneratorData tsGenerationData(cluster); - generator.run(tsGenerationData); + cluster->series.timeSeries = generator.run(tsGenerationData); } - bool archive = study.parameters.timeSeriesToArchive & Data::timeSeriesThermal; - bool doWeWrite = archive && !study.parameters.noOutput; - if (! doWeWrite) - { - logs.info() << "Study parameters forbid writing thermal TS."; - return true; - } + return true; +} +void writeThermalTimeSeries(const std::vector& clusters, + const fs::path& savePath) +{ for (auto* cluster: clusters) { auto areaName = cluster->parentArea->id.to(); auto clusterName = cluster->id(); - auto filePath = fs::path(savePath) / areaName / clusterName += ".txt"; + auto filePath = savePath / areaName / clusterName += ".txt"; - writeTStoDisk(cluster->series.timeSeries, filePath.string()); + writeTStoDisk(cluster->series.timeSeries, filePath); } - - return true; } // gp : we should try to add const identifiers before args here bool generateLinkTimeSeries(std::vector& links, StudyParamsForLinkTS& generalParams, - const std::string& savePath) + const fs::path& savePath) { logs.info(); logs.info() << "Generation of links time-series"; @@ -687,27 +659,19 @@ bool generateLinkTimeSeries(std::vector& links, if (link.forceNoGeneration) continue; // Skipping the link - Data::TimeSeriesNumbers fakeTSnumbers; // gp : to quickly get rid of - Data::TimeSeries ts(fakeTSnumbers); - ts.resize(generalParams.nbLinkTStoGenerate, HOURS_PER_YEAR); - - // DIRECT - AvailabilityTSGeneratorData tsConfigDataDirect(link, ts, link.modulationCapacityDirect, link.namesPair.second); - - generator.run(tsConfigDataDirect); - - std::string filePath = savePath + SEP + link.namesPair.first + SEP + link.namesPair.second - + "_direct.txt"; - writeTStoDisk(ts.timeSeries, filePath); + // === DIRECT ======================= + AvailabilityTSGeneratorData tsConfigDataDirect(link, link.modulationCapacityDirect, link.namesPair.second); + auto generated_ts = generator.run(tsConfigDataDirect); - // INDIRECT - AvailabilityTSGeneratorData tsConfigDataIndirect(link, ts, link.modulationCapacityIndirect, link.namesPair.second); + auto filePath = savePath / link.namesPair.first / link.namesPair.second += "_direct.txt"; + writeTStoDisk(generated_ts, filePath); - generator.run(tsConfigDataIndirect); + // === INDIRECT ======================= + AvailabilityTSGeneratorData tsConfigDataIndirect(link, link.modulationCapacityIndirect, link.namesPair.second); + generated_ts = generator.run(tsConfigDataIndirect); - filePath = savePath + SEP + link.namesPair.first + SEP + link.namesPair.second - + "_indirect.txt"; - writeTStoDisk(ts.timeSeries, filePath); + filePath = savePath / link.namesPair.first / link.namesPair.second += "_indirect.txt"; + writeTStoDisk(generated_ts, filePath); } return true; diff --git a/src/solver/ts-generator/include/antares/solver/ts-generator/generator.h b/src/solver/ts-generator/include/antares/solver/ts-generator/generator.h index 75255206f0..ef55c2a964 100644 --- a/src/solver/ts-generator/include/antares/solver/ts-generator/generator.h +++ b/src/solver/ts-generator/include/antares/solver/ts-generator/generator.h @@ -24,15 +24,13 @@ #include #include -#include #include #include #include #include -#include - #include "xcast/xcast.h" +namespace fs = std::filesystem; using LinkPair = std::pair; using LinkPairs = std::vector; @@ -52,7 +50,7 @@ struct LinkTSgenerationParams { LinkPair namesPair; - unsigned unitCount = 0; + unsigned unitCount = 1; double nominalCapacity = 0; double forcedVolatility = 0.; @@ -77,7 +75,6 @@ class AvailabilityTSGeneratorData explicit AvailabilityTSGeneratorData(Data::ThermalCluster*); AvailabilityTSGeneratorData(LinkTSgenerationParams&, - Data::TimeSeries&, Matrix<>& modulation, const std::string& name); @@ -92,8 +89,6 @@ class AvailabilityTSGeneratorData Data::PreproAvailability* prepro; - Matrix<>& series; - Matrix<>::ColumnType& modulationCapacity; const std::string& name; @@ -111,11 +106,14 @@ bool GenerateTimeSeries(Data::Study& study, uint year, IResultWriter& writer); bool generateThermalTimeSeries(Data::Study& study, const std::vector& clusters, - const std::string& savePath); + MersenneTwister& thermalRandom); + +void writeThermalTimeSeries(const std::vector& clusters, + const fs::path& savePath); bool generateLinkTimeSeries(std::vector& links, StudyParamsForLinkTS&, - const std::string& savePath); + const fs::path& savePath); std::vector getAllClustersToGen(const Data::AreaList& areas, bool globalThermalTSgeneration); diff --git a/src/tools/ts-generator/include/antares/tools/ts-generator/tsGenerationOptions.h b/src/tools/ts-generator/include/antares/tools/ts-generator/tsGenerationOptions.h index dfc58257f5..2942964bd9 100644 --- a/src/tools/ts-generator/include/antares/tools/ts-generator/tsGenerationOptions.h +++ b/src/tools/ts-generator/include/antares/tools/ts-generator/tsGenerationOptions.h @@ -26,4 +26,6 @@ bool parseOptions(int, char*[], Settings&); std::unique_ptr createTsGeneratorParser(Settings&); bool checkOptions(Settings& options); +bool linkTSrequired(Settings& options); +bool thermalTSrequired(Settings& options); } diff --git a/src/tools/ts-generator/linksTSgenerator.cpp b/src/tools/ts-generator/linksTSgenerator.cpp index 87d1e94aad..6c16dfc0a3 100644 --- a/src/tools/ts-generator/linksTSgenerator.cpp +++ b/src/tools/ts-generator/linksTSgenerator.cpp @@ -166,52 +166,56 @@ void readIniProperties(std::vector& linkList, fs::path t } } - bool readLinkPreproTimeSeries(LinkTSgenerationParams& link, - fs::path sourceAreaDir) - { - bool to_return = true; - const auto preproId = link.namesPair.first + "/" + link.namesPair.second; - link.prepro = std::make_unique(preproId, link.unitCount); +fs::path makePreproFile(const fs::path& preproFilePath, const std::string& changingEnd) +{ + auto to_return = preproFilePath; + to_return += changingEnd + ".txt"; + return to_return; +} - auto preproFileRoot = sourceAreaDir / "prepro" / link.namesPair.second; +bool readLinkPreproTimeSeries(LinkTSgenerationParams& link, + fs::path sourceAreaDir) +{ + bool to_return = true; + const auto preproId = link.namesPair.first + "/" + link.namesPair.second; + link.prepro = std::make_unique(preproId, link.unitCount); + + auto preproFileRoot = sourceAreaDir / "prepro" / link.namesPair.second; + + // Testing files existence + auto preproFile = makePreproFile(preproFileRoot, ""); + auto modulationDirectFile = makePreproFile(preproFileRoot, "_mod_direct"); + auto modulationIndirectFile = makePreproFile(preproFileRoot, "_mod_indirect"); + std::vector paths {preproFile, modulationDirectFile, modulationIndirectFile}; + if (std::any_of(paths.begin(), paths.end(), [](auto& path) {return ! fs::exists(path);})) + { + link.hasValidData = false; + return false; + } - auto preproFile = preproFileRoot; - preproFile += ".txt"; - if (fs::exists(preproFile)) - { - to_return = link.prepro->data.loadFromCSVFile( + // Files loading + to_return = link.prepro->data.loadFromCSVFile( preproFile.string(), Data::PreproAvailability::preproAvailabilityMax, DAYS_PER_YEAR) - && link.prepro->validate() - && to_return; - } + && link.prepro->validate() + && to_return; - auto modulationFileDirect = preproFileRoot; - modulationFileDirect += "_mod_direct.txt"; - if (fs::exists(modulationFileDirect)) - { - to_return = link.modulationCapacityDirect.loadFromCSVFile( - modulationFileDirect.string(), + to_return = link.modulationCapacityDirect.loadFromCSVFile( + modulationDirectFile.string(), 1, HOURS_PER_YEAR) - && to_return; - } + && to_return; - auto modulationFileIndirect = preproFileRoot; - modulationFileIndirect += "_mod_indirect.txt"; - if (fs::exists(modulationFileIndirect)) - { - to_return = link.modulationCapacityIndirect.loadFromCSVFile( - modulationFileIndirect.string(), + to_return = link.modulationCapacityIndirect.loadFromCSVFile( + modulationIndirectFile.string(), 1, HOURS_PER_YEAR) - && to_return; - } - // Makes it possible to skip a link's TS generation when time comes - link.hasValidData = link.hasValidData && to_return; - return to_return; - } + && to_return; + + link.hasValidData = link.hasValidData && to_return; + return to_return; +} void readPreproTimeSeries(std::vector& linkList, fs::path toLinksDir) @@ -222,7 +226,7 @@ void readPreproTimeSeries(std::vector& linkList, fs::path sourceAreaDir = toLinksDir / sourceAreaName; if (! readLinkPreproTimeSeries(link, sourceAreaDir)) { - logs.warning() << "Could not load all prepro data for link '" + logs.warning() << "Could not load all prepro/modulation data for link '" << link.namesPair.first << "." << link.namesPair.second << "'"; } } @@ -330,7 +334,7 @@ bool LinksTSgenerator::generate() saveTSpath /= "ts-generator"; saveTSpath /= "links"; - return generateLinkTimeSeries(linkList_, generalParams_, saveTSpath.string()); + return generateLinkTimeSeries(linkList_, generalParams_, saveTSpath); } } \ No newline at end of file diff --git a/src/tools/ts-generator/main.cpp b/src/tools/ts-generator/main.cpp index 74040e3f69..13eee06897 100644 --- a/src/tools/ts-generator/main.cpp +++ b/src/tools/ts-generator/main.cpp @@ -22,15 +22,10 @@ #include #include -#include -#include -#include #include #include #include #include -#include -#include #include "antares/tools/ts-generator/tsGenerationOptions.h" #include "antares/tools/ts-generator/linksTSgenerator.h" @@ -44,9 +39,9 @@ std::vector getClustersToGen(Data::AreaList& areas, const std::string& clustersToGen) { std::vector clusters; - const auto ids = splitStringIntoPairs(clustersToGen, ';', '.'); + const auto pairsAreaCluster = splitStringIntoPairs(clustersToGen, ';', '.'); - for (const auto& [areaID, clusterID]: ids) + for (const auto& [areaID, clusterID]: pairsAreaCluster) { logs.info() << "Searching for area: " << areaID << " and cluster: " << clusterID; @@ -81,61 +76,49 @@ int main(int argc, char* argv[]) if (! checkOptions(settings)) return 1; - auto study = std::make_shared(true); - Data::StudyLoadOptions studyOptions; - studyOptions.prepareOutput = true; + bool return_code {true}; - if (!study->loadFromFolder(settings.studyFolder, studyOptions)) + if (thermalTSrequired(settings)) { - logs.error() << "Invalid study given to the generator"; - return 1; - } - - study->initializeRuntimeInfos(); - // Force the writing of generated TS into output/YYYYMMDD-HHSSeco/ts-generator/thermal[/mc-0] - study->parameters.timeSeriesToArchive |= Antares::Data::timeSeriesThermal; - - try - { - Antares::Check::checkMinStablePower(true, study->areas); - } - catch (Error::InvalidParametersForThermalClusters& ex) - { - Antares::logs.error() << ex.what(); - } + // === Data for TS generation === + auto study = std::make_shared(true); + Data::StudyLoadOptions studyOptions; + if (!study->loadFromFolder(settings.studyFolder, studyOptions)) + { + logs.error() << "Invalid study given to the generator"; + return 1; + } - auto thermalSavePath = fs::path(settings.studyFolder) / "output" / FormattedTime("%Y%m%d-%H%M"); - thermalSavePath /= "ts-generator"; - thermalSavePath /= "thermal"; + std::vector clusters; + if (settings.allThermal) + { + clusters = getAllClustersToGen(study->areas, true); + } + else if (!settings.thermalListToGen.empty()) + { + clusters = getClustersToGen(study->areas, settings.thermalListToGen); + } - // ============ THERMAL : Getting data for generating time-series ========= - std::vector clusters; - if (settings.allThermal) - { - clusters = getAllClustersToGen(study->areas, true); - } - else if (!settings.thermalListToGen.empty()) - { - clusters = getClustersToGen(study->areas, settings.thermalListToGen); + // === TS generation === + MersenneTwister thermalRandom; + thermalRandom.reset(study->parameters.seed[Data::seedTsGenThermal]); + return_code = TSGenerator::generateThermalTimeSeries(*study, + clusters, + thermalRandom); + + // === Writing generated TS on disk === + auto thermalSavePath = fs::path(settings.studyFolder) / "output" / FormattedTime("%Y%m%d-%H%M"); + thermalSavePath /= "ts-generator"; + thermalSavePath /= "thermal"; + writeThermalTimeSeries(clusters, thermalSavePath); } - for (auto& c: clusters) + if (linkTSrequired(settings)) { - logs.debug() << c->id(); + LinksTSgenerator linksTSgenerator(settings); + linksTSgenerator.extractData(); + return_code = linksTSgenerator.generate() && return_code; } - - - LinksTSgenerator linksTSgenerator(settings); - linksTSgenerator.extractData(); - - // ============ TS Generation ============================================= - - bool ret = TSGenerator::generateThermalTimeSeries(*study, - clusters, - thermalSavePath.string()); - - ret = linksTSgenerator.generate() && ret; - - return !ret; // return 0 for success + return !return_code; // return 0 for success } \ No newline at end of file diff --git a/src/tools/ts-generator/tsGenerationOptions.cpp b/src/tools/ts-generator/tsGenerationOptions.cpp index 16750a2173..48ee5b09e4 100644 --- a/src/tools/ts-generator/tsGenerationOptions.cpp +++ b/src/tools/ts-generator/tsGenerationOptions.cpp @@ -63,4 +63,12 @@ bool checkOptions(Settings& options) return true; } +bool linkTSrequired(Settings& options) +{ + return options.allLinks || !options.linksListToGen.empty(); +} +bool thermalTSrequired(Settings& options) +{ + return options.allThermal || !options.thermalListToGen.empty(); +} } \ No newline at end of file From fa17f6ebae5969082762ccb483ed79b2a15b0d9a Mon Sep 17 00:00:00 2001 From: Florian OMNES Date: Tue, 9 Jul 2024 11:10:28 +0200 Subject: [PATCH 27/28] Formatting --- src/solver/application/application.cpp | 5 +- .../antares/solver/simulation/solver.hxx | 6 +- src/solver/ts-generator/availability.cpp | 12 ++- .../antares/solver/ts-generator/generator.h | 1 + .../tools/ts-generator/linksTSgenerator.h | 7 +- .../tools/ts-generator/tsGenerationOptions.h | 2 +- src/tools/ts-generator/linksTSgenerator.cpp | 91 ++++++++++--------- src/tools/ts-generator/main.cpp | 20 ++-- .../ts-generator/tsGenerationOptions.cpp | 25 ++--- 9 files changed, 95 insertions(+), 74 deletions(-) diff --git a/src/solver/application/application.cpp b/src/solver/application/application.cpp index 44b1fc2379..c65e849a15 100644 --- a/src/solver/application/application.cpp +++ b/src/solver/application/application.cpp @@ -445,8 +445,9 @@ void Application::resetLogFilename() const + ". Aborting now."); } - logfile /= "solver-"; // append the filename - logfile += FormattedTime("%Y%m%d-%H%M%S") + ".log"; // complete filename with timestamp and extension + logfile /= "solver-"; // append the filename + logfile += FormattedTime("%Y%m%d-%H%M%S") + + ".log"; // complete filename with timestamp and extension // Assigning the log filename logs.logfile(logfile.string()); diff --git a/src/solver/simulation/include/antares/solver/simulation/solver.hxx b/src/solver/simulation/include/antares/solver/simulation/solver.hxx index 79430754a7..1aa04cb38f 100644 --- a/src/solver/simulation/include/antares/solver/simulation/solver.hxx +++ b/src/solver/simulation/include/antares/solver/simulation/solver.hxx @@ -454,7 +454,9 @@ void ISimulation::regenerateTimeSeries(uint year) if (refreshTSonCurrentYear) { auto clusters = getAllClustersToGen(study.areas, pData.haveToRefreshTSThermal); - generateThermalTimeSeries(study, clusters, study.runtime->random[Data::seedTsGenThermal]); + generateThermalTimeSeries(study, + clusters, + study.runtime->random[Data::seedTsGenThermal]); bool archive = study.parameters.timeSeriesToArchive & Data::timeSeriesThermal; bool doWeWrite = archive && !study.parameters.noOutput; @@ -467,7 +469,9 @@ void ISimulation::regenerateTimeSeries(uint year) // apply the spinning if we generated some in memory clusters for (auto* cluster: clusters) + { cluster->calculationOfSpinning(); + } } }; } diff --git a/src/solver/ts-generator/availability.cpp b/src/solver/ts-generator/availability.cpp index 660a17b0ef..a94fd99e46 100644 --- a/src/solver/ts-generator/availability.cpp +++ b/src/solver/ts-generator/availability.cpp @@ -26,7 +26,6 @@ #include #include #include -#include // For Antares::IO::fileSetContent constexpr double FAILURE_RATE_EQ_1 = 0.999; @@ -591,8 +590,7 @@ std::vector getAllClustersToGen(const Data::AreaList& are return clusters; } -void writeTStoDisk(const Matrix<>& series, - const std::filesystem::path savePath) +void writeTStoDisk(const Matrix<>& series, const std::filesystem::path savePath) { std::string buffer; series.saveToBuffer(buffer, 0); @@ -664,14 +662,18 @@ bool generateLinkTimeSeries(std::vector& links, } // === DIRECT ======================= - AvailabilityTSGeneratorData tsConfigDataDirect(link, link.modulationCapacityDirect, link.namesPair.second); + AvailabilityTSGeneratorData tsConfigDataDirect(link, + link.modulationCapacityDirect, + link.namesPair.second); auto generated_ts = generator.run(tsConfigDataDirect); auto filePath = savePath / link.namesPair.first / link.namesPair.second += "_direct.txt"; writeTStoDisk(generated_ts, filePath); // === INDIRECT ======================= - AvailabilityTSGeneratorData tsConfigDataIndirect(link, link.modulationCapacityIndirect, link.namesPair.second); + AvailabilityTSGeneratorData tsConfigDataIndirect(link, + link.modulationCapacityIndirect, + link.namesPair.second); generated_ts = generator.run(tsConfigDataIndirect); filePath = savePath / link.namesPair.first / link.namesPair.second += "_indirect.txt"; diff --git a/src/solver/ts-generator/include/antares/solver/ts-generator/generator.h b/src/solver/ts-generator/include/antares/solver/ts-generator/generator.h index 57032a9431..5ffea85100 100644 --- a/src/solver/ts-generator/include/antares/solver/ts-generator/generator.h +++ b/src/solver/ts-generator/include/antares/solver/ts-generator/generator.h @@ -28,6 +28,7 @@ #include #include #include + #include "xcast/xcast.h" namespace fs = std::filesystem; diff --git a/src/tools/ts-generator/include/antares/tools/ts-generator/linksTSgenerator.h b/src/tools/ts-generator/include/antares/tools/ts-generator/linksTSgenerator.h index 517d99a633..1feaac5038 100644 --- a/src/tools/ts-generator/include/antares/tools/ts-generator/linksTSgenerator.h +++ b/src/tools/ts-generator/include/antares/tools/ts-generator/linksTSgenerator.h @@ -1,12 +1,13 @@ #pragma once -#include "antares/tools/ts-generator/tsGenerationOptions.h" #include +#include "antares/tools/ts-generator/tsGenerationOptions.h" namespace fs = std::filesystem; -namespace Antares::TSGenerator { +namespace Antares::TSGenerator +{ class LinksTSgenerator { @@ -28,4 +29,4 @@ class LinksTSgenerator StudyParamsForLinkTS generalParams_; }; -} // End Antares::TSGenerator +} // namespace Antares::TSGenerator diff --git a/src/tools/ts-generator/include/antares/tools/ts-generator/tsGenerationOptions.h b/src/tools/ts-generator/include/antares/tools/ts-generator/tsGenerationOptions.h index 2942964bd9..1d83cce593 100644 --- a/src/tools/ts-generator/include/antares/tools/ts-generator/tsGenerationOptions.h +++ b/src/tools/ts-generator/include/antares/tools/ts-generator/tsGenerationOptions.h @@ -28,4 +28,4 @@ std::unique_ptr createTsGeneratorParser(Settings&); bool checkOptions(Settings& options); bool linkTSrequired(Settings& options); bool thermalTSrequired(Settings& options); -} +} // namespace Antares::TSGenerator diff --git a/src/tools/ts-generator/linksTSgenerator.cpp b/src/tools/ts-generator/linksTSgenerator.cpp index 6c16dfc0a3..7ad893194a 100644 --- a/src/tools/ts-generator/linksTSgenerator.cpp +++ b/src/tools/ts-generator/linksTSgenerator.cpp @@ -1,5 +1,6 @@ #include "antares/tools/ts-generator/linksTSgenerator.h" + #include "antares/utils/utils.h" namespace Antares::TSGenerator @@ -28,10 +29,12 @@ bool pairs_match(const LinkPair& p1, const LinkPair& p2) const LinkPair* getMatchingPairInCollection(const LinkPair& pair, const LinkPairs& collection) { - for(const auto& p : collection) + for (const auto& p: collection) { if (pairs_match(pair, p)) + { return &p; + } } return nullptr; } @@ -50,9 +53,11 @@ bool readLinkGeneralProperty(StudyParamsForLinkTS& params, } if (key == "seed-tsgen-links") { - unsigned int seed {0}; - if (! value.to(seed)) + unsigned int seed{0}; + if (!value.to(seed)) + { return false; + } params.random.reset(seed); return true; } @@ -62,7 +67,8 @@ bool readLinkGeneralProperty(StudyParamsForLinkTS& params, std::vector CreateLinkList(const LinkPairs& linksFromCmdLine) { std::vector to_return; - std::for_each(linksFromCmdLine.begin(), linksFromCmdLine.end(), + std::for_each(linksFromCmdLine.begin(), + linksFromCmdLine.end(), [&to_return](const auto& link_pair) { LinkTSgenerationParams link; @@ -75,10 +81,12 @@ std::vector CreateLinkList(const LinkPairs& linksFromCmd LinkTSgenerationParams* findLinkInList(const LinkPair& link_to_find, std::vector& linkList) { - for(auto& link : linkList) + for (auto& link: linkList) { if (link.namesPair == link_to_find) + { return &link; + } } return nullptr; } @@ -124,16 +132,15 @@ bool readLinkIniProperty(LinkTSgenerationParams* link, return true; } -void readLinkIniProperties(LinkTSgenerationParams* link, - IniFile::Section* section) +void readLinkIniProperties(LinkTSgenerationParams* link, IniFile::Section* section) { for (const IniFile::Property* p = section->firstProperty; p; p = p->next) { - if (! readLinkIniProperty(link, p->key, p->value)) + if (!readLinkIniProperty(link, p->key, p->value)) { std::string linkName = link->namesPair.first + "." + link->namesPair.second; - logs.warning() << "Link '" << linkName << "' : reading value of '" - << p->key << "' went wrong"; + logs.warning() << "Link '" << linkName << "' : reading value of '" << p->key + << "' went wrong"; link->hasValidData = false; } } @@ -158,7 +165,7 @@ void readSourceAreaIniFile(fs::path pathToIni, void readIniProperties(std::vector& linkList, fs::path toLinksDir) { - for(auto& link : linkList) + for (auto& link: linkList) { std::string sourceAreaName = link.namesPair.first; fs::path pathToIni = toLinksDir / sourceAreaName / "properties.ini"; @@ -173,8 +180,7 @@ fs::path makePreproFile(const fs::path& preproFilePath, const std::string& chang return to_return; } -bool readLinkPreproTimeSeries(LinkTSgenerationParams& link, - fs::path sourceAreaDir) +bool readLinkPreproTimeSeries(LinkTSgenerationParams& link, fs::path sourceAreaDir) { bool to_return = true; const auto preproId = link.namesPair.first + "/" + link.namesPair.second; @@ -186,45 +192,40 @@ bool readLinkPreproTimeSeries(LinkTSgenerationParams& link, auto preproFile = makePreproFile(preproFileRoot, ""); auto modulationDirectFile = makePreproFile(preproFileRoot, "_mod_direct"); auto modulationIndirectFile = makePreproFile(preproFileRoot, "_mod_indirect"); - std::vector paths {preproFile, modulationDirectFile, modulationIndirectFile}; - if (std::any_of(paths.begin(), paths.end(), [](auto& path) {return ! fs::exists(path);})) + std::vector paths{preproFile, modulationDirectFile, modulationIndirectFile}; + if (std::any_of(paths.begin(), paths.end(), [](auto& path) { return !fs::exists(path); })) { link.hasValidData = false; return false; } // Files loading - to_return = link.prepro->data.loadFromCSVFile( - preproFile.string(), - Data::PreproAvailability::preproAvailabilityMax, - DAYS_PER_YEAR) - && link.prepro->validate() + to_return = link.prepro->data.loadFromCSVFile(preproFile.string(), + Data::PreproAvailability::preproAvailabilityMax, + DAYS_PER_YEAR) + && link.prepro->validate() && to_return; + + to_return = link.modulationCapacityDirect.loadFromCSVFile(modulationDirectFile.string(), + 1, + HOURS_PER_YEAR) && to_return; - to_return = link.modulationCapacityDirect.loadFromCSVFile( - modulationDirectFile.string(), - 1, - HOURS_PER_YEAR) - && to_return; - - to_return = link.modulationCapacityIndirect.loadFromCSVFile( - modulationIndirectFile.string(), - 1, - HOURS_PER_YEAR) + to_return = link.modulationCapacityIndirect.loadFromCSVFile(modulationIndirectFile.string(), + 1, + HOURS_PER_YEAR) && to_return; link.hasValidData = link.hasValidData && to_return; return to_return; } -void readPreproTimeSeries(std::vector& linkList, - fs::path toLinksDir) +void readPreproTimeSeries(std::vector& linkList, fs::path toLinksDir) { - for(auto& link : linkList) + for (auto& link: linkList) { std::string sourceAreaName = link.namesPair.first; fs::path sourceAreaDir = toLinksDir / sourceAreaName; - if (! readLinkPreproTimeSeries(link, sourceAreaDir)) + if (!readLinkPreproTimeSeries(link, sourceAreaDir)) { logs.warning() << "Could not load all prepro/modulation data for link '" << link.namesPair.first << "." << link.namesPair.second << "'"; @@ -235,7 +236,7 @@ void readPreproTimeSeries(std::vector& linkList, // ================== // Class methods // ================== -LinksTSgenerator::LinksTSgenerator(Settings & settings) : +LinksTSgenerator::LinksTSgenerator(Settings& settings): studyFolder_(settings.studyFolder), linksFromCmdLineOptions_(settings.linksListToGen), generateTSforAllLinks_(settings.allLinks) @@ -248,9 +249,13 @@ void LinksTSgenerator::extractData() LinkPairs namesLinksToGenerate; if (generateTSforAllLinks_) + { namesLinksToGenerate = allLinksPairs; + } else + { namesLinksToGenerate = extractLinkNamesFromCmdLine(allLinksPairs); + } linkList_ = CreateLinkList(namesLinksToGenerate); extractLinksSpecificTSparameters(); @@ -262,13 +267,13 @@ LinkPairs LinksTSgenerator::extractLinkNamesFromStudy() { LinkPairs to_return; fs::path linksDir = studyFolder_ / "input" / "links"; - for (auto const& item : fs::directory_iterator{linksDir}) + for (const auto& item: fs::directory_iterator{linksDir}) { if (item.is_directory()) { std::string sourceAreaName = item.path().filename().generic_string(); auto targetAreas = extractTargetAreas(item); - for (auto& targetAreaName : targetAreas) + for (auto& targetAreaName: targetAreas) { auto linkPair = std::make_pair(sourceAreaName, targetAreaName); to_return.push_back(linkPair); @@ -282,7 +287,7 @@ LinkPairs LinksTSgenerator::extractLinkNamesFromCmdLine(const LinkPairs& allLink { LinkPairs to_return; LinkPairs pairsFromCmdLine = splitStringIntoPairs(linksFromCmdLineOptions_, ';', '.'); - for (auto& p : pairsFromCmdLine) + for (auto& p: pairsFromCmdLine) { if (const auto* found_pair = getMatchingPairInCollection(p, allLinks); found_pair) { @@ -307,14 +312,16 @@ StudyParamsForLinkTS LinksTSgenerator::readGeneralParamsForLinksTS() // Skipping sections useless in the current context Yuni::String sectionName = section->name; if (sectionName != "general" && sectionName != "seeds - Mersenne Twister") + { continue; + } for (const IniFile::Property* p = section->firstProperty; p; p = p->next) { - if (! readLinkGeneralProperty(to_return, p->key, p->value)) + if (!readLinkGeneralProperty(to_return, p->key, p->value)) { - logs.warning() << ini.filename() << ": reading value of '" - << p->key << "' went wrong"; + logs.warning() << ini.filename() << ": reading value of '" << p->key + << "' went wrong"; } } } @@ -337,4 +344,4 @@ bool LinksTSgenerator::generate() return generateLinkTimeSeries(linkList_, generalParams_, saveTSpath); } -} \ No newline at end of file +} // namespace Antares::TSGenerator diff --git a/src/tools/ts-generator/main.cpp b/src/tools/ts-generator/main.cpp index 1e8974b7ec..d923d26a6c 100644 --- a/src/tools/ts-generator/main.cpp +++ b/src/tools/ts-generator/main.cpp @@ -26,9 +26,8 @@ #include #include #include - -#include "antares/tools/ts-generator/tsGenerationOptions.h" #include "antares/tools/ts-generator/linksTSgenerator.h" +#include "antares/tools/ts-generator/tsGenerationOptions.h" using namespace Antares::TSGenerator; using namespace Antares::TSGenerator; @@ -70,13 +69,17 @@ int main(int argc, char* argv[]) logs.applicationName("ts-generator"); Settings settings; - if (! parseOptions(argc, argv, settings)) + if (!parseOptions(argc, argv, settings)) + { return 1; + } - if (! checkOptions(settings)) + if (!checkOptions(settings)) + { return 1; + } - bool return_code {true}; + bool return_code{true}; if (thermalTSrequired(settings)) { @@ -102,12 +105,11 @@ int main(int argc, char* argv[]) // === TS generation === MersenneTwister thermalRandom; thermalRandom.reset(study->parameters.seed[Data::seedTsGenThermal]); - return_code = TSGenerator::generateThermalTimeSeries(*study, - clusters, - thermalRandom); + return_code = TSGenerator::generateThermalTimeSeries(*study, clusters, thermalRandom); // === Writing generated TS on disk === - auto thermalSavePath = fs::path(settings.studyFolder) / "output" / FormattedTime("%Y%m%d-%H%M"); + auto thermalSavePath = fs::path(settings.studyFolder) / "output" + / FormattedTime("%Y%m%d-%H%M"); thermalSavePath /= "ts-generator"; thermalSavePath /= "thermal"; writeThermalTimeSeries(clusters, thermalSavePath); diff --git a/src/tools/ts-generator/tsGenerationOptions.cpp b/src/tools/ts-generator/tsGenerationOptions.cpp index 48ee5b09e4..29fa64b6ad 100644 --- a/src/tools/ts-generator/tsGenerationOptions.cpp +++ b/src/tools/ts-generator/tsGenerationOptions.cpp @@ -1,9 +1,12 @@ #include "antares/tools/ts-generator/tsGenerationOptions.h" + #include -namespace Antares::TSGenerator { +namespace Antares::TSGenerator +{ -std::unique_ptr createTsGeneratorParser(Settings &settings) { +std::unique_ptr createTsGeneratorParser(Settings& settings) +{ auto parser = std::make_unique(); parser->addParagraph("Antares Time Series generator\n"); @@ -29,20 +32,19 @@ std::unique_ptr createTsGeneratorParser(Settings &settings return parser; } - bool parseOptions(int argc, char* argv[], Settings& options) { auto parser = createTsGeneratorParser(options); switch (auto ret = parser->operator()(argc, argv); ret) { using namespace Yuni::GetOpt; - case ReturnCode::error: - logs.error() << "Unknown arguments, aborting"; - return false; - case ReturnCode::help: - return false; - default: - break; + case ReturnCode::error: + logs.error() << "Unknown arguments, aborting"; + return false; + case ReturnCode::help: + return false; + default: + break; } return true; } @@ -67,8 +69,9 @@ bool linkTSrequired(Settings& options) { return options.allLinks || !options.linksListToGen.empty(); } + bool thermalTSrequired(Settings& options) { return options.allThermal || !options.thermalListToGen.empty(); } -} \ No newline at end of file +} // namespace Antares::TSGenerator From e7e7270b17072ac5b7535fd4c0eb92eebd1ff815 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Florian=20Omn=C3=A8s?= Date: Tue, 9 Jul 2024 11:11:43 +0200 Subject: [PATCH 28/28] Update src/libs/antares/utils/utils.cpp --- src/libs/antares/utils/utils.cpp | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/libs/antares/utils/utils.cpp b/src/libs/antares/utils/utils.cpp index ff4d2baef0..e188b2b13a 100644 --- a/src/libs/antares/utils/utils.cpp +++ b/src/libs/antares/utils/utils.cpp @@ -105,9 +105,7 @@ std::string FormattedTime(const std::string& format) char time_buffer[256]; std::strftime(time_buffer, sizeof(time_buffer), format.c_str(), &local_time); - std::string currentTime = time_buffer; - - return currentTime; + return std::string(time_buffer); } std::vector> splitStringIntoPairs(const std::string& s,