diff --git a/CMakeLists.txt b/CMakeLists.txt index 74474335a..1f85309ad 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -128,6 +128,7 @@ add_library(faabric $ $ $ + $ $ $ $ diff --git a/include/faabric/batch-scheduler/BatchScheduler.h b/include/faabric/batch-scheduler/BatchScheduler.h index 4f6eac172..576178b8a 100644 --- a/include/faabric/batch-scheduler/BatchScheduler.h +++ b/include/faabric/batch-scheduler/BatchScheduler.h @@ -1,24 +1,21 @@ #pragma once +#include #include -#include #include #define DO_NOT_MIGRATE -98 #define DO_NOT_MIGRATE_DECISION \ - faabric::util::SchedulingDecision(DO_NOT_MIGRATE, DO_NOT_MIGRATE) + SchedulingDecision(DO_NOT_MIGRATE, DO_NOT_MIGRATE) #define NOT_ENOUGH_SLOTS -99 #define NOT_ENOUGH_SLOTS_DECISION \ - faabric::util::SchedulingDecision(NOT_ENOUGH_SLOTS, NOT_ENOUGH_SLOTS) + SchedulingDecision(NOT_ENOUGH_SLOTS, NOT_ENOUGH_SLOTS) namespace faabric::batch_scheduler { -// TODO: move BatchExecuteRequest here - -// TODO: move SchedulingDecision here? typedef std::pair, - std::shared_ptr> + std::shared_ptr> InFlightPair; typedef std::map InFlightReqs; @@ -72,8 +69,7 @@ class BatchScheduler const InFlightReqs& inFlightReqs, std::shared_ptr req); - virtual std::shared_ptr - makeSchedulingDecision( + virtual std::shared_ptr makeSchedulingDecision( const HostMap& hostMap, const InFlightReqs& inFlightReqs, std::shared_ptr req) = 0; @@ -111,8 +107,8 @@ class BatchScheduler // ---------- virtual bool isFirstDecisionBetter( - std::shared_ptr decisionA, - std::shared_ptr decisionB) = 0; + std::shared_ptr decisionA, + std::shared_ptr decisionB) = 0; virtual std::vector getSortedHosts( const HostMap& hostMap, diff --git a/include/faabric/batch-scheduler/BinPackScheduler.h b/include/faabric/batch-scheduler/BinPackScheduler.h index 8bc4670c4..ad7692c36 100644 --- a/include/faabric/batch-scheduler/BinPackScheduler.h +++ b/include/faabric/batch-scheduler/BinPackScheduler.h @@ -1,8 +1,8 @@ #pragma once #include +#include #include -#include #include namespace faabric::batch_scheduler { @@ -10,15 +10,15 @@ namespace faabric::batch_scheduler { class BinPackScheduler final : public BatchScheduler { public: - std::shared_ptr makeSchedulingDecision( + std::shared_ptr makeSchedulingDecision( const HostMap& hostMap, const InFlightReqs& inFlightReqs, std::shared_ptr req) override; private: bool isFirstDecisionBetter( - std::shared_ptr decisionA, - std::shared_ptr decisionB) override; + std::shared_ptr decisionA, + std::shared_ptr decisionB) override; std::vector getSortedHosts( const HostMap& hostMap, diff --git a/include/faabric/batch-scheduler/DecisionCache.h b/include/faabric/batch-scheduler/DecisionCache.h new file mode 100644 index 000000000..9ef6f55a8 --- /dev/null +++ b/include/faabric/batch-scheduler/DecisionCache.h @@ -0,0 +1,52 @@ +#pragma once + +#include + +#include +#include + +namespace faabric::batch_scheduler { +/** + * A record of a decision already taken for the given size of batch request + * for the given function. This doesn't contain the messages themselves, + * just the hosts and group ID that was used. + */ +class CachedDecision +{ + public: + CachedDecision(const std::vector& hostsIn, int groupIdIn); + + std::vector getHosts() { return hosts; } + + int getGroupId() const { return groupId; } + + private: + std::vector hosts; + int groupId = 0; +}; + +/** + * Repository for cached scheduling decisions. Object is not thread safe as we + * assume only a single executor will be caching decisions for a given function + * and size of batch request on one host at a time. + */ +class DecisionCache +{ + public: + std::shared_ptr getCachedDecision( + std::shared_ptr req); + + void addCachedDecision(std::shared_ptr req, + SchedulingDecision& decision); + + void clear(); + + private: + std::string getCacheKey(std::shared_ptr req); + + std::unordered_map> + cachedDecisions; +}; + +DecisionCache& getSchedulingDecisionCache(); +} diff --git a/include/faabric/util/scheduling.h b/include/faabric/batch-scheduler/SchedulingDecision.h similarity index 69% rename from include/faabric/util/scheduling.h rename to include/faabric/batch-scheduler/SchedulingDecision.h index 566dada9d..08660c5b7 100644 --- a/include/faabric/util/scheduling.h +++ b/include/faabric/batch-scheduler/SchedulingDecision.h @@ -1,58 +1,8 @@ #pragma once -#include -#include -#include -#include - #include -#include - -namespace faabric::util { - -class SchedulingDecision -{ - public: - static SchedulingDecision fromPointToPointMappings( - faabric::PointToPointMappings& mappings); - - SchedulingDecision(uint32_t appIdIn, int32_t groupIdIn); - - uint32_t appId = 0; - - int32_t groupId = 0; - - int32_t nFunctions = 0; - - std::vector hosts; - - std::vector messageIds; - - std::vector appIdxs; - - std::vector groupIdxs; - - std::string returnHost; - - /** - * Work out if this decision is all on this host. If the decision is - * completely on *another* host, we still count it as not being on a single - * host, as this host will be the master. - * - * Will always return false if single host optimisations are switched off. - */ - bool isSingleHost(); - - void addMessage(const std::string& host, const faabric::Message& msg); - - void addMessage(const std::string& host, int32_t messageId, int32_t appIdx); - - void addMessage(const std::string& host, - int32_t messageId, - int32_t appIdx, - int32_t groupIdx); -}; +namespace faabric::batch_scheduler { // Scheduling topology hints help the scheduler decide which host to assign new // requests in a batch. // - NONE: bin-packs requests to slots in hosts starting from the master @@ -92,59 +42,60 @@ const std::unordered_map { SchedulingTopologyHint::UNDERFULL, "UNDERFULL" }, }; -/** - * A record of a decision already taken for the given size of batch request - * for the given function. This doesn't contain the messages themselves, - * just the hosts and group ID that was used. - */ -class CachedDecision +// TODO(planner-schedule): remove these strategies +// Migration strategies help the scheduler decide wether the scheduling decision +// for a batch request could be changed with the new set of available resources. +// - BIN_PACK: sort hosts by the number of functions from the batch they are +// running. Bin-pack batches in increasing order to hosts in +// decreasing order. +// - EMPTY_HOSTS: pack batches in increasing order to empty hosts. +enum MigrationStrategy +{ + BIN_PACK, + EMPTY_HOSTS +}; + +class SchedulingDecision { public: - CachedDecision(const std::vector& hostsIn, int groupIdIn); + static SchedulingDecision fromPointToPointMappings( + faabric::PointToPointMappings& mappings); - std::vector getHosts() { return hosts; } + SchedulingDecision(uint32_t appIdIn, int32_t groupIdIn); - int getGroupId() const { return groupId; } + uint32_t appId = 0; + + int32_t groupId = 0; + + int32_t nFunctions = 0; - private: std::vector hosts; - int groupId = 0; -}; -/** - * Repository for cached scheduling decisions. Object is not thread safe as we - * assume only a single executor will be caching decisions for a given function - * and size of batch request on one host at a time. - */ -class DecisionCache -{ - public: - std::shared_ptr getCachedDecision( - std::shared_ptr req); + std::vector messageIds; - void addCachedDecision(std::shared_ptr req, - faabric::util::SchedulingDecision& decision); + std::vector appIdxs; - void clear(); + std::vector groupIdxs; - private: - std::string getCacheKey(std::shared_ptr req); + std::string returnHost; - std::unordered_map> - cachedDecisions; -}; + /** + * Work out if this decision is all on this host. If the decision is + * completely on *another* host, we still count it as not being on a single + * host, as this host will be the master. + * + * Will always return false if single host optimisations are switched off. + */ + bool isSingleHost(); -DecisionCache& getSchedulingDecisionCache(); + void addMessage(const std::string& host, const faabric::Message& msg); -// Migration strategies help the scheduler decide wether the scheduling decision -// for a batch request could be changed with the new set of available resources. -// - BIN_PACK: sort hosts by the number of functions from the batch they are -// running. Bin-pack batches in increasing order to hosts in -// decreasing order. -// - EMPTY_HOSTS: pack batches in increasing order to empty hosts. -enum MigrationStrategy -{ - BIN_PACK, - EMPTY_HOSTS + void addMessage(const std::string& host, int32_t messageId, int32_t appIdx); + + void addMessage(const std::string& host, + int32_t messageId, + int32_t appIdx, + int32_t groupIdx); }; + } diff --git a/include/faabric/scheduler/Scheduler.h b/include/faabric/scheduler/Scheduler.h index 34d69d55c..5e83c3492 100644 --- a/include/faabric/scheduler/Scheduler.h +++ b/include/faabric/scheduler/Scheduler.h @@ -1,5 +1,6 @@ #pragma once +#include #include #include #include @@ -14,7 +15,6 @@ #include #include #include -#include #include #include @@ -26,7 +26,7 @@ namespace faabric::scheduler { typedef std::pair, - std::shared_ptr> + std::shared_ptr> InFlightPair; class Scheduler; @@ -186,17 +186,17 @@ class Scheduler ~Scheduler(); - faabric::util::SchedulingDecision makeSchedulingDecision( + faabric::batch_scheduler::SchedulingDecision makeSchedulingDecision( std::shared_ptr req, - faabric::util::SchedulingTopologyHint topologyHint = - faabric::util::SchedulingTopologyHint::NONE); + faabric::batch_scheduler::SchedulingTopologyHint topologyHint = + faabric::batch_scheduler::SchedulingTopologyHint::NONE); - faabric::util::SchedulingDecision callFunctions( + faabric::batch_scheduler::SchedulingDecision callFunctions( std::shared_ptr req); - faabric::util::SchedulingDecision callFunctions( + faabric::batch_scheduler::SchedulingDecision callFunctions( std::shared_ptr req, - faabric::util::SchedulingDecision& hint); + faabric::batch_scheduler::SchedulingDecision& hint); void reset(); @@ -351,15 +351,15 @@ class Scheduler std::unordered_map> registeredHosts; - faabric::util::SchedulingDecision doSchedulingDecision( + faabric::batch_scheduler::SchedulingDecision doSchedulingDecision( std::shared_ptr req, - faabric::util::SchedulingTopologyHint topologyHint); + faabric::batch_scheduler::SchedulingTopologyHint topologyHint); - faabric::util::SchedulingDecision doCallFunctions( + faabric::batch_scheduler::SchedulingDecision doCallFunctions( std::shared_ptr req, - faabric::util::SchedulingDecision& decision, + faabric::batch_scheduler::SchedulingDecision& decision, faabric::util::FullLock& lock, - faabric::util::SchedulingTopologyHint topologyHint); + faabric::batch_scheduler::SchedulingTopologyHint topologyHint); std::shared_ptr claimExecutor( faabric::Message& msg, @@ -386,15 +386,15 @@ class Scheduler std::vector> doCheckForMigrationOpportunities( - faabric::util::MigrationStrategy migrationStrategy = - faabric::util::MigrationStrategy::BIN_PACK); + faabric::batch_scheduler::MigrationStrategy migrationStrategy = + faabric::batch_scheduler::MigrationStrategy::BIN_PACK); void broadcastPendingMigrations( std::shared_ptr pendingMigrations); void doStartFunctionMigrationThread( std::shared_ptr req, - faabric::util::SchedulingDecision& decision); + faabric::batch_scheduler::SchedulingDecision& decision); }; } diff --git a/include/faabric/transport/PointToPointBroker.h b/include/faabric/transport/PointToPointBroker.h index daddab4b2..3ec6c01b3 100644 --- a/include/faabric/transport/PointToPointBroker.h +++ b/include/faabric/transport/PointToPointBroker.h @@ -1,9 +1,9 @@ #pragma once +#include #include #include #include -#include #include #include @@ -93,10 +93,10 @@ class PointToPointBroker std::string getHostForReceiver(int groupId, int recvIdx); std::set setUpLocalMappingsFromSchedulingDecision( - const faabric::util::SchedulingDecision& decision); + const faabric::batch_scheduler::SchedulingDecision& decision); void setAndSendMappingsFromSchedulingDecision( - const faabric::util::SchedulingDecision& decision); + const faabric::batch_scheduler::SchedulingDecision& decision); void waitForMappingsOnThisHost(int groupId); diff --git a/src/batch-scheduler/BatchScheduler.cpp b/src/batch-scheduler/BatchScheduler.cpp index be4eb3b55..71bbf699b 100644 --- a/src/batch-scheduler/BatchScheduler.cpp +++ b/src/batch-scheduler/BatchScheduler.cpp @@ -1,6 +1,7 @@ #include #include #include +#include namespace faabric::batch_scheduler { diff --git a/src/batch-scheduler/BinPackScheduler.cpp b/src/batch-scheduler/BinPackScheduler.cpp index 90ff92dd3..065ce7d57 100644 --- a/src/batch-scheduler/BinPackScheduler.cpp +++ b/src/batch-scheduler/BinPackScheduler.cpp @@ -1,12 +1,12 @@ #include +#include #include #include -#include namespace faabric::batch_scheduler { static std::map getHostFreqCount( - std::shared_ptr decision) + std::shared_ptr decision) { std::map hostFreqCount; for (auto host : decision->hosts) { @@ -21,13 +21,12 @@ static std::map getHostFreqCount( // This is, we want to keep as many host-message scheduling in the old decision // as possible, and also have the overall locality of the new decision (i.e. // the host-message histogram) -static std::shared_ptr -minimiseNumOfMigrations( - std::shared_ptr newDecision, - std::shared_ptr oldDecision) +static std::shared_ptr minimiseNumOfMigrations( + std::shared_ptr newDecision, + std::shared_ptr oldDecision) { - auto decision = std::make_shared( - oldDecision->appId, oldDecision->groupId); + auto decision = std::make_shared(oldDecision->appId, + oldDecision->groupId); // We want to maintain the new decision's host-message histogram auto hostFreqCount = getHostFreqCount(newDecision); @@ -100,12 +99,11 @@ minimiseNumOfMigrations( // less hosts. In case of a tie, we calculate the number of cross-VM links // (i.e. better locality, or better packing) bool BinPackScheduler::isFirstDecisionBetter( - std::shared_ptr decisionA, - std::shared_ptr decisionB) + std::shared_ptr decisionA, + std::shared_ptr decisionB) { auto getLocalityScore = - [](std::shared_ptr decision) - -> std::pair { + [](std::shared_ptr decision) -> std::pair { // First, calculate the host-message histogram (or frequency count) std::map hostFreqCount; for (auto host : decision->hosts) { @@ -253,15 +251,12 @@ std::vector BinPackScheduler::getSortedHosts( // hosts (i.e. bins) in a specific order (depending on the scheduling type), // and then starts filling bins from begining to end, until it runs out of // messages to schedule -std::shared_ptr -BinPackScheduler::makeSchedulingDecision( +std::shared_ptr BinPackScheduler::makeSchedulingDecision( const HostMap& hostMap, const InFlightReqs& inFlightReqs, std::shared_ptr req) { - // TODO: think about the group id! - auto decision = - std::make_shared(req->appid(), 0); + auto decision = std::make_shared(req->appid(), 0); // Get the sorted list of hosts auto decisionType = getDecisionType(inFlightReqs, req); @@ -296,8 +291,7 @@ BinPackScheduler::makeSchedulingDecision( // If we still have enough slots to schedule, we are out of slots if (numLeftToSchedule > 0) { - return std::make_shared( - NOT_ENOUGH_SLOTS_DECISION); + return std::make_shared(NOT_ENOUGH_SLOTS_DECISION); } // In case of a DIST_CHANGE decision (i.e. migration), we want to make sure @@ -310,8 +304,7 @@ BinPackScheduler::makeSchedulingDecision( return minimiseNumOfMigrations(decision, oldDecision); } - return std::make_shared( - DO_NOT_MIGRATE_DECISION); + return std::make_shared(DO_NOT_MIGRATE_DECISION); } return decision; diff --git a/src/batch-scheduler/CMakeLists.txt b/src/batch-scheduler/CMakeLists.txt index e854960f1..3e2a346e1 100644 --- a/src/batch-scheduler/CMakeLists.txt +++ b/src/batch-scheduler/CMakeLists.txt @@ -1,3 +1,8 @@ +faabric_lib(scheduling_util + DecisionCache.cpp + SchedulingDecision.cpp +) + faabric_lib(batch_scheduler BatchScheduler.cpp BinPackScheduler.cpp @@ -5,4 +10,5 @@ faabric_lib(batch_scheduler target_link_libraries(batch_scheduler PRIVATE faabric::util + faabric::scheduling_util ) diff --git a/src/util/scheduling.cpp b/src/batch-scheduler/DecisionCache.cpp similarity index 54% rename from src/util/scheduling.cpp rename to src/batch-scheduler/DecisionCache.cpp index b670ab777..f52cc487f 100644 --- a/src/util/scheduling.cpp +++ b/src/batch-scheduler/DecisionCache.cpp @@ -1,10 +1,9 @@ -#include +#include #include -#include +#include #include -namespace faabric::util { - +namespace faabric::batch_scheduler { CachedDecision::CachedDecision(const std::vector& hostsIn, int groupIdIn) : hosts(hostsIn) @@ -12,7 +11,7 @@ CachedDecision::CachedDecision(const std::vector& hostsIn, {} std::shared_ptr DecisionCache::getCachedDecision( - std::shared_ptr req) + std::shared_ptr req) { std::string cacheKey = getCacheKey(req); bool hasDecision = cachedDecisions.find(cacheKey) != cachedDecisions.end(); @@ -36,15 +35,14 @@ std::shared_ptr DecisionCache::getCachedDecision( return nullptr; } -void DecisionCache::addCachedDecision( - std::shared_ptr req, - faabric::util::SchedulingDecision& decision) +void DecisionCache::addCachedDecision(std::shared_ptr req, + SchedulingDecision& decision) { std::string cacheKey = getCacheKey(req); if (req->messages_size() != decision.hosts.size()) { SPDLOG_ERROR("Trying to cache decision for {} with wrong size {} != {}", - funcToString(req), + faabric::util::funcToString(req), req->messages_size(), decision.hosts.size()); throw std::runtime_error("Invalid decision caching"); @@ -78,54 +76,4 @@ DecisionCache& getSchedulingDecisionCache() static DecisionCache c; return c; } - -SchedulingDecision::SchedulingDecision(uint32_t appIdIn, int32_t groupIdIn) - : appId(appIdIn) - , groupId(groupIdIn) -{} - -bool SchedulingDecision::isSingleHost() -{ - // Always return false if single-host optimisations are switched off - SystemConfig& conf = getSystemConfig(); - if (conf.noSingleHostOptimisations == 1) { - return false; - } - - std::string thisHost = conf.endpointHost; - return std::all_of(hosts.begin(), hosts.end(), [&](const std::string& s) { - return s == thisHost; - }); -} - -void SchedulingDecision::addMessage(const std::string& host, - const faabric::Message& msg) -{ - addMessage(host, msg.id(), msg.appidx(), msg.groupidx()); -} - -void SchedulingDecision::addMessage(const std::string& host, - int32_t messageId, - int32_t appIdx, - int32_t groupIdx) -{ - nFunctions++; - - hosts.emplace_back(host); - messageIds.emplace_back(messageId); - appIdxs.emplace_back(appIdx); - groupIdxs.emplace_back(groupIdx); -} - -SchedulingDecision SchedulingDecision::fromPointToPointMappings( - faabric::PointToPointMappings& mappings) -{ - SchedulingDecision decision(mappings.appid(), mappings.groupid()); - - for (const auto& m : mappings.mappings()) { - decision.addMessage(m.host(), m.messageid(), m.appidx(), m.groupidx()); - } - - return decision; -} } diff --git a/src/batch-scheduler/SchedulingDecision.cpp b/src/batch-scheduler/SchedulingDecision.cpp new file mode 100644 index 000000000..41325c963 --- /dev/null +++ b/src/batch-scheduler/SchedulingDecision.cpp @@ -0,0 +1,55 @@ +#include +#include + +namespace faabric::batch_scheduler { + +SchedulingDecision::SchedulingDecision(uint32_t appIdIn, int32_t groupIdIn) + : appId(appIdIn) + , groupId(groupIdIn) +{} + +bool SchedulingDecision::isSingleHost() +{ + // Always return false if single-host optimisations are switched off + faabric::util::SystemConfig& conf = faabric::util::getSystemConfig(); + if (conf.noSingleHostOptimisations == 1) { + return false; + } + + std::string thisHost = conf.endpointHost; + return std::all_of(hosts.begin(), hosts.end(), [&](const std::string& s) { + return s == thisHost; + }); +} + +void SchedulingDecision::addMessage(const std::string& host, + const faabric::Message& msg) +{ + addMessage(host, msg.id(), msg.appidx(), msg.groupidx()); +} + +void SchedulingDecision::addMessage(const std::string& host, + int32_t messageId, + int32_t appIdx, + int32_t groupIdx) +{ + nFunctions++; + + hosts.emplace_back(host); + messageIds.emplace_back(messageId); + appIdxs.emplace_back(appIdx); + groupIdxs.emplace_back(groupIdx); +} + +SchedulingDecision SchedulingDecision::fromPointToPointMappings( + faabric::PointToPointMappings& mappings) +{ + SchedulingDecision decision(mappings.appid(), mappings.groupid()); + + for (const auto& m : mappings.mappings()) { + decision.addMessage(m.host(), m.messageid(), m.appidx(), m.groupidx()); + } + + return decision; +} +} diff --git a/src/mpi/MpiWorld.cpp b/src/mpi/MpiWorld.cpp index d73afef94..fc6fe0cff 100644 --- a/src/mpi/MpiWorld.cpp +++ b/src/mpi/MpiWorld.cpp @@ -1,3 +1,4 @@ +#include #include #include #include @@ -7,7 +8,6 @@ #include #include #include -#include #include // Each MPI rank runs in a separate thread, thus we use TLS to maintain the @@ -180,12 +180,13 @@ void MpiWorld::create(faabric::Message& call, int newId, int newSize) // As a result of the call to the scheduler, a point-to-point communcation // group will have been created with id equal to the MPI world's id. if (size > 1) { - faabric::util::SchedulingDecision decision = sch.callFunctions(req); + faabric::batch_scheduler::SchedulingDecision decision = + sch.callFunctions(req); assert(decision.hosts.size() == size - 1); } else { // If world has size one, create the communication group (of size one) // manually. - faabric::util::SchedulingDecision decision(id, id); + faabric::batch_scheduler::SchedulingDecision decision(id, id); call.set_groupidx(0); decision.addMessage(thisHost, call); broker.setAndSendMappingsFromSchedulingDecision(decision); diff --git a/src/scheduler/CMakeLists.txt b/src/scheduler/CMakeLists.txt index 009b0f247..468638b15 100644 --- a/src/scheduler/CMakeLists.txt +++ b/src/scheduler/CMakeLists.txt @@ -8,6 +8,7 @@ faabric_lib(scheduler ) target_link_libraries(scheduler PRIVATE + faabric::scheduling_util faabric::snapshot faabric::state faabric::redis diff --git a/src/scheduler/Executor.cpp b/src/scheduler/Executor.cpp index 2cd529c7a..7db93f8c7 100644 --- a/src/scheduler/Executor.cpp +++ b/src/scheduler/Executor.cpp @@ -1,3 +1,4 @@ +#include #include #include #include @@ -16,7 +17,6 @@ #include #include #include -#include #include #include #include @@ -126,8 +126,9 @@ std::vector> Executor::executeThreads( } // Get the scheduling decision - faabric::util::SchedulingDecision decision = sch.makeSchedulingDecision( - req, faabric::util::SchedulingTopologyHint::CACHED); + faabric::batch_scheduler::SchedulingDecision decision = + sch.makeSchedulingDecision( + req, faabric::batch_scheduler::SchedulingTopologyHint::CACHED); bool isSingleHost = decision.isSingleHost(); // Do snapshotting if not on a single host diff --git a/src/scheduler/Scheduler.cpp b/src/scheduler/Scheduler.cpp index 3fd3fcc73..51ddc02f0 100644 --- a/src/scheduler/Scheduler.cpp +++ b/src/scheduler/Scheduler.cpp @@ -1,3 +1,5 @@ +#include +#include #include #include #include @@ -16,7 +18,6 @@ #include #include #include -#include #include #include #include @@ -354,7 +355,7 @@ void Scheduler::vacateSlot() thisHostUsedSlots.fetch_sub(1, std::memory_order_acq_rel); } -faabric::util::SchedulingDecision Scheduler::callFunctions( +faabric::batch_scheduler::SchedulingDecision Scheduler::callFunctions( std::shared_ptr req) { // We assume all the messages are for the same function and have the @@ -363,13 +364,15 @@ faabric::util::SchedulingDecision Scheduler::callFunctions( std::string masterHost = firstMsg.masterhost(); // Get topology hint from message - faabric::util::SchedulingTopologyHint topologyHint = + faabric::batch_scheduler::SchedulingTopologyHint topologyHint = firstMsg.topologyhint().empty() - ? faabric::util::SchedulingTopologyHint::NONE - : faabric::util::strToTopologyHint.at(firstMsg.topologyhint()); + ? faabric::batch_scheduler::SchedulingTopologyHint::NONE + : faabric::batch_scheduler::strToTopologyHint.at( + firstMsg.topologyhint()); bool isForceLocal = - topologyHint == faabric::util::SchedulingTopologyHint::FORCE_LOCAL; + topologyHint == + faabric::batch_scheduler::SchedulingTopologyHint::FORCE_LOCAL; // If we're not the master host, we need to forward the request back to the // master host. This will only happen if a nested batch execution happens. @@ -378,31 +381,33 @@ faabric::util::SchedulingDecision Scheduler::callFunctions( SPDLOG_DEBUG("Forwarding {} back to master {}", funcStr, masterHost); getFunctionCallClient(masterHost)->executeFunctions(req); - SchedulingDecision decision(firstMsg.appid(), firstMsg.groupid()); + faabric::batch_scheduler::SchedulingDecision decision( + firstMsg.appid(), firstMsg.groupid()); decision.returnHost = masterHost; return decision; } faabric::util::FullLock lock(mx); - SchedulingDecision decision = doSchedulingDecision(req, topologyHint); + faabric::batch_scheduler::SchedulingDecision decision = + doSchedulingDecision(req, topologyHint); // Pass decision as hint return doCallFunctions(req, decision, lock, topologyHint); } -faabric::util::SchedulingDecision Scheduler::makeSchedulingDecision( +faabric::batch_scheduler::SchedulingDecision Scheduler::makeSchedulingDecision( std::shared_ptr req, - faabric::util::SchedulingTopologyHint topologyHint) + faabric::batch_scheduler::SchedulingTopologyHint topologyHint) { faabric::util::FullLock lock(mx); return doSchedulingDecision(req, topologyHint); } -faabric::util::SchedulingDecision Scheduler::doSchedulingDecision( +faabric::batch_scheduler::SchedulingDecision Scheduler::doSchedulingDecision( std::shared_ptr req, - faabric::util::SchedulingTopologyHint topologyHint) + faabric::batch_scheduler::SchedulingTopologyHint topologyHint) { int nMessages = req->messages_size(); faabric::Message& firstMsg = req->mutable_messages()->at(0); @@ -410,18 +415,20 @@ faabric::util::SchedulingDecision Scheduler::doSchedulingDecision( // If topology hints are disabled, unset the provided topology hint if (conf.noTopologyHints == "on" && - topologyHint != faabric::util::SchedulingTopologyHint::NONE) { + topologyHint != + faabric::batch_scheduler::SchedulingTopologyHint::NONE) { SPDLOG_WARN("Ignoring topology hint passed to scheduler as hints are " "disabled in the config"); - topologyHint = faabric::util::SchedulingTopologyHint::NONE; + topologyHint = faabric::batch_scheduler::SchedulingTopologyHint::NONE; } // If requesting a cached decision, look for it now - faabric::util::DecisionCache& decisionCache = - faabric::util::getSchedulingDecisionCache(); - if (topologyHint == faabric::util::SchedulingTopologyHint::CACHED) { - std::shared_ptr cachedDecision = - decisionCache.getCachedDecision(req); + faabric::batch_scheduler::DecisionCache& decisionCache = + faabric::batch_scheduler::getSchedulingDecisionCache(); + if (topologyHint == + faabric::batch_scheduler::SchedulingTopologyHint::CACHED) { + std::shared_ptr + cachedDecision = decisionCache.getCachedDecision(req); if (cachedDecision != nullptr) { int groupId = cachedDecision->getGroupId(); @@ -434,8 +441,8 @@ faabric::util::SchedulingDecision Scheduler::doSchedulingDecision( std::vector hosts = cachedDecision->getHosts(); // Create the scheduling decision - faabric::util::SchedulingDecision decision(firstMsg.appid(), - groupId); + faabric::batch_scheduler::SchedulingDecision decision( + firstMsg.appid(), groupId); for (int i = 0; i < hosts.size(); i++) { // Reuse the group id faabric::Message& m = req->mutable_messages()->at(i); @@ -458,7 +465,8 @@ faabric::util::SchedulingDecision Scheduler::doSchedulingDecision( std::vector hosts; hosts.reserve(nMessages); - if (topologyHint == faabric::util::SchedulingTopologyHint::FORCE_LOCAL) { + if (topologyHint == + faabric::batch_scheduler::SchedulingTopologyHint::FORCE_LOCAL) { // We're forced to execute locally here so we do all the messages SPDLOG_TRACE("Scheduling {}/{} of {} locally (force local)", nMessages, @@ -474,7 +482,8 @@ faabric::util::SchedulingDecision Scheduler::doSchedulingDecision( // Work out how many we can handle locally int slots = thisHostResources.slots(); - if (topologyHint == faabric::util::SchedulingTopologyHint::UNDERFULL) { + if (topologyHint == + faabric::batch_scheduler::SchedulingTopologyHint::UNDERFULL) { slots = slots / 2; } @@ -514,8 +523,8 @@ faabric::util::SchedulingDecision Scheduler::doSchedulingDecision( // Under the NEVER_ALONE topology hint, we never choose a host // unless we can schedule at least two requests in it. - if (topologyHint == - faabric::util::SchedulingTopologyHint::NEVER_ALONE && + if (topologyHint == faabric::batch_scheduler:: + SchedulingTopologyHint::NEVER_ALONE && nOnThisHost < 2) { continue; } @@ -558,8 +567,8 @@ faabric::util::SchedulingDecision Scheduler::doSchedulingDecision( available = std::max(0, available); int nOnThisHost = std::min(available, remainder); - if (topologyHint == - faabric::util::SchedulingTopologyHint::NEVER_ALONE && + if (topologyHint == faabric::batch_scheduler:: + SchedulingTopologyHint::NEVER_ALONE && nOnThisHost < 2) { continue; } @@ -593,8 +602,8 @@ faabric::util::SchedulingDecision Scheduler::doSchedulingDecision( // Under the NEVER_ALONE scheduling topology hint we want to // overload the last host we assigned requests to. - if (topologyHint == - faabric::util::SchedulingTopologyHint::NEVER_ALONE && + if (topologyHint == faabric::batch_scheduler:: + SchedulingTopologyHint::NEVER_ALONE && !hosts.empty()) { overloadedHost = hosts.back(); } @@ -622,33 +631,35 @@ faabric::util::SchedulingDecision Scheduler::doSchedulingDecision( } // Set up decision - SchedulingDecision decision(firstMsg.appid(), firstMsg.groupid()); + faabric::batch_scheduler::SchedulingDecision decision(firstMsg.appid(), + firstMsg.groupid()); for (int i = 0; i < hosts.size(); i++) { decision.addMessage(hosts.at(i), req->messages().at(i)); } // Cache decision for next time if necessary - if (topologyHint == faabric::util::SchedulingTopologyHint::CACHED) { + if (topologyHint == + faabric::batch_scheduler::SchedulingTopologyHint::CACHED) { decisionCache.addCachedDecision(req, decision); } return decision; } -faabric::util::SchedulingDecision Scheduler::callFunctions( +faabric::batch_scheduler::SchedulingDecision Scheduler::callFunctions( std::shared_ptr req, - faabric::util::SchedulingDecision& hint) + faabric::batch_scheduler::SchedulingDecision& hint) { faabric::util::FullLock lock(mx); return doCallFunctions( - req, hint, lock, faabric::util::SchedulingTopologyHint::NONE); + req, hint, lock, faabric::batch_scheduler::SchedulingTopologyHint::NONE); } -faabric::util::SchedulingDecision Scheduler::doCallFunctions( +faabric::batch_scheduler::SchedulingDecision Scheduler::doCallFunctions( std::shared_ptr req, - faabric::util::SchedulingDecision& decision, + faabric::batch_scheduler::SchedulingDecision& decision, faabric::util::FullLock& lock, - faabric::util::SchedulingTopologyHint topologyHint) + faabric::batch_scheduler::SchedulingTopologyHint topologyHint) { faabric::Message& firstMsg = req->mutable_messages()->at(0); std::string funcStr = faabric::util::funcToString(firstMsg, false); @@ -674,7 +685,8 @@ faabric::util::SchedulingDecision Scheduler::doCallFunctions( // execute locally, in which case they will be transmitted from the // master) bool isForceLocal = - topologyHint == faabric::util::SchedulingTopologyHint::FORCE_LOCAL; + topologyHint == + faabric::batch_scheduler::SchedulingTopologyHint::FORCE_LOCAL; if (!isForceLocal && !isMigration && (firstMsg.groupid() > 0)) { if (firstMsg.ismpi()) { // If we are scheduling an MPI message, we want rank 0 to be in the @@ -1326,7 +1338,7 @@ void Scheduler::removePendingMigration(uint32_t appId) std::vector> Scheduler::doCheckForMigrationOpportunities( - faabric::util::MigrationStrategy migrationStrategy) + faabric::batch_scheduler::MigrationStrategy migrationStrategy) { std::vector> pendingMigrationsVec; @@ -1349,7 +1361,8 @@ Scheduler::doCheckForMigrationOpportunities( faabric::PendingMigrations msg; msg.set_appid(originalDecision.appId); - if (migrationStrategy == faabric::util::MigrationStrategy::BIN_PACK) { + if (migrationStrategy == + faabric::batch_scheduler::MigrationStrategy::BIN_PACK) { // We assume the batch was originally scheduled using // bin-packing, thus the scheduling decision has at the begining // (left) the hosts with the most allocated requests, and at the @@ -1437,7 +1450,7 @@ Scheduler::doCheckForMigrationOpportunities( // the actual check period instead to ease with experiments. void Scheduler::doStartFunctionMigrationThread( std::shared_ptr req, - faabric::util::SchedulingDecision& decision) + faabric::batch_scheduler::SchedulingDecision& decision) { bool startMigrationThread = inFlightRequests.size() == 0; faabric::Message& firstMsg = req->mutable_messages()->at(0); @@ -1471,7 +1484,8 @@ void Scheduler::doStartFunctionMigrationThread( } } else { auto decisionPtr = - std::make_shared(decision); + std::make_shared( + decision); inFlightRequests[decision.appId] = std::make_pair(req, decisionPtr); } diff --git a/src/transport/CMakeLists.txt b/src/transport/CMakeLists.txt index cc9218259..e8b7c339d 100644 --- a/src/transport/CMakeLists.txt +++ b/src/transport/CMakeLists.txt @@ -12,4 +12,8 @@ faabric_lib(transport PointToPointServer.cpp ) -target_link_libraries(transport PRIVATE faabric::util faabric::proto) +target_link_libraries(transport PRIVATE + faabric::util + faabric::proto + faabric::scheduling_util +) diff --git a/src/transport/PointToPointBroker.cpp b/src/transport/PointToPointBroker.cpp index 849e0520b..be07df067 100644 --- a/src/transport/PointToPointBroker.cpp +++ b/src/transport/PointToPointBroker.cpp @@ -393,7 +393,7 @@ std::string PointToPointBroker::getHostForReceiver(int groupId, int recvIdx) std::set PointToPointBroker::setUpLocalMappingsFromSchedulingDecision( - const faabric::util::SchedulingDecision& decision) + const faabric::batch_scheduler::SchedulingDecision& decision) { int groupId = decision.groupId; @@ -441,7 +441,7 @@ PointToPointBroker::setUpLocalMappingsFromSchedulingDecision( } void PointToPointBroker::setAndSendMappingsFromSchedulingDecision( - const faabric::util::SchedulingDecision& decision) + const faabric::batch_scheduler::SchedulingDecision& decision) { // Set up locally std::set otherHosts = diff --git a/src/transport/PointToPointServer.cpp b/src/transport/PointToPointServer.cpp index a62a021ec..173fec0bf 100644 --- a/src/transport/PointToPointServer.cpp +++ b/src/transport/PointToPointServer.cpp @@ -87,8 +87,9 @@ std::unique_ptr PointToPointServer::doRecvMappings( { PARSE_MSG(faabric::PointToPointMappings, buffer.data(), buffer.size()) - faabric::util::SchedulingDecision decision = - faabric::util::SchedulingDecision::fromPointToPointMappings(parsedMsg); + faabric::batch_scheduler::SchedulingDecision decision = + faabric::batch_scheduler::SchedulingDecision::fromPointToPointMappings( + parsedMsg); SPDLOG_DEBUG("Receiving {} point-to-point mappings", decision.nFunctions); diff --git a/src/util/CMakeLists.txt b/src/util/CMakeLists.txt index d9bf7d07e..ee674e3cf 100644 --- a/src/util/CMakeLists.txt +++ b/src/util/CMakeLists.txt @@ -22,7 +22,6 @@ faabric_lib(util network.cpp queue.cpp random.cpp - scheduling.cpp snapshot.cpp state.cpp string_tools.cpp diff --git a/tests/dist/mpi/mpi_native.cpp b/tests/dist/mpi/mpi_native.cpp index 7a0ec7c0e..db5ae88b8 100644 --- a/tests/dist/mpi/mpi_native.cpp +++ b/tests/dist/mpi/mpi_native.cpp @@ -844,7 +844,8 @@ void mpiMigrationPoint(int entrypointFuncArg) hostToMigrateTo); // Build decision and send - faabric::util::SchedulingDecision decision(msg.appid(), msg.groupid()); + faabric::batch_scheduler::SchedulingDecision decision(msg.appid(), + msg.groupid()); decision.addMessage(hostToMigrateTo, msg); sch.callFunctions(req, decision); diff --git a/tests/dist/scheduler/test_funcs.cpp b/tests/dist/scheduler/test_funcs.cpp index 313eb0dd8..1e876ba84 100644 --- a/tests/dist/scheduler/test_funcs.cpp +++ b/tests/dist/scheduler/test_funcs.cpp @@ -4,12 +4,12 @@ #include "faabric_utils.h" #include "init.h" +#include #include #include #include #include #include -#include namespace tests { @@ -33,15 +33,16 @@ TEST_CASE_METHOD(DistTestsFixture, // Set up the expectation const faabric::Message firstMsg = req->messages().at(0); - faabric::util::SchedulingDecision expectedDecision(firstMsg.appid(), - firstMsg.groupid()); + faabric::batch_scheduler::SchedulingDecision expectedDecision( + firstMsg.appid(), firstMsg.groupid()); expectedDecision.addMessage(thisHost, req->messages().at(0)); expectedDecision.addMessage(thisHost, req->messages().at(1)); expectedDecision.addMessage(otherHost, req->messages().at(2)); expectedDecision.addMessage(otherHost, req->messages().at(3)); // Call the functions - faabric::util::SchedulingDecision actualDecision = sch.callFunctions(req); + faabric::batch_scheduler::SchedulingDecision actualDecision = + sch.callFunctions(req); // Check decision is as expected checkSchedulingDecisionEquality(actualDecision, expectedDecision); diff --git a/tests/dist/scheduler/test_snapshots.cpp b/tests/dist/scheduler/test_snapshots.cpp index acc21b8ba..95d5e5cfa 100644 --- a/tests/dist/scheduler/test_snapshots.cpp +++ b/tests/dist/scheduler/test_snapshots.cpp @@ -6,6 +6,7 @@ #include +#include #include #include #include @@ -14,7 +15,6 @@ #include #include #include -#include #include namespace tests { @@ -51,7 +51,8 @@ TEST_CASE_METHOD(DistTestsFixture, sch.setThisHostResources(res); std::vector expectedHosts = { getWorkerIP() }; - faabric::util::SchedulingDecision decision = sch.callFunctions(req); + faabric::batch_scheduler::SchedulingDecision decision = + sch.callFunctions(req); std::vector executedHosts = decision.hosts; REQUIRE(expectedHosts == executedHosts); @@ -100,7 +101,8 @@ TEST_CASE_METHOD(DistTestsFixture, sch.setThisHostResources(res); std::vector expectedHosts = { getMasterIP() }; - faabric::util::SchedulingDecision decision = sch.callFunctions(req); + faabric::batch_scheduler::SchedulingDecision decision = + sch.callFunctions(req); std::vector executedHosts = decision.hosts; REQUIRE(expectedHosts == executedHosts); @@ -125,7 +127,8 @@ TEST_CASE_METHOD(DistTestsFixture, sch.setThisHostResources(res); std::vector expectedHosts = { getMasterIP() }; - faabric::util::SchedulingDecision decision = sch.callFunctions(req); + faabric::batch_scheduler::SchedulingDecision decision = + sch.callFunctions(req); std::vector executedHosts = decision.hosts; REQUIRE(expectedHosts == executedHosts); diff --git a/tests/dist/transport/functions.cpp b/tests/dist/transport/functions.cpp index 16e5d166d..e00a4216e 100644 --- a/tests/dist/transport/functions.cpp +++ b/tests/dist/transport/functions.cpp @@ -10,7 +10,6 @@ #include #include #include -#include #include using namespace faabric::transport; @@ -144,8 +143,7 @@ int handleDistributedLock(tests::DistTestExecutor* exec, } faabric::scheduler::Scheduler& sch = faabric::scheduler::getScheduler(); - faabric::util::SchedulingDecision decision = - sch.callFunctions(nestedReq); + auto decision = sch.callFunctions(nestedReq); // Await results bool success = true; diff --git a/tests/dist/transport/test_coordination.cpp b/tests/dist/transport/test_coordination.cpp index 7718892ad..c80bc4ee0 100644 --- a/tests/dist/transport/test_coordination.cpp +++ b/tests/dist/transport/test_coordination.cpp @@ -9,7 +9,6 @@ #include #include #include -#include namespace tests { diff --git a/tests/dist/transport/test_point_to_point.cpp b/tests/dist/transport/test_point_to_point.cpp index 32bd35b32..64d5276bd 100644 --- a/tests/dist/transport/test_point_to_point.cpp +++ b/tests/dist/transport/test_point_to_point.cpp @@ -4,12 +4,12 @@ #include "faabric_utils.h" #include "init.h" +#include #include #include #include #include #include -#include namespace tests { @@ -38,11 +38,12 @@ class PointToPointDistTestFixture : public DistTestsFixture sch.setThisHostResources(res); } - faabric::util::SchedulingDecision prepareRequestReturnDecision( + faabric::batch_scheduler::SchedulingDecision prepareRequestReturnDecision( std::shared_ptr req) { // Prepare expected decision - faabric::util::SchedulingDecision expectedDecision(appId, groupId); + faabric::batch_scheduler::SchedulingDecision expectedDecision(appId, + groupId); std::vector expectedHosts(nFuncs, getWorkerIP()); for (int i = 0; i < nLocalSlots; i++) { expectedHosts.at(i) = getMasterIP(); @@ -67,8 +68,8 @@ class PointToPointDistTestFixture : public DistTestsFixture void checkReturnCodesAndSchedulingDecision( std::shared_ptr req, - faabric::util::SchedulingDecision& expectedDecision, - faabric::util::SchedulingDecision& actualDecision) + faabric::batch_scheduler::SchedulingDecision& expectedDecision, + faabric::batch_scheduler::SchedulingDecision& actualDecision) { checkSchedulingDecisionEquality(actualDecision, expectedDecision); @@ -98,11 +99,12 @@ TEST_CASE_METHOD(PointToPointDistTestFixture, // Set up batch request and scheduling decision std::shared_ptr req = faabric::util::batchExecFactory("ptp", "simple", nFuncs); - faabric::util::SchedulingDecision expectedDecision = + faabric::batch_scheduler::SchedulingDecision expectedDecision = prepareRequestReturnDecision(req); // Call the functions - faabric::util::SchedulingDecision actualDecision = sch.callFunctions(req); + faabric::batch_scheduler::SchedulingDecision actualDecision = + sch.callFunctions(req); // Check for equality checkReturnCodesAndSchedulingDecision( @@ -118,11 +120,12 @@ TEST_CASE_METHOD(PointToPointDistTestFixture, // Set up batch request std::shared_ptr req = faabric::util::batchExecFactory("ptp", "many-msg", nFuncs); - faabric::util::SchedulingDecision expectedDecision = + faabric::batch_scheduler::SchedulingDecision expectedDecision = prepareRequestReturnDecision(req); // Call the functions - faabric::util::SchedulingDecision actualDecision = sch.callFunctions(req); + faabric::batch_scheduler::SchedulingDecision actualDecision = + sch.callFunctions(req); // Check for equality checkReturnCodesAndSchedulingDecision( diff --git a/tests/test/batch-scheduler/test_batch_scheduler.cpp b/tests/test/batch-scheduler/test_batch_scheduler.cpp index 56a867873..aeda8898a 100644 --- a/tests/test/batch-scheduler/test_batch_scheduler.cpp +++ b/tests/test/batch-scheduler/test_batch_scheduler.cpp @@ -62,7 +62,7 @@ TEST_CASE_METHOD(ConfFixture, SECTION("Dist-change decision") { auto decisionPtr = - std::make_shared(ber->appid(), 0); + std::make_shared(ber->appid(), 0); ber->set_type(BatchExecuteRequest_BatchExecuteType_MIGRATION); inFlightReqs[ber->appid()] = std::make_pair(ber, decisionPtr); expectedDecisionType = DecisionType::DIST_CHANGE; @@ -71,7 +71,7 @@ TEST_CASE_METHOD(ConfFixture, SECTION("Scale-change decision") { auto decisionPtr = - std::make_shared(ber->appid(), 0); + std::make_shared(ber->appid(), 0); auto newBer = faabric::util::batchExecFactory("foo", "bar", 1); newBer->set_appid(ber->appid()); inFlightReqs[newBer->appid()] = std::make_pair(newBer, decisionPtr); diff --git a/tests/test/batch-scheduler/test_binpack_scheduler.cpp b/tests/test/batch-scheduler/test_binpack_scheduler.cpp index 883f60bd4..4254852c8 100644 --- a/tests/test/batch-scheduler/test_binpack_scheduler.cpp +++ b/tests/test/batch-scheduler/test_binpack_scheduler.cpp @@ -28,7 +28,7 @@ TEST_CASE_METHOD(BinPackSchedulerTestFixture, BatchSchedulerConfig config = { .hostMap = {}, .inFlightReqs = {}, - .expectedDecision = faabric::util::SchedulingDecision(appId, groupId), + .expectedDecision = SchedulingDecision(appId, groupId), }; SECTION("BinPack scheduler gives up if not enough slots are available") @@ -135,7 +135,7 @@ TEST_CASE_METHOD(BinPackSchedulerTestFixture, BatchSchedulerConfig config = { .hostMap = {}, .inFlightReqs = {}, - .expectedDecision = faabric::util::SchedulingDecision(appId, groupId), + .expectedDecision = SchedulingDecision(appId, groupId), }; // The configs in this test must be read as follows: @@ -302,7 +302,7 @@ TEST_CASE_METHOD(BinPackSchedulerTestFixture, BatchSchedulerConfig config = { .hostMap = {}, .inFlightReqs = {}, - .expectedDecision = faabric::util::SchedulingDecision(appId, groupId), + .expectedDecision = SchedulingDecision(appId, groupId), }; // The configs in this test must be read as follows: diff --git a/tests/test/util/test_scheduling.cpp b/tests/test/batch-scheduler/test_scheduling_decisions.cpp similarity index 91% rename from tests/test/util/test_scheduling.cpp rename to tests/test/batch-scheduler/test_scheduling_decisions.cpp index a5862ee91..56e6816c1 100644 --- a/tests/test/util/test_scheduling.cpp +++ b/tests/test/batch-scheduler/test_scheduling_decisions.cpp @@ -3,11 +3,10 @@ #include "faabric_utils.h" #include "fixtures.h" -#include -#include -#include +#include +#include -using namespace faabric::util; +using namespace faabric::batch_scheduler; namespace tests { @@ -20,7 +19,6 @@ TEST_CASE_METHOD(ConfFixture, "Test building scheduling decisions", "[util]") std::string hostB = "hostB"; std::string hostC = "hostC"; - SystemConfig& conf = getSystemConfig(); std::string thisHost = conf.endpointHost; bool expectSingleHost = false; @@ -54,7 +52,7 @@ TEST_CASE_METHOD(ConfFixture, "Test building scheduling decisions", "[util]") expectSingleHost = false; } - auto req = batchExecFactory("foo", "bar", 3); + auto req = faabric::util::batchExecFactory("foo", "bar", 3); SchedulingDecision decision(appId, groupId); @@ -118,8 +116,7 @@ TEST_CASE("Test converting point-to-point mappings to scheduling decisions", mappingB->set_appidx(appIdxB); mappingB->set_groupidx(groupIdxB); - auto actual = - faabric::util::SchedulingDecision::fromPointToPointMappings(mappings); + auto actual = SchedulingDecision::fromPointToPointMappings(mappings); REQUIRE(actual.appId == appId); REQUIRE(actual.nFunctions == 2); @@ -139,7 +136,7 @@ TEST_CASE_METHOD(CachedDecisionTestFixture, std::string thisHost = faabric::util::getSystemConfig().endpointHost; - auto req = batchExecFactory("foo", "bar", 5); + auto req = faabric::util::batchExecFactory("foo", "bar", 5); std::vector hosts = { "alpha", "alpha", "beta", "gamma", "alpha", }; @@ -172,7 +169,7 @@ TEST_CASE_METHOD(CachedDecisionTestFixture, "Test caching invalid decision causes error", "[util]") { - auto req = batchExecFactory("foo", "bar", 3); + auto req = faabric::util::batchExecFactory("foo", "bar", 3); // Decision with wrong number of hosts std::vector hosts = { "alpha", "alpha" }; @@ -189,8 +186,8 @@ TEST_CASE_METHOD(CachedDecisionTestFixture, "Test caching multiple decisions for same function", "[util]") { - auto reqA = batchExecFactory("foo", "bar", 3); - auto reqB = batchExecFactory("foo", "bar", 5); + auto reqA = faabric::util::batchExecFactory("foo", "bar", 3); + auto reqB = faabric::util::batchExecFactory("foo", "bar", 5); // Decision with wrong number of hosts std::vector hostsA = { "alpha", "alpha", "beta" }; diff --git a/tests/test/scheduler/test_executor.cpp b/tests/test/scheduler/test_executor.cpp index 7eeededeb..eafb069d6 100644 --- a/tests/test/scheduler/test_executor.cpp +++ b/tests/test/scheduler/test_executor.cpp @@ -307,7 +307,7 @@ class TestExecutorFixture std::vector executeWithTestExecutorHint( std::shared_ptr req, - faabric::util::SchedulingDecision hint) + faabric::batch_scheduler::SchedulingDecision hint) { initThreadSnapshot(req); @@ -444,7 +444,7 @@ TEST_CASE_METHOD(TestExecutorFixture, } // Set up a hint to force the scheduler to execute single host or not - SchedulingDecision hint(123, 345); + faabric::batch_scheduler::SchedulingDecision hint(123, 345); std::vector expectedHosts; for (int i = 0; i < nLocally; i++) { expectedHosts.emplace_back(thisHost); @@ -1105,7 +1105,7 @@ TEST_CASE_METHOD(TestExecutorFixture, singleHosts[2] = otherHost; } - SchedulingDecision hint(123, 123); + faabric::batch_scheduler::SchedulingDecision hint(123, 123); for (int i = 0; i < nMessages; i++) { hint.addMessage(singleHosts[i], req->messages().at(i)); msgIds.push_back(req->messages(i).id()); diff --git a/tests/test/scheduler/test_scheduler.cpp b/tests/test/scheduler/test_scheduler.cpp index b11306620..c8c93be62 100644 --- a/tests/test/scheduler/test_scheduler.cpp +++ b/tests/test/scheduler/test_scheduler.cpp @@ -4,6 +4,7 @@ #include "faabric_utils.h" #include "fixtures.h" +#include #include #include #include @@ -19,7 +20,6 @@ #include #include #include -#include #include #include @@ -288,8 +288,8 @@ TEST_CASE_METHOD(SlowExecutorTestFixture, // Set up the messages std::vector reqOneMsgIds; - faabric::util::SchedulingDecision expectedDecisionOne(firstMsg.appid(), - firstMsg.groupid()); + faabric::batch_scheduler::SchedulingDecision expectedDecisionOne( + firstMsg.appid(), firstMsg.groupid()); for (int i = 0; i < nCallsOne; i++) { // Set snapshot key faabric::Message& msg = reqOne->mutable_messages()->at(i); @@ -313,7 +313,7 @@ TEST_CASE_METHOD(SlowExecutorTestFixture, reqOne->set_subtype(expectedSubType); reqOne->set_contextdata(expectedContextData); - faabric::util::SchedulingDecision actualDecisionOne = + faabric::batch_scheduler::SchedulingDecision actualDecisionOne = sch.callFunctions(reqOne); // Check decision is as expected @@ -389,8 +389,8 @@ TEST_CASE_METHOD(SlowExecutorTestFixture, std::vector reqTwoMsgIds; const faabric::Message& firstMsg2 = reqTwo->messages().at(0); - faabric::util::SchedulingDecision expectedDecisionTwo(appId, - firstMsg2.groupid()); + faabric::batch_scheduler::SchedulingDecision expectedDecisionTwo( + appId, firstMsg2.groupid()); for (int i = 0; i < nCallsTwo; i++) { faabric::Message& msg = reqTwo->mutable_messages()->at(i); @@ -411,7 +411,7 @@ TEST_CASE_METHOD(SlowExecutorTestFixture, reqTwo->set_type(execMode); // Schedule the functions - faabric::util::SchedulingDecision actualDecisionTwo = + faabric::batch_scheduler::SchedulingDecision actualDecisionTwo = sch.callFunctions(reqTwo); // Check scheduling decision @@ -507,8 +507,8 @@ TEST_CASE_METHOD(SlowExecutorTestFixture, // Make the request req->set_type(execMode); const faabric::Message firstMsg = req->messages().at(0); - faabric::util::SchedulingDecision expectedDecision(firstMsg.appid(), - firstMsg.groupid()); + faabric::batch_scheduler::SchedulingDecision expectedDecision( + firstMsg.appid(), firstMsg.groupid()); std::vector msgToWait; for (int i = 0; i < nCalls; i++) { faabric::Message& msg = req->mutable_messages()->at(i); @@ -526,7 +526,8 @@ TEST_CASE_METHOD(SlowExecutorTestFixture, } // Submit the request - faabric::util::SchedulingDecision decision = sch.callFunctions(req); + faabric::batch_scheduler::SchedulingDecision decision = + sch.callFunctions(req); checkSchedulingDecisionEquality(decision, expectedDecision); // Check status of local queueing @@ -791,7 +792,8 @@ TEST_CASE_METHOD(SlowExecutorTestFixture, faabric::util::batchExecFactory("blah", "foo", 1); req->mutable_messages()->at(0).set_masterhost(otherHost); - faabric::util::SchedulingDecision decision = sch.callFunctions(req); + faabric::batch_scheduler::SchedulingDecision decision = + sch.callFunctions(req); REQUIRE(decision.hosts.empty()); REQUIRE(decision.returnHost == otherHost); @@ -1018,7 +1020,8 @@ TEST_CASE_METHOD(DummyExecutorTestFixture, expectedHosts = { thisHost, thisHost, thisHost, thisHost }; } - faabric::util::SchedulingDecision expectedDecision(appId, groupId); + faabric::batch_scheduler::SchedulingDecision expectedDecision(appId, + groupId); std::vector msgIds; for (int i = 0; i < req->messages().size(); i++) { @@ -1036,7 +1039,8 @@ TEST_CASE_METHOD(DummyExecutorTestFixture, } // Schedule and check decision - faabric::util::SchedulingDecision actualDecision = sch.callFunctions(req); + faabric::batch_scheduler::SchedulingDecision actualDecision = + sch.callFunctions(req); checkSchedulingDecisionEquality(expectedDecision, actualDecision); // Check mappings set up locally or not diff --git a/tests/test/scheduler/test_scheduling_decisions.cpp b/tests/test/scheduler/test_scheduling_decisions.cpp index ced090a35..92f858dc0 100644 --- a/tests/test/scheduler/test_scheduling_decisions.cpp +++ b/tests/test/scheduler/test_scheduling_decisions.cpp @@ -36,7 +36,7 @@ class SchedulingDecisionTestFixture : public SchedulerFixture std::vector slots; std::vector used; int numReqs; - faabric::util::SchedulingTopologyHint topologyHint; + faabric::batch_scheduler::SchedulingTopologyHint topologyHint; std::vector expectedHosts; }; @@ -51,7 +51,7 @@ class SchedulingDecisionTestFixture : public SchedulerFixture // The first time we run the batch request, we will follow the // unregistered hosts path - faabric::util::SchedulingDecision actualDecision = + faabric::batch_scheduler::SchedulingDecision actualDecision = sch.makeSchedulingDecision(req, config.topologyHint); REQUIRE(actualDecision.hosts == config.expectedHosts); @@ -84,7 +84,7 @@ TEST_CASE_METHOD(SchedulingDecisionTestFixture, .slots = { 1, 1 }, .used = { 0, 0 }, .numReqs = 2, - .topologyHint = faabric::util::SchedulingTopologyHint::NONE, + .topologyHint = faabric::batch_scheduler::SchedulingTopologyHint::NONE, .expectedHosts = { masterHost, "hostA" }, }; @@ -102,7 +102,7 @@ TEST_CASE_METHOD(SchedulingDecisionTestFixture, .hosts = { masterHost, "hostA", "hostB" }, .slots = { 4, 4, 4 }, .used = { 0, 0, 0 }, - .topologyHint = faabric::util::SchedulingTopologyHint::NONE, + .topologyHint = faabric::batch_scheduler::SchedulingTopologyHint::NONE, }; SECTION("Capacity on all hosts") @@ -153,7 +153,7 @@ TEST_CASE_METHOD(SchedulingDecisionTestFixture, .slots = { 1, 1 }, .used = { 0, 0 }, .numReqs = 3, - .topologyHint = faabric::util::SchedulingTopologyHint::NONE, + .topologyHint = faabric::batch_scheduler::SchedulingTopologyHint::NONE, .expectedHosts = { masterHost, "hostA", masterHost }, }; @@ -171,7 +171,8 @@ TEST_CASE_METHOD(SchedulingDecisionTestFixture, .slots = { 1, 1 }, .used = { 0, 0 }, .numReqs = 2, - .topologyHint = faabric::util::SchedulingTopologyHint::FORCE_LOCAL, + .topologyHint = + faabric::batch_scheduler::SchedulingTopologyHint::FORCE_LOCAL, .expectedHosts = { masterHost, "hostA" }, }; @@ -179,14 +180,15 @@ TEST_CASE_METHOD(SchedulingDecisionTestFixture, SECTION("Force local off") { - config.topologyHint = faabric::util::SchedulingTopologyHint::NONE, + config.topologyHint = + faabric::batch_scheduler::SchedulingTopologyHint::NONE, config.expectedHosts = { masterHost, "hostA" }; } SECTION("Force local on") { config.topologyHint = - faabric::util::SchedulingTopologyHint::FORCE_LOCAL, + faabric::batch_scheduler::SchedulingTopologyHint::FORCE_LOCAL, config.expectedHosts = { masterHost, masterHost }; } @@ -202,7 +204,8 @@ TEST_CASE_METHOD(SchedulingDecisionTestFixture, .slots = { 1, 1 }, .used = { 0, 0 }, .numReqs = 2, - .topologyHint = faabric::util::SchedulingTopologyHint::FORCE_LOCAL, + .topologyHint = + faabric::batch_scheduler::SchedulingTopologyHint::FORCE_LOCAL, .expectedHosts = { masterHost, "hostA" }, }; @@ -234,7 +237,7 @@ TEST_CASE_METHOD(SchedulingDecisionTestFixture, .slots = { 0, 2 }, .used = { 0, 0 }, .numReqs = 2, - .topologyHint = faabric::util::SchedulingTopologyHint::NONE, + .topologyHint = faabric::batch_scheduler::SchedulingTopologyHint::NONE, .expectedHosts = { "hostA", "hostA" }, }; @@ -252,7 +255,7 @@ TEST_CASE_METHOD(SchedulingDecisionTestFixture, .slots = { 2, 0, 2 }, .used = { 0, 0, 0 }, .numReqs = 4, - .topologyHint = faabric::util::SchedulingTopologyHint::NONE, + .topologyHint = faabric::batch_scheduler::SchedulingTopologyHint::NONE, .expectedHosts = { masterHost, masterHost, "hostB", "hostB" }, }; @@ -260,13 +263,14 @@ TEST_CASE_METHOD(SchedulingDecisionTestFixture, SECTION("No topology hint") { - config.topologyHint = faabric::util::SchedulingTopologyHint::NONE; + config.topologyHint = + faabric::batch_scheduler::SchedulingTopologyHint::NONE; } SECTION("Never alone topology hint") { config.topologyHint = - faabric::util::SchedulingTopologyHint::NEVER_ALONE; + faabric::batch_scheduler::SchedulingTopologyHint::NEVER_ALONE; } testActualSchedulingDecision(req, config); @@ -281,7 +285,7 @@ TEST_CASE_METHOD(SchedulingDecisionTestFixture, .slots = { 0, 0, 0, 0 }, .used = { 0, 0, 0, 0 }, .numReqs = 8, - .topologyHint = faabric::util::SchedulingTopologyHint::NONE, + .topologyHint = faabric::batch_scheduler::SchedulingTopologyHint::NONE, .expectedHosts = { masterHost, masterHost, masterHost, masterHost }, }; @@ -316,7 +320,8 @@ TEST_CASE_METHOD(SchedulingDecisionTestFixture, SECTION("No topology hint") { - config.topologyHint = faabric::util::SchedulingTopologyHint::NONE; + config.topologyHint = + faabric::batch_scheduler::SchedulingTopologyHint::NONE; config.expectedHosts = { masterHost, masterHost, "hostA", "hostA", "hostB", "hostC", "hostC", masterHost @@ -326,7 +331,7 @@ TEST_CASE_METHOD(SchedulingDecisionTestFixture, SECTION("Never alone topology hint") { config.topologyHint = - faabric::util::SchedulingTopologyHint::NEVER_ALONE; + faabric::batch_scheduler::SchedulingTopologyHint::NEVER_ALONE; config.expectedHosts = { masterHost, masterHost, "hostA", "hostA", "hostC", "hostC", "hostC", "hostC" }; } @@ -344,7 +349,8 @@ TEST_CASE_METHOD(SchedulingDecisionTestFixture, .slots = { 1, 1 }, .used = { 0, 0 }, .numReqs = 2, - .topologyHint = faabric::util::SchedulingTopologyHint::NEVER_ALONE, + .topologyHint = + faabric::batch_scheduler::SchedulingTopologyHint::NEVER_ALONE, .expectedHosts = { masterHost, "hostA" }, }; @@ -416,7 +422,8 @@ TEST_CASE_METHOD(SchedulingDecisionTestFixture, .slots = { 2, 2 }, .used = { 0, 0 }, .numReqs = 2, - .topologyHint = faabric::util::SchedulingTopologyHint::UNDERFULL, + .topologyHint = + faabric::batch_scheduler::SchedulingTopologyHint::UNDERFULL, .expectedHosts = { masterHost, "hostA" }, }; @@ -454,7 +461,8 @@ TEST_CASE_METHOD(SchedulingDecisionTestFixture, .slots = { 2, 2 }, .used = { 0, 0 }, .numReqs = 4, - .topologyHint = faabric::util::SchedulingTopologyHint::CACHED, + .topologyHint = + faabric::batch_scheduler::SchedulingTopologyHint::CACHED, .expectedHosts = { masterHost, masterHost, "hostA", "hostA" }, }; @@ -469,7 +477,8 @@ TEST_CASE_METHOD(SchedulingDecisionTestFixture, .slots = { 0, 10 }, .used = { 0, 0 }, .numReqs = 4, - .topologyHint = faabric::util::SchedulingTopologyHint::CACHED, + .topologyHint = + faabric::batch_scheduler::SchedulingTopologyHint::CACHED, .expectedHosts = { masterHost, masterHost, "hostA", "hostA" }, }; @@ -486,7 +495,8 @@ TEST_CASE_METHOD(SchedulingDecisionTestFixture, .slots = { 0, 10 }, .used = { 0, 0 }, .numReqs = 4, - .topologyHint = faabric::util::SchedulingTopologyHint::CACHED, + .topologyHint = + faabric::batch_scheduler::SchedulingTopologyHint::CACHED, .expectedHosts = { "hostA", "hostA", "hostA", "hostA" }, }; auto missReq = diff --git a/tests/test/snapshot/test_snapshot_client_server.cpp b/tests/test/snapshot/test_snapshot_client_server.cpp index 81bc334f0..5c39cae2c 100644 --- a/tests/test/snapshot/test_snapshot_client_server.cpp +++ b/tests/test/snapshot/test_snapshot_client_server.cpp @@ -47,7 +47,7 @@ class SnapshotClientServerTestFixture { void setUpFunctionGroup(int appId, int groupId) { - SchedulingDecision decision(appId, groupId); + faabric::batch_scheduler::SchedulingDecision decision(appId, groupId); faabric::Message msg = messageFactory("foo", "bar"); msg.set_appid(appId); msg.set_groupid(groupId); diff --git a/tests/test/transport/test_point_to_point.cpp b/tests/test/transport/test_point_to_point.cpp index ac700744d..0dfa37cd2 100644 --- a/tests/test/transport/test_point_to_point.cpp +++ b/tests/test/transport/test_point_to_point.cpp @@ -6,6 +6,7 @@ #include +#include #include #include #include @@ -13,7 +14,6 @@ #include #include #include -#include using namespace faabric::transport; using namespace faabric::util; @@ -99,7 +99,7 @@ TEST_CASE_METHOD(PointToPointClientServerFixture, conf.endpointHost = LOCALHOST; // Register both indexes on this host - faabric::util::SchedulingDecision decision(appId, groupId); + faabric::batch_scheduler::SchedulingDecision decision(appId, groupId); faabric::Message msgA = faabric::util::messageFactory("foo", "bar"); msgA.set_appid(appId); @@ -192,7 +192,7 @@ TEST_CASE_METHOD(PointToPointClientServerFixture, conf.endpointHost = LOCALHOST; // Register both indexes on this host - faabric::util::SchedulingDecision decision(appId, groupId); + faabric::batch_scheduler::SchedulingDecision decision(appId, groupId); faabric::Message msgA = faabric::util::messageFactory("foo", "bar"); msgA.set_appid(appId); @@ -313,7 +313,7 @@ TEST_CASE_METHOD( faabric::Message& msgE = req->mutable_messages()->at(4); faabric::Message& msgF = req->mutable_messages()->at(5); - SchedulingDecision decision(appId, groupId); + faabric::batch_scheduler::SchedulingDecision decision(appId, groupId); decision.addMessage(hostB, msgA); decision.addMessage(hostA, msgB); decision.addMessage(hostC, msgC); @@ -405,7 +405,7 @@ TEST_CASE_METHOD(PointToPointClientServerFixture, int groupId = 345; std::atomic sharedInt = 5; - faabric::util::SchedulingDecision decision(appId, groupId); + faabric::batch_scheduler::SchedulingDecision decision(appId, groupId); faabric::Message msg = faabric::util::messageFactory("foo", "bar"); msg.set_appid(appId); @@ -459,7 +459,7 @@ TEST_CASE_METHOD(PointToPointClientServerFixture, rootMsg.set_groupid(groupId); rootMsg.set_groupidx(POINT_TO_POINT_MASTER_IDX); - faabric::util::SchedulingDecision decision(appId, groupId); + faabric::batch_scheduler::SchedulingDecision decision(appId, groupId); decision.addMessage(thisHost, msg); decision.addMessage(thisHost, rootMsg); diff --git a/tests/test/transport/test_point_to_point_groups.cpp b/tests/test/transport/test_point_to_point_groups.cpp index 45b5b8f34..b3db720b2 100644 --- a/tests/test/transport/test_point_to_point_groups.cpp +++ b/tests/test/transport/test_point_to_point_groups.cpp @@ -47,7 +47,7 @@ class PointToPointGroupFixture { req = faabric::util::batchExecFactory("foo", "bar", groupSize); - faabric::util::SchedulingDecision decision(appId, groupId); + faabric::batch_scheduler::SchedulingDecision decision(appId, groupId); for (int i = 0; i < groupSize; i++) { auto& msg = req->mutable_messages()->at(i); @@ -102,7 +102,7 @@ TEST_CASE_METHOD(PointToPointGroupFixture, int groupId = 345; int groupIdx = 1; - faabric::util::SchedulingDecision decision(appId, groupId); + faabric::batch_scheduler::SchedulingDecision decision(appId, groupId); faabric::Message msgA = faabric::util::messageFactory("foo", "bar"); msgA.set_appid(appId); diff --git a/tests/utils/faabric_utils.h b/tests/utils/faabric_utils.h index 07875f1cf..c6c69375d 100644 --- a/tests/utils/faabric_utils.h +++ b/tests/utils/faabric_utils.h @@ -2,11 +2,11 @@ #include +#include #include #include #include #include -#include #include using namespace faabric; @@ -81,8 +81,8 @@ void checkMessageEquality(const faabric::Message& msgA, const faabric::Message& msgB); void checkSchedulingDecisionEquality( - const faabric::util::SchedulingDecision& decisionA, - const faabric::util::SchedulingDecision& decisionB); + const faabric::batch_scheduler::SchedulingDecision& decisionA, + const faabric::batch_scheduler::SchedulingDecision& decisionB); void checkExecGraphNodeEquality(const faabric::util::ExecGraphNode& nodeA, const faabric::util::ExecGraphNode& nodeB); diff --git a/tests/utils/fixtures.h b/tests/utils/fixtures.h index 1f50cc028..3c5342b7e 100644 --- a/tests/utils/fixtures.h +++ b/tests/utils/fixtures.h @@ -7,6 +7,8 @@ #include #include +#include +#include #include #include #include @@ -32,7 +34,6 @@ #include #include #include -#include #include #include @@ -104,13 +105,13 @@ class CachedDecisionTestFixture { public: CachedDecisionTestFixture() - : decisionCache(faabric::util::getSchedulingDecisionCache()) + : decisionCache(faabric::batch_scheduler::getSchedulingDecisionCache()) {} ~CachedDecisionTestFixture() { decisionCache.clear(); } protected: - faabric::util::DecisionCache& decisionCache; + faabric::batch_scheduler::DecisionCache& decisionCache; }; class PlannerClientServerFixture @@ -542,13 +543,13 @@ class BatchSchedulerFixture : public ConfFixture std::shared_ptr ber; std::shared_ptr batchScheduler; - faabric::util::SchedulingDecision actualDecision; + faabric::batch_scheduler::SchedulingDecision actualDecision; struct BatchSchedulerConfig { faabric::batch_scheduler::HostMap hostMap; faabric::batch_scheduler::InFlightReqs inFlightReqs; - faabric::util::SchedulingDecision expectedDecision; + faabric::batch_scheduler::SchedulingDecision expectedDecision; }; static faabric::batch_scheduler::HostMap buildHostMap( @@ -596,19 +597,19 @@ class BatchSchedulerFixture : public ConfFixture oldBer->set_appid(appId); assert(oldBer->messages_size() == hosts.size()); - inFlightReqs[appId] = - std::make_pair(oldBer, - std::make_shared( - buildExpectedDecision(oldBer, hosts))); + inFlightReqs[appId] = std::make_pair( + oldBer, + std::make_shared( + buildExpectedDecision(oldBer, hosts))); return inFlightReqs; } - static faabric::util::SchedulingDecision buildExpectedDecision( + static faabric::batch_scheduler::SchedulingDecision buildExpectedDecision( std::shared_ptr ber, std::vector hosts) { - faabric::util::SchedulingDecision decision(ber->appid(), 0); + faabric::batch_scheduler::SchedulingDecision decision(ber->appid(), 0); assert(ber->messages_size() == hosts.size()); @@ -620,8 +621,8 @@ class BatchSchedulerFixture : public ConfFixture } static void compareSchedulingDecisions( - const faabric::util::SchedulingDecision& decisionA, - const faabric::util::SchedulingDecision& decisionB) + const faabric::batch_scheduler::SchedulingDecision& decisionA, + const faabric::batch_scheduler::SchedulingDecision& decisionB) { REQUIRE(decisionA.appId == decisionB.appId); REQUIRE(decisionA.groupId == decisionB.groupId); diff --git a/tests/utils/scheduling_utils.cpp b/tests/utils/scheduling_utils.cpp index 4a39d53cc..8d6ccae98 100644 --- a/tests/utils/scheduling_utils.cpp +++ b/tests/utils/scheduling_utils.cpp @@ -5,8 +5,8 @@ namespace tests { void checkSchedulingDecisionEquality( - const faabric::util::SchedulingDecision& decisionA, - const faabric::util::SchedulingDecision& decisionB) + const faabric::batch_scheduler::SchedulingDecision& decisionA, + const faabric::batch_scheduler::SchedulingDecision& decisionB) { REQUIRE(decisionA.appId == decisionB.appId); REQUIRE(decisionA.nFunctions == decisionB.nFunctions);