[projmgr] External Generator PoC - Initial commit

tools/buildmgr/cbuildgen/installer/create_installer.sh

@@ -70,9 +70,19 @@ curl --retry 3 -L ${cpackget_base}_linux_arm64.tar.gz  -o - | tar xzfO - --wildc
 curl --retry 3 -L ${cpackget_base}_windows_arm64.zip   -o temp.zip && unzip -p temp.zip '*/cpackget.exe' > ${distdir}/bin/cpackget.exe-arm64 && rm temp.zip
 curl --retry 3 -L ${cpackget_base}_darwin_arm64.tar.gz -o - | tar xzfO - --wildcards    '*cpackget'     > ${distdir}/bin/cpackget.mac-arm64
 
+# Get cbridge
+cbridge_version="0.9.0-extgen2"
+cbridge_base=https://github.com/brondani/generator-bridge/releases/download/${cbridge_version}/cbridge_${cbridge_version}
+curl --retry 3 -L ${cbridge_base}_linux_amd64.tar.gz  -o - | tar xzfO - --wildcards    '*cbridge'     > ${distdir}/bin/cbridge.lin-amd64
+curl --retry 3 -L ${cbridge_base}_windows_amd64.zip   -o temp.zip && unzip -p temp.zip '*/cbridge.exe' > ${distdir}/bin/cbridge.exe-amd64 && rm temp.zip
+curl --retry 3 -L ${cbridge_base}_darwin_amd64.tar.gz -o - | tar xzfO - --wildcards    '*cbridge'     > ${distdir}/bin/cbridge.mac-amd64
+curl --retry 3 -L ${cbridge_base}_linux_arm64.tar.gz  -o - | tar xzfO - --wildcards    '*cbridge'     > ${distdir}/bin/cbridge.lin-arm64
+curl --retry 3 -L ${cbridge_base}_windows_arm64.zip   -o temp.zip && unzip -p temp.zip '*/cbridge.exe' > ${distdir}/bin/cbridge.exe-arm64 && rm temp.zip
+curl --retry 3 -L ${cbridge_base}_darwin_arm64.tar.gz -o - | tar xzfO - --wildcards    '*cbridge'     > ${distdir}/bin/cbridge.mac-arm64
+
 # Get csolution
-csolution_version="2.1.0"
-csolution_base=https://github.com/Open-CMSIS-Pack/devtools/releases/download/tools%2Fprojmgr%2F${csolution_version}/projmgr.zip
+csolution_version="2.2.0-extgen2"
+csolution_base=https://github.com/brondani/devtools/releases/download/tools%2Fprojmgr%2F${csolution_version}/projmgr.zip
 curl --retry 3 -L ${csolution_base} -o temp.zip && unzip -q -d temp temp.zip
 cp 'temp/bin/linux-amd64/csolution' ${distdir}/bin/csolution.lin-amd64
 cp 'temp/bin/windows-amd64/csolution.exe' ${distdir}/bin/csolution.exe-amd64

tools/buildmgr/cbuildgen/installer/make_deb.sh

@@ -99,9 +99,14 @@ cpackget_version="1.0.1"
 cpackget_base=https://github.com/Open-CMSIS-Pack/cpackget/releases/download/v${cpackget_version}/cpackget_${cpackget_version}
 curl --retry 3 -L ${cpackget_base}_linux_amd64.tar.gz -o - | tar xzfO - --wildcards '*cpackget' > ${input}/bin/cpackget.lin-amd64
 
+# Get generator-bridge
+cbridge_version="0.9.0-extgen2"
+cbridge_base=https://github.com/brondani/generator-bridge/releases/download/${cbridge_version}/cbridge_${cbridge_version}
+curl --retry 3 -L ${cbridge_base}_linux_amd64.tar.gz -o - | tar xzfO - --wildcards '*cpackget' > ${input}/bin/cbridge.lin-amd64
+
 # Get csolution
-csolution_version="2.1.0"
-csolution_base=https://github.com/Open-CMSIS-Pack/devtools/releases/download/tools%2Fprojmgr%2F${csolution_version}/projmgr.zip
+csolution_version="2.2.0-extgen2"
+csolution_base=https://github.com/brondani/devtools/releases/download/tools%2Fprojmgr%2F${csolution_version}/projmgr.zip
 curl --retry 3 -L ${csolution_base} -o temp.zip && unzip -q -d temp temp.zip
 cp 'temp/bin/linux-amd64/csolution' ${input}/bin/csolution.lin-amd64
 cp -r temp/etc/* etc/${PACKAGE_NAME}

tools/projmgr/CMakeLists.txt

@@ -13,12 +13,12 @@ set_property(DIRECTORY PROPERTY VS_STARTUP_PROJECT projmgr)
 SET(PROJMGR_SOURCE_FILES ProjMgr.cpp ProjMgrKernel.cpp ProjMgrCallback.cpp
   ProjMgrParser.cpp ProjMgrWorker.cpp ProjMgrGenerator.cpp ProjMgrXmlParser.cpp
   ProjMgrYamlParser.cpp ProjMgrLogger.cpp ProjMgrYamlSchemaChecker.cpp
-  ProjMgrYamlEmitter.cpp ProjMgrUtils.cpp
+  ProjMgrYamlEmitter.cpp ProjMgrUtils.cpp ProjMgrExtGenerator.cpp
 )
 SET(PROJMGR_HEADER_FILES ProjMgr.h ProjMgrKernel.h ProjMgrCallback.h
   ProjMgrParser.h ProjMgrWorker.h ProjMgrGenerator.h ProjMgrXmlParser.h
   ProjMgrYamlParser.h ProjMgrLogger.h ProjMgrYamlSchemaChecker.h
-  ProjMgrYamlEmitter.h ProjMgrUtils.h
+  ProjMgrYamlEmitter.h ProjMgrUtils.h ProjMgrExtGenerator.h
 )
 
 list(TRANSFORM PROJMGR_SOURCE_FILES PREPEND src/)

tools/projmgr/include/ProjMgr.h

@@ -88,6 +88,7 @@ class ProjMgr {
   ProjMgrWorker m_worker;
   ProjMgrGenerator m_generator;
   ProjMgrYamlEmitter m_emitter;
+  ProjMgrExtGenerator m_extGenerator;
 
   std::string m_csolutionFile;
   std::string m_cdefaultFile;

tools/projmgr/include/ProjMgrExtGenerator.h

@@ -0,0 +1,97 @@
+/*
+ * Copyright (c) 2020-2023 Arm Limited. All rights reserved.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+#ifndef PROJMGREXTGENERATOR_H
+#define PROJMGREXTGENERATOR_H
+
+#include "ProjMgrParser.h"
+#include "ProjMgrUtils.h"
+
+ /**
+  * @brief external generator item containing
+  *        component identifier
+  *        directory for generated files
+  *        project type
+ */
+struct ExtGeneratorItem {
+  std::string componentId;
+  std::string genDir;
+  std::string projectType;
+};
+
+
+struct CbuildGenItem {
+  StrVec forContext;
+  std::string device;
+  std::string board;
+  std::string projectPart;
+};
+
+/**
+ * @brief map of used generators, directories and contexts
+*/
+typedef std::map<std::string, StrVecMap> GeneratorContextVecMap;
+
+/**
+ * @brief vector of external generator items
+*/
+typedef std::vector<ExtGeneratorItem> ExtGeneratorVec;
+
+/**
+ * @brief map of vector of external generator items
+*/
+typedef std::map<std::string, ExtGeneratorVec> ExtGeneratorVecMap;
+
+/**
+ * @brief solution/project types
+*/
+static constexpr const char* TYPE_SINGLE_CORE = "single-core";
+static constexpr const char* TYPE_MULTI_CORE = "multi-core";
+static constexpr const char* TYPE_TRUSTZONE = "trustzone";
+static constexpr const char* PROCESSOR_CORE = "processor-core";
+
+/**
+ * @brief projmgr external generator class responsible for handling global generators
+*/
+class ProjMgrExtGenerator {
+public:
+  /**
+   * @brief class constructor
+  */
+  ProjMgrExtGenerator(ProjMgrParser* parser);
+
+
+  /**
+   * @brief class destructor
+  */
+  ~ProjMgrExtGenerator(void);
+
+
+  bool RetrieveGlobalGenerators(void);
+
+  bool IsGlobalGenerator(const std::string& generatorId);
+  bool CheckGeneratorId(const std::string& generatorId, const std::string& componentId);
+  const std::string& GetGlobalGenDir(const std::string& generatorId);
+  const std::string& GetGlobalGenRunCmd(const std::string& generatorId);
+  void AddUsedGenerator(const std::string& generatorId, const std::string& genDir, const std::string& contextId);
+  const GeneratorContextVecMap& GetUsedGenerators(void);
+  bool GetCgen(const std::string& contextId, ClayerItem& cgen);
+
+protected:
+  ProjMgrParser* m_parser = nullptr;
+  StrVec m_globalGeneratorFiles;
+  std::map<std::string, GlobalGeneratorItem> m_globalGenerators;
+  GeneratorContextVecMap m_usedGenerators;
+
+  bool m_checkSchema;
+  std::string m_compilerRoot;
+
+
+
+
+};
+
+#endif  // PROJMGREXTGENERATOR_H

tools/projmgr/include/ProjMgrParser.h

@@ -140,12 +140,14 @@ struct TargetType {
  * @brief directories item containing
  *        intdir directory,
  *        outdir directory,
+ *        cbuild directory,
  *        cprj directory,
  *        rte directory,
 */
 struct DirectoriesItem {
   std::string intdir;
   std::string outdir;
+  std::string cbuild;
   std::string cprj;
   std::string rte;
 };
@@ -413,6 +415,20 @@ struct CbuildSetItem {
   std::string compiler;
 };
 
+/**
+ * @brief global generator item containing
+ *        generator id,
+ *        download url,
+ *        bridge program,
+ *        output
+*/
+struct GlobalGeneratorItem {
+  std::string id;
+  std::string downloadUrl;
+  std::string run;
+  std::string output;
+};
+
 /**
  * @brief projmgr parser class for public interfacing
 */
@@ -465,6 +481,12 @@ class ProjMgrParser {
   */
   bool ParseCbuildSet(const std::string& input);
 
+  /**
+   * @brief parse global generator
+   * @param input generator.yml file
+  */
+  bool ParseGlobalGenerator(const std::string& input, bool checkSchema);
+
   /**
    * @brief get cdefault
    * @return cdefault item
@@ -500,13 +522,21 @@ class ProjMgrParser {
    * @return cbuildset item
   */
   CbuildSetItem& GetCbuildSetItem(void);
+
+  /**
+   * @brief get global generators
+   * @return global generators map
+  */
+  std::map<std::string, GlobalGeneratorItem>& GetGlobalGenerators(void);
+
 protected:
   CdefaultItem m_cdefault;
   CsolutionItem m_csolution;
   CbuildSetItem m_cbuildSet;
   std::map<std::string, CprojectItem> m_cprojects;
   std::map<std::string, ClayerItem> m_clayers;
   std::map<std::string, ClayerItem> m_genericClayers;
+  std::map<std::string, GlobalGeneratorItem> m_globalGenerators;
 };
 
 #endif  // PROJMGRPARSER_H

tools/projmgr/include/ProjMgrWorker.h

@@ -7,6 +7,7 @@
 #ifndef PROJMGRWORKER_H
 #define PROJMGRWORKER_H
 
+#include "ProjMgrExtGenerator.h"
 #include "ProjMgrKernel.h"
 #include "ProjMgrParser.h"
 #include "ProjMgrUtils.h"
@@ -252,6 +253,7 @@ struct ContextTypesItem {
  *        output type,
  *        device selection,
  *        board selection,
+ *        device item struct,
  *        list of package requirements,
  *        map of required pdsc files and optionally its local path
  *        list of component requirements,
@@ -269,6 +271,7 @@ struct ContextTypesItem {
  *        valid connections,
  *        linker options,
  *        map of variables,
+ *        external generator directory,
  *        boolean processed precedences
 */
 struct ContextItem {
@@ -289,6 +292,7 @@ struct ContextItem {
   OutputTypes outputTypes;
   std::string device;
   std::string board;
+  DeviceItem deviceItem;
   std::vector<PackageItem> packRequirements;
   std::map<std::string, std::pair<std::string, std::string>> pdscFiles;
   std::vector<PackInfo>missingPacks;
@@ -310,6 +314,7 @@ struct ContextItem {
   std::vector<ConnectionsCollectionVec> validConnections;
   LinkerContextItem linker;
   std::map<std::string, std::string> variables;
+  StrMap extGenDir;
   bool precedences;
 };
 
@@ -362,28 +367,22 @@ class ProjMgrWorker {
   /**
    * @brief class constructor
   */
-  ProjMgrWorker(void);
+  ProjMgrWorker(ProjMgrParser* parser, ProjMgrExtGenerator* extGenerator);
 
   /**
    * @brief class destructor
   */
   ~ProjMgrWorker(void);
 
-  /**
-   * @brief set parser
-   * @param pointer to parser
-  */
-  void SetParser(ProjMgrParser* parser);
-
   /**
    * @brief process context
    * @param reference to context
-   * @param loadGpdsc boolean automatically load gpdsc, default true
+   * @param loadGenFiles boolean automatically load generated files, default true
    * @param resolveDependencies boolean automatically resolve dependencies, default true
    * @param updateRteFiles boolean update RTE files, default true
    * @return true if executed successfully
   */
-  bool ProcessContext(ContextItem& context, bool loadGpdsc = true, bool resolveDependencies = true, bool updateRteFiles = true);
+  bool ProcessContext(ContextItem& context, bool loadGenFiles = true, bool resolveDependencies = true, bool updateRteFiles = true);
 
   /**
    * @brief list available packs
@@ -554,6 +553,8 @@ class ProjMgrWorker {
   */
   bool ExecuteGenerator(std::string& generatorId);
 
+  bool ExecuteExtGenerator(std::string& generatorId);
+
   /**
    * @brief initialize model
    * @return true if executed successfully
@@ -612,10 +613,14 @@ class ProjMgrWorker {
   */
   std::string GetSelectedToochain(void);
 
+  bool ProcessGlobalGenerators(ContextItem* selectedContext, const std::string& generatorId,
+    std::string& projectType, StrVec& siblings);
+
 protected:
   ProjMgrParser* m_parser = nullptr;
   ProjMgrKernel* m_kernel = nullptr;
   RteGlobalModel* m_model = nullptr;
+  ProjMgrExtGenerator* m_extGenerator = nullptr;
   std::list<RtePackage*> m_loadedPacks;
   std::vector<ToolchainItem> m_toolchains;
   StrVec m_toolchainConfigFiles;
@@ -647,7 +652,7 @@ class ProjMgrWorker {
   bool GetProjectSetup(ContextItem& context);
   bool InitializeTarget(ContextItem& context);
   bool SetTargetAttributes(ContextItem& context, std::map<std::string, std::string>& attributes);
-  bool ProcessPrecedences(ContextItem& context);
+  bool ProcessPrecedences(ContextItem& context, bool rerun = false);
   bool ProcessPrecedence(StringCollection& item);
   bool ProcessCompilerPrecedence(StringCollection& item, bool acceptRedefinition = false);
   bool ProcessDevice(ContextItem& context);
@@ -661,7 +666,7 @@ class ProjMgrWorker {
   bool ProcessConfigFiles(ContextItem& context);
   bool ProcessComponentFiles(ContextItem& context);
   bool ProcessGroups(ContextItem& context);
-  bool ProcessSequencesRelatives(ContextItem& context);
+  bool ProcessSequencesRelatives(ContextItem& context, bool rerun);
   bool ProcessSequencesRelatives(ContextItem& context, std::vector<std::string>& src, const std::string& ref = std::string(), bool withHeadingDot = false);
   bool ProcessSequencesRelatives(ContextItem& context, BuildType& build, const std::string& ref = std::string());
   bool ProcessSequenceRelative(ContextItem& context, std::string& item, const std::string& ref = std::string(), bool withHeadingDot = false);
@@ -729,10 +734,14 @@ class ProjMgrWorker {
   void UpdatePartialReferencedContext(ContextItem& context, std::string& contextName);
   void ExpandAccessSequence(const ContextItem& context, const ContextItem& refContext, const std::string& sequence, std::string& item, bool withHeadingDot);
   bool GetGeneratorDir(const RteGenerator* generator, ContextItem& context, const std::string& layer, std::string& genDir);
+  bool GetGeneratorDir(const std::string& generatorId, ContextItem& context, const std::string& layer, std::string& genDir);
+  bool GetExtGeneratorDir(const std::string& generatorId, ContextItem& context, const std::string& layer, std::string& genDir);
   bool ParseContextLayers(ContextItem& context);
   bool AddPackRequirements(ContextItem& context, const std::vector<PackItem> packRequirements);
+  void InsertPackRequirements(const std::vector<PackItem>& src, std::vector<PackItem>& dst, std::string base);
   void CheckTypeFilterSpelling(const TypeFilter& typeFilter);
   void CheckCompilerFilterSpelling(const std::string& compiler);
+  bool ProcessGeneratedLayers(ContextItem& context);
 };
 
 #endif  // PROJMGRWORKER_H

tools/projmgr/include/ProjMgrYamlEmitter.h

@@ -33,6 +33,9 @@ class ProjMgrYamlEmitter {
   */
   static bool GenerateCbuildIndex(ProjMgrParser& parser, const std::vector<ContextItem*> contexts, const std::string& outputDir);
 
+  static bool GenerateCbuildGenIndex(ProjMgrParser& parser, const std::vector<ContextItem*> siblings,
+    const std::string& type, const std::string& output, const std::string& gendir);
+
   /**
    * @brief generate cbuild.yml file
    * @param context pointer to the context
@@ -46,6 +49,9 @@ class ProjMgrYamlEmitter {
    * @return true if executed successfully
   */
   static bool GenerateCbuildSet(ProjMgrParser& parser, const std::vector<ContextItem*> contexts, const std::string& selectedCompiler);
+
+  static bool GenerateCbuildGen(ContextItem* context, const std::string& output);
+
 };
 
 #endif  // PROJMGRYAMLEMITTER_H