Merge pull request #2432 from alicevision/dev/dynamicOutputValue

[core] New dynamic output attributes

meshroom/core/__init__.py

@@ -128,6 +128,8 @@ def validateNodeDesc(nodeDesc):
             errors.append(err)
 
     for param in nodeDesc.outputs:
+        if param.value is None:
+            continue
         err = param.checkValueTypes()
         if err:
             errors.append(err)

meshroom/core/attribute.py

@@ -273,11 +273,18 @@ def uid(self, uidIndex=-1):
         # 'uidIndex' should be in 'self.desc.uid' but in the case of linked attribute
         # it will not be the case (so we cannot have an assert).
         if self.isOutput:
-            # only dependent on the hash of its value without the cache folder
-            return hashValue(self._invalidationValue)
+            if self.desc.isDynamicValue:
+                # If the attribute is a dynamic output, the UID is derived from the node UID.
+                # To guarantee that each output attribute receives a unique ID, we add the attribute name to it.
+                return hashValue((self.name, self.node._uids.get(uidIndex)))
+            else:
+                # only dependent on the hash of its value without the cache folder
+                return hashValue(self._invalidationValue)
         if self.isLink:
-            return self.getLinkParam().uid(uidIndex)
+            linkParam = self.getLinkParam(recursive=True)
+            return linkParam.uid(uidIndex)
         if isinstance(self._value, (list, tuple, set,)):
+            # non-exclusive choice param
             # hash of sorted values hashed
             return hashValue([hashValue(v) for v in sorted(self._value)])
         return hashValue(self._value)

meshroom/core/desc.py

@@ -32,16 +32,21 @@ def __init__(self, name, label, description, value, advanced, semantic, uid, gro
         self._errorMessage = errorMessage
         self._visible = visible
         self._isExpression = (isinstance(self._value, str) and "{" in self._value) or isinstance(self._value, types.FunctionType)
+        self._isDynamicValue = (self._value is None)
         self._valueType = None
 
     name = Property(str, lambda self: self._name, constant=True)
     label = Property(str, lambda self: self._label, constant=True)
     description = Property(str, lambda self: self._description, constant=True)
     value = Property(Variant, lambda self: self._value, constant=True)
     # isExpression:
-    #   The value of the attribute's descriptor is a static string expression that should be evaluated at runtime.
+    #   The default value of the attribute's descriptor is a static string expression that should be evaluated at runtime.
     #   This property only makes sense for output attributes.
     isExpression = Property(bool, lambda self: self._isExpression, constant=True)
+    # isDynamicValue
+    #   The default value of the attribute's descriptor is None, so it's not an input value,
+    #   but an output value that is computed during the Node's process execution.
+    isDynamicValue = Property(bool, lambda self: self._isDynamicValue, constant=True)
     uid = Property(Variant, lambda self: self._uid, constant=True)
     group = Property(str, lambda self: self._group, constant=True)
     advanced = Property(bool, lambda self: self._advanced, constant=True)
@@ -99,6 +104,8 @@ def __init__(self, elementDesc, name, label, description, group='allParams', adv
     joinChar = Property(str, lambda self: self._joinChar, constant=True)
 
     def validateValue(self, value):
+        if value is None:
+            return value
         if JSValue is not None and isinstance(value, JSValue):
             # Note: we could use isArray(), property("length").toInt() to retrieve all values
             raise ValueError("ListAttribute.validateValue: cannot recognize QJSValue. Please, use JSON.stringify(value) in QML.")
@@ -138,6 +145,8 @@ def __init__(self, groupDesc, name, label, description, group='allParams', advan
     groupDesc = Property(Variant, lambda self: self._groupDesc, constant=True)
 
     def validateValue(self, value):
+        if value is None:
+            return value
         """ Ensure value is compatible with the group description and convert value if needed. """
         if JSValue is not None and isinstance(value, JSValue):
             # Note: we could use isArray(), property("length").toInt() to retrieve all values
@@ -232,6 +241,8 @@ def __init__(self, name, label, description, value, uid, group='allParams', adva
         self._valueType = str
 
     def validateValue(self, value):
+        if value is None:
+            return value
         if not isinstance(value, str):
             raise ValueError('File only supports string input  (param:{}, value:{}, type:{})'.format(self.name, value, type(value)))
         return os.path.normpath(value).replace('\\', '/') if value else ''
@@ -252,6 +263,8 @@ def __init__(self, name, label, description, value, uid, group='allParams', adva
         self._valueType = bool
 
     def validateValue(self, value):
+        if value is None:
+            return value
         try:
             if isinstance(value, str):
                 # use distutils.util.strtobool to handle (1/0, true/false, on/off, y/n)
@@ -276,6 +289,8 @@ def __init__(self, name, label, description, value, range, uid, group='allParams
         self._valueType = int
 
     def validateValue(self, value):
+        if value is None:
+            return value
         # handle unsigned int values that are translated to int by shiboken and may overflow
         try:
             return int(value)
@@ -300,6 +315,8 @@ def __init__(self, name, label, description, value, range, uid, group='allParams
         self._valueType = float
 
     def validateValue(self, value):
+        if value is None:
+            return value
         try:
             return float(value)
         except:
@@ -320,7 +337,7 @@ def __init__(self, name, label, description, uid, group='allParams', advanced=Fa
         self._valueType = None
 
     def validateValue(self, value):
-        pass
+        return value
     def checkValueTypes(self):
         pass
 
@@ -354,6 +371,8 @@ def conformValue(self, value):
         return self._valueType(value)
 
     def validateValue(self, value):
+        if value is None:
+            return value
         if self.exclusive:
             return self.conformValue(value)
 
@@ -383,6 +402,8 @@ def __init__(self, name, label, description, value, uid, group='allParams', adva
         self._valueType = str
 
     def validateValue(self, value):
+        if value is None:
+            return value
         if not isinstance(value, str):
             raise ValueError('StringParam value should be a string (param:{}, value:{}, type:{})'.format(self.name, value, type(value)))
         return value
@@ -401,6 +422,8 @@ def __init__(self, name, label, description, value, uid, group='allParams', adva
         self._valueType = str
 
     def validateValue(self, value):
+        if value is None:
+            return value
         if not isinstance(value, str) or len(value.split(" ")) > 1:
             raise ValueError('ColorParam value should be a string containing either an SVG name or an hexadecimal '
                              'color code (param: {}, value: {}, type: {})'.format(self.name, value, type(value)))
@@ -594,7 +617,8 @@ class Node(object):
     category = 'Other'
 
     def __init__(self):
-        pass
+        super(Node, self).__init__()
+        self.hasDynamicOutputAttribute = any(output.isDynamicValue for output in self.outputs)
 
     def upgradeAttributeValues(self, attrValues, fromVersion):
         return attrValues
@@ -630,6 +654,9 @@ class InputNode(Node):
     """
     Node that does not need to be processed, it is just a placeholder for inputs.
     """
+    def __init__(self):
+        super(InputNode, self).__init__()
+
     def processChunk(self, chunk):
         pass
 
@@ -641,6 +668,9 @@ class CommandLineNode(Node):
     parallelization = None
     commandLineRange = ''
 
+    def __init__(self):
+        super(CommandLineNode, self).__init__()
+
     def buildCommandLine(self, chunk):
 
         cmdPrefix = ''
@@ -708,6 +738,7 @@ class AVCommandLineNode(CommandLineNode):
     cmdCore = ''
 
     def __init__(self):
+        super(AVCommandLineNode, self).__init__()
 
         if AVCommandLineNode.cgroupParsed is False:
 
@@ -730,9 +761,9 @@ def buildCommandLine(self, chunk):
         return commandLineString + AVCommandLineNode.cmdMem + AVCommandLineNode.cmdCore
 
 # Test abstract node
-class InitNode:
+class InitNode(object):
     def __init__(self):
-        pass
+        super(InitNode, self).__init__()
 
     def initialize(self, node, inputs, recursiveInputs):
         """

meshroom/core/graph.py

@@ -14,7 +14,7 @@
 import meshroom.core
 from meshroom.common import BaseObject, DictModel, Slot, Signal, Property
 from meshroom.core import Version
-from meshroom.core.attribute import Attribute, ListAttribute
+from meshroom.core.attribute import Attribute, ListAttribute, GroupAttribute
 from meshroom.core.exception import StopGraphVisit, StopBranchVisit
 from meshroom.core.node import nodeFactory, Status, Node, CompatibilityNode
 
@@ -320,7 +320,10 @@ def load(self, filepath, setupProjectFile=True, importProject=False, publishOutp
             # that were computed.
             if not isTemplate:  # UIDs are not stored in templates
                 self._evaluateUidConflicts(graphData)
-                self._applyExpr()
+                try:
+                    self._applyExpr()
+                except Exception as e:
+                    logging.warning(e)
 
         return True
 
@@ -338,8 +341,13 @@ def _evaluateUidConflicts(self, data):
         for nodeName, nodeData in sorted(data.items(), key=lambda x: self.getNodeIndexFromName(x[0])):
             node = self.node(nodeName)
 
-            savedUid = nodeData.get("uids", "").get("0", "")  # Node's UID from the graph file
-            graphUid = node._uids.get(0)  # Node's UID from the graph itself
+            savedUid = nodeData.get("uids", {})  # Node's UID from the graph file
+            # JSON enfore keys to be strings, see
+            # https://docs.python.org/3.8/library/json.html#json.dump
+            # We know our keys are integers, so we convert them back to int.
+            savedUid = {int(k): v for k, v in savedUid.items()}
+
+            graphUid = node._uids  # Node's UID from the graph itself
             if savedUid != graphUid and graphUid is not None:
                 # Different UIDs, remove the existing node from the graph and replace it with a CompatibilityNode
                 logging.debug("UID conflict detected for {}".format(nodeName))
@@ -548,12 +556,16 @@ def copyNode(self, srcNode, withEdges=False):
             skippedEdges = {}
             if not withEdges:
                 for n, attr in node.attributes.items():
+                    if attr.isOutput:
+                        # edges are declared in input with an expression linking
+                        # to another param (which could be an output)
+                        continue
                     # find top-level links
                     if Attribute.isLinkExpression(attr.value):
                         skippedEdges[attr] = attr.value
                         attr.resetToDefaultValue()
                     # find links in ListAttribute children
-                    elif isinstance(attr, ListAttribute):
+                    elif isinstance(attr, (ListAttribute, GroupAttribute)):
                         for child in attr.value:
                             if Attribute.isLinkExpression(child.value):
                                 skippedEdges[child] = child.value
@@ -584,6 +596,7 @@ def duplicateNodes(self, srcNodes):
 
             # re-create edges taking into account what has been duplicated
             for attr, linkExpression in duplicateEdges.items():
+                logging.warning("attr={} linkExpression={}".format(attr.fullName, linkExpression))
                 link = linkExpression[1:-1]  # remove starting '{' and trailing '}'
                 # get source node and attribute name
                 edgeSrcNodeName, edgeSrcAttrName = link.split(".", 1)
@@ -1625,6 +1638,7 @@ def executeGraph(graph, toNodes=None, forceCompute=False, forceStatus=False):
 
     for n, node in enumerate(nodes):
         try:
+            node.preprocess()
             multiChunks = len(node.chunks) > 1
             for c, chunk in enumerate(node.chunks):
                 if multiChunks:
@@ -1635,6 +1649,7 @@ def executeGraph(graph, toNodes=None, forceCompute=False, forceStatus=False):
                     print('\n[{node}/{nbNodes}] {nodeName}'.format(
                         node=n + 1, nbNodes=len(nodes), nodeName=node.nodeType))
                 chunk.process(forceCompute)
+            node.postprocess()
         except Exception as e:
             logging.error("Error on node computation: {}".format(e))
             graph.clearSubmittedNodes()