Merge remote-tracking branch 'origin/candidate-9.2.x'

Signed-off-by: Jake Smith <[email protected]> # Conflicts: # helm/hpcc/Chart.yaml # helm/hpcc/templates/_helpers.tpl # helm/hpcc/templates/dafilesrv.yaml # helm/hpcc/templates/dali.yaml # helm/hpcc/templates/dfuserver.yaml # helm/hpcc/templates/eclagent.yaml # helm/hpcc/templates/eclccserver.yaml # helm/hpcc/templates/eclscheduler.yaml # helm/hpcc/templates/esp.yaml # helm/hpcc/templates/localroxie.yaml # helm/hpcc/templates/roxie.yaml # helm/hpcc/templates/sasha.yaml # helm/hpcc/templates/thor.yaml # version.cmake
ghalliday · Aug 11, 2023 · 651fcfa · 651fcfa
2 parents 1e7caef + d583842
commit 651fcfa
Show file tree

Hide file tree

Showing 41 changed files with 476 additions and 194 deletions.
diff --git a/common/deftype/defvalue.cpp b/common/deftype/defvalue.cpp
@@ -2033,6 +2033,7 @@ void RealValue::toMem(void *target)
     RealUnion u;
 
     size32_t size = type->getSize();
+    u.r8 = 0;
     switch (size)
     {
     case 4:
@@ -2050,6 +2051,7 @@ unsigned RealValue::getHash(unsigned initval)
     RealUnion u;
 
     size32_t size = type->getSize();
+    u.r8 = 0;
     switch (size)
     {
     case 4:

diff --git a/common/thorhelper/roxierow.cpp b/common/thorhelper/roxierow.cpp
@@ -323,6 +323,11 @@ class RoxieEngineFixedRowAllocator : public RoxieEngineRowAllocatorBase
         heap->releaseAllRows();
     }
 
+    virtual void emptyCache() override
+    {
+        heap->emptyCache();
+    }
+
 protected:
     Owned<roxiemem::IFixedRowHeap> heap;
 };
@@ -388,6 +393,11 @@ class RoxieEngineVariableRowAllocator : public RoxieEngineRowAllocatorBase
         throwUnexpected();
     }
 
+    virtual void emptyCache() override
+    {
+        //Variable length rows do not support blocked/caching
+    }
+
 protected:
     void * doCreateRow(size32_t initialSize, size32_t & allocatedSize)
     {

diff --git a/common/thorhelper/thorsoapcall.cpp b/common/thorhelper/thorsoapcall.cpp
@@ -2307,6 +2307,7 @@ class CWSCAsyncFor : implements IWSCAsyncFor, public CInterface, public CAsyncFo
                     checkTimeLimitExceeded(&remainingMS);
                     Url &connUrl = master->proxyUrlArray.empty() ? url : master->proxyUrlArray.item(0);
                     ep.set(connUrl.host.get(), connUrl.port);
+                    checkTimeLimitExceeded(&remainingMS);  // after ep.set which might make a potentially long getaddrinfo lookup ...
                     if (strieq(url.method, "https"))
                         proto = PersistentProtocol::ProtoTLS;
                     bool shouldClose = false;

diff --git a/dali/base/dadfs.cpp b/dali/base/dadfs.cpp
@@ -10063,7 +10063,31 @@ class CInitGroups
             addHostsToIPTFunc();
         return cluster.getClear();
     }
-
+    const char *getHostFromClusterEntry(const IPropertyTree &node, const char *clusterName)
+    {
+        const char *computer = node.queryProp("@computer");
+        if (!isEmptyString(computer))
+        {
+            auto it = machineMap.find(computer);
+            if (it == machineMap.end())
+            {
+                OERRLOG("Cannot construct %s, computer name %s not found\n", clusterName, computer);
+                return nullptr;
+            }
+            return it->second.c_str();
+        }
+        else
+        {
+            const char *host = node.queryProp("@netAddress");
+            if (isEmptyString(host))
+            {
+                OERRLOG("Cannot construct %s, missing computer spec on node\n", clusterName);
+                return nullptr;
+            }
+            else
+                return host;
+        }
+    }
     IPropertyTree *createClusterGroupFromEnvCluster(GroupType groupType, const IPropertyTree &cluster, const char *dir, bool realCluster, bool expand)
     {
         const char *processName=nullptr;
@@ -10086,50 +10110,43 @@ class CInitGroups
         }
         std::vector<std::string> hosts;
         Owned<IPropertyTreeIterator> nodes = cluster.getElements(processName);
-        ForEach(*nodes)
+        if (nodes->first())
         {
-            IPropertyTree &node = nodes->query();
-            const char *host = nullptr;
-            if (grp_dropzone == groupType)
-                host = node.queryProp("@server");
-            else
+            do
             {
-                const char *computer = node.queryProp("@computer");
-                if (!isEmptyString(computer))
-                {
-                    auto it = machineMap.find(computer);
-                    if (it == machineMap.end())
-                    {
-                        OERRLOG("Cannot construct %s, computer name %s not found\n", cluster.queryProp("@name"), computer);
-                        return nullptr;
-                    }
-                    host = it->second.c_str();
-                }
+                IPropertyTree &node = nodes->query();
+                const char *host = nullptr;
+                if (grp_dropzone == groupType)
+                    host = node.queryProp("@server");
                 else
+                    host = getHostFromClusterEntry(node, cluster.queryProp("@name"));
+                switch (groupType)
                 {
-                    host = node.queryProp("@netAddress");
-                    if (isEmptyString(host))
-                    {
-                        OERRLOG("Cannot construct %s, missing computer spec on node\n", cluster.queryProp("@name"));
-                        return nullptr;
-                    }
+                    case grp_roxie:
+                        // Redundant copies are located via the flags.
+                        // Old environments may contain duplicated sever information for multiple ports
+                        if (hosts.end() == std::find(hosts.begin(), hosts.end(), host)) // only add if not already there
+                            hosts.push_back(host);
+                        break;
+                    case grp_thor:
+                    case grp_thorspares:
+                    case grp_dropzone:
+                        hosts.push_back(host);
+                        break;
+                    default:
+                        throwUnexpected();
                 }
             }
-            switch (groupType)
+            while (nodes->next());
+        }
+        else if (grp_dropzone == groupType)
+        {
+            // legacy support for DropZone's without ServerList
+            if (cluster.hasProp("@computer") || cluster.hasProp("@netAddress"))
             {
-                case grp_roxie:
-                    // Redundant copies are located via the flags.
-                    // Old environments may contain duplicated sever information for multiple ports
-                    if (hosts.end() == std::find(hosts.begin(), hosts.end(), host)) // only add if not already there
-                        hosts.push_back(host);
-                    break;
-                case grp_thor:
-                case grp_thorspares:
-                case grp_dropzone:
+                const char *host = getHostFromClusterEntry(cluster, cluster.queryProp("@name"));
+                if (!isEmptyString(host))
                     hosts.push_back(host);
-                    break;
-                default:
-                    throwUnexpected();
             }
         }
         if (!hosts.size())

diff --git a/dali/base/dautils.cpp b/dali/base/dautils.cpp
@@ -1138,13 +1138,12 @@ static void convertPosixPathToLfn(StringBuffer &str,const char *path)
 
 void CDfsLogicalFileName::setPlaneExternal(const char *plane,const char *path)
 {
-    if (isEmptyString(path))
-        return;
-    if (isPathSepChar(path[0])&&(path[0]==path[1]))
+    if (!isEmptyString(path)&&isPathSepChar(path[0])&&(path[0]==path[1]))
         throw makeStringExceptionV(-1,"Invalid path %s.",path);
     StringBuffer str(PLANE_SCOPE "::");
     str.append(plane);
-    convertPosixPathToLfn(str,path);
+    if (!isEmptyString(path))
+        convertPosixPathToLfn(str,path);
     set(str.str());
 }
 

diff --git a/dali/dfu/dfurun.cpp b/dali/dfu/dfurun.cpp
@@ -533,6 +533,24 @@ class CDFUengine: public CInterface, implements IDFUengine
         }
     }
 
+    StringBuffer & getFDescName(IFileDescriptor * fd, StringBuffer & result)
+    {
+        fd->getTraceName(result);
+        elideString(result, 255);
+        return result;
+    }
+
+    void ensureFilePermissions(const char * fileName, SecAccessFlags perm, bool write)
+    {
+        if ((write && !HASWRITEPERMISSION(perm)) || (!write && !HASREADPERMISSION(perm)))
+        {
+            if (write)
+                throw makeStringExceptionV(DFSERR_CreateAccessDenied, "Create permission denied for physical file(s): %s", fileName);
+            else
+                throw makeStringExceptionV(DFSERR_LookupAccessDenied, "Lookup permission denied for physical file(s): %s", fileName);
+        }
+    }
+
     Linked<const IPropertyTree> config;
     Owned<IScheduleEventPusher> eventpusher;
     IArrayOf<cDFUlistener> listeners;
@@ -595,16 +613,8 @@ class CDFUengine: public CInterface, implements IDFUengine
             auditflags |= DALI_LDAP_WRITE_WANTED;
 
         SecAccessFlags perm = queryDistributedFileDirectory().getFDescPermissions(fd,user,auditflags);
-        if ((write && !HASWRITEPERMISSION(perm)) || (!write && !HASREADPERMISSION(perm)))
-        {
-            StringBuffer traceName;
-            fd->getTraceName(traceName);
-            elideString(traceName, 255);
-            if (write)
-                throw makeStringExceptionV(DFSERR_CreateAccessDenied, "Create permission denied for physical file(s): %s",traceName.str());
-            else
-                throw makeStringExceptionV(DFSERR_LookupAccessDenied, "Lookup permission denied for physical file(s): %s",traceName.str());
-        }
+        StringBuffer name;
+        ensureFilePermissions(getFDescName(fd,name),perm,write);
     }
 
     void checkForeignFilePermissions(IConstDFUfileSpec *fSpec,IFileDescriptor *fd,IUserDescriptor *user,bool write)
@@ -638,14 +648,55 @@ class CDFUengine: public CInterface, implements IDFUengine
         bool checkLegacyPhysicalPerms = getGlobalConfigSP()->getPropBool("expert/@failOverToLegacyPhysicalPerms",!isContainerized());
         if (((!write&&!HASREADPERMISSION(perm)) || (write&&!HASWRITEPERMISSION(perm))) && checkLegacyPhysicalPerms)
             perm = queryDistributedFileDirectory().getFDescPermissions(fd,user,auditflags);
+        ensureFilePermissions(logicalName,perm,write);
+    }
 
+    void checkPlaneFilePermissions(IFileDescriptor *fd,IUserDescriptor *user,bool write)
+    {
+        //This function checks the scope permissions for a file or files that reside in a single directory on a single plane.
+        //The IFileDescriptor is used to discover the plane and directory.
+        //If the plane is not present, it implies that it is a bare-metal system and useDropZoneRestriction is off, and there
+        //is no matching dropzone in the environment. If this is the case, or the plane permissions are not found
+        //and @failOverToLegacyPhysicalPerms is configured, then the legacy physical file permissions will be checked.
+        unsigned auditflags = (DALI_LDAP_AUDIT_REPORT|DALI_LDAP_READ_WANTED);
         if (write)
+            auditflags |= DALI_LDAP_WRITE_WANTED;
+
+        SecAccessFlags perm;
+        bool checkLegacyPhysicalPerms = getGlobalConfigSP()->getPropBool("expert/@failOverToLegacyPhysicalPerms",!isContainerized());
+        IClusterInfo *iClusterInfo = fd->queryClusterNum(0);
+        const char *planeName = iClusterInfo->queryGroupName();
+        if (!isEmptyString(planeName))
         {
-            if (!HASWRITEPERMISSION(perm))
-                throw makeStringExceptionV(DFSERR_CreateAccessDenied,"Create permission denied for foreign file: %s",logicalName.str());
+            const char *dir = fd->queryDefaultDir();
+            if (isEmptyString(dir))
+                throw makeStringException(-1,"Empty default directory.");
+
+            Owned<IPropertyTree> dropZonePlane = getDropZonePlane(planeName);
+            if (!dropZonePlane)
+                throw makeStringExceptionV(-1,"DropZone %s not found.",planeName);
+            const char *relativePath = getRelativePath(dir,dropZonePlane->queryProp("@prefix"));
+            if (nullptr == relativePath)
+                throw makeStringExceptionV(-1,"Invalid DropZone directory %s.",dir);
+
+            perm = queryDistributedFileDirectory().getDropZoneScopePermissions(planeName,relativePath,user,auditflags);
+            if (((!write&&!HASREADPERMISSION(perm))||(write&&!HASWRITEPERMISSION(perm)))&&checkLegacyPhysicalPerms)
+                perm = queryDistributedFileDirectory().getFDescPermissions(fd,user,auditflags);
+        }
+        else
+        {
+#ifndef _CONTAINERIZED
+            Owned<IEnvironmentFactory> factory = getEnvironmentFactory(true);
+            Owned<IConstEnvironment> env = factory->openEnvironment();
+            if (env->isDropZoneRestrictionEnabled()||!checkLegacyPhysicalPerms)
+                throw makeStringException(-1,"Empty plane name.");
+            perm = queryDistributedFileDirectory().getFDescPermissions(fd,user,auditflags);
+#else
+            throw makeStringException(-1,"Unexpected empty plane name."); // should never be the case in containerized setups
+#endif
         }
-        else if (!HASREADPERMISSION(perm))
-            throw makeStringExceptionV(DFSERR_LookupAccessDenied,"Lookup permission denied for foreign file: %s",logicalName.str());
+        StringBuffer name;
+        ensureFilePermissions(getFDescName(fd,name),perm,write);
     }
 
     void monitorCycle(bool &cancelling)
@@ -1741,7 +1792,7 @@ class CDFUengine: public CInterface, implements IDFUengine
                     if (!replicating) {
                         runningconn.setown(setRunning(runningpath.str()));
                         Owned<IFileDescriptor> fdesc = source->getFileDescriptor();
-                        checkPhysicalFilePermissions(fdesc,userdesc,false);
+                        checkPlaneFilePermissions(fdesc,userdesc,false);
                         checkSourceTarget(fdesc);
                         bool needrep = options->getReplicate();
                         ClusterPartDiskMapSpec mspec;
@@ -1777,7 +1828,7 @@ class CDFUengine: public CInterface, implements IDFUengine
                 {
                     runningconn.setown(setRunning(runningpath.str()));
                     Owned<IFileDescriptor> fdesc = destination->getFileDescriptor(iskey);
-                    checkPhysicalFilePermissions(fdesc,userdesc,true);
+                    checkPlaneFilePermissions(fdesc,userdesc,true);
                     checkSourceTarget(fdesc);
                     fsys.exportFile(srcFile, fdesc, recovery, recoveryconn, filter, opttree, &feedback, &abortnotify, dfuwuid);
                     if (!abortnotify.abortRequested()) {

diff --git a/docs/EN_US/ContainerizedHPCC/ContainerizedHPCCSystemsPlatform.xml b/docs/EN_US/ContainerizedHPCC/ContainerizedHPCCSystemsPlatform.xml
@@ -130,7 +130,7 @@
       few fundamental changes to note.</para>
 
       <sect2 id="processesandpods">
-        <title>Processes and pods, not machines</title>
+        <title>Processes and Pods, not Machines</title>
 
         <para>Anyone familiar with the existing configuration system will know
         that part of the configuration involves creating instances of each
@@ -149,7 +149,7 @@
       </sect2>
 
       <sect2 id="helmcharts001">
-        <title>Helm charts</title>
+        <title>Helm Charts</title>
 
         <para>In the containerized world, the information that the operator
         needs to supply to configure an HPCC Systems environment is greatly
@@ -204,7 +204,7 @@
       </sect2>
 
       <sect2 id="topoclustersvsqueues">
-        <title>Topology settings – Clusters vs queues</title>
+        <title>Topology Settings – Clusters vs Queues</title>
 
         <para>In bare-metal deployments, there is a section called <emphasis
         role="strong">Topology</emphasis> where the various queues that