HPCC-32875 Ensure rowservice can process fields stored as blobs

Signed-off-by: Gavin Halliday <[email protected]>
hpcc-systems · Oct 31, 2024 · fcc5efc · fcc5efc
1 parent cf4b015
commit fcc5efc
Show file tree

Hide file tree

Showing 4 changed files with 40 additions and 8 deletions.
diff --git a/fs/dafsserver/dafsserver.cpp b/fs/dafsserver/dafsserver.cpp
@@ -2278,6 +2278,7 @@ class CRemoteIndexReadActivity : public CRemoteIndexBaseActivity
 
     Owned<const IDynamicTransform> translator;
     unsigned __int64 chooseN = 0;
+    bool cleanupBlobs = false;
 public:
     CRemoteIndexReadActivity(IPropertyTree &config, IFileDescriptor *fileDesc) : PARENT(config, fileDesc)
     {
@@ -2316,6 +2317,12 @@ class CRemoteIndexReadActivity : public CRemoteIndexBaseActivity
                         }
                         dbgassertex(retSz);
 
+                        if (cleanupBlobs)
+                        {
+                            keyManager->releaseBlobs();
+                            cleanupBlobs = false;
+                        }
+
                         const void *ret = outBuilder.getSelf();
                         outBuilder.finishRow(retSz);
                         ++processed;
@@ -2350,6 +2357,13 @@ class CRemoteIndexReadActivity : public CRemoteIndexBaseActivity
     {
         return out.appendf("indexread[%s]", fileName.get());
     }
+
+    virtual const byte * lookupBlob(unsigned __int64 id) override
+    {
+        size32_t dummy;
+        cleanupBlobs = true;
+        return (byte *) keyManager->loadBlob(id, dummy, nullptr);
+    }
 };
 
 

diff --git a/rtl/eclrtl/rtldynfield.cpp b/rtl/eclrtl/rtldynfield.cpp
@@ -1690,6 +1690,7 @@ class GeneralRecordTranslator : public CInterfaceOf<IDynamicTransform>
                         {
                             const RtlRecord *subDest = destRecInfo.queryNested(idx);
                             const RtlRecord *subSrc = sourceRecInfo.queryNested(info.matchIdx);
+                            assertex(subSrc);
                             info.subTrans = new GeneralRecordTranslator(*subDest, *subSrc, binarySource);
                             if (!info.subTrans->needsTranslate())
                             {

diff --git a/rtl/eclrtl/rtlrecord.cpp b/rtl/eclrtl/rtlrecord.cpp
@@ -292,6 +292,12 @@ RtlRecord::RtlRecord(const RtlFieldInfo * const *_fields, bool expandFields) : f
         const RtlTypeInfo *curType = queryType(i);
         if (!curType->isFixedSize() || (fields[i]->flags & RFTMinifblock))
             numVarFields++;
+        if (curType->isBlob())
+        {
+            curType = curType->queryChildType();
+            if (unlikely(!curType))
+                throwUnexpectedX("Blob type has no child type");
+        }
         if (curType->getType()==type_table || curType->getType()==type_record || curType->getType()==type_dictionary)
             numTables++;
     }
@@ -331,6 +337,8 @@ RtlRecord::RtlRecord(const RtlFieldInfo * const *_fields, bool expandFields) : f
             curVariable++;
             fixedOffset = 0;
         }
+        if (curType->isBlob())
+            curType = curType->queryChildType();
         switch (curType->getType())
         {
         case type_table:

diff --git a/system/jhtree/jhtree.cpp b/system/jhtree/jhtree.cpp
@@ -3636,7 +3636,7 @@ class IKeyManagerTest : public CppUnit::TestFixture
     {
         const char *json = variable ?
                 "{ \"ty1\": { \"fieldType\": 4, \"length\": 10 }, "
-                "  \"ty2\": { \"fieldType\": 15, \"length\": 8 }, "
+                "  \"ty2\": { \"fieldType\": 15, \"length\": 8, \"child\": \"ty1\" }, "
                 " \"fieldType\": 13, \"length\": 10, "
                 " \"fields\": [ "
                 " { \"name\": \"f1\", \"type\": \"ty1\", \"flags\": 4 }, "
@@ -3816,13 +3816,22 @@ class IKeyManagerTest : public CppUnit::TestFixture
 
     void testKeys()
     {
-        ASSERT(sizeof(CKeyIdAndPos) == sizeof(unsigned __int64) + sizeof(offset_t));
-        for (bool var : { true, false })
-            for (bool trail : { false, true })
-                for (bool noseek : { false, true })
-                    for (bool quick : { true, false })
-                        for (const char * compression : { (const char *)nullptr, "POC", "inplace" })
-                            testKeys(var, trail, noseek, quick, compression);
+        try
+        {
+            ASSERT(sizeof(CKeyIdAndPos) == sizeof(unsigned __int64) + sizeof(offset_t));
+            for (bool var : { true, false })
+                for (bool trail : { false, true })
+                    for (bool noseek : { false, true })
+                        for (bool quick : { true, false })
+                            for (const char * compression : { (const char *)nullptr, "POC", "inplace" })
+                                testKeys(var, trail, noseek, quick, compression);
+        }
+        catch (IException * e)
+        {
+            StringBuffer s;
+            e->errorMessage(s);
+            CPPUNIT_ASSERT_MESSAGE(s.str(), false);
+        }
     }
 };