HPCC-32111 Use compression in global merge activity

Signed-off-by: Jake Smith <[email protected]>
hpcc-systems · Oct 31, 2024 · a84be5f · a84be5f
1 parent ed05bcf
commit a84be5f
Show file tree

Hide file tree

Showing 3 changed files with 20 additions and 6 deletions.
diff --git a/thorlcr/activities/merge/thmergeslave.cpp b/thorlcr/activities/merge/thmergeslave.cpp
@@ -47,6 +47,7 @@ class GlobalMergeSlaveActivity : public CSlaveActivity
     offset_t *partitionpos;
     size32_t chunkmaxsize;
     unsigned width;
+    unsigned rwFlags = 0x0; // flags for streams (e.g. compression flags)
 
     class cRemoteStream : implements IRowStream, public CSimpleInterface
     {
@@ -220,7 +221,7 @@ class GlobalMergeSlaveActivity : public CSlaveActivity
 
         CThorKeyArray partition(*this, queryRowInterfaces(this),helper->querySerialize(),helper->queryCompare(),helper->queryCompareKey(),helper->queryCompareRowKey());
         partition.deserialize(mb,false);
-        partition.calcPositions(tmpfile,sample);
+        partition.calcPositions(tmpfile, sample, rwFlags);
         partitionpos = new offset_t[width];
         unsigned i;
         for (i=0;i<width;i++) {
@@ -270,6 +271,19 @@ class GlobalMergeSlaveActivity : public CSlaveActivity
     void init(MemoryBuffer &data, MemoryBuffer &slaveData) override
     {
         masterMpTag = container.queryJobChannel().deserializeMPTag(data);
+        rwFlags = DEFAULT_RWFLAGS;
+        if (getOptBool(THOROPT_COMPRESS_SPILLS, true))
+        {
+            StringBuffer compType;
+            getOpt(THOROPT_COMPRESS_SPILL_TYPE, compType);
+            unsigned spillCompInfo = 0x0;
+            setCompFlag(compType, spillCompInfo);
+            if (spillCompInfo)
+            {
+                rwFlags |= rw_compress;
+                rwFlags |= spillCompInfo;
+            }
+        }
     }
 
     void abort()
@@ -322,7 +336,7 @@ class GlobalMergeSlaveActivity : public CSlaveActivity
         StringBuffer tmpname;
         GetTempFilePath(tmpname,"merge");
         tmpfile.setown(createIFile(tmpname.str()));
-        Owned<IRowWriter> writer =  createRowWriter(tmpfile, this);
+        Owned<IRowWriter> writer =  createRowWriter(tmpfile, this, rwFlags);
         CThorKeyArray sample(*this, this, helper->querySerialize(), helper->queryCompare(), helper->queryCompareKey(), helper->queryCompareRowKey());
         sample.setSampling(MERGE_TRANSFER_BUFFER_SIZE);
         ActPrintLog("MERGE: start gather");
@@ -366,7 +380,7 @@ class GlobalMergeSlaveActivity : public CSlaveActivity
         offset_t end = partitionpos[idx];
         if (pos>=end)
             return 0;
-        Owned<IExtRowStream> rs = createRowStreamEx(tmpfile, queryRowInterfaces(this), pos, end); // this is not good
+        Owned<IExtRowStream> rs = createRowStreamEx(tmpfile, queryRowInterfaces(this), pos, end, (unsigned __int64)-1, rwFlags); // this is not good
         offset_t so = rs->getOffset();
         size32_t len = 0;
         size32_t chunksize = chunkmaxsize;

diff --git a/thorlcr/msort/tsorta.cpp b/thorlcr/msort/tsorta.cpp
@@ -443,7 +443,7 @@ offset_t CThorKeyArray::findLessRowPos(const void * row)
     return getFixedFilePos(p);
 }
 
-void CThorKeyArray::calcPositions(IFile *file,CThorKeyArray &sample)
+void CThorKeyArray::calcPositions(IFile *file, CThorKeyArray &sample, unsigned rwFlags)
 {
     // calculates positions based on sample
     // not fast!
@@ -459,7 +459,7 @@ void CThorKeyArray::calcPositions(IFile *file,CThorKeyArray &sample)
         if (pos==(offset_t)-1) 
             pos = 0;
         // should do bin-chop for fixed length but initially do sequential search
-        Owned<IRowStream> s = createRowStreamEx(file, rowif, pos);
+        Owned<IRowStream> s = createRowStreamEx(file, rowif, pos, 0, (offset_t)-1, rwFlags);
         for (;;)
         {
             OwnedConstThorRow rowcmp = s->nextRow();

diff --git a/thorlcr/msort/tsorta.hpp b/thorlcr/msort/tsorta.hpp
@@ -114,7 +114,7 @@ class THORSORT_API CThorKeyArray
     void deserialize(MemoryBuffer &mb,bool append);
     void sort();
     void createSortedPartition(unsigned pn);
-    void calcPositions(IFile *file,CThorKeyArray &sample);
+    void calcPositions(IFile *file, CThorKeyArray &sample, unsigned rwFlags);
     void setSampling(size32_t _maxsamplesize, unsigned _divisor=0);
     int keyCompare(unsigned a,unsigned b);
     offset_t getFixedFilePos(unsigned i);