Skip to content

Commit

Permalink
Merge remote-tracking branch 'origin/candidate-9.8.x'
Browse files Browse the repository at this point in the history 
Signed-off-by: Gordon Smith <[email protected]>

# Conflicts:
#	helm/hpcc/Chart.yaml
#	helm/hpcc/templates/_helpers.tpl
#	version.cmake
  • Loading branch information
@GordonSmith
GordonSmith committed Aug 30, 2024
2 parents 0ef5429 + f9f260a commit 170f1cb
Show file tree
Hide file tree
Showing 56 changed files with 2,887 additions and 497 deletions.
55 changes: 41 additions & 14 deletions .github/workflows/test-helm.yml
View file
Original file line number Diff line number Diff line change
@@ -1,16 +1,6 @@
name: Run helm chart tests

on:
push:
branches:
- "master"
- "candidate-*"
- "!candidate-7.8.*"
- "!candidate-7.6.*"
- "!candidate-7.4.*"
- "!candidate-7.2.*"
- "!candidate-7.0.*"
- "!candidate-6.*"
pull_request:
branches:
- "master"
Expand Down Expand Up @@ -57,8 +47,45 @@ jobs:
- name: Check for changes in helm output for default values
working-directory: .
run: |
helm template helm/hpcc > ${{ runner.temp }}/newoutput.txt
git fetch --no-tags --prune --progress --no-recurse-submodules --depth=1 origin ${{ github.base_ref }}
resultcode=0
mkdir ${{ runner.temp }}/new
mkdir ${{ runner.temp }}/old
#Generate the output for the default values file, and each of the specialised test files.
#Check to see if anything has changed between old and new, and report the differences
helm template helm/hpcc > ${{ runner.temp }}/new/output.txt
for file in testing/helm/tests/*.yaml
do
tail=$(basename $file)
helm template helm/hpcc --values $file > ${{ runner.temp }}/new/$tail.txt
done
git fetch --no-tags --prune --progress --no-recurse-submodules --quiet --depth=1 origin ${{ github.base_ref }}
echo git checkout ${{ github.base_ref }}
git checkout ${{ github.base_ref }}
helm template helm/hpcc > ${{ runner.temp }}/oldoutput.txt
diff ${{ runner.temp }}/oldoutput.txt ${{ runner.temp }}/newoutput.txt
helm template helm/hpcc > ${{ runner.temp }}/old/output.txt
for file in testing/helm/tests/*.yaml
do
tail=$(basename $file)
helm template helm/hpcc --values $file > ${{ runner.temp }}/old/$tail.txt
done
diff ${{ runner.temp }}/old/output.txt ${{ runner.temp }}/new/output.txt
if [ $? -ne 0 ]
then
resultcode=1
else
#Only check for differences in the specialised test files if the default values file is the same
for file in testing/helm/tests/*.yaml
do
tail=$(basename $file)
diff ${{ runner.temp }}/old/$tail.txt ${{ runner.temp }}/new/$tail.txt
if [ $? -ne 0 ]
then
resultcode=1
fi
done
fi
exit $resultcode
26 changes: 17 additions & 9 deletions common/thorhelper/thorsoapcall.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2082,10 +2082,11 @@ class CWSCAsyncFor : implements IWSCAsyncFor, public CInterface, public CAsyncFo
StringBuffer dbgheader, contentEncoding;
bool chunked = false;
size32_t read = 0;
bool sockClosed = false;
do {
checkTimeLimitExceeded(&remainingMS);
checkRoxieAbortMonitor(master->roxieAbortMonitor);
readtmsAllowClose(socket, buffer+read, 1, WSCBUFFERSIZE-read, bytesRead, MIN(master->timeoutMS, remainingMS));
sockClosed = readtmsAllowClose(socket, buffer+read, 1, WSCBUFFERSIZE-read, bytesRead, MIN(master->timeoutMS, remainingMS));
checkTimeLimitExceeded(&remainingMS);
checkRoxieAbortMonitor(master->roxieAbortMonitor);

Expand Down Expand Up @@ -2185,7 +2186,7 @@ class CWSCAsyncFor : implements IWSCAsyncFor, public CInterface, public CAsyncFo
break;
}
}
if (bytesRead == 0) // socket closed
if ( (bytesRead == 0) || (sockClosed) ) // socket closed
break;
} while (bytesRead&&(read<WSCBUFFERSIZE));
if (!chunked)
Expand All @@ -2201,33 +2202,40 @@ class CWSCAsyncFor : implements IWSCAsyncFor, public CInterface, public CAsyncFo
}
if (read)
response.append(read,payload);
if (payloadsize) { // read directly into response
while (read<payloadsize) {
if (payloadsize) // read directly into response
{
while (read<payloadsize)
{
checkTimeLimitExceeded(&remainingMS);
checkRoxieAbortMonitor(master->roxieAbortMonitor);
readtmsAllowClose(socket, response.reserve(payloadsize-read), 1, payloadsize-read, bytesRead, MIN(master->timeoutMS, remainingMS));
sockClosed = readtmsAllowClose(socket, response.reserve(payloadsize-read), 1, payloadsize-read, bytesRead, MIN(master->timeoutMS, remainingMS));
checkTimeLimitExceeded(&remainingMS);
checkRoxieAbortMonitor(master->roxieAbortMonitor);

read += bytesRead;
response.setLength(read);
if (bytesRead==0) {
if ( (bytesRead==0) || (sockClosed && (read < payloadsize)) )
{
master->logctx.CTXLOG("%s: Warning %sHTTP response terminated prematurely", getWsCallTypeName(master->wscType),chunked?"CHUNKED ":"");
break; // oops looks likesocket closed early
}
}
}
else {
for (;;) {
else
{
for (;;)
{
checkTimeLimitExceeded(&remainingMS);
checkRoxieAbortMonitor(master->roxieAbortMonitor);
readtmsAllowClose(socket, buffer, 1, WSCBUFFERSIZE, bytesRead, MIN(master->timeoutMS, remainingMS));
sockClosed = readtmsAllowClose(socket, buffer, 1, WSCBUFFERSIZE, bytesRead, MIN(master->timeoutMS, remainingMS));
checkTimeLimitExceeded(&remainingMS);
checkRoxieAbortMonitor(master->roxieAbortMonitor);

if (bytesRead==0)
break; // rely on socket closing to terminate message
response.append(bytesRead,buffer);
if (sockClosed)
break;
}
}
}
Expand Down
8 changes: 4 additions & 4 deletions common/workunit/workunit.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6430,7 +6430,7 @@ class CDaliWorkUnitFactory : public CWorkUnitFactory, implements IDaliClientShut
return WUStateUnknown;
}

Owned<WorkUnitWaiter> waiter = new WorkUnitWaiter(wuid, SubscribeOptionState);
Owned<WorkUnitWaiter> waiter = new WorkUnitWaiter(wuid, (WUSubscribeOptions)(SubscribeOptionState|SubscribeOptionAbort));
LocalIAbortHandler abortHandler(*waiter);
if (conn)
{
Expand Down Expand Up @@ -6504,7 +6504,7 @@ class CDaliWorkUnitFactory : public CWorkUnitFactory, implements IDaliClientShut
waiter->wait(20000); // recheck state every 20 seconds, in case eclagent has crashed.
if (waiter->isAborted())
{
ret = WUStateUnknown; // MORE - throw an exception?
ret = WUStateAborting;
break;
}
}
Expand Down Expand Up @@ -14482,13 +14482,13 @@ void executeThorGraph(const char * graphName, IConstWorkUnit &workunit, const IP
unsigned runningTimeLimit = workunit.getDebugValueInt("maxRunTime", 0);
runningTimeLimit = runningTimeLimit ? runningTimeLimit : INFINITE;

std::list<WUState> expectedStates = { WUStateRunning, WUStateWait, WUStateFailed };
std::list<WUState> expectedStates = { WUStateRunning, WUStateWait, WUStateAborting, WUStateFailed };
unsigned __int64 blockedTime = 0;
for (unsigned i=0; i<2; i++)
{
WUState state = waitForWorkUnitToComplete(wuid, timelimit*1000, expectedStates);
DBGLOG("Got state: %s", getWorkunitStateStr(state));
if ((WUStateWait == state) || (WUStateFailed == state)) // already finished or failed
if ((WUStateWait == state) || (WUStateFailed == state) || (WUStateAborting == state)) // already finished or failed or aborting
{
// workunit may have spent time in blocked state, but then transitioned to
// wait or failed state quickly such that this code did not see its running state.
Expand Down
23 changes: 19 additions & 4 deletions common/workunit/workunit.ipp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -699,20 +699,33 @@ class WorkUnitWaiter : public CInterface, implements IAbortHandler, implements I
{
Semaphore changed;
Owned<IWorkUnitWatcher> watcher;
bool aborted;
mutable bool aborted = false;
mutable bool abortDirty = false;
StringAttr wuid;
public:
IMPLEMENT_IINTERFACE;
WorkUnitWaiter(const char *wuid, WUSubscribeOptions watchFor)
WorkUnitWaiter(const char *_wuid, WUSubscribeOptions watchFor) : wuid(_wuid)
{
Owned<IWorkUnitFactory> factory = getWorkUnitFactory();
watcher.setown(factory->getWatcher(this, watchFor, wuid));
aborted = false;
if (watchFor & SubscribeOptionAbort)
abortDirty = true;
}
~WorkUnitWaiter()
{
unsubscribe();
}
bool isAborted() const { return aborted; }
bool isAborted() const
{
if (!aborted && abortDirty)
{
Owned<IWorkUnitFactory> factory = getWorkUnitFactory();
if (factory->isAborting(wuid))
aborted = true;
abortDirty = false;
}
return aborted;
}
bool wait(unsigned timeout)
{
return changed.wait(timeout) && !aborted;
Expand All @@ -728,6 +741,8 @@ public:
// IWorkUnitSubscriber
virtual void notify(WUSubscribeOptions flags, unsigned valueLen, const void *valueData) override
{
if (SubscribeOptionAbort == flags)
abortDirty = true;
changed.signal();
}
// IAbortHandler
Expand Down
17 changes: 10 additions & 7 deletions dali/base/dacsds.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2239,6 +2239,7 @@ bool CClientSDSManager::updateEnvironment(IPropertyTree *newEnv, bool forceGroup

//////////////

static bool releaseActiveManager = false;
static ISDSManager * activeSDSManager=NULL;
static ISDSManager * savedSDSManager=NULL;

Expand All @@ -2248,10 +2249,13 @@ MODULE_INIT(INIT_PRIORITY_STANDARD)
}
MODULE_EXIT()
{
delete activeSDSManager;
activeSDSManager = nullptr;
delete savedSDSManager;
savedSDSManager = nullptr;
if (releaseActiveManager)
{
delete activeSDSManager;
activeSDSManager = nullptr;
delete savedSDSManager;
savedSDSManager = nullptr;
}
}

ISDSManager &querySDS()
Expand All @@ -2261,9 +2265,8 @@ ISDSManager &querySDS()
return *activeSDSManager;
else if (!queryCoven().inCoven())
{
if (!activeSDSManager)
activeSDSManager = new CClientSDSManager();

releaseActiveManager = true;
activeSDSManager = new CClientSDSManager();
return *activeSDSManager;
}
else
Expand Down
57 changes: 36 additions & 21 deletions docs/EN_US/HPCCClientTools/CT_Mods/CT_ECL_CLI.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1006,10 +1006,11 @@ ecl publish roxie ArchiveQuery.xml --query-name=FindPersonService --no-activate
</row>

<row>
<entry>--wait=&lt;sec&gt;</entry>
<entry>--wait=&lt;ms&gt;</entry>

<entry>Maximum time to wait for cluster finish
updating</entry>
<entry>Maximum time (in milliseconds) to wait for cluster
finish updating. Any value lower than the minimum of 1000 is
ignored. </entry>
</row>

<row>
Expand Down Expand Up @@ -1594,8 +1595,9 @@ ecl run thor findperson.ecl -I C:\MyECL\
<row>
<entry>--wait=&lt;ms&gt;</entry>

<entry>Maximum time to wait for cluster finish updating (in
ms)</entry>
<entry>Maximum time (in milliseconds) to wait for cluster
finish updating. Any value lower than the minimum of 1000 is
ignored. </entry>
</row>

<row>
Expand Down Expand Up @@ -2803,8 +2805,9 @@ ecl queries copy //192.168.1.10:8010/thor/findperson thor
<row>
<entry>--wait=&lt;ms&gt;</entry>

<entry>Maximum time to wait for cluster finish updating (in
ms)</entry>
<entry>Maximum time (in milliseconds) to wait for cluster
finish updating. Any value lower than the minimum of 1000 is
ignored. </entry>
</row>

<row>
Expand Down Expand Up @@ -3363,10 +3366,11 @@ ecl queries copy-set roxie1 roxie2 --clone-active-state</programlisting>
</row>

<row>
<entry>--wait=&lt;sec&gt;</entry>
<entry>--wait=&lt;ms&gt;</entry>

<entry>Maximum time to wait for cluster finish updating (in
ms)</entry>
<entry>Maximum time (in milliseconds) to wait for cluster
finish updating. Any value lower than the minimum of 1000 is
ignored. </entry>
</row>

<row>
Expand Down Expand Up @@ -3704,10 +3708,11 @@ ecl queries recreate roxie findpeople roxie2</programlisting>
</row>

<row>
<entry>--wait=&lt;sec&gt;</entry>
<entry>--wait=&lt;ms&gt;</entry>

<entry>Maximum time to wait for cluster finish
updating</entry>
<entry>Maximum time (in milliseconds) to wait for cluster
finish updating. Any value lower than the minimum of 1000 is
ignored. </entry>
</row>

<row>
Expand Down Expand Up @@ -6549,7 +6554,9 @@ ecl packagemap copy //192.168.0.100:8010/roxie/MyPkg roxie2
<row>
<entry>--wait=&lt;ms&gt;</entry>

<entry>The maximum time to wait in milliseconds</entry>
<entry>Maximum time (in milliseconds) to wait for cluster
finish updating. Any value lower than the minimum of 1000 is
ignored. </entry>
</row>

<row>
Expand Down Expand Up @@ -6680,7 +6687,9 @@ ecl packagemap copy //192.168.0.100:8010/roxie/MyPkg roxie2
<row>
<entry>--wait=&lt;ms&gt;</entry>

<entry>The maximum time to wait in milliseconds</entry>
<entry>Maximum time (in milliseconds) to wait for cluster
finish updating. Any value lower than the minimum of 1000 is
ignored. </entry>
</row>

<row>
Expand Down Expand Up @@ -6811,7 +6820,9 @@ ecl packagemap copy //192.168.0.100:8010/roxie/MyPkg roxie2
<row>
<entry>--wait=&lt;ms&gt;</entry>

<entry>The maximum time to wait in milliseconds</entry>
<entry>Maximum time (in milliseconds) to wait for cluster
finish updating. Any value lower than the minimum of 1000 is
ignored. </entry>
</row>

<row>
Expand Down Expand Up @@ -6942,7 +6953,9 @@ ecl packagemap copy //192.168.0.100:8010/roxie/MyPkg roxie2
<row>
<entry>--wait=&lt;ms&gt;</entry>

<entry>The maximum time to wait in milliseconds</entry>
<entry>Maximum time (in milliseconds) to wait for cluster
finish updating. Any value lower than the minimum of 1000 is
ignored. </entry>
</row>

<row>
Expand Down Expand Up @@ -7116,7 +7129,9 @@ ecl roxie xref myroxie --queryids=myquery.1,myotherquery.1</programlisting>
<row>
<entry>--wait=&lt;ms&gt;</entry>

<entry>The maximum time to wait in milliseconds</entry>
<entry>Maximum time (in milliseconds) to wait for cluster
finish updating. Any value lower than the minimum of 1000 is
ignored. </entry>
</row>

<row>
Expand Down Expand Up @@ -8652,15 +8667,15 @@ ecl zapgen W20171018-091399 --path ~ --inc-thor-slave-logs --description "Unexpe
</tgroup>
</informaltable>The <emphasis>ecl url-secret-name </emphasis>command
generates a secret name from a URL that can be used to support ECL
SOAPCALL/HTTPCALL automated URL to Secret mapping. </para>
SOAPCALL/HTTPCALL automated URL to Secret mapping.</para>

<para>A username can either be embedded in the URL, such as
https://[email protected], or passed in as a parameter using the
--username=username option. If a username is passed in as a parameter
it overrides a username in the URL. </para>
it overrides a username in the URL.</para>

<para>Passwords embedded in the URL are not needed and will be
ignored. </para>
ignored.</para>

<para>When ECL SOAPCALL URL secret mapping is enabled SOAPCALL will
convert the URL provided into a name of this format. ECL will then
Expand Down
Loading

0 comments on commit 170f1cb

Please sign in to comment.