From 09edeafd7c231ef94e211c425e53ebea30998bac Mon Sep 17 00:00:00 2001
From: Jesun Sahariar Firoz <jesun.firoz@pnnl.gov>
Date: Mon, 2 Dec 2019 16:46:00 -0800
Subject: [PATCH 01/13] Changing WorkQueue to be compatible with Chapel 1.2

---
 src/AggregationBuffer.chpl        | 34 +++++++++++++++----------------
 src/DynamicAggregationBuffer.chpl | 14 ++++++-------
 src/Generation.chpl               |  6 +++---
 src/WorkQueue.chpl                |  6 +++---
 4 files changed, 30 insertions(+), 30 deletions(-)

diff --git a/src/AggregationBuffer.chpl b/src/AggregationBuffer.chpl
index ea78807..f299eb0 100644
--- a/src/AggregationBuffer.chpl
+++ b/src/AggregationBuffer.chpl
@@ -2,7 +2,7 @@
 /*
   TODO: Experiment with expanding buffer sizes
 */
-module AggregationBuffer {
+prototype module AggregationBuffer {
 
   use Time;
   use Random;
@@ -16,7 +16,7 @@ module AggregationBuffer {
   pragma "always RVF"
   record Aggregator {
     type msgType;
-    var instance : unmanaged AggregatorImpl(msgType);
+    var instance : unmanaged AggregatorImpl(msgType)?;
     var pid = -1;
 
     proc init(type msgType, aggregatorBufferSize : int = AggregatorBufferSize, aggregatorMaxBuffers : int = AggregatorMaxBuffers) {
@@ -25,13 +25,13 @@ module AggregationBuffer {
       this.pid = this.instance.pid;
     }
 
-    proc init(type msgType, instance : unmanaged AggregatorImpl(msgType), pid : int) {
+    proc init(type msgType, instance : unmanaged AggregatorImpl(msgType)?, pid : int) {
       this.msgType = msgType;
       this.instance = instance;
       this.pid = pid;
     }
 
-    proc init(other) {
+    proc init=(other) {
       this.msgType = other.msgType;
       this.instance = other.instance;
       this.pid = other.pid;
@@ -65,9 +65,9 @@ module AggregationBuffer {
     type msgType;
     var lock$ : sync bool;
     // Head of list of all buffers that can be recycled.
-    var freeBufferList : unmanaged Buffer(msgType);
+    var freeBufferList : unmanaged Buffer(msgType)?;
     // Head of list of all allocated buffers.
-    var allocatedBufferList : unmanaged Buffer(msgType);
+    var allocatedBufferList : unmanaged Buffer(msgType)?;
     // Number of buffers that are available to be recycled...
     var numFreeBuffers : chpl__processorAtomicType(int);
     // Number of buffers that are currently allocated
@@ -93,24 +93,24 @@ module AggregationBuffer {
     }
 
     inline proc canAllocateBuffer() {
-      return aggregatorMaxBuffers == -1 || numAllocatedBuffers.peek() < aggregatorMaxBuffers;
+      return aggregatorMaxBuffers == -1 || numAllocatedBuffers.read() < aggregatorMaxBuffers;
     }
 
     proc getBuffer() : unmanaged Buffer(msgType) {
-      var buf : unmanaged Buffer(msgType);
+      var buf : unmanaged Buffer(msgType)?;
 
       while buf == nil {
         // Yield while we wait for a free buffer...
-        while numFreeBuffers.peek() == 0 && !canAllocateBuffer() {
+        while numFreeBuffers.read() == 0 && !canAllocateBuffer() {
           debug(here, ": waiting on free buffer..., numAllocatedBuffers(", 
-              numAllocatedBuffers.peek(), ") / maxAllocatedBuffers(", aggregatorMaxBuffers, ")");
+              numAllocatedBuffers.read(), ") / maxAllocatedBuffers(", aggregatorMaxBuffers, ")");
           chpl_task_yield();
         }
 
         lock$ = true;
         // Out of buffers, try to create a new one.
         // Note: Since we do this inside the lock we can relax all atomics
-        if numFreeBuffers.peek() == 0 {
+        if numFreeBuffers.read() == 0 {
           if canAllocateBuffer() {
             var tmp = new unmanaged Buffer(msgType, aggregatorBufferSize);
             numAllocatedBuffers.add(1, memory_order_relaxed);
@@ -128,7 +128,7 @@ module AggregationBuffer {
 
       buf.reset();
       buf._bufferPool = _to_unmanaged(this);
-      return buf;
+      return buf!;
     }
 
     proc recycleBuffer(buf : unmanaged Buffer(msgType)) {
@@ -172,11 +172,11 @@ module AggregationBuffer {
     pragma "no doc"
     var _stolen : chpl__processorAtomicType(bool);
     pragma "no doc"
-    var _nextAllocatedBuffer : unmanaged Buffer(msgType);
+    var _nextAllocatedBuffer : unmanaged Buffer(msgType)?;
     pragma "no doc"
-    var _nextFreeBuffer : unmanaged Buffer(msgType);
+    var _nextFreeBuffer : unmanaged Buffer(msgType)?;
     pragma "no doc"
-    var _bufferPool : unmanaged BufferPool(msgType);
+    var _bufferPool : unmanaged BufferPool(msgType)?;
 
     pragma "no doc"
     proc init(type msgType, aggregatorBufferSize : int) {
@@ -353,9 +353,9 @@ module AggregationBuffer {
       return aggregate(msg, loc.id);
     }
     
-    proc aggregate(msg : msgType, locid : int) : unmanaged Buffer(msgType) {
+    proc aggregate(msg : msgType, locid : int) : unmanaged Buffer(msgType)? {
       // Performs sanity checks to ensure that returned buffer is valid
-      proc doSanityCheck(buf : unmanaged Buffer(msgType)) where AggregatorDebug {
+      proc doSanityCheck(buf : unmanaged Buffer(msgType)?) where AggregatorDebug {
         if buf._stolen.peek() != false then halt("Buffer is still stolen!", buf);
         if buf._claimed.peek() != 0 then halt("Buffer has not had claim reset...", buf);
         if buf._filled.peek() != 0 then halt("Buffer has not had filled reset...", buf);
diff --git a/src/DynamicAggregationBuffer.chpl b/src/DynamicAggregationBuffer.chpl
index cd3e0ba..c57d597 100644
--- a/src/DynamicAggregationBuffer.chpl
+++ b/src/DynamicAggregationBuffer.chpl
@@ -5,7 +5,7 @@
   called, data never gets sent, but it opens the possibility of the user creating
   their own background progress task.
 */
-module DynamicAggregationBuffer {
+prototype module DynamicAggregationBuffer {
   use AggregationBuffer;
 
   proc UninitializedDynamicAggregator(type msgType) return new DynamicAggregator(msgType, instance=nil, pid=-1);
@@ -14,7 +14,7 @@ module DynamicAggregationBuffer {
   record DynamicAggregator {
     type msgType;
     var pid = -1;
-    var instance : unmanaged DynamicAggregatorImpl(msgType);
+    var instance : unmanaged DynamicAggregatorImpl(msgType)?;
 
     proc init(type msgType) {
       this.msgType = msgType;
@@ -23,13 +23,13 @@ module DynamicAggregationBuffer {
       this.instance = instance;
     }
 
-    proc init(type msgType, instance : unmanaged DynamicAggregatorImpl(msgType), pid : int) {
+    proc init(type msgType, instance : unmanaged DynamicAggregatorImpl(msgType)?, pid : int) {
       this.msgType = msgType;
       this.pid = pid;
       this.instance = instance;
     }
 
-    proc init(other) {
+    proc init=(other) {
       this.msgType = other.msgType;
       this.pid = other.pid;
       this.instance = other.instance;
@@ -168,7 +168,7 @@ module DynamicAggregationBuffer {
       }
     }
 
-    iter flushLocal() : (DynamicBuffer(msgType), locale) {
+    iter flushLocal() : (unmanaged DynamicBuffer(msgType), locale) {
      for (buf, loc) in agg.flushLocal() {
         dynamicDestBuffers[loc.id].append(buf.getArray());
         buf.done();
@@ -190,11 +190,11 @@ module DynamicAggregationBuffer {
       }
     }
 
-    iter flushGlobal() : (DynamicBuffer(msgType), locale) {
+    iter flushGlobal() : (unmanaged DynamicBuffer(msgType), locale) {
       halt("Serial 'flush' not implemented, use 'forall'...");
     }
      
-    iter flushGlobal(param tag : iterKind) : (DynamicBuffer(msgType), locale) where tag == iterKind.standalone {
+    iter flushGlobal(param tag : iterKind) : (unmanaged DynamicBuffer(msgType), locale) where tag == iterKind.standalone {
       // Flush aggregator first...
       forall (buf, loc) in agg.flushGlobal() {
         getPrivatizedInstance().dynamicDestBuffers[loc.id].append(buf.getArray());
diff --git a/src/Generation.chpl b/src/Generation.chpl
index dbad99c..ec00598 100644
--- a/src/Generation.chpl
+++ b/src/Generation.chpl
@@ -1,7 +1,7 @@
 module Generation {
 
   use IO;
-  use CHGL;
+  use AdjListHyperGraph;
   use Random;
   use BlockDist;
   use CyclicDist;
@@ -237,14 +237,14 @@ module Generation {
   
   // Compute Table degrees to vertices...
   record DynamicArray {
-    var dom = {0..-1};
+    var dom = {0..0};
     var arr : [dom] int;
     
     proc init() {
 
     }
     
-    proc init(other) {
+    proc init=(other) {
       this.dom = other.dom;
       this.arr = other.arr;
     }
diff --git a/src/WorkQueue.chpl b/src/WorkQueue.chpl
index 85db1ce..a303ff9 100644
--- a/src/WorkQueue.chpl
+++ b/src/WorkQueue.chpl
@@ -671,7 +671,7 @@ class BagSegmentBlock {
 
   // Contiguous memory containing all elements
   var elems :  _ddata(eltType);
-  var next : unmanaged BagSegmentBlock(eltType);
+  var next : unmanaged BagSegmentBlock(eltType)?;
 
   // The capacity of this block.
   var cap : int;
@@ -747,8 +747,8 @@ record BagSegment {
   // Used as a test-and-test-and-set spinlock.
   var status : chpl__processorAtomicType(uint);
 
-  var headBlock : unmanaged BagSegmentBlock(eltType);
-  var tailBlock : unmanaged BagSegmentBlock(eltType);
+  var headBlock : unmanaged BagSegmentBlock(eltType)?;
+  var tailBlock : unmanaged BagSegmentBlock(eltType)?;
 
   var nElems : chpl__processorAtomicType(uint);
 

From e2995a20659e993c840c966df76220ae9518eee6 Mon Sep 17 00:00:00 2001
From: Jesun Sahariar Firoz <jesun.firoz@pnnl.gov>
Date: Tue, 3 Dec 2019 17:21:12 -0800
Subject: [PATCH 02/13] Fixing errors for different test cases.

---
 src/AdjListHyperGraph.chpl  |  4 ++--
 src/AggregationBuffer.chpl  |  2 +-
 src/Generation.chpl         |  3 ++-
 src/Graph.chpl              |  2 +-
 src/PropertyMaps.chpl       | 12 ++++++------
 src/UnrolledLinkedList.chpl |  6 +++---
 test/VectorTest.chpl        |  2 +-
 7 files changed, 16 insertions(+), 15 deletions(-)

diff --git a/src/AdjListHyperGraph.chpl b/src/AdjListHyperGraph.chpl
index baf68ef..cc095fc 100644
--- a/src/AdjListHyperGraph.chpl
+++ b/src/AdjListHyperGraph.chpl
@@ -35,7 +35,7 @@
     var graph = new AdjListHyperGraph(propertyMap, new Cyclic(startIdx=0));
  
 */
-module AdjListHyperGraph {
+prototype module AdjListHyperGraph {
   use IO;
   use CyclicDist;
   use LinkedLists;
@@ -2288,7 +2288,7 @@ module AdjListHyperGraph {
     // but is currently being processed remotely (maybe have a counter
     // determining how many tasks are still processing the buffer), so
     // that user knows when all operations have finished/termination detection.
-    inline proc emptyBuffer(buffer : unmanaged Buffer, loc : locale) {
+    inline proc emptyBuffer(buffer : unmanaged Buffer?, loc : locale) {
       on loc {
         var buf = buffer.getArray();
         buffer.done();
diff --git a/src/AggregationBuffer.chpl b/src/AggregationBuffer.chpl
index f299eb0..9d95dc0 100644
--- a/src/AggregationBuffer.chpl
+++ b/src/AggregationBuffer.chpl
@@ -349,7 +349,7 @@ prototype module AggregationBuffer {
       return sz;
     }
 
-    proc aggregate(msg : msgType, loc : locale) : unmanaged Buffer(msgType) {
+    proc aggregate(msg : msgType, loc : locale) : unmanaged Buffer(msgType)? {
       return aggregate(msg, loc.id);
     }
     
diff --git a/src/Generation.chpl b/src/Generation.chpl
index ec00598..9558e43 100644
--- a/src/Generation.chpl
+++ b/src/Generation.chpl
@@ -1,4 +1,4 @@
-module Generation {
+prototype module Generation {
 
   use IO;
   use AdjListHyperGraph;
@@ -7,6 +7,7 @@ module Generation {
   use CyclicDist;
   use BlockDist;
   use Math;
+  use WorkQueue;
   use Sort;
   use Search;
 
diff --git a/src/Graph.chpl b/src/Graph.chpl
index 6ad26f7..700eccc 100644
--- a/src/Graph.chpl
+++ b/src/Graph.chpl
@@ -4,7 +4,7 @@
  support simple 'addEdge(v1,v2)' and 'forall (v1, v2) in graph.getEdges()'; everything else
  should be forwarded to the underlying Hypergraph.
 */
-module Graph {
+prototype module Graph {
   use AdjListHyperGraph;
   use Utilities;
   use AggregationBuffer;
diff --git a/src/PropertyMaps.chpl b/src/PropertyMaps.chpl
index 3e6a8a9..03401a7 100644
--- a/src/PropertyMaps.chpl
+++ b/src/PropertyMaps.chpl
@@ -1,4 +1,4 @@
-module PropertyMaps {
+prototype module PropertyMaps {
   use AggregationBuffer;
   use HashedDist; // Hashed is not used, but the Mapper is
   use Utilities;
@@ -16,7 +16,7 @@ module PropertyMaps {
       // Type of mapper.
       type mapperType;
       pragma "no doc"
-      var map : unmanaged PropertyMapImpl(propertyType, mapperType);
+      var map : unmanaged PropertyMapImpl(propertyType, mapperType)?;
       pragma "no doc"
       var pid = -1;
       
@@ -51,7 +51,7 @@ module PropertyMaps {
         This initializer is used internally, as it is used to create an uninitialized version of this property map.
       */
       pragma "no doc"
-      proc init(type propertyType, type mapperType, pid : int, map : unmanaged PropertyMapImpl(propertyType, mapperType)) {
+      proc init(type propertyType, type mapperType, pid : int, map : unmanaged PropertyMapImpl(propertyType, mapperType)? ) {
         this.propertyType = propertyType;
         this.mapperType = mapperType;
         this.map = map;        
@@ -109,7 +109,7 @@ module PropertyMaps {
     var setAggregator = UninitializedAggregator((propertyType, int));
     // Aggregation used to batch up potentially remote 'fetches' of properties.
     pragma "no doc"
-    var getAggregator = UninitializedAggregator((propertyType, unmanaged PropertyHandle));
+    var getAggregator = UninitializedAggregator((propertyType, unmanaged PropertyHandle?));
     pragma "no doc"
     var terminationDetector : TerminationDetector;
 
@@ -121,7 +121,7 @@ module PropertyMaps {
       this.mapper = mapper;
       this.complete();
       this.setAggregator = new Aggregator((propertyType, int), 8 * 1024);
-      this.getAggregator = new Aggregator((propertyType, unmanaged PropertyHandle), 8 * 1024);
+      this.getAggregator = new Aggregator((propertyType, unmanaged PropertyHandle?), 8 * 1024);
       this.terminationDetector = new TerminationDetector();
       this.pid = _newPrivatizedClass(this:unmanaged);
     }
@@ -131,7 +131,7 @@ module PropertyMaps {
       this.mapper = other.mapper;
       this.complete();
       this.setAggregator = new Aggregator((propertyType, int), 8 * 1024);
-      this.getAggregator = new Aggregator((propertyType, unmanaged PropertyHandle), 8 * 1024);
+      this.getAggregator = new Aggregator((propertyType, unmanaged PropertyHandle?), 8 * 1024);
       this.terminationDetector = new TerminationDetector();
 
       this.pid = _newPrivatizedClass(this:unmanaged);
diff --git a/src/UnrolledLinkedList.chpl b/src/UnrolledLinkedList.chpl
index 0e7201a..aa4f04a 100644
--- a/src/UnrolledLinkedList.chpl
+++ b/src/UnrolledLinkedList.chpl
@@ -4,14 +4,14 @@ class UnrollBlock {
 	var end : int;
 	var start : int;
 	var data : c_array(eltType, cap);
-	var next : unmanaged UnrollBlock(eltType, cap);
+	var next : unmanaged UnrollBlock(eltType, cap)?;
 }
 
 record UnrolledLinkedList {
 	type eltType;
 	param unrollBlockSize : int;
-    var sz : int;
-	var head : unmanaged UnrollBlock(eltType, unrollBlockSize);
+    	var sz : int;
+	var head : unmanaged UnrollBlock(eltType, unrollBlockSize)?;
 	proc init(type eltType, param unrollBlockSize : int) {
 		this.eltType = eltType;
 		this.unrollBlockSize = unrollBlockSize;
diff --git a/test/VectorTest.chpl b/test/VectorTest.chpl
index d240b16..00fe9fe 100644
--- a/test/VectorTest.chpl
+++ b/test/VectorTest.chpl
@@ -1,7 +1,7 @@
 use CHGL;
 use CyclicDist;
 
-var v1 : owned Vector(int);
+var v1 : owned Vector(int)?;
 v1 = new owned Vector(int, 10);
 v1.append(1);
 writeln(v1[0]);

From 0eb9d281a12fc0447e7ed8bf708a4785fb0bc1df Mon Sep 17 00:00:00 2001
From: Jesun Sahariar Firoz <jesun.firoz@pnnl.gov>
Date: Tue, 31 Dec 2019 11:12:01 -0800
Subject: [PATCH 03/13] Modification of the initializers so as to ensure using
 cyclic distributions for vertices and edges.

---
 src/AdjListHyperGraph.chpl | 35 +++++++++++++++++++++++------------
 1 file changed, 23 insertions(+), 12 deletions(-)

diff --git a/src/AdjListHyperGraph.chpl b/src/AdjListHyperGraph.chpl
index cc095fc..0335d2e 100644
--- a/src/AdjListHyperGraph.chpl
+++ b/src/AdjListHyperGraph.chpl
@@ -177,13 +177,14 @@ prototype module AdjListHyperGraph {
       return chpl_getPrivatizedCopy(instance.type, pid);
     }
 
+
     /*
       Create a new hypergraph with the desired number of vertices and edges. 
       Uses the 'DefaultDist', which is normally the shared-memory 'DefaultRectangularDist'.
     */
-    proc init(numVertices : integral, numEdges : integral) {
+    proc init(numVertices : integral, numEdges : integral, param distributeVertices : bool = true, param distributeEdges : bool = true) {
       var dist = new unmanaged DefaultDist();
-      init(numVertices, numEdges, dist, dist);
+      init(numVertices, numEdges, dist, dist, distributeVertices, distributeEdges);
     }
 
     /*
@@ -259,12 +260,14 @@ prototype module AdjListHyperGraph {
       // Number of edges
       numEdges : integral,
       // Distribution of vertices
-      verticesMappings, 
+      verticesMappings,
       // Distribution of edges
-      edgesMappings
+      edgesMappings,
+      param distributeVertices : bool = true,
+      param distributeEdges : bool = true
     ) {
       instance = new unmanaged AdjListHyperGraphImpl(
-        numVertices, numEdges, verticesMappings, edgesMappings
+        numVertices, numEdges, verticesMappings, edgesMappings, distributeVertices, distributeEdges
       );
       pid = instance.pid;
     }
@@ -906,7 +909,8 @@ prototype module AdjListHyperGraph {
     /*
       The main initializer; all other initializers should call this after filling out the appropriate parameters.
     */
-    proc init(numVertices : int, vPropMap : PropertyMap(?vPropType), vertexMappings, numEdges : int,  ePropMap : PropertyMap(?ePropType), edgeMappings) {
+    proc init(numVertices : int, vPropMap : PropertyMap(?vPropType), vertexMappings, numEdges : int,  ePropMap : PropertyMap(?ePropType), edgeMappings,
+      param distributeVertices: bool = true, param distributeEdges: bool = true) {
       // Ensure that arguments are non-negative
       if numVertices < 0 { 
         halt("numVertices must be between 0..", max(int(64)), " but got ", numVertices);
@@ -917,9 +921,16 @@ prototype module AdjListHyperGraph {
       
       // Initialize vertices and edges domain; once `this.complete()` is invoked, the
       // array itself will also be initialized.
-      this._verticesDomain = {0..#numVertices} dmapped new dmap(vertexMappings);
-      this._edgesDomain = {0..#numEdges} dmapped new dmap(edgeMappings);
-      
+       if (distributeVertices) {
+      	 this._verticesDomain = newCyclicDom({0..#numVertices});
+      } else {
+      	this._verticesDomain = {0..#numVertices};
+      }
+      if (distributeEdges) {
+      	 this._edgesDomain = newCyclicDom({0..#numEdges});
+      } else {
+         this._edgesDomain = {0..#numEdges};
+      }
       this._vPropType = vPropType;
       this._ePropType = ePropType;
       this._destBuffer = new Aggregator((vIndexType, eIndexType, InclusionType), 64 * 1024);
@@ -947,11 +958,11 @@ prototype module AdjListHyperGraph {
 
 
     pragma "no doc"
-    proc init(numVertices = 0, numEdges = 0, vertexMappings, edgeMappings) {
+    proc init(numVertices = 0, numEdges = 0, vertexMappings, edgeMappings, param distributeVertices: bool = true, param distributeEdges: bool = true) {
       const pmap = UninitializedPropertyMap(bool);
-      init(numVertices, pmap, vertexMappings, numEdges, pmap, edgeMappings);
+      init(numVertices, pmap, vertexMappings, numEdges, pmap, edgeMappings, distributeVertices, distributeEdges);
     }
-
+    
     pragma "no doc"
     proc init(vPropertyMap : PropertyMap(?vPropType), vertexMappings, numEdges = 0, edgeMappings) {
       const pmap = UninitializedPropertyMap(bool);

From ff6d2183fd6da155d09161894120c49926f1c46c Mon Sep 17 00:00:00 2001
From: Jesun Sahariar Firoz <jesun.firoz@pnnl.gov>
Date: Sat, 1 Feb 2020 12:46:00 -0800
Subject: [PATCH 04/13] Updating travis to point to Chapel 1.20.0

---
 .travis.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.travis.yml b/.travis.yml
index 18a0487..d86e690 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -8,7 +8,7 @@ services:
 jobs:
   include:
     - script: |
-        docker run -it -v ${TRAVIS_BUILD_DIR}:/repo.git -w /repo.git louisjenkinscs/chapel-experimental:chapel-1.19 /bin/bash -c '
+        docker run -it -v ${TRAVIS_BUILD_DIR}:/repo.git -w /repo.git chapel/chapel:1.20.0 /bin/bash -c '
         pushd $CHPL_HOME
         . util/setchplenv.sh
         popd

From 3466463fbe7592a379c4b9077afdc1837f3a80cb Mon Sep 17 00:00:00 2001
From: Jesun Sahariar Firoz <jesun.firoz@pnnl.gov>
Date: Sat, 1 Feb 2020 13:29:33 -0800
Subject: [PATCH 05/13] Updating travis to point to multi-locale Chapel.

---
 .travis.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.travis.yml b/.travis.yml
index d86e690..2736d9a 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -8,7 +8,7 @@ services:
 jobs:
   include:
     - script: |
-        docker run -it -v ${TRAVIS_BUILD_DIR}:/repo.git -w /repo.git chapel/chapel:1.20.0 /bin/bash -c '
+        docker run -it -v ${TRAVIS_BUILD_DIR}:/repo.git -w /repo.git chapel/chapel-gasnet:1.20.0 /bin/bash -c '
         pushd $CHPL_HOME
         . util/setchplenv.sh
         popd

From 855c081b64d78c4cf36de0ed23901ea7c8620777 Mon Sep 17 00:00:00 2001
From: Jesun Sahariar Firoz <jesun.firoz@pnnl.gov>
Date: Sun, 2 Feb 2020 19:06:24 -0800
Subject: [PATCH 06/13] For the butterfly test, ake a graph argument instead

As per Louis, verticesDomain is accessing the field directly via `this`.verticesDomain where `this` is remote.
`this` is the class instance of the original locale that invoked the method
And when the algo jump to a remote locale, accessing this explicitly or implicitly results in communication.
CylicDist  is optimized to disable locality checks when accessing the DSI methods directly, like we are doing
We use graph.verticesDomain instead it will resolve it through privatization (I.E through record wrapper forwarding).
---
 src/Butterfly.chpl                            | 50 +++++++++----------
 .../Butterfly_and_caterpillar_test.chpl       | 16 +++---
 2 files changed, 33 insertions(+), 33 deletions(-)

diff --git a/src/Butterfly.chpl b/src/Butterfly.chpl
index bd52061..3ec441b 100755
--- a/src/Butterfly.chpl
+++ b/src/Butterfly.chpl
@@ -44,16 +44,16 @@ module Butterfly {
     :rtype: array of int(64)
   
     */
-  proc AdjListHyperGraphImpl.getVertexButterflies() {
-    var butterfliesDom = verticesDomain;
+  proc getVertexButterflies(graph: AdjListHyperGraph) {
+    var butterfliesDom = graph.verticesDomain;
     var butterflies : [butterfliesDom] atomic int(64);
 
     // Look for the pattern (v -> u -> w)
-    forall v in getVertices() {
+    forall v in graph.getVertices() {
       // A two-hop neighbor of v would be w iff (v -> u -> w)
-      var twoHopNeighbors : [verticesDomain] atomic int(64);
-      forall u in incidence(v) {
-        forall w in incidence(u) {
+      var twoHopNeighbors : [graph.verticesDomain] atomic int(64);
+      forall u in graph.incidence(v) {
+        forall w in graph.incidence(u) {
           if w.id != v.id {
             twoHopNeighbors[w.id].fetchAdd(1);
           }
@@ -319,14 +319,14 @@ module Butterfly {
   :returns: An array of range 0..N where N is the highest edge ID int the AdjListHyperGraph object
   :rtype: array of int(64)
   */
-  proc AdjListHyperGraphImpl.getEdgeButterflies() {
-    var butterflies : [edgesDomain] atomic int(64);
+  proc getEdgeButterflies(graph: AdjListHyperGraph) {
+    var butterflies : [graph.edgesDomain] atomic int(64);
     
     // Look for pattern (e -> u -> w)
-    forall e in getEdges() {
-      var twoHopNeighbors : [edgesDomain] atomic int(64);
-      forall u in incidence(e) {
-        forall w in incidence(u) {    
+    forall e in graph.getEdges() {
+      var twoHopNeighbors : [graph.edgesDomain] atomic int(64);
+      forall u in graph.incidence(e) {
+        forall w in graph.incidence(u) {    
           if w.id != e.id {
             twoHopNeighbors[w.id].fetchAdd(1);
           }
@@ -351,14 +351,14 @@ module Butterfly {
   :returns: An array of range 0..n where n is the highest ID of all vertices in the AdjListHyperGraph object
   :rtype: array of int(64)
   */
-  proc AdjListHyperGraphImpl.getVertexCaterpillars() {
-    var caterpillarsDomain = verticesDomain;
+  proc getVertexCaterpillars(graph: AdjListHyperGraph) {
+    var caterpillarsDomain = graph.verticesDomain;
     var caterpillars : [caterpillarsDomain] int(64);
-    forall v in getVertices() {
-      var twoHopNeighbors : [verticesDomain] atomic int(64); //this is C[w] in the paper, which is the number of distinct distance-two paths that connect v and w
+    forall v in graph.getVertices() {
+      var twoHopNeighbors : [graph.verticesDomain] atomic int(64); //this is C[w] in the paper, which is the number of distinct distance-two paths that connect v and w
       //C[w] is equivalent to the number of edges that v and w are both connected to
-      forall u in incidence(v) {
-        forall w in incidence(u) {
+      forall u in graph.incidence(v) {
+        forall w in graph.incidence(u) {
           if w.id != v.id {
             twoHopNeighbors[w.id].fetchAdd(1);
             twoHopNeighbors[v.id].fetchAdd(1); //if this is added then all caterpillars including this vertex will be included in the count
@@ -380,18 +380,18 @@ module Butterfly {
     return  caterpillars;
   }
 
-  /* Calcuates the number of 3-cycles that include each edge
+  /* Calculates the number of 3-cycles that include each edge
   
   :returns: An array of range 0..n where n is the highest ID of all edges in the AdjListHyperGraph object
   :rtype: array of int(64)
   */
-  proc AdjListHyperGraphImpl.getEdgeCaterpillars() {
-    var caterpillars : [edgesDomain] int(64);
+  proc getEdgeCaterpillars(graph : AdjListHyperGraph) {
+    var caterpillars : [graph.edgesDomain] int(64);
     
-    forall e in getEdges() {
-      var twoHopNeighbors : [edgesDomain] atomic int(64); 
-      forall u in incidence(e) {
-        forall w in incidence(u) {
+    forall e in graph.getEdges() {
+      var twoHopNeighbors : [graph.edgesDomain] atomic int(64); 
+      forall u in graph.incidence(e) {
+        forall w in graph.incidence(u) {
           if w.id != e.id {
             twoHopNeighbors[w.id].fetchAdd(1);
             twoHopNeighbors[e.id].fetchAdd(1); //if this is added then all caterpillars including this edge will be included in the count
diff --git a/test/Generation/Butterfly_and_caterpillar_test.chpl b/test/Generation/Butterfly_and_caterpillar_test.chpl
index 7adc079..fbcf7e6 100644
--- a/test/Generation/Butterfly_and_caterpillar_test.chpl
+++ b/test/Generation/Butterfly_and_caterpillar_test.chpl
@@ -15,16 +15,16 @@ graph.addInclusion(3,2);
 graph.addInclusion(3,3);
 graph.addInclusion(4,3);
 var val = 0;
-if graph.getVertexButterflies().equals([2,2,3,1,0]){
+if getVertexButterflies(graph).equals([2,2,3,1,0]){
   val += 1;
-} else writeln("Bad Vertex Butterflies: ", graph.getVertexButterflies(), " expected: ", [2,2,3,1,0]);
-if graph.getVertexCaterpillars().equals([8,8,14,6,4]){
+} else writeln("Bad Vertex Butterflies: ", getVertexButterflies(graph), " expected: ", [2,2,3,1,0]);
+if getVertexCaterpillars(graph).equals([8,8,14,6,4]){
   val += 1;
-} else writeln("Bad Vertex Caterpillars: ", graph.getVertexCaterpillars(), " expected: ", [8,8,14,6,4]);
-if graph.getEdgeButterflies().equals([3,3,1,1]){
+} else writeln("Bad Vertex Caterpillars: ", getVertexCaterpillars(graph), " expected: ", [8,8,14,6,4]);
+if getEdgeButterflies(graph).equals([3,3,1,1]){
   val += 1;
-} else writeln("Bad Edge Butterflies: ", graph.getEdgeButterflies(), " expected: ", [3,3,1,1]);
-if graph.getEdgeCaterpillars().equals([10,10,8,8]){
+} else writeln("Bad Edge Butterflies: ", getEdgeButterflies(graph), " expected: ", [3,3,1,1]);
+if getEdgeCaterpillars(graph).equals([10,10,8,8]){
   val += 1;
-} else writeln("Bad Edge Caterpillars: ", graph.getEdgeCaterpillars(), " expected: ", [10,10,8,8]);
+} else writeln("Bad Edge Caterpillars: ", getEdgeCaterpillars(graph), " expected: ", [10,10,8,8]);
 writeln(val == 4);

From 7f25f7f57dfe809376ebbbf1ae7ab518163405f4 Mon Sep 17 00:00:00 2001
From: Jesun Sahariar Firoz <jesun.firoz@pnnl.gov>
Date: Sun, 2 Feb 2020 19:59:30 -0800
Subject: [PATCH 07/13] We needed to grab privatized instance of the hypergraph
 and use that for accessing getEdges in a forall iterator (since they are
 parallel and distributed) in GraphTest.

---
 src/Graph.chpl | 11 ++++++-----
 1 file changed, 6 insertions(+), 5 deletions(-)

diff --git a/src/Graph.chpl b/src/Graph.chpl
index 700eccc..fd0145b 100644
--- a/src/Graph.chpl
+++ b/src/Graph.chpl
@@ -254,18 +254,19 @@ prototype module Graph {
           for u in vec do yield (hg.toVertex(v),u);
         }
       } else {
-        forall e in hg.getEdges() {
-          var sz = hg.getEdge(e).size.read();
+      	 forall e in hg.getEdges() {
+          var __this = getPrivatizedInstance();
+          var sz = __this.hg.getEdge(e).size.read();
           if sz > 2 {
-            halt("Edge ", e, " is has more than two vertices: ", hg.getEdge(e).incident);
+            halt("Edge ", e, " is has more than two vertices: ", __this.hg.getEdge(e).incident);
           }
           if sz == 0 {
             continue;
           }
 
-          yield (hg.toVertex(hg.getEdge(e).incident[0]), hg.toVertex(hg.getEdge(e).incident[1]));
+          yield (__this.hg.toVertex(__this.hg.getEdge(e).incident[0]), __this.hg.toVertex(__this.hg.getEdge(e).incident[1]));
         }
-      }    
+      }
     }
 
     // Return neighbors of a vertex 'v'

From 8985c529f22a18edce83ca5a56bf72b9d4e164ba Mon Sep 17 00:00:00 2001
From: Jesun Sahariar Firoz <jesun.firoz@pnnl.gov>
Date: Wed, 5 Feb 2020 16:36:51 -0800
Subject: [PATCH 08/13] Fixed errors in ActiveDNS code due to upgrade.

---
 example/activeDNS.chpl     |  2 +-
 src/AdjListHyperGraph.chpl | 40 +++++++++++++++++++++-----------------
 src/PropertyMaps.chpl      |  4 ++--
 3 files changed, 25 insertions(+), 21 deletions(-)

diff --git a/example/activeDNS.chpl b/example/activeDNS.chpl
index 4af0583..0beb682 100644
--- a/example/activeDNS.chpl
+++ b/example/activeDNS.chpl
@@ -378,7 +378,7 @@ wq.flush();
 // Aggregate fetches to properties into another work queue; when we flush
 // each of the property maps, their individual PropertyHandle will be finished.
 // Also send the 'String' so that it can be reclaimed.
-var handleWQ = new WorkQueue((unmanaged PropertyHandle, unmanaged PropertyHandle), 64 * 1024);
+var handleWQ = new WorkQueue((unmanaged PropertyHandle?, unmanaged PropertyHandle?), 64 * 1024);
 var handleTD = new TerminationDetector();
 forall fileName in doWorkLoop(wq, td) {
   for line in getLines(fileName) {
diff --git a/src/AdjListHyperGraph.chpl b/src/AdjListHyperGraph.chpl
index 0335d2e..2a33696 100644
--- a/src/AdjListHyperGraph.chpl
+++ b/src/AdjListHyperGraph.chpl
@@ -1392,6 +1392,7 @@ prototype module AdjListHyperGraph {
       [dup in duplicateVertices] dup.write(-1);
       var newVerticesDomain = __verticesDomain;
       var vertexMappings : [__verticesDomain] int = -1;
+      var dummyNode = new unmanaged NodeData(eDescType, _vPropType);
 
       //writeln("Collapsing Vertices...");
       // Pass 1: Locate duplicates by performing an s-walk where s is the size of current vertex
@@ -1438,7 +1439,7 @@ prototype module AdjListHyperGraph {
           numUnique += 1;
           for follower in eqclass.getCandidates(leader) {
             delete _vertices[follower];
-            _vertices[follower] = nil;
+            _vertices[follower] = dummyNode;
             duplicateVertices[follower].write(leader);
           }
         }
@@ -1461,7 +1462,7 @@ prototype module AdjListHyperGraph {
         if Debug.ALHG_DEBUG {
           forall v in _verticesDomain {
             var vv = v;
-            while _vertices[vv] == nil {
+            while _vertices[vv] == dummyNode {
               vv = duplicateVertices[vv].read();
               assert(vv != -1, "A vertex is nil without a duplicate mapping...");
             }
@@ -1478,7 +1479,7 @@ prototype module AdjListHyperGraph {
               writeln("Broken Dual!");
               var vvv = v;
               write("Link: ", toVertex(v));
-              while _vertices[vvv] == nil {
+              while _vertices[vvv] == dummyNode {
                 vvv = duplicateVertices[vvv].read();
                 write(" -> ", toVertex(vvv));
               }
@@ -1513,7 +1514,7 @@ prototype module AdjListHyperGraph {
         // array; hence, try to first claim indices that are local in _both_ new and old arrays.
         var idx : atomic int;
         forall v in oldVertices.domain {
-          if oldVertices[v] != nil {
+          if oldVertices[v] != dummyNode {
             // TODO
             // When no RDMA atomics are supported, this will have _massive_ communication costs
             // Possibly want to perform explicit `coforall` and create explicit tasks on each
@@ -1531,7 +1532,7 @@ prototype module AdjListHyperGraph {
               delete oldVertices[v];
             }
 
-            oldVertices[v] = nil;
+            oldVertices[v] = dummyNode;
             vertexMappings[v] = ix;
           }
         }
@@ -1602,7 +1603,7 @@ prototype module AdjListHyperGraph {
       [dup in duplicateEdges] dup.write(-1);
       var newEdgesDomain = __edgesDomain;
       var edgeMappings : [__edgesDomain] int = -1;
-
+      var dummyEdge = new unmanaged NodeData(vDescType, _ePropType);
       
 
       //writeln("Collapsing Edges...");
@@ -1640,7 +1641,7 @@ prototype module AdjListHyperGraph {
           numUnique += 1;
           for follower in eqclass.getCandidates(leader) {
             delete _edges[follower];
-            _edges[follower] = nil;
+            _edges[follower] = dummyEdge;
             duplicateEdges[follower].write(leader);
           }
         }
@@ -1662,7 +1663,7 @@ prototype module AdjListHyperGraph {
         if Debug.ALHG_DEBUG {
           forall e in _edgesDomain {
             var ee = e;
-            while _edges[ee] == nil {
+            while _edges[ee] == dummyEdge {
               ee = duplicateEdges[ee].read();
               assert(ee != -1, "An edge is nil without a duplicate mapping...");
             }
@@ -1680,7 +1681,7 @@ prototype module AdjListHyperGraph {
               writeln("Broken Dual!");
               var eee = e;
               write("Link: ", toEdge(e));
-              while _edges[eee] == nil {
+              while _edges[eee] == dummyEdge {
                 eee = duplicateEdges[eee].read();
                 write(" -> ", toEdge(eee));
               }
@@ -1707,7 +1708,7 @@ prototype module AdjListHyperGraph {
       {
         var idx : atomic int;
         forall e in oldEdges.domain {
-          if oldEdges[e] != nil {
+          if oldEdges[e] != dummyEdge {
             var ix = idx.fetchAdd(1);
             
             if oldEdges[e].locale == _edges[ix].locale {
@@ -1717,7 +1718,7 @@ prototype module AdjListHyperGraph {
               delete oldEdges[e];
             }
             
-            oldEdges[e] = nil;
+            oldEdges[e] = dummyEdge;
             edgeMappings[e] = ix;
           }
         }
@@ -1786,6 +1787,7 @@ prototype module AdjListHyperGraph {
       [toplex in toplexEdges] toplex.write(-1);
       var newEdgesDomain = __edgesDomain;
       var edgeMappings : [__edgesDomain] int = -1;
+      var dummyEdge = new unmanaged NodeData(vDescType, _ePropType);
 
       writeln("Collapsing Subset...");
       {
@@ -1830,7 +1832,7 @@ prototype module AdjListHyperGraph {
         forall e in _edgesDomain with (+ reduce numToplex) {
           if toplexEdges[e].read() != -1 {
             delete _edges[e];
-            _edges[e] = nil;
+            _edges[e] = dummyEdge;
           } else {
             numToplex += 1;
           }
@@ -1856,7 +1858,7 @@ prototype module AdjListHyperGraph {
       {
         var idx : atomic int;
         forall e in oldEdges.domain {
-          if oldEdges[e] != nil {
+          if oldEdges[e] != dummyEdge {
             var ix = idx.fetchAdd(1);
             
             if oldEdges[e].locale == _edges[ix].locale {
@@ -1866,7 +1868,7 @@ prototype module AdjListHyperGraph {
               delete oldEdges[e];
             }
             
-            oldEdges[e] = nil;
+            oldEdges[e] = dummyEdge;
             edgeMappings[e] = ix;
           }
         }
@@ -2023,6 +2025,8 @@ prototype module AdjListHyperGraph {
       hyperedges. Returns the number of isolated components.
     */
     proc removeIsolatedComponents() : int {
+      var dummyNode = new unmanaged NodeData(eDescType, _vPropType);
+      var dummyEdge = new unmanaged NodeData(vDescType, _ePropType);
       // Enforce on Locale 0 (presumed master locale...)
       if here != Locales[0] {
         // Cannot jump and return as return type is inferred by compiler.
@@ -2046,11 +2050,11 @@ prototype module AdjListHyperGraph {
             if nn == 1 {
               if _this._ePropMap.isInitialized then _this._ePropMap.setProperty(_this.getEdge(e).property, -1);
               delete _this.getEdge(e);
-              _this.getEdge(e) = nil;
+              _this.getEdge(e) = dummyEdge;
               
               if _this._vPropMap.isInitialized then _this._vPropMap.setProperty(_this.getVertex(v).property, -1);
               delete _this.getVertex(v);
-              _this.getVertex(v) = nil;
+              _this.getVertex(v) = dummyNode;
               
               numIsolatedComponents += 1;
             }
@@ -2095,7 +2099,7 @@ prototype module AdjListHyperGraph {
               delete oldVertices[v];
             }
 
-            oldVertices[v] = nil;
+            oldVertices[v] = dummyNode;
             vertexMappings[v] = ix;
           }
         }
@@ -2116,7 +2120,7 @@ prototype module AdjListHyperGraph {
               delete oldEdges[e];
             }
             
-            oldEdges[e] = nil;
+            oldEdges[e] = dummyEdge;
             edgeMappings[e] = ix;
           }
         }
diff --git a/src/PropertyMaps.chpl b/src/PropertyMaps.chpl
index 03401a7..ae5fd2f 100644
--- a/src/PropertyMaps.chpl
+++ b/src/PropertyMaps.chpl
@@ -287,7 +287,7 @@ prototype module PropertyMaps {
       }
     }
 
-    proc _flushGetAggregatorBuffer(buf : Buffer, loc : locale, param acquireLock = true) {
+    proc _flushGetAggregatorBuffer(buf : Buffer?, loc : locale, param acquireLock = true) {
       // Obtain separate array of properties and handles; we need to isolate properties
       // so we can do a bulk-transfer on the other locales.
       var arr = buf.getArray();
@@ -295,7 +295,7 @@ prototype module PropertyMaps {
       const arrSz = arr.size;
       var properties : [0..#arrSz] propertyType;
       var keys : [0..#arrSz] int;
-      var handles : [0..#arrSz] unmanaged PropertyHandle;
+      var handles : [0..#arrSz] unmanaged PropertyHandle?;
       forall ((prop, hndle), _prop, _hndle) in zip(arr, properties, handles) {
         _prop = prop;
         _hndle = hndle;

From b560f7104f8fff9309d388809df2ad01e350369e Mon Sep 17 00:00:00 2001
From: Jesun Sahariar Firoz <jesun.firoz@pnnl.gov>
Date: Wed, 5 Feb 2020 19:27:15 -0800
Subject: [PATCH 09/13] Updating Jenkins script

---
 jenkins-build.sh | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

diff --git a/jenkins-build.sh b/jenkins-build.sh
index 484f056..fc0d695 100644
--- a/jenkins-build.sh
+++ b/jenkins-build.sh
@@ -6,12 +6,11 @@
 WORKSPACE=/lustre/jenkins/chgl-workspace
 
 # Load required modules
-export MODULEPATH=/home/zale916/software/modules:$MODULEPATH
+export MODULEPATH=/home/firo017/softwares/modules:$MODULEPATH
 module load gcc/8.2.0
 module load openmpi/3.1.3
-module load hdf5/1.10.5
-module load zmq/4.3.1
-module load chapel/1.19.0
+module load clang/8.0.1 
+module load chapel/1.20.0
 
 # Execute peformance tests
 export CHPL_TEST_LAUNCHCMD="$CHPL_HOME/util/test/chpl_launchcmd.py --CHPL_LAUNCHCMD_DEBUG"

From e404d33b869037e3350ac05c00806df13341faee Mon Sep 17 00:00:00 2001
From: Jesun Sahariar Firoz <jesun.firoz@pnnl.gov>
Date: Thu, 6 Feb 2020 13:37:19 -0800
Subject: [PATCH 10/13] Updating rest of the tests for performance regression.

---
 src/AdjListHyperGraph.chpl       | 10 +++++-----
 src/Metrics.chpl                 |  2 +-
 test_performance/BTER.chpl       |  2 +-
 test_performance/ChungLu.chpl    |  2 +-
 test_performance/Privatized.chpl |  1 +
 test_performance/activeDNS.chpl  |  2 +-
 6 files changed, 10 insertions(+), 9 deletions(-)

diff --git a/src/AdjListHyperGraph.chpl b/src/AdjListHyperGraph.chpl
index 2a33696..9446bd1 100644
--- a/src/AdjListHyperGraph.chpl
+++ b/src/AdjListHyperGraph.chpl
@@ -268,7 +268,7 @@ prototype module AdjListHyperGraph {
     ) {
       instance = new unmanaged AdjListHyperGraphImpl(
         numVertices, numEdges, verticesMappings, edgesMappings, distributeVertices, distributeEdges
-      );
+      )?;
       pid = instance.pid;
     }
 
@@ -284,7 +284,7 @@ prototype module AdjListHyperGraph {
     proc init(vPropMap : PropertyMap(?vPropType), vertexMappings, numEdges, edgeMappings) {
       instance = new unmanaged AdjListHyperGraphImpl(
         vPropMap, vertexMappings, numEdges, edgeMappings
-      );
+      )?;
       pid = instance.pid;
     }
 
@@ -301,7 +301,7 @@ prototype module AdjListHyperGraph {
     proc init(numVertices, vertexMappings, ePropMap : PropertyMap(?ePropType), edgeMappings) {
       instance = new unmanaged AdjListHyperGraphImpl(
         numVertices, vertexMappings, ePropMap, edgeMappings
-      );
+      )?;
       pid = instance.pid;
     }
 
@@ -318,7 +318,7 @@ prototype module AdjListHyperGraph {
     proc init(vPropMap : PropertyMap(?vPropType), vertexMappings, ePropMap : PropertyMap(?ePropType), edgeMappings) {
       instance = new unmanaged AdjListHyperGraphImpl(
         vPropMap, vertexMappings, ePropMap, edgeMappings
-      );
+      )?;
       pid = instance.pid;
     }
 
@@ -336,7 +336,7 @@ prototype module AdjListHyperGraph {
     // Code that causes it: init(other.numVertices, other.numEdges, other.verticesDist)
     pragma "no doc"
     proc clone(other : this.type) {
-      instance = new unmanaged AdjListHyperGraphImpl(other._value);
+      instance = new unmanaged AdjListHyperGraphImpl(other._value)?;
       pid = instance.pid;
     }
     
diff --git a/src/Metrics.chpl b/src/Metrics.chpl
index 84b1ac7..0b27d9e 100644
--- a/src/Metrics.chpl
+++ b/src/Metrics.chpl
@@ -1,7 +1,7 @@
 /*
   Compilation of common metrics to be performed on hypergraphs or graphs.
 */
-module Metrics {
+prototype module Metrics {
   use CHGL;
   use Vectors;
   use Utilities;
diff --git a/test_performance/BTER.chpl b/test_performance/BTER.chpl
index 5f79d89..c48e60f 100644
--- a/test_performance/BTER.chpl
+++ b/test_performance/BTER.chpl
@@ -1,7 +1,7 @@
 use AdjListHyperGraph;
 use CommDiagnostics;
 use VisualDebug;
-use Generation;
+use CHGL;
 use Time;
 
 /* Performance Test for BTER algorithm */
diff --git a/test_performance/ChungLu.chpl b/test_performance/ChungLu.chpl
index 2503cb9..4719b6e 100644
--- a/test_performance/ChungLu.chpl
+++ b/test_performance/ChungLu.chpl
@@ -1,7 +1,7 @@
 use AdjListHyperGraph;
 use CommDiagnostics;
 use VisualDebug;
-use Generation;
+use CHGL;
 use Time;
 
 /* Performance Test for ChungLu algorithm */
diff --git a/test_performance/Privatized.chpl b/test_performance/Privatized.chpl
index 623459b..2324e82 100644
--- a/test_performance/Privatized.chpl
+++ b/test_performance/Privatized.chpl
@@ -1,5 +1,6 @@
 use CHGL;
 use ReplicatedDist;
+use ReplicatedVar;
 use Time;
 
 config const N = 1024 * 1024;
diff --git a/test_performance/activeDNS.chpl b/test_performance/activeDNS.chpl
index 907b49c..39d05f9 100644
--- a/test_performance/activeDNS.chpl
+++ b/test_performance/activeDNS.chpl
@@ -81,7 +81,7 @@ t.clear();
 t.start();
 graph.startAggregation();
 // Aggregate fetching of keys
-var handles = new owned Vector((unmanaged PropertyHandle, unmanaged PropertyHandle));
+var handles = new owned Vector((unmanaged PropertyHandle?, unmanaged PropertyHandle?));
 for line in getLines(dataset) {
   var attrs = line.split(",");
   var qname = attrs[1];

From 82254057fdde1d604fee922834aeba69fab84adc Mon Sep 17 00:00:00 2001
From: Jesun Sahariar Firoz <jesun.firoz@pnnl.gov>
Date: Thu, 6 Feb 2020 13:56:20 -0800
Subject: [PATCH 11/13] Updating the jenkins script.

---
 jenkins-build.sh | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/jenkins-build.sh b/jenkins-build.sh
index fc0d695..1a5d8f5 100644
--- a/jenkins-build.sh
+++ b/jenkins-build.sh
@@ -7,6 +7,9 @@ WORKSPACE=/lustre/jenkins/chgl-workspace
 
 # Load required modules
 export MODULEPATH=/home/firo017/softwares/modules:$MODULEPATH
+export CHPL_COMM=gasnet
+export CHPL_COMM_SUBSTRATE=ibv
+source util/setchplenv.bash
 module load gcc/8.2.0
 module load openmpi/3.1.3
 module load clang/8.0.1 

From 3dc03114a908e6c0a24bcfcc75975b1ff421025a Mon Sep 17 00:00:00 2001
From: Jesun Sahariar Firoz <jesun.firoz@pnnl.gov>
Date: Thu, 6 Feb 2020 13:59:53 -0800
Subject: [PATCH 12/13] Updating the jenkins script.

---
 jenkins-build.sh | 1 +
 1 file changed, 1 insertion(+)

diff --git a/jenkins-build.sh b/jenkins-build.sh
index 1a5d8f5..7ce1f54 100644
--- a/jenkins-build.sh
+++ b/jenkins-build.sh
@@ -9,6 +9,7 @@ WORKSPACE=/lustre/jenkins/chgl-workspace
 export MODULEPATH=/home/firo017/softwares/modules:$MODULEPATH
 export CHPL_COMM=gasnet
 export CHPL_COMM_SUBSTRATE=ibv
+export CHPL_LAUNCHER=slurm-gasnetrun_ibv
 source util/setchplenv.bash
 module load gcc/8.2.0
 module load openmpi/3.1.3

From 713f96a2583aec8a5552b40730e53f5ca7cb42f1 Mon Sep 17 00:00:00 2001
From: Jesun Sahariar Firoz <jesun.firoz@pnnl.gov>
Date: Thu, 6 Feb 2020 14:23:57 -0800
Subject: [PATCH 13/13] Making Butterfly module a prototype module.

---
 src/Butterfly.chpl | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/Butterfly.chpl b/src/Butterfly.chpl
index 3ec441b..350eb23 100755
--- a/src/Butterfly.chpl
+++ b/src/Butterfly.chpl
@@ -1,4 +1,4 @@
-module Butterfly {
+prototype module Butterfly {
   use AdjListHyperGraph;
   
   /* Calculates the maximum number of inclusions that the AdjListHyperGraph object can contain without duplicates