From 8858896cbf61b1c81bd03a157de1bafc9fd1a7fa Mon Sep 17 00:00:00 2001
From: pensun <Peng.Sun@amd.com>
Date: Tue, 8 Nov 2016 00:35:35 -0600
Subject: [PATCH] Add options to alloc HSA Coherent system memory

---
 include/hc.hpp           | 24 ++++++++++++++++++++++++
 include/hc_am.hpp        |  1 +
 include/kalmar_runtime.h |  2 ++
 lib/hsa/hc_am.cpp        |  4 +++-
 lib/hsa/mcwamp_hsa.cpp   | 16 +++++++++++++++-
 5 files changed, 45 insertions(+), 2 deletions(-)

diff --git a/include/hc.hpp b/include/hc.hpp
index 5d97a75c7b6..f1267930d72 100644
--- a/include/hc.hpp
+++ b/include/hc.hpp
@@ -402,6 +402,18 @@ class accelerator_view {
         return pQueue->getHSAAMHostRegion();
     }
 
+    /**
+     * Returns an opaque handle which points to the AM system region on the HSA agent.
+     * This region can be used to allocate finegrained system memory which is accessible from the 
+     * specified accelerator.
+     *
+     * @return An opaque handle of the region, if the accelerator is based
+     *         on HSA.  NULL otherwise.
+     */
+    void* get_hsa_am_finegrained_system_region() {
+        return pQueue->getHSACoherentAMHostRegion();
+    }
+
     /**
      * Returns an opaque handle which points to the Kernarg region on the HSA
      * agent.
@@ -876,6 +888,18 @@ class accelerator
         return get_default_view().get_hsa_am_system_region();
     }
 
+    /**
+     * Returns an opaque handle which points to the AM system region on the HSA agent.
+     * This region can be used to allocate finegrained system memory which is accessible from the 
+     * specified accelerator.
+     *
+     * @return An opaque handle of the region, if the accelerator is based
+     *         on HSA.  NULL otherwise.
+     */
+    void* get_hsa_am_finegrained_system_region() const {
+        return get_default_view().get_hsa_am_finegrained_system_region();
+    }
+
     /**
      * Returns an opaque handle which points to the Kernarg region on the HSA
      * agent.
diff --git a/include/hc_am.hpp b/include/hc_am.hpp
index 57f7a32039b..b90d365963f 100644
--- a/include/hc_am.hpp
+++ b/include/hc_am.hpp
@@ -10,6 +10,7 @@ typedef int am_status_t;
 
 // Flags for am_alloc API:
 #define amHostPinned 0x1
+#define amHostCoherent 0x2
 
 
 namespace hc {
diff --git a/include/kalmar_runtime.h b/include/kalmar_runtime.h
index 672892191e3..f56fcc1ded3 100644
--- a/include/kalmar_runtime.h
+++ b/include/kalmar_runtime.h
@@ -216,6 +216,8 @@ class KalmarQueue
   virtual void* getHSAAMRegion() { return nullptr; }
 
   virtual void* getHSAAMHostRegion() { return nullptr; }
+  
+  virtual void* getHSACoherentAMHostRegion() { return nullptr; }
 
   /// get kernarg region handle
   virtual void* getHSAKernargRegion() { return nullptr; }
diff --git a/lib/hsa/hc_am.cpp b/lib/hsa/hc_am.cpp
index 74b765eb01b..eb898770642 100644
--- a/lib/hsa/hc_am.cpp
+++ b/lib/hsa/hc_am.cpp
@@ -196,7 +196,9 @@ auto_voidp am_alloc(size_t sizeBytes, hc::accelerator &acc, unsigned flags)
             hsa_amd_memory_pool_t *alloc_region;
             if (flags & amHostPinned) {
                alloc_region = static_cast<hsa_amd_memory_pool_t*>(acc.get_hsa_am_system_region());
-            } else {
+            } else if (flags & amHostCoherent) {
+               alloc_region = static_cast<hsa_amd_memory_pool_t*>(acc.get_hsa_am_finegrained_system_region());
+            }else {
                alloc_region = static_cast<hsa_amd_memory_pool_t*>(acc.get_hsa_am_region());
             }
 
diff --git a/lib/hsa/mcwamp_hsa.cpp b/lib/hsa/mcwamp_hsa.cpp
index 800b8338d96..3f0c8d6b43b 100644
--- a/lib/hsa/mcwamp_hsa.cpp
+++ b/lib/hsa/mcwamp_hsa.cpp
@@ -694,6 +694,7 @@ struct pool_iterator
 {
     hsa_amd_memory_pool_t _am_memory_pool;
     hsa_amd_memory_pool_t _am_host_memory_pool;
+    hsa_amd_memory_pool_t _am_host_coherent_memory_pool;
 
     hsa_amd_memory_pool_t _kernarg_memory_pool;
     hsa_amd_memory_pool_t _finegrained_system_memory_pool;
@@ -1368,6 +1369,8 @@ class HSAQueue final : public KalmarQueue
     void* getHostAgent() override;
 
     void* getHSAAMRegion() override;
+    
+    void* getHSACoherentHostRegion() override;
 
     void* getHSAAMHostRegion() override;
 
@@ -1847,6 +1850,10 @@ class HSADevice final : public KalmarDevice
         ri._am_host_memory_pool = (ri._found_coarsegrained_system_memory_pool)
                                       ? ri._coarsegrained_system_memory_pool
                                       : ri._finegrained_system_memory_pool;
+        
+        ri._am_host_coherent_memory_pool = (ri._found_finegrained_system_memory_pool)
+                                      ? ri._finegrained_system_memory_pool
+                                      : ri._coarsegrained_system_memory_pool;
 
         /// Query the maximum number of work-items in a workgroup
         status = hsa_agent_get_info(agent, HSA_AGENT_INFO_WORKGROUP_MAX_SIZE, &workgroup_max_size);
@@ -2210,6 +2217,10 @@ class HSADevice final : public KalmarDevice
         return ri._am_host_memory_pool;
     }
 
+    hsa_amd_memory_pool_t& getHSACoherentHostRegion() {
+        return ri._am_host_coherent_memory_pool;
+    }
+
     hsa_amd_memory_pool_t& getHSAAMRegion() {
         return ri._am_memory_pool;
     }
@@ -2969,7 +2980,10 @@ inline void*
 HSAQueue::getHSAAMRegion() override {
     return static_cast<void*>(&(static_cast<HSADevice*>(getDev())->getHSAAMRegion()));
 }
-
+inline void*
+HSAQueue::getHSACoherentHostRegion() override {
+    return static_cast<void*>(&(static_cast<HSADevice*>(getDev())->getHSACoherentHostRegion()));
+}
 inline void*
 HSAQueue::getHSAAMHostRegion() override {
     return static_cast<void*>(&(static_cast<HSADevice*>(getDev())->getHSAAMHostRegion()));