Skip to content

Commit

Permalink
Replace use of API wrapper stream and event with plain CUDA, part 1 (#…
Browse files Browse the repository at this point in the history 
…389)

Replace cuda::stream_t<> with cudaStream_t in client code
Replace cuda::event_t with cudaEvent_t in the client code
Clean up BuildFiles
  • Loading branch information
@makortel @fwyzard
makortel authored and fwyzard committed Dec 29, 2020
1 parent 419aa82 commit 93dd141
Show file tree
Hide file tree
Showing 18 changed files with 69 additions and 72 deletions.
1 change: 0 additions & 1 deletion RecoLocalCalo/EcalRecAlgos/BuildFile.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,6 @@
<use name="cuda"/>
<use name="CUDADataFormats/EcalRecHitSoA"/>
<use name="HeterogeneousCore/CUDAUtilities"/>
<use name="cuda-api-wrappers"/>
<use name="HeterogeneousCore/CUDACore"/>

<export>
Expand Down
4 changes: 2 additions & 2 deletions RecoLocalCalo/EcalRecAlgos/interface/EcalGainRatiosGPU.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@
#include "HeterogeneousCore/CUDACore/interface/CUDAESProduct.h"
#endif

#include <cuda/api_wrappers.h>
#include <cuda_runtime.h>

class EcalGainRatiosGPU {
public:
Expand All @@ -26,7 +26,7 @@ class EcalGainRatiosGPU {
~EcalGainRatiosGPU() = default;

// get device pointers
Product const& getProduct(cuda::stream_t<>&) const;
Product const& getProduct(cudaStream_t) const;

//
static std::string name() { return std::string{"ecalGainRatiosGPU"}; }
Expand Down
4 changes: 2 additions & 2 deletions RecoLocalCalo/EcalRecAlgos/interface/EcalPedestalsGPU.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@
#include "HeterogeneousCore/CUDACore/interface/CUDAESProduct.h"
#endif

#include <cuda/api_wrappers.h>
#include <cuda_runtime.h>

class EcalPedestalsGPU {
public:
Expand All @@ -27,7 +27,7 @@ class EcalPedestalsGPU {
~EcalPedestalsGPU() = default;

// get device pointers
Product const &getProduct(cuda::stream_t<> &) const;
Product const &getProduct(cudaStream_t) const;

//
static std::string name() { return std::string{"ecalPedestalsGPU"}; }
Expand Down
4 changes: 2 additions & 2 deletions RecoLocalCalo/EcalRecAlgos/interface/EcalPulseCovariancesGPU.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@
#include "HeterogeneousCore/CUDACore/interface/CUDAESProduct.h"
#endif

#include <cuda/api_wrappers.h>
#include <cuda_runtime.h>

class EcalPulseCovariancesGPU {
public:
Expand All @@ -25,7 +25,7 @@ class EcalPulseCovariancesGPU {
~EcalPulseCovariancesGPU() = default;

// get device pointers
Product const& getProduct(cuda::stream_t<>&) const;
Product const& getProduct(cudaStream_t) const;

//
static std::string name() { return std::string{"ecalPulseCovariancesGPU"}; }
Expand Down
4 changes: 2 additions & 2 deletions RecoLocalCalo/EcalRecAlgos/interface/EcalPulseShapesGPU.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@
#include "HeterogeneousCore/CUDACore/interface/CUDAESProduct.h"
#endif

#include <cuda/api_wrappers.h>
#include <cuda_runtime.h>

class EcalPulseShapesGPU {
public:
Expand All @@ -25,7 +25,7 @@ class EcalPulseShapesGPU {
~EcalPulseShapesGPU() = default;

// get device pointers
Product const& getProduct(cuda::stream_t<>&) const;
Product const& getProduct(cudaStream_t) const;

//
static std::string name() { return std::string{"ecalPulseShapesGPU"}; }
Expand Down
4 changes: 2 additions & 2 deletions RecoLocalCalo/EcalRecAlgos/interface/EcalSamplesCorrelationGPU.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@
#include "HeterogeneousCore/CUDACore/interface/CUDAESProduct.h"
#endif

#include <cuda/api_wrappers.h>
#include <cuda_runtime.h>

class EcalSamplesCorrelationGPU {
public:
Expand All @@ -26,7 +26,7 @@ class EcalSamplesCorrelationGPU {
~EcalSamplesCorrelationGPU() = default;

// get device pointers
Product const& getProduct(cuda::stream_t<>&) const;
Product const& getProduct(cudaStream_t) const;

//
static std::string name() { return std::string{"ecalSamplesCorrelationGPU"}; }
Expand Down
4 changes: 2 additions & 2 deletions RecoLocalCalo/EcalRecAlgos/interface/EcalTimeBiasCorrectionsGPU.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@
#include "HeterogeneousCore/CUDACore/interface/CUDAESProduct.h"
#endif

#include <cuda/api_wrappers.h>
#include <cuda_runtime.h>

class EcalTimeBiasCorrectionsGPU {
public:
Expand All @@ -28,7 +28,7 @@ class EcalTimeBiasCorrectionsGPU {
~EcalTimeBiasCorrectionsGPU() = default;

// get device pointers
Product const& getProduct(cuda::stream_t<>&) const;
Product const& getProduct(cudaStream_t) const;

//
static std::string name() { return std::string{"ecalTimeBiasCorrectionsGPU"}; }
Expand Down
4 changes: 2 additions & 2 deletions RecoLocalCalo/EcalRecAlgos/interface/EcalTimeCalibConstantsGPU.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@
#include "HeterogeneousCore/CUDACore/interface/CUDAESProduct.h"
#endif

#include <cuda/api_wrappers.h>
#include <cuda_runtime.h>

class EcalTimeCalibConstantsGPU {
public:
Expand All @@ -25,7 +25,7 @@ class EcalTimeCalibConstantsGPU {
~EcalTimeCalibConstantsGPU() = default;

// get device pointers
Product const& getProduct(cuda::stream_t<>&) const;
Product const& getProduct(cudaStream_t) const;

// TODO: do this centrally
// get offset for hashes. equals number of barrel items
Expand Down
8 changes: 4 additions & 4 deletions RecoLocalCalo/EcalRecAlgos/src/EcalGainRatiosGPU.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,9 +27,9 @@ EcalGainRatiosGPU::Product::~Product() {
cudaCheck(cudaFree(gain6Over1));
}

EcalGainRatiosGPU::Product const& EcalGainRatiosGPU::getProduct(cuda::stream_t<>& cudaStream) const {
EcalGainRatiosGPU::Product const& EcalGainRatiosGPU::getProduct(cudaStream_t cudaStream) const {
auto const& product = product_.dataForCurrentDeviceAsync(
cudaStream, [this](EcalGainRatiosGPU::Product& product, cuda::stream_t<>& cudaStream) {
cudaStream, [this](EcalGainRatiosGPU::Product& product, cudaStream_t cudaStream) {
// malloc
cudaCheck(cudaMalloc((void**)&product.gain12Over6, this->gain12Over6_.size() * sizeof(float)));
cudaCheck(cudaMalloc((void**)&product.gain6Over1, this->gain6Over1_.size() * sizeof(float)));
Expand All @@ -38,12 +38,12 @@ EcalGainRatiosGPU::Product const& EcalGainRatiosGPU::getProduct(cuda::stream_t<>
this->gain12Over6_.data(),
this->gain12Over6_.size() * sizeof(float),
cudaMemcpyHostToDevice,
cudaStream.id()));
cudaStream));
cudaCheck(cudaMemcpyAsync(product.gain6Over1,
this->gain6Over1_.data(),
this->gain6Over1_.size() * sizeof(float),
cudaMemcpyHostToDevice,
cudaStream.id()));
cudaStream));
});

return product;
Expand Down
16 changes: 8 additions & 8 deletions RecoLocalCalo/EcalRecAlgos/src/EcalPedestalsGPU.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -44,9 +44,9 @@ EcalPedestalsGPU::Product::~Product() {
cudaCheck(cudaFree(rms_x1));
}

EcalPedestalsGPU::Product const& EcalPedestalsGPU::getProduct(cuda::stream_t<>& cudaStream) const {
EcalPedestalsGPU::Product const& EcalPedestalsGPU::getProduct(cudaStream_t cudaStream) const {
auto const& product = product_.dataForCurrentDeviceAsync(
cudaStream, [this](EcalPedestalsGPU::Product& product, cuda::stream_t<>& cudaStream) {
cudaStream, [this](EcalPedestalsGPU::Product& product, cudaStream_t cudaStream) {
// malloc
cudaCheck(cudaMalloc((void**)&product.mean_x12, this->mean_x12_.size() * sizeof(float)));
cudaCheck(cudaMalloc((void**)&product.rms_x12, this->mean_x12_.size() * sizeof(float)));
Expand All @@ -60,32 +60,32 @@ EcalPedestalsGPU::Product const& EcalPedestalsGPU::getProduct(cuda::stream_t<>&
this->mean_x12_.data(),
this->mean_x12_.size() * sizeof(float),
cudaMemcpyHostToDevice,
cudaStream.id()));
cudaStream));
cudaCheck(cudaMemcpyAsync(product.rms_x12,
this->rms_x12_.data(),
this->rms_x12_.size() * sizeof(float),
cudaMemcpyHostToDevice,
cudaStream.id()));
cudaStream));
cudaCheck(cudaMemcpyAsync(product.mean_x6,
this->mean_x6_.data(),
this->mean_x6_.size() * sizeof(float),
cudaMemcpyHostToDevice,
cudaStream.id()));
cudaStream));
cudaCheck(cudaMemcpyAsync(product.rms_x6,
this->rms_x6_.data(),
this->rms_x6_.size() * sizeof(float),
cudaMemcpyHostToDevice,
cudaStream.id()));
cudaStream));
cudaCheck(cudaMemcpyAsync(product.mean_x1,
this->mean_x1_.data(),
this->mean_x1_.size() * sizeof(float),
cudaMemcpyHostToDevice,
cudaStream.id()));
cudaStream));
cudaCheck(cudaMemcpyAsync(product.rms_x1,
this->rms_x1_.data(),
this->rms_x1_.size() * sizeof(float),
cudaMemcpyHostToDevice,
cudaStream.id()));
cudaStream));
});

return product;
Expand Down
8 changes: 4 additions & 4 deletions RecoLocalCalo/EcalRecAlgos/src/EcalPulseCovariancesGPU.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,9 +11,9 @@ EcalPulseCovariancesGPU::Product::~Product() {
cudaCheck(cudaFree(values));
}

EcalPulseCovariancesGPU::Product const& EcalPulseCovariancesGPU::getProduct(cuda::stream_t<>& cudaStream) const {
EcalPulseCovariancesGPU::Product const& EcalPulseCovariancesGPU::getProduct(cudaStream_t cudaStream) const {
auto const& product = product_.dataForCurrentDeviceAsync(
cudaStream, [this](EcalPulseCovariancesGPU::Product& product, cuda::stream_t<>& cudaStream) {
cudaStream, [this](EcalPulseCovariancesGPU::Product& product, cudaStream_t cudaStream) {
// malloc
cudaCheck(cudaMalloc((void**)&product.values,
(this->valuesEE_.size() + this->valuesEB_.size()) * sizeof(EcalPulseCovariance)));
Expand All @@ -26,14 +26,14 @@ EcalPulseCovariancesGPU::Product const& EcalPulseCovariancesGPU::getProduct(cuda
this->valuesEB_.data(),
this->valuesEB_.size() * sizeof(EcalPulseCovariance),
cudaMemcpyHostToDevice,
cudaStream.id()));
cudaStream));

// transfer ee starting at values + offset
cudaCheck(cudaMemcpyAsync(product.values + offset,
this->valuesEE_.data(),
this->valuesEE_.size() * sizeof(EcalPulseCovariance),
cudaMemcpyHostToDevice,
cudaStream.id()));
cudaStream));
});

return product;
Expand Down
8 changes: 4 additions & 4 deletions RecoLocalCalo/EcalRecAlgos/src/EcalPulseShapesGPU.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,9 +11,9 @@ EcalPulseShapesGPU::Product::~Product() {
cudaCheck(cudaFree(values));
}

EcalPulseShapesGPU::Product const& EcalPulseShapesGPU::getProduct(cuda::stream_t<>& cudaStream) const {
EcalPulseShapesGPU::Product const& EcalPulseShapesGPU::getProduct(cudaStream_t cudaStream) const {
auto const& product = product_.dataForCurrentDeviceAsync(
cudaStream, [this](EcalPulseShapesGPU::Product& product, cuda::stream_t<>& cudaStream) {
cudaStream, [this](EcalPulseShapesGPU::Product& product, cudaStream_t cudaStream) {
// malloc
cudaCheck(cudaMalloc((void**)&product.values,
(this->valuesEE_.size() + this->valuesEB_.size()) * sizeof(EcalPulseShape)));
Expand All @@ -26,14 +26,14 @@ EcalPulseShapesGPU::Product const& EcalPulseShapesGPU::getProduct(cuda::stream_t
this->valuesEB_.data(),
this->valuesEB_.size() * sizeof(EcalPulseShape),
cudaMemcpyHostToDevice,
cudaStream.id()));
cudaStream));

// transfer ee starting at values + offset
cudaCheck(cudaMemcpyAsync(product.values + offset,
this->valuesEE_.data(),
this->valuesEE_.size() * sizeof(EcalPulseShape),
cudaMemcpyHostToDevice,
cudaStream.id()));
cudaStream));
});

return product;
Expand Down
16 changes: 8 additions & 8 deletions RecoLocalCalo/EcalRecAlgos/src/EcalSamplesCorrelationGPU.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,9 +21,9 @@ EcalSamplesCorrelationGPU::Product::~Product() {
cudaCheck(cudaFree(EEG1SamplesCorrelation));
}

EcalSamplesCorrelationGPU::Product const& EcalSamplesCorrelationGPU::getProduct(cuda::stream_t<>& cudaStream) const {
EcalSamplesCorrelationGPU::Product const& EcalSamplesCorrelationGPU::getProduct(cudaStream_t cudaStream) const {
auto const& product = product_.dataForCurrentDeviceAsync(
cudaStream, [this](EcalSamplesCorrelationGPU::Product& product, cuda::stream_t<>& cudaStream) {
cudaStream, [this](EcalSamplesCorrelationGPU::Product& product, cudaStream_t cudaStream) {
// malloc
cudaCheck(cudaMalloc((void**)&product.EBG12SamplesCorrelation,
this->EBG12SamplesCorrelation_.size() * sizeof(double)));
Expand All @@ -42,32 +42,32 @@ EcalSamplesCorrelationGPU::Product const& EcalSamplesCorrelationGPU::getProduct(
this->EBG12SamplesCorrelation_.data(),
this->EBG12SamplesCorrelation_.size() * sizeof(double),
cudaMemcpyHostToDevice,
cudaStream.id()));
cudaStream));
cudaCheck(cudaMemcpyAsync(product.EBG6SamplesCorrelation,
this->EBG6SamplesCorrelation_.data(),
this->EBG6SamplesCorrelation_.size() * sizeof(double),
cudaMemcpyHostToDevice,
cudaStream.id()));
cudaStream));
cudaCheck(cudaMemcpyAsync(product.EBG1SamplesCorrelation,
this->EBG1SamplesCorrelation_.data(),
this->EBG1SamplesCorrelation_.size() * sizeof(double),
cudaMemcpyHostToDevice,
cudaStream.id()));
cudaStream));
cudaCheck(cudaMemcpyAsync(product.EEG12SamplesCorrelation,
this->EEG12SamplesCorrelation_.data(),
this->EEG12SamplesCorrelation_.size() * sizeof(double),
cudaMemcpyHostToDevice,
cudaStream.id()));
cudaStream));
cudaCheck(cudaMemcpyAsync(product.EEG6SamplesCorrelation,
this->EEG6SamplesCorrelation_.data(),
this->EEG6SamplesCorrelation_.size() * sizeof(double),
cudaMemcpyHostToDevice,
cudaStream.id()));
cudaStream));
cudaCheck(cudaMemcpyAsync(product.EEG1SamplesCorrelation,
this->EEG1SamplesCorrelation_.data(),
this->EEG1SamplesCorrelation_.size() * sizeof(double),
cudaMemcpyHostToDevice,
cudaStream.id()));
cudaStream));
});

return product;
Expand Down
12 changes: 6 additions & 6 deletions RecoLocalCalo/EcalRecAlgos/src/EcalTimeBiasCorrectionsGPU.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,9 +17,9 @@ EcalTimeBiasCorrectionsGPU::Product::~Product() {
cudaCheck(cudaFree(EETimeCorrShiftBins));
}

EcalTimeBiasCorrectionsGPU::Product const& EcalTimeBiasCorrectionsGPU::getProduct(cuda::stream_t<>& cudaStream) const {
EcalTimeBiasCorrectionsGPU::Product const& EcalTimeBiasCorrectionsGPU::getProduct(cudaStream_t cudaStream) const {
auto const& product = product_.dataForCurrentDeviceAsync(
cudaStream, [this](EcalTimeBiasCorrectionsGPU::Product& product, cuda::stream_t<>& cudaStream) {
cudaStream, [this](EcalTimeBiasCorrectionsGPU::Product& product, cudaStream_t cudaStream) {
// to get the size of vectors later on
// should be removed and host conditions' objects used directly
product.EBTimeCorrAmplitudeBinsSize = this->EBTimeCorrAmplitudeBins_.size();
Expand All @@ -37,22 +37,22 @@ EcalTimeBiasCorrectionsGPU::Product const& EcalTimeBiasCorrectionsGPU::getProduc
this->EBTimeCorrAmplitudeBins_.data(),
this->EBTimeCorrAmplitudeBins_.size() * sizeof(float),
cudaMemcpyHostToDevice,
cudaStream.id()));
cudaStream));
cudaCheck(cudaMemcpyAsync(product.EBTimeCorrShiftBins,
this->EBTimeCorrShiftBins_.data(),
this->EBTimeCorrShiftBins_.size() * sizeof(float),
cudaMemcpyHostToDevice,
cudaStream.id()));
cudaStream));
cudaCheck(cudaMemcpyAsync(product.EETimeCorrAmplitudeBins,
this->EETimeCorrAmplitudeBins_.data(),
this->EETimeCorrAmplitudeBins_.size() * sizeof(float),
cudaMemcpyHostToDevice,
cudaStream.id()));
cudaStream));
cudaCheck(cudaMemcpyAsync(product.EETimeCorrShiftBins,
this->EETimeCorrShiftBins_.data(),
this->EETimeCorrShiftBins_.size() * sizeof(float),
cudaMemcpyHostToDevice,
cudaStream.id()));
cudaStream));
});

return product;
Expand Down
8 changes: 4 additions & 4 deletions RecoLocalCalo/EcalRecAlgos/src/EcalTimeCalibConstantsGPU.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,9 +11,9 @@ EcalTimeCalibConstantsGPU::Product::~Product() {
cudaCheck(cudaFree(values));
}

EcalTimeCalibConstantsGPU::Product const& EcalTimeCalibConstantsGPU::getProduct(cuda::stream_t<>& cudaStream) const {
EcalTimeCalibConstantsGPU::Product const& EcalTimeCalibConstantsGPU::getProduct(cudaStream_t cudaStream) const {
auto const& product = product_.dataForCurrentDeviceAsync(
cudaStream, [this](EcalTimeCalibConstantsGPU::Product& product, cuda::stream_t<>& cudaStream) {
cudaStream, [this](EcalTimeCalibConstantsGPU::Product& product, cudaStream_t cudaStream) {
// malloc
cudaCheck(
cudaMalloc((void**)&product.values, (this->valuesEB_.size() + this->valuesEE_.size()) * sizeof(float)));
Expand All @@ -26,12 +26,12 @@ EcalTimeCalibConstantsGPU::Product const& EcalTimeCalibConstantsGPU::getProduct(
this->valuesEB_.data(),
this->valuesEB_.size() * sizeof(float),
cudaMemcpyHostToDevice,
cudaStream.id()));
cudaStream));
cudaCheck(cudaMemcpyAsync(product.values + offset,
this->valuesEE_.data(),
this->valuesEE_.size() * sizeof(float),
cudaMemcpyHostToDevice,
cudaStream.id()));
cudaStream));
});

return product;
Expand Down
1 change: 0 additions & 1 deletion RecoLocalCalo/EcalRecProducers/BuildFile.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,6 @@
<use name="CondFormats/EcalObjects"/>
<use name="HeterogeneousCore/CUDACore"/>
<use name="HeterogeneousCore/CUDAUtilities"/>
<use name="cuda-api-wrappers"/>
<use name="cuda"/>
<export>
<lib name="1"/>
Expand Down
Loading

0 comments on commit 93dd141

Please sign in to comment.