openmp_cuda.h

Go to the documentation of this file.

/*
 *            Copyright 2009-2020 The VOTCA Development Team
 *                       (http://www.votca.org)
 *
 *      Licensed under the Apache License, Version 2.0 (the "License")
 *
 * You may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *              http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *
 */
 
#pragma once
#ifndef VOTCA_XTP_OPENMP_CUDA_H
#define VOTCA_XTP_OPENMP_CUDA_H
 
// Standard includes
#include <cassert>
 
// Local VOTCA includes
#include "eigen.h"
 
#ifdef USE_CUDA
#include "cudapipeline.h"
#endif

 
namespace votca {
namespace xtp {
 
class OpenMP_CUDA {
 public:
  OpenMP_CUDA();
  static Index UsingGPUs();
  static Index AvailableGPUs();
  static void SetNoGPUs(Index number);
 
  // 3c multiply
  void setOperators(const std::vector<Eigen::MatrixXd>& tensor,
                    const Eigen::MatrixXd& rightoperator);
  void MultiplyRight(Eigen::MatrixXd& matrix, Index OpenmpThread);
 
  // 3c
  void setOperators(const Eigen::MatrixXd& leftoperator,
                    const Eigen::MatrixXd& rightoperator);
  void MultiplyLeftRight(Eigen::MatrixXd& matrix, Index OpenmpThread);
 
  // RPA
  void createTemporaries(Index rows, Index cols);
  void PushMatrix(const Eigen::MatrixXd& mat, Index OpenmpThread);
  void A_TDA(const Eigen::VectorXd& vec, Index OpenmpThread);
 
  // Hd + Hqp + Hd2
  void createTemporaries(const Eigen::VectorXd& vec,
                         const Eigen::MatrixXd& input, Index rows1, Index rows2,
                         Index cols);
  void PrepareMatrix1(Eigen::MatrixXd& mat, Index OpenmpThread);
  void SetTempZero(Index OpenmpThread);
  void PrepareMatrix2(const Eigen::Block<const Eigen::MatrixXd>& mat, bool Hd2,
                      Index OpenmpThread);
  void Addvec(const Eigen::VectorXd& row, Index OpenmpThread);
  void MultiplyRow(Index row, Index OpenmpThread);
 
  // Hx
  void createAdditionalTemporaries(Index rows, Index cols);
  void PushMatrix1(const Eigen::MatrixXd& mat, Index OpenmpThread);
  void MultiplyBlocks(const Eigen::Block<const Eigen::MatrixXd>& mat, Index i1,
                      Index i2, Index OpenmpThread);
 
  Eigen::MatrixXd getReductionVar();
 
 private:
  template <class T>
  class DefaultReference {
   public:
    DefaultReference() = default;
    DefaultReference(T object) : p(&object) {};
 
    DefaultReference& operator=(const T& object) {
      p = &object;
      return *this;
    }
 
    const T& operator()() {
      assert(p != nullptr && "Dangling reference!");
      return *p;
    }
 
   private:
    const T* p = nullptr;
  };
 
  DefaultReference<const Eigen::MatrixXd> rOP_;
  DefaultReference<const Eigen::MatrixXd> lOP_;
  DefaultReference<const Eigen::VectorXd> vec_;
 
  struct CPU_data {
 
    Eigen::MatrixXd& reduce() { return reduce_mat; }
    void InitializeReduce(Index rows, Index cols) {
      reduce_mat = Eigen::MatrixXd::Zero(rows, cols);
    }
 
    void InitializeVec(Index size) { temp_vec = Eigen::VectorXd::Zero(size); }
 
    DefaultReference<Eigen::MatrixXd> ref_mat;
    Eigen::MatrixXd temp_mat;
    Eigen::VectorXd temp_vec;
    Eigen::MatrixXd reduce_mat;
  };
 
  std::vector<CPU_data> cpus_;
 
  bool inside_Parallel_region_;
  Index threadID_parent_;
 
  static Index number_of_gpus;
 
  Index getParentThreadId(Index OpenmpThreadId) const;
 
  Index getLocalThreadId(Index ParentThreadId) const;
 
  Index getNumberThreads() const;
 
#ifdef USE_CUDA
  bool isGPUthread(Index ParentThreadId) const;
 
  struct GPU_data {
 
    explicit GPU_data(Index i)
        : Id(i), pipeline(std::make_unique<CudaPipeline>(int(i))) {
      ;
    }
 
    Index Id;
    std::unique_ptr<CudaPipeline> pipeline;
    std::vector<std::unique_ptr<CudaMatrix>> temp;
 
    CudaMatrix& Mat(Index i) { return *temp[i]; }
    CudaPipeline& pipe() { return *pipeline; }
    void activateGPU() { checkCuda(cudaSetDevice(pipeline->getDeviceId())); }
 
    void push_back(const Eigen::MatrixXd& m) {
      temp.push_back(std::make_unique<CudaMatrix>(m, pipeline->get_stream()));
    }
    void push_back(Index rows, Index cols) {
      temp.push_back(
          std::make_unique<CudaMatrix>(rows, cols, pipeline->get_stream()));
    }
 
    void resize(Index id, Index rows, Index cols) {
      temp[id] =
          std::make_unique<CudaMatrix>(rows, cols, pipeline->get_stream());
    }
  };
 
  std::vector<GPU_data> gpus_;
  static bool isInVector(Index Id, const std::vector<GPU_data>& vec);
#endif
};
 
}  // namespace xtp
}  // namespace votca
 
#endif  // VOTCA_XTP_OPENMP_CUDA_H