mpi_manager.hpp

Go to the documentation of this file.

#pragma once
 
#include "mpi_data_def.hpp"
#include "utils_macros.hpp"
#include <mpi.h>
 
namespace netdem {
 
class Simulation;
 
class MPIManager {
private:
  MPIDataDefine mpi_data_def;
 
  int self_rank, total_rank;
 
  VecXT<VecXT<Particle *>> particle_proxy_out_list;
 
  VecXT<VecXT<ContactPP *>> contact_pp_proxy_out_list;
  VecXT<VecXT<BondEntry *>> bond_entry_pp_proxy_out_list;
  VecXT<VecXT<CollisionEntry *>> collision_entry_pp_proxy_out_list;
 
  VecXT<VecXT<ContactPW *>> contact_pw_proxy_out_list;
  VecXT<VecXT<BondEntry *>> bond_entry_pw_proxy_out_list;
  VecXT<VecXT<CollisionEntry *>> collision_entry_pw_proxy_out_list;
 
  VecXT<VecXT<ContactPP *>> contact_pp_back_out_list;
  VecXT<VecXT<BondEntry *>> bond_entry_pp_back_out_list;
  VecXT<VecXT<CollisionEntry *>> collision_entry_pp_back_out_list;
 
  VecXT<VecXT<ContactPW *>> contact_pw_back_out_list;
  VecXT<VecXT<BondEntry *>> bond_entry_pw_back_out_list;
  VecXT<VecXT<CollisionEntry *>> collision_entry_pw_back_out_list;
 
  VecXT<VecXT<Particle *>> particle_transfer_out_list;
 
  VecXT<VecXT<ContactPP *>> contact_pp_transfer_out_list;
  VecXT<VecXT<BondEntry *>> bond_entry_pp_transfer_out_list;
  VecXT<VecXT<CollisionEntry *>> collision_entry_pp_transfer_out_list;
 
  VecXT<VecXT<ContactPW *>> contact_pw_transfer_out_list;
  VecXT<VecXT<BondEntry *>> bond_entry_pw_transfer_out_list;
  VecXT<VecXT<CollisionEntry *>> collision_entry_pw_transfer_out_list;
 
  VecXT<VecXT<Particle *>> particle_proxy_in_list;
  VecXT<VecXT<ContactPP *>> contact_pp_proxy_in_list;
  VecXT<VecXT<ContactPW *>> contact_pw_proxy_in_list;
 
  VecXT<VecXT<ContactPP *>> contact_pp_back_in_list;
  VecXT<VecXT<ContactPW *>> contact_pw_back_in_list;
 
  VecXT<VecXT<Particle *>> particle_transfer_in_list;
  VecXT<VecXT<ContactPP *>> contact_pp_transfer_in_list;
  VecXT<VecXT<ContactPW *>> contact_pw_transfer_in_list;
 
  VecXT<VecXT<Particle *>> particle_sync_in_list;
  VecXT<VecXT<ContactPP *>> contact_pp_sync_in_list;
  VecXT<VecXT<ContactPW *>> contact_pw_sync_in_list;
 
public:
  MPIManager();
 
  void Init(Simulation *sim);
 
  void CommitMPIDataType();
 
  int GetSelfRank() const;
 
  int GetTotalRank() const;
 
  void BuildContactRefTable();
 
  void RecycleParticleProxy();
 
  void ExchangeDataTransfer();
 
  void ExchangeDataProxy();
 
  void ExchangeDataBack();
 
  void RecycleParticleGhost();
 
  void ClearContactRefTable();
 
  void SyncShapeToAllProcessors();
 
  void SyncParticleToGhostSubDomains();
 
  void SyncContactToGhostSubDomains();
 
  void SyncWallToGhostSubDomains();
 
private:
  void GatherDataProxy();
 
  void GatherDataBack();
 
  void GatherDataTransfer();
 
  void SendDataProxy();
 
  void SendDataBack();
 
  void SendDataTransfer();
 
  void SendParticleSync();
 
  void SendContactSync();
 
  void RecvDataProxy();
 
  void RecvDataBack();
 
  void RecvDataTransfer();
 
  void RecvParticleSync();
 
  void RecvContactSync();
 
  void MergeShape(int source_rank);
 
  void MergeShapeFast(int source_rank);
 
  void MergeParticleProxy(int source_rank);
 
  void MergeContactPPProxy(int source_rank);
 
  void MergeContactPWProxy(int source_rank);
 
  void MergeContactPPBack(int source_rank);
 
  void MergeContactPWBack(int source_rank);
 
  void MergeParticleTransfer(int source_rank);
 
  void MergeContactPPTransfer(int source_rank);
 
  void MergeContactPWTransfer(int source_rank);
 
  void MergeParticleSync(int source_rank);
 
  void MergeContactPPSync(int source_rank);
 
  void MergeContactPWSync(int source_rank);
 
public:
  static int GetSelfRankStatic();
 
  static int GetTotalRankStatic();
 
  static void SyncDataAmongProcessors(VecXT<int> *const data);
 
  static void SyncDataAmongProcessors(VecXT<double> *const data);
 
  ~MPIManager();
 
private:
  // for basic properties
  void GatherDataShape();
 
  void SendShape(int i_rank);
  void SendParticle(VecXT<Particle *> const &p_list, int i_rank);
 
  void SendBondEntryPP(VecXT<BondEntry *> const &bond_list, int i_rank);
  void SendCollisionEntryPP(VecXT<CollisionEntry *> const &collision_list,
                            int i_rank);
  void SendContactPP(VecXT<ContactPP *> const &cnt_list, int i_rank);
 
  void SendBondEntryPW(VecXT<BondEntry *> const &bond_list, int i_rank);
  void SendCollisionEntryPW(VecXT<CollisionEntry *> const &collision_list,
                            int i_rank);
  void SendContactPW(VecXT<ContactPW *> const &cnt_list, int i_rank);
 
  void RecvShape(MPIManager *obj, void (MPIManager::*MergeFunc)(int));
  void RecvParticle(MPIManager *obj, void (MPIManager::*MergeFunc)(int));
 
  void RecvBondEntryPP();
  void RecvCollisionEntryPP();
  void RecvContactPP(MPIManager *obj, void (MPIManager::*MergeFunc)(int));
 
  void RecvBondEntryPW();
  void RecvCollisionEntryPW();
  void RecvContactPW(MPIManager *obj, void (MPIManager::*MergeFunc)(int));
 
  // for dynamics properties
  void SendParticleProp(VecXT<Particle *> const &p_list, int i_rank);
  void SendContactPPProp(VecXT<ContactPP *> const &cnt_list, int i_rank);
  void SendContactPWProp(VecXT<ContactPW *> const &cnt_list, int i_rank);
 
  void RecvParticleProp(VecXT<VecXT<Particle *>> *const p_list);
  void RecvContactPPProp(VecXT<VecXT<ContactPP *>> *const cnt_list);
  void RecvContactPWProp(VecXT<VecXT<ContactPW *>> *const cnt_list);
 
  void MergeParticleProp(VecXT<Particle *> *const p_list, int source_rank);
  void MergeContactPPProp(VecXT<ContactPP *> *const cnt_list, int source_rank);
  void MergeContactPWProp(VecXT<ContactPW *> *const cnt_list, int source_rank);
 
private:
  bool mpi_initialized_here{false};
 
  Simulation *sim{nullptr};
 
  VecXT<ParticleData *> particle_data_list_send;
  VecXT<std::string> particle_prop_keys_list_send;
  VecXT<VecXT<double>> particle_prop_vals_list_send;
  VecXT<ContactPPData *> contact_pp_data_list_send;
  VecXT<std::string> contact_pp_prop_keys_list_send;
  VecXT<VecXT<double>> contact_pp_prop_vals_list_send;
  VecXT<BondEntryData *> bond_entry_pp_data_list_send;
  VecXT<CollisionEntryData *> collision_entry_pp_data_list_send;
  VecXT<ContactPWData *> contact_pw_data_list_send;
  VecXT<std::string> contact_pw_prop_keys_list_send;
  VecXT<VecXT<double>> contact_pw_prop_vals_list_send;
  VecXT<BondEntryData *> bond_entry_pw_data_list_send;
  VecXT<CollisionEntryData *> collision_entry_pw_data_list_send;
 
  VecXT<int> particle_num_list_send;
  VecXT<int> contact_pp_num_list_send;
  VecXT<int> bond_entry_pp_num_list_send;
  VecXT<int> collision_entry_pp_num_list_send;
  VecXT<int> contact_pw_num_list_send;
  VecXT<int> bond_entry_pw_num_list_send;
  VecXT<int> collision_entry_pw_num_list_send;
 
  VecXT<MPI_Request> particle_req_list_send;
  VecXT<MPI_Request> particle_prop_keys_req_list_send;
  VecXT<MPI_Request> particle_prop_vals_req_list_send;
  VecXT<MPI_Request> contact_pp_req_list_send;
  VecXT<MPI_Request> contact_pp_prop_keys_req_list_send;
  VecXT<MPI_Request> contact_pp_prop_vals_req_list_send;
  VecXT<MPI_Request> bond_entry_pp_req_list_send;
  VecXT<MPI_Request> collision_entry_pp_req_list_send;
  VecXT<MPI_Request> contact_pw_req_list_send;
  VecXT<MPI_Request> contact_pw_prop_keys_req_list_send;
  VecXT<MPI_Request> contact_pw_prop_vals_req_list_send;
  VecXT<MPI_Request> bond_entry_pw_req_list_send;
  VecXT<MPI_Request> collision_entry_pw_req_list_send;
 
  std::string *shape_data_send{nullptr};
  VecXT<MPI_Request> shape_req_list_send;
  VecXT<std::string> shape_data_list_recv;
  VecXT<MPI_Request> shape_req_list_recv;
  VecXT<bool> shape_probed_list;
 
  VecXT<ParticleData *> particle_data_list_recv;
  VecXT<std::string> particle_prop_keys_list_recv;
  VecXT<VecXT<double>> particle_prop_vals_list_recv;
  VecXT<ContactPPData *> contact_pp_data_list_recv;
  VecXT<std::string> contact_pp_prop_keys_list_recv;
  VecXT<VecXT<double>> contact_pp_prop_vals_list_recv;
  VecXT<BondEntryData *> bond_entry_pp_data_list_recv;
  VecXT<CollisionEntryData *> collision_entry_pp_data_list_recv;
  VecXT<ContactPWData *> contact_pw_data_list_recv;
  VecXT<std::string> contact_pw_prop_keys_list_recv;
  VecXT<VecXT<double>> contact_pw_prop_vals_list_recv;
  VecXT<BondEntryData *> bond_entry_pw_data_list_recv;
  VecXT<CollisionEntryData *> collision_entry_pw_data_list_recv;
 
  VecXT<int> particle_num_list_recv;
  VecXT<int> contact_pp_num_list_recv;
  VecXT<int> bond_entry_pp_num_list_recv;
  VecXT<int> collision_entry_pp_num_list_recv;
  VecXT<int> contact_pw_num_list_recv;
  VecXT<int> bond_entry_pw_num_list_recv;
  VecXT<int> collision_entry_pw_num_list_recv;
 
  VecXT<MPI_Request> particle_req_list_recv;
  VecXT<MPI_Request> particle_prop_keys_req_list_recv;
  VecXT<MPI_Request> particle_prop_vals_req_list_recv;
  VecXT<MPI_Request> contact_pp_req_list_recv;
  VecXT<MPI_Request> contact_pp_prop_keys_req_list_recv;
  VecXT<MPI_Request> contact_pp_prop_vals_req_list_recv;
  VecXT<MPI_Request> bond_entry_pp_req_list_recv;
  VecXT<MPI_Request> collision_entry_pp_req_list_recv;
  VecXT<MPI_Request> contact_pw_req_list_recv;
  VecXT<MPI_Request> contact_pw_prop_keys_req_list_recv;
  VecXT<MPI_Request> contact_pw_prop_vals_req_list_recv;
  VecXT<MPI_Request> bond_entry_pw_req_list_recv;
  VecXT<MPI_Request> collision_entry_pw_req_list_recv;
 
  VecXT<bool> particle_probed_list;
  VecXT<bool> particle_prop_keys_probed_list;
  VecXT<bool> particle_prop_vals_probed_list;
  VecXT<bool> contact_pp_probed_list;
  VecXT<bool> contact_pp_prop_keys_probed_list;
  VecXT<bool> contact_pp_prop_vals_probed_list;
  VecXT<bool> bond_entry_pp_probed_list;
  VecXT<bool> collision_entry_pp_probed_list;
  VecXT<bool> contact_pw_probed_list;
  VecXT<bool> contact_pw_prop_keys_probed_list;
  VecXT<bool> contact_pw_prop_vals_probed_list;
  VecXT<bool> bond_entry_pw_probed_list;
  VecXT<bool> collision_entry_pw_probed_list;
 
  List<int> GetOtherRanks();
  void RemoveParticle(int id, VecXT<Particle *> *p_list);
 
  void ClearBuffer();
 
  void ClearBufferShape();
  void ClearBufferParticle();
  void ClearBufferContact();
};
 
} // namespace netdem