DOLFINx
DOLFINx C++ interface
IndexMap.h
1 // Copyright (C) 2015-2019 Chris Richardson, Garth N. Wells and Igor Baratta
2 //
3 // This file is part of DOLFINx (https://www.fenicsproject.org)
4 //
5 // SPDX-License-Identifier: LGPL-3.0-or-later
6 
7 #pragma once
8 
9 #include <array>
10 #include <cstdint>
11 #include <dolfinx/common/MPI.h>
12 #include <dolfinx/graph/AdjacencyList.h>
13 #include <map>
14 #include <memory>
15 #include <tuple>
16 #include <utility>
17 #include <vector>
18 #include <xtl/xspan.hpp>
19 
20 namespace dolfinx::common
21 {
22 // Forward declaration
23 class IndexMap;
24 
34 std::tuple<std::int64_t, std::vector<std::int32_t>,
35  std::vector<std::vector<std::int64_t>>,
36  std::vector<std::vector<int>>>
38  const std::vector<
39  std::pair<std::reference_wrapper<const common::IndexMap>, int>>& maps);
40 
47 
48 class IndexMap
49 {
50 public:
52  enum class Mode
53  {
54  insert,
55  add
56  };
57 
59  enum class Direction
60  {
61  reverse, // Ghost to owner
62  forward, // Owner to ghost
63  symmetric // Symmetric. NOTE: To be removed
64  };
65 
73  IndexMap(MPI_Comm comm, std::int32_t local_size);
74 
87  IndexMap(MPI_Comm mpi_comm, std::int32_t local_size,
88  const std::vector<int>& dest_ranks,
89  const std::vector<std::int64_t>& ghosts,
90  const std::vector<int>& src_ranks);
91 
92  // Copy constructor
93  IndexMap(const IndexMap& map) = delete;
94 
96  IndexMap(IndexMap&& map) = default;
97 
99  ~IndexMap() = default;
100 
102  IndexMap& operator=(IndexMap&& map) = default;
103 
104  // Copy assignment
105  IndexMap& operator=(const IndexMap& map) = delete;
106 
108  std::array<std::int64_t, 2> local_range() const noexcept;
109 
111  std::int32_t num_ghosts() const noexcept;
112 
114  std::int32_t size_local() const noexcept;
115 
117  std::int64_t size_global() const noexcept;
118 
121  const std::vector<std::int64_t>& ghosts() const noexcept;
122 
128  MPI_Comm comm(Direction dir = Direction::symmetric) const;
129 
133  void local_to_global(const xtl::span<const std::int32_t>& local,
134  const xtl::span<std::int64_t>& global) const;
135 
140  void global_to_local(const xtl::span<const std::int64_t>& global,
141  const xtl::span<std::int32_t>& local) const;
142 
146  std::vector<std::int64_t> global_indices() const;
147 
154  const graph::AdjacencyList<std::int32_t>& shared_indices() const noexcept;
155 
157  std::vector<int> ghost_owner_rank() const;
158 
165  std::map<std::int32_t, std::set<int>> compute_shared_indices() const;
166 
177  template <typename T>
178  void scatter_fwd(xtl::span<const T> local_data, xtl::span<T> remote_data,
179  int n) const;
180 
190  template <typename T>
191  void scatter_rev(xtl::span<T> local_data, xtl::span<const T> remote_data,
192  int n, IndexMap::Mode op) const;
193 
194 private:
195  // Range of indices (global) owned by this process
196  std::array<std::int64_t, 2> _local_range;
197 
198  // Number indices across communicator
199  std::int64_t _size_global;
200 
201  // MPI neighborhood communicators
202 
203  // Communicator where the source ranks own the indices in the callers
204  // halo, and the destination ranks 'ghost' indices owned by the
205  // caller. I.e.,
206  // - in-edges (src) are from ranks that own my ghosts
207  // - out-edges (dest) go to ranks that 'ghost' my owned indices
208  dolfinx::MPI::Comm _comm_owner_to_ghost;
209 
210  // Communicator where the source ranks have ghost indices that are
211  // owned by the caller, and the destination ranks are the owners of
212  // indices in the callers halo region. I.e.,
213  // - in-edges (src) are from ranks that 'ghost' my owned indicies
214  // - out-edges (dest) are to the owning ranks of my ghost indices
215  dolfinx::MPI::Comm _comm_ghost_to_owner;
216 
217  // TODO: remove
218  dolfinx::MPI::Comm _comm_symmetric;
219 
220  // Local-to-global map for ghost indices
221  std::vector<std::int64_t> _ghosts;
222 
223  // Owning neighborhood rank (out edge) on '_comm_owner_to_ghost'
224  // communicator for each ghost index
225  std::vector<std::int32_t> _ghost_owners;
226 
227  // List of owned local indices that are in the halo (ghost) region on other
228  // ranks, grouped by rank in the neighbor communicator (destination ranks in
229  // forward communicator and source ranks in the reverse communicator).
230  std::unique_ptr<graph::AdjacencyList<std::int32_t>> _shared_indices;
231 };
232 
233 } // namespace dolfinx::common
This class provides utility functions for easy communication with MPI and handles cases when DOLFINx ...
Definition: MPI.h:31
This class represents the distribution index arrays across processes. An index array is a contiguous ...
Definition: IndexMap.h:49
std::map< std::int32_t, std::set< int > > compute_shared_indices() const
Definition: IndexMap.cpp:546
MPI_Comm comm(Direction dir=Direction::symmetric) const
Return a MPI communicator with attached distributed graph topology information.
Definition: IndexMap.cpp:531
std::vector< int > ghost_owner_rank() const
Owner rank (on global communicator) of each ghost entry.
Definition: IndexMap.cpp:514
void global_to_local(const xtl::span< const std::int64_t > &global, const xtl::span< std::int32_t > &local) const
Compute local indices for array of global indices.
Definition: IndexMap.cpp:469
~IndexMap()=default
Destructor.
std::array< std::int64_t, 2 > local_range() const noexcept
Range of indices (global) owned by this process.
Definition: IndexMap.cpp:433
Mode
Mode for reverse scatter operation.
Definition: IndexMap.h:53
IndexMap(IndexMap &&map)=default
Move constructor.
Direction
Edge directions of neighborhood communicator.
Definition: IndexMap.h:60
std::int64_t size_global() const noexcept
Number indices across communicator.
Definition: IndexMap.cpp:445
std::vector< std::int64_t > global_indices() const
Global indices.
Definition: IndexMap.cpp:495
IndexMap(MPI_Comm comm, std::int32_t local_size)
Create an non-overlapping index map with local_size owned on this process.
Definition: IndexMap.cpp:307
std::int32_t num_ghosts() const noexcept
Number of ghost indices on this process.
Definition: IndexMap.cpp:438
IndexMap & operator=(IndexMap &&map)=default
Move assignment.
void scatter_rev(xtl::span< T > local_data, xtl::span< const T > remote_data, int n, IndexMap::Mode op) const
Send n values for each ghost index to owning to the process.
Definition: IndexMap.cpp:742
void scatter_fwd(xtl::span< const T > local_data, xtl::span< T > remote_data, int n) const
Send n values for each index that is owned to processes that have the index as a ghost....
Definition: IndexMap.cpp:659
const graph::AdjacencyList< std::int32_t > & shared_indices() const noexcept
Definition: IndexMap.cpp:509
const std::vector< std::int64_t > & ghosts() const noexcept
Local-to-global map for ghosts (local indexing beyond end of local range)
Definition: IndexMap.cpp:447
void local_to_global(const xtl::span< const std::int32_t > &local, const xtl::span< std::int64_t > &global) const
Compute global indices for array of local indices.
Definition: IndexMap.cpp:452
std::int32_t size_local() const noexcept
Number of indices owned by on this process.
Definition: IndexMap.cpp:440
Miscellaneous classes, functions and types.
std::tuple< std::int64_t, std::vector< std::int32_t >, std::vector< std::vector< std::int64_t > >, std::vector< std::vector< int > > > stack_index_maps(const std::vector< std::pair< std::reference_wrapper< const common::IndexMap >, int >> &maps)
Compute layout data and ghost indices for a stacked (concatenated) index map, i.e....
Definition: IndexMap.cpp:190