Actual source code: petscdevicetypes.h

  1: #ifndef PETSCDEVICETYPES_H
  2: #define PETSCDEVICETYPES_H

  4: /* for PETSC_HAVE_CUDA/HIP/KOKKOS etc */
  5: #include <petscsys.h>

  7: /*E
  8:   PetscMemType - Memory type of a pointer

 10:   Developer Note:
 11:   Encoding of the bitmask in binary: xxxxyyyz

 13: $ z = 0                - Host memory
 14: $ z = 1                - Device memory
 15: $ yyy = 000            - CUDA-related memory
 16: $ yyy = 001            - HIP-related memory
 17: $ yyy = 010            - SYCL-related memory
 18: $ xxxxyyy1 = 0000,0001 - CUDA memory
 19: $ xxxxyyy1 = 0001,0001 - CUDA NVSHMEM memory
 20: $ xxxxyyy1 = 0000,0011 - HIP memory
 21: $ xxxxyyy1 = 0000,0101 - SYCL memory

 23:   Other types of memory, e.g., CUDA managed memory, can be added when needed.

 25:   Level: beginner

 27:   Notes:
 28:   PETSC_MEMTYPE_KOKKOS depends on the KOKKOS backend configuration

 30: .seealso: VecGetArrayAndMemType(), PetscSFBcastWithMemTypeBegin(), PetscSFReduceWithMemTypeBegin()
 31: E*/
 32: typedef enum {
 33:   PETSC_MEMTYPE_HOST    = 0,
 34:   PETSC_MEMTYPE_DEVICE  = 0x01,
 35:   PETSC_MEMTYPE_CUDA    = 0x01,
 36:   PETSC_MEMTYPE_NVSHMEM = 0x11,
 37:   PETSC_MEMTYPE_HIP     = 0x03,
 38:   PETSC_MEMTYPE_SYCL    = 0x05,
 39: } PetscMemType;
 40: #if PetscDefined(HAVE_CUDA)
 41: # define PETSC_MEMTYPE_KOKKOS PETSC_MEMTYPE_CUDA
 42: #elif PetscDefined(HAVE_HIP)
 43: # define PETSC_MEMTYPE_KOKKOS PETSC_MEMTYPE_HIP
 44: #elif PetscDefined(HAVE_SYCL)
 45: # define PETSC_MEMTYPE_KOKKOS PETSC_MEMTYPE_SYCL
 46: #else
 47: # define PETSC_MEMTYPE_KOKKOS PETSC_MEMTYPE_HOST
 48: #endif

 50: #define PetscMemTypeHost(m)    (((m) & 0x1) == PETSC_MEMTYPE_HOST)
 51: #define PetscMemTypeDevice(m)  (((m) & 0x1) == PETSC_MEMTYPE_DEVICE)
 52: #define PetscMemTypeCUDA(m)    (((m) & 0xF) == PETSC_MEMTYPE_CUDA)
 53: #define PetscMemTypeHIP(m)     (((m) & 0xF) == PETSC_MEMTYPE_HIP)
 54: #define PetscMemTypeSYCL(m)    (((m) & 0xF) == PETSC_MEMTYPE_SYCL)
 55: #define PetscMemTypeNVSHMEM(m) ((m) == PETSC_MEMTYPE_NVSHMEM)

 57: #define PETSC_OFFLOAD_VECKOKKOS_DEPRECATED PETSC_OFFLOAD_VECKOKKOS PETSC_DEPRECATED_ENUM("Use PETSC_OFFLOAD_KOKKOS (since version 3.17.0)")
 58: /*E
 59:   PetscOffloadMask - indicates which memory (CPU, GPU, or none) contains valid data

 61: $ PETSC_OFFLOAD_UNALLOCATED - no memory contains valid matrix entries; NEVER used for vectors
 62: $ PETSC_OFFLOAD_GPU         - GPU has valid vector/matrix entries
 63: $ PETSC_OFFLOAD_CPU         - CPU has valid vector/matrix entries
 64: $ PETSC_OFFLOAD_BOTH        - Both GPU and CPU have valid vector/matrix entries and they match
 65: $ PETSC_OFFLOAD_KOKKOS      - Reserved for Kokkos matrix and vector. It means the offload is managed by Kokkos, thus this flag itself cannot tell you where the valid data is.

 67:   Level: developer
 68: E*/
 69: typedef enum {
 70:   PETSC_OFFLOAD_UNALLOCATED = 0x0,
 71:   PETSC_OFFLOAD_CPU         = 0x1,
 72:   PETSC_OFFLOAD_GPU         = 0x2,
 73:   PETSC_OFFLOAD_BOTH        = 0x3,
 74:   PETSC_OFFLOAD_VECKOKKOS_DEPRECATED = 0x100,
 75:   PETSC_OFFLOAD_KOKKOS      = 0x100
 76: } PetscOffloadMask;

 78: /*E
 79:   PetscDeviceInitType - Initialization strategy for PetscDevice

 81: $ PETSC_DEVICE_INIT_NONE  - PetscDevice is never initialized
 82: $ PETSC_DEVICE_INIT_LAZY  - PetscDevice is initialized on demand
 83: $ PETSC_DEVICE_INIT_EAGER - PetscDevice is initialized as soon as possible

 85:   Notes:
 86:   PETSC_DEVICE_INIT_NONE implies that any initialization of PetscDevice is disallowed and
 87:   doing so results in an error. Useful to ensure that no accelerator is used in a program.

 89:   Level: beginner

 91: .seealso: PetscDevice, PetscDeviceType, PetscDeviceInitialize(), PetscDeviceInitialized(), PetscDeviceCreate()
 92: E*/
 93: typedef enum {
 94:   PETSC_DEVICE_INIT_NONE,
 95:   PETSC_DEVICE_INIT_LAZY,
 96:   PETSC_DEVICE_INIT_EAGER
 97: } PetscDeviceInitType;
 98: PETSC_EXTERN const char *const PetscDeviceInitTypes[];

100: /*E
101:   PetscDeviceType - Kind of accelerator device backend

103: $ PETSC_DEVICE_INVALID - Invalid type, do not use
104: $ PETSC_DEVICE_CUDA    - CUDA enabled GPU
105: $ PETSC_DEVICE_HIP     - ROCM/HIP enabled GPU
106: $ PETSC_DEVICE_SYCL    - SYCL enabled device
107: $ PETSC_DEVICE_DEFAULT - Automatically select backend based on availability
108: $ PETSC_DEVICE_MAX     - Always 1 greater than the largest valid PetscDeviceType, invalid type, do not use

110:   Notes:
111:   PETSC_DEVICE_DEFAULT is selected in the following order: PETSC_DEVICE_HIP, PETSC_DEVICE_CUDA, PETSC_DEVICE_SYCL, PETSC_DEVICE_INVALID.

113:   Level: beginner

115: .seealso: PetscDevice, PetscDeviceInitType, PetscDeviceCreate()
116: E*/
117: typedef enum {
118:   PETSC_DEVICE_INVALID,
119:   PETSC_DEVICE_CUDA,
120:   PETSC_DEVICE_HIP,
121:   PETSC_DEVICE_SYCL,
122:   PETSC_DEVICE_MAX
123: } PetscDeviceType;
124: PETSC_EXTERN const char *const PetscDeviceTypes[];
125: #if defined(PETSC_HAVE_HIP)
126: #  define PETSC_DEVICE_DEFAULT PETSC_DEVICE_HIP
127: #elif defined(PETSC_HAVE_CUDA)
128: #  define PETSC_DEVICE_DEFAULT PETSC_DEVICE_CUDA
129: #elif PetscDefined(HAVE_SYCL)
130: #  define PETSC_DEVICE_DEFAULT PETSC_DEVICE_SYCL
131: #else
132: #  define PETSC_DEVICE_DEFAULT PETSC_DEVICE_INVALID
133: #endif

135: /*S
136:   PetscDevice - Handle to an accelerator "device" (usually a GPU)

138:   Notes:
139:   This object is used to house configuration and state of a device, but does not offer any ability to interact with or
140:   drive device computation. This functionality is facilitated instead by the PetscDeviceContext object.

142:   Level: beginner

144: .seealso: PetscDeviceType, PetscDeviceInitType, PetscDeviceCreate(), PetscDeviceConfigure(), PetscDeviceDestroy(), PetscDeviceContext, PetscDeviceContextSetDevice(), PetscDeviceContextGetDevice()
145: S*/
146: typedef struct _n_PetscDevice *PetscDevice;

148: /*E
149:   PetscStreamType - Stream blocking mode, indicates how a stream implementation will interact with the default "NULL"
150:   stream, which is usually blocking.

152: $ PETSC_STREAM_GLOBAL_BLOCKING    - Alias for NULL stream. Any stream of this type will block the host for all other streams to finish work before starting its operations.
153: $ PETSC_STREAM_DEFAULT_BLOCKING   - Stream will act independent of other streams, but will still be blocked by actions on the NULL stream.
154: $ PETSC_STREAM_GLOBAL_NONBLOCKING - Stream is truly asynchronous, and is blocked by nothing, not even the NULL stream.
155: $ PETSC_STREAM_MAX                - Always 1 greater than the largest PetscStreamType, do not use

157:   Level: intermediate

159: .seealso: PetscDeviceContextSetStreamType(), PetscDeviceContextGetStreamType()
160: E*/
161: typedef enum {
162:   PETSC_STREAM_GLOBAL_BLOCKING,
163:   PETSC_STREAM_DEFAULT_BLOCKING,
164:   PETSC_STREAM_GLOBAL_NONBLOCKING,
165:   PETSC_STREAM_MAX
166: } PetscStreamType;
167: PETSC_EXTERN const char *const PetscStreamTypes[];

169: /*E
170:   PetscDeviceContextJoinMode - Describes the type of join operation to perform in PetscDeviceContextJoin()

172: $ PETSC_DEVICE_CONTEXT_DESTROY - Destroy all incoming sub-contexts after join.
173: $ PETSC_CONTEXT_JOIN_SYNC      - Synchronize incoming sub-contexts after join.
174: $ PETSC_CONTEXT_JOIN_NO_SYNC   - Do not synchronize incoming sub-contexts after join.

176:   Level: beginner

178: .seealso: PetscDeviceContext, PetscDeviceContextFork(), PetscDeviceContextJoin()
179: E*/
180: typedef enum {
181:   PETSC_DEVICE_CONTEXT_JOIN_DESTROY,
182:   PETSC_DEVICE_CONTEXT_JOIN_SYNC,
183:   PETSC_DEVICE_CONTEXT_JOIN_NO_SYNC
184: } PetscDeviceContextJoinMode;
185: PETSC_EXTERN const char *const PetscDeviceContextJoinModes[];

187: /*S
188:   PetscDeviceContext - Container to manage stream dependencies and the various solver handles for asynchronous device compute.

190:   Level: beginner

192: .seealso: PetscDevice, PetscDeviceContextCreate(), PetscDeviceContextSetDevice(), PetscDeviceContextDestroy(),
193: PetscDeviceContextFork(), PetscDeviceContextJoin()
194: S*/
195: typedef struct _n_PetscDeviceContext *PetscDeviceContext;
196: #endif /* PETSCDEVICETYPES_H */