Actual source code: petscdevicetypes.h
1: #ifndef PETSCDEVICETYPES_H
2: #define PETSCDEVICETYPES_H
4: /* for PETSC_HAVE_CUDA/HIP/KOKKOS etc */
5: #include <petscsys.h>
7: /*E
8: PetscMemType - Memory type of a pointer
10: Developer Note:
11: Encoding of the bitmask in binary: xxxxyyyz
13: $ z = 0 - Host memory
14: $ z = 1 - Device memory
15: $ yyy = 000 - CUDA-related memory
16: $ yyy = 001 - HIP-related memory
17: $ yyy = 010 - SYCL-related memory
18: $ xxxxyyy1 = 0000,0001 - CUDA memory
19: $ xxxxyyy1 = 0001,0001 - CUDA NVSHMEM memory
20: $ xxxxyyy1 = 0000,0011 - HIP memory
21: $ xxxxyyy1 = 0000,0101 - SYCL memory
23: Other types of memory, e.g., CUDA managed memory, can be added when needed.
25: Level: beginner
27: Notes:
28: PETSC_MEMTYPE_KOKKOS depends on the KOKKOS backend configuration
30: .seealso: VecGetArrayAndMemType(), PetscSFBcastWithMemTypeBegin(), PetscSFReduceWithMemTypeBegin()
31: E*/
32: typedef enum {
33: PETSC_MEMTYPE_HOST = 0,
34: PETSC_MEMTYPE_DEVICE = 0x01,
35: PETSC_MEMTYPE_CUDA = 0x01,
36: PETSC_MEMTYPE_NVSHMEM = 0x11,
37: PETSC_MEMTYPE_HIP = 0x03,
38: PETSC_MEMTYPE_SYCL = 0x05,
39: } PetscMemType;
40: #if PetscDefined(HAVE_CUDA)
41: # define PETSC_MEMTYPE_KOKKOS PETSC_MEMTYPE_CUDA
42: #elif PetscDefined(HAVE_HIP)
43: # define PETSC_MEMTYPE_KOKKOS PETSC_MEMTYPE_HIP
44: #elif PetscDefined(HAVE_SYCL)
45: # define PETSC_MEMTYPE_KOKKOS PETSC_MEMTYPE_SYCL
46: #else
47: # define PETSC_MEMTYPE_KOKKOS PETSC_MEMTYPE_HOST
48: #endif
50: #define PetscMemTypeHost(m) (((m) & 0x1) == PETSC_MEMTYPE_HOST)
51: #define PetscMemTypeDevice(m) (((m) & 0x1) == PETSC_MEMTYPE_DEVICE)
52: #define PetscMemTypeCUDA(m) (((m) & 0xF) == PETSC_MEMTYPE_CUDA)
53: #define PetscMemTypeHIP(m) (((m) & 0xF) == PETSC_MEMTYPE_HIP)
54: #define PetscMemTypeSYCL(m) (((m) & 0xF) == PETSC_MEMTYPE_SYCL)
55: #define PetscMemTypeNVSHMEM(m) ((m) == PETSC_MEMTYPE_NVSHMEM)
57: #define PETSC_OFFLOAD_VECKOKKOS_DEPRECATED PETSC_OFFLOAD_VECKOKKOS PETSC_DEPRECATED_ENUM("Use PETSC_OFFLOAD_KOKKOS (since version 3.17.0)")
58: /*E
59: PetscOffloadMask - indicates which memory (CPU, GPU, or none) contains valid data
61: $ PETSC_OFFLOAD_UNALLOCATED - no memory contains valid matrix entries; NEVER used for vectors
62: $ PETSC_OFFLOAD_GPU - GPU has valid vector/matrix entries
63: $ PETSC_OFFLOAD_CPU - CPU has valid vector/matrix entries
64: $ PETSC_OFFLOAD_BOTH - Both GPU and CPU have valid vector/matrix entries and they match
65: $ PETSC_OFFLOAD_KOKKOS - Reserved for Kokkos matrix and vector. It means the offload is managed by Kokkos, thus this flag itself cannot tell you where the valid data is.
67: Level: developer
68: E*/
69: typedef enum {
70: PETSC_OFFLOAD_UNALLOCATED = 0x0,
71: PETSC_OFFLOAD_CPU = 0x1,
72: PETSC_OFFLOAD_GPU = 0x2,
73: PETSC_OFFLOAD_BOTH = 0x3,
74: PETSC_OFFLOAD_VECKOKKOS_DEPRECATED = 0x100,
75: PETSC_OFFLOAD_KOKKOS = 0x100
76: } PetscOffloadMask;
78: /*E
79: PetscDeviceInitType - Initialization strategy for PetscDevice
81: $ PETSC_DEVICE_INIT_NONE - PetscDevice is never initialized
82: $ PETSC_DEVICE_INIT_LAZY - PetscDevice is initialized on demand
83: $ PETSC_DEVICE_INIT_EAGER - PetscDevice is initialized as soon as possible
85: Notes:
86: PETSC_DEVICE_INIT_NONE implies that any initialization of PetscDevice is disallowed and
87: doing so results in an error. Useful to ensure that no accelerator is used in a program.
89: Level: beginner
91: .seealso: PetscDevice, PetscDeviceType, PetscDeviceInitialize(), PetscDeviceInitialized(), PetscDeviceCreate()
92: E*/
93: typedef enum {
94: PETSC_DEVICE_INIT_NONE,
95: PETSC_DEVICE_INIT_LAZY,
96: PETSC_DEVICE_INIT_EAGER
97: } PetscDeviceInitType;
98: PETSC_EXTERN const char *const PetscDeviceInitTypes[];
100: /*E
101: PetscDeviceType - Kind of accelerator device backend
103: $ PETSC_DEVICE_INVALID - Invalid type, do not use
104: $ PETSC_DEVICE_CUDA - CUDA enabled GPU
105: $ PETSC_DEVICE_HIP - ROCM/HIP enabled GPU
106: $ PETSC_DEVICE_SYCL - SYCL enabled device
107: $ PETSC_DEVICE_DEFAULT - Automatically select backend based on availability
108: $ PETSC_DEVICE_MAX - Always 1 greater than the largest valid PetscDeviceType, invalid type, do not use
110: Notes:
111: PETSC_DEVICE_DEFAULT is selected in the following order: PETSC_DEVICE_HIP, PETSC_DEVICE_CUDA, PETSC_DEVICE_SYCL, PETSC_DEVICE_INVALID.
113: Level: beginner
115: .seealso: PetscDevice, PetscDeviceInitType, PetscDeviceCreate()
116: E*/
117: typedef enum {
118: PETSC_DEVICE_INVALID,
119: PETSC_DEVICE_CUDA,
120: PETSC_DEVICE_HIP,
121: PETSC_DEVICE_SYCL,
122: PETSC_DEVICE_MAX
123: } PetscDeviceType;
124: PETSC_EXTERN const char *const PetscDeviceTypes[];
125: #if defined(PETSC_HAVE_HIP)
126: # define PETSC_DEVICE_DEFAULT PETSC_DEVICE_HIP
127: #elif defined(PETSC_HAVE_CUDA)
128: # define PETSC_DEVICE_DEFAULT PETSC_DEVICE_CUDA
129: #elif PetscDefined(HAVE_SYCL)
130: # define PETSC_DEVICE_DEFAULT PETSC_DEVICE_SYCL
131: #else
132: # define PETSC_DEVICE_DEFAULT PETSC_DEVICE_INVALID
133: #endif
135: /*S
136: PetscDevice - Handle to an accelerator "device" (usually a GPU)
138: Notes:
139: This object is used to house configuration and state of a device, but does not offer any ability to interact with or
140: drive device computation. This functionality is facilitated instead by the PetscDeviceContext object.
142: Level: beginner
144: .seealso: PetscDeviceType, PetscDeviceInitType, PetscDeviceCreate(), PetscDeviceConfigure(), PetscDeviceDestroy(), PetscDeviceContext, PetscDeviceContextSetDevice(), PetscDeviceContextGetDevice()
145: S*/
146: typedef struct _n_PetscDevice *PetscDevice;
148: /*E
149: PetscStreamType - Stream blocking mode, indicates how a stream implementation will interact with the default "NULL"
150: stream, which is usually blocking.
152: $ PETSC_STREAM_GLOBAL_BLOCKING - Alias for NULL stream. Any stream of this type will block the host for all other streams to finish work before starting its operations.
153: $ PETSC_STREAM_DEFAULT_BLOCKING - Stream will act independent of other streams, but will still be blocked by actions on the NULL stream.
154: $ PETSC_STREAM_GLOBAL_NONBLOCKING - Stream is truly asynchronous, and is blocked by nothing, not even the NULL stream.
155: $ PETSC_STREAM_MAX - Always 1 greater than the largest PetscStreamType, do not use
157: Level: intermediate
159: .seealso: PetscDeviceContextSetStreamType(), PetscDeviceContextGetStreamType()
160: E*/
161: typedef enum {
162: PETSC_STREAM_GLOBAL_BLOCKING,
163: PETSC_STREAM_DEFAULT_BLOCKING,
164: PETSC_STREAM_GLOBAL_NONBLOCKING,
165: PETSC_STREAM_MAX
166: } PetscStreamType;
167: PETSC_EXTERN const char *const PetscStreamTypes[];
169: /*E
170: PetscDeviceContextJoinMode - Describes the type of join operation to perform in PetscDeviceContextJoin()
172: $ PETSC_DEVICE_CONTEXT_DESTROY - Destroy all incoming sub-contexts after join.
173: $ PETSC_CONTEXT_JOIN_SYNC - Synchronize incoming sub-contexts after join.
174: $ PETSC_CONTEXT_JOIN_NO_SYNC - Do not synchronize incoming sub-contexts after join.
176: Level: beginner
178: .seealso: PetscDeviceContext, PetscDeviceContextFork(), PetscDeviceContextJoin()
179: E*/
180: typedef enum {
181: PETSC_DEVICE_CONTEXT_JOIN_DESTROY,
182: PETSC_DEVICE_CONTEXT_JOIN_SYNC,
183: PETSC_DEVICE_CONTEXT_JOIN_NO_SYNC
184: } PetscDeviceContextJoinMode;
185: PETSC_EXTERN const char *const PetscDeviceContextJoinModes[];
187: /*S
188: PetscDeviceContext - Container to manage stream dependencies and the various solver handles for asynchronous device compute.
190: Level: beginner
192: .seealso: PetscDevice, PetscDeviceContextCreate(), PetscDeviceContextSetDevice(), PetscDeviceContextDestroy(),
193: PetscDeviceContextFork(), PetscDeviceContextJoin()
194: S*/
195: typedef struct _n_PetscDeviceContext *PetscDeviceContext;
196: #endif /* PETSCDEVICETYPES_H */