Actual source code: plexland.c

  1: #include <../src/mat/impls/aij/seq/aij.h>
  2: #include <petsc/private/dmpleximpl.h>
  3: #include <petsclandau.h>
  4: #include <petscts.h>
  5: #include <petscdmforest.h>
  6: #include <petscdmcomposite.h>

  8: /* Landau collision operator */

 10: /* relativistic terms */
 11: #if defined(PETSC_USE_REAL_SINGLE)
 12: #define SPEED_OF_LIGHT 2.99792458e8F
 13: #define C_0(v0) (SPEED_OF_LIGHT/v0) /* needed for relativistic tensor on all architectures */
 14: #else
 15: #define SPEED_OF_LIGHT 2.99792458e8
 16: #define C_0(v0) (SPEED_OF_LIGHT/v0) /* needed for relativistic tensor on all architectures */
 17: #endif

 19: #define PETSC_THREAD_SYNC
 20: #include "land_tensors.h"

 22: #if defined(PETSC_HAVE_OPENMP)
 23: #include <omp.h>
 24: #endif

 26: /* vector padding not supported */
 27: #define LANDAU_VL  1

 29: static PetscErrorCode LandauMatMult(Mat A, Vec x, Vec y)
 30: {
 31:   LandauCtx       *ctx;
 32:   PetscContainer  container;

 34:   PetscObjectQuery((PetscObject) A, "LandauCtx", (PetscObject *) &container);
 35:   if (container) {
 36:     PetscContainerGetPointer(container, (void **) &ctx);
 37:     VecScatterBegin(ctx->plex_batch,x,ctx->work_vec,INSERT_VALUES,SCATTER_FORWARD);
 38:     VecScatterEnd(ctx->plex_batch,x,ctx->work_vec,INSERT_VALUES,SCATTER_FORWARD);
 39:     (*ctx->seqaij_mult)(A,ctx->work_vec,y);
 40:     VecCopy(y, ctx->work_vec);
 41:     VecScatterBegin(ctx->plex_batch,ctx->work_vec,y,INSERT_VALUES,SCATTER_REVERSE);
 42:     VecScatterEnd(ctx->plex_batch,ctx->work_vec,y,INSERT_VALUES,SCATTER_REVERSE);
 43:     return 0;
 44:   }
 45:   MatMult(A,x,y);
 46:   return 0;
 47: }

 49: // Computes v3 = v2 + A * v1.
 50: static PetscErrorCode LandauMatMultAdd(Mat A,Vec v1,Vec v2,Vec v3)
 51: {
 52:   SETERRQ(PETSC_COMM_SELF, PETSC_ERR_PLIB, "?????");
 53:   LandauMatMult(A,v1,v3);
 54:   VecAYPX(v3,1,v2);
 55:   return 0;
 56: }

 58: static PetscErrorCode LandauMatMultTranspose(Mat A, Vec x, Vec y)
 59: {
 60:   LandauCtx       *ctx;
 61:   PetscContainer  container;

 63:   PetscObjectQuery((PetscObject) A, "LandauCtx", (PetscObject *) &container);
 64:   if (container) {
 65:     PetscContainerGetPointer(container, (void **) &ctx);
 66:     VecScatterBegin(ctx->plex_batch,x,ctx->work_vec,INSERT_VALUES,SCATTER_FORWARD);
 67:     VecScatterEnd(ctx->plex_batch,x,ctx->work_vec,INSERT_VALUES,SCATTER_FORWARD);
 68:     (*ctx->seqaij_multtranspose)(A,ctx->work_vec,y);
 69:     VecCopy(y, ctx->work_vec);
 70:     VecScatterBegin(ctx->plex_batch,ctx->work_vec,y,INSERT_VALUES,SCATTER_REVERSE);
 71:     VecScatterEnd(ctx->plex_batch,ctx->work_vec,y,INSERT_VALUES,SCATTER_REVERSE);
 72:     return 0;
 73:   }
 74:   MatMultTranspose(A,x,y);
 75:   return 0;
 76: }

 78: static PetscErrorCode LandauMatGetDiagonal(Mat A,Vec x)
 79: {
 80:   LandauCtx       *ctx;
 81:   PetscContainer  container;

 83:   PetscObjectQuery((PetscObject) A, "LandauCtx", (PetscObject *) &container);
 84:   if (container) {
 85:     PetscContainerGetPointer(container, (void **) &ctx);
 86:     (*ctx->seqaij_getdiagonal)(A,ctx->work_vec);
 87:     VecScatterBegin(ctx->plex_batch,ctx->work_vec,x,INSERT_VALUES,SCATTER_REVERSE);
 88:     VecScatterEnd(ctx->plex_batch,ctx->work_vec,x,INSERT_VALUES,SCATTER_REVERSE);
 89:     return 0;
 90:   }
 91:   MatGetDiagonal(A, x);
 92:   return 0;
 93: }

 95: static PetscErrorCode LandauGPUMapsDestroy(void *ptr)
 96: {
 97:   P4estVertexMaps *maps = (P4estVertexMaps*)ptr;
 98:   // free device data
 99:   if (maps[0].deviceType != LANDAU_CPU) {
100: #if defined(PETSC_HAVE_KOKKOS_KERNELS)
101:     if (maps[0].deviceType == LANDAU_KOKKOS) {
102:       LandauKokkosDestroyMatMaps(maps,  maps[0].numgrids); // imples Kokkos does
103:     } // else could be CUDA
104: #elif defined(PETSC_HAVE_CUDA)
105:     if (maps[0].deviceType == LANDAU_CUDA) {
106:       LandauCUDADestroyMatMaps(maps, maps[0].numgrids);
107:     } else SETERRQ(PETSC_COMM_SELF, PETSC_ERR_PLIB, "maps->deviceType %" PetscInt_FMT " ?????",maps->deviceType);
108: #endif
109:   }
110:   // free host data
111:   for (PetscInt grid=0 ; grid < maps[0].numgrids ; grid++) {
112:     PetscFree(maps[grid].c_maps);
113:     PetscFree(maps[grid].gIdx);
114:   }
115:   PetscFree(maps);

117:   return 0;
118: }
119: static PetscErrorCode energy_f(PetscInt dim, PetscReal time, const PetscReal x[], PetscInt Nf_dummy, PetscScalar *u, void *actx)
120: {
121:   PetscReal     v2 = 0;
122:   /* compute v^2 / 2 */
123:   for (int i = 0; i < dim; ++i) v2 += x[i]*x[i];
124:   /* evaluate the Maxwellian */
125:   u[0] = v2/2;
126:   return 0;
127: }

129: /* needs double */
130: static PetscErrorCode gamma_m1_f(PetscInt dim, PetscReal time, const PetscReal x[], PetscInt Nf_dummy, PetscScalar *u, void *actx)
131: {
132:   PetscReal     *c2_0_arr = ((PetscReal*)actx);
133:   double        u2 = 0, c02 = (double)*c2_0_arr, xx;

135:   /* compute u^2 / 2 */
136:   for (int i = 0; i < dim; ++i) u2 += x[i]*x[i];
137:   /* gamma - 1 = g_eps, for conditioning and we only take derivatives */
138:   xx = u2/c02;
139: #if defined(PETSC_USE_DEBUG)
140:   u[0] = PetscSqrtReal(1. + xx);
141: #else
142:   u[0] = xx/(PetscSqrtReal(1. + xx) + 1.) - 1.; // better conditioned. -1 might help condition and only used for derivative
143: #endif
144:   return 0;
145: }

147: /*
148:  LandauFormJacobian_Internal - Evaluates Jacobian matrix.

150:  Input Parameters:
151:  .  globX - input vector
152:  .  actx - optional user-defined context
153:  .  dim - dimension

155:  Output Parameters:
156:  .  J0acP - Jacobian matrix filled, not created
157:  */
158: static PetscErrorCode LandauFormJacobian_Internal(Vec a_X, Mat JacP, const PetscInt dim, PetscReal shift, void *a_ctx)
159: {
160:   LandauCtx         *ctx = (LandauCtx*)a_ctx;
161:   PetscInt          numCells[LANDAU_MAX_GRIDS],Nq,Nb;
162:   PetscQuadrature   quad;
163:   PetscReal         Eq_m[LANDAU_MAX_SPECIES]; // could be static data w/o quench (ex2)
164:   PetscScalar       *cellClosure=NULL;
165:   const PetscScalar *xdata=NULL;
166:   PetscDS           prob;
167:   PetscContainer    container;
168:   P4estVertexMaps   *maps;
169:   Mat               subJ[LANDAU_MAX_GRIDS*LANDAU_MAX_BATCH_SZ];

174:   /* check for matrix container for GPU assembly. Support CPU assembly for debugging */
176:   PetscLogEventBegin(ctx->events[10],0,0,0,0);
177:   DMGetDS(ctx->plex[0], &prob); // same DS for all grids
178:   PetscObjectQuery((PetscObject) JacP, "assembly_maps", (PetscObject *) &container);
179:   if (container) {
181:     PetscContainerGetPointer(container, (void **) &maps);
183:     for (PetscInt i=0;i<ctx->num_grids*ctx->batch_sz;i++) subJ[i] = NULL;
184:   } else {
186:     for (PetscInt tid=0 ; tid<ctx->batch_sz ; tid++) {
187:       for (PetscInt grid=0;grid<ctx->num_grids;grid++) {
188:         DMCreateMatrix(ctx->plex[grid], &subJ[ LAND_PACK_IDX(tid,grid) ]);
189:       }
190:     }
191:     maps = NULL;
192:   }
193:   // get dynamic data (Eq is odd, for quench and Spitzer test) for CPU assembly and raw data for Jacobian GPU assembly. Get host numCells[], Nq (yuck)
194:   PetscFEGetQuadrature(ctx->fe[0], &quad);
195:   PetscQuadratureGetData(quad, NULL, NULL, &Nq, NULL, NULL); Nb = Nq;
197:   // get metadata for collecting dynamic data
198:   for (PetscInt grid=0;grid<ctx->num_grids;grid++) {
199:     PetscInt cStart, cEnd;
201:     DMPlexGetHeightStratum(ctx->plex[grid], 0, &cStart, &cEnd);
202:     numCells[grid] = cEnd - cStart; // grids can have different topology
203:   }
204:   PetscLogEventEnd(ctx->events[10],0,0,0,0);
205:   if (shift==0) { /* create dynamic point data: f_alpha for closure of each cell (cellClosure[nbatch,ngrids,ncells[g],f[Nb,ns[g]]]) or xdata */
206:     DM pack;
207:     VecGetDM(a_X, &pack);
209:     PetscLogEventBegin(ctx->events[1],0,0,0,0);
210:     MatZeroEntries(JacP);
211:     for (PetscInt fieldA=0;fieldA<ctx->num_species;fieldA++) {
212:       Eq_m[fieldA] = ctx->Ez * ctx->t_0 * ctx->charges[fieldA] / (ctx->v_0 * ctx->masses[fieldA]); /* normalize dimensionless */
213:       if (dim==2) Eq_m[fieldA] *=  2 * PETSC_PI; /* add the 2pi term that is not in Landau */
214:     }
215:     if (!ctx->gpu_assembly) {
216:       Vec          *locXArray,*globXArray;
217:       PetscScalar  *cellClosure_it;
218:       PetscInt     cellClosure_sz=0,nDMs,Nf[LANDAU_MAX_GRIDS];
219:       PetscSection section[LANDAU_MAX_GRIDS],globsection[LANDAU_MAX_GRIDS];
220:       for (PetscInt grid=0;grid<ctx->num_grids;grid++) {
221:         DMGetLocalSection(ctx->plex[grid], &section[grid]);
222:         DMGetGlobalSection(ctx->plex[grid], &globsection[grid]);
223:         PetscSectionGetNumFields(section[grid], &Nf[grid]);
224:       }
225:       /* count cellClosure size */
226:       DMCompositeGetNumberDM(pack,&nDMs);
227:       for (PetscInt grid=0 ; grid<ctx->num_grids ; grid++) cellClosure_sz += Nb*Nf[grid]*numCells[grid];
228:       PetscMalloc1(cellClosure_sz*ctx->batch_sz,&cellClosure);
229:       cellClosure_it = cellClosure;
230:       PetscMalloc(sizeof(*locXArray)*nDMs, &locXArray);
231:       PetscMalloc(sizeof(*globXArray)*nDMs, &globXArray);
232:       DMCompositeGetLocalAccessArray(pack, a_X, nDMs, NULL, locXArray);
233:       DMCompositeGetAccessArray(pack, a_X, nDMs, NULL, globXArray);
234:       for (PetscInt b_id = 0 ; b_id < ctx->batch_sz ; b_id++) { // OpenMP (once)
235:         for (PetscInt grid=0 ; grid<ctx->num_grids ; grid++) {
236:           Vec         locX = locXArray[ LAND_PACK_IDX(b_id,grid) ], globX = globXArray[ LAND_PACK_IDX(b_id,grid) ], locX2;
237:           PetscInt    cStart, cEnd, ei;
238:           VecDuplicate(locX,&locX2);
239:           DMGlobalToLocalBegin(ctx->plex[grid], globX, INSERT_VALUES, locX2);
240:           DMGlobalToLocalEnd  (ctx->plex[grid], globX, INSERT_VALUES, locX2);
241:           DMPlexGetHeightStratum(ctx->plex[grid], 0, &cStart, &cEnd);
242:           for (ei = cStart ; ei < cEnd; ++ei) {
243:             PetscScalar *coef = NULL;
244:             DMPlexVecGetClosure(ctx->plex[grid], section[grid], locX2, ei, NULL, &coef);
245:             PetscMemcpy(cellClosure_it,coef,Nb*Nf[grid]*sizeof(*cellClosure_it)); /* change if LandauIPReal != PetscScalar */
246:             DMPlexVecRestoreClosure(ctx->plex[grid], section[grid], locX2, ei, NULL, &coef);
247:             cellClosure_it += Nb*Nf[grid];
248:           }
249:           VecDestroy(&locX2);
250:         }
251:       }
253:       DMCompositeRestoreLocalAccessArray(pack, a_X, nDMs, NULL, locXArray);
254:       DMCompositeRestoreAccessArray(pack, a_X, nDMs, NULL, globXArray);
255:       PetscFree(locXArray);
256:       PetscFree(globXArray);
257:       xdata = NULL;
258:     } else {
259:       PetscMemType mtype;
260:       if (ctx->jacobian_field_major_order) { // get data in batch ordering
261:         VecScatterBegin(ctx->plex_batch,a_X,ctx->work_vec,INSERT_VALUES,SCATTER_FORWARD);
262:         VecScatterEnd(ctx->plex_batch,a_X,ctx->work_vec,INSERT_VALUES,SCATTER_FORWARD);
263:         VecGetArrayReadAndMemType(ctx->work_vec,&xdata,&mtype);
264:       } else {
265:         VecGetArrayReadAndMemType(a_X,&xdata,&mtype);
266:       }
267:       if (mtype!=PETSC_MEMTYPE_HOST && ctx->deviceType == LANDAU_CPU) {
268:         SETERRQ(ctx->comm,PETSC_ERR_ARG_WRONG,"CPU run with device data: use -mat_type aij");
269:       }
270:       cellClosure = NULL;
271:     }
272:     PetscLogEventEnd(ctx->events[1],0,0,0,0);
273:   } else xdata = cellClosure = NULL;

275:   /* do it */
276:   if (ctx->deviceType == LANDAU_CUDA || ctx->deviceType == LANDAU_KOKKOS) {
277:     if (ctx->deviceType == LANDAU_CUDA) {
278: #if defined(PETSC_HAVE_CUDA)
279:       LandauCUDAJacobian(ctx->plex,Nq,ctx->batch_sz,ctx->num_grids,numCells,Eq_m,cellClosure,xdata,&ctx->SData_d,shift,ctx->events,ctx->mat_offset, ctx->species_offset, subJ, JacP);
280: #else
281:       SETERRQ(ctx->comm,PETSC_ERR_ARG_WRONG,"-landau_device_type %s not built","cuda");
282: #endif
283:     } else if (ctx->deviceType == LANDAU_KOKKOS) {
284: #if defined(PETSC_HAVE_KOKKOS_KERNELS)
285:       LandauKokkosJacobian(ctx->plex,Nq,ctx->batch_sz,ctx->num_grids,numCells,Eq_m,cellClosure,xdata,&ctx->SData_d,shift,ctx->events,ctx->mat_offset, ctx->species_offset, subJ,JacP);
286: #else
287:       SETERRQ(ctx->comm,PETSC_ERR_ARG_WRONG,"-landau_device_type %s not built","kokkos");
288: #endif
289:     }
290:   } else {   /* CPU version */
291:     PetscTabulation *Tf; // used for CPU and print info. Same on all grids and all species
292:     PetscInt        ip_offset[LANDAU_MAX_GRIDS+1], ipf_offset[LANDAU_MAX_GRIDS+1], elem_offset[LANDAU_MAX_GRIDS+1],IPf_sz_glb,IPf_sz_tot,num_grids=ctx->num_grids,Nf[LANDAU_MAX_GRIDS];
293:     PetscReal       *ff, *dudx, *dudy, *dudz, *invJ_a = (PetscReal*)ctx->SData_d.invJ, *xx = (PetscReal*)ctx->SData_d.x, *yy = (PetscReal*)ctx->SData_d.y, *zz = (PetscReal*)ctx->SData_d.z, *ww = (PetscReal*)ctx->SData_d.w;
294:     PetscReal       Eq_m[LANDAU_MAX_SPECIES], invMass[LANDAU_MAX_SPECIES], nu_alpha[LANDAU_MAX_SPECIES], nu_beta[LANDAU_MAX_SPECIES];
295:     PetscSection    section[LANDAU_MAX_GRIDS],globsection[LANDAU_MAX_GRIDS];
296:     PetscScalar     *coo_vals=NULL;
297:     for (PetscInt grid=0;grid<ctx->num_grids;grid++) {
298:       DMGetLocalSection(ctx->plex[grid], &section[grid]);
299:       DMGetGlobalSection(ctx->plex[grid], &globsection[grid]);
300:       PetscSectionGetNumFields(section[grid], &Nf[grid]);
301:     }
302:     /* count IPf size, etc */
303:     PetscDSGetTabulation(prob, &Tf); // Bf, &Df same for all grids
304:     const PetscReal *const BB = Tf[0]->T[0], * const DD = Tf[0]->T[1];
305:     ip_offset[0] = ipf_offset[0] = elem_offset[0] = 0;
306:     for (PetscInt grid=0 ; grid<num_grids ; grid++) {
307:       PetscInt nfloc = ctx->species_offset[grid+1] - ctx->species_offset[grid];
308:       elem_offset[grid+1] = elem_offset[grid] + numCells[grid];
309:       ip_offset[grid+1]   = ip_offset[grid]   + numCells[grid]*Nq;
310:       ipf_offset[grid+1]  = ipf_offset[grid]  + Nq*nfloc*numCells[grid];
311:     }
312:     IPf_sz_glb = ipf_offset[num_grids];
313:     IPf_sz_tot = IPf_sz_glb*ctx->batch_sz;
314:     // prep COO
315:     if (ctx->coo_assembly) {
316:       PetscMalloc1(ctx->SData_d.coo_size,&coo_vals); // allocate every time?
317:       PetscInfo(ctx->plex[0], "COO Allocate %" PetscInt_FMT " values\n",ctx->SData_d.coo_size);
318:     }
319:     if (shift==0.0) { /* compute dynamic data f and df and init data for Jacobian */
320: #if defined(PETSC_HAVE_THREADSAFETY)
321:       double         starttime, endtime;
322:       starttime = MPI_Wtime();
323: #endif
324:       PetscLogEventBegin(ctx->events[8],0,0,0,0);
325:       for (PetscInt fieldA=0;fieldA<ctx->num_species;fieldA++) {
326:         invMass[fieldA]  = ctx->m_0/ctx->masses[fieldA];
327:         Eq_m[fieldA]     = ctx->Ez * ctx->t_0 * ctx->charges[fieldA] / (ctx->v_0 * ctx->masses[fieldA]); /* normalize dimensionless */
328:         if (dim==2) Eq_m[fieldA] *=  2 * PETSC_PI; /* add the 2pi term that is not in Landau */
329:         nu_alpha[fieldA] = PetscSqr(ctx->charges[fieldA]/ctx->m_0)*ctx->m_0/ctx->masses[fieldA];
330:         nu_beta[fieldA]  = PetscSqr(ctx->charges[fieldA]/ctx->epsilon0)*ctx->lnLam / (8*PETSC_PI) * ctx->t_0*ctx->n_0/PetscPowReal(ctx->v_0,3);
331:       }
332:       PetscMalloc4(IPf_sz_tot, &ff, IPf_sz_tot, &dudx, IPf_sz_tot, &dudy, dim==3 ? IPf_sz_tot : 0, &dudz);
333:       // F df/dx
334:       for (PetscInt tid = 0 ; tid < ctx->batch_sz*elem_offset[num_grids] ; tid++) { // for each element
335:         const PetscInt b_Nelem = elem_offset[num_grids], b_elem_idx = tid%b_Nelem, b_id = tid/b_Nelem; // b_id == OMP thd_id in batch
336:         // find my grid:
337:         PetscInt       grid = 0;
338:         while (b_elem_idx >= elem_offset[grid+1]) grid++; // yuck search for grid
339:         {
340:           const PetscInt     loc_nip = numCells[grid]*Nq, loc_Nf = ctx->species_offset[grid+1] - ctx->species_offset[grid], loc_elem = b_elem_idx - elem_offset[grid];
341:           const PetscInt     moffset = LAND_MOFFSET(b_id,grid,ctx->batch_sz,ctx->num_grids,ctx->mat_offset); //b_id*b_N + ctx->mat_offset[grid];
342:           PetscScalar        *coef, coef_buff[LANDAU_MAX_SPECIES*LANDAU_MAX_NQ];
343:           PetscReal          *invJe  = &invJ_a[(ip_offset[grid] + loc_elem*Nq)*dim*dim]; // ingJ is static data on batch 0
344:           PetscInt           b,f,q;
345:           if (cellClosure) {
346:             coef = &cellClosure[b_id*IPf_sz_glb + ipf_offset[grid] + loc_elem*Nb*loc_Nf]; // this is const
347:           } else {
348:             coef = coef_buff;
349:             for (f = 0; f < loc_Nf; ++f) {
350:               LandauIdx *const Idxs = &maps[grid].gIdx[loc_elem][f][0];
351:               for (b = 0; b < Nb; ++b) {
352:                 PetscInt idx = Idxs[b];
353:                 if (idx >= 0) {
354:                   coef[f*Nb+b] = xdata[idx+moffset];
355:                 } else {
356:                   idx = -idx - 1;
357:                   coef[f*Nb+b] = 0;
358:                   for (q = 0; q < maps[grid].num_face; q++) {
359:                     PetscInt    id    = maps[grid].c_maps[idx][q].gid;
360:                     PetscScalar scale = maps[grid].c_maps[idx][q].scale;
361:                     coef[f*Nb+b] += scale*xdata[id+moffset];
362:                   }
363:                 }
364:               }
365:             }
366:           }
367:           /* get f and df */
368:           for (PetscInt qi = 0; qi < Nq; qi++) {
369:             const PetscReal  *invJ = &invJe[qi*dim*dim];
370:             const PetscReal  *Bq   = &BB[qi*Nb];
371:             const PetscReal  *Dq   = &DD[qi*Nb*dim];
372:             PetscReal        u_x[LANDAU_DIM];
373:             /* get f & df */
374:             for (f = 0; f < loc_Nf; ++f) {
375:               const PetscInt idx = b_id*IPf_sz_glb + ipf_offset[grid] + f*loc_nip + loc_elem*Nq + qi;
376:               PetscInt       b, e;
377:               PetscReal      refSpaceDer[LANDAU_DIM];
378:               ff[idx] = 0.0;
379:               for (int d = 0; d < LANDAU_DIM; ++d) refSpaceDer[d] = 0.0;
380:               for (b = 0; b < Nb; ++b) {
381:                 const PetscInt    cidx = b;
382:                 ff[idx] += Bq[cidx]*PetscRealPart(coef[f*Nb+cidx]);
383:                 for (int d = 0; d < dim; ++d) {
384:                   refSpaceDer[d] += Dq[cidx*dim+d]*PetscRealPart(coef[f*Nb+cidx]);
385:                 }
386:               }
387:               for (int d = 0; d < LANDAU_DIM; ++d) {
388:                 for (e = 0, u_x[d] = 0.0; e < LANDAU_DIM; ++e) {
389:                   u_x[d] += invJ[e*dim+d]*refSpaceDer[e];
390:                 }
391:               }
392:               dudx[idx] = u_x[0];
393:               dudy[idx] = u_x[1];
394:  #if LANDAU_DIM==3
395:               dudz[idx] = u_x[2];
396: #endif
397:             }
398:           } // q
399:         } // grid
400:       } // grid*batch
401:       PetscLogEventEnd(ctx->events[8],0,0,0,0);
402: #if defined(PETSC_HAVE_THREADSAFETY)
403:       endtime = MPI_Wtime();
404:       if (ctx->stage) ctx->times[LANDAU_F_DF] += (endtime - starttime);
405: #endif
406:     } // Jacobian setup
407:     // assemble Jacobian (or mass)
408:     for (PetscInt tid = 0 ; tid < ctx->batch_sz*elem_offset[num_grids] ; tid++) { // for each element
409:       const PetscInt b_Nelem      = elem_offset[num_grids];
410:       const PetscInt glb_elem_idx = tid%b_Nelem, b_id = tid/b_Nelem;
411:       PetscInt       grid         = 0;
412: #if defined(PETSC_HAVE_THREADSAFETY)
413:       double         starttime, endtime;
414:       starttime                   = MPI_Wtime();
415: #endif
416:       while (glb_elem_idx >= elem_offset[grid+1]) grid++;
417:       {
418:         const PetscInt     loc_Nf  = ctx->species_offset[grid+1] - ctx->species_offset[grid], loc_elem = glb_elem_idx - elem_offset[grid];
419:         const PetscInt     moffset = LAND_MOFFSET(b_id,grid,ctx->batch_sz,ctx->num_grids,ctx->mat_offset), totDim = loc_Nf*Nq, elemMatSize = totDim*totDim;
420:         PetscScalar        *elemMat;
421:          const PetscReal   *invJe  = &invJ_a[(ip_offset[grid] + loc_elem*Nq)*dim*dim];
422:         PetscMalloc1(elemMatSize, &elemMat);
423:         PetscMemzero(elemMat, elemMatSize*sizeof(*elemMat));
424:         if (shift==0.0) { // Jacobian
425:           PetscLogEventBegin(ctx->events[4],0,0,0,0);
426:         } else {          // mass
427:           PetscLogEventBegin(ctx->events[16],0,0,0,0);
428:         }
429:         for (PetscInt qj = 0; qj < Nq; ++qj) {
430:           const PetscInt   jpidx_glb = ip_offset[grid] + qj + loc_elem * Nq;
431:           PetscReal        g0[LANDAU_MAX_SPECIES], g2[LANDAU_MAX_SPECIES][LANDAU_DIM], g3[LANDAU_MAX_SPECIES][LANDAU_DIM][LANDAU_DIM]; // could make a LANDAU_MAX_SPECIES_GRID ~ number of ions - 1
432:           PetscInt         d,d2,dp,d3,IPf_idx;
433:           if (shift==0.0) { // Jacobian
434:             const PetscReal * const invJj = &invJe[qj*dim*dim];
435:             PetscReal               gg2[LANDAU_MAX_SPECIES][LANDAU_DIM],gg3[LANDAU_MAX_SPECIES][LANDAU_DIM][LANDAU_DIM], gg2_temp[LANDAU_DIM], gg3_temp[LANDAU_DIM][LANDAU_DIM];
436:             const PetscReal         vj[3] = {xx[jpidx_glb], yy[jpidx_glb], zz ? zz[jpidx_glb] : 0}, wj = ww[jpidx_glb];
437:             // create g2 & g3
438:             for (d=0;d<LANDAU_DIM;d++) { // clear accumulation data D & K
439:               gg2_temp[d] = 0;
440:               for (d2=0;d2<LANDAU_DIM;d2++) gg3_temp[d][d2] = 0;
441:             }
442:             /* inner beta reduction */
443:             IPf_idx = 0;
444:             for (PetscInt grid_r = 0, f_off = 0, ipidx = 0; grid_r < ctx->num_grids ; grid_r++, f_off = ctx->species_offset[grid_r]) { // IPf_idx += nip_loc_r*Nfloc_r
445:               PetscInt  nip_loc_r = numCells[grid_r]*Nq, Nfloc_r = Nf[grid_r];
446:               for (PetscInt ei_r = 0, loc_fdf_idx = 0; ei_r < numCells[grid_r]; ++ei_r) {
447:                 for (PetscInt qi = 0; qi < Nq; qi++, ipidx++, loc_fdf_idx++) {
448:                   const PetscReal wi       = ww[ipidx], x = xx[ipidx], y = yy[ipidx];
449:                   PetscReal       temp1[3] = {0, 0, 0}, temp2 = 0;
450: #if LANDAU_DIM==2
451:                   PetscReal       Ud[2][2], Uk[2][2], mask = (PetscAbs(vj[0]-x) < 100*PETSC_SQRT_MACHINE_EPSILON && PetscAbs(vj[1]-y) < 100*PETSC_SQRT_MACHINE_EPSILON) ? 0. : 1.;
452:                   LandauTensor2D(vj, x, y, Ud, Uk, mask);
453: #else
454:                   PetscReal U[3][3], z = zz[ipidx], mask = (PetscAbs(vj[0]-x) < 100*PETSC_SQRT_MACHINE_EPSILON && PetscAbs(vj[1]-y) < 100*PETSC_SQRT_MACHINE_EPSILON && PetscAbs(vj[2]-z) < 100*PETSC_SQRT_MACHINE_EPSILON) ? 0. : 1.;
455:                   if (ctx->use_relativistic_corrections) {
456:                     LandauTensor3DRelativistic(vj, x, y, z, U, mask, C_0(ctx->v_0));
457:                   } else {
458:                     LandauTensor3D(vj, x, y, z, U, mask);
459:                   }
460: #endif
461:                   for (int f = 0; f < Nfloc_r ; ++f) {
462:                     const PetscInt idx = b_id*IPf_sz_glb + ipf_offset[grid_r] + f*nip_loc_r + ei_r*Nq + qi;  // IPf_idx + f*nip_loc_r + loc_fdf_idx;
463:                     temp1[0] += dudx[idx]*nu_beta[f+f_off]*invMass[f+f_off];
464:                     temp1[1] += dudy[idx]*nu_beta[f+f_off]*invMass[f+f_off];
465: #if LANDAU_DIM==3
466:                     temp1[2] += dudz[idx]*nu_beta[f+f_off]*invMass[f+f_off];
467: #endif
468:                     temp2    += ff[idx]*nu_beta[f+f_off];
469:                   }
470:                   temp1[0] *= wi;
471:                   temp1[1] *= wi;
472: #if LANDAU_DIM==3
473:                   temp1[2] *= wi;
474: #endif
475:                   temp2    *= wi;
476: #if LANDAU_DIM==2
477:                   for (d2 = 0; d2 < 2; d2++) {
478:                     for (d3 = 0; d3 < 2; ++d3) {
479:                       /* K = U * grad(f): g2=e: i,A */
480:                       gg2_temp[d2] += Uk[d2][d3]*temp1[d3];
481:                       /* D = -U * (I \kron (fx)): g3=f: i,j,A */
482:                       gg3_temp[d2][d3] += Ud[d2][d3]*temp2;
483:                     }
484:                   }
485: #else
486:                   for (d2 = 0; d2 < 3; ++d2) {
487:                     for (d3 = 0; d3 < 3; ++d3) {
488:                       /* K = U * grad(f): g2 = e: i,A */
489:                       gg2_temp[d2] += U[d2][d3]*temp1[d3];
490:                       /* D = -U * (I \kron (fx)): g3 = f: i,j,A */
491:                       gg3_temp[d2][d3] += U[d2][d3]*temp2;
492:                     }
493:                   }
494: #endif
495:                 } // qi
496:               } // ei_r
497:               IPf_idx += nip_loc_r*Nfloc_r;
498:             } /* grid_r - IPs */
500:             // add alpha and put in gg2/3
501:             for (PetscInt fieldA = 0, f_off = ctx->species_offset[grid]; fieldA < loc_Nf; ++fieldA) {
502:               for (d2 = 0; d2 < dim; d2++) {
503:                 gg2[fieldA][d2] = gg2_temp[d2]*nu_alpha[fieldA+f_off];
504:                 for (d3 = 0; d3 < dim; d3++) {
505:                   gg3[fieldA][d2][d3] = -gg3_temp[d2][d3]*nu_alpha[fieldA+f_off]*invMass[fieldA+f_off];
506:                 }
507:               }
508:             }
509:             /* add electric field term once per IP */
510:             for (PetscInt fieldA = 0, f_off = ctx->species_offset[grid] ; fieldA < loc_Nf; ++fieldA) {
511:               gg2[fieldA][dim-1] += Eq_m[fieldA+f_off];
512:             }
513:             /* Jacobian transform - g2, g3 */
514:             for (PetscInt fieldA = 0; fieldA < loc_Nf; ++fieldA) {
515:               for (d = 0; d < dim; ++d) {
516:                 g2[fieldA][d] = 0.0;
517:                 for (d2 = 0; d2 < dim; ++d2) {
518:                   g2[fieldA][d] += invJj[d*dim+d2]*gg2[fieldA][d2];
519:                   g3[fieldA][d][d2] = 0.0;
520:                   for (d3 = 0; d3 < dim; ++d3) {
521:                     for (dp = 0; dp < dim; ++dp) {
522:                       g3[fieldA][d][d2] += invJj[d*dim + d3]*gg3[fieldA][d3][dp]*invJj[d2*dim + dp];
523:                     }
524:                   }
525:                   g3[fieldA][d][d2] *= wj;
526:                 }
527:                 g2[fieldA][d] *= wj;
528:               }
529:             }
530:           } else { // mass
531:             PetscReal wj = ww[jpidx_glb];
532:             /* Jacobian transform - g0 */
533:             for (PetscInt fieldA = 0; fieldA < loc_Nf ; ++fieldA) {
534:               if (dim==2) {
535:                 g0[fieldA] = wj * shift * 2. * PETSC_PI; // move this to below and remove g0
536:               } else {
537:                 g0[fieldA] = wj * shift; // move this to below and remove g0
538:               }
539:             }
540:           }
541:           /* FE matrix construction */
542:           {
543:             PetscInt  fieldA,d,f,d2,g;
544:             const PetscReal *BJq = &BB[qj*Nb], *DIq = &DD[qj*Nb*dim];
545:             /* assemble - on the diagonal (I,I) */
546:             for (fieldA = 0; fieldA < loc_Nf ; fieldA++) {
547:               for (f = 0; f < Nb ; f++) {
548:                 const PetscInt i = fieldA*Nb + f; /* Element matrix row */
549:                 for (g = 0; g < Nb; ++g) {
550:                   const PetscInt j    = fieldA*Nb + g; /* Element matrix column */
551:                   const PetscInt fOff = i*totDim + j;
552:                   if (shift==0.0) {
553:                     for (d = 0; d < dim; ++d) {
554:                       elemMat[fOff] += DIq[f*dim+d]*g2[fieldA][d]*BJq[g];
555:                       for (d2 = 0; d2 < dim; ++d2) {
556:                         elemMat[fOff] += DIq[f*dim + d]*g3[fieldA][d][d2]*DIq[g*dim + d2];
557:                       }
558:                     }
559:                   } else { // mass
560:                     elemMat[fOff] += BJq[f]*g0[fieldA]*BJq[g];
561:                   }
562:                 }
563:               }
564:             }
565:           }
566:         } /* qj loop */
567:         if (shift==0.0) { // Jacobian
568:           PetscLogEventEnd(ctx->events[4],0,0,0,0);
569:         } else {
570:           PetscLogEventEnd(ctx->events[16],0,0,0,0);
571:         }
572: #if defined(PETSC_HAVE_THREADSAFETY)
573:         endtime = MPI_Wtime();
574:         if (ctx->stage) ctx->times[LANDAU_KERNEL] += (endtime - starttime);
575: #endif
576:         /* assemble matrix */
577:         if (!container) {
578:           PetscInt cStart;
579:           PetscLogEventBegin(ctx->events[6],0,0,0,0);
580:           DMPlexGetHeightStratum(ctx->plex[grid], 0, &cStart, NULL);
581:           DMPlexMatSetClosure(ctx->plex[grid], section[grid], globsection[grid], subJ[ LAND_PACK_IDX(b_id,grid) ], loc_elem + cStart, elemMat, ADD_VALUES);
582:           PetscLogEventEnd(ctx->events[6],0,0,0,0);
583:         } else {  // GPU like assembly for debugging
584:           PetscInt      fieldA,q,f,g,d,nr,nc,rows0[LANDAU_MAX_Q_FACE]={0},cols0[LANDAU_MAX_Q_FACE]={0},rows[LANDAU_MAX_Q_FACE],cols[LANDAU_MAX_Q_FACE];
585:           PetscScalar   vals[LANDAU_MAX_Q_FACE*LANDAU_MAX_Q_FACE]={0},row_scale[LANDAU_MAX_Q_FACE]={0},col_scale[LANDAU_MAX_Q_FACE]={0};
586:           LandauIdx     *coo_elem_offsets = (LandauIdx*)ctx->SData_d.coo_elem_offsets, *coo_elem_fullNb = (LandauIdx*)ctx->SData_d.coo_elem_fullNb, (*coo_elem_point_offsets)[LANDAU_MAX_NQ+1] = (LandauIdx (*)[LANDAU_MAX_NQ+1])ctx->SData_d.coo_elem_point_offsets;
587:           /* assemble - from the diagonal (I,I) in this format for DMPlexMatSetClosure */
588:           for (fieldA = 0; fieldA < loc_Nf ; fieldA++) {
589:             LandauIdx *const Idxs = &maps[grid].gIdx[loc_elem][fieldA][0];
590:             for (f = 0; f < Nb ; f++) {
591:               PetscInt idx = Idxs[f];
592:               if (idx >= 0) {
593:                 nr           = 1;
594:                 rows0[0]     = idx;
595:                 row_scale[0] = 1.;
596:               } else {
597:                 idx = -idx - 1;
598:                 for (q = 0, nr = 0; q < maps[grid].num_face; q++, nr++) {
599:                   if (maps[grid].c_maps[idx][q].gid < 0) break;
600:                   rows0[q]     = maps[grid].c_maps[idx][q].gid;
601:                   row_scale[q] = maps[grid].c_maps[idx][q].scale;
602:                 }
603:               }
604:               for (g = 0; g < Nb; ++g) {
605:                 idx = Idxs[g];
606:                 if (idx >= 0) {
607:                   nc = 1;
608:                   cols0[0]     = idx;
609:                   col_scale[0] = 1.;
610:                 } else {
611:                   idx = -idx - 1;
612:                   nc = maps[grid].num_face;
613:                   for (q = 0, nc = 0; q < maps[grid].num_face; q++, nc++) {
614:                     if (maps[grid].c_maps[idx][q].gid < 0) break;
615:                     cols0[q]     = maps[grid].c_maps[idx][q].gid;
616:                     col_scale[q] = maps[grid].c_maps[idx][q].scale;
617:                   }
618:                 }
619:                 const PetscInt    i   = fieldA*Nb + f; /* Element matrix row */
620:                 const PetscInt    j   = fieldA*Nb + g; /* Element matrix column */
621:                 const PetscScalar Aij = elemMat[i*totDim + j];
622:                 if (coo_vals) { // mirror (i,j) in CreateStaticGPUData
623:                   const int fullNb = coo_elem_fullNb[glb_elem_idx],fullNb2=fullNb*fullNb;
624:                   const int idx0   = b_id*coo_elem_offsets[elem_offset[num_grids]] + coo_elem_offsets[glb_elem_idx] + fieldA*fullNb2 + fullNb * coo_elem_point_offsets[glb_elem_idx][f] + nr * coo_elem_point_offsets[glb_elem_idx][g];
625:                   for (int q = 0, idx2 = idx0; q < nr; q++) {
626:                     for (int d = 0; d < nc; d++, idx2++) {
627:                       coo_vals[idx2] = row_scale[q]*col_scale[d]*Aij;
628:                     }
629:                   }
630:                 } else {
631:                   for (q = 0; q < nr; q++) rows[q] = rows0[q] + moffset;
632:                   for (d = 0; d < nc; d++) cols[d] = cols0[d] + moffset;
633:                   for (q = 0; q < nr; q++) {
634:                     for (d = 0; d < nc; d++) {
635:                       vals[q*nc + d] = row_scale[q]*col_scale[d]*Aij;
636:                     }
637:                   }
638:                   MatSetValues(JacP,nr,rows,nc,cols,vals,ADD_VALUES);
639:                 }
640:               }
641:             }
642:           }
643:         }
644:         if (loc_elem==-1) {
645:           PetscPrintf(ctx->comm,"CPU Element matrix\n");
646:           for (int d = 0; d < totDim; ++d) {
647:             for (int f = 0; f < totDim; ++f) PetscPrintf(ctx->comm," %12.5e",  PetscRealPart(elemMat[d*totDim + f]));
648:             PetscPrintf(ctx->comm,"\n");
649:           }
650:           exit(12);
651:         }
652:         PetscFree(elemMat);
653:       } /* grid */
654:     } /* outer element & batch loop */
655:     if (shift==0.0) { // mass
656:       PetscFree4(ff, dudx, dudy, dudz);
657:     }
658:     if (!container) {   // 'CPU' assembly move nest matrix to global JacP
659:       for (PetscInt b_id = 0 ; b_id < ctx->batch_sz ; b_id++) { // OpenMP
660:         for (PetscInt grid=0 ; grid<ctx->num_grids ; grid++) {
661:           const PetscInt    moffset = LAND_MOFFSET(b_id,grid,ctx->batch_sz,ctx->num_grids,ctx->mat_offset); // b_id*b_N + ctx->mat_offset[grid];
662:           PetscInt          nloc, nzl, colbuf[1024], row;
663:           const PetscInt    *cols;
664:           const PetscScalar *vals;
665:           Mat               B = subJ[ LAND_PACK_IDX(b_id,grid) ];
666:           MatAssemblyBegin(B, MAT_FINAL_ASSEMBLY);
667:           MatAssemblyEnd(B, MAT_FINAL_ASSEMBLY);
668:           MatGetSize(B, &nloc, NULL);
669:           for (int i=0 ; i<nloc ; i++) {
670:             MatGetRow(B,i,&nzl,&cols,&vals);
672:             for (int j=0; j<nzl; j++) colbuf[j] = moffset + cols[j];
673:             row  = moffset + i;
674:             MatSetValues(JacP,1,&row,nzl,colbuf,vals,ADD_VALUES);
675:             MatRestoreRow(B,i,&nzl,&cols,&vals);
676:           }
677:           MatDestroy(&B);
678:         }
679:       }
680:     }
681:     if (coo_vals) {
682:       MatSetValuesCOO(JacP,coo_vals,ADD_VALUES);
683:       PetscFree(coo_vals);
684:     }
685:   } /* CPU version */
686:   MatAssemblyBegin(JacP, MAT_FINAL_ASSEMBLY);
687:   MatAssemblyEnd(JacP, MAT_FINAL_ASSEMBLY);
688:   /* clean up */
689:   if (cellClosure) {
690:     PetscFree(cellClosure);
691:   }
692:   if (xdata) {
693:     VecRestoreArrayReadAndMemType(a_X,&xdata);
694:   }
695:   return 0;
696: }

698: #if defined(LANDAU_ADD_BCS)
699: static void zero_bc(PetscInt dim, PetscInt Nf, PetscInt NfAux,
700:                     const PetscInt uOff[], const PetscInt uOff_x[], const PetscScalar u[], const PetscScalar u_t[], const PetscScalar u_x[],
701:                     const PetscInt aOff[], const PetscInt aOff_x[], const PetscScalar a[], const PetscScalar a_t[], const PetscScalar a_x[],
702:                     PetscReal t, const PetscReal x[], PetscInt numConstants, const PetscScalar constants[], PetscScalar uexact[])
703: {
704:   uexact[0] = 0;
705: }
706: #endif

708: #define MATVEC2(__a,__x,__p) {int i,j; for (i=0.; i<2; i++) {__p[i] = 0; for (j=0.; j<2; j++) __p[i] += __a[i][j]*__x[j]; }}
709: static void CircleInflate(PetscReal r1, PetscReal r2, PetscReal r0, PetscInt num_sections, PetscReal x, PetscReal y,
710:                           PetscReal *outX, PetscReal *outY)
711: {
712:   PetscReal rr = PetscSqrtReal(x*x + y*y), outfact, efact;
713:   if (rr < r1 + PETSC_SQRT_MACHINE_EPSILON) {
714:     *outX = x; *outY = y;
715:   } else {
716:     const PetscReal xy[2] = {x,y}, sinphi=y/rr, cosphi=x/rr;
717:     PetscReal       cth,sth,xyprime[2],Rth[2][2],rotcos,newrr;
718:     if (num_sections==2) {
719:       rotcos  = 0.70710678118654;
720:       outfact = 1.5; efact = 2.5;
721:       /* rotate normalized vector into [-pi/4,pi/4) */
722:       if (sinphi >= 0.) {         /* top cell, -pi/2 */
723:         cth = 0.707106781186548; sth = -0.707106781186548;
724:       } else {                    /* bottom cell -pi/8 */
725:         cth = 0.707106781186548; sth = .707106781186548;
726:       }
727:     } else if (num_sections==3) {
728:       rotcos  = 0.86602540378443;
729:       outfact = 1.5; efact = 2.5;
730:       /* rotate normalized vector into [-pi/6,pi/6) */
731:       if (sinphi >= 0.5) {         /* top cell, -pi/3 */
732:         cth = 0.5; sth = -0.866025403784439;
733:       } else if (sinphi >= -.5) {  /* mid cell 0 */
734:         cth = 1.; sth = .0;
735:       } else { /* bottom cell +pi/3 */
736:         cth = 0.5; sth = 0.866025403784439;
737:       }
738:     } else if (num_sections==4) {
739:       rotcos  = 0.9238795325112;
740:       outfact = 1.5; efact = 3;
741:       /* rotate normalized vector into [-pi/8,pi/8) */
742:       if (sinphi >= 0.707106781186548) {         /* top cell, -3pi/8 */
743:         cth = 0.38268343236509;  sth = -0.923879532511287;
744:       } else if (sinphi >= 0.) {                 /* mid top cell -pi/8 */
745:         cth = 0.923879532511287; sth = -.38268343236509;
746:       } else if (sinphi >= -0.707106781186548) { /* mid bottom cell + pi/8 */
747:         cth = 0.923879532511287; sth = 0.38268343236509;
748:       } else {                                   /* bottom cell + 3pi/8 */
749:         cth = 0.38268343236509;  sth = .923879532511287;
750:       }
751:     } else {
752:       cth = 0.; sth = 0.; rotcos = 0; efact = 0;
753:     }
754:     Rth[0][0] = cth; Rth[0][1] =-sth;
755:     Rth[1][0] = sth; Rth[1][1] = cth;
756:     MATVEC2(Rth,xy,xyprime);
757:     if (num_sections==2) {
758:       newrr = xyprime[0]/rotcos;
759:     } else {
760:       PetscReal newcosphi=xyprime[0]/rr, rin = r1, rout = rr - rin;
761:       PetscReal routmax = r0*rotcos/newcosphi - rin, nroutmax = r0 - rin, routfrac = rout/routmax;
762:       newrr = rin + routfrac*nroutmax;
763:     }
764:     *outX = cosphi*newrr; *outY = sinphi*newrr;
765:     /* grade */
766:     PetscReal fact,tt,rs,re, rr = PetscSqrtReal(PetscSqr(*outX) + PetscSqr(*outY));
767:     if (rr > r2) { rs = r2; re = r0; fact = outfact;} /* outer zone */
768:     else {         rs = r1; re = r2; fact = efact;} /* electron zone */
769:     tt = (rs + PetscPowReal((rr - rs)/(re - rs),fact) * (re-rs)) / rr;
770:     *outX *= tt;
771:     *outY *= tt;
772:   }
773: }

775: static PetscErrorCode GeometryDMLandau(DM base, PetscInt point, PetscInt dim, const PetscReal abc[], PetscReal xyz[], void *a_ctx)
776: {
777:   LandauCtx   *ctx = (LandauCtx*)a_ctx;
778:   PetscReal   r = abc[0], z = abc[1];
779:   if (ctx->inflate) {
780:     PetscReal absR, absZ;
781:     absR = PetscAbs(r);
782:     absZ = PetscAbs(z);
783:     CircleInflate(ctx->i_radius[0],ctx->e_radius,ctx->radius[0],ctx->num_sections,absR,absZ,&absR,&absZ); // wrong: how do I know what grid I am on?
784:     r = (r > 0) ? absR : -absR;
785:     z = (z > 0) ? absZ : -absZ;
786:   }
787:   xyz[0] = r;
788:   xyz[1] = z;
789:   if (dim==3) xyz[2] = abc[2];

791:   return 0;
792: }

794: /* create DMComposite of meshes for each species group */
795: static PetscErrorCode LandauDMCreateVMeshes(MPI_Comm comm_self, const PetscInt dim, const char prefix[], LandauCtx *ctx, DM pack)
796: {
797:   { /* p4est, quads */
798:     /* Create plex mesh of Landau domain */
799:     for (PetscInt grid=0;grid<ctx->num_grids;grid++) {
800:       PetscReal radius = ctx->radius[grid];
801:       if (!ctx->sphere) {
802:         PetscInt       cells[] = {2,2,2};
803:         PetscReal      lo[] = {-radius,-radius,-radius}, hi[] = {radius,radius,radius};
804:         DMBoundaryType periodicity[3] = {DM_BOUNDARY_NONE, dim==2 ? DM_BOUNDARY_NONE : DM_BOUNDARY_NONE, DM_BOUNDARY_NONE};
805:         if (dim==2) { lo[0] = 0; cells[0] /* = cells[1] */ = 1; }
806:         DMPlexCreateBoxMesh(comm_self, dim, PETSC_FALSE, cells, lo, hi, periodicity, PETSC_TRUE, &ctx->plex[grid]); // todo: make composite and create dm[grid] here
807:         DMLocalizeCoordinates(ctx->plex[grid]); /* needed for periodic */
808:         if (dim==3) PetscObjectSetName((PetscObject) ctx->plex[grid], "cube");
809:         else PetscObjectSetName((PetscObject) ctx->plex[grid], "half-plane");
810:       } else if (dim==2) { // sphere is all wrong. should just have one inner radius
811:         PetscInt       numCells,cells[16][4],i,j;
812:         PetscInt       numVerts;
813:         PetscReal      inner_radius1 = ctx->i_radius[grid], inner_radius2 = ctx->e_radius;
814:         PetscReal      *flatCoords   = NULL;
815:         PetscInt       *flatCells    = NULL, *pcell;
816:         if (ctx->num_sections==2) {
817: #if 1
818:           numCells = 5;
819:           numVerts = 10;
820:           int cells2[][4] = { {0,1,4,3},
821:                               {1,2,5,4},
822:                               {3,4,7,6},
823:                               {4,5,8,7},
824:                               {6,7,8,9} };
825:           for (i = 0; i < numCells; i++) for (j = 0; j < 4; j++) cells[i][j] = cells2[i][j];
826:           PetscMalloc2(numVerts * 2, &flatCoords, numCells * 4, &flatCells);
827:           {
828:             PetscReal (*coords)[2] = (PetscReal (*) [2]) flatCoords;
829:             for (j = 0; j < numVerts-1; j++) {
830:               PetscReal z, r, theta = -PETSC_PI/2 + (j%3) * PETSC_PI/2;
831:               PetscReal rad = (j >= 6) ? inner_radius1 : (j >= 3) ? inner_radius2 : ctx->radius[grid];
832:               z = rad * PetscSinReal(theta);
833:               coords[j][1] = z;
834:               r = rad * PetscCosReal(theta);
835:               coords[j][0] = r;
836:             }
837:             coords[numVerts-1][0] = coords[numVerts-1][1] = 0;
838:           }
839: #else
840:           numCells = 4;
841:           numVerts = 8;
842:           static int     cells2[][4] = {{0,1,2,3},
843:                                         {4,5,1,0},
844:                                         {5,6,2,1},
845:                                         {6,7,3,2}};
846:           for (i = 0; i < numCells; i++) for (j = 0; j < 4; j++) cells[i][j] = cells2[i][j];
847:           loc2(numVerts * 2, &flatCoords, numCells * 4, &flatCells);
848:           {
849:             PetscReal (*coords)[2] = (PetscReal (*) [2]) flatCoords;
850:             PetscInt j;
851:             for (j = 0; j < 8; j++) {
852:               PetscReal z, r;
853:               PetscReal theta = -PETSC_PI/2 + (j%4) * PETSC_PI/3.;
854:               PetscReal rad = ctx->radius[grid] * ((j < 4) ? 0.5 : 1.0);
855:               z = rad * PetscSinReal(theta);
856:               coords[j][1] = z;
857:               r = rad * PetscCosReal(theta);
858:               coords[j][0] = r;
859:             }
860:           }
861: #endif
862:         } else if (ctx->num_sections==3) {
863:           numCells = 7;
864:           numVerts = 12;
865:           int cells2[][4] = { {0,1,5,4},
866:                               {1,2,6,5},
867:                               {2,3,7,6},
868:                               {4,5,9,8},
869:                               {5,6,10,9},
870:                               {6,7,11,10},
871:                               {8,9,10,11} };
872:           for (i = 0; i < numCells; i++) for (j = 0; j < 4; j++) cells[i][j] = cells2[i][j];
873:           PetscMalloc2(numVerts * 2, &flatCoords, numCells * 4, &flatCells);
874:           {
875:             PetscReal (*coords)[2] = (PetscReal (*) [2]) flatCoords;
876:             for (j = 0; j < numVerts; j++) {
877:               PetscReal z, r, theta = -PETSC_PI/2 + (j%4) * PETSC_PI/3;
878:               PetscReal rad = (j >= 8) ? inner_radius1 : (j >= 4) ? inner_radius2 : ctx->radius[grid];
879:               z = rad * PetscSinReal(theta);
880:               coords[j][1] = z;
881:               r = rad * PetscCosReal(theta);
882:               coords[j][0] = r;
883:             }
884:           }
885:         } else if (ctx->num_sections==4) {
886:           numCells = 10;
887:           numVerts = 16;
888:           int cells2[][4] = { {0,1,6,5},
889:                               {1,2,7,6},
890:                               {2,3,8,7},
891:                               {3,4,9,8},
892:                               {5,6,11,10},
893:                               {6,7,12,11},
894:                               {7,8,13,12},
895:                               {8,9,14,13},
896:                               {10,11,12,15},
897:                               {12,13,14,15}};
898:           for (i = 0; i < numCells; i++) for (j = 0; j < 4; j++) cells[i][j] = cells2[i][j];
899:           PetscMalloc2(numVerts * 2, &flatCoords, numCells * 4, &flatCells);
900:           {
901:             PetscReal (*coords)[2] = (PetscReal (*) [2]) flatCoords;
902:             for (j = 0; j < numVerts-1; j++) {
903:               PetscReal z, r, theta = -PETSC_PI/2 + (j%5) * PETSC_PI/4;
904:               PetscReal rad = (j >= 10) ? inner_radius1 : (j >= 5) ? inner_radius2 : ctx->radius[grid];
905:               z = rad * PetscSinReal(theta);
906:               coords[j][1] = z;
907:               r = rad * PetscCosReal(theta);
908:               coords[j][0] = r;
909:             }
910:             coords[numVerts-1][0] = coords[numVerts-1][1] = 0;
911:           }
912:         } else {
913:           numCells = 0;
914:           numVerts = 0;
915:         }
916:         for (j = 0, pcell = flatCells; j < numCells; j++, pcell += 4) {
917:           pcell[0] = cells[j][0]; pcell[1] = cells[j][1];
918:           pcell[2] = cells[j][2]; pcell[3] = cells[j][3];
919:         }
920:         DMPlexCreateFromCellListPetsc(comm_self,2,numCells,numVerts,4,ctx->interpolate,flatCells,2,flatCoords,&ctx->plex[grid]);
921:         PetscFree2(flatCoords,flatCells);
922:         PetscObjectSetName((PetscObject) ctx->plex[grid], "semi-circle");
923:       } else SETERRQ(ctx->comm, PETSC_ERR_PLIB, "Velocity space meshes does not support cubed sphere");

925:       DMSetFromOptions(ctx->plex[grid]);
926:     } // grid loop
927:     PetscObjectSetOptionsPrefix((PetscObject)pack,prefix);
928:     DMSetFromOptions(pack);

930:     { /* convert to p4est (or whatever), wait for discretization to create pack */
931:       char           convType[256];
932:       PetscBool      flg;

935:       PetscOptionsBegin(ctx->comm, prefix, "Mesh conversion options", "DMPLEX");
936:       PetscOptionsFList("-dm_landau_type","Convert DMPlex to another format (p4est)","plexland.c",DMList,DMPLEX,convType,256,&flg);
937:       PetscOptionsEnd();
938:       if (flg) {
939:         ctx->use_p4est = PETSC_TRUE; /* flag for Forest */
940:         for (PetscInt grid=0;grid<ctx->num_grids;grid++) {
941:           DM dmforest;
942:           DMConvert(ctx->plex[grid],convType,&dmforest);
943:           if (dmforest) {
944:             PetscBool isForest;
945:             PetscObjectSetOptionsPrefix((PetscObject)dmforest,prefix);
946:             DMIsForest(dmforest,&isForest);
947:             if (isForest) {
948:               if (ctx->sphere && ctx->inflate) {
949:                 DMForestSetBaseCoordinateMapping(dmforest,GeometryDMLandau,ctx);
950:               }
951:               DMDestroy(&ctx->plex[grid]);
952:               ctx->plex[grid] = dmforest; // Forest for adaptivity
953:             } else SETERRQ(ctx->comm, PETSC_ERR_PLIB, "Converted to non Forest?");
954:           } else SETERRQ(ctx->comm, PETSC_ERR_PLIB, "Convert failed?");
955:         }
956:       } else ctx->use_p4est = PETSC_FALSE; /* flag for Forest */
957:     }
958:   } /* non-file */
959:   DMSetDimension(pack, dim);
960:   PetscObjectSetName((PetscObject) pack, "Mesh");
961:   DMSetApplicationContext(pack, ctx);

963:   return 0;
964: }

966: static PetscErrorCode SetupDS(DM pack, PetscInt dim, PetscInt grid, LandauCtx *ctx)
967: {
968:   PetscInt        ii,i0;
969:   char            buf[256];
970:   PetscSection    section;

972:   for (ii = ctx->species_offset[grid], i0 = 0 ; ii < ctx->species_offset[grid+1] ; ii++, i0++) {
973:     if (ii==0) PetscSNPrintf(buf, sizeof(buf), "e");
974:     else PetscSNPrintf(buf, sizeof(buf), "i%" PetscInt_FMT, ii);
975:     /* Setup Discretization - FEM */
976:     PetscFECreateDefault(PETSC_COMM_SELF, dim, 1, PETSC_FALSE, NULL, PETSC_DECIDE, &ctx->fe[ii]);
977:     PetscObjectSetName((PetscObject) ctx->fe[ii], buf);
978:     DMSetField(ctx->plex[grid], i0, NULL, (PetscObject) ctx->fe[ii]);
979:   }
980:   DMCreateDS(ctx->plex[grid]);
981:   DMGetSection(ctx->plex[grid], &section);
982:   for (PetscInt ii = ctx->species_offset[grid], i0 = 0 ; ii < ctx->species_offset[grid+1] ; ii++, i0++) {
983:     if (ii==0) PetscSNPrintf(buf, sizeof(buf), "se");
984:     else PetscSNPrintf(buf, sizeof(buf), "si%" PetscInt_FMT, ii);
985:     PetscSectionSetComponentName(section, i0, 0, buf);
986:   }
987:   return 0;
988: }

990: /* Define a Maxwellian function for testing out the operator. */

992: /* Using cartesian velocity space coordinates, the particle */
993: /* density, [1/m^3], is defined according to */

995: /* $$ n=\int_{R^3} dv^3 \left(\frac{m}{2\pi T}\right)^{3/2}\exp [- mv^2/(2T)] $$ */

997: /* Using some constant, c, we normalize the velocity vector into a */
998: /* dimensionless variable according to v=c*x. Thus the density, $n$, becomes */

1000: /* $$ n=\int_{R^3} dx^3 \left(\frac{mc^2}{2\pi T}\right)^{3/2}\exp [- mc^2/(2T)*x^2] $$ */

1002: /* Defining $\theta=2T/mc^2$, we thus find that the probability density */
1003: /* for finding the particle within the interval in a box dx^3 around x is */

1005: /* f(x;\theta)=\left(\frac{1}{\pi\theta}\right)^{3/2} \exp [ -x^2/\theta ] */

1007: typedef struct {
1008:   PetscReal v_0;
1009:   PetscReal kT_m;
1010:   PetscReal n;
1011:   PetscReal shift;
1012: } MaxwellianCtx;

1014: static PetscErrorCode maxwellian(PetscInt dim, PetscReal time, const PetscReal x[], PetscInt Nf_dummy, PetscScalar *u, void *actx)
1015: {
1016:   MaxwellianCtx *mctx = (MaxwellianCtx*)actx;
1017:   PetscInt      i;
1018:   PetscReal     v2 = 0, theta = 2*mctx->kT_m/(mctx->v_0*mctx->v_0); /* theta = 2kT/mc^2 */
1019:   /* compute the exponents, v^2 */
1020:   for (i = 0; i < dim; ++i) v2 += x[i]*x[i];
1021:   /* evaluate the Maxwellian */
1022:   u[0] = mctx->n*PetscPowReal(PETSC_PI*theta,-1.5)*(PetscExpReal(-v2/theta));
1023:   if (mctx->shift!=0.) {
1024:     v2 = 0;
1025:     for (i = 0; i < dim-1; ++i) v2 += x[i]*x[i];
1026:     v2 += (x[dim-1]-mctx->shift)*(x[dim-1]-mctx->shift);
1027:     /* evaluate the shifted Maxwellian */
1028:     u[0] += mctx->n*PetscPowReal(PETSC_PI*theta,-1.5)*(PetscExpReal(-v2/theta));
1029:   }
1030:   return 0;
1031: }

1033: /*@
1034:  DMPlexLandauAddMaxwellians - Add a Maxwellian distribution to a state

1036:  Collective on X

1038:  Input Parameters:
1039:  .   dm - The mesh (local)
1040:  +   time - Current time
1041:  -   temps - Temperatures of each species (global)
1042:  .   ns - Number density of each species (global)
1043:  -   grid - index into current grid - just used for offset into temp and ns
1044:  +   actx - Landau context

1046:  Output Parameter:
1047:  .   X  - The state (local to this grid)

1049:  Level: beginner

1051:  .keywords: mesh
1052:  .seealso: DMPlexLandauCreateVelocitySpace()
1053:  @*/
1054: PetscErrorCode DMPlexLandauAddMaxwellians(DM dm, Vec X, PetscReal time, PetscReal temps[], PetscReal ns[], PetscInt grid, PetscInt b_id, void *actx)
1055: {
1056:   LandauCtx      *ctx = (LandauCtx*)actx;
1057:   PetscErrorCode (*initu[LANDAU_MAX_SPECIES])(PetscInt, PetscReal, const PetscReal [], PetscInt, PetscScalar [], void *);
1058:   PetscInt       dim;
1059:   MaxwellianCtx  *mctxs[LANDAU_MAX_SPECIES], data[LANDAU_MAX_SPECIES];

1061:   DMGetDimension(dm, &dim);
1062:   if (!ctx) DMGetApplicationContext(dm, &ctx);
1063:   for (PetscInt ii = ctx->species_offset[grid], i0 = 0 ; ii < ctx->species_offset[grid+1] ; ii++, i0++) {
1064:     mctxs[i0]      = &data[i0];
1065:     data[i0].v_0   = ctx->v_0; // v_0 same for all grids
1066:     data[i0].kT_m  = ctx->k*temps[ii]/ctx->masses[ii]; /* kT/m */
1067:     data[i0].n     = ns[ii] * (1+(double)b_id/100.0); // make solves a little different to mimic application, n[0] use for Conner-Hastie
1068:     initu[i0]      = maxwellian;
1069:     data[i0].shift = 0;
1070:   }
1071:   data[0].shift = ctx->electronShift;
1072:   /* need to make ADD_ALL_VALUES work - TODO */
1073:   DMProjectFunction(dm, time, initu, (void**)mctxs, INSERT_ALL_VALUES, X);
1074:   return 0;
1075: }

1077: /*
1078:  LandauSetInitialCondition - Addes Maxwellians with context

1080:  Collective on X

1082:  Input Parameters:
1083:  .   dm - The mesh
1084:  -   grid - index into current grid - just used for offset into temp and ns
1085:  +   actx - Landau context with T and n

1087:  Output Parameter:
1088:  .   X  - The state

1090:  Level: beginner

1092:  .keywords: mesh
1093:  .seealso: DMPlexLandauCreateVelocitySpace(), DMPlexLandauAddMaxwellians()
1094:  */
1095: static PetscErrorCode LandauSetInitialCondition(DM dm, Vec X, PetscInt grid, PetscInt b_id, void *actx)
1096: {
1097:   LandauCtx        *ctx = (LandauCtx*)actx;
1098:   if (!ctx) DMGetApplicationContext(dm, &ctx);
1099:   VecZeroEntries(X);
1100:   DMPlexLandauAddMaxwellians(dm, X, 0.0, ctx->thermal_temps, ctx->n, grid, b_id, ctx);
1101:   return 0;
1102: }

1104: // adapt a level once. Forest in/out
1105: static PetscErrorCode adaptToleranceFEM(PetscFE fem, Vec sol, PetscInt type, PetscInt grid, LandauCtx *ctx, DM *newForest)
1106: {
1107:   DM               forest, plex, adaptedDM = NULL;
1108:   PetscDS          prob;
1109:   PetscBool        isForest;
1110:   PetscQuadrature  quad;
1111:   PetscInt         Nq, *Nb, cStart, cEnd, c, dim, qj, k;
1112:   DMLabel          adaptLabel = NULL;

1114:   forest = ctx->plex[grid];
1115:   DMCreateDS(forest);
1116:   DMGetDS(forest, &prob);
1117:   DMGetDimension(forest, &dim);
1118:   DMIsForest(forest, &isForest);
1120:   DMConvert(forest, DMPLEX, &plex);
1121:   DMPlexGetHeightStratum(plex,0,&cStart,&cEnd);
1122:   DMLabelCreate(PETSC_COMM_SELF,"adapt",&adaptLabel);
1123:   PetscFEGetQuadrature(fem, &quad);
1124:   PetscQuadratureGetData(quad, NULL, NULL, &Nq, NULL, NULL);
1126:   PetscDSGetDimensions(prob, &Nb);
1127:   if (type==4) {
1128:     for (c = cStart; c < cEnd; c++) {
1129:       DMLabelSetValue(adaptLabel, c, DM_ADAPT_REFINE);
1130:     }
1131:     PetscInfo(sol, "Phase:%s: Uniform refinement\n","adaptToleranceFEM");
1132:   } else if (type==2) {
1133:     PetscInt  rCellIdx[8], eCellIdx[64], iCellIdx[64], eMaxIdx = -1, iMaxIdx = -1, nr = 0, nrmax = (dim==3) ? 8 : 2;
1134:     PetscReal minRad = PETSC_INFINITY, r, eMinRad = PETSC_INFINITY, iMinRad = PETSC_INFINITY;
1135:     for (c = 0; c < 64; c++) { eCellIdx[c] = iCellIdx[c] = -1; }
1136:     for (c = cStart; c < cEnd; c++) {
1137:       PetscReal    tt, v0[LANDAU_MAX_NQ*3], detJ[LANDAU_MAX_NQ];
1138:       DMPlexComputeCellGeometryFEM(plex, c, quad, v0, NULL, NULL, detJ);
1139:       for (qj = 0; qj < Nq; ++qj) {
1140:         tt = PetscSqr(v0[dim*qj+0]) + PetscSqr(v0[dim*qj+1]) + PetscSqr(((dim==3) ? v0[dim*qj+2] : 0));
1141:         r  = PetscSqrtReal(tt);
1142:         if (r < minRad - PETSC_SQRT_MACHINE_EPSILON*10.) {
1143:           minRad = r;
1144:           nr     = 0;
1145:           rCellIdx[nr++]= c;
1146:           PetscInfo(sol, "\t\tPhase: adaptToleranceFEM Found first inner r=%e, cell %" PetscInt_FMT ", qp %" PetscInt_FMT "/%" PetscInt_FMT "\n", r, c, qj+1, Nq);
1147:         } else if ((r-minRad) < PETSC_SQRT_MACHINE_EPSILON*100. && nr < nrmax) {
1148:           for (k=0;k<nr;k++) if (c == rCellIdx[k]) break;
1149:           if (k==nr) {
1150:             rCellIdx[nr++]= c;
1151:             PetscInfo(sol, "\t\t\tPhase: adaptToleranceFEM Found another inner r=%e, cell %" PetscInt_FMT ", qp %" PetscInt_FMT "/%" PetscInt_FMT ", d=%e\n", r, c, qj+1, Nq, r-minRad);
1152:           }
1153:         }
1154:         if (ctx->sphere) {
1155:           if ((tt=r-ctx->e_radius) > 0) {
1156:             PetscInfo(sol, "\t\t\t %" PetscInt_FMT " cell r=%g\n",c,tt);
1157:             if (tt < eMinRad - PETSC_SQRT_MACHINE_EPSILON*100.) {
1158:               eMinRad = tt;
1159:               eMaxIdx = 0;
1160:               eCellIdx[eMaxIdx++] = c;
1161:             } else if (eMaxIdx > 0 && (tt-eMinRad) <= PETSC_SQRT_MACHINE_EPSILON && c != eCellIdx[eMaxIdx-1]) {
1162:               eCellIdx[eMaxIdx++] = c;
1163:             }
1164:           }
1165:           if ((tt=r-ctx->i_radius[grid]) > 0) {
1166:             if (tt < iMinRad - 1.e-5) {
1167:               iMinRad = tt;
1168:               iMaxIdx = 0;
1169:               iCellIdx[iMaxIdx++] = c;
1170:             } else if (iMaxIdx > 0 && (tt-iMinRad) <= PETSC_SQRT_MACHINE_EPSILON && c != iCellIdx[iMaxIdx-1]) {
1171:               iCellIdx[iMaxIdx++] = c;
1172:             }
1173:           }
1174:         }
1175:       }
1176:     }
1177:     for (k=0;k<nr;k++) {
1178:       DMLabelSetValue(adaptLabel, rCellIdx[k], DM_ADAPT_REFINE);
1179:     }
1180:     if (ctx->sphere) {
1181:       for (c = 0; c < eMaxIdx; c++) {
1182:         DMLabelSetValue(adaptLabel, eCellIdx[c], DM_ADAPT_REFINE);
1183:         PetscInfo(sol, "\t\tPhase:%s: refine sphere e cell %" PetscInt_FMT " r=%g\n","adaptToleranceFEM",eCellIdx[c],eMinRad);
1184:       }
1185:       for (c = 0; c < iMaxIdx; c++) {
1186:         DMLabelSetValue(adaptLabel, iCellIdx[c], DM_ADAPT_REFINE);
1187:         PetscInfo(sol, "\t\tPhase:%s: refine sphere i cell %" PetscInt_FMT " r=%g\n","adaptToleranceFEM",iCellIdx[c],iMinRad);
1188:       }
1189:     }
1190:     PetscInfo(sol, "Phase:%s: Adaptive refine origin cells %" PetscInt_FMT ",%" PetscInt_FMT " r=%g\n","adaptToleranceFEM",rCellIdx[0],rCellIdx[1],minRad);
1191:   } else if (type==0 || type==1 || type==3) { /* refine along r=0 axis */
1192:     PetscScalar  *coef = NULL;
1193:     Vec          coords;
1194:     PetscInt     csize,Nv,d,nz;
1195:     DM           cdm;
1196:     PetscSection cs;
1197:     DMGetCoordinatesLocal(forest, &coords);
1198:     DMGetCoordinateDM(forest, &cdm);
1199:     DMGetLocalSection(cdm, &cs);
1200:     for (c = cStart; c < cEnd; c++) {
1201:       PetscInt doit = 0, outside = 0;
1202:       DMPlexVecGetClosure(cdm, cs, coords, c, &csize, &coef);
1203:       Nv = csize/dim;
1204:       for (nz = d = 0; d < Nv; d++) {
1205:         PetscReal z = PetscRealPart(coef[d*dim + (dim-1)]), x = PetscSqr(PetscRealPart(coef[d*dim + 0])) + ((dim==3) ? PetscSqr(PetscRealPart(coef[d*dim + 1])) : 0);
1206:         x = PetscSqrtReal(x);
1207:         if (x < PETSC_MACHINE_EPSILON*10. && PetscAbs(z)<PETSC_MACHINE_EPSILON*10.) doit = 1;             /* refine origin */
1208:         else if (type==0 && (z < -PETSC_MACHINE_EPSILON*10. || z > ctx->re_radius+PETSC_MACHINE_EPSILON*10.)) outside++;   /* first pass don't refine bottom */
1209:         else if (type==1 && (z > ctx->vperp0_radius1 || z < -ctx->vperp0_radius1)) outside++; /* don't refine outside electron refine radius */
1210:         else if (type==3 && (z > ctx->vperp0_radius2 || z < -ctx->vperp0_radius2)) outside++; /* don't refine outside ion refine radius */
1211:         if (x < PETSC_MACHINE_EPSILON*10.) nz++;
1212:       }
1213:       DMPlexVecRestoreClosure(cdm, cs, coords, c, &csize, &coef);
1214:       if (doit || (outside<Nv && nz)) {
1215:         DMLabelSetValue(adaptLabel, c, DM_ADAPT_REFINE);
1216:       }
1217:     }
1218:     PetscInfo(sol, "Phase:%s: RE refinement\n","adaptToleranceFEM");
1219:   }
1220:   DMDestroy(&plex);
1221:   DMAdaptLabel(forest, adaptLabel, &adaptedDM);
1222:   DMLabelDestroy(&adaptLabel);
1223:   *newForest = adaptedDM;
1224:   if (adaptedDM) {
1225:     if (isForest) {
1226:       DMForestSetAdaptivityForest(adaptedDM,NULL); // ????
1227:     } else exit(33); // ???????
1228:     DMConvert(adaptedDM, DMPLEX, &plex);
1229:     DMPlexGetHeightStratum(plex,0,&cStart,&cEnd);
1230:     PetscInfo(sol, "\tPhase: adaptToleranceFEM: %" PetscInt_FMT " cells, %" PetscInt_FMT " total quadrature points\n",cEnd-cStart,Nq*(cEnd-cStart));
1231:     DMDestroy(&plex);
1232:   } else *newForest = NULL;
1233:   return 0;
1234: }

1236: // forest goes in (ctx->plex[grid]), plex comes out
1237: static PetscErrorCode adapt(PetscInt grid, LandauCtx *ctx, Vec *uu)
1238: {
1239:   PetscInt        adaptIter;

1241:   PetscInt  type, limits[5] = {(grid==0) ? ctx->numRERefine : 0, (grid==0) ? ctx->nZRefine1 : 0, ctx->numAMRRefine[grid], (grid==0) ? ctx->nZRefine2 : 0,ctx->postAMRRefine[grid]};
1242:   for (type=0;type<5;type++) {
1243:     for (adaptIter = 0; adaptIter<limits[type];adaptIter++) {
1244:       DM  newForest = NULL;
1245:       adaptToleranceFEM(ctx->fe[0], *uu, type, grid, ctx, &newForest);
1246:       if (newForest)  {
1247:         DMDestroy(&ctx->plex[grid]);
1248:         VecDestroy(uu);
1249:         DMCreateGlobalVector(newForest,uu);
1250:         PetscObjectSetName((PetscObject) *uu, "uAMR");
1251:         LandauSetInitialCondition(newForest, *uu, grid, 0, ctx);
1252:         ctx->plex[grid] = newForest;
1253:       } else {
1254:         exit(4); // can happen with no AMR and post refinement
1255:       }
1256:     }
1257:   }
1258:   return 0;
1259: }

1261: static PetscErrorCode ProcessOptions(LandauCtx *ctx, const char prefix[])
1262: {
1263:   PetscErrorCode    ierr;
1264:   PetscBool         flg, sph_flg;
1265:   PetscInt          ii,nt,nm,nc,num_species_grid[LANDAU_MAX_GRIDS];
1266:   PetscReal         v0_grid[LANDAU_MAX_GRIDS];
1267:   DM                dummy;

1269:   DMCreate(ctx->comm,&dummy);
1270:   /* get options - initialize context */
1271:   ctx->verbose = 1; // should be 0 for silent compliance
1272: #if defined(PETSC_HAVE_THREADSAFETY)
1273:   ctx->batch_sz = PetscNumOMPThreads;
1274: #else
1275:   ctx->batch_sz = 1;
1276: #endif
1277:   ctx->batch_view_idx = 0;
1278:   ctx->interpolate    = PETSC_TRUE;
1279:   ctx->gpu_assembly   = PETSC_TRUE;
1280:   ctx->aux_bool       = PETSC_FALSE;
1281:   ctx->electronShift  = 0;
1282:   ctx->M              = NULL;
1283:   ctx->J              = NULL;
1284:   /* geometry and grids */
1285:   ctx->sphere         = PETSC_FALSE;
1286:   ctx->inflate        = PETSC_FALSE;
1287:   ctx->aux_bool       = PETSC_FALSE;
1288:   ctx->use_p4est      = PETSC_FALSE;
1289:   ctx->num_sections   = 3; /* 2, 3 or 4 */
1290:   for (PetscInt grid=0;grid<LANDAU_MAX_GRIDS;grid++) {
1291:     ctx->radius[grid]           = 5.; /* thermal radius (velocity) */
1292:     ctx->numAMRRefine[grid]     = 5;
1293:     ctx->postAMRRefine[grid]    = 0;
1294:     ctx->species_offset[grid+1] = 1; // one species default
1295:     num_species_grid[grid]      = 0;
1296:     ctx->plex[grid] = NULL;     /* cache as expensive to Convert */
1297:   }
1298:   ctx->species_offset[0] = 0;
1299:   ctx->re_radius         = 0.;
1300:   ctx->vperp0_radius1    = 0;
1301:   ctx->vperp0_radius2    = 0;
1302:   ctx->nZRefine1         = 0;
1303:   ctx->nZRefine2         = 0;
1304:   ctx->numRERefine       = 0;
1305:   num_species_grid[0]    = 1; // one species default
1306:   /* species - [0] electrons, [1] one ion species eg, duetarium, [2] heavy impurity ion, ... */
1307:   ctx->charges[0]        = -1;  /* electron charge (MKS) */
1308:   ctx->masses[0]         = 1/1835.469965278441013; /* temporary value in proton mass */
1309:   ctx->n[0]              = 1;
1310:   ctx->v_0               = 1; /* thermal velocity, we could start with a scale != 1 */
1311:   ctx->thermal_temps[0]  = 1;
1312:   /* constants, etc. */
1313:   ctx->epsilon0          = 8.8542e-12; /* permittivity of free space (MKS) F/m */
1314:   ctx->k                 = 1.38064852e-23; /* Boltzmann constant (MKS) J/K */
1315:   ctx->lnLam             = 10;         /* cross section ratio large - small angle collisions */
1316:   ctx->n_0               = 1.e20;        /* typical plasma n, but could set it to 1 */
1317:   ctx->Ez                = 0;
1318:   for (PetscInt grid=0;grid<LANDAU_NUM_TIMERS;grid++) ctx->times[grid] = 0;
1319:   ctx->use_matrix_mass   =  PETSC_FALSE;
1320:   ctx->use_relativistic_corrections = PETSC_FALSE;
1321:   ctx->use_energy_tensor_trick      = PETSC_FALSE; /* Use Eero's trick for energy conservation v --> grad(v^2/2) */
1322:   ctx->SData_d.w         = NULL;
1323:   ctx->SData_d.x         = NULL;
1324:   ctx->SData_d.y         = NULL;
1325:   ctx->SData_d.z         = NULL;
1326:   ctx->SData_d.invJ      = NULL;
1327:   ctx->jacobian_field_major_order     = PETSC_FALSE;
1328:   ctx->SData_d.coo_elem_offsets       = NULL;
1329:   ctx->SData_d.coo_elem_point_offsets = NULL;
1330:   ctx->coo_assembly                   = PETSC_FALSE;
1331:   ctx->SData_d.coo_elem_fullNb        = NULL;
1332:   ctx->SData_d.coo_size               = 0;
1333:   PetscOptionsBegin(ctx->comm, prefix, "Options for Fokker-Plank-Landau collision operator", "none");
1334:   {
1335:     char opstring[256];
1336: #if defined(PETSC_HAVE_KOKKOS_KERNELS)
1337:     ctx->deviceType = LANDAU_KOKKOS;
1338:     PetscStrcpy(opstring,"kokkos");
1339: #elif defined(PETSC_HAVE_CUDA)
1340:     ctx->deviceType = LANDAU_CUDA;
1341:     PetscStrcpy(opstring,"cuda");
1342: #else
1343:     ctx->deviceType = LANDAU_CPU;
1344:     PetscStrcpy(opstring,"cpu");
1345: #endif
1346:     PetscOptionsString("-dm_landau_device_type","Use kernels on 'cpu', 'cuda', or 'kokkos'","plexland.c",opstring,opstring,sizeof(opstring),NULL);
1347:     PetscStrcmp("cpu",opstring,&flg);
1348:     if (flg) {
1349:       ctx->deviceType = LANDAU_CPU;
1350:     } else {
1351:       PetscStrcmp("cuda",opstring,&flg);
1352:       if (flg) {
1353:         ctx->deviceType = LANDAU_CUDA;
1354:       } else {
1355:         PetscStrcmp("kokkos",opstring,&flg);
1356:         if (flg) ctx->deviceType = LANDAU_KOKKOS;
1357:         else SETERRQ(ctx->comm,PETSC_ERR_ARG_WRONG,"-dm_landau_device_type %s",opstring);
1358:       }
1359:     }
1360:   }
1361:   PetscOptionsReal("-dm_landau_electron_shift","Shift in thermal velocity of electrons","none",ctx->electronShift,&ctx->electronShift, NULL);
1362:   PetscOptionsInt("-dm_landau_verbose", "Level of verbosity output", "plexland.c", ctx->verbose, &ctx->verbose, NULL);
1363:   PetscOptionsInt("-dm_landau_batch_size", "Number of 'vertices' to batch", "ex2.c", ctx->batch_sz, &ctx->batch_sz, NULL);
1365:   PetscOptionsInt("-dm_landau_batch_view_idx", "Index of batch for diagnostics like plotting", "ex2.c", ctx->batch_view_idx, &ctx->batch_view_idx, NULL);
1367:   PetscOptionsReal("-dm_landau_Ez","Initial parallel electric field in unites of Conner-Hastie critical field","plexland.c",ctx->Ez,&ctx->Ez, NULL);
1368:   PetscOptionsReal("-dm_landau_n_0","Normalization constant for number density","plexland.c",ctx->n_0,&ctx->n_0, NULL);
1369:   PetscOptionsReal("-dm_landau_ln_lambda","Cross section parameter","plexland.c",ctx->lnLam,&ctx->lnLam, NULL);
1370:   PetscOptionsBool("-dm_landau_use_mataxpy_mass", "Use fast but slightly fragile MATAXPY to add mass term", "plexland.c", ctx->use_matrix_mass, &ctx->use_matrix_mass, NULL);
1371:   PetscOptionsBool("-dm_landau_use_relativistic_corrections", "Use relativistic corrections", "plexland.c", ctx->use_relativistic_corrections, &ctx->use_relativistic_corrections, NULL);
1372:   PetscOptionsBool("-dm_landau_use_energy_tensor_trick", "Use Eero's trick of using grad(v^2/2) instead of v as args to Landau tensor to conserve energy with relativistic corrections and Q1 elements", "plexland.c", ctx->use_energy_tensor_trick, &ctx->use_energy_tensor_trick, NULL);

1374:   /* get num species with temperature, set defaults */
1375:   for (ii=1;ii<LANDAU_MAX_SPECIES;ii++) {
1376:     ctx->thermal_temps[ii] = 1;
1377:     ctx->charges[ii]       = 1;
1378:     ctx->masses[ii]        = 1;
1379:     ctx->n[ii]             = 1;
1380:   }
1381:   nt = LANDAU_MAX_SPECIES;
1382:   PetscOptionsRealArray("-dm_landau_thermal_temps", "Temperature of each species [e,i_0,i_1,...] in keV (must be set to set number of species)", "plexland.c", ctx->thermal_temps, &nt, &flg);
1383:   if (flg) {
1384:     PetscInfo(dummy, "num_species set to number of thermal temps provided (%" PetscInt_FMT ")\n",nt);
1385:     ctx->num_species = nt;
1386:   } else SETERRQ(ctx->comm,PETSC_ERR_ARG_WRONG,"-dm_landau_thermal_temps ,t1,t2,.. must be provided to set the number of species");
1387:   for (ii=0;ii<ctx->num_species;ii++) ctx->thermal_temps[ii] *= 1.1604525e7; /* convert to Kelvin */
1388:   nm = LANDAU_MAX_SPECIES-1;
1389:   PetscOptionsRealArray("-dm_landau_ion_masses", "Mass of each species in units of proton mass [i_0=2,i_1=40...]", "plexland.c", &ctx->masses[1], &nm, &flg);
1390:   if (flg && nm != ctx->num_species-1) {
1391:     SETERRQ(ctx->comm,PETSC_ERR_ARG_WRONG,"num ion masses %" PetscInt_FMT " != num species %" PetscInt_FMT "",nm,ctx->num_species-1);
1392:   }
1393:   nm = LANDAU_MAX_SPECIES;
1394:   PetscOptionsRealArray("-dm_landau_n", "Number density of each species = n_s * n_0", "plexland.c", ctx->n, &nm, &flg);
1396:   for (ii=0;ii<LANDAU_MAX_SPECIES;ii++) ctx->masses[ii] *= 1.6720e-27; /* scale by proton mass kg */
1397:   ctx->masses[0] = 9.10938356e-31; /* electron mass kg (should be about right already) */
1398:   ctx->m_0 = ctx->masses[0]; /* arbitrary reference mass, electrons */
1399:   nc = LANDAU_MAX_SPECIES-1;
1400:   PetscOptionsRealArray("-dm_landau_ion_charges", "Charge of each species in units of proton charge [i_0=2,i_1=18,...]", "plexland.c", &ctx->charges[1], &nc, &flg);
1402:   for (ii=0;ii<LANDAU_MAX_SPECIES;ii++) ctx->charges[ii] *= 1.6022e-19; /* electron/proton charge (MKS) */
1403:   /* geometry and grids */
1404:   nt = LANDAU_MAX_GRIDS;
1405:   PetscOptionsIntArray("-dm_landau_num_species_grid","Number of species on each grid: [ 1, ....] or [S, 0 ....] for single grid","plexland.c", num_species_grid, &nt, &flg);
1406:   if (flg) {
1407:     ctx->num_grids = nt;
1408:     for (ii=nt=0;ii<ctx->num_grids;ii++) nt += num_species_grid[ii];
1410:   } else {
1411:     ctx->num_grids = 1; // go back to a single grid run
1412:     num_species_grid[0] = ctx->num_species;
1413:   }
1414:   for (ctx->species_offset[0] = ii = 0; ii < ctx->num_grids ; ii++) ctx->species_offset[ii+1] = ctx->species_offset[ii] + num_species_grid[ii];
1416:   for (PetscInt grid = 0; grid < ctx->num_grids ; grid++) {
1417:     int iii = ctx->species_offset[grid]; // normalize with first (arbitrary) species on grid
1418:     v0_grid[grid] = PetscSqrtReal(ctx->k*ctx->thermal_temps[iii]/ctx->masses[iii]); /* arbitrary units for non-dimensionalization: mean velocity in 1D of first species on grid */
1419:   }
1420:   ii = 0;
1421:   PetscOptionsInt("-dm_landau_v0_grid", "Index of grid to use for setting v_0 (electrons are default). Not recommended to change", "plexland.c", ii, &ii, NULL);
1422:   ctx->v_0 = v0_grid[ii]; /* arbitrary units for non dimensionalization: global mean velocity in 1D of electrons */
1423:   ctx->t_0 = 8*PETSC_PI*PetscSqr(ctx->epsilon0*ctx->m_0/PetscSqr(ctx->charges[0]))/ctx->lnLam/ctx->n_0*PetscPowReal(ctx->v_0,3); /* note, this t_0 makes nu[0,0]=1 */
1424:   /* domain */
1425:   nt = LANDAU_MAX_GRIDS;
1426:   PetscOptionsRealArray("-dm_landau_domain_radius","Phase space size in units of thermal velocity of grid","plexland.c",ctx->radius,&nt, &flg);
1428:   for (PetscInt grid = 0; grid < ctx->num_grids ; grid++) {
1429:     if (flg && ctx->radius[grid] <= 0) { /* negative is ratio of c */
1430:       if (ctx->radius[grid] == 0) ctx->radius[grid] = 0.75;
1431:       else ctx->radius[grid] = -ctx->radius[grid];
1432:       ctx->radius[grid] = ctx->radius[grid]*SPEED_OF_LIGHT/ctx->v_0; // use any species on grid to normalize (v_0 same for all on grid)
1433:       PetscInfo(dummy, "Change domain radius to %g for grid %" PetscInt_FMT "\n",ctx->radius[grid],grid);
1434:     }
1435:     ctx->radius[grid] *= v0_grid[grid]/ctx->v_0; // scale domain by thermal radius relative to v_0
1436:   }
1437:   /* amr parametres */
1438:   nt = LANDAU_MAX_GRIDS;
1439:   PetscOptionsIntArray("-dm_landau_amr_levels_max", "Number of AMR levels of refinement around origin, after (RE) refinements along z", "plexland.c", ctx->numAMRRefine, &nt, &flg);
1441:   nt = LANDAU_MAX_GRIDS;
1442:   PetscOptionsIntArray("-dm_landau_amr_post_refine", "Number of levels to uniformly refine after AMR", "plexland.c", ctx->postAMRRefine, &nt, &flg);
1443:   for (ii=1;ii<ctx->num_grids;ii++)  ctx->postAMRRefine[ii] = ctx->postAMRRefine[0]; // all grids the same now
1444:   PetscOptionsInt("-dm_landau_amr_re_levels", "Number of levels to refine along v_perp=0, z>0", "plexland.c", ctx->numRERefine, &ctx->numRERefine, &flg);
1445:   PetscOptionsInt("-dm_landau_amr_z_refine1",  "Number of levels to refine along v_perp=0", "plexland.c", ctx->nZRefine1, &ctx->nZRefine1, &flg);
1446:   PetscOptionsInt("-dm_landau_amr_z_refine2",  "Number of levels to refine along v_perp=0", "plexland.c", ctx->nZRefine2, &ctx->nZRefine2, &flg);
1447:   PetscOptionsReal("-dm_landau_re_radius","velocity range to refine on positive (z>0) r=0 axis for runaways","plexland.c",ctx->re_radius,&ctx->re_radius, &flg);
1448:   PetscOptionsReal("-dm_landau_z_radius1","velocity range to refine r=0 axis (for electrons)","plexland.c",ctx->vperp0_radius1,&ctx->vperp0_radius1, &flg);
1449:   PetscOptionsReal("-dm_landau_z_radius2","velocity range to refine r=0 axis (for ions) after origin AMR","plexland.c",ctx->vperp0_radius2, &ctx->vperp0_radius2, &flg);
1450:   /* spherical domain (not used) */
1451:   PetscOptionsInt("-dm_landau_num_sections", "Number of tangential section in (2D) grid, 2, 3, of 4", "plexland.c", ctx->num_sections, &ctx->num_sections, NULL);
1452:   PetscOptionsBool("-dm_landau_sphere", "use sphere/semi-circle domain instead of rectangle", "plexland.c", ctx->sphere, &ctx->sphere, &sph_flg);
1453:   PetscOptionsBool("-dm_landau_inflate", "With sphere, inflate for curved edges", "plexland.c", ctx->inflate, &ctx->inflate, &flg);
1454:   PetscOptionsReal("-dm_landau_e_radius","Electron thermal velocity, used for circular meshes","plexland.c",ctx->e_radius, &ctx->e_radius, &flg);
1455:   if (flg && !sph_flg) ctx->sphere = PETSC_TRUE; /* you gave me an e radius but did not set sphere, user error really */
1456:   if (!flg) {
1457:     ctx->e_radius = 1.5*PetscSqrtReal(8*ctx->k*ctx->thermal_temps[0]/ctx->masses[0]/PETSC_PI)/ctx->v_0;
1458:   }
1459:   nt = LANDAU_MAX_GRIDS;
1460:   PetscOptionsRealArray("-dm_landau_i_radius","Ion thermal velocity, used for circular meshes","plexland.c",ctx->i_radius, &nt, &flg);
1461:   if (flg && !sph_flg) ctx->sphere = PETSC_TRUE;
1462:   if (!flg) {
1463:     ctx->i_radius[0] = 1.5*PetscSqrtReal(8*ctx->k*ctx->thermal_temps[1]/ctx->masses[1]/PETSC_PI)/ctx->v_0; // need to correct for ion grid domain
1464:   }
1467:   /* processing options */
1468:   PetscOptionsBool("-dm_landau_gpu_assembly", "Assemble Jacobian on GPU", "plexland.c", ctx->gpu_assembly, &ctx->gpu_assembly, NULL);
1469:   if (ctx->deviceType == LANDAU_CPU || ctx->deviceType == LANDAU_KOKKOS) { // make Kokkos
1470:     PetscOptionsBool("-dm_landau_coo_assembly", "Assemble Jacobian with Kokkos on 'device'", "plexland.c", ctx->coo_assembly, &ctx->coo_assembly, NULL);
1472:   }
1473:   PetscOptionsBool("-dm_landau_jacobian_field_major_order", "Reorder Jacobian for GPU assembly with field major, or block diagonal, ordering", "plexland.c", ctx->jacobian_field_major_order, &ctx->jacobian_field_major_order, NULL);
1475:   PetscOptionsEnd();

1477:   for (ii=ctx->num_species;ii<LANDAU_MAX_SPECIES;ii++) ctx->masses[ii] = ctx->thermal_temps[ii]  = ctx->charges[ii] = 0;
1478:   if (ctx->verbose > 0) {
1479:     PetscPrintf(ctx->comm, "masses:        e=%10.3e; ions in proton mass units:   %10.3e %10.3e ...\n",ctx->masses[0],ctx->masses[1]/1.6720e-27,ctx->num_species>2 ? ctx->masses[2]/1.6720e-27 : 0);
1480:     PetscPrintf(ctx->comm, "charges:       e=%10.3e; charges in elementary units: %10.3e %10.3e\n", ctx->charges[0],-ctx->charges[1]/ctx->charges[0],ctx->num_species>2 ? -ctx->charges[2]/ctx->charges[0] : 0);
1481:     PetscPrintf(ctx->comm, "n:             e: %10.3e                           i: %10.3e %10.3e\n", ctx->n[0],ctx->n[1],ctx->num_species>2 ? ctx->n[2] : 0);
1482:     PetscPrintf(ctx->comm, "thermal T (K): e=%10.3e i=%10.3e %10.3e. v_0=%10.3e (%10.3ec) n_0=%10.3e t_0=%10.3e, %s, %s, %" PetscInt_FMT " batched\n", ctx->thermal_temps[0], ctx->thermal_temps[1], (ctx->num_species>2) ? ctx->thermal_temps[2] : 0, ctx->v_0, ctx->v_0/SPEED_OF_LIGHT, ctx->n_0, ctx->t_0, ctx->use_relativistic_corrections ? "relativistic" : "classical", ctx->use_energy_tensor_trick ? "Use trick" : "Intuitive",ctx->batch_sz);
1483:     PetscPrintf(ctx->comm, "Domain radius (AMR levels) grid %" PetscInt_FMT ": %10.3e (%" PetscInt_FMT ") ",0,ctx->radius[0],ctx->numAMRRefine[0]);
1484:     for (ii=1;ii<ctx->num_grids;ii++) PetscPrintf(ctx->comm, ", %" PetscInt_FMT ": %10.3e (%" PetscInt_FMT ") ",ii,ctx->radius[ii],ctx->numAMRRefine[ii]);
1485:     PetscPrintf(ctx->comm,"\n");
1486:     if (ctx->jacobian_field_major_order) {
1487:       PetscPrintf(ctx->comm,"Using field major order for GPU Jacobian\n");
1488:     } else {
1489:       PetscPrintf(ctx->comm,"Using default Plex order for all matrices\n");
1490:     }
1491:   }
1492:   DMDestroy(&dummy);
1493:   {
1494:     PetscMPIInt    rank;
1495:     MPI_Comm_rank(ctx->comm, &rank);
1496:     ctx->stage = 0;
1497:     PetscLogEventRegister("Landau Create", DM_CLASSID, &ctx->events[13]); /* 13 */
1498:     PetscLogEventRegister(" GPU ass. setup", DM_CLASSID, &ctx->events[2]); /* 2 */
1499:     PetscLogEventRegister(" Build matrix", DM_CLASSID, &ctx->events[12]); /* 12 */
1500:     PetscLogEventRegister(" Assembly maps", DM_CLASSID, &ctx->events[15]); /* 15 */
1501:     PetscLogEventRegister("Landau Mass mat", DM_CLASSID, &ctx->events[14]); /* 14 */
1502:     PetscLogEventRegister("Landau Operator", DM_CLASSID, &ctx->events[11]); /* 11 */
1503:     PetscLogEventRegister("Landau Jacobian", DM_CLASSID, &ctx->events[0]); /* 0 */
1504:     PetscLogEventRegister("Landau Mass", DM_CLASSID, &ctx->events[9]); /* 9 */
1505:     PetscLogEventRegister(" Preamble", DM_CLASSID, &ctx->events[10]); /* 10 */
1506:     PetscLogEventRegister(" static IP Data", DM_CLASSID, &ctx->events[7]); /* 7 */
1507:     PetscLogEventRegister(" dynamic IP-Jac", DM_CLASSID, &ctx->events[1]); /* 1 */
1508:     PetscLogEventRegister(" Kernel-init", DM_CLASSID, &ctx->events[3]); /* 3 */
1509:     PetscLogEventRegister(" Jac-f-df (GPU)", DM_CLASSID, &ctx->events[8]); /* 8 */
1510:     PetscLogEventRegister(" J Kernel (GPU)", DM_CLASSID, &ctx->events[4]); /* 4 */
1511:     PetscLogEventRegister(" M Kernel (GPU)", DM_CLASSID, &ctx->events[16]); /* 16 */
1512:     PetscLogEventRegister(" Copy to CPU", DM_CLASSID, &ctx->events[5]); /* 5 */
1513:     PetscLogEventRegister(" CPU assemble", DM_CLASSID, &ctx->events[6]); /* 6 */

1515:     if (rank) { /* turn off output stuff for duplicate runs - do we need to add the prefix to all this? */
1516:       PetscOptionsClearValue(NULL,"-snes_converged_reason");
1517:       PetscOptionsClearValue(NULL,"-ksp_converged_reason");
1518:       PetscOptionsClearValue(NULL,"-snes_monitor");
1519:       PetscOptionsClearValue(NULL,"-ksp_monitor");
1520:       PetscOptionsClearValue(NULL,"-ts_monitor");
1521:       PetscOptionsClearValue(NULL,"-ts_view");
1522:       PetscOptionsClearValue(NULL,"-ts_adapt_monitor");
1523:       PetscOptionsClearValue(NULL,"-dm_landau_amr_dm_view");
1524:       PetscOptionsClearValue(NULL,"-dm_landau_amr_vec_view");
1525:       PetscOptionsClearValue(NULL,"-dm_landau_mass_dm_view");
1526:       PetscOptionsClearValue(NULL,"-dm_landau_mass_view");
1527:       PetscOptionsClearValue(NULL,"-dm_landau_jacobian_view");
1528:       PetscOptionsClearValue(NULL,"-dm_landau_mat_view");
1529:       PetscOptionsClearValue(NULL,"-pc_bjkokkos_ksp_converged_reason");
1530:       PetscOptionsClearValue(NULL,"-pc_bjkokkos_ksp_monitor");
1531:       PetscOptionsClearValue(NULL,"-");
1532:       PetscOptionsClearValue(NULL,"-info");
1533:     }
1534:   }
1535:   return 0;
1536: }

1538: static PetscErrorCode CreateStaticGPUData(PetscInt dim, IS grid_batch_is_inv[], LandauCtx *ctx)
1539: {
1540:   PetscSection      section[LANDAU_MAX_GRIDS],globsection[LANDAU_MAX_GRIDS];
1541:   PetscQuadrature   quad;
1542:   const PetscReal   *quadWeights;
1543:   PetscInt          numCells[LANDAU_MAX_GRIDS],Nq,Nf[LANDAU_MAX_GRIDS], ncellsTot=0;
1544:   PetscTabulation   *Tf;
1545:   PetscDS           prob;

1547:   DMGetDS(ctx->plex[0], &prob); // same DS for all grids
1548:   PetscDSGetTabulation(prob, &Tf); // Bf, &Df same for all grids
1549:   /* DS, Tab and quad is same on all grids */
1551:   PetscFEGetQuadrature(ctx->fe[0], &quad);
1552:   PetscQuadratureGetData(quad, NULL, NULL, &Nq, NULL,  &quadWeights);
1554:   /* setup each grid */
1555:   for (PetscInt grid=0;grid<ctx->num_grids;grid++) {
1556:     PetscInt cStart, cEnd;
1558:     DMPlexGetHeightStratum(ctx->plex[grid], 0, &cStart, &cEnd);
1559:     numCells[grid] = cEnd - cStart; // grids can have different topology
1560:     DMGetLocalSection(ctx->plex[grid], &section[grid]);
1561:     DMGetGlobalSection(ctx->plex[grid], &globsection[grid]);
1562:     PetscSectionGetNumFields(section[grid], &Nf[grid]);
1563:     ncellsTot += numCells[grid];
1564:   }
1565: #define MAP_BF_SIZE (64*LANDAU_DIM*LANDAU_DIM*LANDAU_MAX_Q_FACE*LANDAU_MAX_SPECIES)
1566:   /* create GPU assembly data */
1567:   if (ctx->gpu_assembly) { /* we need GPU object with GPU assembly */
1568:     PetscContainer          container;
1569:     PetscScalar             elemMatrix[LANDAU_MAX_NQ*LANDAU_MAX_NQ*LANDAU_MAX_SPECIES*LANDAU_MAX_SPECIES], *elMat;
1570:     pointInterpolationP4est pointMaps[MAP_BF_SIZE][LANDAU_MAX_Q_FACE];
1571:     P4estVertexMaps         *maps;
1572:     const PetscInt          *plex_batch=NULL,Nb=Nq; // tensor elements;
1573:     LandauIdx               *coo_elem_offsets=NULL, *coo_elem_fullNb=NULL, (*coo_elem_point_offsets)[LANDAU_MAX_NQ+1] = NULL;
1574:     /* create GPU asssembly data */
1575:     PetscInfo(ctx->plex[0], "Make GPU maps %d\n",1);
1576:     PetscLogEventBegin(ctx->events[2],0,0,0,0);
1577:     PetscMalloc(sizeof(*maps)*ctx->num_grids, &maps);

1579:     if (ctx->coo_assembly) { // setup COO assembly -- put COO metadata directly in ctx->SData_d
1580:       PetscMalloc3(ncellsTot+1,&coo_elem_offsets,ncellsTot,&coo_elem_fullNb,ncellsTot, &coo_elem_point_offsets); // array of integer pointers
1581:       coo_elem_offsets[0] = 0; // finish later
1582:       PetscInfo(ctx->plex[0], "COO initialization, %" PetscInt_FMT " cells\n",ncellsTot);
1583:       ctx->SData_d.coo_n_cellsTot         = ncellsTot;
1584:       ctx->SData_d.coo_elem_offsets       = (void*)coo_elem_offsets;
1585:       ctx->SData_d.coo_elem_fullNb        = (void*)coo_elem_fullNb;
1586:       ctx->SData_d.coo_elem_point_offsets = (void*)coo_elem_point_offsets;
1587:     } else {
1588:       ctx->SData_d.coo_elem_offsets       = ctx->SData_d.coo_elem_fullNb = NULL;
1589:       ctx->SData_d.coo_elem_point_offsets = NULL;
1590:       ctx->SData_d.coo_n_cellsTot         = 0;
1591:     }

1593:     ctx->SData_d.coo_max_fullnb = 0;
1594:     for (PetscInt grid=0,glb_elem_idx=0;grid<ctx->num_grids;grid++) {
1595:       PetscInt cStart, cEnd, Nfloc = Nf[grid], totDim = Nfloc*Nq;
1596:       if (grid_batch_is_inv[grid]) {
1597:         ISGetIndices(grid_batch_is_inv[grid], &plex_batch);
1598:       }
1599:       DMPlexGetHeightStratum(ctx->plex[grid], 0, &cStart, &cEnd);
1600:       // make maps
1601:       maps[grid].d_self       = NULL;
1602:       maps[grid].num_elements = numCells[grid];
1603:       maps[grid].num_face = (PetscInt)(pow(Nq,1./((double)dim))+.001); // Q
1604:       maps[grid].num_face = (PetscInt)(pow(maps[grid].num_face,(double)(dim-1))+.001); // Q^2
1605:       maps[grid].num_reduced  = 0;
1606:       maps[grid].deviceType   = ctx->deviceType;
1607:       maps[grid].numgrids     = ctx->num_grids;
1608:       // count reduced and get
1609:       PetscMalloc(maps[grid].num_elements * sizeof(*maps[grid].gIdx), &maps[grid].gIdx);
1610:       for (int ej = cStart, eidx = 0 ; ej < cEnd; ++ej, ++eidx, glb_elem_idx++) {
1611:         if (coo_elem_offsets) coo_elem_offsets[glb_elem_idx+1] = coo_elem_offsets[glb_elem_idx]; // start with last one, then add
1612:         for (int fieldA=0;fieldA<Nf[grid];fieldA++) {
1613:           int fullNb = 0;
1614:           for (int q = 0; q < Nb; ++q) {
1615:             PetscInt    numindices,*indices;
1616:             PetscScalar *valuesOrig = elMat = elemMatrix;
1617:             PetscArrayzero(elMat, totDim*totDim);
1618:             elMat[ (fieldA*Nb + q)*totDim + fieldA*Nb + q] = 1;
1619:             DMPlexGetClosureIndices(ctx->plex[grid], section[grid], globsection[grid], ej, PETSC_TRUE, &numindices, &indices, NULL, (PetscScalar **) &elMat);
1620:             for (PetscInt f = 0 ; f < numindices ; ++f) { // look for a non-zero on the diagonal
1621:               if (PetscAbs(PetscRealPart(elMat[f*numindices + f])) > PETSC_MACHINE_EPSILON) {
1622:                 // found it
1623:                 if (PetscAbs(PetscRealPart(elMat[f*numindices + f] - 1.)) < PETSC_MACHINE_EPSILON) { // normal vertex 1.0
1624:                   if (plex_batch) {
1625:                     maps[grid].gIdx[eidx][fieldA][q] = (LandauIdx) plex_batch[indices[f]];
1626:                   } else {
1627:                     maps[grid].gIdx[eidx][fieldA][q] = (LandauIdx)indices[f];
1628:                   }
1629:                   fullNb++;
1630:                 } else { //found a constraint
1631:                   int       jj      = 0;
1632:                   PetscReal sum     = 0;
1633:                   const PetscInt ff = f;
1634:                   maps[grid].gIdx[eidx][fieldA][q] = -maps[grid].num_reduced - 1; // store (-)index: id = -(idx+1): idx = -id - 1

1636:                   do {  // constraints are continuous in Plex - exploit that here
1637:                     int ii; // get 'scale'
1638:                     for (ii = 0, pointMaps[maps[grid].num_reduced][jj].scale = 0; ii < maps[grid].num_face; ii++) { // sum row of outer product to recover vector value
1639:                       if (ff + ii < numindices) { // 3D has Q and Q^2 interps so might run off end. We could test that elMat[f*numindices + ff + ii] > 0, and break if not
1640:                         pointMaps[maps[grid].num_reduced][jj].scale += PetscRealPart(elMat[f*numindices + ff + ii]);
1641:                       }
1642:                     }
1643:                     sum += pointMaps[maps[grid].num_reduced][jj].scale; // diagnostic
1644:                     // get 'gid'
1645:                     if (pointMaps[maps[grid].num_reduced][jj].scale == 0) pointMaps[maps[grid].num_reduced][jj].gid = -1; // 3D has Q and Q^2 interps
1646:                     else {
1647:                       if (plex_batch) {
1648:                         pointMaps[maps[grid].num_reduced][jj].gid = plex_batch[indices[f]];
1649:                       } else {
1650:                         pointMaps[maps[grid].num_reduced][jj].gid = indices[f];
1651:                       }
1652:                       fullNb++;
1653:                     }
1654:                   } while (++jj < maps[grid].num_face && ++f < numindices); // jj is incremented if we hit the end
1655:                   while (jj < maps[grid].num_face) {
1656:                     pointMaps[maps[grid].num_reduced][jj].scale = 0;
1657:                     pointMaps[maps[grid].num_reduced][jj].gid = -1;
1658:                     jj++;
1659:                   }
1660:                   if (PetscAbs(sum-1.0) > 10*PETSC_MACHINE_EPSILON) { // debug
1661:                     int       d,f;
1662:                     PetscReal tmp = 0;
1663:                     PetscPrintf(PETSC_COMM_SELF,"\t\t%" PetscInt_FMT ".%" PetscInt_FMT ".%" PetscInt_FMT ") ERROR total I = %22.16e (LANDAU_MAX_Q_FACE=%d, #face=%" PetscInt_FMT ")\n",eidx,q,fieldA,sum,LANDAU_MAX_Q_FACE,maps[grid].num_face);
1664:                     for (d = 0, tmp = 0; d < numindices; ++d) {
1665:                       if (tmp!=0 && PetscAbs(tmp-1.0) > 10*PETSC_MACHINE_EPSILON) PetscPrintf(PETSC_COMM_WORLD,"%3" PetscInt_FMT ") %3" PetscInt_FMT ": ",d,indices[d]);
1666:                       for (f = 0; f < numindices; ++f) {
1667:                         tmp += PetscRealPart(elMat[d*numindices + f]);
1668:                       }
1669:                       if (tmp!=0) PetscPrintf(ctx->comm," | %22.16e\n",tmp);
1670:                     }
1671:                   }
1672:                   maps[grid].num_reduced++;
1674:                 }
1675:                 break;
1676:               }
1677:             }
1678:             // cleanup
1679:             DMPlexRestoreClosureIndices(ctx->plex[grid], section[grid], globsection[grid], ej, PETSC_TRUE, &numindices, &indices, NULL, (PetscScalar **) &elMat);
1680:             if (elMat != valuesOrig) DMRestoreWorkArray(ctx->plex[grid], numindices*numindices, MPIU_SCALAR, &elMat);
1681:           }
1682:           if (ctx->coo_assembly) { // setup COO assembly
1683:             coo_elem_offsets[glb_elem_idx+1] += fullNb*fullNb; // one species block, adds a block for each species, on this element in this grid
1684:             if (fieldA==0) { // cache full Nb for this element, on this grid per species
1685:               coo_elem_fullNb[glb_elem_idx] = fullNb;
1686:               if (fullNb>ctx->SData_d.coo_max_fullnb) ctx->SData_d.coo_max_fullnb = fullNb;
1688:           }
1689:         } // field
1690:       } // cell
1691:       // allocate and copy point data maps[grid].gIdx[eidx][field][q]
1692:       PetscMalloc(maps[grid].num_reduced * sizeof(*maps[grid].c_maps), &maps[grid].c_maps);
1693:       for (int ej = 0; ej < maps[grid].num_reduced; ++ej) {
1694:         for (int q = 0; q < maps[grid].num_face; ++q) {
1695:           maps[grid].c_maps[ej][q].scale = pointMaps[ej][q].scale;
1696:           maps[grid].c_maps[ej][q].gid   = pointMaps[ej][q].gid;
1697:         }
1698:       }
1699: #if defined(PETSC_HAVE_KOKKOS_KERNELS)
1700:       if (ctx->deviceType == LANDAU_KOKKOS) {
1701:         LandauKokkosCreateMatMaps(maps, pointMaps, Nf, Nq, grid); // imples Kokkos does
1702:       } // else could be CUDA
1703: #endif
1704: #if defined(PETSC_HAVE_CUDA)
1705:       if (ctx->deviceType == LANDAU_CUDA) {
1706:         LandauCUDACreateMatMaps(maps, pointMaps, Nf, Nq, grid);
1707:       }
1708: #endif
1709:       if (plex_batch) {
1710:         ISRestoreIndices(grid_batch_is_inv[grid], &plex_batch);
1711:         ISDestroy(&grid_batch_is_inv[grid]); // we are done with this
1712:       }
1713:     } /* grids */
1714:     // finish COO
1715:     if (ctx->coo_assembly) { // setup COO assembly
1716:       PetscInt *oor, *ooc;
1717:       ctx->SData_d.coo_size = coo_elem_offsets[ncellsTot]*ctx->batch_sz;
1718:       PetscMalloc2(ctx->SData_d.coo_size,&oor,ctx->SData_d.coo_size,&ooc);
1719:       for (int i=0;i<ctx->SData_d.coo_size;i++) oor[i] = ooc[i] = -1;
1720:       // get
1721:       for (int grid=0,glb_elem_idx=0;grid<ctx->num_grids;grid++) {
1722:         for (int ej = 0 ; ej < numCells[grid] ; ++ej, glb_elem_idx++) {
1723:           const int              fullNb = coo_elem_fullNb[glb_elem_idx];
1724:           const LandauIdx *const Idxs = &maps[grid].gIdx[ej][0][0]; // just use field-0 maps, They should be the same but this is just for COO storage
1725:           coo_elem_point_offsets[glb_elem_idx][0] = 0;
1726:           for (int f=0, cnt2=0;f<Nb;f++) {
1727:             int idx = Idxs[f];
1728:             coo_elem_point_offsets[glb_elem_idx][f+1] = coo_elem_point_offsets[glb_elem_idx][f]; // start at last
1729:             if (idx >= 0) {
1730:               cnt2++;
1731:               coo_elem_point_offsets[glb_elem_idx][f+1]++; // inc
1732:             } else {
1733:               idx = -idx - 1;
1734:               for (int q = 0 ; q < maps[grid].num_face; q++) {
1735:                 if (maps[grid].c_maps[idx][q].gid < 0) break;
1736:                 cnt2++;
1737:                 coo_elem_point_offsets[glb_elem_idx][f+1]++; // inc
1738:               }
1739:             }
1741:           }
1743:         }
1744:       }
1745:       // set
1746:       for (PetscInt b_id = 0 ; b_id < ctx->batch_sz ; b_id++) {
1747:         for (int grid=0,glb_elem_idx=0;grid<ctx->num_grids;grid++) {
1748:           const PetscInt moffset = LAND_MOFFSET(b_id,grid,ctx->batch_sz,ctx->num_grids,ctx->mat_offset);
1749:           for (int ej = 0 ; ej < numCells[grid] ; ++ej, glb_elem_idx++) {
1750:             const int  fullNb = coo_elem_fullNb[glb_elem_idx],fullNb2=fullNb*fullNb;
1751:             // set (i,j)
1752:             for (int fieldA=0;fieldA<Nf[grid];fieldA++) {
1753:               const LandauIdx *const Idxs = &maps[grid].gIdx[ej][fieldA][0];
1754:               int                    rows[LANDAU_MAX_Q_FACE],cols[LANDAU_MAX_Q_FACE];
1755:               for (int f = 0; f < Nb; ++f) {
1756:                 const int nr =  coo_elem_point_offsets[glb_elem_idx][f+1] - coo_elem_point_offsets[glb_elem_idx][f];
1757:                 if (nr==1) rows[0] = Idxs[f];
1758:                 else {
1759:                   const int idx = -Idxs[f] - 1;
1760:                   for (int q = 0; q < nr; q++) {
1761:                     rows[q] = maps[grid].c_maps[idx][q].gid;
1762:                   }
1763:                 }
1764:                 for (int g = 0; g < Nb; ++g) {
1765:                   const int nc =  coo_elem_point_offsets[glb_elem_idx][g+1] - coo_elem_point_offsets[glb_elem_idx][g];
1766:                   if (nc==1) cols[0] = Idxs[g];
1767:                   else {
1768:                     const int idx = -Idxs[g] - 1;
1769:                     for (int q = 0; q < nc; q++) {
1770:                       cols[q] = maps[grid].c_maps[idx][q].gid;
1771:                     }
1772:                   }
1773:                   const int idx0 = b_id*coo_elem_offsets[ncellsTot] + coo_elem_offsets[glb_elem_idx] + fieldA*fullNb2 + fullNb * coo_elem_point_offsets[glb_elem_idx][f] + nr * coo_elem_point_offsets[glb_elem_idx][g];
1774:                   for (int q = 0, idx = idx0; q < nr; q++) {
1775:                     for (int d = 0; d < nc; d++, idx++) {
1776:                       oor[idx] = rows[q] + moffset;
1777:                       ooc[idx] = cols[d] + moffset;
1778:                     }
1779:                   }
1780:                 }
1781:               }
1782:             }
1783:           } // cell
1784:         } // grid
1785:       } // batch
1786:       MatSetPreallocationCOO(ctx->J,ctx->SData_d.coo_size,oor,ooc);
1787:       PetscFree2(oor,ooc);
1788:     }
1789:     PetscContainerCreate(PETSC_COMM_SELF, &container);
1790:     PetscContainerSetPointer(container, (void *)maps);
1791:     PetscContainerSetUserDestroy(container, LandauGPUMapsDestroy);
1792:     PetscObjectCompose((PetscObject) ctx->J, "assembly_maps", (PetscObject) container);
1793:     PetscContainerDestroy(&container);
1794:     PetscLogEventEnd(ctx->events[2],0,0,0,0);
1795:   } // end GPU assembly
1796:   { /* create static point data, Jacobian called first, only one vertex copy */
1797:     PetscReal      *invJe,*ww,*xx,*yy,*zz=NULL,*invJ_a;
1798:     PetscInt       outer_ipidx, outer_ej,grid, nip_glb = 0;
1799:     PetscFE        fe;
1800:     const PetscInt Nb = Nq;
1801:     PetscLogEventBegin(ctx->events[7],0,0,0,0);
1802:     PetscInfo(ctx->plex[0], "Initialize static data\n");
1803:     for (PetscInt grid=0;grid<ctx->num_grids;grid++) nip_glb += Nq*numCells[grid];
1804:     /* collect f data, first time is for Jacobian, but make mass now */
1805:     if (ctx->verbose > 0) {
1806:       PetscInt ncells = 0, N;
1807:       MatGetSize(ctx->J,&N,NULL);
1808:       for (PetscInt grid=0;grid<ctx->num_grids;grid++) ncells += numCells[grid];
1809:       PetscPrintf(ctx->comm,"%" PetscInt_FMT ") %s %" PetscInt_FMT " IPs, %" PetscInt_FMT " cells total, Nb=%" PetscInt_FMT ", Nq=%" PetscInt_FMT ", dim=%" PetscInt_FMT ", Tab: Nb=%" PetscInt_FMT " Nf=%" PetscInt_FMT " Np=%" PetscInt_FMT " cdim=%" PetscInt_FMT " N=%" PetscInt_FMT "\n",0,"FormLandau",nip_glb,ncells, Nb, Nq, dim, Nb, ctx->num_species, Nb, dim, N);
1810:     }
1811:     PetscMalloc4(nip_glb,&ww,nip_glb,&xx,nip_glb,&yy,nip_glb*dim*dim,&invJ_a);
1812:     if (dim==3) {
1813:       PetscMalloc1(nip_glb,&zz);
1814:     }
1815:     if (ctx->use_energy_tensor_trick) {
1816:       PetscFECreateDefault(PETSC_COMM_SELF, dim, 1, PETSC_FALSE, NULL, PETSC_DECIDE, &fe);
1817:       PetscObjectSetName((PetscObject) fe, "energy");
1818:     }
1819:     /* init each grids static data - no batch */
1820:     for (grid=0, outer_ipidx=0, outer_ej=0 ; grid < ctx->num_grids ; grid++) { // OpenMP (once)
1821:       Vec             v2_2 = NULL; // projected function: v^2/2 for non-relativistic, gamma... for relativistic
1822:       PetscSection    e_section;
1823:       DM              dmEnergy;
1824:       PetscInt        cStart, cEnd, ej;

1826:       DMPlexGetHeightStratum(ctx->plex[grid], 0, &cStart, &cEnd);
1827:       // prep energy trick, get v^2 / 2 vector
1828:       if (ctx->use_energy_tensor_trick) {
1829:         PetscErrorCode (*energyf[1])(PetscInt, PetscReal, const PetscReal [], PetscInt, PetscScalar [], void *) = {ctx->use_relativistic_corrections ? gamma_m1_f : energy_f};
1830:         Vec            glob_v2;
1831:         PetscReal      *c2_0[1], data[1] = {PetscSqr(C_0(ctx->v_0))};

1833:         DMClone(ctx->plex[grid], &dmEnergy);
1834:         PetscObjectSetName((PetscObject) dmEnergy, "energy");
1835:         DMSetField(dmEnergy, 0, NULL, (PetscObject)fe);
1836:         DMCreateDS(dmEnergy);
1837:         DMGetSection(dmEnergy, &e_section);
1838:         DMGetGlobalVector(dmEnergy,&glob_v2);
1839:         PetscObjectSetName((PetscObject) glob_v2, "trick");
1840:         c2_0[0] = &data[0];
1841:         DMProjectFunction(dmEnergy, 0., energyf, (void**)c2_0, INSERT_ALL_VALUES, glob_v2);
1842:         DMGetLocalVector(dmEnergy, &v2_2);
1843:         VecZeroEntries(v2_2); /* zero BCs so don't set */
1844:         DMGlobalToLocalBegin(dmEnergy, glob_v2, INSERT_VALUES, v2_2);
1845:         DMGlobalToLocalEnd  (dmEnergy, glob_v2, INSERT_VALUES, v2_2);
1846:         DMViewFromOptions(dmEnergy,NULL, "-energy_dm_view");
1847:         VecViewFromOptions(glob_v2,NULL, "-energy_vec_view");
1848:         DMRestoreGlobalVector(dmEnergy, &glob_v2);
1849:       }
1850:       /* append part of the IP data for each grid */
1851:       for (ej = 0 ; ej < numCells[grid]; ++ej, ++outer_ej) {
1852:         PetscScalar *coefs = NULL;
1853:         PetscReal    vj[LANDAU_MAX_NQ*LANDAU_DIM],detJj[LANDAU_MAX_NQ], Jdummy[LANDAU_MAX_NQ*LANDAU_DIM*LANDAU_DIM], c0 = C_0(ctx->v_0), c02 = PetscSqr(c0);
1854:         invJe = invJ_a + outer_ej*Nq*dim*dim;
1855:         DMPlexComputeCellGeometryFEM(ctx->plex[grid], ej+cStart, quad, vj, Jdummy, invJe, detJj);
1856:         if (ctx->use_energy_tensor_trick) {
1857:           DMPlexVecGetClosure(dmEnergy, e_section, v2_2, ej+cStart, NULL, &coefs);
1858:         }
1859:         /* create static point data */
1860:         for (PetscInt qj = 0; qj < Nq; qj++, outer_ipidx++) {
1861:           const PetscInt  gidx = outer_ipidx;
1862:           const PetscReal *invJ = &invJe[qj*dim*dim];
1863:           ww    [gidx] = detJj[qj] * quadWeights[qj];
1864:           if (dim==2) ww    [gidx] *=              vj[qj * dim + 0];  /* cylindrical coordinate, w/o 2pi */
1865:           // get xx, yy, zz
1866:           if (ctx->use_energy_tensor_trick) {
1867:             double                  refSpaceDer[3],eGradPhi[3];
1868:             const PetscReal * const DD = Tf[0]->T[1];
1869:             const PetscReal         *Dq = &DD[qj*Nb*dim];
1870:             for (int d = 0; d < 3; ++d) refSpaceDer[d] = eGradPhi[d] = 0.0;
1871:             for (int b = 0; b < Nb; ++b) {
1872:               for (int d = 0; d < dim; ++d) refSpaceDer[d] += Dq[b*dim+d]*PetscRealPart(coefs[b]);
1873:             }
1874:             xx[gidx] = 1e10;
1875:             if (ctx->use_relativistic_corrections) {
1876:               double dg2_c2 = 0;
1877:               //for (int d = 0; d < dim; ++d) refSpaceDer[d] *= c02;
1878:               for (int d = 0; d < dim; ++d) dg2_c2 += PetscSqr(refSpaceDer[d]);
1879:               dg2_c2 *= (double)c02;
1880:               if (dg2_c2 >= .999) {
1881:                 xx[gidx] = vj[qj * dim + 0]; /* coordinate */
1882:                 yy[gidx] = vj[qj * dim + 1];
1883:                 if (dim==3) zz[gidx] = vj[qj * dim + 2];
1884:                 PetscPrintf(ctx->comm,"Error: %12.5e %" PetscInt_FMT ".%" PetscInt_FMT ") dg2/c02 = %12.5e x= %12.5e %12.5e %12.5e\n",PetscSqrtReal(xx[gidx]*xx[gidx] + yy[gidx]*yy[gidx] + zz[gidx]*zz[gidx]), ej, qj, dg2_c2, xx[gidx],yy[gidx],zz[gidx]);
1885:               } else {
1886:                 PetscReal fact = c02/PetscSqrtReal(1. - dg2_c2);
1887:                 for (int d = 0; d < dim; ++d) refSpaceDer[d] *= fact;
1888:                 // could test with other point u' that (grad - grad') * U (refSpaceDer, refSpaceDer') == 0
1889:               }
1890:             }
1891:             if (xx[gidx] == 1e10) {
1892:               for (int d = 0; d < dim; ++d) {
1893:                 for (int e = 0 ; e < dim; ++e) {
1894:                   eGradPhi[d] += invJ[e*dim+d]*refSpaceDer[e];
1895:                 }
1896:               }
1897:               xx[gidx] = eGradPhi[0];
1898:               yy[gidx] = eGradPhi[1];
1899:               if (dim==3) zz[gidx] = eGradPhi[2];
1900:             }
1901:           } else {
1902:             xx[gidx] = vj[qj * dim + 0]; /* coordinate */
1903:             yy[gidx] = vj[qj * dim + 1];
1904:             if (dim==3) zz[gidx] = vj[qj * dim + 2];
1905:           }
1906:         } /* q */
1907:         if (ctx->use_energy_tensor_trick) {
1908:           DMPlexVecRestoreClosure(dmEnergy, e_section, v2_2, ej+cStart, NULL, &coefs);
1909:         }
1910:       } /* ej */
1911:       if (ctx->use_energy_tensor_trick) {
1912:         DMRestoreLocalVector(dmEnergy, &v2_2);
1913:         DMDestroy(&dmEnergy);
1914:       }
1915:     } /* grid */
1916:     if (ctx->use_energy_tensor_trick) {
1917:       PetscFEDestroy(&fe);
1918:     }
1919:     /* cache static data */
1920:     if (ctx->deviceType == LANDAU_CUDA || ctx->deviceType == LANDAU_KOKKOS) {
1921: #if defined(PETSC_HAVE_CUDA) || defined(PETSC_HAVE_KOKKOS_KERNELS)
1922:       PetscReal invMass[LANDAU_MAX_SPECIES],nu_alpha[LANDAU_MAX_SPECIES], nu_beta[LANDAU_MAX_SPECIES];
1923:       for (PetscInt grid = 0; grid < ctx->num_grids ; grid++) {
1924:         for (PetscInt ii=ctx->species_offset[grid];ii<ctx->species_offset[grid+1];ii++) {
1925:           invMass[ii]  = ctx->m_0/ctx->masses[ii];
1926:           nu_alpha[ii] = PetscSqr(ctx->charges[ii]/ctx->m_0)*ctx->m_0/ctx->masses[ii];
1927:           nu_beta[ii]  = PetscSqr(ctx->charges[ii]/ctx->epsilon0)*ctx->lnLam / (8*PETSC_PI) * ctx->t_0*ctx->n_0/PetscPowReal(ctx->v_0,3);
1928:         }
1929:       }
1930:       if (ctx->deviceType == LANDAU_CUDA) {
1931: #if defined(PETSC_HAVE_CUDA)
1932:         PetscCall(LandauCUDAStaticDataSet(ctx->plex[0], Nq, ctx->batch_sz, ctx->num_grids, numCells, ctx->species_offset, ctx->mat_offset,
1933:                                         nu_alpha, nu_beta, invMass, invJ_a, xx, yy, zz, ww, &ctx->SData_d));
1934: #else
1935:         SETERRQ(ctx->comm,PETSC_ERR_ARG_WRONG,"-landau_device_type cuda not built");
1936: #endif
1937:       } else if (ctx->deviceType == LANDAU_KOKKOS) {
1938: #if defined(PETSC_HAVE_KOKKOS_KERNELS)
1939:         PetscCall(LandauKokkosStaticDataSet(ctx->plex[0], Nq, ctx->batch_sz, ctx->num_grids, numCells, ctx->species_offset, ctx->mat_offset,
1940:                                           nu_alpha, nu_beta, invMass,invJ_a,xx,yy,zz,ww,&ctx->SData_d));
1941: #else
1942:         SETERRQ(ctx->comm,PETSC_ERR_ARG_WRONG,"-landau_device_type kokkos not built");
1943: #endif
1944:       }
1945: #endif
1946:       /* free */
1947:       PetscFree4(ww,xx,yy,invJ_a);
1948:       if (dim==3) {
1949:         PetscFree(zz);
1950:       }
1951:     } else { /* CPU version, just copy in, only use part */
1952:       ctx->SData_d.w = (void*)ww;
1953:       ctx->SData_d.x = (void*)xx;
1954:       ctx->SData_d.y = (void*)yy;
1955:       ctx->SData_d.z = (void*)zz;
1956:       ctx->SData_d.invJ = (void*)invJ_a;
1957:     }
1958:     PetscLogEventEnd(ctx->events[7],0,0,0,0);
1959:   } // initialize
1960:   return 0;
1961: }

1963: /* < v, u > */
1964: static void g0_1(PetscInt dim, PetscInt Nf, PetscInt NfAux,
1965:                  const PetscInt uOff[], const PetscInt uOff_x[], const PetscScalar u[], const PetscScalar u_t[], const PetscScalar u_x[],
1966:                  const PetscInt aOff[], const PetscInt aOff_x[], const PetscScalar a[], const PetscScalar a_t[], const PetscScalar a_x[],
1967:                  PetscReal t, PetscReal u_tShift, const PetscReal x[],  PetscInt numConstants, const PetscScalar constants[], PetscScalar g0[])
1968: {
1969:   g0[0] = 1.;
1970: }

1972: /* < v, u > */
1973: static void g0_fake(PetscInt dim, PetscInt Nf, PetscInt NfAux,
1974:                  const PetscInt uOff[], const PetscInt uOff_x[], const PetscScalar u[], const PetscScalar u_t[], const PetscScalar u_x[],
1975:                  const PetscInt aOff[], const PetscInt aOff_x[], const PetscScalar a[], const PetscScalar a_t[], const PetscScalar a_x[],
1976:                  PetscReal t, PetscReal u_tShift, const PetscReal x[],  PetscInt numConstants, const PetscScalar constants[], PetscScalar g0[])
1977: {
1978:   static double ttt = 1;
1979:   g0[0] = ttt++;
1980: }

1982: /* < v, u > */
1983: static void g0_r(PetscInt dim, PetscInt Nf, PetscInt NfAux,
1984:                  const PetscInt uOff[], const PetscInt uOff_x[], const PetscScalar u[], const PetscScalar u_t[], const PetscScalar u_x[],
1985:                  const PetscInt aOff[], const PetscInt aOff_x[], const PetscScalar a[], const PetscScalar a_t[], const PetscScalar a_x[],
1986:                  PetscReal t, PetscReal u_tShift, const PetscReal x[],  PetscInt numConstants, const PetscScalar constants[], PetscScalar g0[])
1987: {
1988:   g0[0] = 2.*PETSC_PI*x[0];
1989: }

1991: static PetscErrorCode MatrixNfDestroy(void *ptr)
1992: {
1993:   PetscInt *nf = (PetscInt *)ptr;
1994:   PetscFree(nf);
1995:   return 0;
1996: }

1998: static PetscErrorCode LandauCreateMatrix(MPI_Comm comm, Vec X, IS grid_batch_is_inv[LANDAU_MAX_GRIDS], LandauCtx *ctx)
1999: {
2000:   PetscInt       *idxs=NULL;
2001:   Mat            subM[LANDAU_MAX_GRIDS];

2003:   if (!ctx->gpu_assembly) { /* we need GPU object with GPU assembly */
2004:     return 0;
2005:   }
2006:   // get the RCM for this grid to separate out species into blocks -- create 'idxs' & 'ctx->batch_is'
2007:   if (ctx->gpu_assembly && ctx->jacobian_field_major_order) {
2008:     PetscMalloc1(ctx->mat_offset[ctx->num_grids]*ctx->batch_sz, &idxs);
2009:   }
2010:   for (PetscInt grid=0 ; grid < ctx->num_grids ; grid++) {
2011:     const PetscInt *values, n = ctx->mat_offset[grid+1] - ctx->mat_offset[grid];
2012:     Mat             gMat;
2013:     DM              massDM;
2014:     PetscDS         prob;
2015:     Vec             tvec;
2016:     // get "mass" matrix for reordering
2017:     DMClone(ctx->plex[grid], &massDM);
2018:     DMCopyFields(ctx->plex[grid], massDM);
2019:     DMCreateDS(massDM);
2020:     DMGetDS(massDM, &prob);
2021:     for (int ix=0, ii=ctx->species_offset[grid];ii<ctx->species_offset[grid+1];ii++,ix++) {
2022:       PetscDSSetJacobian(prob, ix, ix, g0_fake, NULL, NULL, NULL);
2023:     }
2024:     PetscOptionsInsertString(NULL,"-dm_preallocate_only");
2025:     DMSetFromOptions(massDM);
2026:     DMCreateMatrix(massDM, &gMat);
2027:     PetscOptionsInsertString(NULL,"-dm_preallocate_only false");
2028:     MatSetOption(gMat,MAT_STRUCTURALLY_SYMMETRIC, PETSC_TRUE);
2029:     MatSetOption(gMat,MAT_IGNORE_ZERO_ENTRIES,PETSC_TRUE);
2030:     DMCreateLocalVector(ctx->plex[grid],&tvec);
2031:     DMPlexSNESComputeJacobianFEM(massDM, tvec, gMat, gMat, ctx);
2032:     MatViewFromOptions(gMat, NULL, "-dm_landau_reorder_mat_view");
2033:     DMDestroy(&massDM);
2034:     VecDestroy(&tvec);
2035:     subM[grid] = gMat;
2036:     if (ctx->gpu_assembly && ctx->jacobian_field_major_order) {
2037:       MatOrderingType rtype = MATORDERINGRCM;
2038:       IS              isrow,isicol;
2039:       MatGetOrdering(gMat,rtype,&isrow,&isicol);
2040:       ISInvertPermutation(isrow,PETSC_DECIDE,&grid_batch_is_inv[grid]);
2041:       ISGetIndices(isrow, &values);
2042:       for (PetscInt b_id=0 ; b_id < ctx->batch_sz ; b_id++) { // add batch size DMs for this species grid
2043: #if !defined(LANDAU_SPECIES_MAJOR)
2044:         PetscInt N = ctx->mat_offset[ctx->num_grids], n0 = ctx->mat_offset[grid] + b_id*N;
2045:         for (int ii = 0; ii < n; ++ii) idxs[n0+ii] = values[ii] + n0;
2046: #else
2047:         PetscInt n0 = ctx->mat_offset[grid]*ctx->batch_sz + b_id*n;
2048:         for (int ii = 0; ii < n; ++ii) idxs[n0+ii] = values[ii] + n0;
2049: #endif
2050:       }
2051:       ISRestoreIndices(isrow, &values);
2052:       ISDestroy(&isrow);
2053:       ISDestroy(&isicol);
2054:     }
2055:   }
2056:   if (ctx->gpu_assembly && ctx->jacobian_field_major_order) {
2057:     ISCreateGeneral(comm,ctx->mat_offset[ctx->num_grids]*ctx->batch_sz,idxs,PETSC_OWN_POINTER,&ctx->batch_is);
2058:   }
2059:   // get a block matrix
2060:   for (PetscInt grid=0 ; grid<ctx->num_grids ; grid++) {
2061:     Mat               B = subM[grid];
2062:     PetscInt          nloc, nzl, colbuf[1024], row;
2063:     MatGetSize(B, &nloc, NULL);
2064:     for (PetscInt b_id = 0 ; b_id < ctx->batch_sz ; b_id++) {
2065:       const PetscInt    moffset = LAND_MOFFSET(b_id,grid,ctx->batch_sz,ctx->num_grids,ctx->mat_offset);
2066:       const PetscInt    *cols;
2067:       const PetscScalar *vals;
2068:       for (int i=0 ; i<nloc ; i++) {
2069:         MatGetRow(B,i,&nzl,&cols,&vals);
2071:         for (int j=0; j<nzl; j++) colbuf[j] = cols[j] + moffset;
2072:         row = i + moffset;
2073:         MatSetValues(ctx->J,1,&row,nzl,colbuf,vals,INSERT_VALUES);
2074:         MatRestoreRow(B,i,&nzl,&cols,&vals);
2075:       }
2076:     }
2077:   }
2078:   for (PetscInt grid=0 ; grid<ctx->num_grids ; grid++) {
2079:     MatDestroy(&subM[grid]);
2080:   }
2081:   MatAssemblyBegin(ctx->J,MAT_FINAL_ASSEMBLY);
2082:   MatAssemblyEnd(ctx->J,MAT_FINAL_ASSEMBLY);

2084:   if (ctx->gpu_assembly && ctx->jacobian_field_major_order) {
2085:     Mat            mat_block_order;
2086:     MatCreateSubMatrix(ctx->J,ctx->batch_is,ctx->batch_is,MAT_INITIAL_MATRIX,&mat_block_order); // use MatPermute
2087:     MatViewFromOptions(mat_block_order, NULL, "-dm_landau_field_major_mat_view");
2088:     MatDestroy(&ctx->J);
2089:     ctx->J = mat_block_order;
2090:     // override ops to make KSP work in field major space
2091:     ctx->seqaij_mult                  = mat_block_order->ops->mult;
2092:     mat_block_order->ops->mult        = LandauMatMult;
2093:     mat_block_order->ops->multadd     = LandauMatMultAdd;
2094:     ctx->seqaij_solve                 = NULL;
2095:     ctx->seqaij_getdiagonal           = mat_block_order->ops->getdiagonal;
2096:     mat_block_order->ops->getdiagonal = LandauMatGetDiagonal;
2097:     ctx->seqaij_multtranspose         = mat_block_order->ops->multtranspose;
2098:     mat_block_order->ops->multtranspose = LandauMatMultTranspose;
2099:     VecDuplicate(X,&ctx->work_vec);
2100:     VecScatterCreate(X, ctx->batch_is, ctx->work_vec, NULL, &ctx->plex_batch);
2101:   }

2103:   return 0;
2104: }

2106: PetscErrorCode DMPlexLandauCreateMassMatrix(DM pack, Mat *Amat);
2107: /*@C
2108:  DMPlexLandauCreateVelocitySpace - Create a DMPlex velocity space mesh

2110:  Collective on comm

2112:  Input Parameters:
2113:  +   comm  - The MPI communicator
2114:  .   dim - velocity space dimension (2 for axisymmetric, 3 for full 3X + 3V solver)
2115:  -   prefix - prefix for options (not tested)

2117:  Output Parameter:
2118:  .   pack  - The DM object representing the mesh
2119:  +   X - A vector (user destroys)
2120:  -   J - Optional matrix (object destroys)

2122:  Level: beginner

2124:  .keywords: mesh
2125:  .seealso: DMPlexCreate(), DMPlexLandauDestroyVelocitySpace()
2126:  @*/
2127: PetscErrorCode DMPlexLandauCreateVelocitySpace(MPI_Comm comm, PetscInt dim, const char prefix[], Vec *X, Mat *J, DM *pack)
2128: {
2129:   LandauCtx      *ctx;
2130:   Vec            Xsub[LANDAU_MAX_GRIDS];
2131:   IS             grid_batch_is_inv[LANDAU_MAX_GRIDS];

2135:   PetscNew(&ctx);
2136:   ctx->comm = comm; /* used for diagnostics and global errors */
2137:   /* process options */
2138:   ProcessOptions(ctx,prefix);
2139:   if (dim==2) ctx->use_relativistic_corrections = PETSC_FALSE;
2140:   /* Create Mesh */
2141:   DMCompositeCreate(PETSC_COMM_SELF,pack);
2142:   PetscLogEventBegin(ctx->events[13],0,0,0,0);
2143:   PetscLogEventBegin(ctx->events[15],0,0,0,0);
2144:   LandauDMCreateVMeshes(PETSC_COMM_SELF, dim, prefix, ctx, *pack); // creates grids (Forest of AMR)
2145:   for (PetscInt grid=0;grid<ctx->num_grids;grid++) {
2146:     /* create FEM */
2147:     SetupDS(ctx->plex[grid],dim,grid,ctx);
2148:     /* set initial state */
2149:     DMCreateGlobalVector(ctx->plex[grid],&Xsub[grid]);
2150:     PetscObjectSetName((PetscObject) Xsub[grid], "u_orig");
2151:     /* initial static refinement, no solve */
2152:     LandauSetInitialCondition(ctx->plex[grid], Xsub[grid], grid, 0, ctx);
2153:     /* forest refinement - forest goes in (if forest), plex comes out */
2154:     if (ctx->use_p4est) {
2155:       DM plex;
2156:       adapt(grid,ctx,&Xsub[grid]); // forest goes in, plex comes out
2157:       DMViewFromOptions(ctx->plex[grid],NULL,"-dm_landau_amr_dm_view"); // need to differentiate - todo
2158:       VecViewFromOptions(Xsub[grid], NULL, "-dm_landau_amr_vec_view");
2159:       // convert to plex, all done with this level
2160:       DMConvert(ctx->plex[grid], DMPLEX, &plex);
2161:       DMDestroy(&ctx->plex[grid]);
2162:       ctx->plex[grid] = plex;
2163:     }
2164: #if !defined(LANDAU_SPECIES_MAJOR)
2165:     DMCompositeAddDM(*pack,ctx->plex[grid]);
2166: #else
2167:     for (PetscInt b_id=0;b_id<ctx->batch_sz;b_id++) { // add batch size DMs for this species grid
2168:       DMCompositeAddDM(*pack,ctx->plex[grid]);
2169:     }
2170: #endif
2171:     DMSetApplicationContext(ctx->plex[grid], ctx);
2172:   }
2173: #if !defined(LANDAU_SPECIES_MAJOR)
2174:   // stack the batched DMs, could do it all here!!! b_id=0
2175:   for (PetscInt b_id=1;b_id<ctx->batch_sz;b_id++) {
2176:     for (PetscInt grid=0;grid<ctx->num_grids;grid++) {
2177:       DMCompositeAddDM(*pack,ctx->plex[grid]);
2178:     }
2179:   }
2180: #endif
2181:   // create ctx->mat_offset
2182:   ctx->mat_offset[0] = 0;
2183:   for (PetscInt grid=0 ; grid < ctx->num_grids ; grid++) {
2184:     PetscInt    n;
2185:     VecGetLocalSize(Xsub[grid],&n);
2186:     ctx->mat_offset[grid+1] = ctx->mat_offset[grid] + n;
2187:   }
2188:   // creat DM & Jac
2189:   DMSetApplicationContext(*pack, ctx);
2190:   PetscOptionsInsertString(NULL,"-dm_preallocate_only");
2191:   DMSetFromOptions(*pack);
2192:   DMCreateMatrix(*pack, &ctx->J);
2193:   PetscOptionsInsertString(NULL,"-dm_preallocate_only false");
2194:   MatSetOption(ctx->J,MAT_STRUCTURALLY_SYMMETRIC, PETSC_TRUE);
2195:   MatSetOption(ctx->J,MAT_IGNORE_ZERO_ENTRIES,PETSC_TRUE);
2196:   PetscObjectSetName((PetscObject)ctx->J, "Jac");
2197:   // construct initial conditions in X
2198:   DMCreateGlobalVector(*pack,X);
2199:   for (PetscInt grid=0 ; grid < ctx->num_grids ; grid++) {
2200:     PetscInt n;
2201:     VecGetLocalSize(Xsub[grid],&n);
2202:     for (PetscInt b_id = 0 ; b_id < ctx->batch_sz ; b_id++) {
2203:       PetscScalar const *values;
2204:       const PetscInt    moffset = LAND_MOFFSET(b_id,grid,ctx->batch_sz,ctx->num_grids,ctx->mat_offset);
2205:       LandauSetInitialCondition(ctx->plex[grid], Xsub[grid], grid, b_id, ctx);
2206:       VecGetArrayRead(Xsub[grid],&values);
2207:       for (int i=0, idx = moffset; i<n; i++, idx++) {
2208:         VecSetValue(*X,idx,values[i],INSERT_VALUES);
2209:       }
2210:       VecRestoreArrayRead(Xsub[grid],&values);
2211:     }
2212:   }
2213:   // cleanup
2214:   for (PetscInt grid=0 ; grid < ctx->num_grids ; grid++) {
2215:     VecDestroy(&Xsub[grid]);
2216:   }
2217:   /* check for correct matrix type */
2218:   if (ctx->gpu_assembly) { /* we need GPU object with GPU assembly */
2219:     PetscBool flg;
2220:     if (ctx->deviceType == LANDAU_CUDA) {
2221:       PetscObjectTypeCompareAny((PetscObject)ctx->J,&flg,MATSEQAIJCUSPARSE,MATMPIAIJCUSPARSE,MATAIJCUSPARSE,"");
2223:     } else if (ctx->deviceType == LANDAU_KOKKOS) {
2224:       PetscObjectTypeCompareAny((PetscObject)ctx->J,&flg,MATSEQAIJKOKKOS,MATMPIAIJKOKKOS,MATAIJKOKKOS,"");
2225: #if defined(PETSC_HAVE_KOKKOS_KERNELS)
2227: #else
2229: #endif
2230:     }
2231:   }
2232:   PetscLogEventEnd(ctx->events[15],0,0,0,0);
2233:   // create field major ordering

2235:   ctx->work_vec   = NULL;
2236:   ctx->plex_batch = NULL;
2237:   ctx->batch_is   = NULL;
2238:   for (int i=0;i<LANDAU_MAX_GRIDS;i++) grid_batch_is_inv[i] = NULL;
2239:   PetscLogEventBegin(ctx->events[12],0,0,0,0);
2240:   LandauCreateMatrix(comm, *X, grid_batch_is_inv, ctx);
2241:   PetscLogEventEnd(ctx->events[12],0,0,0,0);

2243:   // create AMR GPU assembly maps and static GPU data
2244:   CreateStaticGPUData(dim,grid_batch_is_inv,ctx);

2246:   PetscLogEventEnd(ctx->events[13],0,0,0,0);

2248:   // create mass matrix
2249:   DMPlexLandauCreateMassMatrix(*pack, NULL);

2251:   if (J) *J = ctx->J;

2253:   if (ctx->gpu_assembly && ctx->jacobian_field_major_order) {
2254:     PetscContainer container;
2255:     // cache ctx for KSP with batch/field major Jacobian ordering -ksp_type gmres/etc -dm_landau_jacobian_field_major_order
2256:     PetscContainerCreate(PETSC_COMM_SELF, &container);
2257:     PetscContainerSetPointer(container, (void *)ctx);
2258:     PetscObjectCompose((PetscObject) ctx->J, "LandauCtx", (PetscObject) container);
2259:     PetscContainerDestroy(&container);
2260:     // batch solvers need to map -- can batch solvers work
2261:     PetscContainerCreate(PETSC_COMM_SELF, &container);
2262:     PetscContainerSetPointer(container, (void *)ctx->plex_batch);
2263:     PetscObjectCompose((PetscObject) ctx->J, "plex_batch_is", (PetscObject) container);
2264:     PetscContainerDestroy(&container);
2265:   }
2266:   // for batch solvers
2267:   {
2268:     PetscContainer  container;
2269:     PetscInt        *pNf;
2270:     PetscContainerCreate(PETSC_COMM_SELF, &container);
2271:     PetscMalloc1(sizeof(*pNf), &pNf);
2272:     *pNf = ctx->batch_sz;
2273:     PetscContainerSetPointer(container, (void *)pNf);
2274:     PetscContainerSetUserDestroy(container, MatrixNfDestroy);
2275:     PetscObjectCompose((PetscObject)ctx->J, "batch size", (PetscObject) container);
2276:     PetscContainerDestroy(&container);
2277:   }

2279:   return 0;
2280: }

2282: /*@
2283:  DMPlexLandauDestroyVelocitySpace - Destroy a DMPlex velocity space mesh

2285:  Collective on dm

2287:  Input/Output Parameters:
2288:  .   dm - the dm to destroy

2290:  Level: beginner

2292:  .keywords: mesh
2293:  .seealso: DMPlexLandauCreateVelocitySpace()
2294:  @*/
2295: PetscErrorCode DMPlexLandauDestroyVelocitySpace(DM *dm)
2296: {
2297:   LandauCtx      *ctx;
2298:   DMGetApplicationContext(*dm, &ctx);
2299:   MatDestroy(&ctx->M);
2300:   MatDestroy(&ctx->J);
2301:   for (PetscInt ii=0;ii<ctx->num_species;ii++) PetscFEDestroy(&ctx->fe[ii]);
2302:   ISDestroy(&ctx->batch_is);
2303:   VecDestroy(&ctx->work_vec);
2304:   VecScatterDestroy(&ctx->plex_batch);
2305:   if (ctx->deviceType == LANDAU_CUDA) {
2306: #if defined(PETSC_HAVE_CUDA)
2307:     LandauCUDAStaticDataClear(&ctx->SData_d);
2308: #else
2309:     SETERRQ(ctx->comm,PETSC_ERR_ARG_WRONG,"-landau_device_type %s not built","cuda");
2310: #endif
2311:   } else if (ctx->deviceType == LANDAU_KOKKOS) {
2312: #if defined(PETSC_HAVE_KOKKOS_KERNELS)
2313:     LandauKokkosStaticDataClear(&ctx->SData_d);
2314: #else
2315:     SETERRQ(ctx->comm,PETSC_ERR_ARG_WRONG,"-landau_device_type %s not built","kokkos");
2316: #endif
2317:   } else {
2318:     if (ctx->SData_d.x) { /* in a CPU run */
2319:       PetscReal *invJ = (PetscReal*)ctx->SData_d.invJ, *xx = (PetscReal*)ctx->SData_d.x, *yy = (PetscReal*)ctx->SData_d.y, *zz = (PetscReal*)ctx->SData_d.z, *ww = (PetscReal*)ctx->SData_d.w;
2320:       LandauIdx *coo_elem_offsets = (LandauIdx*)ctx->SData_d.coo_elem_offsets, *coo_elem_fullNb = (LandauIdx*)ctx->SData_d.coo_elem_fullNb, (*coo_elem_point_offsets)[LANDAU_MAX_NQ+1] = (LandauIdx (*)[LANDAU_MAX_NQ+1])ctx->SData_d.coo_elem_point_offsets;
2321:       PetscFree4(ww,xx,yy,invJ);
2322:       if (zz) {
2323:         PetscFree(zz);
2324:       }
2325:       if (coo_elem_offsets) {
2326:         PetscFree3(coo_elem_offsets,coo_elem_fullNb,coo_elem_point_offsets); // could be NULL
2327:       }
2328:     }
2329:   }

2331:   if (ctx->times[LANDAU_MATRIX_TOTAL] > 0) { // OMP timings
2332:     PetscPrintf(ctx->comm, "TSStep               N  1.0 %10.3e\n",ctx->times[LANDAU_EX2_TSSOLVE]);
2333:     PetscPrintf(ctx->comm, "2:           Solve:  %10.3e with %" PetscInt_FMT " threads\n",ctx->times[LANDAU_EX2_TSSOLVE] - ctx->times[LANDAU_MATRIX_TOTAL],ctx->batch_sz);
2334:     PetscPrintf(ctx->comm, "3:          Landau:  %10.3e\n",ctx->times[LANDAU_MATRIX_TOTAL]);
2335:     PetscPrintf(ctx->comm, "Landau Jacobian       %" PetscInt_FMT " 1.0 %10.3e\n",(PetscInt)ctx->times[LANDAU_JACOBIAN_COUNT],ctx->times[LANDAU_JACOBIAN]);
2336:     PetscPrintf(ctx->comm, "Landau Operator       N 1.0  %10.3e\n",ctx->times[LANDAU_OPERATOR]);
2337:     PetscPrintf(ctx->comm, "Landau Mass           N 1.0  %10.3e\n",ctx->times[LANDAU_MASS]);
2338:     PetscPrintf(ctx->comm, " Jac-f-df (GPU)       N 1.0  %10.3e\n",ctx->times[LANDAU_F_DF]);
2339:     PetscPrintf(ctx->comm, " Kernel (GPU)         N 1.0  %10.3e\n",ctx->times[LANDAU_KERNEL]);
2340:     PetscPrintf(ctx->comm, "MatLUFactorNum        X 1.0 %10.3e\n",ctx->times[KSP_FACTOR]);
2341:     PetscPrintf(ctx->comm, "MatSolve              X 1.0 %10.3e\n",ctx->times[KSP_SOLVE]);
2342:   }
2343:   for (PetscInt grid=0 ; grid < ctx->num_grids ; grid++) {
2344:     DMDestroy(&ctx->plex[grid]);
2345:   }
2346:   PetscFree(ctx);
2347:   DMDestroy(dm);
2348:   return 0;
2349: }

2351: /* < v, ru > */
2352: static void f0_s_den(PetscInt dim, PetscInt Nf, PetscInt NfAux,
2353:                      const PetscInt uOff[], const PetscInt uOff_x[], const PetscScalar u[], const PetscScalar u_t[], const PetscScalar u_x[],
2354:                      const PetscInt aOff[], const PetscInt aOff_x[], const PetscScalar a[], const PetscScalar a_t[], const PetscScalar a_x[],
2355:                      PetscReal t, const PetscReal x[],  PetscInt numConstants, const PetscScalar constants[], PetscScalar *f0)
2356: {
2357:   PetscInt ii = (PetscInt)PetscRealPart(constants[0]);
2358:   f0[0] = u[ii];
2359: }

2361: /* < v, ru > */
2362: static void f0_s_mom(PetscInt dim, PetscInt Nf, PetscInt NfAux,
2363:                      const PetscInt uOff[], const PetscInt uOff_x[], const PetscScalar u[], const PetscScalar u_t[], const PetscScalar u_x[],
2364:                      const PetscInt aOff[], const PetscInt aOff_x[], const PetscScalar a[], const PetscScalar a_t[], const PetscScalar a_x[],
2365:                      PetscReal t, const PetscReal x[],  PetscInt numConstants, const PetscScalar constants[], PetscScalar *f0)
2366: {
2367:   PetscInt ii = (PetscInt)PetscRealPart(constants[0]), jj = (PetscInt)PetscRealPart(constants[1]);
2368:   f0[0] = x[jj]*u[ii]; /* x momentum */
2369: }

2371: static void f0_s_v2(PetscInt dim, PetscInt Nf, PetscInt NfAux,
2372:                     const PetscInt uOff[], const PetscInt uOff_x[], const PetscScalar u[], const PetscScalar u_t[], const PetscScalar u_x[],
2373:                     const PetscInt aOff[], const PetscInt aOff_x[], const PetscScalar a[], const PetscScalar a_t[], const PetscScalar a_x[],
2374:                     PetscReal t, const PetscReal x[],  PetscInt numConstants, const PetscScalar constants[], PetscScalar *f0)
2375: {
2376:   PetscInt i, ii = (PetscInt)PetscRealPart(constants[0]);
2377:   double tmp1 = 0.;
2378:   for (i = 0; i < dim; ++i) tmp1 += x[i]*x[i];
2379:   f0[0] = tmp1*u[ii];
2380: }

2382: static PetscErrorCode gamma_n_f(PetscInt dim, PetscReal time, const PetscReal x[], PetscInt Nf, PetscScalar *u, void *actx)
2383: {
2384:   const PetscReal *c2_0_arr = ((PetscReal*)actx);
2385:   const PetscReal c02 = c2_0_arr[0];

2387:   for (int s = 0 ; s < Nf ; s++) {
2388:     PetscReal tmp1 = 0.;
2389:     for (int i = 0; i < dim; ++i) tmp1 += x[i]*x[i];
2390: #if defined(PETSC_USE_DEBUG)
2391:     u[s] = PetscSqrtReal(1. + tmp1/c02);//  u[0] = PetscSqrtReal(1. + xx);
2392: #else
2393:     {
2394:       PetscReal xx = tmp1/c02;
2395:       u[s] = xx/(PetscSqrtReal(1. + xx) + 1.); // better conditioned = xx/(PetscSqrtReal(1. + xx) + 1.)
2396:     }
2397: #endif
2398:   }
2399:   return 0;
2400: }

2402: /* < v, ru > */
2403: static void f0_s_rden(PetscInt dim, PetscInt Nf, PetscInt NfAux,
2404:                       const PetscInt uOff[], const PetscInt uOff_x[], const PetscScalar u[], const PetscScalar u_t[], const PetscScalar u_x[],
2405:                       const PetscInt aOff[], const PetscInt aOff_x[], const PetscScalar a[], const PetscScalar a_t[], const PetscScalar a_x[],
2406:                       PetscReal t, const PetscReal x[],  PetscInt numConstants, const PetscScalar constants[], PetscScalar *f0)
2407: {
2408:   PetscInt ii = (PetscInt)PetscRealPart(constants[0]);
2409:   f0[0] = 2.*PETSC_PI*x[0]*u[ii];
2410: }

2412: /* < v, ru > */
2413: static void f0_s_rmom(PetscInt dim, PetscInt Nf, PetscInt NfAux,
2414:                       const PetscInt uOff[], const PetscInt uOff_x[], const PetscScalar u[], const PetscScalar u_t[], const PetscScalar u_x[],
2415:                       const PetscInt aOff[], const PetscInt aOff_x[], const PetscScalar a[], const PetscScalar a_t[], const PetscScalar a_x[],
2416:                       PetscReal t, const PetscReal x[],  PetscInt numConstants, const PetscScalar constants[], PetscScalar *f0)
2417: {
2418:   PetscInt ii = (PetscInt)PetscRealPart(constants[0]);
2419:   f0[0] = 2.*PETSC_PI*x[0]*x[1]*u[ii];
2420: }

2422: static void f0_s_rv2(PetscInt dim, PetscInt Nf, PetscInt NfAux,
2423:                      const PetscInt uOff[], const PetscInt uOff_x[], const PetscScalar u[], const PetscScalar u_t[], const PetscScalar u_x[],
2424:                      const PetscInt aOff[], const PetscInt aOff_x[], const PetscScalar a[], const PetscScalar a_t[], const PetscScalar a_x[],
2425:                      PetscReal t, const PetscReal x[],  PetscInt numConstants, const PetscScalar constants[], PetscScalar *f0)
2426: {
2427:   PetscInt ii = (PetscInt)PetscRealPart(constants[0]);
2428:   f0[0] =  2.*PETSC_PI*x[0]*(x[0]*x[0] + x[1]*x[1])*u[ii];
2429: }

2431: /*@
2432:  DMPlexLandauPrintNorms - collects moments and prints them

2434:  Collective on dm

2436:  Input Parameters:
2437:  +   X  - the state
2438:  -   stepi - current step to print

2440:  Level: beginner

2442:  .keywords: mesh
2443:  .seealso: DMPlexLandauCreateVelocitySpace()
2444:  @*/
2445: PetscErrorCode DMPlexLandauPrintNorms(Vec X, PetscInt stepi)
2446: {
2447:   LandauCtx      *ctx;
2448:   PetscDS        prob;
2449:   DM             pack;
2450:   PetscInt       cStart, cEnd, dim, ii, i0, nDMs;
2451:   PetscScalar    xmomentumtot=0, ymomentumtot=0, zmomentumtot=0, energytot=0, densitytot=0, tt[LANDAU_MAX_SPECIES];
2452:   PetscScalar    xmomentum[LANDAU_MAX_SPECIES],  ymomentum[LANDAU_MAX_SPECIES],  zmomentum[LANDAU_MAX_SPECIES], energy[LANDAU_MAX_SPECIES], density[LANDAU_MAX_SPECIES];
2453:   Vec            *globXArray;

2455:   VecGetDM(X, &pack);
2457:   DMGetDimension(pack, &dim);
2459:   DMGetApplicationContext(pack, &ctx);
2461:   /* print momentum and energy */
2462:   DMCompositeGetNumberDM(pack,&nDMs);
2464:   PetscMalloc(sizeof(*globXArray)*nDMs, &globXArray);
2465:   DMCompositeGetAccessArray(pack, X, nDMs, NULL, globXArray);
2466:   for (PetscInt grid = 0; grid < ctx->num_grids ; grid++) {
2467:     Vec Xloc = globXArray[ LAND_PACK_IDX(ctx->batch_view_idx,grid) ];
2468:     DMGetDS(ctx->plex[grid], &prob);
2469:     for (ii=ctx->species_offset[grid],i0=0;ii<ctx->species_offset[grid+1];ii++,i0++) {
2470:       PetscScalar user[2] = { (PetscScalar)i0, (PetscScalar)ctx->charges[ii]};
2471:       PetscDSSetConstants(prob, 2, user);
2472:       if (dim==2) { /* 2/3X + 3V (cylindrical coordinates) */
2473:         PetscDSSetObjective(prob, 0, &f0_s_rden);
2474:         DMPlexComputeIntegralFEM(ctx->plex[grid],Xloc,tt,ctx);
2475:         density[ii] = tt[0]*ctx->n_0*ctx->charges[ii];
2476:         PetscDSSetObjective(prob, 0, &f0_s_rmom);
2477:         DMPlexComputeIntegralFEM(ctx->plex[grid],Xloc,tt,ctx);
2478:         zmomentum[ii] = tt[0]*ctx->n_0*ctx->v_0*ctx->masses[ii];
2479:         PetscDSSetObjective(prob, 0, &f0_s_rv2);
2480:         DMPlexComputeIntegralFEM(ctx->plex[grid],Xloc,tt,ctx);
2481:         energy[ii] = tt[0]*0.5*ctx->n_0*ctx->v_0*ctx->v_0*ctx->masses[ii];
2482:         zmomentumtot += zmomentum[ii];
2483:         energytot  += energy[ii];
2484:         densitytot += density[ii];
2485:         PetscPrintf(ctx->comm, "%3D) species-%" PetscInt_FMT ": charge density= %20.13e z-momentum= %20.13e energy= %20.13e",stepi,ii,PetscRealPart(density[ii]),PetscRealPart(zmomentum[ii]),PetscRealPart(energy[ii]));
2486:       } else { /* 2/3Xloc + 3V */
2487:         PetscDSSetObjective(prob, 0, &f0_s_den);
2488:         DMPlexComputeIntegralFEM(ctx->plex[grid],Xloc,tt,ctx);
2489:         density[ii] = tt[0]*ctx->n_0*ctx->charges[ii];
2490:         PetscDSSetObjective(prob, 0, &f0_s_mom);
2491:         user[1] = 0;
2492:         PetscDSSetConstants(prob, 2, user);
2493:         DMPlexComputeIntegralFEM(ctx->plex[grid],Xloc,tt,ctx);
2494:         xmomentum[ii]  = tt[0]*ctx->n_0*ctx->v_0*ctx->masses[ii];
2495:         user[1] = 1;
2496:         PetscDSSetConstants(prob, 2, user);
2497:         DMPlexComputeIntegralFEM(ctx->plex[grid],Xloc,tt,ctx);
2498:         ymomentum[ii] = tt[0]*ctx->n_0*ctx->v_0*ctx->masses[ii];
2499:         user[1] = 2;
2500:         PetscDSSetConstants(prob, 2, user);
2501:         DMPlexComputeIntegralFEM(ctx->plex[grid],Xloc,tt,ctx);
2502:         zmomentum[ii] = tt[0]*ctx->n_0*ctx->v_0*ctx->masses[ii];
2503:         if (ctx->use_relativistic_corrections) {
2504:           /* gamma * M * f */
2505:           if (ii==0 && grid==0) { // do all at once
2506:             Vec            Mf, globGamma, *globMfArray, *globGammaArray;
2507:             PetscErrorCode (*gammaf[1])(PetscInt, PetscReal, const PetscReal [], PetscInt, PetscScalar [], void *) = {gamma_n_f};
2508:             PetscReal      *c2_0[1], data[1];

2510:             VecDuplicate(X,&globGamma);
2511:             VecDuplicate(X,&Mf);
2512:             PetscMalloc(sizeof(*globMfArray)*nDMs, &globMfArray);
2513:             PetscMalloc(sizeof(*globMfArray)*nDMs, &globGammaArray);
2514:             /* M * f */
2515:             MatMult(ctx->M,X,Mf);
2516:             /* gamma */
2517:             DMCompositeGetAccessArray(pack, globGamma, nDMs, NULL, globGammaArray);
2518:             for (PetscInt grid = 0; grid < ctx->num_grids ; grid++) { // yes a grid loop in a grid loop to print nice, need to fix for batching
2519:               Vec v1 = globGammaArray[ LAND_PACK_IDX(ctx->batch_view_idx,grid) ];
2520:               data[0] = PetscSqr(C_0(ctx->v_0));
2521:               c2_0[0] = &data[0];
2522:               DMProjectFunction(ctx->plex[grid], 0., gammaf, (void**)c2_0, INSERT_ALL_VALUES, v1);
2523:             }
2524:             DMCompositeRestoreAccessArray(pack, globGamma, nDMs, NULL, globGammaArray);
2525:             /* gamma * Mf */
2526:             DMCompositeGetAccessArray(pack, globGamma, nDMs, NULL, globGammaArray);
2527:             DMCompositeGetAccessArray(pack, Mf, nDMs, NULL, globMfArray);
2528:             for (PetscInt grid = 0; grid < ctx->num_grids ; grid++) { // yes a grid loop in a grid loop to print nice
2529:               PetscInt Nf = ctx->species_offset[grid+1] - ctx->species_offset[grid], N, bs;
2530:               Vec      Mfsub = globMfArray[ LAND_PACK_IDX(ctx->batch_view_idx,grid) ], Gsub = globGammaArray[ LAND_PACK_IDX(ctx->batch_view_idx,grid) ], v1, v2;
2531:               // get each component
2532:               VecGetSize(Mfsub,&N);
2533:               VecCreate(ctx->comm,&v1);
2534:               VecSetSizes(v1,PETSC_DECIDE,N/Nf);
2535:               VecCreate(ctx->comm,&v2);
2536:               VecSetSizes(v2,PETSC_DECIDE,N/Nf);
2537:               VecSetFromOptions(v1); // ???
2538:               VecSetFromOptions(v2);
2539:               // get each component
2540:               VecGetBlockSize(Gsub,&bs);
2542:               VecGetBlockSize(Mfsub,&bs);
2544:               for (int i=0, ix=ctx->species_offset[grid] ; i<Nf ; i++, ix++) {
2545:                 PetscScalar val;
2546:                 VecStrideGather(Gsub,i,v1,INSERT_VALUES);
2547:                 VecStrideGather(Mfsub,i,v2,INSERT_VALUES);
2548:                 VecDot(v1,v2,&val);
2549:                 energy[ix] = PetscRealPart(val)*ctx->n_0*ctx->v_0*ctx->v_0*ctx->masses[ix];
2550:               }
2551:               VecDestroy(&v1);
2552:               VecDestroy(&v2);
2553:             } /* grids */
2554:             DMCompositeRestoreAccessArray(pack, globGamma, nDMs, NULL, globGammaArray);
2555:             DMCompositeRestoreAccessArray(pack, Mf, nDMs, NULL, globMfArray);
2556:             PetscFree(globGammaArray);
2557:             PetscFree(globMfArray);
2558:             VecDestroy(&globGamma);
2559:             VecDestroy(&Mf);
2560:           }
2561:         } else {
2562:           PetscDSSetObjective(prob, 0, &f0_s_v2);
2563:           DMPlexComputeIntegralFEM(ctx->plex[grid],Xloc,tt,ctx);
2564:           energy[ii]    = 0.5*tt[0]*ctx->n_0*ctx->v_0*ctx->v_0*ctx->masses[ii];
2565:         }
2566:         PetscPrintf(ctx->comm, "%3" PetscInt_FMT ") species %" PetscInt_FMT ": density=%20.13e, x-momentum=%20.13e, y-momentum=%20.13e, z-momentum=%20.13e, energy=%21.13e",stepi,ii,PetscRealPart(density[ii]),PetscRealPart(xmomentum[ii]),PetscRealPart(ymomentum[ii]),PetscRealPart(zmomentum[ii]),PetscRealPart(energy[ii]));
2567:         xmomentumtot += xmomentum[ii];
2568:         ymomentumtot += ymomentum[ii];
2569:         zmomentumtot += zmomentum[ii];
2570:         energytot    += energy[ii];
2571:         densitytot   += density[ii];
2572:       }
2573:       if (ctx->num_species>1) PetscPrintf(ctx->comm, "\n");
2574:     }
2575:   }
2576:   DMCompositeRestoreAccessArray(pack, X, nDMs, NULL, globXArray);
2577:   PetscFree(globXArray);
2578:   /* totals */
2579:   DMPlexGetHeightStratum(ctx->plex[0],0,&cStart,&cEnd);
2580:   if (ctx->num_species>1) {
2581:     if (dim==2) {
2582:       PetscPrintf(ctx->comm, "\t%3" PetscInt_FMT ") Total: charge density=%21.13e, momentum=%21.13e, energy=%21.13e (m_i[0]/m_e = %g, %" PetscInt_FMT " cells on electron grid)",stepi,(double)PetscRealPart(densitytot),(double)PetscRealPart(zmomentumtot),(double)PetscRealPart(energytot),(double)(ctx->masses[1]/ctx->masses[0]),cEnd-cStart);
2583:     } else {
2584:       PetscPrintf(ctx->comm, "\t%3" PetscInt_FMT ") Total: charge density=%21.13e, x-momentum=%21.13e, y-momentum=%21.13e, z-momentum=%21.13e, energy=%21.13e (m_i[0]/m_e = %g, %" PetscInt_FMT " cells)",stepi,(double)PetscRealPart(densitytot),(double)PetscRealPart(xmomentumtot),(double)PetscRealPart(ymomentumtot),(double)PetscRealPart(zmomentumtot),(double)PetscRealPart(energytot),(double)(ctx->masses[1]/ctx->masses[0]),cEnd-cStart);
2585:     }
2586:   } else PetscPrintf(ctx->comm, " -- %" PetscInt_FMT " cells",cEnd-cStart);
2587:   PetscPrintf(ctx->comm,"\n");
2588:   return 0;
2589: }

2591: /*@
2592:  DMPlexLandauCreateMassMatrix - Create mass matrix for Landau in Plex space (not field major order of Jacobian)

2594:  Collective on pack

2596:  Input Parameters:
2597:  . pack     - the DM object

2599:  Output Parameters:
2600:  . Amat - The mass matrix (optional), mass matrix is added to the DM context

2602:  Level: beginner

2604:  .keywords: mesh
2605:  .seealso: DMPlexLandauCreateVelocitySpace()
2606:  @*/
2607: PetscErrorCode DMPlexLandauCreateMassMatrix(DM pack, Mat *Amat)
2608: {
2609:   DM             mass_pack,massDM[LANDAU_MAX_GRIDS];
2610:   PetscDS        prob;
2611:   PetscInt       ii,dim,N1=1,N2;
2612:   LandauCtx      *ctx;
2613:   Mat            packM,subM[LANDAU_MAX_GRIDS];

2617:   DMGetApplicationContext(pack, &ctx);
2619:   PetscLogEventBegin(ctx->events[14],0,0,0,0);
2620:   DMGetDimension(pack, &dim);
2621:   DMCompositeCreate(PetscObjectComm((PetscObject) pack),&mass_pack);
2622:   /* create pack mass matrix */
2623:   for (PetscInt grid=0, ix=0 ; grid<ctx->num_grids ; grid++) {
2624:     DMClone(ctx->plex[grid], &massDM[grid]);
2625:     DMCopyFields(ctx->plex[grid], massDM[grid]);
2626:     DMCreateDS(massDM[grid]);
2627:     DMGetDS(massDM[grid], &prob);
2628:     for (ix=0, ii=ctx->species_offset[grid];ii<ctx->species_offset[grid+1];ii++,ix++) {
2629:       if (dim==3) PetscDSSetJacobian(prob, ix, ix, g0_1, NULL, NULL, NULL);
2630:       else        PetscDSSetJacobian(prob, ix, ix, g0_r, NULL, NULL, NULL);
2631:     }
2632: #if !defined(LANDAU_SPECIES_MAJOR)
2633:     DMCompositeAddDM(mass_pack,massDM[grid]);
2634: #else
2635:     for (PetscInt b_id=0;b_id<ctx->batch_sz;b_id++) { // add batch size DMs for this species grid
2636:       DMCompositeAddDM(mass_pack,massDM[grid]);
2637:     }
2638: #endif
2639:     DMCreateMatrix(massDM[grid], &subM[grid]);
2640:   }
2641: #if !defined(LANDAU_SPECIES_MAJOR)
2642:   // stack the batched DMs
2643:   for (PetscInt b_id=1;b_id<ctx->batch_sz;b_id++) {
2644:     for (PetscInt grid=0;grid<ctx->num_grids;grid++) {
2645:       DMCompositeAddDM(mass_pack, massDM[grid]);
2646:     }
2647:   }
2648: #endif
2649:   PetscOptionsInsertString(NULL,"-dm_preallocate_only");
2650:   DMSetFromOptions(mass_pack);
2651:   DMCreateMatrix(mass_pack, &packM);
2652:   PetscOptionsInsertString(NULL,"-dm_preallocate_only false");
2653:   MatSetOption(packM,MAT_STRUCTURALLY_SYMMETRIC, PETSC_TRUE);
2654:   MatSetOption(packM,MAT_IGNORE_ZERO_ENTRIES,PETSC_TRUE);
2655:   DMDestroy(&mass_pack);
2656:   /* make mass matrix for each block */
2657:   for (PetscInt grid=0;grid<ctx->num_grids;grid++) {
2658:     Vec locX;
2659:     DM  plex = massDM[grid];
2660:     DMGetLocalVector(plex, &locX);
2661:     /* Mass matrix is independent of the input, so no need to fill locX */
2662:     DMPlexSNESComputeJacobianFEM(plex, locX, subM[grid], subM[grid], ctx);
2663:     DMRestoreLocalVector(plex, &locX);
2664:     DMDestroy(&massDM[grid]);
2665:   }
2666:   MatGetSize(ctx->J, &N1, NULL);
2667:   MatGetSize(packM, &N2, NULL);
2669:   /* assemble block diagonals */
2670:   for (PetscInt grid=0 ; grid<ctx->num_grids ; grid++) {
2671:     Mat               B = subM[grid];
2672:     PetscInt          nloc, nzl, colbuf[1024], row;
2673:     MatGetSize(B, &nloc, NULL);
2674:     for (PetscInt b_id = 0 ; b_id < ctx->batch_sz ; b_id++) {
2675:       const PetscInt    moffset = LAND_MOFFSET(b_id,grid,ctx->batch_sz,ctx->num_grids,ctx->mat_offset);
2676:       const PetscInt    *cols;
2677:       const PetscScalar *vals;
2678:       for (int i=0 ; i<nloc ; i++) {
2679:         MatGetRow(B,i,&nzl,&cols,&vals);
2681:         for (int j=0; j<nzl; j++) colbuf[j] = cols[j] + moffset;
2682:         row = i + moffset;
2683:         MatSetValues(packM,1,&row,nzl,colbuf,vals,INSERT_VALUES);
2684:         MatRestoreRow(B,i,&nzl,&cols,&vals);
2685:       }
2686:     }
2687:   }
2688:   // cleanup
2689:   for (PetscInt grid=0 ; grid<ctx->num_grids ; grid++) {
2690:     MatDestroy(&subM[grid]);
2691:   }
2692:   MatAssemblyBegin(packM,MAT_FINAL_ASSEMBLY);
2693:   MatAssemblyEnd(packM,MAT_FINAL_ASSEMBLY);
2694:   PetscObjectSetName((PetscObject)packM, "mass");
2695:   MatViewFromOptions(packM,NULL,"-dm_landau_mass_view");
2696:   ctx->M = packM;
2697:   if (Amat) *Amat = packM;
2698:   PetscLogEventEnd(ctx->events[14],0,0,0,0);
2699:   return 0;
2700: }

2702: /*@
2703:  DMPlexLandauIFunction - TS residual calculation

2705:  Collective on ts

2707:  Input Parameters:
2708:  +   TS  - The time stepping context
2709:  .   time_dummy - current time (not used)
2710:  -   X - Current state
2711:  +   X_t - Time derivative of current state
2712:  .   actx - Landau context

2714:  Output Parameter:
2715:  .   F  - The residual

2717:  Level: beginner

2719:  .keywords: mesh
2720:  .seealso: DMPlexLandauCreateVelocitySpace(), DMPlexLandauIJacobian()
2721:  @*/
2722: PetscErrorCode DMPlexLandauIFunction(TS ts, PetscReal time_dummy, Vec X, Vec X_t, Vec F, void *actx)
2723: {
2724:   LandauCtx      *ctx=(LandauCtx*)actx;
2725:   PetscInt       dim;
2726:   DM             pack;
2727: #if defined(PETSC_HAVE_THREADSAFETY)
2728:   double         starttime, endtime;
2729: #endif

2731:   TSGetDM(ts,&pack);
2732:   DMGetApplicationContext(pack, &ctx);
2734:   if (ctx->stage) {
2735:     PetscLogStagePush(ctx->stage);
2736:   }
2737:   PetscLogEventBegin(ctx->events[11],0,0,0,0);
2738:   PetscLogEventBegin(ctx->events[0],0,0,0,0);
2739: #if defined(PETSC_HAVE_THREADSAFETY)
2740:   starttime = MPI_Wtime();
2741: #endif
2742:   DMGetDimension(pack, &dim);
2743:   if (!ctx->aux_bool) {
2744:     PetscInfo(ts, "Create Landau Jacobian t=%g X=%p %s\n",time_dummy,X_t,ctx->aux_bool ? " -- seems to be in line search" : "");
2745:     LandauFormJacobian_Internal(X,ctx->J,dim,0.0,(void*)ctx);
2746:     MatViewFromOptions(ctx->J, NULL, "-dm_landau_jacobian_view");
2747:     ctx->aux_bool = PETSC_TRUE;
2748:   } else {
2749:     PetscInfo(ts, "Skip forming Jacobian, has not changed (should check norm)\n");
2750:   }
2751:   /* mat vec for op */
2752:   MatMult(ctx->J,X,F); /* C*f */
2753:   /* add time term */
2754:   if (X_t) {
2755:     MatMultAdd(ctx->M,X_t,F,F);
2756:   }
2757: #if defined(PETSC_HAVE_THREADSAFETY)
2758:   if (ctx->stage) {
2759:     endtime = MPI_Wtime();
2760:     ctx->times[LANDAU_OPERATOR] += (endtime - starttime);
2761:     ctx->times[LANDAU_JACOBIAN] += (endtime - starttime);
2762:     ctx->times[LANDAU_JACOBIAN_COUNT] += 1;
2763:   }
2764: #endif
2765:   PetscLogEventEnd(ctx->events[0],0,0,0,0);
2766:   PetscLogEventEnd(ctx->events[11],0,0,0,0);
2767:   if (ctx->stage) {
2768:     PetscLogStagePop();
2769: #if defined(PETSC_HAVE_THREADSAFETY)
2770:     ctx->times[LANDAU_MATRIX_TOTAL] += (endtime - starttime);
2771: #endif
2772:   }
2773:   return 0;
2774: }

2776: /*@
2777:  DMPlexLandauIJacobian - TS Jacobian construction

2779:  Collective on ts

2781:  Input Parameters:
2782:  +   TS  - The time stepping context
2783:  .   time_dummy - current time (not used)
2784:  -   X - Current state
2785:  +   U_tdummy - Time derivative of current state (not used)
2786:  .   shift - shift for du/dt term
2787:  -   actx - Landau context

2789:  Output Parameter:
2790:  .   Amat  - Jacobian
2791:  +   Pmat  - same as Amat

2793:  Level: beginner

2795:  .keywords: mesh
2796:  .seealso: DMPlexLandauCreateVelocitySpace(), DMPlexLandauIFunction()
2797:  @*/
2798: PetscErrorCode DMPlexLandauIJacobian(TS ts, PetscReal time_dummy, Vec X, Vec U_tdummy, PetscReal shift, Mat Amat, Mat Pmat, void *actx)
2799: {
2800:   LandauCtx      *ctx=NULL;
2801:   PetscInt       dim;
2802:   DM             pack;
2803: #if defined(PETSC_HAVE_THREADSAFETY)
2804:   double         starttime, endtime;
2805: #endif
2806:   TSGetDM(ts,&pack);
2807:   DMGetApplicationContext(pack, &ctx);
2810:   DMGetDimension(pack, &dim);
2811:   /* get collision Jacobian into A */
2812:   if (ctx->stage) {
2813:     PetscLogStagePush(ctx->stage);
2814:   }
2815:   PetscLogEventBegin(ctx->events[11],0,0,0,0);
2816:   PetscLogEventBegin(ctx->events[9],0,0,0,0);
2817: #if defined(PETSC_HAVE_THREADSAFETY)
2818:   starttime = MPI_Wtime();
2819: #endif
2820:   PetscInfo(ts, "Adding just mass to Jacobian t=%g, shift=%g\n",(double)time_dummy,(double)shift);
2823:   if (!ctx->use_matrix_mass) {
2824:     LandauFormJacobian_Internal(X,ctx->J,dim,shift,(void*)ctx);
2825:     MatViewFromOptions(ctx->J, NULL, "-dm_landau_mat_view");
2826:   } else { /* add mass */
2827:     MatAXPY(Pmat,shift,ctx->M,SAME_NONZERO_PATTERN);
2828:   }
2829:   ctx->aux_bool = PETSC_FALSE;
2830: #if defined(PETSC_HAVE_THREADSAFETY)
2831:   if (ctx->stage) {
2832:     endtime = MPI_Wtime();
2833:     ctx->times[LANDAU_OPERATOR] += (endtime - starttime);
2834:     ctx->times[LANDAU_MASS] += (endtime - starttime);
2835:   }
2836: #endif
2837:   PetscLogEventEnd(ctx->events[9],0,0,0,0);
2838:   PetscLogEventEnd(ctx->events[11],0,0,0,0);
2839:   if (ctx->stage) {
2840:     PetscLogStagePop();
2841: #if defined(PETSC_HAVE_THREADSAFETY)
2842:     ctx->times[LANDAU_MATRIX_TOTAL] += (endtime - starttime);
2843: #endif
2844:   }
2845:   return 0;
2846: }