Date: 2026-03-31T05:46:16+00:00

Actual source code: bddcprivate.c

      1: #include <../src/mat/impls/aij/seq/aij.h>
      2: #include <petsc/private/pcbddcimpl.h>
      3: #include <petsc/private/pcbddcprivateimpl.h>
      4: #include <petsc/private/kernels/blockinvert.h>
      5: #include <../src/mat/impls/dense/seq/dense.h>
      6: #include <petscdmplex.h>
      7: #include <petscblaslapack.h>
      8: #include <petsc/private/sfimpl.h>
      9: #include <petsc/private/dmpleximpl.h>
     10: #include <petscdmda.h>

     12: static PetscErrorCode MatMPIAIJRestrict(Mat, MPI_Comm, Mat *);

     14: /* if range is true,  it returns B s.t. span{B} = range(A)
     15:    if range is false, it returns B s.t. range(B) _|_ range(A) */
     16: static PetscErrorCode MatDenseOrthogonalRangeOrComplement(Mat A, PetscBool range, PetscInt lw, PetscScalar *work, PetscReal *rwork, Mat *B)
     17: {
     18:   PetscScalar *uwork, *data, *U, ds = 0.;
     19:   PetscReal   *sing;
     20:   PetscBLASInt bM, bN, lwork, lierr, di = 1;
     21:   PetscInt     ulw, i, nr, nc, n;
     22: #if defined(PETSC_USE_COMPLEX)
     23:   PetscReal *rwork2;
     24: #endif

     26:   PetscFunctionBegin;
     27:   PetscCall(MatGetSize(A, &nr, &nc));
     28:   if (!nr || !nc) PetscFunctionReturn(PETSC_SUCCESS);

     30:   /* workspace */
     31:   if (!work) {
     32:     ulw = PetscMax(PetscMax(1, 5 * PetscMin(nr, nc)), 3 * PetscMin(nr, nc) + PetscMax(nr, nc));
     33:     PetscCall(PetscMalloc1(ulw, &uwork));
     34:   } else {
     35:     ulw   = lw;
     36:     uwork = work;
     37:   }
     38:   n = PetscMin(nr, nc);
     39:   if (!rwork) {
     40:     PetscCall(PetscMalloc1(n, &sing));
     41:   } else {
     42:     sing = rwork;
     43:   }

     45:   /* SVD */
     46:   PetscCall(PetscMalloc1(nr * nr, &U));
     47:   PetscCall(PetscBLASIntCast(nr, &bM));
     48:   PetscCall(PetscBLASIntCast(nc, &bN));
     49:   PetscCall(PetscBLASIntCast(ulw, &lwork));
     50:   PetscCall(MatDenseGetArray(A, &data));
     51:   PetscCall(PetscFPTrapPush(PETSC_FP_TRAP_OFF));
     52: #if !defined(PETSC_USE_COMPLEX)
     53:   PetscCallBLAS("LAPACKgesvd", LAPACKgesvd_("A", "N", &bM, &bN, data, &bM, sing, U, &bM, &ds, &di, uwork, &lwork, &lierr));
     54: #else
     55:   PetscCall(PetscMalloc1(5 * n, &rwork2));
     56:   PetscCallBLAS("LAPACKgesvd", LAPACKgesvd_("A", "N", &bM, &bN, data, &bM, sing, U, &bM, &ds, &di, uwork, &lwork, rwork2, &lierr));
     57:   PetscCall(PetscFree(rwork2));
     58: #endif
     59:   PetscCall(PetscFPTrapPop());
     60:   PetscCheck(!lierr, PETSC_COMM_SELF, PETSC_ERR_LIB, "Error in GESVD Lapack routine %" PetscBLASInt_FMT, lierr);
     61:   PetscCall(MatDenseRestoreArray(A, &data));
     62:   for (i = 0; i < n; i++)
     63:     if (sing[i] < PETSC_SMALL) break;
     64:   if (!rwork) PetscCall(PetscFree(sing));
     65:   if (!work) PetscCall(PetscFree(uwork));
     66:   /* create B */
     67:   if (!range) {
     68:     PetscCall(MatCreateSeqDense(PETSC_COMM_SELF, nr, nr - i, NULL, B));
     69:     PetscCall(MatDenseGetArray(*B, &data));
     70:     PetscCall(PetscArraycpy(data, U + nr * i, (nr - i) * nr));
     71:   } else {
     72:     PetscCall(MatCreateSeqDense(PETSC_COMM_SELF, nr, i, NULL, B));
     73:     PetscCall(MatDenseGetArray(*B, &data));
     74:     PetscCall(PetscArraycpy(data, U, i * nr));
     75:   }
     76:   PetscCall(MatDenseRestoreArray(*B, &data));
     77:   PetscCall(PetscFree(U));
     78:   PetscFunctionReturn(PETSC_SUCCESS);
     79: }

     81: /* TODO REMOVE */
     82: #if defined(PRINT_GDET)
     83: static int inc = 0;
     84: static int lev = 0;
     85: #endif

     87: static PetscErrorCode PCBDDCComputeNedelecChangeEdge(Mat lG, IS edge, IS extrow, IS extcol, IS corners, Mat *Gins, Mat *GKins, PetscScalar cvals[2], PetscScalar *work, PetscReal *rwork)
     88: {
     89:   Mat          GE, GEd;
     90:   PetscInt     rsize, csize, esize;
     91:   PetscScalar *ptr;

     93:   PetscFunctionBegin;
     94:   PetscCall(ISGetSize(edge, &esize));
     95:   if (!esize) PetscFunctionReturn(PETSC_SUCCESS);
     96:   PetscCall(ISGetSize(extrow, &rsize));
     97:   PetscCall(ISGetSize(extcol, &csize));

     99:   /* gradients */
    100:   ptr = work + 5 * esize;
    101:   PetscCall(MatCreateSubMatrix(lG, extrow, extcol, MAT_INITIAL_MATRIX, &GE));
    102:   PetscCall(MatCreateSeqDense(PETSC_COMM_SELF, rsize, csize, ptr, Gins));
    103:   PetscCall(MatConvert(GE, MATSEQDENSE, MAT_REUSE_MATRIX, Gins));
    104:   PetscCall(MatDestroy(&GE));

    106:   /* constants */
    107:   ptr += rsize * csize;
    108:   PetscCall(MatCreateSeqDense(PETSC_COMM_SELF, esize, csize, ptr, &GEd));
    109:   PetscCall(MatCreateSubMatrix(lG, edge, extcol, MAT_INITIAL_MATRIX, &GE));
    110:   PetscCall(MatConvert(GE, MATSEQDENSE, MAT_REUSE_MATRIX, &GEd));
    111:   PetscCall(MatDestroy(&GE));
    112:   PetscCall(MatDenseOrthogonalRangeOrComplement(GEd, PETSC_FALSE, 5 * esize, work, rwork, GKins));
    113:   PetscCall(MatDestroy(&GEd));

    115:   if (corners) {
    116:     Mat                GEc;
    117:     const PetscScalar *vals;
    118:     PetscScalar        v;

    120:     PetscCall(MatCreateSubMatrix(lG, edge, corners, MAT_INITIAL_MATRIX, &GEc));
    121:     PetscCall(MatTransposeMatMult(GEc, *GKins, MAT_INITIAL_MATRIX, 1.0, &GEd));
    122:     PetscCall(MatDenseGetArrayRead(GEd, &vals));
    123:     /* v       = PetscAbsScalar(vals[0]); */
    124:     v        = 1.;
    125:     cvals[0] = vals[0] / v;
    126:     cvals[1] = vals[1] / v;
    127:     PetscCall(MatDenseRestoreArrayRead(GEd, &vals));
    128:     PetscCall(MatScale(*GKins, 1. / v));
    129: #if defined(PRINT_GDET)
    130:     {
    131:       PetscViewer viewer;
    132:       char        filename[256];
    133:       PetscCall(PetscSNPrintf(filename, PETSC_STATIC_ARRAY_LENGTH(filename), "Gdet_l%d_r%d_cc%d.m", lev, PetscGlobalRank, inc++));
    134:       PetscCall(PetscViewerASCIIOpen(PETSC_COMM_SELF, filename, &viewer));
    135:       PetscCall(PetscViewerPushFormat(viewer, PETSC_VIEWER_ASCII_MATLAB));
    136:       PetscCall(PetscObjectSetName((PetscObject)GEc, "GEc"));
    137:       PetscCall(MatView(GEc, viewer));
    138:       PetscCall(PetscObjectSetName((PetscObject)*GKins, "GK"));
    139:       PetscCall(MatView(*GKins, viewer));
    140:       PetscCall(PetscObjectSetName((PetscObject)GEd, "Gproj"));
    141:       PetscCall(MatView(GEd, viewer));
    142:       PetscCall(PetscViewerDestroy(&viewer));
    143:     }
    144: #endif
    145:     PetscCall(MatDestroy(&GEd));
    146:     PetscCall(MatDestroy(&GEc));
    147:   }
    148:   PetscFunctionReturn(PETSC_SUCCESS);
    149: }

    151: static PetscErrorCode MatAIJExtractRows(Mat, IS, Mat *);

    153: PetscErrorCode PCBDDCNedelecSupport(PC pc)
    154: {
    155:   PC_BDDC               *pcbddc = (PC_BDDC *)pc->data;
    156:   Mat_IS                *matis  = (Mat_IS *)pc->pmat->data;
    157:   Mat                    G, T, conn, lG, lGt, lGis, lGall, lGe, lGinit;
    158:   PetscSF                sfv;
    159:   ISLocalToGlobalMapping el2g, vl2g, fl2g, al2g;
    160:   MPI_Comm               comm;
    161:   IS                     lned, primals, allprimals, nedfieldlocal, elements_corners = NULL;
    162:   IS                    *eedges, *extrows, *extcols, *alleedges;
    163:   PetscBT                btv, bte, btvc, btb, btbd, btvcand, btvi, btee, bter;
    164:   PetscScalar           *vals, *work;
    165:   PetscReal             *rwork;
    166:   const PetscInt        *idxs, *ii, *jj, *iit, *jjt;
    167:   PetscInt               ne, nv, Lv, order, n, field;
    168:   PetscInt               i, j, extmem, cum, maxsize, nee;
    169:   PetscInt              *extrow, *extrowcum, *marks, *vmarks, *gidxs;
    170:   PetscInt              *sfvleaves, *sfvroots;
    171:   PetscInt              *corners, *cedges;
    172:   PetscInt              *ecount, **eneighs, *vcount, **vneighs;
    173:   PetscInt              *emarks;
    174:   PetscBool              print, eerr, done, lrc[2], conforming, global, setprimal;

    176:   PetscFunctionBegin;
    177:   /* If the discrete gradient is defined for a subset of dofs and global is true,
    178:      it assumes G is given in global ordering for all the dofs.
    179:      Otherwise, the ordering is global for the Nedelec field */
    180:   order      = pcbddc->nedorder;
    181:   conforming = pcbddc->conforming;
    182:   field      = pcbddc->nedfield;
    183:   global     = pcbddc->nedglobal;
    184:   setprimal  = PETSC_FALSE;
    185:   print      = PETSC_FALSE;

    187:   /* Command line customization */
    188:   PetscOptionsBegin(PetscObjectComm((PetscObject)pc), ((PetscObject)pc)->prefix, "BDDC Nedelec options", "PC");
    189:   PetscCall(PetscOptionsBool("-pc_bddc_nedelec_field_primal", "All edge dofs set as primals: Toselli's algorithm C", NULL, setprimal, &setprimal, NULL));
    190:   /* print debug info and adaptive order TODO: to be removed */
    191:   PetscCall(PetscOptionsInt("-pc_bddc_nedelec_order", "Test variable order code (to be removed)", NULL, order, &order, NULL));
    192:   PetscCall(PetscOptionsBool("-pc_bddc_nedelec_print", "Print debug info", NULL, print, &print, NULL));
    193:   PetscOptionsEnd();

    195:   /* Return if there are no edges in the decomposition */
    196:   PetscCall(MatISGetLocalToGlobalMapping(pc->pmat, &al2g, NULL));
    197:   PetscCall(ISLocalToGlobalMappingGetSize(al2g, &n));
    198:   PetscCall(PetscObjectGetComm((PetscObject)pc, &comm));
    199:   PetscCall(VecGetArrayRead(matis->counter, (const PetscScalar **)&vals));
    200:   lrc[0] = PETSC_FALSE;
    201:   for (i = 0; i < n; i++) {
    202:     if (PetscRealPart(vals[i]) > 2.) {
    203:       lrc[0] = PETSC_TRUE;
    204:       break;
    205:     }
    206:   }
    207:   PetscCall(VecRestoreArrayRead(matis->counter, (const PetscScalar **)&vals));
    208:   PetscCallMPI(MPIU_Allreduce(&lrc[0], &lrc[1], 1, MPI_C_BOOL, MPI_LOR, comm));
    209:   if (!lrc[1]) PetscFunctionReturn(PETSC_SUCCESS);

    211:   /* Get Nedelec field */
    212:   PetscCheck(!pcbddc->n_ISForDofsLocal || field < pcbddc->n_ISForDofsLocal, comm, PETSC_ERR_USER, "Invalid field for Nedelec %" PetscInt_FMT ": number of fields is %" PetscInt_FMT, field, pcbddc->n_ISForDofsLocal);
    213:   if (pcbddc->n_ISForDofsLocal && field >= 0) {
    214:     PetscCall(PetscObjectReference((PetscObject)pcbddc->ISForDofsLocal[field]));
    215:     nedfieldlocal = pcbddc->ISForDofsLocal[field];
    216:     PetscCall(ISGetLocalSize(nedfieldlocal, &ne));
    217:   } else if (!pcbddc->n_ISForDofsLocal && field != PETSC_DECIDE) {
    218:     ne            = n;
    219:     nedfieldlocal = NULL;
    220:     global        = PETSC_TRUE;
    221:   } else if (field == PETSC_DECIDE) {
    222:     PetscInt rst, ren, *idx;

    224:     PetscCall(PetscArrayzero(matis->sf_leafdata, n));
    225:     PetscCall(PetscArrayzero(matis->sf_rootdata, pc->pmat->rmap->n));
    226:     PetscCall(MatGetOwnershipRange(pcbddc->discretegradient, &rst, &ren));
    227:     for (i = rst; i < ren; i++) {
    228:       PetscInt nc;

    230:       PetscCall(MatGetRow(pcbddc->discretegradient, i, &nc, NULL, NULL));
    231:       if (nc > 1) matis->sf_rootdata[i - rst] = 1;
    232:       PetscCall(MatRestoreRow(pcbddc->discretegradient, i, &nc, NULL, NULL));
    233:     }
    234:     PetscCall(PetscSFBcastBegin(matis->sf, MPIU_INT, matis->sf_rootdata, matis->sf_leafdata, MPI_REPLACE));
    235:     PetscCall(PetscSFBcastEnd(matis->sf, MPIU_INT, matis->sf_rootdata, matis->sf_leafdata, MPI_REPLACE));
    236:     PetscCall(PetscMalloc1(n, &idx));
    237:     for (i = 0, ne = 0; i < n; i++)
    238:       if (matis->sf_leafdata[i]) idx[ne++] = i;
    239:     PetscCall(ISCreateGeneral(comm, ne, idx, PETSC_OWN_POINTER, &nedfieldlocal));
    240:   } else {
    241:     SETERRQ(comm, PETSC_ERR_USER, "When multiple fields are present, the Nedelec field has to be specified");
    242:   }

    244:   /* Sanity checks */
    245:   PetscCheck(order || conforming, comm, PETSC_ERR_SUP, "Variable order and non-conforming spaces are not supported at the same time");
    246:   PetscCheck(!pcbddc->user_ChangeOfBasisMatrix, comm, PETSC_ERR_SUP, "Cannot generate Nedelec support with user defined change of basis");
    247:   PetscCheck(!order || (ne % order == 0), PETSC_COMM_SELF, PETSC_ERR_USER, "The number of local edge dofs %" PetscInt_FMT " is not a multiple of the order %" PetscInt_FMT, ne, order);

    249:   /* Just set primal dofs and return */
    250:   if (setprimal) {
    251:     IS        enedfieldlocal;
    252:     PetscInt *eidxs;

    254:     PetscCall(PetscMalloc1(ne, &eidxs));
    255:     PetscCall(VecGetArrayRead(matis->counter, (const PetscScalar **)&vals));
    256:     if (nedfieldlocal) {
    257:       PetscCall(ISGetIndices(nedfieldlocal, &idxs));
    258:       for (i = 0, cum = 0; i < ne; i++) {
    259:         if (PetscRealPart(vals[idxs[i]]) > 2.) eidxs[cum++] = idxs[i];
    260:       }
    261:       PetscCall(ISRestoreIndices(nedfieldlocal, &idxs));
    262:     } else {
    263:       for (i = 0, cum = 0; i < ne; i++) {
    264:         if (PetscRealPart(vals[i]) > 2.) eidxs[cum++] = i;
    265:       }
    266:     }
    267:     PetscCall(VecRestoreArrayRead(matis->counter, (const PetscScalar **)&vals));
    268:     PetscCall(ISCreateGeneral(comm, cum, eidxs, PETSC_COPY_VALUES, &enedfieldlocal));
    269:     PetscCall(PCBDDCSetPrimalVerticesLocalIS(pc, enedfieldlocal));
    270:     PetscCall(PetscFree(eidxs));
    271:     PetscCall(ISDestroy(&nedfieldlocal));
    272:     PetscCall(ISDestroy(&enedfieldlocal));
    273:     PetscFunctionReturn(PETSC_SUCCESS);
    274:   }

    276:   /* Compute some l2g maps */
    277:   if (nedfieldlocal) {
    278:     IS is;

    280:     /* need to map from the local Nedelec field to local numbering */
    281:     PetscCall(ISLocalToGlobalMappingCreateIS(nedfieldlocal, &fl2g));
    282:     /* need to map from the local Nedelec field to global numbering for the whole dofs*/
    283:     PetscCall(ISLocalToGlobalMappingApplyIS(al2g, nedfieldlocal, &is));
    284:     PetscCall(ISLocalToGlobalMappingCreateIS(is, &al2g));
    285:     /* need to map from the local Nedelec field to global numbering (for Nedelec only) */
    286:     if (global) {
    287:       PetscCall(PetscObjectReference((PetscObject)al2g));
    288:       el2g = al2g;
    289:     } else {
    290:       IS gis;

    292:       PetscCall(ISRenumber(is, NULL, NULL, &gis));
    293:       PetscCall(ISLocalToGlobalMappingCreateIS(gis, &el2g));
    294:       PetscCall(ISDestroy(&gis));
    295:     }
    296:     PetscCall(ISDestroy(&is));
    297:   } else {
    298:     /* one ref for the destruction of al2g, one for el2g */
    299:     PetscCall(PetscObjectReference((PetscObject)al2g));
    300:     PetscCall(PetscObjectReference((PetscObject)al2g));
    301:     el2g = al2g;
    302:     fl2g = NULL;
    303:   }

    305:   /* Start communication to drop connections for interior edges (for cc analysis only) */
    306:   PetscCall(PetscArrayzero(matis->sf_leafdata, n));
    307:   PetscCall(PetscArrayzero(matis->sf_rootdata, pc->pmat->rmap->n));
    308:   if (nedfieldlocal) {
    309:     PetscCall(ISGetIndices(nedfieldlocal, &idxs));
    310:     for (i = 0; i < ne; i++) matis->sf_leafdata[idxs[i]] = 1;
    311:     PetscCall(ISRestoreIndices(nedfieldlocal, &idxs));
    312:   } else {
    313:     for (i = 0; i < ne; i++) matis->sf_leafdata[i] = 1;
    314:   }
    315:   PetscCall(PetscSFReduceBegin(matis->sf, MPIU_INT, matis->sf_leafdata, matis->sf_rootdata, MPI_SUM));
    316:   PetscCall(PetscSFReduceEnd(matis->sf, MPIU_INT, matis->sf_leafdata, matis->sf_rootdata, MPI_SUM));

    318:   /* There's no way to detect all possible corner candidates in a element-by-element case in a pure algebraic setting
    319:      Firedrake attaches a index set to identify them upfront. If it is present, we assume we are in such a case */
    320:   if (matis->allow_repeated) PetscCall(PetscObjectQuery((PetscObject)pcbddc->discretegradient, "_elements_corners", (PetscObject *)&elements_corners));

    322:   /* drop connections with interior edges to avoid unneeded communications and memory movements */
    323:   PetscCall(MatViewFromOptions(pcbddc->discretegradient, (PetscObject)pc, "-pc_bddc_discrete_gradient_view"));
    324:   PetscCall(MatDuplicate(pcbddc->discretegradient, MAT_COPY_VALUES, &G));
    325:   PetscCall(MatSetOption(G, MAT_KEEP_NONZERO_PATTERN, PETSC_FALSE));
    326:   if (global) {
    327:     PetscInt rst;

    329:     PetscCall(MatGetOwnershipRange(G, &rst, NULL));
    330:     for (i = 0, cum = 0; i < pc->pmat->rmap->n; i++) {
    331:       if (matis->sf_rootdata[i] < 2) matis->sf_rootdata[cum++] = i + rst;
    332:     }
    333:     PetscCall(MatSetOption(G, MAT_NO_OFF_PROC_ZERO_ROWS, PETSC_TRUE));
    334:     PetscCall(MatZeroRows(G, cum, matis->sf_rootdata, 0., NULL, NULL));
    335:   } else {
    336:     PetscInt *tbz;

    338:     PetscCall(PetscMalloc1(ne, &tbz));
    339:     PetscCall(PetscSFBcastBegin(matis->sf, MPIU_INT, matis->sf_rootdata, matis->sf_leafdata, MPI_REPLACE));
    340:     PetscCall(PetscSFBcastEnd(matis->sf, MPIU_INT, matis->sf_rootdata, matis->sf_leafdata, MPI_REPLACE));
    341:     PetscCall(ISGetIndices(nedfieldlocal, &idxs));
    342:     for (i = 0, cum = 0; i < ne; i++)
    343:       if (matis->sf_leafdata[idxs[i]] == 1) tbz[cum++] = i;
    344:     PetscCall(ISRestoreIndices(nedfieldlocal, &idxs));
    345:     PetscCall(ISLocalToGlobalMappingApply(el2g, cum, tbz, tbz));
    346:     PetscCall(MatZeroRows(G, cum, tbz, 0., NULL, NULL));
    347:     PetscCall(PetscFree(tbz));
    348:   }

    350:   /* Extract subdomain relevant rows of G  */
    351:   PetscCall(ISLocalToGlobalMappingGetIndices(el2g, &idxs));
    352:   PetscCall(ISCreateGeneral(comm, ne, idxs, PETSC_USE_POINTER, &lned));
    353:   PetscCall(MatAIJExtractRows(G, lned, &lGall));
    354:   /* PetscCall(MatCreateSubMatrix(G, lned, NULL, MAT_INITIAL_MATRIX, &lGall)); */
    355:   PetscCall(ISLocalToGlobalMappingRestoreIndices(el2g, &idxs));
    356:   PetscCall(ISDestroy(&lned));
    357:   PetscCall(MatConvert(lGall, MATIS, MAT_INITIAL_MATRIX, &lGis));
    358:   PetscCall(MatDestroy(&lGall));
    359:   PetscCall(MatISGetLocalMat(lGis, &lG));
    360:   if (matis->allow_repeated) { /* multi-element support */
    361:     Mat                   *lGn, B;
    362:     IS                    *is_rows, *tcols, tmap, nmap;
    363:     PetscInt               subnv;
    364:     const PetscInt        *subvidxs;
    365:     ISLocalToGlobalMapping mapn;

    367:     PetscCall(PetscCalloc1(pcbddc->n_local_subs * pcbddc->n_local_subs, &lGn));
    368:     PetscCall(PetscMalloc1(pcbddc->n_local_subs, &is_rows));
    369:     PetscCall(PetscMalloc1(pcbddc->n_local_subs, &tcols));
    370:     for (PetscInt i = 0; i < pcbddc->n_local_subs; i++) {
    371:       if (fl2g) {
    372:         PetscCall(ISGlobalToLocalMappingApplyIS(fl2g, IS_GTOLM_MASK, pcbddc->local_subs[i], &is_rows[i]));
    373:       } else {
    374:         PetscCall(PetscObjectReference((PetscObject)pcbddc->local_subs[i]));
    375:         is_rows[i] = pcbddc->local_subs[i];
    376:       }
    377:       PetscCall(MatCreateSubMatrix(lG, is_rows[i], NULL, MAT_INITIAL_MATRIX, &lGn[i * (1 + pcbddc->n_local_subs)]));
    378:       PetscCall(MatSeqAIJCompactOutExtraColumns_SeqAIJ(lGn[i * (1 + pcbddc->n_local_subs)], &mapn));
    379:       PetscCall(ISLocalToGlobalMappingGetSize(mapn, &subnv));
    380:       PetscCall(ISLocalToGlobalMappingGetIndices(mapn, &subvidxs));
    381:       PetscCall(ISCreateGeneral(PETSC_COMM_SELF, subnv, subvidxs, PETSC_COPY_VALUES, &tcols[i]));
    382:       PetscCall(ISLocalToGlobalMappingRestoreIndices(mapn, &subvidxs));
    383:       PetscCall(ISLocalToGlobalMappingDestroy(&mapn));
    384:     }

    386:     /* Create new MATIS with repeated vertices */
    387:     PetscCall(MatCreate(comm, &B));
    388:     PetscCall(MatSetSizes(B, lGis->rmap->n, lGis->cmap->n, lGis->rmap->N, lGis->cmap->N));
    389:     PetscCall(MatSetType(B, MATIS));
    390:     PetscCall(MatISSetAllowRepeated(B, PETSC_TRUE));
    391:     PetscCall(ISConcatenate(PETSC_COMM_SELF, pcbddc->n_local_subs, tcols, &tmap));
    392:     PetscCall(ISLocalToGlobalMappingApplyIS(lGis->cmap->mapping, tmap, &nmap));
    393:     PetscCall(ISDestroy(&tmap));
    394:     PetscCall(ISGetLocalSize(nmap, &subnv));
    395:     PetscCall(ISGetIndices(nmap, &subvidxs));
    396:     PetscCall(ISCreateGeneral(comm, subnv, subvidxs, PETSC_USE_POINTER, &tmap));
    397:     PetscCall(ISRestoreIndices(nmap, &subvidxs));
    398:     PetscCall(ISLocalToGlobalMappingCreateIS(tmap, &mapn));
    399:     PetscCall(ISDestroy(&tmap));
    400:     PetscCall(ISDestroy(&nmap));
    401:     PetscCall(MatSetLocalToGlobalMapping(B, lGis->rmap->mapping, mapn));
    402:     PetscCall(ISLocalToGlobalMappingDestroy(&mapn));
    403:     PetscCall(MatCreateNest(PETSC_COMM_SELF, pcbddc->n_local_subs, is_rows, pcbddc->n_local_subs, NULL, lGn, &lG));
    404:     for (PetscInt i = 0; i < pcbddc->n_local_subs; i++) {
    405:       PetscCall(MatDestroy(&lGn[i * (1 + pcbddc->n_local_subs)]));
    406:       PetscCall(ISDestroy(&is_rows[i]));
    407:       PetscCall(ISDestroy(&tcols[i]));
    408:     }
    409:     PetscCall(MatConvert(lG, MATSEQAIJ, MAT_INPLACE_MATRIX, &lG));
    410:     PetscCall(PetscFree(lGn));
    411:     PetscCall(PetscFree(is_rows));
    412:     PetscCall(PetscFree(tcols));
    413:     PetscCall(MatISSetLocalMat(B, lG));
    414:     PetscCall(MatDestroy(&lG));

    416:     PetscCall(MatDestroy(&lGis));
    417:     lGis = B;

    419:     lGis->assembled = PETSC_TRUE;
    420:   }
    421:   PetscCall(MatViewFromOptions(lGis, (PetscObject)pc, "-pc_bddc_nedelec_init_G_view"));

    423:   /* SF for nodal dofs communications */
    424:   PetscCall(MatGetLocalSize(G, NULL, &Lv));
    425:   PetscCall(MatISGetLocalToGlobalMapping(lGis, NULL, &vl2g));
    426:   PetscCall(PetscObjectReference((PetscObject)vl2g));
    427:   PetscCall(ISLocalToGlobalMappingGetSize(vl2g, &nv));
    428:   PetscCall(PetscSFCreate(comm, &sfv));
    429:   PetscCall(ISLocalToGlobalMappingGetIndices(vl2g, &idxs));
    430:   PetscCall(PetscSFSetGraphLayout(sfv, lGis->cmap, nv, NULL, PETSC_OWN_POINTER, idxs));
    431:   PetscCall(ISLocalToGlobalMappingRestoreIndices(vl2g, &idxs));

    433:   if (elements_corners) {
    434:     IS      tmp;
    435:     Vec     global, local;
    436:     Mat_IS *tGis = (Mat_IS *)lGis->data;

    438:     PetscCall(MatCreateVecs(lGis, &global, NULL));
    439:     PetscCall(MatCreateVecs(tGis->A, &local, NULL));
    440:     PetscCall(PCBDDCGlobalToLocal(tGis->cctx, global, local, elements_corners, &tmp));
    441:     PetscCall(VecDestroy(&global));
    442:     PetscCall(VecDestroy(&local));
    443:     elements_corners = tmp;
    444:   }

    446:   /* Destroy temporary G */
    447:   PetscCall(MatISGetLocalMat(lGis, &lG));
    448:   PetscCall(PetscObjectReference((PetscObject)lG));
    449:   PetscCall(MatDestroy(&G));
    450:   PetscCall(MatDestroy(&lGis));

    452:   if (print) {
    453:     PetscCall(PetscObjectSetName((PetscObject)lG, "initial_lG"));
    454:     PetscCall(MatView(lG, NULL));
    455:   }

    457:   /* Save lG for values insertion in change of basis */
    458:   PetscCall(MatDuplicate(lG, MAT_COPY_VALUES, &lGinit));

    460:   /* Analyze the edge-nodes connections (duplicate lG) */
    461:   PetscCall(MatDuplicate(lG, MAT_COPY_VALUES, &lGe));
    462:   PetscCall(MatSetOption(lGe, MAT_KEEP_NONZERO_PATTERN, PETSC_FALSE));
    463:   PetscCall(PetscBTCreate(nv, &btv));
    464:   PetscCall(PetscBTCreate(ne, &bte));
    465:   PetscCall(PetscBTCreate(ne, &btb));
    466:   PetscCall(PetscBTCreate(ne, &btbd));
    467:   /* need to import the boundary specification to ensure the
    468:      proper detection of coarse edges' endpoints */
    469:   if (pcbddc->DirichletBoundariesLocal) {
    470:     IS is;

    472:     if (fl2g) {
    473:       PetscCall(ISGlobalToLocalMappingApplyIS(fl2g, IS_GTOLM_MASK, pcbddc->DirichletBoundariesLocal, &is));
    474:     } else {
    475:       is = pcbddc->DirichletBoundariesLocal;
    476:     }
    477:     PetscCall(ISGetLocalSize(is, &cum));
    478:     PetscCall(ISGetIndices(is, &idxs));
    479:     for (i = 0; i < cum; i++) {
    480:       if (idxs[i] >= 0 && idxs[i] < ne) {
    481:         PetscCall(PetscBTSet(btb, idxs[i]));
    482:         PetscCall(PetscBTSet(btbd, idxs[i]));
    483:       }
    484:     }
    485:     PetscCall(ISRestoreIndices(is, &idxs));
    486:     if (fl2g) PetscCall(ISDestroy(&is));
    487:   }
    488:   if (pcbddc->NeumannBoundariesLocal) {
    489:     IS is;

    491:     if (fl2g) {
    492:       PetscCall(ISGlobalToLocalMappingApplyIS(fl2g, IS_GTOLM_MASK, pcbddc->NeumannBoundariesLocal, &is));
    493:     } else {
    494:       is = pcbddc->NeumannBoundariesLocal;
    495:     }
    496:     PetscCall(ISGetLocalSize(is, &cum));
    497:     PetscCall(ISGetIndices(is, &idxs));
    498:     for (i = 0; i < cum; i++) {
    499:       if (idxs[i] >= 0 && idxs[i] < ne) PetscCall(PetscBTSet(btb, idxs[i]));
    500:     }
    501:     PetscCall(ISRestoreIndices(is, &idxs));
    502:     if (fl2g) PetscCall(ISDestroy(&is));
    503:   }

    505:   /* Count neighs per dof */
    506:   PetscCall(ISLocalToGlobalMappingGetNodeInfo(el2g, NULL, &ecount, NULL));
    507:   PetscCall(ISLocalToGlobalMappingGetNodeInfo(vl2g, NULL, &vcount, NULL));

    509:   /* need to remove coarse faces' dofs and coarse edges' dirichlet dofs
    510:      for proper detection of coarse edges' endpoints */
    511:   PetscCall(PetscBTCreate(ne, &btee));
    512:   for (i = 0; i < ne; i++) {
    513:     if ((ecount[i] > 2 && !PetscBTLookup(btbd, i)) || (ecount[i] == 2 && PetscBTLookup(btb, i))) PetscCall(PetscBTSet(btee, i));
    514:   }
    515:   PetscCall(PetscMalloc1(ne, &marks));
    516:   if (!conforming) {
    517:     PetscCall(MatTranspose(lGe, MAT_INITIAL_MATRIX, &lGt));
    518:     PetscCall(MatGetRowIJ(lGt, 0, PETSC_FALSE, PETSC_FALSE, &i, &iit, &jjt, &done));
    519:   }
    520:   PetscCall(MatGetRowIJ(lGe, 0, PETSC_FALSE, PETSC_FALSE, &i, &ii, &jj, &done));
    521:   PetscCall(MatSeqAIJGetArray(lGe, &vals));
    522:   cum = 0;
    523:   for (i = 0; i < ne; i++) {
    524:     /* eliminate rows corresponding to edge dofs belonging to coarse faces */
    525:     if (!PetscBTLookup(btee, i)) {
    526:       marks[cum++] = i;
    527:       continue;
    528:     }
    529:     /* set badly connected edge dofs as primal */
    530:     if (!conforming) {
    531:       if (ii[i + 1] - ii[i] != order + 1) { /* every row of G on the coarse edge should list order+1 nodal dofs */
    532:         marks[cum++] = i;
    533:         PetscCall(PetscBTSet(bte, i));
    534:         for (j = ii[i]; j < ii[i + 1]; j++) PetscCall(PetscBTSet(btv, jj[j]));
    535:       } else {
    536:         /* every edge dofs should be connected through a certain number of nodal dofs
    537:            to other edge dofs belonging to coarse edges
    538:            - at most 2 endpoints
    539:            - order-1 interior nodal dofs
    540:            - no undefined nodal dofs (nconn < order)
    541:         */
    542:         PetscInt ends = 0, ints = 0, undef = 0;
    543:         for (j = ii[i]; j < ii[i + 1]; j++) {
    544:           PetscInt v     = jj[j], k;
    545:           PetscInt nconn = iit[v + 1] - iit[v];
    546:           for (k = iit[v]; k < iit[v + 1]; k++)
    547:             if (!PetscBTLookup(btee, jjt[k])) nconn--;
    548:           if (nconn > order) ends++;
    549:           else if (nconn == order) ints++;
    550:           else undef++;
    551:         }
    552:         if (undef || ends > 2 || ints != order - 1) {
    553:           marks[cum++] = i;
    554:           PetscCall(PetscBTSet(bte, i));
    555:           for (j = ii[i]; j < ii[i + 1]; j++) PetscCall(PetscBTSet(btv, jj[j]));
    556:         }
    557:       }
    558:     }
    559:     /* We assume the order on the element edge is ii[i+1]-ii[i]-1 */
    560:     if (!order && ii[i + 1] != ii[i]) {
    561:       PetscScalar val = 1. / (ii[i + 1] - ii[i] - 1);
    562:       for (j = ii[i]; j < ii[i + 1]; j++) vals[j] = val;
    563:     }
    564:   }
    565:   PetscCall(PetscBTDestroy(&btee));
    566:   PetscCall(MatSeqAIJRestoreArray(lGe, &vals));
    567:   PetscCall(MatRestoreRowIJ(lGe, 0, PETSC_FALSE, PETSC_FALSE, &i, &ii, &jj, &done));
    568:   if (!conforming) {
    569:     PetscCall(MatRestoreRowIJ(lGt, 0, PETSC_FALSE, PETSC_FALSE, &i, &iit, &jjt, &done));
    570:     PetscCall(MatDestroy(&lGt));
    571:   }
    572:   PetscCall(MatZeroRows(lGe, cum, marks, 0., NULL, NULL));

    574:   /* identify splitpoints and corner candidates */
    575:   PetscCall(PetscMalloc2(nv, &sfvleaves, Lv, &sfvroots));
    576:   PetscCall(PetscBTCreate(nv, &btvcand));
    577:   if (elements_corners) {
    578:     PetscCall(ISGetLocalSize(elements_corners, &cum));
    579:     PetscCall(ISGetIndices(elements_corners, &idxs));
    580:     for (i = 0; i < cum; i++) PetscCall(PetscBTSet(btvcand, idxs[i]));
    581:     PetscCall(ISRestoreIndices(elements_corners, &idxs));
    582:   }

    584:   if (matis->allow_repeated) { /* assign a uniq global id to edge local subsets and communicate it with nodal space */
    585:     PetscSF   emlsf, vmlsf;
    586:     PetscInt *eleaves, *vleaves, *meleaves, *mvleaves;
    587:     PetscInt  cum_subs = 0, n_subs = pcbddc->n_local_subs, bs, emnr, emnl, vmnr, vmnl;

    589:     PetscCall(ISLocalToGlobalMappingGetBlockSize(el2g, &bs));
    590:     PetscCheck(bs == 1, comm, PETSC_ERR_SUP, "Not coded");
    591:     PetscCall(ISLocalToGlobalMappingGetBlockSize(vl2g, &bs));
    592:     PetscCheck(bs == 1, comm, PETSC_ERR_SUP, "Not coded");

    594:     PetscCall(ISLocalToGlobalMappingGetBlockMultiLeavesSF(el2g, &emlsf));
    595:     PetscCall(ISLocalToGlobalMappingGetBlockMultiLeavesSF(vl2g, &vmlsf));

    597:     PetscCall(PetscSFGetGraph(emlsf, &emnr, &emnl, NULL, NULL));
    598:     for (i = 0, j = 0; i < ne; i++) j += ecount[i];
    599:     PetscCheck(emnr == ne, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Invalid number of roots in edge multi-leaves SF %" PetscInt_FMT " != %" PetscInt_FMT, emnr, ne);
    600:     PetscCheck(emnl == j, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Invalid number of leaves in edge multi-leaves SF %" PetscInt_FMT " != %" PetscInt_FMT, emnl, j);

    602:     PetscCall(PetscSFGetGraph(vmlsf, &vmnr, &vmnl, NULL, NULL));
    603:     for (i = 0, j = 0; i < nv; i++) j += vcount[i];
    604:     PetscCheck(vmnr == nv, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Invalid number of roots in nodal multi-leaves SF %" PetscInt_FMT " != %" PetscInt_FMT, vmnr, nv);
    605:     PetscCheck(vmnl == j, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Invalid number of leaves in nodal multi-leaves SF %" PetscInt_FMT " != %" PetscInt_FMT, vmnl, j);

    607:     PetscCall(PetscMalloc1(ne, &eleaves));
    608:     PetscCall(PetscMalloc1(nv, &vleaves));
    609:     for (i = 0; i < ne; i++) eleaves[i] = PETSC_INT_MAX;
    610:     for (i = 0; i < nv; i++) vleaves[i] = PETSC_INT_MAX;
    611:     PetscCall(PetscMalloc1(emnl, &meleaves));
    612:     PetscCall(PetscMalloc1(vmnl, &mvleaves));

    614:     PetscCallMPI(MPI_Exscan(&n_subs, &cum_subs, 1, MPIU_INT, MPI_SUM, comm));
    615:     PetscCall(MatGetRowIJ(lGinit, 0, PETSC_FALSE, PETSC_FALSE, &i, &ii, &jj, &done));
    616:     for (i = 0; i < n_subs; i++) {
    617:       const PetscInt *idxs;
    618:       const PetscInt  subid = cum_subs + i;
    619:       PetscInt        ns;

    621:       PetscCall(ISGetLocalSize(pcbddc->local_subs[i], &ns));
    622:       PetscCall(ISGetIndices(pcbddc->local_subs[i], &idxs));
    623:       for (j = 0; j < ns; j++) {
    624:         const PetscInt e = idxs[j];

    626:         eleaves[e] = subid;
    627:         for (PetscInt k = ii[e]; k < ii[e + 1]; k++) vleaves[jj[k]] = subid;
    628:       }
    629:       PetscCall(ISRestoreIndices(pcbddc->local_subs[i], &idxs));
    630:     }
    631:     PetscCall(MatRestoreRowIJ(lGinit, 0, PETSC_FALSE, PETSC_FALSE, &i, &ii, &jj, &done));
    632:     PetscCall(PetscSFBcastBegin(emlsf, MPIU_INT, eleaves, meleaves, MPI_REPLACE));
    633:     PetscCall(PetscSFBcastEnd(emlsf, MPIU_INT, eleaves, meleaves, MPI_REPLACE));
    634:     PetscCall(PetscSFBcastBegin(vmlsf, MPIU_INT, vleaves, mvleaves, MPI_REPLACE));
    635:     PetscCall(PetscSFBcastEnd(vmlsf, MPIU_INT, vleaves, mvleaves, MPI_REPLACE));
    636:     PetscCall(PetscFree(eleaves));
    637:     PetscCall(PetscFree(vleaves));

    639:     PetscCall(PetscMalloc1(ne + 1, &eneighs));
    640:     eneighs[0] = meleaves;
    641:     for (i = 0; i < ne; i++) {
    642:       PetscCall(PetscSortInt(ecount[i], eneighs[i]));
    643:       eneighs[i + 1] = eneighs[i] + ecount[i];
    644:     }
    645:     PetscCall(PetscMalloc1(nv + 1, &vneighs));
    646:     vneighs[0] = mvleaves;
    647:     for (i = 0; i < nv; i++) {
    648:       PetscCall(PetscSortInt(vcount[i], vneighs[i]));
    649:       vneighs[i + 1] = vneighs[i] + vcount[i];
    650:     }
    651:   } else {
    652:     PetscCall(ISLocalToGlobalMappingGetNodeInfo(el2g, NULL, NULL, &eneighs));
    653:     PetscCall(ISLocalToGlobalMappingGetNodeInfo(vl2g, NULL, NULL, &vneighs));
    654:   }

    656:   PetscCall(MatTranspose(lGe, MAT_INITIAL_MATRIX, &lGt));
    657:   if (print) {
    658:     PetscCall(PetscObjectSetName((PetscObject)lGe, "edgerestr_lG"));
    659:     PetscCall(MatView(lGe, NULL));
    660:     PetscCall(PetscObjectSetName((PetscObject)lGt, "edgerestr_lGt"));
    661:     PetscCall(MatView(lGt, NULL));
    662:   }
    663:   PetscCall(MatGetRowIJ(lGt, 0, PETSC_FALSE, PETSC_FALSE, &i, &ii, &jj, &done));
    664:   PetscCall(MatSeqAIJGetArray(lGt, &vals));
    665:   for (i = 0; i < nv; i++) {
    666:     PetscInt  ord = order, test = ii[i + 1] - ii[i], vc = vcount[i];
    667:     PetscBool sneighs = PETSC_TRUE, bdir = PETSC_FALSE;
    668:     if (!order) { /* variable order */
    669:       PetscReal vorder = 0.;

    671:       for (j = ii[i]; j < ii[i + 1]; j++) vorder += PetscRealPart(vals[j]);
    672:       test = PetscFloorReal(vorder + 10. * PETSC_SQRT_MACHINE_EPSILON);
    673:       PetscCheck(vorder - test <= PETSC_SQRT_MACHINE_EPSILON, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Unexpected value for vorder: %g (%" PetscInt_FMT ")", (double)vorder, test);
    674:       ord = 1;
    675:     }
    676:     for (j = ii[i]; j < ii[i + 1] && sneighs; j++) {
    677:       const PetscInt e = jj[j];

    679:       if (PetscBTLookup(btbd, e)) {
    680:         bdir = PETSC_TRUE;
    681:         break;
    682:       }
    683:       if (vc != ecount[e]) {
    684:         sneighs = PETSC_FALSE;
    685:       } else {
    686:         const PetscInt *vn = vneighs[i], *en = eneighs[e];

    688:         for (PetscInt k = 0; k < vc; k++) {
    689:           if (vn[k] != en[k]) {
    690:             sneighs = PETSC_FALSE;
    691:             break;
    692:           }
    693:         }
    694:       }
    695:     }
    696:     if (elements_corners) test = 0;
    697:     if (!sneighs || test >= 3 * ord || bdir) { /* splitpoints */
    698:       if (print) PetscCall(PetscPrintf(PETSC_COMM_SELF, "SPLITPOINT %" PetscInt_FMT " (%s %s %s)\n", i, PetscBools[!sneighs], PetscBools[test >= 3 * ord], PetscBools[bdir]));
    699:       PetscCall(PetscBTSet(btv, i));
    700:     } else if (test == ord) {
    701:       if (order == 1 || (!order && ii[i + 1] - ii[i] == 1)) {
    702:         if (print) PetscCall(PetscPrintf(PETSC_COMM_SELF, "ENDPOINT %" PetscInt_FMT "\n", i));
    703:         PetscCall(PetscBTSet(btv, i));
    704:       } else if (!elements_corners) {
    705:         if (print) PetscCall(PetscPrintf(PETSC_COMM_SELF, "CORNER CANDIDATE %" PetscInt_FMT "\n", i));
    706:         PetscCall(PetscBTSet(btvcand, i));
    707:       }
    708:     }
    709:   }
    710:   PetscCall(PetscBTDestroy(&btbd));

    712:   /* a candidate is valid if it is connected to another candidate via a non-primal edge dof */
    713:   if (order != 1) {
    714:     if (print) PetscCall(PetscPrintf(PETSC_COMM_SELF, "INSPECTING CANDIDATES\n"));
    715:     PetscCall(MatGetRowIJ(lGe, 0, PETSC_FALSE, PETSC_FALSE, &i, &iit, &jjt, &done));
    716:     for (i = 0; i < nv; i++) {
    717:       if (PetscBTLookup(btvcand, i)) {
    718:         PetscBool found = PETSC_FALSE;
    719:         for (j = ii[i]; j < ii[i + 1] && !found; j++) {
    720:           PetscInt k, e = jj[j];
    721:           if (PetscBTLookup(bte, e)) continue;
    722:           for (k = iit[e]; k < iit[e + 1]; k++) {
    723:             PetscInt v = jjt[k];
    724:             if (v != i && PetscBTLookup(btvcand, v)) {
    725:               found = PETSC_TRUE;
    726:               break;
    727:             }
    728:           }
    729:         }
    730:         if (!found) {
    731:           if (print) PetscCall(PetscPrintf(PETSC_COMM_SELF, "  CANDIDATE %" PetscInt_FMT " CLEARED\n", i));
    732:           PetscCall(PetscBTClear(btvcand, i));
    733:         } else {
    734:           if (print) PetscCall(PetscPrintf(PETSC_COMM_SELF, "  CANDIDATE %" PetscInt_FMT " ACCEPTED\n", i));
    735:         }
    736:       }
    737:     }
    738:     PetscCall(MatRestoreRowIJ(lGe, 0, PETSC_FALSE, PETSC_FALSE, &i, &iit, &jjt, &done));
    739:   }
    740:   PetscCall(MatSeqAIJRestoreArray(lGt, &vals));
    741:   PetscCall(MatRestoreRowIJ(lGt, 0, PETSC_FALSE, PETSC_FALSE, &i, &ii, &jj, &done));
    742:   PetscCall(MatDestroy(&lGe));

    744:   /* Get the local G^T explicitly */
    745:   PetscCall(MatDestroy(&lGt));
    746:   PetscCall(MatTranspose(lG, MAT_INITIAL_MATRIX, &lGt));
    747:   PetscCall(MatSetOption(lGt, MAT_KEEP_NONZERO_PATTERN, PETSC_FALSE));

    749:   /* Mark shared nodal dofs */
    750:   PetscCall(PetscBTCreate(nv, &btvi));
    751:   for (i = 0; i < nv; i++) {
    752:     if (vcount[i] > 1) PetscCall(PetscBTSet(btvi, i));
    753:   }

    755:   if (matis->allow_repeated) {
    756:     PetscCall(PetscFree(eneighs[0]));
    757:     PetscCall(PetscFree(vneighs[0]));
    758:     PetscCall(PetscFree(eneighs));
    759:     PetscCall(PetscFree(vneighs));
    760:   }
    761:   PetscCall(ISLocalToGlobalMappingRestoreNodeInfo(el2g, NULL, &ecount, &eneighs));
    762:   PetscCall(ISLocalToGlobalMappingRestoreNodeInfo(vl2g, NULL, &vcount, &vneighs));

    764:   /* communicate corners and splitpoints */
    765:   PetscCall(PetscMalloc1(nv, &vmarks));
    766:   PetscCall(PetscArrayzero(sfvleaves, nv));
    767:   PetscCall(PetscArrayzero(sfvroots, Lv));
    768:   for (i = 0; i < nv; i++)
    769:     if (PetscUnlikely(PetscBTLookup(btv, i))) sfvleaves[i] = 1;

    771:   if (print) {
    772:     IS tbz;

    774:     cum = 0;
    775:     for (i = 0; i < nv; i++)
    776:       if (sfvleaves[i]) vmarks[cum++] = i;

    778:     PetscCall(ISCreateGeneral(PETSC_COMM_SELF, cum, vmarks, PETSC_COPY_VALUES, &tbz));
    779:     PetscCall(PetscObjectSetName((PetscObject)tbz, "corners_to_be_zeroed_local"));
    780:     PetscCall(ISView(tbz, NULL));
    781:     PetscCall(ISDestroy(&tbz));
    782:   }

    784:   PetscCall(PetscSFReduceBegin(sfv, MPIU_INT, sfvleaves, sfvroots, MPI_SUM));
    785:   PetscCall(PetscSFReduceEnd(sfv, MPIU_INT, sfvleaves, sfvroots, MPI_SUM));
    786:   PetscCall(PetscSFBcastBegin(sfv, MPIU_INT, sfvroots, sfvleaves, MPI_REPLACE));
    787:   PetscCall(PetscSFBcastEnd(sfv, MPIU_INT, sfvroots, sfvleaves, MPI_REPLACE));

    789:   /* Zero rows of lGt corresponding to identified corners
    790:      and interior nodal dofs */
    791:   cum = 0;
    792:   for (i = 0; i < nv; i++) {
    793:     if (sfvleaves[i]) {
    794:       vmarks[cum++] = i;
    795:       PetscCall(PetscBTSet(btv, i));
    796:     } else if (!PetscBTLookup(btvi, i)) vmarks[cum++] = i;
    797:   }
    798:   PetscCall(PetscBTDestroy(&btvi));
    799:   if (print) {
    800:     IS tbz;

    802:     PetscCall(ISCreateGeneral(PETSC_COMM_SELF, cum, vmarks, PETSC_COPY_VALUES, &tbz));
    803:     PetscCall(PetscObjectSetName((PetscObject)tbz, "corners_to_be_zeroed_with_interior"));
    804:     PetscCall(ISView(tbz, NULL));
    805:     PetscCall(ISDestroy(&tbz));
    806:   }
    807:   PetscCall(MatZeroRows(lGt, cum, vmarks, 0., NULL, NULL));
    808:   PetscCall(PetscFree(vmarks));
    809:   PetscCall(PetscSFDestroy(&sfv));
    810:   PetscCall(PetscFree2(sfvleaves, sfvroots));

    812:   /* Recompute G */
    813:   PetscCall(MatDestroy(&lG));
    814:   PetscCall(MatTranspose(lGt, MAT_INITIAL_MATRIX, &lG));
    815:   if (print) {
    816:     PetscCall(PetscObjectSetName((PetscObject)lG, "used_lG"));
    817:     PetscCall(MatView(lG, NULL));
    818:     PetscCall(PetscObjectSetName((PetscObject)lGt, "used_lGt"));
    819:     PetscCall(MatView(lGt, NULL));
    820:   }

    822:   /* Get primal dofs (if any) */
    823:   cum = 0;
    824:   for (i = 0; i < ne; i++) {
    825:     if (PetscUnlikely(PetscBTLookup(bte, i))) marks[cum++] = i;
    826:   }
    827:   if (fl2g) PetscCall(ISLocalToGlobalMappingApply(fl2g, cum, marks, marks));
    828:   PetscCall(ISCreateGeneral(comm, cum, marks, PETSC_COPY_VALUES, &primals));
    829:   if (print) {
    830:     PetscCall(PetscObjectSetName((PetscObject)primals, "prescribed_primal_dofs"));
    831:     PetscCall(ISView(primals, NULL));
    832:   }
    833:   PetscCall(PetscBTDestroy(&bte));
    834:   /* TODO: what if the user passed in some of them ?  */
    835:   PetscCall(PCBDDCSetPrimalVerticesLocalIS(pc, primals));
    836:   PetscCall(ISDestroy(&primals));

    838:   /* Compute edge connectivity */
    839:   PetscCall(PetscObjectSetOptionsPrefix((PetscObject)lG, "econn_"));

    841:   /* Symbolic conn = lG*lGt */
    842:   if (!elements_corners) { /* if present, we assume we are in the element-by-element case and the CSR graph is not needed */
    843:     PetscCall(MatProductCreate(lG, lGt, NULL, &conn));
    844:     PetscCall(MatProductSetType(conn, MATPRODUCT_AB));
    845:     PetscCall(MatProductSetAlgorithm(conn, "default"));
    846:     PetscCall(MatProductSetFill(conn, PETSC_DEFAULT));
    847:     PetscCall(PetscObjectSetOptionsPrefix((PetscObject)conn, "econn_"));
    848:     PetscCall(MatProductSetFromOptions(conn));
    849:     PetscCall(MatProductSymbolic(conn));
    850:     PetscCall(MatGetRowIJ(conn, 0, PETSC_FALSE, PETSC_FALSE, &i, &ii, &jj, &done));
    851:     if (fl2g) {
    852:       PetscBT   btf;
    853:       PetscInt *iia, *jja, *iiu, *jju;
    854:       PetscBool rest = PETSC_FALSE, free = PETSC_FALSE;

    856:       /* create CSR for all local dofs */
    857:       PetscCall(PetscMalloc1(n + 1, &iia));
    858:       if (pcbddc->mat_graph->nvtxs_csr) { /* the user has passed in a CSR graph */
    859:         PetscCheck(pcbddc->mat_graph->nvtxs_csr == n, PETSC_COMM_SELF, PETSC_ERR_USER, "Invalid size of CSR graph %" PetscInt_FMT ". Should be %" PetscInt_FMT, pcbddc->mat_graph->nvtxs_csr, n);
    860:         iiu = pcbddc->mat_graph->xadj;
    861:         jju = pcbddc->mat_graph->adjncy;
    862:       } else if (pcbddc->use_local_adj) {
    863:         rest = PETSC_TRUE;
    864:         PetscCall(MatGetRowIJ(matis->A, 0, PETSC_TRUE, PETSC_FALSE, &i, (const PetscInt **)&iiu, (const PetscInt **)&jju, &done));
    865:       } else {
    866:         free = PETSC_TRUE;
    867:         PetscCall(PetscMalloc2(n + 1, &iiu, n, &jju));
    868:         iiu[0] = 0;
    869:         for (i = 0; i < n; i++) {
    870:           iiu[i + 1] = i + 1;
    871:           jju[i]     = -1;
    872:         }
    873:       }

    875:       /* import sizes of CSR */
    876:       iia[0] = 0;
    877:       for (i = 0; i < n; i++) iia[i + 1] = iiu[i + 1] - iiu[i];

    879:       /* overwrite entries corresponding to the Nedelec field */
    880:       PetscCall(PetscBTCreate(n, &btf));
    881:       PetscCall(ISGetIndices(nedfieldlocal, &idxs));
    882:       for (i = 0; i < ne; i++) {
    883:         PetscCall(PetscBTSet(btf, idxs[i]));
    884:         iia[idxs[i] + 1] = ii[i + 1] - ii[i];
    885:       }

    887:       /* iia in CSR */
    888:       for (i = 0; i < n; i++) iia[i + 1] += iia[i];

    890:       /* jja in CSR */
    891:       PetscCall(PetscMalloc1(iia[n], &jja));
    892:       for (i = 0; i < n; i++)
    893:         if (!PetscBTLookup(btf, i))
    894:           for (j = 0; j < iiu[i + 1] - iiu[i]; j++) jja[iia[i] + j] = jju[iiu[i] + j];

    896:       /* map edge dofs connectivity */
    897:       if (jj) {
    898:         PetscCall(ISLocalToGlobalMappingApply(fl2g, ii[ne], jj, (PetscInt *)jj));
    899:         for (i = 0; i < ne; i++) {
    900:           PetscInt e = idxs[i];
    901:           for (j = 0; j < ii[i + 1] - ii[i]; j++) jja[iia[e] + j] = jj[ii[i] + j];
    902:         }
    903:       }
    904:       PetscCall(ISRestoreIndices(nedfieldlocal, &idxs));
    905:       PetscCall(PCBDDCSetLocalAdjacencyGraph(pc, n, iia, jja, PETSC_COPY_VALUES));
    906:       if (rest) PetscCall(MatRestoreRowIJ(matis->A, 0, PETSC_TRUE, PETSC_FALSE, &i, (const PetscInt **)&iiu, (const PetscInt **)&jju, &done));
    907:       if (free) PetscCall(PetscFree2(iiu, jju));
    908:       PetscCall(PetscBTDestroy(&btf));
    909:     } else {
    910:       PetscCall(PCBDDCSetLocalAdjacencyGraph(pc, n, ii, jj, PETSC_COPY_VALUES));
    911:     }
    912:     PetscCall(MatRestoreRowIJ(conn, 0, PETSC_FALSE, PETSC_FALSE, &i, &ii, &jj, &done));
    913:     PetscCall(MatDestroy(&conn));
    914:   }

    916:   /* Analyze interface for edge dofs */
    917:   PetscCall(PCBDDCAnalyzeInterface(pc));
    918:   pcbddc->mat_graph->twodim = PETSC_FALSE;

    920:   /* Get coarse edges in the edge space */
    921:   PetscCall(PCBDDCGraphGetCandidatesIS(pcbddc->mat_graph, NULL, NULL, &nee, &alleedges, &allprimals));

    923:   if (fl2g) {
    924:     PetscCall(ISGlobalToLocalMappingApplyIS(fl2g, IS_GTOLM_DROP, allprimals, &primals));
    925:     PetscCall(PetscMalloc1(nee, &eedges));
    926:     for (i = 0; i < nee; i++) PetscCall(ISGlobalToLocalMappingApplyIS(fl2g, IS_GTOLM_DROP, alleedges[i], &eedges[i]));
    927:   } else {
    928:     eedges  = alleedges;
    929:     primals = allprimals;
    930:   }

    932:   /* Mark fine edge dofs with their coarse edge id */
    933:   PetscCall(PetscArrayzero(marks, ne));
    934:   PetscCall(ISGetLocalSize(primals, &cum));
    935:   PetscCall(ISGetIndices(primals, &idxs));
    936:   for (i = 0; i < cum; i++) marks[idxs[i]] = nee + 1;
    937:   PetscCall(ISRestoreIndices(primals, &idxs));
    938:   if (print) {
    939:     PetscCall(PetscObjectSetName((PetscObject)primals, "obtained_primal_dofs"));
    940:     PetscCall(ISView(primals, NULL));
    941:   }

    943:   maxsize = 0;
    944:   for (i = 0; i < nee; i++) {
    945:     PetscInt size, mark = i + 1;

    947:     PetscCall(ISGetLocalSize(eedges[i], &size));
    948:     PetscCall(ISGetIndices(eedges[i], &idxs));
    949:     for (j = 0; j < size; j++) marks[idxs[j]] = mark;
    950:     PetscCall(ISRestoreIndices(eedges[i], &idxs));
    951:     maxsize = PetscMax(maxsize, size);
    952:   }

    954:   /* Find coarse edge endpoints */
    955:   PetscCall(MatGetRowIJ(lG, 0, PETSC_FALSE, PETSC_FALSE, &i, &ii, &jj, &done));
    956:   PetscCall(MatGetRowIJ(lGt, 0, PETSC_FALSE, PETSC_FALSE, &i, &iit, &jjt, &done));
    957:   for (i = 0; i < nee; i++) {
    958:     PetscInt mark = i + 1, size;

    960:     PetscCall(ISGetLocalSize(eedges[i], &size));
    961:     if (!size && nedfieldlocal) continue;
    962:     PetscCheck(size, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Unexpected zero sized edge %" PetscInt_FMT, i);
    963:     PetscCall(ISGetIndices(eedges[i], &idxs));
    964:     if (print) {
    965:       PetscCall(PetscPrintf(PETSC_COMM_SELF, "ENDPOINTS ANALYSIS EDGE %" PetscInt_FMT "\n", i));
    966:       PetscCall(ISView(eedges[i], NULL));
    967:     }
    968:     for (j = 0; j < size; j++) {
    969:       PetscInt k, ee = idxs[j];
    970:       if (print) PetscCall(PetscPrintf(PETSC_COMM_SELF, "  idx %" PetscInt_FMT "\n", ee));
    971:       for (k = ii[ee]; k < ii[ee + 1]; k++) {
    972:         if (print) PetscCall(PetscPrintf(PETSC_COMM_SELF, "    inspect %" PetscInt_FMT "\n", jj[k]));
    973:         if (PetscBTLookup(btv, jj[k])) {
    974:           if (print) PetscCall(PetscPrintf(PETSC_COMM_SELF, "      corner found (already set) %" PetscInt_FMT "\n", jj[k]));
    975:         } else if (PetscBTLookup(btvcand, jj[k])) { /* is it ok? */
    976:           PetscInt  k2;
    977:           PetscBool corner = PETSC_FALSE;
    978:           for (k2 = iit[jj[k]]; k2 < iit[jj[k] + 1]; k2++) {
    979:             if (print) PetscCall(PetscPrintf(PETSC_COMM_SELF, "        INSPECTING %" PetscInt_FMT ": mark %" PetscInt_FMT " (ref mark %" PetscInt_FMT "), boundary %d\n", jjt[k2], marks[jjt[k2]], mark, (int)!!PetscBTLookup(btb, jjt[k2])));
    980:             /* it's a corner if either is connected with an edge dof belonging to a different cc or
    981:                if the edge dof lie on the natural part of the boundary */
    982:             if ((marks[jjt[k2]] && marks[jjt[k2]] != mark) || (!marks[jjt[k2]] && PetscBTLookup(btb, jjt[k2]))) {
    983:               corner = PETSC_TRUE;
    984:               break;
    985:             }
    986:           }
    987:           if (corner) { /* found the nodal dof corresponding to the endpoint of the edge */
    988:             if (print) PetscCall(PetscPrintf(PETSC_COMM_SELF, "        corner found %" PetscInt_FMT "\n", jj[k]));
    989:             PetscCall(PetscBTSet(btv, jj[k]));
    990:           } else {
    991:             if (print) PetscCall(PetscPrintf(PETSC_COMM_SELF, "        no corners found\n"));
    992:           }
    993:         }
    994:       }
    995:     }
    996:     PetscCall(ISRestoreIndices(eedges[i], &idxs));
    997:   }
    998:   PetscCall(MatRestoreRowIJ(lGt, 0, PETSC_FALSE, PETSC_FALSE, &i, &iit, &jjt, &done));
    999:   PetscCall(MatRestoreRowIJ(lG, 0, PETSC_FALSE, PETSC_FALSE, &i, &ii, &jj, &done));
    1000:   PetscCall(PetscBTDestroy(&btb));

    1002:   /* Reset marked primal dofs */
    1003:   PetscCall(ISGetLocalSize(primals, &cum));
    1004:   PetscCall(ISGetIndices(primals, &idxs));
    1005:   for (i = 0; i < cum; i++) marks[idxs[i]] = 0;
    1006:   PetscCall(ISRestoreIndices(primals, &idxs));

    1008:   /* Now use the initial lG */
    1009:   PetscCall(MatDestroy(&lG));
    1010:   PetscCall(MatDestroy(&lGt));
    1011:   lG = lGinit;
    1012:   PetscCall(MatTranspose(lG, MAT_INITIAL_MATRIX, &lGt));

    1014:   /* Compute extended cols indices */
    1015:   PetscCall(PetscBTCreate(nv, &btvc));
    1016:   PetscCall(PetscBTCreate(nee, &bter));
    1017:   PetscCall(MatGetRowIJ(lG, 0, PETSC_FALSE, PETSC_FALSE, &i, &ii, &jj, &done));
    1018:   PetscCall(MatSeqAIJGetMaxRowNonzeros(lG, &i));
    1019:   i *= maxsize;
    1020:   PetscCall(PetscCalloc1(nee, &extcols));
    1021:   PetscCall(PetscMalloc2(i, &extrow, i, &gidxs));
    1022:   eerr = PETSC_FALSE;
    1023:   for (i = 0; i < nee; i++) {
    1024:     PetscInt size, found = 0;

    1026:     cum = 0;
    1027:     PetscCall(ISGetLocalSize(eedges[i], &size));
    1028:     if (!size && nedfieldlocal) continue;
    1029:     PetscCheck(size, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Unexpected zero sized edge %" PetscInt_FMT, i);
    1030:     PetscCall(ISGetIndices(eedges[i], &idxs));
    1031:     PetscCall(PetscBTMemzero(nv, btvc));
    1032:     for (j = 0; j < size; j++) {
    1033:       PetscInt k, ee = idxs[j];
    1034:       for (k = ii[ee]; k < ii[ee + 1]; k++) {
    1035:         PetscInt vv = jj[k];
    1036:         if (!PetscBTLookup(btv, vv)) extrow[cum++] = vv;
    1037:         else if (!PetscBTLookupSet(btvc, vv)) found++;
    1038:       }
    1039:     }
    1040:     PetscCall(ISRestoreIndices(eedges[i], &idxs));
    1041:     PetscCall(PetscSortRemoveDupsInt(&cum, extrow));
    1042:     PetscCall(ISLocalToGlobalMappingApply(vl2g, cum, extrow, gidxs));
    1043:     PetscCall(PetscSortIntWithArray(cum, gidxs, extrow));
    1044:     PetscCall(ISCreateGeneral(PETSC_COMM_SELF, cum, extrow, PETSC_COPY_VALUES, &extcols[i]));
    1045:     /* it may happen that endpoints are not defined at this point
    1046:        if it is the case, mark this edge for a second pass */
    1047:     if (cum != size - 1 || found != 2) {
    1048:       PetscCall(PetscBTSet(bter, i));
    1049:       if (print) {
    1050:         PetscCall(PetscObjectSetName((PetscObject)eedges[i], "error_edge"));
    1051:         PetscCall(ISView(eedges[i], NULL));
    1052:         PetscCall(PetscObjectSetName((PetscObject)extcols[i], "error_extcol"));
    1053:         PetscCall(ISView(extcols[i], NULL));
    1054:       }
    1055:       eerr = PETSC_TRUE;
    1056:     }
    1057:   }
    1058:   /* PetscCheck(!eerr,PETSC_COMM_SELF,PETSC_ERR_PLIB,"Unexpected SIZE OF EDGE > EXTCOL FIRST PASS"); */
    1059:   PetscCallMPI(MPIU_Allreduce(&eerr, &done, 1, MPI_C_BOOL, MPI_LOR, comm));
    1060:   if (done) {
    1061:     PetscInt *newprimals;

    1063:     PetscCall(PetscMalloc1(ne, &newprimals));
    1064:     PetscCall(ISGetLocalSize(primals, &cum));
    1065:     PetscCall(ISGetIndices(primals, &idxs));
    1066:     PetscCall(PetscArraycpy(newprimals, idxs, cum));
    1067:     PetscCall(ISRestoreIndices(primals, &idxs));
    1068:     PetscCall(MatGetRowIJ(lGt, 0, PETSC_FALSE, PETSC_FALSE, &i, &iit, &jjt, &done));
    1069:     if (print) PetscCall(PetscPrintf(PETSC_COMM_SELF, "DOING SECOND PASS (eerr %s)\n", PetscBools[eerr]));
    1070:     for (i = 0; i < nee; i++) {
    1071:       PetscBool has_candidates = PETSC_FALSE;
    1072:       if (PetscBTLookup(bter, i)) {
    1073:         PetscInt size, mark = i + 1;

    1075:         PetscCall(ISGetLocalSize(eedges[i], &size));
    1076:         PetscCall(ISGetIndices(eedges[i], &idxs));
    1077:         /* for (j=0;j<size;j++) newprimals[cum++] = idxs[j]; */
    1078:         for (j = 0; j < size; j++) {
    1079:           PetscInt k, ee = idxs[j];
    1080:           if (print) PetscCall(PetscPrintf(PETSC_COMM_SELF, "Inspecting edge dof %" PetscInt_FMT " [%" PetscInt_FMT " %" PetscInt_FMT ")\n", ee, ii[ee], ii[ee + 1]));
    1081:           for (k = ii[ee]; k < ii[ee + 1]; k++) {
    1082:             /* set all candidates located on the edge as corners */
    1083:             if (PetscBTLookup(btvcand, jj[k])) {
    1084:               PetscInt k2, vv = jj[k];
    1085:               has_candidates = PETSC_TRUE;
    1086:               if (print) PetscCall(PetscPrintf(PETSC_COMM_SELF, "  Candidate set to vertex %" PetscInt_FMT "\n", vv));
    1087:               PetscCall(PetscBTSet(btv, vv));
    1088:               /* set all edge dofs connected to candidate as primals */
    1089:               for (k2 = iit[vv]; k2 < iit[vv + 1]; k2++) {
    1090:                 if (marks[jjt[k2]] == mark) {
    1091:                   PetscInt k3, ee2 = jjt[k2];
    1092:                   if (print) PetscCall(PetscPrintf(PETSC_COMM_SELF, "    Connected edge dof set to primal %" PetscInt_FMT "\n", ee2));
    1093:                   newprimals[cum++] = ee2;
    1094:                   /* finally set the new corners */
    1095:                   for (k3 = ii[ee2]; k3 < ii[ee2 + 1]; k3++) {
    1096:                     if (print) PetscCall(PetscPrintf(PETSC_COMM_SELF, "      Connected nodal dof set to vertex %" PetscInt_FMT "\n", jj[k3]));
    1097:                     PetscCall(PetscBTSet(btv, jj[k3]));
    1098:                   }
    1099:                 }
    1100:               }
    1101:             } else {
    1102:               if (print) PetscCall(PetscPrintf(PETSC_COMM_SELF, "  Not a candidate vertex %" PetscInt_FMT "\n", jj[k]));
    1103:             }
    1104:           }
    1105:         }
    1106:         if (!has_candidates) { /* circular edge */
    1107:           PetscInt k, ee = idxs[0], *tmarks;

    1109:           PetscCall(PetscCalloc1(ne, &tmarks));
    1110:           if (print) PetscCall(PetscPrintf(PETSC_COMM_SELF, "  Circular edge %" PetscInt_FMT "\n", i));
    1111:           for (k = ii[ee]; k < ii[ee + 1]; k++) {
    1112:             PetscInt k2;
    1113:             if (print) PetscCall(PetscPrintf(PETSC_COMM_SELF, "    Set to corner %" PetscInt_FMT "\n", jj[k]));
    1114:             PetscCall(PetscBTSet(btv, jj[k]));
    1115:             for (k2 = iit[jj[k]]; k2 < iit[jj[k] + 1]; k2++) tmarks[jjt[k2]]++;
    1116:           }
    1117:           for (j = 0; j < size; j++) {
    1118:             if (tmarks[idxs[j]] > 1) {
    1119:               if (print) PetscCall(PetscPrintf(PETSC_COMM_SELF, "  Edge dof set to primal %" PetscInt_FMT "\n", idxs[j]));
    1120:               newprimals[cum++] = idxs[j];
    1121:             }
    1122:           }
    1123:           PetscCall(PetscFree(tmarks));
    1124:         }
    1125:         PetscCall(ISRestoreIndices(eedges[i], &idxs));
    1126:       }
    1127:       PetscCall(ISDestroy(&extcols[i]));
    1128:     }
    1129:     PetscCall(PetscFree(extcols));
    1130:     PetscCall(MatRestoreRowIJ(lGt, 0, PETSC_FALSE, PETSC_FALSE, &i, &iit, &jjt, &done));
    1131:     PetscCall(PetscSortRemoveDupsInt(&cum, newprimals));
    1132:     if (fl2g) {
    1133:       PetscCall(ISLocalToGlobalMappingApply(fl2g, cum, newprimals, newprimals));
    1134:       PetscCall(ISDestroy(&primals));
    1135:       for (i = 0; i < nee; i++) PetscCall(ISDestroy(&eedges[i]));
    1136:       PetscCall(PetscFree(eedges));
    1137:     }
    1138:     PetscCall(PCBDDCGraphRestoreCandidatesIS(pcbddc->mat_graph, NULL, NULL, &nee, &alleedges, &allprimals));
    1139:     PetscCall(ISCreateGeneral(comm, cum, newprimals, PETSC_COPY_VALUES, &primals));
    1140:     PetscCall(PetscFree(newprimals));
    1141:     PetscCall(PCBDDCSetPrimalVerticesLocalIS(pc, primals));
    1142:     PetscCall(ISDestroy(&primals));
    1143:     PetscCall(PCBDDCAnalyzeInterface(pc));
    1144:     pcbddc->mat_graph->twodim = PETSC_FALSE;
    1145:     PetscCall(PCBDDCGraphGetCandidatesIS(pcbddc->mat_graph, NULL, NULL, &nee, &alleedges, &allprimals));
    1146:     if (fl2g) {
    1147:       PetscCall(ISGlobalToLocalMappingApplyIS(fl2g, IS_GTOLM_DROP, allprimals, &primals));
    1148:       PetscCall(PetscMalloc1(nee, &eedges));
    1149:       for (i = 0; i < nee; i++) PetscCall(ISGlobalToLocalMappingApplyIS(fl2g, IS_GTOLM_DROP, alleedges[i], &eedges[i]));
    1150:     } else {
    1151:       eedges  = alleedges;
    1152:       primals = allprimals;
    1153:     }
    1154:     PetscCall(PetscCalloc1(nee, &extcols));

    1156:     /* Mark again */
    1157:     PetscCall(PetscArrayzero(marks, ne));
    1158:     for (i = 0; i < nee; i++) {
    1159:       PetscInt size, mark = i + 1;

    1161:       PetscCall(ISGetLocalSize(eedges[i], &size));
    1162:       PetscCall(ISGetIndices(eedges[i], &idxs));
    1163:       for (j = 0; j < size; j++) marks[idxs[j]] = mark;
    1164:       PetscCall(ISRestoreIndices(eedges[i], &idxs));
    1165:     }
    1166:     if (print) {
    1167:       PetscCall(PetscObjectSetName((PetscObject)primals, "obtained_primal_dofs_secondpass"));
    1168:       PetscCall(ISView(primals, NULL));
    1169:     }

    1171:     /* Recompute extended cols */
    1172:     eerr = PETSC_FALSE;
    1173:     for (i = 0; i < nee; i++) {
    1174:       PetscInt size;

    1176:       cum = 0;
    1177:       PetscCall(ISGetLocalSize(eedges[i], &size));
    1178:       if (!size && nedfieldlocal) continue;
    1179:       PetscCheck(size, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Unexpected zero sized edge %" PetscInt_FMT, i);
    1180:       PetscCall(ISGetIndices(eedges[i], &idxs));
    1181:       for (j = 0; j < size; j++) {
    1182:         PetscInt k, ee = idxs[j];
    1183:         for (k = ii[ee]; k < ii[ee + 1]; k++)
    1184:           if (!PetscBTLookup(btv, jj[k])) extrow[cum++] = jj[k];
    1185:       }
    1186:       PetscCall(ISRestoreIndices(eedges[i], &idxs));
    1187:       PetscCall(PetscSortRemoveDupsInt(&cum, extrow));
    1188:       PetscCall(ISLocalToGlobalMappingApply(vl2g, cum, extrow, gidxs));
    1189:       PetscCall(PetscSortIntWithArray(cum, gidxs, extrow));
    1190:       PetscCall(ISCreateGeneral(PETSC_COMM_SELF, cum, extrow, PETSC_COPY_VALUES, &extcols[i]));
    1191:       if (cum != size - 1) {
    1192:         if (print) {
    1193:           PetscCall(PetscObjectSetName((PetscObject)eedges[i], "error_edge_secondpass"));
    1194:           PetscCall(ISView(eedges[i], NULL));
    1195:           PetscCall(PetscObjectSetName((PetscObject)extcols[i], "error_extcol_secondpass"));
    1196:           PetscCall(ISView(extcols[i], NULL));
    1197:         }
    1198:         eerr = PETSC_TRUE;
    1199:       }
    1200:     }
    1201:   }
    1202:   PetscCall(MatRestoreRowIJ(lG, 0, PETSC_FALSE, PETSC_FALSE, &i, &ii, &jj, &done));
    1203:   PetscCall(PetscFree2(extrow, gidxs));
    1204:   PetscCall(PetscBTDestroy(&bter));
    1205:   if (print) PetscCall(PCBDDCGraphASCIIView(pcbddc->mat_graph, 5, PETSC_VIEWER_STDOUT_SELF));
    1206:   /* an error should not occur at this point */
    1207:   PetscCheck(!eerr, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Unexpected SIZE OF EDGE > EXTCOL SECOND PASS");

    1209:   /* Check the number of endpoints */
    1210:   PetscCall(MatGetRowIJ(lG, 0, PETSC_FALSE, PETSC_FALSE, &i, &ii, &jj, &done));
    1211:   PetscCall(PetscMalloc1(2 * nee, &corners));
    1212:   PetscCall(PetscMalloc1(nee, &cedges));
    1213:   for (i = 0; i < nee; i++) {
    1214:     PetscInt size, found = 0, gc[2];

    1216:     /* init with defaults */
    1217:     cedges[i] = corners[i * 2] = corners[i * 2 + 1] = -1;
    1218:     PetscCall(ISGetLocalSize(eedges[i], &size));
    1219:     if (!size && nedfieldlocal) continue;
    1220:     PetscCheck(size, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Unexpected zero sized edge %" PetscInt_FMT, i);
    1221:     PetscCall(ISGetIndices(eedges[i], &idxs));
    1222:     PetscCall(PetscBTMemzero(nv, btvc));
    1223:     for (j = 0; j < size; j++) {
    1224:       PetscInt k, ee = idxs[j];
    1225:       for (k = ii[ee]; k < ii[ee + 1]; k++) {
    1226:         PetscInt vv = jj[k];
    1227:         if (PetscBTLookup(btv, vv) && !PetscBTLookupSet(btvc, vv)) {
    1228:           PetscCheck(found != 2, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Found more than two corners for edge %" PetscInt_FMT, i);
    1229:           corners[i * 2 + found++] = vv;
    1230:         }
    1231:       }
    1232:     }
    1233:     if (found != 2) {
    1234:       PetscInt e;
    1235:       if (fl2g) PetscCall(ISLocalToGlobalMappingApply(fl2g, 1, idxs, &e));
    1236:       else e = idxs[0];
    1237:       SETERRQ(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Found %" PetscInt_FMT " corners for edge %" PetscInt_FMT " (astart %" PetscInt_FMT ", estart %" PetscInt_FMT ")", found, i, e, idxs[0]);
    1238:     }

    1240:     /* get primal dof index on this coarse edge */
    1241:     PetscCall(ISLocalToGlobalMappingApply(vl2g, 2, corners + 2 * i, gc));
    1242:     if (gc[0] > gc[1]) {
    1243:       PetscInt swap      = corners[2 * i];
    1244:       corners[2 * i]     = corners[2 * i + 1];
    1245:       corners[2 * i + 1] = swap;
    1246:     }
    1247:     cedges[i] = idxs[size - 1];
    1248:     PetscCall(ISRestoreIndices(eedges[i], &idxs));
    1249:     if (print) PetscCall(PetscPrintf(PETSC_COMM_SELF, "EDGE %" PetscInt_FMT ": ce %" PetscInt_FMT ", corners (%" PetscInt_FMT ",%" PetscInt_FMT ")\n", i, cedges[i], corners[2 * i], corners[2 * i + 1]));
    1250:   }
    1251:   PetscCall(MatRestoreRowIJ(lG, 0, PETSC_FALSE, PETSC_FALSE, &i, &ii, &jj, &done));
    1252:   PetscCall(PetscBTDestroy(&btvc));

    1254:   if (PetscDefined(USE_DEBUG)) {
    1255:     /* Inspects columns of lG (rows of lGt) and make sure the change of basis will
    1256:      not interfere with neighbouring coarse edges */
    1257:     PetscCall(PetscMalloc1(nee + 1, &emarks));
    1258:     PetscCall(MatGetRowIJ(lGt, 0, PETSC_FALSE, PETSC_FALSE, &i, &ii, &jj, &done));
    1259:     for (i = 0; i < nv; i++) {
    1260:       PetscInt emax = 0, eemax = 0;

    1262:       if (ii[i + 1] == ii[i] || PetscBTLookup(btv, i)) continue;
    1263:       PetscCall(PetscArrayzero(emarks, nee + 1));
    1264:       for (j = ii[i]; j < ii[i + 1]; j++) emarks[marks[jj[j]]]++;
    1265:       for (j = 1; j < nee + 1; j++) {
    1266:         if (emax < emarks[j]) {
    1267:           emax  = emarks[j];
    1268:           eemax = j;
    1269:         }
    1270:       }
    1271:       /* not relevant for edges */
    1272:       if (!eemax) continue;

    1274:       for (j = ii[i]; j < ii[i + 1]; j++) {
    1275:         PetscCheck(!marks[jj[j]] || marks[jj[j]] == eemax, PETSC_COMM_SELF, PETSC_ERR_SUP, "Found 2 coarse edges (id %" PetscInt_FMT " and %" PetscInt_FMT ") connected through the %" PetscInt_FMT " nodal dof at edge dof %" PetscInt_FMT, marks[jj[j]] - 1, eemax, i, jj[j]);
    1276:       }
    1277:     }
    1278:     PetscCall(PetscFree(emarks));
    1279:     PetscCall(MatRestoreRowIJ(lGt, 0, PETSC_FALSE, PETSC_FALSE, &i, &ii, &jj, &done));
    1280:   }

    1282:   /* Compute extended rows indices for edge blocks of the change of basis */
    1283:   PetscCall(MatGetRowIJ(lGt, 0, PETSC_FALSE, PETSC_FALSE, &i, &ii, &jj, &done));
    1284:   PetscCall(MatSeqAIJGetMaxRowNonzeros(lGt, &extmem));
    1285:   extmem *= maxsize;
    1286:   PetscCall(PetscMalloc1(extmem * nee, &extrow));
    1287:   PetscCall(PetscMalloc1(nee, &extrows));
    1288:   PetscCall(PetscCalloc1(nee, &extrowcum));
    1289:   for (i = 0; i < nv; i++) {
    1290:     PetscInt mark = 0, size, start;

    1292:     if (ii[i + 1] == ii[i] || PetscBTLookup(btv, i)) continue;
    1293:     for (j = ii[i]; j < ii[i + 1]; j++)
    1294:       if (marks[jj[j]] && !mark) mark = marks[jj[j]];

    1296:     /* not relevant */
    1297:     if (!mark) continue;

    1299:     /* import extended row */
    1300:     mark--;
    1301:     start = mark * extmem + extrowcum[mark];
    1302:     size  = ii[i + 1] - ii[i];
    1303:     PetscCheck(extrowcum[mark] + size <= extmem, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Not enough memory allocated %" PetscInt_FMT " > %" PetscInt_FMT, extrowcum[mark] + size, extmem);
    1304:     PetscCall(PetscArraycpy(extrow + start, jj + ii[i], size));
    1305:     extrowcum[mark] += size;
    1306:   }
    1307:   PetscCall(MatRestoreRowIJ(lGt, 0, PETSC_FALSE, PETSC_FALSE, &i, &ii, &jj, &done));
    1308:   PetscCall(MatDestroy(&lGt));
    1309:   PetscCall(PetscFree(marks));

    1311:   /* Compress extrows */
    1312:   cum = 0;
    1313:   for (i = 0; i < nee; i++) {
    1314:     PetscInt size = extrowcum[i], *start = extrow + i * extmem;
    1315:     PetscCall(PetscSortRemoveDupsInt(&size, start));
    1316:     PetscCall(ISCreateGeneral(PETSC_COMM_SELF, size, start, PETSC_USE_POINTER, &extrows[i]));
    1317:     cum = PetscMax(cum, size);
    1318:   }
    1319:   PetscCall(PetscFree(extrowcum));
    1320:   PetscCall(PetscBTDestroy(&btv));
    1321:   PetscCall(PetscBTDestroy(&btvcand));

    1323:   /* Workspace for lapack inner calls and VecSetValues */
    1324:   PetscCall(PetscMalloc2((5 + cum + maxsize) * maxsize, &work, maxsize, &rwork));

    1326:   /* Create change of basis matrix (no preallocation) */
    1327:   PetscCall(MatCreate(comm, &T));
    1328:   PetscCall(MatSetLayouts(T, pc->mat->rmap, pc->mat->cmap));
    1329:   PetscCall(MatSetType(T, MATAIJ));
    1330:   PetscCall(MatSetLocalToGlobalMapping(T, al2g, al2g));
    1331:   PetscCall(MatSetOption(T, MAT_ROW_ORIENTED, PETSC_FALSE));
    1332:   PetscCall(MatSetOption(T, MAT_IGNORE_ZERO_ENTRIES, PETSC_TRUE));
    1333:   //PetscCall(MatSeqAIJSetPreallocation(T, maxsize, NULL));
    1334:   //PetscCall(MatMPIAIJSetPreallocation(T, maxsize, NULL, maxsize, NULL));
    1335:   //PetscCall(MatSetOption(T, MAT_NEW_NONZERO_ALLOCATION_ERR, PETSC_FALSE));

    1337:   /* Defaults to identity */
    1338:   {
    1339:     Vec                w;
    1340:     const PetscScalar *wa;

    1342:     PetscCall(MatCreateVecs(T, &w, NULL));
    1343:     PetscCall(VecSetLocalToGlobalMapping(w, al2g));
    1344:     PetscCall(VecSet(w, 1.0));
    1345:     for (i = 0; i < nee; i++) {
    1346:       const PetscInt *idxs;
    1347:       PetscInt        nl;

    1349:       PetscCall(ISGetLocalSize(eedges[i], &nl));
    1350:       PetscCall(ISGetIndices(eedges[i], &idxs));
    1351:       PetscCall(VecSetValuesLocal(w, nl, idxs, NULL, INSERT_VALUES));
    1352:       PetscCall(ISRestoreIndices(eedges[i], &idxs));
    1353:     }
    1354:     PetscCall(VecAssemblyBegin(w));
    1355:     PetscCall(VecAssemblyEnd(w));
    1356:     PetscCall(VecGetArrayRead(w, &wa));
    1357:     for (i = T->rmap->rstart; i < T->rmap->rend; i++)
    1358:       if (PetscAbsScalar(wa[i - T->rmap->rstart])) PetscCall(MatSetValue(T, i, i, 1.0, INSERT_VALUES));
    1359:     PetscCall(VecRestoreArrayRead(w, &wa));
    1360:     PetscCall(VecDestroy(&w));
    1361:   }

    1363:   /* Create discrete gradient for the coarser level if needed */
    1364:   PetscCall(MatDestroy(&pcbddc->nedcG));
    1365:   PetscCall(ISDestroy(&pcbddc->nedclocal));
    1366:   if (pcbddc->current_level < pcbddc->max_levels) {
    1367:     ISLocalToGlobalMapping cel2g, cvl2g;
    1368:     IS                     wis, gwis;
    1369:     PetscInt               cnv, cne;

    1371:     PetscCall(ISCreateGeneral(comm, nee, cedges, PETSC_COPY_VALUES, &wis));
    1372:     if (fl2g) PetscCall(ISLocalToGlobalMappingApplyIS(fl2g, wis, &pcbddc->nedclocal));
    1373:     else {
    1374:       PetscCall(PetscObjectReference((PetscObject)wis));
    1375:       pcbddc->nedclocal = wis;
    1376:     }
    1377:     PetscCall(ISLocalToGlobalMappingApplyIS(el2g, wis, &gwis));
    1378:     PetscCall(ISDestroy(&wis));
    1379:     PetscCall(ISRenumber(gwis, NULL, &cne, &wis));
    1380:     PetscCall(ISLocalToGlobalMappingCreateIS(wis, &cel2g));
    1381:     PetscCall(ISDestroy(&wis));
    1382:     PetscCall(ISDestroy(&gwis));

    1384:     PetscCall(ISCreateGeneral(comm, 2 * nee, corners, PETSC_USE_POINTER, &wis));
    1385:     PetscCall(ISLocalToGlobalMappingApplyIS(vl2g, wis, &gwis));
    1386:     PetscCall(ISDestroy(&wis));
    1387:     PetscCall(ISRenumber(gwis, NULL, &cnv, &wis));
    1388:     PetscCall(ISLocalToGlobalMappingCreateIS(wis, &cvl2g));
    1389:     PetscCall(ISDestroy(&wis));
    1390:     PetscCall(ISDestroy(&gwis));

    1392:     PetscCall(MatCreate(comm, &pcbddc->nedcG));
    1393:     PetscCall(MatSetSizes(pcbddc->nedcG, PETSC_DECIDE, PETSC_DECIDE, cne, cnv));
    1394:     PetscCall(MatSetType(pcbddc->nedcG, MATAIJ));
    1395:     PetscCall(MatSeqAIJSetPreallocation(pcbddc->nedcG, 2, NULL));
    1396:     PetscCall(MatMPIAIJSetPreallocation(pcbddc->nedcG, 2, NULL, 2, NULL));
    1397:     PetscCall(MatSetLocalToGlobalMapping(pcbddc->nedcG, cel2g, cvl2g));
    1398:     PetscCall(ISLocalToGlobalMappingDestroy(&cel2g));
    1399:     PetscCall(ISLocalToGlobalMappingDestroy(&cvl2g));
    1400:   }

    1402:   MatNullSpace nnsp;
    1403:   PetscBool    nnsp_has_const = PETSC_FALSE;
    1404:   const Vec   *nnsp_vecs      = NULL;
    1405:   PetscInt     nnsp_nvecs     = 0;
    1406:   VecScatter   nnsp_vscat     = NULL;
    1407:   PetscCall(MatGetNullSpace(pcbddc->discretegradient, &nnsp));
    1408:   if (nnsp) PetscCall(MatNullSpaceGetVecs(nnsp, &nnsp_has_const, &nnsp_nvecs, &nnsp_vecs));
    1409:   if (nnsp_has_const || nnsp_nvecs) { /* create scatter to import edge constraints */
    1410:     IS                 allextcols, gallextcols, galleedges, is_E_to_zero;
    1411:     Vec                E, V;
    1412:     PetscInt          *eedgesidxs;
    1413:     const PetscScalar *evals;

    1415:     PetscCall(MatCreateVecs(pc->pmat, &E, NULL));
    1416:     PetscCall(MatCreateVecs(pcbddc->discretegradient, &V, NULL));
    1417:     PetscCall(ISConcatenate(PETSC_COMM_SELF, nee, extcols, &allextcols));
    1418:     cum = 0;
    1419:     for (i = 0; i < nee; i++) {
    1420:       PetscInt j;

    1422:       PetscCall(ISGetLocalSize(eedges[i], &j));
    1423:       PetscCheck(j, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Zero sized edge %" PetscInt_FMT, i);
    1424:       cum += j - 1;
    1425:     }
    1426:     PetscCall(PetscMalloc1(PetscMax(cum, pc->pmat->rmap->n), &eedgesidxs));
    1427:     cum = 0;
    1428:     for (i = 0; i < nee; i++) {
    1429:       const PetscInt *idxs;
    1430:       PetscInt        j;

    1432:       PetscCall(ISGetLocalSize(eedges[i], &j));
    1433:       PetscCall(ISGetIndices(eedges[i], &idxs));
    1434:       PetscCall(PetscArraycpy(eedgesidxs + cum, idxs, j - 1)); /* last on the edge is primal */
    1435:       PetscCall(ISRestoreIndices(eedges[i], &idxs));
    1436:       cum += j - 1;
    1437:     }
    1438:     PetscCall(ISLocalToGlobalMappingApply(al2g, cum, eedgesidxs, eedgesidxs));
    1439:     PetscCall(ISCreateGeneral(PETSC_COMM_SELF, cum, eedgesidxs, PETSC_USE_POINTER, &galleedges));
    1440:     PetscCall(ISLocalToGlobalMappingApplyIS(vl2g, allextcols, &gallextcols));
    1441:     PetscCall(VecScatterCreate(V, gallextcols, E, galleedges, &nnsp_vscat));
    1442:     PetscCall(ISDestroy(&allextcols));
    1443:     PetscCall(ISDestroy(&gallextcols));
    1444:     PetscCall(ISDestroy(&galleedges));

    1446:     /* identify dofs we must zero if importing user-defined near nullspace from pmat */
    1447:     PetscCall(VecSet(E, 1.0));
    1448:     PetscCall(VecSetValues(E, cum, eedgesidxs, NULL, INSERT_VALUES));
    1449:     PetscCall(VecAssemblyBegin(E));
    1450:     PetscCall(VecAssemblyEnd(E));
    1451:     PetscCall(VecGetArrayRead(E, &evals));
    1452:     for (i = 0, cum = 0; i < pc->pmat->rmap->n; i++)
    1453:       if (evals[i] == 0.0) eedgesidxs[cum++] = i + pc->pmat->rmap->rstart;
    1454:     PetscCall(VecRestoreArrayRead(E, &evals));
    1455:     PetscCall(ISCreateGeneral(PETSC_COMM_SELF, cum, eedgesidxs, PETSC_COPY_VALUES, &is_E_to_zero));
    1456:     PetscCall(PetscFree(eedgesidxs));

    1458:     PetscCall(PetscObjectCompose((PetscObject)nnsp_vscat, "__V_Vec", (PetscObject)V));
    1459:     PetscCall(PetscObjectCompose((PetscObject)nnsp_vscat, "__E_Vec", (PetscObject)E));
    1460:     PetscCall(PetscObjectCompose((PetscObject)nnsp_vscat, "__E_zero", (PetscObject)is_E_to_zero));
    1461:     PetscCall(ISDestroy(&is_E_to_zero));
    1462:     PetscCall(VecDestroy(&V));
    1463:     PetscCall(VecDestroy(&E));
    1464:   }
    1465: #if defined(PRINT_GDET)
    1466:   inc = 0;
    1467:   lev = pcbddc->current_level;
    1468: #endif

    1470:   /* Insert values in the change of basis matrix */
    1471:   for (i = 0; i < nee; i++) {
    1472:     Mat         Gins = NULL, GKins = NULL;
    1473:     IS          cornersis = NULL;
    1474:     PetscScalar cvals[2];

    1476:     if (pcbddc->nedcG) PetscCall(ISCreateGeneral(PETSC_COMM_SELF, 2, corners + 2 * i, PETSC_USE_POINTER, &cornersis));
    1477:     PetscCall(PCBDDCComputeNedelecChangeEdge(lG, eedges[i], extrows[i], extcols[i], cornersis, &Gins, &GKins, cvals, work, rwork));
    1478:     if (Gins && GKins) {
    1479:       const PetscScalar *data;
    1480:       const PetscInt    *rows, *cols;
    1481:       PetscInt           nrh, nch, nrc, ncc;

    1483:       PetscCall(ISGetIndices(eedges[i], &cols));
    1484:       /* H1 */
    1485:       PetscCall(ISGetIndices(extrows[i], &rows));
    1486:       PetscCall(MatGetSize(Gins, &nrh, &nch));
    1487:       PetscCall(MatDenseGetArrayRead(Gins, &data));
    1488:       PetscCall(MatSetValuesLocal(T, nrh, rows, nch, cols, data, INSERT_VALUES));
    1489:       PetscCall(MatDenseRestoreArrayRead(Gins, &data));
    1490:       PetscCall(ISRestoreIndices(extrows[i], &rows));
    1491:       /* complement */
    1492:       PetscCall(MatGetSize(GKins, &nrc, &ncc));
    1493:       PetscCheck(ncc, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Constant function has not been generated for coarse edge %" PetscInt_FMT, i);
    1494:       PetscCheck(ncc + nch == nrc, PETSC_COMM_SELF, PETSC_ERR_PLIB, "The sum of the number of columns of GKins %" PetscInt_FMT " and Gins %" PetscInt_FMT " does not match %" PetscInt_FMT " for coarse edge %" PetscInt_FMT, ncc, nch, nrc, i);
    1495:       PetscCheck(ncc == 1 || !pcbddc->nedcG, PETSC_COMM_SELF, PETSC_ERR_SUP, "Cannot generate the coarse discrete gradient for coarse edge %" PetscInt_FMT " with ncc %" PetscInt_FMT, i, ncc);
    1496:       PetscCall(MatDenseGetArrayRead(GKins, &data));
    1497:       PetscCall(MatSetValuesLocal(T, nrc, cols, ncc, cols + nch, data, INSERT_VALUES));
    1498:       PetscCall(MatDenseRestoreArrayRead(GKins, &data));

    1500:       /* coarse discrete gradient */
    1501:       if (pcbddc->nedcG) {
    1502:         PetscInt cols[2];

    1504:         cols[0] = 2 * i;
    1505:         cols[1] = 2 * i + 1;
    1506:         PetscCall(MatSetValuesLocal(pcbddc->nedcG, 1, &i, 2, cols, cvals, INSERT_VALUES));
    1507:       }
    1508:       PetscCall(ISRestoreIndices(eedges[i], &cols));
    1509:     }
    1510:     PetscCall(ISDestroy(&extrows[i]));
    1511:     PetscCall(ISDestroy(&extcols[i]));
    1512:     PetscCall(ISDestroy(&cornersis));
    1513:     PetscCall(MatDestroy(&Gins));
    1514:     PetscCall(MatDestroy(&GKins));
    1515:   }

    1517:   /* import edge constraints */
    1518:   if (nnsp_vscat) {
    1519:     Vec          V, E, *quadvecs;
    1520:     PetscInt     nvecs, nvecs_orth;
    1521:     MatNullSpace onnsp           = NULL;
    1522:     PetscBool    onnsp_has_const = PETSC_FALSE;
    1523:     const Vec   *onnsp_vecs      = NULL;
    1524:     PetscInt     onnsp_nvecs     = 0, new_nnsp_nvecs, old_nnsp_nvecs;
    1525:     IS           is_E_to_zero;

    1527:     /* import nearnullspace from preconditioning matrix if user-defined */
    1528:     PetscCall(MatGetNearNullSpace(pc->pmat, &onnsp));
    1529:     if (onnsp) {
    1530:       PetscBool isinternal;

    1532:       PetscCall(PetscStrcmp("_internal_BDDC_nedelec_nnsp", ((PetscObject)onnsp)->name, &isinternal));
    1533:       if (!isinternal) PetscCall(MatNullSpaceGetVecs(onnsp, &onnsp_has_const, &onnsp_nvecs, &onnsp_vecs));
    1534:     }
    1535:     new_nnsp_nvecs = nnsp_nvecs + (nnsp_has_const ? 1 : 0);
    1536:     old_nnsp_nvecs = onnsp_nvecs + (onnsp_has_const ? 1 : 0);
    1537:     nvecs          = old_nnsp_nvecs + new_nnsp_nvecs;
    1538:     PetscCall(PetscMalloc1(nvecs, &quadvecs));

    1540:     PetscCall(PetscObjectQuery((PetscObject)nnsp_vscat, "__V_Vec", (PetscObject *)&V));
    1541:     PetscCall(PetscObjectQuery((PetscObject)nnsp_vscat, "__E_Vec", (PetscObject *)&E));
    1542:     PetscCall(PetscObjectQuery((PetscObject)nnsp_vscat, "__E_zero", (PetscObject *)&is_E_to_zero));
    1543:     for (i = 0; i < nvecs; i++) PetscCall(VecDuplicate(E, &quadvecs[i]));
    1544:     cum = 0;
    1545:     if (nnsp_has_const) {
    1546:       PetscCall(VecSet(V, 1.0));
    1547:       PetscCall(VecScatterBegin(nnsp_vscat, V, quadvecs[0], INSERT_VALUES, SCATTER_FORWARD));
    1548:       PetscCall(VecScatterEnd(nnsp_vscat, V, quadvecs[0], INSERT_VALUES, SCATTER_FORWARD));
    1549:       cum = 1;
    1550:     }
    1551:     for (i = 0; i < nnsp_nvecs; i++) {
    1552:       PetscCall(VecScatterBegin(nnsp_vscat, nnsp_vecs[i], quadvecs[i + cum], INSERT_VALUES, SCATTER_FORWARD));
    1553:       PetscCall(VecScatterEnd(nnsp_vscat, nnsp_vecs[i], quadvecs[i + cum], INSERT_VALUES, SCATTER_FORWARD));
    1554:     }

    1556:     /* Now add old nnsp if present */
    1557:     cum = 0;
    1558:     if (onnsp_has_const) {
    1559:       PetscCall(VecSet(quadvecs[new_nnsp_nvecs], 1.0));
    1560:       PetscCall(VecISSet(quadvecs[new_nnsp_nvecs], is_E_to_zero, 0));
    1561:       cum = 1;
    1562:     }
    1563:     for (i = 0; i < onnsp_nvecs; i++) {
    1564:       PetscCall(VecCopy(onnsp_vecs[i], quadvecs[i + cum + new_nnsp_nvecs]));
    1565:       PetscCall(VecISSet(quadvecs[i + cum + new_nnsp_nvecs], is_E_to_zero, 0));
    1566:     }
    1567:     nvecs_orth = nvecs;
    1568:     PetscCall(PCBDDCOrthonormalizeVecs(&nvecs_orth, quadvecs));
    1569:     PetscCall(MatNullSpaceCreate(PetscObjectComm((PetscObject)pc), PETSC_FALSE, nvecs_orth, quadvecs, &nnsp));
    1570:     for (i = 0; i < nvecs; i++) PetscCall(VecDestroy(&quadvecs[i]));
    1571:     PetscCall(PetscFree(quadvecs));
    1572:     PetscCall(PetscObjectSetName((PetscObject)nnsp, "_internal_BDDC_nedelec_nnsp"));
    1573:     PetscCall(MatSetNearNullSpace(pc->pmat, nnsp));
    1574:     PetscCall(MatNullSpaceDestroy(&nnsp));
    1575:   }
    1576:   PetscCall(VecScatterDestroy(&nnsp_vscat));
    1577:   PetscCall(ISLocalToGlobalMappingDestroy(&vl2g));
    1578:   PetscCall(ISLocalToGlobalMappingDestroy(&el2g));
    1579:   PetscCall(ISLocalToGlobalMappingDestroy(&al2g));

    1581:   /* Start assembling */
    1582:   PetscCall(MatAssemblyBegin(T, MAT_FINAL_ASSEMBLY));
    1583:   if (pcbddc->nedcG) PetscCall(MatAssemblyBegin(pcbddc->nedcG, MAT_FINAL_ASSEMBLY));

    1585:   /* Free */
    1586:   if (fl2g) {
    1587:     PetscCall(ISDestroy(&primals));
    1588:     for (i = 0; i < nee; i++) PetscCall(ISDestroy(&eedges[i]));
    1589:     PetscCall(PetscFree(eedges));
    1590:   }

    1592:   /* hack mat_graph with primal dofs on the coarse edges */
    1593:   {
    1594:     PCBDDCGraph graph  = pcbddc->mat_graph;
    1595:     PetscInt   *oqueue = graph->queue;
    1596:     PetscInt   *ocptr  = graph->cptr;
    1597:     PetscInt    ncc, *idxs;

    1599:     /* find first primal edge */
    1600:     if (pcbddc->nedclocal) PetscCall(ISGetIndices(pcbddc->nedclocal, (const PetscInt **)&idxs));
    1601:     else {
    1602:       if (fl2g) PetscCall(ISLocalToGlobalMappingApply(fl2g, nee, cedges, cedges));
    1603:       idxs = cedges;
    1604:     }
    1605:     cum = 0;
    1606:     while (cum < nee && cedges[cum] < 0) cum++;

    1608:     /* adapt connected components */
    1609:     PetscCall(PetscMalloc2(graph->nvtxs + 1, &graph->cptr, ocptr[graph->ncc], &graph->queue));
    1610:     graph->cptr[0] = 0;
    1611:     for (i = 0, ncc = 0; i < graph->ncc; i++) {
    1612:       PetscInt lc = ocptr[i + 1] - ocptr[i];
    1613:       if (cum != nee && oqueue[ocptr[i + 1] - 1] == cedges[cum]) { /* this cc has a primal dof */
    1614:         graph->cptr[ncc + 1]           = graph->cptr[ncc] + 1;
    1615:         graph->queue[graph->cptr[ncc]] = cedges[cum];
    1616:         ncc++;
    1617:         lc--;
    1618:         cum++;
    1619:         while (cum < nee && cedges[cum] < 0) cum++;
    1620:       }
    1621:       graph->cptr[ncc + 1] = graph->cptr[ncc] + lc;
    1622:       for (j = 0; j < lc; j++) graph->queue[graph->cptr[ncc] + j] = oqueue[ocptr[i] + j];
    1623:       ncc++;
    1624:     }
    1625:     graph->ncc = ncc;
    1626:     if (pcbddc->nedclocal) PetscCall(ISRestoreIndices(pcbddc->nedclocal, (const PetscInt **)&idxs));
    1627:     PetscCall(PetscFree2(ocptr, oqueue));
    1628:   }
    1629:   PetscCall(ISLocalToGlobalMappingDestroy(&fl2g));
    1630:   PetscCall(PCBDDCGraphRestoreCandidatesIS(pcbddc->mat_graph, NULL, NULL, &nee, &alleedges, &allprimals));
    1631:   PetscCall(PCBDDCGraphResetCSR(pcbddc->mat_graph));

    1633:   PetscCall(ISDestroy(&nedfieldlocal));
    1634:   PetscCall(PetscFree(extrow));
    1635:   PetscCall(PetscFree2(work, rwork));
    1636:   PetscCall(PetscFree(corners));
    1637:   PetscCall(PetscFree(cedges));
    1638:   PetscCall(PetscFree(extrows));
    1639:   PetscCall(PetscFree(extcols));
    1640:   PetscCall(MatDestroy(&lG));

    1642:   /* Complete assembling */
    1643:   PetscCall(MatAssemblyEnd(T, MAT_FINAL_ASSEMBLY));
    1644:   PetscCall(MatViewFromOptions(T, (PetscObject)pc, "-pc_bddc_nedelec_change_view"));
    1645:   if (pcbddc->nedcG) {
    1646:     PetscCall(MatAssemblyEnd(pcbddc->nedcG, MAT_FINAL_ASSEMBLY));
    1647:     PetscCall(MatViewFromOptions(pcbddc->nedcG, (PetscObject)pc, "-pc_bddc_nedelec_coarse_change_view"));
    1648:   }

    1650:   PetscCall(ISDestroy(&elements_corners));

    1652:   /* set change of basis */
    1653:   PetscCall(PCBDDCSetChangeOfBasisMat(pc, T, PETSC_FALSE));
    1654:   PetscCall(MatDestroy(&T));
    1655:   PetscFunctionReturn(PETSC_SUCCESS);
    1656: }

    1658: /* the near-null space of BDDC carries information on quadrature weights,
    1659:    and these can be collinear -> so cheat with MatNullSpaceCreate
    1660:    and create a suitable set of basis vectors first */
    1661: PetscErrorCode PCBDDCNullSpaceCreate(MPI_Comm comm, PetscBool has_const, PetscInt nvecs, Vec quad_vecs[], MatNullSpace *nnsp)
    1662: {
    1663:   PetscInt i;

    1665:   PetscFunctionBegin;
    1666:   for (i = 0; i < nvecs; i++) {
    1667:     PetscInt first, last;

    1669:     PetscCall(VecGetOwnershipRange(quad_vecs[i], &first, &last));
    1670:     PetscCheck(last - first >= 2 * nvecs || !has_const, PETSC_COMM_SELF, PETSC_ERR_SUP, "Not implemented");
    1671:     if (i >= first && i < last) {
    1672:       PetscScalar *data;
    1673:       PetscCall(VecGetArray(quad_vecs[i], &data));
    1674:       if (!has_const) {
    1675:         data[i - first] = 1.;
    1676:       } else {
    1677:         data[2 * i - first]     = 1. / PetscSqrtReal(2.);
    1678:         data[2 * i - first + 1] = -1. / PetscSqrtReal(2.);
    1679:       }
    1680:       PetscCall(VecRestoreArray(quad_vecs[i], &data));
    1681:     }
    1682:     PetscCall(PetscObjectStateIncrease((PetscObject)quad_vecs[i]));
    1683:   }
    1684:   PetscCall(MatNullSpaceCreate(comm, has_const, nvecs, quad_vecs, nnsp));
    1685:   for (i = 0; i < nvecs; i++) { /* reset vectors */
    1686:     PetscInt first, last;
    1687:     PetscCall(VecLockReadPop(quad_vecs[i]));
    1688:     PetscCall(VecGetOwnershipRange(quad_vecs[i], &first, &last));
    1689:     if (i >= first && i < last) {
    1690:       PetscScalar *data;
    1691:       PetscCall(VecGetArray(quad_vecs[i], &data));
    1692:       if (!has_const) {
    1693:         data[i - first] = 0.;
    1694:       } else {
    1695:         data[2 * i - first]     = 0.;
    1696:         data[2 * i - first + 1] = 0.;
    1697:       }
    1698:       PetscCall(VecRestoreArray(quad_vecs[i], &data));
    1699:     }
    1700:     PetscCall(PetscObjectStateIncrease((PetscObject)quad_vecs[i]));
    1701:     PetscCall(VecLockReadPush(quad_vecs[i]));
    1702:   }
    1703:   PetscFunctionReturn(PETSC_SUCCESS);
    1704: }

    1706: PetscErrorCode PCBDDCComputeNoNetFlux(Mat A, Mat divudotp, PetscBool transpose, IS vl2l, PCBDDCGraph graph, MatNullSpace *nnsp)
    1707: {
    1708:   Mat                    loc_divudotp;
    1709:   Vec                    p, v, quad_vec;
    1710:   ISLocalToGlobalMapping map;
    1711:   PetscScalar           *array;

    1713:   PetscFunctionBegin;
    1714:   PetscCall(MatCreateVecs(A, &quad_vec, NULL));
    1715:   if (!transpose) {
    1716:     PetscCall(MatISGetLocalToGlobalMapping(A, &map, NULL));
    1717:   } else {
    1718:     PetscCall(MatISGetLocalToGlobalMapping(A, NULL, &map));
    1719:   }
    1720:   PetscCall(PCBDDCNullSpaceCreate(PetscObjectComm((PetscObject)A), PETSC_FALSE, 1, &quad_vec, nnsp));
    1721:   PetscCall(VecLockReadPop(quad_vec));
    1722:   PetscCall(VecSetLocalToGlobalMapping(quad_vec, map));

    1724:   /* compute local quad vec */
    1725:   PetscCall(MatISGetLocalMat(divudotp, &loc_divudotp));
    1726:   if (!transpose) {
    1727:     PetscCall(MatCreateVecs(loc_divudotp, &v, &p));
    1728:   } else {
    1729:     PetscCall(MatCreateVecs(loc_divudotp, &p, &v));
    1730:   }
    1731:   /* the assumption here is that the constant vector interpolates the constant on the L2 conforming space */
    1732:   PetscCall(VecSet(p, 1.));
    1733:   if (!transpose) {
    1734:     PetscCall(MatMultTranspose(loc_divudotp, p, v));
    1735:   } else {
    1736:     PetscCall(MatMult(loc_divudotp, p, v));
    1737:   }
    1738:   PetscCall(VecDestroy(&p));
    1739:   if (vl2l) {
    1740:     Mat        lA;
    1741:     VecScatter sc;
    1742:     Vec        vins;

    1744:     PetscCall(MatISGetLocalMat(A, &lA));
    1745:     PetscCall(MatCreateVecs(lA, &vins, NULL));
    1746:     PetscCall(VecScatterCreate(v, NULL, vins, vl2l, &sc));
    1747:     PetscCall(VecScatterBegin(sc, v, vins, INSERT_VALUES, SCATTER_FORWARD));
    1748:     PetscCall(VecScatterEnd(sc, v, vins, INSERT_VALUES, SCATTER_FORWARD));
    1749:     PetscCall(VecScatterDestroy(&sc));
    1750:     PetscCall(VecDestroy(&v));
    1751:     v = vins;
    1752:   }

    1754:   /* mask summation of interface values */
    1755:   PetscInt        n, *mmask, *mask, *idxs, nmr, nr;
    1756:   const PetscInt *degree;
    1757:   PetscSF         msf;

    1759:   PetscCall(VecGetLocalSize(v, &n));
    1760:   PetscCall(PetscSFGetGraph(graph->interface_subset_sf, &nr, NULL, NULL, NULL));
    1761:   PetscCall(PetscSFGetMultiSF(graph->interface_subset_sf, &msf));
    1762:   PetscCall(PetscSFGetGraph(msf, &nmr, NULL, NULL, NULL));
    1763:   PetscCall(PetscCalloc3(nmr, &mmask, n, &mask, n, &idxs));
    1764:   PetscCall(PetscSFComputeDegreeBegin(graph->interface_subset_sf, &degree));
    1765:   PetscCall(PetscSFComputeDegreeEnd(graph->interface_subset_sf, &degree));
    1766:   for (PetscInt i = 0, c = 0; i < nr; i++) {
    1767:     mmask[c] = 1;
    1768:     c += degree[i];
    1769:   }
    1770:   PetscCall(PetscSFScatterBegin(graph->interface_subset_sf, MPIU_INT, mmask, mask));
    1771:   PetscCall(PetscSFScatterEnd(graph->interface_subset_sf, MPIU_INT, mmask, mask));
    1772:   PetscCall(VecGetArray(v, &array));
    1773:   for (PetscInt i = 0; i < n; i++) {
    1774:     array[i] *= mask[i];
    1775:     idxs[i] = i;
    1776:   }
    1777:   PetscCall(VecSetValuesLocal(quad_vec, n, idxs, array, ADD_VALUES));
    1778:   PetscCall(VecRestoreArray(v, &array));
    1779:   PetscCall(PetscFree3(mmask, mask, idxs));
    1780:   PetscCall(VecDestroy(&v));
    1781:   PetscCall(VecAssemblyBegin(quad_vec));
    1782:   PetscCall(VecAssemblyEnd(quad_vec));
    1783:   PetscCall(VecViewFromOptions(quad_vec, NULL, "-pc_bddc_quad_vec_view"));
    1784:   PetscCall(VecLockReadPush(quad_vec));
    1785:   PetscCall(VecDestroy(&quad_vec));
    1786:   PetscFunctionReturn(PETSC_SUCCESS);
    1787: }

    1789: PetscErrorCode PCBDDCAddPrimalVerticesLocalIS(PC pc, IS primalv)
    1790: {
    1791:   PC_BDDC *pcbddc = (PC_BDDC *)pc->data;

    1793:   PetscFunctionBegin;
    1794:   if (primalv) {
    1795:     if (pcbddc->user_primal_vertices_local) {
    1796:       IS list[2], newp;

    1798:       list[0] = primalv;
    1799:       list[1] = pcbddc->user_primal_vertices_local;
    1800:       PetscCall(ISConcatenate(PetscObjectComm((PetscObject)pc), 2, list, &newp));
    1801:       PetscCall(ISSortRemoveDups(newp));
    1802:       PetscCall(ISDestroy(&list[1]));
    1803:       pcbddc->user_primal_vertices_local = newp;
    1804:     } else {
    1805:       PetscCall(PCBDDCSetPrimalVerticesLocalIS(pc, primalv));
    1806:     }
    1807:   }
    1808:   PetscFunctionReturn(PETSC_SUCCESS);
    1809: }

    1811: static PetscErrorCode func_coords_private(PetscInt dim, PetscReal t, const PetscReal X[], PetscInt Nf, PetscScalar *out, PetscCtx ctx)
    1812: {
    1813:   PetscInt f, *comp = (PetscInt *)ctx;

    1815:   PetscFunctionBegin;
    1816:   for (f = 0; f < Nf; f++) out[f] = X[*comp];
    1817:   PetscFunctionReturn(PETSC_SUCCESS);
    1818: }

    1820: PetscErrorCode PCBDDCComputeLocalTopologyInfo(PC pc)
    1821: {
    1822:   Vec       local, global;
    1823:   PC_BDDC  *pcbddc     = (PC_BDDC *)pc->data;
    1824:   Mat_IS   *matis      = (Mat_IS *)pc->pmat->data;
    1825:   PetscBool monolithic = PETSC_FALSE;

    1827:   PetscFunctionBegin;
    1828:   PetscOptionsBegin(PetscObjectComm((PetscObject)pc), ((PetscObject)pc)->prefix, "BDDC topology options", "PC");
    1829:   PetscCall(PetscOptionsBool("-pc_bddc_monolithic", "Discard any information on dofs splitting", NULL, monolithic, &monolithic, NULL));
    1830:   PetscOptionsEnd();
    1831:   /* need to convert from global to local topology information and remove references to information in global ordering */
    1832:   PetscCall(MatCreateVecs(pc->pmat, &global, NULL));
    1833:   PetscCall(MatCreateVecs(matis->A, &local, NULL));
    1834:   PetscCall(VecBindToCPU(global, PETSC_TRUE));
    1835:   PetscCall(VecBindToCPU(local, PETSC_TRUE));
    1836:   if (monolithic) { /* just get block size to properly compute vertices */
    1837:     if (pcbddc->vertex_size == 1) PetscCall(MatGetBlockSize(pc->pmat, &pcbddc->vertex_size));
    1838:     goto boundary;
    1839:   }

    1841:   if (pcbddc->user_provided_isfordofs) {
    1842:     if (pcbddc->n_ISForDofs) {
    1843:       PetscInt i;

    1845:       PetscCall(PetscMalloc1(pcbddc->n_ISForDofs, &pcbddc->ISForDofsLocal));
    1846:       for (i = 0; i < pcbddc->n_ISForDofs; i++) {
    1847:         PetscInt bs;

    1849:         PetscCall(PCBDDCGlobalToLocal(matis->rctx, global, local, pcbddc->ISForDofs[i], &pcbddc->ISForDofsLocal[i]));
    1850:         PetscCall(ISGetBlockSize(pcbddc->ISForDofs[i], &bs));
    1851:         PetscCall(ISSetBlockSize(pcbddc->ISForDofsLocal[i], bs));
    1852:         PetscCall(ISDestroy(&pcbddc->ISForDofs[i]));
    1853:       }
    1854:       pcbddc->n_ISForDofsLocal = pcbddc->n_ISForDofs;
    1855:       pcbddc->n_ISForDofs      = 0;
    1856:       PetscCall(PetscFree(pcbddc->ISForDofs));
    1857:     }
    1858:   } else {
    1859:     if (!pcbddc->n_ISForDofsLocal) { /* field split not present */
    1860:       DM dm;

    1862:       PetscCall(MatGetDM(pc->pmat, &dm));
    1863:       if (!dm) PetscCall(PCGetDM(pc, &dm));
    1864:       if (dm) {
    1865:         IS      *fields;
    1866:         PetscInt nf, i;

    1868:         PetscCall(DMCreateFieldDecomposition(dm, &nf, NULL, &fields, NULL));
    1869:         PetscCall(PetscMalloc1(nf, &pcbddc->ISForDofsLocal));
    1870:         for (i = 0; i < nf; i++) {
    1871:           PetscInt bs;

    1873:           PetscCall(PCBDDCGlobalToLocal(matis->rctx, global, local, fields[i], &pcbddc->ISForDofsLocal[i]));
    1874:           PetscCall(ISGetBlockSize(fields[i], &bs));
    1875:           PetscCall(ISSetBlockSize(pcbddc->ISForDofsLocal[i], bs));
    1876:           PetscCall(ISDestroy(&fields[i]));
    1877:         }
    1878:         PetscCall(PetscFree(fields));
    1879:         pcbddc->n_ISForDofsLocal = nf;
    1880:       } else { /* See if MATIS has fields attached by the conversion from MatNest */
    1881:         PetscContainer c;

    1883:         PetscCall(PetscObjectQuery((PetscObject)pc->pmat, "_convert_nest_lfields", (PetscObject *)&c));
    1884:         if (c) {
    1885:           MatISLocalFields lf;
    1886:           PetscCall(PetscContainerGetPointer(c, &lf));
    1887:           PetscCall(PCBDDCSetDofsSplittingLocal(pc, lf->nr, lf->rf));
    1888:         } else { /* fallback, create the default fields if bs > 1 */
    1889:           PetscInt i, n = matis->A->rmap->n;
    1890:           PetscCall(MatGetBlockSize(pc->pmat, &i));
    1891:           if (i > 1) {
    1892:             pcbddc->n_ISForDofsLocal = i;
    1893:             PetscCall(PetscMalloc1(pcbddc->n_ISForDofsLocal, &pcbddc->ISForDofsLocal));
    1894:             for (i = 0; i < pcbddc->n_ISForDofsLocal; i++) PetscCall(ISCreateStride(PetscObjectComm((PetscObject)pc), n / pcbddc->n_ISForDofsLocal, i, pcbddc->n_ISForDofsLocal, &pcbddc->ISForDofsLocal[i]));
    1895:           }
    1896:         }
    1897:       }
    1898:     } else {
    1899:       PetscInt i;
    1900:       for (i = 0; i < pcbddc->n_ISForDofsLocal; i++) PetscCall(PCBDDCConsistencyCheckIS(pc, MPI_LAND, &pcbddc->ISForDofsLocal[i]));
    1901:     }
    1902:   }

    1904: boundary:
    1905:   if (!pcbddc->DirichletBoundariesLocal && pcbddc->DirichletBoundaries) {
    1906:     PetscCall(PCBDDCGlobalToLocal(matis->rctx, global, local, pcbddc->DirichletBoundaries, &pcbddc->DirichletBoundariesLocal));
    1907:   } else if (pcbddc->DirichletBoundariesLocal) {
    1908:     PetscCall(PCBDDCConsistencyCheckIS(pc, MPI_LAND, &pcbddc->DirichletBoundariesLocal));
    1909:   }
    1910:   if (!pcbddc->NeumannBoundariesLocal && pcbddc->NeumannBoundaries) {
    1911:     PetscCall(PCBDDCGlobalToLocal(matis->rctx, global, local, pcbddc->NeumannBoundaries, &pcbddc->NeumannBoundariesLocal));
    1912:   } else if (pcbddc->NeumannBoundariesLocal) {
    1913:     PetscCall(PCBDDCConsistencyCheckIS(pc, MPI_LOR, &pcbddc->NeumannBoundariesLocal));
    1914:   }
    1915:   if (!pcbddc->user_primal_vertices_local && pcbddc->user_primal_vertices) PetscCall(PCBDDCGlobalToLocal(matis->rctx, global, local, pcbddc->user_primal_vertices, &pcbddc->user_primal_vertices_local));
    1916:   PetscCall(VecDestroy(&global));
    1917:   PetscCall(VecDestroy(&local));
    1918:   /* detect local disconnected subdomains if requested or needed */
    1919:   if (pcbddc->detect_disconnected || matis->allow_repeated) {
    1920:     IS        primalv = NULL;
    1921:     PetscInt  nel;
    1922:     PetscBool filter = pcbddc->detect_disconnected_filter;

    1924:     for (PetscInt i = 0; i < pcbddc->n_local_subs; i++) PetscCall(ISDestroy(&pcbddc->local_subs[i]));
    1925:     PetscCall(PetscFree(pcbddc->local_subs));
    1926:     PetscCall(MatGetVariableBlockSizes(matis->A, &nel, NULL));
    1927:     if (matis->allow_repeated && nel) {
    1928:       const PetscInt *elsizes;

    1930:       pcbddc->n_local_subs = nel;
    1931:       PetscCall(MatGetVariableBlockSizes(matis->A, NULL, &elsizes));
    1932:       PetscCall(PetscMalloc1(nel, &pcbddc->local_subs));
    1933:       for (PetscInt i = 0, c = 0; i < nel; i++) {
    1934:         PetscCall(ISCreateStride(PETSC_COMM_SELF, elsizes[i], c, 1, &pcbddc->local_subs[i]));
    1935:         c += elsizes[i];
    1936:       }
    1937:     } else {
    1938:       PetscCall(PCBDDCDetectDisconnectedComponents(pc, filter, &pcbddc->n_local_subs, &pcbddc->local_subs, &primalv));
    1939:     }
    1940:     PetscCall(PCBDDCAddPrimalVerticesLocalIS(pc, primalv));
    1941:     PetscCall(ISDestroy(&primalv));
    1942:   }
    1943:   /* early stage corner detection */
    1944:   {
    1945:     DM dm;

    1947:     PetscCall(MatGetDM(pc->pmat, &dm));
    1948:     if (!dm) PetscCall(PCGetDM(pc, &dm));
    1949:     if (dm) {
    1950:       PetscBool isda;

    1952:       PetscCall(PetscObjectTypeCompare((PetscObject)dm, DMDA, &isda));
    1953:       if (isda) {
    1954:         ISLocalToGlobalMapping l2l;
    1955:         IS                     corners;
    1956:         Mat                    lA;
    1957:         PetscBool              gl, lo;

    1959:         {
    1960:           Vec                cvec;
    1961:           const PetscScalar *coords;
    1962:           PetscInt           dof, n, cdim;
    1963:           PetscBool          memc = PetscDefined(USE_COMPLEX) ? PETSC_FALSE : PETSC_TRUE;

    1965:           PetscCall(DMDAGetInfo(dm, NULL, NULL, NULL, NULL, NULL, NULL, NULL, &dof, NULL, NULL, NULL, NULL, NULL));
    1966:           PetscCall(DMGetCoordinates(dm, &cvec));
    1967:           PetscCall(VecGetLocalSize(cvec, &n));
    1968:           PetscCall(VecGetBlockSize(cvec, &cdim));
    1969:           n /= cdim;
    1970:           PetscCall(PetscFree(pcbddc->mat_graph->coords));
    1971:           PetscCall(PetscMalloc1(dof * n * cdim, &pcbddc->mat_graph->coords));
    1972:           PetscCall(VecGetArrayRead(cvec, &coords));
    1973:           if (dof != 1) memc = PETSC_FALSE;
    1974:           if (memc) {
    1975:             PetscCall(PetscArraycpy(pcbddc->mat_graph->coords, coords, cdim * n * dof));
    1976:           } else { /* BDDC graph does not use any blocked information, we need to replicate the data */
    1977:             PetscReal *bcoords = pcbddc->mat_graph->coords;
    1978:             PetscInt   i, b, d;

    1980:             for (i = 0; i < n; i++) {
    1981:               for (b = 0; b < dof; b++) {
    1982:                 for (d = 0; d < cdim; d++) bcoords[i * dof * cdim + b * cdim + d] = PetscRealPart(coords[i * cdim + d]);
    1983:               }
    1984:             }
    1985:           }
    1986:           PetscCall(VecRestoreArrayRead(cvec, &coords));
    1987:           pcbddc->mat_graph->cdim  = cdim;
    1988:           pcbddc->mat_graph->cnloc = dof * n;
    1989:           pcbddc->mat_graph->cloc  = PETSC_FALSE;
    1990:         }
    1991:         PetscCall(DMDAGetSubdomainCornersIS(dm, &corners));
    1992:         PetscCall(MatISGetLocalMat(pc->pmat, &lA));
    1993:         PetscCall(MatGetLocalToGlobalMapping(lA, &l2l, NULL));
    1994:         PetscCall(MatISRestoreLocalMat(pc->pmat, &lA));
    1995:         lo = (PetscBool)(l2l && corners);
    1996:         PetscCallMPI(MPIU_Allreduce(&lo, &gl, 1, MPI_C_BOOL, MPI_LAND, PetscObjectComm((PetscObject)pc)));
    1997:         if (gl) { /* From PETSc's DMDA */
    1998:           const PetscInt *idx;
    1999:           PetscInt        dof, bs, *idxout, n;

    2001:           PetscCall(DMDAGetInfo(dm, NULL, NULL, NULL, NULL, NULL, NULL, NULL, &dof, NULL, NULL, NULL, NULL, NULL));
    2002:           PetscCall(ISLocalToGlobalMappingGetBlockSize(l2l, &bs));
    2003:           PetscCall(ISGetLocalSize(corners, &n));
    2004:           PetscCall(ISGetIndices(corners, &idx));
    2005:           if (bs == dof) {
    2006:             PetscCall(PetscMalloc1(n, &idxout));
    2007:             PetscCall(ISLocalToGlobalMappingApplyBlock(l2l, n, idx, idxout));
    2008:           } else { /* the original DMDA local-to-local map have been modified */
    2009:             PetscInt i, d;

    2011:             PetscCall(PetscMalloc1(dof * n, &idxout));
    2012:             for (i = 0; i < n; i++)
    2013:               for (d = 0; d < dof; d++) idxout[dof * i + d] = dof * idx[i] + d;
    2014:             PetscCall(ISLocalToGlobalMappingApply(l2l, dof * n, idxout, idxout));

    2016:             bs = 1;
    2017:             n *= dof;
    2018:           }
    2019:           PetscCall(ISRestoreIndices(corners, &idx));
    2020:           PetscCall(DMDARestoreSubdomainCornersIS(dm, &corners));
    2021:           PetscCall(ISCreateBlock(PetscObjectComm((PetscObject)pc), bs, n, idxout, PETSC_OWN_POINTER, &corners));
    2022:           PetscCall(PCBDDCAddPrimalVerticesLocalIS(pc, corners));
    2023:           PetscCall(ISDestroy(&corners));
    2024:           pcbddc->corner_selected  = PETSC_TRUE;
    2025:           pcbddc->corner_selection = PETSC_TRUE;
    2026:         }
    2027:         if (corners) PetscCall(DMDARestoreSubdomainCornersIS(dm, &corners));
    2028:       }
    2029:     }
    2030:   }
    2031:   if (pcbddc->corner_selection && !pcbddc->mat_graph->cdim) {
    2032:     DM dm;

    2034:     PetscCall(MatGetDM(pc->pmat, &dm));
    2035:     if (!dm) PetscCall(PCGetDM(pc, &dm));
    2036:     if (dm) { /* this can get very expensive, I need to find a faster alternative */
    2037:       Vec          vcoords;
    2038:       PetscSection section;
    2039:       PetscReal   *coords;
    2040:       PetscInt     d, cdim, nl, nf, **ctxs;
    2041:       PetscErrorCode (**funcs)(PetscInt, PetscReal, const PetscReal *, PetscInt, PetscScalar *, void *);
    2042:       /* debug coordinates */
    2043:       PetscViewer       viewer;
    2044:       PetscBool         flg;
    2045:       PetscViewerFormat format;
    2046:       const char       *prefix;

    2048:       PetscCall(DMGetCoordinateDim(dm, &cdim));
    2049:       PetscCall(DMGetLocalSection(dm, &section));
    2050:       PetscCall(PetscSectionGetNumFields(section, &nf));
    2051:       PetscCall(DMCreateGlobalVector(dm, &vcoords));
    2052:       PetscCall(VecGetLocalSize(vcoords, &nl));
    2053:       PetscCall(PetscMalloc1(nl * cdim, &coords));
    2054:       PetscCall(PetscMalloc2(nf, &funcs, nf, &ctxs));
    2055:       PetscCall(PetscMalloc1(nf, &ctxs[0]));
    2056:       for (d = 0; d < nf; d++) funcs[d] = func_coords_private;
    2057:       for (d = 1; d < nf; d++) ctxs[d] = ctxs[d - 1] + 1;

    2059:       /* debug coordinates */
    2060:       PetscCall(PCGetOptionsPrefix(pc, &prefix));
    2061:       PetscCall(PetscOptionsCreateViewer(PetscObjectComm((PetscObject)vcoords), ((PetscObject)vcoords)->options, prefix, "-pc_bddc_coords_vec_view", &viewer, &format, &flg));
    2062:       if (flg) PetscCall(PetscViewerPushFormat(viewer, format));
    2063:       for (d = 0; d < cdim; d++) {
    2064:         PetscInt           i;
    2065:         const PetscScalar *v;
    2066:         char               name[16];

    2068:         for (i = 0; i < nf; i++) ctxs[i][0] = d;
    2069:         PetscCall(PetscSNPrintf(name, sizeof(name), "bddc_coords_%" PetscInt_FMT, d));
    2070:         PetscCall(PetscObjectSetName((PetscObject)vcoords, name));
    2071:         PetscCall(DMProjectFunction(dm, 0.0, funcs, (void **)ctxs, INSERT_VALUES, vcoords));
    2072:         if (flg) PetscCall(VecView(vcoords, viewer));
    2073:         PetscCall(VecGetArrayRead(vcoords, &v));
    2074:         for (i = 0; i < nl; i++) coords[i * cdim + d] = PetscRealPart(v[i]);
    2075:         PetscCall(VecRestoreArrayRead(vcoords, &v));
    2076:       }
    2077:       PetscCall(VecDestroy(&vcoords));
    2078:       PetscCall(PCSetCoordinates(pc, cdim, nl, coords));
    2079:       PetscCall(PetscFree(coords));
    2080:       PetscCall(PetscFree(ctxs[0]));
    2081:       PetscCall(PetscFree2(funcs, ctxs));
    2082:       if (flg) {
    2083:         PetscCall(PetscViewerPopFormat(viewer));
    2084:         PetscCall(PetscViewerDestroy(&viewer));
    2085:       }
    2086:     }
    2087:   }
    2088:   PetscFunctionReturn(PETSC_SUCCESS);
    2089: }

    2091: PetscErrorCode PCBDDCConsistencyCheckIS(PC pc, MPI_Op mop, IS *is)
    2092: {
    2093:   Mat_IS         *matis = (Mat_IS *)pc->pmat->data;
    2094:   IS              nis;
    2095:   const PetscInt *idxs;
    2096:   PetscInt        i, nd, n = matis->A->rmap->n, *nidxs, nnd;

    2098:   PetscFunctionBegin;
    2099:   PetscCheck(mop == MPI_LAND || mop == MPI_LOR, PetscObjectComm((PetscObject)pc), PETSC_ERR_SUP, "Supported are MPI_LAND and MPI_LOR");
    2100:   if (mop == MPI_LAND) {
    2101:     /* init rootdata with true */
    2102:     for (i = 0; i < pc->pmat->rmap->n; i++) matis->sf_rootdata[i] = 1;
    2103:   } else {
    2104:     PetscCall(PetscArrayzero(matis->sf_rootdata, pc->pmat->rmap->n));
    2105:   }
    2106:   PetscCall(PetscArrayzero(matis->sf_leafdata, n));
    2107:   PetscCall(ISGetLocalSize(*is, &nd));
    2108:   PetscCall(ISGetIndices(*is, &idxs));
    2109:   for (i = 0; i < nd; i++)
    2110:     if (-1 < idxs[i] && idxs[i] < n) matis->sf_leafdata[idxs[i]] = 1;
    2111:   PetscCall(ISRestoreIndices(*is, &idxs));
    2112:   PetscCall(PetscSFReduceBegin(matis->sf, MPIU_INT, matis->sf_leafdata, matis->sf_rootdata, mop));
    2113:   PetscCall(PetscSFReduceEnd(matis->sf, MPIU_INT, matis->sf_leafdata, matis->sf_rootdata, mop));
    2114:   PetscCall(PetscSFBcastBegin(matis->sf, MPIU_INT, matis->sf_rootdata, matis->sf_leafdata, MPI_REPLACE));
    2115:   PetscCall(PetscSFBcastEnd(matis->sf, MPIU_INT, matis->sf_rootdata, matis->sf_leafdata, MPI_REPLACE));
    2116:   if (mop == MPI_LAND) PetscCall(PetscMalloc1(nd, &nidxs));
    2117:   else PetscCall(PetscMalloc1(n, &nidxs));
    2118:   for (i = 0, nnd = 0; i < n; i++)
    2119:     if (matis->sf_leafdata[i]) nidxs[nnd++] = i;
    2120:   PetscCall(ISCreateGeneral(PetscObjectComm((PetscObject)*is), nnd, nidxs, PETSC_OWN_POINTER, &nis));
    2121:   PetscCall(ISDestroy(is));
    2122:   *is = nis;
    2123:   PetscFunctionReturn(PETSC_SUCCESS);
    2124: }

    2126: PetscErrorCode PCBDDCBenignRemoveInterior(PC pc, Vec r, Vec z)
    2127: {
    2128:   PC_IS   *pcis   = (PC_IS *)pc->data;
    2129:   PC_BDDC *pcbddc = (PC_BDDC *)pc->data;

    2131:   PetscFunctionBegin;
    2132:   if (!pcbddc->benign_have_null) PetscFunctionReturn(PETSC_SUCCESS);
    2133:   if (pcbddc->ChangeOfBasisMatrix) {
    2134:     Vec swap;

    2136:     PetscCall(MatMultTranspose(pcbddc->ChangeOfBasisMatrix, r, pcbddc->work_change));
    2137:     swap                = pcbddc->work_change;
    2138:     pcbddc->work_change = r;
    2139:     r                   = swap;
    2140:   }
    2141:   PetscCall(VecScatterBegin(pcis->global_to_D, r, pcis->vec1_D, INSERT_VALUES, SCATTER_FORWARD));
    2142:   PetscCall(VecScatterEnd(pcis->global_to_D, r, pcis->vec1_D, INSERT_VALUES, SCATTER_FORWARD));
    2143:   PetscCall(PetscLogEventBegin(PC_BDDC_Solves[pcbddc->current_level][0], pc, 0, 0, 0));
    2144:   PetscCall(KSPSolve(pcbddc->ksp_D, pcis->vec1_D, pcis->vec2_D));
    2145:   PetscCall(PetscLogEventEnd(PC_BDDC_Solves[pcbddc->current_level][0], pc, 0, 0, 0));
    2146:   PetscCall(KSPCheckSolve(pcbddc->ksp_D, pc, pcis->vec2_D));
    2147:   PetscCall(VecSet(z, 0.));
    2148:   PetscCall(VecScatterBegin(pcis->global_to_D, pcis->vec2_D, z, INSERT_VALUES, SCATTER_REVERSE));
    2149:   PetscCall(VecScatterEnd(pcis->global_to_D, pcis->vec2_D, z, INSERT_VALUES, SCATTER_REVERSE));
    2150:   if (pcbddc->ChangeOfBasisMatrix) {
    2151:     pcbddc->work_change = r;
    2152:     PetscCall(VecCopy(z, pcbddc->work_change));
    2153:     PetscCall(MatMult(pcbddc->ChangeOfBasisMatrix, pcbddc->work_change, z));
    2154:   }
    2155:   PetscFunctionReturn(PETSC_SUCCESS);
    2156: }

    2158: static PetscErrorCode PCBDDCBenignMatMult_Private_Private(Mat A, Vec x, Vec y, PetscBool transpose)
    2159: {
    2160:   PCBDDCBenignMatMult_ctx ctx;
    2161:   PetscBool               apply_right, apply_left, reset_x;

    2163:   PetscFunctionBegin;
    2164:   PetscCall(MatShellGetContext(A, &ctx));
    2165:   if (transpose) {
    2166:     apply_right = ctx->apply_left;
    2167:     apply_left  = ctx->apply_right;
    2168:   } else {
    2169:     apply_right = ctx->apply_right;
    2170:     apply_left  = ctx->apply_left;
    2171:   }
    2172:   reset_x = PETSC_FALSE;
    2173:   if (apply_right) {
    2174:     const PetscScalar *ax;
    2175:     PetscInt           nl, i;

    2177:     PetscCall(VecGetLocalSize(x, &nl));
    2178:     PetscCall(VecGetArrayRead(x, &ax));
    2179:     PetscCall(PetscArraycpy(ctx->work, ax, nl));
    2180:     PetscCall(VecRestoreArrayRead(x, &ax));
    2181:     for (i = 0; i < ctx->benign_n; i++) {
    2182:       PetscScalar     sum, val;
    2183:       const PetscInt *idxs;
    2184:       PetscInt        nz, j;
    2185:       PetscCall(ISGetLocalSize(ctx->benign_zerodiag_subs[i], &nz));
    2186:       PetscCall(ISGetIndices(ctx->benign_zerodiag_subs[i], &idxs));
    2187:       sum = 0.;
    2188:       if (ctx->apply_p0) {
    2189:         val = ctx->work[idxs[nz - 1]];
    2190:         for (j = 0; j < nz - 1; j++) {
    2191:           sum += ctx->work[idxs[j]];
    2192:           ctx->work[idxs[j]] += val;
    2193:         }
    2194:       } else {
    2195:         for (j = 0; j < nz - 1; j++) sum += ctx->work[idxs[j]];
    2196:       }
    2197:       ctx->work[idxs[nz - 1]] -= sum;
    2198:       PetscCall(ISRestoreIndices(ctx->benign_zerodiag_subs[i], &idxs));
    2199:     }
    2200:     PetscCall(VecPlaceArray(x, ctx->work));
    2201:     reset_x = PETSC_TRUE;
    2202:   }
    2203:   if (transpose) {
    2204:     PetscCall(MatMultTranspose(ctx->A, x, y));
    2205:   } else {
    2206:     PetscCall(MatMult(ctx->A, x, y));
    2207:   }
    2208:   if (reset_x) PetscCall(VecResetArray(x));
    2209:   if (apply_left) {
    2210:     PetscScalar *ay;
    2211:     PetscInt     i;

    2213:     PetscCall(VecGetArray(y, &ay));
    2214:     for (i = 0; i < ctx->benign_n; i++) {
    2215:       PetscScalar     sum, val;
    2216:       const PetscInt *idxs;
    2217:       PetscInt        nz, j;
    2218:       PetscCall(ISGetLocalSize(ctx->benign_zerodiag_subs[i], &nz));
    2219:       PetscCall(ISGetIndices(ctx->benign_zerodiag_subs[i], &idxs));
    2220:       val = -ay[idxs[nz - 1]];
    2221:       if (ctx->apply_p0) {
    2222:         sum = 0.;
    2223:         for (j = 0; j < nz - 1; j++) {
    2224:           sum += ay[idxs[j]];
    2225:           ay[idxs[j]] += val;
    2226:         }
    2227:         ay[idxs[nz - 1]] += sum;
    2228:       } else {
    2229:         for (j = 0; j < nz - 1; j++) ay[idxs[j]] += val;
    2230:         ay[idxs[nz - 1]] = 0.;
    2231:       }
    2232:       PetscCall(ISRestoreIndices(ctx->benign_zerodiag_subs[i], &idxs));
    2233:     }
    2234:     PetscCall(VecRestoreArray(y, &ay));
    2235:   }
    2236:   PetscFunctionReturn(PETSC_SUCCESS);
    2237: }

    2239: static PetscErrorCode PCBDDCBenignMatMultTranspose_Private(Mat A, Vec x, Vec y)
    2240: {
    2241:   PetscFunctionBegin;
    2242:   PetscCall(PCBDDCBenignMatMult_Private_Private(A, x, y, PETSC_TRUE));
    2243:   PetscFunctionReturn(PETSC_SUCCESS);
    2244: }

    2246: static PetscErrorCode PCBDDCBenignMatMult_Private(Mat A, Vec x, Vec y)
    2247: {
    2248:   PetscFunctionBegin;
    2249:   PetscCall(PCBDDCBenignMatMult_Private_Private(A, x, y, PETSC_FALSE));
    2250:   PetscFunctionReturn(PETSC_SUCCESS);
    2251: }

    2253: PetscErrorCode PCBDDCBenignShellMat(PC pc, PetscBool restore)
    2254: {
    2255:   PC_IS                  *pcis   = (PC_IS *)pc->data;
    2256:   PC_BDDC                *pcbddc = (PC_BDDC *)pc->data;
    2257:   PCBDDCBenignMatMult_ctx ctx;

    2259:   PetscFunctionBegin;
    2260:   if (!restore) {
    2261:     Mat                A_IB, A_BI;
    2262:     PetscScalar       *work;
    2263:     PCBDDCReuseSolvers reuse = pcbddc->sub_schurs ? pcbddc->sub_schurs->reuse_solver : NULL;

    2265:     PetscCheck(!pcbddc->benign_original_mat, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Benign original mat has not been restored");
    2266:     if (!pcbddc->benign_change || !pcbddc->benign_n || pcbddc->benign_change_explicit) PetscFunctionReturn(PETSC_SUCCESS);
    2267:     PetscCall(PetscMalloc1(pcis->n, &work));
    2268:     PetscCall(MatCreate(PETSC_COMM_SELF, &A_IB));
    2269:     PetscCall(MatSetSizes(A_IB, pcis->n - pcis->n_B, pcis->n_B, PETSC_DECIDE, PETSC_DECIDE));
    2270:     PetscCall(MatSetType(A_IB, MATSHELL));
    2271:     PetscCall(MatShellSetOperation(A_IB, MATOP_MULT, (PetscErrorCodeFn *)PCBDDCBenignMatMult_Private));
    2272:     PetscCall(MatShellSetOperation(A_IB, MATOP_MULT_TRANSPOSE, (PetscErrorCodeFn *)PCBDDCBenignMatMultTranspose_Private));
    2273:     PetscCall(PetscNew(&ctx));
    2274:     PetscCall(MatShellSetContext(A_IB, ctx));
    2275:     ctx->apply_left  = PETSC_TRUE;
    2276:     ctx->apply_right = PETSC_FALSE;
    2277:     ctx->apply_p0    = PETSC_FALSE;
    2278:     ctx->benign_n    = pcbddc->benign_n;
    2279:     if (reuse) {
    2280:       ctx->benign_zerodiag_subs = reuse->benign_zerodiag_subs;
    2281:       ctx->free                 = PETSC_FALSE;
    2282:     } else { /* TODO: could be optimized for successive solves */
    2283:       ISLocalToGlobalMapping N_to_D;
    2284:       PetscInt               i;

    2286:       PetscCall(ISLocalToGlobalMappingCreateIS(pcis->is_I_local, &N_to_D));
    2287:       PetscCall(PetscMalloc1(pcbddc->benign_n, &ctx->benign_zerodiag_subs));
    2288:       for (i = 0; i < pcbddc->benign_n; i++) PetscCall(ISGlobalToLocalMappingApplyIS(N_to_D, IS_GTOLM_DROP, pcbddc->benign_zerodiag_subs[i], &ctx->benign_zerodiag_subs[i]));
    2289:       PetscCall(ISLocalToGlobalMappingDestroy(&N_to_D));
    2290:       ctx->free = PETSC_TRUE;
    2291:     }
    2292:     ctx->A    = pcis->A_IB;
    2293:     ctx->work = work;
    2294:     PetscCall(MatSetUp(A_IB));
    2295:     PetscCall(MatAssemblyBegin(A_IB, MAT_FINAL_ASSEMBLY));
    2296:     PetscCall(MatAssemblyEnd(A_IB, MAT_FINAL_ASSEMBLY));
    2297:     pcis->A_IB = A_IB;

    2299:     /* A_BI as A_IB^T */
    2300:     PetscCall(MatCreateTranspose(A_IB, &A_BI));
    2301:     pcbddc->benign_original_mat = pcis->A_BI;
    2302:     pcis->A_BI                  = A_BI;
    2303:   } else {
    2304:     if (!pcbddc->benign_original_mat) PetscFunctionReturn(PETSC_SUCCESS);
    2305:     PetscCall(MatShellGetContext(pcis->A_IB, &ctx));
    2306:     PetscCall(MatDestroy(&pcis->A_IB));
    2307:     pcis->A_IB = ctx->A;
    2308:     ctx->A     = NULL;
    2309:     PetscCall(MatDestroy(&pcis->A_BI));
    2310:     pcis->A_BI                  = pcbddc->benign_original_mat;
    2311:     pcbddc->benign_original_mat = NULL;
    2312:     if (ctx->free) {
    2313:       PetscInt i;
    2314:       for (i = 0; i < ctx->benign_n; i++) PetscCall(ISDestroy(&ctx->benign_zerodiag_subs[i]));
    2315:       PetscCall(PetscFree(ctx->benign_zerodiag_subs));
    2316:     }
    2317:     PetscCall(PetscFree(ctx->work));
    2318:     PetscCall(PetscFree(ctx));
    2319:   }
    2320:   PetscFunctionReturn(PETSC_SUCCESS);
    2321: }

    2323: /* used just in bddc debug mode */
    2324: static PetscErrorCode PCBDDCBenignProject(PC pc, IS is1, IS is2, Mat *B)
    2325: {
    2326:   PC_BDDC *pcbddc = (PC_BDDC *)pc->data;
    2327:   Mat_IS  *matis  = (Mat_IS *)pc->pmat->data;
    2328:   Mat      An;

    2330:   PetscFunctionBegin;
    2331:   PetscCall(MatPtAP(matis->A, pcbddc->benign_change, MAT_INITIAL_MATRIX, 2.0, &An));
    2332:   PetscCall(MatZeroRowsColumns(An, pcbddc->benign_n, pcbddc->benign_p0_lidx, 1.0, NULL, NULL));
    2333:   if (is1) {
    2334:     PetscCall(MatCreateSubMatrix(An, is1, is2, MAT_INITIAL_MATRIX, B));
    2335:     PetscCall(MatDestroy(&An));
    2336:   } else {
    2337:     *B = An;
    2338:   }
    2339:   PetscFunctionReturn(PETSC_SUCCESS);
    2340: }

    2342: /* TODO: add reuse flag */
    2343: PetscErrorCode MatSeqAIJCompress(Mat A, Mat *B)
    2344: {
    2345:   Mat             Bt;
    2346:   PetscScalar    *a, *bdata;
    2347:   const PetscInt *ii, *ij;
    2348:   PetscInt        m, n, i, nnz, *bii, *bij;
    2349:   PetscBool       flg_row;

    2351:   PetscFunctionBegin;
    2352:   PetscCall(MatGetSize(A, &n, &m));
    2353:   PetscCall(MatGetRowIJ(A, 0, PETSC_FALSE, PETSC_FALSE, &n, &ii, &ij, &flg_row));
    2354:   PetscCall(MatSeqAIJGetArray(A, &a));
    2355:   nnz = n;
    2356:   for (i = 0; i < ii[n]; i++) {
    2357:     if (PetscLikely(PetscAbsScalar(a[i]) > PETSC_SMALL)) nnz++;
    2358:   }
    2359:   PetscCall(PetscMalloc1(n + 1, &bii));
    2360:   PetscCall(PetscMalloc1(nnz, &bij));
    2361:   PetscCall(PetscMalloc1(nnz, &bdata));
    2362:   nnz    = 0;
    2363:   bii[0] = 0;
    2364:   for (i = 0; i < n; i++) {
    2365:     PetscInt j;
    2366:     for (j = ii[i]; j < ii[i + 1]; j++) {
    2367:       PetscScalar entry = a[j];
    2368:       if (PetscLikely(PetscAbsScalar(entry) > PETSC_SMALL) || (n == m && ij[j] == i)) {
    2369:         bij[nnz]   = ij[j];
    2370:         bdata[nnz] = entry;
    2371:         nnz++;
    2372:       }
    2373:     }
    2374:     bii[i + 1] = nnz;
    2375:   }
    2376:   PetscCall(MatSeqAIJRestoreArray(A, &a));
    2377:   PetscCall(MatCreateSeqAIJWithArrays(PetscObjectComm((PetscObject)A), n, m, bii, bij, bdata, &Bt));
    2378:   PetscCall(MatRestoreRowIJ(A, 0, PETSC_FALSE, PETSC_FALSE, &n, &ii, &ij, &flg_row));
    2379:   {
    2380:     Mat_SeqAIJ *b = (Mat_SeqAIJ *)Bt->data;
    2381:     b->free_a     = PETSC_TRUE;
    2382:     b->free_ij    = PETSC_TRUE;
    2383:   }
    2384:   if (*B == A) PetscCall(MatDestroy(&A));
    2385:   *B = Bt;
    2386:   PetscFunctionReturn(PETSC_SUCCESS);
    2387: }

    2389: PetscErrorCode PCBDDCDetectDisconnectedComponents(PC pc, PetscBool filter, PetscInt *ncc, IS *cc[], IS *primalv)
    2390: {
    2391:   Mat                    B = NULL;
    2392:   DM                     dm;
    2393:   IS                     is_dummy, *cc_n;
    2394:   ISLocalToGlobalMapping l2gmap_dummy;
    2395:   PCBDDCGraph            graph;
    2396:   PetscInt              *xadj_filtered = NULL, *adjncy_filtered = NULL;
    2397:   PetscInt               i, n;
    2398:   PetscInt              *xadj, *adjncy;
    2399:   PetscBool              isplex = PETSC_FALSE;

    2401:   PetscFunctionBegin;
    2402:   if (ncc) *ncc = 0;
    2403:   if (cc) *cc = NULL;
    2404:   if (primalv) *primalv = NULL;
    2405:   PetscCall(PCBDDCGraphCreate(&graph));
    2406:   PetscCall(MatGetDM(pc->pmat, &dm));
    2407:   if (!dm) PetscCall(PCGetDM(pc, &dm));
    2408:   if (dm) PetscCall(PetscObjectTypeCompareAny((PetscObject)dm, &isplex, DMPLEX, DMP4EST, DMP8EST, ""));
    2409:   if (filter) isplex = PETSC_FALSE;

    2411:   if (isplex) { /* this code has been modified from plexpartition.c */
    2412:     PetscInt        p, pStart, pEnd, a, adjSize, idx, size, nroots;
    2413:     PetscInt       *adj = NULL;
    2414:     IS              cellNumbering;
    2415:     const PetscInt *cellNum;
    2416:     PetscBool       useCone, useClosure;
    2417:     PetscSection    section;
    2418:     PetscSegBuffer  adjBuffer;
    2419:     PetscSF         sfPoint;

    2421:     PetscCall(DMConvert(dm, DMPLEX, &dm));
    2422:     PetscCall(DMPlexGetHeightStratum(dm, 0, &pStart, &pEnd));
    2423:     PetscCall(DMGetPointSF(dm, &sfPoint));
    2424:     PetscCall(PetscSFGetGraph(sfPoint, &nroots, NULL, NULL, NULL));
    2425:     /* Build adjacency graph via a section/segbuffer */
    2426:     PetscCall(PetscSectionCreate(PetscObjectComm((PetscObject)dm), &section));
    2427:     PetscCall(PetscSectionSetChart(section, pStart, pEnd));
    2428:     PetscCall(PetscSegBufferCreate(sizeof(PetscInt), 1000, &adjBuffer));
    2429:     /* Always use FVM adjacency to create partitioner graph */
    2430:     PetscCall(DMGetBasicAdjacency(dm, &useCone, &useClosure));
    2431:     PetscCall(DMSetBasicAdjacency(dm, PETSC_TRUE, PETSC_FALSE));
    2432:     PetscCall(DMPlexGetCellNumbering(dm, &cellNumbering));
    2433:     PetscCall(ISGetIndices(cellNumbering, &cellNum));
    2434:     for (n = 0, p = pStart; p < pEnd; p++) {
    2435:       /* Skip non-owned cells in parallel (ParMetis expects no overlap) */
    2436:       if (nroots > 0) {
    2437:         if (cellNum[p] < 0) continue;
    2438:       }
    2439:       adjSize = PETSC_DETERMINE;
    2440:       PetscCall(DMPlexGetAdjacency(dm, p, &adjSize, &adj));
    2441:       for (a = 0; a < adjSize; ++a) {
    2442:         const PetscInt point = adj[a];
    2443:         if (pStart <= point && point < pEnd) {
    2444:           PetscInt *PETSC_RESTRICT pBuf;
    2445:           PetscCall(PetscSectionAddDof(section, p, 1));
    2446:           PetscCall(PetscSegBufferGetInts(adjBuffer, 1, &pBuf));
    2447:           *pBuf = point;
    2448:         }
    2449:       }
    2450:       n++;
    2451:     }
    2452:     PetscCall(DMSetBasicAdjacency(dm, useCone, useClosure));
    2453:     /* Derive CSR graph from section/segbuffer */
    2454:     PetscCall(PetscSectionSetUp(section));
    2455:     PetscCall(PetscSectionGetStorageSize(section, &size));
    2456:     PetscCall(PetscMalloc1(n + 1, &xadj));
    2457:     for (idx = 0, p = pStart; p < pEnd; p++) {
    2458:       if (nroots > 0) {
    2459:         if (cellNum[p] < 0) continue;
    2460:       }
    2461:       PetscCall(PetscSectionGetOffset(section, p, &xadj[idx++]));
    2462:     }
    2463:     xadj[n] = size;
    2464:     PetscCall(PetscSegBufferExtractAlloc(adjBuffer, &adjncy));
    2465:     /* Clean up */
    2466:     PetscCall(PetscSegBufferDestroy(&adjBuffer));
    2467:     PetscCall(PetscSectionDestroy(&section));
    2468:     PetscCall(PetscFree(adj));
    2469:     graph->xadj   = xadj;
    2470:     graph->adjncy = adjncy;
    2471:   } else {
    2472:     Mat       A;
    2473:     PetscBool isseqaij, flg_row;

    2475:     PetscCall(MatISGetLocalMat(pc->pmat, &A));
    2476:     if (!A->rmap->N || !A->cmap->N) {
    2477:       PetscCall(PCBDDCGraphDestroy(&graph));
    2478:       PetscFunctionReturn(PETSC_SUCCESS);
    2479:     }
    2480:     PetscCall(PetscObjectBaseTypeCompare((PetscObject)A, MATSEQAIJ, &isseqaij));
    2481:     if (!isseqaij && filter) {
    2482:       PetscBool isseqdense;

    2484:       PetscCall(PetscObjectTypeCompare((PetscObject)A, MATSEQDENSE, &isseqdense));
    2485:       if (!isseqdense) {
    2486:         PetscCall(MatConvert(A, MATSEQAIJ, MAT_INITIAL_MATRIX, &B));
    2487:       } else { /* TODO: rectangular case and LDA */
    2488:         PetscScalar *array;
    2489:         PetscReal    chop = 1.e-6;

    2491:         PetscCall(MatDuplicate(A, MAT_COPY_VALUES, &B));
    2492:         PetscCall(MatDenseGetArray(B, &array));
    2493:         PetscCall(MatGetSize(B, &n, NULL));
    2494:         for (i = 0; i < n; i++) {
    2495:           PetscInt j;
    2496:           for (j = i + 1; j < n; j++) {
    2497:             PetscReal thresh = chop * (PetscAbsScalar(array[i * (n + 1)]) + PetscAbsScalar(array[j * (n + 1)]));
    2498:             if (PetscAbsScalar(array[i * n + j]) < thresh) array[i * n + j] = 0.;
    2499:             if (PetscAbsScalar(array[j * n + i]) < thresh) array[j * n + i] = 0.;
    2500:           }
    2501:         }
    2502:         PetscCall(MatDenseRestoreArray(B, &array));
    2503:         PetscCall(MatConvert(B, MATSEQAIJ, MAT_INPLACE_MATRIX, &B));
    2504:       }
    2505:     } else {
    2506:       PetscCall(PetscObjectReference((PetscObject)A));
    2507:       B = A;
    2508:     }
    2509:     PetscCall(MatGetRowIJ(B, 0, PETSC_TRUE, PETSC_FALSE, &n, (const PetscInt **)&xadj, (const PetscInt **)&adjncy, &flg_row));

    2511:     /* if filter is true, then removes entries lower than PETSC_SMALL in magnitude */
    2512:     if (filter) {
    2513:       PetscScalar *data;
    2514:       PetscInt     j, cum;

    2516:       PetscCall(PetscCalloc2(n + 1, &xadj_filtered, xadj[n], &adjncy_filtered));
    2517:       PetscCall(MatSeqAIJGetArray(B, &data));
    2518:       cum = 0;
    2519:       for (i = 0; i < n; i++) {
    2520:         PetscInt t;

    2522:         for (j = xadj[i]; j < xadj[i + 1]; j++) {
    2523:           if (PetscUnlikely(PetscAbsScalar(data[j]) < PETSC_SMALL)) continue;
    2524:           adjncy_filtered[cum + xadj_filtered[i]++] = adjncy[j];
    2525:         }
    2526:         t                = xadj_filtered[i];
    2527:         xadj_filtered[i] = cum;
    2528:         cum += t;
    2529:       }
    2530:       PetscCall(MatSeqAIJRestoreArray(B, &data));
    2531:       graph->xadj   = xadj_filtered;
    2532:       graph->adjncy = adjncy_filtered;
    2533:     } else {
    2534:       graph->xadj   = xadj;
    2535:       graph->adjncy = adjncy;
    2536:     }
    2537:   }
    2538:   /* compute local connected components using PCBDDCGraph */
    2539:   graph->seq_graph = PETSC_TRUE; /* analyze local connected components (i.e. disconnected subdomains) irrespective of dofs count */
    2540:   PetscCall(ISCreateStride(PETSC_COMM_SELF, n, 0, 1, &is_dummy));
    2541:   PetscCall(ISLocalToGlobalMappingCreateIS(is_dummy, &l2gmap_dummy));
    2542:   PetscCall(ISDestroy(&is_dummy));
    2543:   PetscCall(PCBDDCGraphInit(graph, l2gmap_dummy, n, PETSC_INT_MAX));
    2544:   PetscCall(ISLocalToGlobalMappingDestroy(&l2gmap_dummy));
    2545:   PetscCall(PCBDDCGraphSetUp(graph, 1, NULL, NULL, 0, NULL, NULL));
    2546:   PetscCall(PCBDDCGraphComputeConnectedComponents(graph));

    2548:   /* partial clean up */
    2549:   PetscCall(PetscFree2(xadj_filtered, adjncy_filtered));
    2550:   if (B) {
    2551:     PetscBool flg_row;
    2552:     PetscCall(MatRestoreRowIJ(B, 0, PETSC_TRUE, PETSC_FALSE, &n, (const PetscInt **)&xadj, (const PetscInt **)&adjncy, &flg_row));
    2553:     PetscCall(MatDestroy(&B));
    2554:   }
    2555:   if (isplex) {
    2556:     PetscCall(PetscFree(xadj));
    2557:     PetscCall(PetscFree(adjncy));
    2558:   }

    2560:   /* get back data */
    2561:   if (isplex) {
    2562:     if (ncc) *ncc = graph->ncc;
    2563:     if (cc || primalv) {
    2564:       Mat          A;
    2565:       PetscBT      btv, btvt, btvc;
    2566:       PetscSection subSection;
    2567:       PetscInt    *ids, cum, cump, *cids, *pids;
    2568:       PetscInt     dim, cStart, cEnd, fStart, fEnd, vStart, vEnd, pStart, pEnd;

    2570:       PetscCall(DMGetDimension(dm, &dim));
    2571:       PetscCall(DMPlexGetSubdomainSection(dm, &subSection));
    2572:       PetscCall(DMPlexGetHeightStratum(dm, 1, &fStart, &fEnd));
    2573:       PetscCall(DMPlexGetHeightStratum(dm, 0, &cStart, &cEnd));
    2574:       PetscCall(DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd));
    2575:       PetscCall(DMPlexGetChart(dm, &pStart, &pEnd));
    2576:       PetscCall(MatISGetLocalMat(pc->pmat, &A));
    2577:       PetscCall(PetscMalloc3(A->rmap->n, &ids, graph->ncc + 1, &cids, A->rmap->n, &pids));
    2578:       PetscCall(PetscBTCreate(A->rmap->n, &btv));
    2579:       PetscCall(PetscBTCreate(A->rmap->n, &btvt));
    2580:       PetscCall(PetscBTCreate(pEnd - pStart, &btvc));

    2582:       /* First see if we find corners for the subdomains, i.e. a vertex
    2583:          shared by at least dim subdomain boundary faces. This does not
    2584:          cover all the possible cases with simplices but it is enough
    2585:          for tensor cells */
    2586:       if (vStart != fStart && dim <= 3) {
    2587:         for (PetscInt c = cStart; c < cEnd; c++) {
    2588:           PetscInt        nf, cnt = 0, mcnt = dim, *cfaces;
    2589:           const PetscInt *faces;

    2591:           PetscCall(DMPlexGetConeSize(dm, c, &nf));
    2592:           PetscCall(DMGetWorkArray(dm, nf, MPIU_INT, &cfaces));
    2593:           PetscCall(DMPlexGetCone(dm, c, &faces));
    2594:           for (PetscInt f = 0; f < nf; f++) {
    2595:             PetscInt nc, ff;

    2597:             PetscCall(DMPlexGetSupportSize(dm, faces[f], &nc));
    2598:             PetscCall(DMPlexGetTreeParent(dm, faces[f], &ff, NULL));
    2599:             if (nc == 1 && faces[f] == ff) cfaces[cnt++] = faces[f];
    2600:           }
    2601:           if (cnt >= mcnt) {
    2602:             PetscInt size, *closure = NULL;

    2604:             PetscCall(DMPlexGetTransitiveClosure(dm, c, PETSC_TRUE, &size, &closure));
    2605:             for (PetscInt k = 0; k < 2 * size; k += 2) {
    2606:               PetscInt v = closure[k];
    2607:               if (v >= vStart && v < vEnd) {
    2608:                 PetscInt vsize, *vclosure = NULL;

    2610:                 cnt = 0;
    2611:                 PetscCall(DMPlexGetTransitiveClosure(dm, v, PETSC_FALSE, &vsize, &vclosure));
    2612:                 for (PetscInt vk = 0; vk < 2 * vsize; vk += 2) {
    2613:                   PetscInt f = vclosure[vk];
    2614:                   if (f >= fStart && f < fEnd) {
    2615:                     PetscInt  nc, ff;
    2616:                     PetscBool valid = PETSC_FALSE;

    2618:                     for (PetscInt fk = 0; fk < nf; fk++)
    2619:                       if (f == cfaces[fk]) valid = PETSC_TRUE;
    2620:                     if (!valid) continue;
    2621:                     PetscCall(DMPlexGetSupportSize(dm, f, &nc));
    2622:                     PetscCall(DMPlexGetTreeParent(dm, f, &ff, NULL));
    2623:                     if (nc == 1 && f == ff) cnt++;
    2624:                   }
    2625:                 }
    2626:                 if (cnt >= mcnt) PetscCall(PetscBTSet(btvc, v - pStart));
    2627:                 PetscCall(DMPlexRestoreTransitiveClosure(dm, v, PETSC_FALSE, &vsize, &vclosure));
    2628:               }
    2629:             }
    2630:             PetscCall(DMPlexRestoreTransitiveClosure(dm, c, PETSC_TRUE, &size, &closure));
    2631:           }
    2632:           PetscCall(DMRestoreWorkArray(dm, nf, MPIU_INT, &cfaces));
    2633:         }
    2634:       }

    2636:       cids[0] = 0;
    2637:       for (i = 0, cump = 0, cum = 0; i < graph->ncc; i++) {
    2638:         PetscInt j;

    2640:         PetscCall(PetscBTMemzero(A->rmap->n, btvt));
    2641:         for (j = graph->cptr[i]; j < graph->cptr[i + 1]; j++) {
    2642:           PetscInt k, size, *closure = NULL, cell = graph->queue[j];

    2644:           PetscCall(DMPlexGetTransitiveClosure(dm, cell, PETSC_TRUE, &size, &closure));
    2645:           for (k = 0; k < 2 * size; k += 2) {
    2646:             PetscInt s, pp, p = closure[k], off, dof, cdof;

    2648:             PetscCall(PetscSectionGetConstraintDof(subSection, p, &cdof));
    2649:             PetscCall(PetscSectionGetOffset(subSection, p, &off));
    2650:             PetscCall(PetscSectionGetDof(subSection, p, &dof));
    2651:             for (s = 0; s < dof - cdof; s++) {
    2652:               if (PetscBTLookupSet(btvt, off + s)) continue;
    2653:               if (PetscBTLookup(btvc, p - pStart)) pids[cump++] = off + s; /* subdomain corner */
    2654:               else if (!PetscBTLookup(btv, off + s)) ids[cum++] = off + s;
    2655:               else pids[cump++] = off + s; /* cross-vertex */
    2656:             }
    2657:             PetscCall(DMPlexGetTreeParent(dm, p, &pp, NULL));
    2658:             if (pp != p) {
    2659:               PetscCall(PetscSectionGetConstraintDof(subSection, pp, &cdof));
    2660:               PetscCall(PetscSectionGetOffset(subSection, pp, &off));
    2661:               PetscCall(PetscSectionGetDof(subSection, pp, &dof));
    2662:               for (s = 0; s < dof - cdof; s++) {
    2663:                 if (PetscBTLookupSet(btvt, off + s)) continue;
    2664:                 if (PetscBTLookup(btvc, pp - pStart)) pids[cump++] = off + s; /* subdomain corner */
    2665:                 else if (!PetscBTLookup(btv, off + s)) ids[cum++] = off + s;
    2666:                 else pids[cump++] = off + s; /* cross-vertex */
    2667:               }
    2668:             }
    2669:           }
    2670:           PetscCall(DMPlexRestoreTransitiveClosure(dm, cell, PETSC_TRUE, &size, &closure));
    2671:         }
    2672:         cids[i + 1] = cum;
    2673:         /* mark dofs as already assigned */
    2674:         for (j = cids[i]; j < cids[i + 1]; j++) PetscCall(PetscBTSet(btv, ids[j]));
    2675:       }
    2676:       if (cc) {
    2677:         PetscCall(PetscMalloc1(graph->ncc, &cc_n));
    2678:         for (i = 0; i < graph->ncc; i++) PetscCall(ISCreateGeneral(PETSC_COMM_SELF, cids[i + 1] - cids[i], ids + cids[i], PETSC_COPY_VALUES, &cc_n[i]));
    2679:         *cc = cc_n;
    2680:       }
    2681:       if (primalv) PetscCall(ISCreateGeneral(PetscObjectComm((PetscObject)pc), cump, pids, PETSC_COPY_VALUES, primalv));
    2682:       PetscCall(PetscFree3(ids, cids, pids));
    2683:       PetscCall(PetscBTDestroy(&btv));
    2684:       PetscCall(PetscBTDestroy(&btvt));
    2685:       PetscCall(PetscBTDestroy(&btvc));
    2686:       PetscCall(DMDestroy(&dm));
    2687:     }
    2688:   } else {
    2689:     if (ncc) *ncc = graph->ncc;
    2690:     if (cc) {
    2691:       PetscCall(PetscMalloc1(graph->ncc, &cc_n));
    2692:       for (i = 0; i < graph->ncc; i++) PetscCall(ISCreateGeneral(PETSC_COMM_SELF, graph->cptr[i + 1] - graph->cptr[i], graph->queue + graph->cptr[i], PETSC_COPY_VALUES, &cc_n[i]));
    2693:       *cc = cc_n;
    2694:     }
    2695:   }
    2696:   /* clean up graph */
    2697:   graph->xadj   = NULL;
    2698:   graph->adjncy = NULL;
    2699:   PetscCall(PCBDDCGraphDestroy(&graph));
    2700:   PetscFunctionReturn(PETSC_SUCCESS);
    2701: }

    2703: PetscErrorCode PCBDDCBenignCheck(PC pc, IS zerodiag)
    2704: {
    2705:   PC_BDDC *pcbddc = (PC_BDDC *)pc->data;
    2706:   PC_IS   *pcis   = (PC_IS *)pc->data;
    2707:   IS       dirIS  = NULL;
    2708:   PetscInt i;

    2710:   PetscFunctionBegin;
    2711:   PetscCall(PCBDDCGraphGetDirichletDofs(pcbddc->mat_graph, &dirIS));
    2712:   if (zerodiag) {
    2713:     Mat             A;
    2714:     Vec             vec3_N;
    2715:     PetscScalar    *vals;
    2716:     const PetscInt *idxs;
    2717:     PetscInt        nz, *count;

    2719:     /* p0 */
    2720:     PetscCall(VecSet(pcis->vec1_N, 0.));
    2721:     PetscCall(PetscMalloc1(pcis->n, &vals));
    2722:     PetscCall(ISGetLocalSize(zerodiag, &nz));
    2723:     PetscCall(ISGetIndices(zerodiag, &idxs));
    2724:     for (i = 0; i < nz; i++) vals[i] = 1.;
    2725:     PetscCall(VecSetValues(pcis->vec1_N, nz, idxs, vals, INSERT_VALUES));
    2726:     PetscCall(VecAssemblyBegin(pcis->vec1_N));
    2727:     PetscCall(VecAssemblyEnd(pcis->vec1_N));
    2728:     /* v_I */
    2729:     PetscCall(VecSetRandom(pcis->vec2_N, NULL));
    2730:     for (i = 0; i < nz; i++) vals[i] = 0.;
    2731:     PetscCall(VecSetValues(pcis->vec2_N, nz, idxs, vals, INSERT_VALUES));
    2732:     PetscCall(ISRestoreIndices(zerodiag, &idxs));
    2733:     PetscCall(ISGetIndices(pcis->is_B_local, &idxs));
    2734:     for (i = 0; i < pcis->n_B; i++) vals[i] = 0.;
    2735:     PetscCall(VecSetValues(pcis->vec2_N, pcis->n_B, idxs, vals, INSERT_VALUES));
    2736:     PetscCall(ISRestoreIndices(pcis->is_B_local, &idxs));
    2737:     if (dirIS) {
    2738:       PetscInt n;

    2740:       PetscCall(ISGetLocalSize(dirIS, &n));
    2741:       PetscCall(ISGetIndices(dirIS, &idxs));
    2742:       for (i = 0; i < n; i++) vals[i] = 0.;
    2743:       PetscCall(VecSetValues(pcis->vec2_N, n, idxs, vals, INSERT_VALUES));
    2744:       PetscCall(ISRestoreIndices(dirIS, &idxs));
    2745:     }
    2746:     PetscCall(VecAssemblyBegin(pcis->vec2_N));
    2747:     PetscCall(VecAssemblyEnd(pcis->vec2_N));
    2748:     PetscCall(VecDuplicate(pcis->vec1_N, &vec3_N));
    2749:     PetscCall(VecSet(vec3_N, 0.));
    2750:     PetscCall(MatISGetLocalMat(pc->pmat, &A));
    2751:     PetscCall(MatMult(A, pcis->vec1_N, vec3_N));
    2752:     PetscCall(VecDot(vec3_N, pcis->vec2_N, &vals[0]));
    2753:     PetscCheck(PetscAbsScalar(vals[0]) <= 1.e-1, PETSC_COMM_SELF, PETSC_ERR_SUP, "Benign trick can not be applied! b(v_I,p_0) = %1.6e (should be numerically 0.)", (double)PetscAbsScalar(vals[0]));
    2754:     PetscCall(PetscFree(vals));
    2755:     PetscCall(VecDestroy(&vec3_N));

    2757:     /* there should not be any pressure dofs lying on the interface */
    2758:     PetscCall(PetscCalloc1(pcis->n, &count));
    2759:     PetscCall(ISGetIndices(pcis->is_B_local, &idxs));
    2760:     for (i = 0; i < pcis->n_B; i++) count[idxs[i]]++;
    2761:     PetscCall(ISRestoreIndices(pcis->is_B_local, &idxs));
    2762:     PetscCall(ISGetIndices(zerodiag, &idxs));
    2763:     for (i = 0; i < nz; i++) PetscCheck(!count[idxs[i]], PETSC_COMM_SELF, PETSC_ERR_SUP, "Benign trick can not be applied! pressure dof %" PetscInt_FMT " is an interface dof", idxs[i]);
    2764:     PetscCall(ISRestoreIndices(zerodiag, &idxs));
    2765:     PetscCall(PetscFree(count));
    2766:   }
    2767:   PetscCall(ISDestroy(&dirIS));

    2769:   /* check PCBDDCBenignGetOrSetP0 */
    2770:   PetscCall(VecSetRandom(pcis->vec1_global, NULL));
    2771:   for (i = 0; i < pcbddc->benign_n; i++) pcbddc->benign_p0[i] = -PetscGlobalRank - i;
    2772:   PetscCall(PCBDDCBenignGetOrSetP0(pc, pcis->vec1_global, PETSC_FALSE));
    2773:   for (i = 0; i < pcbddc->benign_n; i++) pcbddc->benign_p0[i] = 1;
    2774:   PetscCall(PCBDDCBenignGetOrSetP0(pc, pcis->vec1_global, PETSC_TRUE));
    2775:   for (i = 0; i < pcbddc->benign_n; i++) {
    2776:     PetscInt val = PetscRealPart(pcbddc->benign_p0[i]);
    2777:     PetscCheck(val == -PetscGlobalRank - i, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Error testing PCBDDCBenignGetOrSetP0! Found %g at %" PetscInt_FMT " instead of %g", (double)PetscRealPart(pcbddc->benign_p0[i]), i, (double)(-PetscGlobalRank - i));
    2778:   }
    2779:   PetscFunctionReturn(PETSC_SUCCESS);
    2780: }

    2782: PetscErrorCode PCBDDCBenignDetectSaddlePoint(PC pc, PetscBool reuse, IS *zerodiaglocal)
    2783: {
    2784:   PC_BDDC  *pcbddc    = (PC_BDDC *)pc->data;
    2785:   Mat_IS   *matis     = (Mat_IS *)pc->pmat->data;
    2786:   IS        pressures = NULL, zerodiag = NULL, *bzerodiag = NULL, zerodiag_save, *zerodiag_subs;
    2787:   PetscInt  nz, n, benign_n, bsp = 1;
    2788:   PetscInt *interior_dofs, n_interior_dofs, nneu;
    2789:   PetscBool sorted, have_null, has_null_pressures, recompute_zerodiag, checkb;

    2791:   PetscFunctionBegin;
    2792:   if (reuse) goto project_b0;
    2793:   PetscCall(PetscSFDestroy(&pcbddc->benign_sf));
    2794:   PetscCall(MatDestroy(&pcbddc->benign_B0));
    2795:   for (n = 0; n < pcbddc->benign_n; n++) PetscCall(ISDestroy(&pcbddc->benign_zerodiag_subs[n]));
    2796:   PetscCall(PetscFree(pcbddc->benign_zerodiag_subs));
    2797:   has_null_pressures = PETSC_TRUE;
    2798:   have_null          = PETSC_TRUE;
    2799:   /* if a local information on dofs is present, gets pressure dofs from command line (uses the last field is not provided)
    2800:      Without local information, it uses only the zerodiagonal dofs (ok if the pressure block is all zero and it is a scalar field)
    2801:      Checks if all the pressure dofs in each subdomain have a zero diagonal
    2802:      If not, a change of basis on pressures is not needed
    2803:      since the local Schur complements are already SPD
    2804:   */
    2805:   if (pcbddc->n_ISForDofsLocal) {
    2806:     IS        iP = NULL;
    2807:     PetscInt  p, *pp;
    2808:     PetscBool flg, blocked = PETSC_FALSE;

    2810:     PetscCall(PetscMalloc1(pcbddc->n_ISForDofsLocal, &pp));
    2811:     n = pcbddc->n_ISForDofsLocal;
    2812:     PetscOptionsBegin(PetscObjectComm((PetscObject)pc), ((PetscObject)pc)->prefix, "BDDC benign options", "PC");
    2813:     PetscCall(PetscOptionsIntArray("-pc_bddc_pressure_field", "Field id for pressures", NULL, pp, &n, &flg));
    2814:     PetscCall(PetscOptionsBool("-pc_bddc_pressure_blocked", "Use blocked pressure fields", NULL, blocked, &blocked, NULL));
    2815:     PetscOptionsEnd();
    2816:     if (!flg) {
    2817:       n     = 1;
    2818:       pp[0] = pcbddc->n_ISForDofsLocal - 1;
    2819:     }

    2821:     bsp = 0;
    2822:     for (p = 0; p < n; p++) {
    2823:       PetscInt bs = 1;

    2825:       PetscCheck(pp[p] >= 0 && pp[p] < pcbddc->n_ISForDofsLocal, PetscObjectComm((PetscObject)pc), PETSC_ERR_USER, "Invalid field id for pressures %" PetscInt_FMT, pp[p]);
    2826:       if (blocked) PetscCall(ISGetBlockSize(pcbddc->ISForDofsLocal[pp[p]], &bs));
    2827:       bsp += bs;
    2828:     }
    2829:     PetscCall(PetscMalloc1(bsp, &bzerodiag));
    2830:     bsp = 0;
    2831:     for (p = 0; p < n; p++) {
    2832:       const PetscInt *idxs;
    2833:       PetscInt        b, bs = 1, npl, *bidxs;

    2835:       if (blocked) PetscCall(ISGetBlockSize(pcbddc->ISForDofsLocal[pp[p]], &bs));
    2836:       PetscCall(ISGetLocalSize(pcbddc->ISForDofsLocal[pp[p]], &npl));
    2837:       PetscCall(ISGetIndices(pcbddc->ISForDofsLocal[pp[p]], &idxs));
    2838:       PetscCall(PetscMalloc1(npl / bs, &bidxs));
    2839:       for (b = 0; b < bs; b++) {
    2840:         PetscInt i;

    2842:         for (i = 0; i < npl / bs; i++) bidxs[i] = idxs[bs * i + b];
    2843:         PetscCall(ISCreateGeneral(PETSC_COMM_SELF, npl / bs, bidxs, PETSC_COPY_VALUES, &bzerodiag[bsp]));
    2844:         bsp++;
    2845:       }
    2846:       PetscCall(PetscFree(bidxs));
    2847:       PetscCall(ISRestoreIndices(pcbddc->ISForDofsLocal[pp[p]], &idxs));
    2848:     }
    2849:     PetscCall(ISConcatenate(PETSC_COMM_SELF, bsp, bzerodiag, &pressures));

    2851:     /* remove zeroed out pressures if we are setting up a BDDC solver for a saddle-point FETI-DP */
    2852:     PetscCall(PetscObjectQuery((PetscObject)pc, "__KSPFETIDP_lP", (PetscObject *)&iP));
    2853:     if (iP) {
    2854:       IS newpressures;

    2856:       PetscCall(ISDifference(pressures, iP, &newpressures));
    2857:       PetscCall(ISDestroy(&pressures));
    2858:       pressures = newpressures;
    2859:     }
    2860:     PetscCall(ISSorted(pressures, &sorted));
    2861:     if (!sorted) PetscCall(ISSort(pressures));
    2862:     PetscCall(PetscFree(pp));
    2863:   }

    2865:   /* pcis has not been setup yet, so get the local size from the subdomain matrix */
    2866:   PetscCall(MatGetLocalSize(pcbddc->local_mat, &n, NULL));
    2867:   if (!n) pcbddc->benign_change_explicit = PETSC_TRUE;
    2868:   PetscCall(MatFindZeroDiagonals(pcbddc->local_mat, &zerodiag));
    2869:   PetscCall(ISSorted(zerodiag, &sorted));
    2870:   if (!sorted) PetscCall(ISSort(zerodiag));
    2871:   PetscCall(PetscObjectReference((PetscObject)zerodiag));
    2872:   zerodiag_save = zerodiag;
    2873:   PetscCall(ISGetLocalSize(zerodiag, &nz));
    2874:   if (!nz) {
    2875:     if (n) have_null = PETSC_FALSE;
    2876:     has_null_pressures = PETSC_FALSE;
    2877:     PetscCall(ISDestroy(&zerodiag));
    2878:   }
    2879:   recompute_zerodiag = PETSC_FALSE;

    2881:   /* in case disconnected subdomains info is present, split the pressures accordingly (otherwise the benign trick could fail) */
    2882:   zerodiag_subs   = NULL;
    2883:   benign_n        = 0;
    2884:   n_interior_dofs = 0;
    2885:   interior_dofs   = NULL;
    2886:   nneu            = 0;
    2887:   if (pcbddc->NeumannBoundariesLocal) PetscCall(ISGetLocalSize(pcbddc->NeumannBoundariesLocal, &nneu));
    2888:   checkb = (PetscBool)(!pcbddc->NeumannBoundariesLocal || pcbddc->current_level);
    2889:   if (checkb) { /* need to compute interior nodes */
    2890:     PetscInt               n, i;
    2891:     PetscInt              *count;
    2892:     ISLocalToGlobalMapping mapping;

    2894:     PetscCall(MatISGetLocalToGlobalMapping(pc->pmat, &mapping, NULL));
    2895:     PetscCall(ISLocalToGlobalMappingGetNodeInfo(mapping, &n, &count, NULL));
    2896:     PetscCall(PetscMalloc1(n, &interior_dofs));
    2897:     for (i = 0; i < n; i++)
    2898:       if (count[i] < 2) interior_dofs[n_interior_dofs++] = i;
    2899:     PetscCall(ISLocalToGlobalMappingRestoreNodeInfo(mapping, &n, &count, NULL));
    2900:   }
    2901:   if (has_null_pressures) {
    2902:     IS             *subs;
    2903:     PetscInt        nsubs, i, j, nl;
    2904:     const PetscInt *idxs;
    2905:     PetscScalar    *array;
    2906:     Vec            *work;

    2908:     subs  = pcbddc->local_subs;
    2909:     nsubs = pcbddc->n_local_subs;
    2910:     /* these vectors are needed to check if the constant on pressures is in the kernel of the local operator B (i.e. B(v_I,p0) should be zero) */
    2911:     if (checkb) {
    2912:       PetscCall(VecDuplicateVecs(matis->y, 2, &work));
    2913:       PetscCall(ISGetLocalSize(zerodiag, &nl));
    2914:       PetscCall(ISGetIndices(zerodiag, &idxs));
    2915:       /* work[0] = 1_p */
    2916:       PetscCall(VecSet(work[0], 0.));
    2917:       PetscCall(VecGetArray(work[0], &array));
    2918:       for (j = 0; j < nl; j++) array[idxs[j]] = 1.;
    2919:       PetscCall(VecRestoreArray(work[0], &array));
    2920:       /* work[0] = 1_v */
    2921:       PetscCall(VecSet(work[1], 1.));
    2922:       PetscCall(VecGetArray(work[1], &array));
    2923:       for (j = 0; j < nl; j++) array[idxs[j]] = 0.;
    2924:       PetscCall(VecRestoreArray(work[1], &array));
    2925:       PetscCall(ISRestoreIndices(zerodiag, &idxs));
    2926:     }

    2928:     if (nsubs > 1 || bsp > 1) {
    2929:       IS      *is;
    2930:       PetscInt b, totb;

    2932:       totb  = bsp;
    2933:       is    = bsp > 1 ? bzerodiag : &zerodiag;
    2934:       nsubs = PetscMax(nsubs, 1);
    2935:       PetscCall(PetscCalloc1(nsubs * totb, &zerodiag_subs));
    2936:       for (b = 0; b < totb; b++) {
    2937:         for (i = 0; i < nsubs; i++) {
    2938:           ISLocalToGlobalMapping l2g;
    2939:           IS                     t_zerodiag_subs;
    2940:           PetscInt               nl;

    2942:           if (subs) {
    2943:             PetscCall(ISLocalToGlobalMappingCreateIS(subs[i], &l2g));
    2944:           } else {
    2945:             IS tis;

    2947:             PetscCall(MatGetLocalSize(pcbddc->local_mat, &nl, NULL));
    2948:             PetscCall(ISCreateStride(PETSC_COMM_SELF, nl, 0, 1, &tis));
    2949:             PetscCall(ISLocalToGlobalMappingCreateIS(tis, &l2g));
    2950:             PetscCall(ISDestroy(&tis));
    2951:           }
    2952:           PetscCall(ISGlobalToLocalMappingApplyIS(l2g, IS_GTOLM_DROP, is[b], &t_zerodiag_subs));
    2953:           PetscCall(ISGetLocalSize(t_zerodiag_subs, &nl));
    2954:           if (nl) {
    2955:             PetscBool valid = PETSC_TRUE;

    2957:             if (checkb) {
    2958:               PetscCall(VecSet(matis->x, 0));
    2959:               PetscCall(ISGetLocalSize(subs[i], &nl));
    2960:               PetscCall(ISGetIndices(subs[i], &idxs));
    2961:               PetscCall(VecGetArray(matis->x, &array));
    2962:               for (j = 0; j < nl; j++) array[idxs[j]] = 1.;
    2963:               PetscCall(VecRestoreArray(matis->x, &array));
    2964:               PetscCall(ISRestoreIndices(subs[i], &idxs));
    2965:               PetscCall(VecPointwiseMult(matis->x, work[0], matis->x));
    2966:               PetscCall(MatMult(matis->A, matis->x, matis->y));
    2967:               PetscCall(VecPointwiseMult(matis->y, work[1], matis->y));
    2968:               PetscCall(VecGetArray(matis->y, &array));
    2969:               for (j = 0; j < n_interior_dofs; j++) {
    2970:                 if (PetscAbsScalar(array[interior_dofs[j]]) > PETSC_SMALL) {
    2971:                   valid = PETSC_FALSE;
    2972:                   break;
    2973:                 }
    2974:               }
    2975:               PetscCall(VecRestoreArray(matis->y, &array));
    2976:             }
    2977:             if (valid && nneu) {
    2978:               const PetscInt *idxs;
    2979:               PetscInt        nzb;

    2981:               PetscCall(ISGetIndices(pcbddc->NeumannBoundariesLocal, &idxs));
    2982:               PetscCall(ISGlobalToLocalMappingApply(l2g, IS_GTOLM_DROP, nneu, idxs, &nzb, NULL));
    2983:               PetscCall(ISRestoreIndices(pcbddc->NeumannBoundariesLocal, &idxs));
    2984:               if (nzb) valid = PETSC_FALSE;
    2985:             }
    2986:             if (valid && pressures) {
    2987:               IS       t_pressure_subs, tmp;
    2988:               PetscInt i1, i2;

    2990:               PetscCall(ISGlobalToLocalMappingApplyIS(l2g, IS_GTOLM_DROP, pressures, &t_pressure_subs));
    2991:               PetscCall(ISEmbed(t_zerodiag_subs, t_pressure_subs, PETSC_TRUE, &tmp));
    2992:               PetscCall(ISGetLocalSize(tmp, &i1));
    2993:               PetscCall(ISGetLocalSize(t_zerodiag_subs, &i2));
    2994:               if (i2 != i1) valid = PETSC_FALSE;
    2995:               PetscCall(ISDestroy(&t_pressure_subs));
    2996:               PetscCall(ISDestroy(&tmp));
    2997:             }
    2998:             if (valid) {
    2999:               PetscCall(ISLocalToGlobalMappingApplyIS(l2g, t_zerodiag_subs, &zerodiag_subs[benign_n]));
    3000:               benign_n++;
    3001:             } else recompute_zerodiag = PETSC_TRUE;
    3002:           }
    3003:           PetscCall(ISDestroy(&t_zerodiag_subs));
    3004:           PetscCall(ISLocalToGlobalMappingDestroy(&l2g));
    3005:         }
    3006:       }
    3007:     } else { /* there's just one subdomain (or zero if they have not been detected */
    3008:       PetscBool valid = PETSC_TRUE;

    3010:       if (nneu) valid = PETSC_FALSE;
    3011:       if (valid && pressures) PetscCall(ISEqual(pressures, zerodiag, &valid));
    3012:       if (valid && checkb) {
    3013:         PetscCall(MatMult(matis->A, work[0], matis->x));
    3014:         PetscCall(VecPointwiseMult(matis->x, work[1], matis->x));
    3015:         PetscCall(VecGetArray(matis->x, &array));
    3016:         for (j = 0; j < n_interior_dofs; j++) {
    3017:           if (PetscAbsScalar(array[interior_dofs[j]]) > PETSC_SMALL) {
    3018:             valid = PETSC_FALSE;
    3019:             break;
    3020:           }
    3021:         }
    3022:         PetscCall(VecRestoreArray(matis->x, &array));
    3023:       }
    3024:       if (valid) {
    3025:         benign_n = 1;
    3026:         PetscCall(PetscMalloc1(benign_n, &zerodiag_subs));
    3027:         PetscCall(PetscObjectReference((PetscObject)zerodiag));
    3028:         zerodiag_subs[0] = zerodiag;
    3029:       }
    3030:     }
    3031:     if (checkb) PetscCall(VecDestroyVecs(2, &work));
    3032:   }
    3033:   PetscCall(PetscFree(interior_dofs));

    3035:   if (!benign_n) {
    3036:     PetscInt n;

    3038:     PetscCall(ISDestroy(&zerodiag));
    3039:     recompute_zerodiag = PETSC_FALSE;
    3040:     PetscCall(MatGetLocalSize(pcbddc->local_mat, &n, NULL));
    3041:     if (n) have_null = PETSC_FALSE;
    3042:   }

    3044:   /* final check for null pressures */
    3045:   if (zerodiag && pressures) PetscCall(ISEqual(pressures, zerodiag, &have_null));

    3047:   if (recompute_zerodiag) {
    3048:     PetscCall(ISDestroy(&zerodiag));
    3049:     if (benign_n == 1) {
    3050:       PetscCall(PetscObjectReference((PetscObject)zerodiag_subs[0]));
    3051:       zerodiag = zerodiag_subs[0];
    3052:     } else {
    3053:       PetscInt i, nzn, *new_idxs;

    3055:       nzn = 0;
    3056:       for (i = 0; i < benign_n; i++) {
    3057:         PetscInt ns;
    3058:         PetscCall(ISGetLocalSize(zerodiag_subs[i], &ns));
    3059:         nzn += ns;
    3060:       }
    3061:       PetscCall(PetscMalloc1(nzn, &new_idxs));
    3062:       nzn = 0;
    3063:       for (i = 0; i < benign_n; i++) {
    3064:         PetscInt ns, *idxs;
    3065:         PetscCall(ISGetLocalSize(zerodiag_subs[i], &ns));
    3066:         PetscCall(ISGetIndices(zerodiag_subs[i], (const PetscInt **)&idxs));
    3067:         PetscCall(PetscArraycpy(new_idxs + nzn, idxs, ns));
    3068:         PetscCall(ISRestoreIndices(zerodiag_subs[i], (const PetscInt **)&idxs));
    3069:         nzn += ns;
    3070:       }
    3071:       PetscCall(PetscSortInt(nzn, new_idxs));
    3072:       PetscCall(ISCreateGeneral(PETSC_COMM_SELF, nzn, new_idxs, PETSC_OWN_POINTER, &zerodiag));
    3073:     }
    3074:     have_null = PETSC_FALSE;
    3075:   }

    3077:   /* determines if the coarse solver will be singular or not */
    3078:   PetscCallMPI(MPIU_Allreduce(&have_null, &pcbddc->benign_null, 1, MPI_C_BOOL, MPI_LAND, PetscObjectComm((PetscObject)pc)));

    3080:   /* Prepare matrix to compute no-net-flux */
    3081:   if (pcbddc->compute_nonetflux && !pcbddc->divudotp) {
    3082:     Mat                    A, loc_divudotp;
    3083:     ISLocalToGlobalMapping rl2g, cl2g, l2gmap;
    3084:     IS                     row, col, isused = NULL;
    3085:     PetscInt               M, N, n, st, n_isused;

    3087:     if (pressures) {
    3088:       isused = pressures;
    3089:     } else {
    3090:       isused = zerodiag_save;
    3091:     }
    3092:     PetscCall(MatISGetLocalToGlobalMapping(pc->pmat, &l2gmap, NULL));
    3093:     PetscCall(MatISGetLocalMat(pc->pmat, &A));
    3094:     PetscCall(MatGetLocalSize(A, &n, NULL));
    3095:     PetscCheck(isused || (n == 0), PETSC_COMM_SELF, PETSC_ERR_USER, "Don't know how to extract div u dot p! Please provide the pressure field");
    3096:     n_isused = 0;
    3097:     if (isused) PetscCall(ISGetLocalSize(isused, &n_isused));
    3098:     PetscCallMPI(MPI_Scan(&n_isused, &st, 1, MPIU_INT, MPI_SUM, PetscObjectComm((PetscObject)pc)));
    3099:     st = st - n_isused;
    3100:     if (n) {
    3101:       const PetscInt *gidxs;

    3103:       PetscCall(MatCreateSubMatrix(A, isused, NULL, MAT_INITIAL_MATRIX, &loc_divudotp));
    3104:       PetscCall(ISLocalToGlobalMappingGetIndices(l2gmap, &gidxs));
    3105:       /* TODO: extend ISCreateStride with st = PETSC_DECIDE */
    3106:       PetscCall(ISCreateStride(PetscObjectComm((PetscObject)pc), n_isused, st, 1, &row));
    3107:       PetscCall(ISCreateGeneral(PetscObjectComm((PetscObject)pc), n, gidxs, PETSC_COPY_VALUES, &col));
    3108:       PetscCall(ISLocalToGlobalMappingRestoreIndices(l2gmap, &gidxs));
    3109:     } else {
    3110:       PetscCall(MatCreateSeqAIJ(PETSC_COMM_SELF, 0, 0, 1, NULL, &loc_divudotp));
    3111:       PetscCall(ISCreateStride(PetscObjectComm((PetscObject)pc), n_isused, st, 1, &row));
    3112:       PetscCall(ISCreateGeneral(PetscObjectComm((PetscObject)pc), 0, NULL, PETSC_COPY_VALUES, &col));
    3113:     }
    3114:     PetscCall(MatGetSize(pc->pmat, NULL, &N));
    3115:     PetscCall(ISGetSize(row, &M));
    3116:     PetscCall(ISLocalToGlobalMappingCreateIS(row, &rl2g));
    3117:     PetscCall(ISLocalToGlobalMappingCreateIS(col, &cl2g));
    3118:     PetscCall(ISDestroy(&row));
    3119:     PetscCall(ISDestroy(&col));
    3120:     PetscCall(MatCreate(PetscObjectComm((PetscObject)pc), &pcbddc->divudotp));
    3121:     PetscCall(MatSetType(pcbddc->divudotp, MATIS));
    3122:     PetscCall(MatSetSizes(pcbddc->divudotp, PETSC_DECIDE, PETSC_DECIDE, M, N));
    3123:     PetscCall(MatSetLocalToGlobalMapping(pcbddc->divudotp, rl2g, cl2g));
    3124:     PetscCall(ISLocalToGlobalMappingDestroy(&rl2g));
    3125:     PetscCall(ISLocalToGlobalMappingDestroy(&cl2g));
    3126:     PetscCall(MatISSetLocalMat(pcbddc->divudotp, loc_divudotp));
    3127:     PetscCall(MatDestroy(&loc_divudotp));
    3128:     PetscCall(MatAssemblyBegin(pcbddc->divudotp, MAT_FINAL_ASSEMBLY));
    3129:     PetscCall(MatAssemblyEnd(pcbddc->divudotp, MAT_FINAL_ASSEMBLY));
    3130:   }
    3131:   PetscCall(ISDestroy(&zerodiag_save));
    3132:   PetscCall(ISDestroy(&pressures));
    3133:   if (bzerodiag) {
    3134:     PetscInt i;

    3136:     for (i = 0; i < bsp; i++) PetscCall(ISDestroy(&bzerodiag[i]));
    3137:     PetscCall(PetscFree(bzerodiag));
    3138:   }
    3139:   pcbddc->benign_n             = benign_n;
    3140:   pcbddc->benign_zerodiag_subs = zerodiag_subs;

    3142:   /* determines if the problem has subdomains with 0 pressure block */
    3143:   have_null = (PetscBool)(!!pcbddc->benign_n);
    3144:   PetscCallMPI(MPIU_Allreduce(&have_null, &pcbddc->benign_have_null, 1, MPI_C_BOOL, MPI_LOR, PetscObjectComm((PetscObject)pc)));

    3146: project_b0:
    3147:   PetscCall(MatGetLocalSize(pcbddc->local_mat, &n, NULL));
    3148:   /* change of basis and p0 dofs */
    3149:   if (pcbddc->benign_n) {
    3150:     PetscInt i, s, *nnz;

    3152:     /* local change of basis for pressures */
    3153:     PetscCall(MatDestroy(&pcbddc->benign_change));
    3154:     PetscCall(MatCreate(PetscObjectComm((PetscObject)pcbddc->local_mat), &pcbddc->benign_change));
    3155:     PetscCall(MatSetType(pcbddc->benign_change, MATAIJ));
    3156:     PetscCall(MatSetSizes(pcbddc->benign_change, n, n, PETSC_DECIDE, PETSC_DECIDE));
    3157:     PetscCall(PetscMalloc1(n, &nnz));
    3158:     for (i = 0; i < n; i++) nnz[i] = 1; /* defaults to identity */
    3159:     for (i = 0; i < pcbddc->benign_n; i++) {
    3160:       const PetscInt *idxs;
    3161:       PetscInt        nzs, j;

    3163:       PetscCall(ISGetLocalSize(pcbddc->benign_zerodiag_subs[i], &nzs));
    3164:       PetscCall(ISGetIndices(pcbddc->benign_zerodiag_subs[i], &idxs));
    3165:       for (j = 0; j < nzs - 1; j++) nnz[idxs[j]] = 2; /* change on pressures */
    3166:       nnz[idxs[nzs - 1]] = nzs;                       /* last local pressure dof in subdomain */
    3167:       PetscCall(ISRestoreIndices(pcbddc->benign_zerodiag_subs[i], &idxs));
    3168:     }
    3169:     PetscCall(MatSeqAIJSetPreallocation(pcbddc->benign_change, 0, nnz));
    3170:     PetscCall(MatSetOption(pcbddc->benign_change, MAT_NEW_NONZERO_ALLOCATION_ERR, PETSC_TRUE));
    3171:     PetscCall(PetscFree(nnz));
    3172:     /* set identity by default */
    3173:     for (i = 0; i < n; i++) PetscCall(MatSetValue(pcbddc->benign_change, i, i, 1., INSERT_VALUES));
    3174:     PetscCall(PetscFree3(pcbddc->benign_p0_lidx, pcbddc->benign_p0_gidx, pcbddc->benign_p0));
    3175:     PetscCall(PetscMalloc3(pcbddc->benign_n, &pcbddc->benign_p0_lidx, pcbddc->benign_n, &pcbddc->benign_p0_gidx, pcbddc->benign_n, &pcbddc->benign_p0));
    3176:     /* set change on pressures */
    3177:     for (s = 0; s < pcbddc->benign_n; s++) {
    3178:       PetscScalar    *array;
    3179:       const PetscInt *idxs;
    3180:       PetscInt        nzs;

    3182:       PetscCall(ISGetLocalSize(pcbddc->benign_zerodiag_subs[s], &nzs));
    3183:       PetscCall(ISGetIndices(pcbddc->benign_zerodiag_subs[s], &idxs));
    3184:       for (i = 0; i < nzs - 1; i++) {
    3185:         PetscScalar vals[2];
    3186:         PetscInt    cols[2];

    3188:         cols[0] = idxs[i];
    3189:         cols[1] = idxs[nzs - 1];
    3190:         vals[0] = 1.;
    3191:         vals[1] = 1.;
    3192:         PetscCall(MatSetValues(pcbddc->benign_change, 1, cols, 2, cols, vals, INSERT_VALUES));
    3193:       }
    3194:       PetscCall(PetscMalloc1(nzs, &array));
    3195:       for (i = 0; i < nzs - 1; i++) array[i] = -1.;
    3196:       array[nzs - 1] = 1.;
    3197:       PetscCall(MatSetValues(pcbddc->benign_change, 1, idxs + nzs - 1, nzs, idxs, array, INSERT_VALUES));
    3198:       /* store local idxs for p0 */
    3199:       pcbddc->benign_p0_lidx[s] = idxs[nzs - 1];
    3200:       PetscCall(ISRestoreIndices(pcbddc->benign_zerodiag_subs[s], &idxs));
    3201:       PetscCall(PetscFree(array));
    3202:     }
    3203:     PetscCall(MatAssemblyBegin(pcbddc->benign_change, MAT_FINAL_ASSEMBLY));
    3204:     PetscCall(MatAssemblyEnd(pcbddc->benign_change, MAT_FINAL_ASSEMBLY));

    3206:     /* project if needed */
    3207:     if (pcbddc->benign_change_explicit) {
    3208:       Mat M;

    3210:       PetscCall(MatPtAP(pcbddc->local_mat, pcbddc->benign_change, MAT_INITIAL_MATRIX, 2.0, &M));
    3211:       PetscCall(MatDestroy(&pcbddc->local_mat));
    3212:       PetscCall(MatSeqAIJCompress(M, &pcbddc->local_mat));
    3213:       PetscCall(MatDestroy(&M));
    3214:     }
    3215:     /* store global idxs for p0 */
    3216:     PetscCall(ISLocalToGlobalMappingApply(matis->rmapping, pcbddc->benign_n, pcbddc->benign_p0_lidx, pcbddc->benign_p0_gidx));
    3217:   }
    3218:   *zerodiaglocal = zerodiag;
    3219:   PetscFunctionReturn(PETSC_SUCCESS);
    3220: }

    3222: PetscErrorCode PCBDDCBenignGetOrSetP0(PC pc, Vec v, PetscBool get)
    3223: {
    3224:   PC_BDDC     *pcbddc = (PC_BDDC *)pc->data;
    3225:   PetscScalar *array;

    3227:   PetscFunctionBegin;
    3228:   if (!pcbddc->benign_sf) {
    3229:     PetscCall(PetscSFCreate(PetscObjectComm((PetscObject)pc), &pcbddc->benign_sf));
    3230:     PetscCall(PetscSFSetGraphLayout(pcbddc->benign_sf, pc->pmat->rmap, pcbddc->benign_n, NULL, PETSC_OWN_POINTER, pcbddc->benign_p0_gidx));
    3231:   }
    3232:   if (get) {
    3233:     PetscCall(VecGetArrayRead(v, (const PetscScalar **)&array));
    3234:     PetscCall(PetscSFBcastBegin(pcbddc->benign_sf, MPIU_SCALAR, array, pcbddc->benign_p0, MPI_REPLACE));
    3235:     PetscCall(PetscSFBcastEnd(pcbddc->benign_sf, MPIU_SCALAR, array, pcbddc->benign_p0, MPI_REPLACE));
    3236:     PetscCall(VecRestoreArrayRead(v, (const PetscScalar **)&array));
    3237:   } else {
    3238:     PetscCall(VecGetArray(v, &array));
    3239:     PetscCall(PetscSFReduceBegin(pcbddc->benign_sf, MPIU_SCALAR, pcbddc->benign_p0, array, MPI_REPLACE));
    3240:     PetscCall(PetscSFReduceEnd(pcbddc->benign_sf, MPIU_SCALAR, pcbddc->benign_p0, array, MPI_REPLACE));
    3241:     PetscCall(VecRestoreArray(v, &array));
    3242:   }
    3243:   PetscFunctionReturn(PETSC_SUCCESS);
    3244: }

    3246: PetscErrorCode PCBDDCBenignPopOrPushB0(PC pc, PetscBool pop)
    3247: {
    3248:   PC_BDDC *pcbddc = (PC_BDDC *)pc->data;

    3250:   PetscFunctionBegin;
    3251:   /* TODO: add error checking
    3252:     - avoid nested pop (or push) calls.
    3253:     - cannot push before pop.
    3254:     - cannot call this if pcbddc->local_mat is NULL
    3255:   */
    3256:   if (!pcbddc->benign_n) PetscFunctionReturn(PETSC_SUCCESS);
    3257:   if (pop) {
    3258:     if (pcbddc->benign_change_explicit) {
    3259:       IS       is_p0;
    3260:       MatReuse reuse;

    3262:       /* extract B_0 */
    3263:       reuse = MAT_INITIAL_MATRIX;
    3264:       if (pcbddc->benign_B0) reuse = MAT_REUSE_MATRIX;
    3265:       PetscCall(ISCreateGeneral(PETSC_COMM_SELF, pcbddc->benign_n, pcbddc->benign_p0_lidx, PETSC_COPY_VALUES, &is_p0));
    3266:       PetscCall(MatCreateSubMatrix(pcbddc->local_mat, is_p0, NULL, reuse, &pcbddc->benign_B0));
    3267:       /* remove rows and cols from local problem */
    3268:       PetscCall(MatSetOption(pcbddc->local_mat, MAT_KEEP_NONZERO_PATTERN, PETSC_TRUE));
    3269:       PetscCall(MatSetOption(pcbddc->local_mat, MAT_NEW_NONZERO_LOCATION_ERR, PETSC_FALSE));
    3270:       PetscCall(MatZeroRowsColumnsIS(pcbddc->local_mat, is_p0, 1.0, NULL, NULL));
    3271:       PetscCall(ISDestroy(&is_p0));
    3272:     } else {
    3273:       Mat_IS      *matis = (Mat_IS *)pc->pmat->data;
    3274:       PetscScalar *vals;
    3275:       PetscInt     i, n, *idxs_ins;

    3277:       PetscCall(VecGetLocalSize(matis->y, &n));
    3278:       PetscCall(PetscMalloc2(n, &idxs_ins, n, &vals));
    3279:       if (!pcbddc->benign_B0) {
    3280:         PetscInt *nnz;
    3281:         PetscCall(MatCreate(PetscObjectComm((PetscObject)pcbddc->local_mat), &pcbddc->benign_B0));
    3282:         PetscCall(MatSetType(pcbddc->benign_B0, MATAIJ));
    3283:         PetscCall(MatSetSizes(pcbddc->benign_B0, pcbddc->benign_n, n, PETSC_DECIDE, PETSC_DECIDE));
    3284:         PetscCall(PetscMalloc1(pcbddc->benign_n, &nnz));
    3285:         for (i = 0; i < pcbddc->benign_n; i++) {
    3286:           PetscCall(ISGetLocalSize(pcbddc->benign_zerodiag_subs[i], &nnz[i]));
    3287:           nnz[i] = n - nnz[i];
    3288:         }
    3289:         PetscCall(MatSeqAIJSetPreallocation(pcbddc->benign_B0, 0, nnz));
    3290:         PetscCall(MatSetOption(pcbddc->benign_B0, MAT_NEW_NONZERO_ALLOCATION_ERR, PETSC_TRUE));
    3291:         PetscCall(PetscFree(nnz));
    3292:       }

    3294:       for (i = 0; i < pcbddc->benign_n; i++) {
    3295:         PetscScalar *array;
    3296:         PetscInt    *idxs, j, nz, cum;

    3298:         PetscCall(VecSet(matis->x, 0.));
    3299:         PetscCall(ISGetLocalSize(pcbddc->benign_zerodiag_subs[i], &nz));
    3300:         PetscCall(ISGetIndices(pcbddc->benign_zerodiag_subs[i], (const PetscInt **)&idxs));
    3301:         for (j = 0; j < nz; j++) vals[j] = 1.;
    3302:         PetscCall(VecSetValues(matis->x, nz, idxs, vals, INSERT_VALUES));
    3303:         PetscCall(VecAssemblyBegin(matis->x));
    3304:         PetscCall(VecAssemblyEnd(matis->x));
    3305:         PetscCall(VecSet(matis->y, 0.));
    3306:         PetscCall(MatMult(matis->A, matis->x, matis->y));
    3307:         PetscCall(VecGetArray(matis->y, &array));
    3308:         cum = 0;
    3309:         for (j = 0; j < n; j++) {
    3310:           if (PetscUnlikely(PetscAbsScalar(array[j]) > PETSC_SMALL)) {
    3311:             vals[cum]     = array[j];
    3312:             idxs_ins[cum] = j;
    3313:             cum++;
    3314:           }
    3315:         }
    3316:         PetscCall(MatSetValues(pcbddc->benign_B0, 1, &i, cum, idxs_ins, vals, INSERT_VALUES));
    3317:         PetscCall(VecRestoreArray(matis->y, &array));
    3318:         PetscCall(ISRestoreIndices(pcbddc->benign_zerodiag_subs[i], (const PetscInt **)&idxs));
    3319:       }
    3320:       PetscCall(MatAssemblyBegin(pcbddc->benign_B0, MAT_FINAL_ASSEMBLY));
    3321:       PetscCall(MatAssemblyEnd(pcbddc->benign_B0, MAT_FINAL_ASSEMBLY));
    3322:       PetscCall(PetscFree2(idxs_ins, vals));
    3323:     }
    3324:   } else { /* push */

    3326:     PetscCheck(pcbddc->benign_change_explicit, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Cannot push B0!");
    3327:     for (PetscInt i = 0; i < pcbddc->benign_n; i++) {
    3328:       PetscScalar *B0_vals;
    3329:       PetscInt    *B0_cols, B0_ncol;

    3331:       PetscCall(MatGetRow(pcbddc->benign_B0, i, &B0_ncol, (const PetscInt **)&B0_cols, (const PetscScalar **)&B0_vals));
    3332:       PetscCall(MatSetValues(pcbddc->local_mat, 1, pcbddc->benign_p0_lidx + i, B0_ncol, B0_cols, B0_vals, INSERT_VALUES));
    3333:       PetscCall(MatSetValues(pcbddc->local_mat, B0_ncol, B0_cols, 1, pcbddc->benign_p0_lidx + i, B0_vals, INSERT_VALUES));
    3334:       PetscCall(MatSetValue(pcbddc->local_mat, pcbddc->benign_p0_lidx[i], pcbddc->benign_p0_lidx[i], 0.0, INSERT_VALUES));
    3335:       PetscCall(MatRestoreRow(pcbddc->benign_B0, i, &B0_ncol, (const PetscInt **)&B0_cols, (const PetscScalar **)&B0_vals));
    3336:     }
    3337:     PetscCall(MatAssemblyBegin(pcbddc->local_mat, MAT_FINAL_ASSEMBLY));
    3338:     PetscCall(MatAssemblyEnd(pcbddc->local_mat, MAT_FINAL_ASSEMBLY));
    3339:   }
    3340:   PetscFunctionReturn(PETSC_SUCCESS);
    3341: }

    3343: PetscErrorCode PCBDDCAdaptiveSelection(PC pc)
    3344: {
    3345:   PC_BDDC        *pcbddc     = (PC_BDDC *)pc->data;
    3346:   PCBDDCSubSchurs sub_schurs = pcbddc->sub_schurs;
    3347:   PetscBLASInt    B_neigs, B_ierr, B_lwork;
    3348:   PetscBLASInt   *B_iwork, *B_ifail;
    3349:   PetscScalar    *work, lwork;
    3350:   PetscScalar    *St, *S, *eigv;
    3351:   PetscScalar    *Sarray, *Starray;
    3352:   PetscReal      *eigs, thresh, lthresh, uthresh;
    3353:   PetscInt        i, nmax, nmin, nv, cum, mss, cum2, cumarray, maxneigs;
    3354:   PetscBool       allocated_S_St, upart;
    3355: #if defined(PETSC_USE_COMPLEX)
    3356:   PetscReal *rwork;
    3357: #endif

    3359:   PetscFunctionBegin;
    3360:   if (!pcbddc->adaptive_selection) PetscFunctionReturn(PETSC_SUCCESS);
    3361:   PetscCheck(sub_schurs, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Adaptive selection of constraints requires SubSchurs data");
    3362:   PetscCheck(sub_schurs->schur_explicit || !sub_schurs->n_subs, PetscObjectComm((PetscObject)pc), PETSC_ERR_SUP, "Adaptive selection of constraints requires MUMPS and/or MKL_CPARDISO");
    3363:   PetscCheck(!sub_schurs->n_subs || sub_schurs->is_symmetric, PETSC_COMM_SELF, PETSC_ERR_SUP, "Adaptive selection not yet implemented for this matrix pencil (herm %d, symm %d, posdef %d)", sub_schurs->is_hermitian, sub_schurs->is_symmetric,
    3364:              sub_schurs->is_posdef);
    3365:   PetscCall(PetscLogEventBegin(PC_BDDC_AdaptiveSetUp[pcbddc->current_level], pc, 0, 0, 0));

    3367:   if (pcbddc->dbg_flag) {
    3368:     if (!pcbddc->dbg_viewer) pcbddc->dbg_viewer = PETSC_VIEWER_STDOUT_(PetscObjectComm((PetscObject)pc));
    3369:     PetscCall(PetscViewerFlush(pcbddc->dbg_viewer));
    3370:     PetscCall(PetscViewerASCIIPrintf(pcbddc->dbg_viewer, "--------------------------------------------------\n"));
    3371:     PetscCall(PetscViewerASCIIPrintf(pcbddc->dbg_viewer, "Check adaptive selection of constraints\n"));
    3372:     PetscCall(PetscViewerASCIIPushSynchronized(pcbddc->dbg_viewer));
    3373:   }

    3375:   if (pcbddc->dbg_flag) PetscCall(PetscViewerASCIISynchronizedPrintf(pcbddc->dbg_viewer, "Subdomain %04d cc %" PetscInt_FMT " (%d,%d).\n", PetscGlobalRank, sub_schurs->n_subs, sub_schurs->is_hermitian, sub_schurs->is_posdef));

    3377:   /* max size of subsets */
    3378:   mss = 0;
    3379:   for (i = 0; i < sub_schurs->n_subs; i++) {
    3380:     PetscInt subset_size;

    3382:     PetscCall(ISGetLocalSize(sub_schurs->is_subs[i], &subset_size));
    3383:     mss = PetscMax(mss, subset_size);
    3384:   }

    3386:   /* min/max and threshold */
    3387:   nmax           = pcbddc->adaptive_nmax > 0 ? pcbddc->adaptive_nmax : mss;
    3388:   nmin           = pcbddc->adaptive_nmin > 0 ? pcbddc->adaptive_nmin : 0;
    3389:   nmax           = PetscMax(nmin, nmax);
    3390:   allocated_S_St = PETSC_FALSE;
    3391:   if (nmin || !sub_schurs->is_posdef) { /* XXX */
    3392:     allocated_S_St = PETSC_TRUE;
    3393:   }

    3395:   /* allocate lapack workspace */
    3396:   cum = cum2 = 0;
    3397:   maxneigs   = 0;
    3398:   for (i = 0; i < sub_schurs->n_subs; i++) {
    3399:     PetscInt n, subset_size;

    3401:     PetscCall(ISGetLocalSize(sub_schurs->is_subs[i], &subset_size));
    3402:     n = PetscMin(subset_size, nmax);
    3403:     cum += subset_size;
    3404:     cum2 += subset_size * n;
    3405:     maxneigs = PetscMax(maxneigs, n);
    3406:   }
    3407:   lwork = 0;
    3408:   if (mss) {
    3409:     PetscScalar  sdummy  = 0.;
    3410:     PetscBLASInt B_itype = 1;
    3411:     PetscBLASInt B_N, idummy = 0;
    3412:     PetscReal    rdummy = 0., zero = 0.0;
    3413:     PetscReal    eps = 0.0; /* dlamch? */

    3415:     PetscCheck(sub_schurs->is_symmetric, PETSC_COMM_SELF, PETSC_ERR_SUP, "Not yet implemented");
    3416:     PetscCall(PetscBLASIntCast(mss, &B_N));
    3417:     B_lwork = -1;
    3418:     /* some implementations may complain about NULL pointers, even if we are querying */
    3419:     S       = &sdummy;
    3420:     St      = &sdummy;
    3421:     eigs    = &rdummy;
    3422:     eigv    = &sdummy;
    3423:     B_iwork = &idummy;
    3424:     B_ifail = &idummy;
    3425: #if defined(PETSC_USE_COMPLEX)
    3426:     rwork = &rdummy;
    3427: #endif
    3428:     thresh = 1.0;
    3429:     PetscCall(PetscFPTrapPush(PETSC_FP_TRAP_OFF));
    3430: #if defined(PETSC_USE_COMPLEX)
    3431:     PetscCallBLAS("LAPACKsygvx", LAPACKsygvx_(&B_itype, "V", "V", "L", &B_N, St, &B_N, S, &B_N, &zero, &thresh, B_iwork, B_iwork, &eps, &B_neigs, eigs, eigv, &B_N, &lwork, &B_lwork, rwork, B_iwork, B_ifail, &B_ierr));
    3432: #else
    3433:     PetscCallBLAS("LAPACKsygvx", LAPACKsygvx_(&B_itype, "V", "V", "L", &B_N, St, &B_N, S, &B_N, &zero, &thresh, B_iwork, B_iwork, &eps, &B_neigs, eigs, eigv, &B_N, &lwork, &B_lwork, B_iwork, B_ifail, &B_ierr));
    3434: #endif
    3435:     PetscCheck(B_ierr == 0, PETSC_COMM_SELF, PETSC_ERR_LIB, "Error in query to SYGVX Lapack routine %" PetscBLASInt_FMT, B_ierr);
    3436:     PetscCall(PetscFPTrapPop());
    3437:   }

    3439:   nv = 0;
    3440:   if (sub_schurs->is_vertices && pcbddc->use_vertices) { /* complement set of active subsets, each entry is a vertex (boundary made by active subsets, vertices and dirichlet dofs) */
    3441:     PetscCall(ISGetLocalSize(sub_schurs->is_vertices, &nv));
    3442:   }
    3443:   PetscCall(PetscBLASIntCast((PetscInt)PetscRealPart(lwork), &B_lwork));
    3444:   if (allocated_S_St) PetscCall(PetscMalloc2(mss * mss, &S, mss * mss, &St));
    3445:   PetscCall(PetscMalloc5(mss * mss, &eigv, mss, &eigs, B_lwork, &work, 5 * mss, &B_iwork, mss, &B_ifail));
    3446: #if defined(PETSC_USE_COMPLEX)
    3447:   PetscCall(PetscMalloc1(7 * mss, &rwork));
    3448: #endif
    3449:   PetscCall(PetscMalloc5(nv + sub_schurs->n_subs, &pcbddc->adaptive_constraints_n, nv + sub_schurs->n_subs + 1, &pcbddc->adaptive_constraints_idxs_ptr, nv + sub_schurs->n_subs + 1, &pcbddc->adaptive_constraints_data_ptr, nv + cum, &pcbddc->adaptive_constraints_idxs, nv + cum2,
    3450:                          &pcbddc->adaptive_constraints_data));
    3451:   PetscCall(PetscArrayzero(pcbddc->adaptive_constraints_n, nv + sub_schurs->n_subs));

    3453:   maxneigs = 0;
    3454:   cum = cumarray                           = 0;
    3455:   pcbddc->adaptive_constraints_idxs_ptr[0] = 0;
    3456:   pcbddc->adaptive_constraints_data_ptr[0] = 0;
    3457:   if (sub_schurs->is_vertices && pcbddc->use_vertices) {
    3458:     const PetscInt *idxs;

    3460:     PetscCall(ISGetIndices(sub_schurs->is_vertices, &idxs));
    3461:     for (cum = 0; cum < nv; cum++) {
    3462:       pcbddc->adaptive_constraints_n[cum]            = 1;
    3463:       pcbddc->adaptive_constraints_idxs[cum]         = idxs[cum];
    3464:       pcbddc->adaptive_constraints_data[cum]         = 1.0;
    3465:       pcbddc->adaptive_constraints_idxs_ptr[cum + 1] = pcbddc->adaptive_constraints_idxs_ptr[cum] + 1;
    3466:       pcbddc->adaptive_constraints_data_ptr[cum + 1] = pcbddc->adaptive_constraints_data_ptr[cum] + 1;
    3467:     }
    3468:     PetscCall(ISRestoreIndices(sub_schurs->is_vertices, &idxs));
    3469:   }

    3471:   if (mss) { /* multilevel */
    3472:     if (sub_schurs->gdsw) {
    3473:       PetscCall(MatSeqAIJGetArray(sub_schurs->sum_S_Ej_all, &Sarray));
    3474:       PetscCall(MatSeqAIJGetArray(sub_schurs->sum_S_Ej_tilda_all, &Starray));
    3475:     } else {
    3476:       PetscCall(MatSeqAIJGetArray(sub_schurs->sum_S_Ej_inv_all, &Sarray));
    3477:       PetscCall(MatSeqAIJGetArray(sub_schurs->sum_S_Ej_tilda_all, &Starray));
    3478:     }
    3479:   }

    3481:   lthresh = pcbddc->adaptive_threshold[0];
    3482:   uthresh = pcbddc->adaptive_threshold[1];
    3483:   upart   = pcbddc->use_deluxe_scaling;
    3484:   for (i = 0; i < sub_schurs->n_subs; i++) {
    3485:     const PetscInt *idxs;
    3486:     PetscReal       upper, lower;
    3487:     PetscInt        j, subset_size, eigs_start = 0;
    3488:     PetscBLASInt    B_N;
    3489:     PetscBool       same_data = PETSC_FALSE;
    3490:     PetscBool       scal      = PETSC_FALSE;

    3492:     if (upart) {
    3493:       upper = PETSC_MAX_REAL;
    3494:       lower = uthresh;
    3495:     } else {
    3496:       if (sub_schurs->gdsw) {
    3497:         upper = uthresh;
    3498:         lower = PETSC_MIN_REAL;
    3499:       } else {
    3500:         PetscCheck(sub_schurs->is_posdef, PETSC_COMM_SELF, PETSC_ERR_SUP, "Not yet implemented without deluxe scaling");
    3501:         upper = 1. / uthresh;
    3502:         lower = 0.;
    3503:       }
    3504:     }
    3505:     PetscCall(ISGetLocalSize(sub_schurs->is_subs[i], &subset_size));
    3506:     PetscCall(ISGetIndices(sub_schurs->is_subs[i], &idxs));
    3507:     PetscCall(PetscBLASIntCast(subset_size, &B_N));
    3508:     /* this is experimental: we assume the dofs have been properly grouped to have
    3509:        the diagonal blocks Schur complements either positive or negative definite (true for Stokes) */
    3510:     if (!sub_schurs->is_posdef) {
    3511:       Mat T;

    3513:       for (j = 0; j < subset_size; j++) {
    3514:         if (PetscRealPart(*(Sarray + cumarray + j * (subset_size + 1))) < 0.0) {
    3515:           PetscCall(MatCreateSeqDense(PETSC_COMM_SELF, subset_size, subset_size, Sarray + cumarray, &T));
    3516:           PetscCall(MatScale(T, -1.0));
    3517:           PetscCall(MatDestroy(&T));
    3518:           PetscCall(MatCreateSeqDense(PETSC_COMM_SELF, subset_size, subset_size, Starray + cumarray, &T));
    3519:           PetscCall(MatScale(T, -1.0));
    3520:           PetscCall(MatDestroy(&T));
    3521:           if (sub_schurs->change_primal_sub) {
    3522:             PetscInt        nz, k;
    3523:             const PetscInt *idxs;

    3525:             PetscCall(ISGetLocalSize(sub_schurs->change_primal_sub[i], &nz));
    3526:             PetscCall(ISGetIndices(sub_schurs->change_primal_sub[i], &idxs));
    3527:             for (k = 0; k < nz; k++) {
    3528:               *(Sarray + cumarray + idxs[k] * (subset_size + 1)) *= -1.0;
    3529:               *(Starray + cumarray + idxs[k] * (subset_size + 1)) = 0.0;
    3530:             }
    3531:             PetscCall(ISRestoreIndices(sub_schurs->change_primal_sub[i], &idxs));
    3532:           }
    3533:           scal = PETSC_TRUE;
    3534:           break;
    3535:         }
    3536:       }
    3537:     }

    3539:     if (allocated_S_St) { /* S and S_t should be copied since we could need them later */
    3540:       if (sub_schurs->is_symmetric) {
    3541:         PetscInt j, k;
    3542:         if (sub_schurs->n_subs == 1) { /* zeroing memory to use PetscArraycmp() later */
    3543:           PetscCall(PetscArrayzero(S, subset_size * subset_size));
    3544:           PetscCall(PetscArrayzero(St, subset_size * subset_size));
    3545:         }
    3546:         for (j = 0; j < subset_size; j++) {
    3547:           for (k = j; k < subset_size; k++) {
    3548:             S[j * subset_size + k]  = Sarray[cumarray + j * subset_size + k];
    3549:             St[j * subset_size + k] = Starray[cumarray + j * subset_size + k];
    3550:           }
    3551:         }
    3552:       } else {
    3553:         PetscCall(PetscArraycpy(S, Sarray + cumarray, subset_size * subset_size));
    3554:         PetscCall(PetscArraycpy(St, Starray + cumarray, subset_size * subset_size));
    3555:       }
    3556:     } else {
    3557:       S  = Sarray + cumarray;
    3558:       St = Starray + cumarray;
    3559:     }
    3560:     /* see if we can save some work */
    3561:     if (sub_schurs->n_subs == 1 && pcbddc->use_deluxe_scaling) PetscCall(PetscArraycmp(S, St, subset_size * subset_size, &same_data));

    3563:     if (same_data && !sub_schurs->change) { /* there's no need of constraints here */
    3564:       B_neigs = 0;
    3565:     } else {
    3566:       PetscBLASInt B_itype = 1, B_IL = 1, B_IU = 0;
    3567:       PetscReal    eps = -1.0; /* dlamch? */
    3568:       PetscInt     nmin_s;
    3569:       PetscBool    compute_range;

    3571:       PetscCheck(sub_schurs->is_symmetric, PETSC_COMM_SELF, PETSC_ERR_SUP, "Not yet implemented");
    3572:       B_neigs       = 0;
    3573:       compute_range = (PetscBool)!same_data;
    3574:       if (nmin >= subset_size) compute_range = PETSC_FALSE;

    3576:       if (pcbddc->dbg_flag) {
    3577:         PetscInt nc = 0, c = pcbddc->mat_graph->nodes[idxs[0]].count, w = pcbddc->mat_graph->nodes[idxs[0]].which_dof;

    3579:         if (sub_schurs->change_primal_sub) PetscCall(ISGetLocalSize(sub_schurs->change_primal_sub[i], &nc));
    3580:         PetscCall(
    3581:           PetscViewerASCIISynchronizedPrintf(pcbddc->dbg_viewer, "Computing for sub %" PetscInt_FMT "/%" PetscInt_FMT " size %" PetscInt_FMT " count %" PetscInt_FMT " fid %" PetscInt_FMT " (range %d) (change %" PetscInt_FMT ").\n", i, sub_schurs->n_subs, subset_size, c, w, compute_range, nc));
    3582:       }

    3584:       PetscCall(PetscFPTrapPush(PETSC_FP_TRAP_OFF));
    3585:       if (compute_range) {
    3586:         /* ask for eigenvalues larger than thresh */
    3587:         if (sub_schurs->is_posdef) {
    3588: #if defined(PETSC_USE_COMPLEX)
    3589:           PetscCallBLAS("LAPACKsygvx", LAPACKsygvx_(&B_itype, "V", "V", "L", &B_N, St, &B_N, S, &B_N, &lower, &upper, &B_IL, &B_IU, &eps, &B_neigs, eigs, eigv, &B_N, work, &B_lwork, rwork, B_iwork, B_ifail, &B_ierr));
    3590: #else
    3591:           PetscCallBLAS("LAPACKsygvx", LAPACKsygvx_(&B_itype, "V", "V", "L", &B_N, St, &B_N, S, &B_N, &lower, &upper, &B_IL, &B_IU, &eps, &B_neigs, eigs, eigv, &B_N, work, &B_lwork, B_iwork, B_ifail, &B_ierr));
    3592: #endif
    3593:           PetscCall(PetscLogFlops((4.0 * subset_size * subset_size * subset_size) / 3.0));
    3594:         } else { /* no theory so far, but it works nicely */
    3595:           PetscInt  recipe = 0, recipe_m = 1;
    3596:           PetscReal bb[2];

    3598:           PetscCall(PetscOptionsGetInt(NULL, ((PetscObject)pc)->prefix, "-pc_bddc_adaptive_recipe", &recipe, NULL));
    3599:           switch (recipe) {
    3600:           case 0:
    3601:             if (scal) {
    3602:               bb[0] = PETSC_MIN_REAL;
    3603:               bb[1] = lthresh;
    3604:             } else {
    3605:               bb[0] = uthresh;
    3606:               bb[1] = PETSC_MAX_REAL;
    3607:             }
    3608: #if defined(PETSC_USE_COMPLEX)
    3609:             PetscCallBLAS("LAPACKsygvx", LAPACKsygvx_(&B_itype, "V", "V", "L", &B_N, St, &B_N, S, &B_N, &bb[0], &bb[1], &B_IL, &B_IU, &eps, &B_neigs, eigs, eigv, &B_N, work, &B_lwork, rwork, B_iwork, B_ifail, &B_ierr));
    3610: #else
    3611:             PetscCallBLAS("LAPACKsygvx", LAPACKsygvx_(&B_itype, "V", "V", "L", &B_N, St, &B_N, S, &B_N, &bb[0], &bb[1], &B_IL, &B_IU, &eps, &B_neigs, eigs, eigv, &B_N, work, &B_lwork, B_iwork, B_ifail, &B_ierr));
    3612: #endif
    3613:             PetscCall(PetscLogFlops((4.0 * subset_size * subset_size * subset_size) / 3.0));
    3614:             break;
    3615:           case 1:
    3616:             bb[0] = PETSC_MIN_REAL;
    3617:             bb[1] = lthresh * lthresh;
    3618: #if defined(PETSC_USE_COMPLEX)
    3619:             PetscCallBLAS("LAPACKsygvx", LAPACKsygvx_(&B_itype, "V", "V", "L", &B_N, St, &B_N, S, &B_N, &bb[0], &bb[1], &B_IL, &B_IU, &eps, &B_neigs, eigs, eigv, &B_N, work, &B_lwork, rwork, B_iwork, B_ifail, &B_ierr));
    3620: #else
    3621:             PetscCallBLAS("LAPACKsygvx", LAPACKsygvx_(&B_itype, "V", "V", "L", &B_N, St, &B_N, S, &B_N, &bb[0], &bb[1], &B_IL, &B_IU, &eps, &B_neigs, eigs, eigv, &B_N, work, &B_lwork, B_iwork, B_ifail, &B_ierr));
    3622: #endif
    3623:             PetscCall(PetscLogFlops((4.0 * subset_size * subset_size * subset_size) / 3.0));
    3624:             if (!scal) {
    3625:               PetscBLASInt B_neigs2 = 0;

    3627:               bb[0] = PetscMax(lthresh * lthresh, uthresh);
    3628:               bb[1] = PETSC_MAX_REAL;
    3629:               PetscCall(PetscArraycpy(S, Sarray + cumarray, subset_size * subset_size));
    3630:               PetscCall(PetscArraycpy(St, Starray + cumarray, subset_size * subset_size));
    3631: #if defined(PETSC_USE_COMPLEX)
    3632:               PetscCallBLAS("LAPACKsygvx", LAPACKsygvx_(&B_itype, "V", "V", "L", &B_N, St, &B_N, S, &B_N, &bb[0], &bb[1], &B_IL, &B_IU, &eps, &B_neigs2, eigs + B_neigs, eigv + B_neigs * B_N, &B_N, work, &B_lwork, rwork, B_iwork, B_ifail, &B_ierr));
    3633: #else
    3634:               PetscCallBLAS("LAPACKsygvx", LAPACKsygvx_(&B_itype, "V", "V", "L", &B_N, St, &B_N, S, &B_N, &bb[0], &bb[1], &B_IL, &B_IU, &eps, &B_neigs2, eigs + B_neigs, eigv + B_neigs * B_N, &B_N, work, &B_lwork, B_iwork, B_ifail, &B_ierr));
    3635: #endif
    3636:               PetscCall(PetscLogFlops((4.0 * subset_size * subset_size * subset_size) / 3.0));
    3637:               B_neigs += B_neigs2;
    3638:             }
    3639:             break;
    3640:           case 2:
    3641:             if (scal) {
    3642:               bb[0] = PETSC_MIN_REAL;
    3643:               bb[1] = 0;
    3644: #if defined(PETSC_USE_COMPLEX)
    3645:               PetscCallBLAS("LAPACKsygvx", LAPACKsygvx_(&B_itype, "V", "V", "L", &B_N, St, &B_N, S, &B_N, &bb[0], &bb[1], &B_IL, &B_IU, &eps, &B_neigs, eigs, eigv, &B_N, work, &B_lwork, rwork, B_iwork, B_ifail, &B_ierr));
    3646: #else
    3647:               PetscCallBLAS("LAPACKsygvx", LAPACKsygvx_(&B_itype, "V", "V", "L", &B_N, St, &B_N, S, &B_N, &bb[0], &bb[1], &B_IL, &B_IU, &eps, &B_neigs, eigs, eigv, &B_N, work, &B_lwork, B_iwork, B_ifail, &B_ierr));
    3648: #endif
    3649:               PetscCall(PetscLogFlops((4.0 * subset_size * subset_size * subset_size) / 3.0));
    3650:             } else {
    3651:               PetscBLASInt B_neigs2 = 0;
    3652:               PetscBool    do_copy  = PETSC_FALSE;

    3654:               lthresh = PetscMax(lthresh, 0.0);
    3655:               if (lthresh > 0.0) {
    3656:                 bb[0] = PETSC_MIN_REAL;
    3657:                 bb[1] = lthresh * lthresh;

    3659:                 do_copy = PETSC_TRUE;
    3660: #if defined(PETSC_USE_COMPLEX)
    3661:                 PetscCallBLAS("LAPACKsygvx", LAPACKsygvx_(&B_itype, "V", "V", "L", &B_N, St, &B_N, S, &B_N, &bb[0], &bb[1], &B_IL, &B_IU, &eps, &B_neigs, eigs, eigv, &B_N, work, &B_lwork, rwork, B_iwork, B_ifail, &B_ierr));
    3662: #else
    3663:                 PetscCallBLAS("LAPACKsygvx", LAPACKsygvx_(&B_itype, "V", "V", "L", &B_N, St, &B_N, S, &B_N, &bb[0], &bb[1], &B_IL, &B_IU, &eps, &B_neigs, eigs, eigv, &B_N, work, &B_lwork, B_iwork, B_ifail, &B_ierr));
    3664: #endif
    3665:                 PetscCall(PetscLogFlops((4.0 * subset_size * subset_size * subset_size) / 3.0));
    3666:               }
    3667:               bb[0] = PetscMax(lthresh * lthresh, uthresh);
    3668:               bb[1] = PETSC_MAX_REAL;
    3669:               if (do_copy) {
    3670:                 PetscCall(PetscArraycpy(S, Sarray + cumarray, subset_size * subset_size));
    3671:                 PetscCall(PetscArraycpy(St, Starray + cumarray, subset_size * subset_size));
    3672:               }
    3673: #if defined(PETSC_USE_COMPLEX)
    3674:               PetscCallBLAS("LAPACKsygvx", LAPACKsygvx_(&B_itype, "V", "V", "L", &B_N, St, &B_N, S, &B_N, &bb[0], &bb[1], &B_IL, &B_IU, &eps, &B_neigs2, eigs + B_neigs, eigv + B_neigs * B_N, &B_N, work, &B_lwork, rwork, B_iwork, B_ifail, &B_ierr));
    3675: #else
    3676:               PetscCallBLAS("LAPACKsygvx", LAPACKsygvx_(&B_itype, "V", "V", "L", &B_N, St, &B_N, S, &B_N, &bb[0], &bb[1], &B_IL, &B_IU, &eps, &B_neigs2, eigs + B_neigs, eigv + B_neigs * B_N, &B_N, work, &B_lwork, B_iwork, B_ifail, &B_ierr));
    3677: #endif
    3678:               PetscCall(PetscLogFlops((4.0 * subset_size * subset_size * subset_size) / 3.0));
    3679:               B_neigs += B_neigs2;
    3680:             }
    3681:             break;
    3682:           case 3:
    3683:             if (scal) {
    3684:               PetscCall(PetscOptionsGetInt(NULL, ((PetscObject)pc)->prefix, "-pc_bddc_adaptive_recipe3_min_scal", &recipe_m, NULL));
    3685:             } else {
    3686:               PetscCall(PetscOptionsGetInt(NULL, ((PetscObject)pc)->prefix, "-pc_bddc_adaptive_recipe3_min", &recipe_m, NULL));
    3687:             }
    3688:             if (!scal) {
    3689:               bb[0] = uthresh;
    3690:               bb[1] = PETSC_MAX_REAL;
    3691: #if defined(PETSC_USE_COMPLEX)
    3692:               PetscCallBLAS("LAPACKsygvx", LAPACKsygvx_(&B_itype, "V", "V", "L", &B_N, St, &B_N, S, &B_N, &bb[0], &bb[1], &B_IL, &B_IU, &eps, &B_neigs, eigs, eigv, &B_N, work, &B_lwork, rwork, B_iwork, B_ifail, &B_ierr));
    3693: #else
    3694:               PetscCallBLAS("LAPACKsygvx", LAPACKsygvx_(&B_itype, "V", "V", "L", &B_N, St, &B_N, S, &B_N, &bb[0], &bb[1], &B_IL, &B_IU, &eps, &B_neigs, eigs, eigv, &B_N, work, &B_lwork, B_iwork, B_ifail, &B_ierr));
    3695: #endif
    3696:               PetscCall(PetscLogFlops((4.0 * subset_size * subset_size * subset_size) / 3.0));
    3697:             }
    3698:             if (recipe_m > 0 && B_N - B_neigs > 0) {
    3699:               PetscBLASInt B_neigs2 = 0;

    3701:               PetscCall(PetscBLASIntCast(PetscMin(recipe_m, B_N - B_neigs), &B_IU));
    3702:               PetscCall(PetscArraycpy(S, Sarray + cumarray, subset_size * subset_size));
    3703:               PetscCall(PetscArraycpy(St, Starray + cumarray, subset_size * subset_size));
    3704: #if defined(PETSC_USE_COMPLEX)
    3705:               PetscCallBLAS("LAPACKsygvx", LAPACKsygvx_(&B_itype, "V", "I", "L", &B_N, St, &B_N, S, &B_N, &lower, &upper, &B_IL, &B_IU, &eps, &B_neigs2, eigs + B_neigs, eigv + B_neigs * B_N, &B_N, work, &B_lwork, rwork, B_iwork, B_ifail, &B_ierr));
    3706: #else
    3707:               PetscCallBLAS("LAPACKsygvx", LAPACKsygvx_(&B_itype, "V", "I", "L", &B_N, St, &B_N, S, &B_N, &lower, &upper, &B_IL, &B_IU, &eps, &B_neigs2, eigs + B_neigs, eigv + B_neigs * B_N, &B_N, work, &B_lwork, B_iwork, B_ifail, &B_ierr));
    3708: #endif
    3709:               PetscCall(PetscLogFlops((4.0 * subset_size * subset_size * subset_size) / 3.0));
    3710:               B_neigs += B_neigs2;
    3711:             }
    3712:             break;
    3713:           case 4:
    3714:             bb[0] = PETSC_MIN_REAL;
    3715:             bb[1] = lthresh;
    3716: #if defined(PETSC_USE_COMPLEX)
    3717:             PetscCallBLAS("LAPACKsygvx", LAPACKsygvx_(&B_itype, "V", "V", "L", &B_N, St, &B_N, S, &B_N, &bb[0], &bb[1], &B_IL, &B_IU, &eps, &B_neigs, eigs, eigv, &B_N, work, &B_lwork, rwork, B_iwork, B_ifail, &B_ierr));
    3718: #else
    3719:             PetscCallBLAS("LAPACKsygvx", LAPACKsygvx_(&B_itype, "V", "V", "L", &B_N, St, &B_N, S, &B_N, &bb[0], &bb[1], &B_IL, &B_IU, &eps, &B_neigs, eigs, eigv, &B_N, work, &B_lwork, B_iwork, B_ifail, &B_ierr));
    3720: #endif
    3721:             PetscCall(PetscLogFlops((4.0 * subset_size * subset_size * subset_size) / 3.0));
    3722:             {
    3723:               PetscBLASInt B_neigs2 = 0;

    3725:               bb[0] = PetscMax(lthresh + PETSC_SMALL, uthresh);
    3726:               bb[1] = PETSC_MAX_REAL;
    3727:               PetscCall(PetscArraycpy(S, Sarray + cumarray, subset_size * subset_size));
    3728:               PetscCall(PetscArraycpy(St, Starray + cumarray, subset_size * subset_size));
    3729: #if defined(PETSC_USE_COMPLEX)
    3730:               PetscCallBLAS("LAPACKsygvx", LAPACKsygvx_(&B_itype, "V", "V", "L", &B_N, St, &B_N, S, &B_N, &bb[0], &bb[1], &B_IL, &B_IU, &eps, &B_neigs2, eigs + B_neigs, eigv + B_neigs * B_N, &B_N, work, &B_lwork, rwork, B_iwork, B_ifail, &B_ierr));
    3731: #else
    3732:               PetscCallBLAS("LAPACKsygvx", LAPACKsygvx_(&B_itype, "V", "V", "L", &B_N, St, &B_N, S, &B_N, &bb[0], &bb[1], &B_IL, &B_IU, &eps, &B_neigs2, eigs + B_neigs, eigv + B_neigs * B_N, &B_N, work, &B_lwork, B_iwork, B_ifail, &B_ierr));
    3733: #endif
    3734:               PetscCall(PetscLogFlops((4.0 * subset_size * subset_size * subset_size) / 3.0));
    3735:               B_neigs += B_neigs2;
    3736:             }
    3737:             break;
    3738:           case 5: /* same as before: first compute all eigenvalues, then filter */
    3739: #if defined(PETSC_USE_COMPLEX)
    3740:             PetscCallBLAS("LAPACKsygvx", LAPACKsygvx_(&B_itype, "V", "A", "L", &B_N, St, &B_N, S, &B_N, &bb[0], &bb[1], &B_IL, &B_IU, &eps, &B_neigs, eigs, eigv, &B_N, work, &B_lwork, rwork, B_iwork, B_ifail, &B_ierr));
    3741: #else
    3742:             PetscCallBLAS("LAPACKsygvx", LAPACKsygvx_(&B_itype, "V", "A", "L", &B_N, St, &B_N, S, &B_N, &bb[0], &bb[1], &B_IL, &B_IU, &eps, &B_neigs, eigs, eigv, &B_N, work, &B_lwork, B_iwork, B_ifail, &B_ierr));
    3743: #endif
    3744:             PetscCall(PetscLogFlops((4.0 * subset_size * subset_size * subset_size) / 3.0));
    3745:             {
    3746:               PetscInt e, k, ne;
    3747:               for (e = 0, ne = 0; e < B_neigs; e++) {
    3748:                 if (eigs[e] < lthresh || eigs[e] > uthresh) {
    3749:                   for (k = 0; k < B_N; k++) S[ne * B_N + k] = eigv[e * B_N + k];
    3750:                   eigs[ne] = eigs[e];
    3751:                   ne++;
    3752:                 }
    3753:               }
    3754:               PetscCall(PetscArraycpy(eigv, S, B_N * ne));
    3755:               PetscCall(PetscBLASIntCast(ne, &B_neigs));
    3756:             }
    3757:             break;
    3758:           default:
    3759:             SETERRQ(PetscObjectComm((PetscObject)pc), PETSC_ERR_SUP, "Unknown recipe %" PetscInt_FMT, recipe);
    3760:           }
    3761:         }
    3762:       } else if (!same_data) { /* this is just to see all the eigenvalues */
    3763:         PetscCall(PetscBLASIntCast(PetscMax(1, PetscMin(B_N, nmax)), &B_IU));
    3764: #if defined(PETSC_USE_COMPLEX)
    3765:         PetscCallBLAS("LAPACKsygvx", LAPACKsygvx_(&B_itype, "V", "I", "L", &B_N, St, &B_N, S, &B_N, &lower, &upper, &B_IL, &B_IU, &eps, &B_neigs, eigs, eigv, &B_N, work, &B_lwork, rwork, B_iwork, B_ifail, &B_ierr));
    3766: #else
    3767:         PetscCallBLAS("LAPACKsygvx", LAPACKsygvx_(&B_itype, "V", "I", "L", &B_N, St, &B_N, S, &B_N, &lower, &upper, &B_IL, &B_IU, &eps, &B_neigs, eigs, eigv, &B_N, work, &B_lwork, B_iwork, B_ifail, &B_ierr));
    3768: #endif
    3769:         PetscCall(PetscLogFlops((4.0 * subset_size * subset_size * subset_size) / 3.0));
    3770:       } else { /* same_data is true, so just get the adaptive functional requested by the user */
    3771:         PetscInt k;
    3772:         PetscCheck(sub_schurs->change_primal_sub, PETSC_COMM_SELF, PETSC_ERR_PLIB, "This should not happen");
    3773:         PetscCall(ISGetLocalSize(sub_schurs->change_primal_sub[i], &nmax));
    3774:         PetscCall(PetscBLASIntCast(nmax, &B_neigs));
    3775:         nmin = nmax;
    3776:         PetscCall(PetscArrayzero(eigv, subset_size * nmax));
    3777:         for (k = 0; k < nmax; k++) {
    3778:           eigs[k]                     = 1. / PETSC_SMALL;
    3779:           eigv[k * (subset_size + 1)] = 1.0;
    3780:         }
    3781:       }
    3782:       PetscCall(PetscFPTrapPop());
    3783:       if (B_ierr) {
    3784:         PetscCheck(B_ierr >= 0, PETSC_COMM_SELF, PETSC_ERR_LIB, "Error in SYGVX Lapack routine: illegal value for argument %" PetscBLASInt_FMT, -B_ierr);
    3785:         PetscCheck(B_ierr > B_N, PETSC_COMM_SELF, PETSC_ERR_LIB, "Error in SYGVX Lapack routine: %" PetscBLASInt_FMT " eigenvalues failed to converge", B_ierr);
    3786:         SETERRQ(PETSC_COMM_SELF, PETSC_ERR_LIB, "Error in SYGVX Lapack routine: leading minor of order %" PetscBLASInt_FMT " is not positive definite", B_ierr - B_N - 1);
    3787:       }

    3789:       if (B_neigs > nmax) {
    3790:         if (pcbddc->dbg_flag) PetscCall(PetscViewerASCIISynchronizedPrintf(pcbddc->dbg_viewer, "   found %" PetscBLASInt_FMT " eigs, more than maximum required %" PetscInt_FMT ".\n", B_neigs, nmax));
    3791:         if (upart) eigs_start = scal ? 0 : B_neigs - nmax;
    3792:         PetscCall(PetscBLASIntCast(nmax, &B_neigs));
    3793:       }

    3795:       nmin_s = PetscMin(nmin, B_N);
    3796:       if (B_neigs < nmin_s) {
    3797:         PetscBLASInt B_neigs2 = 0;

    3799:         if (upart) {
    3800:           if (scal) {
    3801:             PetscCall(PetscBLASIntCast(nmin_s, &B_IU));
    3802:             B_IL = B_neigs + 1;
    3803:           } else {
    3804:             PetscCall(PetscBLASIntCast(B_N - nmin_s + 1, &B_IL));
    3805:             B_IU = B_N - B_neigs;
    3806:           }
    3807:         } else {
    3808:           B_IL = B_neigs + 1;
    3809:           PetscCall(PetscBLASIntCast(nmin_s, &B_IU));
    3810:         }
    3811:         if (pcbddc->dbg_flag) {
    3812:           PetscCall(PetscViewerASCIISynchronizedPrintf(pcbddc->dbg_viewer, "   found %" PetscBLASInt_FMT " eigs, less than minimum required %" PetscInt_FMT ". Asking for %" PetscBLASInt_FMT " to %" PetscBLASInt_FMT " incl (fortran like)\n", B_neigs, nmin, B_IL, B_IU));
    3813:         }
    3814:         if (sub_schurs->is_symmetric) {
    3815:           PetscInt j, k;
    3816:           for (j = 0; j < subset_size; j++) {
    3817:             for (k = j; k < subset_size; k++) {
    3818:               S[j * subset_size + k]  = Sarray[cumarray + j * subset_size + k];
    3819:               St[j * subset_size + k] = Starray[cumarray + j * subset_size + k];
    3820:             }
    3821:           }
    3822:         } else {
    3823:           PetscCall(PetscArraycpy(S, Sarray + cumarray, subset_size * subset_size));
    3824:           PetscCall(PetscArraycpy(St, Starray + cumarray, subset_size * subset_size));
    3825:         }
    3826:         PetscCall(PetscFPTrapPush(PETSC_FP_TRAP_OFF));
    3827: #if defined(PETSC_USE_COMPLEX)
    3828:         PetscCallBLAS("LAPACKsygvx", LAPACKsygvx_(&B_itype, "V", "I", "L", &B_N, St, &B_N, S, &B_N, &lower, &upper, &B_IL, &B_IU, &eps, &B_neigs2, eigs + B_neigs, eigv + B_neigs * subset_size, &B_N, work, &B_lwork, rwork, B_iwork, B_ifail, &B_ierr));
    3829: #else
    3830:         PetscCallBLAS("LAPACKsygvx", LAPACKsygvx_(&B_itype, "V", "I", "L", &B_N, St, &B_N, S, &B_N, &lower, &upper, &B_IL, &B_IU, &eps, &B_neigs2, eigs + B_neigs, eigv + B_neigs * subset_size, &B_N, work, &B_lwork, B_iwork, B_ifail, &B_ierr));
    3831: #endif
    3832:         PetscCall(PetscLogFlops((4.0 * subset_size * subset_size * subset_size) / 3.0));
    3833:         PetscCall(PetscFPTrapPop());
    3834:         B_neigs += B_neigs2;
    3835:       }
    3836:       if (B_ierr) {
    3837:         PetscCheck(B_ierr >= 0, PETSC_COMM_SELF, PETSC_ERR_LIB, "Error in SYGVX Lapack routine: illegal value for argument %" PetscBLASInt_FMT, -B_ierr);
    3838:         PetscCheck(B_ierr > B_N, PETSC_COMM_SELF, PETSC_ERR_LIB, "Error in SYGVX Lapack routine: %" PetscBLASInt_FMT " eigenvalues failed to converge", B_ierr);
    3839:         SETERRQ(PETSC_COMM_SELF, PETSC_ERR_LIB, "Error in SYGVX Lapack routine: leading minor of order %" PetscBLASInt_FMT " is not positive definite", B_ierr - B_N - 1);
    3840:       }
    3841:       if (pcbddc->dbg_flag) {
    3842:         PetscCall(PetscViewerASCIISynchronizedPrintf(pcbddc->dbg_viewer, "   -> Got %" PetscBLASInt_FMT " eigs\n", B_neigs));
    3843:         for (j = 0; j < B_neigs; j++) {
    3844:           if (!sub_schurs->gdsw) {
    3845:             if (eigs[j] == 0.0) {
    3846:               PetscCall(PetscViewerASCIISynchronizedPrintf(pcbddc->dbg_viewer, "     Inf\n"));
    3847:             } else {
    3848:               if (upart) {
    3849:                 PetscCall(PetscViewerASCIISynchronizedPrintf(pcbddc->dbg_viewer, "     %1.6e\n", (double)eigs[j + eigs_start]));
    3850:               } else {
    3851:                 PetscCall(PetscViewerASCIISynchronizedPrintf(pcbddc->dbg_viewer, "     %1.6e\n", (double)(1 / eigs[j + eigs_start])));
    3852:               }
    3853:             }
    3854:           } else {
    3855:             double pg = (double)eigs[j + eigs_start];
    3856:             if (pg < 2 * PETSC_SMALL) pg = 0.0;
    3857:             PetscCall(PetscViewerASCIISynchronizedPrintf(pcbddc->dbg_viewer, "     %1.6e\n", pg));
    3858:           }
    3859:         }
    3860:       }
    3861:     }
    3862:     /* change the basis back to the original one */
    3863:     if (sub_schurs->change) {
    3864:       Mat change, phi, phit;

    3866:       if (pcbddc->dbg_flag > 2) {
    3867:         PetscInt ii;
    3868:         for (ii = 0; ii < B_neigs; ii++) {
    3869:           PetscCall(PetscViewerASCIISynchronizedPrintf(pcbddc->dbg_viewer, "   -> Eigenvector (old basis) %" PetscInt_FMT "/%" PetscBLASInt_FMT " (%" PetscBLASInt_FMT ")\n", ii, B_neigs, B_N));
    3870:           for (j = 0; j < B_N; j++) {
    3871: #if defined(PETSC_USE_COMPLEX)
    3872:             PetscReal r = PetscRealPart(eigv[(ii + eigs_start) * subset_size + j]);
    3873:             PetscReal c = PetscImaginaryPart(eigv[(ii + eigs_start) * subset_size + j]);
    3874:             PetscCall(PetscViewerASCIISynchronizedPrintf(pcbddc->dbg_viewer, "       %1.4e + %1.4e i\n", (double)r, (double)c));
    3875: #else
    3876:             PetscCall(PetscViewerASCIISynchronizedPrintf(pcbddc->dbg_viewer, "       %1.4e\n", (double)(eigv[(ii + eigs_start) * subset_size + j])));
    3877: #endif
    3878:           }
    3879:         }
    3880:       }
    3881:       PetscCall(KSPGetOperators(sub_schurs->change[i], &change, NULL));
    3882:       PetscCall(MatCreateSeqDense(PETSC_COMM_SELF, subset_size, B_neigs, eigv + eigs_start * subset_size, &phit));
    3883:       PetscCall(MatMatMult(change, phit, MAT_INITIAL_MATRIX, PETSC_DETERMINE, &phi));
    3884:       PetscCall(MatCopy(phi, phit, SAME_NONZERO_PATTERN));
    3885:       PetscCall(MatDestroy(&phit));
    3886:       PetscCall(MatDestroy(&phi));
    3887:     }
    3888:     maxneigs                               = PetscMax(B_neigs, maxneigs);
    3889:     pcbddc->adaptive_constraints_n[i + nv] = B_neigs;
    3890:     if (B_neigs) {
    3891:       PetscCall(PetscArraycpy(pcbddc->adaptive_constraints_data + pcbddc->adaptive_constraints_data_ptr[cum], eigv + eigs_start * subset_size, B_neigs * subset_size));

    3893:       if (pcbddc->dbg_flag > 1) {
    3894:         PetscInt ii;
    3895:         for (ii = 0; ii < B_neigs; ii++) {
    3896:           PetscCall(PetscViewerASCIISynchronizedPrintf(pcbddc->dbg_viewer, "   -> Eigenvector %" PetscInt_FMT "/%" PetscBLASInt_FMT " (%" PetscBLASInt_FMT ")\n", ii, B_neigs, B_N));
    3897:           for (j = 0; j < B_N; j++) {
    3898: #if defined(PETSC_USE_COMPLEX)
    3899:             PetscReal r = PetscRealPart(pcbddc->adaptive_constraints_data[ii * subset_size + j + pcbddc->adaptive_constraints_data_ptr[cum]]);
    3900:             PetscReal c = PetscImaginaryPart(pcbddc->adaptive_constraints_data[ii * subset_size + j + pcbddc->adaptive_constraints_data_ptr[cum]]);
    3901:             PetscCall(PetscViewerASCIISynchronizedPrintf(pcbddc->dbg_viewer, "       %1.4e + %1.4e i\n", (double)r, (double)c));
    3902: #else
    3903:             PetscCall(PetscViewerASCIISynchronizedPrintf(pcbddc->dbg_viewer, "       %1.4e\n", (double)PetscRealPart(pcbddc->adaptive_constraints_data[ii * subset_size + j + pcbddc->adaptive_constraints_data_ptr[cum]])));
    3904: #endif
    3905:           }
    3906:         }
    3907:       }
    3908:       PetscCall(PetscArraycpy(pcbddc->adaptive_constraints_idxs + pcbddc->adaptive_constraints_idxs_ptr[cum], idxs, subset_size));
    3909:       pcbddc->adaptive_constraints_idxs_ptr[cum + 1] = pcbddc->adaptive_constraints_idxs_ptr[cum] + subset_size;
    3910:       pcbddc->adaptive_constraints_data_ptr[cum + 1] = pcbddc->adaptive_constraints_data_ptr[cum] + subset_size * B_neigs;
    3911:       cum++;
    3912:     }
    3913:     PetscCall(ISRestoreIndices(sub_schurs->is_subs[i], &idxs));
    3914:     /* shift for next computation */
    3915:     cumarray += subset_size * subset_size;
    3916:   }
    3917:   if (pcbddc->dbg_flag) PetscCall(PetscViewerFlush(pcbddc->dbg_viewer));

    3919:   if (mss) {
    3920:     if (sub_schurs->gdsw) {
    3921:       PetscCall(MatSeqAIJGetArray(sub_schurs->sum_S_Ej_all, &Sarray));
    3922:       PetscCall(MatSeqAIJGetArray(sub_schurs->sum_S_Ej_tilda_all, &Starray));
    3923:     } else {
    3924:       PetscCall(MatSeqAIJRestoreArray(sub_schurs->sum_S_Ej_inv_all, &Sarray));
    3925:       PetscCall(MatSeqAIJRestoreArray(sub_schurs->sum_S_Ej_tilda_all, &Starray));
    3926:       /* destroy matrices (junk) */
    3927:       PetscCall(MatDestroy(&sub_schurs->sum_S_Ej_inv_all));
    3928:       PetscCall(MatDestroy(&sub_schurs->sum_S_Ej_tilda_all));
    3929:     }
    3930:   }
    3931:   if (allocated_S_St) PetscCall(PetscFree2(S, St));
    3932:   PetscCall(PetscFree5(eigv, eigs, work, B_iwork, B_ifail));
    3933: #if defined(PETSC_USE_COMPLEX)
    3934:   PetscCall(PetscFree(rwork));
    3935: #endif
    3936:   if (pcbddc->dbg_flag) {
    3937:     PetscInt maxneigs_r;
    3938:     PetscCallMPI(MPIU_Allreduce(&maxneigs, &maxneigs_r, 1, MPIU_INT, MPI_MAX, PetscObjectComm((PetscObject)pc)));
    3939:     PetscCall(PetscViewerASCIIPrintf(pcbddc->dbg_viewer, "Maximum number of constraints per cc %" PetscInt_FMT "\n", maxneigs_r));
    3940:   }
    3941:   PetscCall(PetscLogEventEnd(PC_BDDC_AdaptiveSetUp[pcbddc->current_level], pc, 0, 0, 0));
    3942:   PetscFunctionReturn(PETSC_SUCCESS);
    3943: }

    3945: PetscErrorCode PCBDDCSetUpSolvers(PC pc)
    3946: {
    3947:   Mat coarse_submat;

    3949:   PetscFunctionBegin;
    3950:   /* Setup local scatters R_to_B and (optionally) R_to_D */
    3951:   /* PCBDDCSetUpLocalWorkVectors should be called first! */
    3952:   PetscCall(PCBDDCSetUpLocalScatters(pc));

    3954:   /* Setup local neumann solver ksp_R */
    3955:   /* PCBDDCSetUpLocalScatters should be called first! */
    3956:   PetscCall(PCBDDCSetUpLocalSolvers(pc, PETSC_FALSE, PETSC_TRUE));

    3958:   /*
    3959:      Setup local correction and local part of coarse basis.
    3960:      Gives back the dense local part of the coarse matrix in column major ordering
    3961:   */
    3962:   PetscCall(PCBDDCSetUpCorrection(pc, &coarse_submat));

    3964:   /* Compute total number of coarse nodes and setup coarse solver */
    3965:   PetscCall(PCBDDCSetUpCoarseSolver(pc, coarse_submat));
    3966:   PetscCall(MatDestroy(&coarse_submat));
    3967:   PetscFunctionReturn(PETSC_SUCCESS);
    3968: }

    3970: PetscErrorCode PCBDDCResetCustomization(PC pc)
    3971: {
    3972:   PC_BDDC *pcbddc = (PC_BDDC *)pc->data;

    3974:   PetscFunctionBegin;
    3975:   PetscCall(ISDestroy(&pcbddc->user_primal_vertices));
    3976:   PetscCall(ISDestroy(&pcbddc->user_primal_vertices_local));
    3977:   PetscCall(ISDestroy(&pcbddc->NeumannBoundaries));
    3978:   PetscCall(ISDestroy(&pcbddc->NeumannBoundariesLocal));
    3979:   PetscCall(ISDestroy(&pcbddc->DirichletBoundaries));
    3980:   PetscCall(MatNullSpaceDestroy(&pcbddc->onearnullspace));
    3981:   PetscCall(PetscFree(pcbddc->onearnullvecs_state));
    3982:   PetscCall(ISDestroy(&pcbddc->DirichletBoundariesLocal));
    3983:   PetscCall(PCBDDCSetDofsSplitting(pc, 0, NULL));
    3984:   PetscCall(PCBDDCSetDofsSplittingLocal(pc, 0, NULL));
    3985:   PetscFunctionReturn(PETSC_SUCCESS);
    3986: }

    3988: PetscErrorCode PCBDDCResetTopography(PC pc)
    3989: {
    3990:   PC_BDDC *pcbddc = (PC_BDDC *)pc->data;
    3991:   PetscInt i;

    3993:   PetscFunctionBegin;
    3994:   PetscCall(MatDestroy(&pcbddc->nedcG));
    3995:   PetscCall(ISDestroy(&pcbddc->nedclocal));
    3996:   PetscCall(MatDestroy(&pcbddc->discretegradient));
    3997:   PetscCall(MatDestroy(&pcbddc->user_ChangeOfBasisMatrix));
    3998:   PetscCall(MatDestroy(&pcbddc->ChangeOfBasisMatrix));
    3999:   PetscCall(MatDestroy(&pcbddc->switch_static_change));
    4000:   PetscCall(VecDestroy(&pcbddc->work_change));
    4001:   PetscCall(MatDestroy(&pcbddc->ConstraintMatrix));
    4002:   PetscCall(MatDestroy(&pcbddc->divudotp));
    4003:   PetscCall(ISDestroy(&pcbddc->divudotp_vl2l));
    4004:   PetscCall(PCBDDCGraphDestroy(&pcbddc->mat_graph));
    4005:   for (i = 0; i < pcbddc->n_local_subs; i++) PetscCall(ISDestroy(&pcbddc->local_subs[i]));
    4006:   pcbddc->n_local_subs = 0;
    4007:   PetscCall(PetscFree(pcbddc->local_subs));
    4008:   PetscCall(PCBDDCSubSchursDestroy(&pcbddc->sub_schurs));
    4009:   pcbddc->graphanalyzed        = PETSC_FALSE;
    4010:   pcbddc->recompute_topography = PETSC_TRUE;
    4011:   pcbddc->corner_selected      = PETSC_FALSE;
    4012:   PetscFunctionReturn(PETSC_SUCCESS);
    4013: }

    4015: PetscErrorCode PCBDDCResetSolvers(PC pc)
    4016: {
    4017:   PC_BDDC *pcbddc = (PC_BDDC *)pc->data;

    4019:   PetscFunctionBegin;
    4020:   PetscCall(VecDestroy(&pcbddc->coarse_vec));
    4021:   PetscCall(MatDestroy(&pcbddc->coarse_phi_B));
    4022:   PetscCall(MatDestroy(&pcbddc->coarse_phi_D));
    4023:   PetscCall(MatDestroy(&pcbddc->coarse_psi_B));
    4024:   PetscCall(MatDestroy(&pcbddc->coarse_psi_D));
    4025:   PetscCall(VecDestroy(&pcbddc->vec1_P));
    4026:   PetscCall(VecDestroy(&pcbddc->vec1_C));
    4027:   PetscCall(MatDestroy(&pcbddc->local_auxmat2));
    4028:   PetscCall(MatDestroy(&pcbddc->local_auxmat1));
    4029:   PetscCall(VecDestroy(&pcbddc->vec1_R));
    4030:   PetscCall(VecDestroy(&pcbddc->vec2_R));
    4031:   PetscCall(ISDestroy(&pcbddc->is_R_local));
    4032:   PetscCall(VecScatterDestroy(&pcbddc->R_to_B));
    4033:   PetscCall(VecScatterDestroy(&pcbddc->R_to_D));
    4034:   PetscCall(VecScatterDestroy(&pcbddc->coarse_loc_to_glob));
    4035:   PetscCall(KSPReset(pcbddc->ksp_D));
    4036:   PetscCall(KSPReset(pcbddc->ksp_R));
    4037:   PetscCall(KSPReset(pcbddc->coarse_ksp));
    4038:   PetscCall(MatDestroy(&pcbddc->local_mat));
    4039:   PetscCall(PetscFree(pcbddc->primal_indices_local_idxs));
    4040:   PetscCall(PetscFree2(pcbddc->local_primal_ref_node, pcbddc->local_primal_ref_mult));
    4041:   PetscCall(PetscFree(pcbddc->global_primal_indices));
    4042:   PetscCall(ISDestroy(&pcbddc->coarse_subassembling));
    4043:   PetscCall(MatDestroy(&pcbddc->benign_change));
    4044:   PetscCall(VecDestroy(&pcbddc->benign_vec));
    4045:   PetscCall(PCBDDCBenignShellMat(pc, PETSC_TRUE));
    4046:   PetscCall(MatDestroy(&pcbddc->benign_B0));
    4047:   PetscCall(PetscSFDestroy(&pcbddc->benign_sf));
    4048:   if (pcbddc->benign_zerodiag_subs) {
    4049:     PetscInt i;
    4050:     for (i = 0; i < pcbddc->benign_n; i++) PetscCall(ISDestroy(&pcbddc->benign_zerodiag_subs[i]));
    4051:     PetscCall(PetscFree(pcbddc->benign_zerodiag_subs));
    4052:   }
    4053:   PetscCall(PetscFree3(pcbddc->benign_p0_lidx, pcbddc->benign_p0_gidx, pcbddc->benign_p0));
    4054:   PetscFunctionReturn(PETSC_SUCCESS);
    4055: }

    4057: PetscErrorCode PCBDDCSetUpLocalWorkVectors(PC pc)
    4058: {
    4059:   PC_BDDC *pcbddc = (PC_BDDC *)pc->data;
    4060:   PC_IS   *pcis   = (PC_IS *)pc->data;
    4061:   VecType  impVecType;
    4062:   PetscInt n_constraints, n_R, old_size;

    4064:   PetscFunctionBegin;
    4065:   n_constraints = pcbddc->local_primal_size - pcbddc->benign_n - pcbddc->n_vertices;
    4066:   n_R           = pcis->n - pcbddc->n_vertices;
    4067:   PetscCall(VecGetType(pcis->vec1_N, &impVecType));
    4068:   /* local work vectors (try to avoid unneeded work)*/
    4069:   /* R nodes */
    4070:   old_size = -1;
    4071:   if (pcbddc->vec1_R) PetscCall(VecGetSize(pcbddc->vec1_R, &old_size));
    4072:   if (n_R != old_size) {
    4073:     PetscCall(VecDestroy(&pcbddc->vec1_R));
    4074:     PetscCall(VecDestroy(&pcbddc->vec2_R));
    4075:     PetscCall(VecCreate(PetscObjectComm((PetscObject)pcis->vec1_N), &pcbddc->vec1_R));
    4076:     PetscCall(VecSetSizes(pcbddc->vec1_R, PETSC_DECIDE, n_R));
    4077:     PetscCall(VecSetType(pcbddc->vec1_R, impVecType));
    4078:     PetscCall(VecDuplicate(pcbddc->vec1_R, &pcbddc->vec2_R));
    4079:   }
    4080:   /* local primal dofs */
    4081:   old_size = -1;
    4082:   if (pcbddc->vec1_P) PetscCall(VecGetSize(pcbddc->vec1_P, &old_size));
    4083:   if (pcbddc->local_primal_size != old_size) {
    4084:     PetscCall(VecDestroy(&pcbddc->vec1_P));
    4085:     PetscCall(VecCreate(PetscObjectComm((PetscObject)pcis->vec1_N), &pcbddc->vec1_P));
    4086:     PetscCall(VecSetSizes(pcbddc->vec1_P, PETSC_DECIDE, pcbddc->local_primal_size));
    4087:     PetscCall(VecSetType(pcbddc->vec1_P, impVecType));
    4088:   }
    4089:   /* local explicit constraints */
    4090:   old_size = -1;
    4091:   if (pcbddc->vec1_C) PetscCall(VecGetSize(pcbddc->vec1_C, &old_size));
    4092:   if (n_constraints && n_constraints != old_size) {
    4093:     PetscCall(VecDestroy(&pcbddc->vec1_C));
    4094:     PetscCall(VecCreate(PetscObjectComm((PetscObject)pcis->vec1_N), &pcbddc->vec1_C));
    4095:     PetscCall(VecSetSizes(pcbddc->vec1_C, PETSC_DECIDE, n_constraints));
    4096:     PetscCall(VecSetType(pcbddc->vec1_C, impVecType));
    4097:   }
    4098:   PetscFunctionReturn(PETSC_SUCCESS);
    4099: }

    4101: static PetscErrorCode MatSetValuesSubMat(Mat A, Mat S, PetscInt nr, const PetscInt rows[], PetscInt nc, const PetscInt cols[], InsertMode imode)
    4102: {
    4103:   PetscBool          flg;
    4104:   const PetscScalar *a;

    4106:   PetscFunctionBegin;
    4107:   PetscCall(PetscObjectBaseTypeCompare((PetscObject)S, MATSEQDENSE, &flg));
    4108:   if (flg) {
    4109:     PetscCall(MatDenseGetArrayRead(S, &a));
    4110:     PetscCall(MatSetOption(A, MAT_ROW_ORIENTED, PETSC_FALSE));
    4111:     PetscCall(MatSetValues(A, nr, rows, nc, cols, a, imode));
    4112:     PetscCall(MatSetOption(A, MAT_ROW_ORIENTED, PETSC_TRUE));
    4113:     PetscCall(MatDenseRestoreArrayRead(S, &a));
    4114:   } else {
    4115:     const PetscInt *ii, *jj;
    4116:     PetscInt        n;
    4117:     PetscInt        buf[8192], *bufc = NULL;
    4118:     PetscBool       freeb = PETSC_FALSE;
    4119:     Mat             Sm    = S;

    4121:     PetscCall(PetscObjectBaseTypeCompare((PetscObject)S, MATSEQAIJ, &flg));
    4122:     if (!flg) PetscCall(MatConvert(S, MATSEQAIJ, MAT_INITIAL_MATRIX, &Sm));
    4123:     else PetscCall(PetscObjectReference((PetscObject)S));
    4124:     PetscCall(MatSeqAIJGetArrayRead(Sm, &a));
    4125:     PetscCall(MatGetRowIJ(Sm, 0, PETSC_FALSE, PETSC_FALSE, &n, &ii, &jj, &flg));
    4126:     PetscCheck(flg, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Cannot get IJ structure");
    4127:     if (nc <= (PetscInt)PETSC_STATIC_ARRAY_LENGTH(buf)) {
    4128:       bufc = buf;
    4129:     } else {
    4130:       PetscCall(PetscMalloc1(nc, &bufc));
    4131:       freeb = PETSC_TRUE;
    4132:     }

    4134:     for (PetscInt i = 0; i < n; i++) {
    4135:       const PetscInt nci = ii[i + 1] - ii[i];

    4137:       for (PetscInt j = 0; j < nci; j++) bufc[j] = cols[jj[ii[i] + j]];
    4138:       PetscCall(MatSetValues(A, 1, rows + i, nci, bufc, a + ii[i], imode));
    4139:     }
    4140:     PetscCall(MatRestoreRowIJ(Sm, 0, PETSC_FALSE, PETSC_FALSE, &n, &ii, &jj, &flg));
    4141:     PetscCall(MatSeqAIJRestoreArrayRead(Sm, &a));
    4142:     PetscCall(MatDestroy(&Sm));
    4143:     if (freeb) PetscCall(PetscFree(bufc));
    4144:   }
    4145:   PetscCall(MatAssemblyBegin(A, MAT_FLUSH_ASSEMBLY));
    4146:   PetscCall(MatAssemblyEnd(A, MAT_FLUSH_ASSEMBLY));
    4147:   PetscFunctionReturn(PETSC_SUCCESS);
    4148: }

    4150: static PetscErrorCode MatCreateSeqAIJFromDenseExpand(Mat D, PetscInt n, const PetscInt j[], Mat *mat)
    4151: {
    4152:   Mat_SeqAIJ        *aij;
    4153:   PetscInt          *ii, *jj;
    4154:   PetscScalar       *aa;
    4155:   PetscInt           nnz = 0, m, nc;
    4156:   const PetscScalar *a;
    4157:   const PetscScalar  zero = 0.0;

    4159:   PetscFunctionBegin;
    4160:   PetscCall(MatGetLocalSize(D, &m, &nc));
    4161:   PetscCall(MatDenseGetArrayRead(D, &a));
    4162:   PetscCall(PetscMalloc1(m + 1, &ii));
    4163:   PetscCall(PetscMalloc1(m * nc, &jj));
    4164:   PetscCall(PetscMalloc1(m * nc, &aa));
    4165:   ii[0] = 0;
    4166:   for (PetscInt k = 0; k < m; k++) {
    4167:     for (PetscInt s = 0; s < nc; s++) {
    4168:       const PetscInt    c = s + k * nc;
    4169:       const PetscScalar v = a[k + s * m];

    4171:       if (PetscUnlikely(j[c] < 0 || v == zero)) continue;
    4172:       jj[nnz] = j[c];
    4173:       aa[nnz] = a[k + s * m];
    4174:       nnz++;
    4175:     }
    4176:     ii[k + 1] = nnz;
    4177:   }

    4179:   PetscCall(MatCreateSeqAIJWithArrays(PetscObjectComm((PetscObject)D), m, n, ii, jj, aa, mat));
    4180:   PetscCall(MatDenseRestoreArrayRead(D, &a));

    4182:   aij          = (Mat_SeqAIJ *)(*mat)->data;
    4183:   aij->free_a  = PETSC_TRUE;
    4184:   aij->free_ij = PETSC_TRUE;
    4185:   PetscFunctionReturn(PETSC_SUCCESS);
    4186: }

    4188: /* adapted from MatInvertVariableBlockDiagonal_SeqAIJ */
    4189: static PetscErrorCode MatSeqAIJInvertVariableBlockDiagonalMat(Mat A, PetscInt nblocks, const PetscInt *bsizes, Mat *B)
    4190: {
    4191:   PetscInt        n = A->rmap->n, ncnt = 0, ncnt2 = 0, bsizemax = 0, *v_pivots = NULL;
    4192:   const PetscBool allowzeropivot    = PETSC_FALSE;
    4193:   PetscBool       zeropivotdetected = PETSC_FALSE;
    4194:   const PetscReal shift             = 0.0;
    4195:   PetscInt        ipvt[5], *ii, *jj, *indi, *indj;
    4196:   PetscScalar     work[25], *v_work = NULL, *aa, *diag;
    4197:   PetscLogDouble  flops = 0.0;

    4199:   PetscFunctionBegin;
    4200:   PetscCheck(A->rmap->n == A->cmap->n, PETSC_COMM_SELF, PETSC_ERR_ARG_SIZ, "Not for rectangular matrices");
    4201:   for (PetscInt i = 0; i < nblocks; i++) {
    4202:     ncnt += bsizes[i];
    4203:     ncnt2 += PetscSqr(bsizes[i]);
    4204:   }
    4205:   PetscCheck(ncnt == n, PETSC_COMM_SELF, PETSC_ERR_ARG_SIZ, "Total blocksizes %" PetscInt_FMT " doesn't match number matrix rows %" PetscInt_FMT, ncnt, n);
    4206:   for (PetscInt i = 0; i < nblocks; i++) bsizemax = PetscMax(bsizemax, bsizes[i]);
    4207:   if (bsizemax > 7) PetscCall(PetscMalloc2(bsizemax, &v_work, bsizemax, &v_pivots));

    4209:   PetscCall(PetscMalloc1(n + 1, &ii));
    4210:   PetscCall(PetscMalloc1(ncnt2, &jj));
    4211:   PetscCall(PetscCalloc1(ncnt2, &aa));

    4213:   ncnt  = 0;
    4214:   ii[0] = 0;
    4215:   indi  = ii;
    4216:   indj  = jj;
    4217:   diag  = aa;
    4218:   for (PetscInt i = 0; i < nblocks; i++) {
    4219:     const PetscInt bs = bsizes[i];

    4221:     for (PetscInt k = 0; k < bs; k++) {
    4222:       indi[k + 1] = indi[k] + bs;
    4223:       for (PetscInt j = 0; j < bs; j++) indj[k * bs + j] = ncnt + j;
    4224:     }
    4225:     PetscCall(MatGetValues(A, bs, indj, bs, indj, diag));
    4226:     switch (bs) {
    4227:     case 1:
    4228:       *diag = 1.0 / (*diag);
    4229:       break;
    4230:     case 2:
    4231:       PetscCall(PetscKernel_A_gets_inverse_A_2(diag, shift, allowzeropivot, &zeropivotdetected));
    4232:       break;
    4233:     case 3:
    4234:       PetscCall(PetscKernel_A_gets_inverse_A_3(diag, shift, allowzeropivot, &zeropivotdetected));
    4235:       break;
    4236:     case 4:
    4237:       PetscCall(PetscKernel_A_gets_inverse_A_4(diag, shift, allowzeropivot, &zeropivotdetected));
    4238:       break;
    4239:     case 5:
    4240:       PetscCall(PetscKernel_A_gets_inverse_A_5(diag, ipvt, work, shift, allowzeropivot, &zeropivotdetected));
    4241:       break;
    4242:     case 6:
    4243:       PetscCall(PetscKernel_A_gets_inverse_A_6(diag, shift, allowzeropivot, &zeropivotdetected));
    4244:       break;
    4245:     case 7:
    4246:       PetscCall(PetscKernel_A_gets_inverse_A_7(diag, shift, allowzeropivot, &zeropivotdetected));
    4247:       break;
    4248:     default:
    4249:       PetscCall(PetscKernel_A_gets_inverse_A(bs, diag, v_pivots, v_work, allowzeropivot, &zeropivotdetected));
    4250:     }
    4251:     ncnt += bs;
    4252:     flops += 2.0 * PetscPowInt(bs, 3) / 3.0;
    4253:     diag += bs * bs;
    4254:     indj += bs * bs;
    4255:     indi += bs;
    4256:   }
    4257:   PetscCall(PetscLogFlops(flops));
    4258:   PetscCall(PetscFree2(v_work, v_pivots));
    4259:   PetscCall(MatCreateSeqAIJWithArrays(PetscObjectComm((PetscObject)A), n, n, ii, jj, aa, B));
    4260:   {
    4261:     Mat_SeqAIJ *aij = (Mat_SeqAIJ *)(*B)->data;
    4262:     aij->free_a     = PETSC_TRUE;
    4263:     aij->free_ij    = PETSC_TRUE;
    4264:   }
    4265:   PetscFunctionReturn(PETSC_SUCCESS);
    4266: }

    4268: PetscErrorCode PCBDDCSetUpCorrection(PC pc, Mat *coarse_submat)
    4269: {
    4270:   PC_IS          *pcis       = (PC_IS *)pc->data;
    4271:   PC_BDDC        *pcbddc     = (PC_BDDC *)pc->data;
    4272:   PCBDDCGraph     graph      = pcbddc->mat_graph;
    4273:   PCBDDCSubSchurs sub_schurs = pcbddc->sub_schurs;
    4274:   /* submatrices of local problem */
    4275:   Mat A_RV = NULL, A_VR, A_VV, local_auxmat2_R = NULL;
    4276:   /* submatrices of local coarse problem */
    4277:   Mat S_CV = NULL, S_VC = NULL, S_CC = NULL;
    4278:   /* working matrices */
    4279:   Mat C_CR;

    4281:   /* additional working stuff */
    4282:   PC              pc_R;
    4283:   IS              is_R, is_V, is_C;
    4284:   const PetscInt *idx_V, *idx_C;
    4285:   Mat             F, Brhs = NULL;
    4286:   Vec             dummy_vec;
    4287:   PetscBool       isPreonly, isLU, isCHOL, need_benign_correction, sparserhs;
    4288:   PetscInt       *idx_V_B;
    4289:   PetscInt        lda_rhs, n_vertices, n_constraints, *p0_lidx_I;
    4290:   PetscInt        n_eff_vertices, n_eff_constraints;
    4291:   PetscInt        i, n_R, n_D, n_B;
    4292:   PetscScalar     one = 1.0, m_one = -1.0;

    4294:   /* Multi-element support */
    4295:   PetscBool multi_element = graph->multi_element;
    4296:   PetscInt *V_to_eff_V = NULL, *C_to_eff_C = NULL;
    4297:   PetscInt *B_eff_V_J = NULL, *R_eff_V_J = NULL, *B_eff_C_J = NULL, *R_eff_C_J = NULL;
    4298:   IS        is_C_perm = NULL;
    4299:   PetscInt  n_C_bss = 0, *C_bss = NULL;
    4300:   Mat       coarse_phi_multi;

    4302:   PetscFunctionBegin;
    4303:   PetscCheck(pcbddc->symmetric_primal || !pcbddc->benign_n, PETSC_COMM_SELF, PETSC_ERR_SUP, "Non-symmetric primal basis computation with benign trick not yet implemented");
    4304:   PetscCall(PetscLogEventBegin(PC_BDDC_CorrectionSetUp[pcbddc->current_level], pc, 0, 0, 0));

    4306:   /* Set Non-overlapping dimensions */
    4307:   n_vertices    = pcbddc->n_vertices;
    4308:   n_constraints = pcbddc->local_primal_size - pcbddc->benign_n - n_vertices;
    4309:   n_B           = pcis->n_B;
    4310:   n_D           = pcis->n - n_B;
    4311:   n_R           = pcis->n - n_vertices;

    4313:   /* vertices in boundary numbering */
    4314:   PetscCall(PetscMalloc1(n_vertices, &idx_V_B));
    4315:   PetscCall(ISGlobalToLocalMappingApply(pcis->BtoNmap, IS_GTOLM_DROP, n_vertices, pcbddc->local_primal_ref_node, &i, idx_V_B));
    4316:   PetscCheck(i == n_vertices, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Error in boundary numbering for BDDC vertices! %" PetscInt_FMT " != %" PetscInt_FMT, n_vertices, i);

    4318:   /* these two cases still need to be optimized */
    4319:   if (pcbddc->benign_saddle_point || !pcbddc->symmetric_primal) multi_element = PETSC_FALSE;

    4321:   /* Subdomain contribution (Non-overlapping) to coarse matrix  */
    4322:   if (multi_element) {
    4323:     PetscCheck(!pcbddc->benign_n, PETSC_COMM_SELF, PETSC_ERR_SUP, "Not yet implemented");

    4325:     PetscCall(MatCreate(PETSC_COMM_SELF, coarse_submat));
    4326:     PetscCall(MatSetSizes(*coarse_submat, pcbddc->local_primal_size, pcbddc->local_primal_size, pcbddc->local_primal_size, pcbddc->local_primal_size));
    4327:     PetscCall(MatSetType(*coarse_submat, MATSEQAIJ));
    4328:     PetscCall(MatSetOption(*coarse_submat, MAT_IGNORE_ZERO_ENTRIES, PETSC_TRUE));
    4329:     PetscCall(MatSetOption(*coarse_submat, MAT_NEW_NONZERO_LOCATION_ERR, PETSC_TRUE));

    4331:     /* group vertices and constraints by subdomain id */
    4332:     const PetscInt *vidxs = pcbddc->primal_indices_local_idxs;
    4333:     const PetscInt *cidxs = pcbddc->primal_indices_local_idxs + n_vertices;
    4334:     PetscInt       *count_eff, *V_eff_to_V, *C_eff_to_C, *nnz;
    4335:     PetscInt        n_el = PetscMax(graph->n_local_subs, 1);

    4337:     PetscCall(PetscCalloc1(2 * n_el, &count_eff));
    4338:     PetscCall(PetscMalloc1(n_vertices, &V_to_eff_V));
    4339:     PetscCall(PetscMalloc1(n_constraints, &C_to_eff_C));
    4340:     for (PetscInt i = 0; i < n_vertices; i++) {
    4341:       PetscInt s = 2 * graph->nodes[vidxs[i]].local_sub;

    4343:       V_to_eff_V[i] = count_eff[s];
    4344:       count_eff[s] += 1;
    4345:     }
    4346:     for (PetscInt i = 0; i < n_constraints; i++) {
    4347:       PetscInt s = 2 * graph->nodes[cidxs[i]].local_sub + 1;

    4349:       C_to_eff_C[i] = count_eff[s];
    4350:       count_eff[s] += 1;
    4351:     }

    4353:     /* preallocation */
    4354:     PetscCall(PetscMalloc1(n_vertices + n_constraints, &nnz));
    4355:     for (PetscInt i = 0; i < n_vertices; i++) {
    4356:       PetscInt s = 2 * graph->nodes[vidxs[i]].local_sub;

    4358:       nnz[i] = count_eff[s] + count_eff[s + 1];
    4359:     }
    4360:     for (PetscInt i = 0; i < n_constraints; i++) {
    4361:       PetscInt s = 2 * graph->nodes[cidxs[i]].local_sub;

    4363:       nnz[i + n_vertices] = count_eff[s] + count_eff[s + 1];
    4364:     }
    4365:     PetscCall(MatSeqAIJSetPreallocation(*coarse_submat, 0, nnz));
    4366:     PetscCall(PetscFree(nnz));

    4368:     n_eff_vertices    = 0;
    4369:     n_eff_constraints = 0;
    4370:     for (PetscInt i = 0; i < n_el; i++) {
    4371:       n_eff_vertices       = PetscMax(n_eff_vertices, count_eff[2 * i]);
    4372:       n_eff_constraints    = PetscMax(n_eff_constraints, count_eff[2 * i + 1]);
    4373:       count_eff[2 * i]     = 0;
    4374:       count_eff[2 * i + 1] = 0;
    4375:     }

    4377:     const PetscInt *idx;
    4378:     PetscCall(PetscMalloc2(n_el * n_eff_vertices, &V_eff_to_V, n_el * n_eff_constraints, &C_eff_to_C));

    4380:     for (PetscInt i = 0; i < n_vertices; i++) {
    4381:       const PetscInt e = graph->nodes[vidxs[i]].local_sub;
    4382:       const PetscInt s = 2 * e;

    4384:       V_eff_to_V[e * n_eff_vertices + count_eff[s]] = i;
    4385:       count_eff[s] += 1;
    4386:     }
    4387:     for (PetscInt i = 0; i < n_constraints; i++) {
    4388:       const PetscInt e = graph->nodes[cidxs[i]].local_sub;
    4389:       const PetscInt s = 2 * e + 1;

    4391:       C_eff_to_C[e * n_eff_constraints + count_eff[s]] = i;
    4392:       count_eff[s] += 1;
    4393:     }

    4395:     PetscCall(PetscMalloc1(n_R * n_eff_vertices, &R_eff_V_J));
    4396:     PetscCall(PetscMalloc1(n_R * n_eff_constraints, &R_eff_C_J));
    4397:     PetscCall(PetscMalloc1(n_B * n_eff_vertices, &B_eff_V_J));
    4398:     PetscCall(PetscMalloc1(n_B * n_eff_constraints, &B_eff_C_J));
    4399:     for (PetscInt i = 0; i < n_R * n_eff_vertices; i++) R_eff_V_J[i] = -1;
    4400:     for (PetscInt i = 0; i < n_R * n_eff_constraints; i++) R_eff_C_J[i] = -1;
    4401:     for (PetscInt i = 0; i < n_B * n_eff_vertices; i++) B_eff_V_J[i] = -1;
    4402:     for (PetscInt i = 0; i < n_B * n_eff_constraints; i++) B_eff_C_J[i] = -1;

    4404:     PetscCall(ISGetIndices(pcbddc->is_R_local, &idx));
    4405:     for (PetscInt i = 0; i < n_R; i++) {
    4406:       const PetscInt e = graph->nodes[idx[i]].local_sub;
    4407:       const PetscInt s = 2 * e;
    4408:       PetscInt       j;

    4410:       for (j = 0; j < count_eff[s]; j++) R_eff_V_J[i * n_eff_vertices + j] = V_eff_to_V[e * n_eff_vertices + j];
    4411:       for (j = 0; j < count_eff[s + 1]; j++) R_eff_C_J[i * n_eff_constraints + j] = C_eff_to_C[e * n_eff_constraints + j];
    4412:     }
    4413:     PetscCall(ISRestoreIndices(pcbddc->is_R_local, &idx));
    4414:     PetscCall(ISGetIndices(pcis->is_B_local, &idx));
    4415:     for (PetscInt i = 0; i < n_B; i++) {
    4416:       const PetscInt e = graph->nodes[idx[i]].local_sub;
    4417:       const PetscInt s = 2 * e;
    4418:       PetscInt       j;

    4420:       for (j = 0; j < count_eff[s]; j++) B_eff_V_J[i * n_eff_vertices + j] = V_eff_to_V[e * n_eff_vertices + j];
    4421:       for (j = 0; j < count_eff[s + 1]; j++) B_eff_C_J[i * n_eff_constraints + j] = C_eff_to_C[e * n_eff_constraints + j];
    4422:     }
    4423:     PetscCall(ISRestoreIndices(pcis->is_B_local, &idx));

    4425:     /* permutation and blocksizes for block invert of S_CC */
    4426:     PetscInt *idxp;

    4428:     PetscCall(PetscMalloc1(n_constraints, &idxp));
    4429:     PetscCall(PetscMalloc1(n_el, &C_bss));
    4430:     n_C_bss = 0;
    4431:     for (PetscInt e = 0, cnt = 0; e < n_el; e++) {
    4432:       const PetscInt nc = count_eff[2 * e + 1];

    4434:       if (nc) C_bss[n_C_bss++] = nc;
    4435:       for (PetscInt c = 0; c < nc; c++) idxp[cnt + c] = C_eff_to_C[e * n_eff_constraints + c];
    4436:       cnt += nc;
    4437:     }

    4439:     PetscCall(ISCreateGeneral(PETSC_COMM_SELF, n_constraints, idxp, PETSC_OWN_POINTER, &is_C_perm));

    4441:     PetscCall(PetscFree2(V_eff_to_V, C_eff_to_C));
    4442:     PetscCall(PetscFree(count_eff));
    4443:   } else {
    4444:     PetscCall(MatCreateSeqDense(PETSC_COMM_SELF, pcbddc->local_primal_size, pcbddc->local_primal_size, NULL, coarse_submat));
    4445:     n_eff_constraints = n_constraints;
    4446:     n_eff_vertices    = n_vertices;
    4447:   }

    4449:   /* determine if can use MatSolve routines instead of calling KSPSolve on ksp_R */
    4450:   PetscCall(KSPGetPC(pcbddc->ksp_R, &pc_R));
    4451:   PetscCall(PCSetUp(pc_R));
    4452:   PetscCall(PetscObjectTypeCompare((PetscObject)pcbddc->ksp_R, KSPPREONLY, &isPreonly));
    4453:   PetscCall(PetscObjectTypeCompare((PetscObject)pc_R, PCLU, &isLU));
    4454:   PetscCall(PetscObjectTypeCompare((PetscObject)pc_R, PCCHOLESKY, &isCHOL));
    4455:   lda_rhs                = n_R;
    4456:   need_benign_correction = PETSC_FALSE;
    4457:   F                      = NULL;
    4458:   if (isPreonly && (isLU || isCHOL)) {
    4459:     PetscCall(PCFactorGetMatrix(pc_R, &F));
    4460:   } else if (sub_schurs && sub_schurs->reuse_solver) {
    4461:     PCBDDCReuseSolvers reuse_solver = sub_schurs->reuse_solver;
    4462:     MatFactorType      type;

    4464:     F = reuse_solver->F;
    4465:     PetscCall(MatGetFactorType(F, &type));
    4466:     if (type == MAT_FACTOR_CHOLESKY) isCHOL = PETSC_TRUE;
    4467:     if (type == MAT_FACTOR_LU) isLU = PETSC_TRUE;
    4468:     PetscCall(MatGetSize(F, &lda_rhs, NULL));
    4469:     need_benign_correction = (PetscBool)(!!reuse_solver->benign_n);
    4470:   }

    4472:   /* determine if we can use a sparse right-hand side */
    4473:   sparserhs = PETSC_FALSE;
    4474:   if (F && !multi_element) {
    4475:     MatSolverType solver;

    4477:     PetscCall(MatFactorGetSolverType(F, &solver));
    4478:     PetscCall(PetscStrcmp(solver, MATSOLVERMUMPS, &sparserhs));
    4479:   }

    4481:   /* create dummy vector to modify rhs and sol of MatMatSolve (work array will never be used) */
    4482:   dummy_vec = NULL;
    4483:   if (need_benign_correction && lda_rhs != n_R && F) {
    4484:     PetscCall(VecCreate(PetscObjectComm((PetscObject)pcis->vec1_N), &dummy_vec));
    4485:     PetscCall(VecSetSizes(dummy_vec, lda_rhs, PETSC_DECIDE));
    4486:     PetscCall(VecSetType(dummy_vec, ((PetscObject)pcis->vec1_N)->type_name));
    4487:   }

    4489:   PetscCall(MatDestroy(&pcbddc->local_auxmat1));
    4490:   PetscCall(MatDestroy(&pcbddc->local_auxmat2));

    4492:   PetscCall(ISCreateStride(PETSC_COMM_SELF, n_R, 0, 1, &is_R));
    4493:   PetscCall(ISCreateStride(PETSC_COMM_SELF, n_vertices, 0, 1, &is_V));
    4494:   PetscCall(ISCreateStride(PETSC_COMM_SELF, n_constraints, n_vertices, 1, &is_C));
    4495:   PetscCall(ISGetIndices(is_V, &idx_V));
    4496:   PetscCall(ISGetIndices(is_C, &idx_C));

    4498:   /* Precompute stuffs needed for preprocessing and application of BDDC*/
    4499:   if (n_constraints) {
    4500:     Mat C_B;

    4502:     /* Extract constraints on R nodes: C_{CR}  */
    4503:     PetscCall(MatCreateSubMatrix(pcbddc->ConstraintMatrix, is_C, pcbddc->is_R_local, MAT_INITIAL_MATRIX, &C_CR));
    4504:     PetscCall(MatCreateSubMatrix(pcbddc->ConstraintMatrix, is_C, pcis->is_B_local, MAT_INITIAL_MATRIX, &C_B));

    4506:     /* Assemble         local_auxmat2_R =        (- A_{RR}^{-1} C^T_{CR}) needed by BDDC setup */
    4507:     /* Assemble pcbddc->local_auxmat2   = R_to_B (- A_{RR}^{-1} C^T_{CR}) needed by BDDC application */
    4508:     if (!sparserhs) {
    4509:       PetscScalar *marr;

    4511:       PetscCall(MatCreateSeqDense(PETSC_COMM_SELF, lda_rhs, n_eff_constraints, NULL, &Brhs));
    4512:       PetscCall(MatDenseGetArrayWrite(Brhs, &marr));
    4513:       for (i = 0; i < n_constraints; i++) {
    4514:         const PetscScalar *row_cmat_values;
    4515:         const PetscInt    *row_cmat_indices;
    4516:         PetscInt           size_of_constraint, j, col = C_to_eff_C ? C_to_eff_C[i] : i;

    4518:         PetscCall(MatGetRow(C_CR, i, &size_of_constraint, &row_cmat_indices, &row_cmat_values));
    4519:         for (j = 0; j < size_of_constraint; j++) marr[row_cmat_indices[j] + col * lda_rhs] = -row_cmat_values[j];
    4520:         PetscCall(MatRestoreRow(C_CR, i, &size_of_constraint, &row_cmat_indices, &row_cmat_values));
    4521:       }
    4522:       PetscCall(MatDenseRestoreArrayWrite(Brhs, &marr));
    4523:     } else {
    4524:       Mat tC_CR;

    4526:       PetscCall(MatScale(C_CR, -1.0));
    4527:       if (lda_rhs != n_R) {
    4528:         PetscScalar *aa;
    4529:         PetscInt     r, *ii, *jj;
    4530:         PetscBool    done;

    4532:         PetscCall(MatGetRowIJ(C_CR, 0, PETSC_FALSE, PETSC_FALSE, &r, (const PetscInt **)&ii, (const PetscInt **)&jj, &done));
    4533:         PetscCheck(done, PETSC_COMM_SELF, PETSC_ERR_PLIB, "GetRowIJ failed");
    4534:         PetscCall(MatSeqAIJGetArray(C_CR, &aa));
    4535:         PetscCall(MatCreateSeqAIJWithArrays(PETSC_COMM_SELF, n_constraints, lda_rhs, ii, jj, aa, &tC_CR));
    4536:         PetscCall(MatRestoreRowIJ(C_CR, 0, PETSC_FALSE, PETSC_FALSE, &r, (const PetscInt **)&ii, (const PetscInt **)&jj, &done));
    4537:         PetscCheck(done, PETSC_COMM_SELF, PETSC_ERR_PLIB, "RestoreRowIJ failed");
    4538:       } else {
    4539:         PetscCall(PetscObjectReference((PetscObject)C_CR));
    4540:         tC_CR = C_CR;
    4541:       }
    4542:       PetscCall(MatCreateTranspose(tC_CR, &Brhs));
    4543:       PetscCall(MatDestroy(&tC_CR));
    4544:     }
    4545:     PetscCall(MatCreateSeqDense(PETSC_COMM_SELF, lda_rhs, n_eff_constraints, NULL, &local_auxmat2_R));
    4546:     if (F) {
    4547:       if (need_benign_correction) {
    4548:         PCBDDCReuseSolvers reuse_solver = sub_schurs->reuse_solver;

    4550:         /* rhs is already zero on interior dofs, no need to change the rhs */
    4551:         PetscCall(PetscArrayzero(reuse_solver->benign_save_vals, pcbddc->benign_n));
    4552:       }
    4553:       PetscCall(MatMatSolve(F, Brhs, local_auxmat2_R));
    4554:       if (need_benign_correction) {
    4555:         PetscScalar       *marr;
    4556:         PCBDDCReuseSolvers reuse_solver = sub_schurs->reuse_solver;

    4558:         /* XXX multi_element? */
    4559:         PetscCall(MatDenseGetArray(local_auxmat2_R, &marr));
    4560:         if (lda_rhs != n_R) {
    4561:           for (i = 0; i < n_eff_constraints; i++) {
    4562:             PetscCall(VecPlaceArray(dummy_vec, marr + i * lda_rhs));
    4563:             PetscCall(PCBDDCReuseSolversBenignAdapt(reuse_solver, dummy_vec, NULL, PETSC_TRUE, PETSC_TRUE));
    4564:             PetscCall(VecResetArray(dummy_vec));
    4565:           }
    4566:         } else {
    4567:           for (i = 0; i < n_eff_constraints; i++) {
    4568:             PetscCall(VecPlaceArray(pcbddc->vec1_R, marr + i * lda_rhs));
    4569:             PetscCall(PCBDDCReuseSolversBenignAdapt(reuse_solver, pcbddc->vec1_R, NULL, PETSC_TRUE, PETSC_TRUE));
    4570:             PetscCall(VecResetArray(pcbddc->vec1_R));
    4571:           }
    4572:         }
    4573:         PetscCall(MatDenseRestoreArray(local_auxmat2_R, &marr));
    4574:       }
    4575:     } else {
    4576:       const PetscScalar *barr;
    4577:       PetscScalar       *marr;

    4579:       PetscCall(MatDenseGetArrayRead(Brhs, &barr));
    4580:       PetscCall(MatDenseGetArray(local_auxmat2_R, &marr));
    4581:       for (i = 0; i < n_eff_constraints; i++) {
    4582:         PetscCall(VecPlaceArray(pcbddc->vec1_R, barr + i * lda_rhs));
    4583:         PetscCall(VecPlaceArray(pcbddc->vec2_R, marr + i * lda_rhs));
    4584:         PetscCall(KSPSolve(pcbddc->ksp_R, pcbddc->vec1_R, pcbddc->vec2_R));
    4585:         PetscCall(KSPCheckSolve(pcbddc->ksp_R, pc, pcbddc->vec2_R));
    4586:         PetscCall(VecResetArray(pcbddc->vec1_R));
    4587:         PetscCall(VecResetArray(pcbddc->vec2_R));
    4588:       }
    4589:       PetscCall(MatDenseRestoreArrayRead(Brhs, &barr));
    4590:       PetscCall(MatDenseRestoreArray(local_auxmat2_R, &marr));
    4591:     }
    4592:     if (sparserhs) PetscCall(MatScale(C_CR, -1.0));
    4593:     PetscCall(MatDestroy(&Brhs));
    4594:     /* Assemble explicitly S_CC = ( C_{CR} A_{RR}^{-1} C^T_{CR})^{-1}  */
    4595:     if (!pcbddc->switch_static) {
    4596:       PetscCall(MatCreateSeqDense(PETSC_COMM_SELF, n_B, n_eff_constraints, NULL, &pcbddc->local_auxmat2));
    4597:       PetscCall(MatDenseScatter_Private(pcbddc->R_to_B, local_auxmat2_R, pcbddc->local_auxmat2, INSERT_VALUES, SCATTER_FORWARD));
    4598:       if (multi_element) {
    4599:         Mat T;

    4601:         PetscCall(MatCreateSeqAIJFromDenseExpand(local_auxmat2_R, n_constraints, R_eff_C_J, &T));
    4602:         PetscCall(MatDestroy(&local_auxmat2_R));
    4603:         local_auxmat2_R = T;
    4604:         PetscCall(MatCreateSeqAIJFromDenseExpand(pcbddc->local_auxmat2, n_constraints, B_eff_C_J, &T));
    4605:         PetscCall(MatDestroy(&pcbddc->local_auxmat2));
    4606:         pcbddc->local_auxmat2 = T;
    4607:       }
    4608:       PetscCall(MatMatMult(C_B, pcbddc->local_auxmat2, MAT_INITIAL_MATRIX, PETSC_DETERMINE, &S_CC));
    4609:     } else {
    4610:       if (multi_element) {
    4611:         Mat T;

    4613:         PetscCall(MatCreateSeqAIJFromDenseExpand(local_auxmat2_R, n_constraints, R_eff_C_J, &T));
    4614:         PetscCall(MatDestroy(&local_auxmat2_R));
    4615:         local_auxmat2_R = T;
    4616:       }
    4617:       if (lda_rhs != n_R) {
    4618:         PetscCall(MatCreateSubMatrix(local_auxmat2_R, is_R, NULL, MAT_INITIAL_MATRIX, &pcbddc->local_auxmat2));
    4619:       } else {
    4620:         PetscCall(PetscObjectReference((PetscObject)local_auxmat2_R));
    4621:         pcbddc->local_auxmat2 = local_auxmat2_R;
    4622:       }
    4623:       PetscCall(MatMatMult(C_CR, pcbddc->local_auxmat2, MAT_INITIAL_MATRIX, PETSC_DETERMINE, &S_CC));
    4624:     }
    4625:     PetscCall(MatScale(S_CC, m_one));
    4626:     if (multi_element) {
    4627:       Mat T, T2;
    4628:       IS  isp, ispi;

    4630:       isp = is_C_perm;

    4632:       PetscCall(ISInvertPermutation(isp, PETSC_DECIDE, &ispi));
    4633:       PetscCall(MatPermute(S_CC, isp, isp, &T));
    4634:       PetscCall(MatSeqAIJInvertVariableBlockDiagonalMat(T, n_C_bss, C_bss, &T2));
    4635:       PetscCall(MatDestroy(&T));
    4636:       PetscCall(MatDestroy(&S_CC));
    4637:       PetscCall(MatPermute(T2, ispi, ispi, &S_CC));
    4638:       PetscCall(MatDestroy(&T2));
    4639:       PetscCall(ISDestroy(&ispi));
    4640:     } else {
    4641:       if (isCHOL) {
    4642:         PetscCall(MatCholeskyFactor(S_CC, NULL, NULL));
    4643:       } else {
    4644:         PetscCall(MatLUFactor(S_CC, NULL, NULL, NULL));
    4645:       }
    4646:       PetscCall(MatSeqDenseInvertFactors_Private(S_CC));
    4647:     }
    4648:     /* Assemble local_auxmat1 = S_CC*C_{CB} needed by BDDC application in KSP and in preproc */
    4649:     PetscCall(MatMatMult(S_CC, C_B, MAT_INITIAL_MATRIX, PETSC_DETERMINE, &pcbddc->local_auxmat1));
    4650:     PetscCall(MatDestroy(&C_B));
    4651:     PetscCall(MatSetValuesSubMat(*coarse_submat, S_CC, n_constraints, idx_C, n_constraints, idx_C, INSERT_VALUES));
    4652:   }

    4654:   /* Get submatrices from subdomain matrix */
    4655:   if (n_vertices) {
    4656: #if defined(PETSC_HAVE_VIENNACL) || defined(PETSC_HAVE_CUDA)
    4657:     PetscBool oldpin;
    4658: #endif
    4659:     IS is_aux;

    4661:     if (sub_schurs && sub_schurs->reuse_solver) { /* is_R_local is not sorted, ISComplement doesn't like it */
    4662:       IS tis;

    4664:       PetscCall(ISDuplicate(pcbddc->is_R_local, &tis));
    4665:       PetscCall(ISSort(tis));
    4666:       PetscCall(ISComplement(tis, 0, pcis->n, &is_aux));
    4667:       PetscCall(ISDestroy(&tis));
    4668:     } else {
    4669:       PetscCall(ISComplement(pcbddc->is_R_local, 0, pcis->n, &is_aux));
    4670:     }
    4671: #if defined(PETSC_HAVE_VIENNACL) || defined(PETSC_HAVE_CUDA)
    4672:     oldpin = pcbddc->local_mat->boundtocpu;
    4673: #endif
    4674:     PetscCall(MatBindToCPU(pcbddc->local_mat, PETSC_TRUE));
    4675:     PetscCall(MatCreateSubMatrix(pcbddc->local_mat, pcbddc->is_R_local, is_aux, MAT_INITIAL_MATRIX, &A_RV));
    4676:     PetscCall(MatCreateSubMatrix(pcbddc->local_mat, is_aux, pcbddc->is_R_local, MAT_INITIAL_MATRIX, &A_VR));
    4677:     /* TODO REMOVE: MatMatMult(A_VR,A_RRmA_RV) below may raise an error */
    4678:     PetscCall(MatConvert(A_VR, MATSEQAIJ, MAT_INPLACE_MATRIX, &A_VR));
    4679:     PetscCall(MatCreateSubMatrix(pcbddc->local_mat, is_aux, is_aux, MAT_INITIAL_MATRIX, &A_VV));
    4680: #if defined(PETSC_HAVE_VIENNACL) || defined(PETSC_HAVE_CUDA)
    4681:     PetscCall(MatBindToCPU(pcbddc->local_mat, oldpin));
    4682: #endif
    4683:     PetscCall(ISDestroy(&is_aux));
    4684:   }
    4685:   PetscCall(ISDestroy(&is_C_perm));
    4686:   PetscCall(PetscFree(C_bss));

    4688:   p0_lidx_I = NULL;
    4689:   if (pcbddc->benign_n && (pcbddc->switch_static || pcbddc->dbg_flag)) {
    4690:     const PetscInt *idxs;

    4692:     PetscCall(ISGetIndices(pcis->is_I_local, &idxs));
    4693:     PetscCall(PetscMalloc1(pcbddc->benign_n, &p0_lidx_I));
    4694:     for (i = 0; i < pcbddc->benign_n; i++) PetscCall(PetscFindInt(pcbddc->benign_p0_lidx[i], pcis->n - pcis->n_B, idxs, &p0_lidx_I[i]));
    4695:     PetscCall(ISRestoreIndices(pcis->is_I_local, &idxs));
    4696:   }

    4698:   /* We are now ready to evaluate coarse basis functions and subdomain contribution to coarse problem */

    4700:   /* Matrices of coarse basis functions (local) */
    4701:   PetscCall(MatDestroy(&pcbddc->coarse_phi_B));
    4702:   PetscCall(MatDestroy(&pcbddc->coarse_psi_B));
    4703:   PetscCall(MatDestroy(&pcbddc->coarse_phi_D));
    4704:   PetscCall(MatDestroy(&pcbddc->coarse_psi_D));
    4705:   if (!multi_element) {
    4706:     PetscCall(MatCreateSeqDense(PETSC_COMM_SELF, n_B, pcbddc->local_primal_size, NULL, &pcbddc->coarse_phi_B));
    4707:     if (pcbddc->switch_static || pcbddc->dbg_flag) PetscCall(MatCreateSeqDense(PETSC_COMM_SELF, n_D, pcbddc->local_primal_size, NULL, &pcbddc->coarse_phi_D));
    4708:     coarse_phi_multi = NULL;
    4709:   } else { /* Create temporary NEST matrix to hold coarse basis functions blocks */
    4710:     IS is_rows[2] = {pcbddc->is_R_local, NULL};
    4711:     IS is_cols[2] = {is_V, is_C};

    4713:     PetscCall(ISCreateGeneral(PETSC_COMM_SELF, n_vertices, pcbddc->local_primal_ref_node, PETSC_USE_POINTER, &is_rows[1]));
    4714:     PetscCall(MatCreateNest(PETSC_COMM_SELF, 2, is_rows, 2, is_cols, NULL, &coarse_phi_multi));
    4715:     PetscCall(ISDestroy(&is_rows[1]));
    4716:   }

    4718:   /* vertices */
    4719:   if (n_vertices) {
    4720:     PetscBool restoreavr = PETSC_FALSE;
    4721:     Mat       A_RRmA_RV  = NULL;

    4723:     PetscCall(MatSetValuesSubMat(*coarse_submat, A_VV, n_vertices, idx_V, n_vertices, idx_V, ADD_VALUES));
    4724:     PetscCall(MatDestroy(&A_VV));

    4726:     if (n_R) {
    4727:       Mat A_RV_bcorr = NULL, S_VV;

    4729:       PetscCall(MatScale(A_RV, m_one));
    4730:       if (need_benign_correction) {
    4731:         ISLocalToGlobalMapping RtoN;
    4732:         IS                     is_p0;
    4733:         PetscInt              *idxs_p0, n;

    4735:         PetscCall(PetscMalloc1(pcbddc->benign_n, &idxs_p0));
    4736:         PetscCall(ISLocalToGlobalMappingCreateIS(pcbddc->is_R_local, &RtoN));
    4737:         PetscCall(ISGlobalToLocalMappingApply(RtoN, IS_GTOLM_DROP, pcbddc->benign_n, pcbddc->benign_p0_lidx, &n, idxs_p0));
    4738:         PetscCheck(n == pcbddc->benign_n, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Error in R numbering for benign p0! %" PetscInt_FMT " != %" PetscInt_FMT, n, pcbddc->benign_n);
    4739:         PetscCall(ISLocalToGlobalMappingDestroy(&RtoN));
    4740:         PetscCall(ISCreateGeneral(PETSC_COMM_SELF, n, idxs_p0, PETSC_OWN_POINTER, &is_p0));
    4741:         PetscCall(MatCreateSubMatrix(A_RV, is_p0, NULL, MAT_INITIAL_MATRIX, &A_RV_bcorr));
    4742:         PetscCall(ISDestroy(&is_p0));
    4743:       }

    4745:       PetscCall(MatCreateSeqDense(PETSC_COMM_SELF, lda_rhs, n_eff_vertices, NULL, &A_RRmA_RV));
    4746:       if (!sparserhs || need_benign_correction) {
    4747:         if (lda_rhs == n_R && !multi_element) {
    4748:           PetscCall(MatConvert(A_RV, MATDENSE, MAT_INPLACE_MATRIX, &A_RV));
    4749:         } else {
    4750:           Mat             T;
    4751:           PetscScalar    *av, *array;
    4752:           const PetscInt *xadj, *adjncy;
    4753:           PetscInt        n;
    4754:           PetscBool       flg_row;

    4756:           PetscCall(MatCreateSeqDense(PETSC_COMM_SELF, lda_rhs, n_eff_vertices, NULL, &T));
    4757:           PetscCall(MatDenseGetArrayWrite(T, &array));
    4758:           PetscCall(MatConvert(A_RV, MATSEQAIJ, MAT_INPLACE_MATRIX, &A_RV));
    4759:           PetscCall(MatGetRowIJ(A_RV, 0, PETSC_FALSE, PETSC_FALSE, &n, &xadj, &adjncy, &flg_row));
    4760:           PetscCall(MatSeqAIJGetArray(A_RV, &av));
    4761:           for (i = 0; i < n; i++) {
    4762:             PetscInt j;
    4763:             for (j = xadj[i]; j < xadj[i + 1]; j++) array[lda_rhs * (V_to_eff_V ? V_to_eff_V[adjncy[j]] : adjncy[j]) + i] = av[j];
    4764:           }
    4765:           PetscCall(MatRestoreRowIJ(A_RV, 0, PETSC_FALSE, PETSC_FALSE, &n, &xadj, &adjncy, &flg_row));
    4766:           PetscCall(MatDenseRestoreArrayWrite(T, &array));
    4767:           PetscCall(MatDestroy(&A_RV));
    4768:           A_RV = T;
    4769:         }
    4770:         if (need_benign_correction) {
    4771:           PCBDDCReuseSolvers reuse_solver = sub_schurs->reuse_solver;
    4772:           PetscScalar       *marr;

    4774:           /* XXX multi_element */
    4775:           PetscCall(MatDenseGetArray(A_RV, &marr));
    4776:           /* need \Phi^T A_RV = (I+L)A_RV, L given by

    4778:                  | 0 0  0 | (V)
    4779:              L = | 0 0 -1 | (P-p0)
    4780:                  | 0 0 -1 | (p0)

    4782:           */
    4783:           for (i = 0; i < reuse_solver->benign_n; i++) {
    4784:             const PetscScalar *vals;
    4785:             const PetscInt    *idxs, *idxs_zero;
    4786:             PetscInt           n, j, nz;

    4788:             PetscCall(ISGetLocalSize(reuse_solver->benign_zerodiag_subs[i], &nz));
    4789:             PetscCall(ISGetIndices(reuse_solver->benign_zerodiag_subs[i], &idxs_zero));
    4790:             PetscCall(MatGetRow(A_RV_bcorr, i, &n, &idxs, &vals));
    4791:             for (j = 0; j < n; j++) {
    4792:               PetscScalar val = vals[j];
    4793:               PetscInt    k, col = idxs[j];
    4794:               for (k = 0; k < nz; k++) marr[idxs_zero[k] + lda_rhs * col] -= val;
    4795:             }
    4796:             PetscCall(MatRestoreRow(A_RV_bcorr, i, &n, &idxs, &vals));
    4797:             PetscCall(ISRestoreIndices(reuse_solver->benign_zerodiag_subs[i], &idxs_zero));
    4798:           }
    4799:           PetscCall(MatDenseRestoreArray(A_RV, &marr));
    4800:         }
    4801:         PetscCall(PetscObjectReference((PetscObject)A_RV));
    4802:         Brhs = A_RV;
    4803:       } else {
    4804:         Mat tA_RVT, A_RVT;

    4806:         if (!pcbddc->symmetric_primal) {
    4807:           /* A_RV already scaled by -1 */
    4808:           PetscCall(MatTranspose(A_RV, MAT_INITIAL_MATRIX, &A_RVT));
    4809:         } else {
    4810:           restoreavr = PETSC_TRUE;
    4811:           PetscCall(MatScale(A_VR, -1.0));
    4812:           PetscCall(PetscObjectReference((PetscObject)A_VR));
    4813:           A_RVT = A_VR;
    4814:         }
    4815:         if (lda_rhs != n_R) {
    4816:           PetscScalar *aa;
    4817:           PetscInt     r, *ii, *jj;
    4818:           PetscBool    done;

    4820:           PetscCall(MatGetRowIJ(A_RVT, 0, PETSC_FALSE, PETSC_FALSE, &r, (const PetscInt **)&ii, (const PetscInt **)&jj, &done));
    4821:           PetscCheck(done, PETSC_COMM_SELF, PETSC_ERR_PLIB, "GetRowIJ failed");
    4822:           PetscCall(MatSeqAIJGetArray(A_RVT, &aa));
    4823:           PetscCall(MatCreateSeqAIJWithArrays(PETSC_COMM_SELF, n_vertices, lda_rhs, ii, jj, aa, &tA_RVT));
    4824:           PetscCall(MatRestoreRowIJ(A_RVT, 0, PETSC_FALSE, PETSC_FALSE, &r, (const PetscInt **)&ii, (const PetscInt **)&jj, &done));
    4825:           PetscCheck(done, PETSC_COMM_SELF, PETSC_ERR_PLIB, "RestoreRowIJ failed");
    4826:         } else {
    4827:           PetscCall(PetscObjectReference((PetscObject)A_RVT));
    4828:           tA_RVT = A_RVT;
    4829:         }
    4830:         PetscCall(MatCreateTranspose(tA_RVT, &Brhs));
    4831:         PetscCall(MatDestroy(&tA_RVT));
    4832:         PetscCall(MatDestroy(&A_RVT));
    4833:       }
    4834:       if (F) {
    4835:         /* need to correct the rhs */
    4836:         if (need_benign_correction) {
    4837:           PCBDDCReuseSolvers reuse_solver = sub_schurs->reuse_solver;
    4838:           PetscScalar       *marr;

    4840:           PetscCall(MatDenseGetArray(Brhs, &marr));
    4841:           if (lda_rhs != n_R) {
    4842:             for (i = 0; i < n_eff_vertices; i++) {
    4843:               PetscCall(VecPlaceArray(dummy_vec, marr + i * lda_rhs));
    4844:               PetscCall(PCBDDCReuseSolversBenignAdapt(reuse_solver, dummy_vec, NULL, PETSC_FALSE, PETSC_TRUE));
    4845:               PetscCall(VecResetArray(dummy_vec));
    4846:             }
    4847:           } else {
    4848:             for (i = 0; i < n_eff_vertices; i++) {
    4849:               PetscCall(VecPlaceArray(pcbddc->vec1_R, marr + i * lda_rhs));
    4850:               PetscCall(PCBDDCReuseSolversBenignAdapt(reuse_solver, pcbddc->vec1_R, NULL, PETSC_FALSE, PETSC_TRUE));
    4851:               PetscCall(VecResetArray(pcbddc->vec1_R));
    4852:             }
    4853:           }
    4854:           PetscCall(MatDenseRestoreArray(Brhs, &marr));
    4855:         }
    4856:         PetscCall(MatMatSolve(F, Brhs, A_RRmA_RV));
    4857:         if (restoreavr) PetscCall(MatScale(A_VR, -1.0));
    4858:         /* need to correct the solution */
    4859:         if (need_benign_correction) {
    4860:           PCBDDCReuseSolvers reuse_solver = sub_schurs->reuse_solver;
    4861:           PetscScalar       *marr;

    4863:           PetscCall(MatDenseGetArray(A_RRmA_RV, &marr));
    4864:           if (lda_rhs != n_R) {
    4865:             for (i = 0; i < n_eff_vertices; i++) {
    4866:               PetscCall(VecPlaceArray(dummy_vec, marr + i * lda_rhs));
    4867:               PetscCall(PCBDDCReuseSolversBenignAdapt(reuse_solver, dummy_vec, NULL, PETSC_TRUE, PETSC_TRUE));
    4868:               PetscCall(VecResetArray(dummy_vec));
    4869:             }
    4870:           } else {
    4871:             for (i = 0; i < n_eff_vertices; i++) {
    4872:               PetscCall(VecPlaceArray(pcbddc->vec1_R, marr + i * lda_rhs));
    4873:               PetscCall(PCBDDCReuseSolversBenignAdapt(reuse_solver, pcbddc->vec1_R, NULL, PETSC_TRUE, PETSC_TRUE));
    4874:               PetscCall(VecResetArray(pcbddc->vec1_R));
    4875:             }
    4876:           }
    4877:           PetscCall(MatDenseRestoreArray(A_RRmA_RV, &marr));
    4878:         }
    4879:       } else {
    4880:         const PetscScalar *barr;
    4881:         PetscScalar       *marr;

    4883:         PetscCall(MatDenseGetArrayRead(Brhs, &barr));
    4884:         PetscCall(MatDenseGetArray(A_RRmA_RV, &marr));
    4885:         for (i = 0; i < n_eff_vertices; i++) {
    4886:           PetscCall(VecPlaceArray(pcbddc->vec1_R, barr + i * lda_rhs));
    4887:           PetscCall(VecPlaceArray(pcbddc->vec2_R, marr + i * lda_rhs));
    4888:           PetscCall(KSPSolve(pcbddc->ksp_R, pcbddc->vec1_R, pcbddc->vec2_R));
    4889:           PetscCall(KSPCheckSolve(pcbddc->ksp_R, pc, pcbddc->vec2_R));
    4890:           PetscCall(VecResetArray(pcbddc->vec1_R));
    4891:           PetscCall(VecResetArray(pcbddc->vec2_R));
    4892:         }
    4893:         PetscCall(MatDenseRestoreArrayRead(Brhs, &barr));
    4894:         PetscCall(MatDenseRestoreArray(A_RRmA_RV, &marr));
    4895:       }
    4896:       PetscCall(MatDestroy(&A_RV));
    4897:       PetscCall(MatDestroy(&Brhs));
    4898:       /* S_VV and S_CV */
    4899:       if (n_constraints) {
    4900:         Mat B;

    4902:         PetscCall(MatCreateSeqDense(PETSC_COMM_SELF, n_B, n_eff_vertices, NULL, &B));
    4903:         PetscCall(MatDenseScatter_Private(pcbddc->R_to_B, A_RRmA_RV, B, INSERT_VALUES, SCATTER_FORWARD));

    4905:         /* S_CV = pcbddc->local_auxmat1 * B */
    4906:         if (multi_element) {
    4907:           Mat T;

    4909:           PetscCall(MatCreateSeqAIJFromDenseExpand(B, n_vertices, B_eff_V_J, &T));
    4910:           PetscCall(MatDestroy(&B));
    4911:           B = T;
    4912:         }
    4913:         PetscCall(MatProductCreate(pcbddc->local_auxmat1, B, NULL, &S_CV));
    4914:         PetscCall(MatProductSetType(S_CV, MATPRODUCT_AB));
    4915:         PetscCall(MatProductSetFromOptions(S_CV));
    4916:         PetscCall(MatProductSymbolic(S_CV));
    4917:         PetscCall(MatProductNumeric(S_CV));
    4918:         PetscCall(MatProductClear(S_CV));
    4919:         PetscCall(MatDestroy(&B));

    4921:         /* B = local_auxmat2_R * S_CV */
    4922:         PetscCall(MatProductCreate(local_auxmat2_R, S_CV, NULL, &B));
    4923:         PetscCall(MatProductSetType(B, MATPRODUCT_AB));
    4924:         PetscCall(MatProductSetFromOptions(B));
    4925:         PetscCall(MatProductSymbolic(B));
    4926:         PetscCall(MatProductNumeric(B));

    4928:         PetscCall(MatScale(S_CV, m_one));
    4929:         PetscCall(MatSetValuesSubMat(*coarse_submat, S_CV, n_constraints, idx_C, n_vertices, idx_V, INSERT_VALUES));

    4931:         if (multi_element) {
    4932:           Mat T;

    4934:           PetscCall(MatCreateSeqAIJFromDenseExpand(A_RRmA_RV, n_vertices, R_eff_V_J, &T));
    4935:           PetscCall(MatDestroy(&A_RRmA_RV));
    4936:           A_RRmA_RV = T;
    4937:         }
    4938:         PetscCall(MatAXPY(A_RRmA_RV, 1.0, B, UNKNOWN_NONZERO_PATTERN)); /* XXX ? */
    4939:         PetscCall(MatDestroy(&B));
    4940:       } else if (multi_element) {
    4941:         Mat T;

    4943:         PetscCall(MatCreateSeqAIJFromDenseExpand(A_RRmA_RV, n_vertices, R_eff_V_J, &T));
    4944:         PetscCall(MatDestroy(&A_RRmA_RV));
    4945:         A_RRmA_RV = T;
    4946:       }

    4948:       if (lda_rhs != n_R) {
    4949:         Mat T;

    4951:         PetscCall(MatCreateSubMatrix(A_RRmA_RV, is_R, NULL, MAT_INITIAL_MATRIX, &T));
    4952:         PetscCall(MatDestroy(&A_RRmA_RV));
    4953:         A_RRmA_RV = T;
    4954:       }

    4956:       /* need A_VR * \Phi * A_RRmA_RV = A_VR * (I+L)^T * A_RRmA_RV, L given as before */
    4957:       if (need_benign_correction) { /* XXX SPARSE */
    4958:         PCBDDCReuseSolvers reuse_solver = sub_schurs->reuse_solver;
    4959:         PetscScalar       *sums;
    4960:         const PetscScalar *marr;

    4962:         PetscCall(MatDenseGetArrayRead(A_RRmA_RV, &marr));
    4963:         PetscCall(PetscMalloc1(n_vertices, &sums));
    4964:         for (i = 0; i < reuse_solver->benign_n; i++) {
    4965:           const PetscScalar *vals;
    4966:           const PetscInt    *idxs, *idxs_zero;
    4967:           PetscInt           n, j, nz;

    4969:           PetscCall(ISGetLocalSize(reuse_solver->benign_zerodiag_subs[i], &nz));
    4970:           PetscCall(ISGetIndices(reuse_solver->benign_zerodiag_subs[i], &idxs_zero));
    4971:           for (j = 0; j < n_vertices; j++) {
    4972:             sums[j] = 0.;
    4973:             for (PetscInt k = 0; k < nz; k++) sums[j] += marr[idxs_zero[k] + j * n_R];
    4974:           }
    4975:           PetscCall(MatGetRow(A_RV_bcorr, i, &n, &idxs, &vals));
    4976:           for (j = 0; j < n; j++) {
    4977:             PetscScalar val = vals[j];
    4978:             for (PetscInt k = 0; k < n_vertices; k++) PetscCall(MatSetValue(*coarse_submat, idx_V[idxs[j]], idx_V[k], val * sums[k], ADD_VALUES));
    4979:           }
    4980:           PetscCall(MatRestoreRow(A_RV_bcorr, i, &n, &idxs, &vals));
    4981:           PetscCall(ISRestoreIndices(reuse_solver->benign_zerodiag_subs[i], &idxs_zero));
    4982:         }
    4983:         PetscCall(PetscFree(sums));
    4984:         PetscCall(MatDestroy(&A_RV_bcorr));
    4985:         PetscCall(MatDenseRestoreArrayRead(A_RRmA_RV, &marr));
    4986:       }

    4988:       PetscCall(MatMatMult(A_VR, A_RRmA_RV, MAT_INITIAL_MATRIX, PETSC_DETERMINE, &S_VV));
    4989:       PetscCall(MatSetValuesSubMat(*coarse_submat, S_VV, n_vertices, idx_V, n_vertices, idx_V, ADD_VALUES));
    4990:       PetscCall(MatDestroy(&S_VV));
    4991:     }

    4993:     /* coarse basis functions */
    4994:     if (coarse_phi_multi) {
    4995:       Mat Vid;

    4997:       PetscCall(MatCreateSeqAIJ(PETSC_COMM_SELF, n_vertices, n_vertices, 1, NULL, &Vid));
    4998:       PetscCall(MatShift_Basic(Vid, 1.0));
    4999:       PetscCall(MatNestSetSubMat(coarse_phi_multi, 0, 0, A_RRmA_RV));
    5000:       PetscCall(MatNestSetSubMat(coarse_phi_multi, 1, 0, Vid));
    5001:       PetscCall(MatDestroy(&Vid));
    5002:     } else {
    5003:       if (A_RRmA_RV) {
    5004:         Mat B;

    5006:         PetscCall(MatDenseGetSubMatrix(pcbddc->coarse_phi_B, PETSC_DECIDE, PETSC_DECIDE, 0, n_vertices, &B));
    5007:         PetscCall(MatDenseScatter_Private(pcbddc->R_to_B, A_RRmA_RV, B, INSERT_VALUES, SCATTER_FORWARD));
    5008:         PetscCall(MatDenseRestoreSubMatrix(pcbddc->coarse_phi_B, &B));
    5009:         if (pcbddc->switch_static || pcbddc->dbg_flag) {
    5010:           PetscCall(MatDenseGetSubMatrix(pcbddc->coarse_phi_D, PETSC_DECIDE, PETSC_DECIDE, 0, n_vertices, &B));
    5011:           PetscCall(MatDenseScatter_Private(pcbddc->R_to_D, A_RRmA_RV, B, INSERT_VALUES, SCATTER_FORWARD));
    5012:           PetscCall(MatDenseRestoreSubMatrix(pcbddc->coarse_phi_D, &B));
    5013:           if (pcbddc->benign_n) {
    5014:             for (i = 0; i < n_vertices; i++) PetscCall(MatSetValues(pcbddc->coarse_phi_D, pcbddc->benign_n, p0_lidx_I, 1, &i, NULL, INSERT_VALUES));
    5015:             PetscCall(MatAssemblyBegin(pcbddc->coarse_phi_D, MAT_FINAL_ASSEMBLY));
    5016:             PetscCall(MatAssemblyEnd(pcbddc->coarse_phi_D, MAT_FINAL_ASSEMBLY));
    5017:           }
    5018:         }
    5019:       }
    5020:       for (i = 0; i < n_vertices; i++) PetscCall(MatSetValues(pcbddc->coarse_phi_B, 1, &idx_V_B[i], 1, &i, &one, INSERT_VALUES));
    5021:       PetscCall(MatAssemblyBegin(pcbddc->coarse_phi_B, MAT_FINAL_ASSEMBLY));
    5022:       PetscCall(MatAssemblyEnd(pcbddc->coarse_phi_B, MAT_FINAL_ASSEMBLY));
    5023:     }
    5024:     PetscCall(MatDestroy(&A_RRmA_RV));
    5025:   }
    5026:   PetscCall(MatDestroy(&A_RV));
    5027:   PetscCall(VecDestroy(&dummy_vec));

    5029:   if (n_constraints) {
    5030:     Mat B, B2;

    5032:     PetscCall(MatScale(S_CC, m_one));
    5033:     PetscCall(MatProductCreate(local_auxmat2_R, S_CC, NULL, &B));
    5034:     PetscCall(MatProductSetType(B, MATPRODUCT_AB));
    5035:     PetscCall(MatProductSetFromOptions(B));
    5036:     PetscCall(MatProductSymbolic(B));
    5037:     PetscCall(MatProductNumeric(B));

    5039:     if (n_vertices) {
    5040:       if (isCHOL || need_benign_correction) { /* if we can solve the interior problem with cholesky, we should also be fine with transposing here */
    5041:         PetscCall(MatTranspose(S_CV, MAT_INITIAL_MATRIX, &S_VC));
    5042:       } else {
    5043:         if (lda_rhs != n_R) {
    5044:           Mat tB;

    5046:           PetscCall(MatCreateSubMatrix(B, is_R, NULL, MAT_INITIAL_MATRIX, &tB));
    5047:           PetscCall(MatDestroy(&B));
    5048:           B = tB;
    5049:         }
    5050:         PetscCall(MatMatMult(A_VR, B, MAT_INITIAL_MATRIX, PETSC_DETERMINE, &S_VC));
    5051:       }
    5052:       PetscCall(MatSetValuesSubMat(*coarse_submat, S_VC, n_vertices, idx_V, n_constraints, idx_C, INSERT_VALUES));
    5053:     }

    5055:     /* coarse basis functions */
    5056:     if (coarse_phi_multi) {
    5057:       PetscCall(MatNestSetSubMat(coarse_phi_multi, 0, 1, B));
    5058:     } else {
    5059:       PetscCall(MatDenseGetSubMatrix(pcbddc->coarse_phi_B, PETSC_DECIDE, PETSC_DECIDE, n_vertices, n_vertices + n_constraints, &B2));
    5060:       PetscCall(MatDenseScatter_Private(pcbddc->R_to_B, B, B2, INSERT_VALUES, SCATTER_FORWARD));
    5061:       PetscCall(MatDenseRestoreSubMatrix(pcbddc->coarse_phi_B, &B2));
    5062:       if (pcbddc->switch_static || pcbddc->dbg_flag) {
    5063:         PetscCall(MatDenseGetSubMatrix(pcbddc->coarse_phi_D, PETSC_DECIDE, PETSC_DECIDE, n_vertices, n_vertices + n_constraints, &B2));
    5064:         PetscCall(MatDenseScatter_Private(pcbddc->R_to_D, B, B2, INSERT_VALUES, SCATTER_FORWARD));
    5065:         if (pcbddc->benign_n) {
    5066:           for (i = 0; i < n_constraints; i++) PetscCall(MatSetValues(B2, pcbddc->benign_n, p0_lidx_I, 1, &i, NULL, INSERT_VALUES));
    5067:         }
    5068:         PetscCall(MatDenseRestoreSubMatrix(pcbddc->coarse_phi_D, &B2));
    5069:       }
    5070:     }
    5071:     PetscCall(MatDestroy(&B));
    5072:   }

    5074:   /* assemble sparse coarse basis functions */
    5075:   if (coarse_phi_multi) {
    5076:     Mat T;

    5078:     PetscCall(MatConvert(coarse_phi_multi, MATSEQAIJ, MAT_INITIAL_MATRIX, &T));
    5079:     PetscCall(MatDestroy(&coarse_phi_multi));
    5080:     PetscCall(MatCreateSubMatrix(T, pcis->is_B_local, NULL, MAT_INITIAL_MATRIX, &pcbddc->coarse_phi_B));
    5081:     if (pcbddc->switch_static || pcbddc->dbg_flag) PetscCall(MatCreateSubMatrix(T, pcis->is_I_local, NULL, MAT_INITIAL_MATRIX, &pcbddc->coarse_phi_D));
    5082:     PetscCall(MatDestroy(&T));
    5083:   }
    5084:   PetscCall(MatDestroy(&local_auxmat2_R));
    5085:   PetscCall(PetscFree(p0_lidx_I));

    5087:   /* coarse matrix entries relative to B_0 */
    5088:   if (pcbddc->benign_n) {
    5089:     Mat                B0_B, B0_BPHI;
    5090:     IS                 is_dummy;
    5091:     const PetscScalar *data;
    5092:     PetscInt           j;

    5094:     PetscCall(ISCreateStride(PETSC_COMM_SELF, pcbddc->benign_n, 0, 1, &is_dummy));
    5095:     PetscCall(MatCreateSubMatrix(pcbddc->benign_B0, is_dummy, pcis->is_B_local, MAT_INITIAL_MATRIX, &B0_B));
    5096:     PetscCall(ISDestroy(&is_dummy));
    5097:     PetscCall(MatMatMult(B0_B, pcbddc->coarse_phi_B, MAT_INITIAL_MATRIX, 1.0, &B0_BPHI));
    5098:     PetscCall(MatConvert(B0_BPHI, MATSEQDENSE, MAT_INPLACE_MATRIX, &B0_BPHI));
    5099:     PetscCall(MatDenseGetArrayRead(B0_BPHI, &data));
    5100:     for (j = 0; j < pcbddc->benign_n; j++) {
    5101:       PetscInt primal_idx = pcbddc->local_primal_size - pcbddc->benign_n + j;
    5102:       for (i = 0; i < pcbddc->local_primal_size; i++) {
    5103:         PetscCall(MatSetValue(*coarse_submat, primal_idx, i, data[i * pcbddc->benign_n + j], INSERT_VALUES));
    5104:         PetscCall(MatSetValue(*coarse_submat, i, primal_idx, data[i * pcbddc->benign_n + j], INSERT_VALUES));
    5105:       }
    5106:     }
    5107:     PetscCall(MatDenseRestoreArrayRead(B0_BPHI, &data));
    5108:     PetscCall(MatDestroy(&B0_B));
    5109:     PetscCall(MatDestroy(&B0_BPHI));
    5110:   }

    5112:   /* compute other basis functions for non-symmetric problems */
    5113:   if (!pcbddc->symmetric_primal) {
    5114:     Mat          B_V = NULL, B_C = NULL;
    5115:     PetscScalar *marray, *work;

    5117:     /* TODO multi_element MatDenseScatter */
    5118:     if (n_constraints) {
    5119:       Mat S_CCT, C_CRT;

    5121:       PetscCall(MatScale(S_CC, m_one));
    5122:       PetscCall(MatTranspose(C_CR, MAT_INITIAL_MATRIX, &C_CRT));
    5123:       PetscCall(MatTranspose(S_CC, MAT_INITIAL_MATRIX, &S_CCT));
    5124:       PetscCall(MatMatMult(C_CRT, S_CCT, MAT_INITIAL_MATRIX, PETSC_DETERMINE, &B_C));
    5125:       PetscCall(MatConvert(B_C, MATDENSE, MAT_INPLACE_MATRIX, &B_C));
    5126:       PetscCall(MatDestroy(&S_CCT));
    5127:       if (n_vertices) {
    5128:         Mat S_VCT;

    5130:         PetscCall(MatTranspose(S_VC, MAT_INITIAL_MATRIX, &S_VCT));
    5131:         PetscCall(MatMatMult(C_CRT, S_VCT, MAT_INITIAL_MATRIX, PETSC_DETERMINE, &B_V));
    5132:         PetscCall(MatDestroy(&S_VCT));
    5133:         PetscCall(MatConvert(B_V, MATDENSE, MAT_INPLACE_MATRIX, &B_V));
    5134:       }
    5135:       PetscCall(MatDestroy(&C_CRT));
    5136:     } else {
    5137:       PetscCall(MatCreateSeqDense(PETSC_COMM_SELF, n_R, n_vertices, NULL, &B_V));
    5138:     }
    5139:     if (n_vertices && n_R) {
    5140:       PetscScalar    *av, *marray;
    5141:       const PetscInt *xadj, *adjncy;
    5142:       PetscInt        n;
    5143:       PetscBool       flg_row;

    5145:       /* B_V = B_V - A_VR^T */
    5146:       PetscCall(MatConvert(A_VR, MATSEQAIJ, MAT_INPLACE_MATRIX, &A_VR));
    5147:       PetscCall(MatGetRowIJ(A_VR, 0, PETSC_FALSE, PETSC_FALSE, &n, &xadj, &adjncy, &flg_row));
    5148:       PetscCall(MatSeqAIJGetArray(A_VR, &av));
    5149:       PetscCall(MatDenseGetArray(B_V, &marray));
    5150:       for (i = 0; i < n; i++) {
    5151:         PetscInt j;
    5152:         for (j = xadj[i]; j < xadj[i + 1]; j++) marray[i * n_R + adjncy[j]] -= av[j];
    5153:       }
    5154:       PetscCall(MatDenseRestoreArray(B_V, &marray));
    5155:       PetscCall(MatRestoreRowIJ(A_VR, 0, PETSC_FALSE, PETSC_FALSE, &n, &xadj, &adjncy, &flg_row));
    5156:       PetscCall(MatDestroy(&A_VR));
    5157:     }

    5159:     /* currently there's no support for MatTransposeMatSolve(F,B,X) */
    5160:     PetscCall(PetscMalloc1(n_R * pcbddc->local_primal_size, &work));
    5161:     if (n_vertices) {
    5162:       PetscCall(MatDenseGetArray(B_V, &marray));
    5163:       for (i = 0; i < n_vertices; i++) {
    5164:         PetscCall(VecPlaceArray(pcbddc->vec1_R, marray + i * n_R));
    5165:         PetscCall(VecPlaceArray(pcbddc->vec2_R, work + i * n_R));
    5166:         PetscCall(KSPSolveTranspose(pcbddc->ksp_R, pcbddc->vec1_R, pcbddc->vec2_R));
    5167:         PetscCall(KSPCheckSolve(pcbddc->ksp_R, pc, pcbddc->vec2_R));
    5168:         PetscCall(VecResetArray(pcbddc->vec1_R));
    5169:         PetscCall(VecResetArray(pcbddc->vec2_R));
    5170:       }
    5171:       PetscCall(MatDenseRestoreArray(B_V, &marray));
    5172:     }
    5173:     if (B_C) {
    5174:       PetscCall(MatDenseGetArray(B_C, &marray));
    5175:       for (i = n_vertices; i < n_constraints + n_vertices; i++) {
    5176:         PetscCall(VecPlaceArray(pcbddc->vec1_R, marray + (i - n_vertices) * n_R));
    5177:         PetscCall(VecPlaceArray(pcbddc->vec2_R, work + i * n_R));
    5178:         PetscCall(KSPSolveTranspose(pcbddc->ksp_R, pcbddc->vec1_R, pcbddc->vec2_R));
    5179:         PetscCall(KSPCheckSolve(pcbddc->ksp_R, pc, pcbddc->vec2_R));
    5180:         PetscCall(VecResetArray(pcbddc->vec1_R));
    5181:         PetscCall(VecResetArray(pcbddc->vec2_R));
    5182:       }
    5183:       PetscCall(MatDenseRestoreArray(B_C, &marray));
    5184:     }
    5185:     /* coarse basis functions */
    5186:     PetscCall(MatCreateSeqDense(PETSC_COMM_SELF, n_B, pcbddc->local_primal_size, NULL, &pcbddc->coarse_psi_B));
    5187:     if (pcbddc->switch_static || pcbddc->dbg_flag) PetscCall(MatCreateSeqDense(PETSC_COMM_SELF, n_D, pcbddc->local_primal_size, NULL, &pcbddc->coarse_psi_D));
    5188:     for (i = 0; i < pcbddc->local_primal_size; i++) {
    5189:       Vec v;

    5191:       PetscCall(VecPlaceArray(pcbddc->vec1_R, work + i * n_R));
    5192:       PetscCall(MatDenseGetColumnVec(pcbddc->coarse_psi_B, i, &v));
    5193:       PetscCall(VecScatterBegin(pcbddc->R_to_B, pcbddc->vec1_R, v, INSERT_VALUES, SCATTER_FORWARD));
    5194:       PetscCall(VecScatterEnd(pcbddc->R_to_B, pcbddc->vec1_R, v, INSERT_VALUES, SCATTER_FORWARD));
    5195:       if (i < n_vertices) {
    5196:         PetscScalar one = 1.0;
    5197:         PetscCall(VecSetValues(v, 1, &idx_V_B[i], &one, INSERT_VALUES));
    5198:         PetscCall(VecAssemblyBegin(v));
    5199:         PetscCall(VecAssemblyEnd(v));
    5200:       }
    5201:       PetscCall(MatDenseRestoreColumnVec(pcbddc->coarse_psi_B, i, &v));

    5203:       if (pcbddc->switch_static || pcbddc->dbg_flag) {
    5204:         PetscCall(MatDenseGetColumnVec(pcbddc->coarse_psi_D, i, &v));
    5205:         PetscCall(VecScatterBegin(pcbddc->R_to_D, pcbddc->vec1_R, v, INSERT_VALUES, SCATTER_FORWARD));
    5206:         PetscCall(VecScatterEnd(pcbddc->R_to_D, pcbddc->vec1_R, v, INSERT_VALUES, SCATTER_FORWARD));
    5207:         PetscCall(MatDenseRestoreColumnVec(pcbddc->coarse_psi_D, i, &v));
    5208:       }
    5209:       PetscCall(VecResetArray(pcbddc->vec1_R));
    5210:     }
    5211:     PetscCall(MatDestroy(&B_V));
    5212:     PetscCall(MatDestroy(&B_C));
    5213:     PetscCall(PetscFree(work));
    5214:   } else {
    5215:     PetscCall(PetscObjectReference((PetscObject)pcbddc->coarse_phi_B));
    5216:     pcbddc->coarse_psi_B = pcbddc->coarse_phi_B;
    5217:     PetscCall(PetscObjectReference((PetscObject)pcbddc->coarse_phi_D));
    5218:     pcbddc->coarse_psi_D = pcbddc->coarse_phi_D;
    5219:   }
    5220:   PetscCall(MatAssemblyBegin(*coarse_submat, MAT_FINAL_ASSEMBLY));
    5221:   PetscCall(MatAssemblyEnd(*coarse_submat, MAT_FINAL_ASSEMBLY));

    5223:   /* free memory */
    5224:   PetscCall(PetscFree(V_to_eff_V));
    5225:   PetscCall(PetscFree(C_to_eff_C));
    5226:   PetscCall(PetscFree(R_eff_V_J));
    5227:   PetscCall(PetscFree(R_eff_C_J));
    5228:   PetscCall(PetscFree(B_eff_V_J));
    5229:   PetscCall(PetscFree(B_eff_C_J));
    5230:   PetscCall(ISDestroy(&is_R));
    5231:   PetscCall(ISRestoreIndices(is_V, &idx_V));
    5232:   PetscCall(ISRestoreIndices(is_C, &idx_C));
    5233:   PetscCall(ISDestroy(&is_V));
    5234:   PetscCall(ISDestroy(&is_C));
    5235:   PetscCall(PetscFree(idx_V_B));
    5236:   PetscCall(MatDestroy(&S_CV));
    5237:   PetscCall(MatDestroy(&S_VC));
    5238:   PetscCall(MatDestroy(&S_CC));
    5239:   if (n_vertices) PetscCall(MatDestroy(&A_VR));
    5240:   if (n_constraints) PetscCall(MatDestroy(&C_CR));
    5241:   PetscCall(PetscLogEventEnd(PC_BDDC_CorrectionSetUp[pcbddc->current_level], pc, 0, 0, 0));

    5243:   /* Checking coarse_sub_mat and coarse basis functions */
    5244:   /* Symmetric case     : It should be \Phi^{(j)^T} A^{(j)} \Phi^{(j)}=coarse_sub_mat */
    5245:   /* Non-symmetric case : It should be \Psi^{(j)^T} A^{(j)} \Phi^{(j)}=coarse_sub_mat */
    5246:   if (pcbddc->dbg_flag) {
    5247:     Mat       AUXMAT, TM1, TM2, TM3, TM4;
    5248:     Mat       coarse_phi_D, coarse_phi_B;
    5249:     Mat       coarse_psi_D, coarse_psi_B;
    5250:     Mat       A_II, A_BB, A_IB, A_BI;
    5251:     Mat       C_B, CPHI;
    5252:     IS        is_dummy;
    5253:     Vec       mones;
    5254:     MatType   checkmattype = MATSEQAIJ;
    5255:     PetscReal real_value;

    5257:     if (pcbddc->benign_n && !pcbddc->benign_change_explicit) {
    5258:       Mat A;
    5259:       PetscCall(PCBDDCBenignProject(pc, NULL, NULL, &A));
    5260:       PetscCall(MatCreateSubMatrix(A, pcis->is_I_local, pcis->is_I_local, MAT_INITIAL_MATRIX, &A_II));
    5261:       PetscCall(MatCreateSubMatrix(A, pcis->is_I_local, pcis->is_B_local, MAT_INITIAL_MATRIX, &A_IB));
    5262:       PetscCall(MatCreateSubMatrix(A, pcis->is_B_local, pcis->is_I_local, MAT_INITIAL_MATRIX, &A_BI));
    5263:       PetscCall(MatCreateSubMatrix(A, pcis->is_B_local, pcis->is_B_local, MAT_INITIAL_MATRIX, &A_BB));
    5264:       PetscCall(MatDestroy(&A));
    5265:     } else {
    5266:       PetscCall(MatConvert(pcis->A_II, checkmattype, MAT_INITIAL_MATRIX, &A_II));
    5267:       PetscCall(MatConvert(pcis->A_IB, checkmattype, MAT_INITIAL_MATRIX, &A_IB));
    5268:       PetscCall(MatConvert(pcis->A_BI, checkmattype, MAT_INITIAL_MATRIX, &A_BI));
    5269:       PetscCall(MatConvert(pcis->A_BB, checkmattype, MAT_INITIAL_MATRIX, &A_BB));
    5270:     }
    5271:     PetscCall(MatConvert(pcbddc->coarse_phi_D, checkmattype, MAT_INITIAL_MATRIX, &coarse_phi_D));
    5272:     PetscCall(MatConvert(pcbddc->coarse_phi_B, checkmattype, MAT_INITIAL_MATRIX, &coarse_phi_B));
    5273:     if (!pcbddc->symmetric_primal) {
    5274:       PetscCall(MatConvert(pcbddc->coarse_psi_D, checkmattype, MAT_INITIAL_MATRIX, &coarse_psi_D));
    5275:       PetscCall(MatConvert(pcbddc->coarse_psi_B, checkmattype, MAT_INITIAL_MATRIX, &coarse_psi_B));
    5276:     }
    5277:     PetscCall(PetscViewerASCIIPrintf(pcbddc->dbg_viewer, "--------------------------------------------------\n"));
    5278:     PetscCall(PetscViewerASCIIPrintf(pcbddc->dbg_viewer, "Check coarse sub mat computation (symmetric %d)\n", pcbddc->symmetric_primal));
    5279:     PetscCall(PetscViewerFlush(pcbddc->dbg_viewer));
    5280:     if (!pcbddc->symmetric_primal) {
    5281:       PetscCall(MatMatMult(A_II, coarse_phi_D, MAT_INITIAL_MATRIX, 1.0, &AUXMAT));
    5282:       PetscCall(MatTransposeMatMult(coarse_psi_D, AUXMAT, MAT_INITIAL_MATRIX, 1.0, &TM1));
    5283:       PetscCall(MatDestroy(&AUXMAT));
    5284:       PetscCall(MatMatMult(A_BB, coarse_phi_B, MAT_INITIAL_MATRIX, 1.0, &AUXMAT));
    5285:       PetscCall(MatTransposeMatMult(coarse_psi_B, AUXMAT, MAT_INITIAL_MATRIX, 1.0, &TM2));
    5286:       PetscCall(MatDestroy(&AUXMAT));
    5287:       PetscCall(MatMatMult(A_IB, coarse_phi_B, MAT_INITIAL_MATRIX, 1.0, &AUXMAT));
    5288:       PetscCall(MatTransposeMatMult(coarse_psi_D, AUXMAT, MAT_INITIAL_MATRIX, 1.0, &TM3));
    5289:       PetscCall(MatDestroy(&AUXMAT));
    5290:       PetscCall(MatMatMult(A_BI, coarse_phi_D, MAT_INITIAL_MATRIX, 1.0, &AUXMAT));
    5291:       PetscCall(MatTransposeMatMult(coarse_psi_B, AUXMAT, MAT_INITIAL_MATRIX, 1.0, &TM4));
    5292:       PetscCall(MatDestroy(&AUXMAT));
    5293:     } else {
    5294:       PetscCall(MatPtAP(A_II, coarse_phi_D, MAT_INITIAL_MATRIX, 1.0, &TM1));
    5295:       PetscCall(MatPtAP(A_BB, coarse_phi_B, MAT_INITIAL_MATRIX, 1.0, &TM2));
    5296:       PetscCall(MatMatMult(A_IB, coarse_phi_B, MAT_INITIAL_MATRIX, 1.0, &AUXMAT));
    5297:       PetscCall(MatTransposeMatMult(coarse_phi_D, AUXMAT, MAT_INITIAL_MATRIX, 1.0, &TM3));
    5298:       PetscCall(MatDestroy(&AUXMAT));
    5299:       PetscCall(MatMatMult(A_BI, coarse_phi_D, MAT_INITIAL_MATRIX, 1.0, &AUXMAT));
    5300:       PetscCall(MatTransposeMatMult(coarse_phi_B, AUXMAT, MAT_INITIAL_MATRIX, 1.0, &TM4));
    5301:       PetscCall(MatDestroy(&AUXMAT));
    5302:     }
    5303:     PetscCall(MatAXPY(TM1, one, TM2, DIFFERENT_NONZERO_PATTERN));
    5304:     PetscCall(MatAXPY(TM1, one, TM3, DIFFERENT_NONZERO_PATTERN));
    5305:     PetscCall(MatAXPY(TM1, one, TM4, DIFFERENT_NONZERO_PATTERN));
    5306:     PetscCall(MatConvert(TM1, MATSEQDENSE, MAT_INPLACE_MATRIX, &TM1));
    5307:     if (pcbddc->benign_n) {
    5308:       Mat                B0_B, B0_BPHI;
    5309:       const PetscScalar *data2;
    5310:       PetscScalar       *data;
    5311:       PetscInt           j;

    5313:       PetscCall(ISCreateStride(PETSC_COMM_SELF, pcbddc->benign_n, 0, 1, &is_dummy));
    5314:       PetscCall(MatCreateSubMatrix(pcbddc->benign_B0, is_dummy, pcis->is_B_local, MAT_INITIAL_MATRIX, &B0_B));
    5315:       PetscCall(MatMatMult(B0_B, coarse_phi_B, MAT_INITIAL_MATRIX, 1.0, &B0_BPHI));
    5316:       PetscCall(MatConvert(B0_BPHI, MATSEQDENSE, MAT_INPLACE_MATRIX, &B0_BPHI));
    5317:       PetscCall(MatDenseGetArray(TM1, &data));
    5318:       PetscCall(MatDenseGetArrayRead(B0_BPHI, &data2));
    5319:       for (j = 0; j < pcbddc->benign_n; j++) {
    5320:         PetscInt primal_idx = pcbddc->local_primal_size - pcbddc->benign_n + j;
    5321:         for (i = 0; i < pcbddc->local_primal_size; i++) {
    5322:           data[primal_idx * pcbddc->local_primal_size + i] += data2[i * pcbddc->benign_n + j];
    5323:           data[i * pcbddc->local_primal_size + primal_idx] += data2[i * pcbddc->benign_n + j];
    5324:         }
    5325:       }
    5326:       PetscCall(MatDenseRestoreArray(TM1, &data));
    5327:       PetscCall(MatDenseRestoreArrayRead(B0_BPHI, &data2));
    5328:       PetscCall(MatDestroy(&B0_B));
    5329:       PetscCall(ISDestroy(&is_dummy));
    5330:       PetscCall(MatDestroy(&B0_BPHI));
    5331:     }
    5332:     PetscCall(MatAXPY(TM1, m_one, *coarse_submat, DIFFERENT_NONZERO_PATTERN));
    5333:     PetscCall(MatNorm(TM1, NORM_FROBENIUS, &real_value));
    5334:     PetscCall(PetscViewerASCIIPushSynchronized(pcbddc->dbg_viewer));
    5335:     PetscCall(PetscViewerASCIISynchronizedPrintf(pcbddc->dbg_viewer, "Subdomain %04d          matrix error % 1.14e\n", PetscGlobalRank, (double)real_value));

    5337:     /* check constraints */
    5338:     PetscCall(ISCreateStride(PETSC_COMM_SELF, pcbddc->local_primal_size - pcbddc->benign_n, 0, 1, &is_dummy));
    5339:     PetscCall(MatCreateSubMatrix(pcbddc->ConstraintMatrix, is_dummy, pcis->is_B_local, MAT_INITIAL_MATRIX, &C_B));
    5340:     if (!pcbddc->benign_n) { /* TODO: add benign case */
    5341:       PetscCall(MatMatMult(C_B, coarse_phi_B, MAT_INITIAL_MATRIX, 1.0, &CPHI));
    5342:     } else {
    5343:       PetscScalar *data;
    5344:       Mat          tmat;
    5345:       PetscCall(MatDenseGetArray(pcbddc->coarse_phi_B, &data));
    5346:       PetscCall(MatCreateSeqDense(PETSC_COMM_SELF, pcis->n_B, pcbddc->local_primal_size - pcbddc->benign_n, data, &tmat));
    5347:       PetscCall(MatDenseRestoreArray(pcbddc->coarse_phi_B, &data));
    5348:       PetscCall(MatMatMult(C_B, tmat, MAT_INITIAL_MATRIX, 1.0, &CPHI));
    5349:       PetscCall(MatDestroy(&tmat));
    5350:     }
    5351:     PetscCall(MatCreateVecs(CPHI, &mones, NULL));
    5352:     PetscCall(VecSet(mones, -1.0));
    5353:     PetscCall(MatDiagonalSet(CPHI, mones, ADD_VALUES));
    5354:     PetscCall(MatNorm(CPHI, NORM_FROBENIUS, &real_value));
    5355:     PetscCall(PetscViewerASCIISynchronizedPrintf(pcbddc->dbg_viewer, "Subdomain %04d phi constraints error % 1.14e\n", PetscGlobalRank, (double)real_value));
    5356:     if (!pcbddc->symmetric_primal) {
    5357:       PetscCall(MatMatMult(C_B, coarse_psi_B, MAT_REUSE_MATRIX, 1.0, &CPHI));
    5358:       PetscCall(VecSet(mones, -1.0));
    5359:       PetscCall(MatDiagonalSet(CPHI, mones, ADD_VALUES));
    5360:       PetscCall(MatNorm(CPHI, NORM_FROBENIUS, &real_value));
    5361:       PetscCall(PetscViewerASCIISynchronizedPrintf(pcbddc->dbg_viewer, "Subdomain %04d psi constraints error % 1.14e\n", PetscGlobalRank, (double)real_value));
    5362:     }
    5363:     PetscCall(MatDestroy(&C_B));
    5364:     PetscCall(MatDestroy(&CPHI));
    5365:     PetscCall(ISDestroy(&is_dummy));
    5366:     PetscCall(VecDestroy(&mones));
    5367:     PetscCall(PetscViewerFlush(pcbddc->dbg_viewer));
    5368:     PetscCall(MatDestroy(&A_II));
    5369:     PetscCall(MatDestroy(&A_BB));
    5370:     PetscCall(MatDestroy(&A_IB));
    5371:     PetscCall(MatDestroy(&A_BI));
    5372:     PetscCall(MatDestroy(&TM1));
    5373:     PetscCall(MatDestroy(&TM2));
    5374:     PetscCall(MatDestroy(&TM3));
    5375:     PetscCall(MatDestroy(&TM4));
    5376:     PetscCall(MatDestroy(&coarse_phi_D));
    5377:     PetscCall(MatDestroy(&coarse_phi_B));
    5378:     if (!pcbddc->symmetric_primal) {
    5379:       PetscCall(MatDestroy(&coarse_psi_D));
    5380:       PetscCall(MatDestroy(&coarse_psi_B));
    5381:     }
    5382:   }

    5384: #if 0
    5385:   {
    5386:     PetscViewer viewer;
    5387:     char filename[256];

    5389:     PetscCall(PetscSNPrintf(filename, PETSC_STATIC_ARRAY_LENGTH(filename), "details_local_coarse_mat%d_level%d.m",PetscGlobalRank,pcbddc->current_level));
    5390:     PetscCall(PetscViewerASCIIOpen(PETSC_COMM_SELF,filename,&viewer));
    5391:     PetscCall(PetscViewerPushFormat(viewer,PETSC_VIEWER_ASCII_MATLAB));
    5392:     PetscCall(PetscObjectSetName((PetscObject)*coarse_submat,"coarse submat"));
    5393:     PetscCall(MatView(*coarse_submat,viewer));
    5394:     if (pcbddc->coarse_phi_B) {
    5395:       PetscCall(PetscObjectSetName((PetscObject)pcbddc->coarse_phi_B,"phi_B"));
    5396:       PetscCall(MatView(pcbddc->coarse_phi_B,viewer));
    5397:     }
    5398:     if (pcbddc->coarse_phi_D) {
    5399:       PetscCall(PetscObjectSetName((PetscObject)pcbddc->coarse_phi_D,"phi_D"));
    5400:       PetscCall(MatView(pcbddc->coarse_phi_D,viewer));
    5401:     }
    5402:     if (pcbddc->coarse_psi_B) {
    5403:       PetscCall(PetscObjectSetName((PetscObject)pcbddc->coarse_psi_B,"psi_B"));
    5404:       PetscCall(MatView(pcbddc->coarse_psi_B,viewer));
    5405:     }
    5406:     if (pcbddc->coarse_psi_D) {
    5407:       PetscCall(PetscObjectSetName((PetscObject)pcbddc->coarse_psi_D,"psi_D"));
    5408:       PetscCall(MatView(pcbddc->coarse_psi_D,viewer));
    5409:     }
    5410:     PetscCall(PetscObjectSetName((PetscObject)pcbddc->local_mat,"A"));
    5411:     PetscCall(MatView(pcbddc->local_mat,viewer));
    5412:     PetscCall(PetscObjectSetName((PetscObject)pcbddc->ConstraintMatrix,"C"));
    5413:     PetscCall(MatView(pcbddc->ConstraintMatrix,viewer));
    5414:     PetscCall(PetscObjectSetName((PetscObject)pcis->is_I_local,"I"));
    5415:     PetscCall(ISView(pcis->is_I_local,viewer));
    5416:     PetscCall(PetscObjectSetName((PetscObject)pcis->is_B_local,"B"));
    5417:     PetscCall(ISView(pcis->is_B_local,viewer));
    5418:     PetscCall(PetscObjectSetName((PetscObject)pcbddc->is_R_local,"R"));
    5419:     PetscCall(ISView(pcbddc->is_R_local,viewer));
    5420:     PetscCall(PetscViewerDestroy(&viewer));
    5421:   }
    5422: #endif

    5424:   /* device support */
    5425:   {
    5426:     PetscBool iscuda, iship, iskokkos;
    5427:     MatType   mtype = NULL;

    5429:     PetscCall(PetscObjectTypeCompareAny((PetscObject)pcis->vec1_N, &iscuda, VECCUDA, VECMPICUDA, VECSEQCUDA, ""));
    5430:     PetscCall(PetscObjectTypeCompareAny((PetscObject)pcis->vec1_N, &iship, VECHIP, VECMPIHIP, VECSEQHIP, ""));
    5431:     PetscCall(PetscObjectTypeCompareAny((PetscObject)pcis->vec1_N, &iskokkos, VECKOKKOS, VECMPIKOKKOS, VECSEQKOKKOS, ""));
    5432:     if (iskokkos) {
    5433:       if (PetscDefined(HAVE_MACRO_KOKKOS_ENABLE_CUDA)) iscuda = PETSC_TRUE;
    5434:       else if (PetscDefined(HAVE_MACRO_KOKKOS_ENABLE_HIP)) iship = PETSC_TRUE;
    5435:     }
    5436:     if (iskokkos) mtype = multi_element ? MATSEQAIJKOKKOS : (iscuda ? MATSEQDENSECUDA : MATSEQDENSEHIP);
    5437:     else if (iship) mtype = multi_element ? MATSEQAIJHIPSPARSE : MATSEQDENSEHIP;
    5438:     else if (iscuda) mtype = multi_element ? MATSEQAIJCUSPARSE : MATSEQDENSECUDA;
    5439:     if (mtype) {
    5440:       if (pcbddc->local_auxmat1) PetscCall(MatConvert(pcbddc->local_auxmat1, mtype, MAT_INPLACE_MATRIX, &pcbddc->local_auxmat1));
    5441:       if (pcbddc->local_auxmat2) PetscCall(MatConvert(pcbddc->local_auxmat2, mtype, MAT_INPLACE_MATRIX, &pcbddc->local_auxmat2));
    5442:       if (pcbddc->coarse_phi_B) PetscCall(MatConvert(pcbddc->coarse_phi_B, mtype, MAT_INPLACE_MATRIX, &pcbddc->coarse_phi_B));
    5443:       if (pcbddc->coarse_phi_D) PetscCall(MatConvert(pcbddc->coarse_phi_D, mtype, MAT_INPLACE_MATRIX, &pcbddc->coarse_phi_D));
    5444:       if (pcbddc->coarse_psi_B) PetscCall(MatConvert(pcbddc->coarse_psi_B, mtype, MAT_INPLACE_MATRIX, &pcbddc->coarse_psi_B));
    5445:       if (pcbddc->coarse_psi_D) PetscCall(MatConvert(pcbddc->coarse_psi_D, mtype, MAT_INPLACE_MATRIX, &pcbddc->coarse_psi_D));
    5446:     }
    5447:   }
    5448:   PetscFunctionReturn(PETSC_SUCCESS);
    5449: }

    5451: PetscErrorCode MatCreateSubMatrixUnsorted(Mat A, IS isrow, IS iscol, Mat *B)
    5452: {
    5453:   Mat      *work_mat;
    5454:   IS        isrow_s, iscol_s;
    5455:   PetscBool rsorted, csorted;
    5456:   PetscInt  rsize, *idxs_perm_r = NULL, csize, *idxs_perm_c = NULL;

    5458:   PetscFunctionBegin;
    5459:   PetscCall(ISSorted(isrow, &rsorted));
    5460:   PetscCall(ISSorted(iscol, &csorted));
    5461:   PetscCall(ISGetLocalSize(isrow, &rsize));
    5462:   PetscCall(ISGetLocalSize(iscol, &csize));

    5464:   if (!rsorted) {
    5465:     const PetscInt *idxs;
    5466:     PetscInt       *idxs_sorted, i;

    5468:     PetscCall(PetscMalloc1(rsize, &idxs_perm_r));
    5469:     PetscCall(PetscMalloc1(rsize, &idxs_sorted));
    5470:     for (i = 0; i < rsize; i++) idxs_perm_r[i] = i;
    5471:     PetscCall(ISGetIndices(isrow, &idxs));
    5472:     PetscCall(PetscSortIntWithPermutation(rsize, idxs, idxs_perm_r));
    5473:     for (i = 0; i < rsize; i++) idxs_sorted[i] = idxs[idxs_perm_r[i]];
    5474:     PetscCall(ISRestoreIndices(isrow, &idxs));
    5475:     PetscCall(ISCreateGeneral(PETSC_COMM_SELF, rsize, idxs_sorted, PETSC_OWN_POINTER, &isrow_s));
    5476:   } else {
    5477:     PetscCall(PetscObjectReference((PetscObject)isrow));
    5478:     isrow_s = isrow;
    5479:   }

    5481:   if (!csorted) {
    5482:     if (isrow == iscol) {
    5483:       PetscCall(PetscObjectReference((PetscObject)isrow_s));
    5484:       iscol_s = isrow_s;
    5485:     } else {
    5486:       const PetscInt *idxs;
    5487:       PetscInt       *idxs_sorted, i;

    5489:       PetscCall(PetscMalloc1(csize, &idxs_perm_c));
    5490:       PetscCall(PetscMalloc1(csize, &idxs_sorted));
    5491:       for (i = 0; i < csize; i++) idxs_perm_c[i] = i;
    5492:       PetscCall(ISGetIndices(iscol, &idxs));
    5493:       PetscCall(PetscSortIntWithPermutation(csize, idxs, idxs_perm_c));
    5494:       for (i = 0; i < csize; i++) idxs_sorted[i] = idxs[idxs_perm_c[i]];
    5495:       PetscCall(ISRestoreIndices(iscol, &idxs));
    5496:       PetscCall(ISCreateGeneral(PETSC_COMM_SELF, csize, idxs_sorted, PETSC_OWN_POINTER, &iscol_s));
    5497:     }
    5498:   } else {
    5499:     PetscCall(PetscObjectReference((PetscObject)iscol));
    5500:     iscol_s = iscol;
    5501:   }

    5503:   PetscCall(MatCreateSubMatrices(A, 1, &isrow_s, &iscol_s, MAT_INITIAL_MATRIX, &work_mat));

    5505:   if (!rsorted || !csorted) {
    5506:     Mat new_mat;
    5507:     IS  is_perm_r, is_perm_c;

    5509:     if (!rsorted) {
    5510:       PetscInt *idxs_r, i;
    5511:       PetscCall(PetscMalloc1(rsize, &idxs_r));
    5512:       for (i = 0; i < rsize; i++) idxs_r[idxs_perm_r[i]] = i;
    5513:       PetscCall(PetscFree(idxs_perm_r));
    5514:       PetscCall(ISCreateGeneral(PETSC_COMM_SELF, rsize, idxs_r, PETSC_OWN_POINTER, &is_perm_r));
    5515:     } else {
    5516:       PetscCall(ISCreateStride(PETSC_COMM_SELF, rsize, 0, 1, &is_perm_r));
    5517:     }
    5518:     PetscCall(ISSetPermutation(is_perm_r));

    5520:     if (!csorted) {
    5521:       if (isrow_s == iscol_s) {
    5522:         PetscCall(PetscObjectReference((PetscObject)is_perm_r));
    5523:         is_perm_c = is_perm_r;
    5524:       } else {
    5525:         PetscInt *idxs_c, i;
    5526:         PetscCheck(idxs_perm_c, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Permutation array not present");
    5527:         PetscCall(PetscMalloc1(csize, &idxs_c));
    5528:         for (i = 0; i < csize; i++) idxs_c[idxs_perm_c[i]] = i;
    5529:         PetscCall(PetscFree(idxs_perm_c));
    5530:         PetscCall(ISCreateGeneral(PETSC_COMM_SELF, csize, idxs_c, PETSC_OWN_POINTER, &is_perm_c));
    5531:       }
    5532:     } else {
    5533:       PetscCall(ISCreateStride(PETSC_COMM_SELF, csize, 0, 1, &is_perm_c));
    5534:     }
    5535:     PetscCall(ISSetPermutation(is_perm_c));

    5537:     PetscCall(MatPermute(work_mat[0], is_perm_r, is_perm_c, &new_mat));
    5538:     PetscCall(MatDestroy(&work_mat[0]));
    5539:     work_mat[0] = new_mat;
    5540:     PetscCall(ISDestroy(&is_perm_r));
    5541:     PetscCall(ISDestroy(&is_perm_c));
    5542:   }

    5544:   PetscCall(PetscObjectReference((PetscObject)work_mat[0]));
    5545:   *B = work_mat[0];
    5546:   PetscCall(MatDestroyMatrices(1, &work_mat));
    5547:   PetscCall(ISDestroy(&isrow_s));
    5548:   PetscCall(ISDestroy(&iscol_s));
    5549:   PetscFunctionReturn(PETSC_SUCCESS);
    5550: }

    5552: static PetscErrorCode MatPtAPWithPrefix_Private(Mat A, Mat P, PetscReal fill, const char *prefix, Mat *C)
    5553: {
    5554:   PetscFunctionBegin;
    5555:   PetscCall(MatProductCreate(A, P, NULL, C));
    5556:   PetscCall(MatProductSetType(*C, MATPRODUCT_PtAP));
    5557:   PetscCall(MatProductSetAlgorithm(*C, "default"));
    5558:   PetscCall(MatProductSetFill(*C, fill));
    5559:   PetscCall(MatSetOptionsPrefix(*C, prefix));
    5560:   PetscCall(MatProductSetFromOptions(*C));
    5561:   PetscCall(MatProductSymbolic(*C));
    5562:   PetscCall(MatProductNumeric(*C));
    5563:   (*C)->symmetric = A->symmetric;
    5564:   (*C)->spd       = A->spd;
    5565:   PetscFunctionReturn(PETSC_SUCCESS);
    5566: }

    5568: PetscErrorCode PCBDDCComputeLocalMatrix(PC pc, Mat ChangeOfBasisMatrix)
    5569: {
    5570:   Mat_IS   *matis  = (Mat_IS *)pc->pmat->data;
    5571:   PC_BDDC  *pcbddc = (PC_BDDC *)pc->data;
    5572:   Mat       new_mat, lA;
    5573:   IS        is_local, is_global;
    5574:   PetscInt  local_size;
    5575:   PetscBool isseqaij, issym, isset;
    5576:   char      ptapprefix[256];

    5578:   PetscFunctionBegin;
    5579:   PetscCall(MatDestroy(&pcbddc->local_mat));
    5580:   PetscCall(MatGetSize(matis->A, &local_size, NULL));
    5581:   if (pcbddc->mat_graph->multi_element) {
    5582:     Mat     *mats, *bdiags;
    5583:     IS      *gsubs;
    5584:     PetscInt nsubs = pcbddc->n_local_subs;

    5586:     PetscCall(PetscCalloc1(nsubs * nsubs, &mats));
    5587: #if 1
    5588:     PetscCall(PetscMalloc1(nsubs, &gsubs));
    5589:     for (PetscInt i = 0; i < nsubs; i++) PetscCall(ISLocalToGlobalMappingApplyIS(matis->rmapping, pcbddc->local_subs[i], &gsubs[i]));
    5590:     PetscCall(MatCreateSubMatrices(ChangeOfBasisMatrix, nsubs, gsubs, gsubs, MAT_INITIAL_MATRIX, &bdiags));
    5591:     for (PetscInt i = 0; i < nsubs; i++) PetscCall(ISDestroy(&gsubs[i]));
    5592:     PetscCall(PetscFree(gsubs));
    5593: #else /* this does not work since MatCreateSubMatrices does not support repeated indices */
    5594:     Mat *tmats;
    5595:     PetscCall(ISCreateStride(PetscObjectComm((PetscObject)matis->A), local_size, 0, 1, &is_local));
    5596:     PetscCall(ISLocalToGlobalMappingApplyIS(matis->rmapping, is_local, &is_global));
    5597:     PetscCall(ISDestroy(&is_local));
    5598:     PetscCall(MatSetOption(ChangeOfBasisMatrix, MAT_SUBMAT_SINGLEIS, PETSC_TRUE));
    5599:     PetscCall(MatCreateSubMatrices(ChangeOfBasisMatrix, 1, &is_global, &is_global, MAT_INITIAL_MATRIX, &tmats));
    5600:     PetscCall(ISDestroy(&is_global));
    5601:     PetscCall(MatCreateSubMatrices(tmats[0], nsubs, pcbddc->local_subs, pcbddc->local_subs, MAT_INITIAL_MATRIX, &bdiags));
    5602:     PetscCall(MatDestroySubMatrices(1, &tmats));
    5603: #endif
    5604:     for (PetscInt i = 0; i < nsubs; i++) mats[i * (1 + nsubs)] = bdiags[i];
    5605:     PetscCall(MatCreateNest(PETSC_COMM_SELF, nsubs, pcbddc->local_subs, nsubs, pcbddc->local_subs, mats, &new_mat));
    5606:     PetscCall(MatConvert(new_mat, MATSEQAIJ, MAT_INPLACE_MATRIX, &new_mat));
    5607:     PetscCall(MatDestroySubMatrices(nsubs, &bdiags));
    5608:     PetscCall(PetscFree(mats));
    5609:   } else {
    5610:     PetscCall(ISCreateStride(PetscObjectComm((PetscObject)matis->A), local_size, 0, 1, &is_local));
    5611:     PetscCall(ISLocalToGlobalMappingApplyIS(matis->rmapping, is_local, &is_global));
    5612:     PetscCall(ISDestroy(&is_local));
    5613:     PetscCall(MatCreateSubMatrixUnsorted(ChangeOfBasisMatrix, is_global, is_global, &new_mat));
    5614:     PetscCall(ISDestroy(&is_global));
    5615:   }
    5616:   if (pcbddc->dbg_flag) {
    5617:     Vec       x, x_change;
    5618:     PetscReal error;

    5620:     PetscCall(MatCreateVecs(ChangeOfBasisMatrix, &x, &x_change));
    5621:     PetscCall(VecSetRandom(x, NULL));
    5622:     PetscCall(MatMult(ChangeOfBasisMatrix, x, x_change));
    5623:     PetscCall(VecScatterBegin(matis->cctx, x, matis->x, INSERT_VALUES, SCATTER_FORWARD));
    5624:     PetscCall(VecScatterEnd(matis->cctx, x, matis->x, INSERT_VALUES, SCATTER_FORWARD));
    5625:     PetscCall(MatMult(new_mat, matis->x, matis->y));
    5626:     if (!pcbddc->change_interior) {
    5627:       const PetscScalar *x, *y, *v;
    5628:       PetscReal          lerror = 0.;
    5629:       PetscInt           i;

    5631:       PetscCall(VecGetArrayRead(matis->x, &x));
    5632:       PetscCall(VecGetArrayRead(matis->y, &y));
    5633:       PetscCall(VecGetArrayRead(matis->counter, &v));
    5634:       for (i = 0; i < local_size; i++)
    5635:         if (PetscRealPart(v[i]) < 1.5 && PetscAbsScalar(x[i] - y[i]) > lerror) lerror = PetscAbsScalar(x[i] - y[i]);
    5636:       PetscCall(VecRestoreArrayRead(matis->x, &x));
    5637:       PetscCall(VecRestoreArrayRead(matis->y, &y));
    5638:       PetscCall(VecRestoreArrayRead(matis->counter, &v));
    5639:       PetscCallMPI(MPIU_Allreduce(&lerror, &error, 1, MPIU_REAL, MPIU_MAX, PetscObjectComm((PetscObject)pc)));
    5640:       if (error > PETSC_SMALL) {
    5641:         if (!pcbddc->user_ChangeOfBasisMatrix || pcbddc->current_level) {
    5642:           SETERRQ(PetscObjectComm((PetscObject)pc), PETSC_ERR_PLIB, "Error global vs local change on I: %1.6e", (double)error);
    5643:         } else {
    5644:           SETERRQ(PetscObjectComm((PetscObject)pc), PETSC_ERR_USER, "Error global vs local change on I: %1.6e", (double)error);
    5645:         }
    5646:       }
    5647:     }
    5648:     PetscCall(VecScatterBegin(matis->rctx, matis->y, x, INSERT_VALUES, SCATTER_REVERSE));
    5649:     PetscCall(VecScatterEnd(matis->rctx, matis->y, x, INSERT_VALUES, SCATTER_REVERSE));
    5650:     PetscCall(VecAXPY(x, -1.0, x_change));
    5651:     PetscCall(VecNorm(x, NORM_INFINITY, &error));
    5652:     if (error > PETSC_SMALL) {
    5653:       if (!pcbddc->user_ChangeOfBasisMatrix || pcbddc->current_level) {
    5654:         SETERRQ(PetscObjectComm((PetscObject)pc), PETSC_ERR_PLIB, "Error global vs local change on N: %1.6e", (double)error);
    5655:       } else {
    5656:         SETERRQ(PetscObjectComm((PetscObject)pc), PETSC_ERR_USER, "Error global vs local change on N: %1.6e", (double)error);
    5657:       }
    5658:     }
    5659:     PetscCall(VecDestroy(&x));
    5660:     PetscCall(VecDestroy(&x_change));
    5661:   }

    5663:   /* lA is present if we are setting up an inner BDDC for a saddle point FETI-DP */
    5664:   PetscCall(PetscObjectQuery((PetscObject)pc, "__KSPFETIDP_lA", (PetscObject *)&lA));

    5666:   /* TODO: HOW TO WORK WITH BAIJ and SBAIJ and SEQDENSE? */
    5667:   if (((PetscObject)pc)->prefix) PetscCall(PetscSNPrintf(ptapprefix, sizeof(ptapprefix), "%spc_bddc_change_", ((PetscObject)pc)->prefix));
    5668:   else PetscCall(PetscSNPrintf(ptapprefix, sizeof(ptapprefix), "pc_bddc_change_"));
    5669:   PetscCall(PetscObjectBaseTypeCompare((PetscObject)matis->A, MATSEQAIJ, &isseqaij));
    5670:   if (isseqaij) {
    5671:     PetscCall(MatDestroy(&pcbddc->local_mat));
    5672:     PetscCall(MatPtAPWithPrefix_Private(matis->A, new_mat, PETSC_DEFAULT, ptapprefix, &pcbddc->local_mat));
    5673:     if (lA) {
    5674:       Mat work;
    5675:       PetscCall(MatPtAPWithPrefix_Private(lA, new_mat, PETSC_DEFAULT, ptapprefix, &work));
    5676:       PetscCall(PetscObjectCompose((PetscObject)pc, "__KSPFETIDP_lA", (PetscObject)work));
    5677:       PetscCall(MatDestroy(&work));
    5678:     }
    5679:   } else {
    5680:     Mat work_mat;

    5682:     PetscCall(MatDestroy(&pcbddc->local_mat));
    5683:     PetscCall(MatConvert(matis->A, MATSEQAIJ, MAT_INITIAL_MATRIX, &work_mat));
    5684:     PetscCall(MatPtAPWithPrefix_Private(work_mat, new_mat, PETSC_DEFAULT, ptapprefix, &pcbddc->local_mat));
    5685:     PetscCall(MatDestroy(&work_mat));
    5686:     if (lA) {
    5687:       Mat work;
    5688:       PetscCall(MatConvert(lA, MATSEQAIJ, MAT_INITIAL_MATRIX, &work_mat));
    5689:       PetscCall(MatPtAPWithPrefix_Private(work_mat, new_mat, PETSC_DEFAULT, ptapprefix, &work));
    5690:       PetscCall(PetscObjectCompose((PetscObject)pc, "__KSPFETIDP_lA", (PetscObject)work));
    5691:       PetscCall(MatDestroy(&work));
    5692:     }
    5693:   }
    5694:   PetscCall(MatIsSymmetricKnown(matis->A, &isset, &issym));
    5695:   if (isset) PetscCall(MatSetOption(pcbddc->local_mat, MAT_SYMMETRIC, issym));
    5696:   PetscCall(MatDestroy(&new_mat));
    5697:   PetscFunctionReturn(PETSC_SUCCESS);
    5698: }

    5700: PetscErrorCode PCBDDCSetUpLocalScatters(PC pc)
    5701: {
    5702:   PC_IS          *pcis        = (PC_IS *)pc->data;
    5703:   PC_BDDC        *pcbddc      = (PC_BDDC *)pc->data;
    5704:   PCBDDCSubSchurs sub_schurs  = pcbddc->sub_schurs;
    5705:   PetscInt       *idx_R_local = NULL;
    5706:   PetscInt        n_vertices, i, j, n_R, n_D, n_B;
    5707:   PetscInt        vbs, bs;
    5708:   PetscBT         bitmask = NULL;

    5710:   PetscFunctionBegin;
    5711:   /*
    5712:     No need to setup local scatters if
    5713:       - primal space is unchanged
    5714:         AND
    5715:       - we actually have locally some primal dofs (could not be true in multilevel or for isolated subdomains)
    5716:         AND
    5717:       - we are not in debugging mode (this is needed since there are Synchronized prints at the end of the subroutine
    5718:   */
    5719:   if (!pcbddc->new_primal_space_local && pcbddc->local_primal_size && !pcbddc->dbg_flag) PetscFunctionReturn(PETSC_SUCCESS);
    5720:   /* destroy old objects */
    5721:   PetscCall(ISDestroy(&pcbddc->is_R_local));
    5722:   PetscCall(VecScatterDestroy(&pcbddc->R_to_B));
    5723:   PetscCall(VecScatterDestroy(&pcbddc->R_to_D));
    5724:   /* Set Non-overlapping dimensions */
    5725:   n_B        = pcis->n_B;
    5726:   n_D        = pcis->n - n_B;
    5727:   n_vertices = pcbddc->n_vertices;

    5729:   /* Dohrmann's notation: dofs split in R (Remaining: all dofs but the vertices) and V (Vertices) */

    5731:   /* create auxiliary bitmask and allocate workspace */
    5732:   if (!sub_schurs || !sub_schurs->reuse_solver) {
    5733:     PetscCall(PetscMalloc1(pcis->n - n_vertices, &idx_R_local));
    5734:     PetscCall(PetscBTCreate(pcis->n, &bitmask));
    5735:     for (i = 0; i < n_vertices; i++) PetscCall(PetscBTSet(bitmask, pcbddc->local_primal_ref_node[i]));

    5737:     for (i = 0, n_R = 0; i < pcis->n; i++) {
    5738:       if (!PetscBTLookup(bitmask, i)) idx_R_local[n_R++] = i;
    5739:     }
    5740:   } else { /* A different ordering (already computed) is present if we are reusing the Schur solver */
    5741:     PCBDDCReuseSolvers reuse_solver = sub_schurs->reuse_solver;

    5743:     PetscCall(ISGetIndices(reuse_solver->is_R, (const PetscInt **)&idx_R_local));
    5744:     PetscCall(ISGetLocalSize(reuse_solver->is_R, &n_R));
    5745:   }

    5747:   /* Block code */
    5748:   vbs = 1;
    5749:   PetscCall(MatGetBlockSize(pcbddc->local_mat, &bs));
    5750:   if (bs > 1 && !(n_vertices % bs)) {
    5751:     PetscBool is_blocked = PETSC_TRUE;
    5752:     PetscInt *vary;
    5753:     if (!sub_schurs || !sub_schurs->reuse_solver) {
    5754:       PetscCall(PetscMalloc1(pcis->n / bs, &vary));
    5755:       PetscCall(PetscArrayzero(vary, pcis->n / bs));
    5756:       /* Verify that the vertex indices correspond to each element in a block (code taken from sbaij2.c) */
    5757:       /* it is ok to check this way since local_primal_ref_node are always sorted by local numbering and idx_R_local is obtained as a complement */
    5758:       for (i = 0; i < n_vertices; i++) vary[pcbddc->local_primal_ref_node[i] / bs]++;
    5759:       for (i = 0; i < pcis->n / bs; i++) {
    5760:         if (vary[i] != 0 && vary[i] != bs) {
    5761:           is_blocked = PETSC_FALSE;
    5762:           break;
    5763:         }
    5764:       }
    5765:       PetscCall(PetscFree(vary));
    5766:     } else {
    5767:       /* Verify directly the R set */
    5768:       for (i = 0; i < n_R / bs; i++) {
    5769:         PetscInt j, node = idx_R_local[bs * i];
    5770:         for (j = 1; j < bs; j++) {
    5771:           if (node != idx_R_local[bs * i + j] - j) {
    5772:             is_blocked = PETSC_FALSE;
    5773:             break;
    5774:           }
    5775:         }
    5776:       }
    5777:     }
    5778:     if (is_blocked) { /* build compressed IS for R nodes (complement of vertices) */
    5779:       vbs = bs;
    5780:       for (i = 0; i < n_R / vbs; i++) idx_R_local[i] = idx_R_local[vbs * i] / vbs;
    5781:     }
    5782:   }
    5783:   PetscCall(ISCreateBlock(PETSC_COMM_SELF, vbs, n_R / vbs, idx_R_local, PETSC_COPY_VALUES, &pcbddc->is_R_local));
    5784:   if (sub_schurs && sub_schurs->reuse_solver) {
    5785:     PCBDDCReuseSolvers reuse_solver = sub_schurs->reuse_solver;

    5787:     PetscCall(ISRestoreIndices(reuse_solver->is_R, (const PetscInt **)&idx_R_local));
    5788:     PetscCall(ISDestroy(&reuse_solver->is_R));
    5789:     PetscCall(PetscObjectReference((PetscObject)pcbddc->is_R_local));
    5790:     reuse_solver->is_R = pcbddc->is_R_local;
    5791:   } else {
    5792:     PetscCall(PetscFree(idx_R_local));
    5793:   }

    5795:   /* print some info if requested */
    5796:   if (pcbddc->dbg_flag) {
    5797:     PetscCall(PetscViewerASCIIPrintf(pcbddc->dbg_viewer, "--------------------------------------------------\n"));
    5798:     PetscCall(PetscViewerFlush(pcbddc->dbg_viewer));
    5799:     PetscCall(PetscViewerASCIIPushSynchronized(pcbddc->dbg_viewer));
    5800:     PetscCall(PetscViewerASCIISynchronizedPrintf(pcbddc->dbg_viewer, "Subdomain %04d local dimensions\n", PetscGlobalRank));
    5801:     PetscCall(PetscViewerASCIISynchronizedPrintf(pcbddc->dbg_viewer, "local_size = %" PetscInt_FMT ", dirichlet_size = %" PetscInt_FMT ", boundary_size = %" PetscInt_FMT "\n", pcis->n, n_D, n_B));
    5802:     PetscCall(PetscViewerASCIISynchronizedPrintf(pcbddc->dbg_viewer, "r_size = %" PetscInt_FMT ", v_size = %" PetscInt_FMT ", constraints = %" PetscInt_FMT ", local_primal_size = %" PetscInt_FMT "\n", n_R, n_vertices,
    5803:                                                  pcbddc->local_primal_size - n_vertices - pcbddc->benign_n, pcbddc->local_primal_size));
    5804:     PetscCall(PetscViewerFlush(pcbddc->dbg_viewer));
    5805:   }

    5807:   /* VecScatters pcbddc->R_to_B and (optionally) pcbddc->R_to_D */
    5808:   if (!sub_schurs || !sub_schurs->reuse_solver) {
    5809:     IS        is_aux1, is_aux2;
    5810:     PetscInt *aux_array1, *aux_array2, *is_indices, *idx_R_local;

    5812:     PetscCall(ISGetIndices(pcbddc->is_R_local, (const PetscInt **)&idx_R_local));
    5813:     PetscCall(PetscMalloc1(pcis->n_B - n_vertices, &aux_array1));
    5814:     PetscCall(PetscMalloc1(pcis->n_B - n_vertices, &aux_array2));
    5815:     PetscCall(ISGetIndices(pcis->is_I_local, (const PetscInt **)&is_indices));
    5816:     for (i = 0; i < n_D; i++) PetscCall(PetscBTSet(bitmask, is_indices[i]));
    5817:     PetscCall(ISRestoreIndices(pcis->is_I_local, (const PetscInt **)&is_indices));
    5818:     for (i = 0, j = 0; i < n_R; i++) {
    5819:       if (!PetscBTLookup(bitmask, idx_R_local[i])) aux_array1[j++] = i;
    5820:     }
    5821:     PetscCall(ISCreateGeneral(PETSC_COMM_SELF, j, aux_array1, PETSC_OWN_POINTER, &is_aux1));
    5822:     PetscCall(ISGetIndices(pcis->is_B_local, (const PetscInt **)&is_indices));
    5823:     for (i = 0, j = 0; i < n_B; i++) {
    5824:       if (!PetscBTLookup(bitmask, is_indices[i])) aux_array2[j++] = i;
    5825:     }
    5826:     PetscCall(ISRestoreIndices(pcis->is_B_local, (const PetscInt **)&is_indices));
    5827:     PetscCall(ISCreateGeneral(PETSC_COMM_SELF, j, aux_array2, PETSC_OWN_POINTER, &is_aux2));
    5828:     PetscCall(VecScatterCreate(pcbddc->vec1_R, is_aux1, pcis->vec1_B, is_aux2, &pcbddc->R_to_B));
    5829:     PetscCall(ISDestroy(&is_aux1));
    5830:     PetscCall(ISDestroy(&is_aux2));

    5832:     if (pcbddc->switch_static || pcbddc->dbg_flag) {
    5833:       PetscCall(PetscMalloc1(n_D, &aux_array1));
    5834:       for (i = 0, j = 0; i < n_R; i++) {
    5835:         if (PetscBTLookup(bitmask, idx_R_local[i])) aux_array1[j++] = i;
    5836:       }
    5837:       PetscCall(ISCreateGeneral(PETSC_COMM_SELF, j, aux_array1, PETSC_OWN_POINTER, &is_aux1));
    5838:       PetscCall(VecScatterCreate(pcbddc->vec1_R, is_aux1, pcis->vec1_D, (IS)0, &pcbddc->R_to_D));
    5839:       PetscCall(ISDestroy(&is_aux1));
    5840:     }
    5841:     PetscCall(PetscBTDestroy(&bitmask));
    5842:     PetscCall(ISRestoreIndices(pcbddc->is_R_local, (const PetscInt **)&idx_R_local));
    5843:   } else {
    5844:     PCBDDCReuseSolvers reuse_solver = sub_schurs->reuse_solver;
    5845:     IS                 tis;
    5846:     PetscInt           schur_size;

    5848:     PetscCall(ISGetLocalSize(reuse_solver->is_B, &schur_size));
    5849:     PetscCall(ISCreateStride(PETSC_COMM_SELF, schur_size, n_D, 1, &tis));
    5850:     PetscCall(VecScatterCreate(pcbddc->vec1_R, tis, pcis->vec1_B, reuse_solver->is_B, &pcbddc->R_to_B));
    5851:     PetscCall(ISDestroy(&tis));
    5852:     if (pcbddc->switch_static || pcbddc->dbg_flag) {
    5853:       PetscCall(ISCreateStride(PETSC_COMM_SELF, n_D, 0, 1, &tis));
    5854:       PetscCall(VecScatterCreate(pcbddc->vec1_R, tis, pcis->vec1_D, (IS)0, &pcbddc->R_to_D));
    5855:       PetscCall(ISDestroy(&tis));
    5856:     }
    5857:   }
    5858:   PetscFunctionReturn(PETSC_SUCCESS);
    5859: }

    5861: PetscErrorCode MatNullSpacePropagateAny_Private(Mat A, IS is, Mat B)
    5862: {
    5863:   MatNullSpace NullSpace;
    5864:   Mat          dmat;
    5865:   const Vec   *nullvecs;
    5866:   Vec          v, v2, *nullvecs2;
    5867:   VecScatter   sct = NULL;
    5868:   PetscScalar *ddata;
    5869:   PetscInt     k, nnsp_size, bsiz, bsiz2, n, N, bs;
    5870:   PetscBool    nnsp_has_cnst;

    5872:   PetscFunctionBegin;
    5873:   if (!is && !B) { /* MATIS */
    5874:     Mat_IS *matis = (Mat_IS *)A->data;

    5876:     if (!B) PetscCall(MatISGetLocalMat(A, &B));
    5877:     sct = matis->cctx;
    5878:     PetscCall(PetscObjectReference((PetscObject)sct));
    5879:   } else {
    5880:     PetscCall(MatGetNullSpace(B, &NullSpace));
    5881:     if (!NullSpace) PetscCall(MatGetNearNullSpace(B, &NullSpace));
    5882:     if (NullSpace) PetscFunctionReturn(PETSC_SUCCESS);
    5883:   }
    5884:   PetscCall(MatGetNullSpace(A, &NullSpace));
    5885:   if (!NullSpace) PetscCall(MatGetNearNullSpace(A, &NullSpace));
    5886:   if (!NullSpace) PetscFunctionReturn(PETSC_SUCCESS);

    5888:   PetscCall(MatCreateVecs(A, &v, NULL));
    5889:   PetscCall(MatCreateVecs(B, &v2, NULL));
    5890:   if (!sct) PetscCall(VecScatterCreate(v, is, v2, NULL, &sct));
    5891:   PetscCall(MatNullSpaceGetVecs(NullSpace, &nnsp_has_cnst, &nnsp_size, &nullvecs));
    5892:   bsiz = bsiz2 = nnsp_size + !!nnsp_has_cnst;
    5893:   PetscCall(PetscMalloc1(bsiz, &nullvecs2));
    5894:   PetscCall(VecGetBlockSize(v2, &bs));
    5895:   PetscCall(VecGetSize(v2, &N));
    5896:   PetscCall(VecGetLocalSize(v2, &n));
    5897:   PetscCall(PetscMalloc1(n * bsiz, &ddata));
    5898:   for (k = 0; k < nnsp_size; k++) {
    5899:     PetscCall(VecCreateMPIWithArray(PetscObjectComm((PetscObject)B), bs, n, N, ddata + n * k, &nullvecs2[k]));
    5900:     PetscCall(VecScatterBegin(sct, nullvecs[k], nullvecs2[k], INSERT_VALUES, SCATTER_FORWARD));
    5901:     PetscCall(VecScatterEnd(sct, nullvecs[k], nullvecs2[k], INSERT_VALUES, SCATTER_FORWARD));
    5902:   }
    5903:   if (nnsp_has_cnst) {
    5904:     PetscCall(VecCreateMPIWithArray(PetscObjectComm((PetscObject)B), bs, n, N, ddata + n * nnsp_size, &nullvecs2[nnsp_size]));
    5905:     PetscCall(VecSet(nullvecs2[nnsp_size], 1.0));
    5906:   }
    5907:   PetscCall(PCBDDCOrthonormalizeVecs(&bsiz2, nullvecs2));
    5908:   PetscCall(MatNullSpaceCreate(PetscObjectComm((PetscObject)B), PETSC_FALSE, bsiz2, nullvecs2, &NullSpace));

    5910:   PetscCall(MatCreateDense(PetscObjectComm((PetscObject)B), n, PETSC_DECIDE, N, bsiz2, ddata, &dmat));
    5911:   PetscCall(PetscObjectContainerCompose((PetscObject)dmat, "_PBDDC_Null_dmat_arr", ddata, PetscCtxDestroyDefault));
    5912:   PetscCall(PetscObjectCompose((PetscObject)NullSpace, "_PBDDC_Null_dmat", (PetscObject)dmat));
    5913:   PetscCall(MatDestroy(&dmat));

    5915:   for (k = 0; k < bsiz; k++) PetscCall(VecDestroy(&nullvecs2[k]));
    5916:   PetscCall(PetscFree(nullvecs2));
    5917:   PetscCall(MatSetNearNullSpace(B, NullSpace));
    5918:   PetscCall(MatNullSpaceDestroy(&NullSpace));
    5919:   PetscCall(VecDestroy(&v));
    5920:   PetscCall(VecDestroy(&v2));
    5921:   PetscCall(VecScatterDestroy(&sct));
    5922:   PetscFunctionReturn(PETSC_SUCCESS);
    5923: }

    5925: PetscErrorCode PCBDDCSetUpLocalSolvers(PC pc, PetscBool dirichlet, PetscBool neumann)
    5926: {
    5927:   PC_BDDC     *pcbddc = (PC_BDDC *)pc->data;
    5928:   PC_IS       *pcis   = (PC_IS *)pc->data;
    5929:   PC           pc_temp;
    5930:   Mat          A_RR;
    5931:   MatNullSpace nnsp;
    5932:   MatReuse     reuse;
    5933:   PetscScalar  m_one = -1.0;
    5934:   PetscReal    value;
    5935:   PetscInt     n_D, n_R;
    5936:   PetscBool    issbaij, opts, isset, issym;
    5937:   PetscBool    f = PETSC_FALSE;
    5938:   char         dir_prefix[256], neu_prefix[256], str_level[16];
    5939:   size_t       len;

    5941:   PetscFunctionBegin;
    5942:   PetscCall(PetscLogEventBegin(PC_BDDC_LocalSolvers[pcbddc->current_level], pc, 0, 0, 0));
    5943:   /* approximate solver, propagate NearNullSpace if needed */
    5944:   if (!pc->setupcalled && (pcbddc->NullSpace_corr[0] || pcbddc->NullSpace_corr[2])) {
    5945:     MatNullSpace gnnsp1, gnnsp2;
    5946:     PetscBool    lhas, ghas;

    5948:     PetscCall(MatGetNearNullSpace(pcbddc->local_mat, &nnsp));
    5949:     PetscCall(MatGetNearNullSpace(pc->pmat, &gnnsp1));
    5950:     PetscCall(MatGetNullSpace(pc->pmat, &gnnsp2));
    5951:     lhas = nnsp ? PETSC_TRUE : PETSC_FALSE;
    5952:     PetscCallMPI(MPIU_Allreduce(&lhas, &ghas, 1, MPI_C_BOOL, MPI_LOR, PetscObjectComm((PetscObject)pc)));
    5953:     if (!ghas && (gnnsp1 || gnnsp2)) PetscCall(MatNullSpacePropagateAny_Private(pc->pmat, NULL, NULL));
    5954:   }

    5956:   /* compute prefixes */
    5957:   PetscCall(PetscStrncpy(dir_prefix, "", sizeof(dir_prefix)));
    5958:   PetscCall(PetscStrncpy(neu_prefix, "", sizeof(neu_prefix)));
    5959:   if (!pcbddc->current_level) {
    5960:     PetscCall(PetscStrncpy(dir_prefix, ((PetscObject)pc)->prefix, sizeof(dir_prefix)));
    5961:     PetscCall(PetscStrncpy(neu_prefix, ((PetscObject)pc)->prefix, sizeof(neu_prefix)));
    5962:     PetscCall(PetscStrlcat(dir_prefix, "pc_bddc_dirichlet_", sizeof(dir_prefix)));
    5963:     PetscCall(PetscStrlcat(neu_prefix, "pc_bddc_neumann_", sizeof(neu_prefix)));
    5964:   } else {
    5965:     PetscCall(PetscSNPrintf(str_level, sizeof(str_level), "l%" PetscInt_FMT "_", pcbddc->current_level));
    5966:     PetscCall(PetscStrlen(((PetscObject)pc)->prefix, &len));
    5967:     len -= 15;                                /* remove "pc_bddc_coarse_" */
    5968:     if (pcbddc->current_level > 1) len -= 3;  /* remove "lX_" with X level number */
    5969:     if (pcbddc->current_level > 10) len -= 1; /* remove another char from level number */
    5970:     /* Nonstandard use of PetscStrncpy() to only copy a portion of the input string */
    5971:     PetscCall(PetscStrncpy(dir_prefix, ((PetscObject)pc)->prefix, len + 1));
    5972:     PetscCall(PetscStrncpy(neu_prefix, ((PetscObject)pc)->prefix, len + 1));
    5973:     PetscCall(PetscStrlcat(dir_prefix, "pc_bddc_dirichlet_", sizeof(dir_prefix)));
    5974:     PetscCall(PetscStrlcat(neu_prefix, "pc_bddc_neumann_", sizeof(neu_prefix)));
    5975:     PetscCall(PetscStrlcat(dir_prefix, str_level, sizeof(dir_prefix)));
    5976:     PetscCall(PetscStrlcat(neu_prefix, str_level, sizeof(neu_prefix)));
    5977:   }

    5979:   /* DIRICHLET PROBLEM */
    5980:   if (dirichlet) {
    5981:     PCBDDCSubSchurs sub_schurs = pcbddc->sub_schurs;
    5982:     if (pcbddc->benign_n && !pcbddc->benign_change_explicit) {
    5983:       PetscCheck(sub_schurs && sub_schurs->reuse_solver, PETSC_COMM_SELF, PETSC_ERR_SUP, "Not yet implemented");
    5984:       if (pcbddc->dbg_flag) {
    5985:         Mat A_IIn;

    5987:         PetscCall(PCBDDCBenignProject(pc, pcis->is_I_local, pcis->is_I_local, &A_IIn));
    5988:         PetscCall(MatDestroy(&pcis->A_II));
    5989:         pcis->A_II = A_IIn;
    5990:       }
    5991:     }
    5992:     PetscCall(MatIsSymmetricKnown(pcbddc->local_mat, &isset, &issym));
    5993:     if (isset) PetscCall(MatSetOption(pcis->A_II, MAT_SYMMETRIC, issym));

    5995:     /* Matrix for Dirichlet problem is pcis->A_II */
    5996:     n_D  = pcis->n - pcis->n_B;
    5997:     opts = PETSC_FALSE;
    5998:     if (!pcbddc->ksp_D) { /* create object if not yet build */
    5999:       opts = PETSC_TRUE;
    6000:       PetscCall(KSPCreate(PETSC_COMM_SELF, &pcbddc->ksp_D));
    6001:       PetscCall(KSPSetNestLevel(pcbddc->ksp_D, pc->kspnestlevel));
    6002:       PetscCall(PetscObjectIncrementTabLevel((PetscObject)pcbddc->ksp_D, (PetscObject)pc, 1));
    6003:       /* default */
    6004:       PetscCall(KSPSetType(pcbddc->ksp_D, KSPPREONLY));
    6005:       PetscCall(KSPSetOptionsPrefix(pcbddc->ksp_D, dir_prefix));
    6006:       PetscCall(PetscObjectTypeCompare((PetscObject)pcis->pA_II, MATSEQSBAIJ, &issbaij));
    6007:       PetscCall(KSPGetPC(pcbddc->ksp_D, &pc_temp));
    6008:       if (issbaij) PetscCall(PCSetType(pc_temp, PCCHOLESKY));
    6009:       else PetscCall(PCSetType(pc_temp, PCLU));
    6010:       PetscCall(KSPSetErrorIfNotConverged(pcbddc->ksp_D, pc->erroriffailure));
    6011:     }
    6012:     PetscCall(MatSetOptionsPrefix(pcis->pA_II, ((PetscObject)pcbddc->ksp_D)->prefix));
    6013:     PetscCall(MatViewFromOptions(pcis->pA_II, NULL, "-mat_view"));
    6014:     PetscCall(KSPSetOperators(pcbddc->ksp_D, pcis->A_II, pcis->pA_II));
    6015:     /* Allow user's customization */
    6016:     if (opts) PetscCall(KSPSetFromOptions(pcbddc->ksp_D));
    6017:     PetscCall(MatGetNearNullSpace(pcis->pA_II, &nnsp));
    6018:     if (pcbddc->NullSpace_corr[0] && !nnsp) { /* approximate solver, propagate NearNullSpace */
    6019:       PetscCall(MatNullSpacePropagateAny_Private(pcbddc->local_mat, pcis->is_I_local, pcis->pA_II));
    6020:     }
    6021:     PetscCall(MatGetNearNullSpace(pcis->pA_II, &nnsp));
    6022:     PetscCall(KSPGetPC(pcbddc->ksp_D, &pc_temp));
    6023:     PetscCall(PetscObjectHasFunction((PetscObject)pc_temp, "PCSetCoordinates_C", &f));
    6024:     if (f && pcbddc->mat_graph->cloc && !nnsp) {
    6025:       PetscReal      *coords = pcbddc->mat_graph->coords, *scoords;
    6026:       const PetscInt *idxs;
    6027:       PetscInt        cdim = pcbddc->mat_graph->cdim, nl, i, d;

    6029:       PetscCall(ISGetLocalSize(pcis->is_I_local, &nl));
    6030:       PetscCall(ISGetIndices(pcis->is_I_local, &idxs));
    6031:       PetscCall(PetscMalloc1(nl * cdim, &scoords));
    6032:       for (i = 0; i < nl; i++) {
    6033:         for (d = 0; d < cdim; d++) scoords[i * cdim + d] = coords[idxs[i] * cdim + d];
    6034:       }
    6035:       PetscCall(ISRestoreIndices(pcis->is_I_local, &idxs));
    6036:       PetscCall(PCSetCoordinates(pc_temp, cdim, nl, scoords));
    6037:       PetscCall(PetscFree(scoords));
    6038:     }
    6039:     if (sub_schurs && sub_schurs->reuse_solver) {
    6040:       PCBDDCReuseSolvers reuse_solver = sub_schurs->reuse_solver;

    6042:       PetscCall(KSPSetPC(pcbddc->ksp_D, reuse_solver->interior_solver));
    6043:     }

    6045:     /* umfpack interface has a bug when matrix dimension is zero. TODO solve from umfpack interface */
    6046:     if (!n_D) {
    6047:       PetscCall(KSPGetPC(pcbddc->ksp_D, &pc_temp));
    6048:       PetscCall(PCSetType(pc_temp, PCNONE));
    6049:     }
    6050:     PetscCall(KSPSetUp(pcbddc->ksp_D));
    6051:     /* set ksp_D into pcis data */
    6052:     PetscCall(PetscObjectReference((PetscObject)pcbddc->ksp_D));
    6053:     PetscCall(KSPDestroy(&pcis->ksp_D));
    6054:     pcis->ksp_D = pcbddc->ksp_D;
    6055:   }

    6057:   /* NEUMANN PROBLEM */
    6058:   A_RR = NULL;
    6059:   if (neumann) {
    6060:     PCBDDCSubSchurs sub_schurs = pcbddc->sub_schurs;
    6061:     PetscInt        ibs, mbs;
    6062:     PetscBool       issbaij, reuse_neumann_solver, isset, issym;
    6063:     Mat_IS         *matis = (Mat_IS *)pc->pmat->data;

    6065:     reuse_neumann_solver = PETSC_FALSE;
    6066:     if (sub_schurs && sub_schurs->reuse_solver) {
    6067:       IS iP;

    6069:       reuse_neumann_solver = PETSC_TRUE;
    6070:       PetscCall(PetscObjectQuery((PetscObject)sub_schurs->A, "__KSPFETIDP_iP", (PetscObject *)&iP));
    6071:       if (iP) reuse_neumann_solver = PETSC_FALSE;
    6072:     }
    6073:     /* Matrix for Neumann problem is A_RR -> we need to create/reuse it at this point */
    6074:     PetscCall(ISGetSize(pcbddc->is_R_local, &n_R));
    6075:     if (pcbddc->ksp_R) { /* already created ksp */
    6076:       PetscInt nn_R;
    6077:       PetscCall(KSPGetOperators(pcbddc->ksp_R, NULL, &A_RR));
    6078:       PetscCall(PetscObjectReference((PetscObject)A_RR));
    6079:       PetscCall(MatGetSize(A_RR, &nn_R, NULL));
    6080:       if (nn_R != n_R) { /* old ksp is not reusable, so reset it */
    6081:         PetscCall(KSPReset(pcbddc->ksp_R));
    6082:         PetscCall(MatDestroy(&A_RR));
    6083:         reuse = MAT_INITIAL_MATRIX;
    6084:       } else {                                /* same sizes, but nonzero pattern depend on primal vertices so it can be changed */
    6085:         if (pcbddc->new_primal_space_local) { /* we are not sure the matrix will have the same nonzero pattern */
    6086:           PetscCall(MatDestroy(&A_RR));
    6087:           reuse = MAT_INITIAL_MATRIX;
    6088:         } else { /* safe to reuse the matrix */
    6089:           reuse = MAT_REUSE_MATRIX;
    6090:         }
    6091:       }
    6092:       /* last check */
    6093:       if (pc->flag == DIFFERENT_NONZERO_PATTERN) {
    6094:         PetscCall(MatDestroy(&A_RR));
    6095:         reuse = MAT_INITIAL_MATRIX;
    6096:       }
    6097:     } else { /* first time, so we need to create the matrix */
    6098:       reuse = MAT_INITIAL_MATRIX;
    6099:     }
    6100:     /* convert pcbddc->local_mat if needed later in PCBDDCSetUpCorrection
    6101:        TODO: Get Rid of these conversions */
    6102:     PetscCall(MatGetBlockSize(pcbddc->local_mat, &mbs));
    6103:     PetscCall(ISGetBlockSize(pcbddc->is_R_local, &ibs));
    6104:     PetscCall(PetscObjectTypeCompare((PetscObject)pcbddc->local_mat, MATSEQSBAIJ, &issbaij));
    6105:     if (ibs != mbs) { /* need to convert to SEQAIJ to extract any submatrix with is_R_local */
    6106:       if (matis->A == pcbddc->local_mat) {
    6107:         PetscCall(MatDestroy(&pcbddc->local_mat));
    6108:         PetscCall(MatConvert(matis->A, MATSEQAIJ, MAT_INITIAL_MATRIX, &pcbddc->local_mat));
    6109:       } else {
    6110:         PetscCall(MatConvert(pcbddc->local_mat, MATSEQAIJ, MAT_INPLACE_MATRIX, &pcbddc->local_mat));
    6111:       }
    6112:     } else if (issbaij) { /* need to convert to BAIJ to get off-diagonal blocks */
    6113:       if (matis->A == pcbddc->local_mat) {
    6114:         PetscCall(MatDestroy(&pcbddc->local_mat));
    6115:         PetscCall(MatConvert(matis->A, mbs > 1 ? MATSEQBAIJ : MATSEQAIJ, MAT_INITIAL_MATRIX, &pcbddc->local_mat));
    6116:       } else {
    6117:         PetscCall(MatConvert(pcbddc->local_mat, mbs > 1 ? MATSEQBAIJ : MATSEQAIJ, MAT_INPLACE_MATRIX, &pcbddc->local_mat));
    6118:       }
    6119:     }
    6120:     /* extract A_RR */
    6121:     if (reuse_neumann_solver) {
    6122:       PCBDDCReuseSolvers reuse_solver = sub_schurs->reuse_solver;

    6124:       if (pcbddc->dbg_flag) { /* we need A_RR to test the solver later */
    6125:         PetscCall(MatDestroy(&A_RR));
    6126:         if (reuse_solver->benign_n) { /* we are not using the explicit change of basis on the pressures */
    6127:           PetscCall(PCBDDCBenignProject(pc, pcbddc->is_R_local, pcbddc->is_R_local, &A_RR));
    6128:         } else {
    6129:           PetscCall(MatCreateSubMatrix(pcbddc->local_mat, pcbddc->is_R_local, pcbddc->is_R_local, MAT_INITIAL_MATRIX, &A_RR));
    6130:         }
    6131:       } else {
    6132:         PetscCall(MatDestroy(&A_RR));
    6133:         PetscCall(PCGetOperators(reuse_solver->correction_solver, &A_RR, NULL));
    6134:         PetscCall(PetscObjectReference((PetscObject)A_RR));
    6135:       }
    6136:     } else { /* we have to build the neumann solver, so we need to extract the relevant matrix */
    6137:       PetscCall(MatCreateSubMatrix(pcbddc->local_mat, pcbddc->is_R_local, pcbddc->is_R_local, reuse, &A_RR));
    6138:     }
    6139:     PetscCall(MatIsSymmetricKnown(pcbddc->local_mat, &isset, &issym));
    6140:     if (isset) PetscCall(MatSetOption(A_RR, MAT_SYMMETRIC, issym));
    6141:     opts = PETSC_FALSE;
    6142:     if (!pcbddc->ksp_R) { /* create object if not present */
    6143:       opts = PETSC_TRUE;
    6144:       PetscCall(KSPCreate(PETSC_COMM_SELF, &pcbddc->ksp_R));
    6145:       PetscCall(KSPSetNestLevel(pcbddc->ksp_R, pc->kspnestlevel));
    6146:       PetscCall(PetscObjectIncrementTabLevel((PetscObject)pcbddc->ksp_R, (PetscObject)pc, 1));
    6147:       /* default */
    6148:       PetscCall(KSPSetType(pcbddc->ksp_R, KSPPREONLY));
    6149:       PetscCall(KSPSetOptionsPrefix(pcbddc->ksp_R, neu_prefix));
    6150:       PetscCall(KSPGetPC(pcbddc->ksp_R, &pc_temp));
    6151:       PetscCall(PetscObjectTypeCompare((PetscObject)A_RR, MATSEQSBAIJ, &issbaij));
    6152:       if (issbaij) PetscCall(PCSetType(pc_temp, PCCHOLESKY));
    6153:       else PetscCall(PCSetType(pc_temp, PCLU));
    6154:       PetscCall(KSPSetErrorIfNotConverged(pcbddc->ksp_R, pc->erroriffailure));
    6155:     }
    6156:     PetscCall(MatSetOptionsPrefix(A_RR, ((PetscObject)pcbddc->ksp_R)->prefix));
    6157:     PetscCall(MatViewFromOptions(A_RR, NULL, "-mat_view"));
    6158:     PetscCall(KSPSetOperators(pcbddc->ksp_R, A_RR, A_RR));
    6159:     if (opts) { /* Allow user's customization once */
    6160:       PetscCall(KSPSetFromOptions(pcbddc->ksp_R));
    6161:     }
    6162:     PetscCall(MatGetNearNullSpace(A_RR, &nnsp));
    6163:     if (pcbddc->NullSpace_corr[2] && !nnsp) { /* approximate solver, propagate NearNullSpace */
    6164:       PetscCall(MatNullSpacePropagateAny_Private(pcbddc->local_mat, pcbddc->is_R_local, A_RR));
    6165:     }
    6166:     PetscCall(MatGetNearNullSpace(A_RR, &nnsp));
    6167:     PetscCall(KSPGetPC(pcbddc->ksp_R, &pc_temp));
    6168:     PetscCall(PetscObjectHasFunction((PetscObject)pc_temp, "PCSetCoordinates_C", &f));
    6169:     if (f && pcbddc->mat_graph->cloc && !nnsp) {
    6170:       PetscReal      *coords = pcbddc->mat_graph->coords, *scoords;
    6171:       const PetscInt *idxs;
    6172:       PetscInt        cdim = pcbddc->mat_graph->cdim, nl, i, d;

    6174:       PetscCall(ISGetLocalSize(pcbddc->is_R_local, &nl));
    6175:       PetscCall(ISGetIndices(pcbddc->is_R_local, &idxs));
    6176:       PetscCall(PetscMalloc1(nl * cdim, &scoords));
    6177:       for (i = 0; i < nl; i++) {
    6178:         for (d = 0; d < cdim; d++) scoords[i * cdim + d] = coords[idxs[i] * cdim + d];
    6179:       }
    6180:       PetscCall(ISRestoreIndices(pcbddc->is_R_local, &idxs));
    6181:       PetscCall(PCSetCoordinates(pc_temp, cdim, nl, scoords));
    6182:       PetscCall(PetscFree(scoords));
    6183:     }

    6185:     /* umfpack interface has a bug when matrix dimension is zero. TODO solve from umfpack interface */
    6186:     if (!n_R) {
    6187:       PetscCall(KSPGetPC(pcbddc->ksp_R, &pc_temp));
    6188:       PetscCall(PCSetType(pc_temp, PCNONE));
    6189:     }
    6190:     /* Reuse solver if it is present */
    6191:     if (reuse_neumann_solver) {
    6192:       PCBDDCReuseSolvers reuse_solver = sub_schurs->reuse_solver;

    6194:       PetscCall(KSPSetPC(pcbddc->ksp_R, reuse_solver->correction_solver));
    6195:     }
    6196:     PetscCall(KSPSetUp(pcbddc->ksp_R));
    6197:   }

    6199:   if (pcbddc->dbg_flag) {
    6200:     PetscCall(PetscViewerFlush(pcbddc->dbg_viewer));
    6201:     PetscCall(PetscViewerASCIIPushSynchronized(pcbddc->dbg_viewer));
    6202:     PetscCall(PetscViewerASCIIPrintf(pcbddc->dbg_viewer, "--------------------------------------------------\n"));
    6203:   }
    6204:   PetscCall(PetscLogEventEnd(PC_BDDC_LocalSolvers[pcbddc->current_level], pc, 0, 0, 0));

    6206:   /* adapt Dirichlet and Neumann solvers if a nullspace correction has been requested */
    6207:   if (pcbddc->NullSpace_corr[0]) PetscCall(PCBDDCSetUseExactDirichlet(pc, PETSC_FALSE));
    6208:   if (dirichlet && pcbddc->NullSpace_corr[0] && !pcbddc->switch_static) PetscCall(PCBDDCNullSpaceAssembleCorrection(pc, PETSC_TRUE, pcbddc->NullSpace_corr[1]));
    6209:   if (neumann && pcbddc->NullSpace_corr[2]) PetscCall(PCBDDCNullSpaceAssembleCorrection(pc, PETSC_FALSE, pcbddc->NullSpace_corr[3]));
    6210:   /* check Dirichlet and Neumann solvers */
    6211:   if (pcbddc->dbg_flag) {
    6212:     if (dirichlet) { /* Dirichlet */
    6213:       PetscCall(VecSetRandom(pcis->vec1_D, NULL));
    6214:       PetscCall(MatMult(pcis->A_II, pcis->vec1_D, pcis->vec2_D));
    6215:       PetscCall(KSPSolve(pcbddc->ksp_D, pcis->vec2_D, pcis->vec2_D));
    6216:       PetscCall(KSPCheckSolve(pcbddc->ksp_D, pc, pcis->vec2_D));
    6217:       PetscCall(VecAXPY(pcis->vec1_D, m_one, pcis->vec2_D));
    6218:       PetscCall(VecNorm(pcis->vec1_D, NORM_INFINITY, &value));
    6219:       PetscCall(PetscViewerASCIISynchronizedPrintf(pcbddc->dbg_viewer, "Subdomain %04d infinity error for Dirichlet solve (%s) = % 1.14e \n", PetscGlobalRank, ((PetscObject)pcbddc->ksp_D)->prefix, (double)value));
    6220:       PetscCall(PetscViewerFlush(pcbddc->dbg_viewer));
    6221:     }
    6222:     if (neumann) { /* Neumann */
    6223:       PetscCall(VecSetRandom(pcbddc->vec1_R, NULL));
    6224:       PetscCall(MatMult(A_RR, pcbddc->vec1_R, pcbddc->vec2_R));
    6225:       PetscCall(KSPSolve(pcbddc->ksp_R, pcbddc->vec2_R, pcbddc->vec2_R));
    6226:       PetscCall(KSPCheckSolve(pcbddc->ksp_R, pc, pcbddc->vec2_R));
    6227:       PetscCall(VecAXPY(pcbddc->vec1_R, m_one, pcbddc->vec2_R));
    6228:       PetscCall(VecNorm(pcbddc->vec1_R, NORM_INFINITY, &value));
    6229:       PetscCall(PetscViewerASCIISynchronizedPrintf(pcbddc->dbg_viewer, "Subdomain %04d infinity error for Neumann solve (%s) = % 1.14e\n", PetscGlobalRank, ((PetscObject)pcbddc->ksp_R)->prefix, (double)value));
    6230:       PetscCall(PetscViewerFlush(pcbddc->dbg_viewer));
    6231:     }
    6232:   }
    6233:   /* free Neumann problem's matrix */
    6234:   PetscCall(MatDestroy(&A_RR));
    6235:   PetscFunctionReturn(PETSC_SUCCESS);
    6236: }

    6238: static PetscErrorCode PCBDDCSolveSubstructureCorrection(PC pc, Vec inout_B, Vec inout_D, PetscBool applytranspose)
    6239: {
    6240:   PC_BDDC        *pcbddc       = (PC_BDDC *)pc->data;
    6241:   PCBDDCSubSchurs sub_schurs   = pcbddc->sub_schurs;
    6242:   PetscBool       reuse_solver = sub_schurs ? (sub_schurs->reuse_solver ? PETSC_TRUE : PETSC_FALSE) : PETSC_FALSE;

    6244:   PetscFunctionBegin;
    6245:   if (!reuse_solver) PetscCall(VecSet(pcbddc->vec1_R, 0.));
    6246:   if (!pcbddc->switch_static) {
    6247:     if (applytranspose && pcbddc->local_auxmat1) {
    6248:       PetscCall(MatMultTranspose(pcbddc->local_auxmat2, inout_B, pcbddc->vec1_C));
    6249:       PetscCall(MatMultTransposeAdd(pcbddc->local_auxmat1, pcbddc->vec1_C, inout_B, inout_B));
    6250:     }
    6251:     if (!reuse_solver) {
    6252:       PetscCall(VecScatterBegin(pcbddc->R_to_B, inout_B, pcbddc->vec1_R, INSERT_VALUES, SCATTER_REVERSE));
    6253:       PetscCall(VecScatterEnd(pcbddc->R_to_B, inout_B, pcbddc->vec1_R, INSERT_VALUES, SCATTER_REVERSE));
    6254:     } else {
    6255:       PCBDDCReuseSolvers reuse_solver = sub_schurs->reuse_solver;

    6257:       PetscCall(VecScatterBegin(reuse_solver->correction_scatter_B, inout_B, reuse_solver->rhs_B, INSERT_VALUES, SCATTER_FORWARD));
    6258:       PetscCall(VecScatterEnd(reuse_solver->correction_scatter_B, inout_B, reuse_solver->rhs_B, INSERT_VALUES, SCATTER_FORWARD));
    6259:     }
    6260:   } else {
    6261:     PetscCall(VecScatterBegin(pcbddc->R_to_B, inout_B, pcbddc->vec1_R, INSERT_VALUES, SCATTER_REVERSE));
    6262:     PetscCall(VecScatterEnd(pcbddc->R_to_B, inout_B, pcbddc->vec1_R, INSERT_VALUES, SCATTER_REVERSE));
    6263:     PetscCall(VecScatterBegin(pcbddc->R_to_D, inout_D, pcbddc->vec1_R, INSERT_VALUES, SCATTER_REVERSE));
    6264:     PetscCall(VecScatterEnd(pcbddc->R_to_D, inout_D, pcbddc->vec1_R, INSERT_VALUES, SCATTER_REVERSE));
    6265:     if (applytranspose && pcbddc->local_auxmat1) {
    6266:       PetscCall(MatMultTranspose(pcbddc->local_auxmat2, pcbddc->vec1_R, pcbddc->vec1_C));
    6267:       PetscCall(MatMultTransposeAdd(pcbddc->local_auxmat1, pcbddc->vec1_C, inout_B, inout_B));
    6268:       PetscCall(VecScatterBegin(pcbddc->R_to_B, inout_B, pcbddc->vec1_R, INSERT_VALUES, SCATTER_REVERSE));
    6269:       PetscCall(VecScatterEnd(pcbddc->R_to_B, inout_B, pcbddc->vec1_R, INSERT_VALUES, SCATTER_REVERSE));
    6270:     }
    6271:   }
    6272:   PetscCall(PetscLogEventBegin(PC_BDDC_Solves[pcbddc->current_level][1], pc, 0, 0, 0));
    6273:   if (!reuse_solver || pcbddc->switch_static) {
    6274:     if (applytranspose) {
    6275:       PetscCall(KSPSolveTranspose(pcbddc->ksp_R, pcbddc->vec1_R, pcbddc->vec1_R));
    6276:     } else {
    6277:       PetscCall(KSPSolve(pcbddc->ksp_R, pcbddc->vec1_R, pcbddc->vec1_R));
    6278:     }
    6279:     PetscCall(KSPCheckSolve(pcbddc->ksp_R, pc, pcbddc->vec1_R));
    6280:   } else {
    6281:     PCBDDCReuseSolvers reuse_solver = sub_schurs->reuse_solver;

    6283:     if (applytranspose) {
    6284:       PetscCall(MatFactorSolveSchurComplementTranspose(reuse_solver->F, reuse_solver->rhs_B, reuse_solver->sol_B));
    6285:     } else {
    6286:       PetscCall(MatFactorSolveSchurComplement(reuse_solver->F, reuse_solver->rhs_B, reuse_solver->sol_B));
    6287:     }
    6288:   }
    6289:   PetscCall(PetscLogEventEnd(PC_BDDC_Solves[pcbddc->current_level][1], pc, 0, 0, 0));
    6290:   PetscCall(VecSet(inout_B, 0.));
    6291:   if (!pcbddc->switch_static) {
    6292:     if (!reuse_solver) {
    6293:       PetscCall(VecScatterBegin(pcbddc->R_to_B, pcbddc->vec1_R, inout_B, INSERT_VALUES, SCATTER_FORWARD));
    6294:       PetscCall(VecScatterEnd(pcbddc->R_to_B, pcbddc->vec1_R, inout_B, INSERT_VALUES, SCATTER_FORWARD));
    6295:     } else {
    6296:       PCBDDCReuseSolvers reuse_solver = sub_schurs->reuse_solver;

    6298:       PetscCall(VecScatterBegin(reuse_solver->correction_scatter_B, reuse_solver->sol_B, inout_B, INSERT_VALUES, SCATTER_REVERSE));
    6299:       PetscCall(VecScatterEnd(reuse_solver->correction_scatter_B, reuse_solver->sol_B, inout_B, INSERT_VALUES, SCATTER_REVERSE));
    6300:     }
    6301:     if (!applytranspose && pcbddc->local_auxmat1) {
    6302:       PetscCall(MatMult(pcbddc->local_auxmat1, inout_B, pcbddc->vec1_C));
    6303:       PetscCall(MatMultAdd(pcbddc->local_auxmat2, pcbddc->vec1_C, inout_B, inout_B));
    6304:     }
    6305:   } else {
    6306:     PetscCall(VecScatterBegin(pcbddc->R_to_B, pcbddc->vec1_R, inout_B, INSERT_VALUES, SCATTER_FORWARD));
    6307:     PetscCall(VecScatterEnd(pcbddc->R_to_B, pcbddc->vec1_R, inout_B, INSERT_VALUES, SCATTER_FORWARD));
    6308:     PetscCall(VecScatterBegin(pcbddc->R_to_D, pcbddc->vec1_R, inout_D, INSERT_VALUES, SCATTER_FORWARD));
    6309:     PetscCall(VecScatterEnd(pcbddc->R_to_D, pcbddc->vec1_R, inout_D, INSERT_VALUES, SCATTER_FORWARD));
    6310:     if (!applytranspose && pcbddc->local_auxmat1) {
    6311:       PetscCall(MatMult(pcbddc->local_auxmat1, inout_B, pcbddc->vec1_C));
    6312:       PetscCall(MatMultAdd(pcbddc->local_auxmat2, pcbddc->vec1_C, pcbddc->vec1_R, pcbddc->vec1_R));
    6313:     }
    6314:     PetscCall(VecScatterBegin(pcbddc->R_to_B, pcbddc->vec1_R, inout_B, INSERT_VALUES, SCATTER_FORWARD));
    6315:     PetscCall(VecScatterEnd(pcbddc->R_to_B, pcbddc->vec1_R, inout_B, INSERT_VALUES, SCATTER_FORWARD));
    6316:     PetscCall(VecScatterBegin(pcbddc->R_to_D, pcbddc->vec1_R, inout_D, INSERT_VALUES, SCATTER_FORWARD));
    6317:     PetscCall(VecScatterEnd(pcbddc->R_to_D, pcbddc->vec1_R, inout_D, INSERT_VALUES, SCATTER_FORWARD));
    6318:   }
    6319:   PetscFunctionReturn(PETSC_SUCCESS);
    6320: }

    6322: /* parameter apply transpose determines if the interface preconditioner should be applied transposed or not */
    6323: PetscErrorCode PCBDDCApplyInterfacePreconditioner(PC pc, PetscBool applytranspose)
    6324: {
    6325:   PC_BDDC          *pcbddc = (PC_BDDC *)pc->data;
    6326:   PC_IS            *pcis   = (PC_IS *)pc->data;
    6327:   const PetscScalar zero   = 0.0;

    6329:   PetscFunctionBegin;
    6330:   /* Application of PSI^T or PHI^T (depending on applytranspose, see comment above) */
    6331:   if (!pcbddc->benign_apply_coarse_only) {
    6332:     if (applytranspose) {
    6333:       PetscCall(MatMultTranspose(pcbddc->coarse_phi_B, pcis->vec1_B, pcbddc->vec1_P));
    6334:       if (pcbddc->switch_static) PetscCall(MatMultTransposeAdd(pcbddc->coarse_phi_D, pcis->vec1_D, pcbddc->vec1_P, pcbddc->vec1_P));
    6335:     } else {
    6336:       PetscCall(MatMultTranspose(pcbddc->coarse_psi_B, pcis->vec1_B, pcbddc->vec1_P));
    6337:       if (pcbddc->switch_static) PetscCall(MatMultTransposeAdd(pcbddc->coarse_psi_D, pcis->vec1_D, pcbddc->vec1_P, pcbddc->vec1_P));
    6338:     }
    6339:   } else {
    6340:     PetscCall(VecSet(pcbddc->vec1_P, zero));
    6341:   }

    6343:   /* add p0 to the last value of vec1_P holding the coarse dof relative to p0 */
    6344:   if (pcbddc->benign_n) {
    6345:     PetscScalar *array;
    6346:     PetscInt     j;

    6348:     PetscCall(VecGetArray(pcbddc->vec1_P, &array));
    6349:     for (j = 0; j < pcbddc->benign_n; j++) array[pcbddc->local_primal_size - pcbddc->benign_n + j] += pcbddc->benign_p0[j];
    6350:     PetscCall(VecRestoreArray(pcbddc->vec1_P, &array));
    6351:   }

    6353:   /* start communications from local primal nodes to rhs of coarse solver */
    6354:   PetscCall(VecSet(pcbddc->coarse_vec, zero));
    6355:   PetscCall(PCBDDCScatterCoarseDataBegin(pc, ADD_VALUES, SCATTER_FORWARD));
    6356:   PetscCall(PCBDDCScatterCoarseDataEnd(pc, ADD_VALUES, SCATTER_FORWARD));

    6358:   /* Coarse solution -> rhs and sol updated inside PCBDDCScattarCoarseDataBegin/End */
    6359:   PetscCall(PetscLogEventBegin(PC_BDDC_Solves[pcbddc->current_level][2], pc, 0, 0, 0));
    6360:   if (pcbddc->coarse_ksp) {
    6361:     Mat          coarse_mat;
    6362:     Vec          rhs, sol;
    6363:     MatNullSpace nullsp;
    6364:     PetscBool    isbddc = PETSC_FALSE;

    6366:     if (pcbddc->benign_have_null) {
    6367:       PC coarse_pc;

    6369:       PetscCall(KSPGetPC(pcbddc->coarse_ksp, &coarse_pc));
    6370:       PetscCall(PetscObjectTypeCompare((PetscObject)coarse_pc, PCBDDC, &isbddc));
    6371:       /* we need to propagate to coarser levels the need for a possible benign correction */
    6372:       if (isbddc && pcbddc->benign_apply_coarse_only && !pcbddc->benign_skip_correction) {
    6373:         PC_BDDC *coarsepcbddc                  = (PC_BDDC *)coarse_pc->data;
    6374:         coarsepcbddc->benign_skip_correction   = PETSC_FALSE;
    6375:         coarsepcbddc->benign_apply_coarse_only = PETSC_TRUE;
    6376:       }
    6377:     }
    6378:     PetscCall(KSPGetRhs(pcbddc->coarse_ksp, &rhs));
    6379:     PetscCall(KSPGetSolution(pcbddc->coarse_ksp, &sol));
    6380:     PetscCall(KSPGetOperators(pcbddc->coarse_ksp, &coarse_mat, NULL));
    6381:     if (applytranspose) {
    6382:       PetscCheck(!pcbddc->benign_apply_coarse_only, PetscObjectComm((PetscObject)pcbddc->coarse_ksp), PETSC_ERR_SUP, "Not yet implemented");
    6383:       PetscCall(KSPSolveTranspose(pcbddc->coarse_ksp, rhs, sol));
    6384:       PetscCall(KSPCheckSolve(pcbddc->coarse_ksp, pc, sol));
    6385:       PetscCall(MatGetTransposeNullSpace(coarse_mat, &nullsp));
    6386:       if (nullsp) PetscCall(MatNullSpaceRemove(nullsp, sol));
    6387:     } else {
    6388:       PetscCall(MatGetNullSpace(coarse_mat, &nullsp));
    6389:       if (pcbddc->benign_apply_coarse_only && isbddc) { /* need just to apply the coarse preconditioner during presolve */
    6390:         PC coarse_pc;

    6392:         if (nullsp) PetscCall(MatNullSpaceRemove(nullsp, rhs));
    6393:         PetscCall(KSPGetPC(pcbddc->coarse_ksp, &coarse_pc));
    6394:         PetscCall(PCPreSolve(coarse_pc, pcbddc->coarse_ksp));
    6395:         PetscCall(PCBDDCBenignRemoveInterior(coarse_pc, rhs, sol));
    6396:         PetscCall(PCPostSolve(coarse_pc, pcbddc->coarse_ksp));
    6397:       } else {
    6398:         PetscCall(KSPSolve(pcbddc->coarse_ksp, rhs, sol));
    6399:         PetscCall(KSPCheckSolve(pcbddc->coarse_ksp, pc, sol));
    6400:         if (nullsp) PetscCall(MatNullSpaceRemove(nullsp, sol));
    6401:       }
    6402:     }
    6403:     /* we don't need the benign correction at coarser levels anymore */
    6404:     if (pcbddc->benign_have_null && isbddc) {
    6405:       PC       coarse_pc;
    6406:       PC_BDDC *coarsepcbddc;

    6408:       PetscCall(KSPGetPC(pcbddc->coarse_ksp, &coarse_pc));
    6409:       coarsepcbddc                           = (PC_BDDC *)coarse_pc->data;
    6410:       coarsepcbddc->benign_skip_correction   = PETSC_TRUE;
    6411:       coarsepcbddc->benign_apply_coarse_only = PETSC_FALSE;
    6412:     }
    6413:   }
    6414:   PetscCall(PetscLogEventEnd(PC_BDDC_Solves[pcbddc->current_level][2], pc, 0, 0, 0));

    6416:   /* Local solution on R nodes */
    6417:   if (!pcbddc->benign_apply_coarse_only) PetscCall(PCBDDCSolveSubstructureCorrection(pc, pcis->vec1_B, pcis->vec1_D, applytranspose));
    6418:   /* communications from coarse sol to local primal nodes */
    6419:   PetscCall(PCBDDCScatterCoarseDataBegin(pc, INSERT_VALUES, SCATTER_REVERSE));
    6420:   PetscCall(PCBDDCScatterCoarseDataEnd(pc, INSERT_VALUES, SCATTER_REVERSE));

    6422:   /* Sum contributions from the two levels */
    6423:   if (!pcbddc->benign_apply_coarse_only) {
    6424:     if (applytranspose) {
    6425:       PetscCall(MatMultAdd(pcbddc->coarse_psi_B, pcbddc->vec1_P, pcis->vec1_B, pcis->vec1_B));
    6426:       if (pcbddc->switch_static) PetscCall(MatMultAdd(pcbddc->coarse_psi_D, pcbddc->vec1_P, pcis->vec1_D, pcis->vec1_D));
    6427:     } else {
    6428:       PetscCall(MatMultAdd(pcbddc->coarse_phi_B, pcbddc->vec1_P, pcis->vec1_B, pcis->vec1_B));
    6429:       if (pcbddc->switch_static) PetscCall(MatMultAdd(pcbddc->coarse_phi_D, pcbddc->vec1_P, pcis->vec1_D, pcis->vec1_D));
    6430:     }
    6431:     /* store p0 */
    6432:     if (pcbddc->benign_n) {
    6433:       PetscScalar *array;
    6434:       PetscInt     j;

    6436:       PetscCall(VecGetArray(pcbddc->vec1_P, &array));
    6437:       for (j = 0; j < pcbddc->benign_n; j++) pcbddc->benign_p0[j] = array[pcbddc->local_primal_size - pcbddc->benign_n + j];
    6438:       PetscCall(VecRestoreArray(pcbddc->vec1_P, &array));
    6439:     }
    6440:   } else { /* expand the coarse solution */
    6441:     if (applytranspose) {
    6442:       PetscCall(MatMult(pcbddc->coarse_psi_B, pcbddc->vec1_P, pcis->vec1_B));
    6443:     } else {
    6444:       PetscCall(MatMult(pcbddc->coarse_phi_B, pcbddc->vec1_P, pcis->vec1_B));
    6445:     }
    6446:   }
    6447:   PetscFunctionReturn(PETSC_SUCCESS);
    6448: }

    6450: PetscErrorCode PCBDDCScatterCoarseDataBegin(PC pc, InsertMode imode, ScatterMode smode)
    6451: {
    6452:   PC_BDDC           *pcbddc = (PC_BDDC *)pc->data;
    6453:   Vec                from, to;
    6454:   const PetscScalar *array;

    6456:   PetscFunctionBegin;
    6457:   if (smode == SCATTER_REVERSE) { /* from global to local -> get data from coarse solution */
    6458:     from = pcbddc->coarse_vec;
    6459:     to   = pcbddc->vec1_P;
    6460:     if (pcbddc->coarse_ksp) { /* get array from coarse processes */
    6461:       Vec tvec;

    6463:       PetscCall(KSPGetRhs(pcbddc->coarse_ksp, &tvec));
    6464:       PetscCall(VecResetArray(tvec));
    6465:       PetscCall(KSPGetSolution(pcbddc->coarse_ksp, &tvec));
    6466:       PetscCall(VecGetArrayRead(tvec, &array));
    6467:       PetscCall(VecPlaceArray(from, array));
    6468:       PetscCall(VecRestoreArrayRead(tvec, &array));
    6469:     }
    6470:   } else { /* from local to global -> put data in coarse right-hand side */
    6471:     from = pcbddc->vec1_P;
    6472:     to   = pcbddc->coarse_vec;
    6473:   }
    6474:   PetscCall(VecScatterBegin(pcbddc->coarse_loc_to_glob, from, to, imode, smode));
    6475:   PetscFunctionReturn(PETSC_SUCCESS);
    6476: }

    6478: PetscErrorCode PCBDDCScatterCoarseDataEnd(PC pc, InsertMode imode, ScatterMode smode)
    6479: {
    6480:   PC_BDDC           *pcbddc = (PC_BDDC *)pc->data;
    6481:   Vec                from, to;
    6482:   const PetscScalar *array;

    6484:   PetscFunctionBegin;
    6485:   if (smode == SCATTER_REVERSE) { /* from global to local -> get data from coarse solution */
    6486:     from = pcbddc->coarse_vec;
    6487:     to   = pcbddc->vec1_P;
    6488:   } else { /* from local to global -> put data in coarse right-hand side */
    6489:     from = pcbddc->vec1_P;
    6490:     to   = pcbddc->coarse_vec;
    6491:   }
    6492:   PetscCall(VecScatterEnd(pcbddc->coarse_loc_to_glob, from, to, imode, smode));
    6493:   if (smode == SCATTER_FORWARD) {
    6494:     if (pcbddc->coarse_ksp) { /* get array from coarse processes */
    6495:       Vec tvec;

    6497:       PetscCall(KSPGetRhs(pcbddc->coarse_ksp, &tvec));
    6498:       PetscCall(VecGetArrayRead(to, &array));
    6499:       PetscCall(VecPlaceArray(tvec, array));
    6500:       PetscCall(VecRestoreArrayRead(to, &array));
    6501:     }
    6502:   } else {
    6503:     if (pcbddc->coarse_ksp) { /* restore array of pcbddc->coarse_vec */
    6504:       PetscCall(VecResetArray(from));
    6505:     }
    6506:   }
    6507:   PetscFunctionReturn(PETSC_SUCCESS);
    6508: }

    6510: PetscErrorCode PCBDDCConstraintsSetUp(PC pc)
    6511: {
    6512:   PC_IS   *pcis   = (PC_IS *)pc->data;
    6513:   PC_BDDC *pcbddc = (PC_BDDC *)pc->data;
    6514:   Mat_IS  *matis  = (Mat_IS *)pc->pmat->data;
    6515:   /* one and zero */
    6516:   PetscScalar one = 1.0, zero = 0.0;
    6517:   /* space to store constraints and their local indices */
    6518:   PetscScalar *constraints_data;
    6519:   PetscInt    *constraints_idxs, *constraints_idxs_B;
    6520:   PetscInt    *constraints_idxs_ptr, *constraints_data_ptr;
    6521:   PetscInt    *constraints_n;
    6522:   /* iterators */
    6523:   PetscInt i, j, k, total_counts, total_counts_cc, cum;
    6524:   /* BLAS integers */
    6525:   PetscBLASInt lwork, lierr;
    6526:   PetscBLASInt Blas_N, Blas_M, Blas_K, Blas_one = 1;
    6527:   PetscBLASInt Blas_LDA, Blas_LDB, Blas_LDC;
    6528:   /* reuse */
    6529:   PetscInt  olocal_primal_size, olocal_primal_size_cc;
    6530:   PetscInt *olocal_primal_ref_node, *olocal_primal_ref_mult;
    6531:   /* change of basis */
    6532:   PetscBool qr_needed;
    6533:   PetscBT   change_basis, qr_needed_idx;
    6534:   /* auxiliary stuff */
    6535:   PetscInt *nnz, *is_indices;
    6536:   PetscInt  ncc;
    6537:   /* some quantities */
    6538:   PetscInt  n_vertices, total_primal_vertices, valid_constraints;
    6539:   PetscInt  size_of_constraint, max_size_of_constraint = 0, max_constraints, temp_constraints;
    6540:   PetscReal tol; /* tolerance for retaining eigenmodes */

    6542:   PetscFunctionBegin;
    6543:   tol = PetscSqrtReal(PETSC_SMALL);
    6544:   /* Destroy Mat objects computed previously */
    6545:   PetscCall(MatDestroy(&pcbddc->ChangeOfBasisMatrix));
    6546:   PetscCall(MatDestroy(&pcbddc->ConstraintMatrix));
    6547:   PetscCall(MatDestroy(&pcbddc->switch_static_change));
    6548:   /* save info on constraints from previous setup (if any) */
    6549:   olocal_primal_size    = pcbddc->local_primal_size;
    6550:   olocal_primal_size_cc = pcbddc->local_primal_size_cc;
    6551:   PetscCall(PetscMalloc2(olocal_primal_size_cc, &olocal_primal_ref_node, olocal_primal_size_cc, &olocal_primal_ref_mult));
    6552:   PetscCall(PetscArraycpy(olocal_primal_ref_node, pcbddc->local_primal_ref_node, olocal_primal_size_cc));
    6553:   PetscCall(PetscArraycpy(olocal_primal_ref_mult, pcbddc->local_primal_ref_mult, olocal_primal_size_cc));
    6554:   PetscCall(PetscFree2(pcbddc->local_primal_ref_node, pcbddc->local_primal_ref_mult));
    6555:   PetscCall(PetscFree(pcbddc->primal_indices_local_idxs));

    6557:   if (!pcbddc->adaptive_selection) {
    6558:     IS           ISForVertices, *ISForFaces, *ISForEdges;
    6559:     MatNullSpace nearnullsp;
    6560:     const Vec   *nearnullvecs;
    6561:     Vec         *localnearnullsp;
    6562:     PetscScalar *array;
    6563:     PetscInt     n_ISForFaces, n_ISForEdges, nnsp_size, o_nf, o_ne;
    6564:     PetscBool    nnsp_has_cnst;
    6565:     /* LAPACK working arrays for SVD or POD */
    6566:     PetscBool    skip_lapack, boolforchange;
    6567:     PetscScalar *work;
    6568:     PetscReal   *singular_vals;
    6569: #if defined(PETSC_USE_COMPLEX)
    6570:     PetscReal *rwork;
    6571: #endif
    6572:     PetscScalar *temp_basis = NULL, *correlation_mat = NULL;
    6573:     PetscBLASInt dummy_int    = 1;
    6574:     PetscScalar  dummy_scalar = 1.;
    6575:     PetscBool    use_pod      = PetscDefined(MISSING_LAPACK_GESVD) || PetscDefined(HAVE_MKL_LIBS) ? PETSC_TRUE : PETSC_FALSE; /* MKL SVD with same input gives different results on different processes! */

    6577:     /* Get index sets for faces, edges and vertices from graph */
    6578:     PetscCall(PCBDDCGraphGetCandidatesIS(pcbddc->mat_graph, &n_ISForFaces, &ISForFaces, &n_ISForEdges, &ISForEdges, &ISForVertices));
    6579:     o_nf       = n_ISForFaces;
    6580:     o_ne       = n_ISForEdges;
    6581:     n_vertices = 0;
    6582:     if (ISForVertices) PetscCall(ISGetSize(ISForVertices, &n_vertices));
    6583:     /* print some info */
    6584:     if (pcbddc->dbg_flag && (!pcbddc->sub_schurs || pcbddc->sub_schurs_rebuild)) {
    6585:       if (!pcbddc->dbg_viewer) pcbddc->dbg_viewer = PETSC_VIEWER_STDOUT_(PetscObjectComm((PetscObject)pc));
    6586:       PetscCall(PCBDDCGraphASCIIView(pcbddc->mat_graph, pcbddc->dbg_flag, pcbddc->dbg_viewer));
    6587:       PetscCall(PetscViewerASCIIPushSynchronized(pcbddc->dbg_viewer));
    6588:       PetscCall(PetscViewerASCIISynchronizedPrintf(pcbddc->dbg_viewer, "--------------------------------------------------------------\n"));
    6589:       PetscCall(PetscViewerASCIISynchronizedPrintf(pcbddc->dbg_viewer, "Subdomain %04d got %02" PetscInt_FMT " local candidate vertices (%d)\n", PetscGlobalRank, n_vertices, pcbddc->use_vertices));
    6590:       PetscCall(PetscViewerASCIISynchronizedPrintf(pcbddc->dbg_viewer, "Subdomain %04d got %02" PetscInt_FMT " local candidate edges    (%d)\n", PetscGlobalRank, n_ISForEdges, pcbddc->use_edges));
    6591:       PetscCall(PetscViewerASCIISynchronizedPrintf(pcbddc->dbg_viewer, "Subdomain %04d got %02" PetscInt_FMT " local candidate faces    (%d)\n", PetscGlobalRank, n_ISForFaces, pcbddc->use_faces));
    6592:       PetscCall(PetscViewerFlush(pcbddc->dbg_viewer));
    6593:       PetscCall(PetscViewerASCIIPopSynchronized(pcbddc->dbg_viewer));
    6594:     }

    6596:     if (!pcbddc->use_vertices) n_vertices = 0;
    6597:     if (!pcbddc->use_edges) n_ISForEdges = 0;
    6598:     if (!pcbddc->use_faces) n_ISForFaces = 0;

    6600:     /* check if near null space is attached to global mat */
    6601:     if (pcbddc->use_nnsp) PetscCall(MatGetNearNullSpace(pc->pmat, &nearnullsp));
    6602:     else nearnullsp = NULL;

    6604:     if (nearnullsp) {
    6605:       PetscCall(MatNullSpaceGetVecs(nearnullsp, &nnsp_has_cnst, &nnsp_size, &nearnullvecs));
    6606:       /* remove any stored info */
    6607:       PetscCall(MatNullSpaceDestroy(&pcbddc->onearnullspace));
    6608:       PetscCall(PetscFree(pcbddc->onearnullvecs_state));
    6609:       /* store information for BDDC solver reuse */
    6610:       PetscCall(PetscObjectReference((PetscObject)nearnullsp));
    6611:       pcbddc->onearnullspace = nearnullsp;
    6612:       PetscCall(PetscMalloc1(nnsp_size, &pcbddc->onearnullvecs_state));
    6613:       for (i = 0; i < nnsp_size; i++) PetscCall(PetscObjectStateGet((PetscObject)nearnullvecs[i], &pcbddc->onearnullvecs_state[i]));
    6614:     } else { /* if near null space is not provided BDDC uses constants by default */
    6615:       nnsp_size     = 0;
    6616:       nnsp_has_cnst = PETSC_TRUE;
    6617:     }
    6618:     /* get max number of constraints on a single cc */
    6619:     max_constraints = nnsp_size;
    6620:     if (nnsp_has_cnst) max_constraints++;

    6622:     /*
    6623:          Evaluate maximum storage size needed by the procedure
    6624:          - Indices for connected component i stored at "constraints_idxs + constraints_idxs_ptr[i]"
    6625:          - Values for constraints on connected component i stored at "constraints_data + constraints_data_ptr[i]"
    6626:          There can be multiple constraints per connected component
    6627:                                                                                                                                                            */
    6628:     ncc = n_vertices + n_ISForFaces + n_ISForEdges;
    6629:     PetscCall(PetscMalloc3(ncc + 1, &constraints_idxs_ptr, ncc + 1, &constraints_data_ptr, ncc, &constraints_n));

    6631:     total_counts = n_ISForFaces + n_ISForEdges;
    6632:     total_counts *= max_constraints;
    6633:     total_counts += n_vertices;
    6634:     PetscCall(PetscBTCreate(total_counts, &change_basis));

    6636:     total_counts           = 0;
    6637:     max_size_of_constraint = 0;
    6638:     for (i = 0; i < n_ISForEdges + n_ISForFaces; i++) {
    6639:       IS used_is;
    6640:       if (i < n_ISForEdges) {
    6641:         used_is = ISForEdges[i];
    6642:       } else {
    6643:         used_is = ISForFaces[i - n_ISForEdges];
    6644:       }
    6645:       PetscCall(ISGetSize(used_is, &j));
    6646:       total_counts += j;
    6647:       max_size_of_constraint = PetscMax(j, max_size_of_constraint);
    6648:     }
    6649:     PetscCall(PetscMalloc3(total_counts * max_constraints + n_vertices, &constraints_data, total_counts + n_vertices, &constraints_idxs, total_counts + n_vertices, &constraints_idxs_B));

    6651:     /* get local part of global near null space vectors */
    6652:     PetscCall(PetscMalloc1(nnsp_size, &localnearnullsp));
    6653:     for (k = 0; k < nnsp_size; k++) {
    6654:       PetscCall(VecDuplicate(pcis->vec1_N, &localnearnullsp[k]));
    6655:       PetscCall(VecScatterBegin(matis->rctx, nearnullvecs[k], localnearnullsp[k], INSERT_VALUES, SCATTER_FORWARD));
    6656:       PetscCall(VecScatterEnd(matis->rctx, nearnullvecs[k], localnearnullsp[k], INSERT_VALUES, SCATTER_FORWARD));
    6657:     }

    6659:     /* whether or not to skip lapack calls */
    6660:     skip_lapack = PETSC_TRUE;
    6661:     if (n_ISForFaces + n_ISForEdges && max_constraints > 1 && !pcbddc->use_nnsp_true) skip_lapack = PETSC_FALSE;

    6663:     /* First we issue queries to allocate optimal workspace for LAPACKgesvd (or LAPACKsyev if SVD is missing) */
    6664:     if (!skip_lapack) {
    6665:       PetscScalar temp_work;

    6667:       if (use_pod) {
    6668:         /* Proper Orthogonal Decomposition (POD) using the snapshot method */
    6669:         PetscCall(PetscMalloc1(max_constraints * max_constraints, &correlation_mat));
    6670:         PetscCall(PetscMalloc1(max_constraints, &singular_vals));
    6671:         PetscCall(PetscMalloc1(max_size_of_constraint * max_constraints, &temp_basis));
    6672: #if defined(PETSC_USE_COMPLEX)
    6673:         PetscCall(PetscMalloc1(3 * max_constraints, &rwork));
    6674: #endif
    6675:         /* now we evaluate the optimal workspace using query with lwork=-1 */
    6676:         PetscCall(PetscBLASIntCast(max_constraints, &Blas_N));
    6677:         PetscCall(PetscBLASIntCast(max_constraints, &Blas_LDA));
    6678:         lwork = -1;
    6679:         PetscCall(PetscFPTrapPush(PETSC_FP_TRAP_OFF));
    6680: #if !defined(PETSC_USE_COMPLEX)
    6681:         PetscCallBLAS("LAPACKsyev", LAPACKsyev_("V", "U", &Blas_N, correlation_mat, &Blas_LDA, singular_vals, &temp_work, &lwork, &lierr));
    6682: #else
    6683:         PetscCallBLAS("LAPACKsyev", LAPACKsyev_("V", "U", &Blas_N, correlation_mat, &Blas_LDA, singular_vals, &temp_work, &lwork, rwork, &lierr));
    6684: #endif
    6685:         PetscCall(PetscFPTrapPop());
    6686:         PetscCheck(!lierr, PETSC_COMM_SELF, PETSC_ERR_LIB, "Error in query to SYEV Lapack routine %" PetscBLASInt_FMT, lierr);
    6687:       } else {
    6688: #if !defined(PETSC_MISSING_LAPACK_GESVD)
    6689:         /* SVD */
    6690:         PetscInt max_n, min_n;
    6691:         max_n = max_size_of_constraint;
    6692:         min_n = max_constraints;
    6693:         if (max_size_of_constraint < max_constraints) {
    6694:           min_n = max_size_of_constraint;
    6695:           max_n = max_constraints;
    6696:         }
    6697:         PetscCall(PetscMalloc1(min_n, &singular_vals));
    6698:   #if defined(PETSC_USE_COMPLEX)
    6699:         PetscCall(PetscMalloc1(5 * min_n, &rwork));
    6700:   #endif
    6701:         /* now we evaluate the optimal workspace using query with lwork=-1 */
    6702:         lwork = -1;
    6703:         PetscCall(PetscBLASIntCast(max_n, &Blas_M));
    6704:         PetscCall(PetscBLASIntCast(min_n, &Blas_N));
    6705:         PetscCall(PetscBLASIntCast(max_n, &Blas_LDA));
    6706:         PetscCall(PetscFPTrapPush(PETSC_FP_TRAP_OFF));
    6707:   #if !defined(PETSC_USE_COMPLEX)
    6708:         PetscCallBLAS("LAPACKgesvd", LAPACKgesvd_("O", "N", &Blas_M, &Blas_N, &constraints_data[0], &Blas_LDA, singular_vals, &dummy_scalar, &dummy_int, &dummy_scalar, &dummy_int, &temp_work, &lwork, &lierr));
    6709:   #else
    6710:         PetscCallBLAS("LAPACKgesvd", LAPACKgesvd_("O", "N", &Blas_M, &Blas_N, &constraints_data[0], &Blas_LDA, singular_vals, &dummy_scalar, &dummy_int, &dummy_scalar, &dummy_int, &temp_work, &lwork, rwork, &lierr));
    6711:   #endif
    6712:         PetscCall(PetscFPTrapPop());
    6713:         PetscCheck(!lierr, PETSC_COMM_SELF, PETSC_ERR_LIB, "Error in query to GESVD Lapack routine %" PetscBLASInt_FMT, lierr);
    6714: #else
    6715:         SETERRQ(PETSC_COMM_SELF, PETSC_ERR_LIB, "This should not happen");
    6716: #endif /* on missing GESVD */
    6717:       }
    6718:       /* Allocate optimal workspace */
    6719:       PetscCall(PetscBLASIntCast((PetscInt)PetscRealPart(temp_work), &lwork));
    6720:       PetscCall(PetscMalloc1(lwork, &work));
    6721:     }
    6722:     /* Now we can loop on constraining sets */
    6723:     total_counts            = 0;
    6724:     constraints_idxs_ptr[0] = 0;
    6725:     constraints_data_ptr[0] = 0;
    6726:     /* vertices */
    6727:     if (n_vertices) {
    6728:       PetscCall(ISGetIndices(ISForVertices, (const PetscInt **)&is_indices));
    6729:       PetscCall(PetscArraycpy(constraints_idxs, is_indices, n_vertices));
    6730:       for (i = 0; i < n_vertices; i++) {
    6731:         constraints_n[total_counts]            = 1;
    6732:         constraints_data[total_counts]         = 1.0;
    6733:         constraints_idxs_ptr[total_counts + 1] = constraints_idxs_ptr[total_counts] + 1;
    6734:         constraints_data_ptr[total_counts + 1] = constraints_data_ptr[total_counts] + 1;
    6735:         total_counts++;
    6736:       }
    6737:       PetscCall(ISRestoreIndices(ISForVertices, (const PetscInt **)&is_indices));
    6738:     }

    6740:     /* edges and faces */
    6741:     total_counts_cc = total_counts;
    6742:     for (ncc = 0; ncc < n_ISForEdges + n_ISForFaces; ncc++) {
    6743:       IS        used_is;
    6744:       PetscBool idxs_copied = PETSC_FALSE;

    6746:       if (ncc < n_ISForEdges) {
    6747:         used_is       = ISForEdges[ncc];
    6748:         boolforchange = pcbddc->use_change_of_basis; /* change or not the basis on the edge */
    6749:       } else {
    6750:         used_is       = ISForFaces[ncc - n_ISForEdges];
    6751:         boolforchange = (PetscBool)(pcbddc->use_change_of_basis && pcbddc->use_change_on_faces); /* change or not the basis on the face */
    6752:       }
    6753:       temp_constraints = 0; /* zero the number of constraints I have on this conn comp */

    6755:       PetscCall(ISGetSize(used_is, &size_of_constraint));
    6756:       if (!size_of_constraint) continue;
    6757:       PetscCall(ISGetIndices(used_is, (const PetscInt **)&is_indices));
    6758:       if (nnsp_has_cnst) {
    6759:         PetscScalar quad_value;

    6761:         PetscCall(PetscArraycpy(constraints_idxs + constraints_idxs_ptr[total_counts_cc], is_indices, size_of_constraint));
    6762:         idxs_copied = PETSC_TRUE;

    6764:         if (!pcbddc->use_nnsp_true) {
    6765:           quad_value = (PetscScalar)(1.0 / PetscSqrtReal((PetscReal)size_of_constraint));
    6766:         } else {
    6767:           quad_value = 1.0;
    6768:         }
    6769:         for (j = 0; j < size_of_constraint; j++) constraints_data[constraints_data_ptr[total_counts_cc] + j] = quad_value;
    6770:         temp_constraints++;
    6771:         total_counts++;
    6772:       }
    6773:       for (k = 0; k < nnsp_size; k++) {
    6774:         PetscReal    real_value;
    6775:         PetscScalar *ptr_to_data;

    6777:         PetscCall(VecGetArrayRead(localnearnullsp[k], (const PetscScalar **)&array));
    6778:         ptr_to_data = &constraints_data[constraints_data_ptr[total_counts_cc] + temp_constraints * size_of_constraint];
    6779:         for (j = 0; j < size_of_constraint; j++) ptr_to_data[j] = array[is_indices[j]];
    6780:         PetscCall(VecRestoreArrayRead(localnearnullsp[k], (const PetscScalar **)&array));
    6781:         /* check if array is null on the connected component */
    6782:         PetscCall(PetscBLASIntCast(size_of_constraint, &Blas_N));
    6783:         PetscCallBLAS("BLASasum", real_value = BLASasum_(&Blas_N, ptr_to_data, &Blas_one));
    6784:         if (real_value > tol * size_of_constraint) { /* keep indices and values */
    6785:           temp_constraints++;
    6786:           total_counts++;
    6787:           if (!idxs_copied) {
    6788:             PetscCall(PetscArraycpy(constraints_idxs + constraints_idxs_ptr[total_counts_cc], is_indices, size_of_constraint));
    6789:             idxs_copied = PETSC_TRUE;
    6790:           }
    6791:         }
    6792:       }
    6793:       PetscCall(ISRestoreIndices(used_is, (const PetscInt **)&is_indices));
    6794:       valid_constraints = temp_constraints;
    6795:       if (!pcbddc->use_nnsp_true && temp_constraints) {
    6796:         if (temp_constraints == 1) { /* just normalize the constraint */
    6797:           PetscScalar norm, *ptr_to_data;

    6799:           ptr_to_data = &constraints_data[constraints_data_ptr[total_counts_cc]];
    6800:           PetscCall(PetscBLASIntCast(size_of_constraint, &Blas_N));
    6801:           PetscCallBLAS("BLASdot", norm = BLASdot_(&Blas_N, ptr_to_data, &Blas_one, ptr_to_data, &Blas_one));
    6802:           norm = 1.0 / PetscSqrtReal(PetscRealPart(norm));
    6803:           PetscCallBLAS("BLASscal", BLASscal_(&Blas_N, &norm, ptr_to_data, &Blas_one));
    6804:         } else { /* perform SVD */
    6805:           PetscScalar *ptr_to_data = &constraints_data[constraints_data_ptr[total_counts_cc]];

    6807:           if (use_pod) {
    6808:             /* SVD: Y = U*S*V^H                -> U (eigenvectors of Y*Y^H) = Y*V*(S)^\dag
    6809:                POD: Y^H*Y = V*D*V^H, D = S^H*S -> U = Y*V*D^(-1/2)
    6810:                -> When PETSC_USE_COMPLEX and PETSC_MISSING_LAPACK_GESVD are defined
    6811:                   the constraints basis will differ (by a complex factor with absolute value equal to 1)
    6812:                   from that computed using LAPACKgesvd
    6813:                -> This is due to a different computation of eigenvectors in LAPACKheev
    6814:                -> The quality of the POD-computed basis will be the same */
    6815:             PetscCall(PetscArrayzero(correlation_mat, temp_constraints * temp_constraints));
    6816:             /* Store upper triangular part of correlation matrix */
    6817:             PetscCall(PetscBLASIntCast(size_of_constraint, &Blas_N));
    6818:             PetscCall(PetscFPTrapPush(PETSC_FP_TRAP_OFF));
    6819:             for (j = 0; j < temp_constraints; j++) {
    6820:               for (k = 0; k < j + 1; k++) PetscCallBLAS("BLASdot", correlation_mat[j * temp_constraints + k] = BLASdot_(&Blas_N, ptr_to_data + k * size_of_constraint, &Blas_one, ptr_to_data + j * size_of_constraint, &Blas_one));
    6821:             }
    6822:             /* compute eigenvalues and eigenvectors of correlation matrix */
    6823:             PetscCall(PetscBLASIntCast(temp_constraints, &Blas_N));
    6824:             PetscCall(PetscBLASIntCast(temp_constraints, &Blas_LDA));
    6825: #if !defined(PETSC_USE_COMPLEX)
    6826:             PetscCallBLAS("LAPACKsyev", LAPACKsyev_("V", "U", &Blas_N, correlation_mat, &Blas_LDA, singular_vals, work, &lwork, &lierr));
    6827: #else
    6828:             PetscCallBLAS("LAPACKsyev", LAPACKsyev_("V", "U", &Blas_N, correlation_mat, &Blas_LDA, singular_vals, work, &lwork, rwork, &lierr));
    6829: #endif
    6830:             PetscCall(PetscFPTrapPop());
    6831:             PetscCheck(!lierr, PETSC_COMM_SELF, PETSC_ERR_LIB, "Error in SYEV Lapack routine %" PetscBLASInt_FMT, lierr);
    6832:             /* retain eigenvalues greater than tol: note that LAPACKsyev gives eigs in ascending order */
    6833:             j = 0;
    6834:             while (j < temp_constraints && singular_vals[j] / singular_vals[temp_constraints - 1] < tol) j++;
    6835:             total_counts      = total_counts - j;
    6836:             valid_constraints = temp_constraints - j;
    6837:             /* scale and copy POD basis into used quadrature memory */
    6838:             PetscCall(PetscBLASIntCast(size_of_constraint, &Blas_M));
    6839:             PetscCall(PetscBLASIntCast(temp_constraints, &Blas_N));
    6840:             PetscCall(PetscBLASIntCast(temp_constraints, &Blas_K));
    6841:             PetscCall(PetscBLASIntCast(size_of_constraint, &Blas_LDA));
    6842:             PetscCall(PetscBLASIntCast(temp_constraints, &Blas_LDB));
    6843:             PetscCall(PetscBLASIntCast(size_of_constraint, &Blas_LDC));
    6844:             if (j < temp_constraints) {
    6845:               PetscInt ii;
    6846:               for (k = j; k < temp_constraints; k++) singular_vals[k] = 1.0 / PetscSqrtReal(singular_vals[k]);
    6847:               PetscCall(PetscFPTrapPush(PETSC_FP_TRAP_OFF));
    6848:               PetscCallBLAS("BLASgemm", BLASgemm_("N", "N", &Blas_M, &Blas_N, &Blas_K, &one, ptr_to_data, &Blas_LDA, correlation_mat, &Blas_LDB, &zero, temp_basis, &Blas_LDC));
    6849:               PetscCall(PetscFPTrapPop());
    6850:               for (k = 0; k < temp_constraints - j; k++) {
    6851:                 for (ii = 0; ii < size_of_constraint; ii++) ptr_to_data[k * size_of_constraint + ii] = singular_vals[temp_constraints - 1 - k] * temp_basis[(temp_constraints - 1 - k) * size_of_constraint + ii];
    6852:               }
    6853:             }
    6854:           } else {
    6855: #if !defined(PETSC_MISSING_LAPACK_GESVD)
    6856:             PetscCall(PetscBLASIntCast(size_of_constraint, &Blas_M));
    6857:             PetscCall(PetscBLASIntCast(temp_constraints, &Blas_N));
    6858:             PetscCall(PetscBLASIntCast(size_of_constraint, &Blas_LDA));
    6859:             PetscCall(PetscFPTrapPush(PETSC_FP_TRAP_OFF));
    6860:   #if !defined(PETSC_USE_COMPLEX)
    6861:             PetscCallBLAS("LAPACKgesvd", LAPACKgesvd_("O", "N", &Blas_M, &Blas_N, ptr_to_data, &Blas_LDA, singular_vals, &dummy_scalar, &dummy_int, &dummy_scalar, &dummy_int, work, &lwork, &lierr));
    6862:   #else
    6863:             PetscCallBLAS("LAPACKgesvd", LAPACKgesvd_("O", "N", &Blas_M, &Blas_N, ptr_to_data, &Blas_LDA, singular_vals, &dummy_scalar, &dummy_int, &dummy_scalar, &dummy_int, work, &lwork, rwork, &lierr));
    6864:   #endif
    6865:             PetscCheck(!lierr, PETSC_COMM_SELF, PETSC_ERR_LIB, "Error in GESVD Lapack routine %" PetscBLASInt_FMT, lierr);
    6866:             PetscCall(PetscFPTrapPop());
    6867:             /* retain eigenvalues greater than tol: note that LAPACKgesvd gives eigs in descending order */
    6868:             k = temp_constraints;
    6869:             if (k > size_of_constraint) k = size_of_constraint;
    6870:             j = 0;
    6871:             while (j < k && singular_vals[k - j - 1] / singular_vals[0] < tol) j++;
    6872:             valid_constraints = k - j;
    6873:             total_counts      = total_counts - temp_constraints + valid_constraints;
    6874: #else
    6875:             SETERRQ(PETSC_COMM_SELF, PETSC_ERR_LIB, "This should not happen");
    6876: #endif /* on missing GESVD */
    6877:           }
    6878:         }
    6879:       }
    6880:       /* update pointers information */
    6881:       if (valid_constraints) {
    6882:         constraints_n[total_counts_cc]            = valid_constraints;
    6883:         constraints_idxs_ptr[total_counts_cc + 1] = constraints_idxs_ptr[total_counts_cc] + size_of_constraint;
    6884:         constraints_data_ptr[total_counts_cc + 1] = constraints_data_ptr[total_counts_cc] + size_of_constraint * valid_constraints;
    6885:         /* set change_of_basis flag */
    6886:         if (boolforchange) PetscCall(PetscBTSet(change_basis, total_counts_cc));
    6887:         total_counts_cc++;
    6888:       }
    6889:     }
    6890:     /* free workspace */
    6891:     if (!skip_lapack) {
    6892:       PetscCall(PetscFree(work));
    6893: #if defined(PETSC_USE_COMPLEX)
    6894:       PetscCall(PetscFree(rwork));
    6895: #endif
    6896:       PetscCall(PetscFree(singular_vals));
    6897:       PetscCall(PetscFree(correlation_mat));
    6898:       PetscCall(PetscFree(temp_basis));
    6899:     }
    6900:     for (k = 0; k < nnsp_size; k++) PetscCall(VecDestroy(&localnearnullsp[k]));
    6901:     PetscCall(PetscFree(localnearnullsp));
    6902:     /* free index sets of faces, edges and vertices */
    6903:     PetscCall(PCBDDCGraphRestoreCandidatesIS(pcbddc->mat_graph, &o_nf, &ISForFaces, &o_ne, &ISForEdges, &ISForVertices));
    6904:   } else {
    6905:     PCBDDCSubSchurs sub_schurs = pcbddc->sub_schurs;

    6907:     total_counts = 0;
    6908:     n_vertices   = 0;
    6909:     if (sub_schurs->is_vertices && pcbddc->use_vertices) PetscCall(ISGetLocalSize(sub_schurs->is_vertices, &n_vertices));
    6910:     max_constraints = 0;
    6911:     total_counts_cc = 0;
    6912:     for (i = 0; i < sub_schurs->n_subs + n_vertices; i++) {
    6913:       total_counts += pcbddc->adaptive_constraints_n[i];
    6914:       if (pcbddc->adaptive_constraints_n[i]) total_counts_cc++;
    6915:       max_constraints = PetscMax(max_constraints, pcbddc->adaptive_constraints_n[i]);
    6916:     }
    6917:     constraints_idxs_ptr = pcbddc->adaptive_constraints_idxs_ptr;
    6918:     constraints_data_ptr = pcbddc->adaptive_constraints_data_ptr;
    6919:     constraints_idxs     = pcbddc->adaptive_constraints_idxs;
    6920:     constraints_data     = pcbddc->adaptive_constraints_data;
    6921:     /* constraints_n differs from pcbddc->adaptive_constraints_n */
    6922:     PetscCall(PetscMalloc1(total_counts_cc, &constraints_n));
    6923:     total_counts_cc = 0;
    6924:     for (i = 0; i < sub_schurs->n_subs + n_vertices; i++) {
    6925:       if (pcbddc->adaptive_constraints_n[i]) constraints_n[total_counts_cc++] = pcbddc->adaptive_constraints_n[i];
    6926:     }

    6928:     max_size_of_constraint = 0;
    6929:     for (i = 0; i < total_counts_cc; i++) max_size_of_constraint = PetscMax(max_size_of_constraint, constraints_idxs_ptr[i + 1] - constraints_idxs_ptr[i]);
    6930:     PetscCall(PetscMalloc1(constraints_idxs_ptr[total_counts_cc], &constraints_idxs_B));
    6931:     /* Change of basis */
    6932:     PetscCall(PetscBTCreate(total_counts_cc, &change_basis));
    6933:     if (pcbddc->use_change_of_basis) {
    6934:       for (i = 0; i < sub_schurs->n_subs; i++) {
    6935:         if (PetscBTLookup(sub_schurs->is_edge, i) || pcbddc->use_change_on_faces) PetscCall(PetscBTSet(change_basis, i + n_vertices));
    6936:       }
    6937:     }
    6938:   }
    6939:   pcbddc->local_primal_size = total_counts;
    6940:   PetscCall(PetscMalloc1(pcbddc->local_primal_size + pcbddc->benign_n, &pcbddc->primal_indices_local_idxs));

    6942:   /* map constraints_idxs in boundary numbering */
    6943:   if (pcbddc->use_change_of_basis) {
    6944:     PetscCall(ISGlobalToLocalMappingApply(pcis->BtoNmap, IS_GTOLM_DROP, constraints_idxs_ptr[total_counts_cc], constraints_idxs, &i, constraints_idxs_B));
    6945:     PetscCheck(i == constraints_idxs_ptr[total_counts_cc], PETSC_COMM_SELF, PETSC_ERR_PLIB, "Error in boundary numbering for constraints indices %" PetscInt_FMT " != %" PetscInt_FMT, constraints_idxs_ptr[total_counts_cc], i);
    6946:   }

    6948:   /* Create constraint matrix */
    6949:   PetscCall(MatCreate(PETSC_COMM_SELF, &pcbddc->ConstraintMatrix));
    6950:   PetscCall(MatSetType(pcbddc->ConstraintMatrix, MATAIJ));
    6951:   PetscCall(MatSetSizes(pcbddc->ConstraintMatrix, pcbddc->local_primal_size, pcis->n, pcbddc->local_primal_size, pcis->n));

    6953:   /* find primal_dofs: subdomain corners plus dofs selected as primal after change of basis */
    6954:   /* determine if a QR strategy is needed for change of basis */
    6955:   qr_needed = pcbddc->use_qr_single;
    6956:   PetscCall(PetscBTCreate(total_counts_cc, &qr_needed_idx));
    6957:   total_primal_vertices        = 0;
    6958:   pcbddc->local_primal_size_cc = 0;
    6959:   for (i = 0; i < total_counts_cc; i++) {
    6960:     size_of_constraint = constraints_idxs_ptr[i + 1] - constraints_idxs_ptr[i];
    6961:     if (size_of_constraint == 1 && pcbddc->mat_graph->custom_minimal_size) {
    6962:       pcbddc->primal_indices_local_idxs[total_primal_vertices++] = constraints_idxs[constraints_idxs_ptr[i]];
    6963:       pcbddc->local_primal_size_cc += 1;
    6964:     } else if (PetscBTLookup(change_basis, i)) {
    6965:       for (k = 0; k < constraints_n[i]; k++) pcbddc->primal_indices_local_idxs[total_primal_vertices++] = constraints_idxs[constraints_idxs_ptr[i] + k];
    6966:       pcbddc->local_primal_size_cc += constraints_n[i];
    6967:       if (constraints_n[i] > 1 || pcbddc->use_qr_single) {
    6968:         PetscCall(PetscBTSet(qr_needed_idx, i));
    6969:         qr_needed = PETSC_TRUE;
    6970:       }
    6971:     } else {
    6972:       pcbddc->local_primal_size_cc += 1;
    6973:     }
    6974:   }
    6975:   /* note that the local variable n_vertices used below stores the number of pointwise constraints */
    6976:   pcbddc->n_vertices = total_primal_vertices;
    6977:   /* permute indices in order to have a sorted set of vertices */
    6978:   PetscCall(PetscSortInt(total_primal_vertices, pcbddc->primal_indices_local_idxs));
    6979:   PetscCall(PetscMalloc2(pcbddc->local_primal_size_cc + pcbddc->benign_n, &pcbddc->local_primal_ref_node, pcbddc->local_primal_size_cc + pcbddc->benign_n, &pcbddc->local_primal_ref_mult));
    6980:   PetscCall(PetscArraycpy(pcbddc->local_primal_ref_node, pcbddc->primal_indices_local_idxs, total_primal_vertices));
    6981:   for (i = 0; i < total_primal_vertices; i++) pcbddc->local_primal_ref_mult[i] = 1;

    6983:   /* nonzero structure of constraint matrix */
    6984:   /* and get reference dof for local constraints */
    6985:   PetscCall(PetscMalloc1(pcbddc->local_primal_size, &nnz));
    6986:   for (i = 0; i < total_primal_vertices; i++) nnz[i] = 1;

    6988:   j            = total_primal_vertices;
    6989:   total_counts = total_primal_vertices;
    6990:   cum          = total_primal_vertices;
    6991:   for (i = n_vertices; i < total_counts_cc; i++) {
    6992:     if (!PetscBTLookup(change_basis, i)) {
    6993:       pcbddc->local_primal_ref_node[cum] = constraints_idxs[constraints_idxs_ptr[i]];
    6994:       pcbddc->local_primal_ref_mult[cum] = constraints_n[i];
    6995:       cum++;
    6996:       size_of_constraint = constraints_idxs_ptr[i + 1] - constraints_idxs_ptr[i];
    6997:       for (k = 0; k < constraints_n[i]; k++) {
    6998:         pcbddc->primal_indices_local_idxs[total_counts++] = constraints_idxs[constraints_idxs_ptr[i] + k];
    6999:         nnz[j + k]                                        = size_of_constraint;
    7000:       }
    7001:       j += constraints_n[i];
    7002:     }
    7003:   }
    7004:   PetscCall(MatSeqAIJSetPreallocation(pcbddc->ConstraintMatrix, 0, nnz));
    7005:   PetscCall(MatSetOption(pcbddc->ConstraintMatrix, MAT_NEW_NONZERO_ALLOCATION_ERR, PETSC_TRUE));
    7006:   PetscCall(MatSetOption(pcbddc->ConstraintMatrix, MAT_IGNORE_ZERO_ENTRIES, PETSC_TRUE));
    7007:   PetscCall(PetscFree(nnz));

    7009:   /* set values in constraint matrix */
    7010:   for (i = 0; i < total_primal_vertices; i++) PetscCall(MatSetValue(pcbddc->ConstraintMatrix, i, pcbddc->local_primal_ref_node[i], 1.0, INSERT_VALUES));
    7011:   total_counts = total_primal_vertices;
    7012:   for (i = n_vertices; i < total_counts_cc; i++) {
    7013:     if (!PetscBTLookup(change_basis, i)) {
    7014:       PetscInt *cols;

    7016:       size_of_constraint = constraints_idxs_ptr[i + 1] - constraints_idxs_ptr[i];
    7017:       cols               = constraints_idxs + constraints_idxs_ptr[i];
    7018:       for (k = 0; k < constraints_n[i]; k++) {
    7019:         PetscInt     row = total_counts + k;
    7020:         PetscScalar *vals;

    7022:         vals = constraints_data + constraints_data_ptr[i] + k * size_of_constraint;
    7023:         PetscCall(MatSetValues(pcbddc->ConstraintMatrix, 1, &row, size_of_constraint, cols, vals, INSERT_VALUES));
    7024:       }
    7025:       total_counts += constraints_n[i];
    7026:     }
    7027:   }
    7028:   /* assembling */
    7029:   PetscCall(MatAssemblyBegin(pcbddc->ConstraintMatrix, MAT_FINAL_ASSEMBLY));
    7030:   PetscCall(MatAssemblyEnd(pcbddc->ConstraintMatrix, MAT_FINAL_ASSEMBLY));
    7031:   PetscCall(MatViewFromOptions(pcbddc->ConstraintMatrix, (PetscObject)pc, "-pc_bddc_constraint_mat_view"));

    7033:   /* Create matrix for change of basis. We don't need it in case pcbddc->use_change_of_basis is FALSE */
    7034:   if (pcbddc->use_change_of_basis) {
    7035:     /* dual and primal dofs on a single cc */
    7036:     PetscInt dual_dofs, primal_dofs;
    7037:     /* working stuff for GEQRF */
    7038:     PetscScalar *qr_basis = NULL, *qr_tau = NULL, *qr_work = NULL, lqr_work_t;
    7039:     PetscBLASInt lqr_work;
    7040:     /* working stuff for UNGQR */
    7041:     PetscScalar *gqr_work = NULL, lgqr_work_t = 0.0;
    7042:     PetscBLASInt lgqr_work;
    7043:     /* working stuff for TRTRS */
    7044:     PetscScalar *trs_rhs = NULL;
    7045:     PetscBLASInt Blas_NRHS;
    7046:     /* pointers for values insertion into change of basis matrix */
    7047:     PetscInt    *start_rows, *start_cols;
    7048:     PetscScalar *start_vals;
    7049:     /* working stuff for values insertion */
    7050:     PetscBT   is_primal;
    7051:     PetscInt *aux_primal_numbering_B;
    7052:     /* matrix sizes */
    7053:     PetscInt global_size, local_size;
    7054:     /* temporary change of basis */
    7055:     Mat localChangeOfBasisMatrix;
    7056:     /* extra space for debugging */
    7057:     PetscScalar *dbg_work = NULL;

    7059:     PetscCall(MatCreate(PETSC_COMM_SELF, &localChangeOfBasisMatrix));
    7060:     PetscCall(MatSetType(localChangeOfBasisMatrix, MATAIJ));
    7061:     PetscCall(MatSetSizes(localChangeOfBasisMatrix, pcis->n, pcis->n, pcis->n, pcis->n));
    7062:     /* nonzeros for local mat */
    7063:     PetscCall(PetscMalloc1(pcis->n, &nnz));
    7064:     if (!pcbddc->benign_change || pcbddc->fake_change) {
    7065:       for (i = 0; i < pcis->n; i++) nnz[i] = 1;
    7066:     } else {
    7067:       const PetscInt *ii;
    7068:       PetscInt        n;
    7069:       PetscBool       flg_row;
    7070:       PetscCall(MatGetRowIJ(pcbddc->benign_change, 0, PETSC_FALSE, PETSC_FALSE, &n, &ii, NULL, &flg_row));
    7071:       for (i = 0; i < n; i++) nnz[i] = ii[i + 1] - ii[i];
    7072:       PetscCall(MatRestoreRowIJ(pcbddc->benign_change, 0, PETSC_FALSE, PETSC_FALSE, &n, &ii, NULL, &flg_row));
    7073:     }
    7074:     for (i = n_vertices; i < total_counts_cc; i++) {
    7075:       if (PetscBTLookup(change_basis, i)) {
    7076:         size_of_constraint = constraints_idxs_ptr[i + 1] - constraints_idxs_ptr[i];
    7077:         if (PetscBTLookup(qr_needed_idx, i)) {
    7078:           for (j = 0; j < size_of_constraint; j++) nnz[constraints_idxs[constraints_idxs_ptr[i] + j]] = size_of_constraint;
    7079:         } else {
    7080:           nnz[constraints_idxs[constraints_idxs_ptr[i]]] = size_of_constraint;
    7081:           for (j = 1; j < size_of_constraint; j++) nnz[constraints_idxs[constraints_idxs_ptr[i] + j]] = 2;
    7082:         }
    7083:       }
    7084:     }
    7085:     PetscCall(MatSeqAIJSetPreallocation(localChangeOfBasisMatrix, 0, nnz));
    7086:     PetscCall(MatSetOption(localChangeOfBasisMatrix, MAT_NEW_NONZERO_ALLOCATION_ERR, PETSC_TRUE));
    7087:     PetscCall(PetscFree(nnz));
    7088:     /* Set interior change in the matrix */
    7089:     if (!pcbddc->benign_change || pcbddc->fake_change) {
    7090:       for (i = 0; i < pcis->n; i++) PetscCall(MatSetValue(localChangeOfBasisMatrix, i, i, 1.0, INSERT_VALUES));
    7091:     } else {
    7092:       const PetscInt *ii, *jj;
    7093:       PetscScalar    *aa;
    7094:       PetscInt        n;
    7095:       PetscBool       flg_row;
    7096:       PetscCall(MatGetRowIJ(pcbddc->benign_change, 0, PETSC_FALSE, PETSC_FALSE, &n, &ii, &jj, &flg_row));
    7097:       PetscCall(MatSeqAIJGetArray(pcbddc->benign_change, &aa));
    7098:       for (i = 0; i < n; i++) PetscCall(MatSetValues(localChangeOfBasisMatrix, 1, &i, ii[i + 1] - ii[i], jj + ii[i], aa + ii[i], INSERT_VALUES));
    7099:       PetscCall(MatSeqAIJRestoreArray(pcbddc->benign_change, &aa));
    7100:       PetscCall(MatRestoreRowIJ(pcbddc->benign_change, 0, PETSC_FALSE, PETSC_FALSE, &n, &ii, &jj, &flg_row));
    7101:     }

    7103:     if (pcbddc->dbg_flag) {
    7104:       PetscCall(PetscViewerASCIISynchronizedPrintf(pcbddc->dbg_viewer, "--------------------------------------------------------------\n"));
    7105:       PetscCall(PetscViewerASCIISynchronizedPrintf(pcbddc->dbg_viewer, "Checking change of basis computation for subdomain %04d\n", PetscGlobalRank));
    7106:     }

    7108:     /* Now we loop on the constraints which need a change of basis */
    7109:     /*
    7110:        Change of basis matrix is evaluated similarly to the FIRST APPROACH in
    7111:        Klawonn and Widlund, Dual-primal FETI-DP methods for linear elasticity, (see Sect 6.2.1)

    7113:        Basic blocks of change of basis matrix T computed:

    7115:           - By using the following block transformation if there is only a primal dof on the cc (and -pc_bddc_use_qr_single is not specified)

    7117:             | 1        0   ...        0         s_1/S |
    7118:             | 0        1   ...        0         s_2/S |
    7119:             |              ...                        |
    7120:             | 0        ...            1     s_{n-1}/S |
    7121:             | -s_1/s_n ...    -s_{n-1}/s_n      s_n/S |

    7123:             with S = \sum_{i=1}^n s_i^2
    7124:             NOTE: in the above example, the primal dof is the last one of the edge in LOCAL ordering
    7125:                   in the current implementation, the primal dof is the first one of the edge in GLOBAL ordering

    7127:           - QR decomposition of constraints otherwise
    7128:     */
    7129:     if (qr_needed && max_size_of_constraint) {
    7130:       /* space to store Q */
    7131:       PetscCall(PetscMalloc1(max_size_of_constraint * max_size_of_constraint, &qr_basis));
    7132:       /* array to store scaling factors for reflectors */
    7133:       PetscCall(PetscMalloc1(max_constraints, &qr_tau));
    7134:       /* first we issue queries for optimal work */
    7135:       PetscCall(PetscBLASIntCast(max_size_of_constraint, &Blas_M));
    7136:       PetscCall(PetscBLASIntCast(max_constraints, &Blas_N));
    7137:       PetscCall(PetscBLASIntCast(max_size_of_constraint, &Blas_LDA));
    7138:       lqr_work = -1;
    7139:       PetscCallBLAS("LAPACKgeqrf", LAPACKgeqrf_(&Blas_M, &Blas_N, qr_basis, &Blas_LDA, qr_tau, &lqr_work_t, &lqr_work, &lierr));
    7140:       PetscCheck(!lierr, PETSC_COMM_SELF, PETSC_ERR_LIB, "Error in query to GEQRF Lapack routine %" PetscBLASInt_FMT, lierr);
    7141:       PetscCall(PetscBLASIntCast((PetscInt)PetscRealPart(lqr_work_t), &lqr_work));
    7142:       PetscCall(PetscMalloc1(lqr_work, &qr_work));
    7143:       lgqr_work = -1;
    7144:       PetscCall(PetscBLASIntCast(max_size_of_constraint, &Blas_M));
    7145:       PetscCall(PetscBLASIntCast(max_size_of_constraint, &Blas_N));
    7146:       PetscCall(PetscBLASIntCast(max_constraints, &Blas_K));
    7147:       PetscCall(PetscBLASIntCast(max_size_of_constraint, &Blas_LDA));
    7148:       if (Blas_K > Blas_M) Blas_K = Blas_M; /* adjust just for computing optimal work */
    7149:       PetscCallBLAS("LAPACKorgqr", LAPACKorgqr_(&Blas_M, &Blas_N, &Blas_K, qr_basis, &Blas_LDA, qr_tau, &lgqr_work_t, &lgqr_work, &lierr));
    7150:       PetscCheck(!lierr, PETSC_COMM_SELF, PETSC_ERR_LIB, "Error in query to ORGQR/UNGQR Lapack routine %" PetscBLASInt_FMT, lierr);
    7151:       PetscCall(PetscBLASIntCast((PetscInt)PetscRealPart(lgqr_work_t), &lgqr_work));
    7152:       PetscCall(PetscMalloc1(lgqr_work, &gqr_work));
    7153:       /* array to store rhs and solution of triangular solver */
    7154:       PetscCall(PetscMalloc1(max_constraints * max_constraints, &trs_rhs));
    7155:       /* allocating workspace for check */
    7156:       if (pcbddc->dbg_flag) PetscCall(PetscMalloc1(max_size_of_constraint * (max_constraints + max_size_of_constraint), &dbg_work));
    7157:     }
    7158:     /* array to store whether a node is primal or not */
    7159:     PetscCall(PetscBTCreate(pcis->n_B, &is_primal));
    7160:     PetscCall(PetscMalloc1(total_primal_vertices, &aux_primal_numbering_B));
    7161:     PetscCall(ISGlobalToLocalMappingApply(pcis->BtoNmap, IS_GTOLM_DROP, total_primal_vertices, pcbddc->local_primal_ref_node, &i, aux_primal_numbering_B));
    7162:     PetscCheck(i == total_primal_vertices, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Error in boundary numbering for BDDC vertices! %" PetscInt_FMT " != %" PetscInt_FMT, total_primal_vertices, i);
    7163:     for (i = 0; i < total_primal_vertices; i++) PetscCall(PetscBTSet(is_primal, aux_primal_numbering_B[i]));
    7164:     PetscCall(PetscFree(aux_primal_numbering_B));

    7166:     /* loop on constraints and see whether or not they need a change of basis and compute it */
    7167:     for (total_counts = n_vertices; total_counts < total_counts_cc; total_counts++) {
    7168:       size_of_constraint = constraints_idxs_ptr[total_counts + 1] - constraints_idxs_ptr[total_counts];
    7169:       if (PetscBTLookup(change_basis, total_counts)) {
    7170:         /* get constraint info */
    7171:         primal_dofs = constraints_n[total_counts];
    7172:         dual_dofs   = size_of_constraint - primal_dofs;

    7174:         if (pcbddc->dbg_flag) PetscCall(PetscViewerASCIISynchronizedPrintf(pcbddc->dbg_viewer, "Constraints %" PetscInt_FMT ": %" PetscInt_FMT " need a change of basis (size %" PetscInt_FMT ")\n", total_counts, primal_dofs, size_of_constraint));

    7176:         if (PetscBTLookup(qr_needed_idx, total_counts)) { /* QR */

    7178:           /* copy quadrature constraints for change of basis check */
    7179:           if (pcbddc->dbg_flag) PetscCall(PetscArraycpy(dbg_work, &constraints_data[constraints_data_ptr[total_counts]], size_of_constraint * primal_dofs));
    7180:           /* copy temporary constraints into larger work vector (in order to store all columns of Q) */
    7181:           PetscCall(PetscArraycpy(qr_basis, &constraints_data[constraints_data_ptr[total_counts]], size_of_constraint * primal_dofs));

    7183:           /* compute QR decomposition of constraints */
    7184:           PetscCall(PetscBLASIntCast(size_of_constraint, &Blas_M));
    7185:           PetscCall(PetscBLASIntCast(primal_dofs, &Blas_N));
    7186:           PetscCall(PetscBLASIntCast(size_of_constraint, &Blas_LDA));
    7187:           PetscCall(PetscFPTrapPush(PETSC_FP_TRAP_OFF));
    7188:           PetscCallBLAS("LAPACKgeqrf", LAPACKgeqrf_(&Blas_M, &Blas_N, qr_basis, &Blas_LDA, qr_tau, qr_work, &lqr_work, &lierr));
    7189:           PetscCheck(!lierr, PETSC_COMM_SELF, PETSC_ERR_LIB, "Error in GEQRF Lapack routine %" PetscBLASInt_FMT, lierr);
    7190:           PetscCall(PetscFPTrapPop());

    7192:           /* explicitly compute R^-T */
    7193:           PetscCall(PetscArrayzero(trs_rhs, primal_dofs * primal_dofs));
    7194:           for (j = 0; j < primal_dofs; j++) trs_rhs[j * (primal_dofs + 1)] = 1.0;
    7195:           PetscCall(PetscBLASIntCast(primal_dofs, &Blas_N));
    7196:           PetscCall(PetscBLASIntCast(primal_dofs, &Blas_NRHS));
    7197:           PetscCall(PetscBLASIntCast(size_of_constraint, &Blas_LDA));
    7198:           PetscCall(PetscBLASIntCast(primal_dofs, &Blas_LDB));
    7199:           PetscCall(PetscFPTrapPush(PETSC_FP_TRAP_OFF));
    7200:           PetscCallBLAS("LAPACKtrtrs", LAPACKtrtrs_("U", "T", "N", &Blas_N, &Blas_NRHS, qr_basis, &Blas_LDA, trs_rhs, &Blas_LDB, &lierr));
    7201:           PetscCheck(!lierr, PETSC_COMM_SELF, PETSC_ERR_LIB, "Error in TRTRS Lapack routine %" PetscBLASInt_FMT, lierr);
    7202:           PetscCall(PetscFPTrapPop());

    7204:           /* explicitly compute all columns of Q (Q = [Q1 | Q2]) overwriting QR factorization in qr_basis */
    7205:           PetscCall(PetscBLASIntCast(size_of_constraint, &Blas_M));
    7206:           PetscCall(PetscBLASIntCast(size_of_constraint, &Blas_N));
    7207:           PetscCall(PetscBLASIntCast(primal_dofs, &Blas_K));
    7208:           PetscCall(PetscBLASIntCast(size_of_constraint, &Blas_LDA));
    7209:           PetscCall(PetscFPTrapPush(PETSC_FP_TRAP_OFF));
    7210:           PetscCallBLAS("LAPACKorgqr", LAPACKorgqr_(&Blas_M, &Blas_N, &Blas_K, qr_basis, &Blas_LDA, qr_tau, gqr_work, &lgqr_work, &lierr));
    7211:           PetscCheck(!lierr, PETSC_COMM_SELF, PETSC_ERR_LIB, "Error in ORGQR/UNGQR Lapack routine %" PetscBLASInt_FMT, lierr);
    7212:           PetscCall(PetscFPTrapPop());

    7214:           /* first primal_dofs columns of Q need to be re-scaled in order to be unitary w.r.t constraints
    7215:              i.e. C_{pxn}*Q_{nxn} should be equal to [I_pxp | 0_pxd] (see check below)
    7216:              where n=size_of_constraint, p=primal_dofs, d=dual_dofs (n=p+d), I and 0 identity and null matrix resp. */
    7217:           PetscCall(PetscBLASIntCast(size_of_constraint, &Blas_M));
    7218:           PetscCall(PetscBLASIntCast(primal_dofs, &Blas_N));
    7219:           PetscCall(PetscBLASIntCast(primal_dofs, &Blas_K));
    7220:           PetscCall(PetscBLASIntCast(size_of_constraint, &Blas_LDA));
    7221:           PetscCall(PetscBLASIntCast(primal_dofs, &Blas_LDB));
    7222:           PetscCall(PetscBLASIntCast(size_of_constraint, &Blas_LDC));
    7223:           PetscCall(PetscFPTrapPush(PETSC_FP_TRAP_OFF));
    7224:           PetscCallBLAS("BLASgemm", BLASgemm_("N", "N", &Blas_M, &Blas_N, &Blas_K, &one, qr_basis, &Blas_LDA, trs_rhs, &Blas_LDB, &zero, constraints_data + constraints_data_ptr[total_counts], &Blas_LDC));
    7225:           PetscCall(PetscFPTrapPop());
    7226:           PetscCall(PetscArraycpy(qr_basis, &constraints_data[constraints_data_ptr[total_counts]], size_of_constraint * primal_dofs));

    7228:           /* insert values in change of basis matrix respecting global ordering of new primal dofs */
    7229:           start_rows = &constraints_idxs[constraints_idxs_ptr[total_counts]];
    7230:           /* insert cols for primal dofs */
    7231:           for (j = 0; j < primal_dofs; j++) {
    7232:             start_vals = &qr_basis[j * size_of_constraint];
    7233:             start_cols = &constraints_idxs[constraints_idxs_ptr[total_counts] + j];
    7234:             PetscCall(MatSetValues(localChangeOfBasisMatrix, size_of_constraint, start_rows, 1, start_cols, start_vals, INSERT_VALUES));
    7235:           }
    7236:           /* insert cols for dual dofs */
    7237:           for (j = 0, k = 0; j < dual_dofs; k++) {
    7238:             if (!PetscBTLookup(is_primal, constraints_idxs_B[constraints_idxs_ptr[total_counts] + k])) {
    7239:               start_vals = &qr_basis[(primal_dofs + j) * size_of_constraint];
    7240:               start_cols = &constraints_idxs[constraints_idxs_ptr[total_counts] + k];
    7241:               PetscCall(MatSetValues(localChangeOfBasisMatrix, size_of_constraint, start_rows, 1, start_cols, start_vals, INSERT_VALUES));
    7242:               j++;
    7243:             }
    7244:           }

    7246:           /* check change of basis */
    7247:           if (pcbddc->dbg_flag) {
    7248:             PetscInt  ii, jj;
    7249:             PetscBool valid_qr = PETSC_TRUE;
    7250:             PetscCall(PetscBLASIntCast(primal_dofs, &Blas_M));
    7251:             PetscCall(PetscBLASIntCast(size_of_constraint, &Blas_N));
    7252:             PetscCall(PetscBLASIntCast(size_of_constraint, &Blas_K));
    7253:             PetscCall(PetscBLASIntCast(size_of_constraint, &Blas_LDA));
    7254:             PetscCall(PetscBLASIntCast(size_of_constraint, &Blas_LDB));
    7255:             PetscCall(PetscBLASIntCast(primal_dofs, &Blas_LDC));
    7256:             PetscCall(PetscFPTrapPush(PETSC_FP_TRAP_OFF));
    7257:             PetscCallBLAS("BLASgemm", BLASgemm_("T", "N", &Blas_M, &Blas_N, &Blas_K, &one, dbg_work, &Blas_LDA, qr_basis, &Blas_LDB, &zero, &dbg_work[size_of_constraint * primal_dofs], &Blas_LDC));
    7258:             PetscCall(PetscFPTrapPop());
    7259:             for (jj = 0; jj < size_of_constraint; jj++) {
    7260:               for (ii = 0; ii < primal_dofs; ii++) {
    7261:                 if (ii != jj && PetscAbsScalar(dbg_work[size_of_constraint * primal_dofs + jj * primal_dofs + ii]) > 1.e-12) valid_qr = PETSC_FALSE;
    7262:                 if (ii == jj && PetscAbsScalar(dbg_work[size_of_constraint * primal_dofs + jj * primal_dofs + ii] - (PetscReal)1) > 1.e-12) valid_qr = PETSC_FALSE;
    7263:               }
    7264:             }
    7265:             if (!valid_qr) {
    7266:               PetscCall(PetscViewerASCIISynchronizedPrintf(pcbddc->dbg_viewer, "\t-> wrong change of basis!\n"));
    7267:               for (jj = 0; jj < size_of_constraint; jj++) {
    7268:                 for (ii = 0; ii < primal_dofs; ii++) {
    7269:                   if (ii != jj && PetscAbsScalar(dbg_work[size_of_constraint * primal_dofs + jj * primal_dofs + ii]) > 1.e-12) {
    7270:                     PetscCall(PetscViewerASCIISynchronizedPrintf(pcbddc->dbg_viewer, "\tQr basis function %" PetscInt_FMT " is not orthogonal to constraint %" PetscInt_FMT " (%1.14e)!\n", jj, ii, (double)PetscAbsScalar(dbg_work[size_of_constraint * primal_dofs + jj * primal_dofs + ii])));
    7271:                   }
    7272:                   if (ii == jj && PetscAbsScalar(dbg_work[size_of_constraint * primal_dofs + jj * primal_dofs + ii] - (PetscReal)1) > 1.e-12) {
    7273:                     PetscCall(PetscViewerASCIISynchronizedPrintf(pcbddc->dbg_viewer, "\tQr basis function %" PetscInt_FMT " is not unitary w.r.t constraint %" PetscInt_FMT " (%1.14e)!\n", jj, ii, (double)PetscAbsScalar(dbg_work[size_of_constraint * primal_dofs + jj * primal_dofs + ii])));
    7274:                   }
    7275:                 }
    7276:               }
    7277:             } else {
    7278:               PetscCall(PetscViewerASCIISynchronizedPrintf(pcbddc->dbg_viewer, "\t-> right change of basis!\n"));
    7279:             }
    7280:           }
    7281:         } else { /* simple transformation block */
    7282:           PetscInt    row, col;
    7283:           PetscScalar val, norm;

    7285:           PetscCall(PetscBLASIntCast(size_of_constraint, &Blas_N));
    7286:           PetscCallBLAS("BLASdot", norm = BLASdot_(&Blas_N, constraints_data + constraints_data_ptr[total_counts], &Blas_one, constraints_data + constraints_data_ptr[total_counts], &Blas_one));
    7287:           for (j = 0; j < size_of_constraint; j++) {
    7288:             PetscInt row_B = constraints_idxs_B[constraints_idxs_ptr[total_counts] + j];
    7289:             row            = constraints_idxs[constraints_idxs_ptr[total_counts] + j];
    7290:             if (!PetscBTLookup(is_primal, row_B)) {
    7291:               col = constraints_idxs[constraints_idxs_ptr[total_counts]];
    7292:               PetscCall(MatSetValue(localChangeOfBasisMatrix, row, row, 1.0, INSERT_VALUES));
    7293:               PetscCall(MatSetValue(localChangeOfBasisMatrix, row, col, constraints_data[constraints_data_ptr[total_counts] + j] / norm, INSERT_VALUES));
    7294:             } else {
    7295:               for (k = 0; k < size_of_constraint; k++) {
    7296:                 col = constraints_idxs[constraints_idxs_ptr[total_counts] + k];
    7297:                 if (row != col) {
    7298:                   val = -constraints_data[constraints_data_ptr[total_counts] + k] / constraints_data[constraints_data_ptr[total_counts]];
    7299:                 } else {
    7300:                   val = constraints_data[constraints_data_ptr[total_counts]] / norm;
    7301:                 }
    7302:                 PetscCall(MatSetValue(localChangeOfBasisMatrix, row, col, val, INSERT_VALUES));
    7303:               }
    7304:             }
    7305:           }
    7306:           if (pcbddc->dbg_flag) PetscCall(PetscViewerASCIISynchronizedPrintf(pcbddc->dbg_viewer, "\t-> using standard change of basis\n"));
    7307:         }
    7308:       } else {
    7309:         if (pcbddc->dbg_flag) PetscCall(PetscViewerASCIISynchronizedPrintf(pcbddc->dbg_viewer, "Constraint %" PetscInt_FMT " does not need a change of basis (size %" PetscInt_FMT ")\n", total_counts, size_of_constraint));
    7310:       }
    7311:     }

    7313:     /* free workspace */
    7314:     if (qr_needed) {
    7315:       if (pcbddc->dbg_flag) PetscCall(PetscFree(dbg_work));
    7316:       PetscCall(PetscFree(trs_rhs));
    7317:       PetscCall(PetscFree(qr_tau));
    7318:       PetscCall(PetscFree(qr_work));
    7319:       PetscCall(PetscFree(gqr_work));
    7320:       PetscCall(PetscFree(qr_basis));
    7321:     }
    7322:     PetscCall(PetscBTDestroy(&is_primal));
    7323:     PetscCall(MatAssemblyBegin(localChangeOfBasisMatrix, MAT_FINAL_ASSEMBLY));
    7324:     PetscCall(MatAssemblyEnd(localChangeOfBasisMatrix, MAT_FINAL_ASSEMBLY));

    7326:     /* assembling of global change of variable */
    7327:     if (!pcbddc->fake_change) {
    7328:       Mat tmat;

    7330:       PetscCall(VecGetSize(pcis->vec1_global, &global_size));
    7331:       PetscCall(VecGetLocalSize(pcis->vec1_global, &local_size));
    7332:       PetscCall(MatDuplicate(pc->pmat, MAT_DO_NOT_COPY_VALUES, &tmat));
    7333:       PetscCall(MatISSetLocalMat(tmat, localChangeOfBasisMatrix));
    7334:       PetscCall(MatAssemblyBegin(tmat, MAT_FINAL_ASSEMBLY));
    7335:       PetscCall(MatAssemblyEnd(tmat, MAT_FINAL_ASSEMBLY));
    7336:       PetscCall(MatConvert(tmat, MATAIJ, MAT_INITIAL_MATRIX, &pcbddc->ChangeOfBasisMatrix));
    7337:       PetscCall(MatDestroy(&tmat));
    7338:       PetscCall(VecSet(pcis->vec1_global, 0.0));
    7339:       PetscCall(VecSet(pcis->vec1_N, 1.0));
    7340:       PetscCall(VecScatterBegin(matis->rctx, pcis->vec1_N, pcis->vec1_global, ADD_VALUES, SCATTER_REVERSE));
    7341:       PetscCall(VecScatterEnd(matis->rctx, pcis->vec1_N, pcis->vec1_global, ADD_VALUES, SCATTER_REVERSE));
    7342:       PetscCall(VecReciprocal(pcis->vec1_global));
    7343:       PetscCall(MatDiagonalScale(pcbddc->ChangeOfBasisMatrix, pcis->vec1_global, NULL));

    7345:       /* check */
    7346:       if (pcbddc->dbg_flag) {
    7347:         PetscReal error;
    7348:         Vec       x, x_change;

    7350:         PetscCall(VecDuplicate(pcis->vec1_global, &x));
    7351:         PetscCall(VecDuplicate(pcis->vec1_global, &x_change));
    7352:         PetscCall(VecSetRandom(x, NULL));
    7353:         PetscCall(VecCopy(x, pcis->vec1_global));
    7354:         PetscCall(VecScatterBegin(matis->rctx, x, pcis->vec1_N, INSERT_VALUES, SCATTER_FORWARD));
    7355:         PetscCall(VecScatterEnd(matis->rctx, x, pcis->vec1_N, INSERT_VALUES, SCATTER_FORWARD));
    7356:         PetscCall(MatMult(localChangeOfBasisMatrix, pcis->vec1_N, pcis->vec2_N));
    7357:         PetscCall(VecScatterBegin(matis->rctx, pcis->vec2_N, x, INSERT_VALUES, SCATTER_REVERSE));
    7358:         PetscCall(VecScatterEnd(matis->rctx, pcis->vec2_N, x, INSERT_VALUES, SCATTER_REVERSE));
    7359:         PetscCall(MatMult(pcbddc->ChangeOfBasisMatrix, pcis->vec1_global, x_change));
    7360:         PetscCall(VecAXPY(x, -1.0, x_change));
    7361:         PetscCall(VecNorm(x, NORM_INFINITY, &error));
    7362:         PetscCheck(error <= PETSC_SMALL, PetscObjectComm((PetscObject)pc), PETSC_ERR_PLIB, "Error global vs local change on N: %1.6e", (double)error);
    7363:         PetscCall(VecDestroy(&x));
    7364:         PetscCall(VecDestroy(&x_change));
    7365:       }
    7366:       /* adapt sub_schurs computed (if any) */
    7367:       if (pcbddc->use_deluxe_scaling) {
    7368:         PCBDDCSubSchurs sub_schurs = pcbddc->sub_schurs;

    7370:         PetscCheck(!pcbddc->use_change_of_basis || !pcbddc->adaptive_userdefined, PetscObjectComm((PetscObject)pc), PETSC_ERR_SUP, "Cannot mix automatic change of basis, adaptive selection and user-defined constraints");
    7371:         if (sub_schurs && sub_schurs->S_Ej_all) {
    7372:           Mat S_new, tmat;
    7373:           IS  is_all_N, is_V_Sall = NULL;

    7375:           PetscCall(ISLocalToGlobalMappingApplyIS(pcis->BtoNmap, sub_schurs->is_Ej_all, &is_all_N));
    7376:           PetscCall(MatCreateSubMatrix(localChangeOfBasisMatrix, is_all_N, is_all_N, MAT_INITIAL_MATRIX, &tmat));
    7377:           if (pcbddc->deluxe_zerorows) {
    7378:             ISLocalToGlobalMapping NtoSall;
    7379:             IS                     is_V;
    7380:             PetscCall(ISCreateGeneral(PETSC_COMM_SELF, pcbddc->n_vertices, pcbddc->local_primal_ref_node, PETSC_COPY_VALUES, &is_V));
    7381:             PetscCall(ISLocalToGlobalMappingCreateIS(is_all_N, &NtoSall));
    7382:             PetscCall(ISGlobalToLocalMappingApplyIS(NtoSall, IS_GTOLM_DROP, is_V, &is_V_Sall));
    7383:             PetscCall(ISLocalToGlobalMappingDestroy(&NtoSall));
    7384:             PetscCall(ISDestroy(&is_V));
    7385:           }
    7386:           PetscCall(ISDestroy(&is_all_N));
    7387:           PetscCall(MatPtAP(sub_schurs->S_Ej_all, tmat, MAT_INITIAL_MATRIX, 1.0, &S_new));
    7388:           PetscCall(MatDestroy(&sub_schurs->S_Ej_all));
    7389:           PetscCall(PetscObjectReference((PetscObject)S_new));
    7390:           if (pcbddc->deluxe_zerorows) {
    7391:             const PetscScalar *array;
    7392:             const PetscInt    *idxs_V, *idxs_all;
    7393:             PetscInt           i, n_V;

    7395:             PetscCall(MatZeroRowsColumnsIS(S_new, is_V_Sall, 1., NULL, NULL));
    7396:             PetscCall(ISGetLocalSize(is_V_Sall, &n_V));
    7397:             PetscCall(ISGetIndices(is_V_Sall, &idxs_V));
    7398:             PetscCall(ISGetIndices(sub_schurs->is_Ej_all, &idxs_all));
    7399:             PetscCall(VecGetArrayRead(pcis->D, &array));
    7400:             for (i = 0; i < n_V; i++) {
    7401:               PetscScalar val;
    7402:               PetscInt    idx;

    7404:               idx = idxs_V[i];
    7405:               val = array[idxs_all[idxs_V[i]]];
    7406:               PetscCall(MatSetValue(S_new, idx, idx, val, INSERT_VALUES));
    7407:             }
    7408:             PetscCall(MatAssemblyBegin(S_new, MAT_FINAL_ASSEMBLY));
    7409:             PetscCall(MatAssemblyEnd(S_new, MAT_FINAL_ASSEMBLY));
    7410:             PetscCall(VecRestoreArrayRead(pcis->D, &array));
    7411:             PetscCall(ISRestoreIndices(sub_schurs->is_Ej_all, &idxs_all));
    7412:             PetscCall(ISRestoreIndices(is_V_Sall, &idxs_V));
    7413:           }
    7414:           sub_schurs->S_Ej_all = S_new;
    7415:           PetscCall(MatDestroy(&S_new));
    7416:           if (sub_schurs->sum_S_Ej_all) {
    7417:             PetscCall(MatPtAP(sub_schurs->sum_S_Ej_all, tmat, MAT_INITIAL_MATRIX, 1.0, &S_new));
    7418:             PetscCall(MatDestroy(&sub_schurs->sum_S_Ej_all));
    7419:             PetscCall(PetscObjectReference((PetscObject)S_new));
    7420:             if (pcbddc->deluxe_zerorows) PetscCall(MatZeroRowsColumnsIS(S_new, is_V_Sall, 1., NULL, NULL));
    7421:             sub_schurs->sum_S_Ej_all = S_new;
    7422:             PetscCall(MatDestroy(&S_new));
    7423:           }
    7424:           PetscCall(ISDestroy(&is_V_Sall));
    7425:           PetscCall(MatDestroy(&tmat));
    7426:         }
    7427:         /* destroy any change of basis context in sub_schurs */
    7428:         if (sub_schurs && sub_schurs->change) {
    7429:           PetscInt i;

    7431:           for (i = 0; i < sub_schurs->n_subs; i++) PetscCall(KSPDestroy(&sub_schurs->change[i]));
    7432:           PetscCall(PetscFree(sub_schurs->change));
    7433:         }
    7434:       }
    7435:       if (pcbddc->switch_static) { /* need to save the local change */
    7436:         pcbddc->switch_static_change = localChangeOfBasisMatrix;
    7437:       } else {
    7438:         PetscCall(MatDestroy(&localChangeOfBasisMatrix));
    7439:       }
    7440:       /* determine if any process has changed the pressures locally */
    7441:       pcbddc->change_interior = pcbddc->benign_have_null;
    7442:     } else { /* fake change (get back change of basis into ConstraintMatrix and info on qr) */
    7443:       PetscCall(MatDestroy(&pcbddc->ConstraintMatrix));
    7444:       pcbddc->ConstraintMatrix = localChangeOfBasisMatrix;
    7445:       pcbddc->use_qr_single    = qr_needed;
    7446:     }
    7447:   } else if (pcbddc->user_ChangeOfBasisMatrix || pcbddc->benign_saddle_point) {
    7448:     if (!pcbddc->benign_have_null && pcbddc->user_ChangeOfBasisMatrix) {
    7449:       PetscCall(PetscObjectReference((PetscObject)pcbddc->user_ChangeOfBasisMatrix));
    7450:       pcbddc->ChangeOfBasisMatrix = pcbddc->user_ChangeOfBasisMatrix;
    7451:     } else {
    7452:       Mat benign_global = NULL;
    7453:       if (pcbddc->benign_have_null) {
    7454:         Mat M;

    7456:         pcbddc->change_interior = PETSC_TRUE;
    7457:         PetscCall(VecCopy(matis->counter, pcis->vec1_N));
    7458:         PetscCall(VecReciprocal(pcis->vec1_N));
    7459:         PetscCall(MatDuplicate(pc->pmat, MAT_DO_NOT_COPY_VALUES, &benign_global));
    7460:         if (pcbddc->benign_change) {
    7461:           PetscCall(MatDuplicate(pcbddc->benign_change, MAT_COPY_VALUES, &M));
    7462:           PetscCall(MatDiagonalScale(M, pcis->vec1_N, NULL));
    7463:         } else {
    7464:           PetscCall(MatCreateSeqAIJ(PETSC_COMM_SELF, pcis->n, pcis->n, 1, NULL, &M));
    7465:           PetscCall(MatDiagonalSet(M, pcis->vec1_N, INSERT_VALUES));
    7466:         }
    7467:         PetscCall(MatISSetLocalMat(benign_global, M));
    7468:         PetscCall(MatDestroy(&M));
    7469:         PetscCall(MatAssemblyBegin(benign_global, MAT_FINAL_ASSEMBLY));
    7470:         PetscCall(MatAssemblyEnd(benign_global, MAT_FINAL_ASSEMBLY));
    7471:       }
    7472:       if (pcbddc->user_ChangeOfBasisMatrix) {
    7473:         PetscCall(MatMatMult(pcbddc->user_ChangeOfBasisMatrix, benign_global, MAT_INITIAL_MATRIX, PETSC_DETERMINE, &pcbddc->ChangeOfBasisMatrix));
    7474:         PetscCall(MatDestroy(&benign_global));
    7475:       } else if (pcbddc->benign_have_null) {
    7476:         pcbddc->ChangeOfBasisMatrix = benign_global;
    7477:       }
    7478:     }
    7479:     if (pcbddc->switch_static && pcbddc->ChangeOfBasisMatrix) { /* need to save the local change */
    7480:       IS              is_global;
    7481:       const PetscInt *gidxs;

    7483:       PetscCall(ISLocalToGlobalMappingGetIndices(matis->rmapping, &gidxs));
    7484:       PetscCall(ISCreateGeneral(PetscObjectComm((PetscObject)pc), pcis->n, gidxs, PETSC_COPY_VALUES, &is_global));
    7485:       PetscCall(ISLocalToGlobalMappingRestoreIndices(matis->rmapping, &gidxs));
    7486:       PetscCall(MatCreateSubMatrixUnsorted(pcbddc->ChangeOfBasisMatrix, is_global, is_global, &pcbddc->switch_static_change));
    7487:       PetscCall(ISDestroy(&is_global));
    7488:     }
    7489:   }
    7490:   if (!pcbddc->fake_change && pcbddc->ChangeOfBasisMatrix && !pcbddc->work_change) PetscCall(VecDuplicate(pcis->vec1_global, &pcbddc->work_change));

    7492:   if (!pcbddc->fake_change) {
    7493:     /* add pressure dofs to set of primal nodes for numbering purposes */
    7494:     for (i = 0; i < pcbddc->benign_n; i++) {
    7495:       pcbddc->local_primal_ref_node[pcbddc->local_primal_size_cc]  = pcbddc->benign_p0_lidx[i];
    7496:       pcbddc->primal_indices_local_idxs[pcbddc->local_primal_size] = pcbddc->benign_p0_lidx[i];
    7497:       pcbddc->local_primal_ref_mult[pcbddc->local_primal_size_cc]  = 1;
    7498:       pcbddc->local_primal_size_cc++;
    7499:       pcbddc->local_primal_size++;
    7500:     }

    7502:     /* check if a new primal space has been introduced (also take into account benign trick) */
    7503:     pcbddc->new_primal_space_local = PETSC_TRUE;
    7504:     if (olocal_primal_size == pcbddc->local_primal_size) {
    7505:       PetscCall(PetscArraycmp(pcbddc->local_primal_ref_node, olocal_primal_ref_node, olocal_primal_size_cc, &pcbddc->new_primal_space_local));
    7506:       pcbddc->new_primal_space_local = (PetscBool)(!pcbddc->new_primal_space_local);
    7507:       if (!pcbddc->new_primal_space_local) {
    7508:         PetscCall(PetscArraycmp(pcbddc->local_primal_ref_mult, olocal_primal_ref_mult, olocal_primal_size_cc, &pcbddc->new_primal_space_local));
    7509:         pcbddc->new_primal_space_local = (PetscBool)(!pcbddc->new_primal_space_local);
    7510:       }
    7511:     }
    7512:     /* new_primal_space will be used for numbering of coarse dofs, so it should be the same across all subdomains */
    7513:     PetscCallMPI(MPIU_Allreduce(&pcbddc->new_primal_space_local, &pcbddc->new_primal_space, 1, MPI_C_BOOL, MPI_LOR, PetscObjectComm((PetscObject)pc)));
    7514:   }
    7515:   PetscCall(PetscFree2(olocal_primal_ref_node, olocal_primal_ref_mult));

    7517:   /* flush dbg viewer */
    7518:   if (pcbddc->dbg_flag) PetscCall(PetscViewerFlush(pcbddc->dbg_viewer));

    7520:   /* free workspace */
    7521:   PetscCall(PetscBTDestroy(&qr_needed_idx));
    7522:   PetscCall(PetscBTDestroy(&change_basis));
    7523:   if (!pcbddc->adaptive_selection) {
    7524:     PetscCall(PetscFree3(constraints_idxs_ptr, constraints_data_ptr, constraints_n));
    7525:     PetscCall(PetscFree3(constraints_data, constraints_idxs, constraints_idxs_B));
    7526:   } else {
    7527:     PetscCall(PetscFree5(pcbddc->adaptive_constraints_n, pcbddc->adaptive_constraints_idxs_ptr, pcbddc->adaptive_constraints_data_ptr, pcbddc->adaptive_constraints_idxs, pcbddc->adaptive_constraints_data));
    7528:     PetscCall(PetscFree(constraints_n));
    7529:     PetscCall(PetscFree(constraints_idxs_B));
    7530:   }
    7531:   PetscFunctionReturn(PETSC_SUCCESS);
    7532: }

    7534: PetscErrorCode PCBDDCAnalyzeInterface(PC pc)
    7535: {
    7536:   ISLocalToGlobalMapping map;
    7537:   PC_BDDC               *pcbddc = (PC_BDDC *)pc->data;
    7538:   Mat_IS                *matis  = (Mat_IS *)pc->pmat->data;
    7539:   PetscInt               i, N;
    7540:   PetscBool              rcsr = PETSC_FALSE;

    7542:   PetscFunctionBegin;
    7543:   if (pcbddc->recompute_topography) {
    7544:     pcbddc->graphanalyzed = PETSC_FALSE;
    7545:     /* Reset previously computed graph */
    7546:     PetscCall(PCBDDCGraphReset(pcbddc->mat_graph));
    7547:     /* Init local Graph struct */
    7548:     PetscCall(MatGetSize(pc->pmat, &N, NULL));
    7549:     PetscCall(MatISGetLocalToGlobalMapping(pc->pmat, &map, NULL));
    7550:     PetscCall(PCBDDCGraphInit(pcbddc->mat_graph, map, N, pcbddc->graphmaxcount));

    7552:     if (pcbddc->user_primal_vertices_local && !pcbddc->user_primal_vertices) PetscCall(PCBDDCConsistencyCheckIS(pc, MPI_LOR, &pcbddc->user_primal_vertices_local));
    7553:     /* Check validity of the csr graph passed in by the user */
    7554:     PetscCheck(!pcbddc->mat_graph->nvtxs_csr || pcbddc->mat_graph->nvtxs_csr == pcbddc->mat_graph->nvtxs, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Invalid size of local CSR graph! Found %" PetscInt_FMT ", expected %" PetscInt_FMT, pcbddc->mat_graph->nvtxs_csr,
    7555:                pcbddc->mat_graph->nvtxs);

    7557:     /* Set default CSR adjacency of local dofs if not provided by the user with PCBDDCSetLocalAdjacencyGraph */
    7558:     if (!pcbddc->mat_graph->xadj && pcbddc->use_local_adj) {
    7559:       PetscInt *xadj, *adjncy;
    7560:       PetscInt  nvtxs;
    7561:       PetscBool flg_row;
    7562:       Mat       A;

    7564:       PetscCall(PetscObjectReference((PetscObject)matis->A));
    7565:       A = matis->A;
    7566:       for (PetscInt i = 0; i < pcbddc->local_adj_square; i++) {
    7567:         Mat AtA;

    7569:         PetscCall(MatProductCreate(A, A, NULL, &AtA));
    7570:         PetscCall(MatSetOptionsPrefix(AtA, "pc_bddc_graph_"));
    7571:         PetscCall(MatProductSetType(AtA, MATPRODUCT_AtB));
    7572:         PetscCall(MatProductSetFromOptions(AtA));
    7573:         PetscCall(MatProductSymbolic(AtA));
    7574:         PetscCall(MatProductClear(AtA));
    7575:         /* we only need the sparsity, cheat and tell PETSc the matrix has been assembled */
    7576:         AtA->assembled = PETSC_TRUE;
    7577:         PetscCall(MatDestroy(&A));
    7578:         A = AtA;
    7579:       }
    7580:       PetscCall(MatGetRowIJ(A, 0, PETSC_TRUE, PETSC_FALSE, &nvtxs, (const PetscInt **)&xadj, (const PetscInt **)&adjncy, &flg_row));
    7581:       if (flg_row) {
    7582:         PetscCall(PCBDDCSetLocalAdjacencyGraph(pc, nvtxs, xadj, adjncy, PETSC_COPY_VALUES));
    7583:         pcbddc->computed_rowadj = PETSC_TRUE;
    7584:         PetscCall(MatRestoreRowIJ(A, 0, PETSC_TRUE, PETSC_FALSE, &nvtxs, (const PetscInt **)&xadj, (const PetscInt **)&adjncy, &flg_row));
    7585:         rcsr = PETSC_TRUE;
    7586:       }
    7587:       PetscCall(MatDestroy(&A));
    7588:     }
    7589:     if (pcbddc->dbg_flag) PetscCall(PetscViewerFlush(pcbddc->dbg_viewer));

    7591:     if (pcbddc->mat_graph->cdim && !pcbddc->mat_graph->cloc) {
    7592:       PetscReal   *lcoords;
    7593:       PetscInt     n;
    7594:       MPI_Datatype dimrealtype;
    7595:       PetscMPIInt  cdimi;

    7597:       /* TODO: support for blocked */
    7598:       PetscCheck(pcbddc->mat_graph->cnloc == pc->pmat->rmap->n, PETSC_COMM_SELF, PETSC_ERR_USER, "Invalid number of local coordinates! Got %" PetscInt_FMT ", expected %" PetscInt_FMT, pcbddc->mat_graph->cnloc, pc->pmat->rmap->n);
    7599:       PetscCall(MatGetLocalSize(matis->A, &n, NULL));
    7600:       PetscCall(PetscMalloc1(pcbddc->mat_graph->cdim * n, &lcoords));
    7601:       PetscCall(PetscMPIIntCast(pcbddc->mat_graph->cdim, &cdimi));
    7602:       PetscCallMPI(MPI_Type_contiguous(cdimi, MPIU_REAL, &dimrealtype));
    7603:       PetscCallMPI(MPI_Type_commit(&dimrealtype));
    7604:       PetscCall(PetscSFBcastBegin(matis->sf, dimrealtype, pcbddc->mat_graph->coords, lcoords, MPI_REPLACE));
    7605:       PetscCall(PetscSFBcastEnd(matis->sf, dimrealtype, pcbddc->mat_graph->coords, lcoords, MPI_REPLACE));
    7606:       PetscCallMPI(MPI_Type_free(&dimrealtype));
    7607:       PetscCall(PetscFree(pcbddc->mat_graph->coords));

    7609:       pcbddc->mat_graph->coords = lcoords;
    7610:       pcbddc->mat_graph->cloc   = PETSC_TRUE;
    7611:       pcbddc->mat_graph->cnloc  = n;
    7612:     }
    7613:     PetscCheck(!pcbddc->mat_graph->cnloc || pcbddc->mat_graph->cnloc == pcbddc->mat_graph->nvtxs, PETSC_COMM_SELF, PETSC_ERR_USER, "Invalid number of local subdomain coordinates! Got %" PetscInt_FMT ", expected %" PetscInt_FMT, pcbddc->mat_graph->cnloc,
    7614:                pcbddc->mat_graph->nvtxs);
    7615:     pcbddc->mat_graph->active_coords = (PetscBool)(pcbddc->corner_selection && pcbddc->mat_graph->cdim && !pcbddc->corner_selected);

    7617:     /* attach info on disconnected subdomains if present */
    7618:     if (pcbddc->n_local_subs) {
    7619:       PetscInt *local_subs, n, totn;

    7621:       PetscCall(MatGetLocalSize(matis->A, &n, NULL));
    7622:       PetscCall(PetscMalloc1(n, &local_subs));
    7623:       for (i = 0; i < n; i++) local_subs[i] = pcbddc->n_local_subs;
    7624:       for (i = 0; i < pcbddc->n_local_subs; i++) {
    7625:         const PetscInt *idxs;
    7626:         PetscInt        nl, j;

    7628:         PetscCall(ISGetLocalSize(pcbddc->local_subs[i], &nl));
    7629:         PetscCall(ISGetIndices(pcbddc->local_subs[i], &idxs));
    7630:         for (j = 0; j < nl; j++) local_subs[idxs[j]] = i;
    7631:         PetscCall(ISRestoreIndices(pcbddc->local_subs[i], &idxs));
    7632:       }
    7633:       for (i = 0, totn = 0; i < n; i++) totn = PetscMax(totn, local_subs[i]);
    7634:       pcbddc->mat_graph->n_local_subs = totn + 1;
    7635:       pcbddc->mat_graph->local_subs   = local_subs;
    7636:     }

    7638:     /* Setup of Graph */
    7639:     PetscCall(PCBDDCGraphSetUp(pcbddc->mat_graph, pcbddc->vertex_size, pcbddc->NeumannBoundariesLocal, pcbddc->DirichletBoundariesLocal, pcbddc->n_ISForDofsLocal, pcbddc->ISForDofsLocal, pcbddc->user_primal_vertices_local));
    7640:   }

    7642:   if (!pcbddc->graphanalyzed) {
    7643:     /* Graph's connected components analysis */
    7644:     PetscCall(PCBDDCGraphComputeConnectedComponents(pcbddc->mat_graph));
    7645:     pcbddc->graphanalyzed   = PETSC_TRUE;
    7646:     pcbddc->corner_selected = pcbddc->corner_selection;
    7647:   }
    7648:   if (rcsr) pcbddc->mat_graph->nvtxs_csr = 0;
    7649:   PetscFunctionReturn(PETSC_SUCCESS);
    7650: }

    7652: PetscErrorCode PCBDDCOrthonormalizeVecs(PetscInt *nio, Vec vecs[])
    7653: {
    7654:   PetscInt     i, j, n;
    7655:   PetscScalar *alphas;
    7656:   PetscReal    norm, *onorms;

    7658:   PetscFunctionBegin;
    7659:   n = *nio;
    7660:   if (!n) PetscFunctionReturn(PETSC_SUCCESS);
    7661:   PetscCall(PetscMalloc2(n, &alphas, n, &onorms));
    7662:   PetscCall(VecNormalize(vecs[0], &norm));
    7663:   if (norm < PETSC_SMALL) {
    7664:     onorms[0] = 0.0;
    7665:     PetscCall(VecSet(vecs[0], 0.0));
    7666:   } else {
    7667:     onorms[0] = norm;
    7668:   }

    7670:   for (i = 1; i < n; i++) {
    7671:     PetscCall(VecMDot(vecs[i], i, vecs, alphas));
    7672:     for (j = 0; j < i; j++) alphas[j] = PetscConj(-alphas[j]);
    7673:     PetscCall(VecMAXPY(vecs[i], i, alphas, vecs));
    7674:     PetscCall(VecNormalize(vecs[i], &norm));
    7675:     if (norm < PETSC_SMALL) {
    7676:       onorms[i] = 0.0;
    7677:       PetscCall(VecSet(vecs[i], 0.0));
    7678:     } else {
    7679:       onorms[i] = norm;
    7680:     }
    7681:   }
    7682:   /* push nonzero vectors at the beginning */
    7683:   for (i = 0; i < n; i++) {
    7684:     if (onorms[i] == 0.0) {
    7685:       for (j = i + 1; j < n; j++) {
    7686:         if (onorms[j] != 0.0) {
    7687:           PetscCall(VecCopy(vecs[j], vecs[i]));
    7688:           onorms[i] = onorms[j];
    7689:           onorms[j] = 0.0;
    7690:           break;
    7691:         }
    7692:       }
    7693:     }
    7694:   }
    7695:   for (i = 0, *nio = 0; i < n; i++) *nio += onorms[i] != 0.0 ? 1 : 0;
    7696:   PetscCall(PetscFree2(alphas, onorms));
    7697:   PetscFunctionReturn(PETSC_SUCCESS);
    7698: }

    7700: static PetscErrorCode PCBDDCMatISGetSubassemblingPattern(Mat mat, PetscInt *n_subdomains, PetscInt redprocs, IS *is_sends, PetscBool *have_void)
    7701: {
    7702:   ISLocalToGlobalMapping mapping;
    7703:   Mat                    A;
    7704:   PetscInt               n_neighs, *neighs, *n_shared, **shared;
    7705:   PetscMPIInt            size, rank, color;
    7706:   PetscInt              *xadj, *adjncy;
    7707:   PetscInt              *adjncy_wgt, *v_wgt, *ranks_send_to_idx;
    7708:   PetscInt               im_active, active_procs, N, n, i, j, threshold = 2;
    7709:   PetscInt               void_procs, *procs_candidates = NULL;
    7710:   PetscInt               xadj_count, *count;
    7711:   PetscBool              ismatis, use_vwgt = PETSC_FALSE;
    7712:   PetscSubcomm           psubcomm;
    7713:   MPI_Comm               subcomm;

    7715:   PetscFunctionBegin;
    7717:   PetscCall(PetscObjectTypeCompare((PetscObject)mat, MATIS, &ismatis));
    7718:   PetscCheck(ismatis, PetscObjectComm((PetscObject)mat), PETSC_ERR_SUP, "Cannot use %s on a matrix object which is not of type MATIS", PETSC_FUNCTION_NAME);
    7721:   PetscCheck(*n_subdomains > 0, PetscObjectComm((PetscObject)mat), PETSC_ERR_ARG_WRONG, "Invalid number of subdomains requested %" PetscInt_FMT, *n_subdomains);

    7723:   if (have_void) *have_void = PETSC_FALSE;
    7724:   PetscCallMPI(MPI_Comm_size(PetscObjectComm((PetscObject)mat), &size));
    7725:   PetscCallMPI(MPI_Comm_rank(PetscObjectComm((PetscObject)mat), &rank));
    7726:   PetscCall(MatISGetLocalMat(mat, &A));
    7727:   PetscCall(MatGetLocalSize(A, &n, NULL));
    7728:   im_active = !!n;
    7729:   PetscCallMPI(MPIU_Allreduce(&im_active, &active_procs, 1, MPIU_INT, MPI_SUM, PetscObjectComm((PetscObject)mat)));
    7730:   void_procs = size - active_procs;
    7731:   /* get ranks of non-active processes in mat communicator */
    7732:   if (void_procs) {
    7733:     PetscInt ncand;

    7735:     if (have_void) *have_void = PETSC_TRUE;
    7736:     PetscCall(PetscMalloc1(size, &procs_candidates));
    7737:     PetscCallMPI(MPI_Allgather(&im_active, 1, MPIU_INT, procs_candidates, 1, MPIU_INT, PetscObjectComm((PetscObject)mat)));
    7738:     for (i = 0, ncand = 0; i < size; i++) {
    7739:       if (!procs_candidates[i]) procs_candidates[ncand++] = i;
    7740:     }
    7741:     /* force n_subdomains to be not greater that the number of non-active processes */
    7742:     *n_subdomains = PetscMin(void_procs, *n_subdomains);
    7743:   }

    7745:   /* number of subdomains requested greater than active processes or matrix size -> just shift the matrix
    7746:      number of subdomains requested 1 -> send to rank-0 or first candidate in voids  */
    7747:   PetscCall(MatGetSize(mat, &N, NULL));
    7748:   if (active_procs < *n_subdomains || *n_subdomains == 1 || N <= *n_subdomains) {
    7749:     PetscInt  issize, isidx, dest;
    7750:     PetscBool default_sub;

    7752:     if (*n_subdomains == 1) dest = 0;
    7753:     else dest = rank;
    7754:     if (im_active) {
    7755:       issize = 1;
    7756:       if (procs_candidates) { /* shift the pattern on non-active candidates (if any) */
    7757:         isidx = procs_candidates[dest];
    7758:       } else {
    7759:         isidx = dest;
    7760:       }
    7761:     } else {
    7762:       issize = 0;
    7763:       isidx  = rank;
    7764:     }
    7765:     if (*n_subdomains != 1) *n_subdomains = active_procs;
    7766:     PetscCall(ISCreateGeneral(PetscObjectComm((PetscObject)mat), issize, &isidx, PETSC_COPY_VALUES, is_sends));
    7767:     default_sub = (PetscBool)(isidx == rank);
    7768:     PetscCallMPI(MPIU_Allreduce(MPI_IN_PLACE, &default_sub, 1, MPI_C_BOOL, MPI_LAND, PetscObjectComm((PetscObject)mat)));
    7769:     if (default_sub) PetscCall(PetscObjectSetName((PetscObject)*is_sends, "default subassembling"));
    7770:     PetscCall(PetscFree(procs_candidates));
    7771:     PetscFunctionReturn(PETSC_SUCCESS);
    7772:   }
    7773:   PetscCall(PetscOptionsGetBool(NULL, ((PetscObject)A)->prefix, "-mat_is_partitioning_use_vwgt", &use_vwgt, NULL));
    7774:   PetscCall(PetscOptionsGetInt(NULL, ((PetscObject)A)->prefix, "-mat_is_partitioning_threshold", &threshold, NULL));
    7775:   threshold = PetscMax(threshold, 2);

    7777:   /* Get info on mapping */
    7778:   PetscCall(MatISGetLocalToGlobalMapping(mat, &mapping, NULL));
    7779:   PetscCall(ISLocalToGlobalMappingGetInfo(mapping, &n_neighs, &neighs, &n_shared, &shared));

    7781:   /* build local CSR graph of subdomains' connectivity */
    7782:   PetscCall(PetscMalloc1(2, &xadj));
    7783:   xadj[0] = 0;
    7784:   xadj[1] = PetscMax(n_neighs - 1, 0);
    7785:   PetscCall(PetscMalloc1(xadj[1], &adjncy));
    7786:   PetscCall(PetscMalloc1(xadj[1], &adjncy_wgt));
    7787:   PetscCall(PetscCalloc1(n, &count));
    7788:   for (i = 1; i < n_neighs; i++)
    7789:     for (j = 0; j < n_shared[i]; j++) count[shared[i][j]] += 1;

    7791:   xadj_count = 0;
    7792:   for (i = 1; i < n_neighs; i++) {
    7793:     for (j = 0; j < n_shared[i]; j++) {
    7794:       if (count[shared[i][j]] < threshold) {
    7795:         adjncy[xadj_count]     = neighs[i];
    7796:         adjncy_wgt[xadj_count] = n_shared[i];
    7797:         xadj_count++;
    7798:         break;
    7799:       }
    7800:     }
    7801:   }
    7802:   xadj[1] = xadj_count;
    7803:   PetscCall(PetscFree(count));
    7804:   PetscCall(ISLocalToGlobalMappingRestoreInfo(mapping, &n_neighs, &neighs, &n_shared, &shared));
    7805:   PetscCall(PetscSortIntWithArray(xadj[1], adjncy, adjncy_wgt));

    7807:   PetscCall(PetscMalloc1(1, &ranks_send_to_idx));

    7809:   /* Restrict work on active processes only */
    7810:   PetscCall(PetscMPIIntCast(im_active, &color));
    7811:   if (void_procs) {
    7812:     PetscCall(PetscSubcommCreate(PetscObjectComm((PetscObject)mat), &psubcomm));
    7813:     PetscCall(PetscSubcommSetNumber(psubcomm, 2)); /* 2 groups, active process and not active processes */
    7814:     PetscCall(PetscSubcommSetTypeGeneral(psubcomm, color, rank));
    7815:     subcomm = PetscSubcommChild(psubcomm);
    7816:   } else {
    7817:     psubcomm = NULL;
    7818:     subcomm  = PetscObjectComm((PetscObject)mat);
    7819:   }

    7821:   v_wgt = NULL;
    7822:   if (!color) {
    7823:     PetscCall(PetscFree(xadj));
    7824:     PetscCall(PetscFree(adjncy));
    7825:     PetscCall(PetscFree(adjncy_wgt));
    7826:   } else {
    7827:     Mat             subdomain_adj;
    7828:     IS              new_ranks, new_ranks_contig;
    7829:     MatPartitioning partitioner;
    7830:     PetscInt        rstart, rend;
    7831:     PetscMPIInt     irstart = 0, irend = 0;
    7832:     PetscInt       *is_indices, *oldranks;
    7833:     PetscMPIInt     size;
    7834:     PetscBool       aggregate;

    7836:     PetscCallMPI(MPI_Comm_size(subcomm, &size));
    7837:     if (void_procs) {
    7838:       PetscInt prank = rank;
    7839:       PetscCall(PetscMalloc1(size, &oldranks));
    7840:       PetscCallMPI(MPI_Allgather(&prank, 1, MPIU_INT, oldranks, 1, MPIU_INT, subcomm));
    7841:       for (i = 0; i < xadj[1]; i++) PetscCall(PetscFindInt(adjncy[i], size, oldranks, &adjncy[i]));
    7842:       PetscCall(PetscSortIntWithArray(xadj[1], adjncy, adjncy_wgt));
    7843:     } else {
    7844:       oldranks = NULL;
    7845:     }
    7846:     aggregate = ((redprocs > 0 && redprocs < size) ? PETSC_TRUE : PETSC_FALSE);
    7847:     if (aggregate) { /* TODO: all this part could be made more efficient */
    7848:       PetscInt     lrows, row, ncols, *cols;
    7849:       PetscMPIInt  nrank;
    7850:       PetscScalar *vals;

    7852:       PetscCallMPI(MPI_Comm_rank(subcomm, &nrank));
    7853:       lrows = 0;
    7854:       if (nrank < redprocs) {
    7855:         lrows = size / redprocs;
    7856:         if (nrank < size % redprocs) lrows++;
    7857:       }
    7858:       PetscCall(MatCreateAIJ(subcomm, lrows, lrows, size, size, 50, NULL, 50, NULL, &subdomain_adj));
    7859:       PetscCall(MatGetOwnershipRange(subdomain_adj, &rstart, &rend));
    7860:       PetscCall(PetscMPIIntCast(rstart, &irstart));
    7861:       PetscCall(PetscMPIIntCast(rend, &irend));
    7862:       PetscCall(MatSetOption(subdomain_adj, MAT_NEW_NONZERO_LOCATION_ERR, PETSC_FALSE));
    7863:       PetscCall(MatSetOption(subdomain_adj, MAT_NEW_NONZERO_ALLOCATION_ERR, PETSC_FALSE));
    7864:       row   = nrank;
    7865:       ncols = xadj[1] - xadj[0];
    7866:       cols  = adjncy;
    7867:       PetscCall(PetscMalloc1(ncols, &vals));
    7868:       for (i = 0; i < ncols; i++) vals[i] = adjncy_wgt[i];
    7869:       PetscCall(MatSetValues(subdomain_adj, 1, &row, ncols, cols, vals, INSERT_VALUES));
    7870:       PetscCall(MatAssemblyBegin(subdomain_adj, MAT_FINAL_ASSEMBLY));
    7871:       PetscCall(MatAssemblyEnd(subdomain_adj, MAT_FINAL_ASSEMBLY));
    7872:       PetscCall(PetscFree(xadj));
    7873:       PetscCall(PetscFree(adjncy));
    7874:       PetscCall(PetscFree(adjncy_wgt));
    7875:       PetscCall(PetscFree(vals));
    7876:       if (use_vwgt) {
    7877:         Vec                v;
    7878:         const PetscScalar *array;
    7879:         PetscInt           nl;

    7881:         PetscCall(MatCreateVecs(subdomain_adj, &v, NULL));
    7882:         PetscCall(VecSetValue(v, row, (PetscScalar)n, INSERT_VALUES));
    7883:         PetscCall(VecAssemblyBegin(v));
    7884:         PetscCall(VecAssemblyEnd(v));
    7885:         PetscCall(VecGetLocalSize(v, &nl));
    7886:         PetscCall(VecGetArrayRead(v, &array));
    7887:         PetscCall(PetscMalloc1(nl, &v_wgt));
    7888:         for (i = 0; i < nl; i++) v_wgt[i] = (PetscInt)PetscRealPart(array[i]);
    7889:         PetscCall(VecRestoreArrayRead(v, &array));
    7890:         PetscCall(VecDestroy(&v));
    7891:       }
    7892:     } else {
    7893:       PetscCall(MatCreateMPIAdj(subcomm, 1, size, xadj, adjncy, adjncy_wgt, &subdomain_adj));
    7894:       if (use_vwgt) {
    7895:         PetscCall(PetscMalloc1(1, &v_wgt));
    7896:         v_wgt[0] = n;
    7897:       }
    7898:     }
    7899:     /* PetscCall(MatView(subdomain_adj,0)); */

    7901:     /* Partition */
    7902:     PetscCall(MatPartitioningCreate(subcomm, &partitioner));
    7903: #if defined(PETSC_HAVE_PTSCOTCH)
    7904:     PetscCall(MatPartitioningSetType(partitioner, MATPARTITIONINGPTSCOTCH));
    7905: #elif defined(PETSC_HAVE_PARMETIS)
    7906:     PetscCall(MatPartitioningSetType(partitioner, MATPARTITIONINGPARMETIS));
    7907: #else
    7908:     PetscCall(MatPartitioningSetType(partitioner, MATPARTITIONINGAVERAGE));
    7909: #endif
    7910:     PetscCall(MatPartitioningSetAdjacency(partitioner, subdomain_adj));
    7911:     if (v_wgt) PetscCall(MatPartitioningSetVertexWeights(partitioner, v_wgt));
    7912:     *n_subdomains = PetscMin(size, *n_subdomains);
    7913:     PetscCall(MatPartitioningSetNParts(partitioner, *n_subdomains));
    7914:     PetscCall(MatPartitioningSetFromOptions(partitioner));
    7915:     PetscCall(MatPartitioningApply(partitioner, &new_ranks));
    7916:     /* PetscCall(MatPartitioningView(partitioner,0)); */

    7918:     /* renumber new_ranks to avoid "holes" in new set of processors */
    7919:     PetscCall(ISRenumber(new_ranks, NULL, NULL, &new_ranks_contig));
    7920:     PetscCall(ISDestroy(&new_ranks));
    7921:     PetscCall(ISGetIndices(new_ranks_contig, (const PetscInt **)&is_indices));
    7922:     if (!aggregate) {
    7923:       if (procs_candidates) { /* shift the pattern on non-active candidates (if any) */
    7924:         PetscAssert(oldranks, PETSC_COMM_SELF, PETSC_ERR_PLIB, "This should not happen");
    7925:         ranks_send_to_idx[0] = procs_candidates[oldranks[is_indices[0]]];
    7926:       } else if (oldranks) {
    7927:         ranks_send_to_idx[0] = oldranks[is_indices[0]];
    7928:       } else {
    7929:         ranks_send_to_idx[0] = is_indices[0];
    7930:       }
    7931:     } else {
    7932:       PetscInt     idx = 0;
    7933:       PetscMPIInt  tag;
    7934:       MPI_Request *reqs;

    7936:       PetscCall(PetscObjectGetNewTag((PetscObject)subdomain_adj, &tag));
    7937:       PetscCall(PetscMalloc1(rend - rstart, &reqs));
    7938:       for (PetscMPIInt i = irstart; i < irend; i++) PetscCallMPI(MPIU_Isend(is_indices + i - rstart, 1, MPIU_INT, i, tag, subcomm, &reqs[i - rstart]));
    7939:       PetscCallMPI(MPIU_Recv(&idx, 1, MPIU_INT, MPI_ANY_SOURCE, tag, subcomm, MPI_STATUS_IGNORE));
    7940:       PetscCallMPI(MPI_Waitall(irend - irstart, reqs, MPI_STATUSES_IGNORE));
    7941:       PetscCall(PetscFree(reqs));
    7942:       if (procs_candidates) { /* shift the pattern on non-active candidates (if any) */
    7943:         PetscAssert(oldranks, PETSC_COMM_SELF, PETSC_ERR_PLIB, "This should not happen");
    7944:         ranks_send_to_idx[0] = procs_candidates[oldranks[idx]];
    7945:       } else if (oldranks) {
    7946:         ranks_send_to_idx[0] = oldranks[idx];
    7947:       } else {
    7948:         ranks_send_to_idx[0] = idx;
    7949:       }
    7950:     }
    7951:     PetscCall(ISRestoreIndices(new_ranks_contig, (const PetscInt **)&is_indices));
    7952:     /* clean up */
    7953:     PetscCall(PetscFree(oldranks));
    7954:     PetscCall(ISDestroy(&new_ranks_contig));
    7955:     PetscCall(MatDestroy(&subdomain_adj));
    7956:     PetscCall(MatPartitioningDestroy(&partitioner));
    7957:   }
    7958:   PetscCall(PetscSubcommDestroy(&psubcomm));
    7959:   PetscCall(PetscFree(procs_candidates));

    7961:   /* assemble parallel IS for sends */
    7962:   i = 1;
    7963:   if (!color) i = 0;
    7964:   PetscCall(ISCreateGeneral(PetscObjectComm((PetscObject)mat), i, ranks_send_to_idx, PETSC_OWN_POINTER, is_sends));
    7965:   PetscFunctionReturn(PETSC_SUCCESS);
    7966: }

    7968: typedef enum {
    7969:   MATDENSE_PRIVATE = 0,
    7970:   MATAIJ_PRIVATE,
    7971:   MATBAIJ_PRIVATE,
    7972:   MATSBAIJ_PRIVATE
    7973: } MatTypePrivate;

    7975: static PetscErrorCode PCBDDCMatISSubassemble(Mat mat, IS is_sends, PetscInt n_subdomains, PetscBool restrict_comm, PetscBool restrict_full, PetscBool reuse, Mat *mat_n, PetscInt nis, IS isarray[], PetscInt nvecs, Vec nnsp_vec[])
    7976: {
    7977:   Mat                    local_mat;
    7978:   IS                     is_sends_internal;
    7979:   PetscInt               rows, cols, new_local_rows;
    7980:   PetscInt               i, bs, buf_size_idxs, buf_size_idxs_is, buf_size_vals, buf_size_vecs;
    7981:   PetscBool              ismatis, isdense, newisdense, destroy_mat;
    7982:   ISLocalToGlobalMapping l2gmap;
    7983:   PetscInt              *l2gmap_indices;
    7984:   const PetscInt        *is_indices;
    7985:   MatType                new_local_type;
    7986:   /* buffers */
    7987:   PetscInt          *ptr_idxs, *send_buffer_idxs, *recv_buffer_idxs;
    7988:   PetscInt          *ptr_idxs_is, *send_buffer_idxs_is, *recv_buffer_idxs_is;
    7989:   PetscInt          *recv_buffer_idxs_local;
    7990:   PetscScalar       *ptr_vals, *recv_buffer_vals;
    7991:   const PetscScalar *send_buffer_vals;
    7992:   PetscScalar       *ptr_vecs, *send_buffer_vecs, *recv_buffer_vecs;
    7993:   /* MPI */
    7994:   MPI_Comm     comm, comm_n;
    7995:   PetscSubcomm subcomm;
    7996:   PetscMPIInt  n_sends, n_recvs, size;
    7997:   PetscMPIInt *iflags, *ilengths_idxs, *ilengths_vals, *ilengths_idxs_is;
    7998:   PetscMPIInt *onodes, *onodes_is, *olengths_idxs, *olengths_idxs_is, *olengths_vals;
    7999:   PetscMPIInt  len, tag_idxs, tag_idxs_is, tag_vals, tag_vecs, source_dest;
    8000:   MPI_Request *send_req_idxs, *send_req_idxs_is, *send_req_vals, *send_req_vecs;
    8001:   MPI_Request *recv_req_idxs, *recv_req_idxs_is, *recv_req_vals, *recv_req_vecs;

    8003:   PetscFunctionBegin;
    8005:   PetscCall(PetscObjectTypeCompare((PetscObject)mat, MATIS, &ismatis));
    8006:   PetscCheck(ismatis, PetscObjectComm((PetscObject)mat), PETSC_ERR_SUP, "Cannot use %s on a matrix object which is not of type MATIS", PETSC_FUNCTION_NAME);
    8013:   if (nvecs) {
    8014:     PetscCheck(nvecs <= 1, PetscObjectComm((PetscObject)mat), PETSC_ERR_SUP, "Just 1 vector supported");
    8016:   }
    8017:   /* further checks */
    8018:   PetscCall(MatISGetLocalMat(mat, &local_mat));
    8019:   PetscCall(PetscObjectTypeCompare((PetscObject)local_mat, MATSEQDENSE, &isdense));
    8020:   PetscCheck(isdense, PetscObjectComm((PetscObject)mat), PETSC_ERR_SUP, "Currently cannot subassemble MATIS when local matrix type is not of type SEQDENSE");

    8022:   PetscCall(MatGetSize(local_mat, &rows, &cols));
    8023:   PetscCheck(rows == cols, PetscObjectComm((PetscObject)mat), PETSC_ERR_SUP, "Local MATIS matrices should be square");
    8024:   if (reuse && *mat_n) {
    8025:     PetscInt mrows, mcols, mnrows, mncols;
    8027:     PetscCall(PetscObjectTypeCompare((PetscObject)*mat_n, MATIS, &ismatis));
    8028:     PetscCheck(ismatis, PetscObjectComm((PetscObject)*mat_n), PETSC_ERR_SUP, "Cannot reuse a matrix which is not of type MATIS");
    8029:     PetscCall(MatGetSize(mat, &mrows, &mcols));
    8030:     PetscCall(MatGetSize(*mat_n, &mnrows, &mncols));
    8031:     PetscCheck(mrows == mnrows, PetscObjectComm((PetscObject)mat), PETSC_ERR_SUP, "Cannot reuse matrix! Wrong number of rows %" PetscInt_FMT " != %" PetscInt_FMT, mrows, mnrows);
    8032:     PetscCheck(mcols == mncols, PetscObjectComm((PetscObject)mat), PETSC_ERR_SUP, "Cannot reuse matrix! Wrong number of cols %" PetscInt_FMT " != %" PetscInt_FMT, mcols, mncols);
    8033:   }
    8034:   PetscCall(MatGetBlockSize(local_mat, &bs));

    8037:   /* prepare IS for sending if not provided */
    8038:   if (!is_sends) {
    8039:     PetscCheck(n_subdomains, PetscObjectComm((PetscObject)mat), PETSC_ERR_SUP, "You should specify either an IS or a target number of subdomains");
    8040:     PetscCall(PCBDDCMatISGetSubassemblingPattern(mat, &n_subdomains, 0, &is_sends_internal, NULL));
    8041:   } else {
    8042:     PetscCall(PetscObjectReference((PetscObject)is_sends));
    8043:     is_sends_internal = is_sends;
    8044:   }

    8046:   /* get comm */
    8047:   PetscCall(PetscObjectGetComm((PetscObject)mat, &comm));

    8049:   /* compute number of sends */
    8050:   PetscCall(ISGetLocalSize(is_sends_internal, &i));
    8051:   PetscCall(PetscMPIIntCast(i, &n_sends));

    8053:   /* compute number of receives */
    8054:   PetscCallMPI(MPI_Comm_size(comm, &size));
    8055:   PetscCall(PetscMalloc1(size, &iflags));
    8056:   PetscCall(PetscArrayzero(iflags, size));
    8057:   PetscCall(ISGetIndices(is_sends_internal, &is_indices));
    8058:   for (i = 0; i < n_sends; i++) iflags[is_indices[i]] = 1;
    8059:   PetscCall(PetscGatherNumberOfMessages(comm, iflags, NULL, &n_recvs));
    8060:   PetscCall(PetscFree(iflags));

    8062:   /* restrict comm if requested */
    8063:   subcomm     = NULL;
    8064:   destroy_mat = PETSC_FALSE;
    8065:   if (restrict_comm) {
    8066:     PetscMPIInt color, subcommsize;

    8068:     color = 0;
    8069:     if (restrict_full) {
    8070:       if (!n_recvs) color = 1; /* processes not receiving anything will not participate in new comm (full restriction) */
    8071:     } else {
    8072:       if (!n_recvs && n_sends) color = 1; /* just those processes that are sending but not receiving anything will not participate in new comm */
    8073:     }
    8074:     PetscCallMPI(MPIU_Allreduce(&color, &subcommsize, 1, MPI_INT, MPI_SUM, comm));
    8075:     subcommsize = size - subcommsize;
    8076:     /* check if reuse has been requested */
    8077:     if (reuse) {
    8078:       if (*mat_n) {
    8079:         PetscMPIInt subcommsize2;
    8080:         PetscCallMPI(MPI_Comm_size(PetscObjectComm((PetscObject)*mat_n), &subcommsize2));
    8081:         PetscCheck(subcommsize == subcommsize2, PetscObjectComm((PetscObject)*mat_n), PETSC_ERR_PLIB, "Cannot reuse matrix! wrong subcomm size %d != %d", subcommsize, subcommsize2);
    8082:         comm_n = PetscObjectComm((PetscObject)*mat_n);
    8083:       } else {
    8084:         comm_n = PETSC_COMM_SELF;
    8085:       }
    8086:     } else { /* MAT_INITIAL_MATRIX */
    8087:       PetscMPIInt rank;

    8089:       PetscCallMPI(MPI_Comm_rank(comm, &rank));
    8090:       PetscCall(PetscSubcommCreate(comm, &subcomm));
    8091:       PetscCall(PetscSubcommSetNumber(subcomm, 2));
    8092:       PetscCall(PetscSubcommSetTypeGeneral(subcomm, color, rank));
    8093:       comm_n = PetscSubcommChild(subcomm);
    8094:     }
    8095:     /* flag to destroy *mat_n if not significative */
    8096:     if (color) destroy_mat = PETSC_TRUE;
    8097:   } else {
    8098:     comm_n = comm;
    8099:   }

    8101:   /* prepare send/receive buffers */
    8102:   PetscCall(PetscMalloc1(size, &ilengths_idxs));
    8103:   PetscCall(PetscArrayzero(ilengths_idxs, size));
    8104:   PetscCall(PetscMalloc1(size, &ilengths_vals));
    8105:   PetscCall(PetscArrayzero(ilengths_vals, size));
    8106:   if (nis) PetscCall(PetscCalloc1(size, &ilengths_idxs_is));

    8108:   /* Get data from local matrices */
    8109:   PetscCheck(isdense, PetscObjectComm((PetscObject)mat), PETSC_ERR_SUP, "Subassembling of AIJ local matrices not yet implemented");
    8110:   /* TODO: See below some guidelines on how to prepare the local buffers */
    8111:   /*
    8112:        send_buffer_vals should contain the raw values of the local matrix
    8113:        send_buffer_idxs should contain:
    8114:        - MatType_PRIVATE type
    8115:        - PetscInt        size_of_l2gmap
    8116:        - PetscInt        global_row_indices[size_of_l2gmap]
    8117:        - PetscInt        all_other_info_which_is_needed_to_compute_preallocation_and_set_values
    8118:     */
    8119:   {
    8120:     ISLocalToGlobalMapping mapping;

    8122:     PetscCall(MatISGetLocalToGlobalMapping(mat, &mapping, NULL));
    8123:     PetscCall(MatDenseGetArrayRead(local_mat, &send_buffer_vals));
    8124:     PetscCall(ISLocalToGlobalMappingGetSize(mapping, &i));
    8125:     PetscCall(PetscMalloc1(i + 2, &send_buffer_idxs));
    8126:     send_buffer_idxs[0] = (PetscInt)MATDENSE_PRIVATE;
    8127:     send_buffer_idxs[1] = i;
    8128:     PetscCall(ISLocalToGlobalMappingGetIndices(mapping, (const PetscInt **)&ptr_idxs));
    8129:     PetscCall(PetscArraycpy(&send_buffer_idxs[2], ptr_idxs, i));
    8130:     PetscCall(ISLocalToGlobalMappingRestoreIndices(mapping, (const PetscInt **)&ptr_idxs));
    8131:     PetscCall(PetscMPIIntCast(i, &len));
    8132:     for (i = 0; i < n_sends; i++) {
    8133:       ilengths_vals[is_indices[i]] = len * len;
    8134:       ilengths_idxs[is_indices[i]] = len + 2;
    8135:     }
    8136:   }
    8137:   PetscCall(PetscGatherMessageLengths2(comm, n_sends, n_recvs, ilengths_idxs, ilengths_vals, &onodes, &olengths_idxs, &olengths_vals));
    8138:   /* additional is (if any) */
    8139:   if (nis) {
    8140:     PetscMPIInt psum;
    8141:     PetscInt    j;
    8142:     for (j = 0, psum = 0; j < nis; j++) {
    8143:       PetscInt plen;
    8144:       PetscCall(ISGetLocalSize(isarray[j], &plen));
    8145:       PetscCall(PetscMPIIntCast(plen, &len));
    8146:       psum += len + 1; /* indices + length */
    8147:     }
    8148:     PetscCall(PetscMalloc1(psum, &send_buffer_idxs_is));
    8149:     for (j = 0, psum = 0; j < nis; j++) {
    8150:       PetscInt        plen;
    8151:       const PetscInt *is_array_idxs;
    8152:       PetscCall(ISGetLocalSize(isarray[j], &plen));
    8153:       send_buffer_idxs_is[psum] = plen;
    8154:       PetscCall(ISGetIndices(isarray[j], &is_array_idxs));
    8155:       PetscCall(PetscArraycpy(&send_buffer_idxs_is[psum + 1], is_array_idxs, plen));
    8156:       PetscCall(ISRestoreIndices(isarray[j], &is_array_idxs));
    8157:       psum += plen + 1; /* indices + length */
    8158:     }
    8159:     for (i = 0; i < n_sends; i++) ilengths_idxs_is[is_indices[i]] = psum;
    8160:     PetscCall(PetscGatherMessageLengths(comm, n_sends, n_recvs, ilengths_idxs_is, &onodes_is, &olengths_idxs_is));
    8161:   }
    8162:   PetscCall(MatISRestoreLocalMat(mat, &local_mat));

    8164:   buf_size_idxs    = 0;
    8165:   buf_size_vals    = 0;
    8166:   buf_size_idxs_is = 0;
    8167:   buf_size_vecs    = 0;
    8168:   for (i = 0; i < n_recvs; i++) {
    8169:     buf_size_idxs += olengths_idxs[i];
    8170:     buf_size_vals += olengths_vals[i];
    8171:     if (nis) buf_size_idxs_is += olengths_idxs_is[i];
    8172:     if (nvecs) buf_size_vecs += olengths_idxs[i];
    8173:   }
    8174:   PetscCall(PetscMalloc1(buf_size_idxs, &recv_buffer_idxs));
    8175:   PetscCall(PetscMalloc1(buf_size_vals, &recv_buffer_vals));
    8176:   PetscCall(PetscMalloc1(buf_size_idxs_is, &recv_buffer_idxs_is));
    8177:   PetscCall(PetscMalloc1(buf_size_vecs, &recv_buffer_vecs));

    8179:   /* get new tags for clean communications */
    8180:   PetscCall(PetscObjectGetNewTag((PetscObject)mat, &tag_idxs));
    8181:   PetscCall(PetscObjectGetNewTag((PetscObject)mat, &tag_vals));
    8182:   PetscCall(PetscObjectGetNewTag((PetscObject)mat, &tag_idxs_is));
    8183:   PetscCall(PetscObjectGetNewTag((PetscObject)mat, &tag_vecs));

    8185:   /* allocate for requests */
    8186:   PetscCall(PetscMalloc1(n_sends, &send_req_idxs));
    8187:   PetscCall(PetscMalloc1(n_sends, &send_req_vals));
    8188:   PetscCall(PetscMalloc1(n_sends, &send_req_idxs_is));
    8189:   PetscCall(PetscMalloc1(n_sends, &send_req_vecs));
    8190:   PetscCall(PetscMalloc1(n_recvs, &recv_req_idxs));
    8191:   PetscCall(PetscMalloc1(n_recvs, &recv_req_vals));
    8192:   PetscCall(PetscMalloc1(n_recvs, &recv_req_idxs_is));
    8193:   PetscCall(PetscMalloc1(n_recvs, &recv_req_vecs));

    8195:   /* communications */
    8196:   ptr_idxs    = recv_buffer_idxs;
    8197:   ptr_vals    = recv_buffer_vals;
    8198:   ptr_idxs_is = recv_buffer_idxs_is;
    8199:   ptr_vecs    = recv_buffer_vecs;
    8200:   for (i = 0; i < n_recvs; i++) {
    8201:     PetscCallMPI(MPIU_Irecv(ptr_idxs, olengths_idxs[i], MPIU_INT, onodes[i], tag_idxs, comm, &recv_req_idxs[i]));
    8202:     PetscCallMPI(MPIU_Irecv(ptr_vals, olengths_vals[i], MPIU_SCALAR, onodes[i], tag_vals, comm, &recv_req_vals[i]));
    8203:     ptr_idxs += olengths_idxs[i];
    8204:     ptr_vals += olengths_vals[i];
    8205:     if (nis) {
    8206:       PetscCallMPI(MPIU_Irecv(ptr_idxs_is, olengths_idxs_is[i], MPIU_INT, onodes_is[i], tag_idxs_is, comm, &recv_req_idxs_is[i]));
    8207:       ptr_idxs_is += olengths_idxs_is[i];
    8208:     }
    8209:     if (nvecs) {
    8210:       PetscCallMPI(MPIU_Irecv(ptr_vecs, olengths_idxs[i] - 2, MPIU_SCALAR, onodes[i], tag_vecs, comm, &recv_req_vecs[i]));
    8211:       ptr_vecs += olengths_idxs[i] - 2;
    8212:     }
    8213:   }
    8214:   for (i = 0; i < n_sends; i++) {
    8215:     PetscCall(PetscMPIIntCast(is_indices[i], &source_dest));
    8216:     PetscCallMPI(MPIU_Isend(send_buffer_idxs, ilengths_idxs[source_dest], MPIU_INT, source_dest, tag_idxs, comm, &send_req_idxs[i]));
    8217:     PetscCallMPI(MPIU_Isend(send_buffer_vals, ilengths_vals[source_dest], MPIU_SCALAR, source_dest, tag_vals, comm, &send_req_vals[i]));
    8218:     if (nis) PetscCallMPI(MPIU_Isend(send_buffer_idxs_is, ilengths_idxs_is[source_dest], MPIU_INT, source_dest, tag_idxs_is, comm, &send_req_idxs_is[i]));
    8219:     if (nvecs) {
    8220:       PetscCall(VecGetArray(nnsp_vec[0], &send_buffer_vecs));
    8221:       PetscCallMPI(MPIU_Isend(send_buffer_vecs, ilengths_idxs[source_dest] - 2, MPIU_SCALAR, source_dest, tag_vecs, comm, &send_req_vecs[i]));
    8222:     }
    8223:   }
    8224:   PetscCall(ISRestoreIndices(is_sends_internal, &is_indices));
    8225:   PetscCall(ISDestroy(&is_sends_internal));

    8227:   /* assemble new l2g map */
    8228:   PetscCallMPI(MPI_Waitall(n_recvs, recv_req_idxs, MPI_STATUSES_IGNORE));
    8229:   ptr_idxs       = recv_buffer_idxs;
    8230:   new_local_rows = 0;
    8231:   for (i = 0; i < n_recvs; i++) {
    8232:     new_local_rows += *(ptr_idxs + 1); /* second element is the local size of the l2gmap */
    8233:     ptr_idxs += olengths_idxs[i];
    8234:   }
    8235:   PetscCall(PetscMalloc1(new_local_rows, &l2gmap_indices));
    8236:   ptr_idxs       = recv_buffer_idxs;
    8237:   new_local_rows = 0;
    8238:   for (i = 0; i < n_recvs; i++) {
    8239:     PetscCall(PetscArraycpy(&l2gmap_indices[new_local_rows], ptr_idxs + 2, *(ptr_idxs + 1)));
    8240:     new_local_rows += *(ptr_idxs + 1); /* second element is the local size of the l2gmap */
    8241:     ptr_idxs += olengths_idxs[i];
    8242:   }
    8243:   PetscCall(PetscSortRemoveDupsInt(&new_local_rows, l2gmap_indices));
    8244:   PetscCall(ISLocalToGlobalMappingCreate(comm_n, 1, new_local_rows, l2gmap_indices, PETSC_COPY_VALUES, &l2gmap));
    8245:   PetscCall(PetscFree(l2gmap_indices));

    8247:   /* infer new local matrix type from received local matrices type */
    8248:   /* currently if all local matrices are of type X, then the resulting matrix will be of type X, except for the dense case */
    8249:   /* it also assumes that if the block size is set, than it is the same among all local matrices (see checks at the beginning of the function) */
    8250:   if (n_recvs) {
    8251:     MatTypePrivate new_local_type_private = (MatTypePrivate)send_buffer_idxs[0];
    8252:     ptr_idxs                              = recv_buffer_idxs;
    8253:     for (i = 0; i < n_recvs; i++) {
    8254:       if ((PetscInt)new_local_type_private != *ptr_idxs) {
    8255:         new_local_type_private = MATAIJ_PRIVATE;
    8256:         break;
    8257:       }
    8258:       ptr_idxs += olengths_idxs[i];
    8259:     }
    8260:     switch (new_local_type_private) {
    8261:     case MATDENSE_PRIVATE:
    8262:       new_local_type = MATSEQAIJ;
    8263:       bs             = 1;
    8264:       break;
    8265:     case MATAIJ_PRIVATE:
    8266:       new_local_type = MATSEQAIJ;
    8267:       bs             = 1;
    8268:       break;
    8269:     case MATBAIJ_PRIVATE:
    8270:       new_local_type = MATSEQBAIJ;
    8271:       break;
    8272:     case MATSBAIJ_PRIVATE:
    8273:       new_local_type = MATSEQSBAIJ;
    8274:       break;
    8275:     default:
    8276:       SETERRQ(comm, PETSC_ERR_SUP, "Unsupported private type %d in %s", new_local_type_private, PETSC_FUNCTION_NAME);
    8277:     }
    8278:   } else { /* by default, new_local_type is seqaij */
    8279:     new_local_type = MATSEQAIJ;
    8280:     bs             = 1;
    8281:   }

    8283:   /* create MATIS object if needed */
    8284:   if (!reuse) {
    8285:     PetscCall(MatGetSize(mat, &rows, &cols));
    8286:     PetscCall(MatCreateIS(comm_n, bs, PETSC_DECIDE, PETSC_DECIDE, rows, cols, l2gmap, l2gmap, mat_n));
    8287:   } else {
    8288:     /* it also destroys the local matrices */
    8289:     if (*mat_n) PetscCall(MatSetLocalToGlobalMapping(*mat_n, l2gmap, l2gmap));
    8290:     else PetscCall(MatCreateIS(comm_n, bs, PETSC_DECIDE, PETSC_DECIDE, rows, cols, l2gmap, l2gmap, mat_n)); /* this is a fake object */
    8291:   }
    8292:   PetscCall(MatISGetLocalMat(*mat_n, &local_mat));
    8293:   PetscCall(MatSetType(local_mat, new_local_type));

    8295:   PetscCallMPI(MPI_Waitall(n_recvs, recv_req_vals, MPI_STATUSES_IGNORE));

    8297:   /* Global to local map of received indices */
    8298:   PetscCall(PetscMalloc1(buf_size_idxs, &recv_buffer_idxs_local)); /* needed for values insertion */
    8299:   PetscCall(ISGlobalToLocalMappingApply(l2gmap, IS_GTOLM_MASK, buf_size_idxs, recv_buffer_idxs, &i, recv_buffer_idxs_local));
    8300:   PetscCall(ISLocalToGlobalMappingDestroy(&l2gmap));

    8302:   /* restore attributes -> type of incoming data and its size */
    8303:   buf_size_idxs = 0;
    8304:   for (i = 0; i < n_recvs; i++) {
    8305:     recv_buffer_idxs_local[buf_size_idxs]     = recv_buffer_idxs[buf_size_idxs];
    8306:     recv_buffer_idxs_local[buf_size_idxs + 1] = recv_buffer_idxs[buf_size_idxs + 1];
    8307:     buf_size_idxs += olengths_idxs[i];
    8308:   }
    8309:   PetscCall(PetscFree(recv_buffer_idxs));

    8311:   /* set preallocation */
    8312:   PetscCall(PetscObjectTypeCompare((PetscObject)local_mat, MATSEQDENSE, &newisdense));
    8313:   if (!newisdense) {
    8314:     PetscInt *new_local_nnz = NULL;

    8316:     ptr_idxs = recv_buffer_idxs_local;
    8317:     if (n_recvs) PetscCall(PetscCalloc1(new_local_rows, &new_local_nnz));
    8318:     for (i = 0; i < n_recvs; i++) {
    8319:       PetscInt j;
    8320:       if (*ptr_idxs == (PetscInt)MATDENSE_PRIVATE) { /* preallocation provided for dense case only */
    8321:         for (j = 0; j < *(ptr_idxs + 1); j++) new_local_nnz[*(ptr_idxs + 2 + j)] += *(ptr_idxs + 1);
    8322:       } else {
    8323:         /* TODO */
    8324:       }
    8325:       ptr_idxs += olengths_idxs[i];
    8326:     }
    8327:     if (new_local_nnz) {
    8328:       for (i = 0; i < new_local_rows; i++) new_local_nnz[i] = PetscMin(new_local_nnz[i], new_local_rows);
    8329:       PetscCall(MatSeqAIJSetPreallocation(local_mat, 0, new_local_nnz));
    8330:       for (i = 0; i < new_local_rows; i++) new_local_nnz[i] /= bs;
    8331:       PetscCall(MatSeqBAIJSetPreallocation(local_mat, bs, 0, new_local_nnz));
    8332:       for (i = 0; i < new_local_rows; i++) new_local_nnz[i] = PetscMax(new_local_nnz[i] - i, 0);
    8333:       PetscCall(MatSeqSBAIJSetPreallocation(local_mat, bs, 0, new_local_nnz));
    8334:     } else {
    8335:       PetscCall(MatSetUp(local_mat));
    8336:     }
    8337:     PetscCall(PetscFree(new_local_nnz));
    8338:   } else {
    8339:     PetscCall(MatSetUp(local_mat));
    8340:   }

    8342:   /* set values */
    8343:   ptr_vals = recv_buffer_vals;
    8344:   ptr_idxs = recv_buffer_idxs_local;
    8345:   for (i = 0; i < n_recvs; i++) {
    8346:     if (*ptr_idxs == (PetscInt)MATDENSE_PRIVATE) { /* values insertion provided for dense case only */
    8347:       PetscCall(MatSetOption(local_mat, MAT_ROW_ORIENTED, PETSC_FALSE));
    8348:       PetscCall(MatSetValues(local_mat, *(ptr_idxs + 1), ptr_idxs + 2, *(ptr_idxs + 1), ptr_idxs + 2, ptr_vals, ADD_VALUES));
    8349:       PetscCall(MatAssemblyBegin(local_mat, MAT_FLUSH_ASSEMBLY));
    8350:       PetscCall(MatAssemblyEnd(local_mat, MAT_FLUSH_ASSEMBLY));
    8351:       PetscCall(MatSetOption(local_mat, MAT_ROW_ORIENTED, PETSC_TRUE));
    8352:     } else {
    8353:       /* TODO */
    8354:     }
    8355:     ptr_idxs += olengths_idxs[i];
    8356:     ptr_vals += olengths_vals[i];
    8357:   }
    8358:   PetscCall(MatAssemblyBegin(local_mat, MAT_FINAL_ASSEMBLY));
    8359:   PetscCall(MatAssemblyEnd(local_mat, MAT_FINAL_ASSEMBLY));
    8360:   PetscCall(MatISRestoreLocalMat(*mat_n, &local_mat));
    8361:   PetscCall(MatAssemblyBegin(*mat_n, MAT_FINAL_ASSEMBLY));
    8362:   PetscCall(MatAssemblyEnd(*mat_n, MAT_FINAL_ASSEMBLY));
    8363:   PetscCall(PetscFree(recv_buffer_vals));

    8365: #if 0
    8366:   if (!restrict_comm) { /* check */
    8367:     Vec       lvec,rvec;
    8368:     PetscReal infty_error;

    8370:     PetscCall(MatCreateVecs(mat,&rvec,&lvec));
    8371:     PetscCall(VecSetRandom(rvec,NULL));
    8372:     PetscCall(MatMult(mat,rvec,lvec));
    8373:     PetscCall(VecScale(lvec,-1.0));
    8374:     PetscCall(MatMultAdd(*mat_n,rvec,lvec,lvec));
    8375:     PetscCall(VecNorm(lvec,NORM_INFINITY,&infty_error));
    8376:     PetscCall(PetscPrintf(PetscObjectComm((PetscObject)mat),"Infinity error subassembling %1.6e\n",infty_error));
    8377:     PetscCall(VecDestroy(&rvec));
    8378:     PetscCall(VecDestroy(&lvec));
    8379:   }
    8380: #endif

    8382:   /* assemble new additional is (if any) */
    8383:   if (nis) {
    8384:     PetscInt **temp_idxs, *count_is, j, psum;

    8386:     PetscCallMPI(MPI_Waitall(n_recvs, recv_req_idxs_is, MPI_STATUSES_IGNORE));
    8387:     PetscCall(PetscCalloc1(nis, &count_is));
    8388:     ptr_idxs = recv_buffer_idxs_is;
    8389:     psum     = 0;
    8390:     for (i = 0; i < n_recvs; i++) {
    8391:       for (j = 0; j < nis; j++) {
    8392:         PetscInt plen = *(ptr_idxs); /* first element is the local size of IS's indices */
    8393:         count_is[j] += plen;         /* increment counting of buffer for j-th IS */
    8394:         psum += plen;
    8395:         ptr_idxs += plen + 1; /* shift pointer to received data */
    8396:       }
    8397:     }
    8398:     PetscCall(PetscMalloc1(nis, &temp_idxs));
    8399:     PetscCall(PetscMalloc1(psum, &temp_idxs[0]));
    8400:     for (i = 1; i < nis; i++) temp_idxs[i] = PetscSafePointerPlusOffset(temp_idxs[i - 1], count_is[i - 1]);
    8401:     PetscCall(PetscArrayzero(count_is, nis));
    8402:     ptr_idxs = recv_buffer_idxs_is;
    8403:     for (i = 0; i < n_recvs; i++) {
    8404:       for (j = 0; j < nis; j++) {
    8405:         PetscInt plen = *(ptr_idxs); /* first element is the local size of IS's indices */
    8406:         PetscCall(PetscArraycpy(&temp_idxs[j][count_is[j]], ptr_idxs + 1, plen));
    8407:         count_is[j] += plen;  /* increment starting point of buffer for j-th IS */
    8408:         ptr_idxs += plen + 1; /* shift pointer to received data */
    8409:       }
    8410:     }
    8411:     for (i = 0; i < nis; i++) {
    8412:       PetscCall(ISDestroy(&isarray[i]));
    8413:       PetscCall(PetscSortRemoveDupsInt(&count_is[i], temp_idxs[i]));
    8414:       PetscCall(ISCreateGeneral(comm_n, count_is[i], temp_idxs[i], PETSC_COPY_VALUES, &isarray[i]));
    8415:     }
    8416:     PetscCall(PetscFree(count_is));
    8417:     PetscCall(PetscFree(temp_idxs[0]));
    8418:     PetscCall(PetscFree(temp_idxs));
    8419:   }
    8420:   /* free workspace */
    8421:   PetscCall(PetscFree(recv_buffer_idxs_is));
    8422:   PetscCallMPI(MPI_Waitall(n_sends, send_req_idxs, MPI_STATUSES_IGNORE));
    8423:   PetscCall(PetscFree(send_buffer_idxs));
    8424:   PetscCallMPI(MPI_Waitall(n_sends, send_req_vals, MPI_STATUSES_IGNORE));
    8425:   if (isdense) {
    8426:     PetscCall(MatISGetLocalMat(mat, &local_mat));
    8427:     PetscCall(MatDenseRestoreArrayRead(local_mat, &send_buffer_vals));
    8428:     PetscCall(MatISRestoreLocalMat(mat, &local_mat));
    8429:   } else {
    8430:     /* PetscCall(PetscFree(send_buffer_vals)); */
    8431:   }
    8432:   if (nis) {
    8433:     PetscCallMPI(MPI_Waitall(n_sends, send_req_idxs_is, MPI_STATUSES_IGNORE));
    8434:     PetscCall(PetscFree(send_buffer_idxs_is));
    8435:   }

    8437:   if (nvecs) {
    8438:     PetscCallMPI(MPI_Waitall(n_recvs, recv_req_vecs, MPI_STATUSES_IGNORE));
    8439:     PetscCallMPI(MPI_Waitall(n_sends, send_req_vecs, MPI_STATUSES_IGNORE));
    8440:     PetscCall(VecRestoreArray(nnsp_vec[0], &send_buffer_vecs));
    8441:     PetscCall(VecDestroy(&nnsp_vec[0]));
    8442:     PetscCall(VecCreate(comm_n, &nnsp_vec[0]));
    8443:     PetscCall(VecSetSizes(nnsp_vec[0], new_local_rows, PETSC_DECIDE));
    8444:     PetscCall(VecSetType(nnsp_vec[0], VECSTANDARD));
    8445:     /* set values */
    8446:     ptr_vals = recv_buffer_vecs;
    8447:     ptr_idxs = recv_buffer_idxs_local;
    8448:     PetscCall(VecGetArray(nnsp_vec[0], &send_buffer_vecs));
    8449:     for (i = 0; i < n_recvs; i++) {
    8450:       PetscInt j;
    8451:       for (j = 0; j < *(ptr_idxs + 1); j++) send_buffer_vecs[*(ptr_idxs + 2 + j)] += *(ptr_vals + j);
    8452:       ptr_idxs += olengths_idxs[i];
    8453:       ptr_vals += olengths_idxs[i] - 2;
    8454:     }
    8455:     PetscCall(VecRestoreArray(nnsp_vec[0], &send_buffer_vecs));
    8456:     PetscCall(VecAssemblyBegin(nnsp_vec[0]));
    8457:     PetscCall(VecAssemblyEnd(nnsp_vec[0]));
    8458:   }

    8460:   PetscCall(PetscFree(recv_buffer_vecs));
    8461:   PetscCall(PetscFree(recv_buffer_idxs_local));
    8462:   PetscCall(PetscFree(recv_req_idxs));
    8463:   PetscCall(PetscFree(recv_req_vals));
    8464:   PetscCall(PetscFree(recv_req_vecs));
    8465:   PetscCall(PetscFree(recv_req_idxs_is));
    8466:   PetscCall(PetscFree(send_req_idxs));
    8467:   PetscCall(PetscFree(send_req_vals));
    8468:   PetscCall(PetscFree(send_req_vecs));
    8469:   PetscCall(PetscFree(send_req_idxs_is));
    8470:   PetscCall(PetscFree(ilengths_vals));
    8471:   PetscCall(PetscFree(ilengths_idxs));
    8472:   PetscCall(PetscFree(olengths_vals));
    8473:   PetscCall(PetscFree(olengths_idxs));
    8474:   PetscCall(PetscFree(onodes));
    8475:   if (nis) {
    8476:     PetscCall(PetscFree(ilengths_idxs_is));
    8477:     PetscCall(PetscFree(olengths_idxs_is));
    8478:     PetscCall(PetscFree(onodes_is));
    8479:   }
    8480:   PetscCall(PetscSubcommDestroy(&subcomm));
    8481:   if (destroy_mat) { /* destroy mat is true only if restrict comm is true and process will not participate */
    8482:     PetscCall(MatDestroy(mat_n));
    8483:     for (i = 0; i < nis; i++) PetscCall(ISDestroy(&isarray[i]));
    8484:     if (nvecs) { /* need to match VecDestroy nnsp_vec called in the other code path */
    8485:       PetscCall(VecDestroy(&nnsp_vec[0]));
    8486:     }
    8487:     *mat_n = NULL;
    8488:   }
    8489:   PetscFunctionReturn(PETSC_SUCCESS);
    8490: }

    8492: /* temporary hack into ksp private data structure */
    8493: #include <petsc/private/kspimpl.h>

    8495: PetscErrorCode PCBDDCSetUpCoarseSolver(PC pc, Mat coarse_submat)
    8496: {
    8497:   PC_BDDC               *pcbddc = (PC_BDDC *)pc->data;
    8498:   PC_IS                 *pcis   = (PC_IS *)pc->data;
    8499:   PCBDDCGraph            graph  = pcbddc->mat_graph;
    8500:   Mat                    coarse_mat, coarse_mat_is;
    8501:   Mat                    coarsedivudotp = NULL;
    8502:   Mat                    coarseG, t_coarse_mat_is;
    8503:   MatNullSpace           CoarseNullSpace = NULL;
    8504:   ISLocalToGlobalMapping coarse_islg;
    8505:   IS                     coarse_is, *isarray, corners;
    8506:   PetscInt               i, im_active = -1, active_procs = -1;
    8507:   PetscInt               nis, nisdofs, nisneu, nisvert;
    8508:   PetscInt               coarse_eqs_per_proc, coarsening_ratio;
    8509:   PC                     pc_temp;
    8510:   PCType                 coarse_pc_type;
    8511:   KSPType                coarse_ksp_type;
    8512:   PetscBool              multilevel_requested, multilevel_allowed;
    8513:   PetscBool              coarse_reuse, multi_element = graph->multi_element;
    8514:   PetscInt               ncoarse, nedcfield;
    8515:   PetscBool              compute_vecs = PETSC_FALSE;
    8516:   PetscScalar           *array;
    8517:   MatReuse               coarse_mat_reuse;
    8518:   PetscBool              restr, full_restr, have_void;
    8519:   PetscMPIInt            size;

    8521:   PetscFunctionBegin;
    8522:   PetscCall(PetscLogEventBegin(PC_BDDC_CoarseSetUp[pcbddc->current_level], pc, 0, 0, 0));
    8523:   /* Assign global numbering to coarse dofs */
    8524:   if (pcbddc->new_primal_space || pcbddc->coarse_size == -1) { /* a new primal space is present or it is the first initialization, so recompute global numbering */
    8525:     PetscInt ocoarse_size;
    8526:     compute_vecs = PETSC_TRUE;

    8528:     pcbddc->new_primal_space = PETSC_TRUE;
    8529:     ocoarse_size             = pcbddc->coarse_size;
    8530:     PetscCall(PetscFree(pcbddc->global_primal_indices));
    8531:     PetscCall(PCBDDCComputePrimalNumbering(pc, &pcbddc->coarse_size, &pcbddc->global_primal_indices));
    8532:     /* see if we can avoid some work */
    8533:     if (pcbddc->coarse_ksp) { /* coarse ksp has already been created */
    8534:       /* if the coarse size is different or we are using adaptive selection, better to not reuse the coarse matrix */
    8535:       if (ocoarse_size != pcbddc->coarse_size || pcbddc->adaptive_selection) {
    8536:         PetscCall(KSPReset(pcbddc->coarse_ksp));
    8537:         coarse_reuse = PETSC_FALSE;
    8538:       } else { /* we can safely reuse already computed coarse matrix */
    8539:         coarse_reuse = PETSC_TRUE;
    8540:       }
    8541:     } else { /* there's no coarse ksp, so we need to create the coarse matrix too */
    8542:       coarse_reuse = PETSC_FALSE;
    8543:     }
    8544:     /* reset any subassembling information */
    8545:     if (!coarse_reuse || pcbddc->recompute_topography) PetscCall(ISDestroy(&pcbddc->coarse_subassembling));
    8546:   } else { /* primal space is unchanged, so we can reuse coarse matrix */
    8547:     coarse_reuse = PETSC_TRUE;
    8548:   }
    8549:   if (coarse_reuse && pcbddc->coarse_ksp) {
    8550:     PetscCall(KSPGetOperators(pcbddc->coarse_ksp, &coarse_mat, NULL));
    8551:     PetscCall(PetscObjectReference((PetscObject)coarse_mat));
    8552:     coarse_mat_reuse = MAT_REUSE_MATRIX;
    8553:   } else {
    8554:     coarse_mat       = NULL;
    8555:     coarse_mat_reuse = MAT_INITIAL_MATRIX;
    8556:   }

    8558:   /* creates temporary l2gmap and IS for coarse indexes */
    8559:   PetscCall(ISCreateGeneral(PetscObjectComm((PetscObject)pc), pcbddc->local_primal_size, pcbddc->global_primal_indices, PETSC_COPY_VALUES, &coarse_is));
    8560:   PetscCall(ISLocalToGlobalMappingCreateIS(coarse_is, &coarse_islg));

    8562:   /* creates temporary MATIS object for coarse matrix */
    8563:   PetscCall(MatCreate(PetscObjectComm((PetscObject)pc), &t_coarse_mat_is));
    8564:   PetscCall(MatSetType(t_coarse_mat_is, MATIS));
    8565:   PetscCall(MatSetSizes(t_coarse_mat_is, PETSC_DECIDE, PETSC_DECIDE, pcbddc->coarse_size, pcbddc->coarse_size));
    8566:   PetscCall(MatISSetAllowRepeated(t_coarse_mat_is, multi_element));
    8567:   PetscCall(MatSetLocalToGlobalMapping(t_coarse_mat_is, coarse_islg, coarse_islg));
    8568:   PetscCall(MatISSetLocalMat(t_coarse_mat_is, coarse_submat));
    8569:   PetscCall(MatAssemblyBegin(t_coarse_mat_is, MAT_FINAL_ASSEMBLY));
    8570:   PetscCall(MatAssemblyEnd(t_coarse_mat_is, MAT_FINAL_ASSEMBLY));
    8571:   PetscCall(MatViewFromOptions(t_coarse_mat_is, (PetscObject)pc, "-pc_bddc_coarse_mat_is_view"));

    8573:   /* count "active" (i.e. with positive local size) and "void" processes */
    8574:   im_active = !!pcis->n;
    8575:   PetscCallMPI(MPIU_Allreduce(&im_active, &active_procs, 1, MPIU_INT, MPI_SUM, PetscObjectComm((PetscObject)pc)));

    8577:   /* determine number of processes partecipating to coarse solver and compute subassembling pattern */
    8578:   /* restr : whether we want to exclude senders (which are not receivers) from the subassembling pattern */
    8579:   /* full_restr : just use the receivers from the subassembling pattern */
    8580:   PetscCallMPI(MPI_Comm_size(PetscObjectComm((PetscObject)pc), &size));
    8581:   coarse_mat_is        = NULL;
    8582:   multilevel_allowed   = PETSC_FALSE;
    8583:   multilevel_requested = PETSC_FALSE;
    8584:   coarse_eqs_per_proc  = PetscMin(PetscMax(pcbddc->coarse_size, 1), pcbddc->coarse_eqs_per_proc);
    8585:   if (coarse_eqs_per_proc < 0 || size == 1) coarse_eqs_per_proc = PetscMax(pcbddc->coarse_size, 1);
    8586:   if (pcbddc->current_level < pcbddc->max_levels) multilevel_requested = PETSC_TRUE;
    8587:   if (pcbddc->coarse_size <= pcbddc->coarse_eqs_limit) multilevel_requested = PETSC_FALSE;
    8588:   coarsening_ratio = multi_element ? 1 : pcbddc->coarsening_ratio;
    8589:   if (multilevel_requested) {
    8590:     ncoarse    = active_procs / coarsening_ratio;
    8591:     restr      = PETSC_FALSE;
    8592:     full_restr = PETSC_FALSE;
    8593:   } else {
    8594:     ncoarse    = pcbddc->coarse_size / coarse_eqs_per_proc + !!(pcbddc->coarse_size % coarse_eqs_per_proc);
    8595:     restr      = PETSC_TRUE;
    8596:     full_restr = PETSC_TRUE;
    8597:   }
    8598:   if (!pcbddc->coarse_size || (size == 1 && !multi_element)) multilevel_allowed = multilevel_requested = restr = full_restr = PETSC_FALSE;
    8599:   ncoarse = PetscMax(1, ncoarse);
    8600:   if (!pcbddc->coarse_subassembling) {
    8601:     if (coarsening_ratio > 1) {
    8602:       if (multilevel_requested) {
    8603:         PetscCall(PCBDDCMatISGetSubassemblingPattern(pc->pmat, &ncoarse, pcbddc->coarse_adj_red, &pcbddc->coarse_subassembling, &have_void));
    8604:       } else {
    8605:         PetscCall(PCBDDCMatISGetSubassemblingPattern(t_coarse_mat_is, &ncoarse, pcbddc->coarse_adj_red, &pcbddc->coarse_subassembling, &have_void));
    8606:       }
    8607:     } else {
    8608:       PetscMPIInt rank;

    8610:       PetscCallMPI(MPI_Comm_rank(PetscObjectComm((PetscObject)pc), &rank));
    8611:       have_void = (active_procs == size) ? PETSC_FALSE : PETSC_TRUE;
    8612:       PetscCall(ISCreateStride(PetscObjectComm((PetscObject)pc), 1, rank, 1, &pcbddc->coarse_subassembling));
    8613:       PetscCall(PetscObjectSetName((PetscObject)pcbddc->coarse_subassembling, "default subassembling"));
    8614:     }
    8615:   } else { /* if a subassembling pattern exists, then we can reuse the coarse ksp and compute the number of process involved */
    8616:     PetscInt psum;
    8617:     if (pcbddc->coarse_ksp) psum = 1;
    8618:     else psum = 0;
    8619:     PetscCallMPI(MPIU_Allreduce(&psum, &ncoarse, 1, MPIU_INT, MPI_SUM, PetscObjectComm((PetscObject)pc)));
    8620:     have_void = ncoarse < size ? PETSC_TRUE : PETSC_FALSE;
    8621:   }
    8622:   /* determine if we can go multilevel */
    8623:   if (multilevel_requested) {
    8624:     if (ncoarse > 1) multilevel_allowed = PETSC_TRUE; /* found enough processes */
    8625:     else restr = full_restr = PETSC_TRUE;             /* 1 subdomain, use a direct solver */
    8626:   }
    8627:   if (multilevel_allowed && have_void) restr = PETSC_TRUE;

    8629:   /* dump subassembling pattern */
    8630:   if (pcbddc->dbg_flag && multilevel_allowed) PetscCall(ISView(pcbddc->coarse_subassembling, pcbddc->dbg_viewer));
    8631:   /* compute dofs splitting and neumann boundaries for coarse dofs */
    8632:   nedcfield = -1;
    8633:   corners   = NULL;
    8634:   if (multilevel_allowed && !coarse_reuse && (pcbddc->n_ISForDofsLocal || pcbddc->NeumannBoundariesLocal || pcbddc->nedclocal || pcbddc->corner_selected)) { /* protects from unneeded computations */
    8635:     PetscInt              *tidxs, *tidxs2, nout, tsize, i;
    8636:     const PetscInt        *idxs;
    8637:     ISLocalToGlobalMapping tmap;

    8639:     /* create map between primal indices (in local representative ordering) and local primal numbering */
    8640:     PetscCall(ISLocalToGlobalMappingCreate(PETSC_COMM_SELF, 1, pcbddc->local_primal_size, pcbddc->primal_indices_local_idxs, PETSC_COPY_VALUES, &tmap));
    8641:     /* allocate space for temporary storage */
    8642:     PetscCall(PetscMalloc1(pcbddc->local_primal_size, &tidxs));
    8643:     PetscCall(PetscMalloc1(pcbddc->local_primal_size, &tidxs2));
    8644:     /* allocate for IS array */
    8645:     nisdofs = pcbddc->n_ISForDofsLocal;
    8646:     if (pcbddc->nedclocal) {
    8647:       if (pcbddc->nedfield > -1) {
    8648:         nedcfield = pcbddc->nedfield;
    8649:       } else {
    8650:         nedcfield = 0;
    8651:         PetscCheck(!nisdofs, PetscObjectComm((PetscObject)pc), PETSC_ERR_PLIB, "This should not happen (%" PetscInt_FMT ")", nisdofs);
    8652:         nisdofs = 1;
    8653:       }
    8654:     }
    8655:     nisneu  = !!pcbddc->NeumannBoundariesLocal;
    8656:     nisvert = 0; /* nisvert is not used */
    8657:     nis     = nisdofs + nisneu + nisvert;
    8658:     PetscCall(PetscMalloc1(nis, &isarray));
    8659:     /* dofs splitting */
    8660:     for (i = 0; i < nisdofs; i++) {
    8661:       /* PetscCall(ISView(pcbddc->ISForDofsLocal[i],0)); */
    8662:       if (nedcfield != i) {
    8663:         PetscCall(ISGetLocalSize(pcbddc->ISForDofsLocal[i], &tsize));
    8664:         PetscCall(ISGetIndices(pcbddc->ISForDofsLocal[i], &idxs));
    8665:         PetscCall(ISGlobalToLocalMappingApply(tmap, IS_GTOLM_DROP, tsize, idxs, &nout, tidxs));
    8666:         PetscCall(ISRestoreIndices(pcbddc->ISForDofsLocal[i], &idxs));
    8667:       } else {
    8668:         PetscCall(ISGetLocalSize(pcbddc->nedclocal, &tsize));
    8669:         PetscCall(ISGetIndices(pcbddc->nedclocal, &idxs));
    8670:         PetscCall(ISGlobalToLocalMappingApply(tmap, IS_GTOLM_DROP, tsize, idxs, &nout, tidxs));
    8671:         PetscCheck(tsize == nout, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Failed when mapping coarse nedelec field! %" PetscInt_FMT " != %" PetscInt_FMT, tsize, nout);
    8672:         PetscCall(ISRestoreIndices(pcbddc->nedclocal, &idxs));
    8673:       }
    8674:       PetscCall(ISLocalToGlobalMappingApply(coarse_islg, nout, tidxs, tidxs2));
    8675:       PetscCall(ISCreateGeneral(PetscObjectComm((PetscObject)pc), nout, tidxs2, PETSC_COPY_VALUES, &isarray[i]));
    8676:       /* PetscCall(ISView(isarray[i],0)); */
    8677:     }
    8678:     /* neumann boundaries */
    8679:     if (pcbddc->NeumannBoundariesLocal) {
    8680:       /* PetscCall(ISView(pcbddc->NeumannBoundariesLocal,0)); */
    8681:       PetscCall(ISGetLocalSize(pcbddc->NeumannBoundariesLocal, &tsize));
    8682:       PetscCall(ISGetIndices(pcbddc->NeumannBoundariesLocal, &idxs));
    8683:       PetscCall(ISGlobalToLocalMappingApply(tmap, IS_GTOLM_DROP, tsize, idxs, &nout, tidxs));
    8684:       PetscCall(ISRestoreIndices(pcbddc->NeumannBoundariesLocal, &idxs));
    8685:       PetscCall(ISLocalToGlobalMappingApply(coarse_islg, nout, tidxs, tidxs2));
    8686:       PetscCall(ISCreateGeneral(PetscObjectComm((PetscObject)pc), nout, tidxs2, PETSC_COPY_VALUES, &isarray[nisdofs]));
    8687:       /* PetscCall(ISView(isarray[nisdofs],0)); */
    8688:     }
    8689:     /* coordinates */
    8690:     if (pcbddc->corner_selected) {
    8691:       PetscCall(PCBDDCGraphGetCandidatesIS(pcbddc->mat_graph, NULL, NULL, NULL, NULL, &corners));
    8692:       PetscCall(ISGetLocalSize(corners, &tsize));
    8693:       PetscCall(ISGetIndices(corners, &idxs));
    8694:       PetscCall(ISGlobalToLocalMappingApply(tmap, IS_GTOLM_DROP, tsize, idxs, &nout, tidxs));
    8695:       PetscCheck(tsize == nout, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Failed when mapping corners! %" PetscInt_FMT " != %" PetscInt_FMT, tsize, nout);
    8696:       PetscCall(ISRestoreIndices(corners, &idxs));
    8697:       PetscCall(PCBDDCGraphRestoreCandidatesIS(pcbddc->mat_graph, NULL, NULL, NULL, NULL, &corners));
    8698:       PetscCall(ISLocalToGlobalMappingApply(coarse_islg, nout, tidxs, tidxs2));
    8699:       PetscCall(ISCreateGeneral(PetscObjectComm((PetscObject)pc), nout, tidxs2, PETSC_COPY_VALUES, &corners));
    8700:     }
    8701:     PetscCall(PetscFree(tidxs));
    8702:     PetscCall(PetscFree(tidxs2));
    8703:     PetscCall(ISLocalToGlobalMappingDestroy(&tmap));
    8704:   } else {
    8705:     nis     = 0;
    8706:     nisdofs = 0;
    8707:     nisneu  = 0;
    8708:     nisvert = 0;
    8709:     isarray = NULL;
    8710:   }
    8711:   /* destroy no longer needed map */
    8712:   PetscCall(ISLocalToGlobalMappingDestroy(&coarse_islg));

    8714:   /* subassemble */
    8715:   if (multilevel_allowed) {
    8716:     Vec       vp[1];
    8717:     PetscInt  nvecs = 0;
    8718:     PetscBool reuse;

    8720:     vp[0] = NULL;
    8721:     /* XXX HDIV also */
    8722:     if (pcbddc->benign_have_null) { /* propagate no-net-flux quadrature to coarser level */
    8723:       PetscCall(VecCreate(PetscObjectComm((PetscObject)pc), &vp[0]));
    8724:       PetscCall(VecSetSizes(vp[0], pcbddc->local_primal_size, PETSC_DECIDE));
    8725:       PetscCall(VecSetType(vp[0], VECSTANDARD));
    8726:       nvecs = 1;

    8728:       if (pcbddc->divudotp) {
    8729:         Mat      B, loc_divudotp;
    8730:         Vec      v, p;
    8731:         IS       dummy;
    8732:         PetscInt np;

    8734:         PetscCall(MatISGetLocalMat(pcbddc->divudotp, &loc_divudotp));
    8735:         PetscCall(MatGetSize(loc_divudotp, &np, NULL));
    8736:         PetscCall(ISCreateStride(PETSC_COMM_SELF, np, 0, 1, &dummy));
    8737:         PetscCall(MatCreateSubMatrix(loc_divudotp, dummy, pcis->is_B_local, MAT_INITIAL_MATRIX, &B));
    8738:         PetscCall(MatCreateVecs(B, &v, &p));
    8739:         PetscCall(VecSet(p, 1.));
    8740:         PetscCall(MatMultTranspose(B, p, v));
    8741:         PetscCall(VecDestroy(&p));
    8742:         PetscCall(MatDestroy(&B));
    8743:         PetscCall(VecGetArray(vp[0], &array));
    8744:         PetscCall(VecPlaceArray(pcbddc->vec1_P, array));
    8745:         PetscCall(MatMultTranspose(pcbddc->coarse_phi_B, v, pcbddc->vec1_P));
    8746:         PetscCall(VecResetArray(pcbddc->vec1_P));
    8747:         PetscCall(VecRestoreArray(vp[0], &array));
    8748:         PetscCall(ISDestroy(&dummy));
    8749:         PetscCall(VecDestroy(&v));
    8750:       }
    8751:     }
    8752:     if (coarse_mat) reuse = PETSC_TRUE;
    8753:     else reuse = PETSC_FALSE;
    8754:     if (multi_element) {
    8755:       PetscCall(PetscObjectReference((PetscObject)t_coarse_mat_is));
    8756:       coarse_mat_is = t_coarse_mat_is;
    8757:     } else {
    8758:       PetscCallMPI(MPIU_Allreduce(MPI_IN_PLACE, &reuse, 1, MPI_C_BOOL, MPI_LOR, PetscObjectComm((PetscObject)pc)));
    8759:       if (reuse) {
    8760:         PetscCall(PCBDDCMatISSubassemble(t_coarse_mat_is, pcbddc->coarse_subassembling, 0, restr, full_restr, PETSC_TRUE, &coarse_mat, nis, isarray, nvecs, vp));
    8761:       } else {
    8762:         PetscCall(PCBDDCMatISSubassemble(t_coarse_mat_is, pcbddc->coarse_subassembling, 0, restr, full_restr, PETSC_FALSE, &coarse_mat_is, nis, isarray, nvecs, vp));
    8763:       }
    8764:       if (vp[0]) { /* vp[0] could have been placed on a different set of processes */
    8765:         PetscScalar       *arraym;
    8766:         const PetscScalar *arrayv;
    8767:         PetscInt           nl;
    8768:         PetscCall(VecGetLocalSize(vp[0], &nl));
    8769:         PetscCall(MatCreateSeqDense(PETSC_COMM_SELF, 1, nl, NULL, &coarsedivudotp));
    8770:         PetscCall(MatDenseGetArray(coarsedivudotp, &arraym));
    8771:         PetscCall(VecGetArrayRead(vp[0], &arrayv));
    8772:         PetscCall(PetscArraycpy(arraym, arrayv, nl));
    8773:         PetscCall(VecRestoreArrayRead(vp[0], &arrayv));
    8774:         PetscCall(MatDenseRestoreArray(coarsedivudotp, &arraym));
    8775:         PetscCall(VecDestroy(&vp[0]));
    8776:       } else {
    8777:         PetscCall(MatCreateSeqAIJ(PETSC_COMM_SELF, 0, 0, 1, NULL, &coarsedivudotp));
    8778:       }
    8779:     }
    8780:   } else {
    8781:     PetscBool default_sub;

    8783:     PetscCall(PetscStrcmp(((PetscObject)pcbddc->coarse_subassembling)->name, "default subassembling", &default_sub));
    8784:     if (!default_sub) PetscCall(PCBDDCMatISSubassemble(t_coarse_mat_is, pcbddc->coarse_subassembling, 0, restr, full_restr, PETSC_FALSE, &coarse_mat_is, 0, NULL, 0, NULL));
    8785:     else {
    8786:       PetscCall(PetscObjectReference((PetscObject)t_coarse_mat_is));
    8787:       coarse_mat_is = t_coarse_mat_is;
    8788:     }
    8789:   }
    8790:   if (coarse_mat_is || coarse_mat) {
    8791:     if (!multilevel_allowed) {
    8792:       PetscCall(MatConvert(coarse_mat_is, MATAIJ, coarse_mat_reuse, &coarse_mat));
    8793:     } else {
    8794:       /* if this matrix is present, it means we are not reusing the coarse matrix */
    8795:       if (coarse_mat_is) {
    8796:         PetscCheck(!coarse_mat, PetscObjectComm((PetscObject)coarse_mat_is), PETSC_ERR_PLIB, "This should not happen");
    8797:         PetscCall(PetscObjectReference((PetscObject)coarse_mat_is));
    8798:         coarse_mat = coarse_mat_is;
    8799:       }
    8800:     }
    8801:   }
    8802:   PetscCall(MatDestroy(&t_coarse_mat_is));
    8803:   PetscCall(MatDestroy(&coarse_mat_is));

    8805:   /* create local to global scatters for coarse problem */
    8806:   if (compute_vecs) {
    8807:     PetscInt lrows;
    8808:     PetscCall(VecDestroy(&pcbddc->coarse_vec));
    8809:     if (coarse_mat) {
    8810:       PetscCall(MatGetLocalSize(coarse_mat, &lrows, NULL));
    8811:     } else {
    8812:       lrows = 0;
    8813:     }
    8814:     PetscCall(VecCreate(PetscObjectComm((PetscObject)pc), &pcbddc->coarse_vec));
    8815:     PetscCall(VecSetSizes(pcbddc->coarse_vec, lrows, PETSC_DECIDE));
    8816:     PetscCall(VecSetType(pcbddc->coarse_vec, coarse_mat ? coarse_mat->defaultvectype : VECSTANDARD));
    8817:     PetscCall(VecScatterDestroy(&pcbddc->coarse_loc_to_glob));
    8818:     PetscCall(VecScatterCreate(pcbddc->vec1_P, NULL, pcbddc->coarse_vec, coarse_is, &pcbddc->coarse_loc_to_glob));
    8819:   }
    8820:   PetscCall(ISDestroy(&coarse_is));

    8822:   /* set defaults for coarse KSP and PC */
    8823:   if (multilevel_allowed) {
    8824:     coarse_ksp_type = KSPRICHARDSON;
    8825:     coarse_pc_type  = PCBDDC;
    8826:   } else {
    8827:     coarse_ksp_type = KSPPREONLY;
    8828:     coarse_pc_type  = PCREDUNDANT;
    8829:   }

    8831:   /* print some info if requested */
    8832:   if (pcbddc->dbg_flag) {
    8833:     if (!multilevel_allowed) {
    8834:       PetscCall(PetscViewerASCIIPrintf(pcbddc->dbg_viewer, "--------------------------------------------------\n"));
    8835:       if (multilevel_requested) {
    8836:         PetscCall(PetscViewerASCIIPrintf(pcbddc->dbg_viewer, "Not enough active processes on level %" PetscInt_FMT " (active processes %" PetscInt_FMT ", coarsening ratio %" PetscInt_FMT ")\n", pcbddc->current_level, active_procs, coarsening_ratio));
    8837:       } else if (pcbddc->max_levels) {
    8838:         PetscCall(PetscViewerASCIIPrintf(pcbddc->dbg_viewer, "Maximum number of requested levels reached (%" PetscInt_FMT ")\n", pcbddc->max_levels));
    8839:       }
    8840:       PetscCall(PetscViewerFlush(pcbddc->dbg_viewer));
    8841:     }
    8842:   }

    8844:   /* communicate coarse discrete gradient */
    8845:   coarseG = NULL;
    8846:   if (pcbddc->nedcG && multilevel_allowed) {
    8847:     MPI_Comm ccomm;
    8848:     if (coarse_mat) {
    8849:       ccomm = PetscObjectComm((PetscObject)coarse_mat);
    8850:     } else {
    8851:       ccomm = MPI_COMM_NULL;
    8852:     }
    8853:     PetscCall(MatMPIAIJRestrict(pcbddc->nedcG, ccomm, &coarseG));
    8854:   }

    8856:   /* create the coarse KSP object only once with defaults */
    8857:   if (coarse_mat) {
    8858:     PetscBool   isredundant, isbddc, force, valid;
    8859:     PetscViewer dbg_viewer = NULL;
    8860:     PetscBool   isset, issym, isher, isspd;

    8862:     if (pcbddc->dbg_flag) {
    8863:       dbg_viewer = PETSC_VIEWER_STDOUT_(PetscObjectComm((PetscObject)coarse_mat));
    8864:       PetscCall(PetscViewerASCIIAddTab(dbg_viewer, 2 * pcbddc->current_level));
    8865:     }
    8866:     if (!pcbddc->coarse_ksp) {
    8867:       char   prefix[256], str_level[16];
    8868:       size_t len;

    8870:       PetscCall(KSPCreate(PetscObjectComm((PetscObject)coarse_mat), &pcbddc->coarse_ksp));
    8871:       PetscCall(KSPSetNestLevel(pcbddc->coarse_ksp, pc->kspnestlevel));
    8872:       PetscCall(KSPSetErrorIfNotConverged(pcbddc->coarse_ksp, pc->erroriffailure));
    8873:       PetscCall(PetscObjectIncrementTabLevel((PetscObject)pcbddc->coarse_ksp, (PetscObject)pc, 1));
    8874:       PetscCall(KSPSetTolerances(pcbddc->coarse_ksp, PETSC_CURRENT, PETSC_CURRENT, PETSC_CURRENT, 1));
    8875:       PetscCall(KSPSetOperators(pcbddc->coarse_ksp, coarse_mat, coarse_mat));
    8876:       PetscCall(KSPSetType(pcbddc->coarse_ksp, coarse_ksp_type));
    8877:       PetscCall(KSPSetNormType(pcbddc->coarse_ksp, KSP_NORM_NONE));
    8878:       PetscCall(KSPGetPC(pcbddc->coarse_ksp, &pc_temp));
    8879:       /* TODO is this logic correct? should check for coarse_mat type */
    8880:       PetscCall(PCSetType(pc_temp, coarse_pc_type));
    8881:       /* prefix */
    8882:       PetscCall(PetscStrncpy(prefix, "", sizeof(prefix)));
    8883:       PetscCall(PetscStrncpy(str_level, "", sizeof(str_level)));
    8884:       if (!pcbddc->current_level) {
    8885:         PetscCall(PetscStrncpy(prefix, ((PetscObject)pc)->prefix, sizeof(prefix)));
    8886:         PetscCall(PetscStrlcat(prefix, "pc_bddc_coarse_", sizeof(prefix)));
    8887:       } else {
    8888:         PetscCall(PetscStrlen(((PetscObject)pc)->prefix, &len));
    8889:         if (pcbddc->current_level > 1) len -= 3;  /* remove "lX_" with X level number */
    8890:         if (pcbddc->current_level > 10) len -= 1; /* remove another char from level number */
    8891:         /* Nonstandard use of PetscStrncpy() to copy only a portion of the string */
    8892:         PetscCall(PetscStrncpy(prefix, ((PetscObject)pc)->prefix, len + 1));
    8893:         PetscCall(PetscSNPrintf(str_level, sizeof(str_level), "l%" PetscInt_FMT "_", pcbddc->current_level));
    8894:         PetscCall(PetscStrlcat(prefix, str_level, sizeof(prefix)));
    8895:       }
    8896:       PetscCall(KSPSetOptionsPrefix(pcbddc->coarse_ksp, prefix));
    8897:       /* propagate BDDC info to the next level (these are dummy calls if pc_temp is not of type PCBDDC) */
    8898:       PetscCall(PCBDDCSetLevel(pc_temp, pcbddc->current_level + 1));
    8899:       PetscCall(PCBDDCSetCoarseningRatio(pc_temp, pcbddc->coarsening_ratio));
    8900:       PetscCall(PCBDDCSetLevels(pc_temp, pcbddc->max_levels));
    8901:       /* allow user customization */
    8902:       PetscCall(KSPSetFromOptions(pcbddc->coarse_ksp));
    8903:       /* get some info after set from options */
    8904:       PetscCall(KSPGetPC(pcbddc->coarse_ksp, &pc_temp));
    8905:       /* multilevel cannot be done with coarse PC different from BDDC, NN, HPDDM, unless forced to */
    8906:       force = PETSC_FALSE;
    8907:       PetscCall(PetscOptionsGetBool(NULL, ((PetscObject)pc_temp)->prefix, "-pc_type_forced", &force, NULL));
    8908:       PetscCall(PetscObjectTypeCompareAny((PetscObject)pc_temp, &valid, PCBDDC, PCNN, PCHPDDM, ""));
    8909:       PetscCall(PetscObjectTypeCompare((PetscObject)pc_temp, PCBDDC, &isbddc));
    8910:       if (multilevel_allowed && !force && !valid) {
    8911:         isbddc = PETSC_TRUE;
    8912:         PetscCall(PCSetType(pc_temp, PCBDDC));
    8913:         PetscCall(PCBDDCSetLevel(pc_temp, pcbddc->current_level + 1));
    8914:         PetscCall(PCBDDCSetCoarseningRatio(pc_temp, pcbddc->coarsening_ratio));
    8915:         PetscCall(PCBDDCSetLevels(pc_temp, pcbddc->max_levels));
    8916:         if (pc_temp->ops->setfromoptions) { /* need to setfromoptions again, skipping the pc_type */
    8917:           PetscObjectOptionsBegin((PetscObject)pc_temp);
    8918:           PetscCall((*pc_temp->ops->setfromoptions)(pc_temp, PetscOptionsObject));
    8919:           PetscCall(PetscObjectProcessOptionsHandlers((PetscObject)pc_temp, PetscOptionsObject));
    8920:           PetscOptionsEnd();
    8921:           pc_temp->setfromoptionscalled++;
    8922:         }
    8923:       }
    8924:     }
    8925:     /* propagate BDDC info to the next level (these are dummy calls if pc_temp is not of type PCBDDC) */
    8926:     PetscCall(KSPGetPC(pcbddc->coarse_ksp, &pc_temp));
    8927:     if (nisdofs) {
    8928:       PetscCall(PCBDDCSetDofsSplitting(pc_temp, nisdofs, isarray));
    8929:       for (i = 0; i < nisdofs; i++) PetscCall(ISDestroy(&isarray[i]));
    8930:     }
    8931:     if (nisneu) {
    8932:       PetscCall(PCBDDCSetNeumannBoundaries(pc_temp, isarray[nisdofs]));
    8933:       PetscCall(ISDestroy(&isarray[nisdofs]));
    8934:     }
    8935:     if (nisvert) {
    8936:       PetscCall(PCBDDCSetPrimalVerticesIS(pc_temp, isarray[nis - 1]));
    8937:       PetscCall(ISDestroy(&isarray[nis - 1]));
    8938:     }
    8939:     if (coarseG) PetscCall(PCBDDCSetDiscreteGradient(pc_temp, coarseG, 1, nedcfield, PETSC_FALSE, PETSC_TRUE));

    8941:     /* get some info after set from options */
    8942:     PetscCall(PetscObjectTypeCompare((PetscObject)pc_temp, PCBDDC, &isbddc));

    8944:     /* multilevel can only be requested via -pc_bddc_levels or PCBDDCSetLevels */
    8945:     if (isbddc && !multilevel_allowed) PetscCall(PCSetType(pc_temp, coarse_pc_type));
    8946:     /* multilevel cannot be done with coarse PC different from BDDC, NN, HPDDM, unless forced to */
    8947:     force = PETSC_FALSE;
    8948:     PetscCall(PetscOptionsGetBool(NULL, ((PetscObject)pc_temp)->prefix, "-pc_type_forced", &force, NULL));
    8949:     PetscCall(PetscObjectTypeCompareAny((PetscObject)pc_temp, &valid, PCBDDC, PCNN, PCHPDDM, ""));
    8950:     if (multilevel_requested && multilevel_allowed && !valid && !force) PetscCall(PCSetType(pc_temp, PCBDDC));
    8951:     PetscCall(PetscObjectTypeCompare((PetscObject)pc_temp, PCREDUNDANT, &isredundant));
    8952:     if (isredundant) {
    8953:       KSP inner_ksp;
    8954:       PC  inner_pc;

    8956:       PetscCall(PCRedundantGetKSP(pc_temp, &inner_ksp));
    8957:       PetscCall(KSPGetPC(inner_ksp, &inner_pc));
    8958:     }

    8960:     /* parameters which miss an API */
    8961:     PetscCall(PetscObjectTypeCompare((PetscObject)pc_temp, PCBDDC, &isbddc));
    8962:     if (isbddc) {
    8963:       PC_BDDC *pcbddc_coarse = (PC_BDDC *)pc_temp->data;

    8965:       pcbddc_coarse->detect_disconnected = PETSC_TRUE;
    8966:       pcbddc_coarse->coarse_eqs_per_proc = pcbddc->coarse_eqs_per_proc;
    8967:       pcbddc_coarse->coarse_eqs_limit    = pcbddc->coarse_eqs_limit;
    8968:       pcbddc_coarse->benign_saddle_point = pcbddc->benign_have_null;
    8969:       if (pcbddc_coarse->benign_saddle_point) {
    8970:         Mat                    coarsedivudotp_is;
    8971:         ISLocalToGlobalMapping l2gmap, rl2g, cl2g;
    8972:         IS                     row, col;
    8973:         const PetscInt        *gidxs;
    8974:         PetscInt               n, st, M, N;

    8976:         PetscCall(MatGetSize(coarsedivudotp, &n, NULL));
    8977:         PetscCallMPI(MPI_Scan(&n, &st, 1, MPIU_INT, MPI_SUM, PetscObjectComm((PetscObject)coarse_mat)));
    8978:         st = st - n;
    8979:         PetscCall(ISCreateStride(PetscObjectComm((PetscObject)coarse_mat), 1, st, 1, &row));
    8980:         PetscCall(MatISGetLocalToGlobalMapping(coarse_mat, &l2gmap, NULL));
    8981:         PetscCall(ISLocalToGlobalMappingGetSize(l2gmap, &n));
    8982:         PetscCall(ISLocalToGlobalMappingGetIndices(l2gmap, &gidxs));
    8983:         PetscCall(ISCreateGeneral(PetscObjectComm((PetscObject)coarse_mat), n, gidxs, PETSC_COPY_VALUES, &col));
    8984:         PetscCall(ISLocalToGlobalMappingRestoreIndices(l2gmap, &gidxs));
    8985:         PetscCall(ISLocalToGlobalMappingCreateIS(row, &rl2g));
    8986:         PetscCall(ISLocalToGlobalMappingCreateIS(col, &cl2g));
    8987:         PetscCall(ISGetSize(row, &M));
    8988:         PetscCall(MatGetSize(coarse_mat, &N, NULL));
    8989:         PetscCall(ISDestroy(&row));
    8990:         PetscCall(ISDestroy(&col));
    8991:         PetscCall(MatCreate(PetscObjectComm((PetscObject)coarse_mat), &coarsedivudotp_is));
    8992:         PetscCall(MatSetType(coarsedivudotp_is, MATIS));
    8993:         PetscCall(MatSetSizes(coarsedivudotp_is, PETSC_DECIDE, PETSC_DECIDE, M, N));
    8994:         PetscCall(MatSetLocalToGlobalMapping(coarsedivudotp_is, rl2g, cl2g));
    8995:         PetscCall(ISLocalToGlobalMappingDestroy(&rl2g));
    8996:         PetscCall(ISLocalToGlobalMappingDestroy(&cl2g));
    8997:         PetscCall(MatISSetLocalMat(coarsedivudotp_is, coarsedivudotp));
    8998:         PetscCall(MatDestroy(&coarsedivudotp));
    8999:         PetscCall(PCBDDCSetDivergenceMat(pc_temp, coarsedivudotp_is, PETSC_FALSE, NULL));
    9000:         PetscCall(MatDestroy(&coarsedivudotp_is));
    9001:         pcbddc_coarse->adaptive_userdefined = PETSC_TRUE;
    9002:         if (pcbddc->adaptive_threshold[0] == 0.0) pcbddc_coarse->deluxe_zerorows = PETSC_TRUE;
    9003:       }
    9004:     }

    9006:     /* propagate symmetry info of coarse matrix */
    9007:     PetscCall(MatSetOption(coarse_mat, MAT_STRUCTURALLY_SYMMETRIC, PETSC_TRUE));
    9008:     PetscCall(MatIsSymmetricKnown(pc->pmat, &isset, &issym));
    9009:     if (isset) PetscCall(MatSetOption(coarse_mat, MAT_SYMMETRIC, issym));
    9010:     PetscCall(MatIsHermitianKnown(pc->pmat, &isset, &isher));
    9011:     if (isset) PetscCall(MatSetOption(coarse_mat, MAT_HERMITIAN, isher));
    9012:     PetscCall(MatIsSPDKnown(pc->pmat, &isset, &isspd));
    9013:     if (isset) PetscCall(MatSetOption(coarse_mat, MAT_SPD, isspd));

    9015:     if (pcbddc->benign_saddle_point && !pcbddc->benign_have_null) PetscCall(MatSetOption(coarse_mat, MAT_SPD, PETSC_TRUE));
    9016:     /* set operators */
    9017:     PetscCall(MatViewFromOptions(coarse_mat, (PetscObject)pc, "-pc_bddc_coarse_mat_view"));
    9018:     PetscCall(MatSetOptionsPrefix(coarse_mat, ((PetscObject)pcbddc->coarse_ksp)->prefix));
    9019:     PetscCall(KSPSetOperators(pcbddc->coarse_ksp, coarse_mat, coarse_mat));
    9020:     if (pcbddc->dbg_flag) PetscCall(PetscViewerASCIISubtractTab(dbg_viewer, 2 * pcbddc->current_level));
    9021:   }
    9022:   PetscCall(MatDestroy(&coarseG));
    9023:   PetscCall(PetscFree(isarray));
    9024: #if 0
    9025:   {
    9026:     PetscViewer viewer;
    9027:     char filename[256];
    9028:     PetscCall(PetscSNPrintf(filename, PETSC_STATIC_ARRAY_LENGTH(filename), "coarse_mat_level%d.m",pcbddc->current_level));
    9029:     PetscCall(PetscViewerASCIIOpen(PetscObjectComm((PetscObject)coarse_mat),filename,&viewer));
    9030:     PetscCall(PetscViewerPushFormat(viewer,PETSC_VIEWER_ASCII_MATLAB));
    9031:     PetscCall(MatView(coarse_mat,viewer));
    9032:     PetscCall(PetscViewerPopFormat(viewer));
    9033:     PetscCall(PetscViewerDestroy(&viewer));
    9034:   }
    9035: #endif

    9037:   if (corners) {
    9038:     Vec             gv;
    9039:     IS              is;
    9040:     const PetscInt *idxs;
    9041:     PetscInt        i, d, N, n, cdim = pcbddc->mat_graph->cdim;
    9042:     PetscScalar    *coords;

    9044:     PetscCheck(pcbddc->mat_graph->cloc, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Missing local coordinates");
    9045:     PetscCall(VecGetSize(pcbddc->coarse_vec, &N));
    9046:     PetscCall(VecGetLocalSize(pcbddc->coarse_vec, &n));
    9047:     PetscCall(VecCreate(PetscObjectComm((PetscObject)pcbddc->coarse_vec), &gv));
    9048:     PetscCall(VecSetBlockSize(gv, cdim));
    9049:     PetscCall(VecSetSizes(gv, n * cdim, N * cdim));
    9050:     PetscCall(VecSetType(gv, VECSTANDARD));
    9051:     PetscCall(VecSetFromOptions(gv));
    9052:     PetscCall(VecSet(gv, PETSC_MAX_REAL)); /* we only propagate coordinates from vertices constraints */

    9054:     PetscCall(PCBDDCGraphGetCandidatesIS(pcbddc->mat_graph, NULL, NULL, NULL, NULL, &is));
    9055:     PetscCall(ISGetLocalSize(is, &n));
    9056:     PetscCall(ISGetIndices(is, &idxs));
    9057:     PetscCall(PetscMalloc1(n * cdim, &coords));
    9058:     for (i = 0; i < n; i++) {
    9059:       for (d = 0; d < cdim; d++) coords[cdim * i + d] = pcbddc->mat_graph->coords[cdim * idxs[i] + d];
    9060:     }
    9061:     PetscCall(ISRestoreIndices(is, &idxs));
    9062:     PetscCall(PCBDDCGraphRestoreCandidatesIS(pcbddc->mat_graph, NULL, NULL, NULL, NULL, &is));

    9064:     PetscCall(ISGetLocalSize(corners, &n));
    9065:     PetscCall(ISGetIndices(corners, &idxs));
    9066:     PetscCall(VecSetValuesBlocked(gv, n, idxs, coords, INSERT_VALUES));
    9067:     PetscCall(ISRestoreIndices(corners, &idxs));
    9068:     PetscCall(PetscFree(coords));
    9069:     PetscCall(VecAssemblyBegin(gv));
    9070:     PetscCall(VecAssemblyEnd(gv));
    9071:     PetscCall(VecGetArray(gv, &coords));
    9072:     if (pcbddc->coarse_ksp) {
    9073:       PC        coarse_pc;
    9074:       PetscBool isbddc;

    9076:       PetscCall(KSPGetPC(pcbddc->coarse_ksp, &coarse_pc));
    9077:       PetscCall(PetscObjectTypeCompare((PetscObject)coarse_pc, PCBDDC, &isbddc));
    9078:       if (isbddc) { /* coarse coordinates have PETSC_MAX_REAL, specific for BDDC */
    9079:         PetscReal *realcoords;

    9081:         PetscCall(VecGetLocalSize(gv, &n));
    9082: #if defined(PETSC_USE_COMPLEX)
    9083:         PetscCall(PetscMalloc1(n, &realcoords));
    9084:         for (i = 0; i < n; i++) realcoords[i] = PetscRealPart(coords[i]);
    9085: #else
    9086:         realcoords = coords;
    9087: #endif
    9088:         PetscCall(PCSetCoordinates(coarse_pc, cdim, n / cdim, realcoords));
    9089: #if defined(PETSC_USE_COMPLEX)
    9090:         PetscCall(PetscFree(realcoords));
    9091: #endif
    9092:       }
    9093:     }
    9094:     PetscCall(VecRestoreArray(gv, &coords));
    9095:     PetscCall(VecDestroy(&gv));
    9096:   }
    9097:   PetscCall(ISDestroy(&corners));

    9099:   if (pcbddc->coarse_ksp) {
    9100:     Vec crhs, csol;

    9102:     PetscCall(KSPGetSolution(pcbddc->coarse_ksp, &csol));
    9103:     PetscCall(KSPGetRhs(pcbddc->coarse_ksp, &crhs));
    9104:     if (!csol) PetscCall(MatCreateVecs(coarse_mat, &pcbddc->coarse_ksp->vec_sol, NULL));
    9105:     if (!crhs) PetscCall(MatCreateVecs(coarse_mat, NULL, &pcbddc->coarse_ksp->vec_rhs));
    9106:   }
    9107:   PetscCall(MatDestroy(&coarsedivudotp));

    9109:   /* compute null space for coarse solver if the benign trick has been requested */
    9110:   if (pcbddc->benign_null) {
    9111:     PetscCall(VecSet(pcbddc->vec1_P, 0.));
    9112:     for (i = 0; i < pcbddc->benign_n; i++) PetscCall(VecSetValue(pcbddc->vec1_P, pcbddc->local_primal_size - pcbddc->benign_n + i, 1.0, INSERT_VALUES));
    9113:     PetscCall(VecAssemblyBegin(pcbddc->vec1_P));
    9114:     PetscCall(VecAssemblyEnd(pcbddc->vec1_P));
    9115:     PetscCall(VecScatterBegin(pcbddc->coarse_loc_to_glob, pcbddc->vec1_P, pcbddc->coarse_vec, INSERT_VALUES, SCATTER_FORWARD));
    9116:     PetscCall(VecScatterEnd(pcbddc->coarse_loc_to_glob, pcbddc->vec1_P, pcbddc->coarse_vec, INSERT_VALUES, SCATTER_FORWARD));
    9117:     if (coarse_mat) {
    9118:       Vec          nullv;
    9119:       PetscScalar *array, *array2;
    9120:       PetscInt     nl;

    9122:       PetscCall(MatCreateVecs(coarse_mat, &nullv, NULL));
    9123:       PetscCall(VecGetLocalSize(nullv, &nl));
    9124:       PetscCall(VecGetArrayRead(pcbddc->coarse_vec, (const PetscScalar **)&array));
    9125:       PetscCall(VecGetArray(nullv, &array2));
    9126:       PetscCall(PetscArraycpy(array2, array, nl));
    9127:       PetscCall(VecRestoreArray(nullv, &array2));
    9128:       PetscCall(VecRestoreArrayRead(pcbddc->coarse_vec, (const PetscScalar **)&array));
    9129:       PetscCall(VecNormalize(nullv, NULL));
    9130:       PetscCall(MatNullSpaceCreate(PetscObjectComm((PetscObject)coarse_mat), PETSC_FALSE, 1, &nullv, &CoarseNullSpace));
    9131:       PetscCall(VecDestroy(&nullv));
    9132:     }
    9133:   }
    9134:   PetscCall(PetscLogEventEnd(PC_BDDC_CoarseSetUp[pcbddc->current_level], pc, 0, 0, 0));

    9136:   PetscCall(PetscLogEventBegin(PC_BDDC_CoarseSolver[pcbddc->current_level], pc, 0, 0, 0));
    9137:   if (pcbddc->coarse_ksp) {
    9138:     PetscBool ispreonly;

    9140:     if (CoarseNullSpace) {
    9141:       PetscBool isnull;

    9143:       PetscCall(MatNullSpaceTest(CoarseNullSpace, coarse_mat, &isnull));
    9144:       if (isnull) PetscCall(MatSetNullSpace(coarse_mat, CoarseNullSpace));
    9145:       /* TODO: add local nullspaces (if any) */
    9146:     }
    9147:     /* setup coarse ksp */
    9148:     PetscCall(KSPSetUp(pcbddc->coarse_ksp));
    9149:     /* Check coarse problem if in debug mode or if solving with an iterative method */
    9150:     PetscCall(PetscObjectTypeCompare((PetscObject)pcbddc->coarse_ksp, KSPPREONLY, &ispreonly));
    9151:     if (pcbddc->dbg_flag || (!ispreonly && pcbddc->use_coarse_estimates)) {
    9152:       KSP         check_ksp;
    9153:       KSPType     check_ksp_type;
    9154:       PC          check_pc;
    9155:       Vec         check_vec, coarse_vec;
    9156:       PetscReal   abs_infty_error, infty_error, lambda_min = 1.0, lambda_max = 1.0;
    9157:       PetscInt    its;
    9158:       PetscBool   compute_eigs;
    9159:       PetscReal  *eigs_r, *eigs_c;
    9160:       PetscInt    neigs;
    9161:       const char *prefix;

    9163:       /* Create ksp object suitable for estimation of extreme eigenvalues */
    9164:       PetscCall(KSPCreate(PetscObjectComm((PetscObject)pcbddc->coarse_ksp), &check_ksp));
    9165:       PetscCall(KSPSetNestLevel(check_ksp, pc->kspnestlevel));
    9166:       PetscCall(PetscObjectIncrementTabLevel((PetscObject)check_ksp, (PetscObject)pcbddc->coarse_ksp, 0));
    9167:       PetscCall(KSPSetErrorIfNotConverged(pcbddc->coarse_ksp, PETSC_FALSE));
    9168:       PetscCall(KSPSetOperators(check_ksp, coarse_mat, coarse_mat));
    9169:       PetscCall(KSPSetTolerances(check_ksp, 1.e-12, 1.e-12, PETSC_CURRENT, pcbddc->coarse_size));
    9170:       /* prevent from setup unneeded object */
    9171:       PetscCall(KSPGetPC(check_ksp, &check_pc));
    9172:       PetscCall(PCSetType(check_pc, PCNONE));
    9173:       if (ispreonly) {
    9174:         check_ksp_type = KSPPREONLY;
    9175:         compute_eigs   = PETSC_FALSE;
    9176:       } else {
    9177:         check_ksp_type = KSPGMRES;
    9178:         compute_eigs   = PETSC_TRUE;
    9179:       }
    9180:       PetscCall(KSPSetType(check_ksp, check_ksp_type));
    9181:       PetscCall(KSPSetComputeSingularValues(check_ksp, compute_eigs));
    9182:       PetscCall(KSPSetComputeEigenvalues(check_ksp, compute_eigs));
    9183:       PetscCall(KSPGMRESSetRestart(check_ksp, pcbddc->coarse_size + 1));
    9184:       PetscCall(KSPGetOptionsPrefix(pcbddc->coarse_ksp, &prefix));
    9185:       PetscCall(KSPSetOptionsPrefix(check_ksp, prefix));
    9186:       PetscCall(KSPAppendOptionsPrefix(check_ksp, "check_"));
    9187:       PetscCall(KSPSetFromOptions(check_ksp));
    9188:       PetscCall(KSPSetUp(check_ksp));
    9189:       PetscCall(KSPGetPC(pcbddc->coarse_ksp, &check_pc));
    9190:       PetscCall(KSPSetPC(check_ksp, check_pc));
    9191:       /* create random vec */
    9192:       PetscCall(MatCreateVecs(coarse_mat, &coarse_vec, &check_vec));
    9193:       PetscCall(VecSetRandom(check_vec, NULL));
    9194:       PetscCall(MatMult(coarse_mat, check_vec, coarse_vec));
    9195:       /* solve coarse problem */
    9196:       PetscCall(KSPSolve(check_ksp, coarse_vec, coarse_vec));
    9197:       PetscCall(KSPCheckSolve(check_ksp, pc, coarse_vec));
    9198:       /* set eigenvalue estimation if preonly has not been requested */
    9199:       if (compute_eigs) {
    9200:         PetscCall(PetscMalloc1(pcbddc->coarse_size + 1, &eigs_r));
    9201:         PetscCall(PetscMalloc1(pcbddc->coarse_size + 1, &eigs_c));
    9202:         PetscCall(KSPComputeEigenvalues(check_ksp, pcbddc->coarse_size + 1, eigs_r, eigs_c, &neigs));
    9203:         if (neigs) {
    9204:           lambda_max = eigs_r[neigs - 1];
    9205:           lambda_min = eigs_r[0];
    9206:           if (pcbddc->use_coarse_estimates) {
    9207:             if (lambda_max >= lambda_min) { /* using PETSC_SMALL since lambda_max == lambda_min is not allowed by KSPChebyshevSetEigenvalues */
    9208:               PetscCall(KSPChebyshevSetEigenvalues(pcbddc->coarse_ksp, lambda_max + PETSC_SMALL, lambda_min));
    9209:               PetscCall(KSPRichardsonSetScale(pcbddc->coarse_ksp, 2.0 / (lambda_max + lambda_min)));
    9210:             }
    9211:           }
    9212:         }
    9213:       }

    9215:       /* check coarse problem residual error */
    9216:       if (pcbddc->dbg_flag) {
    9217:         PetscViewer dbg_viewer = PETSC_VIEWER_STDOUT_(PetscObjectComm((PetscObject)pcbddc->coarse_ksp));
    9218:         PetscCall(PetscViewerASCIIAddTab(dbg_viewer, 2 * (pcbddc->current_level + 1)));
    9219:         PetscCall(VecAXPY(check_vec, -1.0, coarse_vec));
    9220:         PetscCall(VecNorm(check_vec, NORM_INFINITY, &infty_error));
    9221:         PetscCall(MatMult(coarse_mat, check_vec, coarse_vec));
    9222:         PetscCall(VecNorm(coarse_vec, NORM_INFINITY, &abs_infty_error));
    9223:         PetscCall(PetscViewerASCIIPrintf(dbg_viewer, "Coarse problem details (use estimates %d)\n", pcbddc->use_coarse_estimates));
    9224:         PetscCall(PetscObjectPrintClassNamePrefixType((PetscObject)pcbddc->coarse_ksp, dbg_viewer));
    9225:         PetscCall(PetscObjectPrintClassNamePrefixType((PetscObject)check_pc, dbg_viewer));
    9226:         PetscCall(PetscViewerASCIIPrintf(dbg_viewer, "Coarse problem exact infty_error   : %1.6e\n", (double)infty_error));
    9227:         PetscCall(PetscViewerASCIIPrintf(dbg_viewer, "Coarse problem residual infty_error: %1.6e\n", (double)abs_infty_error));
    9228:         if (CoarseNullSpace) PetscCall(PetscViewerASCIIPrintf(dbg_viewer, "Coarse problem is singular\n"));
    9229:         if (compute_eigs) {
    9230:           PetscReal          lambda_max_s, lambda_min_s;
    9231:           KSPConvergedReason reason;
    9232:           PetscCall(KSPGetType(check_ksp, &check_ksp_type));
    9233:           PetscCall(KSPGetIterationNumber(check_ksp, &its));
    9234:           PetscCall(KSPGetConvergedReason(check_ksp, &reason));
    9235:           PetscCall(KSPComputeExtremeSingularValues(check_ksp, &lambda_max_s, &lambda_min_s));
    9236:           PetscCall(PetscViewerASCIIPrintf(dbg_viewer, "Coarse problem eigenvalues (estimated with %" PetscInt_FMT " iterations of %s, conv reason %d): %1.6e %1.6e (%1.6e %1.6e)\n", its, check_ksp_type, reason, (double)lambda_min, (double)lambda_max, (double)lambda_min_s, (double)lambda_max_s));
    9237:           for (i = 0; i < neigs; i++) PetscCall(PetscViewerASCIIPrintf(dbg_viewer, "%1.6e %1.6ei\n", (double)eigs_r[i], (double)eigs_c[i]));
    9238:         }
    9239:         PetscCall(PetscViewerFlush(dbg_viewer));
    9240:         PetscCall(PetscViewerASCIISubtractTab(dbg_viewer, 2 * (pcbddc->current_level + 1)));
    9241:       }
    9242:       PetscCall(VecDestroy(&check_vec));
    9243:       PetscCall(VecDestroy(&coarse_vec));
    9244:       PetscCall(KSPDestroy(&check_ksp));
    9245:       if (compute_eigs) {
    9246:         PetscCall(PetscFree(eigs_r));
    9247:         PetscCall(PetscFree(eigs_c));
    9248:       }
    9249:     }
    9250:   }
    9251:   PetscCall(MatNullSpaceDestroy(&CoarseNullSpace));
    9252:   /* print additional info */
    9253:   if (pcbddc->dbg_flag) {
    9254:     /* waits until all processes reaches this point */
    9255:     PetscCall(PetscBarrier((PetscObject)pc));
    9256:     PetscCall(PetscViewerASCIIPrintf(pcbddc->dbg_viewer, "Coarse solver setup completed at level %" PetscInt_FMT "\n", pcbddc->current_level));
    9257:     PetscCall(PetscViewerFlush(pcbddc->dbg_viewer));
    9258:   }

    9260:   /* free memory */
    9261:   PetscCall(MatDestroy(&coarse_mat));
    9262:   PetscCall(PetscLogEventEnd(PC_BDDC_CoarseSolver[pcbddc->current_level], pc, 0, 0, 0));
    9263:   PetscFunctionReturn(PETSC_SUCCESS);
    9264: }

    9266: PetscErrorCode PCBDDCComputePrimalNumbering(PC pc, PetscInt *coarse_size_n, PetscInt **local_primal_indices_n)
    9267: {
    9268:   PC_BDDC        *pcbddc = (PC_BDDC *)pc->data;
    9269:   PC_IS          *pcis   = (PC_IS *)pc->data;
    9270:   IS              subset, subset_mult, subset_n;
    9271:   PetscInt        local_size, coarse_size = 0;
    9272:   PetscInt       *local_primal_indices = NULL;
    9273:   const PetscInt *t_local_primal_indices;

    9275:   PetscFunctionBegin;
    9276:   /* Compute global number of coarse dofs */
    9277:   PetscCheck(!pcbddc->local_primal_size || pcbddc->local_primal_ref_node, PETSC_COMM_SELF, PETSC_ERR_PLIB, "BDDC ConstraintsSetUp should be called first");
    9278:   PetscCall(ISCreateGeneral(PetscObjectComm((PetscObject)pc->pmat), pcbddc->local_primal_size_cc, pcbddc->local_primal_ref_node, PETSC_COPY_VALUES, &subset_n));
    9279:   PetscCall(ISLocalToGlobalMappingApplyIS(pcis->mapping, subset_n, &subset));
    9280:   PetscCall(ISDestroy(&subset_n));
    9281:   PetscCall(ISCreateGeneral(PetscObjectComm((PetscObject)pc->pmat), pcbddc->local_primal_size_cc, pcbddc->local_primal_ref_mult, PETSC_COPY_VALUES, &subset_mult));
    9282:   PetscCall(ISRenumber(subset, subset_mult, &coarse_size, &subset_n));
    9283:   PetscCall(ISDestroy(&subset));
    9284:   PetscCall(ISDestroy(&subset_mult));
    9285:   PetscCall(ISGetLocalSize(subset_n, &local_size));
    9286:   PetscCheck(local_size == pcbddc->local_primal_size, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Invalid number of local primal indices computed %" PetscInt_FMT " != %" PetscInt_FMT, local_size, pcbddc->local_primal_size);
    9287:   PetscCall(PetscMalloc1(local_size, &local_primal_indices));
    9288:   PetscCall(ISGetIndices(subset_n, &t_local_primal_indices));
    9289:   PetscCall(PetscArraycpy(local_primal_indices, t_local_primal_indices, local_size));
    9290:   PetscCall(ISRestoreIndices(subset_n, &t_local_primal_indices));
    9291:   PetscCall(ISDestroy(&subset_n));

    9293:   if (pcbddc->dbg_flag) {
    9294:     PetscCall(PetscViewerFlush(pcbddc->dbg_viewer));
    9295:     PetscCall(PetscViewerASCIIPrintf(pcbddc->dbg_viewer, "--------------------------------------------------\n"));
    9296:     PetscCall(PetscViewerASCIIPrintf(pcbddc->dbg_viewer, "Size of coarse problem is %" PetscInt_FMT "\n", coarse_size));
    9297:     PetscCall(PetscViewerFlush(pcbddc->dbg_viewer));
    9298:   }

    9300:   /* get back data */
    9301:   *coarse_size_n          = coarse_size;
    9302:   *local_primal_indices_n = local_primal_indices;
    9303:   PetscFunctionReturn(PETSC_SUCCESS);
    9304: }

    9306: PetscErrorCode PCBDDCGlobalToLocal(VecScatter g2l_ctx, Vec gwork, Vec lwork, IS globalis, IS *localis)
    9307: {
    9308:   IS           localis_t;
    9309:   PetscInt     i, lsize, *idxs, n;
    9310:   PetscScalar *vals;

    9312:   PetscFunctionBegin;
    9313:   /* get indices in local ordering exploiting local to global map */
    9314:   PetscCall(ISGetLocalSize(globalis, &lsize));
    9315:   PetscCall(PetscMalloc1(lsize, &vals));
    9316:   for (i = 0; i < lsize; i++) vals[i] = 1.0;
    9317:   PetscCall(ISGetIndices(globalis, (const PetscInt **)&idxs));
    9318:   PetscCall(VecSet(gwork, 0.0));
    9319:   PetscCall(VecSet(lwork, 0.0));
    9320:   if (idxs) { /* multilevel guard */
    9321:     PetscCall(VecSetOption(gwork, VEC_IGNORE_NEGATIVE_INDICES, PETSC_TRUE));
    9322:     PetscCall(VecSetValues(gwork, lsize, idxs, vals, INSERT_VALUES));
    9323:   }
    9324:   PetscCall(VecAssemblyBegin(gwork));
    9325:   PetscCall(ISRestoreIndices(globalis, (const PetscInt **)&idxs));
    9326:   PetscCall(PetscFree(vals));
    9327:   PetscCall(VecAssemblyEnd(gwork));
    9328:   /* now compute set in local ordering */
    9329:   PetscCall(VecScatterBegin(g2l_ctx, gwork, lwork, INSERT_VALUES, SCATTER_FORWARD));
    9330:   PetscCall(VecScatterEnd(g2l_ctx, gwork, lwork, INSERT_VALUES, SCATTER_FORWARD));
    9331:   PetscCall(VecGetArrayRead(lwork, (const PetscScalar **)&vals));
    9332:   PetscCall(VecGetSize(lwork, &n));
    9333:   for (i = 0, lsize = 0; i < n; i++) {
    9334:     if (PetscRealPart(vals[i]) > 0.5) lsize++;
    9335:   }
    9336:   PetscCall(PetscMalloc1(lsize, &idxs));
    9337:   for (i = 0, lsize = 0; i < n; i++) {
    9338:     if (PetscRealPart(vals[i]) > 0.5) idxs[lsize++] = i;
    9339:   }
    9340:   PetscCall(VecRestoreArrayRead(lwork, (const PetscScalar **)&vals));
    9341:   PetscCall(ISCreateGeneral(PetscObjectComm((PetscObject)gwork), lsize, idxs, PETSC_OWN_POINTER, &localis_t));
    9342:   *localis = localis_t;
    9343:   PetscFunctionReturn(PETSC_SUCCESS);
    9344: }

    9346: PetscErrorCode PCBDDCComputeFakeChange(PC pc, PetscBool constraints, PCBDDCGraph graph, PCBDDCSubSchurs schurs, Mat *change, IS *change_primal, IS *change_primal_mult, PetscBool *change_with_qr)
    9347: {
    9348:   PC_IS   *pcis   = (PC_IS *)pc->data;
    9349:   PC_BDDC *pcbddc = (PC_BDDC *)pc->data;
    9350:   PC_IS   *pcisf;
    9351:   PC_BDDC *pcbddcf;
    9352:   PC       pcf;

    9354:   PetscFunctionBegin;
    9355:   PetscCall(PCCreate(PetscObjectComm((PetscObject)pc), &pcf));
    9356:   PetscCall(PCSetOperators(pcf, pc->mat, pc->pmat));
    9357:   PetscCall(PCSetType(pcf, PCBDDC));

    9359:   pcisf   = (PC_IS *)pcf->data;
    9360:   pcbddcf = (PC_BDDC *)pcf->data;

    9362:   pcisf->is_B_local = pcis->is_B_local;
    9363:   pcisf->vec1_N     = pcis->vec1_N;
    9364:   pcisf->BtoNmap    = pcis->BtoNmap;
    9365:   pcisf->n          = pcis->n;
    9366:   pcisf->n_B        = pcis->n_B;

    9368:   PetscCall(PetscFree(pcbddcf->mat_graph));
    9369:   PetscCall(PetscFree(pcbddcf->sub_schurs));
    9370:   pcbddcf->mat_graph             = graph ? graph : pcbddc->mat_graph;
    9371:   pcbddcf->sub_schurs            = schurs;
    9372:   pcbddcf->adaptive_selection    = schurs ? PETSC_TRUE : PETSC_FALSE;
    9373:   pcbddcf->adaptive_threshold[0] = pcbddc->adaptive_threshold[0];
    9374:   pcbddcf->adaptive_threshold[1] = pcbddc->adaptive_threshold[1];
    9375:   pcbddcf->adaptive_nmin         = pcbddc->adaptive_nmin;
    9376:   pcbddcf->adaptive_nmax         = pcbddc->adaptive_nmax;
    9377:   pcbddcf->use_faces             = PETSC_TRUE;
    9378:   pcbddcf->use_change_of_basis   = (PetscBool)!constraints;
    9379:   pcbddcf->use_change_on_faces   = (PetscBool)!constraints;
    9380:   pcbddcf->use_qr_single         = (PetscBool)!constraints;
    9381:   pcbddcf->fake_change           = PETSC_TRUE;
    9382:   pcbddcf->dbg_flag              = pcbddc->dbg_flag;

    9384:   PetscCall(PCBDDCAdaptiveSelection(pcf));
    9385:   PetscCall(PCBDDCConstraintsSetUp(pcf));

    9387:   *change = pcbddcf->ConstraintMatrix;
    9388:   if (change_primal) PetscCall(ISCreateGeneral(PetscObjectComm((PetscObject)pc->pmat), pcbddcf->local_primal_size_cc, pcbddcf->local_primal_ref_node, PETSC_COPY_VALUES, change_primal));
    9389:   if (change_primal_mult) PetscCall(ISCreateGeneral(PetscObjectComm((PetscObject)pc->pmat), pcbddcf->local_primal_size_cc, pcbddcf->local_primal_ref_mult, PETSC_COPY_VALUES, change_primal_mult));
    9390:   if (change_with_qr) *change_with_qr = pcbddcf->use_qr_single;

    9392:   if (schurs) pcbddcf->sub_schurs = NULL;
    9393:   pcbddcf->ConstraintMatrix = NULL;
    9394:   pcbddcf->mat_graph        = NULL;
    9395:   pcisf->is_B_local         = NULL;
    9396:   pcisf->vec1_N             = NULL;
    9397:   pcisf->BtoNmap            = NULL;
    9398:   PetscCall(PCDestroy(&pcf));
    9399:   PetscFunctionReturn(PETSC_SUCCESS);
    9400: }

    9402: PetscErrorCode PCBDDCSetUpSubSchurs(PC pc)
    9403: {
    9404:   PC_IS          *pcis       = (PC_IS *)pc->data;
    9405:   PC_BDDC        *pcbddc     = (PC_BDDC *)pc->data;
    9406:   PCBDDCSubSchurs sub_schurs = pcbddc->sub_schurs;
    9407:   Mat             S_j;
    9408:   PetscInt       *used_xadj, *used_adjncy;
    9409:   PetscBool       free_used_adj;

    9411:   PetscFunctionBegin;
    9412:   PetscCall(PetscLogEventBegin(PC_BDDC_Schurs[pcbddc->current_level], pc, 0, 0, 0));
    9413:   /* decide the adjacency to be used for determining internal problems for local schur on subsets */
    9414:   free_used_adj = PETSC_FALSE;
    9415:   if (pcbddc->sub_schurs_layers == -1) {
    9416:     used_xadj   = NULL;
    9417:     used_adjncy = NULL;
    9418:   } else {
    9419:     if (pcbddc->sub_schurs_use_useradj && pcbddc->mat_graph->xadj) {
    9420:       used_xadj   = pcbddc->mat_graph->xadj;
    9421:       used_adjncy = pcbddc->mat_graph->adjncy;
    9422:     } else if (pcbddc->computed_rowadj) {
    9423:       used_xadj   = pcbddc->mat_graph->xadj;
    9424:       used_adjncy = pcbddc->mat_graph->adjncy;
    9425:     } else {
    9426:       PetscBool       flg_row = PETSC_FALSE;
    9427:       const PetscInt *xadj, *adjncy;
    9428:       PetscInt        nvtxs;

    9430:       PetscCall(MatGetRowIJ(pcbddc->local_mat, 0, PETSC_TRUE, PETSC_FALSE, &nvtxs, &xadj, &adjncy, &flg_row));
    9431:       if (flg_row) {
    9432:         PetscCall(PetscMalloc2(nvtxs + 1, &used_xadj, xadj[nvtxs], &used_adjncy));
    9433:         PetscCall(PetscArraycpy(used_xadj, xadj, nvtxs + 1));
    9434:         PetscCall(PetscArraycpy(used_adjncy, adjncy, xadj[nvtxs]));
    9435:         free_used_adj = PETSC_TRUE;
    9436:       } else {
    9437:         pcbddc->sub_schurs_layers = -1;
    9438:         used_xadj                 = NULL;
    9439:         used_adjncy               = NULL;
    9440:       }
    9441:       PetscCall(MatRestoreRowIJ(pcbddc->local_mat, 0, PETSC_TRUE, PETSC_FALSE, &nvtxs, &xadj, &adjncy, &flg_row));
    9442:     }
    9443:   }

    9445:   /* setup sub_schurs data */
    9446:   PetscCall(MatCreateSchurComplement(pcis->A_II, pcis->pA_II, pcis->A_IB, pcis->A_BI, pcis->A_BB, &S_j));
    9447:   if (!sub_schurs->schur_explicit) {
    9448:     /* pcbddc->ksp_D up to date only if not using MatFactor with Schur complement support */
    9449:     PetscCall(MatSchurComplementSetKSP(S_j, pcbddc->ksp_D));
    9450:     PetscCall(PCBDDCSubSchursSetUp(sub_schurs, NULL, S_j, PETSC_FALSE, used_xadj, used_adjncy, pcbddc->sub_schurs_layers, NULL, pcbddc->adaptive_selection, PETSC_FALSE, PETSC_FALSE, 0, NULL, NULL, NULL, NULL));
    9451:   } else {
    9452:     Mat       change        = NULL;
    9453:     Vec       scaling       = NULL;
    9454:     IS        change_primal = NULL, iP;
    9455:     PetscInt  benign_n;
    9456:     PetscBool reuse_solvers     = (PetscBool)!pcbddc->use_change_of_basis;
    9457:     PetscBool need_change       = PETSC_FALSE;
    9458:     PetscBool discrete_harmonic = PETSC_FALSE;

    9460:     if (!pcbddc->use_vertices && reuse_solvers) {
    9461:       PetscInt n_vertices;

    9463:       PetscCall(ISGetLocalSize(sub_schurs->is_vertices, &n_vertices));
    9464:       reuse_solvers = (PetscBool)!n_vertices;
    9465:     }
    9466:     if (!pcbddc->benign_change_explicit) {
    9467:       benign_n = pcbddc->benign_n;
    9468:     } else {
    9469:       benign_n = 0;
    9470:     }
    9471:     /* sub_schurs->change is a local object; instead, PCBDDCConstraintsSetUp and the quantities used in the test below are logically collective on pc.
    9472:        We need a global reduction to avoid possible deadlocks.
    9473:        We assume that sub_schurs->change is created once, and then reused for different solves, unless the topography has been recomputed */
    9474:     if (pcbddc->adaptive_userdefined || (pcbddc->deluxe_zerorows && !pcbddc->use_change_of_basis)) {
    9475:       PetscBool have_loc_change = (PetscBool)(!!sub_schurs->change);
    9476:       PetscCallMPI(MPIU_Allreduce(&have_loc_change, &need_change, 1, MPI_C_BOOL, MPI_LOR, PetscObjectComm((PetscObject)pc)));
    9477:       need_change = (PetscBool)(!need_change);
    9478:     }
    9479:     /* If the user defines additional constraints, we import them here */
    9480:     if (need_change) {
    9481:       PetscCheck(!pcbddc->sub_schurs_rebuild, PETSC_COMM_SELF, PETSC_ERR_SUP, "Cannot compute change of basis with a different graph");
    9482:       PetscCall(PCBDDCComputeFakeChange(pc, PETSC_FALSE, NULL, NULL, &change, &change_primal, NULL, &sub_schurs->change_with_qr));
    9483:     }
    9484:     if (!pcbddc->use_deluxe_scaling) scaling = pcis->D;

    9486:     PetscCall(PetscObjectQuery((PetscObject)pc, "__KSPFETIDP_iP", (PetscObject *)&iP));
    9487:     if (iP) {
    9488:       PetscOptionsBegin(PetscObjectComm((PetscObject)iP), sub_schurs->prefix, "BDDC sub_schurs options", "PC");
    9489:       PetscCall(PetscOptionsBool("-sub_schurs_discrete_harmonic", NULL, NULL, discrete_harmonic, &discrete_harmonic, NULL));
    9490:       PetscOptionsEnd();
    9491:     }
    9492:     if (discrete_harmonic) {
    9493:       Mat A;
    9494:       PetscCall(MatDuplicate(pcbddc->local_mat, MAT_COPY_VALUES, &A));
    9495:       PetscCall(MatZeroRowsColumnsIS(A, iP, 1.0, NULL, NULL));
    9496:       PetscCall(PetscObjectCompose((PetscObject)A, "__KSPFETIDP_iP", (PetscObject)iP));
    9497:       PetscCall(PCBDDCSubSchursSetUp(sub_schurs, A, S_j, pcbddc->sub_schurs_exact_schur, used_xadj, used_adjncy, pcbddc->sub_schurs_layers, scaling, pcbddc->adaptive_selection, reuse_solvers, pcbddc->benign_saddle_point, benign_n, pcbddc->benign_p0_lidx,
    9498:                                      pcbddc->benign_zerodiag_subs, change, change_primal));
    9499:       PetscCall(MatDestroy(&A));
    9500:     } else {
    9501:       PetscCall(PCBDDCSubSchursSetUp(sub_schurs, pcbddc->local_mat, S_j, pcbddc->sub_schurs_exact_schur, used_xadj, used_adjncy, pcbddc->sub_schurs_layers, scaling, pcbddc->adaptive_selection, reuse_solvers, pcbddc->benign_saddle_point, benign_n,
    9502:                                      pcbddc->benign_p0_lidx, pcbddc->benign_zerodiag_subs, change, change_primal));
    9503:     }
    9504:     PetscCall(MatDestroy(&change));
    9505:     PetscCall(ISDestroy(&change_primal));
    9506:   }
    9507:   PetscCall(MatDestroy(&S_j));

    9509:   /* free adjacency */
    9510:   if (free_used_adj) PetscCall(PetscFree2(used_xadj, used_adjncy));
    9511:   PetscCall(PetscLogEventEnd(PC_BDDC_Schurs[pcbddc->current_level], pc, 0, 0, 0));
    9512:   PetscFunctionReturn(PETSC_SUCCESS);
    9513: }

    9515: PetscErrorCode PCBDDCInitSubSchurs(PC pc)
    9516: {
    9517:   PC_IS      *pcis   = (PC_IS *)pc->data;
    9518:   PC_BDDC    *pcbddc = (PC_BDDC *)pc->data;
    9519:   PCBDDCGraph graph;

    9521:   PetscFunctionBegin;
    9522:   /* attach interface graph for determining subsets */
    9523:   if (pcbddc->sub_schurs_rebuild) { /* in case rebuild has been requested, it uses a graph generated only by the neighbouring information */
    9524:     IS       verticesIS, verticescomm;
    9525:     PetscInt vsize, *idxs;

    9527:     PetscCall(PCBDDCGraphGetCandidatesIS(pcbddc->mat_graph, NULL, NULL, NULL, NULL, &verticesIS));
    9528:     PetscCall(ISGetSize(verticesIS, &vsize));
    9529:     PetscCall(ISGetIndices(verticesIS, (const PetscInt **)&idxs));
    9530:     PetscCall(ISCreateGeneral(PetscObjectComm((PetscObject)pc), vsize, idxs, PETSC_COPY_VALUES, &verticescomm));
    9531:     PetscCall(ISRestoreIndices(verticesIS, (const PetscInt **)&idxs));
    9532:     PetscCall(PCBDDCGraphRestoreCandidatesIS(pcbddc->mat_graph, NULL, NULL, NULL, NULL, &verticesIS));
    9533:     PetscCall(PCBDDCGraphCreate(&graph));
    9534:     PetscCall(PCBDDCGraphInit(graph, pcbddc->mat_graph->l2gmap, pcbddc->mat_graph->nvtxs_global, pcbddc->graphmaxcount));
    9535:     PetscCall(PCBDDCGraphSetUp(graph, pcbddc->mat_graph->custom_minimal_size, NULL, pcbddc->DirichletBoundariesLocal, 0, NULL, verticescomm));
    9536:     PetscCall(ISDestroy(&verticescomm));
    9537:     PetscCall(PCBDDCGraphComputeConnectedComponents(graph));
    9538:   } else {
    9539:     graph = pcbddc->mat_graph;
    9540:   }
    9541:   /* print some info */
    9542:   if (pcbddc->dbg_flag && !pcbddc->sub_schurs_rebuild) {
    9543:     IS       vertices;
    9544:     PetscInt nv, nedges, nfaces;
    9545:     PetscCall(PCBDDCGraphASCIIView(graph, pcbddc->dbg_flag, pcbddc->dbg_viewer));
    9546:     PetscCall(PCBDDCGraphGetCandidatesIS(graph, &nfaces, NULL, &nedges, NULL, &vertices));
    9547:     PetscCall(ISGetSize(vertices, &nv));
    9548:     PetscCall(PetscViewerASCIIPushSynchronized(pcbddc->dbg_viewer));
    9549:     PetscCall(PetscViewerASCIISynchronizedPrintf(pcbddc->dbg_viewer, "--------------------------------------------------------------\n"));
    9550:     PetscCall(PetscViewerASCIISynchronizedPrintf(pcbddc->dbg_viewer, "Subdomain %04d got %02" PetscInt_FMT " local candidate vertices (%d)\n", PetscGlobalRank, nv, pcbddc->use_vertices));
    9551:     PetscCall(PetscViewerASCIISynchronizedPrintf(pcbddc->dbg_viewer, "Subdomain %04d got %02" PetscInt_FMT " local candidate edges    (%d)\n", PetscGlobalRank, nedges, pcbddc->use_edges));
    9552:     PetscCall(PetscViewerASCIISynchronizedPrintf(pcbddc->dbg_viewer, "Subdomain %04d got %02" PetscInt_FMT " local candidate faces    (%d)\n", PetscGlobalRank, nfaces, pcbddc->use_faces));
    9553:     PetscCall(PetscViewerFlush(pcbddc->dbg_viewer));
    9554:     PetscCall(PetscViewerASCIIPopSynchronized(pcbddc->dbg_viewer));
    9555:     PetscCall(PCBDDCGraphRestoreCandidatesIS(graph, &nfaces, NULL, &nedges, NULL, &vertices));
    9556:   }

    9558:   /* sub_schurs init */
    9559:   if (!pcbddc->sub_schurs) PetscCall(PCBDDCSubSchursCreate(&pcbddc->sub_schurs));
    9560:   PetscCall(PCBDDCSubSchursInit(pcbddc->sub_schurs, ((PetscObject)pc)->prefix, pcis->is_I_local, pcis->is_B_local, graph, pcis->BtoNmap, pcbddc->sub_schurs_rebuild, PETSC_FALSE));

    9562:   /* free graph struct */
    9563:   if (pcbddc->sub_schurs_rebuild) PetscCall(PCBDDCGraphDestroy(&graph));
    9564:   PetscFunctionReturn(PETSC_SUCCESS);
    9565: }

    9567: static PetscErrorCode PCBDDCViewGlobalIS(PC pc, IS is, PetscViewer viewer)
    9568: {
    9569:   Mat_IS         *matis = (Mat_IS *)pc->pmat->data;
    9570:   PetscInt        n     = pc->pmat->rmap->n, ln, ni, st;
    9571:   const PetscInt *idxs;
    9572:   IS              gis;

    9574:   PetscFunctionBegin;
    9575:   if (!is) PetscFunctionReturn(PETSC_SUCCESS);
    9576:   PetscCall(MatGetOwnershipRange(pc->pmat, &st, NULL));
    9577:   PetscCall(MatGetLocalSize(matis->A, NULL, &ln));
    9578:   PetscCall(PetscArrayzero(matis->sf_leafdata, ln));
    9579:   PetscCall(PetscArrayzero(matis->sf_rootdata, n));
    9580:   PetscCall(ISGetLocalSize(is, &ni));
    9581:   PetscCall(ISGetIndices(is, &idxs));
    9582:   for (PetscInt i = 0; i < ni; i++) {
    9583:     if (idxs[i] < 0 || idxs[i] >= ln) continue;
    9584:     matis->sf_leafdata[idxs[i]] = 1;
    9585:   }
    9586:   PetscCall(ISRestoreIndices(is, &idxs));
    9587:   PetscCall(PetscSFReduceBegin(matis->sf, MPIU_INT, matis->sf_leafdata, matis->sf_rootdata, MPI_SUM));
    9588:   PetscCall(PetscSFReduceEnd(matis->sf, MPIU_INT, matis->sf_leafdata, matis->sf_rootdata, MPI_SUM));
    9589:   ln = 0;
    9590:   for (PetscInt i = 0; i < n; i++) {
    9591:     if (matis->sf_rootdata[i]) matis->sf_rootdata[ln++] = i + st;
    9592:   }
    9593:   PetscCall(ISCreateGeneral(PetscObjectComm((PetscObject)pc), ln, matis->sf_rootdata, PETSC_USE_POINTER, &gis));
    9594:   PetscCall(ISView(gis, viewer));
    9595:   PetscCall(ISDestroy(&gis));
    9596:   PetscFunctionReturn(PETSC_SUCCESS);
    9597: }

    9599: PetscErrorCode PCBDDCLoadOrViewCustomization(PC pc, PetscBool load, const char *outfile)
    9600: {
    9601:   PetscInt    header[11];
    9602:   PC_BDDC    *pcbddc = (PC_BDDC *)pc->data;
    9603:   PetscViewer viewer;
    9604:   MPI_Comm    comm = PetscObjectComm((PetscObject)pc);

    9606:   PetscFunctionBegin;
    9607:   PetscCall(PetscViewerBinaryOpen(comm, outfile ? outfile : "bddc_dump.dat", load ? FILE_MODE_READ : FILE_MODE_WRITE, &viewer));
    9608:   if (load) {
    9609:     IS  is;
    9610:     Mat A;

    9612:     PetscCall(PetscViewerBinaryRead(viewer, header, PETSC_STATIC_ARRAY_LENGTH(header), NULL, PETSC_INT));
    9613:     PetscCheck(header[0] == 0 || header[0] == 1, PETSC_COMM_SELF, PETSC_ERR_FILE_UNEXPECTED, "Not a BDDC dump next in file");
    9614:     PetscCheck(header[1] == 0 || header[1] == 1, PETSC_COMM_SELF, PETSC_ERR_FILE_UNEXPECTED, "Not a BDDC dump next in file");
    9615:     PetscCheck(header[2] >= 0, PETSC_COMM_SELF, PETSC_ERR_FILE_UNEXPECTED, "Not a BDDC dump next in file");
    9616:     PetscCheck(header[3] == 0 || header[3] == 1, PETSC_COMM_SELF, PETSC_ERR_FILE_UNEXPECTED, "Not a BDDC dump next in file");
    9617:     PetscCheck(header[4] == 0 || header[4] == 1, PETSC_COMM_SELF, PETSC_ERR_FILE_UNEXPECTED, "Not a BDDC dump next in file");
    9618:     PetscCheck(header[5] >= 0, PETSC_COMM_SELF, PETSC_ERR_FILE_UNEXPECTED, "Not a BDDC dump next in file");
    9619:     PetscCheck(header[7] == 0 || header[7] == 1, PETSC_COMM_SELF, PETSC_ERR_FILE_UNEXPECTED, "Not a BDDC dump next in file");
    9620:     PetscCheck(header[8] == 0 || header[8] == 1, PETSC_COMM_SELF, PETSC_ERR_FILE_UNEXPECTED, "Not a BDDC dump next in file");
    9621:     PetscCheck(header[9] == 0 || header[9] == 1, PETSC_COMM_SELF, PETSC_ERR_FILE_UNEXPECTED, "Not a BDDC dump next in file");
    9622:     PetscCheck(header[10] == 0 || header[10] == 1, PETSC_COMM_SELF, PETSC_ERR_FILE_UNEXPECTED, "Not a BDDC dump next in file");
    9623:     if (header[0]) {
    9624:       PetscCall(ISCreate(comm, &is));
    9625:       PetscCall(ISLoad(is, viewer));
    9626:       PetscCall(PCBDDCSetDirichletBoundaries(pc, is));
    9627:       PetscCall(ISDestroy(&is));
    9628:     }
    9629:     if (header[1]) {
    9630:       PetscCall(ISCreate(comm, &is));
    9631:       PetscCall(ISLoad(is, viewer));
    9632:       PetscCall(PCBDDCSetNeumannBoundaries(pc, is));
    9633:       PetscCall(ISDestroy(&is));
    9634:     }
    9635:     if (header[2]) {
    9636:       IS *isarray;

    9638:       PetscCall(PetscMalloc1(header[2], &isarray));
    9639:       for (PetscInt i = 0; i < header[2]; i++) {
    9640:         PetscCall(ISCreate(comm, &isarray[i]));
    9641:         PetscCall(ISLoad(isarray[i], viewer));
    9642:       }
    9643:       PetscCall(PCBDDCSetDofsSplitting(pc, header[2], isarray));
    9644:       for (PetscInt i = 0; i < header[2]; i++) PetscCall(ISDestroy(&isarray[i]));
    9645:       PetscCall(PetscFree(isarray));
    9646:     }
    9647:     if (header[3]) {
    9648:       PetscCall(ISCreate(comm, &is));
    9649:       PetscCall(ISLoad(is, viewer));
    9650:       PetscCall(PCBDDCSetPrimalVerticesIS(pc, is));
    9651:       PetscCall(ISDestroy(&is));
    9652:     }
    9653:     if (header[4]) {
    9654:       PetscCall(MatCreate(comm, &A));
    9655:       PetscCall(MatSetType(A, MATAIJ));
    9656:       PetscCall(MatLoad(A, viewer));
    9657:       PetscCall(PCBDDCSetDiscreteGradient(pc, A, header[5], header[6], (PetscBool)header[7], (PetscBool)header[8]));
    9658:       PetscCall(MatDestroy(&A));
    9659:     }
    9660:     if (header[9]) {
    9661:       PetscCall(MatCreate(comm, &A));
    9662:       PetscCall(MatSetType(A, MATIS));
    9663:       PetscCall(MatLoad(A, viewer));
    9664:       PetscCall(PCBDDCSetDivergenceMat(pc, A, (PetscBool)header[10], NULL));
    9665:       PetscCall(MatDestroy(&A));
    9666:     }
    9667:   } else {
    9668:     header[0]  = (PetscInt)!!pcbddc->DirichletBoundariesLocal;
    9669:     header[1]  = (PetscInt)!!pcbddc->NeumannBoundariesLocal;
    9670:     header[2]  = pcbddc->n_ISForDofsLocal;
    9671:     header[3]  = (PetscInt)!!pcbddc->user_primal_vertices_local;
    9672:     header[4]  = (PetscInt)!!pcbddc->discretegradient;
    9673:     header[5]  = pcbddc->nedorder;
    9674:     header[6]  = pcbddc->nedfield;
    9675:     header[7]  = (PetscInt)pcbddc->nedglobal;
    9676:     header[8]  = (PetscInt)pcbddc->conforming;
    9677:     header[9]  = (PetscInt)!!pcbddc->divudotp;
    9678:     header[10] = (PetscInt)pcbddc->divudotp_trans;
    9679:     if (header[4]) header[3] = 0;

    9681:     PetscCall(PetscViewerBinaryWrite(viewer, header, PETSC_STATIC_ARRAY_LENGTH(header), PETSC_INT));
    9682:     PetscCall(PCBDDCViewGlobalIS(pc, pcbddc->DirichletBoundariesLocal, viewer));
    9683:     PetscCall(PCBDDCViewGlobalIS(pc, pcbddc->NeumannBoundariesLocal, viewer));
    9684:     for (PetscInt i = 0; i < header[2]; i++) PetscCall(PCBDDCViewGlobalIS(pc, pcbddc->ISForDofsLocal[i], viewer));
    9685:     if (header[3]) PetscCall(PCBDDCViewGlobalIS(pc, pcbddc->user_primal_vertices_local, viewer));
    9686:     if (header[4]) PetscCall(MatView(pcbddc->discretegradient, viewer));
    9687:     if (header[9]) PetscCall(MatView(pcbddc->divudotp, viewer));
    9688:   }
    9689:   PetscCall(PetscViewerDestroy(&viewer));
    9690:   PetscFunctionReturn(PETSC_SUCCESS);
    9691: }

    9693: #include <../src/mat/impls/aij/mpi/mpiaij.h>
    9694: static PetscErrorCode MatMPIAIJRestrict(Mat A, MPI_Comm ccomm, Mat *B)
    9695: {
    9696:   Mat         At;
    9697:   IS          rows;
    9698:   PetscInt    rst, ren;
    9699:   PetscLayout rmap;

    9701:   PetscFunctionBegin;
    9702:   rst = ren = 0;
    9703:   if (ccomm != MPI_COMM_NULL) {
    9704:     PetscCall(PetscLayoutCreate(ccomm, &rmap));
    9705:     PetscCall(PetscLayoutSetSize(rmap, A->rmap->N));
    9706:     PetscCall(PetscLayoutSetBlockSize(rmap, 1));
    9707:     PetscCall(PetscLayoutSetUp(rmap));
    9708:     PetscCall(PetscLayoutGetRange(rmap, &rst, &ren));
    9709:   }
    9710:   PetscCall(ISCreateStride(PetscObjectComm((PetscObject)A), ren - rst, rst, 1, &rows));
    9711:   PetscCall(MatCreateSubMatrix(A, rows, NULL, MAT_INITIAL_MATRIX, &At));
    9712:   PetscCall(ISDestroy(&rows));

    9714:   if (ccomm != MPI_COMM_NULL) {
    9715:     Mat_MPIAIJ *a, *b;
    9716:     IS          from, to;
    9717:     Vec         gvec;
    9718:     PetscInt    lsize;

    9720:     PetscCall(MatCreate(ccomm, B));
    9721:     PetscCall(MatSetSizes(*B, ren - rst, PETSC_DECIDE, PETSC_DECIDE, At->cmap->N));
    9722:     PetscCall(MatSetType(*B, MATAIJ));
    9723:     PetscCall(PetscLayoutDestroy(&(*B)->rmap));
    9724:     PetscCall(PetscLayoutSetUp((*B)->cmap));
    9725:     a = (Mat_MPIAIJ *)At->data;
    9726:     b = (Mat_MPIAIJ *)(*B)->data;
    9727:     PetscCallMPI(MPI_Comm_size(ccomm, &b->size));
    9728:     PetscCallMPI(MPI_Comm_rank(ccomm, &b->rank));
    9729:     PetscCall(PetscObjectReference((PetscObject)a->A));
    9730:     PetscCall(PetscObjectReference((PetscObject)a->B));
    9731:     b->A = a->A;
    9732:     b->B = a->B;

    9734:     b->donotstash   = a->donotstash;
    9735:     b->roworiented  = a->roworiented;
    9736:     b->rowindices   = NULL;
    9737:     b->rowvalues    = NULL;
    9738:     b->getrowactive = PETSC_FALSE;

    9740:     (*B)->rmap         = rmap;
    9741:     (*B)->factortype   = A->factortype;
    9742:     (*B)->assembled    = PETSC_TRUE;
    9743:     (*B)->insertmode   = NOT_SET_VALUES;
    9744:     (*B)->preallocated = PETSC_TRUE;

    9746:     if (a->colmap) {
    9747: #if defined(PETSC_USE_CTABLE)
    9748:       PetscCall(PetscHMapIDuplicate(a->colmap, &b->colmap));
    9749: #else
    9750:       PetscCall(PetscMalloc1(At->cmap->N, &b->colmap));
    9751:       PetscCall(PetscArraycpy(b->colmap, a->colmap, At->cmap->N));
    9752: #endif
    9753:     } else b->colmap = NULL;
    9754:     if (a->garray) {
    9755:       PetscInt len;
    9756:       len = a->B->cmap->n;
    9757:       PetscCall(PetscMalloc1(len + 1, &b->garray));
    9758:       if (len) PetscCall(PetscArraycpy(b->garray, a->garray, len));
    9759:     } else b->garray = NULL;

    9761:     PetscCall(PetscObjectReference((PetscObject)a->lvec));
    9762:     b->lvec = a->lvec;

    9764:     /* cannot use VecScatterCopy */
    9765:     PetscCall(VecGetLocalSize(b->lvec, &lsize));
    9766:     PetscCall(ISCreateGeneral(ccomm, lsize, b->garray, PETSC_USE_POINTER, &from));
    9767:     PetscCall(ISCreateStride(PETSC_COMM_SELF, lsize, 0, 1, &to));
    9768:     PetscCall(MatCreateVecs(*B, &gvec, NULL));
    9769:     PetscCall(VecScatterCreate(gvec, from, b->lvec, to, &b->Mvctx));
    9770:     PetscCall(ISDestroy(&from));
    9771:     PetscCall(ISDestroy(&to));
    9772:     PetscCall(VecDestroy(&gvec));
    9773:   }
    9774:   PetscCall(MatDestroy(&At));
    9775:   PetscFunctionReturn(PETSC_SUCCESS);
    9776: }

    9778: /* same as MatCreateSubMatrix(A, rows, NULL,...) but allows repeated rows */
    9779: static PetscErrorCode MatAIJExtractRows(Mat A, IS rows, Mat *sA)
    9780: {
    9781:   PetscBool isaij;
    9782:   MPI_Comm  comm;

    9784:   PetscFunctionBegin;
    9785:   PetscCall(PetscObjectGetComm((PetscObject)A, &comm));
    9786:   PetscCall(PetscObjectBaseTypeCompareAny((PetscObject)A, &isaij, MATSEQAIJ, MATMPIAIJ, ""));
    9787:   PetscCheck(isaij, comm, PETSC_ERR_SUP, "Not implemented");
    9788:   PetscCall(PetscObjectBaseTypeCompare((PetscObject)A, MATSEQAIJ, &isaij));
    9789:   if (isaij) { /* SeqAIJ supports repeated rows */
    9790:     PetscCall(MatCreateSubMatrix(A, rows, NULL, MAT_INITIAL_MATRIX, sA));
    9791:   } else {
    9792:     Mat                A_loc;
    9793:     Mat_SeqAIJ        *da;
    9794:     PetscSF            sf;
    9795:     PetscInt           ni, *di, *dj, m = A->rmap->n, c, *ldata, *rdata;
    9796:     PetscScalar       *daa;
    9797:     const PetscInt    *idxs;
    9798:     const PetscSFNode *iremotes;
    9799:     PetscSFNode       *remotes;

    9801:     /* SF for incoming rows */
    9802:     PetscCall(PetscSFCreate(comm, &sf));
    9803:     PetscCall(ISGetLocalSize(rows, &ni));
    9804:     PetscCall(ISGetIndices(rows, &idxs));
    9805:     PetscCall(PetscSFSetGraphLayout(sf, A->rmap, ni, NULL, PETSC_USE_POINTER, idxs));
    9806:     PetscCall(ISRestoreIndices(rows, &idxs));

    9808:     PetscCall(MatMPIAIJGetLocalMat(A, MAT_INITIAL_MATRIX, &A_loc));
    9809:     da = (Mat_SeqAIJ *)A_loc->data;
    9810:     PetscCall(PetscMalloc2(2 * ni, &ldata, 2 * m, &rdata));
    9811:     for (PetscInt i = 0; i < m; i++) {
    9812:       rdata[2 * i + 0] = da->i[i + 1] - da->i[i];
    9813:       rdata[2 * i + 1] = da->i[i];
    9814:     }
    9815:     PetscCall(PetscSFBcastBegin(sf, MPIU_2INT, rdata, ldata, MPI_REPLACE));
    9816:     PetscCall(PetscSFBcastEnd(sf, MPIU_2INT, rdata, ldata, MPI_REPLACE));
    9817:     PetscCall(PetscMalloc1(ni + 1, &di));
    9818:     di[0] = 0;
    9819:     for (PetscInt i = 0; i < ni; i++) di[i + 1] = di[i] + ldata[2 * i + 0];
    9820:     PetscCall(PetscMalloc1(di[ni], &dj));
    9821:     PetscCall(PetscMalloc1(di[ni], &daa));
    9822:     PetscCall(PetscMalloc1(di[ni], &remotes));

    9824:     PetscCall(PetscSFGetGraph(sf, NULL, NULL, NULL, &iremotes));

    9826:     /* SF graph for nonzeros */
    9827:     c = 0;
    9828:     for (PetscInt i = 0; i < ni; i++) {
    9829:       const PetscInt rank  = iremotes[i].rank;
    9830:       const PetscInt rsize = ldata[2 * i];
    9831:       for (PetscInt j = 0; j < rsize; j++) {
    9832:         remotes[c].rank  = rank;
    9833:         remotes[c].index = ldata[2 * i + 1] + j;
    9834:         c++;
    9835:       }
    9836:     }
    9837:     PetscCheck(c == di[ni], PETSC_COMM_SELF, PETSC_ERR_PLIB, "Invalid number of local nonzeros %" PetscInt_FMT " != %" PetscInt_FMT, c, di[ni]);
    9838:     PetscCall(PetscSFSetGraph(sf, da->i[m], di[ni], NULL, PETSC_USE_POINTER, remotes, PETSC_USE_POINTER));
    9839:     PetscCall(PetscSFBcastBegin(sf, MPIU_INT, da->j, dj, MPI_REPLACE));
    9840:     PetscCall(PetscSFBcastEnd(sf, MPIU_INT, da->j, dj, MPI_REPLACE));
    9841:     PetscCall(PetscSFBcastBegin(sf, MPIU_SCALAR, da->a, daa, MPI_REPLACE));
    9842:     PetscCall(PetscSFBcastEnd(sf, MPIU_SCALAR, da->a, daa, MPI_REPLACE));

    9844:     PetscCall(MatCreateMPIAIJWithArrays(comm, ni, A->cmap->n, PETSC_DECIDE, A->cmap->N, di, dj, daa, sA));
    9845:     PetscCall(MatDestroy(&A_loc));
    9846:     PetscCall(PetscSFDestroy(&sf));
    9847:     PetscCall(PetscFree(di));
    9848:     PetscCall(PetscFree(dj));
    9849:     PetscCall(PetscFree(daa));
    9850:     PetscCall(PetscFree(remotes));
    9851:     PetscCall(PetscFree2(ldata, rdata));
    9852:   }
    9853:   PetscFunctionReturn(PETSC_SUCCESS);
    9854: }