Actual source code: ex76.c
petsc-3.15.0 2021-03-30
Report Typos and Errors

      1: #include <petscksp.h>

      3: static char help[] = "Solves a linear system using PCHPDDM.\n\n";

      5: int main(int argc,char **args)
      6: {
      7:   Vec                x,b;        /* computed solution and RHS */
      8:   Mat                A,aux,X,B;  /* linear system matrix */
      9:   KSP                ksp;        /* linear solver context */
     10:   PC                 pc;
     11:   IS                 is,sizes;
     12:   const PetscInt     *idx;
     13:   PetscMPIInt        rank,size;
     14:   PetscInt           m,N = 1;
     15:   const char         *deft = MATAIJ;
     16:   PetscViewer        viewer;
     17:   char               dir[PETSC_MAX_PATH_LEN],name[PETSC_MAX_PATH_LEN],type[256];
     18:   PetscBool          flg;
     19: #if defined(PETSC_USE_LOG)
     20:   PetscLogEvent      event;
     21: #endif
     22:   PetscEventPerfInfo info1,info2;
     23:   PetscErrorCode     ierr;

     25:   PetscInitialize(&argc,&args,NULL,help);if (ierr) return ierr;
     26:   PetscLogDefaultBegin();
     27:   MPI_Comm_size(PETSC_COMM_WORLD,&size);
     28:   if (size != 4) SETERRQ(PETSC_COMM_WORLD,PETSC_ERR_USER,"This example requires 4 processes");
     29:   PetscOptionsGetInt(NULL,NULL,"-rhs",&N,NULL);
     30:   MPI_Comm_rank(PETSC_COMM_WORLD,&rank);
     31:   MatCreate(PETSC_COMM_WORLD,&A);
     32:   MatCreate(PETSC_COMM_SELF,&aux);
     33:   ISCreate(PETSC_COMM_SELF,&is);
     34:   PetscStrcpy(dir,".");
     35:   PetscOptionsGetString(NULL,NULL,"-load_dir",dir,sizeof(dir),NULL);
     36:   /* loading matrices */
     37:   PetscSNPrintf(name,sizeof(name),"%s/sizes_%d_%d.dat",dir,rank,size);
     38:   PetscViewerBinaryOpen(PETSC_COMM_SELF,name,FILE_MODE_READ,&viewer);
     39:   ISCreate(PETSC_COMM_SELF,&sizes);
     40:   ISLoad(sizes,viewer);
     41:   ISGetIndices(sizes,&idx);
     42:   MatSetSizes(A,idx[0],idx[1],idx[2],idx[3]);
     43:   ISRestoreIndices(sizes,&idx);
     44:   ISDestroy(&sizes);
     45:   PetscViewerDestroy(&viewer);
     46:   MatSetUp(A);
     47:   PetscSNPrintf(name,sizeof(name),"%s/A.dat",dir);
     48:   PetscViewerBinaryOpen(PETSC_COMM_WORLD,name,FILE_MODE_READ,&viewer);
     49:   MatLoad(A,viewer);
     50:   PetscViewerDestroy(&viewer);
     51:   PetscSNPrintf(name,sizeof(name),"%s/is_%d_%d.dat",dir,rank,size);
     52:   PetscViewerBinaryOpen(PETSC_COMM_SELF,name,FILE_MODE_READ,&viewer);
     53:   ISLoad(is,viewer);
     54:   PetscViewerDestroy(&viewer);
     55:   PetscSNPrintf(name,sizeof(name),"%s/Neumann_%d_%d.dat",dir,rank,size);
     56:   PetscViewerBinaryOpen(PETSC_COMM_SELF,name,FILE_MODE_READ,&viewer);
     57:   MatLoad(aux,viewer);
     58:   PetscViewerDestroy(&viewer);
     59:   flg = PETSC_FALSE;
     60:   PetscOptionsGetBool(NULL,NULL,"-pc_hpddm_levels_1_st_share_sub_ksp",&flg,NULL);
     61:   if (flg) { /* PETSc LU/Cholesky is struggling numerically for bs > 1          */
     62:              /* only set the proper bs for the geneo_share_* tests, 1 otherwise */
     63:     MatSetBlockSizesFromMats(aux,A,A);
     64:   }
     65:   MatSetOption(A,MAT_SYMMETRIC,PETSC_TRUE);
     66:   MatSetOption(aux,MAT_SYMMETRIC,PETSC_TRUE);
     67:   /* ready for testing */
     68:   PetscOptionsBegin(PETSC_COMM_WORLD,"","","");
     69:   PetscOptionsFList("-mat_type","Matrix type","MatSetType",MatList,deft,type,256,&flg);
     70:   PetscOptionsEnd();
     71:   if (flg) {
     72:     MatConvert(A,type,MAT_INPLACE_MATRIX,&A);
     73:     MatConvert(aux,type,MAT_INPLACE_MATRIX,&aux);
     74:   }
     75:   KSPCreate(PETSC_COMM_WORLD,&ksp);
     76:   KSPSetOperators(ksp,A,A);
     77:   KSPGetPC(ksp,&pc);
     78:   PCSetType(pc,PCHPDDM);
     79: #if defined(PETSC_HAVE_HPDDM) && defined(PETSC_HAVE_DYNAMIC_LIBRARIES) && defined(PETSC_USE_SHARED_LIBRARIES)
     80:   PCHPDDMSetAuxiliaryMat(pc,is,aux,NULL,NULL);
     81:   PCHPDDMHasNeumannMat(pc,PETSC_FALSE); /* PETSC_TRUE is fine as well, just testing */
     82: #endif
     83:   ISDestroy(&is);
     84:   MatDestroy(&aux);
     85:   KSPSetFromOptions(ksp);
     86:   MatCreateVecs(A,&x,&b);
     87:   VecSet(b,1.0);
     88:   KSPSolve(ksp,b,x);
     89:   VecGetLocalSize(x,&m);
     90:   VecDestroy(&x);
     91:   VecDestroy(&b);
     92:   if (N > 1) {
     93:     KSPType type;
     94:     PetscOptionsClearValue(NULL,"-ksp_converged_reason");
     95:     KSPSetFromOptions(ksp);
     96:     MatCreateDense(PETSC_COMM_WORLD,m,PETSC_DECIDE,PETSC_DECIDE,N,NULL,&B);
     97:     MatCreateDense(PETSC_COMM_WORLD,m,PETSC_DECIDE,PETSC_DECIDE,N,NULL,&X);
     98:     MatSetRandom(B,NULL);
     99:     /* this is algorithmically optimal in the sense that blocks of vectors are coarsened or interpolated using matrix--matrix operations */
    100:     /* PCHPDDM however heavily relies on MPI[S]BAIJ format for which there is no efficient MatProduct implementation */
    101:     KSPMatSolve(ksp,B,X);
    102:     KSPGetType(ksp,&type);
    103:     PetscStrcmp(type,KSPHPDDM,&flg);
    104: #if defined(PETSC_HAVE_HPDDM)
    105:     if (flg) {
    106:       PetscReal    norm;
    107:       KSPHPDDMType type;
    108:       KSPHPDDMGetType(ksp,&type);
    109:       if (type == KSP_HPDDM_TYPE_PREONLY || type == KSP_HPDDM_TYPE_CG || type == KSP_HPDDM_TYPE_GMRES || type == KSP_HPDDM_TYPE_GCRODR) {
    110:         Mat C;
    111:         MatDuplicate(X,MAT_DO_NOT_COPY_VALUES,&C);
    112:         KSPSetMatSolveBatchSize(ksp,1);
    113:         KSPMatSolve(ksp,B,C);
    114:         MatAYPX(C,-1.0,X,SAME_NONZERO_PATTERN);
    115:         MatNorm(C,NORM_INFINITY,&norm);
    116:         MatDestroy(&C);
    117:         if (norm > 100*PETSC_MACHINE_EPSILON) SETERRQ2(PetscObjectComm((PetscObject)pc),PETSC_ERR_PLIB,"KSPMatSolve() and KSPSolve() difference has nonzero norm %g with pseudo-block KSPHPDDMType %s",(double)norm,KSPHPDDMTypes[type]);
    118:       }
    119:     }
    120: #endif
    121:     MatDestroy(&X);
    122:     MatDestroy(&B);
    123:   }
    124:   PetscObjectTypeCompare((PetscObject)pc,PCHPDDM,&flg);
    125: #if defined(PETSC_HAVE_HPDDM) && defined(PETSC_HAVE_DYNAMIC_LIBRARIES) && defined(PETSC_USE_SHARED_LIBRARIES)
    126:   if (flg) {
    127:     PCHPDDMGetSTShareSubKSP(pc,&flg);
    128:   }
    129: #endif
    130:   if (flg && PetscDefined(USE_LOG)) {
    131:     PetscLogEventRegister("MatLUFactorSym",PC_CLASSID,&event);
    132:     PetscLogEventGetPerfInfo(PETSC_DETERMINE,event,&info1);
    133:     PetscLogEventRegister("MatLUFactorNum",PC_CLASSID,&event);
    134:     PetscLogEventGetPerfInfo(PETSC_DETERMINE,event,&info2);
    135:     if (info1.count || info2.count) {
    136:       if (info2.count <= info1.count) SETERRQ2(PETSC_COMM_SELF,PETSC_ERR_PLIB,"LU numerical factorization (%d) not called more times than LU symbolic factorization (%d), broken -pc_hpddm_levels_1_st_share_sub_ksp",info2.count,info1.count);
    137:     } else {
    138:       PetscLogEventRegister("MatCholFctrSym",PC_CLASSID,&event);
    139:       PetscLogEventGetPerfInfo(PETSC_DETERMINE,event,&info1);
    140:       PetscLogEventRegister("MatCholFctrNum",PC_CLASSID,&event);
    141:       PetscLogEventGetPerfInfo(PETSC_DETERMINE,event,&info2);
    142:       if (info2.count <= info1.count) SETERRQ2(PETSC_COMM_SELF,PETSC_ERR_PLIB,"Cholesky numerical factorization (%d) not called more times than Cholesky symbolic factorization (%d), broken -pc_hpddm_levels_1_st_share_sub_ksp",info2.count,info1.count);
    143:     }
    144:   }
    145:   KSPDestroy(&ksp);
    146:   MatDestroy(&A);
    147:   PetscFinalize();
    148:   return ierr;
    149: }

    151: /*TEST

    153:    test:
    154:       requires: hpddm slepc datafilespath double !complex !define(PETSC_USE_64BIT_INDICES) define(PETSC_HAVE_DYNAMIC_LIBRARIES) define(PETSC_USE_SHARED_LIBRARIES)
    155:       nsize: 4
    156:       args: -ksp_rtol 1e-3 -ksp_converged_reason -pc_type {{bjacobi hpddm}shared output} -pc_hpddm_coarse_sub_pc_type lu -sub_pc_type lu -options_left no -load_dir ${DATAFILESPATH}/matrices/hpddm/GENEO

    158:    test:
    159:       requires: hpddm slepc datafilespath double !complex !define(PETSC_USE_64BIT_INDICES) define(PETSC_HAVE_DYNAMIC_LIBRARIES) define(PETSC_USE_SHARED_LIBRARIES)
    160:       suffix: geneo
    161:       nsize: 4
    162:       args: -ksp_converged_reason -pc_type hpddm -pc_hpddm_levels_1_sub_pc_type cholesky -pc_hpddm_levels_1_eps_nev {{5 15}separate output} -pc_hpddm_levels_1_st_pc_type cholesky -pc_hpddm_coarse_p {{1 2}shared output} -pc_hpddm_coarse_pc_type redundant -mat_type {{aij baij sbaij}shared output} -load_dir ${DATAFILESPATH}/matrices/hpddm/GENEO

    164:    testset:
    165:       requires: hpddm slepc datafilespath double !complex !define(PETSC_USE_64BIT_INDICES) define(PETSC_HAVE_DYNAMIC_LIBRARIES) define(PETSC_USE_SHARED_LIBRARIES)
    166:       nsize: 4
    167:       args: -ksp_converged_reason -ksp_max_it 150 -pc_type hpddm -pc_hpddm_levels_1_eps_nev 5 -pc_hpddm_coarse_p 1 -pc_hpddm_coarse_pc_type redundant -load_dir ${DATAFILESPATH}/matrices/hpddm/GENEO -pc_hpddm_define_subdomains -pc_hpddm_has_neumann -pc_hpddm_levels_1_st_share_sub_ksp {{false true}shared output}
    168:       test:
    169:         suffix: geneo_share_cholesky
    170:         output_file: output/ex76_geneo_share.out
    171:         # extra -pc_hpddm_levels_1_eps_gen_non_hermitian needed to avoid failures with PETSc Cholesky
    172:         args: -pc_hpddm_levels_1_sub_pc_type cholesky -pc_hpddm_levels_1_st_pc_type cholesky -mat_type {{aij baij sbaij}shared output} -pc_hpddm_levels_1_eps_gen_non_hermitian
    173:       test:
    174:         requires: mumps
    175:         suffix: geneo_share_lu
    176:         output_file: output/ex76_geneo_share.out
    177:         # extra -pc_factor_mat_solver_type mumps needed to avoid failures with PETSc LU
    178:         args: -pc_hpddm_levels_1_sub_pc_type lu -pc_hpddm_levels_1_st_pc_type lu -mat_type baij -pc_hpddm_levels_1_st_pc_factor_mat_solver_type mumps -pc_hpddm_levels_1_sub_pc_factor_mat_solver_type mumps

    180:    test:
    181:       requires: hpddm slepc datafilespath double !complex !define(PETSC_USE_64BIT_INDICES) define(PETSC_HAVE_DYNAMIC_LIBRARIES) define(PETSC_USE_SHARED_LIBRARIES)
    182:       suffix: fgmres_geneo_20_p_2
    183:       nsize: 4
    184:       args: -ksp_converged_reason -pc_type hpddm -pc_hpddm_levels_1_sub_pc_type lu -pc_hpddm_levels_1_eps_nev 20 -pc_hpddm_coarse_p 2 -pc_hpddm_coarse_pc_type redundant -ksp_type fgmres -pc_hpddm_coarse_mat_type {{baij sbaij}shared output} -pc_hpddm_log_separate {{false true}shared output} -load_dir ${DATAFILESPATH}/matrices/hpddm/GENEO

    186:    test:
    187:       requires: hpddm slepc datafilespath double !complex !define(PETSC_USE_64BIT_INDICES) define(PETSC_HAVE_DYNAMIC_LIBRARIES) define(PETSC_USE_SHARED_LIBRARIES)
    188:       suffix: fgmres_geneo_20_p_2_geneo
    189:       output_file: output/ex76_fgmres_geneo_20_p_2.out
    190:       nsize: 4
    191:       args: -ksp_converged_reason -pc_type hpddm -pc_hpddm_levels_1_sub_pc_type cholesky -pc_hpddm_levels_1_eps_nev 20 -pc_hpddm_levels_2_p 2 -pc_hpddm_levels_2_mat_type {{baij sbaij}shared output} -pc_hpddm_levels_2_eps_nev {{5 20}shared output} -pc_hpddm_levels_2_sub_pc_type cholesky -pc_hpddm_levels_2_ksp_type gmres -ksp_type fgmres -pc_hpddm_coarse_mat_type {{baij sbaij}shared output} -mat_type {{aij sbaij}shared output} -load_dir ${DATAFILESPATH}/matrices/hpddm/GENEO
    192:    # PCHPDDM + KSPHPDDM test to exercise multilevel + multiple RHS in one go
    193:    test:
    194:       requires: hpddm slepc datafilespath double !complex !define(PETSC_USE_64BIT_INDICES) define(PETSC_HAVE_DYNAMIC_LIBRARIES) define(PETSC_USE_SHARED_LIBRARIES)
    195:       suffix: fgmres_geneo_20_p_2_geneo_rhs
    196:       output_file: output/ex76_fgmres_geneo_20_p_2.out
    197:       # for -pc_hpddm_coarse_correction additive
    198:       filter: sed -e "s/Linear solve converged due to CONVERGED_RTOL iterations 37/Linear solve converged due to CONVERGED_RTOL iterations 25/g"
    199:       nsize: 4
    200:       args: -ksp_converged_reason -pc_type hpddm -pc_hpddm_levels_1_sub_pc_type cholesky -pc_hpddm_levels_1_eps_nev 20 -pc_hpddm_levels_2_p 2 -pc_hpddm_levels_2_mat_type baij -pc_hpddm_levels_2_eps_nev 5 -pc_hpddm_levels_2_sub_pc_type cholesky -pc_hpddm_levels_2_ksp_max_it 10 -pc_hpddm_levels_2_ksp_type hpddm -pc_hpddm_levels_2_ksp_hpddm_type gmres -ksp_type hpddm -ksp_hpddm_variant flexible -pc_hpddm_coarse_mat_type baij -mat_type aij -load_dir ${DATAFILESPATH}/matrices/hpddm/GENEO -rhs 4 -pc_hpddm_coarse_correction {{additive deflated balanced}shared output}

    202: TEST*/