40submodule(
mma) mma_hdf5_io
42 use mpi_f08,
only: mpi_scan, mpi_info_null, mpi_integer8, mpi_max, mpi_min, &
44 use math,
only: abscmp
45 use comm,
only: pe_size
46 use hdf5_session,
only: hdf5_session_init, hdf5_session_finalize
59 module subroutine mma_save_checkpoint_hdf5(object, filename, overwrite)
60 class(mma_t),
intent(inout) :: object
61 character(len=*),
intent(in) :: filename
62 logical,
intent(in),
optional :: overwrite
63 integer(hid_t) :: fapl_id, xf_id, file_id, dset_id, filespace, memspace, &
64 attr_id, grp_id, str_type, mma_grp_id
65 integer(hid_t) :: H5T_NEKO_REAL
66 integer(hsize_t),
dimension(1) :: ddim, dcount, doffset
67 integer :: ierr, info, drank
68 logical :: file_exists, mma_exists, overwrite_flag
69 integer :: n_accum, n_array(pe_size)
71 overwrite_flag = .false.
72 if (
present(overwrite)) overwrite_flag = overwrite
76 call object%copy_from(device_to_host, sync = .true.)
80 n_array(pe_rank + 1) = object%n
81 call mpi_allreduce(mpi_in_place, n_array, pe_size, mpi_integer, mpi_max, &
87 call hdf5_session_init()
88 call h5pcreate_f(h5p_file_access_f, fapl_id, ierr)
89 info = mpi_info_null%mpi_val
90 call h5pset_fapl_mpio_f(fapl_id, neko_comm%mpi_val, info, ierr)
94 inquire(file=trim(filename), exist=file_exists)
95 if (.not. file_exists)
then
96 call h5fcreate_f(trim(filename), h5f_acc_trunc_f, file_id, ierr, &
100 call h5gcreate_f(file_id,
"MMA", mma_grp_id, ierr)
101 call h5gcreate_f(mma_grp_id,
"checkpoint", grp_id, ierr)
104 call h5fopen_f(trim(filename), h5f_acc_rdwr_f, file_id, ierr, fapl_id)
108 call h5lexists_f(file_id,
'/MMA', mma_exists, ierr)
110 if (.not. mma_exists)
then
112 call h5gcreate_f(file_id,
"MMA", mma_grp_id, ierr)
113 call h5gcreate_f(mma_grp_id,
"checkpoint", grp_id, ierr)
116 call h5gopen_f(file_id,
'MMA', mma_grp_id, ierr)
117 call h5lexists_f(mma_grp_id,
'checkpoint', mma_exists, ierr)
119 if (mma_exists .and. overwrite_flag)
then
120 call h5gunlink_f(mma_grp_id,
"checkpoint", ierr)
121 call h5gcreate_f(mma_grp_id,
"checkpoint", grp_id, ierr)
123 call neko_error(
'mma: HDF5 file "' // trim(filename) //
'" ' // &
124 'already contains MMA group; use overwrite option to replace')
128 call h5gclose_f(mma_grp_id, ierr)
133 h5t_neko_real = h5t_native_double
135 h5t_neko_real = h5t_native_real
137 call neko_error(
'mma: unsupported real kind for HDF5')
145 call h5screate_f(h5s_scalar_f, filespace, ierr)
147 call h5acreate_f(grp_id,
'n_global', h5t_native_integer, filespace, &
148 attr_id, ierr, h5p_default_f, h5p_default_f)
149 call h5awrite_f(attr_id, h5t_native_integer, object%n_global, ddim, ierr)
150 call h5aclose_f(attr_id, ierr)
152 call h5acreate_f(grp_id,
'm', h5t_native_integer, filespace, attr_id, &
153 ierr, h5p_default_f, h5p_default_f)
154 call h5awrite_f(attr_id, h5t_native_integer, object%m, ddim, ierr)
155 call h5aclose_f(attr_id, ierr)
157 call h5acreate_f(grp_id,
'max_iter', h5t_native_integer, filespace, &
158 attr_id, ierr, h5p_default_f, h5p_default_f)
159 call h5awrite_f(attr_id, h5t_native_integer, object%max_iter, ddim, ierr)
160 call h5aclose_f(attr_id, ierr)
164 call h5acreate_f(grp_id,
'asyinit', h5t_neko_real, filespace, attr_id, &
165 ierr, h5p_default_f, h5p_default_f)
166 call h5awrite_f(attr_id, h5t_neko_real, object%asyinit, ddim, ierr)
167 call h5aclose_f(attr_id, ierr)
169 call h5acreate_f(grp_id,
'asyincr', h5t_neko_real, filespace, attr_id, &
170 ierr, h5p_default_f, h5p_default_f)
171 call h5awrite_f(attr_id, h5t_neko_real, object%asyincr, ddim, ierr)
172 call h5aclose_f(attr_id, ierr)
174 call h5acreate_f(grp_id,
'asydecr', h5t_neko_real, filespace, attr_id, &
175 ierr, h5p_default_f, h5p_default_f)
176 call h5awrite_f(attr_id, h5t_neko_real, object%asydecr, ddim, ierr)
177 call h5aclose_f(attr_id, ierr)
179 call h5acreate_f(grp_id,
'epsimin', h5t_neko_real, filespace, attr_id, &
180 ierr, h5p_default_f, h5p_default_f)
181 call h5awrite_f(attr_id, h5t_neko_real, object%epsimin, ddim, ierr)
182 call h5aclose_f(attr_id, ierr)
184 call h5acreate_f(grp_id,
'move_limit', h5t_neko_real, filespace, attr_id, &
185 ierr, h5p_default_f, h5p_default_f)
186 call h5awrite_f(attr_id, h5t_neko_real, object%move_limit, ddim, ierr)
187 call h5aclose_f(attr_id, ierr)
192 call h5tcopy_f(h5t_fortran_s1, str_type, ierr)
193 call h5tset_strpad_f(str_type, h5t_str_spacepad_f, ierr)
195 ddim(1) = len_trim(object%bcknd)
196 call h5tset_size_f(str_type, ddim(1), ierr)
197 call h5acreate_f(grp_id,
'bcknd', str_type, filespace, attr_id, &
198 ierr, h5p_default_f, h5p_default_f)
199 call h5awrite_f(attr_id, str_type, object%bcknd, ddim, ierr)
200 call h5aclose_f(attr_id, ierr)
202 ddim(1) = len_trim(object%subsolver)
203 call h5tset_size_f(str_type, ddim(1), ierr)
204 call h5acreate_f(grp_id,
'subsolver', str_type, filespace, attr_id, &
205 ierr, h5p_default_f, h5p_default_f)
206 call h5awrite_f(attr_id, str_type, object%subsolver, ddim, ierr)
207 call h5aclose_f(attr_id, ierr)
210 call h5tclose_f(str_type, ierr)
213 call h5sclose_f(filespace, ierr)
218 call h5screate_simple_f(drank, ddim, filespace, ierr)
220 call h5acreate_f(grp_id,
'n', h5t_native_integer, filespace, attr_id, ierr)
221 call h5awrite_f(attr_id, h5t_native_integer, n_array, ddim, ierr)
222 call h5aclose_f(attr_id, ierr)
224 call h5sclose_f(filespace, ierr)
230 call h5screate_f(h5s_scalar_f, filespace, ierr)
232 call h5dcreate_f(grp_id,
'a0', h5t_neko_real, filespace, dset_id, ierr)
233 call h5dwrite_f(dset_id, h5t_neko_real, object%a0, ddim, ierr)
234 call h5dclose_f(dset_id, ierr)
235 call h5sclose_f(filespace, ierr)
241 call h5screate_simple_f(drank, ddim, filespace, ierr)
243 call h5dcreate_f(grp_id,
'a', h5t_neko_real, filespace, dset_id, ierr)
244 call h5dwrite_f(dset_id, h5t_neko_real, object%a%x, ddim, ierr)
245 call h5dclose_f(dset_id, ierr)
247 call h5dcreate_f(grp_id,
'c', h5t_neko_real, filespace, dset_id, ierr)
248 call h5dwrite_f(dset_id, h5t_neko_real, object%c%x, ddim, ierr)
249 call h5dclose_f(dset_id, ierr)
251 call h5dcreate_f(grp_id,
'd', h5t_neko_real, filespace, dset_id, ierr)
252 call h5dwrite_f(dset_id, h5t_neko_real, object%d%x, ddim, ierr)
253 call h5dclose_f(dset_id, ierr)
256 call h5sclose_f(filespace, ierr)
262 call mpi_scan(object%n, n_accum, 1, mpi_integer, mpi_sum, neko_comm, ierr)
266 doffset(1) = n_accum - object%n
267 ddim(1) = object%n_global
270 call h5screate_simple_f(drank, ddim, filespace, ierr)
271 call h5screate_simple_f(drank, dcount, memspace, ierr)
274 call h5pcreate_f(h5p_dataset_xfer_f, xf_id, ierr)
275 call h5pset_dxpl_mpio_f(xf_id, h5fd_mpio_collective_f, ierr)
278 call h5sselect_hyperslab_f(filespace, h5s_select_set_f, doffset, dcount, &
282 call h5dcreate_f(grp_id,
'xmin', h5t_neko_real, &
283 filespace, dset_id, ierr)
284 call h5dwrite_f(dset_id, h5t_neko_real, object%xmin%x, ddim, ierr, &
285 file_space_id = filespace, mem_space_id = memspace, xfer_prp = xf_id)
286 call h5dclose_f(dset_id, ierr)
289 call h5dcreate_f(grp_id,
'xmax', h5t_neko_real, &
290 filespace, dset_id, ierr)
291 call h5dwrite_f(dset_id, h5t_neko_real, object%xmax%x, ddim, ierr, &
292 file_space_id = filespace, mem_space_id = memspace, xfer_prp = xf_id)
293 call h5dclose_f(dset_id, ierr)
296 call h5dcreate_f(grp_id,
'xold1', h5t_neko_real, &
297 filespace, dset_id, ierr)
298 call h5dwrite_f(dset_id, h5t_neko_real, object%xold1%x, ddim, ierr, &
299 file_space_id = filespace, mem_space_id = memspace, xfer_prp = xf_id)
300 call h5dclose_f(dset_id, ierr)
303 call h5dcreate_f(grp_id,
'xold2', h5t_neko_real, &
304 filespace, dset_id, ierr)
305 call h5dwrite_f(dset_id, h5t_neko_real, object%xold2%x, ddim, ierr, &
306 file_space_id = filespace, mem_space_id = memspace, xfer_prp = xf_id)
307 call h5dclose_f(dset_id, ierr)
310 call h5dcreate_f(grp_id,
'low', h5t_neko_real, &
311 filespace, dset_id, ierr)
312 call h5dwrite_f(dset_id, h5t_neko_real, object%low%x, ddim, ierr, &
313 file_space_id = filespace, mem_space_id = memspace, xfer_prp = xf_id)
314 call h5dclose_f(dset_id, ierr)
317 call h5dcreate_f(grp_id,
'upp', h5t_neko_real, &
318 filespace, dset_id, ierr)
319 call h5dwrite_f(dset_id, h5t_neko_real, object%upp%x, ddim, ierr, &
320 file_space_id = filespace, mem_space_id = memspace, xfer_prp = xf_id)
321 call h5dclose_f(dset_id, ierr)
324 call h5sclose_f(filespace, ierr)
325 call h5sclose_f(memspace, ierr)
326 call h5pclose_f(xf_id, ierr)
331 call h5gclose_f(grp_id, ierr)
332 call h5fclose_f(file_id, ierr)
333 call h5pclose_f(fapl_id, ierr)
335 call hdf5_session_finalize()
337 end subroutine mma_save_checkpoint_hdf5
345 module subroutine mma_load_checkpoint_hdf5(object, filename)
346 class(mma_t),
intent(inout) :: object
347 character(len=*),
intent(in) :: filename
348 integer(hid_t) :: fapl_id, file_id, dset_id, attr_id, grp_id
349 integer(hid_t) :: str_type, filespace, memspace, xf_id
350 integer(hid_t) :: H5T_NEKO_REAL
351 integer(hsize_t),
dimension(1) :: ddim, dcount, doffset
352 integer :: ierr, info, mpi_comm
353 integer :: n, n_global, m, max_iter, n_accum, n_array(pe_size)
354 real(kind=rp) :: asyinit, asyincr, asydecr, epsimin, move_limit
355 logical :: move_limit_exists
357 character(len=*),
parameter :: h5_group =
'/MMA/checkpoint'
358 character(len=12) :: bcknd, subsolver
368 move_limit = object%move_limit
374 info = mpi_info_null%mpi_val
375 mpi_comm = neko_comm%mpi_val
378 call hdf5_session_init()
379 call h5pcreate_f(h5p_file_access_f, fapl_id, ierr)
380 call h5pset_fapl_mpio_f(fapl_id, mpi_comm, info, ierr)
381 call h5fopen_f(trim(filename), h5f_acc_rdonly_f, file_id, ierr, &
382 access_prp = fapl_id)
383 call h5gopen_f(file_id, h5_group, grp_id, ierr)
385 if (ierr .ne. 0)
then
386 call neko_error(
'mma: unable to open HDF5 file "' // trim(filename) // &
387 '" or group "' // trim(h5_group) //
'".')
393 h5t_neko_real = h5t_native_double
395 h5t_neko_real = h5t_native_real
397 call neko_error(
'mma: unsupported real kind for HDF5')
405 call h5aopen_f(grp_id,
'n_global', attr_id, ierr)
406 call h5aread_f(attr_id, h5t_native_integer, n_global, ddim, ierr)
407 call h5aclose_f(attr_id, ierr)
409 call h5aopen_f(grp_id,
'm', attr_id, ierr)
410 call h5aread_f(attr_id, h5t_native_integer, m, ddim, ierr)
411 call h5aclose_f(attr_id, ierr)
413 call h5aopen_f(grp_id,
'max_iter', attr_id, ierr)
414 call h5aread_f(attr_id, h5t_native_integer, max_iter, ddim, ierr)
415 call h5aclose_f(attr_id, ierr)
417 call h5aopen_f(grp_id,
'asyinit', attr_id, ierr)
418 call h5aread_f(attr_id, h5t_neko_real, asyinit, ddim, ierr)
419 call h5aclose_f(attr_id, ierr)
421 call h5aopen_f(grp_id,
'asyincr', attr_id, ierr)
422 call h5aread_f(attr_id, h5t_neko_real, asyincr, ddim, ierr)
423 call h5aclose_f(attr_id, ierr)
425 call h5aopen_f(grp_id,
'asydecr', attr_id, ierr)
426 call h5aread_f(attr_id, h5t_neko_real, asydecr, ddim, ierr)
427 call h5aclose_f(attr_id, ierr)
429 call h5aopen_f(grp_id,
'epsimin', attr_id, ierr)
430 call h5aread_f(attr_id, h5t_neko_real, epsimin, ddim, ierr)
431 call h5aclose_f(attr_id, ierr)
433 move_limit_exists = .false.
434 call h5aexists_f(grp_id,
'move_limit', move_limit_exists, ierr)
435 if (move_limit_exists)
then
436 call h5aopen_f(grp_id,
'move_limit', attr_id, ierr)
437 call h5aread_f(attr_id, h5t_neko_real, move_limit, ddim, ierr)
438 call h5aclose_f(attr_id, ierr)
442 call h5aopen_f(grp_id,
'bcknd', attr_id, ierr)
443 call h5aget_type_f(attr_id, str_type, ierr)
444 call h5aread_f(attr_id, str_type, bcknd, ddim, ierr)
445 call h5tclose_f(str_type, ierr)
446 call h5aclose_f(attr_id, ierr)
448 call h5aopen_f(grp_id,
'subsolver', attr_id, ierr)
449 call h5aget_type_f(attr_id, str_type, ierr)
450 call h5aread_f(attr_id, str_type, subsolver, ddim, ierr)
451 call h5tclose_f(str_type, ierr)
452 call h5aclose_f(attr_id, ierr)
458 call h5aopen_f(grp_id,
'n', attr_id, ierr)
459 call h5aread_f(attr_id, h5t_native_integer, n_array, ddim, ierr)
460 call h5aclose_f(attr_id, ierr)
465 if (n_array(pe_rank + 1) .ne. object%n)
then
466 call neko_error(
'mma: mismatch in n during HDF5 read')
468 if (n_global .ne. object%n_global)
then
469 call neko_error(
'mma: mismatch in n_global during HDF5 read')
471 if (m .ne. object%m)
then
472 call neko_error(
'mma: mismatch in m during HDF5 read')
474 if (max_iter .ne. object%max_iter)
then
475 call neko_error(
'mma: mismatch in max_iter during HDF5 read')
477 if (.not. abscmp(asyinit, object%asyinit))
then
478 call neko_error(
'mma: mismatch in asyinit during HDF5 read')
480 if (.not. abscmp(asyincr, object%asyincr))
then
481 call neko_error(
'mma: mismatch in asyincr during HDF5 read')
483 if (.not. abscmp(asydecr, object%asydecr))
then
484 call neko_error(
'mma: mismatch in asydecr during HDF5 read')
486 if (.not. abscmp(epsimin, object%epsimin))
then
487 call neko_error(
'mma: mismatch in epsimin during HDF5 read')
489 if (move_limit_exists)
then
490 if (.not. abscmp(move_limit, object%move_limit))
then
491 call neko_error(
'mma: mismatch in move_limit during HDF5 read')
494 if (trim(bcknd) .ne. trim(object%bcknd))
then
495 call neko_error(
'mma: mismatch in bcknd during HDF5 read')
497 if (trim(subsolver) .ne. trim(object%subsolver))
then
498 call neko_error(
'mma: mismatch in subsolver during HDF5 read')
506 call h5dopen_f(grp_id,
'a0', dset_id, ierr)
507 call h5dread_f(dset_id, h5t_neko_real, object%a0, ddim, ierr)
508 call h5dclose_f(dset_id, ierr)
511 call h5dopen_f(grp_id,
'a', dset_id, ierr)
512 call h5dread_f(dset_id, h5t_neko_real, object%a%x, ddim, ierr)
513 call h5dclose_f(dset_id, ierr)
515 call h5dopen_f(grp_id,
'c', dset_id, ierr)
516 call h5dread_f(dset_id, h5t_neko_real, object%c%x, ddim, ierr)
517 call h5dclose_f(dset_id, ierr)
519 call h5dopen_f(grp_id,
'd', dset_id, ierr)
520 call h5dread_f(dset_id, h5t_neko_real, object%d%x, ddim, ierr)
521 call h5dclose_f(dset_id, ierr)
529 call mpi_scan(object%n, n_accum, 1, mpi_integer, mpi_sum, neko_comm, ierr)
532 doffset(1) = n_accum - object%n
533 ddim(1) = object%n_global
536 call h5screate_simple_f(1, ddim, filespace, ierr)
537 call h5screate_simple_f(1, dcount, memspace, ierr)
540 call h5pcreate_f(h5p_dataset_xfer_f, xf_id, ierr)
541 call h5pset_dxpl_mpio_f(xf_id, h5fd_mpio_collective_f, ierr)
544 call h5sselect_hyperslab_f(filespace, h5s_select_set_f, doffset, dcount, &
547 call h5dopen_f(grp_id,
'xmin', dset_id, ierr)
548 call h5dread_f(dset_id, h5t_neko_real, object%xmin%x, dcount, ierr, &
549 file_space_id = filespace, mem_space_id = memspace, xfer_prp = xf_id)
550 call h5dclose_f(dset_id, ierr)
552 call h5dopen_f(grp_id,
'xmax', dset_id, ierr)
553 call h5dread_f(dset_id, h5t_neko_real, object%xmax%x, dcount, ierr, &
554 file_space_id = filespace, mem_space_id = memspace, xfer_prp = xf_id)
555 call h5dclose_f(dset_id, ierr)
557 call h5dopen_f(grp_id,
'xold1', dset_id, ierr)
558 call h5dread_f(dset_id, h5t_neko_real, object%xold1%x, dcount, ierr, &
559 file_space_id = filespace, mem_space_id = memspace, xfer_prp = xf_id)
560 call h5dclose_f(dset_id, ierr)
562 call h5dopen_f(grp_id,
'xold2', dset_id, ierr)
563 call h5dread_f(dset_id, h5t_neko_real, object%xold2%x, dcount, ierr, &
564 file_space_id = filespace, mem_space_id = memspace, xfer_prp = xf_id)
565 call h5dclose_f(dset_id, ierr)
567 call h5dopen_f(grp_id,
'low', dset_id, ierr)
568 call h5dread_f(dset_id, h5t_neko_real, object%low%x, dcount, ierr, &
569 file_space_id = filespace, mem_space_id = memspace, xfer_prp = xf_id)
570 call h5dclose_f(dset_id, ierr)
572 call h5dopen_f(grp_id,
'upp', dset_id, ierr)
573 call h5dread_f(dset_id, h5t_neko_real, object%upp%x, dcount, ierr, &
574 file_space_id = filespace, mem_space_id = memspace, xfer_prp = xf_id)
575 call h5dclose_f(dset_id, ierr)
578 call h5pclose_f(xf_id, ierr)
579 call h5sclose_f(memspace, ierr)
580 call h5sclose_f(filespace, ierr)
585 call h5gclose_f(grp_id, ierr)
586 call h5fclose_f(file_id, ierr)
587 call h5pclose_f(fapl_id, ierr)
589 call hdf5_session_finalize()
593 call object%copy_from(host_to_device, sync = .true.)
595 end subroutine mma_load_checkpoint_hdf5
597end submodule mma_hdf5_io