Neko-TOP
A portable framework for high-order spectral element flow toplogy optimization.
Loading...
Searching...
No Matches
steepest_descent_hdf5_io.f90
1
39
46submodule(steepest_descent_optimizer) steepest_descent_hdf5_io
47 use hdf5
48 use hdf5_session, only: hdf5_session_init, hdf5_session_finalize
49 use num_types, only: dp, sp
50 use mpi_f08, only: mpi_allreduce, mpi_scan, mpi_in_place, mpi_info_null, &
51 mpi_integer, mpi_sum, mpi_max
52 use comm, only: neko_comm, pe_rank, pe_size
53 use device, only: device_to_host, host_to_device
54
55 implicit none
56
58 character(len=*), parameter :: SD_GROUP = '/SteepestDescent/checkpoint'
59
60contains
61
68 module subroutine sd_optimizer_save_checkpoint_hdf5(object, filename, &
69 overwrite)
70 class(steepest_descent_optimizer_t), intent(inout) :: object
71 character(len=*), intent(in) :: filename
72 logical, intent(in), optional :: overwrite
73 integer(hid_t) :: fapl_id, file_id, sd_grp_id, grp_id, filespace, attr_id
74 integer(hsize_t) :: ddim(1)
75 integer :: ierr, info, n, n_global, n_accum, n_array(pe_size)
76 logical :: overwrite_flag, link_exists
77
78 overwrite_flag = .false.
79 if (present(overwrite)) overwrite_flag = overwrite
80
81 ! Ensure device state is on host
82 call object%xold1%copy_from(device_to_host, .true.)
83 call object%xold2%copy_from(device_to_host, .true.)
84 call object%delta%copy_from(device_to_host, .true.)
85
86 ! Gather the per-rank sizes and the offset of this rank
87 n = object%xold1%size()
88 n_array = 0
89 n_array(pe_rank + 1) = n
90 call mpi_allreduce(mpi_in_place, n_array, pe_size, mpi_integer, mpi_sum, &
91 neko_comm, ierr)
92 n_global = sum(n_array)
93 call mpi_scan(n, n_accum, 1, mpi_integer, mpi_sum, neko_comm, ierr)
94
95 ! ------------------------------------------------------------------------ !
96 ! Open the file and prepare the checkpoint group
97
98 call hdf5_session_init()
99 call h5pcreate_f(h5p_file_access_f, fapl_id, ierr)
100 info = mpi_info_null%mpi_val
101 call h5pset_fapl_mpio_f(fapl_id, neko_comm%mpi_val, info, ierr)
102
103 call h5fopen_f(trim(filename), h5f_acc_rdwr_f, file_id, ierr, &
104 access_prp = fapl_id)
105 if (ierr .ne. 0) then
106 call neko_error('steepest_descent: unable to open HDF5 file "' // &
107 trim(filename) // '"')
108 end if
109
110 link_exists = .false.
111 call h5lexists_f(file_id, 'SteepestDescent', link_exists, ierr)
112 if (link_exists) then
113 call h5gopen_f(file_id, 'SteepestDescent', sd_grp_id, ierr)
114 else
115 call h5gcreate_f(file_id, 'SteepestDescent', sd_grp_id, ierr)
116 end if
117
118 link_exists = .false.
119 call h5lexists_f(sd_grp_id, 'checkpoint', link_exists, ierr)
120 if (link_exists) then
121 if (.not. overwrite_flag) then
122 call neko_error('steepest_descent: HDF5 file "' // &
123 trim(filename) // '" already contains a steepest descent ' // &
124 'checkpoint; use overwrite option to replace')
125 end if
126 call h5ldelete_f(sd_grp_id, 'checkpoint', ierr)
127 end if
128 call h5gcreate_f(sd_grp_id, 'checkpoint', grp_id, ierr)
129 call h5gclose_f(sd_grp_id, ierr)
130
131 ! ------------------------------------------------------------------------ !
132 ! Write the size attributes
133
134 ddim(1) = 1
135 call h5screate_f(h5s_scalar_f, filespace, ierr)
136 call h5acreate_f(grp_id, 'n_global', h5t_native_integer, filespace, &
137 attr_id, ierr, h5p_default_f, h5p_default_f)
138 call h5awrite_f(attr_id, h5t_native_integer, n_global, ddim, ierr)
139 call h5aclose_f(attr_id, ierr)
140 call h5sclose_f(filespace, ierr)
141
142 ddim(1) = int(pe_size, hsize_t)
143 call h5screate_simple_f(1, ddim, filespace, ierr)
144 call h5acreate_f(grp_id, 'n', h5t_native_integer, filespace, attr_id, &
145 ierr)
146 call h5awrite_f(attr_id, h5t_native_integer, n_array, ddim, ierr)
147 call h5aclose_f(attr_id, ierr)
148 call h5sclose_f(filespace, ierr)
149
150 ! ------------------------------------------------------------------------ !
151 ! Write the distributed datasets
152
153 call sd_write_dist(grp_id, 'xold1', object%xold1, n_accum - n, n_global)
154 call sd_write_dist(grp_id, 'xold2', object%xold2, n_accum - n, n_global)
155 call sd_write_dist(grp_id, 'delta', object%delta, n_accum - n, n_global)
156
157 ! ------------------------------------------------------------------------ !
158 ! Close the group and file
159
160 call h5gclose_f(grp_id, ierr)
161 call h5fclose_f(file_id, ierr)
162 call h5pclose_f(fapl_id, ierr)
163 call hdf5_session_finalize()
164
165 end subroutine sd_optimizer_save_checkpoint_hdf5
166
171 module subroutine sd_optimizer_load_checkpoint_hdf5(object, filename)
172 class(steepest_descent_optimizer_t), intent(inout) :: object
173 character(len=*), intent(in) :: filename
174 integer(hid_t) :: fapl_id, file_id, grp_id, attr_id, space_id
175 integer(hsize_t) :: ddim(1), n_stored
176 integer :: ierr, info, n, n_global, n_accum, n_array(pe_size)
177 integer :: n_global_file, mismatch
178
179 ! Local and global sizes of the current run
180 n = object%xold1%size()
181 call mpi_allreduce(n, n_global, 1, mpi_integer, mpi_sum, neko_comm, ierr)
182 call mpi_scan(n, n_accum, 1, mpi_integer, mpi_sum, neko_comm, ierr)
183
184 ! ------------------------------------------------------------------------ !
185 ! Open the file and the checkpoint group
186
187 call hdf5_session_init()
188 call h5pcreate_f(h5p_file_access_f, fapl_id, ierr)
189 info = mpi_info_null%mpi_val
190 call h5pset_fapl_mpio_f(fapl_id, neko_comm%mpi_val, info, ierr)
191 call h5fopen_f(trim(filename), h5f_acc_rdonly_f, file_id, ierr, &
192 access_prp = fapl_id)
193 call h5gopen_f(file_id, sd_group, grp_id, ierr)
194
195 if (ierr .ne. 0) then
196 call neko_error('steepest_descent: unable to open HDF5 file "' // &
197 trim(filename) // '" or group "' // sd_group // '"')
198 end if
199
200 ! ------------------------------------------------------------------------ !
201 ! Read and check the size attributes
202
203 n_global_file = -1
204 ddim(1) = 1
205 call h5aopen_f(grp_id, 'n_global', attr_id, ierr)
206 call h5aread_f(attr_id, h5t_native_integer, n_global_file, ddim, ierr)
207 call h5aclose_f(attr_id, ierr)
208
209 ! The read fills the stored extent, so it must match the rank count
210 call h5aopen_f(grp_id, 'n', attr_id, ierr)
211 call h5aget_space_f(attr_id, space_id, ierr)
212 call h5sget_simple_extent_npoints_f(space_id, n_stored, ierr)
213 call h5sclose_f(space_id, ierr)
214 if (n_stored .ne. int(pe_size, hsize_t)) then
215 call neko_error('steepest_descent: checkpoint was written with a ' // &
216 'different number of ranks')
217 end if
218
219 n_array = -1
220 ddim(1) = int(pe_size, hsize_t)
221 call h5aread_f(attr_id, h5t_native_integer, n_array, ddim, ierr)
222 call h5aclose_f(attr_id, ierr)
223
224 if (n_global_file .ne. n_global) then
225 call neko_error('steepest_descent: mismatch in n_global during ' // &
226 'HDF5 read')
227 end if
228
229 ! The local sizes differ per rank, so agree on the outcome before the
230 ! collective reads
231 mismatch = 0
232 if (n_array(pe_rank + 1) .ne. n) mismatch = 1
233 call mpi_allreduce(mpi_in_place, mismatch, 1, mpi_integer, mpi_max, &
234 neko_comm, ierr)
235 if (mismatch .ne. 0) then
236 call neko_error('steepest_descent: mismatch in n during HDF5 read')
237 end if
238
239 ! ------------------------------------------------------------------------ !
240 ! Read the distributed datasets
241
242 call sd_read_dist(grp_id, 'xold1', object%xold1, n_accum - n, n_global)
243 call sd_read_dist(grp_id, 'xold2', object%xold2, n_accum - n, n_global)
244 call sd_read_dist(grp_id, 'delta', object%delta, n_accum - n, n_global)
245
246 ! ------------------------------------------------------------------------ !
247 ! Close the group and file
248
249 call h5gclose_f(grp_id, ierr)
250 call h5fclose_f(file_id, ierr)
251 call h5pclose_f(fapl_id, ierr)
252 call hdf5_session_finalize()
253
254 ! Ensure device state is updated
255 call object%xold1%copy_from(host_to_device, .true.)
256 call object%xold2%copy_from(host_to_device, .true.)
257 call object%delta%copy_from(host_to_device, .true.)
258
259 end subroutine sd_optimizer_load_checkpoint_hdf5
260
261 ! -------------------------------------------------------------------------- !
262 ! Private helpers
263
266 function sd_h5t_real() result(h5t_real)
267 integer(hid_t) :: h5t_real
268
269 select case (rp)
270 case (dp)
271 h5t_real = h5t_native_double
272 case (sp)
273 h5t_real = h5t_native_real
274 case default
275 call neko_error('steepest_descent: unsupported real kind for HDF5')
276 end select
277
278 end function sd_h5t_real
279
289 subroutine sd_write_dist(grp_id, name, vec, offset, n_global)
290 integer(hid_t), intent(in) :: grp_id
291 character(len=*), intent(in) :: name
292 type(vector_t), intent(in) :: vec
293 integer, intent(in) :: offset, n_global
294 integer(hid_t) :: filespace, memspace, xf_id, dset_id
295 integer(hsize_t) :: ddim(1), dcount(1), doffset(1)
296 real(kind=rp) :: empty(1)
297 integer :: ierr
298
299 empty = 0.0_rp
300 ddim(1) = int(n_global, hsize_t)
301 dcount(1) = int(vec%size(), hsize_t)
302 doffset(1) = int(offset, hsize_t)
303
304 call h5screate_simple_f(1, ddim, filespace, ierr)
305 call h5screate_simple_f(1, dcount, memspace, ierr)
306 call h5sselect_hyperslab_f(filespace, h5s_select_set_f, doffset, dcount, &
307 ierr)
308
309 call h5pcreate_f(h5p_dataset_xfer_f, xf_id, ierr)
310 call h5pset_dxpl_mpio_f(xf_id, h5fd_mpio_collective_f, ierr)
311
312 call h5dcreate_f(grp_id, name, sd_h5t_real(), filespace, dset_id, ierr)
313 if (vec%size() .gt. 0) then
314 call h5dwrite_f(dset_id, sd_h5t_real(), vec%x, dcount, ierr, &
315 file_space_id = filespace, mem_space_id = memspace, &
316 xfer_prp = xf_id)
317 else
318 call h5dwrite_f(dset_id, sd_h5t_real(), empty, dcount, ierr, &
319 file_space_id = filespace, mem_space_id = memspace, &
320 xfer_prp = xf_id)
321 end if
322 call h5dclose_f(dset_id, ierr)
323
324 call h5pclose_f(xf_id, ierr)
325 call h5sclose_f(memspace, ierr)
326 call h5sclose_f(filespace, ierr)
327
328 end subroutine sd_write_dist
329
339 subroutine sd_read_dist(grp_id, name, vec, offset, n_global)
340 integer(hid_t), intent(in) :: grp_id
341 character(len=*), intent(in) :: name
342 type(vector_t), intent(inout) :: vec
343 integer, intent(in) :: offset, n_global
344 integer(hid_t) :: filespace, memspace, xf_id, dset_id
345 integer(hsize_t) :: ddim(1), dcount(1), doffset(1)
346 real(kind=rp) :: empty(1)
347 integer :: ierr
348
349 ddim(1) = int(n_global, hsize_t)
350 dcount(1) = int(vec%size(), hsize_t)
351 doffset(1) = int(offset, hsize_t)
352
353 call h5screate_simple_f(1, ddim, filespace, ierr)
354 call h5screate_simple_f(1, dcount, memspace, ierr)
355 call h5sselect_hyperslab_f(filespace, h5s_select_set_f, doffset, dcount, &
356 ierr)
357
358 call h5pcreate_f(h5p_dataset_xfer_f, xf_id, ierr)
359 call h5pset_dxpl_mpio_f(xf_id, h5fd_mpio_collective_f, ierr)
360
361 call h5dopen_f(grp_id, name, dset_id, ierr)
362 if (vec%size() .gt. 0) then
363 call h5dread_f(dset_id, sd_h5t_real(), vec%x, dcount, ierr, &
364 file_space_id = filespace, mem_space_id = memspace, &
365 xfer_prp = xf_id)
366 else
367 call h5dread_f(dset_id, sd_h5t_real(), empty, dcount, ierr, &
368 file_space_id = filespace, mem_space_id = memspace, &
369 xfer_prp = xf_id)
370 end if
371 call h5dclose_f(dset_id, ierr)
372
373 call h5pclose_f(xf_id, ierr)
374 call h5sclose_f(memspace, ierr)
375 call h5sclose_f(filespace, ierr)
376
377 end subroutine sd_read_dist
378
379end submodule steepest_descent_hdf5_io
steepest_descent_optimizer
Steepest descent optimizer with move limits.
Definition steepest_descent_optimizer.f90:54