adjoint_scalar_pnpn.f90

Go to the documentation of this file.

 
!
 
module adjoint_scalar_pnpn
  use comm, only: neko_comm
  use utils, only: neko_error
  use num_types, only: rp, dp
  use, intrinsic :: iso_fortran_env, only: error_unit
  use rhs_maker, only : rhs_maker_bdf_t, rhs_maker_ext_t, rhs_maker_oifs_t, &
       rhs_maker_ext_fctry, rhs_maker_bdf_fctry, rhs_maker_oifs_fctry
  use adjoint_scalar_scheme, only : adjoint_scalar_scheme_t
  use checkpoint_payload, only : checkpoint_payload_t
  use checkpoint, only : chkp_t
  use field, only : field_t
  use scalar_bc_projector, only : scalar_bc_projector_t
  use mesh, only : mesh_t
  use coefs, only : coef_t
  use device, only : host_to_device, device_memcpy
  use gather_scatter, only : gs_t, gs_op_add
  use scalar_residual, only : scalar_residual_t, scalar_residual_factory
  use ax_product, only : ax_t, ax_helm_allocator
  use field_series, only: field_series_t
  use facet_normal, only : facet_normal_t
  use krylov, only : ksp_monitor_t
  use device_math, only : device_add2s2, device_col2
  use time_scheme_controller, only : time_scheme_controller_t
  use projection, only : projection_t
  use math, only : glsc2, col2, add2s2
  use field_math, only : field_col3, field_col2
  use logger, only : neko_log, log_size, neko_log_debug
  use advection_adjoint, only : advection_adjoint_t, advection_adjoint_factory
  use profiler, only : profiler_start_region, profiler_end_region
  use json_utils, only : json_get, json_get_or_default, json_extract_item
  use json_module, only : json_file, json_core, json_value
  use user_intf, only : user_t
  use neko_config, only : neko_bcknd_device
  use time_step_controller, only : time_step_controller_t
  use scratch_registry, only : neko_scratch_registry
  use time_state, only : time_state_t
  use bc, only : bc_t, bc_dirichlet
  use mpi_f08, only: mpi_integer, mpi_sum, mpi_max
  implicit none
  private
 
 
  type, public, extends(adjoint_scalar_scheme_t) :: adjoint_scalar_pnpn_t
 
     type(field_t) :: s_adj_res
 
     type(field_t) :: ds_adj
 
     class(ax_t), allocatable :: ax
 
     type(projection_t) :: proj_s
 
     type(scalar_bc_projector_t) :: bc_projector
 
     class(advection_adjoint_t), allocatable :: adv
 
     ! Time interpolation scheme
     logical :: oifs
 
     ! Advection terms for the oifs method
     type(field_t) :: advs
 
     class(scalar_residual_t), allocatable :: res
 
     class(rhs_maker_ext_t), allocatable :: makeext
 
     class(rhs_maker_bdf_t), allocatable :: makebdf
 
     class(rhs_maker_oifs_t), allocatable :: makeoifs
 
   contains
     procedure, pass(this) :: init => adjoint_scalar_pnpn_init
     procedure, pass(this) :: register_checkpoint => &
          adjoint_scalar_pnpn_register_checkpoint
     procedure, pass(this) :: restart => adjoint_scalar_pnpn_restart
     procedure, pass(this) :: free => adjoint_scalar_pnpn_free
     procedure, pass(this) :: step => adjoint_scalar_pnpn_step
     procedure, pass(this) :: setup_bcs_ => adjoint_scalar_pnpn_setup_bcs_
  end type adjoint_scalar_pnpn_t
 
  interface adjoint_bc_factory
     module subroutine adjoint_bc_factory(object, scheme, json, coef, user)
       class(bc_t), pointer, intent(inout) :: object
       type(adjoint_scalar_pnpn_t), intent(in) :: scheme
       type(json_file), intent(inout) :: json
       type(coef_t), target, intent(in) :: coef
       type(user_t), target, intent(in) :: user
     end subroutine adjoint_bc_factory
  end interface adjoint_bc_factory
 
contains
 
  subroutine adjoint_scalar_pnpn_init(this, msh, coef, gs, params_adjoint, &
       params_primal, numerics_params, user, chkp, ulag, vlag, wlag, &
       time_scheme, rho)
    class(adjoint_scalar_pnpn_t), target, intent(inout) :: this
    type(mesh_t), target, intent(in) :: msh
    type(coef_t), target, intent(in) :: coef
    type(gs_t), target, intent(inout) :: gs
    type(json_file), target, intent(inout) :: params_adjoint
    type(json_file), target, intent(inout) :: params_primal
    type(json_file), target, intent(inout) :: numerics_params
    type(user_t), target, intent(in) :: user
    type(chkp_t), target, intent(inout) :: chkp
    type(field_series_t), target, intent(in) :: ulag, vlag, wlag
    type(time_scheme_controller_t), target, intent(in) :: time_scheme
    type(field_t), target, intent(in) :: rho
    integer :: i
    class(bc_t), pointer :: bc_i
    character(len=15), parameter :: scheme = 'Modular (Pn/Pn)'
    logical :: advection
    real(kind=dp), pointer :: tlag(:), dtlag(:)
 
    call this%free()
 
    ! Initiliaze base type.
    call this%scheme_init(msh, coef, gs, params_adjoint, params_primal, &
         scheme, user, rho)
 
    ! Setup backend dependent Ax routines
    call ax_helm_allocator(this%ax, type_name = "standard")
 
    ! Setup backend dependent scalar residual routines
    call scalar_residual_factory(this%res)
 
    ! Setup backend dependent summation of extrapolation scheme
    call rhs_maker_ext_fctry(this%makeext)
 
    ! Setup backend dependent contributions to F from lagged BD terms
    call rhs_maker_bdf_fctry(this%makebdf)
 
    ! Setup backend dependent contributions of the OIFS scheme
    call rhs_maker_oifs_fctry(this%makeoifs)
 
    ! Initialize variables specific to this plan
    associate(xh_lx => this%Xh%lx, xh_ly => this%Xh%ly, xh_lz => this%Xh%lz, &
         dm_xh => this%dm_Xh, nelv => this%msh%nelv)
 
      call this%s_adj_res%init(dm_xh, "s_adj_res")
 
      call this%abx1%init(dm_xh, "abx1")
 
      call this%abx2%init(dm_xh, "abx2")
 
      call this%advs%init(dm_xh, "advs")
 
      call this%ds_adj%init(dm_xh, 'ds_adj')
 
    end associate
 
    ! Set up boundary conditions
    call this%setup_bcs_(user)
 
    ! Collect the Dirichlet dofs of all bcs into the increment projector.
    do i = 1, this%bcs%size()
       if (this%bcs%bc_type(i) .eq. bc_dirichlet) then
          bc_i => this%bcs%get(i)
          call this%bc_projector%mark(bc_i)
       end if
    end do
 
 
    ! Initialize projection space
    call this%proj_s%init(this%dm_Xh%size(), this%projection_dim, &
         this%projection_activ_step)
 
    ! Determine the time-interpolation scheme
    call json_get_or_default(numerics_params, 'oifs', this%oifs, .false.)
 
    ! Point to case checkpoint
    this%chkp => chkp
 
    ! Initialize advection factory
    call json_get_or_default(params_adjoint, 'advection', advection, .true.)
 
    call chkp%get_time_history(tlag, dtlag)
    call advection_adjoint_factory(this%adv, numerics_params, this%c_Xh, &
         ulag, vlag, wlag, dtlag, &
         tlag, time_scheme, .not. advection, &
         this%s_adj_lag)
  end subroutine adjoint_scalar_pnpn_init
 
  subroutine adjoint_scalar_pnpn_register_checkpoint(this, chkp)
    class(adjoint_scalar_pnpn_t), target, intent(inout) :: this
    type(chkp_t), intent(inout) :: chkp
    type(checkpoint_payload_t), pointer :: payload
 
    payload => chkp%add_payload("adjoint_scalars/" // trim(this%name))
    call payload%add_field(this%s_adj)
    call payload%add_series(this%s_adj_lag)
    call payload%add_field(this%abx1)
    call payload%add_field(this%abx2)
 
  end subroutine adjoint_scalar_pnpn_register_checkpoint
 
  subroutine adjoint_scalar_pnpn_restart(this, chkp)
    class(adjoint_scalar_pnpn_t), target, intent(inout) :: this
    type(chkp_t), intent(inout) :: chkp
    logical :: interpolated
    integer :: n
 
    n = this%s_adj%dof%size()
 
    ! Lag fields are restored through the checkpoint's fsp mechanism
 
    ! The restored fields are continuous unless the checkpoint was written
    ! on another mesh or at another polynomial order and was interpolated
    ! on the way in. Only then do the copies of a node shared between
    ! elements need averaging: scale by the inverse multiplicity, then sum
    ! the copies with a gather-scatter. On a plain restart that is the
    ! identity in exact arithmetic but not in floating point, and would put
    ! about one ulp of error on every shared node. Same guard as the fluid.
    interpolated = allocated(chkp%previous_mesh%elements) .or. &
         chkp%previous_Xh%lx .ne. this%Xh%lx
 
    if (interpolated) then
       call col2(this%s_adj%x, this%c_Xh%mult, n)
       call col2(this%s_adj_lag%lf(1)%x, this%c_Xh%mult, n)
       call col2(this%s_adj_lag%lf(2)%x, this%c_Xh%mult, n)
    end if
    if (neko_bcknd_device .eq. 1) then
       call device_memcpy(this%s_adj%x, this%s_adj%x_d, &
            n, host_to_device, sync = .false.)
       call device_memcpy(this%s_adj_lag%lf(1)%x, this%s_adj_lag%lf(1)%x_d, &
            n, host_to_device, sync = .false.)
       call device_memcpy(this%s_adj_lag%lf(2)%x, this%s_adj_lag%lf(2)%x_d, &
            n, host_to_device, sync = .false.)
       call device_memcpy(this%abx1%x, this%abx1%x_d, &
            n, host_to_device, sync = .false.)
       call device_memcpy(this%abx2%x, this%abx2%x_d, &
            n, host_to_device, sync = .false.)
       call device_memcpy(this%advs%x, this%advs%x_d, &
            n, host_to_device, sync = .false.)
    end if
 
    if (interpolated) then
       call this%gs_Xh%op(this%s_adj, gs_op_add)
       call this%gs_Xh%op(this%s_adj_lag%lf(1), gs_op_add)
       call this%gs_Xh%op(this%s_adj_lag%lf(2), gs_op_add)
    end if
 
  end subroutine adjoint_scalar_pnpn_restart
 
  subroutine adjoint_scalar_pnpn_free(this)
    class(adjoint_scalar_pnpn_t), intent(inout) :: this
 
    !Deallocate scalar field
    call this%scheme_free()
 
    call this%bc_projector%free()
    call this%proj_s%free()
 
    call this%s_adj_res%free()
 
    call this%ds_adj%free()
 
    call this%abx1%free()
    call this%abx2%free()
 
    call this%advs%free()
 
    if (allocated(this%Ax)) then
       deallocate(this%Ax)
    end if
 
    if (allocated(this%res)) then
       deallocate(this%res)
    end if
 
    if (allocated(this%makeext)) then
       deallocate(this%makeext)
    end if
 
    if (allocated(this%makebdf)) then
       deallocate(this%makebdf)
    end if
 
    if (allocated(this%makeoifs)) then
       deallocate(this%makeoifs)
    end if
 
  end subroutine adjoint_scalar_pnpn_free
 
  subroutine adjoint_scalar_pnpn_step(this, time, ext_bdf, dt_controller, &
       ksp_results)
    class(adjoint_scalar_pnpn_t), intent(inout) :: this
    type(time_state_t), intent(in) :: time
    type(time_scheme_controller_t), intent(in) :: ext_bdf
    type(time_step_controller_t), intent(in) :: dt_controller
    type(ksp_monitor_t), intent(inout) :: ksp_results
    type(field_t), pointer :: rho_cp
    integer :: rho_cp_index
    ! Number of degrees of freedom
    integer :: n
 
    if (this%freeze) return
 
    n = this%dm_Xh%size()
    call neko_scratch_registry%request_field(rho_cp, rho_cp_index, .false.)
 
    call profiler_start_region('Adjoint Scalar')
    associate(u => this%u, v => this%v, w => this%w, s_adj => this%s_adj, &
         cp => this%cp, rho => this%rho, lambda => this%lambda, &
         ds_adj => this%ds_adj, &
         s_adj_res => this%s_adj_res, &
         ax => this%Ax, f_xh => this%f_Xh, xh => this%Xh, &
         c_xh => this%c_Xh, dm_xh => this%dm_Xh, gs_xh => this%gs_Xh, &
         s_adj_lag => this%s_adj_lag, oifs => this%oifs, &
         projection_dim => this%projection_dim, &
         msh => this%msh, res => this%res, makeoifs => this%makeoifs, &
         makeext => this%makeext, makebdf => this%makebdf, &
         t => time%t, tstep => time%tstep, dt => time%dt)
 
      ! Logs extra information the log level is NEKO_LOG_DEBUG or above.
      call print_debug(this)
 
      ! Update material properties and their pointwise product.
      ! This MUST happen before rho_cp is used below (by makebdf and by the
      ! source/advection scaling). It used to sit after those uses, which left
      ! makebdf reading an uninitialised scratch field while res%compute used
      ! the correct value -- so the BDF mass terms could not cancel at steady
      ! state and the converged adjoint scalar came out proportional to dt.
      ! Mirrors the ordering in Neko's forward scalar_pnpn.
      call this%update_material_properties(time)
      call field_col3(rho_cp, rho, cp)
 
      ! Compute the source terms that are extrapolated in time below
      call this%source_term%compute(time, extrapolate = .true.)
 
      ! if (oifs) then
      !    call neko_error("oifs not implemented for adjoint scalar")
      ! ! Add the advection operators to the right-hans-side.
      ! call this%adv%compute_scalar(u, v, w, s_adj, this%advs, &
      !                           Xh, this%c_Xh, dm_Xh%size())
 
      ! call makeext%compute_scalar(this%abx1, this%abx2, f_Xh%x, rho, &
      !                             ext_bdf%advection_coeffs, n)
 
      ! call makeoifs%compute_scalar(this%advs%x, f_Xh%x, rho, dt, n)
      ! else
      ! Add the advection operators to the right-hans-side.
      call this%adv%compute_adjoint_scalar(u, v, w, s_adj, f_xh, &
           xh, this%c_Xh, dm_xh%size())
 
      ! Scale the volumetric source and advection terms by rho * cp, then add
      ! the weak boundary fluxes without that scaling -- same split as the
      ! forward scalar. (The weak BC application was previously done before
      ! the advection term and never scaled, so it could not follow this
      ! convention.)
      call field_col2(f_xh, rho_cp)
 
      ! Apply weak boundary conditions, that contribute to the source terms.
      call this%bcs%apply_scalar(this%f_Xh%x, dm_xh%size(), time, .false.)
 
      ! At this point the RHS contains the sum of the advection operator,
      ! Neumann boundary sources and additional source terms, evaluated using
      ! the scalar field from the previous time-step.
      ! Now, this value is used in the explicit time scheme to advance these
      ! terms in time.
      call makeext%compute_scalar(this%abx1, this%abx2, f_xh%x, &
           ext_bdf%advection_coeffs%x, n)
 
      ! Add the RHS contributions coming from the BDF scheme.
      call makebdf%compute_scalar(s_adj_lag, f_xh%x, s_adj, c_xh%B, &
           rho_cp, dt, ext_bdf%diffusion_coeffs%x, ext_bdf%ndiff, n)
      ! end if
 
      ! Source terms that must not be extrapolated in time are added as
      ! computed, weighted by the mass matrix and rho * cp like the
      ! extrapolated terms above.
      call this%source_term%compute(time, extrapolate = .false., &
           scale = rho_cp)
 
      call s_adj_lag%update()
 
      call this%bcs%apply_scalar(this%s_adj%x, this%dm_Xh%size(), time, &
           .true.)
 
      ! Compute scalar residual.
      call profiler_start_region('Adjoint_scalar_residual')
      call res%compute(ax, s_adj, s_adj_res, f_xh, c_xh, msh, xh, &
           lambda, rho_cp, ext_bdf%diffusion_coeffs%x(1), &
           dt, dm_xh%size())
 
      call gs_xh%op(s_adj_res, gs_op_add)
 
 
      ! Zero-out residual at Dirichlet nodes before solving.
      call this%bc_projector%apply(s_adj_res%x, dm_xh%size())
 
      call profiler_end_region('Adjoint_scalar_residual')
 
      call this%proj_s%pre_solving(s_adj_res%x, tstep, c_xh, n, dt_controller)
 
      call this%pc%update()
      call profiler_start_region('Adjoint_scalar_solve')
      ksp_results = this%ksp%solve(ax, ds_adj, s_adj_res%x, n, &
           c_xh, this%bc_projector, gs_xh)
      call profiler_end_region('Adjoint_scalar_solve')
 
      ksp_results%name = 'Adjoint Scalar'
 
      call this%proj_s%post_solving(ds_adj%x, ax, c_xh, this%bc_projector, &
           gs_xh, n, tstep, dt_controller)
 
      ! Update the solution
      if (neko_bcknd_device .eq. 1) then
         call device_add2s2(s_adj%x_d, ds_adj%x_d, 1.0_rp, n)
      else
         call add2s2(s_adj%x, ds_adj%x, 1.0_rp, n)
      end if
 
    end associate
    call neko_scratch_registry%relinquish_field(rho_cp_index)
    call profiler_end_region('Adjoint Scalar')
  end subroutine adjoint_scalar_pnpn_step
 
  subroutine print_debug(this)
    class(adjoint_scalar_pnpn_t), intent(inout) :: this
    ! character(len=LOG_SIZE) :: log_buf
    integer :: n
 
    n = this%dm_Xh%size()
    ! TODO come back to this
    !write(log_buf,'(A,A,E15.7,A,E15.7,A,E15.7)') 'Adjoint scalar debug', &
    !   ' l2norm s_adj', glsc2(this%s_adj%x, this%s_adj%x, n), &
    !   ' slag1', glsc2(this%s_adj_lag%lf(1)%x, this%s_adj_lag%lf(1)%x, n), &
    !   ' slag2', glsc2(this%s_adj_lag%lf(2)%x, this%s_adj_lag%lf(2)%x, n)
    !call neko_log%message(log_buf, lvl=NEKO_LOG_DEBUG)
    !write(log_buf,'(A,A,E15.7,A,E15.7)') 'Adjoint scalar debug2', &
    !   ' l2norm abx1', glsc2(this%abx1%x, this%abx1%x, n), &
    !   ' abx2', glsc2(this%abx2%x, this%abx2%x, n)
    !call neko_log%message(log_buf, lvl=NEKO_LOG_DEBUG)
  end subroutine print_debug
 
  subroutine adjoint_scalar_pnpn_setup_bcs_(this, user)
    class(adjoint_scalar_pnpn_t), target, intent(inout) :: this
    type(user_t), target, intent(in) :: user
    integer :: i, j, n_bcs, zone_size, global_zone_size, ierr
    type(json_core) :: core
    type(json_value), pointer :: bc_object
    type(json_file) :: bc_subdict
    class(bc_t), pointer :: bc_i
    logical :: found
    ! Monitor which boundary zones have been marked
    logical, allocatable :: marked_zones(:)
    integer, allocatable :: zone_indices(:)
 
    if (this%params%valid_path('boundary_conditions')) then
       call this%params%info('boundary_conditions', &
            n_children = n_bcs)
       call this%params%get_core(core)
       call this%params%get('boundary_conditions', bc_object, found)
 
       call this%bcs%init(n_bcs)
 
       allocate(marked_zones(size(this%msh%labeled_zones)))
       marked_zones = .false.
 
       do i = 1, n_bcs
          ! Create a new json containing just the subdict for this bc
          call json_extract_item(core, bc_object, i, bc_subdict)
 
          ! Check that we are not trying to assing a bc to zone, for which one
          ! has already been assigned and that the zone has more than 0 size
          ! in the mesh.
          call json_get(bc_subdict, "zone_indices", zone_indices)
 
          do j = 1, size(zone_indices)
             zone_size = this%msh%labeled_zones(zone_indices(j))%size
             call mpi_allreduce(zone_size, global_zone_size, 1, &
                  mpi_integer, mpi_max, neko_comm, ierr)
 
             if (global_zone_size .eq. 0) then
                write(error_unit, '(A, A, I0, A, A, I0, A)') "*** ERROR ***: ",&
                     "Zone index ", zone_indices(j), &
                     " is invalid as this zone has 0 size, meaning it ", &
                     "does not exist in the mesh. Check adjoint scalar BC ", &
                     i, "."
                error stop
             end if
 
             if (marked_zones(zone_indices(j)) .eqv. .true.) then
                write(error_unit, '(A, A, I0, A, A, A, A)') "*** ERROR ***: ", &
                     "Zone with index ", zone_indices(j), &
                     " has already been assigned a boundary condition. ", &
                     "Please check your boundary_conditions entry for the ", &
                     "adjoint scalar and make sure that each zone index ", &
                     "appears only in a single boundary condition."
                error stop
             else
                marked_zones(zone_indices(j)) = .true.
             end if
          end do
 
          bc_i => null()
 
          call adjoint_bc_factory(bc_i, this, bc_subdict, this%c_Xh, user)
          call this%bcs%append(bc_i)
       end do
 
       ! Make sure all labeled zones with non-zero size have been marked
       do i = 1, size(this%msh%labeled_zones)
          if ((this%msh%labeled_zones(i)%size .gt. 0) .and. &
               (marked_zones(i) .eqv. .false.)) then
             write(error_unit, '(A, A, I0)') "*** ERROR ***: ", &
                  "No adjoint scalar boundary condition assigned to zone ", i
             error stop
          end if
       end do
    else
       ! Check that there are no labeled zones, i.e. all are periodic.
       do i = 1, size(this%msh%labeled_zones)
          if (this%msh%labeled_zones(i)%size .gt. 0) then
             write(error_unit, '(A, A, A)') "*** ERROR ***: ", &
                  "No boundary_conditions entry in the case file for " // &
                  " adjoint scalar ", &
                  this%s%name
             error stop
          end if
       end do
    end if
  end subroutine adjoint_scalar_pnpn_setup_bcs_
 
end module adjoint_scalar_pnpn