yowexchangeModule.F90

Go to the documentation of this file.

!PDLIB Software License
!
!Software, as understood herein, shall be broadly interpreted as being inclusive of algorithms,
!source code, object code, data bases and related documentation, all of which shall be furnished
!free of charge to the Licensee. Corrections, upgrades or enhancements may be furnished and, if
!furnished, shall also be furnished to the Licensee without charge. NOAA, however, is not
!required to develop or furnish such corrections, upgrades or enhancements.
!Roland & Partner software, whether that initially furnished or corrections or upgrades,
!are furnished "as is". Roland & Partner furnishes its software without any warranty
!whatsoever and is not responsible for any direct, indirect or consequential damages
!that may be incurred by the Licensee. Warranties of merchantability, fitness for any
!particular purpose, title, and non-infringement, are specifically negated.
!The Licensee is not required to develop any software related to the licensed software.
!However, in the event that the Licensee does so, the Licensee is required to offer same
!to Roland & Partner for inclusion under the instant licensing terms with Roland & Partner
!licensed software along with documentation regarding its principles, use and its advantages.
!This includes changes to the wave model proper including numerical and physical approaches
!to wave modeling, and boundary layer parameterizations embedded in the wave model
!A Licensee may reproduce sufficient software to satisfy its needs.
!All copies shall bear the name of the software with any version number
!as well as replicas of any applied copyright notice, trademark notice,
!other notices and credit lines. Additionally, if the copies have been modified,
!e.g. with deletions or additions, this shall be so stated and identified.
!All of Licensee's employees who have a need to use the software may have access
!to the software but only after reading the instant license and stating, in writing,
!that they have read and understood the license and have agreed to its terms.
!Licensee is responsible for employing reasonable efforts to assure
!that only those of its employees that should have access to the software, in fact, have access.
!The Licensee may use the software for any purpose relating to sea state prediction.
!No disclosure of any portion of the software, whether by means of a media or verbally,
!may be made to any third party by the Licensee or the Licensee's employees
!The Licensee is responsible for compliance with any applicable export or
!import control laws of the United States, the European Union and Germany.
!
!© 2009 Roland&Partner, Georgenstr.32, 64297 Germany. All rights reserved.
!PDLIB is a trademark of Roland & Partner. No unauthorized use without permission.
!
module yowexchangemodule
  use yowdatapool, only: rkind
  use mpi, only: mpi_datatype_null
  implicit none
  private
  public :: initnbrdomains, creatempitypes, setdimsize
  public :: finalizeexchangemodule, pdlib_exchange1dreal
  public :: pdlib_exchange2dreal, pdlib_exchange2dreal_zero
 
  type, public :: t_neighbordomain
 
    integer :: domainid = 0
 
    integer :: numnodestoreceive = 0
 
    integer, allocatable :: nodestoreceive(:)
 
    integer :: numnodestosend = 0
 
    integer, allocatable :: nodestosend(:)
 
    ! MPI datatypes for size(U) == npa+1  U(0:npa)
 
    integer :: p1drsendtype_zero = mpi_datatype_null
    integer :: p1drrecvtype_zero = mpi_datatype_null
 
    integer :: p2drsendtype_zero = mpi_datatype_null
    integer :: p2drrecvtype_zero = mpi_datatype_null
 
    ! MPI datatypes for size(U) == npa  U(1:npa)
    integer :: p1drsendtype = mpi_datatype_null
    integer :: p1drrecvtype = mpi_datatype_null
    integer :: p2drsendtype1 = mpi_datatype_null
    integer :: p2drrecvtype1 = mpi_datatype_null
    integer :: p2drsendtype2 = mpi_datatype_null
    integer :: p2drrecvtype2 = mpi_datatype_null
 
  contains
    !     procedure :: exchangeGhostIds
    !     final :: finalizeNeighborDomain
    procedure :: finalize
    procedure :: creatempitype
 
  end type t_neighbordomain
 
  type(t_neighbordomain), public, allocatable :: neighbordomains(:)
 
  integer, public :: nconndomains = 0
 
  integer, public :: n2nddim = 1
 
  integer, public :: nnthdim = 1
 
 
contains
 
 
  subroutine finalize(this)
    use yowerr
    use mpi
    implicit none
    class(t_neighbordomain), intent(inout) :: this
    integer :: ierr
 
    if(allocated(this%nodesToSend))    deallocate(this%nodesToSend)
    if(allocated(this%nodesToReceive)) deallocate(this%nodesToReceive)
 
    call mpi_type_free(this%p1DRsendType_zero, ierr)
    if(ierr /= mpi_success) CALL parallel_abort("freeMPItype", ierr)
    call mpi_type_free(this%p1DRrecvType_zero, ierr)
    if(ierr /= mpi_success) CALL parallel_abort("freeMPItype", ierr)
    call mpi_type_free(this%p2DRsendType_zero, ierr)
    if(ierr /= mpi_success) CALL parallel_abort("freeMPItype", ierr)
    call mpi_type_free(this%p2DRrecvType_zero, ierr)
    if(ierr /= mpi_success) CALL parallel_abort("freeMPItype", ierr)
    call mpi_type_free(this%p1DRsendType, ierr)
    if(ierr /= mpi_success) CALL parallel_abort("freeMPItype", ierr)
    call mpi_type_free(this%p1DRrecvType, ierr)
    if(ierr /= mpi_success) CALL parallel_abort("freeMPItype", ierr)
    call mpi_type_free(this%p2DRsendType1, ierr)
    if(ierr /= mpi_success) CALL parallel_abort("freeMPItype", ierr)
    call mpi_type_free(this%p2DRrecvType1, ierr)
    if(ierr /= mpi_success) CALL parallel_abort("freeMPItype", ierr)
    call mpi_type_free(this%p2DRsendType2, ierr)
    if(ierr /= mpi_success) CALL parallel_abort("freeMPItype", ierr)
    call mpi_type_free(this%p2DRrecvType2, ierr)
    if(ierr /= mpi_success) CALL parallel_abort("freeMPItype", ierr)
  end subroutine finalize
 
  ! create MPI indexed datatype for this neighborDomain
  subroutine creatempitype(this)
    use yowerr
    use mpi
    use yownodepool, only: ghostgl, np, ipgl
    use yowdatapool, only: rtype, itype
    implicit none
    class(t_neighbordomain), intent(inout) :: this
 
    integer :: ierr
    integer :: dsplSend(this%numNodesToSend)
    integer :: dsplRecv(this%numNodesToReceive)
 
    ! MPI datatypes for size(U) == npa+1  U(0:npa)
    dsplsend = ipgl(this%nodesToSend)
    dsplrecv = ghostgl(this%nodesToReceive) + np
 
    ! p1D real
    call mpi_type_create_indexed_block(this%numNodesToSend, 1, dsplsend, rtype, this%p1DRsendType_zero, ierr)
    if(ierr /= mpi_success) CALL parallel_abort("createMPIType", ierr)
    call mpi_type_commit(this%p1DRsendType_zero,ierr)
    if(ierr /= mpi_success) CALL parallel_abort("createMPIType", ierr)
 
    call mpi_type_create_indexed_block(this%numNodesToReceive, 1, dsplrecv, rtype, this%p1DRrecvType_zero, ierr)
    if(ierr /= mpi_success) CALL parallel_abort("createMPIType", ierr)
    call mpi_type_commit(this%p1DRrecvType_zero,ierr)
    if(ierr /= mpi_success) CALL parallel_abort("createMPIType", ierr)
 
    ! MPI datatypes for size(U) == npa  U(1:npa)
    dsplsend(:) = dsplsend(:) - 1 ! C count from 0; FORTRAN count from 1
    dsplrecv(:) = dsplrecv(:) - 1
 
    ! p1D real
    call mpi_type_create_indexed_block(this%numNodesToSend, 1, dsplsend, rtype, this%p1DRsendType,ierr)
    if(ierr /= mpi_success) CALL parallel_abort("createMPIType", ierr)
    call mpi_type_commit(this%p1DRsendType,ierr)
    if(ierr /= mpi_success) CALL parallel_abort("createMPIType", ierr)
 
    call mpi_type_create_indexed_block(this%numNodesToReceive, 1, dsplrecv, rtype, this%p1DRrecvType,ierr)
    if(ierr /= mpi_success) CALL parallel_abort("createMPIType", ierr)
    call mpi_type_commit(this%p1DRrecvType,ierr)
    if(ierr /= mpi_success) CALL parallel_abort("createMPIType", ierr)
 
    ! MPI datatypes for size(U) == npa+1  U(0:npa)
    ! p2D real second dim is n2ndDim long
    dsplsend = (ipgl(this%nodesToSend)) * n2nddim
    dsplrecv = (ghostgl(this%nodesToReceive) + np) * n2nddim
    call mpi_type_create_indexed_block(this%numNodesToSend, n2nddim, dsplsend, rtype, this%p2DRsendType_zero,ierr)
    if(ierr /= mpi_success) CALL parallel_abort("createMPIType", ierr)
    call mpi_type_commit(this%p2DRsendType_zero,ierr)
    if(ierr /= mpi_success) CALL parallel_abort("createMPIType", ierr)
 
    call mpi_type_create_indexed_block(this%numNodesToReceive, n2nddim, dsplrecv, rtype, this%p2DRrecvType_zero,ierr)
    if(ierr /= mpi_success) CALL parallel_abort("createMPIType", ierr)
    call mpi_type_commit(this%p2DRrecvType_zero,ierr)
    if(ierr /= mpi_success) CALL parallel_abort("createMPIType", ierr)
 
    ! MPI datatypes for size(U) == npa  U(1:npa)
    ! p2D real second dim is n2ndDim long
    dsplsend = (ipgl(this%nodesToSend)-1) * n2nddim
    dsplrecv = (ghostgl(this%nodesToReceive) + np -1) * n2nddim
    call mpi_type_create_indexed_block(this%numNodesToSend, n2nddim, dsplsend, rtype, this%p2DRsendType1,ierr)
    if(ierr /= mpi_success) CALL parallel_abort("createMPIType", ierr)
    call mpi_type_commit(this%p2DRsendType1,ierr)
    if(ierr /= mpi_success) CALL parallel_abort("createMPIType", ierr)
 
    call mpi_type_create_indexed_block(this%numNodesToReceive, n2nddim, dsplrecv, rtype, this%p2DRrecvType1,ierr)
    if(ierr /= mpi_success) CALL parallel_abort("createMPIType", ierr)
    call mpi_type_commit(this%p2DRrecvType1,ierr)
    if(ierr /= mpi_success) CALL parallel_abort("createMPIType", ierr)
 
 
  end subroutine creatempitype
 
  subroutine initnbrdomains(nConnD)
    use yowerr
    implicit none
    integer, intent(in) :: nconnd
    integer :: stat
 
    call finalizeexchangemodule()
    nconndomains = nconnd
    allocate(neighbordomains(nconndomains), stat=stat)
    if(stat/=0)  CALL abort('neighborDomains allocation failure')
  end subroutine initnbrdomains
 
  subroutine creatempitypes()
    implicit none
    integer :: i
 
    do i=1, nconndomains
      call neighbordomains(i)%createMPIType()
    end do
  end subroutine creatempitypes
 
  subroutine pdlib_exchange1dreal(U)
    use yowdatapool, only: comm, myrank, rkind
    use yownodepool, only: t_node, nodes_global, np, ng, ghosts, npa
    use yowerr
    use mpi
    implicit none
    real(kind=rkind), intent(inout) :: u(:)
 
    integer :: i, ierr, tag
    integer :: sendrqst(nconndomains), recvrqst(nconndomains)
    integer :: recvstat(mpi_status_size, nconndomains), sendstat(mpi_status_size, nconndomains)
    character(len=140) :: errmsg
 
    if(size(u) /= npa) then
      WRITE(errmsg, *) 'size(U)=', size(u), ' but npa=', npa
      CALL abort(errmsg)
    endif
 
    ! post receives
    do i=1, nconndomains
      tag = 10000 + myrank
      call mpi_irecv(u, 1, neighbordomains(i)%p1DRrecvType, &
           neighbordomains(i)%domainID-1, tag, comm, &
           recvrqst(i), ierr)
      if(ierr/=mpi_success) then
        CALL parallel_abort("MPI_IRecv", ierr)
      endif
    enddo
 
    ! post sends
    do i=1, nconndomains
      tag = 10000 + (neighbordomains(i)%domainID-1)
      call mpi_isend(u, 1, neighbordomains(i)%p1DRsendType, &
           neighbordomains(i)%domainID-1, tag, comm, &
           sendrqst(i), ierr);
      if(ierr/=mpi_success) then
        CALL parallel_abort("MPI_ISend", ierr)
      endif
    end do
 
    ! Wait for completion
    call mpi_waitall(nconndomains, recvrqst, recvstat,ierr)
    if(ierr/=mpi_success) CALL parallel_abort("waitall", ierr)
    call mpi_waitall(nconndomains, sendrqst, sendstat,ierr)
    if(ierr/=mpi_success) CALL parallel_abort("waitall", ierr)
  end subroutine pdlib_exchange1dreal
 
 
  subroutine pdlib_exchange2dreal(U)
    use yowdatapool, only: comm, myrank, rkind
    use yownodepool, only: t_node, nodes_global, np, ng, ghosts, npa
    use yowerr
    use mpi
    USE w3odatmd, only : iaproc
    implicit none
    real(kind=rkind), intent(inout) :: u(:,:)
 
    integer :: i, ierr, tag
    integer :: sendrqst(nconndomains), recvrqst(nconndomains)
    integer :: recvstat(mpi_status_size, nconndomains), sendstat(mpi_status_size, nconndomains)
 
 
#ifdef W3_DEBUGEXCH
    WRITE(740+iaproc,*) 'PDLIB_exchange2Dreal, step 3'
    FLUSH(740+iaproc)
#endif
 
    ! post receives
#ifdef W3_DEBUGEXCH
    WRITE(740+iaproc,*) 'PDLIB_exchange2Dreal, step 4'
    FLUSH(740+iaproc)
#endif
    do i=1, nconndomains
      tag = 30000 + myrank
      call mpi_irecv(u, 1, neighbordomains(i)%p2DRrecvType1, &
           neighbordomains(i)%domainID-1, tag, comm, &
           recvrqst(i), ierr)
      if(ierr/=mpi_success) then
        CALL parallel_abort("MPI_IRecv", ierr)
      endif
    enddo
#ifdef W3_DEBUGEXCH
    WRITE(740+iaproc,*) 'PDLIB_exchange2Dreal, step 5'
    FLUSH(740+iaproc)
#endif
 
    ! post sends
    do i=1, nconndomains
      tag = 30000 + (neighbordomains(i)%domainID-1)
      call mpi_isend(u, 1, neighbordomains(i)%p2DRsendType1, &
           neighbordomains(i)%domainID-1, tag, comm, &
           sendrqst(i), ierr)
      if(ierr/=mpi_success) then
        CALL parallel_abort("MPI_ISend", ierr)
      endif
    end do
#ifdef W3_DEBUGEXCH
    WRITE(740+iaproc,*) 'PDLIB_exchange2Dreal, step 6'
    FLUSH(740+iaproc)
#endif
 
    ! Wait for completion
    call mpi_waitall(nconndomains, recvrqst, recvstat,ierr)
    if(ierr/=mpi_success) CALL parallel_abort("waitall", ierr)
#ifdef W3_DEBUGEXCH
    WRITE(740+iaproc,*) 'PDLIB_exchange2Dreal, step 11'
    FLUSH(740+iaproc)
#endif
    call mpi_waitall(nconndomains, sendrqst, sendstat,ierr)
    if(ierr/=mpi_success) CALL parallel_abort("waitall", ierr)
#ifdef W3_DEBUGEXCH
    WRITE(740+iaproc,*) 'PDLIB_exchange2Dreal, step 12'
    FLUSH(740+iaproc)
#endif
  end subroutine pdlib_exchange2dreal
 
 
  subroutine setdimsize(second)
    implicit none
    integer, intent(in) :: second
    n2nddim = second
  end subroutine setdimsize
 
  subroutine finalizeexchangemodule()
    implicit none
    integer :: i
 
    if(allocated(neighbordomains)) then
      do i=1, size(neighbordomains)
        call neighbordomains(i)%finalize()
      end do
      deallocate(neighbordomains)
    endif
  end subroutine finalizeexchangemodule
  subroutine pdlib_exchange1dreal_zero(U)
    use yowdatapool, only: comm, myrank, rkind
    use yownodepool, only: npa
    use yowerr
    use mpi
    implicit none
    real(kind=rkind), intent(inout) :: u(0:npa)
 
    integer :: i, ierr, tag
    integer :: sendrqst(nconndomains), recvrqst(nconndomains)
    integer :: recvstat(mpi_status_size, nconndomains), sendstat(mpi_status_size, nconndomains)
    character(len=200) errstr
 
    ! It is impossible to add these range checks because assumed shape array start vom 1:npa+1 even if you allocate it from 0:npa
 
    !     if(size(U) /= npa+1) then
    !       write(errstr, *) "size(U) /= npa+1", size(U), "should be", npa+1
    !       ABORT(errstr)
    !     endif
    !
    !     if(ubound(U,1) /= npa) then
    !       write(errstr, *) "ubound(U) /= npa", ubound(U), "should be", npa
    !       ABORT(errstr)
    !     endif
    !
    !     if(lbound(U,1) /= 0) then
    !       write(errstr, *) "lbound(U) /= 0", lbound(U), "should be 0"
    !       ABORT(errstr)
    !     endif
 
    ! post receives
    do i=1, nconndomains
      tag = 10001 + myrank
      call mpi_irecv(u, &
           1, &
           neighbordomains(i)%p1DRrecvType_zero, &
           neighbordomains(i)%domainID-1, &
           tag, &
           comm, &
           recvrqst(i), &
           ierr)
      if(ierr/=mpi_success) then
        CALL parallel_abort("MPI_IRecv", ierr)
      endif
    enddo
 
    ! post sends
    do i=1, nconndomains
      tag = 10001 + (neighbordomains(i)%domainID-1)
      call mpi_isend(u, &
           1, &
           neighbordomains(i)%p1DRsendType_zero, &
           neighbordomains(i)%domainID-1, &
           tag, &
           comm, &
           sendrqst(i), &
           ierr);
      if(ierr/=mpi_success) then
        CALL parallel_abort("MPI_ISend", ierr)
      endif
    end do
 
    ! Wait for completion
    call mpi_waitall(nconndomains, recvrqst, recvstat,ierr)
    if(ierr/=mpi_success) CALL parallel_abort("waitall", ierr)
    call mpi_waitall(nconndomains, sendrqst, sendstat,ierr)
    if(ierr/=mpi_success) CALL parallel_abort("waitall", ierr)
  end subroutine pdlib_exchange1dreal_zero
  subroutine pdlib_exchange2dreal_zero(U)
    use yowdatapool, only: comm, myrank, rkind
    use yownodepool, only: npa
    use yowerr
    use mpi
    implicit none
    real(kind=rkind), intent(inout) :: u(n2nddim,0:npa)
 
    integer :: i, ierr, tag
    integer :: sendrqst(nconndomains), recvrqst(nconndomains)
    integer :: recvstat(mpi_status_size, nconndomains), sendstat(mpi_status_size, nconndomains)
    character(len=200) errstr
 
    ! It is impossible to add these range checks because assumed shape array start vom 1:npa+1 even if you allocate it from 0:npa
    !     if(size(U,2) /= npa+1) then
    !       write(errstr, *) "size(U,2) /= npa+1", size(U,2), "should be", npa+1
    !       ABORT(errstr)
    !     endif
    !
    !     if(ubound(U,2) /= npa) then
    !       write(errstr, *) "ubound(U,2) /= npa", ubound(U,2), "should be", npa
    !       ABORT(errstr)
    !     endif
    !
    !     if(lbound(U,2) /= 0) then
    !       write(errstr, *) "lbound(U,2) /= 0", lbound(U,2), "should be 0"
    !       ABORT(errstr)
    !     endif
 
    !    if((size(U,1) /= n2ndDim) ) then
    !        write(errstr, *) "size(U,1) /= n2ndDim size(U,1)=", size(U,1), " n2ndDim=", n2ndDim
    !        ABORT(errstr)
    !    endif
 
    ! post receives
    do i=1, nconndomains
      tag = 30001 + myrank
      call mpi_irecv(u, &
           1, &
           neighbordomains(i)%p2DRrecvType_zero, &
           neighbordomains(i)%domainID-1, &
           tag, &
           comm, &
           recvrqst(i), &
           ierr)
      if(ierr/=mpi_success) then
        CALL parallel_abort("MPI_IRecv", ierr)
      endif
    enddo
 
    ! post sends
    do i=1, nconndomains
      tag = 30001 + (neighbordomains(i)%domainID-1)
      call mpi_isend(u, &
           1, &
           neighbordomains(i)%p2DRsendType_zero, &
           neighbordomains(i)%domainID-1, &
           tag, &
           comm, &
           sendrqst(i), &
           ierr);
      if(ierr/=mpi_success) then
        CALL parallel_abort("MPI_ISend", ierr)
      endif
    end do
 
 
    ! Wait for completion
    call mpi_waitall(nconndomains, recvrqst, recvstat,ierr)
    if(ierr/=mpi_success) CALL parallel_abort("waitall", ierr)
    call mpi_waitall(nconndomains, sendrqst, sendstat,ierr)
    if(ierr/=mpi_success) CALL parallel_abort("waitall", ierr)
  end subroutine pdlib_exchange2dreal_zero
 
 
end module yowexchangemodule