setting strategy to mimic rmpi/dompi #40

Activity

1. yonicd commented on Mar 8, 2019

   

   yonicd

   on Mar 8, 2019

   Author

   More actions

   I think this is the equiv

   cl <- parallel::makeCluster(Sys.getenv()["SLURM_NTASKS"], type = "MPI") 
   
   future::plan(future::cluster, workers = cl)
   
   furrr::future_map(1:4,.f=function(i){
   paste(i,Sys.info()["nodename"], sep = ":") 
   })
   
   parallel::stopCluster(cl)
   

   not sure how to mimic Rmpi::mpi.comm.rank()
2. yonicd commented on Apr 25, 2019

   

   yonicd

   on Apr 25, 2019

   Last edited by yonicd

   Author

   More actions

   I have this version of multiprocess/sge/mpi working, which somewhat answers my initial query. Is there any way to float up the cl <- parallel::makeCluster(2, type = "MPI") into the plan step?

   the sge template is one that is referenced in issue #39 (comment)

   library(future)
   library(batchtools)
   library(future.batchtools)
   
   sge <- future::tweak(
     future.batchtools::batchtools_sge,
     label = 'test',
     template = 'batchtools.sge-mrg.tmpl',
     resources = list(slots = 4)
   )
   
   future::plan(list(future::multiprocess,sge))
   
   f <- future::future({
     future.apply::future_lapply(1:4,FUN=function(i){
       
       hello_world <- function() {
         ## Print the hostname and MPI worker rank.
         paste(i,Sys.info()["nodename"],Rmpi::mpi.comm.rank(), sep = ":")
       }
   
       cl <- parallel::makeCluster(2, type = "MPI")
       on.exit(parallel::stopCluster(cl),add = TRUE)
   
       parallel::clusterCall(cl, hello_world)
       
       
     })
   })
   
   future::value(f)
   
   # 2 slaves are spawned successfully. 0 failed.
   # 2 slaves are spawned successfully. 0 failed.
   # 2 slaves are spawned successfully. 0 failed.
   # 2 slaves are spawned successfully. 0 failed.
   # Loading required namespace: Rmpi
   # Loading required namespace: Rmpi
   # Loading required namespace: Rmpi
   # Loading required namespace: Rmpi
   # [[1]]
   # [[1]][[1]]
   # [1] "1:ip-172-16-2-22:1"
   # 
   # [[1]][[2]]
   # [1] "1:ip-172-16-2-22:2"
   # 
   # 
   # [[2]]
   # [[2]][[1]]
   # [1] "2:ip-172-16-2-129:1"
   # 
   # [[2]][[2]]
   # [1] "2:ip-172-16-2-129:2"
   # 
   # 
   # [[3]]
   # [[3]][[1]]
   # [1] "3:ip-172-16-2-39:1"
   # 
   # [[3]][[2]]
   # [1] "3:ip-172-16-2-39:2"
   # 
   # 
   # [[4]]
   # [[4]][[1]]
   # [1] "4:ip-172-16-2-22:1"
   # 
   # [[4]][[2]]
   # [1] "4:ip-172-16-2-22:2"