让我们假设有一个大型的气候数据集，这些数据在每月的时间步长上，涉及世界上的大量点。然后数据集被塑造为以下类型的data.frame：
伦敦， 纬度， data_month_1_yr_1， ...， data_month_12_yr_100
例：
set.seed(123)
data<- data.frame(cbind(runif(10000,-180,180), runif(10000,-90,90))
, replicate(1200, runif(10000,0,150)))
我想在每个空间点的每月时间序列上执行曼-肯德尔测试(使用 trend::mk.test(，并在data.frame中获取主要统计数据。为了加快这个非常长的过程，我并行化了我的代码并编写了如下内容：
coords<-data[,1:2] #get the coordinates out of the initial dataset
names(coords)<-c("lon","lat")
data_t<- as.data.frame(t(data[,3:1202])) #each column is now the time series associated to a point
data_t$month<-rep(seq(1,12,1),100) # month index as last column of the data frame
# start the parallel processing
library(foreach)
library(doParallel)
cores=detectCores() #count cores
cl <- makeCluster(cores[1]-1) #take all the cores minus 1 not to overload the pc
registerDoParallel(cl)
mk_out<- foreach(m=1:12, .combine = rbind) %:%
foreach (a =1:10000, .combine = rbind) %dopar% {
data_m<-data_t[which(data_t$month==m),]
library(trend) #need to load this all the times otherwise I get an error (don't know why)
test<-mk.test(data_m[,a])
mk_out_temp <- data.frame("lon"=coords[a,1],
"lat"=coords[a,2],
"p.value" = as.numeric(test$p.value),
"z_stat" = as.numeric(test$statistic),
"tau" = as.numeric(test$estimates[3]),
"month"= as.numeric(m))
mk_out_temp
}
stopCluster(cl)
head(mk_out)
lon lat p.value z_stat tau month
1 -76.47209 -34.09350 0.57759040 -0.5569078 -0.03797980 1
2 103.78985 -31.58639 0.64436238 0.4616081 0.03151515 1
3 -32.76831 66.64575 0.11793238 1.5635113 0.10626263 1
4 137.88627 -30.83872 0.79096910 0.2650524 0.01818182 1
5 158.56822 -67.37378 0.09595919 -1.6647673 -0.11313131 1
6 -163.59966 -25.88014 0.82325630 0.2233588 0.01535354 1
这运行良好，给了我我所追求的：一个矩阵，报告每个坐标和月份组合的 M-K 统计数据。尽管该过程是并行的，但是，计算仍然需要相当长的时间。
有没有办法加快这个过程？有使用apply系列功能的空间吗？
您注意到您已经解决了问题。可通过以下步骤之一获取：
1：使用 .packages 和 .export 将必要的对象复制到 foreach 循环中。这可确保每个实例在尝试访问同一内存时不会发生冲突。
2：利用高性能库，如 tidyverse of data.table 来执行子集和计算。
后者有点复杂，但极大地提高了我小型笔记本电脑的性能。(对整个数据集执行所有计算大约 1.5 分钟。
下面是我添加的代码。请注意，我用并行包中的单个 parLapply 函数替换了 foreach。
set.seed(123)
data<- data.frame(cbind(runif(10000,-180,180), runif(10000,-90,90))
, replicate(1200, runif(10000,0,150)))
coords<-data[,1:2] #get the coordinates out of the initial dataset
names(coords)<-c("lon","lat")
data_t<- as.data.frame(t(data[,3:1202])) #each column is now the time series associated to a point
data_t$month<-rep(seq(1,12,1),100) # month index as last column of the data frame
# start the parallel processing
library(data.table)
library(parallel)
library(trend)
setDT(data_t)
setDT(coords)
cores=detectCores() #count cores
cl <- makeCluster(cores[1]-1) #take all the cores minus 1 not to overload the pc
#user system elapsed
#17.80 35.12 98.72
system.time({
test <- data_t[,parLapply(cl,
.SD, function(x){
(
unlist(
trend::mk.test(x)[c("p.value","statistic","estimates")]
)
)
}
), by = month] #Perform the calculations across each month
#create a column that indicates what each row is measuring
rows <- rep(c("p.value","statistic.z","estimates.S","estimates.var","estimates.tau"),12)
final_tests <- dcast( #Cast the melted structure to a nice form
melt(cbind(test,rowname = rows), #Melt the data for a better structure
id.vars = c("rowname","month"), #Grouping variables
measure.vars = paste0("V",seq.int(1,10000))), #variable names
month + variable ~ rowname, #LHS groups the data along rows, RHS decides the value columns
value.var = "value", #Which column contain values?
drop = TRUE) #should we drop unused columns? (doesnt matter here)
#rename the columns as desired
names(final_tests) <- c("month","variable","S","tau","var","p.value","z_stat")
#finally add the coordinates
final_tests <- cbind(final_form,coords)
})
最后，通过用lapply函数替换第二个循环(受此答案的启发(，可以轻松解决问题。执行时间现在控制在几秒钟内。矢量化仍然是 R 中执行时间的最佳解决方案(请参阅这篇文章和这篇文章(
我在下面分享最终代码以供参考：
set.seed(123)
data<- data.frame(cbind(runif(10000,-180,180), runif(10000,-90,90)), replicate(1200, runif(10000,0,150)))
coords<-data[,1:2]
names(coords)<-c("lon","lat")
data_t<- as.data.frame(t(data[,3:1202]))
data_t$month<-rep(seq(1,12,1),100)
library(foreach)
library(doParallel)
cores=detectCores()
cl <- makeCluster(cores[1]-1) #take all the cores minus 1
registerDoParallel(cl)
mk_out<- foreach(m=1:12, .combine = rbind) %dopar% {
data_m<-data_t[which(data_t$month==m),]
library(trend)
mk_out_temp <- do.call(rbind,lapply(data_m[1:100],function(x)unlist(mk.test(x))))
mk_out_temp <-cbind(coords,mk_out_temp,rep(m,dim(coords)[1]))
mk_out_temp
}
stopCluster(cl)
head(mk_out)
head(mk_out)
lon lat data.name p.value statistic.z null.value.S parameter.n estimates.S estimates.varS
1 -76.47209 -34.09350 x 0.577590398263635 -0.556907839290681 0 100 -188 112750
2 103.78985 -31.58639 x 0.644362383361713 0.461608102085858 0 100 156 112750
3 -32.76831 66.64575 x 0.117932376736468 1.56351131351662 0 100 526 112750
4 137.88627 -30.83872 x 0.79096910003836 0.265052394100912 0 100 90 112750
5 158.56822 -67.37378 x 0.0959591933285242 -1.66476728429674 0 100 -560 112750
6 -163.59966 -25.88014 x 0.823256299016955 0.223358759073802 0 100 76 112750
estimates.tau alternative method pvalg rep(m, dim(coords)[1])
1 -0.037979797979798 two.sided Mann-Kendall trend test 0.577590398263635 1
2 0.0315151515151515 two.sided Mann-Kendall trend test 0.644362383361713 1
3 0.106262626262626 two.sided Mann-Kendall trend test 0.117932376736468 1
4 0.0181818181818182 two.sided Mann-Kendall trend test 0.79096910003836 1
5 -0.113131313131313 two.sided Mann-Kendall trend test 0.0959591933285242 1
6 0.0153535353535354 two.sided Mann-Kendall trend test 0.823256299016955 1