From 65f2a62ba855b45124d720bec9ee0a9738768fcd Mon Sep 17 00:00:00 2001 From: corentinchoisy Date: Thu, 25 Apr 2024 17:37:18 +0200 Subject: [PATCH] corrected rosali bug --- Modules/rosali_custom/rosali_original.ado | 12 +- RProject/Scripts/aggregation.R | 258 ++++++++---------- RProject/Scripts/resali_generate_newdata.R | 144 ++++++++++ .../Analysis/DIF-RESIDUS/pcm_dif_residus.do | 2 +- 4 files changed, 265 insertions(+), 151 deletions(-) diff --git a/Modules/rosali_custom/rosali_original.ado b/Modules/rosali_custom/rosali_original.ado index 7aa7b91..a864920 100644 --- a/Modules/rosali_custom/rosali_original.ado +++ b/Modules/rosali_custom/rosali_original.ado @@ -767,11 +767,12 @@ if "`group'"!="" & "`nodif'"=="" { // PARTIE 1 = Slmt si option group & pas de " local nb_stepC = 0 qui lrtest modeldifA modeldifB local diftestp=r(p) + local lrt_passed=0 if `diftestp'<0.05{ /*If pvalue(LRtest)<0.05 then step C*/ di di as input "PROCESSING STEP C" di - + local lrt_passed=1 /*test DIF pour chaque item*/ local boucle = 1 local stop = 0 @@ -1135,11 +1136,18 @@ else if "`group'" == "" { di } -matrix dif_detect = J(1,`nbitems',.) +matrix dif_detect = J(1,2*`nbitems'+1,.) local numdif=1 +matrix dif_detect[1,`nbitems'+`nbitems'+1]=`lrt_passed' forvalues j=1/`nbitems' { if dif_rc[`j',1] != . { matrix dif_detect[1,`numdif']=`j' + if dif_rc[`j',2] == 0 { + matrix dif_detect[1,`nbitems'+`numdif']=0 + } + if dif_rc[`j',2] != 0 { + matrix dif_detect[1,`nbitems'+`numdif']=1 + } local numdif = `numdif'+1 } } diff --git a/RProject/Scripts/aggregation.R b/RProject/Scripts/aggregation.R index fd32bce..b059396 100644 --- a/RProject/Scripts/aggregation.R +++ b/RProject/Scripts/aggregation.R @@ -417,10 +417,25 @@ compile_simulation2_rosali <- function(scenario) { J <- max(which(sapply(1:7,function(x) paste0('item',x) %in% colnames(s) | paste0('item',x,'_1') %in% colnames(s)))) M <- 1+sum(sapply(1:3,function(x) paste0('item1_',x) %in% colnames(s) )) if (M==1) {M <- 2} - nb.dif <- max(which(sapply(1:3,function(x) paste0('dif',x) %in% colnames(s) | paste0('dif',x,'_1') %in% colnames(s)))) + nb.dif.true <- ifelse(name<=4,0,ifelse(name<=8,1,ifelse(name<=16,2,3))) + if (name %in% c(3,4,13:20)) { + m.nb.dif.detect <- mean(ifelse(is.na(s$dif_1_1),0, + ifelse(is.na(s$dif_2_1),1, + ifelse(is.na(s$dif_3_1),2, + ifelse(is.na(s$dif_4_1),3, + ifelse(is.na(s$dif_5_1),4, + ifelse(is.na(s$dif_6_1),5, + ifelse(is.na(s$dif_7_1),6,7)))))))) + } + if (!(name %in% c(3,4,13:20))) { + m.nb.dif.detect <- mean(ifelse(is.na(s$dif_1_1),0, + ifelse(is.na(s$dif_2_1),1, + ifelse(is.na(s$dif_3_1),2, + ifelse(is.na(s$dif_4_1),3,4))))) + } if (J==4) { if (M==2) { - a <- data.frame(m.item1=mean(s$item1),m.item2=mean(s$item2),m.item3=mean(s$item3),m.item4=mean(s$item4)) + a <- data.frame(m.item1=mean(s$item1_1),m.item2=mean(s$item2_1),m.item3=mean(s$item3_1),m.item4=mean(s$item4_1)) } else { a <- data.frame(m.item1_1=mean(s$item1_1),m.item1_2=mean(s$item1_2),m.item1_3=mean(s$item1_3), m.item2_1=mean(s$item2_1),m.item2_2=mean(s$item2_2),m.item2_3=mean(s$item2_3), @@ -430,8 +445,8 @@ compile_simulation2_rosali <- function(scenario) { } } else { if (M==2) { - a <- data.frame(m.item1=mean(s$item1),m.item2=mean(s$item2),m.item3=mean(s$item3),m.item4=mean(s$item4), - m.item5=mean(s$item5),m.item6=mean(s$item6),m.item7=mean(s$item7)) + a <- data.frame(m.item1=mean(s$item1_1),m.item2=mean(s$item2_1),m.item3=mean(s$item3_1),m.item4=mean(s$item4_1), + m.item5=mean(s$item5_1),m.item6=mean(s$item6_1),m.item7=mean(s$item7_1)) } else { a <- data.frame(m.item1_1=mean(s$item1_1),m.item1_2=mean(s$item1_2),m.item1_3=mean(s$item1_3), m.item2_1=mean(s$item2_1),m.item2_2=mean(s$item2_2),m.item2_3=mean(s$item2_3), @@ -453,12 +468,97 @@ compile_simulation2_rosali <- function(scenario) { J=J, M=M, eff.size=eff.size, - nb.dif=nb.dif, + nb.dif=nb.dif.true, + m.nb.dif.detect=m.nb.dif.detect, dif.size=dif.size ) true.value.in.ci <- eff.size <= s$beta+1.96*s$se_beta & eff.size >= s$beta-1.96*s$se_beta beta.same.sign.truebeta.p <- ifelse(rep(eff.size,nrow(s))==0,NA,(rep(eff.size,nrow(s))/s$beta)>0) num.reject <- which((s$beta-1.96*s$se_beta)>0 | (s$beta+1.96*s$se_beta)<0) + dif.d <- mean(sapply(1:1000,function(x) any(!is.na(s[x,paste0("dif_",1:unique(b$J),"_1")])))) + if (nb.dif.true==0) { + prop.perfect <- NA + flexible.detect <- NA + moreflexible.detect <- NA + } + if (nb.dif.true==1 & unique(b$J)==4 & unique(b$M)==4) { + perfect.detection <- sapply(1:1000,function(x) ifelse(sum(!is.na(s[x,paste0("dif_detect_",1:4)]))==1,s[x,"dif_detect_unif_1"]==1 & s[x,paste0('dif_detect_',which(sapply(paste0("dif_detect_",1:4),function(y) !is.na(s[x,y])))[1])]%in%c(s[x,c("real_dif_1")]) + ,0) ) + prop.perfect <- mean(perfect.detection) + flexible.detect <- sapply(1:1000,function(x) ifelse(sum(!is.na(s[x,paste0("dif_detect_",1:4)]))==1,s[x,paste0('dif_detect_',which(sapply(paste0("dif_detect_",1:4),function(y) !is.na(s[x,y])))[1])]%in%c(s[x,c("real_dif_1")]) + ,0) ) + flexible.detect <- mean(flexible.detect) + moreflexible.detect <- sapply(1:1000,function(x) ifelse(sum(!is.na(s[x,paste0("dif_detect_",1:4)]))!=0,s[x,"real_dif_1"]%in%c(s[x,paste0("dif_detect_",1:4)]),0) ) + moreflexible.detect <- mean(moreflexible.detect) + } + if (nb.dif.true==2 & unique(b$J)==4 & unique(b$M)==4) { + perfect.detection <- sapply(1:1000,function(x) ifelse(sum(!is.na(s[x,paste0("dif_detect_",1:4)]))==2,s[x,"dif_detect_unif_1"]==1 & s[x,"dif_detect_unif_2"]==1 & s[x,paste0('dif_detect_',which(sapply(paste0("dif_detect_",1:4),function(y) !is.na(s[x,y])))[1])]%in%c(s[x,c("real_dif_1","real_dif_2")]) & s[x,paste0('dif_detect_',which(sapply(paste0("dif_detect_",1:4),function(y) !is.na(s[x,y])))[2])]%in%c(s[x,c("real_dif_1","real_dif_2")]) + ,0) ) + prop.perfect <- mean(perfect.detection) + flexible.detect <- sapply(1:1000,function(x) ifelse(sum(!is.na(s[x,paste0("dif_detect_",1:4)]))==2,s[x,paste0('dif_detect_',which(sapply(paste0("dif_detect_",1:4),function(y) !is.na(s[x,y])))[1])]%in%c(s[x,c("real_dif_1","real_dif_2")]) & s[x,paste0('dif_detect_',which(sapply(paste0("dif_detect_",1:4),function(y) !is.na(s[x,y])))[2])]%in%c(s[x,c("real_dif_1","real_dif_2")]) + ,0) ) + flexible.detect <- mean(flexible.detect) + moreflexible.detect <- sapply(1:1000,function(x) ifelse(sum(!is.na(s[x,paste0("dif_detect_",1:4)]))!=0,s[x,"real_dif_1"]%in%c(s[x,paste0("dif_detect_",1:4)]) & + s[x,"real_dif_2"]%in%c(s[x,paste0("dif_detect_",1:4)]),0) ) + moreflexible.detect <- mean(moreflexible.detect) + } + if (nb.dif.true==2 & unique(b$J)==7 & unique(b$M)==4) { + perfect.detection <- sapply(1:1000,function(x) ifelse(sum(!is.na(s[x,paste0("dif_detect_",1:7)]))==2,s[x,"dif_detect_unif_1"]==1 & s[x,"dif_detect_unif_2"]==1 & s[x,paste0('dif_detect_',which(sapply(paste0("dif_detect_",1:7),function(y) !is.na(s[x,y])))[1])]%in%c(s[x,c("real_dif_1","real_dif_2")]) & s[x,paste0('dif_detect_',which(sapply(paste0("dif_detect_",1:7),function(y) !is.na(s[x,y])))[2])]%in%c(s[x,c("real_dif_1","real_dif_2")]) + ,0) ) + prop.perfect <- mean(perfect.detection) + flexible.detect <- sapply(1:1000,function(x) ifelse(sum(!is.na(s[x,paste0("dif_detect_",1:7)]))==2,s[x,paste0('dif_detect_',which(sapply(paste0("dif_detect_",1:7),function(y) !is.na(s[x,y])))[1])]%in%c(s[x,c("real_dif_1","real_dif_2")]) & s[x,paste0('dif_detect_',which(sapply(paste0("dif_detect_",1:7),function(y) !is.na(s[x,y])))[2])]%in%c(s[x,c("real_dif_1","real_dif_2")]) + ,0) ) + flexible.detect <- mean(flexible.detect) + moreflexible.detect <- sapply(1:1000,function(x) ifelse(sum(!is.na(s[x,paste0("dif_detect_",1:7)]))!=0,s[x,"real_dif_1"]%in%c(s[x,paste0("dif_detect_",1:7)]) & + s[x,"real_dif_2"]%in%c(s[x,paste0("dif_detect_",1:7)]),0) ) + moreflexible.detect <- mean(moreflexible.detect) + } + if (nb.dif.true==3 & unique(b$J)==7 & unique(b$M)==4) { + perfect.detection <- sapply(1:1000,function(x) ifelse(sum(!is.na(s[x,paste0("dif_detect_",1:7)]))==3,s[x,"dif_detect_unif_1"]==1 & s[x,"dif_detect_unif_2"]==1 & s[x,"dif_detect_unif_3"]==1 & s[x,paste0('dif_detect_',which(sapply(paste0("dif_detect_",1:7),function(y) !is.na(s[x,y])))[1])]%in%c(s[x,c("real_dif_1","real_dif_2","real_dif_3")]) & s[x,paste0('dif_detect_',which(sapply(paste0("dif_detect_",1:7),function(y) !is.na(s[x,y])))[2])]%in%c(s[x,c("real_dif_1","real_dif_2","real_dif_3")]) & s[x,paste0('dif_detect_',which(sapply(paste0("dif_detect_",1:7),function(y) !is.na(s[x,y])))[3])]%in%c(s[x,c("real_dif_1","real_dif_2","real_dif_3")]) + ,0) ) + prop.perfect <- mean(perfect.detection) + flexible.detect <- sapply(1:1000,function(x) ifelse(sum(!is.na(s[x,paste0("dif_detect_",1:7)]))==3,s[x,paste0('dif_detect_',which(sapply(paste0("dif_detect_",1:7),function(y) !is.na(s[x,y])))[1])]%in%c(s[x,c("real_dif_1","real_dif_2","real_dif_3")]) & s[x,paste0('dif_detect_',which(sapply(paste0("dif_detect_",1:7),function(y) !is.na(s[x,y])))[2])]%in%c(s[x,c("real_dif_1","real_dif_2","real_dif_3")]) & s[x,paste0('dif_detect_',which(sapply(paste0("dif_detect_",1:7),function(y) !is.na(s[x,y])))[3])]%in%c(s[x,c("real_dif_1","real_dif_2","real_dif_3")]) + ,0) ) + flexible.detect <- mean(flexible.detect) + moreflexible.detect <- sapply(1:1000,function(x) ifelse(sum(!is.na(s[x,paste0("dif_detect_",1:7)]))!=0,s[x,"real_dif_1"]%in%c(s[x,paste0("dif_detect_",1:7)]) & + s[x,"real_dif_2"]%in%c(s[x,paste0("dif_detect_",1:7)]) & s[x,"real_dif_3"]%in%c(s[x,paste0("dif_detect_",1:7)]),0) ) + moreflexible.detect <- mean(moreflexible.detect) + } + if (nb.dif.true==1 & unique(b$J)==4 & unique(b$M)==2) { + perfect.detection <- sapply(1:1000,function(x) ifelse(sum(!is.na(s[x,paste0("dif_detect_",1:4)]))==1, s[x,paste0('dif_detect_',which(sapply(paste0("dif_detect_",1:4),function(y) !is.na(s[x,y])))[1])]%in%c(s[x,c("real_dif_1")]) + ,0) ) + prop.perfect <- mean(perfect.detection) + flexible.detect <- prop.perfect + moreflexible.detect <- sapply(1:1000,function(x) ifelse(sum(!is.na(s[x,paste0("dif_detect_",1:4)]))!=0,s[x,"real_dif_1"]%in%c(s[x,paste0("dif_detect_",1:4)]),0) ) + moreflexible.detect <- mean(moreflexible.detect) + } + if (nb.dif.true==2 & unique(b$J)==4 & unique(b$M)==2) { + perfect.detection <- sapply(1:1000,function(x) ifelse(sum(!is.na(s[x,paste0("dif_detect_",1:4)]))==2,s[x,paste0('dif_detect_',which(sapply(paste0("dif_detect_",1:4),function(y) !is.na(s[x,y])))[1])]%in%c(s[x,c("real_dif_1","real_dif_2")]) & s[x,paste0('dif_detect_',which(sapply(paste0("dif_detect_",1:4),function(y) !is.na(s[x,y])))[2])]%in%c(s[x,c("real_dif_1","real_dif_2")]) + ,0) ) + prop.perfect <- mean(perfect.detection) + flexible.detect <- prop.perfect + moreflexible.detect <- sapply(1:1000,function(x) ifelse(sum(!is.na(s[x,paste0("dif_detect_",1:4)]))!=0,s[x,"real_dif_1"]%in%c(s[x,paste0("dif_detect_",1:4)]) & + s[x,"real_dif_2"]%in%c(s[x,paste0("dif_detect_",1:4)]),0) ) + moreflexible.detect <- mean(moreflexible.detect) + } + if (nb.dif.true==2 & unique(b$J)==7 & unique(b$M)==2) { + perfect.detection <- sapply(1:1000,function(x) ifelse(sum(!is.na(s[x,paste0("dif_detect_",1:7)]))==2,s[x,paste0('dif_detect_',which(sapply(paste0("dif_detect_",1:7),function(y) !is.na(s[x,y])))[1])]%in%c(s[x,c("real_dif_1","real_dif_2")]) & s[x,paste0('dif_detect_',which(sapply(paste0("dif_detect_",1:7),function(y) !is.na(s[x,y])))[2])]%in%c(s[x,c("real_dif_1","real_dif_2")]) + ,0) ) + prop.perfect <- mean(perfect.detection) + flexible.detect <- prop.perfect + moreflexible.detect <- sapply(1:1000,function(x) ifelse(sum(!is.na(s[x,paste0("dif_detect_",1:7)]))!=0,s[x,"real_dif_1"]%in%c(s[x,paste0("dif_detect_",1:7)]) & + s[x,"real_dif_2"]%in%c(s[x,paste0("dif_detect_",1:7)]),0) ) + moreflexible.detect <- mean(moreflexible.detect) + } + if (nb.dif.true==3 & unique(b$J)==7 & unique(b$M)==2) { + perfect.detection <- sapply(1:1000,function(x) ifelse(sum(!is.na(s[x,paste0("dif_detect_",1:7)]))==3,s[x,paste0('dif_detect_',which(sapply(paste0("dif_detect_",1:7),function(y) !is.na(s[x,y])))[1])]%in%c(s[x,c("real_dif_1","real_dif_2","real_dif_3")]) & s[x,paste0('dif_detect_',which(sapply(paste0("dif_detect_",1:7),function(y) !is.na(s[x,y])))[2])]%in%c(s[x,c("real_dif_1","real_dif_2","real_dif_3")]) & s[x,paste0('dif_detect_',which(sapply(paste0("dif_detect_",1:7),function(y) !is.na(s[x,y])))[3])]%in%c(s[x,c("real_dif_1","real_dif_2","real_dif_3")]) + ,0) ) + prop.perfect <- mean(perfect.detection) + flexible.detect <- prop.perfect + moreflexible.detect <- sapply(1:1000,function(x) ifelse(sum(!is.na(s[x,paste0("dif_detect_",1:7)]))!=0,s[x,"real_dif_1"]%in%c(s[x,paste0("dif_detect_",1:7)]) & + s[x,"real_dif_2"]%in%c(s[x,paste0("dif_detect_",1:7)]) & s[x,"real_dif_3"]%in%c(s[x,paste0("dif_detect_",1:7)]),0) ) + moreflexible.detect <- mean(moreflexible.detect) + } z <- data.frame(m.beta=mean(s$beta), se.empirical.beta=sd(s$beta), se.analytical.beta=mean(s$se_beta), @@ -467,7 +567,11 @@ compile_simulation2_rosali <- function(scenario) { true.value.in.ci.p=mean(true.value.in.ci), h0.rejected.p=mean( (s$beta-1.96*s$se_beta)>0 | (s$beta+1.96*s$se_beta)<0 ), beta.same.sign.truebeta.p=mean(beta.same.sign.truebeta.p), - beta.same.sign.truebeta.signif.p=mean(beta.same.sign.truebeta.p[num.reject]) + beta.same.sign.truebeta.signif.p=mean(beta.same.sign.truebeta.p[num.reject]), + dif.detected=dif.d, + prop.perfect=prop.perfect, + flexible.detect=flexible.detect, + moreflexible.detect=moreflexible.detect ) d <- cbind(b,a,z) d$prop. @@ -487,148 +591,6 @@ for (x in results[seq(2,length(results))]) { res.dat.dif.rosali$bias <- res.dat.dif.rosali$eff.size-res.dat.dif.rosali$m.beta -############################################################################## -#----------------------------------------------------------------------------# -################################### RESALI ################################### -#----------------------------------------------------------------------------# -############################################################################## - -generate_resali <- function(scenario=NULL,grp=NULL) { - scen <- as.numeric(gsub("[A,B,C,D,E,F,G,_]","",substr(scenario,0,3))) - if (substr(scenario,start=nchar(scenario)-1,stop=nchar(scenario))=="50") { - N <- 50 - } - if (substr(scenario,start=nchar(scenario)-2,stop=nchar(scenario))=="100") { - N <- 100 - } - if (substr(scenario,start=nchar(scenario)-2,stop=nchar(scenario))=="200") { - N <- 200 - } - if (substr(scenario,start=nchar(scenario)-2,stop=nchar(scenario))=="300") { - N <- 300 - } - if (scen<5) { - dat <- read.csv(paste0('/home/corentin/Documents/These/Recherche/Simulations/Data/NoDIF/N',N,'/scenario_',scenario,'.csv')) - } - if (scen>=5) { - dat <- read.csv(paste0('/home/corentin/Documents/These/Recherche/Simulations/Data/DIF/N',N,'/scenario_',scenario,'.csv')) - } - if (scen%in%c(3,4,13:20)) { - res <- resali(df=dat[dat$replication==1,],items = seq(1,7),group=grp,verbose=FALSE) - df_res <- data.frame(dif.detect.1=ifelse(length(res$dif.items)>=1,res$dif.items[1],NA), - dif.detect.2=ifelse(length(res$dif.items)>=2,res$dif.items[2],NA), - dif.detect.3=ifelse(length(res$dif.items)>=3,res$dif.items[3],NA), - dif.detect.4=ifelse(length(res$dif.items)>=4,res$dif.items[4],NA), - dif.detect.5=ifelse(length(res$dif.items)>=5,res$dif.items[5],NA), - dif.detect.6=ifelse(length(res$dif.items)>=6,res$dif.items[6],NA), - dif.detect.7=ifelse(length(res$dif.items)>=7,res$dif.items[7],NA), - dif.detect.unif.1=ifelse(length(res$uniform)>=1,res$uniform[1],NA), - dif.detect.unif.2=ifelse(length(res$uniform)>=2,res$uniform[2],NA), - dif.detect.unif.3=ifelse(length(res$uniform)>=3,res$uniform[3],NA), - dif.detect.unif.4=ifelse(length(res$uniform)>=4,res$uniform[4],NA), - dif.detect.unif.5=ifelse(length(res$uniform)>=5,res$uniform[5],NA), - dif.detect.unif.6=ifelse(length(res$uniform)>=6,res$uniform[6],NA), - dif.detect.unif.7=ifelse(length(res$uniform)>=7,res$uniform[7],NA), - N=N, - nbdif=ifelse(scen<=4,0,ifelse(scen<=16,2,3)), - true.dif.1=ifelse(scen<=4,NA,unique(dat[dat$replication==1,]$dif1)), - true.dif.2=ifelse(scen<=4,NA,unique(dat[dat$replication==1,]$dif2)), - true.dif.3=ifelse(scen<=16,NA,unique(dat[dat$replication==1,]$dif3)) - ) - for (k in 2:1000) { - if (k%%100==0) { - cat(paste0('N=',k,'/1000\n')) - } - res <- resali(df=dat[dat$replication==k,],items = seq(1,7),group=grp,verbose=FALSE) - df_res2 <- data.frame(dif.detect.1=ifelse(length(res$dif.items)>=1,res$dif.items[1],NA), - dif.detect.2=ifelse(length(res$dif.items)>=2,res$dif.items[2],NA), - dif.detect.3=ifelse(length(res$dif.items)>=3,res$dif.items[3],NA), - dif.detect.4=ifelse(length(res$dif.items)>=4,res$dif.items[4],NA), - dif.detect.5=ifelse(length(res$dif.items)>=5,res$dif.items[5],NA), - dif.detect.6=ifelse(length(res$dif.items)>=6,res$dif.items[6],NA), - dif.detect.7=ifelse(length(res$dif.items)>=7,res$dif.items[7],NA), - dif.detect.unif.1=ifelse(length(res$uniform)>=1,res$uniform[1],NA), - dif.detect.unif.2=ifelse(length(res$uniform)>=2,res$uniform[2],NA), - dif.detect.unif.3=ifelse(length(res$uniform)>=3,res$uniform[3],NA), - dif.detect.unif.4=ifelse(length(res$uniform)>=4,res$uniform[4],NA), - dif.detect.unif.5=ifelse(length(res$uniform)>=5,res$uniform[5],NA), - dif.detect.unif.6=ifelse(length(res$uniform)>=6,res$uniform[6],NA), - dif.detect.unif.7=ifelse(length(res$uniform)>=7,res$uniform[7],NA), - N=N, - nbdif=ifelse(scen<=4,0,ifelse(scen<=16,2,3)), - true.dif.1=ifelse(scen<=4,NA,unique(dat[dat$replication==k,]$dif1)), - true.dif.2=ifelse(scen<=4,NA,unique(dat[dat$replication==k,]$dif2)), - true.dif.3=ifelse(scen<=16,NA,unique(dat[dat$replication==k,]$dif3))) - df_res <- rbind(df_res,df_res2) - } - } - else if (scen%in%c(1,2,5:12)) { - res <- resali(df=dat[dat$replication==1,],items = seq(1,4),group=grp,verbose=FALSE) - df_res <- data.frame(dif.detect.1=ifelse(length(res$dif.items)>=1,res$dif.items[1],NA), - dif.detect.2=ifelse(length(res$dif.items)>=2,res$dif.items[2],NA), - dif.detect.3=ifelse(length(res$dif.items)>=3,res$dif.items[3],NA), - dif.detect.4=ifelse(length(res$dif.items)>=4,res$dif.items[4],NA), - dif.detect.unif.1=ifelse(length(res$uniform)>=1,res$uniform[1],NA), - dif.detect.unif.2=ifelse(length(res$uniform)>=2,res$uniform[2],NA), - dif.detect.unif.3=ifelse(length(res$uniform)>=3,res$uniform[3],NA), - dif.detect.unif.4=ifelse(length(res$uniform)>=4,res$uniform[4],NA), - N=N, - nbdif=ifelse(scen<=4,0,ifelse(scen<=8,1,2)), - true.dif.1=ifelse(scen<=4,NA,unique(dat[dat$replication==1,]$dif1)), - true.dif.2=ifelse(scen<=8,NA,unique(dat[dat$replication==1,]$dif2)) - ) - for (k in 2:1000) { - if (k%%100==0) { - cat(paste0('N=',k,'/1000\n')) - } - res <- resali(df=dat[dat$replication==k,],items = seq(1,4),group=grp,verbose=FALSE) - df_res2 <- data.frame(dif.detect.1=ifelse(length(res$dif.items)>=1,res$dif.items[1],NA), - dif.detect.2=ifelse(length(res$dif.items)>=2,res$dif.items[2],NA), - dif.detect.3=ifelse(length(res$dif.items)>=3,res$dif.items[3],NA), - dif.detect.4=ifelse(length(res$dif.items)>=4,res$dif.items[4],NA), - dif.detect.unif.1=ifelse(length(res$uniform)>=1,res$uniform[1],NA), - dif.detect.unif.2=ifelse(length(res$uniform)>=2,res$uniform[2],NA), - dif.detect.unif.3=ifelse(length(res$uniform)>=3,res$uniform[3],NA), - dif.detect.unif.4=ifelse(length(res$uniform)>=4,res$uniform[4],NA), - N=N, - nbdif=ifelse(scen<=4,0,ifelse(scen<=8,1,2)), - true.dif.1=ifelse(scen<=4,NA,unique(dat[dat$replication==k,]$dif1)), - true.dif.2=ifelse(scen<=8,NA,unique(dat[dat$replication==k,]$dif2))) - df_res <- rbind(df_res,df_res2) - } - } - return(df_res) -} - - - - - - -results <- c(sapply(1:4,function(x) paste0(x,c('A','B','C','D','E'))),sapply(5:9,function(x) paste0(x,c('A','B','C','D','E','F','G')))) - -results2 <- c(sapply(10:20,function(x) paste0(x,c('A','B','C','D','E','F','G')))) - -results <- c(sapply(c(50,100,200,300),function(x) paste0(results,'_',x))) - -results2 <- c(sapply(c(50,100,200,300),function(x) paste0(results2,'_',x))) - -results <- sort(results) - -results2 <- sort(results2) - -results <- c(results,results2) - -for (r in results) { - cat(paste0(r,"\n")) - cat(paste0("-------------------------------------------","\n")) - write.csv(generate_resali(r,"TT"),paste0("/home/corentin/Documents/These/Recherche/Simulations/Analysis/RESALI/Detection/",r,".csv")) - cat(paste0("-------------------------------------------","\n")) -} - - - - ############################################################################## #----------------------------------------------------------------------------# ####################### AGGREGATION DIF MATRICES RESALI ###################### diff --git a/RProject/Scripts/resali_generate_newdata.R b/RProject/Scripts/resali_generate_newdata.R index fd60de5..f70009e 100644 --- a/RProject/Scripts/resali_generate_newdata.R +++ b/RProject/Scripts/resali_generate_newdata.R @@ -134,3 +134,147 @@ for (r in results) { analyse <- analyse[,!names(analyse) %in% c("X","X.1","X.2")] write.csv(analyse,paste0("/home/corentin/Documents/These/Recherche/Simulations/Analysis/RESALI/Detection_data/",r,".csv")) } + + +############################################################################## +#----------------------------------------------------------------------------# +################################### RESALI ################################### +#----------------------------------------------------------------------------# +############################################################################## + +generate_resali <- function(scenario=NULL,grp=NULL) { + scen <- as.numeric(gsub("[A,B,C,D,E,F,G,_]","",substr(scenario,0,3))) + if (substr(scenario,start=nchar(scenario)-1,stop=nchar(scenario))=="50") { + N <- 50 + } + if (substr(scenario,start=nchar(scenario)-2,stop=nchar(scenario))=="100") { + N <- 100 + } + if (substr(scenario,start=nchar(scenario)-2,stop=nchar(scenario))=="200") { + N <- 200 + } + if (substr(scenario,start=nchar(scenario)-2,stop=nchar(scenario))=="300") { + N <- 300 + } + if (scen<5) { + dat <- read.csv(paste0('/home/corentin/Documents/These/Recherche/Simulations/Data/NoDIF/N',N,'/scenario_',scenario,'.csv')) + } + if (scen>=5) { + dat <- read.csv(paste0('/home/corentin/Documents/These/Recherche/Simulations/Data/DIF/N',N,'/scenario_',scenario,'.csv')) + } + if (scen%in%c(3,4,13:20)) { + res <- resali(df=dat[dat$replication==1,],items = seq(1,7),group=grp,verbose=FALSE) + df_res <- data.frame(dif.detect.1=ifelse(length(res$dif.items)>=1,res$dif.items[1],NA), + dif.detect.2=ifelse(length(res$dif.items)>=2,res$dif.items[2],NA), + dif.detect.3=ifelse(length(res$dif.items)>=3,res$dif.items[3],NA), + dif.detect.4=ifelse(length(res$dif.items)>=4,res$dif.items[4],NA), + dif.detect.5=ifelse(length(res$dif.items)>=5,res$dif.items[5],NA), + dif.detect.6=ifelse(length(res$dif.items)>=6,res$dif.items[6],NA), + dif.detect.7=ifelse(length(res$dif.items)>=7,res$dif.items[7],NA), + dif.detect.unif.1=ifelse(length(res$uniform)>=1,res$uniform[1],NA), + dif.detect.unif.2=ifelse(length(res$uniform)>=2,res$uniform[2],NA), + dif.detect.unif.3=ifelse(length(res$uniform)>=3,res$uniform[3],NA), + dif.detect.unif.4=ifelse(length(res$uniform)>=4,res$uniform[4],NA), + dif.detect.unif.5=ifelse(length(res$uniform)>=5,res$uniform[5],NA), + dif.detect.unif.6=ifelse(length(res$uniform)>=6,res$uniform[6],NA), + dif.detect.unif.7=ifelse(length(res$uniform)>=7,res$uniform[7],NA), + N=N, + nbdif=ifelse(scen<=4,0,ifelse(scen<=16,2,3)), + true.dif.1=ifelse(scen<=4,NA,unique(dat[dat$replication==1,]$dif1)), + true.dif.2=ifelse(scen<=4,NA,unique(dat[dat$replication==1,]$dif2)), + true.dif.3=ifelse(scen<=16,NA,unique(dat[dat$replication==1,]$dif3)) + ) + for (k in 2:1000) { + if (k%%100==0) { + cat(paste0('N=',k,'/1000\n')) + } + res <- resali(df=dat[dat$replication==k,],items = seq(1,7),group=grp,verbose=FALSE) + df_res2 <- data.frame(dif.detect.1=ifelse(length(res$dif.items)>=1,res$dif.items[1],NA), + dif.detect.2=ifelse(length(res$dif.items)>=2,res$dif.items[2],NA), + dif.detect.3=ifelse(length(res$dif.items)>=3,res$dif.items[3],NA), + dif.detect.4=ifelse(length(res$dif.items)>=4,res$dif.items[4],NA), + dif.detect.5=ifelse(length(res$dif.items)>=5,res$dif.items[5],NA), + dif.detect.6=ifelse(length(res$dif.items)>=6,res$dif.items[6],NA), + dif.detect.7=ifelse(length(res$dif.items)>=7,res$dif.items[7],NA), + dif.detect.unif.1=ifelse(length(res$uniform)>=1,res$uniform[1],NA), + dif.detect.unif.2=ifelse(length(res$uniform)>=2,res$uniform[2],NA), + dif.detect.unif.3=ifelse(length(res$uniform)>=3,res$uniform[3],NA), + dif.detect.unif.4=ifelse(length(res$uniform)>=4,res$uniform[4],NA), + dif.detect.unif.5=ifelse(length(res$uniform)>=5,res$uniform[5],NA), + dif.detect.unif.6=ifelse(length(res$uniform)>=6,res$uniform[6],NA), + dif.detect.unif.7=ifelse(length(res$uniform)>=7,res$uniform[7],NA), + N=N, + nbdif=ifelse(scen<=4,0,ifelse(scen<=16,2,3)), + true.dif.1=ifelse(scen<=4,NA,unique(dat[dat$replication==k,]$dif1)), + true.dif.2=ifelse(scen<=4,NA,unique(dat[dat$replication==k,]$dif2)), + true.dif.3=ifelse(scen<=16,NA,unique(dat[dat$replication==k,]$dif3))) + df_res <- rbind(df_res,df_res2) + } + } + else if (scen%in%c(1,2,5:12)) { + res <- resali(df=dat[dat$replication==1,],items = seq(1,4),group=grp,verbose=FALSE) + df_res <- data.frame(dif.detect.1=ifelse(length(res$dif.items)>=1,res$dif.items[1],NA), + dif.detect.2=ifelse(length(res$dif.items)>=2,res$dif.items[2],NA), + dif.detect.3=ifelse(length(res$dif.items)>=3,res$dif.items[3],NA), + dif.detect.4=ifelse(length(res$dif.items)>=4,res$dif.items[4],NA), + dif.detect.unif.1=ifelse(length(res$uniform)>=1,res$uniform[1],NA), + dif.detect.unif.2=ifelse(length(res$uniform)>=2,res$uniform[2],NA), + dif.detect.unif.3=ifelse(length(res$uniform)>=3,res$uniform[3],NA), + dif.detect.unif.4=ifelse(length(res$uniform)>=4,res$uniform[4],NA), + N=N, + nbdif=ifelse(scen<=4,0,ifelse(scen<=8,1,2)), + true.dif.1=ifelse(scen<=4,NA,unique(dat[dat$replication==1,]$dif1)), + true.dif.2=ifelse(scen<=8,NA,unique(dat[dat$replication==1,]$dif2)) + ) + for (k in 2:1000) { + if (k%%100==0) { + cat(paste0('N=',k,'/1000\n')) + } + res <- resali(df=dat[dat$replication==k,],items = seq(1,4),group=grp,verbose=FALSE) + df_res2 <- data.frame(dif.detect.1=ifelse(length(res$dif.items)>=1,res$dif.items[1],NA), + dif.detect.2=ifelse(length(res$dif.items)>=2,res$dif.items[2],NA), + dif.detect.3=ifelse(length(res$dif.items)>=3,res$dif.items[3],NA), + dif.detect.4=ifelse(length(res$dif.items)>=4,res$dif.items[4],NA), + dif.detect.unif.1=ifelse(length(res$uniform)>=1,res$uniform[1],NA), + dif.detect.unif.2=ifelse(length(res$uniform)>=2,res$uniform[2],NA), + dif.detect.unif.3=ifelse(length(res$uniform)>=3,res$uniform[3],NA), + dif.detect.unif.4=ifelse(length(res$uniform)>=4,res$uniform[4],NA), + N=N, + nbdif=ifelse(scen<=4,0,ifelse(scen<=8,1,2)), + true.dif.1=ifelse(scen<=4,NA,unique(dat[dat$replication==k,]$dif1)), + true.dif.2=ifelse(scen<=8,NA,unique(dat[dat$replication==k,]$dif2))) + df_res <- rbind(df_res,df_res2) + } + } + return(df_res) +} + + + + + + +results <- c(sapply(1:4,function(x) paste0(x,c('A','B','C','D','E'))),sapply(5:9,function(x) paste0(x,c('A','B','C','D','E','F','G')))) + +results2 <- c(sapply(10:20,function(x) paste0(x,c('A','B','C','D','E','F','G')))) + +results <- c(sapply(c(50,100,200,300),function(x) paste0(results,'_',x))) + +results2 <- c(sapply(c(50,100,200,300),function(x) paste0(results2,'_',x))) + +results <- sort(results) + +results2 <- sort(results2) + +results <- c(results,results2) + +for (r in results) { + cat(paste0(r,"\n")) + cat(paste0("-------------------------------------------","\n")) + write.csv(generate_resali(r,"TT"),paste0("/home/corentin/Documents/These/Recherche/Simulations/Analysis/RESALI/Detection/",r,".csv")) + cat(paste0("-------------------------------------------","\n")) +} + + + + diff --git a/Scripts/Analysis/DIF-RESIDUS/pcm_dif_residus.do b/Scripts/Analysis/DIF-RESIDUS/pcm_dif_residus.do index 707054f..df499fe 100644 --- a/Scripts/Analysis/DIF-RESIDUS/pcm_dif_residus.do +++ b/Scripts/Analysis/DIF-RESIDUS/pcm_dif_residus.do @@ -12,7 +12,7 @@ adopath+"/home/corentin/Documents/These/Recherche/ROSALI-SIM/Modules/rosali_cust local N = "50 100 200 300" - local ss = "1 2 3 4 5 6 7 8 9 10 11 1 13 14 15 16 17 18 19 20" + local ss = "1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20" foreach s in `ss' { foreach Nnn in `N' { local Nn = `Nnn'