o Yi|^@sddlZddlmZddlmZddlmZmZm Z m Z m Z m Z m Z mZmZmZmZmZmZmZmZmZmZmZd,ddZdd Zd d Zd d ZddZddZddZddZ ddZ!ddZ"ddZ#ddZ$ddZ%d d!Z&d"d#Z'd$d%Z(d&d'Z)d(d)Z*d*d+Z+dS)-N)assert_allclosefmin) CensoredDatabetacauchychi2expongammagumbel_lgumbel_rinvgauss invweibulllaplacelogisticlognormnctncx2norm weibull_max weibull_mincCst||||dddS)Ng-q=)argsdispxtolftolr)funcx0rrrrU/tmp/pip-target-1s0edx8b/lib/python/scipy/stats/tests/test_continuous_fit_censored.py optimizersrcCsttddgddgddgddggd }tj|d d td \}}}}t|d ddt|ddd|d ks2J|d ks8JdS)a Test fitting beta shape parameters to interval-censored data. Calculation in R: > library(fitdistrplus) > data <- data.frame(left=c(0.10, 0.50, 0.75, 0.80), + right=c(0.20, 0.55, 0.90, 0.95)) > result = fitdistcens(data, 'beta', control=list(reltol=1e-14)) > result Fitting of the distribution ' beta ' on censored data by maximum likelihood Parameters: estimate shape1 1.419941 shape2 1.027066 > result$sd shape1 shape2 0.9914177 0.6866565 皙?皙??皙???皙?ffffff?intervalrflocfscalerg ?h㈵>rtolg*V n?N)rrfitrr)dataablocscalerrr test_betas r7cCsDtddgdgd}tj|td\}}t|dddt|d ddd S) a Test fitting the Cauchy distribution to right-censored data. Calculation in R, with two values not censored [1, 10] and one right-censored value [30]. > library(fitdistrplus) > data <- data.frame(left=c(1, 10, 30), right=c(1, 10, NA)) > result = fitdistcens(data, 'cauchy', control=list(reltol=1e-14)) > result Fitting of the distribution ' cauchy ' on censored data by maximum likelihood Parameters: estimate location 7.100001 scale 7.455866 r* ) uncensoredrightrg}if@r.r/gne@Nrrr1rrr2r5r6rrrtest_cauchy_right_censored5sr?cCsPtddgdgdgddggd}tj|td\}}t|dd d t|d d d d S) a\ Test fitting the Cauchy distribution to data with mixed censoring. Calculation in R, with: * two values not censored [1, 10], * one left-censored [1], * one right-censored [30], and * one interval-censored [[4, 8]]. > library(fitdistrplus) > data <- data.frame(left=c(NA, 1, 4, 10, 30), right=c(1, 1, 8, 10, NA)) > result = fitdistcens(data, 'cauchy', control=list(reltol=1e-14)) > result Fitting of the distribution ' cauchy ' on censored data by maximum likelihood Parameters: estimate location 4.605150 scale 5.900852 r*r8r9r:leftr;r)r<g qk@r.r/g%Zx@Nr=r>rrrtest_cauchy_mixedMs rDcCs`tddgdgdgddggd}tj|ddtd\}}}t|d d d |dks(J|dks.Jd S) a= Test fitting just the shape parameter (df) of chi2 to mixed data. Calculation in R, with: * two values not censored [1, 10], * one left-censored [1], * one right-censored [30], and * one interval-censored [[4, 8]]. > library(fitdistrplus) > data <- data.frame(left=c(NA, 1, 4, 10, 30), right=c(1, 1, 8, 10, NA)) > result = fitdistcens(data, 'chisq', control=list(reltol=1e-14)) > result Fitting of the distribution ' chisq ' on censored data by maximum likelihood Parameters: estimate df 5.060329 r*r8r9r@rArBrr+ge=@r.r/N)rrr1rr)r2dfr5r6rrrtest_chi2_mixedis rFcCsgd}dgddgd}t||}tj|dtd\}}|dks$Jt||}|j|j }||}t ||dd S) a For the exponential distribution with loc=0, the exact solution for fitting n uncensored points x[0]...x[n-1] and m right-censored points x[n]..x[n+m-1] is scale = sum(x)/n That is, divide the sum of all the values (not censored and right-censored) by the number of uncensored values. (See, for example, https://en.wikipedia.org/wiki/Censoring_(statistics)#Likelihood.) The second derivative of the log-likelihood function is n/scale**2 - 2*sum(x)/scale**3 from which the estimate of the standard error can be computed. ----- Calculation in R, for reference only. The R results are not used in the test. > library(fitdistrplus) > dexps <- function(x, scale) { + return(dexp(x, 1/scale)) + } > pexps <- function(q, scale) { + return(pexp(q, 1/scale)) + } > left <- c(1, 2.5, 3, 6, 7.5, 10, 12, 12, 14.5, 15, + 16, 16, 20, 20, 21, 22) > right <- c(1, 2.5, 3, 6, 7.5, 10, 12, 12, 14.5, 15, + NA, NA, NA, NA, NA, NA) > result = fitdistcens(data, 'exps', start=list(scale=mean(data$left)), + control=list(reltol=1e-14)) > result Fitting of the distribution ' exps ' on censored data by maximum likelihood Parameters: estimate scale 19.85 > result$sd scale 6.277119 )r*@g@r8 rJg-@rLrMFr8TrIrr,rg:0yE>N) rright_censoredr r1rlen num_censored _uncensoredsum_rightr)obscensr2r5r6ntotalexpectedrrrtest_expon_right_censoreds.  r\cCs^tgddgddg}tj|dtd\}}}t|ddd|dks&Jt|d ddd S) a Fit gamma shape and scale to data with one right-censored value. Calculation in R: > library(fitdistrplus) > data <- data.frame(left=c(2.5, 2.9, 3.8, 9.1, 9.3, 12.0, 23.0, 25.0), + right=c(2.5, 2.9, 3.8, 9.1, 9.3, 12.0, 23.0, NA)) > result = fitdistcens(data, 'gamma', start=list(shape=1, scale=10), + control=list(reltol=1e-13)) > result Fitting of the distribution ' gamma ' on censored data by maximum likelihood Parameters: estimate shape 1.447623 scale 8.360197 > result$sd shape scale 0.7053086 5.1016531 )rGg333333@gffffff@g333333"@g"@g(@g7@g9@rr*rPgv)?r.r/g library(evd) > library(fitdistrplus) > data = data.frame(left=c(0, 2, 3, 9, 10, 10), + right=c(1, 2, 3, 9, NA, NA)) > result = fitdistcens(data, 'gumbel', + control=list(reltol=1e-14), + start=list(loc=4, scale=5)) > result Fitting of the distribution ' gumbel ' on censored data by maximum likelihood Parameters: estimate loc 4.487853 scale 4.843640 rH r8rr*)r;r)r<g@r.r/gc*_@N)rCr)g)nparrayrr r1rrr ) r:r;r)r2r5r6data2loc2scale2rrr test_gumbels ricCsgd}t|dgdgd}tj|ddtd\}}}t|dd d |dks&J|dks,Jtj|dtd \}}}t|d d d |dksDJt|d d d dS)a Fit just the shape parameter of invgauss to data with one value left-censored and one value right-censored. Calculation in R; using a fixed dispersion parameter amounts to fixing the scale to be 1. > library(statmod) > library(fitdistrplus) > left <- c(NA, 0.4813096, 0.5571880, 0.5132463, 0.3801414, 0.5904386, + 0.4822340, 0.3478597, 3, 0.7191797, 1.5810902, 0.4442299) > right <- c(0.15, 0.4813096, 0.5571880, 0.5132463, 0.3801414, 0.5904386, + 0.4822340, 0.3478597, NA, 0.7191797, 1.5810902, 0.4442299) > data <- data.frame(left=left, right=right) > result = fitdistcens(data, 'invgauss', control=list(reltol=1e-12), + fix.arg=list(dispersion=1), start=list(mean=3)) > result Fitting of the distribution ' invgauss ' on censored data by maximum likelihood Parameters: estimate mean 0.853469 Fixed parameters: value dispersion 1 > result$sd mean 0.247636 Here's the R calculation with the dispersion as a free parameter to be fit. > result = fitdistcens(data, 'invgauss', control=list(reltol=1e-12), + start=list(mean=3, dispersion=1)) > result Fitting of the distribution ' invgauss ' on censored data by maximum likelihood Parameters: estimate mean 0.8699819 dispersion 1.2261362 The parametrization of the inverse Gaussian distribution in the `statmod` package is not the same as in SciPy (see https://arxiv.org/abs/1603.06687 for details). The translation from R to SciPy is scale = 1/dispersion mu = mean * dispersion > 1/result$estimate['dispersion'] # 1/dispersion dispersion 0.8155701 > result$estimate['mean'] * result$estimate['dispersion'] mean 1.066716 Those last two values are the SciPy scale and shape parameters. ) g ?g ({?gL`)l?g 6 library(evd) > library(fitdistrplus) > data = data.frame(left=c(0, 2, 3, 9, 10, 10), + right=c(1, 2, 3, 9, NA, NA)) > result = fitdistcens(data, 'frechet', + control=list(reltol=1e-14), + start=list(loc=4, scale=5)) > result Fitting of the distribution ' frechet ' on censored data by maximum likelihood Parameters: estimate scale 2.7902200 shape 0.6379845 Fixed parameters: value loc 0 r`r8rr*r:r;r)rPgp[[x^j?r.r/g5)^R@N)rrr1rr)r2cr5r6rrrtest_invweibull^s rocCs~tgd}||dk|dk@}||dk}||dk}t|||d}tj|ddtd\}}t|dd d t|d d d d S) a Fir the Laplace distribution to left- and right-censored data. Calculation in R: > library(fitdistrplus) > dlaplace <- function(x, location=0, scale=1) { + return(0.5*exp(-abs((x - location)/scale))/scale) + } > plaplace <- function(q, location=0, scale=1) { + z <- (q - location)/scale + s <- sign(z) + f <- -s*0.5*exp(-abs(z)) + (s+1)/2 + return(f) + } > left <- c(NA, -41.564, 50.0, 15.7384, 50.0, 10.0452, -2.0684, + -19.5399, 50.0, 9.0005, 27.1227, 4.3113, -3.7372, + 25.3111, 14.7987, 34.0887, 50.0, 42.8496, 18.5862, + 32.8921, 9.0448, -27.4591, NA, 19.5083, -9.7199) > right <- c(-50.0, -41.564, NA, 15.7384, NA, 10.0452, -2.0684, + -19.5399, NA, 9.0005, 27.1227, 4.3113, -3.7372, + 25.3111, 14.7987, 34.0887, NA, 42.8496, 18.5862, + 32.8921, 9.0448, -27.4591, -50.0, 19.5083, -9.7199) > data <- data.frame(left=left, right=right) > result <- fitdistcens(data, 'laplace', start=list(location=10, scale=10), + control=list(reltol=1e-13)) > result Fitting of the distribution ' laplace ' on censored data by maximum likelihood Parameters: estimate location 14.79870 scale 30.93601 > result$sd location scale 0.1758864 7.0972125 )Igx&1DI@gz/@rqgSt$$@g_LgC63rqgK7A"@g8gDi;@g\m>@g g?O9@b4-@gޓZ A@rqg?W[lE@gK42@g|a2U0r@@g"@gݓu;rpg3@gǘp#rp2rqrjr8r5r6rrrr.r/gY>@N)rdrerrr1rr)rWr^rCr;r2r5r6rrr test_laplaces'  rucCsRtgd}tj||dkd}tj|td\}}t|dddt|dd dd S) a Fit the logistic distribution to left-censored data. Calculation in R: > library(fitdistrplus) > left = c(13.5401, 37.4235, 11.906 , 13.998 , NA , 0.4023, NA , + 10.9044, 21.0629, 9.6985, NA , 12.9016, 39.164 , 34.6396, + NA , 20.3665, 16.5889, 18.0952, 45.3818, 35.3306, 8.4949, + 3.4041, NA , 7.2828, 37.1265, 6.5969, 17.6868, 17.4977, + 16.3391, 36.0541) > right = c(13.5401, 37.4235, 11.906 , 13.998 , 0. , 0.4023, 0. , + 10.9044, 21.0629, 9.6985, 0. , 12.9016, 39.164 , 34.6396, + 0. , 20.3665, 16.5889, 18.0952, 45.3818, 35.3306, 8.4949, + 3.4041, 0. , 7.2828, 37.1265, 6.5969, 17.6868, 17.4977, + 16.3391, 36.0541) > data = data.frame(left=left, right=right) > result = fitdistcens(data, 'logis', control=list(reltol=1e-14)) > result Fitting of the distribution ' logis ' on censored data by maximum likelihood Parameters: estimate location 14.633459 scale 9.232736 > result$sd location scale 2.931505 1.546879 )g#+@g|?5B@gZd;'@g"+@g:H?rvg;M %@g65@gʡe#@rvg%䃞)@gEC@gBiQA@rvg]4@gI&–0@gF_2@gpΈްF@g_QA@gec @gAǘ; @rvg6r/g&4I,)w"@r.N)rdrer left_censoredrr1rr)r^r2r5r6rrr test_logistics rycCs|gd}t|dgddgt|dgd}tj|dd\}}}|dks)Jt|}t|ddd t|d d d d S) a# Ref: https://math.montana.edu/jobo/st528/documents/relc.pdf The data is the locomotive control time to failure example that starts on page 8. That's the 8th page in the PDF; the page number shown in the text is 270). The document includes SAS output for the data. )%g6@gB@gG@g@H@gI@gJ@g@K@gL@gP@gQ@g`Q@g S@g@S@gS@gT@g`T@gT@gT@gU@gV@g`W@gY@gZ@g [@g \@g`\@g]@g@]@g]@g]@g^@g^@g^@g_@g``@g`@g`@;rr*r,g3w@Mb@?r/g~jt?g{Gzt?N)rrQrRrr1rdlogr) miles_to_failr2sigmar5r6rkrrr test_lognorms   rcCstgdgd}tjddtj|ddtd\}}}}Wdn1s(wYt|d d d t|d d d |dksAJ|dksGJdS) a  Test fitting the noncentral t distribution to censored data. Calculation in R: > library(fitdistrplus) > data <- data.frame(left=c(1, 2, 3, 5, 8, 10, 25, 25), + right=c(1, 2, 3, 5, 8, 10, NA, NA)) > result = fitdistcens(data, 't', control=list(reltol=1e-14), + start=list(df=1, ncp=2)) > result Fitting of the distribution ' t ' on censored data by maximum likelihood Parameters: estimate df 0.5432336 ncp 2.8893565 )r*rarHrAr8r)rrrrrrr*r*ignoreoverrr*r+NgBn+b?r.r/gTf@)rrQrderrstaterr1rr)r2rEncr5r6rrrtest_ncts   rcCstgdddgddggd}tjddtj|dd td \}}}}Wd n1s,wYt|d d dt|dd d|dksEJ|d ksKJd S)a Test fitting the shape parameters (df, ncp) of ncx2 to mixed data. Calculation in R, with * 5 not censored values [2.7, 0.2, 6.5, 0.4, 0.1], * 1 interval-censored value [[0.6, 1.0]], and * 2 right-censored values [8, 8]. > library(fitdistrplus) > data <- data.frame(left=c(2.7, 0.2, 6.5, 0.4, 0.1, 0.6, 8, 8), + right=c(2.7, 0.2, 6.5, 0.4, 0.1, 1.0, NA, NA)) > result = fitdistcens(data, 'chisq', control=list(reltol=1e-14), + start=list(df=1, ncp=2)) > result Fitting of the distribution ' chisq ' on censored data by maximum likelihood Parameters: estimate df 1.052871 ncp 2.362934 )g@r!g@g?r rAg333333?g?rmrrrr*r+NgQB?r.r/g I@)rrdrrr1rr)r2rEncpr5r6rrr test_ncx2s  rcCsTtddgddgddgddggd }tj|td \}}t|d d d t|dd d dS)a Test fitting the normal distribution to interval-censored data. Calculation in R: > library(fitdistrplus) > data <- data.frame(left=c(0.10, 0.50, 0.75, 0.80), + right=c(0.20, 0.55, 0.90, 0.95)) > result = fitdistcens(data, 'norm', control=list(reltol=1e-14)) > result Fitting of the distribution ' norm ' on censored data by maximum likelihood Parameters: estimate mean 0.5919990 sd 0.2868042 > result$sd mean sd 0.1444432 0.1029451 r r!r"r#r$r%r&r'r(r<gux?r.r/g[?N)rrr1rrr>rrr test_norm>src Csd}tdddd|dDD\}}t||}tj|dd\}}}t|dd d |dks2Jt|d d d tt | |}t j|dd\}} } t|dd d | dksZJt| d d d dS) Nz>3,5,6*,8,10*,11*,15,20*,22,23,27*,29,32,35,40,26,28,33*,21,24*cSs$g|]}t|dt|dkfqS)rra)floatrR).0trrr dsz*test_weibull_censored1..cSsg|]}|dqS)*)splitrwrrrres,rr|gx&1@gMbP?r/g= ףp<@) ziprrrQrr1rrxrdrer) stimesrXr2rnr5r6rfc2rgrhrrrtest_weibull_censored1^s   rcCsd}tdd|Dddj\}}|d}t||}tj|dt d\}}}t |d d d t |d d d |dks>JdS)Na 450 0 460 1 1150 0 1150 0 1560 1 1600 0 1660 1 1850 1 1850 1 1850 1 1850 1 1850 1 2030 1 2030 1 2030 1 2070 0 2070 0 2080 0 2200 1 3000 1 3000 1 3000 1 3000 1 3100 0 3200 1 3450 0 3750 1 3750 1 4150 1 4150 1 4150 1 4150 1 4300 1 4300 1 4300 1 4300 1 4600 0 4850 1 4850 1 4850 1 4850 1 5000 1 5000 1 5000 1 6100 1 6100 0 6100 1 6100 1 6300 1 6450 1 6450 1 6700 1 7450 1 7800 1 7800 1 8100 1 8100 1 8200 1 8500 1 8500 1 8500 1 8750 1 8750 0 8750 1 9400 1 9900 1 10100 1 10100 1 10100 1 11500 1 cSsg|]}t|qSr)intrrrrrsz)test_weibull_min_sas1..rcrag@@rrPga4?g-C6?r/gsK:@gh㈵>) rdrerreshapeTrrQrr1rr)textliferXr2rnr5r6rrrtest_weibull_min_sas1{s& rcCsztgd}t|dgt|ddgd}tj|dddtd\}}}t|ddd t|d dd t|d dd dS) N)ri i0rrrar*drtgTR'@r}r/gR^@gHz[@) rdrerrQrRrr1rr)daysr2rnr5r6rrrtest_weibull_min_sas2s$  r)rr),numpyrd numpy.testingrscipy.optimizer scipy.statsrrrrr r r r r rrrrrrrrrrr7r?rDrFr\r_rirlroruryrrrrrrrrrrrs.  P #>"(Q$4)!  !