@article {2733, title = {Stratified Item Selection Methods in Cognitive Diagnosis Computerized Adaptive Testing}, journal = {Applied Psychological Measurement}, volume = {44}, number = {5}, year = {2020}, pages = {346-361}, abstract = {Cognitive diagnostic computerized adaptive testing (CD-CAT) aims to obtain more useful diagnostic information by taking advantages of computerized adaptive testing (CAT). Cognitive diagnosis models (CDMs) have been developed to classify examinees into the correct proficiency classes so as to get more efficient remediation, whereas CAT tailors optimal items to the examinee{\textquoteright}s mastery profile. The item selection method is the key factor of the CD-CAT procedure. In recent years, a large number of parametric/nonparametric item selection methods have been proposed. In this article, the authors proposed a series of stratified item selection methods in CD-CAT, which are combined with posterior-weighted Kullback{\textendash}Leibler (PWKL), nonparametric item selection (NPS), and weighted nonparametric item selection (WNPS) methods, and named S-PWKL, S-NPS, and S-WNPS, respectively. Two different types of stratification indices were used: original versus novel. The performances of the proposed item selection methods were evaluated via simulation studies and compared with the PWKL, NPS, and WNPS methods without stratification. Manipulated conditions included calibration sample size, item quality, number of attributes, number of strata, and data generation models. Results indicated that the S-WNPS and S-NPS methods performed similarly, and both outperformed the S-PWKL method. And item selection methods with novel stratification indices performed slightly better than the ones with original stratification indices, and those without stratification performed the worst.}, doi = {10.1177/0146621619893783}, url = {https://doi.org/10.1177/0146621619893783}, author = {Jing Yang and Hua-Hua Chang and Jian Tao and Ningzhong Shi} } @article {2610, title = {Some recommendations for developing multidimensional computerized adaptive tests for patient-reported outcomes}, journal = {Quality of Life Research}, volume = {27}, number = {4}, year = {2018}, month = {Apr}, pages = {1055{\textendash}1063}, abstract = {Multidimensional item response theory and computerized adaptive testing (CAT) are increasingly used in mental health, quality of life (QoL), and patient-reported outcome measurement. Although multidimensional assessment techniques hold promises, they are more challenging in their application than unidimensional ones. The authors comment on minimal standards when developing multidimensional CATs.}, issn = {1573-2649}, doi = {10.1007/s11136-018-1821-8}, url = {https://doi.org/10.1007/s11136-018-1821-8}, author = {Smits, Niels and Paap, Muirne C. S. and B{\"o}hnke, Jan R.} } @conference {2630, title = {Scripted On-the-fly Multistage Testing}, booktitle = {IACAT 2017 Conference}, year = {2017}, month = {08/2017}, publisher = {Niigata Seiryo University}, organization = {Niigata Seiryo University}, address = {Niigata, Japan}, abstract = {

On-the-fly multistage testing (OMST) was introduced recently as a promising alternative to preassembled MST. A decidedly appealing feature of both is the reviewability of items within the current stage. However, the fundamental difference is that, instead of routing to a preassembled module, OMST adaptively assembles a module at each stage according to an interim ability estimate. This produces more individualized forms with finer measurement precision, but imposing nonstatistical constraints and controlling item exposure become more cumbersome. One recommendation is to use the maximum priority index followed by a remediation step to satisfy content constraints, and the Sympson-Hetter method with a stratified item bank for exposure control.

However, these methods can be computationally expensive, thereby impeding practical implementation. Therefore, this study investigated the script method as a simpler solution to the challenge of strict content balancing and effective item exposure control in OMST. The script method was originally devised as an item selection algorithm for CAT and generally proceeds as follows: For a test with m items, there are m slots to be filled, and an item is selected according to pre-defined rules for each slot. For the first slot, randomly select an item from a designated content area (collection). For each subsequent slot, 1) Discard any enemies of items already administered in previous slots; 2) Draw a designated number of candidate items (selection length) from the designated collection according to the current ability estimate; 3) Randomly select one item from the set of candidates. There are two distinct features of the script method. First, a predetermined sequence of collections guarantees meeting content specifications. The specific ordering may be determined either randomly or deliberately by content experts. Second, steps 2 and 3 depict a method of exposure control, in which selection length balances item usage at the possible expense of ability estimation accuracy. The adaptation of the script method to OMST is straightforward. For the first module, randomly select each item from a designated collection. For each subsequent module, the process is the same as in scripted CAT (SCAT) except the same ability estimate is used for the selection of all items within the module. A series of simulations was conducted to evaluate the performance of scripted OMST (SOMST, with 3 or 4 evenly divided stages) relative to SCAT under various item exposure restrictions. In all conditions, reliability was maximized by programming an optimization algorithm that searches for the smallest possible selection length for each slot within the constraints. Preliminary results indicated that SOMST is certainly a capable design with performance comparable to that of SCAT. The encouraging findings and ease of implementation highly motivate the prospect of operational use for large-scale assessments.

Presentation Video

}, keywords = {CAT, multistage testing, On-the-fly testing}, url = {https://drive.google.com/open?id=1wKuAstITLXo6BM4APf2mPsth1BymNl-y}, author = {Edison Choe and Bruce Williams and Sung-Hyuck Lee} } @conference {2639, title = {A Simulation Study to Compare Classification Method in Cognitive Diagnosis Computerized Adaptive Testing}, booktitle = {IACAT 2017 Conference}, year = {2017}, month = {08/2017}, publisher = {Niigata Seiryo University}, organization = {Niigata Seiryo University}, address = {Niigata, Japan}, abstract = {

Cognitive Diagnostic Computerized Adaptive Testing (CD-CAT) combines the strengths of both CAT and cognitive diagnosis. Cognitive diagnosis models that can be viewed as restricted latent class models have been developed to classify the examinees into the correct profile of skills that have been mastered and those that have not so as to get more efficient remediation. Chiu \& Douglas (2013) introduces a nonparametric procedure that only requires specification of Q-matrix to classify by proximity to ideal response pattern. In this article, we compare nonparametric procedure with common profile estimation method like maximum a posterior (MAP) in CD-CAT. Simulation studies consider a variety of Q-matrix structure, the number of attributes, ways to generate attribute profiles,\ and item quality. Results indicate that nonparametric procedure consistently gets the higher pattern and attribute recovery rate in nearly all conditions.

References

Chiu, C.-Y., \& Douglas, J. (2013). A nonparametric approach to cognitive diagnosis by proximity to ideal response patterns. Journal of Classification, 30, 225-250. doi: 10.1007/s00357-013-9132-9

Session Video

}, url = {https://drive.google.com/open?id=1jCL3fPZLgzIdwvEk20D-FliZ15OTUtpr}, author = {Jing Yang and Jian Tao and Hua-Hua Chang and Ning-Zhong Shi} } @article {2459, title = {Stochastic Curtailment of Questionnaires for Three-Level Classification: Shortening the CES-D for Assessing Low, Moderate, and High Risk of Depression}, journal = {Applied Psychological Measurement}, volume = {40}, number = {1}, year = {2016}, pages = {22-36}, abstract = {In clinical assessment, efficient screeners are needed to ensure low respondent burden. In this article, Stochastic Curtailment (SC), a method for efficient computerized testing for classification into two classes for observable outcomes, was extended to three classes. In a post hoc simulation study using the item scores on the Center for Epidemiologic Studies{\textendash}Depression Scale (CES-D) of a large sample, three versions of SC, SC via Empirical Proportions (SC-EP), SC via Simple Ordinal Regression (SC-SOR), and SC via Multiple Ordinal Regression (SC-MOR) were compared at both respondent burden and classification accuracy. All methods were applied under the regular item order of the CES-D and under an ordering that was optimal in terms of the predictive power of the items. Under the regular item ordering, the three methods were equally accurate, but SC-SOR and SC-MOR needed less items. Under the optimal ordering, additional gains in efficiency were found, but SC-MOR suffered from capitalization on chance substantially. It was concluded that SC-SOR is an efficient and accurate method for clinical screening. Strengths and weaknesses of the methods are discussed.}, doi = {10.1177/0146621615592294}, url = {http://apm.sagepub.com/content/40/1/22.abstract}, author = {Smits, Niels and Finkelman, Matthew D. and Kelderman, Henk} } @article {2394, title = {Stochastic Curtailment in Adaptive Mastery Testing: Improving the Efficiency of Confidence Interval{\textendash}Based Stopping Rules}, journal = {Applied Psychological Measurement}, volume = {39}, number = {4}, year = {2015}, pages = {278-292}, abstract = {A well-known stopping rule in adaptive mastery testing is to terminate the assessment once the examinee{\textquoteright}s ability confidence interval lies entirely above or below the cut-off score. This article proposes new procedures that seek to improve such a variable-length stopping rule by coupling it with curtailment and stochastic curtailment. Under the new procedures, test termination can occur earlier if the probability is high enough that the current classification decision remains the same should the test continue. Computation of this probability utilizes normality of an asymptotically equivalent version of the maximum likelihood ability estimate. In two simulation sets, the new procedures showed a substantial reduction in average test length while maintaining similar classification accuracy to the original method.}, doi = {10.1177/0146621614561314}, url = {http://apm.sagepub.com/content/39/4/278.abstract}, author = {Sie, Haskell and Finkelman, Matthew D. and Bartroff, Jay and Thompson, Nathan A.} } @article {2352, title = {The Sequential Probability Ratio Test and Binary Item Response Models}, journal = {Journal of Educational and Behavioral Statistics}, volume = {39}, number = {3}, year = {2014}, pages = {203-230}, abstract = {

The sequential probability ratio test (SPRT) is a common method for terminating item response theory (IRT)-based adaptive classification tests. To decide whether a classification test should stop, the SPRT compares a simple log-likelihood ratio, based on the classification bound separating two categories, to prespecified critical values. As has been previously noted (Spray \& Reckase, 1994), the SPRT test statistic is not necessarily monotonic with respect to the classification bound when item response functions have nonzero lower asymptotes. Because of nonmonotonicity, several researchers (including Spray \& Reckase, 1994) have recommended selecting items at the classification bound rather than the current ability estimate when terminating SPRT-based classification tests. Unfortunately, this well-worn advice is a bit too simplistic. Items yielding optimal evidence for classification depend on the IRT model, item parameters, and location of an examinee with respect to the classification bound. The current study illustrates, in depth, the relationship between the SPRT test statistic and classification evidence in binary IRT models. Unlike earlier studies, we examine the form of the SPRT-based log-likelihood ratio while altering the classification bound and item difficulty. These investigations motivate a novel item selection algorithm based on optimizing the expected SPRT criterion given the current ability estimate. The new expected log-likelihood ratio algorithm results in test lengths noticeably shorter than current, commonly used algorithms, and with no loss in classification accuracy.

}, doi = {10.3102/1076998614524824}, url = {http://jeb.sagepub.com/cgi/content/abstract/39/3/203}, author = {Nydick, Steven W.} } @article {2321, title = {A Sequential Procedure for Detecting Compromised Items in the Item Pool of a CAT System}, journal = {Applied Psychological Measurement}, volume = {38}, number = {2}, year = {2014}, pages = {87-104}, abstract = {

To maintain the validity of a continuous testing system, such as computerized adaptive testing (CAT), items should be monitored to ensure that the performance of test items has not gone through any significant changes during their lifetime in an item pool. In this article, the author developed a sequentially monitoring procedure based on a series of statistical hypothesis tests to examine whether the statistical characteristics of individual items have changed significantly during test administration. Simulation studies show that under the simulated setting, by choosing an appropriate cutoff point, the procedure can control the rate of Type I errors at any reasonable significance level and meanwhile, has a very low rate of Type II errors.

}, doi = {10.1177/0146621613510062}, url = {http://apm.sagepub.com/content/38/2/87.abstract}, author = {Zhang, Jinming} } @article {2341, title = {Stratified Item Selection and Exposure Control in Unidimensional Adaptive Testing in the Presence of Two-Dimensional Data}, journal = {Applied Psychological Measurement}, volume = {38}, number = {7}, year = {2014}, pages = {563-576}, abstract = {

It is not uncommon to use unidimensional item response theory models to estimate ability in multidimensional data with computerized adaptive testing (CAT). The current Monte Carlo study investigated the penalty of this model misspecification in CAT implementations using different item selection methods and exposure control strategies. Three item selection methods\—maximum information (MAXI), a-stratification (STRA), and a-stratification with b-blocking (STRB) with and without Sympson\–Hetter (SH) exposure control strategy\—were investigated. Calibrating multidimensional items as unidimensional items resulted in inaccurate item parameter estimates. Therefore, MAXI performed better than STRA and STRB in estimating the ability parameters. However, all three methods had relatively large standard errors. SH exposure control had no impact on the number of overexposed items. Existing unidimensional CAT implementations might consider using MAXI only if recalibration as multidimensional model is too expensive. Otherwise, building a CAT pool by calibrating multidimensional data as unidimensional is not recommended.

}, doi = {10.1177/0146621614536768}, url = {http://apm.sagepub.com/content/38/7/563.abstract}, author = {Kalinowski, Kevin E. and Natesan, Prathiba and Henson, Robin K.} } @article {2312, title = {A Semiparametric Model for Jointly Analyzing Response Times and Accuracy in Computerized Testing}, journal = {Journal of Educational and Behavioral Statistics}, volume = {38}, number = {4}, year = {2013}, pages = {381-417}, abstract = {

The item response times (RTs) collected from computerized testing represent an underutilized type of information about items and examinees. In addition to knowing the examinees\’ responses to each item, we can investigate the amount of time examinees spend on each item. Current models for RTs mainly focus on parametric models, which have the advantage of conciseness, but may suffer from reduced flexibility to fit real data. We propose a semiparametric approach, specifically, the Cox proportional hazards model with a latent speed covariate to model the RTs, embedded within the hierarchical framework proposed by van der Linden to model the RTs and response accuracy simultaneously. This semiparametric approach combines the flexibility of nonparametric modeling and the brevity and interpretability of the parametric modeling. A Markov chain Monte Carlo method for parameter estimation is given and may be used with sparse data obtained by computerized adaptive testing. Both simulation studies and real data analysis are carried out to demonstrate the applicability of the new model.

}, doi = {10.3102/1076998612461831}, url = {http://jeb.sagepub.com/cgi/content/abstract/38/4/381}, author = {Wang, Chun and Fan, Zhewen and Chang, Hua-Hua and Douglas, Jeffrey A.} } @article {2313, title = {Speededness and Adaptive Testing}, journal = {Journal of Educational and Behavioral Statistics}, volume = {38}, number = {4}, year = {2013}, pages = {418-438}, abstract = {

Two simple constraints on the item parameters in a response\–time model are proposed to control the speededness of an adaptive test. As the constraints are additive, they can easily be included in the constraint set for a shadow-test approach (STA) to adaptive testing. Alternatively, a simple heuristic is presented to control speededness in plain adaptive testing without any constraints. Both types of control are easy to implement and do not require any other real-time parameter estimation during the test than the regular update of the test taker\’s ability estimate. Evaluation of the two approaches using simulated adaptive testing showed that the STA was especially effective. It guaranteed testing times that differed less than 10 seconds from a reference test across a variety of conditions.

}, doi = {10.3102/1076998612466143}, url = {http://jeb.sagepub.com/cgi/content/abstract/38/4/418}, author = {van der Linden, Wim J. and Xiong, Xinhui} } @article {2167, title = {A Stochastic Method for Balancing Item Exposure Rates in Computerized Classification Tests}, journal = {Applied Psychological Measurement}, volume = {36}, number = {3}, year = {2012}, pages = {181-188}, abstract = {

Computerized classification tests (CCTs) classify examinees into categories such as pass/fail, master/nonmaster, and so on. This article proposes the use of stochastic methods from sequential analysis to address item overexposure, a practical concern in operational CCTs. Item overexposure is traditionally dealt with in CCTs by the Sympson-Hetter (SH) method, but this method is unable to restrict the exposure of the most informative items to the desired level. The authors\’ new method of stochastic item exposure balance (SIEB) works in conjunction with the SH method and is shown to greatly reduce the number of overexposed items in a pool and improve overall exposure balance while maintaining classification accuracy comparable with using the SH method alone. The method is demonstrated using a simulation study.

}, doi = {10.1177/0146621612439932}, url = {http://apm.sagepub.com/content/36/3/181.abstract}, author = {Huebner, Alan and Li, Zhushan} } @article {2110, title = {A Stochastic Method for Balancing Item Exposure Rates in Computerized Classification Tests}, journal = {Applied Psychological Measurement}, volume = {36}, number = {3}, year = {2012}, pages = {181-188}, abstract = {

Computerized classification tests (CCTs) classify examinees into categories such as pass/fail, master/nonmaster, and so on. This article proposes the use of stochastic methods from sequential analysis to address item overexposure, a practical concern in operational CCTs. Item overexposure is traditionally dealt with in CCTs by the Sympson-Hetter (SH) method, but this method is unable to restrict the exposure of the most informative items to the desired level. The authors\’ new method of stochastic item exposure balance (SIEB) works in conjunction with the SH method and is shown to greatly reduce the number of overexposed items in a pool and improve overall exposure balance while maintaining classification accuracy comparable with using the SH method alone. The method is demonstrated using a simulation study.

}, doi = {10.1177/0146621612439932}, url = {http://apm.sagepub.com/content/36/3/181.abstract}, author = {Huebner, Alan and Li, Zhushan} } @conference {2076, title = {Small-Sample Shadow Testing}, booktitle = {Annual Conference of the International Association for Computerized Adaptive Testing}, year = {2011}, keywords = {CAT, shadow test}, author = {Wallace Judd} } @inbook {2053, title = {Sequencing an Adaptive Test Battery}, booktitle = {Elements of Adaptive Testing}, year = {2010}, chapter = {5}, doi = {10.1007/978-0-387-85461-8}, author = {van der Linden, W. J.} } @article {2123, title = {SimulCAT: Windows application that simulates computerized adaptive test administration}, year = {2010}, url = {http://www.hantest.net/simulcat}, author = {Han, K. T.} } @article {2203, title = {Stratified and Maximum Information Item Selection Procedures in Computer Adaptive Testing}, journal = {Journal of Educational Measurement}, volume = {47}, number = {2}, year = {2010}, pages = {202{\textendash}226}, abstract = {

In this study we evaluated and compared three item selection procedures: the maximum Fisher information procedure (F), the a-stratified multistage computer adaptive testing (CAT) (STR), and a refined stratification procedure that allows more items to be selected from the high a strata and fewer items from the low a strata (USTR), along with completely random item selection (RAN). The comparisons were with respect to error variances, reliability of ability estimates and item usage through CATs simulated under nine test conditions of various practical constraints and item selection space. The results showed that F had an apparent precision advantage over STR and USTR under unconstrained item selection, but with very poor item usage. USTR reduced error variances for STR under various conditions, with small compromises in item usage. Compared to F, USTR enhanced item usage while achieving comparable precision in ability estimates; it achieved a precision level similar to F with improved item usage when items were selected under exposure control and with limited item selection space. The results provide implications for choosing an appropriate item selection procedure in applied settings.

}, issn = {1745-3984}, doi = {10.1111/j.1745-3984.2010.00109.x}, url = {http://dx.doi.org/10.1111/j.1745-3984.2010.00109.x}, author = {Deng, Hui and Ansley, Timothy and Chang, Hua-Hua} } @article {2117, title = {Stratified and maximum information item selection procedures in computer adaptive testing}, journal = {Journal of Educational Measurement}, volume = {47}, year = {2010}, pages = {202-226}, author = {Deng, H. and Ansley, T. and Chang, H.-H.} } @article {2185, title = {Studying the Equivalence of Computer-Delivered and Paper-Based Administrations of the Raven Standard Progressive Matrices Test}, journal = {Educational and Psychological Measurement}, volume = {69}, number = {5}, year = {2009}, pages = {855-867}, abstract = {

This study investigates the effect of mode of administration of the Raven Standard Progressive Matrices test on distribution, accuracy, and meaning of raw scores. A random sample of high school students take counterbalanced paper-and-pencil and computer-based administrations of the test and answer a questionnaire surveying preferences for computer-delivered test administrations. Administration mode effect is studied with repeated measures multivariate analysis of variance, internal consistency reliability estimates, and confirmatory factor analysis approaches. Results show a lack of test mode effect on distribution, accuracy, and meaning of raw scores. Participants indicate their preferences for the computer-delivered administration of the test. The article discusses findings in light of previous studies of the Raven Standard Progressive Matrices test.

}, doi = {10.1177/0013164409332219}, url = {http://epm.sagepub.com/content/69/5/855.abstract}, author = {Arce-Ferrer, Alvaro J. and Mart{\'\i}nez Guzm{\'a}n, Elvira} } @article {2152, title = {Severity of Organized Item Theft in Computerized Adaptive Testing: A Simulation Study}, journal = {Applied Psychological Measurement}, volume = {32}, number = {7}, year = {2008}, pages = {543-558}, abstract = {

Criteria had been proposed for assessing the severity of possible test security violations for computerized tests with high-stakes outcomes. However, these criteria resulted from theoretical derivations that assumed uniformly randomized item selection. This study investigated potential damage caused by organized item theft in computerized adaptive testing (CAT) for two realistic item selection methods, maximum item information and a-stratified with content blocking, using the randomized method as a baseline for comparison. Damage caused by organized item theft was evaluated by the number of compromised items each examinee could encounter and the impact of the compromised items on examinees\&$\#$39; ability estimates. Severity of test security violation was assessed under self-organized and organized item theft simulation scenarios. Results indicated that though item theft could cause severe damage to CAT with either item selection method, the maximum item information method was more vulnerable to the organized item theft simulation than was the a-stratified method.

}, doi = {10.1177/0146621607311336}, url = {http://apm.sagepub.com/content/32/7/543.abstract}, author = {Qing Yi, and Jinming Zhang, and Chang, Hua-Hua} } @article {400, title = {Some new developments in adaptive testing technology}, journal = {Zeitschrift f{\"u}r Psychologie}, volume = {216}, number = {1}, year = {2008}, pages = {3-11}, abstract = {

In an ironic twist of history, modern psychological testing has returned to an adaptive format quite common when testing was not yet standardized. Important stimuli to the renewed interest in adaptive testing have been the development of item-response theory in psychometrics, which models the responses on test items using separate parameters for the items and test takers, and the use of computers in test administration, which enables us to estimate the parameter for a test taker and select the items in real time. This article reviews a selection from the latest developments in the technology of adaptive testing, such as constrained adaptive item selection, adaptive testing using rule-based item generation, multidimensional adaptive testing, adaptive use of test batteries, and the use of response times in adaptive testing.

}, keywords = {computerized adaptive testing}, author = {van der Linden, W. J.} } @article {95, title = {Strategies for controlling item exposure in computerized adaptive testing with the partial credit model}, journal = {Journal of Applied Measurement}, volume = {9}, number = {1}, year = {2008}, note = {Davis, Laurie LaughlinDodd, Barbara GUnited StatesJournal of applied measurementJ Appl Meas. 2008;9(1):1-17.}, pages = {1-17}, edition = {2008/01/09}, abstract = {Exposure control research with polytomous item pools has determined that randomization procedures can be very effective for controlling test security in computerized adaptive testing (CAT). The current study investigated the performance of four procedures for controlling item exposure in a CAT under the partial credit model. In addition to a no exposure control baseline condition, the Kingsbury-Zara, modified-within-.10-logits, Sympson-Hetter, and conditional Sympson-Hetter procedures were implemented to control exposure rates. The Kingsbury-Zara and the modified-within-.10-logits procedures were implemented with 3 and 6 item candidate conditions. The results show that the Kingsbury-Zara and modified-within-.10-logits procedures with 6 item candidates performed as well as the conditional Sympson-Hetter in terms of exposure rates, overlap rates, and pool utilization. These two procedures are strongly recommended for use with partial credit CATs due to their simplicity and strength of their results.}, keywords = {*Algorithms, *Computers, *Educational Measurement/statistics \& numerical data, Humans, Questionnaires/*standards, United States}, isbn = {1529-7713 (Print)1529-7713 (Linking)}, author = {Davis, L. L. and Dodd, B. G.} } @article {2180, title = {A Strategy for Controlling Item Exposure in Multidimensional Computerized Adaptive Testing}, journal = {Educational and Psychological Measurement}, volume = {68}, number = {2}, year = {2008}, pages = {215-232}, abstract = {

Although computerized adaptive tests have enjoyed tremendous growth, solutions for important problems remain unavailable. One problem is the control of item exposure rate. Because adaptive algorithms are designed to select optimal items, they choose items with high discriminating power. Thus, these items are selected more often than others, leading to both overexposure and underutilization of some parts of the item pool. Overused items are often compromised, creating a security problem that could threaten the validity of a test. Building on a previously proposed stratification scheme to control the exposure rate for one-dimensional tests, the authors extend their method to multidimensional tests. A strategy is proposed based on stratification in accordance with a functional of the vector of the discrimination parameter, which can be implemented with minimal computational overhead. Both theoretical and empirical validation studies are provided. Empirical results indicate significant improvement over the commonly used method of controlling exposure rate that requires only a reasonable sacrifice in efficiency.

}, doi = {10.1177/0013164407307007}, url = {http://epm.sagepub.com/content/68/2/215.abstract}, author = {Lee, Yi-Hsuan and Ip, Edward H. and Fuh, Cheng-Der} } @inbook {1924, title = {The shadow-test approach: A universal framework for implementing adaptive testing}, year = {2007}, note = {{PDF file, 461 KB}}, address = {D. J. Weiss (Ed.), Proceedings of the 2007 GMAC Conference on Computerized Adaptive Testing.}, author = {van der Linden, W. J.} } @inbook {1821, title = {Some thoughts on controlling item exposure in adaptive testing}, year = {2007}, note = {{PDF file, 357 KB}}, address = {D. J. Weiss (Ed.), Proceedings of the 2007 GMAC Conference on Computerized Adaptive Testing.}, author = {Lewis, C.} } @inbook {1926, title = {Statistical aspects of adaptive testing}, year = {2007}, address = {C. R. Rao and S. Sinharay (Eds.), Handbook of statistics (Vol. 27: Psychometrics) (pp. 801838). Amsterdam: North-Holland.}, author = {van der Linden, W. J. and Glas, C. A. W.} } @article {54, title = {A system for interactive assessment and management in palliative care}, journal = {Journal of Pain Symptom Management}, volume = {33}, number = {6}, year = {2007}, note = {Chang, Chih-HungBoni-Saenz, Alexander ADurazo-Arvizu, Ramon ADesHarnais, SusanLau, Denys TEmanuel, Linda LR21CA113191/CA/United States NCIResearch Support, N.I.H., ExtramuralReviewUnited StatesJournal of pain and symptom managementJ Pain Symptom Manage. 2007 Jun;33(6):745-55. Epub 2007 Mar 23.}, pages = {745-55}, edition = {2007/03/16}, abstract = {The availability of psychometrically sound and clinically relevant screening, diagnosis, and outcome evaluation tools is essential to high-quality palliative care assessment and management. Such data will enable us to improve patient evaluations, prognoses, and treatment selections, and to increase patient satisfaction and quality of life. To accomplish these goals, medical care needs more precise, efficient, and comprehensive tools for data acquisition, analysis, interpretation, and management. We describe a system for interactive assessment and management in palliative care (SIAM-PC), which is patient centered, model driven, database derived, evidence based, and technology assisted. The SIAM-PC is designed to reliably measure the multiple dimensions of patients{\textquoteright} needs for palliative care, and then to provide information to clinicians, patients, and the patients{\textquoteright} families to achieve optimal patient care, while improving our capacity for doing palliative care research. This system is innovative in its application of the state-of-the-science approaches, such as item response theory and computerized adaptive testing, to many of the significant clinical problems related to palliative care.}, keywords = {*Needs Assessment, Humans, Medical Informatics/*organization \& administration, Palliative Care/*organization \& administration}, isbn = {0885-3924 (Print)}, author = {Chang, C-H. and Boni-Saenz, A. A. and Durazo-Arvizu, R. A. and DesHarnais, S. and Lau, D. T. and Emanuel, L. L.} } @article {2130, title = {Sensitivity of a computer adaptive assessment for measuring functional mobility changes in children enrolled in a community fitness programme}, journal = {Clin Rehabil}, volume = {20}, number = {6}, year = {2006}, pages = {616-622}, author = {Haley, S. M. and Fragala-Pinkham, M. A. and Ni, P.} } @article {2232, title = {Sequential Computerized Mastery Tests{\^a}{\texteuro}{\textquotedblright}Three Simulation Studies}, journal = {International Journal of Testing}, volume = {6}, number = {1}, year = {2006}, pages = {41-55}, doi = {10.1207/s15327574ijt0601_3}, url = {http://www.tandfonline.com/doi/abs/10.1207/s15327574ijt0601_3}, author = {Wiberg, Marie} } @article {319, title = {SIMCAT 1.0: A SAS computer program for simulating computer adaptive testing}, journal = {Applied Psychological Measurement}, volume = {30}, number = {1}, year = {2006}, pages = {60-61}, publisher = {Sage Publications: US}, abstract = {Monte Carlo methodologies are frequently applied to study the sampling distribution of the estimated proficiency level in adaptive testing. These methods eliminate real situational constraints. However, these Monte Carlo methodologies are not currently supported by the available software programs, and when these programs are available, their flexibility is limited. SIMCAT 1.0 is aimed at the simulation of adaptive testing sessions under different adaptive expected a posteriori (EAP) proficiency-level estimation methods (Blais \& Ra{\^\i}che, 2005; Ra{\^\i}che \& Blais, 2005) based on the one-parameter Rasch logistic model. These methods are all adaptive in the a priori proficiency-level estimation, the proficiency-level estimation bias correction, the integration interval, or a combination of these factors. The use of these adaptive EAP estimation methods diminishes considerably the shrinking, and therefore biasing, effect of the estimated a priori proficiency level encountered when this a priori is fixed at a constant value independently of the computed previous value of the proficiency level. SIMCAT 1.0 also computes empirical and estimated skewness and kurtosis coefficients, such as the standard error, of the estimated proficiency-level sampling distribution. In this way, the program allows one to compare empirical and estimated properties of the estimated proficiency-level sampling distribution under different variations of the EAP estimation method: standard error and bias, like the skewness and kurtosis coefficients. (PsycINFO Database Record (c) 2007 APA, all rights reserved)}, keywords = {computer adaptive testing, computer program, estimated proficiency level, Monte Carlo methodologies, Rasch logistic model}, isbn = {0146-6216 (Print)}, author = {Ra{\^\i}che, G. and Blais, J-G.} } @article {566, title = {Simulated computerized adaptive test for patients with lumbar spine impairments was efficient and produced valid measures of function}, journal = {Journal of Clinical Epidemiology}, volume = {59}, year = {2006}, pages = {947-956}, author = {Hart, D. and Mioduski, J. and Werenke, M. and Stratford, P.} } @article {2073, title = {Simulated computerized adaptive test for patients with lumbar spine impairments was efficient and produced valid measures of function}, journal = {Journal of Clinical Epidemiology}, volume = {59}, year = {2006}, pages = {947{\textendash}956}, abstract = {Objective: To equate physical functioning (PF) items with Back Pain Functional Scale (BPFS) items, develop a computerized adaptive test (CAT) designed to assess lumbar spine functional status (LFS) in people with lumbar spine impairments, and compare discriminant validity of LFS measures (qIRT) generated using all items analyzed with a rating scale Item Response Theory model (RSM) and measures generated using the simulated CAT (qCAT). Methods: We performed a secondary analysis of retrospective intake rehabilitation data. Results: Unidimensionality and local independence of 25 BPFS and PF items were supported. Differential item functioning was negligible for levels of symptom acuity, gender, age, and surgical history. The RSM fit the data well. A lumbar spine specific CAT was developed that was 72\% more efficient than using all 25 items to estimate LFS measures. qIRT and qCAT measures did not discriminate patients by symptom acuity, age, or gender, but discriminated patients by surgical history in similar clinically logical ways. qCAT measures were as precise as qIRT measures. Conclusion: A body part specific simulated CAT developed from an LFS item bank was efficient and produced precise measures of LFS without eroding discriminant validity.}, keywords = {Back Pain Functional Scale, computerized adaptive testing, Item Response Theory, Lumbar spine, Rehabilitation, True-score equating}, doi = {10.1016/j.jclinepi.2005.10.017}, author = {Hart, D. L. and Mioduski, J. E. and Werneke, M. W. and Stratford, P. W.} } @article {2074, title = {Simulated computerized adaptive test for patients with shoulder impairments was efficient and produced valid measures of function}, journal = {Journal of Clinical Epidemiology}, volume = {59}, year = {2006}, pages = {290-298}, abstract = {

Background and Objective: To test unidimensionality and local independence of a set of shoulder functional status (SFS) items,
develop a computerized adaptive test (CAT) of the items using a rating scale item response theory model (RSM), and compare discriminant validity of measures generated using all items (qIRT) and measures generated using the simulated CAT (qCAT).
Study Design and Setting: We performed a secondary analysis of data collected prospectively during rehabilitation of 400 patients
with shoulder impairments who completed 60 SFS items.
Results: Factor analytic techniques supported that the 42 SFS items formed a unidimensional scale and were locally independent. Except for five items, which were deleted, the RSM fit the data well. The remaining 37 SFS items were used to generate the CAT. On average, 6 items on were needed to estimate precise measures of function using the SFS CAT, compared with all 37 SFS items. The qIRT and qCAT measures were highly correlated (r 5 .96) and resulted in similar classifications of patients.
Conclusion: The simulated SFS CAT was efficient and produced precise, clinically relevant measures of functional status with good
discriminating ability.\ 

}, keywords = {computerized adaptive testing, Flexilevel Scale of Shoulder Function, Item Response Theory, Rehabilitation}, author = {Hart, D. L. and Cook, K. F. and Mioduski, J. E. and Teal, C. R. and Crane, P. K.} } @article {184, title = {Simulated computerized adaptive test for patients with shoulder impairments was efficient and produced valid measures of function}, journal = {Journal of Clinical Epidemiology}, volume = {59}, number = {3}, year = {2006}, note = {0895-4356 (Print)Journal ArticleValidation Studies}, pages = {290-8}, abstract = {BACKGROUND AND OBJECTIVE: To test unidimensionality and local independence of a set of shoulder functional status (SFS) items, develop a computerized adaptive test (CAT) of the items using a rating scale item response theory model (RSM), and compare discriminant validity of measures generated using all items (theta(IRT)) and measures generated using the simulated CAT (theta(CAT)). STUDY DESIGN AND SETTING: We performed a secondary analysis of data collected prospectively during rehabilitation of 400 patients with shoulder impairments who completed 60 SFS items. RESULTS: Factor analytic techniques supported that the 42 SFS items formed a unidimensional scale and were locally independent. Except for five items, which were deleted, the RSM fit the data well. The remaining 37 SFS items were used to generate the CAT. On average, 6 items were needed to estimate precise measures of function using the SFS CAT, compared with all 37 SFS items. The theta(IRT) and theta(CAT) measures were highly correlated (r = .96) and resulted in similar classifications of patients. CONCLUSION: The simulated SFS CAT was efficient and produced precise, clinically relevant measures of functional status with good discriminating ability.}, keywords = {*Computer Simulation, *Range of Motion, Articular, Activities of Daily Living, Adult, Aged, Aged, 80 and over, Factor Analysis, Statistical, Female, Humans, Male, Middle Aged, Prospective Studies, Reproducibility of Results, Research Support, N.I.H., Extramural, Research Support, U.S. Gov{\textquoteright}t, Non-P.H.S., Shoulder Dislocation/*physiopathology/psychology/rehabilitation, Shoulder Pain/*physiopathology/psychology/rehabilitation, Shoulder/*physiopathology, Sickness Impact Profile, Treatment Outcome}, author = {Hart, D. L. and Cook, K. F. and Mioduski, J. E. and Teal, C. R. and Crane, P. K.} } @article {185, title = {Simulated computerized adaptive tests for measuring functional status were efficient with good discriminant validity in patients with hip, knee, or foot/ankle impairments}, journal = {Journal of Clinical Epidemiology}, volume = {58}, number = {6}, year = {2005}, note = {0895-4356 (Print)Journal ArticleMulticenter StudyValidation Studies}, pages = {629-38}, abstract = {BACKGROUND AND OBJECTIVE: To develop computerized adaptive tests (CATs) designed to assess lower extremity functional status (FS) in people with lower extremity impairments using items from the Lower Extremity Functional Scale and compare discriminant validity of FS measures generated using all items analyzed with a rating scale Item Response Theory model (theta(IRT)) and measures generated using the simulated CATs (theta(CAT)). METHODS: Secondary analysis of retrospective intake rehabilitation data. RESULTS: Unidimensionality of items was strong, and local independence of items was adequate. Differential item functioning (DIF) affected item calibration related to body part, that is, hip, knee, or foot/ankle, but DIF did not affect item calibration for symptom acuity, gender, age, or surgical history. Therefore, patients were separated into three body part specific groups. The rating scale model fit all three data sets well. Three body part specific CATs were developed: each was 70\% more efficient than using all LEFS items to estimate FS measures. theta(IRT) and theta(CAT) measures discriminated patients by symptom acuity, age, and surgical history in similar ways. theta(CAT) measures were as precise as theta(IRT) measures. CONCLUSION: Body part-specific simulated CATs were efficient and produced precise measures of FS with good discriminant validity.}, keywords = {*Health Status Indicators, Activities of Daily Living, Adolescent, Adult, Aged, Aged, 80 and over, Ankle Joint/physiopathology, Diagnosis, Computer-Assisted/*methods, Female, Hip Joint/physiopathology, Humans, Joint Diseases/physiopathology/*rehabilitation, Knee Joint/physiopathology, Lower Extremity/*physiopathology, Male, Middle Aged, Research Support, N.I.H., Extramural, Research Support, U.S. Gov{\textquoteright}t, P.H.S., Retrospective Studies}, author = {Hart, D. L. and Mioduski, J. E. and Stratford, P. W.} } @article {282, title = {Somministrazione di test computerizzati di tipo adattivo: Un{\textquoteright} applicazione del modello di misurazione di Rasch [Administration of computerized and adaptive tests: An application of the Rasch Model]}, journal = {Testing Psicometria Metodologia}, volume = {12}, number = {3}, year = {2005}, pages = {131-149}, abstract = {The aim of the present study is to describe the characteristics of a procedure for administering computerized and adaptive tests (Computer Adaptive Testing or CAT). Items to be asked to the individuals are interactively chosen and are selected from a "bank" in which they were previously calibrated and recorded on the basis of their difficulty level. The selection of items is performed by increasingly more accurate estimates of the examinees{\textquoteright} ability. The building of an item-bank on Psychometrics and the implementation of this procedure allow a first validation through Monte Carlo simulations. (PsycINFO Database Record (c) 2006 APA ) (journal abstract)}, keywords = {Adaptive Testing, Computer Assisted Testing, Item Response Theory computerized adaptive testing, Models, Psychometrics}, author = {Miceli, R. and Molinengo, G.} } @booklet {1358, title = {Strategies for controlling item exposure in computerized adaptive testing with the partial credit model}, year = {2005}, address = {Pearson Educational Measurement Research Report 05-01}, author = {Davis, L. L. and Dodd, B.} } @article {167, title = {Score comparability of short forms and computerized adaptive testing: Simulation study with the activity measure for post-acute care}, journal = {Archives of Physical Medicine and Rehabilitation}, volume = {85}, number = {4}, year = {2004}, note = {Haley, Stephen MCoster, Wendy JAndres, Patricia LKosinski, MarkNi, PengshengR01 hd43568/hd/nichdComparative StudyMulticenter StudyResearch Support, U.S. Gov{\textquoteright}t, Non-P.H.S.Research Support, U.S. Gov{\textquoteright}t, P.H.S.United StatesArchives of physical medicine and rehabilitationArch Phys Med Rehabil. 2004 Apr;85(4):661-6.}, month = {Apr}, pages = {661-6}, edition = {2004/04/15}, abstract = {OBJECTIVE: To compare simulated short-form and computerized adaptive testing (CAT) scores to scores obtained from complete item sets for each of the 3 domains of the Activity Measure for Post-Acute Care (AM-PAC). DESIGN: Prospective study. SETTING: Six postacute health care networks in the greater Boston metropolitan area, including inpatient acute rehabilitation, transitional care units, home care, and outpatient services. PARTICIPANTS: A convenience sample of 485 adult volunteers who were receiving skilled rehabilitation services. INTERVENTIONS: Not applicable. MAIN OUTCOME MEASURES: Inpatient and community-based short forms and CAT applications were developed for each of 3 activity domains (physical \& mobility, personal care \& instrumental, applied cognition) using item pools constructed from new items and items from existing postacute care instruments. RESULTS: Simulated CAT scores correlated highly with score estimates from the total item pool in each domain (4- and 6-item CAT r range,.90-.95; 10-item CAT r range,.96-.98). Scores on the 10-item short forms constructed for inpatient and community settings also provided good estimates of the AM-PAC item pool scores for the physical \& movement and personal care \& instrumental domains, but were less consistent in the applied cognition domain. Confidence intervals around individual scores were greater in the short forms than for the CATs. CONCLUSIONS: Accurate scoring estimates for AM-PAC domains can be obtained with either the setting-specific short forms or the CATs. The strong relationship between CAT and item pool scores can be attributed to the CAT{\textquoteright}s ability to select specific items to match individual responses. The CAT may have additional advantages over short forms in practicality, efficiency, and the potential for providing more precise scoring estimates for individuals.}, keywords = {Boston, Factor Analysis, Statistical, Humans, Outcome Assessment (Health Care)/*methods, Prospective Studies, Questionnaires/standards, Rehabilitation/*standards, Subacute Care/*standards}, isbn = {0003-9993 (Print)}, author = {Haley, S. M. and Coster, W. J. and Andres, P. L. and Kosinski, M. and Ni, P.} } @conference {1211, title = {A sequential Bayesian procedure for item calibration in multistage testing}, booktitle = {Paper presented at the annual meeting of the National Council on Measurement in Education}, year = {2004}, address = {San Diego CA}, author = {van der Linden, W. J. and Alan D Mead} } @article {61, title = {Sequential estimation in variable length computerized adaptive testing}, journal = {Journal of Statistical Planning and Inference}, volume = {121}, number = {2}, year = {2004}, pages = {249-264}, abstract = {With the advent of modern computer technology, there have been growing e3orts in recent years to computerize standardized tests, including the popular Graduate Record Examination (GRE), the Graduate Management Admission Test (GMAT) and the Test of English as a Foreign Language (TOEFL). Many of such computer-based tests are known as the computerized adaptive tests, a major feature of which is that, depending on their performance in the course of testing, di3erent examinees may be given with di3erent sets of items (questions). In doing so, items can be e>ciently utilized to yield maximum accuracy for estimation of examinees{\textquoteright} ability traits. We consider, in this article, one type of such tests where test lengths vary with examinees to yield approximately same predetermined accuracy for all ability traits. A comprehensive large sample theory is developed for the expected test length and the sequential point and interval estimates of the latent trait. Extensive simulations are conducted with results showing that the large sample approximations are adequate for realistic sample sizes. }, isbn = {03783758}, author = {Chang, I. Y.} } @article {355, title = {A sharing item response theory model for computerized adaptive testing}, journal = {Journal of Educational and Behavioral Statistics}, volume = {29}, number = {4}, year = {2004}, note = {References .American Educational Research Assn, US}, month = {Win}, pages = {439-460}, abstract = {A new sharing item response theory (SIRT) model is presented which explicitly models the effects of sharing item content between informants and testtakers. This model is used to construct adaptive item selection and scoring rules that provide increased precision and reduced score gains in instances where sharing occurs. The adaptive item selection rules are expressed as functions of the item{\textquoteright}s exposure rate in addition to other commonly used properties (characterized by difficulty, discrimination, and guessing parameters). Based on the results of simulated item responses, the new item selection and scoring algorithms compare favorably to the Sympson-Hetter exposure control method. The new SIRT approach provides higher reliability and lower score gains in instances where sharing occurs.}, author = {Segall, D. O.} } @article {82, title = {Siette: a web-based tool for adaptive testing}, journal = {International Journal of Artificial Intelligence in Education}, volume = {14}, number = {1}, year = {2004}, pages = {29-61}, keywords = {computerized adaptive testing}, author = {Conejo, R and Guzm{\'a}n, E and Mill{\'a}n, E and Trella, M and P{\'e}rez-De-La-Cruz, JL and R{\'\i}os, A} } @inbook {1771, title = {State-of-the-art and adaptive open-closed items in adaptive foreign language assessment}, year = {2004}, address = {Proceedings 4th Hellenic Conference with ternational Participation: Informational and Communication Technologies in Education, Athens,747-756}, author = {Giouroglou, H. and Economides, A. A.} } @article {624, title = {Statistics for detecting disclosed items in a CAT environment}, journal = {Metodologia de Las Ciencias del Comportamiento.}, volume = {5}, year = {2004}, chapter = {p{\'a}gs. 225-242}, issn = {1575-9105}, author = {Lu, Y., and Hambleton, R. K.} } @article {2115, title = {Strategies for Controlling Item Exposure in Computerized Adaptive Testing With the Generalized Partial Credit Model}, journal = {Applied Psychological Measurement}, volume = {28}, number = {3}, year = {2004}, pages = {165-185}, abstract = {

Choosing a strategy for controlling item exposure has become an integral part of test development for computerized adaptive testing (CAT). This study investigated the performance of six procedures for controlling item exposure in a series of simulated CATs under the generalized partial credit model. In addition to a no-exposure control baseline condition, the randomesque, modified-within-.10-logits, Sympson-Hetter, conditional Sympson-Hetter, a-stratified with multiple-stratification, and enhanced a-stratified with multiple-stratification procedures were implemented to control exposure rates. Two variations of the randomesque and modified-within-.10-logits procedures were examined, which varied the size of the item group from which the next item to be administered was randomly selected. The results indicate that although the conditional Sympson-Hetter provides somewhat lower maximum exposure rates, the randomesque and modified-within-.10-logits procedures with the six-item group variation have great utility for controlling overlap rates and increasing pool utilization and should be given further consideration.

}, doi = {10.1177/0146621604264133}, url = {http://apm.sagepub.com/content/28/3/165.abstract}, author = {Davis, Laurie Laughlin} } @article {93, title = {Strategies for controlling item exposure in computerized adaptive testing with the generalized partial credit model}, journal = {Applied Psychological Measurement}, volume = {28}, number = {3}, year = {2004}, pages = {165-185}, publisher = {Sage Publications: US}, abstract = {Choosing a strategy for controlling item exposure has become an integral part of test development for computerized adaptive testing (CAT). This study investigated the performance of six procedures for controlling item exposure in a series of simulated CATs under the generalized partial credit model. In addition to a no-exposure control baseline condition, the randomesque, modified-within-.10-logits, Sympson-Hetter, conditional Sympson-Hetter, a-stratified with multiple-stratification, and enhanced a-stratified with multiple-stratification procedures were implemented to control exposure rates. Two variations of the randomesque and modified-within-.10-logits procedures were examined, which varied the size of the item group from which the next item to be administered was randomly selected. The results indicate that although the conditional Sympson-Hetter provides somewhat lower maximum exposure rates, the randomesque and modified-within-.10-logits procedures with the six-item group variation have great utility for controlling overlap rates and increasing pool utilization and should be given further consideration. (PsycINFO Database Record (c) 2007 APA, all rights reserved)}, keywords = {computerized adaptive testing, generalized partial credit model, item exposure}, isbn = {0146-6216 (Print)}, author = {Davis, L. L.} } @article {38, title = {Strategies for controlling testlet exposure rates in computerized adaptive testing systems}, journal = {Dissertation Abstracts International: Section B: The Sciences \& Engineering}, volume = {64}, number = {11-B}, year = {2004}, pages = {5835}, abstract = {Exposure control procedures in computerized adaptive testing (CAT) systems protect item pools from being compromised, however, this impacts measurement precision. Previous research indicates that exposure control procedures perform differently for dichotomously scored versus polytomously scored CAT systems. For dichotomously scored CATs, conditional selection procedures are often the optimal choice, while randomization procedures perform best for polytomously scored CATs. CAT systems modeled with testlet response theory have not been examined to determine optimal exposure control procedures. This dissertation examined various exposure control procedures in testlet-based CAT systems using the three-parameter logistic testlet response theory model and the partial credit model. The exposure control procedures were the randomesque procedure, the modified within .10 logits procedure, two levels of the progressive restricted procedure, and two levels of the Sympson-Hetter procedure. Each of these was compared to a baseline no exposure control procedure, maximum information. The testlets were reading passages with six to ten multiple-choice items. The CAT systems consisted of maximum information testlet selection contingent on an exposure control procedure and content balancing for passage type and the number of items per passage; expected a posteriori ability estimation; and a fixed length stopping rule of seven testlets totaling fifty multiple-choice items. Measurement precision and exposure rates were examined to evaluate the effectiveness of the exposure control procedures for each measurement model. The exposure control procedures yielded similar results for measurement precision within the models. The exposure rates distinguished which exposure control procedures were most effective. The Sympson-Hetter conditions, which are conditional procedures, maintained the pre-specified maximum exposure rate, but performed very poorly in terms of pool utilization. The randomization procedures, randomesque and modified within .10 logits, yielded low maximum exposure rates, but used only about 70\% of the testlet pool. Surprisingly, the progressive restricted procedure, which is a combination of both a conditional and randomization procedure, yielded the best results in its ability to maintain and control the maximum exposure rate and it used the entire testlet pool. The progressive restricted conditions were the optimal procedures for both the partial credit CAT systems and the three-parameter logistic testlet response theory CAT systems. (PsycINFO Database Record (c) 2004 APA, all rights reserved).}, author = {Boyd, Aimee Michelle} } @conference {830, title = {A study of multiple stage adaptive test designs}, booktitle = {Paper presented at the annual meeting of the National Council on Measurement in Education}, year = {2004}, note = {{PDF file, 288 KB}}, address = {San Diego CA}, author = {Armstrong, R. D. and Edmonds, J.} } @booklet {1536, title = {A sequential Bayes procedure for item calibration in multi-stage testing}, year = {2003}, address = {Manuscript in preparation}, author = {van der Linden, W. J. and Alan D Mead} } @conference {1275, title = {A simulation study to compare CAT strategies for cognitive diagnosis}, booktitle = {Paper presented at the annual meeting of the National Council on Measurement in Education}, year = {2003}, note = {{PDF file, 250 KB}}, address = {Chicago IL}, author = {Xu, X. and Chang, Hua-Hua and Douglas, J.} } @article {379, title = {Small sample estimation in dichotomous item response models: Effect of priors based on judgmental information on the accuracy of item parameter estimates}, journal = {Applied Psychological Measurement}, volume = {27}, number = {1}, year = {2003}, note = {Sage Publications, US}, pages = {27-51}, abstract = {Large item banks with properly calibrated test items are essential for ensuring the validity of computer-based tests. At the same time, item calibrations with small samples are desirable to minimize the amount of pretesting and limit item exposure. Bayesian estimation procedures show considerable promise with small examinee samples. The purposes of the study were (a) to examine how prior information for Bayesian item parameter estimation can be specified and (b) to investigate the relationship between sample size and the specification of prior information on the accuracy of item parameter estimates. The results of the simulation study were clear: Estimation of item response theory (IRT) model item parameters can be improved considerably. Improvements in the one-parameter model were modest; considerable improvements with the two- and three-parameter models were observed. Both the study of different forms of priors and ways to improve the judgmental data used in forming the priors appear to be promising directions for future research. }, author = {Swaminathan, H. and Hambleton, R. K. and Sireci, S. G. and Xing, D. and Rizavi, S. M.} } @article {397, title = {Some alternatives to Sympson-Hetter item-exposure control in computerized adaptive testing}, journal = {Journal of Educational and Behavioral Statistics}, volume = {28}, number = {3}, year = {2003}, pages = {249-265}, abstract = {TheHetter and Sympson (1997; 1985) method is a method of probabilistic item-exposure control in computerized adaptive testing. Setting its control parameters to admissible values requires an iterative process of computer simulations that has been found to be time consuming, particularly if the parameters have to be set conditional on a realistic set of values for the examinees{\textquoteright} ability parameter. Formal properties of the method are identified that help us explain why this iterative process can be slow and does not guarantee admissibility. In addition, some alternatives to the SH method are introduced. The behavior of these alternatives was estimated for an adaptive test from an item pool from the Law School Admission Test (LSAT). Two of the alternatives showed attractive behavior and converged smoothly to admissibility for all items in a relatively small number of iteration steps. }, keywords = {Adaptive Testing, Computer Assisted Testing, Test Items computerized adaptive testing}, author = {van der Linden, W. J.} } @conference {937, title = {Standard-setting issues in computerized-adaptive testing}, booktitle = {Paper Prepared for Presentation at the Annual Conference of the Canadian Society for Studies in Education}, year = {2003}, address = {Halifax, Nova Scotia, May 30th, 2003}, author = {Gushta, M. M.} } @article {124, title = {Statistical detection and estimation of differential item functioning in computerized adaptive testing}, journal = {Dissertation Abstracts International: Section B: The Sciences \& Engineering}, volume = {64}, number = {6-B}, year = {2003}, pages = {2736}, abstract = {Differential item functioning (DIF) is an important issue in large scale standardized testing. DIF refers to the unexpected difference in item performances among groups of equally proficient examinees, usually classified by ethnicity or gender. Its presence could seriously affect the validity of inferences drawn from a test. Various statistical methods have been proposed to detect and estimate DIF. This dissertation addresses DIF analysis in the context of computerized adaptive testing (CAT), whose item selection algorithm adapts to the ability level of each individual examinee. In a CAT, a DIF item may be more consequential and more detrimental be cause fewer items are administered in a CAT than in a traditional paper-and-pencil test and because the remaining sequence of items presented to examinees depends in part on their responses to the DIF item. Consequently, an efficient, stable and flexible method to detect and estimate CAT DIF becomes necessary and increasingly important. We propose simultaneous implementations of online calibration and DIF testing. The idea is to perform online calibration of an item of interest separately in the focal and reference groups. Under any specific parametric IRT model, we can use the (online) estimated latent traits as covariates and fit a nonlinear regression model to each of the two groups. Because of the use of the estimated, not the true , the regression fit has to adjust for the covariate "measurement errors". It turns out that this situation fits nicely into the framework of nonlinear error-in-variable modelling, which has been extensively studied in statistical literature. We develop two bias-correction methods using asymptotic expansion and conditional score theory. After correcting the bias caused by measurement error, one can perform a significance test to detect DIF with the parameter estimates for different groups. This dissertation also discusses some general techniques to handle measurement error modelling with different IRT models, including the three-parameter normal ogive model and polytomous response models. Several methods of estimating DIF are studied as well. Large sample properties are established to justify the proposed methods. Extensive simulation studies show that the resulting methods perform well in terms of Type-I error rate control, accuracy in estimating DIF and power against both unidirectional and crossing DIF. (PsycINFO Database Record (c) 2004 APA, all rights reserved).}, author = {Feng, X.} } @article {92, title = {Strategies for controlling item exposure in computerized adaptive testing with polytomously scored items}, journal = {Dissertation Abstracts International: Section B: The Sciences \& Engineering}, volume = {64}, number = {1-B}, year = {2003}, pages = {458}, abstract = {Choosing a strategy for controlling the exposure of items to examinees has become an integral part of test development for computerized adaptive testing (CAT). Item exposure can be controlled through the use of a variety of algorithms which modify the CAT item selection process. This may be done through a randomization, conditional selection, or stratification approach. The effectiveness of each procedure as well as the degree to which measurement precision is sacrificed has been extensively studied with dichotomously scored item pools. However, only recently have researchers begun to examine these procedures in polytomously scored item pools. The current study investigated the performance of six different exposure control mechanisms under three polytomous IRT models in terms of measurement precision, test security, and ease of implementation. The three models examined in the current study were the partial credit, generalized partial credit, and graded response models. In addition to a no exposure control baseline condition, the randomesque, within .10 logits, Sympson-Hetter, conditional Sympson-Hetter, a-Stratified, and enhanced a-Stratified procedures were implemented to control item exposure rates. The a-Stratified and enhanced a-Stratified procedures were not evaluated with the partial credit model. Two variations of the randomesque and within .10 logits procedures were also examined which varied the size of the item group from which the next item to be administered was randomly selected. The results of this study were remarkably similar for all three models and indicated that the randomesque and within .10 logits procedures, when implemented with the six item group variation, provide the best option for controlling exposure rates when impact to measurement precision and ease of implementation are considered. The three item group variations of the procedures were, however, ineffective in controlling exposure, overlap, and pool utilization rates to desired levels. The Sympson-Hetter and conditional Sympson-Hetter procedures were difficult and time consuming to implement, and while they did control exposure rates to the target level, their performance in terms of item overlap (for the Sympson-Hetter) and pool utilization were disappointing. The a-Stratified and enhanced a-Stratified procedures both turned in surprisingly poor performances across all variables. (PsycINFO Database Record (c) 2004 APA, all rights reserved).}, author = {Davis, L. L.} } @conference {896, title = {Strategies for controlling item exposure in computerized adaptive testing with the generalized partial credit model}, booktitle = {Paper presented at the annual meeting of the National Council on Measurement in Education}, year = {2003}, note = {PDF file, 620 K}, address = {Chicago IL}, author = {Davis, L. L.} } @book {1650, title = {Strategies for controlling testlet exposure rates in computerized adaptive testing systems}, year = {2003}, note = {{PDF file, 485 KB}}, address = {Unpublished Ph.D. Dissertation, The University of Texas at Austin.}, author = {Boyd, A. M} } @article {193, title = {Student modeling and ab initio language learning}, journal = {System}, volume = {31}, number = {4}, year = {2003}, pages = {519-535}, abstract = {Provides examples of student modeling techniques that have been employed in computer-assisted language learning over the past decade. Describes two systems for learning German: "German Tutor" and "Geroline." Shows how a student model can support computerized adaptive language testing for diagnostic purposes in a Web-based language learning environment that does not rely on parsing technology. (Author/VWL)}, author = {Heift, T. and Schulze, M.} } @article {456, title = {A study of the feasibility of Internet administration of a computerized health survey: The Headache Impact Test (HIT)}, journal = {Quality of Life Research}, volume = { 12}, year = {2003}, pages = { 953-961}, author = {Bayliss, M.S. and Dewey, J.E. and Dunlap, I and et. al.} } @article {811, title = {Self-adapted testing: An overview}, journal = {International Journal of Continuing Engineering Education and Life-Long Learning}, volume = {12}, year = {2002}, pages = {107-122}, author = {Wise, S. L. and Ponsoda, V. and Olea, J.} } @conference {1101, title = {Some features of the estimated sampling distribution of the ability estimate in computerized adaptive testing according to two stopping rules}, booktitle = {Communication propos{\'e}e au 11e Biannual International objective measurement workshop. New-Orleans : International Objective Measurement Workshops.}, year = {2002}, author = {Ra{\^\i}che, G. and Blais, J. G.} } @conference {861, title = {Some features of the sampling distribution of the ability estimate in computerized adaptive testing according to two stopping rules}, booktitle = {Paper presented at the annual meeting of the International Objective Measurement Workshops-XI}, year = {2002}, note = {{PDF file, 38 KB}}, address = {New Orleans, LA}, author = {Blais, J-G. and Raiche, G.} } @booklet {1508, title = {STAR Math 2 Computer-Adaptive Math Test and Database: Technical Manual}, year = {2002}, address = {Wisconsin Rapids, WI: Author}, author = {Renaissance-Learning-Inc.} } @conference {1294, title = {Statistical indexes for monitoring item behavior under computer adaptive testing environment}, booktitle = {(Original title: Detecting item misfit in computerized adaptive testing.) Paper presented at the annual meeting of the American Educational Research Association}, year = {2002}, note = {{PDF file, 2.287 MB}}, address = {New Orleans LA}, author = {Zhu, R. and Yu, F. and Liu, S. M.} } @book {1661, title = {Strategies for controlling item exposure in computerized adaptive testing with polytomously scored items}, year = {2002}, note = {{PDF file, 1.83 MB}}, address = {Unpublished doctoral dissertation, University of Texas, Austin}, author = {Davis, L. L.} } @booklet {1436, title = {A strategy for controlling item exposure in multidimensional computerized adaptive testing}, year = {2002}, address = {Available from http://www3. tat.sinica.edu.tw/library/c_tec_rep/c-2002-11.pdf}, author = {Lee, Y. H. and Ip, E.H. and Fuh, C.D.} } @article {346, title = {A structure-based approach to psychological measurement: Matching measurement models to latent structure}, journal = {Assessment}, volume = {9}, number = {1}, year = {2002}, pages = {4-16}, abstract = {The present article sets forth the argument that psychological assessment should be based on a construct{\textquoteright}s latent structure. The authors differentiate dimensional (continuous) and taxonic (categorical) structures at the latent and manifest levels and describe the advantages of matching the assessment approach to the latent structure of a construct. A proper match will decrease measurement error, increase statistical power, clarify statistical relationships, and facilitate the location of an efficient cutting score when applicable. Thus, individuals will be placed along a continuum or assigned to classes more accurately. The authors briefly review the methods by which latent structure can be determined and outline a structure-based approach to assessment that builds on dimensional scaling models, such as item response theory, while incorporating classification methods as appropriate. Finally, the authors empirically demonstrate the utility of their approach and discuss its compatibility with traditional assessment methods and with computerized adaptive testing. (PsycINFO Database Record (c) 2005 APA ) (journal abstract)}, keywords = {Adaptive Testing, Assessment, Classification (Cognitive Process), Computer Assisted, Item Response Theory, Psychological, Scaling (Testing), Statistical Analysis computerized adaptive testing, Taxonomies, Testing}, author = {Ruscio, John and Ruscio, Ayelet Meron} } @booklet {1564, title = {Scoring alternatives for incomplete computerized adaptive tests (Research Report 01-20)}, year = {2001}, address = {Princeton NJ: Educational Testing Service}, author = {Way, W. D. and Gawlick, L. A. and Eignor, D. R.} } @booklet {1507, title = {STAR Early Literacy Computer-Adaptive Diagnostic Assessment: Technical Manual}, year = {2001}, address = {Wisconsin Rapids, WI: Author}, author = {Renaissance-Learning-Inc.} } @conference {1232, title = {A system for on-the-fly adaptive testing}, booktitle = {Paper presented at the annual meeting of the National Council on Measurement in Education}, year = {2001}, address = {Seattle WA}, author = {Wagner, M. E.} } @booklet {1509, title = {A selection procedure for polytomous items in computerized adaptive testing (Measurement and Research Department Reports 2000-5)}, year = {2000}, address = {Arnhem, The Netherlands: Cito}, author = {Rijn, P. W. van, and Theo Eggen and Hemker, B. T. and Sanders, P. F.} } @conference {999, title = {Solving complex constraints in a-stratified computerized adaptive testing designs}, booktitle = {Paper presented at the annual meeting of the National Council on Measurement in Education}, year = {2000}, note = {PDF file, 384 K}, address = {New Orleans, USA}, author = {Leung, C-K.. and Chang, Hua-Hua and Hau, K-T.} } @conference {1137, title = {Some considerations for improving accuracy of estimation of item characteristic curves in online calibration of computerized adaptive testing}, booktitle = {Paper presented at the annual meeting of the National Council on Measurement in Education}, year = {2000}, address = {New Orleans LA}, author = {Samejima, F.} } @conference {888, title = {Specific information item selection for adaptive testing}, booktitle = {Paper presented at the annual meeting of the National Council on Measurement in Education}, year = {2000}, address = {New Orleans}, author = {Davey, T. and Fan, M.} } @booklet {1506, title = {STAR Reading 2 Computer-Adaptive Reading Test and Database: Technical Manual}, year = {2000}, address = {Wisconsin Rapids, WI: Author}, author = {Renaissance-Learning-Inc.} } @conference {1085, title = {Sufficient simplicity or comprehensive complexity? A comparison of probabilitic and stratification methods of exposure control}, booktitle = {Paper presented at the annual meeting of the National Council on Measurement in Education}, year = {2000}, address = {New Orleans LA}, author = {Parshall, C. G. and Kromrey, J. D. and Hogarty, K. Y.} } @booklet {1557, title = {Some relationship among issues in CAT item pool management}, year = {1999}, note = {$\#$WA99-03}, author = {Wang, T.} } @article {294, title = {Some reliability estimates for computerized adaptive tests}, journal = {Applied Psychological Measurement}, volume = {23}, number = {3}, year = {1999}, pages = {239-47}, abstract = {Three reliability estimates are derived for the Bayes modal estimate (BME) and the maximum likelihood estimate (MLE) of θin computerized adaptive tests (CAT). Each reliability estimate is a function of test information. Two of the estimates are shown to be upper bounds to true reliability. The three reliability estimates and the true reliabilities of both MLE and BME were computed for seven simulated CATs. Results showed that the true reliabilities for MLE and BME were nearly identical in all seven tests. The three reliability estimates never differed from the true reliabilities by more than .02 (.01 in most cases). A simple implementation of one reliability estimate was found to accurately estimate reliability in CATs. }, author = {Nicewander, W. A. and Thomasson, G. L.} } @conference {971, title = {Standard errors of proficiency estimates in stratum scored CAT}, booktitle = {Paper presented at the annual meeting of the National Council on Measurement in Education}, year = {1999}, address = {Montreal, Canada}, author = {Kingsbury, G. G.} } @conference {956, title = {Study of methods to detect aberrant response patterns in computerized testing}, booktitle = {Paper presented at the annual meeting of the National Council on Measurement in Education}, year = {1999}, address = {Montreal, Canada}, author = {Iwamoto, C. K. and Nungester, R. J. and Luecht, RM} } @booklet {1585, title = {Simulating nonmodel-fitting responses in a CAT Environment (Research Report 98-10)}, year = {1998}, note = {$\#$YI-98-10}, address = {Iowa City IA: ACT Inc. (Also presented at National Council on Measurement in Education, 1999: ERIC No. ED 427 042)}, author = {Yi, Q. and Nering, M, L.} } @booklet {1475, title = {Simulating the null distribution of person-fit statistics for conventional and adaptive tests (Research Report 98-02)}, year = {1998}, address = {Enschede, The Netherlands: University of Twente, Faculty of Educational Science and Technology, Department of Measurement and Data Analysis}, author = {Meijer, R. R. and van Krimpen-Stoop, E. M. L. A.} } @article {373, title = {Simulating the use of disclosed items in computerized adaptive testing}, journal = {Journal of Educational Measurement}, volume = {35}, number = {1}, year = {1998}, note = {National Council on Measurement in Education, US}, pages = {48-68}, abstract = {Regular use of questions previously made available to the public (i.e., disclosed items) may provide one way to meet the requirement for large numbers of questions in a continuous testing environment, that is, an environment in which testing is offered at test taker convenience throughout the year rather than on a few prespecified test dates. First it must be shown that such use has effects on test scores small enough to be acceptable. In this study simulations are used to explore the use of disclosed items under a worst-case scenario which assumes that disclosed items are always answered correctly. Some item pool and test designs were identified in which the use of disclosed items produces effects on test scores that may be viewed as negligible.}, keywords = {computerized adaptive testing}, author = {Stocking, M. L. and W. C. Ward and Potenza, M. T.} } @conference {1136, title = {Some considerations for eliminating biases in ability estimation in computerized adaptive testing}, booktitle = {Paper presented at the annual meeting of the American Educational Research Association}, year = {1998}, author = {Samejima, F.} } @conference {1196, title = {Some item response theory to provide scale scores based on linear combinations of testlet scores, for computerized adaptive tests}, booktitle = {Paper presented at the annual meeting of the Psychometric Society}, year = {1998}, address = {Urbana, IL}, author = {Thissen, D.} } @article {627, title = {Some practical examples of computerized adaptive sequential testing}, journal = {Journal of Educational Measurement}, volume = {35}, year = {1998}, pages = {229-249}, author = {Luecht, RM and Nungester, R. J.} } @conference {1074, title = {Some reliability estimators for computerized adaptive tests}, booktitle = {Paper presented at the annual meeting of the Psychometric Society}, year = {1998}, address = {Urbana, IL}, author = {Nicewander, W. A. and Thomasson, G. L.} } @article {155, title = {Statistical tests for person misfit in computerized adaptive testing}, number = {98-01}, year = {1998}, pages = {28}, institution = {Faculty of Educational Science and Technology, Univeersity of Twente}, address = {Enschede, The Netherlands}, isbn = {98-01}, author = {Glas, C. A. W. and Meijer, R. R. and van Krimpen-Stoop, E. M.} } @booklet {1385, title = {Statistical tests for person misfit in computerized adaptive testing (Research Report 98-01)}, year = {1998}, address = {Enschede, The Netherlands : University of Twente, Faculty of Educational Science and Technology, Department of Measurement and Data Analysis}, author = {Glas, C. A. W. and Meijer, R. R. and van Krimpen-Stoop, E. M. L. A.} } @article {739, title = {Stochastic order in dichotomous item response models for fixed, adaptive, and multidimensional tests}, journal = {Psychometrika}, volume = {63}, year = {1998}, pages = {211-226}, author = {van der Linden, W. J.} } @article {632, title = {Swedish Enlistment Battery: Construct validity and latent variable estimation of cognitive abilities by the CAT-SEB}, journal = {International Journal of Selection and Assessment}, volume = {6}, year = {1998}, pages = {107-114}, author = {Mardberg, B. and Carlstedt, B.} } @article {683, title = {Self-adapted testing: Improving performance by modifying tests instead of examinees}, journal = {Stress \& Coping: An International Journal}, volume = {10(1)}, year = {1997}, pages = {83-104}, abstract = {This paper describes self-adapted testing and some of the evidence concerning its effects, presents possible theoretical explanations for those effects, and discusses some of the practical concerns regarding self-adapted testing. Self-adapted testing is a variant of computerized adapted testing in which the examine makes dynamic choices about the difficulty of the items he or she attempts. Self-adapted testing generates scores that are, in constrast to computerized adapted test and fixed-item tests, uncorrelated with a measure of trait test anxiety. This lack of correlation with an irrelevant attribute of the examine is evidence of an improvement in the construct validity of the scores. This improvement comes at the cost of a decrease in testing efficiency. The interaction between test anxiety and test administration mode is more consistent with an interference theory of test anxiety than a deficit theory. Some of the practical concerns regarding self-adapted testing can be ruled out logically, but others await empirical investigation.}, author = {Rocklin, T.} } @booklet {1628, title = {Simulating the use of disclosed items in computerized adaptive testing (Research Report 97-10)}, year = {1997}, address = {Princeton NJ: Educational Testing Service}, author = {Stocking, M. L. and W. C. Ward and Potenza, M. T.} } @conference {1072, title = {Simulation of realistic ability vectors}, booktitle = {Paper presented at the Psychometric Society meeting}, year = {1997}, address = {Gatlinburg TN}, author = {Nering, M. and Thompson, T.D. and Davey, T.} } @conference {1128, title = {A simulation study of the use of the Mantel-Haenszel and logistic regression procedures for assessing DIF in a CAT environment}, booktitle = {Paper presented at the annual meeting of the National Council on Measurement in Education}, year = {1997}, address = {Chicago IL}, author = {Ross, L. P. and Nandakumar, R, and Clauser, B. E.} } @article {754, title = {Some new item selection criteria for adaptive testing}, journal = {Journal of Educational and Behavioral Statistics}, volume = {22}, year = {1997}, pages = {203-226}, author = {Veerkamp, W. J. J., and Berger, M. P. F.} } @article {477, title = {Some new item selection criteria for adaptive testing}, journal = {Journal of Educational and Behavioral Statistics}, volume = {22}, year = {1997}, pages = {203-226}, author = {Berger, M. P. F., and Veerkamp, W. J. J.} } @conference {970, title = {Some questions that must be addressed to develop and maintain an item pool for use in an adaptive test}, booktitle = {Paper presented at the annual meeting of the National Council on Measurement in Education}, year = {1997}, address = {Chicago IL}, author = {Kingsbury, G. G.} } @book {1707, title = {Statistical methods for computerized adaptive testing}, year = {1997}, address = {Unpublished doctoral dissertation, University of Twente, Enschede, The Netherlands}, author = {Veerkamp, W. J. J.} } @conference {1166, title = {A search procedure to determine sets of decision points when using testlet-based Bayesian sequential testing procedures}, booktitle = {Paper presented at the annual meeting of National Council on Measurement in Education}, year = {1996}, address = {New York}, author = {Smith, R. and Lewis, C.} } @booklet {1454, title = {Some practical examples of computerized adaptive sequential testing (Internal Report)}, year = {1996}, address = {Philadelphia: National Board of Medical Examiners}, author = {Luecht, RM and Nungester, R. J.} } @conference {1253, title = {Strategies for managing item pools to maximize item security}, booktitle = {Paper presented at the annual meeting of the National Council on Measurement in Education}, year = {1996}, address = {San Diego}, author = {Way, W. D. and A Zara and Leahy, J.} } @article {579, title = {Shortfall of questions curbs use of computerized graduate exam}, journal = {The Chronicle of Higher Education}, year = {1995}, chapter = {A23.}, author = {Jacobson, R. L.} } @booklet {1452, title = {Some alternative CAT item selection heuristics (Internal report)}, year = {1995}, address = {Philadelphia PA: National Board of Medical Examiners}, author = {Luecht, RM} } @conference {1154, title = {Some new methods for content balancing adaptive tests}, booktitle = {Paper presented at the annual meeting of the Psychometric Society}, year = {1995}, address = {Minneapolis MN}, author = {Segall, D. O. and Davey, T. C.} } @article {144, title = {A study of psychologically optimal level of item difficulty}, journal = {Shinrigaku Kenkyu}, volume = {65}, number = {6}, year = {1995}, note = {Fujimori, SClinical TrialControlled Clinical TrialEnglish AbstractJapanShinrigaku kenkyu : The Japanese journal of psychologyShinrigaku Kenkyu. 1995 Feb;65(6):446-53.}, month = {Feb}, pages = {446-53}, edition = {1995/02/01}, abstract = {For the purpose of selecting items in a test, this study presented a viewpoint of psychologically optimal difficulty level, as well as measurement efficiency, of items. A paper-and-pencil test (P \& P) composed of hard, moderate and easy subtests was administered to 298 students at a university. A computerized adaptive test (CAT) was also administered to 79 students. The items of both tests were selected from Shiba{\textquoteright}s Word Meaning Comprehension Test, for which the estimates of parameters of two-parameter item response model were available. The results of P \& P research showed that the psychologically optimal success level would be such that the proportion of right answers is somewhere between .75 and .85. A similar result was obtained from CAT research, where the proportion of about .8 might be desirable. Traditionally a success rate of .5 has been recommended in adaptive testing. In this study, however, it was suggested that the items of such level would be too hard psychologically for many examinees.}, keywords = {*Adaptation, Psychological, *Psychological Tests, Adult, Female, Humans, Male}, isbn = {0021-5236 (Print)0021-5236 (Linking)}, author = {Fujimori, S.} } @conference {1113, title = {The selection of test items for decision making with a computer adaptive test}, booktitle = {Paper presented at the annual meeting of the National Council on Measurement in Education}, year = {1994}, address = {New Orleans LA}, author = {Reckase, M. D. and Spray, J. A.} } @conference {1178, title = {The selection of test items for decision making with a computer adaptive test}, booktitle = {Paper presented at the national meeting of the National Council on Measurement in Education}, year = {1994}, note = {$\#$SP94-01}, address = {New Orleans LA}, author = {Spray, J. A. and Reckase, M. D.} } @article {682, title = {Self-adapted testing}, journal = {Applied Measurement in Education}, volume = {7}, year = {1994}, pages = {3-14}, author = {Rocklin, T. R.} } @booklet {1544, title = {A simple and fast item selection procedure for adaptive testing}, year = {1994}, address = {Research (Report 94-13). University of Twente.}, author = {Veerkamp, W. J. J.} } @article {2028, title = {A Simulation Study of Methods for Assessing Differential Item Functioning in Computerized Adaptive Tests}, journal = {Applied Psychological Measurement}, volume = {18}, number = {2}, year = {1994}, pages = {121-140}, author = {Zwick, R. and Thayer, D. T. and Wingersky, M.} } @booklet {1563, title = {A simulation study of the Mantel-Haenszel procedure for detecting DIF with the NCLEX using CAT (Technical Report xx-xx)}, year = {1994}, address = {Princeton NJ: Educational Testing Service}, author = {Way, W. D.} } @booklet {1545, title = {Some new item selection criteria for adaptive testing (Research Rep 94-6)}, year = {1994}, address = {Enschede, The Netherlands: University of Twente, Department of Educational Measurement and Data Analysis.}, author = {Veerkamp, W. J. and Berger, M. P. F.} } @conference {1118, title = {A simulated comparison of testlets and a content balancing procedure for an adaptive certification examination}, booktitle = {Paper presented at the annual meeting of the National Council on Measurement in Education}, year = {1993}, address = {Atlanta}, author = {Reshetar, R. A. and Norcini, J. J. and Shea, J. A.} } @conference {1119, title = {A simulated comparison of two content balancing and maximum information item selection procedures for an adaptive certification examination}, booktitle = {Paper presented at the annual meeting of the National Council on Measurement in Education}, year = {1993}, address = {Atlanta}, author = {Reshetar, R. A. and Norcini, J. J. and Shea, J. A.} } @booklet {1590, title = {A simulation study of methods for assessing differential item functioning in computer-adaptive tests (Educational Testing Service Research Rep No RR 93-11)}, year = {1993}, address = {Princeton NJ: Educational Testing Service.}, author = {Zwick, R. and Thayer, D. and Wingersky, M.} } @conference {1162, title = {Some initial experiments with adaptive survey designs for structured questionnaires}, booktitle = {Paper presented at the New Methods and Applications in Consumer Research Conference}, year = {1993}, address = {Cambridge MA}, author = {Singh, J.} } @article {775, title = {Some practical considerations when converting a linearly administered test to an adaptive format}, journal = {Educational Measurement: Issues and Practice}, volume = {12 (1)}, year = {1993}, pages = {15-20}, author = {Wainer, H.,} } @conference {947, title = {Scaling of two-stage adaptive test configurations for achievement testing}, booktitle = {Paper presented at the annual meeting of the National Council on Measurement in Education}, year = {1992}, address = {New Orleans LA}, author = {Hendrickson, A. B. and Kolen, M. J.} } @booklet {1553, title = {Some practical considerations when converting a linearly administered test to an adaptive format (Research Report 92-21 or 13?)}, year = {1992}, address = {Princeton NJ: Educational Testing Service}, author = {Wainer, H.,} } @conference {833, title = {Student attitudes toward computer-adaptive test administration}, booktitle = {Paper presented at the annual meeting of the American Educational Research Association}, year = {1992}, address = {San Francisco CA}, author = {Baghi, H and Ferrara, S. F and Gabrys, R.} } @booklet {1407, title = {A simulation study of some simple approaches to the study of DIF for CATs}, year = {1991}, address = {Internal memorandum, Educational Testing Service}, author = {Holland, P. W. and Zwick, R.} } @booklet {1554, title = {Some empirical guidelines for building testlets (Technical Report 91-56)}, year = {1991}, address = {Princeton NJ: Educational Testing Service, Program Statistics Research}, author = {Wainer, H., and Kaplan, B. and Lewis, C.} } @article {731, title = {Sequential item response models with an ordered response}, journal = {British Journal of Mathematical and Statistical Psychology}, volume = {43}, year = {1990}, pages = {39-55}, author = {Tutz, G.} } @article {98, title = {A simulation and comparison of flexilevel and Bayesian computerized adaptive testing}, journal = {Journal of Educational Measurement}, volume = {27}, number = {3}, year = {1990}, pages = {227-239}, abstract = {Computerized adaptive testing (CAT) is a testing procedure that adapts an examination to an examinee{\textquoteright}s ability by administering only items of appropriate difficulty for the examinee. In this study, the authors compared Lord{\textquoteright}s flexilevel testing procedure (flexilevel CAT) with an item response theory-based CAT using Bayesian estimation of ability (Bayesian CAT). Three flexilevel CATs, which differed in test length (36, 18, and 11 items), and three Bayesian CATs were simulated; the Bayesian CATs differed from one another in the standard error of estimate (SEE) used for terminating the test (0.25, 0.10, and 0.05). Results showed that the flexilevel 36- and 18-item CATs produced ability estimates that may be considered as accurate as those of the Bayesian CAT with SEE = 0.10 and comparable to the Bayesian CAT with SEE = 0.05. The authors discuss the implications for classroom testing and for item response theory-based CAT.}, keywords = {computerized adaptive testing}, author = {De Ayala, R. J., and Dodd, B. G. and Koch, W. R.} } @article {659, title = {Software review: MicroCAT Testing System Version 3}, journal = {Journal of Educational Measurement}, volume = {7}, year = {1990}, pages = {82-88}, author = {Patience, W. M.} } @conference {847, title = {The stability of Rasch pencil and paper item calibrations on computer adaptive tests}, booktitle = {Paper presented at the Midwest Objective Measurement Seminar}, year = {1990}, address = {Chicago IL}, author = {Bergstrom, Betty A. and Lunz, M. E.} } @article {749, title = {Some procedures for computerized ability testing}, journal = {International Journal of Educational Research}, volume = {13(2)}, year = {1989}, pages = {175-187}, author = {van der Linden, W. J. and Zwarts, M. A.} } @booklet {1611, title = {Scale drift in on-line calibration (Research Report RR-88-28-ONR)}, number = {(Research Report RR-88-28-ONR)}, year = {1988}, address = {Princeton NJ: Educational Testing Service}, author = {Stocking, M. L.} } @booklet {1617, title = {Scale drift in on-line calibration (Tech Rep. No. ERIC ED389710)}, year = {1988}, address = {Educational Testing Service, Princeton, N.J.}, author = {Stocking, M. L.} } @conference {1018, title = {Simple and effective algorithms [for] computer-adaptive testing}, booktitle = {Paper presented at the annual meeting of the American Educational Research Association}, year = {1988}, note = {$\#$LI88-01}, address = {New Orleans LA}, author = {Linacre, J. M.} } @booklet {1612, title = {Some considerations in maintaining adaptive test item pools (Research Report 88-33-ONR)}, year = {1988}, address = {Princeton NJ: Educational Testing Service}, author = {Stocking, M. L.} } @booklet {1618, title = {Some considerations in maintaining adaptive test item pools (Tech Rep. No. ERIC ED391814)}, year = {1988}, address = {Educational Testing Service, Princeton, N.J.}, author = {Stocking, M. L.} } @article {684, title = {Self-adapted testing: A performance improving variation of computerized adaptive testing}, journal = {Journal of Educational Psychology}, volume = {79}, year = {1987}, pages = {315-319}, author = {Rocklin, T. R., and O{\textquoteright}Donnell, A. M.} } @article {2016, title = {Some Applications of Optimization Algorithms in Test Design and Adaptive Testing}, journal = {Applied Psychological Measurement}, volume = {10}, year = {1986}, pages = {381-389}, author = {Theunissen, T. J. J. M.} } @article {722, title = {Some applications of optimization algorithms in test design and adaptive testing}, journal = {Applied Psychological Measurement}, volume = {10}, year = {1986}, pages = {381-389}, author = {Theunissen, T. J. J. M.} } @book {1699, title = {Sequential analysis: Tests and confidence intervals}, year = {1985}, address = {New York: Springer-Verlag}, author = {Siegmund, D.} } @article {91, title = {A structural comparison of conventional and adaptive versions of the ASVAB}, journal = {Multivariate Behavioral Research}, volume = {20}, number = {3}, year = {1985}, note = {Lawrence Erlbaum, US}, pages = {305-322}, abstract = {Examined several structural models of similarity between the Armed Services Vocational Aptitude Battery (ASVAB) and a battery of computerized adaptive tests designed to measure the same aptitudes. 12 plausible models were fitted to sample data in a double cross-validation design. 1,411 US Navy recruits completed 10 ASVAB subtests. A computerized adaptive test version of the ASVAB subtests was developed on item pools of approximately 200 items each. The items were pretested using applicants from military entrance processing stations across the US, resulting in a total calibration sample size of approximately 60,000 for the computerized adaptive tests. Three of the 12 models provided reasonable summaries of the data. One model with a multiplicative structure (M. W. Browne; see record 1984-24964-001) performed quite well. This model provides an estimate of the disattenuated method correlation between conventional testing and adaptive testing. In the present data, this correlation was estimated to be 0.97 and 0.98 in the 2 halves of the data. Results support computerized adaptive tests as replacements for conventional tests. (33 ref) (PsycINFO Database Record (c) 2004 APA, all rights reserved).}, author = {Cudeck, R.} } @conference {1176, title = {The selection of items for decision making with a computer adaptive test}, booktitle = {Paper presented at the annual meeting of the National Council on Measurement in Education}, year = {1984}, address = {New Orleans LA}, author = {Spray, J. A. and Reckase, M. D.} } @inbook {257, title = {Small N justifies Rasch model}, booktitle = {New horizons in testing: Latent trait test theory and computerized adaptive testing}, year = {1983}, pages = {51-61}, publisher = {Academic Press}, organization = {Academic Press}, address = {New York, NY. USA}, author = {Lord, F. M.,}, editor = {Bock, R. D.} } @book {1704, title = {The stochastic modeling of elementary psychological processes}, year = {1983}, address = {Cambridge: Cambridge University Press}, author = {Townsend, J. T. and Ashby, G. F.} } @booklet {1571, title = {The stratified adaptive computerized ability test (Research Report 73-3)}, year = {1983}, address = {Minneapolis: University of Minnesota, Department of Psychology, Computerized Adaptive Testing Laboratory}, author = {Weiss, D. J.} } @article {2011, title = {Sequential Testing for Selection}, journal = {Applied Psychological Measurement}, volume = {6}, year = {1982}, pages = {337-351}, author = {Weitzman, R. A.} } @article {803, title = {Sequential testing for selection}, journal = {Applied Psychological Measurement}, volume = {6}, year = {1982}, pages = {337-351}, author = {Weitzman, R. A.} } @article {651, title = {A simple form of tailored testing}, journal = {British Journal of Educational Psychology}, volume = {50}, year = {1980}, pages = {301-303}, author = {Nisbet, J. and Adams, M. and Arthur, J.} } @inbook {1875, title = {Some decision procedures for use with tailored testing}, year = {1980}, address = {D. J. Weiss (Ed.), Proceedings of the 1979 Computerized Adaptive Testing Conference (pp. 79-100). Minneapolis MN: University of Minnesota, Department of Psychology, Psychometric Methods Program, Computerized Adaptive Testing Laboratory.}, author = {Reckase, M. D.} } @inbook {1831, title = {Some how and which for practical tailored testing}, year = {1980}, address = {L. J. T. van der Kamp, W. F. Langerak and D.N.M. de Gruijter (Eds): Psychometrics for educational debates (pp. 189-206). New York: John Wiley and Sons. Computer-Assisted Instruction, Testing, and Guidance (pp. 139-183). New York: Harper and Row.}, author = {Lord, F. M.,} } @booklet {1473, title = {A successful application of latent trait theory to tailored achievement testing (Research Report 80-1)}, year = {1980}, address = {University of Missouri, Department of Educational Psychology, Tailored Testing Research Laboratory}, author = {McKinley, R. L. and Reckase, M. D.} } @conference {960, title = {Student reaction to computerized adaptive testing in the classroom}, booktitle = {Paper presented at the 87th annual meeting of the American Psychological Association}, year = {1979}, note = {$\#$JO79-01}, address = {New York}, author = {Johnson, M. J.} } @article {735, title = {The stratified adaptive ability test as a tool for personnel selection and placement}, journal = {TIMS Studies in the Management Sciences}, volume = {8}, year = {1978}, pages = {135-151}, author = {Vale, C. D. and Weiss, D. J.} } @article {2240, title = {A stratified adaptive test of verbal ability}, journal = {Japanese Journal of Educational Psychology}, volume = {26}, year = {1978}, pages = {229-238}, author = {Shiba, S. and Noguchi, H. and Haebra, T.} } @article {635, title = {Some properties of a Bayesian adaptive ability testing strategy}, journal = {Applied Psychological Measurement}, volume = {1}, year = {1977}, pages = {121-140}, author = {J. R. McBride} } @article {2003, title = {Some Properties of a Bayesian Adaptive Ability Testing Strategy}, journal = {Applied Psychological Measurement}, volume = {1}, year = {1977}, pages = {121-140}, author = {J. R. McBride} } @inbook {1813, title = {Student attitudes toward tailored testing}, year = {1977}, address = {D. J. Weiss (Ed.), Proceedings of the 1977 Computerized Adaptive Testing Conference. Minneapolis MN: University of Minnesota, Department of Psychology, Psychometric Methods Program.}, author = {Koch, W. R. and Patience, W. M.} } @book {1683, title = {Simulation studies of adaptive testing: A comparative evaluation}, year = {1976}, address = {Unpublished doctoral dissertation, University of Minnesota, Minneapolis, MN}, author = {J. R. McBride} } @inbook {1829, title = {Some likelihood functions found in tailored testing}, year = {1976}, note = {{PDF file, 166 KB}}, address = {C. K. Clark (Ed.), Proceedings of the First Conference on Computerized Adaptive Testing (pp. 79-81). Washington DC: U.S. Government Printing Office.}, author = {Lord, F. M.,} } @booklet {1472, title = {Some properties of a Bayesian adaptive ability testing strategy (Research Report 76-1)}, year = {1976}, address = {Minneapolis MN: Department of Psychology, Computerized Adaptive Testing Laboratory}, author = {J. R. McBride and Weiss, D. J.} } @inbook {1838, title = {Scoring adaptive tests}, year = {1975}, note = {{PDF file, 442 KB}}, address = {D. J. Weiss (Ed.), Computerized adaptive trait measurement: Problems and Prospects (Research Report 75-5), pp. 17-25. Minneapolis MN: University of Minnesota, Department of Psychology, Psychometric Methods Program.}, author = {J. R. McBride} } @article {724, title = {Sequential testing for instructional classification}, journal = {Journal of Computer-Based Instruction.}, volume = {1}, year = {1975}, pages = {92-99}, author = {Thomas, D. B.} } @booklet {1527, title = {A simulation study of stradaptive ability testing (Research Report 75-6)}, year = {1975}, address = {Minneapolis: University of Minnesota, Department of Psychology, Psychometric Methods Program}, author = {Vale, C. D. and Weiss, D. J.} } @inbook {1917, title = {Strategies of branching through an item pool}, year = {1975}, note = {$\#$VA75-01 {PDF file, 600 KB}}, address = {D. J. Weiss (Ed.), Computerized adaptive trait measurement: Problems and Prospects (Research Report 75-5), pp. 1-16. Minneapolis: University of Minnesota, Department of Psychology, Psychometric Methods Program.}, author = {Vale, C. D.} } @booklet {1526, title = {A study of computer-administered stradaptive ability testing (Research Report 75-4)}, year = {1975}, address = {Minneapolis: University of Minnesota, Department of Psychology, Psychometric Methods Program}, author = {Vale, C. D. and Weiss, D. J.} } @booklet {1331, title = {Simulation studies of two-stage ability testing (Research Report 74-4)}, year = {1974}, note = {{PDF file, 2.92 MB}}, address = {Minneapolis: Department of Psychology, Psychometric Methods Program}, author = {Betz, N. E. and Weiss, D. J.} } @booklet {1567, title = {Strategies of adaptive ability measurement (Research Report 74-5)}, year = {1974}, note = {{PDF file, 5.555 MB}}, address = {Minneapolis: University of Minnesota, Department of Psychology, Psychometric Methods Program}, author = {Weiss, D. J.} } @booklet {1566, title = {The stratified adaptive computerized ability test (Research Report 73-3)}, year = {1973}, note = {{PDF file, 2.498 MB}}, address = {Minneapolis: University of Minnesota, Department of Psychology, Psychometric Methods Program}, author = {Weiss, D. J.} } @article {256, title = {Sequential testing for dichotomous decisions. }, journal = {Educational and Psychological Measurement}, volume = {32}, number = {1}, year = {1972}, pages = {85-95.}, keywords = {CCAT, CLASSIFICATION Computerized Adaptive Testing, sequential probability ratio testing, SPRT}, author = {Linn, R. L. and Rock, D. A. and Cleary, T. A.} } @article {621, title = {The self-scoring flexilevel test}, journal = {Journal of Educational Measurement}, volume = {8}, year = {1971}, pages = {147-151}, author = {Lord, F. M.,} } @booklet {1445, title = {The self-scoring flexilevel test (RB-7043)}, year = {1970}, address = {Princeton NJ: Educational Testing Service}, author = {Lord, F. M.,} } @booklet {1444, title = {Sequential testing for dichotomous decisions. College Entrance Examination Board Research and Development Report (RDR 69-70, No 3", and Educational Testing Service RB-70-31)}, year = {1970}, note = {$\#$LI70-31}, address = {Princeton NJ: Educational Testing Service.}, author = {Linn, R. L. and Rock, D. A. and Cleary, T. A.} } @inbook {1825, title = {Some test theory for tailored testing}, year = {1970}, note = {$\#$LO70139}, address = {W. H. Holtzman (Ed.), Computer-assisted instruction, testing, and guidance (pp.139-183). New York: Harper and Row.}, author = {Lord, F. M.,} } @booklet {1609, title = {Short tailored tests (RB-69-63)}, number = {(RB-69-63)}, year = {1969}, address = {Princeton NJ: Educational Testing Service}, author = {Stocking, M. L.} } @article {600, title = {The sequential item test}, journal = {American Psychologist}, volume = {2}, year = {1956}, pages = {419}, author = {Krathwohl, D. R. and Huyser, R. J.} } @article {439, title = { Sequential analysis with more than two alternative hypotheses, and its relation to discriminant function analysis}, volume = {12}, year = {1950}, pages = {137-144}, author = {Armitage, P.} } @article {647, title = {Some empirical aspects of the sequential analysis technique as applied to an achievement examination}, journal = {Journal of Experimental Education}, volume = {18}, year = {1950}, pages = {195-207}, author = {Moonan, W. J.} }