@article {2624, title = {Latent Class Analysis of Recurrent Events in Problem-Solving Items}, journal = {Applied Psychological Measurement}, volume = {42}, number = {6}, year = {2018}, pages = {478-498}, abstract = {Computer-based assessment of complex problem-solving abilities is becoming more and more popular. In such an assessment, the entire problem-solving process of an examinee is recorded, providing detailed information about the individual, such as behavioral patterns, speed, and learning trajectory. The problem-solving processes are recorded in a computer log file which is a time-stamped documentation of events related to task completion. As opposed to cross-sectional response data from traditional tests, process data in log files are massive and irregularly structured, calling for effective exploratory data analysis methods. Motivated by a specific complex problem-solving item {\textquotedblleft}Climate Control{\textquotedblright} in the 2012 Programme for International Student Assessment, the authors propose a latent class analysis approach to analyzing the events occurred in the problem-solving processes. The exploratory latent class analysis yields meaningful latent classes. Simulation studies are conducted to evaluate the proposed approach.}, doi = {10.1177/0146621617748325}, url = {https://doi.org/10.1177/0146621617748325}, author = {Haochen Xu and Guanhua Fang and Yunxiao Chen and Jingchen Liu and Zhiliang Ying} } @conference {2643, title = {A Large-Scale Progress Monitoring Application with Computerized Adaptive Testing}, booktitle = {IACAT 2017 Conference}, year = {2017}, month = {08/2017}, publisher = {Niigata Seiryo University}, organization = {Niigata Seiryo University}, address = {Niigata, Japan}, abstract = {

Many conventional assessment tools are available to teachers in schools for monitoring student progress in a formative manner. The outcomes of these assessment tools are essential to teachers\’ instructional modifications and schools\’ data-driven educational strategies, such as using remedial activities and planning instructional interventions for students with learning difficulties. When measuring student progress toward instructional goals or outcomes, assessments should be not only considerably precise but also sensitive to individual change in learning. Unlike conventional paper-pencil assessments that are usually not appropriate for every student, computerized adaptive tests (CATs) are highly capable of estimating growth consistently with minimum and consistent error. Therefore, CATs can be used as a progress monitoring tool in measuring student growth.

This study focuses on an operational CAT assessment that has been used for measuring student growth in reading during the academic school year. The sample of this study consists of nearly 7 million students from the 1st grade to the 12th grade in the US. The students received a CAT-based reading assessment periodically during the school year. The purpose of these periodical assessments is to measure the growth in students\’ reading achievement and identify the students who may need additional instructional support (e.g., academic interventions). Using real data, this study aims to address the following research questions: (1) How many CAT administrations are necessary to make psychometrically sound decisions about the need for instructional changes in the classroom or when to provide academic interventions?; (2) What is the ideal amount of time between CAT administrations to capture student growth for the purpose of producing meaningful decisions from assessment results?

To address these research questions, we first used the Theil-Sen estimator for robustly fitting a regression line to each student\’s test scores obtained from a series of CAT administrations. Next, we\ used the conditional standard error of measurement (cSEM) from the CAT administrations to create an error band around the Theil-Sen slope (i.e., student growth rate). This process resulted in the normative slope values across all the grade levels. The optimal number of CAT administrations was established from grade-level regression results. The amount of time needed for progress monitoring was determined by calculating the amount of time required for a student to show growth beyond the median cSEM value for each grade level. The results showed that the normative slope values were the highest for lower grades and declined steadily as grade level increased. The results also suggested that the CAT-based reading assessment is most useful for grades 1 through 4, since most struggling readers requiring an intervention appear to be within this grade range. Because CAT yielded very similar cSEM values across administrations, the amount of error in the progress monitoring decisions did not seem to depend on the number of CAT administrations.

Session Video

}, keywords = {CAT, Large-Scale tests, Process monitoring}, url = {https://drive.google.com/open?id=1uGbCKenRLnqTxImX1fZicR2c7GRV6Udc}, author = {Okan Bulut and Damien Cormier} } @article {2529, title = {Latent-Class-Based Item Selection for Computerized Adaptive Progress Tests}, journal = {Journal of Computerized Adaptive Testing}, volume = {5}, year = {2017}, pages = {22-43}, keywords = {computerized adaptive progress test, item selection method, Kullback-Leibler information, Latent class analysis, log-odds scoring}, issn = {2165-6592}, doi = {10.7333/1704-0502022}, url = {http://iacat.org/jcat/index.php/jcat/article/view/62/29}, author = {van Buuren, Nikky and Eggen, Theo J. H. M.} } @article {2320, title = {Longitudinal Multistage Testing}, journal = {Journal of Educational Measurement}, volume = {50}, number = {4}, year = {2013}, pages = {447{\textendash}468}, abstract = {

This article introduces longitudinal multistage testing (lMST), a special form of multistage testing (MST), as a method for adaptive testing in longitudinal large-scale studies. In lMST designs, test forms of different difficulty levels are used, whereas the values on a pretest determine the routing to these test forms. Since lMST allows for testing in paper and pencil mode, lMST may represent an alternative to conventional testing (CT) in assessments for which other adaptive testing designs are not applicable. In this article the performance of lMST is compared to CT in terms of test targeting as well as bias and efficiency of ability and change estimates. Using a simulation study, the effect of the stability of ability across waves, the difficulty level of the different test forms, and the number of link items between the test forms were investigated.

}, issn = {1745-3984}, doi = {10.1111/jedm.12028}, url = {http://dx.doi.org/10.1111/jedm.12028}, author = {Pohl, Steffi} } @inbook {1822, title = {Limiting item exposure for target difficulty ranges in a high-stakes CAT}, year = {2009}, note = {MB}}, address = {D. J. Weiss (Ed.), Proceedings of the 2009 GMAC Conference on Computerized Adaptive Testing. {PDF File, 1.}, author = {Li, X. and Becker, K. and Gorham, J. and Woo, A.} } @article {342, title = {Logistics of collecting patient-reported outcomes (PROs) in clinical practice: an overview and practical examples}, journal = {Quality of Life Research}, volume = {18}, number = {1}, year = {2009}, note = {Rose, MatthiasBezjak, AndreaNetherlandsQuality of life research : an international journal of quality of life aspects of treatment, care and rehabilitationQual Life Res. 2009 Feb;18(1):125-36. Epub 2009 Jan 20.}, month = {Feb}, pages = {125-36}, edition = {2009/01/20}, abstract = {PURPOSE: Interest in collecting patient-reported outcomes (PROs), such as health-related quality of life (HRQOL), health status reports, and patient satisfaction is on the rise and practical aspects of collecting PROs in clinical practice are becoming more important. The purpose of this paper is to draw the attention to a number of issues relevant for a successful integration of PRO measures into the daily work flow of busy clinical settings. METHODS: The paper summarizes the results from a breakout session held at an ISOQOL special topic conference for PRO measures in clinical practice in 2007. RESULTS: Different methodologies of collecting PROs are discussed, and the support needed for each methodology is highlighted. The discussion is illustrated by practical real-life examples from early adaptors who administered paper-pencil, or electronic PRO assessments (ePRO) for more than a decade. The paper also reports about new experiences with more recent technological developments, such as SmartPens and Computer Adaptive Tests (CATs) in daily practice. CONCLUSIONS: Methodological and logistical issues determine the resources needed for a successful integration of PRO measures into daily work flow procedures and influence significantly the usefulness of PRO data for clinical practice.}, isbn = {0962-9343 (Print)}, author = {Rose, M. and Bezjak, A.} } @article {84, title = {Letting the CAT out of the bag: Comparing computer adaptive tests and an 11-item short form of the Roland-Morris Disability Questionnaire}, journal = {Spine}, volume = {33}, number = {12}, year = {2008}, note = {Cook, Karon FChoi, Seung WCrane, Paul KDeyo, Richard AJohnson, Kurt LAmtmann, Dagmar5 P60-AR48093/AR/United States NIAMS5U01AR052171-03/AR/United States NIAMSComparative StudyResearch Support, N.I.H., ExtramuralUnited StatesSpineSpine. 2008 May 20;33(12):1378-83.}, month = {May 20}, pages = {1378-83}, edition = {2008/05/23}, abstract = {STUDY DESIGN: A post hoc simulation of a computer adaptive administration of the items of a modified version of the Roland-Morris Disability Questionnaire. OBJECTIVE: To evaluate the effectiveness of adaptive administration of back pain-related disability items compared with a fixed 11-item short form. SUMMARY OF BACKGROUND DATA: Short form versions of the Roland-Morris Disability Questionnaire have been developed. An alternative to paper-and-pencil short forms is to administer items adaptively so that items are presented based on a person{\textquoteright}s responses to previous items. Theoretically, this allows precise estimation of back pain disability with administration of only a few items. MATERIALS AND METHODS: Data were gathered from 2 previously conducted studies of persons with back pain. An item response theory model was used to calibrate scores based on all items, items of a paper-and-pencil short form, and several computer adaptive tests (CATs). RESULTS: Correlations between each CAT condition and scores based on a 23-item version of the Roland-Morris Disability Questionnaire ranged from 0.93 to 0.98. Compared with an 11-item short form, an 11-item CAT produced scores that were significantly more highly correlated with scores based on the 23-item scale. CATs with even fewer items also produced scores that were highly correlated with scores based on all items. For example, scores from a 5-item CAT had a correlation of 0.93 with full scale scores. Seven- and 9-item CATs correlated at 0.95 and 0.97, respectively. A CAT with a standard-error-based stopping rule produced scores that correlated at 0.95 with full scale scores. CONCLUSION: A CAT-based back pain-related disability measure may be a valuable tool for use in clinical and research contexts. Use of CAT for other common measures in back pain research, such as other functional scales or measures of psychological distress, may offer similar advantages.}, keywords = {*Disability Evaluation, *Health Status Indicators, Adult, Aged, Aged, 80 and over, Back Pain/*diagnosis/psychology, Calibration, Computer Simulation, Diagnosis, Computer-Assisted/*standards, Humans, Middle Aged, Models, Psychological, Predictive Value of Tests, Questionnaires/*standards, Reproducibility of Results}, isbn = {1528-1159 (Electronic)}, author = {Cook, K. F. and Choi, S. W. and Crane, P. K. and Deyo, R. A. and Johnson, K. L. and Amtmann, D.} } @article {2200, title = {Local Dependence in an Operational CAT: Diagnosis and Implications}, journal = {Journal of Educational Measurement}, volume = {45}, number = {3}, year = {2008}, pages = {201{\textendash}223}, abstract = {

The accuracy of CAT scores can be negatively affected by local dependence if the CAT utilizes parameters that are misspecified due to the presence of local dependence and/or fails to control for local dependence in responses during the administration stage. This article evaluates the existence and effect of local dependence in a test of Mathematics Knowledge. Diagnostic tools were first used to evaluate the existence of local dependence in items that were calibrated under a 3PL model. A simulation study was then used to evaluate the effect of local dependence on the precision of examinee CAT scores when the 3PL model was used for selection and scoring. The diagnostic evaluation showed strong evidence for local dependence. The simulation suggested that local dependence in parameters had a minimal effect on CAT score precision, while local dependence in responses had a substantial effect on score precision, depending on the degree of local dependence present.

}, issn = {1745-3984}, doi = {10.1111/j.1745-3984.2008.00061.x}, url = {http://dx.doi.org/10.1111/j.1745-3984.2008.00061.x}, author = {Pommerich, Mary and Segall, Daniel O.} } @article {289, title = {La Validez desde una {\'o}ptica psicom{\'e}trica [Validity from a psychometric perspective]}, journal = {Acta Comportamentalia}, volume = {13}, number = {1}, year = {2005}, pages = {9-20}, abstract = {El estudio de la validez constituye el eje central de los an{\'a}lisis psicom{\'e}tricos de los instrumentos de medida. En esta comunicaci{\'o}n se traza una breve nota hist{\'o}rica de los distintos modos de concebir la validez a lo largo de los tiempos, se comentan las l{\'\i}neas actuales, y se tratan de vislumbrar posibles v{\'\i}as futuras, teniendo en cuenta el impacto que las nuevas tecnolog{\'\i}as inform{\'a}ticas est{\'a}n ejerciendo sobre los propios instrumentos de medida en Psicolog{\'\i}a y Educaci{\'o}n. Cuestiones como los nuevos formatos multimedia de los {\'\i}tems, la evaluaci{\'o}n a distancia, el uso intercultural de las pruebas, las consecuencias de su uso, o los tests adaptativos informatizados, reclaman nuevas formas de evaluar y conceptualizar la validez. Tambi{\'e}n se analizan cr{\'\i}ticamente algunos planteamientos recientes sobre el concepto de validez. The study of validity constitutes a central axis of psychometric analyses of measurement instruments. This paper presents a historical sketch of different modes of conceiving validity, with commentary on current views, and it attempts to predict future lines of research by considering the impact of new computerized technologies on measurement instruments in psychology and education. Factors such as the new multimedia format of items, distance assessment, the intercultural use of tests, the consequences of the latter, or the development of computerized adaptive tests demand new ways of conceiving and evaluating validity. Some recent thoughts about the concept of validity are also critically analyzed. (PsycINFO Database Record (c) 2005 APA ) (journal abstract)}, keywords = {Factor Analysis, Measurement, Psychometrics, Scaling (Testing), Statistical, Technology, Test Validity}, author = {Mu{\~n}iz, J.} } @conference {2251, title = {A learning environment for english for academic purposes based on adaptive tests and task-based systems}, booktitle = {Intelligent Tutoring Systems. }, year = {2004}, publisher = { Springer Berlin Heidelberg}, organization = { Springer Berlin Heidelberg}, author = {PITON-GON{\c C}ALVES, J. and ALUISIO, S. M. and MENDONCA, L. H. and NOVAES, O. O.} } @inbook {2201, title = {A Learning Environment for English for Academic Purposes Based on Adaptive Tests and Task-Based Systems}, booktitle = {Intelligent Tutoring Systems}, series = {Lecture Notes in Computer Science}, volume = {3220}, year = {2004}, pages = {1-11}, publisher = {Springer Berlin / Heidelberg}, organization = {Springer Berlin / Heidelberg}, isbn = {978-3-540-22948-3}, doi = {10.1007/978-3-540-30139-4_1}, url = {http://dx.doi.org/10.1007/978-3-540-30139-4_1}, author = {Gon{\c c}alves, Jean P. and Aluisio, Sandra M. and de Oliveira, Leandro H.M. and Oliveira Jr., Osvaldo N.}, editor = {Lester, James C. and Vicari, Rosa Maria and Paragua{\c c}u, F{\'a}bio} } @article {671, title = {La simulation d{\textquoteright}un test adaptatif bas{\'e} sur le mod{\`e}le de Rasch [Simulation of a Rasch-based adaptive test]}, journal = {Mesure et {\'e}valuation en {\'e}ducation.}, year = {2002}, note = {(In French) {PDF file, 30 KB}}, author = {Ra{\^\i}che, G.} } @inbook {1871, title = {Le testing adaptatif [Adaptive testing]}, year = {2002}, note = {(In French) {PDF file, 191 KB}}, address = {D. R. Bertrand and J.G. Blais (Eds) : Les th{\'e}ories modernes de la mesure [Modern theories of measurement]. Sainte-Foy: Presses de l{\textquoteright}Universit{\'e} du Qu{\'e}bec.}, author = {Ra{\^\i}che, G.} } @book {1693, title = {La distribution dchantillonnage en testing adaptatif en fonction de deux rgles darrt : selon lerreur type et selon le nombre ditems administrs [Sampling distribution of the proficiency estimate in computerized adaptive testing according to two stopping...}, year = {2000}, note = {.}, address = {Doctoral thesis, Montreal: University of Montreal}, author = {Rache, G.} } @article {232, title = {Lagrangian relaxation for constrained curve-fitting with binary variables: Applications in educational testing}, journal = {Dissertation Abstracts International Section A: Humanities and Social Sciences}, volume = {61}, number = {3-A}, year = {2000}, pages = {1063}, abstract = {This dissertation offers a mathematical programming approach to curve fitting with binary variables. Various Lagrangian Relaxation (LR) techniques are applied to constrained curve fitting. Applications in educational testing with respect to test assembly are utilized. In particular, techniques are applied to both static exams (i.e. conventional paper-and-pencil (P\&P)) and adaptive exams (i.e. a hybrid computerized adaptive test (CAT) called a multiple-forms structure (MFS)). This dissertation focuses on the development of mathematical models to represent these test assembly problems as constrained curve-fitting problems with binary variables and solution techniques for the test development. Mathematical programming techniques are used to generate parallel test forms with item characteristics based on item response theory. A binary variable is used to represent whether or not an item is present on a form. The problem of creating a test form is modeled as a network flow problem with additional constraints. In order to meet the target information and the test characteristic curves, a Lagrangian relaxation heuristic is applied to the problem. The Lagrangian approach works by multiplying the constraint by a "Lagrange multiplier" and adding it to the objective. By systematically varying the multiplier, the test form curves approach the targets. This dissertation explores modifications to Lagrangian Relaxation as it is applied to the classical paper-and-pencil exams. For the P\&P exams, LR techniques are also utilized to include additional practical constraints to the network problem, which limit the item selection. An MFS is a type of a computerized adaptive test. It is a hybrid of a standard CAT and a P\&P exam. The concept of an MFS will be introduced in this dissertation, as well as, the application of LR as it is applied to constructing parallel MFSs. The approach is applied to the Law School Admission Test for the assembly of the conventional P\&P test as well as an experimental computerized test using MFSs. (PsycINFO Database Record (c) 2005 APA )}, keywords = {Analysis, Educational Measurement, Mathematical Modeling, Statistical}, author = {Koppel, N. B.} } @book {1662, title = {Learning Potential Computerised Adaptive Test (LPCAT): Technical Manual}, year = {2000}, note = {$\#$deBE00-01}, address = {Pretoria: UNISA}, author = {De Beer, M.} } @book {1663, title = {Learning Potential Computerised Adaptive Test (LPCAT): User{\textquoteright}s Manual}, year = {2000}, note = {$\#$deBE00-02}, address = {Pretoria: UNISA}, author = {De Beer, M.} } @article {764, title = {Limiting answer review and change on computerized adaptive vocabulary tests: Psychometric and attitudinal results}, journal = {Journal of Educational Measurement}, volume = {37}, year = {2000}, pages = {21-38}, author = {Vispoel, W. P. and Hendrickson, A. B. and Bleiler, T.} } @article {203, title = {Los tests adaptativos informatizados en la frontera del siglo XXI: Una revisi{\'o}n [Computerized adaptive tests at the turn of the 21st century: A review]}, journal = {Metodolog{\'\i}a de las Ciencias del Comportamiento}, volume = {2}, number = {2}, year = {2000}, pages = {183-216}, keywords = {computerized adaptive testing}, isbn = {1575-9105}, author = {Hontangas, P. and Ponsoda, V. and Olea, J. and Abad, F. J.} } @conference {1227, title = {Limiting answer review and change on computerized adaptive vocabulary tests: Psychometric and attitudinal results}, booktitle = {Paper presented at the annual meeting of the National Council on Measurement in Education}, year = {1999}, note = {$\#$VI99-01}, address = {Montreal, Canada}, author = {Vispoel, W. P. and Hendrickson, A. and Bleiler, T. and Widiatmo, H. and Shrairi, S. and Ihrig, D.} } @booklet {1434, title = {Linking scores for computer-adaptive and paper-and-pencil administrations of the SAT (Research Report No 97-12)}, year = {1997}, note = {$\#$LA97-12}, address = {Princeton NJ: Educational Testing Service}, author = {Lawrence, I. and Feigenbaum, M.} } @booklet {1499, title = {La simulation de mod{\`e}le sur ordinateur en tant que m{\'e}thode de recherche : le cas concret de l{\textquoteright}{\'e}tude de la distribution d{\textquoteright}{\'e}chantillonnage de l{\textquoteright}estimateur du niveau d{\textquoteright}habilet{\'e} en testing adaptatif en fonction de deux r{\`e}gles d{\textquoteright}arr{\^e}t}, year = {1994}, address = {Actes du 6e colloque de l{\textquoteleft}Association pour la recherche au coll{\'e}gial. Montr{\'e}al : Association pour la recherche au coll{\'e}gial, ARC}, author = {Ra{\^\i}che, G.} } @conference {1099, title = {L{\textquoteright}{\'e}valuation nationale individualis{\'e}e et assist{\'e}e par ordinateur [Large scale assessment: Tailored and computerized]}, booktitle = {Qu{\'e}bec: Proceeding of the 14th Congress of the Association qu{\'e}b{\'e}coise de p{\'e}dagogie coll{\'e}giale. Montr{\'e}al: Association qu{\'e}b{\'e}coise de p{\'e}dagogie coll{\'e}giale (AQPC).}, year = {1994}, author = {Ra{\^\i}che, G. and B{\'e}land, A.} } @booklet {1433, title = {Les tests adaptatifs en langue seconde}, year = {1993}, address = {Communication lors de la 16e session d{\textquoteright}{\'e}tude de l{\textquoteright}ADM{\'E}{\'E} {\`a} Laval. Montr{\'e}al: Association pour le d{\'e}veloppement de la mesure et de l{\textquoteright}{\'e}valuation en {\'e}ducation.}, author = {Laurier, M.} } @article {719, title = {Linking the standard and advanced forms of the Ravens Progressive Matrices in both the paper-and-pencil and computer-adaptive-testing formats}, journal = {Educational and Psychological Measurement}, volume = {53}, year = {1993}, pages = {905-925}, author = {Styles, I. and Andrich, D.} } @booklet {1411, title = {The Language Training Division{\textquoteright}s computer adaptive reading proficiency test}, year = {1992}, address = {Provo, UT: Language Training Division, Office of Training and Education}, author = {Janczewski, D. and Lowe, P.} } @article {440, title = {Le testing adaptatif avec interpr{\'e}tation crit{\'e}rielle, une exp{\'e}rience de praticabilit{\'e} du TAM pour l{\textquoteright}{\'e}valuation sommative des apprentissages au Qu{\'e}bec.}, journal = {Mesure et {\'e}valuation en {\'e}ducation}, volume = {15-1 et 2}, year = {1992}, chapter = {10}, author = {Auger, R.}, editor = {Seguin, S. P.} } @article {729, title = {Latent structure and item sampling models for testing}, journal = {Annual Review of Psychology}, volume = {36}, year = {1985}, pages = {19-48}, author = {Traub, R. E. and Lam, Y. R.} } @conference {1247, title = {Legal and political considerations in large-scale adaptive testing}, booktitle = {Paper presented at the 23rd conference of the Military Testing Association.}, year = {1982}, author = {B. K. Waters and Lee, G. C.} } @booklet {1424, title = {A live tailored testing comparison study of the one- and three-parameter logistic models (Research Report 78-1)}, year = {1978}, address = {Columbia MO: University of Missouri, Department of Psychology}, author = {Koch, W. J. and Reckase, M. D.} } @inbook {1971, title = {A Low-Cost Terminal Usable for Computerized Adaptive Testing}, year = {1977}, address = {D. J. Weiss (Ed.), Proceedings of the 1977 Computerized Adaptive Testing Conference. Minneapolis MN: University of Minnesota, Department of Psychology, Psychometric Methods Program}, author = {Lamos, J. P. and B. K. Waters} } @article {484, title = {Le development de lintelligence chez les enfants}, journal = {LAnee Psychologique}, volume = {14}, year = {1908}, note = {In French}, pages = {1-94}, author = {Binet, A. and Simon, T.} }