\relax 
\citation{weyeuker08,halstead77,mccabe76,chapman02,me04g,nagappan05,hall00,nikora03,conf/icse/NagappanB05a,khoshgoftaar01,conf/ictai/TangK04,journals/ese/KhoshgoftaarS03,me03a,me02e,me03k,me03q,porter90,tiang95,khoshgoftaar99,srinivasan95,me07b}
\citation{me03q}
\newlabel{@firstpg}{{}{1}}
\@writefile{toc}{\contentsline {section}{\numberline {1}Introduction}{1}}
\citation{me07b}
\citation{lessmann09}
\citation{milton08}
\citation{me03q,me09b}
\citation{me08d}
\@writefile{toc}{\contentsline {section}{\numberline {2}Background}{2}}
\newlabel{sec:back}{{2}{2}}
\citation{leven95}
\citation{lutz03}
\citation{voas95}
\citation{lowrey98}
\citation{me99q,me00y}
\citation{bradley98scaling}
\citation{fenton99}
\@writefile{toc}{\contentsline {subsection}{\numberline {2.1}Blind Spots}{3}}
\@writefile{toc}{\contentsline {subsection}{\numberline {2.2}Lightweight Sampling}{3}}
\@writefile{toc}{\contentsline {subsubsection}{\numberline {2.2.1}Data Mining}{3}}
\citation{domingos97optimality}
\citation{cohen95}
\citation{quinlan92}
\citation{briemann01,jiang08a}
\@writefile{lof}{\contentsline {figure}{\numberline {1}{\ignorespaces Static code features.}}{4}}
\newlabel{fig:features}{{1}{4}}
\newlabel{sec:scf}{{2.2.2}{4}}
\@writefile{toc}{\contentsline {subsubsection}{\numberline {2.2.2}Static Code Features}{4}}
\citation{nikora04}
\citation{fenton94}
\citation{musa87,littlewood97}
\citation{nach08}
\citation{weyeuker08}
\@writefile{toc}{\contentsline {subsection}{\numberline {2.3}Frequently Asked Questions}{5}}
\@writefile{toc}{\contentsline {subsubsection}{\numberline {2.3.1}Why Binary Classifications?}{5}}
\@writefile{toc}{\contentsline {subsubsection}{\numberline {2.3.2}Why Static Code Features?}{5}}
\citation{nagappan05}
\citation{me02f}
\citation{rakitin01}
\citation{fenton97}
\citation{shepperd94}
\@writefile{toc}{\contentsline {subsubsection}{\numberline {2.3.3}What Can be Learned from Static Code Features?}{6}}
\citation{fenton97}
\citation{me09b}
\@writefile{lof}{\contentsline {figure}{\numberline {2}{\ignorespaces Tables of data.}}{7}}
\newlabel{fig:data}{{2}{7}}
\citation{me07e}
\citation{me09b}
\citation{me09b}
\@writefile{lof}{\contentsline {figure}{\numberline {3}{\ignorespaces Performance measures}}{8}}
\newlabel{fig:perform}{{3}{8}}
\citation{shu02}
\citation{fagan86}
\citation{shull00a}
\citation{fagan76}
\citation{raffo05}
\@writefile{lof}{\contentsline {figure}{\numberline {4}{\ignorespaces Results of round-robin and self experiments.}}{9}}
\newlabel{fig:rrself}{{4}{9}}
\@writefile{toc}{\contentsline {subsection}{\numberline {2.4}Summary}{9}}
\newlabel{sec:summary}{{2.4}{9}}
\citation{porter90}
\citation{me07b}
\citation{jiang08a}
\citation{jiang08a}
\citation{lessmann09}
\citation{lessmann09}
\citation{jiang08a}
\citation{yang06}
\citation{brieman96}
\citation{FreSch97}
\citation{cover67}
\citation{quinlan92}
\citation{cohen95r}
\citation{breiman84}
\citation{briemann01}
\@writefile{lof}{\contentsline {figure}{\numberline {5}{\ignorespaces Box plot for AUC(pf,pd) seen with 9 learners.}}{10}}
\newlabel{fig:logandnom}{{5}{10}}
\@writefile{toc}{\contentsline {section}{\numberline {3}Ceiling Effects in Defect Predictors}{10}}
\newlabel{sec:ceiling}{{3}{10}}
\citation{lessmann09}
\@writefile{lof}{\contentsline {figure}{\numberline {6}{\ignorespaces Range of AUC(pf,pd) ranks seen in 19 learners.}}{11}}
\newlabel{fig:all}{{6}{11}}
\@writefile{lof}{\contentsline {figure}{\numberline {7}{\ignorespaces $Pf$-vs-$pd$-vs-$effort$.}}{12}}
\newlabel{fig:pdpf}{{7}{12}}
\@writefile{toc}{\contentsline {section}{\numberline {4}Breaking Through the Ceiling}{12}}
\newlabel{sec:b}{{4}{12}}
\citation{arisholm06}
\citation{jiang08a,jiang08b,me07b}
\citation{cohen95}
\citation{koru07}
\@writefile{toc}{\contentsline {subsection}{\numberline {4.1}Experimental Set Up}{13}}
\@writefile{toc}{\contentsline {subsubsection}{\numberline {4.1.1}Operationalizing AUC(effort,pd)}{13}}
\@writefile{toc}{\contentsline {subsubsection}{\numberline {4.1.2}Upper and Lower Bounds on Performance}{13}}
\@writefile{lof}{\contentsline {figure}{\numberline {8}{\ignorespaces $Effort$-vs-$PD$.}}{14}}
\newlabel{fig:effort}{{8}{14}}
\citation{quinlan92}
\citation{elkan01}
\citation{fawcett01}
\citation{jiang08b,me08h}
\@writefile{toc}{\contentsline {subsubsection}{\numberline {4.1.3}Details}{15}}
\@writefile{toc}{\contentsline {subsection}{\numberline {4.2}Initial results}{15}}
\citation{milton08}
\citation{Blake+Merz:1998}
\citation{Blake+Merz:1998}
\citation{Blake+Merz:1998}
\citation{milton08}
\citation{williams03}
\citation{me02c,me03l}
\@writefile{toc}{\contentsline {section}{\numberline {5}WHICH}{16}}
\@writefile{toc}{\contentsline {subsection}{\numberline {5.1}Details}{16}}
\@writefile{lof}{\contentsline {figure}{\numberline {9}{\ignorespaces Top-of-stack scores of the WHICH}}{17}}
\newlabel{fig:picks}{{9}{17}}
\newlabel{eq:pprime}{{1}{17}}
\citation{jiang07,me00p,me02g,me02k,me03q,me04c,me05d,me07b,me07e}
\citation{me08d}
\newlabel{eq:weights}{{2}{18}}
\@writefile{toc}{\contentsline {subsection}{\numberline {5.2}Results}{18}}
\@writefile{toc}{\contentsline {subsubsection}{\numberline {5.2.1}Overall Results}{18}}
\citation{me07b}
\citation{jiang07,me00p,me02g,me02k,me03q,me04c,me05d,me07b,me07e}
\@writefile{lof}{\contentsline {figure}{\numberline {10}{\ignorespaces Results from all data sets of Figure\nobreakspace  {}2\hbox {}.}}{19}}
\newlabel{fig:allr}{{10}{19}}
\@writefile{toc}{\contentsline {subsubsection}{\numberline {5.2.2}Individual Results}{19}}
\citation{lessmann09}
\citation{me07b}
\@writefile{lof}{\contentsline {figure}{\numberline {11}{\ignorespaces Four examples of pattern \#1}}{20}}
\newlabel{fig:bands1a}{{11}{20}}
\@writefile{lof}{\contentsline {figure}{\numberline {12}{\ignorespaces Three more examples of pattern \#1}}{21}}
\newlabel{fig:bands1b}{{12}{21}}
\@writefile{toc}{\contentsline {subsection}{\numberline {5.3}External Validity}{21}}
\@writefile{lof}{\contentsline {figure}{\numberline {13}{\ignorespaces Two examples of pattern \#2.}}{22}}
\newlabel{fig:bands2}{{13}{22}}
\@writefile{lof}{\contentsline {figure}{\numberline {14}{\ignorespaces The only example of pattern \#3.}}{22}}
\newlabel{fig:bands3}{{14}{22}}
\citation{pugh90}
\@writefile{toc}{\contentsline {section}{\numberline {6}Discussion}{23}}
\@writefile{toc}{\contentsline {section}{\numberline {7}Conclusion}{23}}
\bibstyle{acmtrans}
\bibdata{refs}
\bibcite{arisholm06}{\citeauthoryear {Arisholm and Briand}{Arisholm and Briand}{2006}}
\bibcite{Blake+Merz:1998}{\citeauthoryear {Blake and Merz}{Blake and Merz}{1998}}
\bibcite{bradley98scaling}{\citeauthoryear {Bradley, Fayyad, and Reina}{Bradley et\nobreakspace  {}al\unhbox \voidb@x \hbox {.}}{1998}}
\bibcite{breiman84}{\citeauthoryear {Breiman, Friedman, Olshen, and Stone}{Breiman et\nobreakspace  {}al\unhbox \voidb@x \hbox {.}}{1984}}
\bibcite{briemann01}{\citeauthoryear {Breimann}{Breimann}{2001}}
\bibcite{brieman96}{\citeauthoryear {Brieman}{Brieman}{1996}}
\bibcite{chapman02}{\citeauthoryear {Chapman and Solomon}{Chapman and Solomon}{2002}}
\bibcite{cohen95}{\citeauthoryear {Cohen}{Cohen}{1995a}}
\bibcite{cohen95r}{\citeauthoryear {Cohen}{Cohen}{1995b}}
\bibcite{cover67}{\citeauthoryear {Cover and Hart}{Cover and Hart}{1967}}
\bibcite{demsar06}{\citeauthoryear {Demsar}{Demsar}{2006}}
\bibcite{dietterich97}{\citeauthoryear {Dietterich}{Dietterich}{1997}}
\bibcite{domingos97optimality}{\citeauthoryear {Domingos and Pazzani}{Domingos and Pazzani}{1997}}
\bibcite{elkan01}{\citeauthoryear {Elkan}{Elkan}{2001}}
\bibcite{fagan76}{\citeauthoryear {Fagan}{Fagan}{1976}}
\bibcite{fagan86}{\citeauthoryear {Fagan}{Fagan}{1986}}
\bibcite{fawcett01}{\citeauthoryear {Fawcett}{Fawcett}{2001}}
\bibcite{fenton94}{\citeauthoryear {Fenton, Pfleeger, and Glass}{Fenton et\nobreakspace  {}al\unhbox \voidb@x \hbox {.}}{1994}}
\bibcite{fenton99}{\citeauthoryear {Fenton and Neil}{Fenton and Neil}{1999}}
\bibcite{fenton96}{\citeauthoryear {Fenton and Pfleeger}{Fenton and Pfleeger}{1995}}
\bibcite{fenton97}{\citeauthoryear {Fenton and Pfleeger}{Fenton and Pfleeger}{1997}}
\bibcite{FreSch97}{\citeauthoryear {Freund and Schapire}{Freund and Schapire}{1997}}
\bibcite{hall00}{\citeauthoryear {Hall and Munson}{Hall and Munson}{2000}}
\bibcite{halstead77}{\citeauthoryear {Halstead}{Halstead}{1977}}
\bibcite{huang05}{\citeauthoryear {Huang and Ling}{Huang and Ling}{2005}}
\bibcite{jiang08b}{\citeauthoryear {Jiang, Cukic, and Ma}{Jiang et\nobreakspace  {}al\unhbox \voidb@x \hbox {.}}{2008}}
\bibcite{jiang07}{\citeauthoryear {Jiang, Cukic, and Menzies}{Jiang et\nobreakspace  {}al\unhbox \voidb@x \hbox {.}}{2007}}
\bibcite{jiang08a}{\citeauthoryear {Jiang, Cukic, and Menzies}{Jiang et\nobreakspace  {}al\unhbox \voidb@x \hbox {.}}{2008}}
\bibcite{khoshgoftaar01}{\citeauthoryear {Khoshgoftaar}{Khoshgoftaar}{2001}}
\bibcite{khoshgoftaar99}{\citeauthoryear {Khoshgoftaar and Allen}{Khoshgoftaar and Allen}{2001}}
\bibcite{journals/ese/KhoshgoftaarS03}{\citeauthoryear {Khoshgoftaar and Seliya}{Khoshgoftaar and Seliya}{2003}}
\bibcite{koru07}{\citeauthoryear {Koru, Zhang, and Liu}{Koru et\nobreakspace  {}al\unhbox \voidb@x \hbox {.}}{2007}}
\bibcite{lessmann09}{\citeauthoryear {Lessmann, Baesens, Mues, and Pietsch}{Lessmann et\nobreakspace  {}al\unhbox \voidb@x \hbox {.}}{2008}}
\bibcite{leven95}{\citeauthoryear {Leveson}{Leveson}{1995}}
\bibcite{littlewood97}{\citeauthoryear {Littlewood and Wright}{Littlewood and Wright}{1997}}
\bibcite{lowrey98}{\citeauthoryear {Lowry, Boyd, and Kulkarni}{Lowry et\nobreakspace  {}al\unhbox \voidb@x \hbox {.}}{1998}}
\bibcite{lutz03}{\citeauthoryear {Lutz and Mikulski}{Lutz and Mikulski}{2003}}
\bibcite{mccabe76}{\citeauthoryear {McCabe}{McCabe}{1976}}
\bibcite{me00p}{\citeauthoryear {Menzies}{Menzies}{2001}}
\bibcite{me02c}{\citeauthoryear {Menzies, Chiang, Feather, Hu, and Kiper}{Menzies et\nobreakspace  {}al\unhbox \voidb@x \hbox {.}}{2003}}
\bibcite{me99q}{\citeauthoryear {Menzies and Cukic}{Menzies and Cukic}{2000}}
\bibcite{me00y}{\citeauthoryear {Menzies and Cukic}{Menzies and Cukic}{2002}}
\bibcite{me07e}{\citeauthoryear {Menzies, Dekhtyar, Distefano, and Greenwald}{Menzies et\nobreakspace  {}al\unhbox \voidb@x \hbox {.}}{2007}}
\bibcite{me04g}{\citeauthoryear {Menzies, DiStefano, Orrego, and Chapman}{Menzies et\nobreakspace  {}al\unhbox \voidb@x \hbox {.}}{2004}}
\bibcite{me02e}{\citeauthoryear {Menzies, DiStefeno, Chapman, and Mcgill}{Menzies et\nobreakspace  {}al\unhbox \voidb@x \hbox {.}}{2002}}
\bibcite{me07b}{\citeauthoryear {Menzies, Greenwald, and Frank}{Menzies et\nobreakspace  {}al\unhbox \voidb@x \hbox {.}}{2007}}
\bibcite{me02g}{\citeauthoryear {Menzies, Lutz, and Mikulski}{Menzies et\nobreakspace  {}al\unhbox \voidb@x \hbox {.}}{2003}}
\bibcite{me05d}{\citeauthoryear {Menzies, Port, Chen, Hihn, and Stukes}{Menzies et\nobreakspace  {}al\unhbox \voidb@x \hbox {.}}{2005}}
\bibcite{me02f}{\citeauthoryear {Menzies, Raffo, on\nobreakspace  {}Setamanit, Hu, and Tootoonian}{Menzies et\nobreakspace  {}al\unhbox \voidb@x \hbox {.}}{2002}}
\bibcite{me03l}{\citeauthoryear {Menzies and Singh}{Menzies and Singh}{2003}}
\bibcite{me02k}{\citeauthoryear {Menzies and Stefano}{Menzies and Stefano}{2003a}}
\bibcite{me03a}{\citeauthoryear {Menzies, Stefano, Ammar, McGill, Callis, Chapman, and J}{Menzies et\nobreakspace  {}al\unhbox \voidb@x \hbox {.}}{2003}}
\bibcite{me03k}{\citeauthoryear {Menzies, Stefano, and Chapman}{Menzies et\nobreakspace  {}al\unhbox \voidb@x \hbox {.}}{2003}}
\bibcite{me03q}{\citeauthoryear {Menzies and Stefano}{Menzies and Stefano}{2003b}}
\bibcite{me04c}{\citeauthoryear {Menzies, Stefano, Cunanan, and Chapman}{Menzies et\nobreakspace  {}al\unhbox \voidb@x \hbox {.}}{2004}}
\bibcite{me08d}{\citeauthoryear {Menzies, Turhan, Bener, Gay, Cukic, and Jiang}{Menzies et\nobreakspace  {}al\unhbox \voidb@x \hbox {.}}{2008}}
\bibcite{milton08}{\citeauthoryear {Milton}{Milton}{2008}}
\bibcite{musa87}{\citeauthoryear {Musa, Iannino, and Okumoto}{Musa et\nobreakspace  {}al\unhbox \voidb@x \hbox {.}}{1987}}
\bibcite{nagappan05}{\citeauthoryear {Nagappan and Ball}{Nagappan and Ball}{2005a}}
\bibcite{conf/icse/NagappanB05a}{\citeauthoryear {Nagappan and Ball}{Nagappan and Ball}{2005b}}
\bibcite{nach08}{\citeauthoryear {Nagappan, Murphy, and V}{Nagappan et\nobreakspace  {}al\unhbox \voidb@x \hbox {.}}{2008}}
\bibcite{nikora04}{\citeauthoryear {Nikora}{Nikora}{2004}}
\bibcite{nikora03}{\citeauthoryear {Nikora and Munson}{Nikora and Munson}{2003}}
\bibcite{porter90}{\citeauthoryear {Porter and Selby}{Porter and Selby}{1990}}
\bibcite{pugh90}{\citeauthoryear {Pugh}{Pugh}{1990}}
\bibcite{quinlan92}{\citeauthoryear {Quinlan}{Quinlan}{1992}}
\bibcite{raffo05}{\citeauthoryear {Raffo}{Raffo}{2005}}
\bibcite{rakitin01}{\citeauthoryear {Rakitin}{Rakitin}{2001}}
\bibcite{shepperd94}{\citeauthoryear {Shepperd and Ince}{Shepperd and Ince}{1994}}
\bibcite{shu02}{\citeauthoryear {Shull, ad\nobreakspace  {}B.\nobreakspace  {}Boehm, Brown, Costa, Lindvall, Port, Rus, Tesoriero, and Zelkowitz}{Shull et\nobreakspace  {}al\unhbox \voidb@x \hbox {.}}{2002}}
\bibcite{shull00a}{\citeauthoryear {Shull, Rus, and Basili}{Shull et\nobreakspace  {}al\unhbox \voidb@x \hbox {.}}{2000}}
\bibcite{srinivasan95}{\citeauthoryear {Srinivasan and Fisher}{Srinivasan and Fisher}{1995}}
\bibcite{conf/ictai/TangK04}{\citeauthoryear {Tang and Khoshgoftaar}{Tang and Khoshgoftaar}{2004}}
\bibcite{tiang95}{\citeauthoryear {Tian and Zelkowitz}{Tian and Zelkowitz}{1995}}
\bibcite{me09b}{\citeauthoryear {Turhan, Menzies, Bener, and Distefano}{Turhan et\nobreakspace  {}al\unhbox \voidb@x \hbox {.}}{2009}}
\bibcite{voas95}{\citeauthoryear {Voas and Miller}{Voas and Miller}{1995}}
\bibcite{weyeuker08}{\citeauthoryear {Weyuker, Ostrand, and Bell}{Weyuker et\nobreakspace  {}al\unhbox \voidb@x \hbox {.}}{2008}}
\bibcite{williams03}{\citeauthoryear {Williams, Gomes, and Selman}{Williams et\nobreakspace  {}al\unhbox \voidb@x \hbox {.}}{2003}}
\bibcite{witten05}{\citeauthoryear {Witten and Frank}{Witten and Frank}{2005}}
\bibcite{me08h}{\citeauthoryear {Y.\nobreakspace  {}Jiang and Menzies}{Y.\nobreakspace  {}Jiang and Menzies}{2008}}
\bibcite{yang06}{\citeauthoryear {Yang, Webb, Cerquides, Korb, Boughton, and Ting}{Yang et\nobreakspace  {}al\unhbox \voidb@x \hbox {.}}{2006}}
\citation{witten05}
\citation{witten05}
\citation{domingos97optimality}
\citation{domingos97optimality}
\citation{quinlan92}
\citation{witten05}
\citation{quinlan92}
\citation{cohen95r}
\citation{dietterich97}
\citation{halstead77}
\citation{mccabe76}
\citation{fenton96}
\citation{fenton96}
\citation{fenton96}
\citation{demsar06}
\citation{demsar06,huang05}
\citation{demsar06}
\newlabel{@lastpg}{{7}{30}}