Does X cause Y? An in-depth evidence review var siteUrl = 'https://www.cold-takes.com'; var localTheme = localStorage.getItem('dawn\_theme'); switch (localTheme) { case 'dark': document.documentElement.classList.add('theme-dark'); break; case 'light': document.documentElement.classList.add('theme-light'); break; default: break; } { "@context": "https://schema.org", "@type": "Article", "publisher": { "@type": "Organization", "name": "Cold Takes", "url": "https://www.cold-takes.com/", "logo": { "@type": "ImageObject", "url": "https://www.cold-takes.com/favicon.ico", "width": 48, "height": 48 } }, "author": { "@type": "Person", "name": "Holden Karnofsky", "image": { "@type": "ImageObject", "url": "https://www.cold-takes.com/content/images/2021/07/HK-headshot-current---compressed-1.jpg", "width": 1600, "height": 1067 }, "url": "https://www.cold-takes.com/author/holden/", "sameAs": [ "https://www.cold-takes.com/about/" ] }, "headline": "Does X cause Y? An in-depth evidence review", "url": "https://www.cold-takes.com/does-x-cause-y-an-in-depth-evidence-review/", "datePublished": "2021-07-28T18:31:28.000Z", "dateModified": "2021-10-30T19:58:29.000Z", "image": { "@type": "ImageObject", "url": "https://www.cold-takes.com/content/images/2021/07/does-x-cause-y-wider-2.jpg", "width": 1500, "height": 642 }, "description": "

There's an interesting theory out there that X causes Y. If this were true, it
would be pretty important. So I did a deep-dive into the academic literature on
whether X causes Y. Here's what I found.

(Embarrassingly, I can't actually remember what X and Y are. I think maybe X was
enriched preschool, or just school itself, or eating fish while pregnant, or the
Paleo diet, or lead exposure, or a clever "nudge" policy trying to get people to
save more, or some self-help technique, or some micron", "mainEntityOfPage": { "@type": "WebPage", "@id": "https://www.cold-takes.com/" } } .gh-post-upgrade-cta-content, .gh-post-upgrade-cta { display: flex; flex-direction: column; align-items: center; font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, Oxygen, Ubuntu, Cantarell, 'Open Sans', 'Helvetica Neue', sans-serif; text-align: center; width: 100%; color: #ffffff; font-size: 16px; } .gh-post-upgrade-cta-content { border-radius: 8px; padding: 40px 4vw; } .gh-post-upgrade-cta h2 { color: #ffffff; font-size: 28px; letter-spacing: -0.2px; margin: 0; padding: 0; } .gh-post-upgrade-cta p { margin: 20px 0 0; padding: 0; } .gh-post-upgrade-cta small { font-size: 16px; letter-spacing: -0.2px; } .gh-post-upgrade-cta a { color: #ffffff; cursor: pointer; font-weight: 500; box-shadow: none; text-decoration: underline; } .gh-post-upgrade-cta a:hover { color: #ffffff; opacity: 0.8; box-shadow: none; text-decoration: underline; } .gh-post-upgrade-cta a.gh-btn { display: block; background: #ffffff; text-decoration: none; margin: 28px 0 0; padding: 8px 18px; border-radius: 4px; font-size: 16px; font-weight: 600; } .gh-post-upgrade-cta a.gh-btn:hover { opacity: 0.92; } .shaded-table { color: #000000; border: 1px solid; vertical-align: top; } .shaded-table a { color: #0000dd; } .sticky-subscribe-div { z-index: 1000; position: fixed; bottom: 25px; right: 25px; max-width: 500px; overflow: hidden; opacity: 1; } .sticky-subscribe-button { cursor: pointer; padding: 20px; background: var(--brand-color); color: var(--white-color); border-radius: 50px; border-style: none; } .subscribe-overlay { position: fixed; top: 0; right: 0; bottom: 0; left: 0; z-index: 2000; display: flex; justify-content: center; align-items: center; background: var(--white-color); opacity: 0; transition: opacity 0.3s ease-in; pointer-events: none; } .subscribe-overlay:target { z-index: 2001; opacity: 0.95; pointer-events: auto; } .subscribe-form { margin: 1.5em 0; padding: 6.5vh 7vw 8vh; border-style: solid !important; border: var(--mid-gray-color); border-radius: 10px; text-align: center; } .subscribe-overlay-title { display: inline-block; margin: 0 0 10px 0; font-size: 3rem; line-height: 1.15em; color: var(--darker-gray-color); } .subscribe-overlay-description { margin: 0 auto 50px; max-width: 500px; color: var(--darker-gray-color); font-size: 2rem; line-height: 1.3em; opacity: 0.8; } .subscribe-form .form-group { align-self: stretch; display: flex; } .subscribe-email { padding: 10px; border: 1px solid; border-color: var(--darker-gray-color); font-size: 1.8rem; line-height: 1em; font-weight: normal; user-select: text; transition: border-color 0.15s linear; -webkit-appearance: none; } .subscribe-form .subscribe-overlay-button { word-break: normal; position: relative; display: inline-block; margin: 0 0 0 10px; padding: 0 20px; height: 43px; outline: none; font-size: 1.5rem; line-height: 39px; font-weight: 400; background: var(--brand-color); border-radius: 5px; color: var(--white-color); } .subscribe-form .button-loader, .subscribe-form .message-success, .subscribe-form .message-error { display: none; } .subscribe-form .loading .button-content { visibility: hidden; } .subscribe-form .loading .button-loader { position: absolute; top: 0; left: 50%; display: inline-block; margin-left: -19px; transform: scale(0.7); } .subscribe-overlay .loading .button-loader { top: 5px; } .subscribe-form .button-loader svg path, .subscribe-form .button-loader svg rect { fill: var(--white-color); } .subscribe-form .success .message-success, .subscribe-form .invalid .message-error, .subscribe-form .error .message-error { display: block; margin: 1em auto 0; max-width: 400px; color: var(--red-color); font-size: 1.6rem; line-height: 1.5em; text-align: center; } .subscribe-form .success .message-success { display: block; color: var(--darker-gray-color); } .subscribe-form .invalid .message-error, .subscribe-form .error .message-error { display: block; } @media (max-width: 650px) { .subscribe-form-title { font-size: 2.4rem; } .subscribe-form-description { font-size: 1.6rem; } } @media (max-width: 500px) { .subscribe-form form { display: flex; flex-direction: column; } .subscribe-form .subscribe-overlay-button { margin: 0 0 0 0; } .subscribe-form .form-group { display: flex; flex-direction: column; width: 100%; } .subscribe-form button { display: flex; margin: 0 0 0 0; width: 100%; } } .subscribe-close-button { position: absolute; top: 16px; right: 20px; z-index: 2000; display: block; width: 40px; height: 40px; } .subscribe-close-button:before { content: ""; position: absolute; top: 20px; right: 4px; display: block; width: 32px; height: 1px; background: var(--darker-gray-color); opacity: 0.8; transform: rotate(45deg); } .subscribe-close-button:after { content: ""; position: absolute; top: 20px; right: 4px; display: block; width: 32px; height: 1px; background: var(--darker-gray-color); opacity: 0.8; transform: rotate(-45deg); } .subscribe-close-button:hover { cursor: default; } /\*.message-success, .message-error, .message-loading { display: none; }\*/ /\*.message-success .success { visibility: visible; display: block; opacity: 1; } .message-error.error { visibility: visible; display: block; opacity: 1; }\*/ /\*.success .message-success, .error .message-error, .loading .message-loading { visibility: visible; display: block; opacity: 1; }\*/ /\*For TOC\*/ @media (min-width: 1200px) { .sticky-visible .sticky-toc { visibility: visible; opacity: 1; transform: translateY(0); } .toc-opened .sticky-toc { visibility: hidden; } } /\*.gh-portal-triggerbtn-iframe { } .gh-portal-triggerbtn-wrapper { background: #0000ff !important; } .gh-portal-triggerbtn-container { background: #0000ff !important; height: 10px !important; } \*/ .theme-dark:root { \--green-color: #66ff33 !important; \--orange-color: #ff5c33 !important; \--red-color: #ff3333 !important; \--purple-color: #ff00ff !important; \--brand-color: #5ceaff !important; \--white-color: #18141f !important; \--primary-text-color: #fff !important; \--secondary-text-color: #fff !important; } .feed-visibility-false { display: none; } /\*.feed-visibility-true { display: none; }\*/ :root { \--brand-color:#0000dd; \--white-color:#ffffff; \--primary-text-color:#1a1a1a; \--secondary-text-color:#1a1a1a; \--orange-color: #ff6600; \--purple-color: #cc33ff; } blockquote { font-weight: unset; letter-spacing: normal; font-style: normal; text-align: inherit; } table { border: 1px solid; border-collapse: collapse; white-space: unset !important; width=200px !important; } table td { border: 1px solid; border-collapse: collapse; } .theme-dark .CT-thumb { transform: translate(20px,0px); } .theme-light .CT-thumb { transform: translate(0px,0px); } /\*.u-text-format h3 { color: var(--darker-gray-color) ; }\*/ //Detect theme var existingTheme = localStorage.getItem('dawn\_theme'); if (!existingTheme) { if(navigator.platform.indexOf('Win') > -1) { existingTheme = "light"; } else if(window.matchMedia && window.matchMedia('(prefers-color-scheme: dark)').matches) { existingTheme = "dark"; } else if(window.matchMedia && window.matchMedia('(prefers-color-scheme: light)').matches) { existingTheme = "light"; } } //Implement light/dark theme if(existingTheme == "dark") { ctdark(); } else if(existingTheme == 'light') { ctlight(); } window.dataLayer = window.dataLayer || []; function gtag(){dataLayer.push(arguments);} gtag('js', new Date()); gtag('config', 'G-TZK0WDB1DH'); var clicky\_site\_ids = clicky\_site\_ids || []; clicky\_site\_ids.push(101323562); ![Clicky](//in.getclicky.com/101323562ns.gif) :root {--ghost-accent-color: #888888;} [Cold Takes](https://www.cold-takes.com) [About (+highlights)](https://www.cold-takes.com/about/) [Most Important Century](https://www.cold-takes.com/most-important-century/) [About (+highlights)](https://www.cold-takes.com/about/) [Most Important Century](https://www.cold-takes.com/most-important-century/) [Subscribe (free)](#subscribe) [Subscribe (free)](#subscribe) [Subscribe (free)](#subscribe) [Cold Takes](https://www.cold-takes.com) - Does X cause Y? An in-depth evidence review By [Holden Karnofsky](https://www.cold-takes.com/about/ "Holden Karnofsky") Jul 28, 2021 9 min read Does X cause Y? An in-depth evidence review =========================================== ![](https://www.cold-takes.com/content/images/2021/07/does-x-cause-y-1.jpg) There's an interesting theory out there that X causes Y. If this were true, it would be pretty important. So I did a deep-dive into the academic literature on whether X causes Y. Here's what I found. (Embarrassingly, I can't actually remember what X and Y are. I think maybe X was enriched preschool, or just school itself, or eating fish while pregnant, or the Paleo diet, or lead exposure, or a clever "nudge" policy trying to get people to save more, or some self-help technique, or some micronutrient or public health intervention, or democracy, or free trade, or some approach to intellectual property law. And Y was ... lifetime earnings, or risk of ADHD diagnosis, or IQ in adulthood, or weight loss, or violent crime, or peaceful foreign policy, or GDP per capita, or innovation. Sorry about that! Hope you enjoy the post anyway! Fortunately, I think **what I'm about to write is correct for pretty much any (X,Y) from those sorts of lists.**) In brief: * There are hundreds of studies on whether X causes Y, but most of them are simple observational studies that are just essentially saying "People/countries with more X also have more Y." For reasons discussed below, we can't really learn much from these studies. * There are 1-5 more interesting studies on whether X causes Y. Each study looks really clever, informative and rigorous at first glance. However, the more closely you look at them, the more confusing the picture gets. * We ultimately need to choose between (a) believing some overly complicated theory of the relationship between X and Y, which reconciles all of the wildly conflicting and often implausible things we're seeing in the studies; (b) more-or-less reverting to what we would've guessed about the relationship between X and Y in the absence of any research. The chaff: lots of unhelpful studies that I'm disregarding \---------------------------------------------------------- First, the good news: there are hundreds of studies on whether X causes Y. The bad news? We need to throw most of them out. Many have comically small sample sizes (like studying 20 people) and/or comically short time horizons (like looking at weight loss over two weeks),[1](#fn1) or unhelpful outcome measures (like intelligence tests in children under 5).[2](#fn2) But by far the most common problem is that **most of the studies on whether X causes Y are simple observational studies: they essentially just find that people/countries with more X also have more Y.** Why is this a problem? There could be a *confounder* - some third thing, Z, that is correlated with both X and Y. And there are specific reasons we should expect confounders to be common: * In general, people/countries that have more X also have more of lots of other helpful things - they're richer, they're more educated, etc. For example, if we're asking whether higher-quality schooling leads to higher earnings down the line, an issue is that people with higher-quality schooling also tend to come from better-off families with lots of other advantages. * In fact, the *very fact that people in upper-class intellectual circles think X causes Y* means that richer, more educated people/countries tend to deliberately get more X, and also try to do a lot of other things to get more Y. For example, more educated families tend to eat more fish (complicating the attempt to see whether eating fish in pregnancy is good for the baby).[3](#fn3) Now, a lot of these studies try to "control for" the problem I just stated - they say things like "We examined the effect of X and Y, while controlling for Z [e.g., how wealthy or educated the people/countries/whatever are]." How do they do this? The short answer is, well, hm, jeez. Well you see, to simplify matters a bit, just try to imagine ... uh ... shit. Uh. The only high-level way I can put this is: * They use a technique called [regression analysis](https://en.wikipedia.org/wiki/Regression_analysis) that, as far as I can determine, cannot be explained in a simple, intuitive way (especially not in terms of how it "controls for" confounders). * The "controlling for" thing relies on a lot of subtle assumptions and can break in all kinds of weird ways. [Here's](https://journals.plos.org/plosone/article?id=10.1371/journal.pone.<PHONE>) a technical explanation of some of the pitfalls; [here's](https://www.cold-takes.com/phil-birnbaums-regression-analysis/) a set of deconstructions of regressions that break in weird ways. * None of the observational studies about whether X causes Y discuss the pitfalls of "controlling for" things and whether they apply here. I don't think we can trust these papers, and to really pick them all apart (given how many there are) would take too much time. So let's focus on a smaller number of better studies. The wheat: 1-5 more interesting studies \--------------------------------------- Digging through the sea of unhelpful studies, I found 1-5 of them that are actually really interesting! For example, one study examines some strange historical event you've never heard of (perhaps a [surge in Cuban emigration triggered by Fidel Castro suddenly allowing it](https://davidcard.berkeley.edu/papers/mariel-impact.pdf), or [John Rockefeller's decision to fund a hookworm eradication campaign](https://www.ncbi.nlm.nih.gov/pmc/articles/PMC3800113/), or a [sudden collective pardon leading to release of a third of prison inmates in Italy](https://www.aeaweb.org/articles?id=10.1257/aer.103.6.2437)), where for abstruse and idiosyncratic reasons, X got distributed in what seems to be almost a random way. This study is really clever, and the authors were incredibly thorough in examining seemingly every way their results could have been wrong. They conclude that X causes Y! But on closer inspection, I have a bunch of reservations. For example: * The paper doesn't make it easy to replicate its analysis, and when someone does manage to sort-of replicate it, they may [get different results](https://blog.givewell.org/2017/12/07/questioning-evidence-hookworm-eradication-american-south/). * There was other weird stuff going on (e.g., changes in census data collection methods[5](#fn5)), during the strange historical event, so it's a little hard to generalize. * In a response to the study, another academic advances a complex theory of how the study could actually have gotten a misleading result. This led to an intense back-and-forth between the original authors and the skeptic, stretched out over years because each response had to be published in a journal, and by the time I got to the end of it I didn't have any idea what to think anymore.[6](#fn6) I found 0-4 other interesting studies. I can't remember all of the details, but they may have included: * A study comparing siblings, or maybe "very similar countries," that got more or less of X.[7](#fn7)* A study using a complex mathematical technique claiming to cleanly isolate the effect of X and Y. I can't really follow what it's doing, and I'm guessing there are a lot of weird assumptions baked into this analysis.[8](#fn8)* A study with actual randomization: some people were randomly assigned to receive more X than others, and the researchers looked at who ended up with more Y. This sounds awesome! However, there are issues here too: \+ It's kind of ambiguous whether the assignment to X was really "random."[9](#fn9)+ Extremely weird things happened during the study (for example, [generational levels of flooding](https://www.givewell.org/international/technical/programs/deworming/reanalysis#Our_initial_reservations)), so it's not clear how well it generalizes to other settings. \+ The result seems fragile ([simply adding more data weakens it a lot](https://www.givewell.org/international/technical/programs/deworming#Other_studies_of_long-term_effects)) and/or just hard to believe (like [schoolchildren doing noticeably better on a cognition test after a few weeks of being given fish instead of meat with their lunch, even though they mostly didn't eat the fish](https://nutritionj.biomedcentral.com/articles/10.1186/s12937-<PHONE>-9)). Compounding the problem, the 1-5 studies I found tell very different stories about the relationship between X and Y. How could this make sense? Is there a unified theory that can reconcile all the results? Well, one possibility is that X causes Y sometimes, but only under very particular conditions, and the effect can be masked by some other thing going on. So - if you meet one of 7 criteria, you should do X to get more Y, but if you meet one of 9 other criteria, you should actually avoid X! Conclusion \---------- I have to say, this all was simultaneously more fascinating and less *informative* than I expected it would be going in. I thought I would find some nice studies about the relationship between X and Y and be done. Instead, I've learned a ton about weird historical events and about the ins and outs of different measures of X and Y, but I feel just super confused about whether X causes Y. I guess my bottom line is that X does cause Y, because it intuitively seems like it would. I'm glad I did all this research, though. It's good to know that social science research can go haywire in all kinds of strange ways. And it's good to know that despite the confident proclamations of pro- and anti-X people, it's legitimately just super unclear whether X causes Y. I mean, how else could I have learned that? Appendix: based on a true story \------------------------------- This piece was inspired by: * Most evidence reviews GiveWell has done, especially of [deworming](https://www.givewell.org/international/technical/programs/deworming)* Many evidence reviews by David Roodman, particularly [Macro Aid Effectiveness Research: a Guide for the Perplexed](https://www.cgdev.org/publication/macro-aid-effectiveness-research-guide-perplexed-working-paper-135); [Due Diligence: an Impertinent Inquiry into Microfinance](https://smile.amazon.com/dp/B0095V3J0S/); and [Reasonable Doubt: A New Look at Whether Prison Growth Cuts Crime](https://www.openphilanthropy.org/blog/reasonable-doubt-new-look-whether-prison-growth-cuts-crime). * Many evidence reviews by Slate Star Codex, collected [here](https://slatestarcodex.com/tag/much-more-than-you-wanted-to-know/). * Informal evidence reviews I've done for e.g. personal medical decisions. The basic patterns above apply to most of these, and the bottom line usually has the kind of frustrating ambiguity seen in this conclusion. There are cases where things seem a bit less ambiguous and the bottom line seems clearer. Speaking broadly, I think the main things that contribute to this are: * Actual randomization. For years I've nodded along when people say "You shouldn't be dogmatic about randomization, there are many ways for a study to be informative," but each year I've become a bit more dogmatic. Even the most sophisticated-, appealing-seeming alternatives to randomization in studies seem to have a way of falling apart. Randomized studies almost always have problems and drawbacks too. But I'd rather have a randomized study with drawbacks than a non-randomized study with drawbacks. * Extreme thoroughness, such as Roodman's attempt to reconstruct the data and code for key studies in [Reasonable Doubt](https://www.openphilanthropy.org/blog/reasonable-doubt-new-look-whether-prison-growth-cuts-crime). This sometimes leads to outright dismissing a number of studies, leaving a smaller, more consistent set remaining. [![Twitter](https://www.cold-takes.com/content/images/2021/06/ct-twitter-square.png)](https://api.addthis.com/oexchange/0.8/forward/twitter/offer?url=https%3A%2F%2Fwww.cold-takes.com%2Fdoes-x-cause-y-an-in-depth-evidence-review%2F&pubid;=ra-60a178324cffc42e&title;=Cold%20Takes%20-%20Does%20X%20cause%20Y%3F%20An%20in-depth%20evidence%20review&ct;=1) [![Facebook](https://www.cold-takes.com/content/images/2021/06/ct-facebook-square.png)](https://api.addthis.com/oexchange/0.8/forward/facebook/offer?url=https%3A%2F%2Fwww.cold-takes.com%2Fdoes-x-cause-y-an-in-depth-evidence-review%2F&pubid;=ra-60a178324cffc42e&title;=Cold%20Takes%20-%20Does%20X%20cause%20Y%3F%20An%20in-depth%20evidence%20review&ct;=1) [![Reddit](https://www.cold-takes.com/content/images/2021/06/ct-reddit-square.png)](https://api.addthis.com/oexchange/0.8/forward/reddit/offer?url=https%3A%2F%2Fwww.cold-takes.com%2Fdoes-x-cause-y-an-in-depth-evidence-review%2F&pubid;=ra-60a178324cffc42e&title;=Cold%20Takes%20-%20Does%20X%20cause%20Y%3F%20An%20in-depth%20evidence%20review&ct;=1) [![More](https://www.cold-takes.com/content/images/2021/06/ct-addthis-square.png)](https://api.addthis.com/oexchange/0.8/forward/menu/offer?url=https%3A%2F%2Fwww.cold-takes.com%2Fdoes-x-cause-y-an-in-depth-evidence-review%2F&pubid;=ra-60a178324cffc42e&title;=Cold%20Takes%20-%20Does%20X%20cause%20Y%3F%20An%20in-depth%20evidence%20review&ct;=1) [Subscribe](https://www.cold-takes.com/does-x-cause-y-an-in-depth-evidence-review/#/portal/signup/free) [Feedback](https://www.guidedtrack.com/programs/4kal2ue/run?posttitle=Does%20X%20cause%20Y%3F%20An%20in-depth%20evidence%20review) \--- Footnotes \--------- 1\. Both of these show up in studies from [this review on the Paleo diet](https://www.ncbi.nlm.nih.gov/pmc/articles/PMC4588744/). To be fair, small studies can theoretically be aggregated for larger numbers, but that's often hard to do in practice when the studies are all a bit different from each other. [↩](#fnref1)- I don't have a great cite for this, but it's pretty common in studies on things like how in vitro fertilization affects child development. [↩](#fnref2)- See studies cited in [this literature review](https://pubmed.ncbi.nlm.nih.gov/31739098/). [↩](#fnref3)- (Footnote deleted) \- "Borjas's paper ... separately measured the wages of two slices of that larger group ... But it was in that act of slicing the data that the spurious result was generated. It created data samples that, exactly in 1980, suddenly included far more low-wage black males—accounting for the whole wage decline in those samples relative to other cities. Understanding how that happened requires understanding the raw data ... Right in 1980, the Census Bureau—which ran the CPS surveys—improved its survey methods to cover more low-skill black men. The 1970 census and again the 1980 census had greatly undercounted low-skill black men, both by failing to identify their residences and by failing to sufficiently probe survey respondents about marginal or itinerant household members. There was massive legislative and judicial pressure to count blacks better, particularly in Miami." [↩](#fnref5)- E.g., the [Mariel boatlift debate](https://davidroodman.com/blog/2017/05/25/four-points-on-the-debate-over-the-impact-of-mariel-boatlift/). [↩](#fnref6)- For example, sibling analysis features prominently in [Slate Star Codex's examination of preschool impacts](https://slatestarcodex.com/2018/11/13/preschool-much-more-than-you-wanted-to-know/), while comparisons between Sweden and other Scandinavian/European countries is [prominent in its analysis of lockdowns](https://astralcodexten.substack.com/p/lockdown-effectiveness-much-more). [↩](#fnref7)- E.g., [this attempt to gauge the impacts of microfinance](https://blog.givewell.org/2011/05/13/microlending-debate-an-example-of-why-academic-research-should-be-used-with-caution/), or "[generalized method of moments](https://onlinelibrary.wiley.com/doi/full/10.1111/j.<PHONE>.2008.00542.x)" approaches to cross-country analysis (of e.g. the effectiveness of aid). [↩](#fnref8)- This is a surprisingly common issue. E.g. see debates over [whether charter school lotteries are really random](https://freddiedeboer.substack.com/p/charter-school-lotteries-are-a-black), whether ["random assignment to small or large class size" can be interpreted as "random assignment to a teacher,"](https://www.econlib.org/archives/2011/10/teachers_and_in.html) discussion of "judge randomization" and possible randomization failure [here](https://www.openphilanthropy.org/blog/reasonable-doubt-new-look-whether-prison-growth-cuts-crime) (particularly section 9.9). A separate issue: sometimes randomization occurs by "cluster" (instead of randomizing which *individuals* receive some treatment, perhaps particular *schools* or *groups* are chosen to receive it), which can complicate the analysis. [↩](#fnref9) Powered by [Ghost](https://ghost.org/) #### Popular tags [sports](/tag/sports/) [innovationstagnation](/tag/innovationstagnation/) [2star](/tag/2star/) [utopia](/tag/utopia/) Subscribe to Cold Takes ======================= For audio version, search for "Cold Takes Audio" in your podcast app Subscribe **Great!** Check your inbox and click the link to confirm your subscription. **If you don't see it, check your spam!** Please enter a valid email address! $("#lower-right-subscribe").click(function(f) { clicky.goal('1458'); /\*LO.events.track('Subscribe-LR');\*/ setTimeout(function(){$("#subscribe-input").focus();},500); }); $("#upper-right-subscribe-1").click(function(f) { clicky.goal('1471'); /\*LO.events.track('Subscribe-LR');\*/ setTimeout(function(){$("#subscribe-input").focus();},500); }); $("#upper-right-subscribe-2").click(function(f) { clicky.goal('1471'); /\*LO.events.track('Subscribe-LR');\*/ setTimeout(function(){$("#subscribe-input").focus();},500); }); $("#cover-subscribe").click(function(f) { clicky.goal('1472'); /\*LO.events.track('Subscribe-LR');\*/ setTimeout(function(){$("#subscribe-input").focus();},500); }); $("#footer-subscribe").click(function(f) { clicky.goal('1473'); /\*LO.events.track('Subscribe-LR');\*/ setTimeout(function(){$("#subscribe-input").focus();},500); }); $("#final-subscribe").click(function(f) { clicky.goal('1474'); }); //When someone clicks an anchor link, I want the sticky nav to go away. $("a").click(function(f) { body.removeClass('sticky-visible'); lastSt = 9999999999999}); function ctlight() { html.removeClass('theme-dark').addClass('theme-light'); localStorage.setItem('dawn\_theme', 'light'); //$(".gh-portal-triggerbtn-container").css("background","#0000dd"); /\* document.documentElement.style.setProperty('--brand-color', '#0000dd'); document.documentElement.style.setProperty('--white-color', '#ffffff'); document.documentElement.style.setProperty('--primary-text-color', '#1a1a1a'); document.documentElement.style.setProperty('--secondary-text-color', '#1a1a1a');\*/ // $("#CT-thumbID").css("transform","translate(0px,0)"); // $(".single-content").css("font-weight","400"); existingTheme = "light"; } function ctdark() { html.removeClass('theme-light').addClass('theme-dark'); localStorage.setItem('dawn\_theme', 'dark'); /\*document.documentElement.style.setProperty('--brand-color', '#5ceaff'); document.documentElement.style.setProperty('--white-color', '#18141f'); document.documentElement.style.setProperty('--primary-text-color', '#fff'); document.documentElement.style.setProperty('--secondary-text-color', '#fff');\*/ //document.documentElement.style.setProperty('--white-color', '#14101b'); //document.documentElement.style.setProperty('--white-color', '#121212'); //document.documentElement.style.setProperty('--white-color', '#282c35'); //$(".single-content").css("font-weight","300"); // $("#CT-thumbID").css("transform","translate(19px,0)"); existingTheme = "dark"; } //For light/dark toggle $(".CT-toggle").click(function(g) { g.preventDefault(); if(existingTheme == "dark") { ctlight(); } else if(existingTheme == 'light') { ctdark(); } } ); // Parse the URL parameter, then log goal if it is a completed subscription. Weird that it takes this much code to do this. function getParameterByName(name, url) { if (!url) url = window.location.href; name = name.replace(/[\[\]]/g, "\\$&"); var regex = new RegExp("[?&]" + name + "(=([^&#]\*)|&|#|$)"), results = regex.exec(url); if (!results) return null; if (!results[2]) return ''; return decodeURIComponent(results[2].replace(/\+/g, " ")); } // Give the parameter a variable name var action = getParameterByName('action'); var success = getParameterByName('success'); $(document).ready(function () { // Success === null is here for backwards compatibility if (action == 'subscribe' && (success === null || success == 'true')) { clicky.goal('1476'); } });

Does X cause Y? An in-depth evidence review var siteUrl = 'https://www.cold-takes.com'; var localTheme = localStorage.getItem('dawn\\_theme'); switch (localTheme) { case 'dark': document.documentElement.classList.add('theme-dark'); break; case 'light': document.documentElement.classList.add('theme-light'); break; default: break; } { "@context": "https://schema.org", "@type": "Article", "publisher": { "@type": "Organization", "name": "Cold Takes", "url": "https://www.cold-takes.com/", "logo": { "@type": "ImageObject", "url": "https://www.cold-takes.com/favicon.ico", "width": 48, "height": 48 } }, "author": { "@type": "Person", "name": "Holden Karnofsky", "image": { "@type": "ImageObject", "url": "https://www.cold-takes.com/content/images/2021/07/HK-headshot-current---compressed-1.jpg", "width": 1600, "height": 1067 }, "url": "https://www.cold-takes.com/author/holden/", "sameAs": [ "https://www.cold-takes.com/about/" ] }, "headline": "Does X cause Y? An in-depth evidence review", "url": "https://www.cold-takes.com/does-x-cause-y-an-in-depth-evidence-review/", "datePublished": "2021-07-28T18:31:28.000Z", "dateModified": "2021-10-30T19:58:29.000Z", "image": { "@type": "ImageObject", "url": "https://www.cold-takes.com/content/images/2021/07/does-x-cause-y-wider-2.jpg", "width": 1500, "height": 642 }, "description": "\n\nThere's an interesting theory out there that X causes Y. If this were true, it\nwould be pretty important. So I did a deep-dive into the academic literature on\nwhether X causes Y. Here's what I found.\n\n(Embarrassingly, I can't actually remember what X and Y are. I think maybe X was\nenriched preschool, or just school itself, or eating fish while pregnant, or the\nPaleo diet, or lead exposure, or a clever "nudge" policy trying to get people to\nsave more, or some self-help technique, or some micron", "mainEntityOfPage": { "@type": "WebPage", "@id": "https://www.cold-takes.com/" } } .gh-post-upgrade-cta-content, .gh-post-upgrade-cta { display: flex; flex-direction: column; align-items: center; font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, Oxygen, Ubuntu, Cantarell, 'Open Sans', 'Helvetica Neue', sans-serif; text-align: center; width: 100%; color: #ffffff; font-size: 16px; } .gh-post-upgrade-cta-content { border-radius: 8px; padding: 40px 4vw; } .gh-post-upgrade-cta h2 { color: #ffffff; font-size: 28px; letter-spacing: -0.2px; margin: 0; padding: 0; } .gh-post-upgrade-cta p { margin: 20px 0 0; padding: 0; } .gh-post-upgrade-cta small { font-size: 16px; letter-spacing: -0.2px; } .gh-post-upgrade-cta a { color: #ffffff; cursor: pointer; font-weight: 500; box-shadow: none; text-decoration: underline; } .gh-post-upgrade-cta a:hover { color: #ffffff; opacity: 0.8; box-shadow: none; text-decoration: underline; } .gh-post-upgrade-cta a.gh-btn { display: block; background: #ffffff; text-decoration: none; margin: 28px 0 0; padding: 8px 18px; border-radius: 4px; font-size: 16px; font-weight: 600; } .gh-post-upgrade-cta a.gh-btn:hover { opacity: 0.92; } .shaded-table { color: #000000; border: 1px solid; vertical-align: top; } .shaded-table a { color: #0000dd; } .sticky-subscribe-div { z-index: 1000; position: fixed; bottom: 25px; right: 25px; max-width: 500px; overflow: hidden; opacity: 1; } .sticky-subscribe-button { cursor: pointer; padding: 20px; background: var(--brand-color); color: var(--white-color); border-radius: 50px; border-style: none; } .subscribe-overlay { position: fixed; top: 0; right: 0; bottom: 0; left: 0; z-index: 2000; display: flex; justify-content: center; align-items: center; background: var(--white-color); opacity: 0; transition: opacity 0.3s ease-in; pointer-events: none; } .subscribe-overlay:target { z-index: 2001; opacity: 0.95; pointer-events: auto; } .subscribe-form { margin: 1.5em 0; padding: 6.5vh 7vw 8vh; border-style: solid !important; border: var(--mid-gray-color); border-radius: 10px; text-align: center; } .subscribe-overlay-title { display: inline-block; margin: 0 0 10px 0; font-size: 3rem; line-height: 1.15em; color: var(--darker-gray-color); } .subscribe-overlay-description { margin: 0 auto 50px; max-width: 500px; color: var(--darker-gray-color); font-size: 2rem; line-height: 1.3em; opacity: 0.8; } .subscribe-form .form-group { align-self: stretch; display: flex; } .subscribe-email { padding: 10px; border: 1px solid; border-color: var(--darker-gray-color); font-size: 1.8rem; line-height: 1em; font-weight: normal; user-select: text; transition: border-color 0.15s linear; -webkit-appearance: none; } .subscribe-form .subscribe-overlay-button { word-break: normal; position: relative; display: inline-block; margin: 0 0 0 10px; padding: 0 20px; height: 43px; outline: none; font-size: 1.5rem; line-height: 39px; font-weight: 400; background: var(--brand-color); border-radius: 5px; color: var(--white-color); } .subscribe-form .button-loader, .subscribe-form .message-success, .subscribe-form .message-error { display: none; } .subscribe-form .loading .button-content { visibility: hidden; } .subscribe-form .loading .button-loader { position: absolute; top: 0; left: 50%; display: inline-block; margin-left: -19px; transform: scale(0.7); } .subscribe-overlay .loading .button-loader { top: 5px; } .subscribe-form .button-loader svg path, .subscribe-form .button-loader svg rect { fill: var(--white-color); } .subscribe-form .success .message-success, .subscribe-form .invalid .message-error, .subscribe-form .error .message-error { display: block; margin: 1em auto 0; max-width: 400px; color: var(--red-color); font-size: 1.6rem; line-height: 1.5em; text-align: center; } .subscribe-form .success .message-success { display: block; color: var(--darker-gray-color); } .subscribe-form .invalid .message-error, .subscribe-form .error .message-error { display: block; } @media (max-width: 650px) { .subscribe-form-title { font-size: 2.4rem; } .subscribe-form-description { font-size: 1.6rem; } } @media (max-width: 500px) { .subscribe-form form { display: flex; flex-direction: column; } .subscribe-form .subscribe-overlay-button { margin: 0 0 0 0; } .subscribe-form .form-group { display: flex; flex-direction: column; width: 100%; } .subscribe-form button { display: flex; margin: 0 0 0 0; width: 100%; } } .subscribe-close-button { position: absolute; top: 16px; right: 20px; z-index: 2000; display: block; width: 40px; height: 40px; } .subscribe-close-button:before { content: ""; position: absolute; top: 20px; right: 4px; display: block; width: 32px; height: 1px; background: var(--darker-gray-color); opacity: 0.8; transform: rotate(45deg); } .subscribe-close-button:after { content: ""; position: absolute; top: 20px; right: 4px; display: block; width: 32px; height: 1px; background: var(--darker-gray-color); opacity: 0.8; transform: rotate(-45deg); } .subscribe-close-button:hover { cursor: default; } /\\*.message-success, .message-error, .message-loading { display: none; }\\*/ /\\*.message-success .success { visibility: visible; display: block; opacity: 1; } .message-error.error { visibility: visible; display: block; opacity: 1; }\\*/ /\\*.success .message-success, .error .message-error, .loading .message-loading { visibility: visible; display: block; opacity: 1; }\\*/ /\\*For TOC\\*/ @media (min-width: 1200px) { .sticky-visible .sticky-toc { visibility: visible; opacity: 1; transform: translateY(0); } .toc-opened .sticky-toc { visibility: hidden; } } /\\*.gh-portal-triggerbtn-iframe { } .gh-portal-triggerbtn-wrapper { background: #0000ff !important; } .gh-portal-triggerbtn-container { background: #0000ff !important; height: 10px !important; } \\*/ .theme-dark:root { \--green-color: #66ff33 !important; \--orange-color: #ff5c33 !important; \--red-color: #ff3333 !important; \--purple-color: #ff00ff !important; \--brand-color: #5ceaff !important; \--white-color: #18141f !important; \--primary-text-color: #fff !important; \--secondary-text-color: #fff !important; } .feed-visibility-false { display: none; } /\\*.feed-visibility-true { display: none; }\\*/ :root { \--brand-color:#0000dd; \--white-color:#ffffff; \--primary-text-color:#1a1a1a; \--secondary-text-color:#1a1a1a; \--orange-color: #ff6600; \--purple-color: #cc33ff; } blockquote { font-weight: unset; letter-spacing: normal; font-style: normal; text-align: inherit; } table { border: 1px solid; border-collapse: collapse; white-space: unset !important; width=200px !important; } table td { border: 1px solid; border-collapse: collapse; } .theme-dark .CT-thumb { transform: translate(20px,0px); } .theme-light .CT-thumb { transform: translate(0px,0px); } /\\*.u-text-format h3 { color: var(--darker-gray-color) ; }\\*/ //Detect theme var existingTheme = localStorage.getItem('dawn\\_theme'); if (!existingTheme) { if(navigator.platform.indexOf('Win') > -1) { existingTheme = "light"; } else if(window.matchMedia && window.matchMedia('(prefers-color-scheme: dark)').matches) { existingTheme = "dark"; } else if(window.matchMedia && window.matchMedia('(prefers-color-scheme: light)').matches) { existingTheme = "light"; } } //Implement light/dark theme if(existingTheme == "dark") { ctdark(); } else if(existingTheme == 'light') { ctlight(); } window.dataLayer = window.dataLayer || []; function gtag(){dataLayer.push(arguments);} gtag('js', new Date()); gtag('config', 'G-TZK0WDB1DH'); var clicky\\_site\\_ids = clicky\\_site\\_ids || []; clicky\\_site\\_ids.push(101323562); ![Clicky](//in.getclicky.com/101323562ns.gif) :root {--ghost-accent-color: #888888;} [Cold Takes](https://www.cold-takes.com) [About (+highlights)](https://www.cold-takes.com/about/) [Most Important Century](https://www.cold-takes.com/most-important-century/) [About (+highlights)](https://www.cold-takes.com/about/) [Most Important Century](https://www.cold-takes.com/most-important-century/) [Subscribe (free)](#subscribe) [Subscribe (free)](#subscribe) [Subscribe (free)](#subscribe) [Cold Takes](https://www.cold-takes.com) - Does X cause Y? An in-depth evidence review By [Holden Karnofsky](https://www.cold-takes.com/about/ "Holden Karnofsky") Jul 28, 2021 9 min read Does X cause Y? An in-depth evidence review =========================================== ![](https://www.cold-takes.com/content/images/2021/07/does-x-cause-y-1.jpg) There's an interesting theory out there that X causes Y. If this were true, it would be pretty important. So I did a deep-dive into the academic literature on whether X causes Y. Here's what I found. (Embarrassingly, I can't actually remember what X and Y are. I think maybe X was enriched preschool, or just school itself, or eating fish while pregnant, or the Paleo diet, or lead exposure, or a clever "nudge" policy trying to get people to save more, or some self-help technique, or some micronutrient or public health intervention, or democracy, or free trade, or some approach to intellectual property law. And Y was ... lifetime earnings, or risk of ADHD diagnosis, or IQ in adulthood, or weight loss, or violent crime, or peaceful foreign policy, or GDP per capita, or innovation. Sorry about that! Hope you enjoy the post anyway! Fortunately, I think **what I'm about to write is correct for pretty much any (X,Y) from those sorts of lists.**) In brief: * There are hundreds of studies on whether X causes Y, but most of them are simple observational studies that are just essentially saying "People/countries with more X also have more Y." For reasons discussed below, we can't really learn much from these studies. * There are 1-5 more interesting studies on whether X causes Y. Each study looks really clever, informative and rigorous at first glance. However, the more closely you look at them, the more confusing the picture gets. * We ultimately need to choose between (a) believing some overly complicated theory of the relationship between X and Y, which reconciles all of the wildly conflicting and often implausible things we're seeing in the studies; (b) more-or-less reverting to what we would've guessed about the relationship between X and Y in the absence of any research. The chaff: lots of unhelpful studies that I'm disregarding \---------------------------------------------------------- First, the good news: there are hundreds of studies on whether X causes Y. The bad news? We need to throw most of them out. Many have comically small sample sizes (like studying 20 people) and/or comically short time horizons (like looking at weight loss over two weeks),[1](#fn1) or unhelpful outcome measures (like intelligence tests in children under 5).[2](#fn2) But by far the most common problem is that **most of the studies on whether X causes Y are simple observational studies: they essentially just find that people/countries with more X also have more Y.** Why is this a problem? There could be a *confounder* - some third thing, Z, that is correlated with both X and Y. And there are specific reasons we should expect confounders to be common: * In general, people/countries that have more X also have more of lots of other helpful things - they're richer, they're more educated, etc. For example, if we're asking whether higher-quality schooling leads to higher earnings down the line, an issue is that people with higher-quality schooling also tend to come from better-off families with lots of other advantages. * In fact, the *very fact that people in upper-class intellectual circles think X causes Y* means that richer, more educated people/countries tend to deliberately get more X, and also try to do a lot of other things to get more Y. For example, more educated families tend to eat more fish (complicating the attempt to see whether eating fish in pregnancy is good for the baby).[3](#fn3) Now, a lot of these studies try to "control for" the problem I just stated - they say things like "We examined the effect of X and Y, while controlling for Z [e.g., how wealthy or educated the people/countries/whatever are]." How do they do this? The short answer is, well, hm, jeez. Well you see, to simplify matters a bit, just try to imagine ... uh ... shit. Uh. The only high-level way I can put this is: * They use a technique called [regression analysis](https://en.wikipedia.org/wiki/Regression_analysis) that, as far as I can determine, cannot be explained in a simple, intuitive way (especially not in terms of how it "controls for" confounders). * The "controlling for" thing relies on a lot of subtle assumptions and can break in all kinds of weird ways. [Here's](https://journals.plos.org/plosone/article?id=10.1371/journal.pone.0152719) a technical explanation of some of the pitfalls; [here's](https://www.cold-takes.com/phil-birnbaums-regression-analysis/) a set of deconstructions of regressions that break in weird ways. * None of the observational studies about whether X causes Y discuss the pitfalls of "controlling for" things and whether they apply here. I don't think we can trust these papers, and to really pick them all apart (given how many there are) would take too much time. So let's focus on a smaller number of better studies. The wheat: 1-5 more interesting studies \--------------------------------------- Digging through the sea of unhelpful studies, I found 1-5 of them that are actually really interesting! For example, one study examines some strange historical event you've never heard of (perhaps a [surge in Cuban emigration triggered by Fidel Castro suddenly allowing it](https://davidcard.berkeley.edu/papers/mariel-impact.pdf), or [John Rockefeller's decision to fund a hookworm eradication campaign](https://www.ncbi.nlm.nih.gov/pmc/articles/PMC3800113/), or a [sudden collective pardon leading to release of a third of prison inmates in Italy](https://www.aeaweb.org/articles?id=10.1257/aer.103.6.2437)), where for abstruse and idiosyncratic reasons, X got distributed in what seems to be almost a random way. This study is really clever, and the authors were incredibly thorough in examining seemingly every way their results could have been wrong. They conclude that X causes Y! But on closer inspection, I have a bunch of reservations. For example: * The paper doesn't make it easy to replicate its analysis, and when someone does manage to sort-of replicate it, they may [get different results](https://blog.givewell.org/2017/12/07/questioning-evidence-hookworm-eradication-american-south/). * There was other weird stuff going on (e.g., changes in census data collection methods[5](#fn5)), during the strange historical event, so it's a little hard to generalize. * In a response to the study, another academic advances a complex theory of how the study could actually have gotten a misleading result. This led to an intense back-and-forth between the original authors and the skeptic, stretched out over years because each response had to be published in a journal, and by the time I got to the end of it I didn't have any idea what to think anymore.[6](#fn6) I found 0-4 other interesting studies. I can't remember all of the details, but they may have included: * A study comparing siblings, or maybe "very similar countries," that got more or less of X.[7](#fn7)* A study using a complex mathematical technique claiming to cleanly isolate the effect of X and Y. I can't really follow what it's doing, and I’m guessing there are a lot of weird assumptions baked into this analysis.[8](#fn8)* A study with actual randomization: some people were randomly assigned to receive more X than others, and the researchers looked at who ended up with more Y. This sounds awesome! However, there are issues here too: \+ It's kind of ambiguous whether the assignment to X was really "random."[9](#fn9)+ Extremely weird things happened during the study (for example, [generational levels of flooding](https://www.givewell.org/international/technical/programs/deworming/reanalysis#Our_initial_reservations)), so it's not clear how well it generalizes to other settings. \+ The result seems fragile ([simply adding more data weakens it a lot](https://www.givewell.org/international/technical/programs/deworming#Other_studies_of_long-term_effects)) and/or just hard to believe (like [schoolchildren doing noticeably better on a cognition test after a few weeks of being given fish instead of meat with their lunch, even though they mostly didn't eat the fish](https://nutritionj.biomedcentral.com/articles/10.1186/s12937-017-0287-9)). Compounding the problem, the 1-5 studies I found tell very different stories about the relationship between X and Y. How could this make sense? Is there a unified theory that can reconcile all the results? Well, one possibility is that X causes Y sometimes, but only under very particular conditions, and the effect can be masked by some other thing going on. So - if you meet one of 7 criteria, you should do X to get more Y, but if you meet one of 9 other criteria, you should actually avoid X! Conclusion \---------- I have to say, this all was simultaneously more fascinating and less *informative* than I expected it would be going in. I thought I would find some nice studies about the relationship between X and Y and be done. Instead, I've learned a ton about weird historical events and about the ins and outs of different measures of X and Y, but I feel just super confused about whether X causes Y. I guess my bottom line is that X does cause Y, because it intuitively seems like it would. I'm glad I did all this research, though. It's good to know that social science research can go haywire in all kinds of strange ways. And it's good to know that despite the confident proclamations of pro- and anti-X people, it's legitimately just super unclear whether X causes Y. I mean, how else could I have learned that? Appendix: based on a true story \------------------------------- This piece was inspired by: * Most evidence reviews GiveWell has done, especially of [deworming](https://www.givewell.org/international/technical/programs/deworming)* Many evidence reviews by David Roodman, particularly [Macro Aid Effectiveness Research: a Guide for the Perplexed](https://www.cgdev.org/publication/macro-aid-effectiveness-research-guide-perplexed-working-paper-135); [Due Diligence: an Impertinent Inquiry into Microfinance](https://smile.amazon.com/dp/B0095V3J0S/); and [Reasonable Doubt: A New Look at Whether Prison Growth Cuts Crime](https://www.openphilanthropy.org/blog/reasonable-doubt-new-look-whether-prison-growth-cuts-crime). * Many evidence reviews by Slate Star Codex, collected [here](https://slatestarcodex.com/tag/much-more-than-you-wanted-to-know/). * Informal evidence reviews I've done for e.g. personal medical decisions. The basic patterns above apply to most of these, and the bottom line usually has the kind of frustrating ambiguity seen in this conclusion. There are cases where things seem a bit less ambiguous and the bottom line seems clearer. Speaking broadly, I think the main things that contribute to this are: * Actual randomization. For years I've nodded along when people say "You shouldn't be dogmatic about randomization, there are many ways for a study to be informative," but each year I've become a bit more dogmatic. Even the most sophisticated-, appealing-seeming alternatives to randomization in studies seem to have a way of falling apart. Randomized studies almost always have problems and drawbacks too. But I’d rather have a randomized study with drawbacks than a non-randomized study with drawbacks. * Extreme thoroughness, such as Roodman's attempt to reconstruct the data and code for key studies in [Reasonable Doubt](https://www.openphilanthropy.org/blog/reasonable-doubt-new-look-whether-prison-growth-cuts-crime). This sometimes leads to outright dismissing a number of studies, leaving a smaller, more consistent set remaining. [![Twitter](https://www.cold-takes.com/content/images/2021/06/ct-twitter-square.png)](https://api.addthis.com/oexchange/0.8/forward/twitter/offer?url=https%3A%2F%2Fwww.cold-takes.com%2Fdoes-x-cause-y-an-in-depth-evidence-review%2F&pubid;=ra-60a178324cffc42e&title;=Cold%20Takes%20-%20Does%20X%20cause%20Y%3F%20An%20in-depth%20evidence%20review&ct;=1) [![Facebook](https://www.cold-takes.com/content/images/2021/06/ct-facebook-square.png)](https://api.addthis.com/oexchange/0.8/forward/facebook/offer?url=https%3A%2F%2Fwww.cold-takes.com%2Fdoes-x-cause-y-an-in-depth-evidence-review%2F&pubid;=ra-60a178324cffc42e&title;=Cold%20Takes%20-%20Does%20X%20cause%20Y%3F%20An%20in-depth%20evidence%20review&ct;=1) [![Reddit](https://www.cold-takes.com/content/images/2021/06/ct-reddit-square.png)](https://api.addthis.com/oexchange/0.8/forward/reddit/offer?url=https%3A%2F%2Fwww.cold-takes.com%2Fdoes-x-cause-y-an-in-depth-evidence-review%2F&pubid;=ra-60a178324cffc42e&title;=Cold%20Takes%20-%20Does%20X%20cause%20Y%3F%20An%20in-depth%20evidence%20review&ct;=1) [![More](https://www.cold-takes.com/content/images/2021/06/ct-addthis-square.png)](https://api.addthis.com/oexchange/0.8/forward/menu/offer?url=https%3A%2F%2Fwww.cold-takes.com%2Fdoes-x-cause-y-an-in-depth-evidence-review%2F&pubid;=ra-60a178324cffc42e&title;=Cold%20Takes%20-%20Does%20X%20cause%20Y%3F%20An%20in-depth%20evidence%20review&ct;=1) [Subscribe](https://www.cold-takes.com/does-x-cause-y-an-in-depth-evidence-review/#/portal/signup/free) [Feedback](https://www.guidedtrack.com/programs/4kal2ue/run?posttitle=Does%20X%20cause%20Y%3F%20An%20in-depth%20evidence%20review) \--- Footnotes \--------- 1\. Both of these show up in studies from [this review on the Paleo diet](https://www.ncbi.nlm.nih.gov/pmc/articles/PMC4588744/). To be fair, small studies can theoretically be aggregated for larger numbers, but that's often hard to do in practice when the studies are all a bit different from each other. [↩](#fnref1)- I don't have a great cite for this, but it's pretty common in studies on things like how in vitro fertilization affects child development. [↩](#fnref2)- See studies cited in [this literature review](https://pubmed.ncbi.nlm.nih.gov/31739098/). [↩](#fnref3)- (Footnote deleted) \- "Borjas’s paper ... separately measured the wages of two slices of that larger group ... But it was in that act of slicing the data that the spurious result was generated. It created data samples that, exactly in 1980, suddenly included far more low-wage black males—accounting for the whole wage decline in those samples relative to other cities. Understanding how that happened requires understanding the raw data ... Right in 1980, the Census Bureau—which ran the CPS surveys—improved its survey methods to cover more low-skill black men. The 1970 census and again the 1980 census had greatly undercounted low-skill black men, both by failing to identify their residences and by failing to sufficiently probe survey respondents about marginal or itinerant household members. There was massive legislative and judicial pressure to count blacks better, particularly in Miami." [↩](#fnref5)- E.g., the [Mariel boatlift debate](https://davidroodman.com/blog/2017/05/25/four-points-on-the-debate-over-the-impact-of-mariel-boatlift/). [↩](#fnref6)- For example, sibling analysis features prominently in [Slate Star Codex's examination of preschool impacts](https://slatestarcodex.com/2018/11/13/preschool-much-more-than-you-wanted-to-know/), while comparisons between Sweden and other Scandinavian/European countries is [prominent in its analysis of lockdowns](https://astralcodexten.substack.com/p/lockdown-effectiveness-much-more). [↩](#fnref7)- E.g., [this attempt to gauge the impacts of microfinance](https://blog.givewell.org/2011/05/13/microlending-debate-an-example-of-why-academic-research-should-be-used-with-caution/), or "[generalized method of moments](https://onlinelibrary.wiley.com/doi/full/10.1111/j.1468-0084.2008.00542.x)" approaches to cross-country analysis (of e.g. the effectiveness of aid). [↩](#fnref8)- This is a surprisingly common issue. E.g. see debates over [whether charter school lotteries are really random](https://freddiedeboer.substack.com/p/charter-school-lotteries-are-a-black), whether ["random assignment to small or large class size" can be interpreted as "random assignment to a teacher,"](https://www.econlib.org/archives/2011/10/teachers_and_in.html) discussion of "judge randomization" and possible randomization failure [here](https://www.openphilanthropy.org/blog/reasonable-doubt-new-look-whether-prison-growth-cuts-crime) (particularly section 9.9). A separate issue: sometimes randomization occurs by "cluster" (instead of randomizing which *individuals* receive some treatment, perhaps particular *schools* or *groups* are chosen to receive it), which can complicate the analysis. [↩](#fnref9) Powered by [Ghost](https://ghost.org/) #### Popular tags [sports](/tag/sports/) [innovationstagnation](/tag/innovationstagnation/) [2star](/tag/2star/) [utopia](/tag/utopia/) Subscribe to Cold Takes ======================= For audio version, search for "Cold Takes Audio" in your podcast app Subscribe **Great!** Check your inbox and click the link to confirm your subscription. **If you don't see it, check your spam!** Please enter a valid email address! $("#lower-right-subscribe").click(function(f) { clicky.goal('1458'); /\\*LO.events.track('Subscribe-LR');\\*/ setTimeout(function(){$("#subscribe-input").focus();},500); }); $("#upper-right-subscribe-1").click(function(f) { clicky.goal('1471'); /\\*LO.events.track('Subscribe-LR');\\*/ setTimeout(function(){$("#subscribe-input").focus();},500); }); $("#upper-right-subscribe-2").click(function(f) { clicky.goal('1471'); /\\*LO.events.track('Subscribe-LR');\\*/ setTimeout(function(){$("#subscribe-input").focus();},500); }); $("#cover-subscribe").click(function(f) { clicky.goal('1472'); /\\*LO.events.track('Subscribe-LR');\\*/ setTimeout(function(){$("#subscribe-input").focus();},500); }); $("#footer-subscribe").click(function(f) { clicky.goal('1473'); /\\*LO.events.track('Subscribe-LR');\\*/ setTimeout(function(){$("#subscribe-input").focus();},500); }); $("#final-subscribe").click(function(f) { clicky.goal('1474'); }); //When someone clicks an anchor link, I want the sticky nav to go away. $("a").click(function(f) { body.removeClass('sticky-visible'); lastSt = 9999999999999}); function ctlight() { html.removeClass('theme-dark').addClass('theme-light'); localStorage.setItem('dawn\\_theme', 'light'); //$(".gh-portal-triggerbtn-container").css("background","#0000dd"); /\\* document.documentElement.style.setProperty('--brand-color', '#0000dd'); document.documentElement.style.setProperty('--white-color', '#ffffff'); document.documentElement.style.setProperty('--primary-text-color', '#1a1a1a'); document.documentElement.style.setProperty('--secondary-text-color', '#1a1a1a');\\*/ // $("#CT-thumbID").css("transform","translate(0px,0)"); // $(".single-content").css("font-weight","400"); existingTheme = "light"; } function ctdark() { html.removeClass('theme-light').addClass('theme-dark'); localStorage.setItem('dawn\\_theme', 'dark'); /\\*document.documentElement.style.setProperty('--brand-color', '#5ceaff'); document.documentElement.style.setProperty('--white-color', '#18141f'); document.documentElement.style.setProperty('--primary-text-color', '#fff'); document.documentElement.style.setProperty('--secondary-text-color', '#fff');\\*/ //document.documentElement.style.setProperty('--white-color', '#14101b'); //document.documentElement.style.setProperty('--white-color', '#121212'); //document.documentElement.style.setProperty('--white-color', '#282c35'); //$(".single-content").css("font-weight","300"); // $("#CT-thumbID").css("transform","translate(19px,0)"); existingTheme = "dark"; } //For light/dark toggle $(".CT-toggle").click(function(g) { g.preventDefault(); if(existingTheme == "dark") { ctlight(); } else if(existingTheme == 'light') { ctdark(); } } ); // Parse the URL parameter, then log goal if it is a completed subscription. Weird that it takes this much code to do this. function getParameterByName(name, url) { if (!url) url = window.location.href; name = name.replace(/[\\[\\]]/g, "\\\$&"); var regex = new RegExp("[?&]" + name + "(=([^&#]\\*)|&|#|$)"), results = regex.exec(url); if (!results) return null; if (!results[2]) return ''; return decodeURIComponent(results[2].replace(/\\+/g, " ")); } // Give the parameter a variable name var action = getParameterByName('action'); var success = getParameterByName('success'); $(document).ready(function () { // Success === null is here for backwards compatibility if (action == 'subscribe' && (success === null || success == 'true')) { clicky.goal('1476'); } });


Phil Birnbaum's "bad regression" puzzles var siteUrl = 'https://www.cold-takes.com'; var localTheme = localStorage.getItem('dawn\_theme'); switch (localTheme) { case 'dark': document.documentElement.classList.add('theme-dark'); break; case 'light': document.documentElement.classList.add('theme-light'); break; default: break; } { "@context": "https://schema.org", "@type": "Article", "publisher": { "@type": "Organization", "name": "Cold Takes", "url": "https://www.cold-takes.com/", "logo": { "@type": "ImageObject", "url": "https://www.cold-takes.com/favicon.ico", "width": 48, "height": 48 } }, "author": { "@type": "Person", "name": "Holden Karnofsky", "image": { "@type": "ImageObject", "url": "https://www.cold-takes.com/content/images/2021/07/HK-headshot-current---compressed-1.jpg", "width": 1600, "height": 1067 }, "url": "https://www.cold-takes.com/author/holden/", "sameAs": [ "https://www.cold-takes.com/about/" ] }, "headline": "Phil Birnbaum's "bad regression" puzzles", "url": "https://www.cold-takes.com/phil-birnbaums-regression-analysis/", "datePublished": "2021-07-15T22:18:23.000Z", "dateModified": "2021-10-01T22:22:35.000Z", "image": { "@type": "ImageObject", "url": "https://www.cold-takes.com/content/images/2021/10/birnbaum-twitter-img-2.png", "width": 1152, "height": 576 }, "description": "Painstaking, even beautiful deconstructions of misleading regressions.", "mainEntityOfPage": { "@type": "WebPage", "@id": "https://www.cold-takes.com/" } } .gh-post-upgrade-cta-content, .gh-post-upgrade-cta { display: flex; flex-direction: column; align-items: center; font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, Oxygen, Ubuntu, Cantarell, 'Open Sans', 'Helvetica Neue', sans-serif; text-align: center; width: 100%; color: #ffffff; font-size: 16px; } .gh-post-upgrade-cta-content { border-radius: 8px; padding: 40px 4vw; } .gh-post-upgrade-cta h2 { color: #ffffff; font-size: 28px; letter-spacing: -0.2px; margin: 0; padding: 0; } .gh-post-upgrade-cta p { margin: 20px 0 0; padding: 0; } .gh-post-upgrade-cta small { font-size: 16px; letter-spacing: -0.2px; } .gh-post-upgrade-cta a { color: #ffffff; cursor: pointer; font-weight: 500; box-shadow: none; text-decoration: underline; } .gh-post-upgrade-cta a:hover { color: #ffffff; opacity: 0.8; box-shadow: none; text-decoration: underline; } .gh-post-upgrade-cta a.gh-btn { display: block; background: #ffffff; text-decoration: none; margin: 28px 0 0; padding: 8px 18px; border-radius: 4px; font-size: 16px; font-weight: 600; } .gh-post-upgrade-cta a.gh-btn:hover { opacity: 0.92; } .shaded-table { color: #000000; border: 1px solid; vertical-align: top; } .shaded-table a { color: #0000dd; } .sticky-subscribe-div { z-index: 1000; position: fixed; bottom: 25px; right: 25px; max-width: 500px; overflow: hidden; opacity: 1; } .sticky-subscribe-button { cursor: pointer; padding: 20px; background: var(--brand-color); color: var(--white-color); border-radius: 50px; border-style: none; } .subscribe-overlay { position: fixed; top: 0; right: 0; bottom: 0; left: 0; z-index: 2000; display: flex; justify-content: center; align-items: center; background: var(--white-color); opacity: 0; transition: opacity 0.3s ease-in; pointer-events: none; } .subscribe-overlay:target { z-index: 2001; opacity: 0.95; pointer-events: auto; } .subscribe-form { margin: 1.5em 0; padding: 6.5vh 7vw 8vh; border-style: solid !important; border: var(--mid-gray-color); border-radius: 10px; text-align: center; } .subscribe-overlay-title { display: inline-block; margin: 0 0 10px 0; font-size: 3rem; line-height: 1.15em; color: var(--darker-gray-color); } .subscribe-overlay-description { margin: 0 auto 50px; max-width: 500px; color: var(--darker-gray-color); font-size: 2rem; line-height: 1.3em; opacity: 0.8; } .subscribe-form .form-group { align-self: stretch; display: flex; } .subscribe-email { padding: 10px; border: 1px solid; border-color: var(--darker-gray-color); font-size: 1.8rem; line-height: 1em; font-weight: normal; user-select: text; transition: border-color 0.15s linear; -webkit-appearance: none; } .subscribe-form .subscribe-overlay-button { word-break: normal; position: relative; display: inline-block; margin: 0 0 0 10px; padding: 0 20px; height: 43px; outline: none; font-size: 1.5rem; line-height: 39px; font-weight: 400; background: var(--brand-color); border-radius: 5px; color: var(--white-color); } .subscribe-form .button-loader, .subscribe-form .message-success, .subscribe-form .message-error { display: none; } .subscribe-form .loading .button-content { visibility: hidden; } .subscribe-form .loading .button-loader { position: absolute; top: 0; left: 50%; display: inline-block; margin-left: -19px; transform: scale(0.7); } .subscribe-overlay .loading .button-loader { top: 5px; } .subscribe-form .button-loader svg path, .subscribe-form .button-loader svg rect { fill: var(--white-color); } .subscribe-form .success .message-success, .subscribe-form .invalid .message-error, .subscribe-form .error .message-error { display: block; margin: 1em auto 0; max-width: 400px; color: var(--red-color); font-size: 1.6rem; line-height: 1.5em; text-align: center; } .subscribe-form .success .message-success { display: block; color: var(--darker-gray-color); } .subscribe-form .invalid .message-error, .subscribe-form .error .message-error { display: block; } @media (max-width: 650px) { .subscribe-form-title { font-size: 2.4rem; } .subscribe-form-description { font-size: 1.6rem; } } @media (max-width: 500px) { .subscribe-form form { display: flex; flex-direction: column; } .subscribe-form .subscribe-overlay-button { margin: 0 0 0 0; } .subscribe-form .form-group { display: flex; flex-direction: column; width: 100%; } .subscribe-form button { display: flex; margin: 0 0 0 0; width: 100%; } } .subscribe-close-button { position: absolute; top: 16px; right: 20px; z-index: 2000; display: block; width: 40px; height: 40px; } .subscribe-close-button:before { content: ""; position: absolute; top: 20px; right: 4px; display: block; width: 32px; height: 1px; background: var(--darker-gray-color); opacity: 0.8; transform: rotate(45deg); } .subscribe-close-button:after { content: ""; position: absolute; top: 20px; right: 4px; display: block; width: 32px; height: 1px; background: var(--darker-gray-color); opacity: 0.8; transform: rotate(-45deg); } .subscribe-close-button:hover { cursor: default; } /\*.message-success, .message-error, .message-loading { display: none; }\*/ /\*.message-success .success { visibility: visible; display: block; opacity: 1; } .message-error.error { visibility: visible; display: block; opacity: 1; }\*/ /\*.success .message-success, .error .message-error, .loading .message-loading { visibility: visible; display: block; opacity: 1; }\*/ /\*For TOC\*/ @media (min-width: 1200px) { .sticky-visible .sticky-toc { visibility: visible; opacity: 1; transform: translateY(0); } .toc-opened .sticky-toc { visibility: hidden; } } /\*.gh-portal-triggerbtn-iframe { } .gh-portal-triggerbtn-wrapper { background: #0000ff !important; } .gh-portal-triggerbtn-container { background: #0000ff !important; height: 10px !important; } \*/ .theme-dark:root { \--green-color: #66ff33 !important; \--orange-color: #ff5c33 !important; \--red-color: #ff3333 !important; \--purple-color: #ff00ff !important; \--brand-color: #5ceaff !important; \--white-color: #18141f !important; \--primary-text-color: #fff !important; \--secondary-text-color: #fff !important; } .feed-visibility-false { display: none; } /\*.feed-visibility-true { display: none; }\*/ :root { \--brand-color:#0000dd; \--white-color:#ffffff; \--primary-text-color:#1a1a1a; \--secondary-text-color:#1a1a1a; \--orange-color: #ff6600; \--purple-color: #cc33ff; } blockquote { font-weight: unset; letter-spacing: normal; font-style: normal; text-align: inherit; } table { border: 1px solid; border-collapse: collapse; white-space: unset !important; width=200px !important; } table td { border: 1px solid; border-collapse: collapse; } .theme-dark .CT-thumb { transform: translate(20px,0px); } .theme-light .CT-thumb { transform: translate(0px,0px); } /\*.u-text-format h3 { color: var(--darker-gray-color) ; }\*/ //Detect theme var existingTheme = localStorage.getItem('dawn\_theme'); if (!existingTheme) { if(navigator.platform.indexOf('Win') > -1) { existingTheme = "light"; } else if(window.matchMedia && window.matchMedia('(prefers-color-scheme: dark)').matches) { existingTheme = "dark"; } else if(window.matchMedia && window.matchMedia('(prefers-color-scheme: light)').matches) { existingTheme = "light"; } } //Implement light/dark theme if(existingTheme == "dark") { ctdark(); } else if(existingTheme == 'light') { ctlight(); } window.dataLayer = window.dataLayer || []; function gtag(){dataLayer.push(arguments);} gtag('js', new Date()); gtag('config', 'G-TZK0WDB1DH'); var clicky\_site\_ids = clicky\_site\_ids || []; clicky\_site\_ids.push(101323562); ![Clicky](//in.getclicky.com/101323562ns.gif) :root {--ghost-accent-color: #888888;} [Cold Takes](https://www.cold-takes.com) [About (+highlights)](https://www.cold-takes.com/about/) [Most Important Century](https://www.cold-takes.com/most-important-century/) [About (+highlights)](https://www.cold-takes.com/about/) [Most Important Century](https://www.cold-takes.com/most-important-century/) [Subscribe (free)](#subscribe) [Subscribe (free)](#subscribe) [Subscribe (free)](#subscribe) [Cold Takes](https://www.cold-takes.com) - Phil Birnbaum's "bad regression" puzzles By [Holden Karnofsky](https://www.cold-takes.com/about/ "Holden Karnofsky") Jul 15, 2021 3 min read Phil Birnbaum's "bad regression" puzzles ======================================== Painstaking, even beautiful deconstructions of misleading regressions. If you've ever wanted to see someone painstakingly deconstruct a regression analysis and show all the subtle reasons it can generate wild, weird and completely wrong results, there is good stuff at [Sabermetric Research - Phil Birnbaum](http://blog.philbirnbaum.com/). It's a sports blog, but sports knowledge isn't needed (knowledge of regression analysis generally is, if you want to follow all the details). Birnbaum's not exactly the only person to do takedowns of bad studies. But when Birnbaum notices that something is "off," he doesn't just point it out and move on. He isn't satisfied with "This conclusion is implausible" or "This conclusion isn't robust to sensitivity analysis." He digs all the way to the bottom to understand *exactly* how a study got its wrong result. His deconstructions of bad regressions are like four-star meals or masterful jazz solos ... I don't want to besmirch them by trying to explain them, so if you're into regression deconstructions you should just click through the links below. (I'm not going to explain what regression analysis is today, for which I apologize; if I ever do, I will link back to this post. It's very hard to explain it compactly and clearly, as you can see from [Wikipedia's attempt](https://en.wikipedia.org/wiki/Regression_analysis), but it is VERY common in social science research. Kind of a bad combination IMO. If you hear "This study shows [something about people]," it's more likely than not that the study relies on regression analysis.) Some good (old) ones: * [Estimating whether Aaron Rodger's contract overpays or underpays him by making a scatterplot of pay and value-added with other quarterbacks and seeing whether he's above or below the regression line.](https://web.archive.org/web/20130524132245/http://www.advancednflstats.com/2013/05/point-counterpoint-on-rodgers-extension.html) The answer changes completely when you switch the x- and y-axes. Which one is right, and what *exactly* is wrong with the other one? (Birnbaum linked to this, but it's now dead and I am linking directly to an Internet Archive version. Birnbaum's "solution" is down in the comments, just search for his name.) * [Deconstructing an NBA time-zone regression](http://blog.philbirnbaum.com/2017/08/deconstructing-nba-time-zone-regression.html): the key coefficients turn out to be literally meaningless. * Do younger brothers steal more bases? Parts [I](http://blog.philbirnbaum.com/2010/05/do-younger-brothers-steal-more-bases.html), [II](http://blog.philbirnbaum.com/2010/06/do-younger-brothers-steal-more-bases.html), [III](http://blog.philbirnbaum.com/2010/06/do-younger-brothers-steal-more-bases_18.html) although I think it's OK to skip part II (and Part I is short). * The OBP/SLG regression puzzle: parts [I](http://blog.philbirnbaum.com/2013/05/the-obpslg-regression-puzzle.html), [II](http://blog.philbirnbaum.com/2013/05/the-obpslg-regression-puzzle-part-ii.html), [III](http://blog.philbirnbaum.com/2013/05/the-obpslg-regression-puzzle-part-iii.html), [IV](http://blog.philbirnbaum.com/2013/06/the-obpslg-regression-puzzle-part-iv.html), [V](http://blog.philbirnbaum.com/2013/06/the-obpslg-regression-puzzle-part-v.html). This one is very weedsy and you'll probably want to skip parts, though it's also kind of glorious to see just how doggedly he digs on every strange numerical result. He also makes an effort to explain what's going on for people who don't know baseball. The essence of the puzzle: OBP and SLG are both indicators of a team's performance, but when one regresses a team's performance on OBP and SLG, the ratio of the coefficients is pretty far off from what the "true" value for (value of OBP / value of SLG) is separately known to be. I think the issues here are extremely general, and not good news for the practice of treating regression coefficients as effect sizes. [![Twitter](https://www.cold-takes.com/content/images/2021/06/ct-twitter-square.png)](https://api.addthis.com/oexchange/0.8/forward/twitter/offer?url=https%3A%2F%2Fwww.cold-takes.com%2Fphil-birnbaums-regression-analysis&pubid;=ra-60a178324cffc42e&title;=Cold%20Takes%20-%20Phil%20Birnbaum's%20%22bad%20regression%22%20puzzles&ct;=1) [![Facebook](https://www.cold-takes.com/content/images/2021/06/ct-facebook-square.png)](https://api.addthis.com/oexchange/0.8/forward/facebook/offer?url=https%3A%2F%2Fwww.cold-takes.com%2Fphil-birnbaums-regression-analysis&pubid;=ra-60a178324cffc42e&title;=Cold%20Takes%20-%20Phil%20Birnbaum's%20%22bad%20regression%22%20puzzles&ct;=1) [![Reddit](https://www.cold-takes.com/content/images/2021/06/ct-reddit-square.png)](https://api.addthis.com/oexchange/0.8/forward/reddit/offer?url=https%3A%2F%2Fwww.cold-takes.com%2Fphil-birnbaums-regression-analysis&pubid;=ra-60a178324cffc42e&title;=Cold%20Takes%20-%20Phil%20Birnbaum's%20%22bad%20regression%22%20puzzles&ct;=1) [![More](https://www.cold-takes.com/content/images/2021/06/ct-addthis-square.png)](https://api.addthis.com/oexchange/0.8/forward/menu/offer?url=https%3A%2F%2Fwww.cold-takes.com%2Fphil-birnbaums-regression-analysis&pubid;=ra-60a178324cffc42e&title;=Cold%20Takes%20-%20Phil%20Birnbaum's%20%22bad%20regression%22%20puzzles&ct;=1) [Subscribe](https://www.cold-takes.com/phil-birnbaums-regression-analysis/#/portal/signup/free) [Feedback](https://www.guidedtrack.com/programs/4kal2ue/run?posttitle=Phil%20Birnbaum's%20%22bad%20regression%22%20puzzles) Powered by [Ghost](https://ghost.org/) #### Popular tags [sports](/tag/sports/) [innovationstagnation](/tag/innovationstagnation/) [2star](/tag/2star/) [utopia](/tag/utopia/) Subscribe to Cold Takes ======================= For audio version, search for "Cold Takes Audio" in your podcast app Subscribe **Great!** Check your inbox and click the link to confirm your subscription. **If you don't see it, check your spam!** Please enter a valid email address! $("#lower-right-subscribe").click(function(f) { clicky.goal('1458'); /\*LO.events.track('Subscribe-LR');\*/ setTimeout(function(){$("#subscribe-input").focus();},500); }); $("#upper-right-subscribe-1").click(function(f) { clicky.goal('1471'); /\*LO.events.track('Subscribe-LR');\*/ setTimeout(function(){$("#subscribe-input").focus();},500); }); $("#upper-right-subscribe-2").click(function(f) { clicky.goal('1471'); /\*LO.events.track('Subscribe-LR');\*/ setTimeout(function(){$("#subscribe-input").focus();},500); }); $("#cover-subscribe").click(function(f) { clicky.goal('1472'); /\*LO.events.track('Subscribe-LR');\*/ setTimeout(function(){$("#subscribe-input").focus();},500); }); $("#footer-subscribe").click(function(f) { clicky.goal('1473'); /\*LO.events.track('Subscribe-LR');\*/ setTimeout(function(){$("#subscribe-input").focus();},500); }); $("#final-subscribe").click(function(f) { clicky.goal('1474'); }); //When someone clicks an anchor link, I want the sticky nav to go away. $("a").click(function(f) { body.removeClass('sticky-visible'); lastSt = 9999999999999}); function ctlight() { html.removeClass('theme-dark').addClass('theme-light'); localStorage.setItem('dawn\_theme', 'light'); //$(".gh-portal-triggerbtn-container").css("background","#0000dd"); /\* document.documentElement.style.setProperty('--brand-color', '#0000dd'); document.documentElement.style.setProperty('--white-color', '#ffffff'); document.documentElement.style.setProperty('--primary-text-color', '#1a1a1a'); document.documentElement.style.setProperty('--secondary-text-color', '#1a1a1a');\*/ // $("#CT-thumbID").css("transform","translate(0px,0)"); // $(".single-content").css("font-weight","400"); existingTheme = "light"; } function ctdark() { html.removeClass('theme-light').addClass('theme-dark'); localStorage.setItem('dawn\_theme', 'dark'); /\*document.documentElement.style.setProperty('--brand-color', '#5ceaff'); document.documentElement.style.setProperty('--white-color', '#18141f'); document.documentElement.style.setProperty('--primary-text-color', '#fff'); document.documentElement.style.setProperty('--secondary-text-color', '#fff');\*/ //document.documentElement.style.setProperty('--white-color', '#14101b'); //document.documentElement.style.setProperty('--white-color', '#121212'); //document.documentElement.style.setProperty('--white-color', '#282c35'); //$(".single-content").css("font-weight","300"); // $("#CT-thumbID").css("transform","translate(19px,0)"); existingTheme = "dark"; } //For light/dark toggle $(".CT-toggle").click(function(g) { g.preventDefault(); if(existingTheme == "dark") { ctlight(); } else if(existingTheme == 'light') { ctdark(); } } ); // Parse the URL parameter, then log goal if it is a completed subscription. Weird that it takes this much code to do this. function getParameterByName(name, url) { if (!url) url = window.location.href; name = name.replace(/[\[\]]/g, "\\$&"); var regex = new RegExp("[?&]" + name + "(=([^&#]\*)|&|#|$)"), results = regex.exec(url); if (!results) return null; if (!results[2]) return ''; return decodeURIComponent(results[2].replace(/\+/g, " ")); } // Give the parameter a variable name var action = getParameterByName('action'); var success = getParameterByName('success'); $(document).ready(function () { // Success === null is here for backwards compatibility if (action == 'subscribe' && (success === null || success == 'true')) { clicky.goal('1476'); } });

Phil Birnbaum's "bad regression" puzzles var siteUrl = 'https://www.cold-takes.com'; var localTheme = localStorage.getItem('dawn\\_theme'); switch (localTheme) { case 'dark': document.documentElement.classList.add('theme-dark'); break; case 'light': document.documentElement.classList.add('theme-light'); break; default: break; } { "@context": "https://schema.org", "@type": "Article", "publisher": { "@type": "Organization", "name": "Cold Takes", "url": "https://www.cold-takes.com/", "logo": { "@type": "ImageObject", "url": "https://www.cold-takes.com/favicon.ico", "width": 48, "height": 48 } }, "author": { "@type": "Person", "name": "Holden Karnofsky", "image": { "@type": "ImageObject", "url": "https://www.cold-takes.com/content/images/2021/07/HK-headshot-current---compressed-1.jpg", "width": 1600, "height": 1067 }, "url": "https://www.cold-takes.com/author/holden/", "sameAs": [ "https://www.cold-takes.com/about/" ] }, "headline": "Phil Birnbaum's "bad regression" puzzles", "url": "https://www.cold-takes.com/phil-birnbaums-regression-analysis/", "datePublished": "2021-07-15T22:18:23.000Z", "dateModified": "2021-10-01T22:22:35.000Z", "image": { "@type": "ImageObject", "url": "https://www.cold-takes.com/content/images/2021/10/birnbaum-twitter-img-2.png", "width": 1152, "height": 576 }, "description": "Painstaking, even beautiful deconstructions of misleading regressions.", "mainEntityOfPage": { "@type": "WebPage", "@id": "https://www.cold-takes.com/" } } .gh-post-upgrade-cta-content, .gh-post-upgrade-cta { display: flex; flex-direction: column; align-items: center; font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, Oxygen, Ubuntu, Cantarell, 'Open Sans', 'Helvetica Neue', sans-serif; text-align: center; width: 100%; color: #ffffff; font-size: 16px; } .gh-post-upgrade-cta-content { border-radius: 8px; padding: 40px 4vw; } .gh-post-upgrade-cta h2 { color: #ffffff; font-size: 28px; letter-spacing: -0.2px; margin: 0; padding: 0; } .gh-post-upgrade-cta p { margin: 20px 0 0; padding: 0; } .gh-post-upgrade-cta small { font-size: 16px; letter-spacing: -0.2px; } .gh-post-upgrade-cta a { color: #ffffff; cursor: pointer; font-weight: 500; box-shadow: none; text-decoration: underline; } .gh-post-upgrade-cta a:hover { color: #ffffff; opacity: 0.8; box-shadow: none; text-decoration: underline; } .gh-post-upgrade-cta a.gh-btn { display: block; background: #ffffff; text-decoration: none; margin: 28px 0 0; padding: 8px 18px; border-radius: 4px; font-size: 16px; font-weight: 600; } .gh-post-upgrade-cta a.gh-btn:hover { opacity: 0.92; } .shaded-table { color: #000000; border: 1px solid; vertical-align: top; } .shaded-table a { color: #0000dd; } .sticky-subscribe-div { z-index: 1000; position: fixed; bottom: 25px; right: 25px; max-width: 500px; overflow: hidden; opacity: 1; } .sticky-subscribe-button { cursor: pointer; padding: 20px; background: var(--brand-color); color: var(--white-color); border-radius: 50px; border-style: none; } .subscribe-overlay { position: fixed; top: 0; right: 0; bottom: 0; left: 0; z-index: 2000; display: flex; justify-content: center; align-items: center; background: var(--white-color); opacity: 0; transition: opacity 0.3s ease-in; pointer-events: none; } .subscribe-overlay:target { z-index: 2001; opacity: 0.95; pointer-events: auto; } .subscribe-form { margin: 1.5em 0; padding: 6.5vh 7vw 8vh; border-style: solid !important; border: var(--mid-gray-color); border-radius: 10px; text-align: center; } .subscribe-overlay-title { display: inline-block; margin: 0 0 10px 0; font-size: 3rem; line-height: 1.15em; color: var(--darker-gray-color); } .subscribe-overlay-description { margin: 0 auto 50px; max-width: 500px; color: var(--darker-gray-color); font-size: 2rem; line-height: 1.3em; opacity: 0.8; } .subscribe-form .form-group { align-self: stretch; display: flex; } .subscribe-email { padding: 10px; border: 1px solid; border-color: var(--darker-gray-color); font-size: 1.8rem; line-height: 1em; font-weight: normal; user-select: text; transition: border-color 0.15s linear; -webkit-appearance: none; } .subscribe-form .subscribe-overlay-button { word-break: normal; position: relative; display: inline-block; margin: 0 0 0 10px; padding: 0 20px; height: 43px; outline: none; font-size: 1.5rem; line-height: 39px; font-weight: 400; background: var(--brand-color); border-radius: 5px; color: var(--white-color); } .subscribe-form .button-loader, .subscribe-form .message-success, .subscribe-form .message-error { display: none; } .subscribe-form .loading .button-content { visibility: hidden; } .subscribe-form .loading .button-loader { position: absolute; top: 0; left: 50%; display: inline-block; margin-left: -19px; transform: scale(0.7); } .subscribe-overlay .loading .button-loader { top: 5px; } .subscribe-form .button-loader svg path, .subscribe-form .button-loader svg rect { fill: var(--white-color); } .subscribe-form .success .message-success, .subscribe-form .invalid .message-error, .subscribe-form .error .message-error { display: block; margin: 1em auto 0; max-width: 400px; color: var(--red-color); font-size: 1.6rem; line-height: 1.5em; text-align: center; } .subscribe-form .success .message-success { display: block; color: var(--darker-gray-color); } .subscribe-form .invalid .message-error, .subscribe-form .error .message-error { display: block; } @media (max-width: 650px) { .subscribe-form-title { font-size: 2.4rem; } .subscribe-form-description { font-size: 1.6rem; } } @media (max-width: 500px) { .subscribe-form form { display: flex; flex-direction: column; } .subscribe-form .subscribe-overlay-button { margin: 0 0 0 0; } .subscribe-form .form-group { display: flex; flex-direction: column; width: 100%; } .subscribe-form button { display: flex; margin: 0 0 0 0; width: 100%; } } .subscribe-close-button { position: absolute; top: 16px; right: 20px; z-index: 2000; display: block; width: 40px; height: 40px; } .subscribe-close-button:before { content: ""; position: absolute; top: 20px; right: 4px; display: block; width: 32px; height: 1px; background: var(--darker-gray-color); opacity: 0.8; transform: rotate(45deg); } .subscribe-close-button:after { content: ""; position: absolute; top: 20px; right: 4px; display: block; width: 32px; height: 1px; background: var(--darker-gray-color); opacity: 0.8; transform: rotate(-45deg); } .subscribe-close-button:hover { cursor: default; } /\\*.message-success, .message-error, .message-loading { display: none; }\\*/ /\\*.message-success .success { visibility: visible; display: block; opacity: 1; } .message-error.error { visibility: visible; display: block; opacity: 1; }\\*/ /\\*.success .message-success, .error .message-error, .loading .message-loading { visibility: visible; display: block; opacity: 1; }\\*/ /\\*For TOC\\*/ @media (min-width: 1200px) { .sticky-visible .sticky-toc { visibility: visible; opacity: 1; transform: translateY(0); } .toc-opened .sticky-toc { visibility: hidden; } } /\\*.gh-portal-triggerbtn-iframe { } .gh-portal-triggerbtn-wrapper { background: #0000ff !important; } .gh-portal-triggerbtn-container { background: #0000ff !important; height: 10px !important; } \\*/ .theme-dark:root { \--green-color: #66ff33 !important; \--orange-color: #ff5c33 !important; \--red-color: #ff3333 !important; \--purple-color: #ff00ff !important; \--brand-color: #5ceaff !important; \--white-color: #18141f !important; \--primary-text-color: #fff !important; \--secondary-text-color: #fff !important; } .feed-visibility-false { display: none; } /\\*.feed-visibility-true { display: none; }\\*/ :root { \--brand-color:#0000dd; \--white-color:#ffffff; \--primary-text-color:#1a1a1a; \--secondary-text-color:#1a1a1a; \--orange-color: #ff6600; \--purple-color: #cc33ff; } blockquote { font-weight: unset; letter-spacing: normal; font-style: normal; text-align: inherit; } table { border: 1px solid; border-collapse: collapse; white-space: unset !important; width=200px !important; } table td { border: 1px solid; border-collapse: collapse; } .theme-dark .CT-thumb { transform: translate(20px,0px); } .theme-light .CT-thumb { transform: translate(0px,0px); } /\\*.u-text-format h3 { color: var(--darker-gray-color) ; }\\*/ //Detect theme var existingTheme = localStorage.getItem('dawn\\_theme'); if (!existingTheme) { if(navigator.platform.indexOf('Win') > -1) { existingTheme = "light"; } else if(window.matchMedia && window.matchMedia('(prefers-color-scheme: dark)').matches) { existingTheme = "dark"; } else if(window.matchMedia && window.matchMedia('(prefers-color-scheme: light)').matches) { existingTheme = "light"; } } //Implement light/dark theme if(existingTheme == "dark") { ctdark(); } else if(existingTheme == 'light') { ctlight(); } window.dataLayer = window.dataLayer || []; function gtag(){dataLayer.push(arguments);} gtag('js', new Date()); gtag('config', 'G-TZK0WDB1DH'); var clicky\\_site\\_ids = clicky\\_site\\_ids || []; clicky\\_site\\_ids.push(101323562); ![Clicky](//in.getclicky.com/101323562ns.gif) :root {--ghost-accent-color: #888888;} [Cold Takes](https://www.cold-takes.com) [About (+highlights)](https://www.cold-takes.com/about/) [Most Important Century](https://www.cold-takes.com/most-important-century/) [About (+highlights)](https://www.cold-takes.com/about/) [Most Important Century](https://www.cold-takes.com/most-important-century/) [Subscribe (free)](#subscribe) [Subscribe (free)](#subscribe) [Subscribe (free)](#subscribe) [Cold Takes](https://www.cold-takes.com) - Phil Birnbaum's "bad regression" puzzles By [Holden Karnofsky](https://www.cold-takes.com/about/ "Holden Karnofsky") Jul 15, 2021 3 min read Phil Birnbaum's "bad regression" puzzles ======================================== Painstaking, even beautiful deconstructions of misleading regressions. If you've ever wanted to see someone painstakingly deconstruct a regression analysis and show all the subtle reasons it can generate wild, weird and completely wrong results, there is good stuff at [Sabermetric Research - Phil Birnbaum](http://blog.philbirnbaum.com/). It's a sports blog, but sports knowledge isn't needed (knowledge of regression analysis generally is, if you want to follow all the details). Birnbaum's not exactly the only person to do takedowns of bad studies. But when Birnbaum notices that something is "off," he doesn't just point it out and move on. He isn't satisfied with "This conclusion is implausible" or "This conclusion isn't robust to sensitivity analysis." He digs all the way to the bottom to understand *exactly* how a study got its wrong result. His deconstructions of bad regressions are like four-star meals or masterful jazz solos ... I don't want to besmirch them by trying to explain them, so if you're into regression deconstructions you should just click through the links below. (I'm not going to explain what regression analysis is today, for which I apologize; if I ever do, I will link back to this post. It's very hard to explain it compactly and clearly, as you can see from [Wikipedia's attempt](https://en.wikipedia.org/wiki/Regression_analysis), but it is VERY common in social science research. Kind of a bad combination IMO. If you hear "This study shows [something about people]," it's more likely than not that the study relies on regression analysis.) Some good (old) ones: * [Estimating whether Aaron Rodger's contract overpays or underpays him by making a scatterplot of pay and value-added with other quarterbacks and seeing whether he's above or below the regression line.](https://web.archive.org/web/20130524132245/http://www.advancednflstats.com/2013/05/point-counterpoint-on-rodgers-extension.html) The answer changes completely when you switch the x- and y-axes. Which one is right, and what *exactly* is wrong with the other one? (Birnbaum linked to this, but it's now dead and I am linking directly to an Internet Archive version. Birnbaum's "solution" is down in the comments, just search for his name.) * [Deconstructing an NBA time-zone regression](http://blog.philbirnbaum.com/2017/08/deconstructing-nba-time-zone-regression.html): the key coefficients turn out to be literally meaningless. * Do younger brothers steal more bases? Parts [I](http://blog.philbirnbaum.com/2010/05/do-younger-brothers-steal-more-bases.html), [II](http://blog.philbirnbaum.com/2010/06/do-younger-brothers-steal-more-bases.html), [III](http://blog.philbirnbaum.com/2010/06/do-younger-brothers-steal-more-bases_18.html) although I think it's OK to skip part II (and Part I is short). * The OBP/SLG regression puzzle: parts [I](http://blog.philbirnbaum.com/2013/05/the-obpslg-regression-puzzle.html), [II](http://blog.philbirnbaum.com/2013/05/the-obpslg-regression-puzzle-part-ii.html), [III](http://blog.philbirnbaum.com/2013/05/the-obpslg-regression-puzzle-part-iii.html), [IV](http://blog.philbirnbaum.com/2013/06/the-obpslg-regression-puzzle-part-iv.html), [V](http://blog.philbirnbaum.com/2013/06/the-obpslg-regression-puzzle-part-v.html). This one is very weedsy and you'll probably want to skip parts, though it's also kind of glorious to see just how doggedly he digs on every strange numerical result. He also makes an effort to explain what's going on for people who don't know baseball. The essence of the puzzle: OBP and SLG are both indicators of a team's performance, but when one regresses a team's performance on OBP and SLG, the ratio of the coefficients is pretty far off from what the "true" value for (value of OBP / value of SLG) is separately known to be. I think the issues here are extremely general, and not good news for the practice of treating regression coefficients as effect sizes. [![Twitter](https://www.cold-takes.com/content/images/2021/06/ct-twitter-square.png)](https://api.addthis.com/oexchange/0.8/forward/twitter/offer?url=https%3A%2F%2Fwww.cold-takes.com%2Fphil-birnbaums-regression-analysis&pubid;=ra-60a178324cffc42e&title;=Cold%20Takes%20-%20Phil%20Birnbaum's%20%22bad%20regression%22%20puzzles&ct;=1) [![Facebook](https://www.cold-takes.com/content/images/2021/06/ct-facebook-square.png)](https://api.addthis.com/oexchange/0.8/forward/facebook/offer?url=https%3A%2F%2Fwww.cold-takes.com%2Fphil-birnbaums-regression-analysis&pubid;=ra-60a178324cffc42e&title;=Cold%20Takes%20-%20Phil%20Birnbaum's%20%22bad%20regression%22%20puzzles&ct;=1) [![Reddit](https://www.cold-takes.com/content/images/2021/06/ct-reddit-square.png)](https://api.addthis.com/oexchange/0.8/forward/reddit/offer?url=https%3A%2F%2Fwww.cold-takes.com%2Fphil-birnbaums-regression-analysis&pubid;=ra-60a178324cffc42e&title;=Cold%20Takes%20-%20Phil%20Birnbaum's%20%22bad%20regression%22%20puzzles&ct;=1) [![More](https://www.cold-takes.com/content/images/2021/06/ct-addthis-square.png)](https://api.addthis.com/oexchange/0.8/forward/menu/offer?url=https%3A%2F%2Fwww.cold-takes.com%2Fphil-birnbaums-regression-analysis&pubid;=ra-60a178324cffc42e&title;=Cold%20Takes%20-%20Phil%20Birnbaum's%20%22bad%20regression%22%20puzzles&ct;=1) [Subscribe](https://www.cold-takes.com/phil-birnbaums-regression-analysis/#/portal/signup/free) [Feedback](https://www.guidedtrack.com/programs/4kal2ue/run?posttitle=Phil%20Birnbaum's%20%22bad%20regression%22%20puzzles) Powered by [Ghost](https://ghost.org/) #### Popular tags [sports](/tag/sports/) [innovationstagnation](/tag/innovationstagnation/) [2star](/tag/2star/) [utopia](/tag/utopia/) Subscribe to Cold Takes ======================= For audio version, search for "Cold Takes Audio" in your podcast app Subscribe **Great!** Check your inbox and click the link to confirm your subscription. **If you don't see it, check your spam!** Please enter a valid email address! $("#lower-right-subscribe").click(function(f) { clicky.goal('1458'); /\\*LO.events.track('Subscribe-LR');\\*/ setTimeout(function(){$("#subscribe-input").focus();},500); }); $("#upper-right-subscribe-1").click(function(f) { clicky.goal('1471'); /\\*LO.events.track('Subscribe-LR');\\*/ setTimeout(function(){$("#subscribe-input").focus();},500); }); $("#upper-right-subscribe-2").click(function(f) { clicky.goal('1471'); /\\*LO.events.track('Subscribe-LR');\\*/ setTimeout(function(){$("#subscribe-input").focus();},500); }); $("#cover-subscribe").click(function(f) { clicky.goal('1472'); /\\*LO.events.track('Subscribe-LR');\\*/ setTimeout(function(){$("#subscribe-input").focus();},500); }); $("#footer-subscribe").click(function(f) { clicky.goal('1473'); /\\*LO.events.track('Subscribe-LR');\\*/ setTimeout(function(){$("#subscribe-input").focus();},500); }); $("#final-subscribe").click(function(f) { clicky.goal('1474'); }); //When someone clicks an anchor link, I want the sticky nav to go away. $("a").click(function(f) { body.removeClass('sticky-visible'); lastSt = 9999999999999}); function ctlight() { html.removeClass('theme-dark').addClass('theme-light'); localStorage.setItem('dawn\\_theme', 'light'); //$(".gh-portal-triggerbtn-container").css("background","#0000dd"); /\\* document.documentElement.style.setProperty('--brand-color', '#0000dd'); document.documentElement.style.setProperty('--white-color', '#ffffff'); document.documentElement.style.setProperty('--primary-text-color', '#1a1a1a'); document.documentElement.style.setProperty('--secondary-text-color', '#1a1a1a');\\*/ // $("#CT-thumbID").css("transform","translate(0px,0)"); // $(".single-content").css("font-weight","400"); existingTheme = "light"; } function ctdark() { html.removeClass('theme-light').addClass('theme-dark'); localStorage.setItem('dawn\\_theme', 'dark'); /\\*document.documentElement.style.setProperty('--brand-color', '#5ceaff'); document.documentElement.style.setProperty('--white-color', '#18141f'); document.documentElement.style.setProperty('--primary-text-color', '#fff'); document.documentElement.style.setProperty('--secondary-text-color', '#fff');\\*/ //document.documentElement.style.setProperty('--white-color', '#14101b'); //document.documentElement.style.setProperty('--white-color', '#121212'); //document.documentElement.style.setProperty('--white-color', '#282c35'); //$(".single-content").css("font-weight","300"); // $("#CT-thumbID").css("transform","translate(19px,0)"); existingTheme = "dark"; } //For light/dark toggle $(".CT-toggle").click(function(g) { g.preventDefault(); if(existingTheme == "dark") { ctlight(); } else if(existingTheme == 'light') { ctdark(); } } ); // Parse the URL parameter, then log goal if it is a completed subscription. Weird that it takes this much code to do this. function getParameterByName(name, url) { if (!url) url = window.location.href; name = name.replace(/[\\[\\]]/g, "\\\$&"); var regex = new RegExp("[?&]" + name + "(=([^&#]\\*)|&|#|$)"), results = regex.exec(url); if (!results) return null; if (!results[2]) return ''; return decodeURIComponent(results[2].replace(/\\+/g, " ")); } // Give the parameter a variable name var action = getParameterByName('action'); var success = getParameterByName('success'); $(document).ready(function () { // Success === null is here for backwards compatibility if (action == 'subscribe' && (success === null || success == 'true')) { clicky.goal('1476'); } });