Call to Vigilance var siteUrl = 'https://www.cold-takes.com'; var localTheme = localStorage.getItem('dawn\_theme'); switch (localTheme) { case 'dark': document.documentElement.classList.add('theme-dark'); break; case 'light': document.documentElement.classList.add('theme-light'); break; default: break; } { "@context": "https://schema.org", "@type": "Article", "publisher": { "@type": "Organization", "name": "Cold Takes", "url": "https://www.cold-takes.com/", "logo": { "@type": "ImageObject", "url": "https://www.cold-takes.com/favicon.ico", "width": 48, "height": 48 } }, "author": { "@type": "Person", "name": "Holden Karnofsky", "image": { "@type": "ImageObject", "url": "https://www.cold-takes.com/content/images/2021/07/HK-headshot-current---compressed-1.jpg", "width": 1600, "height": 1067 }, "url": "https://www.cold-takes.com/author/holden/", "sameAs": [ "https://www.cold-takes.com/about/" ] }, "headline": "Call to Vigilance", "url": "https://www.cold-takes.com/call-to-vigilance/", "datePublished": "2021-09-15T18:46:27.000Z", "dateModified": "2021-12-15T10:27:19.000Z", "image": { "@type": "ImageObject", "url": "https://www.cold-takes.com/content/images/2021/09/airplane-launch.gif", "width": 569, "height": 320 }, "description": "Click lower right to download or find on Apple Podcasts, Spotify, Stitcher, etc.

Today's worldTransformative AIDigital peopleWorld ofMisaligned AIWorld run by
Something elseoror Stable, galaxy-widecivilization
[https://www.cold-takes.com/roadmap-for-the-most-important-century-series/]This
is the final piece in the "most important century" series
[https://www.cold-takes.com/roadmap-for-the-most-important-century-series/],
which has argued that there's a high probability
[https://www.cold-takes.c", "mainEntityOfPage": { "@type": "WebPage", "@id": "https://www.cold-takes.com/" } } .gh-post-upgrade-cta-content, .gh-post-upgrade-cta { display: flex; flex-direction: column; align-items: center; font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, Oxygen, Ubuntu, Cantarell, 'Open Sans', 'Helvetica Neue', sans-serif; text-align: center; width: 100%; color: #ffffff; font-size: 16px; } .gh-post-upgrade-cta-content { border-radius: 8px; padding: 40px 4vw; } .gh-post-upgrade-cta h2 { color: #ffffff; font-size: 28px; letter-spacing: -0.2px; margin: 0; padding: 0; } .gh-post-upgrade-cta p { margin: 20px 0 0; padding: 0; } .gh-post-upgrade-cta small { font-size: 16px; letter-spacing: -0.2px; } .gh-post-upgrade-cta a { color: #ffffff; cursor: pointer; font-weight: 500; box-shadow: none; text-decoration: underline; } .gh-post-upgrade-cta a:hover { color: #ffffff; opacity: 0.8; box-shadow: none; text-decoration: underline; } .gh-post-upgrade-cta a.gh-btn { display: block; background: #ffffff; text-decoration: none; margin: 28px 0 0; padding: 8px 18px; border-radius: 4px; font-size: 16px; font-weight: 600; } .gh-post-upgrade-cta a.gh-btn:hover { opacity: 0.92; } .shaded-table { color: #000000; border: 1px solid; vertical-align: top; } .shaded-table a { color: #0000dd; } .sticky-subscribe-div { z-index: 1000; position: fixed; bottom: 25px; right: 25px; max-width: 500px; overflow: hidden; opacity: 1; } .sticky-subscribe-button { cursor: pointer; padding: 20px; background: var(--brand-color); color: var(--white-color); border-radius: 50px; border-style: none; } .subscribe-overlay { position: fixed; top: 0; right: 0; bottom: 0; left: 0; z-index: 2000; display: flex; justify-content: center; align-items: center; background: var(--white-color); opacity: 0; transition: opacity 0.3s ease-in; pointer-events: none; } .subscribe-overlay:target { z-index: 2001; opacity: 0.95; pointer-events: auto; } .subscribe-form { margin: 1.5em 0; padding: 6.5vh 7vw 8vh; border-style: solid !important; border: var(--mid-gray-color); border-radius: 10px; text-align: center; } .subscribe-overlay-title { display: inline-block; margin: 0 0 10px 0; font-size: 3rem; line-height: 1.15em; color: var(--darker-gray-color); } .subscribe-overlay-description { margin: 0 auto 50px; max-width: 500px; color: var(--darker-gray-color); font-size: 2rem; line-height: 1.3em; opacity: 0.8; } .subscribe-form .form-group { align-self: stretch; display: flex; } .subscribe-email { padding: 10px; border: 1px solid; border-color: var(--darker-gray-color); font-size: 1.8rem; line-height: 1em; font-weight: normal; user-select: text; transition: border-color 0.15s linear; -webkit-appearance: none; } .subscribe-form .subscribe-overlay-button { word-break: normal; position: relative; display: inline-block; margin: 0 0 0 10px; padding: 0 20px; height: 43px; outline: none; font-size: 1.5rem; line-height: 39px; font-weight: 400; background: var(--brand-color); border-radius: 5px; color: var(--white-color); } .subscribe-form .button-loader, .subscribe-form .message-success, .subscribe-form .message-error { display: none; } .subscribe-form .loading .button-content { visibility: hidden; } .subscribe-form .loading .button-loader { position: absolute; top: 0; left: 50%; display: inline-block; margin-left: -19px; transform: scale(0.7); } .subscribe-overlay .loading .button-loader { top: 5px; } .subscribe-form .button-loader svg path, .subscribe-form .button-loader svg rect { fill: var(--white-color); } .subscribe-form .success .message-success, .subscribe-form .invalid .message-error, .subscribe-form .error .message-error { display: block; margin: 1em auto 0; max-width: 400px; color: var(--red-color); font-size: 1.6rem; line-height: 1.5em; text-align: center; } .subscribe-form .success .message-success { display: block; color: var(--darker-gray-color); } .subscribe-form .invalid .message-error, .subscribe-form .error .message-error { display: block; } @media (max-width: 650px) { .subscribe-form-title { font-size: 2.4rem; } .subscribe-form-description { font-size: 1.6rem; } } @media (max-width: 500px) { .subscribe-form form { display: flex; flex-direction: column; } .subscribe-form .subscribe-overlay-button { margin: 0 0 0 0; } .subscribe-form .form-group { display: flex; flex-direction: column; width: 100%; } .subscribe-form button { display: flex; margin: 0 0 0 0; width: 100%; } } .subscribe-close-button { position: absolute; top: 16px; right: 20px; z-index: 2000; display: block; width: 40px; height: 40px; } .subscribe-close-button:before { content: ""; position: absolute; top: 20px; right: 4px; display: block; width: 32px; height: 1px; background: var(--darker-gray-color); opacity: 0.8; transform: rotate(45deg); } .subscribe-close-button:after { content: ""; position: absolute; top: 20px; right: 4px; display: block; width: 32px; height: 1px; background: var(--darker-gray-color); opacity: 0.8; transform: rotate(-45deg); } .subscribe-close-button:hover { cursor: default; } /\*.message-success, .message-error, .message-loading { display: none; }\*/ /\*.message-success .success { visibility: visible; display: block; opacity: 1; } .message-error.error { visibility: visible; display: block; opacity: 1; }\*/ /\*.success .message-success, .error .message-error, .loading .message-loading { visibility: visible; display: block; opacity: 1; }\*/ /\*For TOC\*/ @media (min-width: 1200px) { .sticky-visible .sticky-toc { visibility: visible; opacity: 1; transform: translateY(0); } .toc-opened .sticky-toc { visibility: hidden; } } /\*.gh-portal-triggerbtn-iframe { } .gh-portal-triggerbtn-wrapper { background: #0000ff !important; } .gh-portal-triggerbtn-container { background: #0000ff !important; height: 10px !important; } \*/ .theme-dark:root { \--green-color: #66ff33 !important; \--orange-color: #ff5c33 !important; \--red-color: #ff3333 !important; \--purple-color: #ff00ff !important; \--brand-color: #5ceaff !important; \--white-color: #18141f !important; \--primary-text-color: #fff !important; \--secondary-text-color: #fff !important; } .feed-visibility-false { display: none; } /\*.feed-visibility-true { display: none; }\*/ :root { \--brand-color:#0000dd; \--white-color:#ffffff; \--primary-text-color:#1a1a1a; \--secondary-text-color:#1a1a1a; \--orange-color: #ff6600; \--purple-color: #cc33ff; } blockquote { font-weight: unset; letter-spacing: normal; font-style: normal; text-align: inherit; } table { border: 1px solid; border-collapse: collapse; white-space: unset !important; width=200px !important; } table td { border: 1px solid; border-collapse: collapse; } .theme-dark .CT-thumb { transform: translate(20px,0px); } .theme-light .CT-thumb { transform: translate(0px,0px); } /\*.u-text-format h3 { color: var(--darker-gray-color) ; }\*/ //Detect theme var existingTheme = localStorage.getItem('dawn\_theme'); if (!existingTheme) { if(navigator.platform.indexOf('Win') > -1) { existingTheme = "light"; } else if(window.matchMedia && window.matchMedia('(prefers-color-scheme: dark)').matches) { existingTheme = "dark"; } else if(window.matchMedia && window.matchMedia('(prefers-color-scheme: light)').matches) { existingTheme = "light"; } } //Implement light/dark theme if(existingTheme == "dark") { ctdark(); } else if(existingTheme == 'light') { ctlight(); } window.dataLayer = window.dataLayer || []; function gtag(){dataLayer.push(arguments);} gtag('js', new Date()); gtag('config', 'G-TZK0WDB1DH'); var clicky\_site\_ids = clicky\_site\_ids || []; clicky\_site\_ids.push(101323562); ![Clicky](//in.getclicky.com/101323562ns.gif) :root {--ghost-accent-color: #888888;} [Cold Takes](https://www.cold-takes.com) [About (+highlights)](https://www.cold-takes.com/about/) [Most Important Century](https://www.cold-takes.com/most-important-century/) [About (+highlights)](https://www.cold-takes.com/about/) [Most Important Century](https://www.cold-takes.com/most-important-century/) [Subscribe (free)](#subscribe) [Subscribe (free)](#subscribe) [Subscribe (free)](#subscribe) [Cold Takes](https://www.cold-takes.com) - Call to Vigilance By [Holden Karnofsky](https://www.cold-takes.com/about/ "Holden Karnofsky") Sep 15, 2021 5 min read Call to Vigilance ================= *Click lower right to download or find on Apple Podcasts, Spotify, Stitcher, etc.* [Today's world Transformative AI Digital people World of Misaligned AI World run by Something else or or Stable, galaxy-wide civilization](https://www.cold-takes.com/roadmap-for-the-most-important-century-series/) This is the final piece in the ["most important century" series](https://www.cold-takes.com/roadmap-for-the-most-important-century-series/), which has argued that there's a [high probability](https://www.cold-takes.com/forecasting-transformative-ai-whats-the-burden-of-proof/#some-rough-probabilities)[1](#fn1) that the coming decades will see: \- The development of a technology like [PASTA](https://www.cold-takes.com/transformative-ai-timelines-part-1-of-4-what-kind-of-ai/) (process for automating scientific and technological advancement). \- A resulting [productivity explosion](https://www.cold-takes.com/transformative-ai-timelines-part-1-of-4-what-kind-of-ai/#explosive-scientific-and-technological-advancement) leading to development of further transformative technologies. \- The seed of a [stable galaxy-wide civilization](https://www.cold-takes.com/all-possible-views-about-humanitys-future-are-wild/), possibly featuring [digital people](https://www.cold-takes.com/how-digital-people-could-change-the-world/), or possibly run by [misaligned AI](https://www.cold-takes.com/transformative-ai-timelines-part-1-of-4-what-kind-of-ai/#misaligned-ai-mysterious-potentially-dangerous-objectives). When trying to call attention to an underrated problem, **it's typical to close on a "call to action":** a tangible, concrete action readers can take to help. But this is challenging, because as I argued [previously](https://www.cold-takes.com/p/f5eba675-6b0e-4f19-b2ad-b63a6bafc8fd/), there are a lot of [open questions about what actions are helpful vs. harmful](https://www.cold-takes.com/p/f5eba675-6b0e-4f19-b2ad-b63a6bafc8fd/#key-open-questions-for-). (Although we can identify some [actions that seem robustly helpful today](https://www.cold-takes.com/p/f5eba675-6b0e-4f19-b2ad-b63a6bafc8fd/#robustly-helpful-actions).) This makes for a somewhat awkward situation. When confronting the "most important century" hypothesis, my attitude doesn't match the familiar ones of "excitement and motion" or "fear and avoidance." Instead, I feel an **odd mix of intensity, urgency, confusion and hesitance.** I'm looking at something bigger than I ever expected to confront, feeling underqualified and ignorant about what to do next. This is a hard mood to share and spread, but I'm trying. | | | | --- | --- | | **Situation** | **Appropriate reaction (IMO)** | | "This could be a billion-dollar company!" | "Woohoo, let's GO for it!" | | "This could be the most important century!" | "... Oh ... wow ... I don't know what to say and I somewhat want to vomit ... I have to sit down and think about this one." | So instead of a call to action, I want to make a **call to vigilance.** If you're convinced by the arguments in this piece, then don't rush to "do something" and then move on. Instead, take whatever [robustly good actions](https://www.cold-takes.com/p/f5eba675-6b0e-4f19-b2ad-b63a6bafc8fd/#robustly-helpful-actions) you can today, and otherwise put yourself in a better position to take important actions when the time comes. This could mean: * Finding ways to interact more with, and learn more about, key topics/fields/industries such as AI (for obvious reasons), science and technology generally (as a lot of the "most important century" hypothesis runs through an [explosion in scientific and technological advancement](https://www.cold-takes.com/transformative-ai-timelines-part-1-of-4-what-kind-of-ai/#impacts-of-pasta)), and relevant areas of policy and national security. * Taking opportunities (when you see them) to move your career in a direction that is more likely to be relevant (some thoughts of mine on this are [here](https://forum.effectivealtruism.org/posts/bud2ssJLQ33pSemKH/my-current-impressions-on-career-choice-for-longtermists); also see [80,000 Hours](https://www.80000hours.org)). * Connecting with other people interested in these topics (I believe this has been one of the biggest drivers of people coming to do high-impact work in the past). Currently, I think the [effective altruism](https://en.wikipedia.org/wiki/Effective_altruism) community is the best venue for this, and you can learn about how to connect with people via the [Centre for Effective Altruism](https://www.centreforeffectivealtruism.org/) (see the "Get involved" dropdown). If new ways of connecting with people come up in the future, I will likely post them on Cold Takes. * And of course, taking any opportunities you see for [robustly helpful actions.](https://www.cold-takes.com/p/f5eba675-6b0e-4f19-b2ad-b63a6bafc8fd/#robustly-helpful-actions) Buttons you can click \--------------------- Here's something you can do right now that would be genuinely helpful, though maybe not as viscerally satisfying as signing a petition or making a donation. In my [day job](https://www.openphilanthropy.org), I have a lot of moments where I - or someone I'm working with - is looking for a particular kind of person (perhaps to fill a job opening with a grantee, or to lend expertise on some topic, or something else). Over time, I expect there to be more and more opportunities for people with specific skills, interests, expertise, etc. to take actions that [help make the best of the most important century](https://www.cold-takes.com/p/f5eba675-6b0e-4f19-b2ad-b63a6bafc8fd/). And I think a major challenge will simply be **knowing who's out there** - who's interested in this cause, and wants to help, and what skills and interests they have. If you're a person we might wish we could find in the future, you can help now by sending in information about yourself via **[this simple form](https://forms.gle/z7mexiTd6wCJsuEv6)**. I vouch that your information won't be sold or otherwise used to make money, that your communication preferences (which the form asks about in detail) will be respected, and that you'll always be able to opt out of any communications. Sharing a headspace \------------------- In [This Can't Go On](https://www.cold-takes.com/this-cant-go-on/), I analogized the world to people on a plane blasting down the runway, without knowing why they're moving so fast or what's coming next: ![Animated image of the view out the airplane window as it blasts down the runway.](https://www.cold-takes.com/content/images/2021/07/airplane-launch-compressed.gif) As someone sitting on this plane, I'd love to be able to tell you I've figured out exactly what's going on and what future we need to be planning for. But I haven't. Lacking answers, I've tried to at least show you what I do see: * Dim outlines of the most important events in humanity's past or future. * A case that they're approaching us more quickly than it seems - whether or not we're ready. * A sense that the world and the rules we're all used to can't be relied on. That we need to lift our gaze above the daily torrent of tangible, relatable news - and try to wrap our heads around weirder, wilder matters that are more likely to be seen as the **headlines about this era billions of years from now.** There's a lot I don't know. But if this is the most important century, I do feel confident that we as a civilization aren't yet up to the challenges it presents. If that's going to change, it needs to start with more people seeing the situation for what it is, taking it seriously, taking action when they can - and when not, staying vigilant. [![Twitter](https://www.cold-takes.com/content/images/2021/06/ct-twitter-square.png)](https://api.addthis.com/oexchange/0.8/forward/twitter/offer?url=https%3A%2F%2Fwww.cold-takes.com%2Fcall-to-vigilance%2F&pubid;=ra-60a178324cffc42e&title;=Cold%20Takes%20-%20Call%20to%20Vigilance&ct;=1) [![Facebook](https://www.cold-takes.com/content/images/2021/06/ct-facebook-square.png)](https://api.addthis.com/oexchange/0.8/forward/facebook/offer?url=https%3A%2F%2Fwww.cold-takes.com%2Fcall-to-vigilance%2F&pubid;=ra-60a178324cffc42e&title;=Cold%20Takes%20-%20Call%20to%20Vigilance&ct;=1) [![Reddit](https://www.cold-takes.com/content/images/2021/06/ct-reddit-square.png)](https://api.addthis.com/oexchange/0.8/forward/reddit/offer?url=https%3A%2F%2Fwww.cold-takes.com%2Fcall-to-vigilance%2F&pubid;=ra-60a178324cffc42e&title;=Cold%20Takes%20-%20Call%20to%20Vigilance&ct;=1) [![More](https://www.cold-takes.com/content/images/2021/06/ct-addthis-square.png)](https://api.addthis.com/oexchange/0.8/forward/menu/offer?url=https%3A%2F%2Fwww.cold-takes.com%2Fcall-to-vigilance%2F&pubid;=ra-60a178324cffc42e&title;=Cold%20Takes%20-%20Call%20to%20Vigilance&ct;=1) [Subscribe](https://www.cold-takes.com/call-to-vigilance/#/portal/signup/free) [Feedback](https://www.guidedtrack.com/programs/4kal2ue/run?posttitle=Call%20to%20Vigilance) [Forum](https://forum.effectivealtruism.org/posts/uNczpTHFvjt4KNxEF/call-to-vigilance#comments)*Use "Feedback" if you have comments/suggestions you want me to see, or if you're up for giving some quick feedback about this post (which I greatly appreciate!) Use "Forum" if you want to discuss this post publicly on the Effective Altruism Forum.* \--- Footnotes \--------- 1\. "I am forecasting more than a 10% chance transformative AI will be developed within 15 years (by 2036); a ~50% chance it will be developed within 40 years (by 2060); and a ~2/3 chance it will be developed this century (by 2100)." [↩](#fnref1) Powered by [Ghost](https://ghost.org/) #### Popular tags [sports](/tag/sports/) [innovationstagnation](/tag/innovationstagnation/) [2star](/tag/2star/) [utopia](/tag/utopia/) Subscribe to Cold Takes ======================= For audio version, search for "Cold Takes Audio" in your podcast app Subscribe **Great!** Check your inbox and click the link to confirm your subscription. **If you don't see it, check your spam!** Please enter a valid email address! $("#lower-right-subscribe").click(function(f) { clicky.goal('1458'); /\*LO.events.track('Subscribe-LR');\*/ setTimeout(function(){$("#subscribe-input").focus();},500); }); $("#upper-right-subscribe-1").click(function(f) { clicky.goal('1471'); /\*LO.events.track('Subscribe-LR');\*/ setTimeout(function(){$("#subscribe-input").focus();},500); }); $("#upper-right-subscribe-2").click(function(f) { clicky.goal('1471'); /\*LO.events.track('Subscribe-LR');\*/ setTimeout(function(){$("#subscribe-input").focus();},500); }); $("#cover-subscribe").click(function(f) { clicky.goal('1472'); /\*LO.events.track('Subscribe-LR');\*/ setTimeout(function(){$("#subscribe-input").focus();},500); }); $("#footer-subscribe").click(function(f) { clicky.goal('1473'); /\*LO.events.track('Subscribe-LR');\*/ setTimeout(function(){$("#subscribe-input").focus();},500); }); $("#final-subscribe").click(function(f) { clicky.goal('1474'); }); //When someone clicks an anchor link, I want the sticky nav to go away. $("a").click(function(f) { body.removeClass('sticky-visible'); lastSt = 9999999999999}); function ctlight() { html.removeClass('theme-dark').addClass('theme-light'); localStorage.setItem('dawn\_theme', 'light'); //$(".gh-portal-triggerbtn-container").css("background","#0000dd"); /\* document.documentElement.style.setProperty('--brand-color', '#0000dd'); document.documentElement.style.setProperty('--white-color', '#ffffff'); document.documentElement.style.setProperty('--primary-text-color', '#1a1a1a'); document.documentElement.style.setProperty('--secondary-text-color', '#1a1a1a');\*/ // $("#CT-thumbID").css("transform","translate(0px,0)"); // $(".single-content").css("font-weight","400"); existingTheme = "light"; } function ctdark() { html.removeClass('theme-light').addClass('theme-dark'); localStorage.setItem('dawn\_theme', 'dark'); /\*document.documentElement.style.setProperty('--brand-color', '#5ceaff'); document.documentElement.style.setProperty('--white-color', '#18141f'); document.documentElement.style.setProperty('--primary-text-color', '#fff'); document.documentElement.style.setProperty('--secondary-text-color', '#fff');\*/ //document.documentElement.style.setProperty('--white-color', '#14101b'); //document.documentElement.style.setProperty('--white-color', '#121212'); //document.documentElement.style.setProperty('--white-color', '#282c35'); //$(".single-content").css("font-weight","300"); // $("#CT-thumbID").css("transform","translate(19px,0)"); existingTheme = "dark"; } //For light/dark toggle $(".CT-toggle").click(function(g) { g.preventDefault(); if(existingTheme == "dark") { ctlight(); } else if(existingTheme == 'light') { ctdark(); } } ); // Parse the URL parameter, then log goal if it is a completed subscription. Weird that it takes this much code to do this. function getParameterByName(name, url) { if (!url) url = window.location.href; name = name.replace(/[\[\]]/g, "\\$&"); var regex = new RegExp("[?&]" + name + "(=([^&#]\*)|&|#|$)"), results = regex.exec(url); if (!results) return null; if (!results[2]) return ''; return decodeURIComponent(results[2].replace(/\+/g, " ")); } // Give the parameter a variable name var action = getParameterByName('action'); var success = getParameterByName('success'); $(document).ready(function () { // Success === null is here for backwards compatibility if (action == 'subscribe' && (success === null || success == 'true')) { clicky.goal('1476'); } });

Call to Vigilance var siteUrl = 'https://www.cold-takes.com'; var localTheme = localStorage.getItem('dawn\\_theme'); switch (localTheme) { case 'dark': document.documentElement.classList.add('theme-dark'); break; case 'light': document.documentElement.classList.add('theme-light'); break; default: break; } { "@context": "https://schema.org", "@type": "Article", "publisher": { "@type": "Organization", "name": "Cold Takes", "url": "https://www.cold-takes.com/", "logo": { "@type": "ImageObject", "url": "https://www.cold-takes.com/favicon.ico", "width": 48, "height": 48 } }, "author": { "@type": "Person", "name": "Holden Karnofsky", "image": { "@type": "ImageObject", "url": "https://www.cold-takes.com/content/images/2021/07/HK-headshot-current---compressed-1.jpg", "width": 1600, "height": 1067 }, "url": "https://www.cold-takes.com/author/holden/", "sameAs": [ "https://www.cold-takes.com/about/" ] }, "headline": "Call to Vigilance", "url": "https://www.cold-takes.com/call-to-vigilance/", "datePublished": "2021-09-15T18:46:27.000Z", "dateModified": "2021-12-15T10:27:19.000Z", "image": { "@type": "ImageObject", "url": "https://www.cold-takes.com/content/images/2021/09/airplane-launch.gif", "width": 569, "height": 320 }, "description": "Click lower right to download or find on Apple Podcasts, Spotify, Stitcher, etc.\n\nToday’s worldTransformative AIDigital peopleWorld ofMisaligned AIWorld run by\nSomething elseoror Stable, galaxy-widecivilization\n[https://www.cold-takes.com/roadmap-for-the-most-important-century-series/]This\nis the final piece in the "most important century" series\n[https://www.cold-takes.com/roadmap-for-the-most-important-century-series/],\nwhich has argued that there's a high probability\n[https://www.cold-takes.c", "mainEntityOfPage": { "@type": "WebPage", "@id": "https://www.cold-takes.com/" } } .gh-post-upgrade-cta-content, .gh-post-upgrade-cta { display: flex; flex-direction: column; align-items: center; font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, Oxygen, Ubuntu, Cantarell, 'Open Sans', 'Helvetica Neue', sans-serif; text-align: center; width: 100%; color: #ffffff; font-size: 16px; } .gh-post-upgrade-cta-content { border-radius: 8px; padding: 40px 4vw; } .gh-post-upgrade-cta h2 { color: #ffffff; font-size: 28px; letter-spacing: -0.2px; margin: 0; padding: 0; } .gh-post-upgrade-cta p { margin: 20px 0 0; padding: 0; } .gh-post-upgrade-cta small { font-size: 16px; letter-spacing: -0.2px; } .gh-post-upgrade-cta a { color: #ffffff; cursor: pointer; font-weight: 500; box-shadow: none; text-decoration: underline; } .gh-post-upgrade-cta a:hover { color: #ffffff; opacity: 0.8; box-shadow: none; text-decoration: underline; } .gh-post-upgrade-cta a.gh-btn { display: block; background: #ffffff; text-decoration: none; margin: 28px 0 0; padding: 8px 18px; border-radius: 4px; font-size: 16px; font-weight: 600; } .gh-post-upgrade-cta a.gh-btn:hover { opacity: 0.92; } .shaded-table { color: #000000; border: 1px solid; vertical-align: top; } .shaded-table a { color: #0000dd; } .sticky-subscribe-div { z-index: 1000; position: fixed; bottom: 25px; right: 25px; max-width: 500px; overflow: hidden; opacity: 1; } .sticky-subscribe-button { cursor: pointer; padding: 20px; background: var(--brand-color); color: var(--white-color); border-radius: 50px; border-style: none; } .subscribe-overlay { position: fixed; top: 0; right: 0; bottom: 0; left: 0; z-index: 2000; display: flex; justify-content: center; align-items: center; background: var(--white-color); opacity: 0; transition: opacity 0.3s ease-in; pointer-events: none; } .subscribe-overlay:target { z-index: 2001; opacity: 0.95; pointer-events: auto; } .subscribe-form { margin: 1.5em 0; padding: 6.5vh 7vw 8vh; border-style: solid !important; border: var(--mid-gray-color); border-radius: 10px; text-align: center; } .subscribe-overlay-title { display: inline-block; margin: 0 0 10px 0; font-size: 3rem; line-height: 1.15em; color: var(--darker-gray-color); } .subscribe-overlay-description { margin: 0 auto 50px; max-width: 500px; color: var(--darker-gray-color); font-size: 2rem; line-height: 1.3em; opacity: 0.8; } .subscribe-form .form-group { align-self: stretch; display: flex; } .subscribe-email { padding: 10px; border: 1px solid; border-color: var(--darker-gray-color); font-size: 1.8rem; line-height: 1em; font-weight: normal; user-select: text; transition: border-color 0.15s linear; -webkit-appearance: none; } .subscribe-form .subscribe-overlay-button { word-break: normal; position: relative; display: inline-block; margin: 0 0 0 10px; padding: 0 20px; height: 43px; outline: none; font-size: 1.5rem; line-height: 39px; font-weight: 400; background: var(--brand-color); border-radius: 5px; color: var(--white-color); } .subscribe-form .button-loader, .subscribe-form .message-success, .subscribe-form .message-error { display: none; } .subscribe-form .loading .button-content { visibility: hidden; } .subscribe-form .loading .button-loader { position: absolute; top: 0; left: 50%; display: inline-block; margin-left: -19px; transform: scale(0.7); } .subscribe-overlay .loading .button-loader { top: 5px; } .subscribe-form .button-loader svg path, .subscribe-form .button-loader svg rect { fill: var(--white-color); } .subscribe-form .success .message-success, .subscribe-form .invalid .message-error, .subscribe-form .error .message-error { display: block; margin: 1em auto 0; max-width: 400px; color: var(--red-color); font-size: 1.6rem; line-height: 1.5em; text-align: center; } .subscribe-form .success .message-success { display: block; color: var(--darker-gray-color); } .subscribe-form .invalid .message-error, .subscribe-form .error .message-error { display: block; } @media (max-width: 650px) { .subscribe-form-title { font-size: 2.4rem; } .subscribe-form-description { font-size: 1.6rem; } } @media (max-width: 500px) { .subscribe-form form { display: flex; flex-direction: column; } .subscribe-form .subscribe-overlay-button { margin: 0 0 0 0; } .subscribe-form .form-group { display: flex; flex-direction: column; width: 100%; } .subscribe-form button { display: flex; margin: 0 0 0 0; width: 100%; } } .subscribe-close-button { position: absolute; top: 16px; right: 20px; z-index: 2000; display: block; width: 40px; height: 40px; } .subscribe-close-button:before { content: ""; position: absolute; top: 20px; right: 4px; display: block; width: 32px; height: 1px; background: var(--darker-gray-color); opacity: 0.8; transform: rotate(45deg); } .subscribe-close-button:after { content: ""; position: absolute; top: 20px; right: 4px; display: block; width: 32px; height: 1px; background: var(--darker-gray-color); opacity: 0.8; transform: rotate(-45deg); } .subscribe-close-button:hover { cursor: default; } /\\*.message-success, .message-error, .message-loading { display: none; }\\*/ /\\*.message-success .success { visibility: visible; display: block; opacity: 1; } .message-error.error { visibility: visible; display: block; opacity: 1; }\\*/ /\\*.success .message-success, .error .message-error, .loading .message-loading { visibility: visible; display: block; opacity: 1; }\\*/ /\\*For TOC\\*/ @media (min-width: 1200px) { .sticky-visible .sticky-toc { visibility: visible; opacity: 1; transform: translateY(0); } .toc-opened .sticky-toc { visibility: hidden; } } /\\*.gh-portal-triggerbtn-iframe { } .gh-portal-triggerbtn-wrapper { background: #0000ff !important; } .gh-portal-triggerbtn-container { background: #0000ff !important; height: 10px !important; } \\*/ .theme-dark:root { \--green-color: #66ff33 !important; \--orange-color: #ff5c33 !important; \--red-color: #ff3333 !important; \--purple-color: #ff00ff !important; \--brand-color: #5ceaff !important; \--white-color: #18141f !important; \--primary-text-color: #fff !important; \--secondary-text-color: #fff !important; } .feed-visibility-false { display: none; } /\\*.feed-visibility-true { display: none; }\\*/ :root { \--brand-color:#0000dd; \--white-color:#ffffff; \--primary-text-color:#1a1a1a; \--secondary-text-color:#1a1a1a; \--orange-color: #ff6600; \--purple-color: #cc33ff; } blockquote { font-weight: unset; letter-spacing: normal; font-style: normal; text-align: inherit; } table { border: 1px solid; border-collapse: collapse; white-space: unset !important; width=200px !important; } table td { border: 1px solid; border-collapse: collapse; } .theme-dark .CT-thumb { transform: translate(20px,0px); } .theme-light .CT-thumb { transform: translate(0px,0px); } /\\*.u-text-format h3 { color: var(--darker-gray-color) ; }\\*/ //Detect theme var existingTheme = localStorage.getItem('dawn\\_theme'); if (!existingTheme) { if(navigator.platform.indexOf('Win') > -1) { existingTheme = "light"; } else if(window.matchMedia && window.matchMedia('(prefers-color-scheme: dark)').matches) { existingTheme = "dark"; } else if(window.matchMedia && window.matchMedia('(prefers-color-scheme: light)').matches) { existingTheme = "light"; } } //Implement light/dark theme if(existingTheme == "dark") { ctdark(); } else if(existingTheme == 'light') { ctlight(); } window.dataLayer = window.dataLayer || []; function gtag(){dataLayer.push(arguments);} gtag('js', new Date()); gtag('config', 'G-TZK0WDB1DH'); var clicky\\_site\\_ids = clicky\\_site\\_ids || []; clicky\\_site\\_ids.push(101323562); ![Clicky](//in.getclicky.com/101323562ns.gif) :root {--ghost-accent-color: #888888;} [Cold Takes](https://www.cold-takes.com) [About (+highlights)](https://www.cold-takes.com/about/) [Most Important Century](https://www.cold-takes.com/most-important-century/) [About (+highlights)](https://www.cold-takes.com/about/) [Most Important Century](https://www.cold-takes.com/most-important-century/) [Subscribe (free)](#subscribe) [Subscribe (free)](#subscribe) [Subscribe (free)](#subscribe) [Cold Takes](https://www.cold-takes.com) - Call to Vigilance By [Holden Karnofsky](https://www.cold-takes.com/about/ "Holden Karnofsky") Sep 15, 2021 5 min read Call to Vigilance ================= *Click lower right to download or find on Apple Podcasts, Spotify, Stitcher, etc.* [Today’s world Transformative AI Digital people World of Misaligned AI World run by Something else or or Stable, galaxy-wide civilization](https://www.cold-takes.com/roadmap-for-the-most-important-century-series/) This is the final piece in the ["most important century" series](https://www.cold-takes.com/roadmap-for-the-most-important-century-series/), which has argued that there's a [high probability](https://www.cold-takes.com/forecasting-transformative-ai-whats-the-burden-of-proof/#some-rough-probabilities)[1](#fn1) that the coming decades will see: \- The development of a technology like [PASTA](https://www.cold-takes.com/transformative-ai-timelines-part-1-of-4-what-kind-of-ai/) (process for automating scientific and technological advancement). \- A resulting [productivity explosion](https://www.cold-takes.com/transformative-ai-timelines-part-1-of-4-what-kind-of-ai/#explosive-scientific-and-technological-advancement) leading to development of further transformative technologies. \- The seed of a [stable galaxy-wide civilization](https://www.cold-takes.com/all-possible-views-about-humanitys-future-are-wild/), possibly featuring [digital people](https://www.cold-takes.com/how-digital-people-could-change-the-world/), or possibly run by [misaligned AI](https://www.cold-takes.com/transformative-ai-timelines-part-1-of-4-what-kind-of-ai/#misaligned-ai-mysterious-potentially-dangerous-objectives). When trying to call attention to an underrated problem, **it's typical to close on a "call to action":** a tangible, concrete action readers can take to help. But this is challenging, because as I argued [previously](https://www.cold-takes.com/p/f5eba675-6b0e-4f19-b2ad-b63a6bafc8fd/), there are a lot of [open questions about what actions are helpful vs. harmful](https://www.cold-takes.com/p/f5eba675-6b0e-4f19-b2ad-b63a6bafc8fd/#key-open-questions-for-). (Although we can identify some [actions that seem robustly helpful today](https://www.cold-takes.com/p/f5eba675-6b0e-4f19-b2ad-b63a6bafc8fd/#robustly-helpful-actions).) This makes for a somewhat awkward situation. When confronting the "most important century" hypothesis, my attitude doesn't match the familiar ones of "excitement and motion" or "fear and avoidance." Instead, I feel an **odd mix of intensity, urgency, confusion and hesitance.** I'm looking at something bigger than I ever expected to confront, feeling underqualified and ignorant about what to do next. This is a hard mood to share and spread, but I'm trying. | | | | --- | --- | | **Situation** | **Appropriate reaction (IMO)** | | "This could be a billion-dollar company!" | "Woohoo, let's GO for it!" | | "This could be the most important century!" | "... Oh ... wow ... I don't know what to say and I somewhat want to vomit ... I have to sit down and think about this one." | So instead of a call to action, I want to make a **call to vigilance.** If you're convinced by the arguments in this piece, then don't rush to "do something" and then move on. Instead, take whatever [robustly good actions](https://www.cold-takes.com/p/f5eba675-6b0e-4f19-b2ad-b63a6bafc8fd/#robustly-helpful-actions) you can today, and otherwise put yourself in a better position to take important actions when the time comes. This could mean: * Finding ways to interact more with, and learn more about, key topics/fields/industries such as AI (for obvious reasons), science and technology generally (as a lot of the "most important century" hypothesis runs through an [explosion in scientific and technological advancement](https://www.cold-takes.com/transformative-ai-timelines-part-1-of-4-what-kind-of-ai/#impacts-of-pasta)), and relevant areas of policy and national security. * Taking opportunities (when you see them) to move your career in a direction that is more likely to be relevant (some thoughts of mine on this are [here](https://forum.effectivealtruism.org/posts/bud2ssJLQ33pSemKH/my-current-impressions-on-career-choice-for-longtermists); also see [80,000 Hours](https://www.80000hours.org)). * Connecting with other people interested in these topics (I believe this has been one of the biggest drivers of people coming to do high-impact work in the past). Currently, I think the [effective altruism](https://en.wikipedia.org/wiki/Effective_altruism) community is the best venue for this, and you can learn about how to connect with people via the [Centre for Effective Altruism](https://www.centreforeffectivealtruism.org/) (see the "Get involved" dropdown). If new ways of connecting with people come up in the future, I will likely post them on Cold Takes. * And of course, taking any opportunities you see for [robustly helpful actions.](https://www.cold-takes.com/p/f5eba675-6b0e-4f19-b2ad-b63a6bafc8fd/#robustly-helpful-actions) Buttons you can click \--------------------- Here's something you can do right now that would be genuinely helpful, though maybe not as viscerally satisfying as signing a petition or making a donation. In my [day job](https://www.openphilanthropy.org), I have a lot of moments where I - or someone I'm working with - is looking for a particular kind of person (perhaps to fill a job opening with a grantee, or to lend expertise on some topic, or something else). Over time, I expect there to be more and more opportunities for people with specific skills, interests, expertise, etc. to take actions that [help make the best of the most important century](https://www.cold-takes.com/p/f5eba675-6b0e-4f19-b2ad-b63a6bafc8fd/). And I think a major challenge will simply be **knowing who's out there** - who's interested in this cause, and wants to help, and what skills and interests they have. If you're a person we might wish we could find in the future, you can help now by sending in information about yourself via **[this simple form](https://forms.gle/z7mexiTd6wCJsuEv6)**. I vouch that your information won't be sold or otherwise used to make money, that your communication preferences (which the form asks about in detail) will be respected, and that you'll always be able to opt out of any communications. Sharing a headspace \------------------- In [This Can't Go On](https://www.cold-takes.com/this-cant-go-on/), I analogized the world to people on a plane blasting down the runway, without knowing why they're moving so fast or what's coming next: ![Animated image of the view out the airplane window as it blasts down the runway.](https://www.cold-takes.com/content/images/2021/07/airplane-launch-compressed.gif) As someone sitting on this plane, I'd love to be able to tell you I've figured out exactly what's going on and what future we need to be planning for. But I haven't. Lacking answers, I've tried to at least show you what I do see: * Dim outlines of the most important events in humanity's past or future. * A case that they're approaching us more quickly than it seems - whether or not we're ready. * A sense that the world and the rules we're all used to can't be relied on. That we need to lift our gaze above the daily torrent of tangible, relatable news - and try to wrap our heads around weirder, wilder matters that are more likely to be seen as the **headlines about this era billions of years from now.** There's a lot I don't know. But if this is the most important century, I do feel confident that we as a civilization aren't yet up to the challenges it presents. If that's going to change, it needs to start with more people seeing the situation for what it is, taking it seriously, taking action when they can - and when not, staying vigilant. [![Twitter](https://www.cold-takes.com/content/images/2021/06/ct-twitter-square.png)](https://api.addthis.com/oexchange/0.8/forward/twitter/offer?url=https%3A%2F%2Fwww.cold-takes.com%2Fcall-to-vigilance%2F&pubid;=ra-60a178324cffc42e&title;=Cold%20Takes%20-%20Call%20to%20Vigilance&ct;=1) [![Facebook](https://www.cold-takes.com/content/images/2021/06/ct-facebook-square.png)](https://api.addthis.com/oexchange/0.8/forward/facebook/offer?url=https%3A%2F%2Fwww.cold-takes.com%2Fcall-to-vigilance%2F&pubid;=ra-60a178324cffc42e&title;=Cold%20Takes%20-%20Call%20to%20Vigilance&ct;=1) [![Reddit](https://www.cold-takes.com/content/images/2021/06/ct-reddit-square.png)](https://api.addthis.com/oexchange/0.8/forward/reddit/offer?url=https%3A%2F%2Fwww.cold-takes.com%2Fcall-to-vigilance%2F&pubid;=ra-60a178324cffc42e&title;=Cold%20Takes%20-%20Call%20to%20Vigilance&ct;=1) [![More](https://www.cold-takes.com/content/images/2021/06/ct-addthis-square.png)](https://api.addthis.com/oexchange/0.8/forward/menu/offer?url=https%3A%2F%2Fwww.cold-takes.com%2Fcall-to-vigilance%2F&pubid;=ra-60a178324cffc42e&title;=Cold%20Takes%20-%20Call%20to%20Vigilance&ct;=1) [Subscribe](https://www.cold-takes.com/call-to-vigilance/#/portal/signup/free) [Feedback](https://www.guidedtrack.com/programs/4kal2ue/run?posttitle=Call%20to%20Vigilance) [Forum](https://forum.effectivealtruism.org/posts/uNczpTHFvjt4KNxEF/call-to-vigilance#comments)*Use "Feedback" if you have comments/suggestions you want me to see, or if you're up for giving some quick feedback about this post (which I greatly appreciate!) Use "Forum" if you want to discuss this post publicly on the Effective Altruism Forum.* \--- Footnotes \--------- 1\. "I am forecasting more than a 10% chance transformative AI will be developed within 15 years (by 2036); a ~50% chance it will be developed within 40 years (by 2060); and a ~2/3 chance it will be developed this century (by 2100)." [↩](#fnref1) Powered by [Ghost](https://ghost.org/) #### Popular tags [sports](/tag/sports/) [innovationstagnation](/tag/innovationstagnation/) [2star](/tag/2star/) [utopia](/tag/utopia/) Subscribe to Cold Takes ======================= For audio version, search for "Cold Takes Audio" in your podcast app Subscribe **Great!** Check your inbox and click the link to confirm your subscription. **If you don't see it, check your spam!** Please enter a valid email address! $("#lower-right-subscribe").click(function(f) { clicky.goal('1458'); /\\*LO.events.track('Subscribe-LR');\\*/ setTimeout(function(){$("#subscribe-input").focus();},500); }); $("#upper-right-subscribe-1").click(function(f) { clicky.goal('1471'); /\\*LO.events.track('Subscribe-LR');\\*/ setTimeout(function(){$("#subscribe-input").focus();},500); }); $("#upper-right-subscribe-2").click(function(f) { clicky.goal('1471'); /\\*LO.events.track('Subscribe-LR');\\*/ setTimeout(function(){$("#subscribe-input").focus();},500); }); $("#cover-subscribe").click(function(f) { clicky.goal('1472'); /\\*LO.events.track('Subscribe-LR');\\*/ setTimeout(function(){$("#subscribe-input").focus();},500); }); $("#footer-subscribe").click(function(f) { clicky.goal('1473'); /\\*LO.events.track('Subscribe-LR');\\*/ setTimeout(function(){$("#subscribe-input").focus();},500); }); $("#final-subscribe").click(function(f) { clicky.goal('1474'); }); //When someone clicks an anchor link, I want the sticky nav to go away. $("a").click(function(f) { body.removeClass('sticky-visible'); lastSt = 9999999999999}); function ctlight() { html.removeClass('theme-dark').addClass('theme-light'); localStorage.setItem('dawn\\_theme', 'light'); //$(".gh-portal-triggerbtn-container").css("background","#0000dd"); /\\* document.documentElement.style.setProperty('--brand-color', '#0000dd'); document.documentElement.style.setProperty('--white-color', '#ffffff'); document.documentElement.style.setProperty('--primary-text-color', '#1a1a1a'); document.documentElement.style.setProperty('--secondary-text-color', '#1a1a1a');\\*/ // $("#CT-thumbID").css("transform","translate(0px,0)"); // $(".single-content").css("font-weight","400"); existingTheme = "light"; } function ctdark() { html.removeClass('theme-light').addClass('theme-dark'); localStorage.setItem('dawn\\_theme', 'dark'); /\\*document.documentElement.style.setProperty('--brand-color', '#5ceaff'); document.documentElement.style.setProperty('--white-color', '#18141f'); document.documentElement.style.setProperty('--primary-text-color', '#fff'); document.documentElement.style.setProperty('--secondary-text-color', '#fff');\\*/ //document.documentElement.style.setProperty('--white-color', '#14101b'); //document.documentElement.style.setProperty('--white-color', '#121212'); //document.documentElement.style.setProperty('--white-color', '#282c35'); //$(".single-content").css("font-weight","300"); // $("#CT-thumbID").css("transform","translate(19px,0)"); existingTheme = "dark"; } //For light/dark toggle $(".CT-toggle").click(function(g) { g.preventDefault(); if(existingTheme == "dark") { ctlight(); } else if(existingTheme == 'light') { ctdark(); } } ); // Parse the URL parameter, then log goal if it is a completed subscription. Weird that it takes this much code to do this. function getParameterByName(name, url) { if (!url) url = window.location.href; name = name.replace(/[\\[\\]]/g, "\\\$&"); var regex = new RegExp("[?&]" + name + "(=([^&#]\\*)|&|#|$)"), results = regex.exec(url); if (!results) return null; if (!results[2]) return ''; return decodeURIComponent(results[2].replace(/\\+/g, " ")); } // Give the parameter a variable name var action = getParameterByName('action'); var success = getParameterByName('success'); $(document).ready(function () { // Success === null is here for backwards compatibility if (action == 'subscribe' && (success === null || success == 'true')) { clicky.goal('1476'); } });