[{"data":1,"prerenderedAt":366},["ShallowReactive",2],{"navigation_docs":3,"-providers-commoncrawl":88,"-providers-commoncrawl-surround":361},[4,40],{"title":5,"path":6,"stem":7,"children":8,"icon":39},"Guide","\u002Fguide","1.guide\u002F1.index",[9,11,15,19,23,27,31,35],{"title":10,"path":6,"stem":7},"Getting Started",{"title":12,"path":13,"stem":14},"Listing Captures","\u002Fguide\u002Fsnapshots","1.guide\u002F2.snapshots",{"title":16,"path":17,"stem":18},"Reading Captures","\u002Fguide\u002Fcontent","1.guide\u002F3.content",{"title":20,"path":21,"stem":22},"Comparing Versions","\u002Fguide\u002Fdiff","1.guide\u002F4.diff",{"title":24,"path":25,"stem":26},"Configuration","\u002Fguide\u002Fconfiguration","1.guide\u002F5.configuration",{"title":28,"path":29,"stem":30},"Agents","\u002Fguide\u002Fagents","1.guide\u002F6.agents",{"title":32,"path":33,"stem":34},"Custom Providers","\u002Fguide\u002Fcustom","1.guide\u002F7.custom",{"title":36,"path":37,"stem":38},"Explorer","\u002Fguide\u002Fexplorer","1.guide\u002F8.explorer","i-lucide-book-open",{"title":41,"path":42,"stem":43,"children":44,"icon":87},"Providers","\u002Fproviders","2.providers\u002F0.index",[45,47,51,55,59,63,67,71,75,79,83],{"title":46,"path":42,"stem":43},"Overview",{"title":48,"path":49,"stem":50},"Wayback Machine","\u002Fproviders\u002Fwayback","2.providers\u002F01.wayback",{"title":52,"path":53,"stem":54},"Arquivo.pt","\u002Fproviders\u002Farquivo","2.providers\u002F02.arquivo",{"title":56,"path":57,"stem":58},"Webarchiv Österreich","\u002Fproviders\u002Fwebarchiv","2.providers\u002F03.webarchiv",{"title":60,"path":61,"stem":62},"Archive.today","\u002Fproviders\u002Farchive-today","2.providers\u002F04.archive-today",{"title":64,"path":65,"stem":66},"Common Crawl","\u002Fproviders\u002Fcommoncrawl","2.providers\u002F05.commoncrawl",{"title":68,"path":69,"stem":70},"WebCite","\u002Fproviders\u002Fwebcite","2.providers\u002F06.webcite",{"title":72,"path":73,"stem":74},"Memento","\u002Fproviders\u002Fmemento","2.providers\u002F07.memento",{"title":76,"path":77,"stem":78},"Archive-It","\u002Fproviders\u002Farchive-it","2.providers\u002F08.archive-it",{"title":80,"path":81,"stem":82},"Conifer","\u002Fproviders\u002Fconifer","2.providers\u002F09.conifer",{"title":84,"path":85,"stem":86},"Perma.cc","\u002Fproviders\u002Fpermacc","2.providers\u002F10.permacc","i-lucide-library",{"id":89,"title":64,"body":90,"description":354,"extension":355,"links":356,"meta":357,"navigation":160,"path":65,"seo":359,"stem":66,"__hash__":360},"docs\u002F2.providers\u002F05.commoncrawl.md",{"type":91,"value":92,"toc":347},"minimark",[93,97,102,233,237,277,284,288,302,306,330,334,343],[94,95],"provider-facts",{"slug":96},"commoncrawl",[98,99,101],"h2",{"id":100},"load-it","Load it",[103,104,109],"pre",{"className":105,"code":106,"language":107,"meta":108,"style":108},"language-ts shiki shiki-themes vitesse-light vitesse-light vesper","import { createArchive, providers } from \"@agntn\u002Farchives\";\n\nconst latest = createArchive(providers.commoncrawl());\nconst pinned = createArchive(providers.commoncrawl({ collection: \"CC-MAIN-2023-50\" }));\n","ts","",[110,111,112,155,162,193],"code",{"__ignoreMap":108},[113,114,117,121,125,129,132,135,138,141,145,149,152],"span",{"class":115,"line":116},"line",1,[113,118,120],{"class":119},"sB1ot","import",[113,122,124],{"class":123},"sldxI"," {",[113,126,128],{"class":127},"sA8fy"," createArchive",[113,130,131],{"class":123},",",[113,133,134],{"class":127}," providers",[113,136,137],{"class":123}," }",[113,139,140],{"class":119}," from",[113,142,144],{"class":143},"sbwUx"," \"",[113,146,148],{"class":147},"sHyde","@agntn\u002Farchives",[113,150,151],{"class":143},"\"",[113,153,154],{"class":123},";\n",[113,156,158],{"class":115,"line":157},2,[113,159,161],{"emptyLinePlaceholder":160},true,"\n",[113,163,165,169,172,176,179,182,185,188,190],{"class":115,"line":164},3,[113,166,168],{"class":167},"s8Cdk","const",[113,170,171],{"class":127}," latest",[113,173,175],{"class":174},"sUeik"," =",[113,177,128],{"class":178},"shFES",[113,180,181],{"class":123},"(",[113,183,184],{"class":127},"providers",[113,186,187],{"class":123},".",[113,189,96],{"class":178},[113,191,192],{"class":123},"());\n",[113,194,196,198,201,203,205,207,209,211,213,216,220,223,225,228,230],{"class":115,"line":195},4,[113,197,168],{"class":167},[113,199,200],{"class":127}," pinned",[113,202,175],{"class":174},[113,204,128],{"class":178},[113,206,181],{"class":123},[113,208,184],{"class":127},[113,210,187],{"class":123},[113,212,96],{"class":178},[113,214,215],{"class":123},"({ ",[113,217,219],{"class":218},"sYv7f","collection",[113,221,222],{"class":123},": ",[113,224,151],{"class":143},[113,226,227],{"class":147},"CC-MAIN-2023-50",[113,229,151],{"class":143},[113,231,232],{"class":123}," }));\n",[98,234,236],{"id":235},"what-it-lists","What it lists",[238,239,240,241,243,244,247,248,250,251,254,255,254,258,261,262,264,265,268,269,272,273,276],"p",{},"Common Crawl publishes one index per crawl. Without ",[110,242,219],{},", the provider reads ",[110,245,246],{},"index.commoncrawl.org\u002Fcollinfo.json"," and uses the newest crawl it advertises; with ",[110,249,219],{},", it queries that crawl's CDX API directly. Rows carry ",[110,252,253],{},"digest",", ",[110,256,257],{},"mime",[110,259,260],{},"length"," and the ",[110,263,219],{}," in ",[110,266,267],{},"_meta",", plus the WARC ",[110,270,271],{},"filename"," and ",[110,274,275],{},"offset"," that locate the body.",[238,278,279,280,283],{},"A 404 with ",[110,281,282],{},"No Captures found"," is the index's way of saying the URL was never crawled; the provider turns it into an empty listing rather than a failure. Every other 404 keeps meaning failure.",[98,285,287],{"id":286},"reading-bodies","Reading bodies",[238,289,290,293,294,297,298,301],{},[110,291,292],{},"content()"," requests the byte range ",[110,295,296],{},"offset..offset+length"," of the WARC file on ",[110,299,300],{},"data.commoncrawl.org",", then parses the WARC record and the HTTP response inside it, including chunked transfer encoding and gzip. The result is the original response body, decoded with the charset the record declares.",[98,303,305],{"id":304},"gotchas","Gotchas",[307,308,309,313,319],"ul",{},[310,311,312],"li",{},"One crawl is a snapshot in time, not a history. A URL crawled once appears once; for a timeline across years, query several collections or another archive.",[310,314,315,318],{},[110,316,317],{},"collinfo.json"," is fetched on every uncached listing when no collection is pinned. Pin one in services that run for long.",[310,320,321,322,325,326,329],{},"The index host is occasionally unreachable; when every archive is asked, that shows up as ",[110,323,324],{},"commoncrawl: … fetch failed"," under ",[110,327,328],{},"_meta.errors",", and the other providers still answer.",[98,331,333],{"id":332},"where-it-lives","Where it lives",[238,335,336,339,340,187],{},[110,337,338],{},"src\u002Fproviders\u002Fcommoncrawl.ts",". WARC parsing lives in ",[110,341,342],{},"src\u002Futils\u002F_content.ts",[344,345,346],"style",{},"html pre.shiki code .sB1ot, html code.shiki .sB1ot{--shiki-light:#1E754F;--shiki-default:#1E754F;--shiki-dark:#A0A0A0}html pre.shiki code .sldxI, html code.shiki .sldxI{--shiki-light:#999999;--shiki-default:#999999;--shiki-dark:#FFF}html pre.shiki code .sA8fy, html code.shiki .sA8fy{--shiki-light:#B07D48;--shiki-default:#B07D48;--shiki-dark:#FFF}html pre.shiki code .sbwUx, html code.shiki .sbwUx{--shiki-light:#B5695977;--shiki-default:#B5695977;--shiki-dark:#99FFE4}html pre.shiki code .sHyde, html code.shiki .sHyde{--shiki-light:#B56959;--shiki-default:#B56959;--shiki-dark:#99FFE4}html pre.shiki code .s8Cdk, html code.shiki .s8Cdk{--shiki-light:#AB5959;--shiki-default:#AB5959;--shiki-dark:#A0A0A0}html pre.shiki code .sUeik, html code.shiki .sUeik{--shiki-light:#999999;--shiki-default:#999999;--shiki-dark:#A0A0A0}html pre.shiki code .shFES, html code.shiki .shFES{--shiki-light:#59873A;--shiki-default:#59873A;--shiki-dark:#FFC799}html pre.shiki code .sYv7f, html code.shiki .sYv7f{--shiki-light:#998418;--shiki-default:#998418;--shiki-dark:#FFF}html .light .shiki span {color: var(--shiki-light);background: var(--shiki-light-bg);font-style: var(--shiki-light-font-style);font-weight: var(--shiki-light-font-weight);text-decoration: var(--shiki-light-text-decoration);}html.light .shiki span {color: var(--shiki-light);background: var(--shiki-light-bg);font-style: var(--shiki-light-font-style);font-weight: var(--shiki-light-font-weight);text-decoration: var(--shiki-light-text-decoration);}html .default .shiki span {color: var(--shiki-default);background: var(--shiki-default-bg);font-style: var(--shiki-default-font-style);font-weight: var(--shiki-default-font-weight);text-decoration: var(--shiki-default-text-decoration);}html .shiki span {color: var(--shiki-default);background: var(--shiki-default-bg);font-style: var(--shiki-default-font-style);font-weight: var(--shiki-default-font-weight);text-decoration: var(--shiki-default-text-decoration);}html .dark .shiki span {color: var(--shiki-dark);background: var(--shiki-dark-bg);font-style: var(--shiki-dark-font-style);font-weight: var(--shiki-dark-font-weight);text-decoration: var(--shiki-dark-text-decoration);}html.dark .shiki span {color: var(--shiki-dark);background: var(--shiki-dark-bg);font-style: var(--shiki-dark-font-style);font-weight: var(--shiki-dark-font-weight);text-decoration: var(--shiki-dark-text-decoration);}",{"title":108,"searchDepth":157,"depth":157,"links":348},[349,350,351,352,353],{"id":100,"depth":157,"text":101},{"id":235,"depth":157,"text":236},{"id":286,"depth":157,"text":287},{"id":304,"depth":157,"text":305},{"id":332,"depth":157,"text":333},"One CDX index per crawl, and bodies read straight from the WARC byte range.","md",null,{"icon":358},"i-lucide-archive",{"title":64,"description":354},"IEu1_KUKQ2ATJDS_IFN1IyomDfrzb0n_dVmrP-aOc6g",[362,364],{"title":60,"path":61,"stem":62,"description":363,"children":-1},"archive.is through its Memento TimeMap; bodies are the rendered wrapper page, not the original bytes.",{"title":68,"path":69,"stem":70,"description":365,"children":-1},"An archive with no listing API. It answers unsupported, with the reason, instead of pretending.",1788446777799]