{ "_id": "6abbef3199f7aa8c6aa494d7", "Package": "crawlee", "Title": "Tidy Interface for Reproducible Web Crawling", "Version": "0.1.1", "Authors@R": "c(\nperson(\"André\", \"Leite\", email = \"leite@castlab.org\", role = c(\"aut\", \"cre\"),\ncomment = c(ORCID = \"0000-0002-4718-9766\")),\nperson(\"Marcos\", \"Wasiliew\", email = \"marcos.wasiliew@gmail.com\", role = \"aut\",\ncomment = c(ORCID = \"0009-0004-4694-3159\")),\nperson(\"Hugo\", \"Vasconcelos\", email = \"hugo.vasconcelos@ufpe.br\", role = \"aut\",\ncomment = c(ORCID = \"0000-0001-6249-0920\")),\nperson(\"Carlos\", \"Amorim\", email = \"carlos.agaf@ufpe.br\", role = \"aut\",\ncomment = c(ORCID = \"0000-0001-6315-8305\")),\nperson(\"Diogo\", \"Bezerra\", email = \"diogo.bezerra@ufpe.br\", role = \"aut\",\ncomment = c(ORCID = \"0000-0002-1216-8674\")),\nperson(\"Júlia\", \"Nascimento Barreto\", email = \"juliabarreto@gd.seplag.pe.gov.br\", role = \"aut\",\ncomment = c(ORCID = \"0009-0004-2851-7770\")))", "Description": "A tidy, pipe-friendly toolkit for reproducible web\ncrawling and structured data collection, inspired by the\narchitecture of the 'Crawlee' library. Provides a unified\ncrawler with a deduplicating, resumable request queue,\ncontent-type aware handlers, structured storage backends and\nrich console logging via 'cli'. Supports crawling HTML pages,\nsitemaps, RSS and Atom feeds and PDF documents, with optional\nheadless-browser rendering and helpers for retrieval-augmented\ngeneration.", "License": "MIT + file LICENSE", "URL": "https://github.com/StrategicProjects/crawlee,\nhttps://strategicprojects.github.io/crawlee/", "BugReports": "https://github.com/StrategicProjects/crawlee/issues", "VignetteBuilder": "knitr", "Config/testthat/edition": "3", "Encoding": "UTF-8", "Language": "en-US", "Roxygen": "list(markdown = TRUE)", "RoxygenNote": "8.0.0", "Config/pak/sysreqs": "libxml2-dev libssl-dev", "Repository": "https://strategicprojects.r-universe.dev", "Date/Publication": "2026-09-29 14:26:52 UTC", "RemoteUrl": "https://github.com/strategicprojects/crawlee", "RemoteRef": "HEAD", "RemoteSha": "89889d112baed55955cbd9ebd70ac93dc8c55018", "NeedsCompilation": "no", "Packaged": { "Date": "2026-09-29 16:11:32 UTC", "User": "root" }, "Author": "André Leite [aut, cre] (ORCID: ),\nMarcos Wasiliew [aut] (ORCID: ),\nHugo Vasconcelos [aut] (ORCID: ),\nCarlos Amorim [aut] (ORCID: ),\nDiogo Bezerra [aut] (ORCID: ),\nJúlia Nascimento Barreto [aut] (ORCID:\n)", "Maintainer": "André Leite ", "_user": "strategicprojects", "_type": "src", "_file": "crawlee_0.1.1.tar.gz", "_fileid": "https://r2.ropensci.org/1b323594495c5424b38fb6ebe98a1131ef0b03fb20cca319b09534fec4a5d1a4", "_filesize": 250649, "_sha256": "1b323594495c5424b38fb6ebe98a1131ef0b03fb20cca319b09534fec4a5d1a4", "_expires": "2027-01-07T17:02:39.000Z", "_created": "2026-09-29T16:11:32.000Z", "_published": "2026-09-29T17:02:41.795Z", "_jobs": [ { "job": 109500741075, "time": 152, "config": "linux-devel-x86_64", "r": "4.7.0", "check": "OK", "artifact": "11045239208" }, { "job": 109500741313, "time": 139, "config": "linux-release-x86_64", "r": "4.6.1", "check": "OK", "artifact": "11045722446" }, { "job": 109500741344, "time": 90, "config": "macos-oldrel-arm64", "r": "4.5.3", "check": "OK", "artifact": "11049149462" }, { "job": 109500741245, "time": 107, "config": "macos-release-arm64", "r": "4.6.1", "check": "OK", "artifact": "11048484102" }, { "job": 109499188989, "time": 218, "config": "source", "r": "4.6.1", "check": "OK", "artifact": "11046316469" }, { "job": 109500741194, "time": 137, "config": "wasm-release", "r": "4.6.0", "check": "OK", "artifact": "11046381610" }, { "job": 109500741264, "time": 2931, "config": "windows-devel-x86_64", "r": "4.7.0", "check": "OK", "artifact": "11048689834" }, { "job": 109500741319, "time": 2957, "config": "windows-oldrel-x86_64", "r": "4.5.3", "check": "OK", "artifact": "11050235100" }, { "job": 109500741592, "time": 2450, "config": "windows-release-x86_64", "r": "4.6.1", "check": "OK", "artifact": "11049143904" } ], "_host": "GitHub-Actions", "_buildurl": "https://github.com/r-universe/strategicprojects/actions/runs/36595444735", "_status": "success", "_upstream": "https://github.com/strategicprojects/crawlee", "_commit": { "id": "89889d112baed55955cbd9ebd70ac93dc8c55018", "author": "André Leite ", "committer": "André Leite ", "message": "Add ORCID identifiers for the authors\n\nCo-Authored-By: Claude Opus 5.5 \n", "time": 1790692012 }, "_maintainer": { "name": "André Leite", "email": "leite@castlab.org", "orcid": "0000-0002-4718-9766" }, "_distro": "resolute", "_registered": true, "_dependencies": [ { "package": "R", "version": ">= 4.1.0", "role": "Depends" }, { "package": "cli", "role": "Imports" }, { "package": "httr2", "role": "Imports" }, { "package": "R6", "role": "Imports" }, { "package": "rlang", "role": "Imports" }, { "package": "rvest", "role": "Imports" }, { "package": "tibble", "role": "Imports" }, { "package": "vctrs", "role": "Imports" }, { "package": "xml2", "role": "Imports" }, { "package": "arrow", "role": "Suggests" }, { "package": "chromote", "role": "Suggests" }, { "package": "DBI", "role": "Suggests" }, { "package": "dplyr", "role": "Suggests" }, { "package": "duckdb", "role": "Suggests" }, { "package": "httptest2", "role": "Suggests" }, { "package": "jsonlite", "role": "Suggests" }, { "package": "knitr", "role": "Suggests" }, { "package": "later", "role": "Suggests" }, { "package": "nanoparquet", "role": "Suggests" }, { "package": "pdftools", "role": "Suggests" }, { "package": "promises", "role": "Suggests" }, { "package": "rmarkdown", "role": "Suggests" }, { "package": "testthat", "version": ">= 3.0.0", "role": "Suggests" } ], "_owner": "strategicprojects", "_selfowned": true, "_usedby": 0, "_updates": [ { "week": "2026-26", "n": 22 }, { "week": "2026-27", "n": 1 }, { "week": "2026-40", "n": 5 } ], "_tags": [], "_topics": [ "crawlee", "crawler", "govtech", "httr2", "open-data", "rvest", "web-scraping" ], "_stars": 2, "_contributors": [ { "user": "milkway", "count": 19, "uuid": 7375849 } ], "_userbio": { "uuid": 127886513, "type": "organization", "name": "Projetos Estratégicos", "followers": 14, "description": "Repositório de código da Assessoria Especial de Projetos Estratégicos." }, "_downloads": { "count": 379, "source": "https://cranlogs.r-pkg.org/downloads/total/last-month/crawlee" }, "_devurl": "https://github.com/strategicprojects/crawlee", "_pkgdown": "https://strategicprojects.github.io/crawlee/", "_searchresults": 0, "_rbuild": "4.6.1", "_assets": [ "extra/citation.cff", "extra/citation.html", "extra/citation.json", "extra/citation.txt", "extra/contents.json", "extra/crawlee.html", "extra/manual.pdf", "extra/NEWS.html", "extra/NEWS.txt", "extra/readme.html", "extra/readme.md", "LICENSE" ], "_homeurl": "https://github.com/strategicprojects/crawlee", "_realowner": "strategicprojects", "_cranurl": true, "_releases": [ { "version": "0.1.0", "date": "2026-07-03" } ], "_exports": [ "cr_autoscale", "cr_chunk", "cr_close", "cr_collect", "cr_dataset", "cr_embed", "cr_export", "cr_from_rss", "cr_from_sitemap", "cr_normalize_url", "cr_on_html", "cr_on_pdf", "cr_options", "cr_parallel", "cr_persist", "cr_run", "cr_store", "cr_stream", "cr_use_browser", "cr_use_http", "crawler", "Crawler", "Dataset", "KeyValueStore", "RequestQueue" ], "_help": [ { "page": "cr_autoscale", "title": "Enable autoscaled parallel fetching", "topics": [ "cr_autoscale" ] }, { "page": "cr_chunk", "title": "Chunk text for retrieval-augmented generation", "topics": [ "cr_chunk" ] }, { "page": "cr_close", "title": "Release a crawler's resources", "topics": [ "cr_close" ] }, { "page": "cr_collect", "title": "Collect crawl results", "topics": [ "cr_collect" ] }, { "page": "cr_dataset", "title": "Configure the dataset backend", "topics": [ "cr_dataset" ] }, { "page": "cr_embed", "title": "Attach embeddings to chunks", "topics": [ "cr_embed" ] }, { "page": "cr_export", "title": "Export chunks (and embeddings) for retrieval", "topics": [ "cr_export" ] }, { "page": "cr_from_rss", "title": "Discover URLs from an RSS or Atom feed", "topics": [ "cr_from_rss" ] }, { "page": "cr_from_sitemap", "title": "Discover URLs from a sitemap", "topics": [ "cr_from_sitemap" ] }, { "page": "cr_normalize_url", "title": "Normalise a URL into a canonical form", "topics": [ "cr_normalize_url" ] }, { "page": "cr_on_html", "title": "Register an HTML handler", "topics": [ "cr_on_html" ] }, { "page": "cr_on_pdf", "title": "Register a PDF handler", "topics": [ "cr_on_pdf" ] }, { "page": "cr_options", "title": "Set crawler options", "topics": [ "cr_options" ] }, { "page": "cr_parallel", "title": "Enable parallel (concurrent) fetching", "topics": [ "cr_parallel" ] }, { "page": "cr_persist", "title": "Persist a crawl to a run directory (and resume it)", "topics": [ "cr_persist" ] }, { "page": "cr_run", "title": "Run a crawl", "topics": [ "cr_run" ] }, { "page": "cr_store", "title": "Configure the key-value store for binary content", "topics": [ "cr_store" ] }, { "page": "cr_stream", "title": "Enable the streaming scheduler", "topics": [ "cr_stream" ] }, { "page": "cr_use_browser", "title": "Use the headless-browser fetch backend", "topics": [ "cr_use_browser" ] }, { "page": "cr_use_http", "title": "Use the HTTP fetch backend", "topics": [ "cr_use_http" ] }, { "page": "crawler", "title": "Create a crawler", "topics": [ "crawler" ] }, { "page": "Crawler-class", "title": "Crawler", "topics": [ "Crawler", "Crawler-class" ] }, { "page": "Dataset", "title": "Dataset", "topics": [ "Dataset" ] }, { "page": "KeyValueStore", "title": "Key-value store", "topics": [ "KeyValueStore" ] }, { "page": "RequestQueue", "title": "Request queue", "topics": [ "RequestQueue" ] } ], "_pkglogo": "https://github.com/strategicprojects/crawlee/raw/HEAD/man/figures/logo.svg", "_readme": "https://github.com/strategicprojects/crawlee/raw/HEAD/README.md", "_rundeps": [ "askpass", "cli", "curl", "glue", "httr", "httr2", "jsonlite", "lifecycle", "magrittr", "mime", "openssl", "pillar", "pkgconfig", "R6", "rlang", "rvest", "selectr", "sys", "tibble", "utf8", "vctrs", "withr", "xml2" ], "_vignettes": [ { "source": "rag-pipeline.Rmd", "filename": "rag-pipeline.html", "title": "A RAG pipeline", "engine": "knitr::rmarkdown", "headings": [ "1. Crawl and collect text", "2. Chunk", "3. Embed", "4. Export for retrieval", "End to end" ], "created": "2026-06-27 18:50:17", "modified": "2026-06-27 19:06:52", "commits": 2 }, { "source": "crawling-a-website.Rmd", "filename": "crawling-a-website.html", "title": "Crawling a website", "engine": "knitr::rmarkdown", "headings": [ "The model", "Your first crawler", "Following links", "Controlling scope", "Routing different page types", "Crawling from a sitemap", "Rendering JavaScript pages", "Where next" ], "created": "2026-06-27 18:42:17", "modified": "2026-06-27 18:42:17", "commits": 1 }, { "source": "crawlee.Rmd", "filename": "crawlee.html", "title": "Getting started with crawlee", "engine": "knitr::rmarkdown", "headings": [ "The mental model", "A minimal crawl", "The handler context", "Controlling link discovery", "Reproducibility" ], "created": "2026-06-27 16:43:28", "modified": "2026-06-27 18:42:17", "commits": 2 }, { "source": "scaling-and-politeness.Rmd", "filename": "scaling-and-politeness.html", "title": "Scaling and politeness", "engine": "knitr::rmarkdown", "headings": [ "Being a good web citizen", "Going faster", "Fixed-concurrency batches — cr_parallel()", "Adaptive batches — cr_autoscale()", "Continuous streaming pool — cr_stream()", "Choosing an engine", "Combining with persistence" ], "created": "2026-06-27 18:50:17", "modified": "2026-06-27 18:50:17", "commits": 1 }, { "source": "storage-and-resumable-runs.Rmd", "filename": "storage-and-resumable-runs.html", "title": "Storage and resumable runs", "engine": "knitr::rmarkdown", "headings": [ "The dataset", "The key-value store", "The request queue and reproducibility", "One-call setup: cr_persist()", "Resuming" ], "created": "2026-06-27 18:50:17", "modified": "2026-06-27 18:50:17", "commits": 1 } ], "_score": 5.176091259055681, "_indexed": true, "_nocasepkg": "crawlee", "_universes": [ "strategicprojects" ], "_previous": "0.1.0", "_binaries": [ { "r": "4.7.0", "os": "linux", "version": "0.1.1", "date": "2026-09-29T16:14:58.000Z", "distro": "resolute", "commit": "89889d112baed55955cbd9ebd70ac93dc8c55018", "fileid": "https://r2.ropensci.org/a2b868c413122bf1156e8116923b1a20441aa679a7f3dcdb7948833018692c88", "status": "success", "check": "OK", "buildurl": "https://github.com/r-universe/strategicprojects/actions/runs/36595444735" }, { "r": "4.6.1", "os": "linux", "version": "0.1.1", "date": "2026-09-29T16:14:54.000Z", "distro": "resolute", "commit": "89889d112baed55955cbd9ebd70ac93dc8c55018", "fileid": "https://r2.ropensci.org/ed747413bde22509e1bad242047457777d22c01f77221a09ef1a96984a72b1d0", "status": "success", "check": "OK", "buildurl": "https://github.com/r-universe/strategicprojects/actions/runs/36595444735" }, { "r": "4.5.3", "os": "mac", "version": "0.1.1", "date": "2026-09-29T16:57:14.000Z", "commit": "89889d112baed55955cbd9ebd70ac93dc8c55018", "fileid": "https://r2.ropensci.org/ed65bc8c2580f6e310c7a5bd58ee1360c11e0bd73c3f92abef9535f278c7b656", "status": "success", "check": "OK", "buildurl": "https://github.com/r-universe/strategicprojects/actions/runs/36595444735" }, { "r": "4.6.1", "os": "mac", "version": "0.1.1", "date": "2026-09-29T16:56:12.000Z", "commit": "89889d112baed55955cbd9ebd70ac93dc8c55018", "fileid": "https://r2.ropensci.org/2f9c75e8ef9868c7c950bad48b2a1d562511407f457d68ac004eeb19ca0bab9a", "status": "success", "check": "OK", "buildurl": "https://github.com/r-universe/strategicprojects/actions/runs/36595444735" }, { "r": "4.6.0", "os": "wasm", "version": "0.1.1", "date": "2026-09-29T16:15:04.000Z", "commit": "89889d112baed55955cbd9ebd70ac93dc8c55018", "fileid": "https://r2.ropensci.org/7412c9251e21cab98384c118819832dd14fdd1b647612e4843d47da7104e8a23", "status": "success", "buildurl": "https://github.com/r-universe/strategicprojects/actions/runs/36595444735" }, { "r": "4.7.0", "os": "win", "version": "0.1.1", "date": "2026-09-29T17:01:16.000Z", "commit": "89889d112baed55955cbd9ebd70ac93dc8c55018", "fileid": "https://r2.ropensci.org/6d56f6a437353c6ce27e9761e4254acf15bd3599396a8c10915c24eb9720fee3", "status": "success", "check": "OK", "buildurl": "https://github.com/r-universe/strategicprojects/actions/runs/36595444735" }, { "r": "4.5.3", "os": "win", "version": "0.1.1", "date": "2026-09-29T17:01:49.000Z", "commit": "89889d112baed55955cbd9ebd70ac93dc8c55018", "fileid": "https://r2.ropensci.org/1efb1997b06bdae8903beb42f6942984fba501ea9e761e2e33f75cea61a7aaff", "status": "success", "check": "OK", "buildurl": "https://github.com/r-universe/strategicprojects/actions/runs/36595444735" }, { "r": "4.6.1", "os": "win", "version": "0.1.1", "date": "2026-09-29T16:53:21.000Z", "commit": "89889d112baed55955cbd9ebd70ac93dc8c55018", "fileid": "https://r2.ropensci.org/8868d8695a4e32274faa0e793d0335fbdef42c16678b1a8eb46edebaca8aad53", "status": "success", "check": "OK", "buildurl": "https://github.com/r-universe/strategicprojects/actions/runs/36595444735" } ] }