{ "_id": "6a48d99cb5e98c40829fffea", "Package": "crawlee", "Title": "Tidy Interface for Reproducible Web Crawling", "Version": "0.1.0", "Authors@R": "c(\nperson(\"Andre\", \"Leite\", email = \"leite@castlab.org\", role = c(\"aut\", \"cre\")),\nperson(\"Marcos\", \"Wasilew\", email = \"marcos.wasilew@gmail.com\", role = \"aut\"),\nperson(\"Hugo\", \"Vasconcelos\", email = \"hugo.vasconcelos@ufpe.br\", role = \"aut\"),\nperson(\"Carlos\", \"Amorim\", email = \"carlos.agaf@ufpe.br\", role = \"aut\"),\nperson(\"Diogo\", \"Bezerra\", email = \"diogo.bezerra@ufpe.br\", role = \"aut\"))", "Description": "A tidy, pipe-friendly toolkit for reproducible web\ncrawling and structured data collection, inspired by the\narchitecture of the 'Crawlee' library. Provides a unified\ncrawler with a deduplicating, resumable request queue,\ncontent-type aware handlers, structured storage backends and\nrich console logging via 'cli'. Supports crawling HTML pages,\nsitemaps, RSS and Atom feeds and PDF documents, with optional\nheadless-browser rendering and helpers for retrieval-augmented\ngeneration.", "License": "MIT + file LICENSE", "URL": "https://github.com/StrategicProjects/crawlee,\nhttps://strategicprojects.github.io/crawlee/", "BugReports": "https://github.com/StrategicProjects/crawlee/issues", "VignetteBuilder": "knitr", "Config/testthat/edition": "3", "Encoding": "UTF-8", "Language": "en-US", "Roxygen": "list(markdown = TRUE)", "RoxygenNote": "8.0.0", "Config/pak/sysreqs": "libxml2-dev libssl-dev", "Repository": "https://strategicprojects.r-universe.dev", "Date/Publication": "2026-06-29 12:10:28 UTC", "RemoteUrl": "https://github.com/strategicprojects/crawlee", "RemoteRef": "HEAD", "RemoteSha": "fef3fd8e7c115fff2dccc4b4091209f16ca7e573", "NeedsCompilation": "no", "Packaged": { "Date": "2026-07-04 09:23:37 UTC", "User": "root" }, "Author": "Andre Leite [aut, cre],\nMarcos Wasilew [aut],\nHugo Vasconcelos [aut],\nCarlos Amorim [aut],\nDiogo Bezerra [aut]", "Maintainer": "Andre Leite ", "_user": "strategicprojects", "_type": "src", "_file": "crawlee_0.1.0.tar.gz", "_fileid": "https://r2.ropensci.org/4845e90f14191820a4f972a021ce2863ca7b86bb208b264a69f76673ec3b1816", "_filesize": 250617, "_sha256": "4845e90f14191820a4f972a021ce2863ca7b86bb208b264a69f76673ec3b1816", "_expires": "2026-10-12T09:59:55.000Z", "_created": "2026-07-04T09:23:37.000Z", "_published": "2026-07-04T09:59:56.828Z", "_jobs": [ { "job": 85121066271, "time": 157, "config": "linux-devel-x86_64", "r": "4.7.0", "check": "OK", "artifact": "8080444041" }, { "job": 85121066273, "time": 159, "config": "linux-release-x86_64", "r": "4.6.1", "check": "OK", "artifact": "8080444181" }, { "job": 85121066289, "time": 81, "config": "macos-oldrel-arm64", "r": "4.5.3", "check": "OK", "artifact": "8080672758" }, { "job": 85121066270, "time": 84, "config": "macos-release-arm64", "r": "4.6.1", "check": "OK", "artifact": "8080670264" }, { "job": 85120848249, "time": 220, "config": "source", "r": "4.6.1", "check": "OK", "artifact": "8080426077" }, { "job": 85121066248, "time": 153, "config": "wasm-release", "r": "4.6.0", "check": "OK", "artifact": "8080443625" }, { "job": 85121066274, "time": 96, "config": "windows-devel", "r": "4.7.0", "check": "OK", "artifact": "8080437234" }, { "job": 85121066292, "time": 79, "config": "windows-oldrel", "r": "4.5.3", "check": "OK", "artifact": "8080435407" }, { "job": 85121066300, "time": 86, "config": "windows-release", "r": "4.6.1", "check": "OK", "artifact": "8080436277" } ], "_host": "GitHub-Actions", "_buildurl": "https://github.com/r-universe/strategicprojects/actions/runs/28701748405", "_status": "success", "_upstream": "https://github.com/strategicprojects/crawlee", "_commit": { "id": "fef3fd8e7c115fff2dccc4b4091209f16ca7e573", "author": "André Leite ", "committer": "André Leite ", "message": "docs: fix author name spelling Carlos Amorin -> Carlos Amorim\n\nMatch the author's GitHub profile (@cagaf). Email unchanged.\n\nCo-Authored-By: Claude Opus 4.8 (1M context) \n", "time": 1782735028 }, "_maintainer": { "name": "Andre Leite", "email": "leite@castlab.org" }, "_distro": "resolute", "_registered": true, "_dependencies": [ { "package": "R", "version": ">= 4.1.0", "role": "Depends" }, { "package": "cli", "role": "Imports" }, { "package": "httr2", "role": "Imports" }, { "package": "R6", "role": "Imports" }, { "package": "rlang", "role": "Imports" }, { "package": "rvest", "role": "Imports" }, { "package": "tibble", "role": "Imports" }, { "package": "vctrs", "role": "Imports" }, { "package": "xml2", "role": "Imports" }, { "package": "arrow", "role": "Suggests" }, { "package": "chromote", "role": "Suggests" }, { "package": "DBI", "role": "Suggests" }, { "package": "dplyr", "role": "Suggests" }, { "package": "duckdb", "role": "Suggests" }, { "package": "httptest2", "role": "Suggests" }, { "package": "jsonlite", "role": "Suggests" }, { "package": "knitr", "role": "Suggests" }, { "package": "later", "role": "Suggests" }, { "package": "nanoparquet", "role": "Suggests" }, { "package": "pdftools", "role": "Suggests" }, { "package": "polite", "role": "Suggests" }, { "package": "promises", "role": "Suggests" }, { "package": "rmarkdown", "role": "Suggests" }, { "package": "testthat", "version": ">= 3.0.0", "role": "Suggests" } ], "_owner": "strategicprojects", "_selfowned": true, "_usedby": 0, "_updates": [ { "week": "2026-26", "n": 22 }, { "week": "2026-27", "n": 1 } ], "_tags": [], "_topics": [ "crawlee", "crawler", "govtech", "httr2", "open-data", "rvest", "web-scraping" ], "_stars": 0, "_contributors": [ { "user": "milkway", "count": 14, "uuid": 7375849 } ], "_userbio": { "uuid": 127886513, "type": "organization", "name": "Projetos Estratégicos", "followers": 4, "description": "Repositório de código da Assessoria Especial de Projetos Estratégicos." }, "_downloads": { "count": 0, "source": "https://cranlogs.r-pkg.org/downloads/total/last-month/crawlee" }, "_devurl": "https://github.com/strategicprojects/crawlee", "_pkgdown": "https://strategicprojects.github.io/crawlee/", "_searchresults": 0, "_rbuild": "4.6.1", "_assets": [ "extra/citation.cff", "extra/citation.html", "extra/citation.json", "extra/citation.txt", "extra/contents.json", "extra/crawlee.html", "extra/NEWS.html", "extra/NEWS.txt", "extra/readme.html", "extra/readme.md", "LICENSE", "manual.pdf" ], "_homeurl": "https://github.com/strategicprojects/crawlee", "_realowner": "strategicprojects", "_cranurl": true, "_releases": [ { "version": "0.1.0", "date": "2026-07-03" } ], "_exports": [ "cr_autoscale", "cr_chunk", "cr_close", "cr_collect", "cr_dataset", "cr_embed", "cr_export", "cr_from_rss", "cr_from_sitemap", "cr_normalize_url", "cr_on_html", "cr_on_pdf", "cr_options", "cr_parallel", "cr_persist", "cr_run", "cr_store", "cr_stream", "cr_use_browser", "cr_use_http", "crawler", "Crawler", "Dataset", "KeyValueStore", "RequestQueue" ], "_help": [ { "page": "cr_autoscale", "title": "Enable autoscaled parallel fetching", "topics": [ "cr_autoscale" ] }, { "page": "cr_chunk", "title": "Chunk text for retrieval-augmented generation", "topics": [ "cr_chunk" ] }, { "page": "cr_close", "title": "Release a crawler's resources", "topics": [ "cr_close" ] }, { "page": "cr_collect", "title": "Collect crawl results", "topics": [ "cr_collect" ] }, { "page": "cr_dataset", "title": "Configure the dataset backend", "topics": [ "cr_dataset" ] }, { "page": "cr_embed", "title": "Attach embeddings to chunks", "topics": [ "cr_embed" ] }, { "page": "cr_export", "title": "Export chunks (and embeddings) for retrieval", "topics": [ "cr_export" ] }, { "page": "cr_from_rss", "title": "Discover URLs from an RSS or Atom feed", "topics": [ "cr_from_rss" ] }, { "page": "cr_from_sitemap", "title": "Discover URLs from a sitemap", "topics": [ "cr_from_sitemap" ] }, { "page": "cr_normalize_url", "title": "Normalise a URL into a canonical form", "topics": [ "cr_normalize_url" ] }, { "page": "cr_on_html", "title": "Register an HTML handler", "topics": [ "cr_on_html" ] }, { "page": "cr_on_pdf", "title": "Register a PDF handler", "topics": [ "cr_on_pdf" ] }, { "page": "cr_options", "title": "Set crawler options", "topics": [ "cr_options" ] }, { "page": "cr_parallel", "title": "Enable parallel (concurrent) fetching", "topics": [ "cr_parallel" ] }, { "page": "cr_persist", "title": "Persist a crawl to a run directory (and resume it)", "topics": [ "cr_persist" ] }, { "page": "cr_run", "title": "Run a crawl", "topics": [ "cr_run" ] }, { "page": "cr_store", "title": "Configure the key-value store for binary content", "topics": [ "cr_store" ] }, { "page": "cr_stream", "title": "Enable the streaming scheduler", "topics": [ "cr_stream" ] }, { "page": "cr_use_browser", "title": "Use the headless-browser fetch backend", "topics": [ "cr_use_browser" ] }, { "page": "cr_use_http", "title": "Use the HTTP fetch backend", "topics": [ "cr_use_http" ] }, { "page": "crawler", "title": "Create a crawler", "topics": [ "crawler" ] }, { "page": "Crawler-class", "title": "Crawler", "topics": [ "Crawler", "Crawler-class" ] }, { "page": "Dataset", "title": "Dataset", "topics": [ "Dataset" ] }, { "page": "KeyValueStore", "title": "Key-value store", "topics": [ "KeyValueStore" ] }, { "page": "RequestQueue", "title": "Request queue", "topics": [ "RequestQueue" ] } ], "_pkglogo": "https://github.com/strategicprojects/crawlee/raw/HEAD/man/figures/logo.svg", "_readme": "https://github.com/strategicprojects/crawlee/raw/HEAD/README.md", "_rundeps": [ "askpass", "cli", "curl", "glue", "httr", "httr2", "jsonlite", "lifecycle", "magrittr", "mime", "openssl", "pillar", "pkgconfig", "R6", "rappdirs", "rlang", "rvest", "selectr", "sys", "tibble", "utf8", "vctrs", "withr", "xml2" ], "_vignettes": [ { "source": "rag-pipeline.Rmd", "filename": "rag-pipeline.html", "title": "A RAG pipeline", "engine": "knitr::rmarkdown", "headings": [ "1. Crawl and collect text", "2. Chunk", "3. Embed", "4. Export for retrieval", "End to end" ], "created": "2026-06-27 18:50:17", "modified": "2026-06-27 19:06:52", "commits": 2 }, { "source": "crawling-a-website.Rmd", "filename": "crawling-a-website.html", "title": "Crawling a website", "engine": "knitr::rmarkdown", "headings": [ "The model", "Your first crawler", "Following links", "Controlling scope", "Routing different page types", "Crawling from a sitemap", "Rendering JavaScript pages", "Where next" ], "created": "2026-06-27 18:42:17", "modified": "2026-06-27 18:42:17", "commits": 1 }, { "source": "crawlee.Rmd", "filename": "crawlee.html", "title": "Getting started with crawlee", "engine": "knitr::rmarkdown", "headings": [ "The mental model", "A minimal crawl", "The handler context", "Controlling link discovery", "Reproducibility" ], "created": "2026-06-27 16:43:28", "modified": "2026-06-27 18:42:17", "commits": 2 }, { "source": "scaling-and-politeness.Rmd", "filename": "scaling-and-politeness.html", "title": "Scaling and politeness", "engine": "knitr::rmarkdown", "headings": [ "Being a good web citizen", "Going faster", "Fixed-concurrency batches — cr_parallel()", "Adaptive batches — cr_autoscale()", "Continuous streaming pool — cr_stream()", "Choosing an engine", "Combining with persistence" ], "created": "2026-06-27 18:50:17", "modified": "2026-06-27 18:50:17", "commits": 1 }, { "source": "storage-and-resumable-runs.Rmd", "filename": "storage-and-resumable-runs.html", "title": "Storage and resumable runs", "engine": "knitr::rmarkdown", "headings": [ "The dataset", "The key-value store", "The request queue and reproducibility", "One-call setup: cr_persist()", "Resuming" ], "created": "2026-06-27 18:50:17", "modified": "2026-06-27 18:50:17", "commits": 1 } ], "_score": 4.698970004336019, "_indexed": true, "_nocasepkg": "crawlee", "_universes": [ "strategicprojects" ], "_binaries": [ { "r": "4.7.0", "os": "linux", "version": "0.1.0", "date": "2026-07-04T09:26:15.000Z", "distro": "resolute", "commit": "fef3fd8e7c115fff2dccc4b4091209f16ca7e573", "fileid": "https://r2.ropensci.org/ca124369ac23250e4e1279ac0ba1df06ce60d1bead35af786054f2307ce57082", "status": "success", "check": "OK", "buildurl": "https://github.com/r-universe/strategicprojects/actions/runs/28701748405" }, { "r": "4.6.1", "os": "linux", "version": "0.1.0", "date": "2026-07-04T09:26:15.000Z", "distro": "resolute", "commit": "fef3fd8e7c115fff2dccc4b4091209f16ca7e573", "fileid": "https://r2.ropensci.org/ca03799128c40dc6aca80028050e86da002b55ce401f4301a59610da00cd470b", "status": "success", "check": "OK", "buildurl": "https://github.com/r-universe/strategicprojects/actions/runs/28701748405" }, { "r": "4.5.3", "os": "mac", "version": "0.1.0", "date": "2026-07-04T09:59:21.000Z", "commit": "fef3fd8e7c115fff2dccc4b4091209f16ca7e573", "fileid": "https://r2.ropensci.org/e58dceaa2c06d286b6a2bdf32a9d85bcf7d5a8cf16b731a37f449e34df1898ce", "status": "success", "check": "OK", "buildurl": "https://github.com/r-universe/strategicprojects/actions/runs/28701748405" }, { "r": "4.6.1", "os": "mac", "version": "0.1.0", "date": "2026-07-04T09:58:57.000Z", "commit": "fef3fd8e7c115fff2dccc4b4091209f16ca7e573", "fileid": "https://r2.ropensci.org/259bc8c17048530ccabf08497e821b074970a3c60c74503057ac2f912e319bc2", "status": "success", "check": "OK", "buildurl": "https://github.com/r-universe/strategicprojects/actions/runs/28701748405" }, { "r": "4.6.0", "os": "wasm", "version": "0.1.0", "date": "2026-07-04T09:26:33.000Z", "commit": "fef3fd8e7c115fff2dccc4b4091209f16ca7e573", "fileid": "https://r2.ropensci.org/e6446b08f7502eaf5ce08ed46cd96fdd3c96c381ec175a73b48ed873b6649b81", "status": "success", "buildurl": "https://github.com/r-universe/strategicprojects/actions/runs/28701748405" }, { "r": "4.7.0", "os": "win", "version": "0.1.0", "date": "2026-07-04T09:25:08.000Z", "commit": "fef3fd8e7c115fff2dccc4b4091209f16ca7e573", "fileid": "https://r2.ropensci.org/872cdfc48256e412b1c6d80d6014bc851e7fe7688172ea3f7b4761bd5b9b7330", "status": "success", "check": "OK", "buildurl": "https://github.com/r-universe/strategicprojects/actions/runs/28701748405" }, { "r": "4.5.3", "os": "win", "version": "0.1.0", "date": "2026-07-04T09:24:53.000Z", "commit": "fef3fd8e7c115fff2dccc4b4091209f16ca7e573", "fileid": "https://r2.ropensci.org/4148cc090eba59bc2bd361aa8df04cd6447d0efc4df54cc2029f225cfa08b7ec", "status": "success", "check": "OK", "buildurl": "https://github.com/r-universe/strategicprojects/actions/runs/28701748405" }, { "r": "4.6.1", "os": "win", "version": "0.1.0", "date": "2026-07-04T09:25:00.000Z", "commit": "fef3fd8e7c115fff2dccc4b4091209f16ca7e573", "fileid": "https://r2.ropensci.org/4a3a96beacbc6cf9962fc1e6c47d8d1a54ed3e2ea1be55b6e8ff25d41426a04f", "status": "success", "check": "OK", "buildurl": "https://github.com/r-universe/strategicprojects/actions/runs/28701748405" } ] }