lychee_lib/ratelimit/
mod.rs

1//! Per-host rate limiting and concurrency control.
2//!
3//! This module provides adaptive rate limiting for HTTP requests on a per-host basis.
4//! It prevents overwhelming servers with too many concurrent requests and respects
5//! server-provided rate limit headers.
6//!
7//! # Architecture
8//!
9//! - [`HostKey`]: Represents a hostname/domain for rate limiting
10//! - [`Host`]: Manages rate limiting, concurrency, and caching for a specific host
11//! - [`HostPool`]: Coordinates multiple hosts and routes requests appropriately
12//! - [`HostConfig`]: Configuration for per-host behavior
13//! - [`HostStats`]: Statistics tracking for each host
14
15mod config;
16mod headers;
17mod host;
18mod pool;
19
20pub use config::{HostConfig, HostConfigs, RateLimitConfig};
21pub use host::{Host, HostKey, HostStats, HostStatsMap};
22use http::HeaderMap;
23pub use pool::{ClientMap, HostPool};
24use reqwest::Response;
25use url::Url;
26
27use crate::{ErrorKind, Result};
28
29/// The result of a HTTP request, used for internal per-host caching.
30/// This abstraction exists, because [`Response`] cannot easily be cached
31/// since it does not implement [`Clone`].
32#[derive(Debug, Clone)]
33pub(crate) struct CacheableResponse {
34    pub(crate) status: reqwest::StatusCode,
35    pub(crate) text: Option<String>,
36    pub(crate) headers: HeaderMap,
37    pub(crate) url: Url,
38}
39
40impl CacheableResponse {
41    async fn from_response(response: Response, needs_body: bool) -> Result<Self> {
42        let status = response.status();
43        let headers = response.headers().clone();
44        let url = response.url().clone();
45        let text = if needs_body {
46            Some(response.text().await.map_err(ErrorKind::ReadResponseBody)?)
47        } else {
48            None
49        };
50
51        Ok(Self {
52            status,
53            text,
54            headers,
55            url,
56        })
57    }
58}
lychee_lib/ratelimit/mod.rs

lychee_lib/ratelimit/
mod.rs
