378 lines
9.8 KiB
Python
378 lines
9.8 KiB
Python
"""
|
|
Async HTTP Client
|
|
|
|
Provides async HTTP client to replace synchronous requests library in FastAPI endpoints.
|
|
This prevents blocking the event loop and improves concurrency.
|
|
|
|
Usage:
|
|
from web.backend.core.http_client import http_client, get_http_client
|
|
|
|
# In async endpoint
|
|
async def my_endpoint():
|
|
async with get_http_client() as client:
|
|
response = await client.get("https://api.example.com/data")
|
|
return response.json()
|
|
|
|
# Or use the singleton
|
|
response = await http_client.get("https://api.example.com/data")
|
|
"""
|
|
|
|
import httpx
|
|
from typing import Optional, Dict, Any, Union
|
|
from contextlib import asynccontextmanager
|
|
import asyncio
|
|
|
|
from .config import settings
|
|
from .exceptions import NetworkError, ServiceError
|
|
|
|
|
|
# Default timeouts
|
|
DEFAULT_TIMEOUT = httpx.Timeout(
|
|
connect=10.0,
|
|
read=30.0,
|
|
write=10.0,
|
|
pool=10.0
|
|
)
|
|
|
|
# Longer timeout for slow operations
|
|
LONG_TIMEOUT = httpx.Timeout(
|
|
connect=10.0,
|
|
read=120.0,
|
|
write=30.0,
|
|
pool=10.0
|
|
)
|
|
|
|
|
|
class AsyncHTTPClient:
|
|
"""
|
|
Async HTTP client wrapper with retry logic and error handling.
|
|
|
|
Features:
|
|
- Connection pooling
|
|
- Automatic retries
|
|
- Timeout handling
|
|
- Error conversion to custom exceptions
|
|
"""
|
|
|
|
def __init__(
|
|
self,
|
|
base_url: Optional[str] = None,
|
|
timeout: Optional[httpx.Timeout] = None,
|
|
headers: Optional[Dict[str, str]] = None,
|
|
max_retries: int = 3
|
|
):
|
|
self.base_url = base_url
|
|
self.timeout = timeout or DEFAULT_TIMEOUT
|
|
self.headers = headers or {}
|
|
self.max_retries = max_retries
|
|
self._client: Optional[httpx.AsyncClient] = None
|
|
|
|
async def _get_client(self) -> httpx.AsyncClient:
|
|
"""Get or create the async client"""
|
|
if self._client is None or self._client.is_closed:
|
|
client_kwargs = {
|
|
"timeout": self.timeout,
|
|
"headers": self.headers,
|
|
"follow_redirects": True,
|
|
"limits": httpx.Limits(
|
|
max_keepalive_connections=20,
|
|
max_connections=100,
|
|
keepalive_expiry=30.0
|
|
)
|
|
}
|
|
# Only add base_url if it's not None
|
|
if self.base_url is not None:
|
|
client_kwargs["base_url"] = self.base_url
|
|
|
|
self._client = httpx.AsyncClient(**client_kwargs)
|
|
return self._client
|
|
|
|
async def close(self):
|
|
"""Close the client connection"""
|
|
if self._client and not self._client.is_closed:
|
|
await self._client.aclose()
|
|
self._client = None
|
|
|
|
async def _request(
|
|
self,
|
|
method: str,
|
|
url: str,
|
|
**kwargs
|
|
) -> httpx.Response:
|
|
"""
|
|
Make an HTTP request with retry logic.
|
|
|
|
Args:
|
|
method: HTTP method (GET, POST, etc.)
|
|
url: URL to request
|
|
**kwargs: Additional arguments for httpx
|
|
|
|
Returns:
|
|
httpx.Response
|
|
|
|
Raises:
|
|
NetworkError: On network failures
|
|
ServiceError: On HTTP errors
|
|
"""
|
|
client = await self._get_client()
|
|
last_error = None
|
|
|
|
for attempt in range(self.max_retries):
|
|
try:
|
|
response = await client.request(method, url, **kwargs)
|
|
|
|
# Raise for 4xx/5xx status codes
|
|
if response.status_code >= 400:
|
|
if response.status_code >= 500:
|
|
raise ServiceError(
|
|
f"Server error: {response.status_code}",
|
|
{"url": url, "status": response.status_code}
|
|
)
|
|
# Don't retry client errors
|
|
return response
|
|
|
|
return response
|
|
|
|
except httpx.ConnectError as e:
|
|
last_error = NetworkError(
|
|
f"Connection failed: {e}",
|
|
{"url": url, "attempt": attempt + 1}
|
|
)
|
|
except httpx.TimeoutException as e:
|
|
last_error = NetworkError(
|
|
f"Request timed out: {e}",
|
|
{"url": url, "attempt": attempt + 1}
|
|
)
|
|
except httpx.HTTPStatusError as e:
|
|
last_error = ServiceError(
|
|
f"HTTP error: {e}",
|
|
{"url": url, "status": e.response.status_code}
|
|
)
|
|
# Don't retry client errors
|
|
if e.response.status_code < 500:
|
|
raise last_error
|
|
except Exception as e:
|
|
last_error = NetworkError(
|
|
f"Request failed: {e}",
|
|
{"url": url, "attempt": attempt + 1}
|
|
)
|
|
|
|
# Wait before retry (exponential backoff)
|
|
if attempt < self.max_retries - 1:
|
|
await asyncio.sleep(2 ** attempt)
|
|
|
|
raise last_error
|
|
|
|
# Convenience methods
|
|
|
|
async def get(
|
|
self,
|
|
url: str,
|
|
params: Optional[Dict[str, Any]] = None,
|
|
headers: Optional[Dict[str, str]] = None,
|
|
timeout: Optional[float] = None,
|
|
**kwargs
|
|
) -> httpx.Response:
|
|
"""Make GET request"""
|
|
request_timeout = httpx.Timeout(timeout) if timeout else None
|
|
return await self._request(
|
|
"GET",
|
|
url,
|
|
params=params,
|
|
headers=headers,
|
|
timeout=request_timeout,
|
|
**kwargs
|
|
)
|
|
|
|
async def post(
|
|
self,
|
|
url: str,
|
|
data: Optional[Dict[str, Any]] = None,
|
|
json: Optional[Dict[str, Any]] = None,
|
|
headers: Optional[Dict[str, str]] = None,
|
|
timeout: Optional[float] = None,
|
|
**kwargs
|
|
) -> httpx.Response:
|
|
"""Make POST request"""
|
|
request_timeout = httpx.Timeout(timeout) if timeout else None
|
|
return await self._request(
|
|
"POST",
|
|
url,
|
|
data=data,
|
|
json=json,
|
|
headers=headers,
|
|
timeout=request_timeout,
|
|
**kwargs
|
|
)
|
|
|
|
async def put(
|
|
self,
|
|
url: str,
|
|
data: Optional[Dict[str, Any]] = None,
|
|
json: Optional[Dict[str, Any]] = None,
|
|
headers: Optional[Dict[str, str]] = None,
|
|
**kwargs
|
|
) -> httpx.Response:
|
|
"""Make PUT request"""
|
|
return await self._request(
|
|
"PUT",
|
|
url,
|
|
data=data,
|
|
json=json,
|
|
headers=headers,
|
|
**kwargs
|
|
)
|
|
|
|
async def delete(
|
|
self,
|
|
url: str,
|
|
headers: Optional[Dict[str, str]] = None,
|
|
**kwargs
|
|
) -> httpx.Response:
|
|
"""Make DELETE request"""
|
|
return await self._request(
|
|
"DELETE",
|
|
url,
|
|
headers=headers,
|
|
**kwargs
|
|
)
|
|
|
|
async def head(
|
|
self,
|
|
url: str,
|
|
headers: Optional[Dict[str, str]] = None,
|
|
**kwargs
|
|
) -> httpx.Response:
|
|
"""Make HEAD request"""
|
|
return await self._request(
|
|
"HEAD",
|
|
url,
|
|
headers=headers,
|
|
**kwargs
|
|
)
|
|
|
|
|
|
# Singleton client for general use
|
|
http_client = AsyncHTTPClient()
|
|
|
|
|
|
@asynccontextmanager
|
|
async def get_http_client(
|
|
base_url: Optional[str] = None,
|
|
timeout: Optional[httpx.Timeout] = None,
|
|
headers: Optional[Dict[str, str]] = None
|
|
):
|
|
"""
|
|
Context manager for creating a temporary HTTP client.
|
|
|
|
Usage:
|
|
async with get_http_client(base_url="https://api.example.com") as client:
|
|
response = await client.get("/endpoint")
|
|
"""
|
|
client = AsyncHTTPClient(
|
|
base_url=base_url,
|
|
timeout=timeout,
|
|
headers=headers
|
|
)
|
|
try:
|
|
yield client
|
|
finally:
|
|
await client.close()
|
|
|
|
|
|
# Helper functions for common patterns
|
|
|
|
async def fetch_json(
|
|
url: str,
|
|
params: Optional[Dict[str, Any]] = None,
|
|
headers: Optional[Dict[str, str]] = None
|
|
) -> Dict[str, Any]:
|
|
"""
|
|
Fetch JSON from URL.
|
|
|
|
Args:
|
|
url: URL to fetch
|
|
params: Query parameters
|
|
headers: Request headers
|
|
|
|
Returns:
|
|
Parsed JSON response
|
|
"""
|
|
response = await http_client.get(url, params=params, headers=headers)
|
|
return response.json()
|
|
|
|
|
|
async def post_json(
|
|
url: str,
|
|
data: Dict[str, Any],
|
|
headers: Optional[Dict[str, str]] = None
|
|
) -> Dict[str, Any]:
|
|
"""
|
|
POST JSON to URL and return JSON response.
|
|
|
|
Args:
|
|
url: URL to post to
|
|
data: JSON data to send
|
|
headers: Request headers
|
|
|
|
Returns:
|
|
Parsed JSON response
|
|
"""
|
|
response = await http_client.post(url, json=data, headers=headers)
|
|
return response.json()
|
|
|
|
|
|
async def check_url_accessible(url: str, timeout: float = 5.0) -> bool:
|
|
"""
|
|
Check if a URL is accessible.
|
|
|
|
Args:
|
|
url: URL to check
|
|
timeout: Request timeout
|
|
|
|
Returns:
|
|
True if URL is accessible
|
|
"""
|
|
try:
|
|
response = await http_client.head(url, timeout=timeout)
|
|
return response.status_code < 400
|
|
except Exception:
|
|
return False
|
|
|
|
|
|
async def download_file(
|
|
url: str,
|
|
save_path: str,
|
|
headers: Optional[Dict[str, str]] = None,
|
|
chunk_size: int = 8192
|
|
) -> int:
|
|
"""
|
|
Download file from URL.
|
|
|
|
Args:
|
|
url: URL to download
|
|
save_path: Path to save file
|
|
headers: Request headers
|
|
chunk_size: Size of chunks for streaming
|
|
|
|
Returns:
|
|
Number of bytes downloaded
|
|
"""
|
|
from pathlib import Path
|
|
|
|
client = await http_client._get_client()
|
|
total_bytes = 0
|
|
|
|
async with client.stream("GET", url, headers=headers) as response:
|
|
response.raise_for_status()
|
|
|
|
# Ensure directory exists
|
|
Path(save_path).parent.mkdir(parents=True, exist_ok=True)
|
|
|
|
with open(save_path, 'wb') as f:
|
|
async for chunk in response.aiter_bytes(chunk_size):
|
|
f.write(chunk)
|
|
total_bytes += len(chunk)
|
|
|
|
return total_bytes
|