Todd
377
web/backend/core/http_client.py
Normal file
377
web/backend/core/http_client.py
Normal file
@@ -0,0 +1,377 @@
|
||||
"""
|
||||
Async HTTP Client
|
||||
|
||||
Provides async HTTP client to replace synchronous requests library in FastAPI endpoints.
|
||||
This prevents blocking the event loop and improves concurrency.
|
||||
|
||||
Usage:
|
||||
from web.backend.core.http_client import http_client, get_http_client
|
||||
|
||||
# In async endpoint
|
||||
async def my_endpoint():
|
||||
async with get_http_client() as client:
|
||||
response = await client.get("https://api.example.com/data")
|
||||
return response.json()
|
||||
|
||||
# Or use the singleton
|
||||
response = await http_client.get("https://api.example.com/data")
|
||||
"""
|
||||
|
||||
import httpx
|
||||
from typing import Optional, Dict, Any, Union
|
||||
from contextlib import asynccontextmanager
|
||||
import asyncio
|
||||
|
||||
from .config import settings
|
||||
from .exceptions import NetworkError, ServiceError
|
||||
|
||||
|
||||
# Default timeouts
|
||||
DEFAULT_TIMEOUT = httpx.Timeout(
|
||||
connect=10.0,
|
||||
read=30.0,
|
||||
write=10.0,
|
||||
pool=10.0
|
||||
)
|
||||
|
||||
# Longer timeout for slow operations
|
||||
LONG_TIMEOUT = httpx.Timeout(
|
||||
connect=10.0,
|
||||
read=120.0,
|
||||
write=30.0,
|
||||
pool=10.0
|
||||
)
|
||||
|
||||
|
||||
class AsyncHTTPClient:
|
||||
"""
|
||||
Async HTTP client wrapper with retry logic and error handling.
|
||||
|
||||
Features:
|
||||
- Connection pooling
|
||||
- Automatic retries
|
||||
- Timeout handling
|
||||
- Error conversion to custom exceptions
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
base_url: Optional[str] = None,
|
||||
timeout: Optional[httpx.Timeout] = None,
|
||||
headers: Optional[Dict[str, str]] = None,
|
||||
max_retries: int = 3
|
||||
):
|
||||
self.base_url = base_url
|
||||
self.timeout = timeout or DEFAULT_TIMEOUT
|
||||
self.headers = headers or {}
|
||||
self.max_retries = max_retries
|
||||
self._client: Optional[httpx.AsyncClient] = None
|
||||
|
||||
async def _get_client(self) -> httpx.AsyncClient:
|
||||
"""Get or create the async client"""
|
||||
if self._client is None or self._client.is_closed:
|
||||
client_kwargs = {
|
||||
"timeout": self.timeout,
|
||||
"headers": self.headers,
|
||||
"follow_redirects": True,
|
||||
"limits": httpx.Limits(
|
||||
max_keepalive_connections=20,
|
||||
max_connections=100,
|
||||
keepalive_expiry=30.0
|
||||
)
|
||||
}
|
||||
# Only add base_url if it's not None
|
||||
if self.base_url is not None:
|
||||
client_kwargs["base_url"] = self.base_url
|
||||
|
||||
self._client = httpx.AsyncClient(**client_kwargs)
|
||||
return self._client
|
||||
|
||||
async def close(self):
|
||||
"""Close the client connection"""
|
||||
if self._client and not self._client.is_closed:
|
||||
await self._client.aclose()
|
||||
self._client = None
|
||||
|
||||
async def _request(
|
||||
self,
|
||||
method: str,
|
||||
url: str,
|
||||
**kwargs
|
||||
) -> httpx.Response:
|
||||
"""
|
||||
Make an HTTP request with retry logic.
|
||||
|
||||
Args:
|
||||
method: HTTP method (GET, POST, etc.)
|
||||
url: URL to request
|
||||
**kwargs: Additional arguments for httpx
|
||||
|
||||
Returns:
|
||||
httpx.Response
|
||||
|
||||
Raises:
|
||||
NetworkError: On network failures
|
||||
ServiceError: On HTTP errors
|
||||
"""
|
||||
client = await self._get_client()
|
||||
last_error = None
|
||||
|
||||
for attempt in range(self.max_retries):
|
||||
try:
|
||||
response = await client.request(method, url, **kwargs)
|
||||
|
||||
# Raise for 4xx/5xx status codes
|
||||
if response.status_code >= 400:
|
||||
if response.status_code >= 500:
|
||||
raise ServiceError(
|
||||
f"Server error: {response.status_code}",
|
||||
{"url": url, "status": response.status_code}
|
||||
)
|
||||
# Don't retry client errors
|
||||
return response
|
||||
|
||||
return response
|
||||
|
||||
except httpx.ConnectError as e:
|
||||
last_error = NetworkError(
|
||||
f"Connection failed: {e}",
|
||||
{"url": url, "attempt": attempt + 1}
|
||||
)
|
||||
except httpx.TimeoutException as e:
|
||||
last_error = NetworkError(
|
||||
f"Request timed out: {e}",
|
||||
{"url": url, "attempt": attempt + 1}
|
||||
)
|
||||
except httpx.HTTPStatusError as e:
|
||||
last_error = ServiceError(
|
||||
f"HTTP error: {e}",
|
||||
{"url": url, "status": e.response.status_code}
|
||||
)
|
||||
# Don't retry client errors
|
||||
if e.response.status_code < 500:
|
||||
raise last_error
|
||||
except Exception as e:
|
||||
last_error = NetworkError(
|
||||
f"Request failed: {e}",
|
||||
{"url": url, "attempt": attempt + 1}
|
||||
)
|
||||
|
||||
# Wait before retry (exponential backoff)
|
||||
if attempt < self.max_retries - 1:
|
||||
await asyncio.sleep(2 ** attempt)
|
||||
|
||||
raise last_error
|
||||
|
||||
# Convenience methods
|
||||
|
||||
async def get(
|
||||
self,
|
||||
url: str,
|
||||
params: Optional[Dict[str, Any]] = None,
|
||||
headers: Optional[Dict[str, str]] = None,
|
||||
timeout: Optional[float] = None,
|
||||
**kwargs
|
||||
) -> httpx.Response:
|
||||
"""Make GET request"""
|
||||
request_timeout = httpx.Timeout(timeout) if timeout else None
|
||||
return await self._request(
|
||||
"GET",
|
||||
url,
|
||||
params=params,
|
||||
headers=headers,
|
||||
timeout=request_timeout,
|
||||
**kwargs
|
||||
)
|
||||
|
||||
async def post(
|
||||
self,
|
||||
url: str,
|
||||
data: Optional[Dict[str, Any]] = None,
|
||||
json: Optional[Dict[str, Any]] = None,
|
||||
headers: Optional[Dict[str, str]] = None,
|
||||
timeout: Optional[float] = None,
|
||||
**kwargs
|
||||
) -> httpx.Response:
|
||||
"""Make POST request"""
|
||||
request_timeout = httpx.Timeout(timeout) if timeout else None
|
||||
return await self._request(
|
||||
"POST",
|
||||
url,
|
||||
data=data,
|
||||
json=json,
|
||||
headers=headers,
|
||||
timeout=request_timeout,
|
||||
**kwargs
|
||||
)
|
||||
|
||||
async def put(
|
||||
self,
|
||||
url: str,
|
||||
data: Optional[Dict[str, Any]] = None,
|
||||
json: Optional[Dict[str, Any]] = None,
|
||||
headers: Optional[Dict[str, str]] = None,
|
||||
**kwargs
|
||||
) -> httpx.Response:
|
||||
"""Make PUT request"""
|
||||
return await self._request(
|
||||
"PUT",
|
||||
url,
|
||||
data=data,
|
||||
json=json,
|
||||
headers=headers,
|
||||
**kwargs
|
||||
)
|
||||
|
||||
async def delete(
|
||||
self,
|
||||
url: str,
|
||||
headers: Optional[Dict[str, str]] = None,
|
||||
**kwargs
|
||||
) -> httpx.Response:
|
||||
"""Make DELETE request"""
|
||||
return await self._request(
|
||||
"DELETE",
|
||||
url,
|
||||
headers=headers,
|
||||
**kwargs
|
||||
)
|
||||
|
||||
async def head(
|
||||
self,
|
||||
url: str,
|
||||
headers: Optional[Dict[str, str]] = None,
|
||||
**kwargs
|
||||
) -> httpx.Response:
|
||||
"""Make HEAD request"""
|
||||
return await self._request(
|
||||
"HEAD",
|
||||
url,
|
||||
headers=headers,
|
||||
**kwargs
|
||||
)
|
||||
|
||||
|
||||
# Singleton client for general use
|
||||
http_client = AsyncHTTPClient()
|
||||
|
||||
|
||||
@asynccontextmanager
|
||||
async def get_http_client(
|
||||
base_url: Optional[str] = None,
|
||||
timeout: Optional[httpx.Timeout] = None,
|
||||
headers: Optional[Dict[str, str]] = None
|
||||
):
|
||||
"""
|
||||
Context manager for creating a temporary HTTP client.
|
||||
|
||||
Usage:
|
||||
async with get_http_client(base_url="https://api.example.com") as client:
|
||||
response = await client.get("/endpoint")
|
||||
"""
|
||||
client = AsyncHTTPClient(
|
||||
base_url=base_url,
|
||||
timeout=timeout,
|
||||
headers=headers
|
||||
)
|
||||
try:
|
||||
yield client
|
||||
finally:
|
||||
await client.close()
|
||||
|
||||
|
||||
# Helper functions for common patterns
|
||||
|
||||
async def fetch_json(
|
||||
url: str,
|
||||
params: Optional[Dict[str, Any]] = None,
|
||||
headers: Optional[Dict[str, str]] = None
|
||||
) -> Dict[str, Any]:
|
||||
"""
|
||||
Fetch JSON from URL.
|
||||
|
||||
Args:
|
||||
url: URL to fetch
|
||||
params: Query parameters
|
||||
headers: Request headers
|
||||
|
||||
Returns:
|
||||
Parsed JSON response
|
||||
"""
|
||||
response = await http_client.get(url, params=params, headers=headers)
|
||||
return response.json()
|
||||
|
||||
|
||||
async def post_json(
|
||||
url: str,
|
||||
data: Dict[str, Any],
|
||||
headers: Optional[Dict[str, str]] = None
|
||||
) -> Dict[str, Any]:
|
||||
"""
|
||||
POST JSON to URL and return JSON response.
|
||||
|
||||
Args:
|
||||
url: URL to post to
|
||||
data: JSON data to send
|
||||
headers: Request headers
|
||||
|
||||
Returns:
|
||||
Parsed JSON response
|
||||
"""
|
||||
response = await http_client.post(url, json=data, headers=headers)
|
||||
return response.json()
|
||||
|
||||
|
||||
async def check_url_accessible(url: str, timeout: float = 5.0) -> bool:
|
||||
"""
|
||||
Check if a URL is accessible.
|
||||
|
||||
Args:
|
||||
url: URL to check
|
||||
timeout: Request timeout
|
||||
|
||||
Returns:
|
||||
True if URL is accessible
|
||||
"""
|
||||
try:
|
||||
response = await http_client.head(url, timeout=timeout)
|
||||
return response.status_code < 400
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
|
||||
async def download_file(
|
||||
url: str,
|
||||
save_path: str,
|
||||
headers: Optional[Dict[str, str]] = None,
|
||||
chunk_size: int = 8192
|
||||
) -> int:
|
||||
"""
|
||||
Download file from URL.
|
||||
|
||||
Args:
|
||||
url: URL to download
|
||||
save_path: Path to save file
|
||||
headers: Request headers
|
||||
chunk_size: Size of chunks for streaming
|
||||
|
||||
Returns:
|
||||
Number of bytes downloaded
|
||||
"""
|
||||
from pathlib import Path
|
||||
|
||||
client = await http_client._get_client()
|
||||
total_bytes = 0
|
||||
|
||||
async with client.stream("GET", url, headers=headers) as response:
|
||||
response.raise_for_status()
|
||||
|
||||
# Ensure directory exists
|
||||
Path(save_path).parent.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
with open(save_path, 'wb') as f:
|
||||
async for chunk in response.aiter_bytes(chunk_size):
|
||||
f.write(chunk)
|
||||
total_bytes += len(chunk)
|
||||
|
||||
return total_bytes
|
||||
Reference in New Issue
Block a user