FetcherSession
A factory context manager that provides configured fetcher sessions for HTTP requests. Useful for maintaining persistent connections and reusing configurations across multiple requests.
from scrapling import FetcherSession
with FetcherSession(impersonate='chrome', timeout=60) as session:
response1 = session.get('https://example.com/page1')
response2 = session.get('https://example.com/page2')
# Session is automatically closed after the context
Constructor
FetcherSession(
impersonate='chrome',
http3=False,
stealthy_headers=True,
proxies=None,
proxy=None,
proxy_auth=None,
timeout=30,
headers=None,
retries=3,
retry_delay=1,
follow_redirects=True,
max_redirects=30,
verify=True,
cert=None,
selector_config=None,
proxy_rotator=None
)
impersonate
str | list
default:"chrome"
Browser version to impersonate. Can be a single browser string or a list of browser strings for random selection
Whether to use HTTP3. Might be problematic if used with impersonate
If enabled, creates and adds real browser headers. Also sets the referer header as if this request came from a Google search of URL’s domain
Dict of proxies to use. Format: {"http": proxy_url, "https": proxy_url}
Proxy URL to use. Format: "http://username:password@localhost:8030". Cannot be used together with the proxies parameter
HTTP basic auth for proxy, tuple of (username, password)
Number of seconds to wait before timing out
Headers to include in the session with every request
Number of seconds to wait between retry attempts
Whether to follow redirects
Maximum number of redirects. Use -1 for unlimited
Whether to verify HTTPS certificates
Tuple of (cert, key) filenames for the client certificate
Arguments passed when creating the final Selector class
A ProxyRotator instance for automatic proxy rotation
Methods
When used as a context manager, FetcherSession yields a session object with the following methods:
get()
session.get(url: str, **kwargs) -> Response
Fetcher.get().
post()
session.post(url: str, **kwargs) -> Response
Fetcher.post().
put()
session.put(url: str, **kwargs) -> Response
Fetcher.put().
delete()
session.delete(url: str, **kwargs) -> Response
Fetcher.delete().
Async Usage
from scrapling import FetcherSession
async with FetcherSession() as session:
response1 = await session.get('https://example.com/page1')
response2 = await session.get('https://example.com/page2')
StealthySession
A stealthy browser session manager with page pooling. Maintains a persistent browser context for multiple requests.
from scrapling import StealthySession
with StealthySession(headless=True, solve_cloudflare=True) as session:
response1 = session.fetch('https://example.com/page1')
response2 = session.fetch('https://example.com/page2')
# Browser is automatically closed after the context
Constructor
Accepts all parameters from StealthyFetcher.fetch(). The session maintains these settings across all fetch calls.
Methods
start()
Manually start the browser session. Automatically called when using as a context manager.
fetch()
session.fetch(url: str, **kwargs) -> Response
StealthyFetcher.fetch(), which can override session defaults.
Static proxy to override rotator and session proxy. A new browser context will be created and used with it
StealthyFetcher.fetch().
AsyncStealthySession
Asynchronous version of StealthySession.
import asyncio
from scrapling import AsyncStealthySession
async def main():
async with AsyncStealthySession(headless=True) as session:
response1 = await session.fetch('https://example.com/page1')
response2 = await session.fetch('https://example.com/page2')
asyncio.run(main())
Methods
start()
await session.start() -> None
fetch()
await session.fetch(url: str, **kwargs) -> Response
DynamicSession
A browser session manager with page pooling using Playwright.
from scrapling import DynamicSession
with DynamicSession(headless=True) as session:
response1 = session.fetch('https://example.com/page1')
response2 = session.fetch('https://example.com/page2')
Constructor
Accepts all parameters from DynamicFetcher.fetch(). The session maintains these settings across all fetch calls.
Methods
start()
Manually start the browser session.
fetch()
session.fetch(url: str, **kwargs) -> Response
DynamicFetcher.fetch().
AsyncDynamicSession
Asynchronous version of DynamicSession.
import asyncio
from scrapling import AsyncDynamicSession
async def main():
async with AsyncDynamicSession(headless=True) as session:
response1 = await session.fetch('https://example.com/page1')
response2 = await session.fetch('https://example.com/page2')
asyncio.run(main())
Constructor
The maximum number of tabs to be opened at the same time. Used in rotation through a PagePool
DynamicFetcher.fetch().
Methods
start()
await session.start() -> None
fetch()
await session.fetch(url: str, **kwargs) -> Response
Usage Examples
HTTP Session with Shared Config
from scrapling import FetcherSession
with FetcherSession(
headers={'User-Agent': 'MyBot/1.0'},
timeout=60,
retries=5
) as session:
# All requests share the same headers and settings
response1 = session.get('https://api.example.com/users')
response2 = session.post('https://api.example.com/data', json={'key': 'value'})
Browser Session with Cookies Persistence
from scrapling import StealthySession
with StealthySession() as session:
# Login
login_response = session.fetch(
'https://example.com/login',
page_action=lambda page: (
page.fill('#username', 'user'),
page.fill('#password', 'pass'),
page.click('#submit')
)
)
# Cookies are maintained across requests
dashboard = session.fetch('https://example.com/dashboard')
Async Session for Concurrent Requests
import asyncio
from scrapling import AsyncStealthySession
async def scrape_multiple():
async with AsyncStealthySession(headless=True) as session:
# Fetch multiple pages concurrently with shared browser context
tasks = [
session.fetch(f'https://example.com/page{i}')
for i in range(1, 6)
]
responses = await asyncio.gather(*tasks)
return responses
results = asyncio.run(scrape_multiple())
Session with Proxy Rotation
from scrapling import FetcherSession, ProxyRotator
rotator = ProxyRotator([
'http://proxy1.example.com:8080',
'http://proxy2.example.com:8080',
'http://proxy3.example.com:8080'
])
with FetcherSession(proxy_rotator=rotator) as session:
# Each request automatically uses a different proxy
response1 = session.get('https://example.com/page1')
response2 = session.get('https://example.com/page2')
response3 = session.get('https://example.com/page3')