Data Extraction

def get() -> TextHandler

from scrapling import Fetcher

page = Fetcher.fetch('https://example.com')

# Get single element content
header = page.css('h1').first
if header:
    content = header.get()
    print(content)  # Returns HTML: <h1>Title</h1>

def getall() -> TextHandlers

extract = getall
extract_first = get

# These are equivalent
text1 = page.css('p').extract_first()
text2 = page.css('p').get()

# These are equivalent
texts1 = page.css('p').extract()
texts2 = page.css('p').getall()

@property
def text() -> TextHandler

# Get text content
title = page.css('h1').first
if title:
    print(title.text)  # Returns: "Title" (not <h1>Title</h1>)

# Text is a TextHandler with useful methods
price_text = page.css('.price').first.text
price_clean = price_text.strip().replace('$', '')
price_value = float(price_clean)

def get_all_text(
    separator: str = "\n",
    strip: bool = False,
    ignore_tags: Tuple = ("script", "style"),
    valid_values: bool = True,
) -> TextHandler

@property
def html_content() -> TextHandler

# Get inner HTML
container = page.css('.container').first
inner_html = container.html_content
print(inner_html)  # HTML inside the container

def prettify() -> TextHandler

# Get prettified HTML
element = page.css('div.complex').first
pretty_html = element.prettify()
print(pretty_html)
# Output:
# <div class="complex">
#   <p>Text</p>
#   <span>More text</span>
# </div>

@property
def body() -> str | bytes

# Get raw body
raw_content = page.body

@property
def attrib() -> AttributesHandler

def __getitem__(key: str) -> TextHandler

# Direct access
link = page.css('a').first
href = link['href']
class_name = link['class']

# Raises KeyError if attribute doesn't exist
try:
    missing = link['data-missing']
except KeyError:
    print("Attribute not found")

def __contains__(key: str) -> bool

# Check if attribute exists
link = page.css('a').first

if 'href' in link:
    print(f"Link points to: {link['href']}")

if 'target' in link:
    print(f"Opens in: {link['target']}")
else:
    print("Opens in same window")

@property
def tag() -> str

# Get tag name
element = page.css('.content').first
print(element.tag)  # e.g., "div"

# For text nodes
text_node = page.xpath('//p/text()').first
print(text_node.tag)  # "#text"

def has_class(class_name: str) -> bool

# Check for class
element = page.css('.container').first

if element.has_class('active'):
    print("Element is active")

if element.has_class('container'):
    print("Element is a container")

def re(
    regex: str | Pattern[str],
    replace_entities: bool = True,
    clean_match: bool = False,
    case_sensitive: bool = True,
) -> TextHandlers

def re_first(
    regex: str | Pattern[str],
    default=None,
    replace_entities: bool = True,
    clean_match: bool = False,
    case_sensitive: bool = True,
) -> TextHandler

import re

# Get first price or default
price = page.get_all_text().re_first(r'\$\d+\.\d{2}', default='$0.00')

# Extract product code
code = page.css('.product-info').first.text.re_first(
    r'SKU: ([A-Z0-9-]+)',
    default='N/A'
)

def json() -> Dict

def urljoin(relative_url: str) -> str

# Convert relative URLs to absolute
links = page.css('a')

for link in links:
    relative_href = link.attrib.get('href', '')
    absolute_url = link.urljoin(relative_href)
    print(absolute_url)

# Example:
# page.url = 'https://example.com/products/'
# relative = '../about'
# result = 'https://example.com/about'

# Extract structured product data
products = []

for item in page.css('.product-card'):
    product = {
        'name': item.css('.product-name').first.text.strip(),
        'price': item.css('.price').first.text.re_first(r'\d+\.\d{2}'),
        'image': item.css('img').first['src'],
        'url': item.urljoin(item.css('a').first['href']),
        'in_stock': item.has_class('in-stock'),
        'rating': float(item.css('[data-rating]').first['data-rating'])
    }
    products.append(product)

print(f"Found {len(products)} products")

# Extract article with metadata
article = page.css('article').first

if article:
    data = {
        'title': article.css('h1').first.text.strip(),
        'author': article.css('.author').first.text.strip(),
        'date': article.css('time').first['datetime'],
        'content': article.css('.content').first.get_all_text(
            separator='\n\n',
            strip=True,
            ignore_tags=('script', 'style', 'aside')
        ),
        'tags': [tag.text for tag in article.css('.tag')],
        'image_url': article.css('img').first['src'] if article.css('img') else None
    }

# Extract data from HTML table
table = page.css('table.data-table').first

if table:
    headers = [th.text.strip() for th in table.css('thead th')]
    
    rows = []
    for tr in table.css('tbody tr'):
        row = {}
        cells = tr.css('td')
        for i, cell in enumerate(cells):
            header = headers[i] if i < len(headers) else f'column_{i}'
            row[header] = cell.text.strip()
        rows.append(row)
    
    print(f"Extracted {len(rows)} rows")

import re

# Extract various contact information
contact_section = page.css('.contact-info').first

if contact_section:
    text = contact_section.get_all_text()
    
    contact = {
        'emails': text.re(r'[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}'),
        'phones': text.re(r'\+?\d[\d\s\-\(\)]{8,}\d'),
        'address': contact_section.css('.address').first.text if contact_section.css('.address') else None,
    }