Extract Web Text with Python: Trafilatura API Guide

# Core extraction functions
from trafilatura import extract, bare_extraction, extract_with_metadata, html2txt, baseline

# Downloads
from trafilatura import fetch_url, fetch_response

# Settings class for batch configuration
from trafilatura.settings import Extractor

# Readability check
from trafilatura.readability_lxml import is_probably_readerable

# Memory management
from trafilatura.meta import reset_caches

from trafilatura import fetch_url, extract

url = "https://github.blog/2019-03-29-leader-spotlight-erin-spiceland/"
downloaded = fetch_url(url)

# plain text output (default)
result = extract(downloaded)

# XML output with metadata
result = extract(downloaded, output_format="xml", with_metadata=True)

# pass the URL to enable relative-to-absolute link conversion and date extraction from URL
result = extract(downloaded, url=url)

from trafilatura import bare_extraction, fetch_url

downloaded = fetch_url("https://www.example.org/article")

# returns a Document object
doc = bare_extraction(downloaded)

# access attributes directly
print(doc.title)
print(doc.date)
print(doc.text)

# convert to a plain dict
data = doc.as_dict()
print(data["title"], data["url"])

from trafilatura import extract_with_metadata, fetch_url

downloaded = fetch_url("https://www.example.org/article")

# returns a Document object with text attribute containing formatted output
doc = extract_with_metadata(downloaded, output_format="txt")

if doc is not None:
    print(doc.title)
    print(doc.author)
    print(doc.text)   # formatted text in the chosen output_format

from trafilatura import html2txt, fetch_url

downloaded = fetch_url("https://www.example.org")
text = html2txt(downloaded)

from trafilatura import baseline, fetch_url

downloaded = fetch_url("https://www.example.org/article")
postbody, text, length = baseline(downloaded)

print(f"Extracted {length} characters")
print(text)

# plain text (default)
extract(downloaded)

# Markdown with formatting preserved
extract(downloaded, output_format="markdown")

# JSON with metadata
extract(downloaded, output_format="json", with_metadata=True)

# CSV
extract(downloaded, output_format="csv")

# HTML (v1.11+)
extract(downloaded, output_format="html")

# XML with structure
extract(downloaded, output_format="xml")

# TEI-XML (Text Encoding Initiative)
extract(downloaded, output_format="xmltei")

# comments are included by default — disable them
result = extract(downloaded, include_comments=False)

# tables are included by default — disable them
result = extract(downloaded, include_tables=False)

# preserve bold, italic, and other inline formatting
result = extract(downloaded, include_formatting=True)

# keep link targets (href values), converting relative to absolute when url is provided
result = extract(downloaded, output_format="xml", include_links=True, url=url)

# include image alt text, src, and title attributes
result = extract(downloaded, output_format="xml", include_images=True)

# fewer results, less noise
result = extract(downloaded, url=url, favor_precision=True)

# more content captured, possibly more noise
result = extract(downloaded, url=url, favor_recall=True)

# only return result if document language is German
result = extract(downloaded, url=url, target_language="de")

# skip fallback detection
result = extract(downloaded, fast=True)

# combine with other options for maximum speed
result = extract(downloaded, include_comments=False, include_tables=False, fast=True)

from trafilatura.settings import Extractor
from trafilatura import extract

# create a reusable configuration object
options = Extractor(output_format="json", with_metadata=True)

# adjust individual settings after creation
options.formatting = True   # same as include_formatting=True
options.source = "My Source"  # useful for debugging and provenance

# pass to extract()
result = extract(my_doc, options=options)

# include metadata fields in output
result = extract(downloaded, url=url, with_metadata=True)

# discard documents missing title, URL, or date
result = extract(downloaded, url=url, only_with_metadata=True)

---
title: Example Article Title
author: Jane Doe
url: https://www.example.org/article
date: 2024-02-01
---
The main article text begins here...

result = extract(
    downloaded,
    output_format="xml",
    date_extraction_params={
        "extensive_search": True,   # broader heuristics, higher recall
        "original_date": True,      # look for the original publication date
        "outputformat": "%Y-%m-%d", # custom date format string
        "max_date": "2024-01-01",   # discard dates after this cutoff (YYYY-MM-DD)
    }
)

from trafilatura.meta import reset_caches

# call at any point to clear all internal caches and run garbage collection
reset_caches()

from trafilatura import extract

html = "<html><body><article><p>Hello world.</p></article></body></html>"
result = extract(html)
# → 'Hello world.'

from lxml import html as lxml_html
from trafilatura import extract

my_doc = "<html><body><article><p>Here is the main text.</p></article></body></html>"
mytree = lxml_html.fromstring(my_doc)

result = extract(mytree)
# → 'Here is the main text.'

from bs4 import BeautifulSoup
from lxml.html.soupparser import convert_tree
from trafilatura import extract

soup = BeautifulSoup(
    "<html><body><article><p>Article content here.</p></article></body></html>",
    "lxml"
)
lxml_tree = convert_tree(soup)[0]
result = extract(lxml_tree)

from trafilatura import fetch_response, bare_extraction

response = fetch_response("https://www.example.org")

# response.url gives the final URL after any redirects
result = bare_extraction(response.data, url=response.url)

from trafilatura.readability_lxml import is_probably_readerable
from trafilatura import fetch_url

downloaded = fetch_url("https://www.example.org/article")

if is_probably_readerable(downloaded):
    result = extract(downloaded)