Unsupervised Knowledge Graph Construction with Prefect

from prefect import flow, task

@task
def scrape_article(url: str) -> str:
    # fetch page HTML, extract main text
    ...

@task
def link_to_wikidata(mentions: list[str]) -> dict:
    # call Wikidata search API for each mention
    ...

@task
def load_to_neo4j(uri_map: dict, source_url: str):
    # MERGE nodes and relationships into Neo4j
    ...

@flow
def build_kg(urls: list[str]):
    for url in urls:
        text = scrape_article(url)
        entities = link_to_wikidata(extract_mentions(text))
        load_to_neo4j(entities, url)

MERGE (e:Entity { uri: $wikidataUri })
SET e.label = $label, e.wikidataId = $qid
MERGE (a:Article { url: $sourceUrl })
MERGE (a)-[:MENTIONS]->(e)

// Total entity and article counts
MATCH (e:Entity) RETURN count(e) AS entityCount
MATCH (a:Article) RETURN count(a) AS articleCount

// Average mentions per article (coverage density)
MATCH (a:Article)-[:MENTIONS]->(e:Entity)
RETURN avg(size((a)-[:MENTIONS]->())) AS avgMentionsPerArticle

// Entities with the most cross-article coverage
MATCH (e:Entity)<-[:MENTIONS]-(a:Article)
RETURN e.label, count(a) AS articleCount
ORDER BY articleCount DESC LIMIT 10