Blueprints for KG Construction from Unstructured Data

from pypdf import PdfReader
from openai import OpenAI
from utils import getNLOntology
from rdflib import Graph
import os
from neo4jconnector import Neo4jConnection

# STEP 1: Extract text from PDF
filename = "data/SimplicityEsportsGamingCompany.pdf"
reader = PdfReader(filename)
text = ""
for page in reader.pages:
    text += page.extract_text()

# STEP 2: Parse the OWL ontology
g = Graph()
g.parse("ontos/contract.ttl")

# OPTION 1: Ontology in standard Turtle serialisation (used by default)
ontology = g.serialize(format="ttl")

# OPTION 2: Natural language description of the ontology
# ontology = getNLOntology(g)

# STEP 3: Prompt GPT-4o for Cypher extraction
client = OpenAI(api_key=os.environ.get("MY_OPENAI_KEY"))

system = (
    "You are an expert in extracting structured information out of natural language text. "
    "You extract entities with their attributes and relationships between entities. "
    "You can produce the output as RDF triples or as Cypher write statements on request."
)

prompt = (
    "Given the ontology below run your best entity extraction over the content.\n"
    " The extracted entities and relationships must be described using exclusively the terms in the ontology "
    "and in the way they are defined. This means that for attributes and relationships you will respect "
    "the domain and range constraints.\n"
    " You will never use terms not defined in the ontology.\n"
    "Return the output as Cypher using merge to allow for linkage of nodes from multiple passes.\n"
    "Absolutely no comments on the output. Just the structured output. "
    + '\n\nONTOLOGY: \n ' + ontology + '\n\nCONTENT: \n ' + text
)

chat_completion = client.chat.completions.create(
    messages=[
        {'role': 'system', 'content': system},
        {'role': 'user', 'content': prompt},
    ],
    model="gpt-4o",
)

cypher_script = chat_completion.choices[0].message.content[3:-3]

# STEP 4: Write to Neo4j (named database)
uri = "bolt://localhost:7687"
user = "neo4j"
password = "neoneoneo"
dbname = "gm3"
conn = Neo4jConnection(uri, user, password, dbname)
result = conn.run_cypher(cypher_script, {})
conn.close()

WITH $jsondata AS value

// Create the Agreement node
MERGE (agreement:Agreement {
    agreement_type: value.agreement.agreement_type,
    contract_id: value.agreement.contract_id,
    effective_date: value.agreement.effective_date,
    expiration_date: value.agreement.expiration_date,
    renewal_term: value.agreement.renewal_term,
    name: value.agreement.name
})

// Create the Country and Governed By Law relationship
MERGE (country:Country {name: value.governed_by_law.country.name})
MERGE (agreement)-[:GOVERNED_BY_LAW]->(country)

// Create Party nodes and relationships
WITH value, agreement
UNWIND value.parties AS party
MERGE (org:Organization {name: party.name, role: party.role})
MERGE (country:Country {name: party.incorporated_in.country.name})
MERGE (org)-[:INCORPORATED_IN]->(country)
MERGE (org)-[:IS_PARTY_TO]->(agreement)

// Create Clause nodes and relationships
WITH value, agreement
UNWIND value.clauses AS clause
MERGE (contractClause:ContractClause {name: clause.name})
MERGE (clauseType:ClauseType {name: clause.clause_type})
MERGE (contractClause)-[:HAS_TYPE]->(clauseType)
MERGE (agreement)-[:HAS_CLAUSE]->(contractClause)

// Create Excerpt nodes and relationships
WITH clause, contractClause
UNWIND clause.excerpts AS excerpt
MERGE (excerptNode:Excerpt {text: excerpt.text})
MERGE (contractClause)-[:HAS_EXCERPT]->(excerptNode);