ronaldbvirinyangwe/african-clinical-nlp
Domain:
healthcarenatural language processingRecord type:
software
Creator:
ron
Host:
"""
Integration tests: full pipeline on synthetic clinical notes.
Tests de-identification + code mapping working together end-to-end.
"""
import sys
from pathlib import Path
import pytest
sys.path.insert(0, str(Path(__file__).parents[2] / "data"))
from synthetic.clinical_notes import SYNTHETIC_NOTES, get_note, SyntheticNote
from african_clinical_nlp.pipeline.deidentification import DeIdentifier, PII_NER_LABELS
from african_clinical_nlp.pipeline.coding import ClinicalCodeMapper, map_codes
from african_clinical_nlp.pipeline.document import Document, Entity
# ---------------------------------------------------------------------------
# Helpers
# ---------------------------------------------------------------------------
def build_doc_from_note(note: SyntheticNote) -> Document:
"""Build a Document from a SyntheticNote for pipeline testing."""
doc = Document()
doc.raw_text = note.text
doc.cleaned_text = note.text
doc.detected_language = note.language
# Inject synthetic ground-truth entities
entities = []
offset = 0
for label, texts in note.ground_truth.items():
if label.startswith("icd10") or not isinstance(texts, list):
continue
for text in texts:
pos = note.text.lower().find(text.lower(), offset)
if pos == -1:
pos = 0
entities.append(Entity(
label=label,
text=text,
normalized=text.lower().strip(),
start=pos,
end=pos + len(text),
confidence=0.9,
language=note.language,
))
doc.entities = entities
return doc
# ---------------------------------------------------------------------------
# De-identification integration
# ---------------------------------------------------------------------------
class TestDeidentificationIntegration:
@pytest.mark.parametrize("note", SYNTHETIC_NOTES)
def test_pii_removed_from_all_notes(self, note: SyntheticNote):
"""PII entities in ground truth should not appear in deidentified text."""
doc = build_doc_from_note(note)
result = DeIdentifier(strategy="mask").process(doc)
pii_labels = {"PATIENT_NAME", "DATE_OF_BIRTH", "FACILITY_ID", " …