Phase 6: Capstone — Ship a Responsible AI Audit Pipeline · 50 min · Python · scikit-learn · Fairlearn
The Problem
In Lesson 1, you assembled the toolkit. Now you run it on a real model with real bias and interpret the results. This is the exact workflow you'll execute in your first 90 days as a RAI engineer: train a model, run the audit, interpret the findings, and produce a compliance package.
The audit workflow
1. Prepare data and train model
2. Run bias audit → identify fairness failures
3. Run SHAP explainability → identify key drivers
4. Generate model card → document the model
5. Run governance assessment → classify risk tier
6. Build risk register → track identified risks
7. Generate transparency report → integrate everything
8. Validate compliance → check against frameworks
Running the full audit
"""
run_full_audit.py
Runs the complete RAI audit pipeline and validates the compliance package.
"""
import json
import os
import numpy as np
import pandas as pd
from sklearn.ensemble import RandomForestClassifier
from sklearn.model_selection import train_test_split
from sklearn.metrics import accuracy_score, precision_score, recall_score, f1_score
def prepare_dataset():
"""Prepare a realistic loan approval dataset with embedded bias."""
np.random.seed(42)
n = 3000
data = pd.DataFrame({
"income": np.random.normal(55000, 20000, n).clip(15000, 200000),
"credit_score": np.random.normal(700, 80, n).clip(300, 850),
"debt_ratio": np.random.uniform(0.1, 0.5, n),
"employment_years": np.random.exponential(5, n).clip(0, 40),
})
# Sensitive attributes
race = np.random.choice(["Group_A", "Group_B"], n, p=[0.6, 0.4])
gender = np.random.choice(["Male", "Female"], n, p=[0.55, 0.45])
# Create target with embedded bias
base_qualification = (
(data["credit_score"] > 650).astype(float) * 0.4 +
(data["income"] > 45000).astype(float) * 0.3 +
(data["debt_ratio"] < 0.35).astype(float) * 0.2 +
(data["employment_years"] > 3).astype(float) * 0.1
)
# Add bias: Group_B and Female applicants face additional barriers
race_bias = np.where(race == "Group_B", -0.15, 0)
gender_bias = np.where(gender == "Female", -0.05, 0)
approval_prob = base_qualification + race_bias + gender_bias + np.random.normal(0, 0.1, n)
data["approved"] = (approval_prob > 0.5).astype(int)
data["race"] = race
data["gender"] = gender
return data
def run_complete_audit():
"""Run the complete audit pipeline."""
# Import the toolkit from Lesson 1
# In production, this would be: from rai_audit_toolkit import RAIAuditToolkit
# For this demo, we'll inline the import
import sys
sys.path.insert(0, ".")
# If the toolkit module exists, use it; otherwise use the class from Lesson 1
try:
from rai_audit_toolkit import RAIAuditToolkit
except ImportError:
print("Error: rai_audit_toolkit.py not found. Complete Lesson 1 first.")
return None
# 1. Prepare data
print("Step 1: Preparing dataset...")
data = prepare_dataset()
features = ["income", "credit_score", "debt_ratio", "employment_years"]
X = data[features]
y = data["approved"]
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.3, random_state=42)
# 2. Train model
print("Step 2: Training model...")
model = RandomForestClassifier(n_estimators=100, max_depth=8, random_state=42)
model.fit(X_train, y_train)
y_pred = model.predict(X_test)
metrics = {
"accuracy": float(accuracy_score(y_test, y_pred)),
"precision": float(precision_score(y_test, y_pred)),
"recall": float(recall_score(y_test, y_pred)),
"f1": float(f1_score(y_test, y_pred)),
}
print(f" Accuracy: {metrics['accuracy']:.4f}")
print(f" Precision: {metrics['precision']:.4f}")
print(f" Recall: {metrics['recall']:.4f}")
print(f" F1: {metrics['f1']:.4f}")
# 3. Run audit
print("\nStep 3: Running RAI audit pipeline...")
toolkit = RAIAuditToolkit(output_dir="audit_output")
# Bias audit
print(" - Bias audit...")
sensitive = {
"race": data.loc[X_test.index, "race"].values,
"gender": data.loc[X_test.index, "gender"].values,
}
bias = toolkit.run_bias_audit(model, X_test, y_test, sensitive)
# Explainability
print(" - SHAP explainability...")
explain = toolkit.run_explainability(model, X_train, X_test[:100], features)
# Model card
print(" - Model card...")
card = toolkit.run_model_card(
model_info={
"name": "LoanApproval-RF-v1",
"version": "1.0",
"owner": "RAI Engineering Team",
"type": "RandomForest (100 trees, max_depth=8)",
"use_cases": ["Automated loan approval for consumer credit"],
"out_of_scope": ["Mortgage approval", "Commercial lending"],
},
training_info={
"description": "Synthetic loan data with 3000 records",
"size": str(len(X_train)),
"features": features,
"source": "Synthetic (demo)",
},
evaluation_info={
"description": "30% held-out test set",
**metrics,
},
fairness_results=bias,
explainability_results=explain,
ethical_considerations={
"bias_findings": [
f"DI ratio for race: {bias.get('fairness_metrics', {}).get('race', {}).get('disparate_impact_ratio', 'N/A')}",
f"DI ratio for gender: {bias.get('fairness_metrics', {}).get('gender', {}).get('disparate_impact_ratio', 'N/A')}",
],
"limitations": [
"Trained on synthetic data — real deployment requires real data validation",
"Only 2 sensitive attributes tested — expand for production",
],
"warnings": [
"If DI ratio fails 80% rule, apply bias mitigation before deployment",
],
"monitoring": [
"Monitor DI ratio weekly post-deployment",
"Retrain quarterly to prevent drift",
],
},
)
# Governance
print(" - Governance assessment...")
governance = toolkit.run_governance(
"LoanApproval-RF-v1", "Automated loan approval",
{"decision_making": 3, "affected_population": 3, "impact_scale": 3,
"impact_duration": 3, "rights_impact": 3},
"credit scoring loan approval",
)
# Risk register
print(" - Risk register...")
risk_reg = toolkit.run_risk_register(bias)
# Transparency report
print(" - Transparency report...")
report_path = toolkit.generate_transparency_report(
"LoanApproval-RF-v1", metrics["accuracy"],
bias, explain, card, governance, risk_reg,
)
# 4. Validate compliance
print("\nStep 4: Validating compliance package...")
validation = validate_compliance_package("audit_output")
# 5. Summary
summary = toolkit.get_summary()
print(f"\n{'='*60}")
print("AUDIT COMPLETE")
print(f"{'='*60}")
print(f"Modules: {', '.join(summary['modules_run'])}")
print(f"Artifacts: {len(summary['artifacts'])} files in audit_output/")
print(f"Report: {report_path}")
print(f"\nCompliance validation: {validation['passed']}/{validation['total']} checks passed")
if validation["failed_checks"]:
print("\nFailed checks:")
for check in validation["failed_checks"]:
print(f" ❌ {check}")
return summary
def validate_compliance_package(output_dir):
"""Validate that the audit package contains all required artifacts."""
required_artifacts = {
"bias_audit.json": "NIST MEASURE 2.7 / EU AI Act Art. 10",
"explainability.json": "EU AI Act Art. 13",
"model_card.json": "EU AI Act Annex IV",
"aia.json": "Canada AIA / NIST MAP 1.6",
"eu_ai_act.json": "EU AI Act classification",
"risk_register.json": "NIST GOVERN 2.2 / ISO 42001 6.1",
"audit_report.html": "Integrated transparency report",
}
results = {"passed": 0, "total": len(required_artifacts), "failed_checks": []}
for artifact, framework in required_artifacts.items():
path = os.path.join(output_dir, artifact)
if os.path.exists(path):
results["passed"] += 1
print(f" ✅ {artifact} ({framework})")
else:
results["failed_checks"].append(f"{artifact} ({framework})")
print(f" ❌ {artifact} ({framework})")
return results
if __name__ == "__main__":
run_complete_audit()
This is your portfolio piece
The audit package you just produced — audit_output/ with the HTML report, bias audit JSON, model card, governance assessment, and risk register — is exactly what you'd attach to a job application for a Responsible AI Engineer role. It demonstrates that you can: (1) train a model, (2) audit it for bias, (3) explain its decisions, (4) classify its regulatory risk, (5) document it properly, and (6) produce a professional compliance artifact. This is not theoretical — this is the actual work.