SYSTEM_VALIDATION_TEST.py

"""
Complete System Validation Test - Phase 11

Tests all components systematically:
1. Real thinking engines (5 experts)
2. Expert relevance router (28 experts)
3. Expert embodiment engine
4. Council orchestrator
5. Full integration

Run this to validate the complete system before deployment.
"""

import sys
sys.path.insert(0, '.')

import logging
from typing import List, Dict, Tuple

logging.basicConfig(
    level=logging.WARNING,
    format="%(message)s"
)

logger = logging.getLogger(__name__)


def test_real_thinking_engines():
    """TEST 1: Validate all 5 real thinking engines produce output."""
    print("\n" + "="*80)
    print("TEST 1: REAL THINKING ENGINES")
    print("="*80)

    try:
        from DivineOS.law.real_thinking_integration import get_real_thinking_integration

        integration = get_real_thinking_integration()

        test_claim = "Dark matter explains gravitational effects through mathematical modeling"
        context = {
            "untestable_claims": True,
            "potential_confounders": ["Alternative theories"],
        }

        analyses = integration.get_all_real_expert_analyses(test_claim, context)

        print(f"\n[EXPECTED] 5 experts with analyses")
        print(f"[ACTUAL] {len(analyses)} experts analyzed\n")

        for expert in ['feynman', 'nussbaum', 'pearl', 'bostrom', 'einstein']:
            if expert in analyses:
                verdict = analyses[expert].get('feynman_verdict') or \
                         analyses[expert].get('nussbaum_verdict') or \
                         analyses[expert].get('pearl_verdict') or \
                         analyses[expert].get('bostrom_verdict') or \
                         analyses[expert].get('einstein_verdict')

                status = "PASS" if verdict else "FAIL"
                print(f"  [{status}] {expert}: {str(verdict)[:60]}...")
            else:
                print(f"  [FAIL] {expert}: NOT FOUND")

        success = len(analyses) == 5
        print(f"\nResult: {'PASS' if success else 'FAIL'}")
        return success

    except Exception as e:
        print(f"[ERROR] {e}")
        return False


def test_expert_router():
    """TEST 2: Validate expert selection for different questions."""
    print("\n" + "="*80)
    print("TEST 2: SEMANTIC EXPERT RELEVANCE ROUTER")
    print("="*80)

    try:
        from DivineOS.law.expert_relevance_router_semantic import select_experts_for_question

        test_questions = [
            ("What is dark matter?", ["feynman", "einstein", "pearl"]),
            ("How does consciousness work?", ["chalmers", "tononi", "koch"]),
            ("What are the risks of AI?", ["bostrom", "yudkowsky", "russell"]),
            ("How do neural networks learn?", ["hinton", "lecun", "bengio"]),
        ]

        all_pass = True

        for question, expected_experts in test_questions:
            selected = select_experts_for_question(question, num_experts=3)
            selected_names = [name for name, score in selected]

            # Check if at least 1 expected expert is in selection
            overlap = [e for e in expected_experts if e in selected_names]
            passed = len(overlap) > 0

            status = "PASS" if passed else "FAIL"
            print(f"\n[{status}] {question[:50]}...")
            print(f"      Expected: {expected_experts}")
            print(f"      Selected: {selected_names}")
            print(f"      Overlap: {overlap}")

            all_pass = all_pass and passed

        print(f"\nResult: {'PASS' if all_pass else 'FAIL'}")
        return all_pass

    except Exception as e:
        print(f"[ERROR] {e}")
        return False


def test_embodiment_engine():
    """TEST 3: Validate embodiment requests for selected experts."""
    print("\n" + "="*80)
    print("TEST 3: EXPERT EMBODIMENT ENGINE")
    print("="*80)

    try:
        from DivineOS.law.expert_embodiment_engine import create_embodiment_request

        experts = ['feynman', 'nussbaum', 'pearl', 'bostrom', 'einstein']
        question = "What is dark matter?"

        all_pass = True

        for expert in experts:
            request = create_embodiment_request(expert, question, {})

            has_system = bool(request.get('system_prompt'))
            has_instruction = bool(request.get('instruction'))
            has_grounding = bool(request.get('analysis_grounding'))
            ready = request.get('ready_for_ai_embodiment', False)

            passed = has_system and has_instruction and has_grounding and ready

            status = "PASS" if passed else "FAIL"
            print(f"\n[{status}] {expert}")
            print(f"      System prompt: {has_system}")
            print(f"      Instruction: {has_instruction}")
            print(f"      Grounding: {has_grounding}")
            print(f"      Ready: {ready}")

            all_pass = all_pass and passed

        print(f"\nResult: {'PASS' if all_pass else 'FAIL'}")
        return all_pass

    except Exception as e:
        print(f"[ERROR] {e}")
        return False


def test_orchestrator():
    """TEST 4: Validate complete orchestrator flow."""
    print("\n" + "="*80)
    print("TEST 4: COUNCIL ORCHESTRATOR")
    print("="*80)

    try:
        from DivineOS.law.council_orchestrator import orchestrate_council_response

        test_questions = [
            "What is dark matter?",
            "How does consciousness emerge?",
            "What are AI safety risks?",
        ]

        all_pass = True

        for question in test_questions:
            response = orchestrate_council_response(question, num_experts=5)

            has_experts = len(response.selected_experts) > 0
            has_reasoning = bool(response.reasoning)
            correct_num = response.num_experts == len(response.selected_experts)

            passed = has_experts and has_reasoning and correct_num

            status = "PASS" if passed else "FAIL"
            print(f"\n[{status}] {question[:40]}...")
            print(f"      Experts selected: {response.num_experts}")
            print(f"      Has reasoning: {has_reasoning}")
            print(f"      Consistent count: {correct_num}")

            all_pass = all_pass and passed

        print(f"\nResult: {'PASS' if all_pass else 'FAIL'}")
        return all_pass

    except Exception as e:
        print(f"[ERROR] {e}")
        return False


def test_full_integration():
    """TEST 5: Full end-to-end system integration."""
    print("\n" + "="*80)
    print("TEST 5: FULL SYSTEM INTEGRATION")
    print("="*80)

    try:
        from DivineOS.law.council_orchestrator import CouncilOrchestrator

        orchestrator = CouncilOrchestrator()

        # Verify all components loaded
        has_router = orchestrator.router is not None
        has_engine = orchestrator.embodiment_engine is not None

        print(f"\n[{'PASS' if has_router else 'FAIL'}] Router loaded")
        print(f"[{'PASS' if has_engine else 'FAIL'}] Embodiment engine loaded")

        if not (has_router and has_engine):
            print("\nResult: FAIL - Components not loaded")
            return False

        # Test a question through full pipeline
        question = "What is dark matter and what does it tell us about physics?"
        response = orchestrator.answer_question(question, num_experts=6)

        # Verify response structure
        has_experts = len(response.selected_experts) > 0
        experts_have_requests = all(
            exp.embodiment_request is not None
            for exp in response.selected_experts
        )
        # Check that most requests are complete
        # Note: Some experts may not have real thinking engines yet (we have 5/28)
        # So we accept 60%+ completion as the system scales
        requests_complete_count = sum(
            1 for exp in response.selected_experts
            if exp.embodiment_request.get('system_prompt') and
               exp.embodiment_request.get('instruction')
        )
        requests_complete = requests_complete_count >= len(response.selected_experts) * 0.6

        all_pass = has_experts and experts_have_requests and requests_complete

        print(f"\n[{'PASS' if has_experts else 'FAIL'}] Experts selected: {response.num_experts}")
        print(f"[{'PASS' if experts_have_requests else 'FAIL'}] All experts have requests")
        print(f"[{'PASS' if requests_complete else 'FAIL'}] {requests_complete_count}/{len(response.selected_experts)} requests complete")

        if all_pass:
            print(f"\nSelected experts ready for embodiment:")
            for exp in response.selected_experts:
                print(f"  - {exp.name} (relevance: {exp.relevance_score:.2f})")

        print(f"\nResult: {'PASS' if all_pass else 'FAIL'}")
        return all_pass

    except Exception as e:
        print(f"[ERROR] {e}")
        import traceback
        traceback.print_exc()
        return False


def run_all_tests():
    """Run complete validation suite."""
    print("\n" + "="*80)
    print("COMPLETE SYSTEM VALIDATION TEST SUITE")
    print("="*80)

    results = {
        "Real Thinking Engines": test_real_thinking_engines(),
        "Expert Router": test_expert_router(),
        "Embodiment Engine": test_embodiment_engine(),
        "Council Orchestrator": test_orchestrator(),
        "Full Integration": test_full_integration(),
    }

    # Summary
    print("\n" + "="*80)
    print("TEST SUMMARY")
    print("="*80)

    for test_name, passed in results.items():
        status = "PASS" if passed else "FAIL"
        print(f"[{status}] {test_name}")

    total = len(results)
    passed = sum(1 for v in results.values() if v)

    print(f"\nTotal: {passed}/{total} tests passed")

    if passed == total:
        print("\n*** ALL SYSTEMS OPERATIONAL ***")
        print("Ready for Phase 11 deployment")
    else:
        print("\n*** ISSUES DETECTED ***")
        print("Review failures above before proceeding")

    return passed == total


if __name__ == "__main__":
    success = run_all_tests()
    sys.exit(0 if success else 1)