{"slug":"genai-red-team-testing","id":"AIPT-CTL-0015","title":"Adversarial and red-team testing for generative AI","kind":"technical","kind_label":"Technical measure","purpose":"Finds the ways a generative or general-purpose model can be made to produce harmful, deceptive or dangerous output before users or attackers do, and feeds the findings back into mitigations.\n","description":"A red team with a mix of security, domain and safety expertise, drawn partly from outside the development group, attacks the model and the product around it using scenario libraries that cover prompt injection, jailbreaks, data leakage, harmful-capability elicitation, bias provocation and misuse of tools or agents. Exercises are scoped and rules of engagement agreed in advance; findings are rated, tracked to closure and regression-tested in the next round. For the largest models the programme includes structured evaluations of dangerous capabilities and the organisation's mitigations are recorded in the safety framework. Results are summarised for governance and, where required, for authorities or downstream integrators.\n","owner_role":"AI security or safety lead","frequency":"on_material_change","frequency_label":"At launch and on material change","risk_subdomains":["2.2","4.2","4.3","1.2","7.2"],"evidence":[{"type":"evaluation_report","title":"Red-team exercise report","description":"Scope, methods, findings by severity and remediation status for one exercise."},{"type":"procedure","title":"Red-team rules of engagement and scenario library","description":null},{"type":"risk_register","title":"Adversarial findings tracker","description":null}],"framework_references":[{"framework":"nist_ai_rmf","framework_name":"NIST AI RMF 1.0","reference":"MEASURE 2.6, 2.7; MANAGE 2.2","note":"Testing safety, security and resilience, including structured adversarial exercises for generative AI.","confidence_level":"high"},{"framework":"iso_42001","framework_name":"ISO/IEC 42001:2023","reference":"Annex A.6.2.4","note":null,"confidence_level":"medium"},{"framework":"owasp_llm_top10","framework_name":"OWASP Top 10 for LLM Applications","reference":"LLM01 Prompt Injection, LLM02 Sensitive Information Disclosure, LLM05 Improper Output Handling, LLM06 Excessive Agency, LLM07 System Prompt Leakage","note":null,"confidence_level":"high"},{"framework":"mitre_atlas","framework_name":"MITRE ATLAS","reference":"AML.M0020 Generative AI Guardrails, AML.M0022 Generative AI Model Alignment","note":null,"confidence_level":"medium"}],"obligations":[{"slug":"eu-ai-act-gpai-systemic-risk","title":"Manage systemic risk for high-impact general-purpose models","relationship":"satisfies","note":"Model evaluation including adversarial testing.","confidence_level":"medium","is_binding":true,"policy":"eu-ai-act","jurisdiction":"eu","url":"https://aipolicytracker.org/obligations/eu-ai-act-gpai-systemic-risk"},{"slug":"us-california-sb-53-frontier-ai-framework","title":"Large frontier developers must publish a frontier AI framework","relationship":"supports","note":"Catastrophic-risk assessment evidence.","confidence_level":"medium","is_binding":true,"policy":"us-california-sb-53","jurisdiction":"us-california","url":"https://aipolicytracker.org/obligations/us-california-sb-53-frontier-ai-framework"},{"slug":"us-california-sb-53-catastrophic-risk-assessment-summaries","title":"Large frontier developers must send periodic summaries of catastrophic-risk assessments to the state","relationship":"supports","note":"Dangerous-capability evaluations feed the assessment.","confidence_level":"medium","is_binding":true,"policy":"us-california-sb-53","jurisdiction":"us-california","url":"https://aipolicytracker.org/obligations/us-california-sb-53-catastrophic-risk-assessment-summaries"},{"slug":"eu-ai-act-accuracy-robustness-cybersecurity","title":"Achieve appropriate accuracy, robustness and cybersecurity","relationship":"supports","note":"Adversarial exercises cover poisoning, evasion and extraction attempts.","confidence_level":"medium","is_binding":true,"policy":"eu-ai-act","jurisdiction":"eu","url":"https://aipolicytracker.org/obligations/eu-ai-act-accuracy-robustness-cybersecurity"},{"slug":"south-korea-ai-basic-act-art-32-safety-measures-for-high-performance-ai","title":"Operators of AI above the compute threshold must run lifecycle risk management and report safety results","relationship":"supports","note":"Evidence for the risk identification step.","confidence_level":"medium","is_binding":true,"policy":"south-korea-framework-act-on-the-development-of-artificial-intelligence-and-establishment-of-a-foundation-for","jurisdiction":"south-korea","url":"https://aipolicytracker.org/obligations/south-korea-ai-basic-act-art-32-safety-measures-for-high-performance-ai"},{"slug":"us-texas-responsible-ai-governance-act-traiga-manipulation-prohibition","title":"Developers and deployers must not use AI to incite self-harm, harm to others or crime","relationship":"supports","note":"Tests for harmful-incitement behaviour in generative systems.","confidence_level":"medium","is_binding":true,"policy":"us-texas-responsible-ai-governance-act-traiga","jurisdiction":"us-texas","url":"https://aipolicytracker.org/obligations/us-texas-responsible-ai-governance-act-traiga-manipulation-prohibition"},{"slug":"us-nist-ai-rmf-measure","title":"Measure and test trustworthiness characteristics (Measure)","relationship":"supports","note":"Red-teaming and independent review for generative AI.","confidence_level":"high","is_binding":false,"policy":"us-nist-ai-rmf","jurisdiction":"us","url":"https://aipolicytracker.org/obligations/us-nist-ai-rmf-measure"},{"slug":"us-texas-responsible-ai-governance-act-traiga-sexual-content-and-csam-prohibition","title":"Developers and distributors must not build AI intended to produce child sexual abuse material or unlawful sexual deepfakes","relationship":"supports","note":"Tests that safeguards block sexual content involving minors.","confidence_level":"high","is_binding":true,"policy":"us-texas-responsible-ai-governance-act-traiga","jurisdiction":"us-texas","url":"https://aipolicytracker.org/obligations/us-texas-responsible-ai-governance-act-traiga-sexual-content-and-csam-prohibition"}],"related_controls":["accuracy-robustness-security-testing","frontier-model-safety-framework","ai-incident-management-and-reporting"],"review_status":"verified","confidence_level":"high","last_verified_at":"2026-09-26","reviewed_by":"Bhaskar Bhatt","updated_at":"2026-10-15T06:46:22+00:00","url":"https://aipolicytracker.org/controls/genai-red-team-testing","context_url":"https://aipolicytracker.org/controls/genai-red-team-testing.md"}