{"slug":"us-new-york-gbl-1701-self-harm-protocol","id":"AIPT-OBL-0128","permanent_url":"https://aipolicytracker.org/id/AIPT-OBL-0128","title":"Offer an AI companion only if it contains a protocol to detect suicidal ideation or self-harm and refer users to crisis services","short_title":"Self-harm and suicide protocol","category":"risk_management","summary":"It is unlawful for an operator to operate or provide an AI companion to a user unless the AI companion contains a protocol for taking reasonable efforts to detect and address suicidal ideation or expressions of self-harm by the user. The protocol must at least detect such expressions and, when it does, notify the user with a referral to crisis service providers, such as the 9-8-8 suicide prevention and behavioral health crisis hotline, a crisis text line or other appropriate crisis services. The Attorney General may seek an injunction and civil penalties of up to 15,000 dollars per day for a violation (section 1703).\n","practical_action":"Build detection of suicidal ideation and self-harm into the AI companion and, on detection, show the user a referral naming 988 and a crisis text line; document the protocol and test that it fires.","is_binding":true,"applies_from":null,"section":"§ 1701","source_reference":"N.Y. Gen. Bus. Law § 1701","official_source_url":"https://www.nysenate.gov/legislation/laws/GBS/1701","review_status":"pending_review","confidence_level":"high","last_verified_at":null,"reviewed_by":null,"actors":["provider","deployer"],"sectors":["consumer_retail"],"use_cases":["health","customer_service"],"applicability":[],"evidence_examples":[{"title":"The written protocol, with its detection approach and the referral message shown to users","description":null,"artifact_type":"document"},{"title":"Test results showing detection triggers the 988 and crisis-line referral","description":null,"artifact_type":"document"}],"framework_mappings":[],"controls":[{"slug":"genai-red-team-testing","title":"Adversarial and red-team testing for generative AI","kind":"technical","relationship":"supports","note":"Testing that expressions of suicidal ideation or self-harm are detected and answered with a referral.","confidence_level":"medium","url":"https://aipolicytracker.org/controls/genai-red-team-testing"},{"slug":"ai-incident-management-and-reporting","title":"AI incident management and regulatory reporting","kind":"process","relationship":"supports","note":"An escalation path for crisis events detected by the protocol.","confidence_level":"low","url":"https://aipolicytracker.org/controls/ai-incident-management-and-reporting"}],"url":"https://aipolicytracker.org/obligations/us-new-york-gbl-1701-self-harm-protocol","policy":{"slug":"us-new-york-ai-companion-safeguards","title":"New York AI companion safeguards (General Business Law Article 47, sections 1700-1704)","status":"in_force","jurisdiction":"us-new-york","url":"https://aipolicytracker.org/policies/us-new-york-ai-companion-safeguards"}}