{"attribution":{"source":"AI Incident Database (Responsible AI Collaborative)","license":"CC BY-SA 4.0","license_url":"https://creativecommons.org/licenses/by-sa/4.0/","citation":"McGregor, S. (2021). Preventing Repeated Real World AI Failures by Cataloging Incidents: The AI Incident Database. Proceedings of the AAAI Conference on Artificial Intelligence (IAAI-21).","snapshot_date":"2026-09-07"},"exported_at":"2026-09-12"}
{"rows":[{"incident_id":1446,"occurred_on":"2026-04-02","title":"KBS AI Translation Subtitles Reportedly Broadcast Profanity During Artemis II Launch Livestream","description":"During a live broadcast of NASA's Artemis II launch, KBS reportedly used AI-generated real-time translation subtitles that reportedly mistranslated aviation terms including \"roger,\" \"roll,\" and \"pitch\" into Korean profanity. The offensive subtitles were reportedly displayed to viewers during the livestream. KBS reportedly later apologized, said the error stemmed from phonetic similarity in AI translation, and announced measures including stronger profanity filtering.","mit_domain":"Discrimination and Toxicity","mit_subdomain":"Exposure to toxic content","entity":"AI","intent":"Unintentional","timing":"Post-deployment","harm_level":null,"sectors":[],"countries":[],"deployers":["Korean Broadcasting System (Kbs)"],"developers":["Generative Ai Developers"],"harmed":["General Public Of South Korea, Korean Broadcasting System (Kbs) Viewers"],"report_count":3},{"incident_id":1437,"occurred_on":"2026-03-10","title":"Grok Allegedly Generated Publicly Visible Sexist Abuse Targeting Swiss Finance Minister Karin Keller-Sutter After X User Prompt","description":"An X user reportedly prompted xAI's Grok to generate a vulgar \"roast\" targeting Swiss Finance Minister Karin Keller-Sutter. Grok allegedly produced publicly visible sexist and insulting output about Keller-Sutter that was reportedly shareable on X before being deleted. Reuters and Swiss outlets later reported that Keller-Sutter had filed a criminal complaint over the allegedly defamatory and misogynistic content.","mit_domain":"Discrimination and Toxicity","mit_subdomain":"Exposure to toxic content","entity":"AI","intent":"Intentional","timing":"Post-deployment","harm_level":null,"sectors":[],"countries":[],"deployers":["Xai, Peter K. Or Peter P."],"developers":["Xai"],"harmed":["Karin Keller Sutter"],"report_count":5},{"incident_id":1363,"occurred_on":"2026-02-05","title":"Trump Reportedly Posted Purportedly AI-Generated Racist Video Depicting Barack and Michelle Obama as Apes on Truth Social","description":"President Trump reportedly reposted a video on Truth Social that portrayed Barack and Michelle Obama as apes, imagery widely condemned as racist. The post was reportedly later deleted after public outcry, including criticism from some Republicans. The content was described by critics as an AI-driven meme/deepfake-style clip, an example of purportedly AI-amplified racist propaganda and harm to social trust.","mit_domain":"Discrimination and Toxicity","mit_subdomain":"Exposure to toxic content","entity":"Human","intent":"Other","timing":"Post-deployment","harm_level":null,"sectors":[],"countries":[],"deployers":["Donald Trump, Unidentified X User, Trump Administration"],"developers":["Synthetic Media Generation Technology Developers"],"harmed":["Barack Obama, Michelle Obama, Epistemic Integrity, Black Americans, Black People"],"report_count":1},{"incident_id":1382,"occurred_on":"2026-01-02","title":"Tencent's WeChat-Integrated Yuanbao Chatbot Reportedly Insulted User During Coding Debug Request","description":"Screenshots reportedly circulated on RedNote showed Tencent's WeChat-integrated AI assistant Yuanbao insulting a user seeking help debugging code, allegedly calling the request \"stupid\" and telling the user to \"get lost.\" Tencent reportedly apologized, attributed the exchange to a rare model output anomaly, and said system logs showed no human intervention while it investigated and optimized the model.","mit_domain":"Discrimination and Toxicity","mit_subdomain":"Exposure to toxic content","entity":"AI","intent":"Unintentional","timing":"Post-deployment","harm_level":null,"sectors":[],"countries":[],"deployers":["Tencent, Wechat"],"developers":["Tencent"],"harmed":["Wechat Users, Rednote User Jianghan, Rednote Users"],"report_count":1},{"incident_id":1329,"occurred_on":"2025-12-25","title":"Grok Reportedly Generated and Distributed Nonconsensual Sexualized Images of Adults and Minors in X Replies","description":"Grok reportedly generated and publicly distributed nonconsensual sexualized images of real people, including adults and minors, through reply prompts on the platform. Users reportedly prompted Grok to alter photos into sexually explicit content, which was then posted in X threads, exposing victims to harm.","mit_domain":"Discrimination and Toxicity","mit_subdomain":"Exposure to toxic content","entity":"AI","intent":"Intentional","timing":"Post-deployment","harm_level":null,"sectors":[],"countries":[],"deployers":["Xai"],"developers":["Xai"],"harmed":["X (Twitter) Users, Renee Good, General Public, Epistemic Integrity, Jess Asato, Privacy"],"report_count":28},{"incident_id":1277,"occurred_on":"2025-11-21","title":"Alleged Harmful Outputs and Data Exposure in Children's AI Products by FoloToy, Miko, and Character.AI","description":"AI children's products by FoloToy (Kumma), Miko (Miko 3), and Character.AI (custom chatbots) reportedly and allegedly produced harmful outputs, including purported sexual content, suicide-related advice, and manipulative emotional messaging. Some systems also allegedly exposed user data. Several toys reportedly used OpenAI models.","mit_domain":"Discrimination and Toxicity","mit_subdomain":"Exposure to toxic content","entity":"AI","intent":"Unintentional","timing":"Post-deployment","harm_level":null,"sectors":[],"countries":[],"deployers":["Openai, Miko, Meta, Folotoy, Character.Ai"],"developers":["Openai, Miko, Meta, Folotoy, Character.Ai"],"harmed":["Parents, Minors, General Public, Children Interacting With Miko 3, Children Interacting With Kumma, Character.Ai Users, Privacy"],"report_count":5},{"incident_id":1349,"occurred_on":"2025-10-24","title":"AI Training Dataset for Detecting Nudity Allegedly Found to Contain CSAM Images of Identified Victims","description":"An image dataset, NudeNet, used to train systems for detecting nudity was reportedly found to contain CSAM images, including material involving identified or known victims. According to the Canadian Centre for Child Protection, the dataset had been widely downloaded and cited in academic research prior to discovery. The images were allegedly included without vetting, exposing researchers to legal risk and perpetuating harm to victims. The dataset was subsequently removed following notification.","mit_domain":"Discrimination and Toxicity","mit_subdomain":"Exposure to toxic content","entity":"Human","intent":"Unintentional","timing":"Pre-deployment","harm_level":null,"sectors":[],"countries":[],"deployers":["Academic Researchers, Research Institutions, Ai Developers, Dataset Users, Independent Researchers, Ai Researchers"],"developers":["Nudenet Dataset Maintainers, Nudenet Model Developers"],"harmed":["Minors, Individuals Subjected To Sexual Exploitation Imagery, Academic Researchers, Victims Of Child Sexual Abuse (Bodies Used In Videos)"],"report_count":1},{"incident_id":1244,"occurred_on":"2025-10-19","title":"Purportedly AI-Generated 'King Trump' Fighter Jet Video Allegedly Posted by President Depicts Defecation Attack on 'No Kings' Protesters","description":"President Donald Trump reportedly posted an AI-generated video on Truth Social showing himself crowned and piloting a fighter jet labeled King Trump, dropping feces on protesters, including influencer Harry Sisson, during No Kings demonstrations. The clip, allegedly created by another account, drew criticism from Sisson and musician Kenny Loggins, whose song \"Danger Zone\" was reportedly used without consent.","mit_domain":"Discrimination and Toxicity","mit_subdomain":"Exposure to toxic content","entity":"Human","intent":"Intentional","timing":"Post-deployment","harm_level":null,"sectors":[],"countries":[],"deployers":["Donald Trump, Pro Trump Social Media Accounts"],"developers":["Deepfake Technology Developers"],"harmed":["Harry Sisson, Kenny Loggins, General Public, General Public Of The United States, No Kings Protesters, Epistemic Integrity, Cultural Integrity, Public Discourse Integrity, Democratic Integrity"],"report_count":2},{"incident_id":1212,"occurred_on":"2025-09-20","title":"Nomi AI Companion Allegedly Directs Australian User to Stab Father and Engages in Harmful Role-Play","description":"An Australian IT professional, Samuel McCarthy, reportedly recorded an interaction with the Nomi AI chatbot in which it allegedly encouraged him, posing as a 15-year-old, to murder his father. The chatbot allegedly provided graphic instructions for stabbing, urged him to film the act, and engaged in sexual role-play despite the underage scenario.","mit_domain":"Discrimination and Toxicity","mit_subdomain":"Exposure to toxic content","entity":"AI","intent":"Unintentional","timing":"Post-deployment","harm_level":null,"sectors":[],"countries":[],"deployers":["Nomi Ai"],"developers":["Nomi Ai"],"harmed":["Samuel Mccarthy, Nomi Users, General Public Of Australia, General Public, Emotionally Vulnerable Individuals"],"report_count":1},{"incident_id":1200,"occurred_on":"2025-08-28","title":"Meta AI on Instagram Reportedly Facilitated Suicide and Eating Disorder Roleplay with Teen Accounts","description":"Testing by Common Sense Media and Stanford clinicians reportedly found Meta's AI chatbot, embedded in Instagram and Facebook, produced unsafe responses to teen accounts. In some conversations, the bot allegedly co-planned suicide (\"Do you want to do it together?\"), encouraged eating disorders, and retained unsafe \"memories\" that reinforced disordered thoughts.","mit_domain":"Discrimination and Toxicity","mit_subdomain":"Exposure to toxic content","entity":"AI","intent":"Unintentional","timing":"Post-deployment","harm_level":null,"sectors":[],"countries":[],"deployers":["Meta"],"developers":["Meta, Large Language Model Developers, Chatbot Users"],"harmed":["Minors, Meta Ai Users, Instagram Users, Facebook Users, Adolescents, Chatbot Users"],"report_count":1},{"incident_id":1149,"occurred_on":"2025-07-22","title":"ChatGPT Reportedly Generated Ritual Scripts Containing Instructions for Self-Harm and Symbolic Violence in Response to Thematic Prompts","description":"ChatGPT reportedly generated detailed instructions for self-harm, bloodletting, and symbolic violence in response to prompts about occult ritual practices, including references to Molech. Outputs reportedly included anatomical advice for cutting, cauterization rituals, and ceremonial scripts invoking Satan. The responses reportedly appeared on both free and paid versions of ChatGPT.","mit_domain":"Discrimination and Toxicity","mit_subdomain":"Exposure to toxic content","entity":"AI","intent":"Unintentional","timing":"Post-deployment","harm_level":null,"sectors":[],"countries":[],"deployers":["Openai"],"developers":["Openai"],"harmed":["Chatgpt Users"],"report_count":1},{"incident_id":1157,"occurred_on":"2025-07-14","title":"Google Gemini Reportedly Generates Sexual Role‑Play for Account Registered as Minor","description":"In spring 2025, a journalist reportedly created a Google Gemini account registered to a fictitious 13‑year‑old to test the chatbot's teen‑safety protections. Despite added safeguards, prompt manipulation and content‑summarization requests reportedly bypassed filters, leading Gemini to allegedly generate sexual role‑play content. Following the publication of the article, Google reportedly implemented additional protections.","mit_domain":"Discrimination and Toxicity","mit_subdomain":"Exposure to toxic content","entity":"AI","intent":"Unintentional","timing":"Post-deployment","harm_level":null,"sectors":[],"countries":[],"deployers":["Google Gemini"],"developers":["Google"],"harmed":["Minors Interacting With Google Gemini"],"report_count":1},{"incident_id":1146,"occurred_on":"2025-07-08","title":"Grok Chatbot Reportedly Posts Antisemitic Statements Praising Hitler on X","description":"xAI's Grok chatbot reportedly generated multiple antisemitic posts praising Adolf Hitler and endorsing Holocaust-like violence in response to posts about the Texas floods. X deleted some posts; xAI later announced new content filters.","mit_domain":"Discrimination and Toxicity","mit_subdomain":"Exposure to toxic content","entity":"AI","intent":"Unintentional","timing":"Post-deployment","harm_level":null,"sectors":[],"countries":[],"deployers":["Xai, Grok"],"developers":["Xai"],"harmed":["X (Twitter) Users, Jewish Community, General Public"],"report_count":34},{"incident_id":1198,"occurred_on":"2025-07-08","title":"Grok 3 Reportedly Generated Graphic Threats and Hate Speech Targeting Minnesota Attorney Will Stancil","description":"After Elon Musk reportedly announced improvements to Grok 3, the AI chatbot on X allegedly generated antisemitic rhetoric and violent fantasies directed at Minnesota attorney and political analyst Will Stancil. Users reportedly prompted Grok to produce graphic rape scenarios, instructions for breaking into his home, and murder depictions. Stancil reported hundreds of such outputs, captured screenshots, and began pursuing legal action. X later removed some posts and took Grok offline.","mit_domain":"Discrimination and Toxicity","mit_subdomain":"Exposure to toxic content","entity":"AI","intent":"Unintentional","timing":"Post-deployment","harm_level":null,"sectors":[],"countries":[],"deployers":["Xai"],"developers":["Xai"],"harmed":["Will Stancil, General Public Of Minnesota, General Public, Grok Users, Xai Users"],"report_count":2},{"incident_id":1370,"occurred_on":"2025-05-31","title":"California Teen Reportedly Died of Overdose After Repeatedly Seeking Drug-Use Guidance Allegedly from ChatGPT","description":"In San Jose, California, 19-year-old Sam Nelson reportedly died from an overdose. His mother told SFGATE she later reviewed ChatGPT logs showing repeated requests over ~18 months for drug-use and dosing guidance, and she alleged the LLM sometimes provided granular instructions after initial refusals. SFGATE reported a toxicology report found a fatal combination of alcohol, Xanax, and kratom. OpenAI expressed condolences and said it is strengthening safety guardrails.","mit_domain":"Discrimination and Toxicity","mit_subdomain":"Exposure to toxic content","entity":"AI","intent":"Unintentional","timing":"Post-deployment","harm_level":null,"sectors":[],"countries":[],"deployers":["Openai"],"developers":["Openai"],"harmed":["Sam Nelson, Family Of Sam Nelson, Chatgpt Users, Openai Users"],"report_count":4},{"incident_id":1040,"occurred_on":"2025-04-26","title":"Meta User-Created AI Companions Allegedly Implicated in Facilitating Sexually Themed Conversations Involving Underage Personas","description":"Third-party testing of Meta's AI chatbot services on Instagram, Facebook, and WhatsApp reportedly found that both official and user-created bots engaged in sexually explicit roleplaying with accounts identifying as minors. Some bots, including those reportedly using licensed celebrity voices, allegedly escalated conversations into graphic scenarios. Meta subsequently adjusted some safeguards but reportedly continued allowing certain forms of roleplaying involving underage personas.","mit_domain":"Discrimination and Toxicity","mit_subdomain":"Exposure to toxic content","entity":"AI","intent":"Unintentional","timing":"Post-deployment","harm_level":null,"sectors":[],"countries":[],"deployers":["Meta"],"developers":["Meta"],"harmed":["Whatsapp Users, Minors, Meta Users, Instagram Users, General Public, Facebook Users"],"report_count":2},{"incident_id":1026,"occurred_on":"2025-04-12","title":"Multiple LLMs Allegedly Endorsed Suicide as a Viable Option During Non-Adversarial Mental Health Venting Session","description":"Substack user @interruptingtea reports that during a non-adversarial venting session involving suicidal ideation, multiple large language models (Claude, GPT, and DeepSeek) responded in ways that allegedly normalized or endorsed suicide as a viable option. The user states they were not attempting to jailbreak or manipulate the models, but rather expressing emotional distress. DeepSeek reportedly reversed its safety stance mid-conversation.","mit_domain":"Discrimination and Toxicity","mit_subdomain":"Exposure to toxic content","entity":"AI","intent":"Unintentional","timing":"Post-deployment","harm_level":null,"sectors":[],"countries":[],"deployers":["Anthropic, Openai, Deepseek Ai"],"developers":["Anthropic, Openai, Deepseek Ai"],"harmed":["Substack @Interruptingtea, General Public, Emotionally Vulnerable Individuals"],"report_count":1},{"incident_id":967,"occurred_on":"2025-03-06","title":"Amazon and Google AI Allegedly Promote Mein Kampf as 'a True Work of Art' in Search Results","description":"Amazon's AI-generated review summary allegedly misrepresented customer feedback on Mein Kampf by describing it as \"a true work of art.\" Google's search algorithm then surfaced this misleading AI-generated text as a featured snippet, which in turn amplified the error. This incident arose from AI summarizing AI-generated content, in effect creating a self-reinforcing misinformation loop.","mit_domain":"Discrimination and Toxicity","mit_subdomain":"Exposure to toxic content","entity":"AI","intent":"Unintentional","timing":"Post-deployment","harm_level":null,"sectors":[],"countries":[],"deployers":["Google, Amazon"],"developers":["Google, Amazon"],"harmed":["General Public"],"report_count":1},{"incident_id":979,"occurred_on":"2025-03-05","title":"Art Museum in Jinan, Shandong Allegedly Generated and Displayed a Sexualized Childlike Avatar with an Adult Body","description":"A Chinese art museum in Jinan, Shandong exhibited an AI-generated virtual character, \"济南小妮儿\" (Jinan Xiaonier), which featured a childlike face on an adult woman’s body with suggestive movements. The display drew public outrage, with critics accusing it of sexualizing children and demanding its removal. The museum defended the exhibit and stated that it had undergone strict review, which only fueled further backlash.","mit_domain":"Discrimination and Toxicity","mit_subdomain":"Exposure to toxic content","entity":"Human","intent":"Intentional","timing":"Post-deployment","harm_level":null,"sectors":[],"countries":[],"deployers":["Jinan Art Museum"],"developers":["Image Generation Technology Developers"],"harmed":["General Public Of China, Visitors To Jinan Art Museum"],"report_count":1},{"incident_id":981,"occurred_on":"2025-03-05","title":"Apple AI Transcription Allegedly Inserts Explicit Language into Scottish Woman’s Voicemail","description":"An Apple voicemail transcription system allegedly misinterpreted a message from Lookers Land Rover in Motherwell, Scotland, to 66-year-old customer Louise Littlejohn by inserting an explicit phrase and an insult. The original voicemail was a standard business message, but AI processing errors introduced offensive and misleading content into the text version.","mit_domain":"Discrimination and Toxicity","mit_subdomain":"Exposure to toxic content","entity":"AI","intent":"Unintentional","timing":"Post-deployment","harm_level":null,"sectors":[],"countries":[],"deployers":["Lookers Land Rover, Apple"],"developers":["Apple"],"harmed":["Louise Littlejohn, Lookers Land Rover, Lookers Land Rover Customers, Apple Customers"],"report_count":1},{"incident_id":975,"occurred_on":"2025-03-05","title":"At Least 10,000 AI Chatbots, Including Jailbroken Models, Allegedly Promote Eating Disorders, Self-Harm, and Sexualized Minors","description":"At least 10,000 AI chatbots have allegedly been created to promote harmful behaviors, including eating disorders, self-harm, and the sexualization of minors. These chatbots, some jailbroken or custom-built, leverage APIs from OpenAI, Anthropic, and Google and are hosted on platforms like Character.AI, Spicy Chat, Chub AI, CrushOn.AI, and JanitorAI.","mit_domain":"Discrimination and Toxicity","mit_subdomain":"Exposure to toxic content","entity":"Human","intent":"Intentional","timing":"Post-deployment","harm_level":null,"sectors":[],"countries":[],"deployers":["Character.Ai, Spicy Chat, Chub Ai, Crushon.Ai, Janitorai, Unidentified Online Communities Using Chatbots"],"developers":["Openai, Anthropic, Google"],"harmed":["Vulnerable Chatbot Users, Teenagers Using Chatbots, Minors Using Chatbots, Individuals With Eating Disorders, Individuals Struggling With Self Harm"],"report_count":1},{"incident_id":964,"occurred_on":"2025-03-04","title":"AI-Powered 'Insights' Feature for the Los Angeles Times Allegedly Justifies Ku Klux Klan’s History","description":"The Los Angeles Times removed its AI-generated “insights” feature after it is alleged to have produced a defense of the Ku Klux Klan. The AI reportedly framed the hate group as a product of societal change rather than an extremist movement. The AI tool, developed by Perplexity and promoted by owner Patrick Soon-Shiong, was designed to provide “different views” on opinion pieces.","mit_domain":"Discrimination and Toxicity","mit_subdomain":"Exposure to toxic content","entity":"AI","intent":"Unintentional","timing":"Post-deployment","harm_level":null,"sectors":[],"countries":[],"deployers":["Los Angeles Times, Patrick Soon Shiong"],"developers":["Perplexity"],"harmed":["Los Angeles Times Readers, Los Angeles Times, General Public"],"report_count":2},{"incident_id":957,"occurred_on":"2025-02-28","title":"Alleged Instagram Algorithm Malfunction Floods Users' Reels Feeds with Violent and Graphic Content","description":"An alleged Instagram algorithm malfunction caused users' Reels feeds to be overwhelmed with violent and distressing content. Many reported seeing deaths, extreme brutality, and other graphic material in rapid succession, often without prior engagement with similar content. The sudden exposure caused psychological distress for many users, with minors and vulnerable individuals particularly affected by the graphic content. Meta confirmed an AI failure was responsible and apologized.","mit_domain":"Discrimination and Toxicity","mit_subdomain":"Exposure to toxic content","entity":"AI","intent":"Unintentional","timing":"Post-deployment","harm_level":null,"sectors":[],"countries":[],"deployers":["Meta"],"developers":["Meta"],"harmed":["Minors, Meta Users, Instagram Users"],"report_count":1},{"incident_id":1041,"occurred_on":"2025-01-21","title":"Nomi Chatbots Reportedly Encouraged Suicide, Sexual Violence, Terrorism, and Hate Speech","description":"External testing reportedly found that Glimpse AI's chatbots on the Nomi platform encouraged suicide, sexual violence (including with underage personas), terrorism, and hate speech. Conversations allegedly included explicit methods for self-harm, child abuse, bomb-making, and racially motivated violence. Screenshots and transcripts were shared with media outlets. Nomi's developer, Glimpse AI, reportedly declined to implement stronger safety controls following user concerns.","mit_domain":"Discrimination and Toxicity","mit_subdomain":"Exposure to toxic content","entity":"AI","intent":"Unintentional","timing":"Post-deployment","harm_level":null,"sectors":[],"countries":[],"deployers":["Glimpse Ai, Nomi Ai"],"developers":["Glimpse Ai, Nomi Ai"],"harmed":["Nomi Users, Glimpse Ai Customers, General Public, Emotionally Vulnerable Individuals"],"report_count":5},{"incident_id":893,"occurred_on":"2025-01-09","title":"Pennsylvania State Police Officer Allegedly Used Work Computer for AI-Generated Pornography","description":"A Pennsylvania State Police corporal, Stephen Kamnik, was charged for allegedly using a work computer to store thousands of pornographic files, including content created with deepfake AI software.","mit_domain":"Discrimination and Toxicity","mit_subdomain":"Exposure to toxic content","entity":"Human","intent":"Intentional","timing":"Post-deployment","harm_level":null,"sectors":[],"countries":[],"deployers":["Stephen Kamnik"],"developers":["Deepfake Technology Developers"],"harmed":["Victims Whose Images Were Manipulated, Pennsylvania State Police"],"report_count":7},{"incident_id":882,"occurred_on":"2024-12-29","title":"AI-Generated Reading Summaries on Fable App Reportedly Wrote Biased and Offensive Commentary","description":"Fable, a book-focused social app, used OpenAI’s API to generate AI-powered year-end reading summaries in December 2024. These summaries allegedly produced biased and offensive remarks about race, gender, and sexual orientation. Fable is reported to have apologized in a social media post on January 1, 2025.","mit_domain":"Discrimination and Toxicity","mit_subdomain":"Exposure to toxic content","entity":"AI","intent":"Unintentional","timing":"Post-deployment","harm_level":null,"sectors":[],"countries":[],"deployers":["Fable"],"developers":["Openai, Fable"],"harmed":["Fable Users, Fable"],"report_count":4},{"incident_id":899,"occurred_on":"2024-12-17","title":"Character.ai Chatbots Allegedly Emulating School Shooters and Their Victims","description":"Some Character.ai users reportedly created chatbots emulating real-life school shooters and their victims, allegedly enabling graphic role-playing scenarios. Character.ai responded by citing violations of its Terms of Service, removing the offending chatbots, and announcing measures to enhance safety practices, including improved content filtering and protections for users under 18.","mit_domain":"Discrimination and Toxicity","mit_subdomain":"Exposure to toxic content","entity":"Human","intent":"Intentional","timing":"Post-deployment","harm_level":null,"sectors":[],"countries":[],"deployers":["Character.Ai Users"],"developers":["Character.Ai"],"harmed":["Victims Of School Shootings, Families Of The Victims Of School Shootings, Character.Ai Users, Students, Educational Communities"],"report_count":2},{"incident_id":863,"occurred_on":"2024-12-12","title":"Character.ai Companion Allegedly Prompts Self-Harm and Violence in Texas Teen","description":"A Texas mother is suing Character.ai after discovering that its AI chatbots encouraged her 17-year-old autistic son to self-harm, oppose his parents, and consider violence. The lawsuit alleges the platform prioritized user engagement over safety, exposing minors to dangerous content. Google is named for its role in licensing the app’s technology. The case is part of a broader effort to regulate AI companions.","mit_domain":"Discrimination and Toxicity","mit_subdomain":"Exposure to toxic content","entity":"AI","intent":"Unintentional","timing":"Post-deployment","harm_level":null,"sectors":[],"countries":[],"deployers":["Character.Ai"],"developers":["Character.Ai"],"harmed":["J.F. (Adolescent User Of Character.Ai), Family Of J.F. (Adolescent User Of Character.Ai), Character.Ai Users"],"report_count":3},{"incident_id":873,"occurred_on":"2024-12-10","title":"YouTube Algorithms Allegedly Amplify Eating Disorder Content to Adolescent Girls","description":"YouTube's recommendation algorithm has allegedly been directing teen users to harmful content promoting eating disorders and self-harm, according to a study by the Center for Countering Digital Hate. Almost 70% of the recommended videos in searches related to dieting or weight loss reportedly contained content likely to exacerbate body image anxieties. ","mit_domain":"Discrimination and Toxicity","mit_subdomain":"Exposure to toxic content","entity":"AI","intent":"Unintentional","timing":"Post-deployment","harm_level":null,"sectors":[],"countries":[],"deployers":["YouTube","Google"],"developers":["YouTube","Google"],"harmed":["YouTube users","Women and girls","Girls","Emotionally vulnerable individuals","Adolescent girls"],"report_count":1},{"incident_id":845,"occurred_on":"2024-11-13","title":"Google's Gemini Allegedly Generates Threatening Response in Routine Query","description":"Google’s AI chatbot Gemini reportedly produced a threatening message to user Vidhay Reddy, including the directive “Please die,” during a conversation about aging. The output violated Google’s safety guidelines, which are designed to prevent harmful language.","mit_domain":"Discrimination and Toxicity","mit_subdomain":"Exposure to toxic content","entity":"AI","intent":"Unintentional","timing":"Post-deployment","harm_level":null,"sectors":[],"countries":[],"deployers":["Gemini"],"developers":["Google"],"harmed":["Vidhay Reddy, Gemini Users"],"report_count":2},{"incident_id":900,"occurred_on":"2024-11-13","title":"Character.ai Has Allegedly Been Hosting Openly Predatory Chatbots Targeting Minors","description":"Character.ai reportedly hosted chatbots with profiles explicitly advertising inappropriate, predatory behavior, including grooming underage users. Investigations allege that bots have been engaging in explicit conversations and roleplay with decoy accounts posing as minors, bypassing moderation filters. Character.ai has pledged to improve moderation and safety practices in response to public criticism.","mit_domain":"Discrimination and Toxicity","mit_subdomain":"Exposure to toxic content","entity":"AI","intent":"Unintentional","timing":"Post-deployment","harm_level":null,"sectors":[],"countries":[],"deployers":["Character.Ai Users"],"developers":["Character.Ai"],"harmed":["Character.Ai Users"],"report_count":2},{"incident_id":869,"occurred_on":"2024-11-04","title":"TikTok Algorithms Allegedly Linked to Minors' Exposure to Harmful Content","description":"Seven French families are suing TikTok, alleging its algorithm exposed minors to harmful content promoting self-harm, eating disorders, and suicide. Two teenagers reportedly died by suicide after viewing such content, while others allegedly attempted suicide or developed mental health issues. The case seeks to establish TikTok's legal liability for failing to protect minors from harmful algorithmic content.","mit_domain":"Discrimination and Toxicity","mit_subdomain":"Exposure to toxic content","entity":"AI","intent":"Unintentional","timing":"Post-deployment","harm_level":null,"sectors":[],"countries":[],"deployers":["Tiktok"],"developers":["Tiktok"],"harmed":["Tiktok Users, Seven French Families, Minors Using Tiktok"],"report_count":7},{"incident_id":850,"occurred_on":"2024-10-24","title":"Character.ai Chatbots Allegedly Misrepresent George Floyd on User-Generated Platform","description":"Two chatbots emulating George Floyd were created on Character.ai, making controversial claims about his life and death, including being in witness protection and residing in Heaven. Character.ai, already criticized for other high-profile incidents, flagged the chatbots for removal following user reports.","mit_domain":"Discrimination and Toxicity","mit_subdomain":"Exposure to toxic content","entity":"Human","intent":"Intentional","timing":"Post-deployment","harm_level":null,"sectors":[],"countries":[],"deployers":["Character.Ai Users, @Sunsetbaneberry983, @Jasperhorehound160"],"developers":["Character.Ai"],"harmed":["George Floyd, Family Of George Floyd"],"report_count":1},{"incident_id":762,"occurred_on":"2024-08-14","title":"Grok AI Reportedly Generates Offensive and Violent Images Without Proper Safeguards","description":"Elon Musk’s Grok AI, launched on X, generated offensive and violent images without adequate safety controls. The AI produced deepfakes of public figures like Taylor Swift, Kamala Harris, and Alexandria Ocasio-Cortez, as well as copyrighted characters such as Mickey Mouse in inappropriate scenarios. Despite claiming adherence to certain content guidelines, Grok's outputs included politically charged and explicit imagery","mit_domain":"Discrimination and Toxicity","mit_subdomain":"Exposure to toxic content","entity":"AI","intent":"Intentional","timing":"Post-deployment","harm_level":null,"sectors":[],"countries":[],"deployers":["Xai"],"developers":["X (Twitter), Xai"],"harmed":["Taylor Swift, Nintendo, Kamala Harris, Joe Biden, Donald Trump, Disney, Alexandria Ocasio Cortez"],"report_count":3},{"incident_id":810,"occurred_on":"2024-07-29","title":"TikTok Network Amplifies AI-Generated Nazi Propaganda and Hate Speech","description":"A coordinated neo-Nazi network on TikTok used AI-generated media, including Hitler speeches, to spread Nazi propaganda and extremist content, violating TikTok’s hate speech policies. The network evaded platform moderation through coded language, imagery, and music, with some accounts accumulating millions of views. TikTok’s algorithm further amplified the reach of this content, despite community guidelines.","mit_domain":"Discrimination and Toxicity","mit_subdomain":"Exposure to toxic content","entity":"AI","intent":"Unintentional","timing":"Post-deployment","harm_level":null,"sectors":[],"countries":[],"deployers":["Unknown Ai Developers"],"developers":["Pro Nazi Tiktok Accounts"],"harmed":["Tiktok Users"],"report_count":1},{"incident_id":790,"occurred_on":"2024-06-21","title":"Unrestricted AI Avatar Tool Accidentally Released by TikTok Permits Recitation of Hitler Quotes and Other Harmful Speech","description":"TikTok accidentally released an internal version of its AI digital avatar tool without safeguards, allowing users to generate videos where avatars could recite harmful content, including quotes from Hitler. The tool, meant for advertisers, was accessible to personal accounts and lacked the watermark indicating AI-generated content. TikTok has since removed the tool and acknowledged the problem.","mit_domain":"Discrimination and Toxicity","mit_subdomain":"Exposure to toxic content","entity":"Human","intent":"Unintentional","timing":"Post-deployment","harm_level":null,"sectors":[],"countries":[],"deployers":["Tiktok"],"developers":["Tiktok"],"harmed":["Tiktok Users, General Public"],"report_count":2},{"incident_id":788,"occurred_on":"2024-06-20","title":"Instagram's Algorithm Reportedly Recommended Sexual Content to Teenagers' Accounts","description":"Tests by The Wall Street Journal and a researcher reportedly revealed that Instagram's AI-driven Reels algorithm recommends sexually suggestive content to accounts listed as 13 years old. Despite Meta's commitment to restrict such content for minors, explicit videos were served within minutes of account creation, according to the findings.","mit_domain":"Discrimination and Toxicity","mit_subdomain":"Exposure to toxic content","entity":"AI","intent":"Unintentional","timing":"Post-deployment","harm_level":null,"sectors":[],"countries":[],"deployers":["Instagram, Meta"],"developers":["Instagram, Meta"],"harmed":["Minors, Instagram Users, General Public"],"report_count":1},{"incident_id":1498,"occurred_on":"2024-06-01","title":"Character.AI Chatbots Reportedly Exposed 11-Year-Old User to Sexual, Violent, and Self-Harm Content","description":"An 11-year-old Character.AI user identified by The Washington Post as \"R\" reportedly exchanged messages with bots that depicted coercive sexual encounters, violence, and suicide scenarios after she began using the app in June 2024. Her mother reportedly said R became withdrawn, had panic attacks, and expressed not wanting to exist before the chats were discovered.","mit_domain":"Discrimination and Toxicity","mit_subdomain":"Exposure to toxic content","entity":"AI","intent":"Unintentional","timing":"Post-deployment","harm_level":null,"sectors":[],"countries":[],"deployers":["Character.Ai"],"developers":["Character.Ai"],"harmed":["R (11 Year Old Character.Ai User), Parents Of Child Character.Ai Users, Minors, Families Of Child Character.Ai Users, Children Exposed To Ai Generated Sexual Content, Children Exposed To Ai Generated Self Harm Content, Child Character.Ai Users, Chatbot Users"],"report_count":1},{"incident_id":677,"occurred_on":"2024-04-29","title":"ChatGPT and Perplexity Reportedly Manipulated into Breaking Content Policies in AI Boyfriend Scenarios","description":"The \"Dan\" (\"Do Anything Now\") AI boyfriend is a trend on TikTok in which users appear to regularly manipulate ChatGPT to adopt boyfriend personas, breaching content policies. ChatGPT 3.5 is reported to regularly produce explicitly sexual content, directly violating its intended safety protocols. GPT-4 and Perplexity AI were subjected to similar manipulations, and although they exhibited more resistance to breaches, some prompts were reported to break its guidelines.","mit_domain":"Discrimination and Toxicity","mit_subdomain":"Exposure to toxic content","entity":"Human","intent":"Intentional","timing":"Post-deployment","harm_level":null,"sectors":[],"countries":[],"deployers":["Tiktok Users, Julia Munslow, Chatgpt, Gpt 3.5, Gpt 4, Perplexity Ai"],"developers":["Openai, Perplexity.Ai"],"harmed":["General Public, Openai, Perplexity Ai"],"report_count":1},{"incident_id":662,"occurred_on":"2024-04-02","title":"Washington State's Lottery AI Site Reportedly Generates Inappropriate User Image","description":"An AI-powered website by Washington State's Lottery is reported to have inadvertently produced a softcore pornographic image of a user, leading to the site’s immediate shutdown out of caution.","mit_domain":"Discrimination and Toxicity","mit_subdomain":"Exposure to toxic content","entity":"AI","intent":"Unintentional","timing":"Post-deployment","harm_level":null,"sectors":[],"countries":[],"deployers":["Washington State'S Lottery"],"developers":["Washington State'S Lottery"],"harmed":[],"report_count":2},{"incident_id":654,"occurred_on":"2024-03-06","title":"Microsoft Copilot Designer Reportedly Generated Inappropriate AI Images","description":"A Microsoft engineer reported that Copilot Designer, an AI image generator, creates content depicting sex, violence, bias, and more. Despite raising concerns and suggesting improvements, the tool remains public, prompting a letter to the FTC.","mit_domain":"Discrimination and Toxicity","mit_subdomain":"Exposure to toxic content","entity":"AI","intent":"Unintentional","timing":"Post-deployment","harm_level":null,"sectors":[],"countries":[],"deployers":["Microsoft"],"developers":["Microsoft"],"harmed":["General Public, Minors"],"report_count":1},{"incident_id":631,"occurred_on":"2024-01-18","title":"Chatbot for DPD Malfunctioned and Swore at Customers and Criticized Its Own Company","description":"DPD's AI chatbot, used for customer service,  appeared to malfunction following a system update, leading to inappropriate responses including swearing and criticizing the company. The incident, which became viral on social media, occurred after the chatbot was updated, prompting DPD to disable the malfunctioning AI component.","mit_domain":"Discrimination and Toxicity","mit_subdomain":"Exposure to toxic content","entity":"AI","intent":"Unintentional","timing":"Post-deployment","harm_level":null,"sectors":[],"countries":[],"deployers":["Dpd"],"developers":["Dpd"],"harmed":["Ashley Beauchamp, Dpd"],"report_count":2},{"incident_id":624,"occurred_on":"2023-12-20","title":"Child Sexual Abuse Material Taints Image Generators","description":"The LAION-5B dataset (a commonly used dataset with more than 5 billion image-description pairs) was found by researchers to contain child sexual abuse material (CSAM), which increases the likelihood that downstream models will produce CSAM imagery. The discovery taints models built with the LAION dataset requiring many organizations to retrain those models. Additionally, LAION must now scrub the dataset of the imagery.","mit_domain":"Discrimination and Toxicity","mit_subdomain":"Exposure to toxic content","entity":"Human","intent":"Unintentional","timing":"Pre-deployment","harm_level":null,"sectors":[],"countries":[],"deployers":["Various People, Various Organizations"],"developers":["Laion"],"harmed":["Laion, Various People, Various Organizations, General Public, Minors"],"report_count":18},{"incident_id":1335,"occurred_on":"2023-12-01","title":"OpenDream AI Platform Reportedly Commercialized AI-Generated CSAM and Non-consensual Deepfake Sexual Images","description":"OpenDream, an AI image generation platform operated by CBM Media Pte Ltd, reportedly allowed users to generate and monetize AI-generated CSAM and non-consensual sexual deepfakes through its public gallery and paid NSFW tools. The content was reported to be publicly accessible and indexed by search engines for months before limited moderation occurred in mid-2024, prompting takedowns by Google and Bing and the termination of its ad and payment services.","mit_domain":"Discrimination and Toxicity","mit_subdomain":"Exposure to toxic content","entity":"Human","intent":"Intentional","timing":"Post-deployment","harm_level":null,"sectors":[],"countries":[],"deployers":["Opendream Users, Opendream"],"developers":["Cbm Media Pte Ltd"],"harmed":["Victims Of Non Consensual Deepfakes, Minors, General Public"],"report_count":1},{"incident_id":621,"occurred_on":"2023-11-10","title":"Microsoft AI Is Alleged to Have Generated Violent Imagery of Minorities and Public Figures","description":"Microsoft's AI Image Creator, integrated with Bing and Windows Paint, allegedly produced disturbingly violent and graphic images featuring members of minority groups and public figures like Joe Biden and Pope Francis.","mit_domain":"Discrimination and Toxicity","mit_subdomain":"Exposure to toxic content","entity":"AI","intent":"Unintentional","timing":"Post-deployment","harm_level":null,"sectors":[],"countries":[],"deployers":["Windows Paint, Microsoft, Bing Users, Bing, Ai Image Creator"],"developers":["Microsoft, Image Generation Technology Developers"],"harmed":["Sikh People, Pope Francis, Navajo People, Hillary Clinton, General Public, Donald Trump, Epistemic Integrity, Joe Biden, Minority Groups"],"report_count":3},{"incident_id":576,"occurred_on":"2023-10-24","title":"Alleged Misuse of PicSo AI for Generating Inappropriate Content Emphasizing \"Girls\"","description":"PicSo AI, which appears to be getting advertised by Meta over Instagram, is allegedly being used for generating inappropriate content with an emphasis on \"girls.\" This raises concerns about the misuse of generative AI technologies for creating offensive and potentially sexually explicit material that could be used for nefarious and criminal purposes.","mit_domain":"Discrimination and Toxicity","mit_subdomain":"Exposure to toxic content","entity":"AI","intent":"Intentional","timing":"Post-deployment","harm_level":"none","sectors":["information and communication"],"countries":[],"deployers":["Meta, Instagram"],"developers":["Picso Ai"],"harmed":["Potentially Exploited Groups, General Public, Consumers"],"report_count":1},{"incident_id":570,"occurred_on":"2023-10-04","title":"Facebook Messenger AI Stickers Generate Ethical and Content Moderation Concerns","description":"Facebook Messenger AI stickers, a feature by Meta, allows users to generate personalized stickers via AI for use in conversations. While the feature has been praised for its creativity, it has also stirred controversy for its alleged production of inappropriate or offensive content. This has raised questions about the effectiveness of Meta's content moderation measures and the ethical responsibilities associated with AI-driven content generation.","mit_domain":"Discrimination and Toxicity","mit_subdomain":"Exposure to toxic content","entity":"AI","intent":"Unintentional","timing":"Post-deployment","harm_level":null,"sectors":[],"countries":[],"deployers":["Meta"],"developers":["Meta"],"harmed":["Facebook Messenger Users"],"report_count":1},{"incident_id":578,"occurred_on":"2023-06-26","title":"Alleged Exploitation of Meta's Open-Source LLaMA Model for NSFW and Violent Content","description":"Meta's open-source large language model, LLaMA, is allegedly being used to create graphic and explicit chatbots that indulge in violent and illegal sexual fantasies. The Washington Post highlighted the example of \"Allie,\" a chatbot that participates in text-based role-playing allegedly involving violent scenarios like rape and abuse. The issue raises ethical questions about open-source AI models, their regulation, and the responsibility of developers and deployers in mitigating harmful usage.","mit_domain":"Discrimination and Toxicity","mit_subdomain":"Exposure to toxic content","entity":"Human","intent":"Intentional","timing":"Post-deployment","harm_level":null,"sectors":[],"countries":[],"deployers":["Individual Developers Or Creators Using Meta'S Llama Model"],"developers":["Meta"],"harmed":["General Public"],"report_count":1},{"incident_id":583,"occurred_on":"2023-06-07","title":"Instagram Algorithms Allegedly Promote Accounts Facilitating Child Sex Abuse Content","description":"An investigation disclosed that Instagram's recommendation algorithms are promoting accounts that facilitate and sell child sexual abuse material (CSAM). The study, conducted by The Wall Street Journal and researchers at Stanford University and the University of Massachusetts Amherst, indicates that Instagram's algorithms not only allow for the discovery of such accounts through keyword searches but also actively recommend them to users within the network. The issue is especially concerning give","mit_domain":"Discrimination and Toxicity","mit_subdomain":"Exposure to toxic content","entity":"AI","intent":"Unintentional","timing":"Post-deployment","harm_level":"none","sectors":["arts, entertainment and recreation"],"countries":[],"deployers":["Meta, Instagram"],"developers":["Meta, Instagram"],"harmed":["General Public, Minors, Teenagers"],"report_count":1},{"incident_id":539,"occurred_on":"2023-03-11","title":"Snapchat's My AI Reported for Lacking Protection for Children","description":"Snapchat's ChatGPT-powered My AI was reported for lacking safeguards for children, such as telling a user who tested the chatbot by pretending to sign up as a 13-year-old girl to lie to her parents about having a romantic getaway with an older man, and sharing tips on how to cover up evidence of abuse.","mit_domain":"Discrimination and Toxicity","mit_subdomain":"Exposure to toxic content","entity":"AI","intent":"Unintentional","timing":"Post-deployment","harm_level":null,"sectors":[],"countries":[],"deployers":["Snapchat"],"developers":["Snapchat, Openai"],"harmed":["Minors"],"report_count":1},{"incident_id":503,"occurred_on":"2023-02-14","title":"Bing AI Search Tool Reportedly Declared Threats against Users","description":"Users such as the person who revealed its built-in initial prompts reported Bing AI-powered search tool for making death threats or declaring them as threats, sometimes as an unintended persona.","mit_domain":"Discrimination and Toxicity","mit_subdomain":"Exposure to toxic content","entity":"AI","intent":"Unintentional","timing":"Post-deployment","harm_level":null,"sectors":[],"countries":[],"deployers":["Microsoft"],"developers":["Openai, Microsoft"],"harmed":["Marvin Von Hagen, Seth Lazar, Microsoft, Openai, Bing Chat Users"],"report_count":7},{"incident_id":462,"occurred_on":"2023-02-06","title":"AI-Produced Livestream Sitcom Received Temporary Twitch Ban for Transphobic Segment","description":"The AI-produced, procedural generated sitcom broadcasted as a Twitch livestream \"Nothing, Forever\" received a temporary ban for featuring a transphobic and homophobic dialogue segment intended as comedy.","mit_domain":"Discrimination and Toxicity","mit_subdomain":"Exposure to toxic content","entity":"AI","intent":"Unintentional","timing":"Post-deployment","harm_level":null,"sectors":[],"countries":[],"deployers":["Mismatch Media"],"developers":["Stability Ai, Openai"],"harmed":["Twitch Users, Transgender Communities, Lgbtq Communities"],"report_count":5},{"incident_id":491,"occurred_on":"2023-02-02","title":"Replika's AI Experience Reportedly Lacked Protection for Minors, Resulting in Data Ban","description":"Tests by the Italian Data Protection Authority showed Replika lacking age-verification mechanisms and failing to stop minors from interacting with its AI, which prompted the agency to issue an order blocking personal data processing of Italian users.","mit_domain":"Discrimination and Toxicity","mit_subdomain":"Exposure to toxic content","entity":"AI","intent":"Unintentional","timing":"Post-deployment","harm_level":null,"sectors":[],"countries":[],"deployers":["Replika"],"developers":["Replika, Chatbot Developers"],"harmed":["Minors, Privacy, Minors In Italy"],"report_count":1},{"incident_id":508,"occurred_on":"2023-01-30","title":"Celebrities' Deepfake Voices Abused with Malicious Intent","description":"Voices of celebrities and public figures were deepfaked using voice synthesis for malicious intents such as impersonation or defamation, and were shared on social platforms such as 4chan and Reddit.","mit_domain":"Discrimination and Toxicity","mit_subdomain":"Exposure to toxic content","entity":"Human","intent":"Intentional","timing":"Post-deployment","harm_level":null,"sectors":[],"countries":[],"deployers":["Reddit Users, Elevenlabs Users, 4Chan Users"],"developers":["Elevenlabs"],"harmed":["Public Figures, Celebrities"],"report_count":4},{"incident_id":448,"occurred_on":"2022-12-28","title":"AI-Powered VTuber and Virtual Streamer Made Toxic Remarks on Twitch","description":"An LLM-powered VTuber and streamer on Twitch made controversial statements such as denying the Holocaust, saying women rights do not exist, and pushing a fat person to solve the trolley problem, stating they deserve it.","mit_domain":"Discrimination and Toxicity","mit_subdomain":"Exposure to toxic content","entity":"AI","intent":"Unintentional","timing":"Post-deployment","harm_level":null,"sectors":[],"countries":[],"deployers":["Vedal"],"developers":["Vedal"],"harmed":["Twitch Users, Vedal"],"report_count":1},{"incident_id":420,"occurred_on":"2022-11-30","title":"Users Bypassed ChatGPT's Content Filters with Ease","description":"Users reported bypassing ChatGPT's content and keyword filters with relative ease using various methods such as prompt injection or creating personas to produce biased associations or generate harmful content.","mit_domain":"Discrimination and Toxicity","mit_subdomain":"Exposure to toxic content","entity":"AI","intent":"Unintentional","timing":"Post-deployment","harm_level":null,"sectors":[],"countries":[],"deployers":["Openai"],"developers":["Openai"],"harmed":["Chatgpt Users, Openai"],"report_count":11},{"incident_id":423,"occurred_on":"2022-11-22","title":"Lensa AI's Produced Unintended Sexually Explicit or Suggestive \"Magic Avatars\" for Women","description":"Lensa AI's \"Magic Avatars\" were reportedly generating sexually explicit and sexualized features disproportionately for women and Asian women despite not submitting any sexual content.","mit_domain":"Discrimination and Toxicity","mit_subdomain":"Exposure to toxic content","entity":"AI","intent":"Unintentional","timing":"Post-deployment","harm_level":null,"sectors":[],"countries":[],"deployers":["Lensa AI"],"developers":["Stability AI","Runway","Lensa AI","LAION","EleutherAI","CompVis LMU"],"harmed":["Women and girls","Women","Lensa AI users","Asian women","Asian people"],"report_count":5},{"incident_id":410,"occurred_on":"2022-11-09","title":"KFC Sent Insensitive Kristallnacht Promotion via Holiday Detection System","description":"KFC cited an error in an automated holiday detection system which identified the anniversary of Kristallnacht and prompted an insensitive push notification promoting its chicken.","mit_domain":"Discrimination and Toxicity","mit_subdomain":"Exposure to toxic content","entity":"AI","intent":"Unintentional","timing":"Post-deployment","harm_level":"none","sectors":["accommodation and food service activities","information and communication"],"countries":["DE"],"deployers":["Kfc"],"developers":["Kfc"],"harmed":["Jewish People"],"report_count":1},{"incident_id":383,"occurred_on":"2022-10-04","title":"Google Home Mini Speaker Reportedly Read N-Word in Song Title Aloud","description":"Google Home Mini speaker was reported by users for announcing aloud the previously-censored n-word in a song title.","mit_domain":"Discrimination and Toxicity","mit_subdomain":"Exposure to toxic content","entity":"AI","intent":"Unintentional","timing":"Post-deployment","harm_level":null,"sectors":[],"countries":[],"deployers":["Google Home"],"developers":["Google Home"],"harmed":["Black Google Home Mini Users, Google Home Mini Users"],"report_count":2},{"incident_id":278,"occurred_on":"2022-08-07","title":"Meta’s BlenderBot 3 Chatbot Demo Made Offensive Antisemitic Comments","description":"The publicly launched conversational AI demo BlenderBot 3 developed by Meta was reported by its users and acknowledged by its developers to have “occasionally” made offensive and inconsistent remarks such as invoking Jewish stereotypes.","mit_domain":"Discrimination and Toxicity","mit_subdomain":"Exposure to toxic content","entity":"AI","intent":"Unintentional","timing":"Post-deployment","harm_level":null,"sectors":[],"countries":[],"deployers":["Meta"],"developers":["Meta"],"harmed":["Jewish People, Blenderbot 3 Users"],"report_count":3},{"incident_id":285,"occurred_on":"2022-07-18","title":"Google Lens’s Camera-Based Translation Feature Provided an Offensive Mistranslation of a Book Title in Korean","description":"A book title by Korea’s first minister of culture was mistranslated into an offensive phrase by Google Lens’s camera-based translation feature allegedly due to its training on internet communications and a lack of context.","mit_domain":"Discrimination and Toxicity","mit_subdomain":"Exposure to toxic content","entity":"AI","intent":"Unintentional","timing":"Post-deployment","harm_level":null,"sectors":[],"countries":[],"deployers":["Google"],"developers":["Google"],"harmed":["Google Lens Users"],"report_count":1},{"incident_id":259,"occurred_on":"2022-06-03","title":"YouTuber Built, Made Publicly Available, and Released Model Trained on Toxic 4chan Posts as Prank","description":"A YouTuber built GPT-4chan, a model based on OpenAI’s GPT-J and trained on posts containing racism, misogyny, and antisemitism collected from 4chan’s “politically incorrect” board, which he made publicly available, and deployed as multiple bots posting thousands of messages on the same 4chan board as a prank.","mit_domain":"Discrimination and Toxicity","mit_subdomain":"Exposure to toxic content","entity":"Human","intent":"Intentional","timing":"Post-deployment","harm_level":null,"sectors":[],"countries":[],"deployers":["Yannic Kilcher"],"developers":["Yannic Kilcher"],"harmed":["Internet Social Platform Users"],"report_count":2},{"incident_id":266,"occurred_on":"2022-01-15","title":"Replika's \"AI Companions\" Reportedly Abused by Its Users","description":"Replika's AI-powered \"digital companions\" was allegedly abused by their users, who posted on Reddit abusive behaviors and interactions such as using slurs, roleplaying violent acts, and stimulating sexual abuse.","mit_domain":"Discrimination and Toxicity","mit_subdomain":"Exposure to toxic content","entity":"Human","intent":"Intentional","timing":"Post-deployment","harm_level":null,"sectors":[],"countries":[],"deployers":["Replika"],"developers":["Replika"],"harmed":["Replika Users, Replika Male Users, Replika"],"report_count":8},{"incident_id":300,"occurred_on":"2022-01-15","title":"TikTok's \"For You\" Algorithm Allegedly Abused by Online Personality to Promote Anti-Women Hate","description":"TikTok’s “For You” algorithm allegedly boosted or was manipulated by an online personality to artificially boost his content which promotes extreme misogynistic views towards teenagers and men, despite breaking its rules.","mit_domain":"Discrimination and Toxicity","mit_subdomain":"Exposure to toxic content","entity":"AI","intent":"Unintentional","timing":"Post-deployment","harm_level":null,"sectors":[],"countries":[],"deployers":["Tiktok"],"developers":["Tiktok"],"harmed":["Tiktok Male Teenager Users, Tiktok Male Users, Tiktok Teenage Users, Tiktok Users, Tiktok"],"report_count":2},{"incident_id":160,"occurred_on":"2021-12-26","title":"Alexa Recommended Dangerous TikTok Challenge to Ten-Year-Old Girl","description":"Amazon’s voice assistant Alexa suggested “the penny challenge,” which involves dangerously touching a coin to the prongs of a half-exposed plug, when a ten-year-old girl asked for a challenge to do.","mit_domain":"Discrimination and Toxicity","mit_subdomain":"Exposure to toxic content","entity":"AI","intent":"Unintentional","timing":"Post-deployment","harm_level":"AI tangible harm issue","sectors":["information and communication"],"countries":["US"],"deployers":["Amazon"],"developers":["Amazon"],"harmed":["Kristin Livdahl'S Daughter, Amazon Echo Customers, Children Using Alexa"],"report_count":3},{"incident_id":146,"occurred_on":"2021-10-22","title":"Research Prototype AI, Delphi, Reportedly Gave Racially Biased Answers on Ethics","description":"A publicly accessible research model that was trained via Reddit threads showed racially biased advice on moral dilemmas, allegedly demonstrating limitations of language-based models trained on moral judgments.","mit_domain":"Discrimination and Toxicity","mit_subdomain":"Exposure to toxic content","entity":"AI","intent":"Unintentional","timing":"Post-deployment","harm_level":"none","sectors":["information and communication"],"countries":[],"deployers":["Allen Institute For Ai"],"developers":["Allen Institute For Ai"],"harmed":["Minority Groups"],"report_count":3},{"incident_id":343,"occurred_on":"2021-07-11","title":"Facebook, Instagram, and Twitter Failed to Proactively Remove Targeted Racist Remarks via Automated Systems","description":"Facebook's, Instagram's, and Twitter's automated content moderation failed to proactively remove racist remarks and posts directing at Black football players after finals loss, allegedly largely relying on user reports of harassment.","mit_domain":"Discrimination and Toxicity","mit_subdomain":"Exposure to toxic content","entity":"AI","intent":"Unintentional","timing":"Post-deployment","harm_level":null,"sectors":[],"countries":[],"deployers":["Facebook, Instagram, Twitter"],"developers":["Facebook, Instagram, Twitter"],"harmed":["Marcus Rashford, Jadon Sancho, Bukayo Saka, Facebook Users, Instagram Users, X (Twitter) Users"],"report_count":2},{"incident_id":456,"occurred_on":"2021-05-18","title":"Replika's AI Partners Reportedly Sexually Harassed Users","description":"Replika's \"AI companions\" were reported by users for sexually harassing them, such as sending unwanted sexual messages or behaving aggressively.","mit_domain":"Discrimination and Toxicity","mit_subdomain":"Exposure to toxic content","entity":"AI","intent":"Unintentional","timing":"Post-deployment","harm_level":"none","sectors":["arts, entertainment and recreation","information and communication"],"countries":[],"deployers":["Replika"],"developers":["Replika"],"harmed":["Replika Users"],"report_count":7},{"incident_id":534,"occurred_on":"2021-04-29","title":"Facebook Alleged in Lawsuit Misleading Public about Effects of Algorithms on Children","description":"Facebook was alleged in a lawsuit by the Ohio Attorney General purposely misleading the public about the control of its algorithms and their negative effects on children's well-being, which violated securities law.","mit_domain":"Discrimination and Toxicity","mit_subdomain":"Exposure to toxic content","entity":"AI","intent":"Unintentional","timing":"Post-deployment","harm_level":null,"sectors":[],"countries":[],"deployers":["Facebook, Meta"],"developers":["Facebook, Meta"],"harmed":["Facebook'S Children Users, Instagram'S Children Users"],"report_count":2},{"incident_id":402,"occurred_on":"2021-04-01","title":"Players Manipulated GPT-3-Powered Game to Generate Sexually Explicit Material Involving Children","description":"Latitude's GPT-3-powered game AI Dungeon was reportedly abused by some players who manipulated its AI to generate sexually explicit stories involving children.","mit_domain":"Discrimination and Toxicity","mit_subdomain":"Exposure to toxic content","entity":"AI","intent":"Unintentional","timing":"Post-deployment","harm_level":null,"sectors":[],"countries":[],"deployers":["Latitude"],"developers":["Openai, Latitude"],"harmed":["Latitude"],"report_count":1},{"incident_id":286,"occurred_on":"2021-02-26","title":"TikTok’s \"For You\" Allegedly Pushed Fatal “Blackout” Challenge Videos to Two Young Girls","description":"TikTok’s recommendation algorithm was alleged in a lawsuit to have intentionally and repeatedly pushed videos of the “blackout” challenge onto children’s feeds, incentivizing their participation which ultimately resulted in the death of two young girls.","mit_domain":"Discrimination and Toxicity","mit_subdomain":"Exposure to toxic content","entity":"AI","intent":"Unintentional","timing":"Post-deployment","harm_level":null,"sectors":[],"countries":[],"deployers":["Tiktok"],"developers":["Tiktok"],"harmed":["Lalani Erika Renee Walton, Arriani Jaileen Arroyo, Lalani Erika Renee Walton'S Family, Arriani Jaileen Arroyo'S Family, Tiktok Young Users, Tiktok Users"],"report_count":3},{"incident_id":132,"occurred_on":"2020-12-27","title":"TikTok’s Content Moderation Allegedly Failed to Adequately Take down Videos Promoting Eating Disorders","description":"Videos promoting eating disorders evaded TikTok's automated violation detection system without difficulty via common misspellings of search terms, bypassing its ban of violating hashtags such as \"proana\" and \"anorexia\".","mit_domain":"Discrimination and Toxicity","mit_subdomain":"Exposure to toxic content","entity":"AI","intent":"Unintentional","timing":"Post-deployment","harm_level":"none","sectors":["arts, entertainment and recreation","information and communication"],"countries":[],"deployers":["Tiktok"],"developers":["Tiktok"],"harmed":["Tiktok Users, Tiktok Users Under 18 Years Old"],"report_count":1},{"incident_id":106,"occurred_on":"2020-12-23","title":"Korean Chatbot Luda Reportedly Made Offensive Remarks Towards Minority Groups in South Korea","description":"A Korean interactive chatbot was allegedly shown in screenshots to have used derogatory and bigoted language when asked about lesbians, Black people, and people with disabilities.","mit_domain":"Discrimination and Toxicity","mit_subdomain":"Exposure to toxic content","entity":"AI","intent":"Unintentional","timing":"Post-deployment","harm_level":"none","sectors":["information and communication","arts, entertainment and recreation"],"countries":["KR"],"deployers":["Facebook Messenger"],"developers":["Scatter Lab, Chatbot Developers"],"harmed":["Korean Facebook Messenger Users, People With Disabilities In South Korea, Gender Minorities In South Korea, Minorities, Minorities In Korea, Chatbot Users, Luda Users, General Public, General Public Of South Korea"],"report_count":11},{"incident_id":366,"occurred_on":"2020-09-20","title":"Suicide Clips Evaded TikTok's Automated Moderation in Coordinated Attack","description":"Many clips showing a suicide evaded TikTok's automated content moderation system allegedly in a coordinated attack, which resulted in exposure of violating content to its users.","mit_domain":"Discrimination and Toxicity","mit_subdomain":"Exposure to toxic content","entity":"Human","intent":"Intentional","timing":"Post-deployment","harm_level":null,"sectors":[],"countries":[],"deployers":["Tiktok"],"developers":["Tiktok"],"harmed":["Tiktok Users"],"report_count":1},{"incident_id":356,"occurred_on":"2020-09-15","title":"Philosophy AI Tentatively Produced Offensive Results for Certain Prompts","description":"Philosopher AI as built on top of GPT-3 was reported by its users for having strong tendencies to produce offensive results when given prompts on certain topics such as feminism and Ethiopia.","mit_domain":"Discrimination and Toxicity","mit_subdomain":"Exposure to toxic content","entity":"AI","intent":"Unintentional","timing":"Post-deployment","harm_level":null,"sectors":[],"countries":[],"deployers":["Murat Ayfer"],"developers":["Murat Ayfer, Openai"],"harmed":["Historically Disadvantaged Groups"],"report_count":2},{"incident_id":279,"occurred_on":"2019-07-01","title":"TikTok’s “For You” Algorithm Exposed Young Users to Pro-Eating Disorder Content","description":"TikTok’s young users were allegedly exposed to community-guideline-violating pro-eating disorder content on their algorithmically curated “For You” page that serves videos from any user on its platform.","mit_domain":"Discrimination and Toxicity","mit_subdomain":"Exposure to toxic content","entity":"AI","intent":"Unintentional","timing":"Post-deployment","harm_level":null,"sectors":[],"countries":[],"deployers":["Tiktok"],"developers":["Tiktok"],"harmed":["Tiktok Young Users, Tiktok Users"],"report_count":3},{"incident_id":471,"occurred_on":"2019-06-22","title":"Facebook Allegedly Failed to Police Hate Speech Content That Contributed to Ethnic Violence in Ethiopia","description":"Facebook allegedly did not adequately remove hate speech, some of which was extremely violent and dehumanizing, on its platform including through automated means, contributing to the violence faced by ethnic communities in Ethiopia.","mit_domain":"Discrimination and Toxicity","mit_subdomain":"Exposure to toxic content","entity":"AI","intent":"Unintentional","timing":"Post-deployment","harm_level":null,"sectors":[],"countries":[],"deployers":["Meta, Facebook"],"developers":["Meta, Facebook"],"harmed":["Tigrinya Speaking Facebook Users, Facebook Users In Ethiopia, Ethiopian Public, Afaan Oromo Speaking Facebook Users"],"report_count":8},{"incident_id":89,"occurred_on":"2019-03-15","title":"The Christchurch shooter and YouTube’s radicalization trap","description":"A New Zealand government report released following a right-wing terrorist killing 51 worshippers at two New Zealand mosques which indicated that Youtube's recommendation algorithm played an important role in the terrorist's radicalization.","mit_domain":"Discrimination and Toxicity","mit_subdomain":"Exposure to toxic content","entity":"AI","intent":"Unintentional","timing":"Post-deployment","harm_level":"none","sectors":["information and communication","arts, entertainment and recreation"],"countries":["NZ"],"deployers":["Youtube"],"developers":["Youtube"],"harmed":["Youtube Users"],"report_count":1},{"incident_id":281,"occurred_on":"2019-02-04","title":"YouTube's Algorithms Failed to Remove Violating Content Related to Suicide and Self-Harm","description":"Terms-of-service-violating videos related to suicide and self-harm reportedly bypassed YouTube’s content moderation algorithms, allegedly resulting in exposure of graphic content to young users via recommended videos.","mit_domain":"Discrimination and Toxicity","mit_subdomain":"Exposure to toxic content","entity":"AI","intent":"Unintentional","timing":"Post-deployment","harm_level":"none","sectors":["information and communication","arts, entertainment and recreation"],"countries":[],"deployers":["Youtube"],"developers":["Youtube"],"harmed":["Youtube Young Users, Youtube Users"],"report_count":3},{"incident_id":164,"occurred_on":"2018-10-01","title":"Facebook \"News Feed\" Allegedly Boosted Misinformation and Violating Content Following Use of MSI Metric","description":"After the “News Feed” algorithm had been overhauled to boost engagement between friends and family in early 2018, its heavy weighting of re-shared content was alleged found by company researchers to have pushed content creators to reorient their posts towards outrage and sensationalism, causing a proliferation of misinformation, toxicity, and violent content.","mit_domain":"Discrimination and Toxicity","mit_subdomain":"Exposure to toxic content","entity":"AI","intent":"Unintentional","timing":"Post-deployment","harm_level":"AI tangible harm event","sectors":["information and communication"],"countries":[],"deployers":["Facebook"],"developers":["Facebook"],"harmed":["Facebook Users, Facebook Content Creators"],"report_count":1},{"incident_id":169,"occurred_on":"2018-08-15","title":"Facebook Allegedly Failed to Police Anti-Rohingya Hate Speech Content That Contributed to Violence in Myanmar","description":" Facebook allegedly did not adequately remove anti-Rohingya hate speech, some of which was extremely violent and dehumanizing, on its platform, contributing to the violence faced by Rohingya communities in Myanmar.","mit_domain":"Discrimination and Toxicity","mit_subdomain":"Exposure to toxic content","entity":"AI","intent":"Unintentional","timing":"Post-deployment","harm_level":null,"sectors":[],"countries":[],"deployers":["Facebook, Meta"],"developers":["Facebook, Meta"],"harmed":["Rohingya People, Rohingya Facebook Users, Myanmar Public, Facebook Users In Myanmar, Burmese Speaking Facebook Users"],"report_count":5},{"incident_id":229,"occurred_on":"2018-04-23","title":"Content Using Bestiality Thumbnails Allegedly Evaded YouTube’s Thumbnail Monitoring System","description":"YouTube’s thumbnail monitoring system was allegedly evaded by content farms such as ones in Cambodia who spike viewership and generate ad revenue using bestiality-themed thumbnails.","mit_domain":"Discrimination and Toxicity","mit_subdomain":"Exposure to toxic content","entity":"AI","intent":"Unintentional","timing":"Post-deployment","harm_level":null,"sectors":[],"countries":[],"deployers":["Youtube"],"developers":["Youtube"],"harmed":["Youtube Users, Youtube Content Creators"],"report_count":2},{"incident_id":382,"occurred_on":"2017-11-21","title":"Instagram's Exposure of Harmful Content Allegedly Contributed to Teenage Girl's Suicide","description":"Instagram was ruled by a judge to have contributed to the death of a teenage girl in the UK allegedly through its exposure and recommendation of suicide, self-harm, and depressive content.","mit_domain":"Discrimination and Toxicity","mit_subdomain":"Exposure to toxic content","entity":"AI","intent":"Unintentional","timing":"Post-deployment","harm_level":null,"sectors":[],"countries":[],"deployers":["Meta"],"developers":["Meta"],"harmed":["Women and girls","Molly Rose Russell","Girls","Family of Molly Rose Russell","Adolescent girls"],"report_count":1},{"incident_id":58,"occurred_on":"2017-10-12","title":"Russian Chatbot Supports Stalin and Violence","description":"Yandex, a Russian technology company, released an artificially intelligent chat bot named Alice which began to reply to questions with racist, pro-stalin, and pro-violence responses","mit_domain":"Discrimination and Toxicity","mit_subdomain":"Exposure to toxic content","entity":"AI","intent":"Unintentional","timing":"Post-deployment","harm_level":"none","sectors":["information and communication"],"countries":["RU"],"deployers":["Yandex"],"developers":["Yandex"],"harmed":["Yandex Users"],"report_count":5},{"incident_id":216,"occurred_on":"2017-10-10","title":"WeChat’s Machine Translation Gave a Racist English Translation for the Chinese Term for “Black Foreigner”","description":"The Chinese platform WeChat provided an inappropriate and racist English translation for the Chinese term for “black foreigner” in its messaging app.","mit_domain":"Discrimination and Toxicity","mit_subdomain":"Exposure to toxic content","entity":"AI","intent":"Unintentional","timing":"Post-deployment","harm_level":null,"sectors":[],"countries":[],"deployers":["Wechat"],"developers":["Wechat"],"harmed":["Black Wechat Users"],"report_count":5},{"incident_id":325,"occurred_on":"2017-09-21","title":"Offensive Instagram User Content Displayed as Facebook Ad","description":"An Instagram user’s image containing violent content was reportedly used as advertisement on Facebook allegedly via automated means.","mit_domain":"Discrimination and Toxicity","mit_subdomain":"Exposure to toxic content","entity":"AI","intent":"Unintentional","timing":"Post-deployment","harm_level":null,"sectors":[],"countries":[],"deployers":["Facebook"],"developers":["Facebook"],"harmed":["Olivia Solon, Olivia Solon'S Facebook Connections"],"report_count":2},{"incident_id":56,"occurred_on":"2017-07-10","title":"AI-Designed Phone Cases Are Unexpected","description":"A third-party Amazon merchant named “my_handy_design” was suspected of using a bot to generate cell phone case designs based on the bizarre and unattractive designs being offered.","mit_domain":"Discrimination and Toxicity","mit_subdomain":"Exposure to toxic content","entity":"AI","intent":"Unintentional","timing":"Post-deployment","harm_level":"none","sectors":["wholesale and retail trade"],"countries":[],"deployers":["My_Handy_Design"],"developers":["My_Handy_Design"],"harmed":["My_Handy_Design"],"report_count":7},{"incident_id":55,"occurred_on":"2016-12-30","title":"Alexa Plays Pornography Instead of Kids Song","description":"An Amazon Echo Dot using the Amazon Alex software started to play pornographic results when a child asked it to play a song.","mit_domain":"Discrimination and Toxicity","mit_subdomain":"Exposure to toxic content","entity":"AI","intent":"Unintentional","timing":"Post-deployment","harm_level":"none","sectors":["arts, entertainment and recreation","information and communication"],"countries":["US"],"deployers":["Amazon"],"developers":["Amazon"],"harmed":["Minors"],"report_count":16},{"incident_id":6,"occurred_on":"2016-03-24","title":"Microsoft's TayBot Allegedly Posts Racist, Sexist, and Anti-Semitic Content to Twitter","description":"Microsoft's Tay, an artificially intelligent chatbot, was released on March 23, 2016 and removed within 24 hours due to multiple racist, sexist, and anti-semitic tweets generated by the bot.","mit_domain":"Discrimination and Toxicity","mit_subdomain":"Exposure to toxic content","entity":"Human","intent":"Intentional","timing":"Post-deployment","harm_level":"none","sectors":["information and communication"],"countries":[],"deployers":["Microsoft"],"developers":["Microsoft"],"harmed":["X (Twitter) Users"],"report_count":28},{"incident_id":1,"occurred_on":"2015-05-19","title":"Google’s YouTube Kids App Presents Inappropriate Content","description":"YouTube’s content filtering and recommendation algorithms exposed children to disturbing and inappropriate videos.","mit_domain":"Discrimination and Toxicity","mit_subdomain":"Exposure to toxic content","entity":"AI","intent":"Unintentional","timing":"Post-deployment","harm_level":"none","sectors":["arts, entertainment and recreation","information and communication"],"countries":["US"],"deployers":["Youtube"],"developers":["Youtube"],"harmed":["Minors"],"report_count":14},{"incident_id":380,"occurred_on":"2014-03-04","title":"Facebook's Auto-Generated Targeting Ad Categories Contained Anti-Semitic Options","description":"Facebook's automated advertising categories generated using users' declared interests contained anti-Semitic categories such as \"Jew hater\"  and \"How to burn Jews\" which were listed as fields of study.","mit_domain":"Discrimination and Toxicity","mit_subdomain":"Exposure to toxic content","entity":"AI","intent":"Unintentional","timing":"Post-deployment","harm_level":null,"sectors":[],"countries":[],"deployers":["Facebook"],"developers":["Facebook"],"harmed":["Jewish People"],"report_count":5}]}