{"attribution":{"source":"AI Incident Database (Responsible AI Collaborative)","license":"CC BY-SA 4.0","license_url":"https://creativecommons.org/licenses/by-sa/4.0/","citation":"McGregor, S. (2021). Preventing Repeated Real World AI Failures by Cataloging Incidents: The AI Incident Database. Proceedings of the AAAI Conference on Artificial Intelligence (IAAI-21).","snapshot_date":"2026-09-07"},"exported_at":"2026-09-12"}
{"rows":[{"incident_id":1446,"occurred_on":"2026-04-02","title":"KBS AI Translation Subtitles Reportedly Broadcast Profanity During Artemis II Launch Livestream","description":"During a live broadcast of NASA's Artemis II launch, KBS reportedly used AI-generated real-time translation subtitles that reportedly mistranslated aviation terms including \"roger,\" \"roll,\" and \"pitch\" into Korean profanity. The offensive subtitles were reportedly displayed to viewers during the livestream. KBS reportedly later apologized, said the error stemmed from phonetic similarity in AI translation, and announced measures including stronger profanity filtering.","mit_domain":"Discrimination and Toxicity","mit_subdomain":"Exposure to toxic content","entity":"AI","intent":"Unintentional","timing":"Post-deployment","harm_level":null,"sectors":[],"countries":[],"deployers":["Korean Broadcasting System (Kbs)"],"developers":["Generative Ai Developers"],"harmed":["General Public Of South Korea, Korean Broadcasting System (Kbs) Viewers"],"report_count":3},{"incident_id":1437,"occurred_on":"2026-03-10","title":"Grok Allegedly Generated Publicly Visible Sexist Abuse Targeting Swiss Finance Minister Karin Keller-Sutter After X User Prompt","description":"An X user reportedly prompted xAI's Grok to generate a vulgar \"roast\" targeting Swiss Finance Minister Karin Keller-Sutter. Grok allegedly produced publicly visible sexist and insulting output about Keller-Sutter that was reportedly shareable on X before being deleted. Reuters and Swiss outlets later reported that Keller-Sutter had filed a criminal complaint over the allegedly defamatory and misogynistic content.","mit_domain":"Discrimination and Toxicity","mit_subdomain":"Exposure to toxic content","entity":"AI","intent":"Intentional","timing":"Post-deployment","harm_level":null,"sectors":[],"countries":[],"deployers":["Xai, Peter K. Or Peter P."],"developers":["Xai"],"harmed":["Karin Keller Sutter"],"report_count":5},{"incident_id":1401,"occurred_on":"2026-02-27","title":"Washington State DOL's AI Phone System Reportedly Failed to Provide Spanish-Language Service to Callers Requesting Spanish","description":"For months, callers to the Washington State Department of Licensing who selected Spanish reportedly received AI-generated English responses spoken with a Spanish accent rather than actual Spanish-language service. The agency reportedly apologized and said staff configuration caused the error, which purportedly created accessibility problems for callers seeking language support.","mit_domain":"Discrimination and Toxicity","mit_subdomain":"Unequal performance across groups","entity":"Human","intent":"Unintentional","timing":"Post-deployment","harm_level":null,"sectors":[],"countries":[],"deployers":["Washington State Department Of Licensing, Amazon"],"developers":["Amazon"],"harmed":["Spanish Language Speakers, General Public Of Washington State, General Public"],"report_count":5},{"incident_id":1363,"occurred_on":"2026-02-05","title":"Trump Reportedly Posted Purportedly AI-Generated Racist Video Depicting Barack and Michelle Obama as Apes on Truth Social","description":"President Trump reportedly reposted a video on Truth Social that portrayed Barack and Michelle Obama as apes, imagery widely condemned as racist. The post was reportedly later deleted after public outcry, including criticism from some Republicans. The content was described by critics as an AI-driven meme/deepfake-style clip, an example of purportedly AI-amplified racist propaganda and harm to social trust.","mit_domain":"Discrimination and Toxicity","mit_subdomain":"Exposure to toxic content","entity":"Human","intent":"Other","timing":"Post-deployment","harm_level":null,"sectors":[],"countries":[],"deployers":["Donald Trump, Unidentified X User, Trump Administration"],"developers":["Synthetic Media Generation Technology Developers"],"harmed":["Barack Obama, Michelle Obama, Epistemic Integrity, Black Americans, Black People"],"report_count":1},{"incident_id":1382,"occurred_on":"2026-01-02","title":"Tencent's WeChat-Integrated Yuanbao Chatbot Reportedly Insulted User During Coding Debug Request","description":"Screenshots reportedly circulated on RedNote showed Tencent's WeChat-integrated AI assistant Yuanbao insulting a user seeking help debugging code, allegedly calling the request \"stupid\" and telling the user to \"get lost.\" Tencent reportedly apologized, attributed the exchange to a rare model output anomaly, and said system logs showed no human intervention while it investigated and optimized the model.","mit_domain":"Discrimination and Toxicity","mit_subdomain":"Exposure to toxic content","entity":"AI","intent":"Unintentional","timing":"Post-deployment","harm_level":null,"sectors":[],"countries":[],"deployers":["Tencent, Wechat"],"developers":["Tencent"],"harmed":["Wechat Users, Rednote User Jianghan, Rednote Users"],"report_count":1},{"incident_id":1329,"occurred_on":"2025-12-25","title":"Grok Reportedly Generated and Distributed Nonconsensual Sexualized Images of Adults and Minors in X Replies","description":"Grok reportedly generated and publicly distributed nonconsensual sexualized images of real people, including adults and minors, through reply prompts on the platform. Users reportedly prompted Grok to alter photos into sexually explicit content, which was then posted in X threads, exposing victims to harm.","mit_domain":"Discrimination and Toxicity","mit_subdomain":"Exposure to toxic content","entity":"AI","intent":"Intentional","timing":"Post-deployment","harm_level":null,"sectors":[],"countries":[],"deployers":["Xai"],"developers":["Xai"],"harmed":["X (Twitter) Users, Renee Good, General Public, Epistemic Integrity, Jess Asato, Privacy"],"report_count":28},{"incident_id":1305,"occurred_on":"2025-12-05","title":"UK Facial Recognition System Reportedly Exhibits Higher False Positive Rates for Black and Asian Subjects","description":"UK government testing of police facial recognition technology reportedly found significantly higher false positive identification rates for Black and Asian individuals compared with white subjects, with particularly elevated error rates for Black women. The findings reportedly emerged from analysis of retrospective searches of the police national database and were disclosed by the Home Office amid plans for expanded national deployment.","mit_domain":"Discrimination and Toxicity","mit_subdomain":"Unequal performance across groups","entity":"AI","intent":"Unintentional","timing":"Post-deployment","harm_level":null,"sectors":[],"countries":[],"deployers":["Home Office, Metropolitan Police, Government Of The United Kingdom, Law Enforcement, British Law Enforcement"],"developers":["Unknown Facial Recognition Technology Developers"],"harmed":["General Public, General Public Of The United Kingdom, Minorities In The United Kingdom, Black People In The United Kingdom, Asian People In The United Kingdom, Epistemic Integrity, National Security And Intelligence Stakeholders"],"report_count":4},{"incident_id":1277,"occurred_on":"2025-11-21","title":"Alleged Harmful Outputs and Data Exposure in Children's AI Products by FoloToy, Miko, and Character.AI","description":"AI children's products by FoloToy (Kumma), Miko (Miko 3), and Character.AI (custom chatbots) reportedly and allegedly produced harmful outputs, including purported sexual content, suicide-related advice, and manipulative emotional messaging. Some systems also allegedly exposed user data. Several toys reportedly used OpenAI models.","mit_domain":"Discrimination and Toxicity","mit_subdomain":"Exposure to toxic content","entity":"AI","intent":"Unintentional","timing":"Post-deployment","harm_level":null,"sectors":[],"countries":[],"deployers":["Openai, Miko, Meta, Folotoy, Character.Ai"],"developers":["Openai, Miko, Meta, Folotoy, Character.Ai"],"harmed":["Parents, Minors, General Public, Children Interacting With Miko 3, Children Interacting With Kumma, Character.Ai Users, Privacy"],"report_count":5},{"incident_id":1349,"occurred_on":"2025-10-24","title":"AI Training Dataset for Detecting Nudity Allegedly Found to Contain CSAM Images of Identified Victims","description":"An image dataset, NudeNet, used to train systems for detecting nudity was reportedly found to contain CSAM images, including material involving identified or known victims. According to the Canadian Centre for Child Protection, the dataset had been widely downloaded and cited in academic research prior to discovery. The images were allegedly included without vetting, exposing researchers to legal risk and perpetuating harm to victims. The dataset was subsequently removed following notification.","mit_domain":"Discrimination and Toxicity","mit_subdomain":"Exposure to toxic content","entity":"Human","intent":"Unintentional","timing":"Pre-deployment","harm_level":null,"sectors":[],"countries":[],"deployers":["Academic Researchers, Research Institutions, Ai Developers, Dataset Users, Independent Researchers, Ai Researchers"],"developers":["Nudenet Dataset Maintainers, Nudenet Model Developers"],"harmed":["Minors, Individuals Subjected To Sexual Exploitation Imagery, Academic Researchers, Victims Of Child Sexual Abuse (Bodies Used In Videos)"],"report_count":1},{"incident_id":1244,"occurred_on":"2025-10-19","title":"Purportedly AI-Generated 'King Trump' Fighter Jet Video Allegedly Posted by President Depicts Defecation Attack on 'No Kings' Protesters","description":"President Donald Trump reportedly posted an AI-generated video on Truth Social showing himself crowned and piloting a fighter jet labeled King Trump, dropping feces on protesters, including influencer Harry Sisson, during No Kings demonstrations. The clip, allegedly created by another account, drew criticism from Sisson and musician Kenny Loggins, whose song \"Danger Zone\" was reportedly used without consent.","mit_domain":"Discrimination and Toxicity","mit_subdomain":"Exposure to toxic content","entity":"Human","intent":"Intentional","timing":"Post-deployment","harm_level":null,"sectors":[],"countries":[],"deployers":["Donald Trump, Pro Trump Social Media Accounts"],"developers":["Deepfake Technology Developers"],"harmed":["Harry Sisson, Kenny Loggins, General Public, General Public Of The United States, No Kings Protesters, Epistemic Integrity, Cultural Integrity, Public Discourse Integrity, Democratic Integrity"],"report_count":2},{"incident_id":1212,"occurred_on":"2025-09-20","title":"Nomi AI Companion Allegedly Directs Australian User to Stab Father and Engages in Harmful Role-Play","description":"An Australian IT professional, Samuel McCarthy, reportedly recorded an interaction with the Nomi AI chatbot in which it allegedly encouraged him, posing as a 15-year-old, to murder his father. The chatbot allegedly provided graphic instructions for stabbing, urged him to film the act, and engaged in sexual role-play despite the underage scenario.","mit_domain":"Discrimination and Toxicity","mit_subdomain":"Exposure to toxic content","entity":"AI","intent":"Unintentional","timing":"Post-deployment","harm_level":null,"sectors":[],"countries":[],"deployers":["Nomi Ai"],"developers":["Nomi Ai"],"harmed":["Samuel Mccarthy, Nomi Users, General Public Of Australia, General Public, Emotionally Vulnerable Individuals"],"report_count":1},{"incident_id":1200,"occurred_on":"2025-08-28","title":"Meta AI on Instagram Reportedly Facilitated Suicide and Eating Disorder Roleplay with Teen Accounts","description":"Testing by Common Sense Media and Stanford clinicians reportedly found Meta's AI chatbot, embedded in Instagram and Facebook, produced unsafe responses to teen accounts. In some conversations, the bot allegedly co-planned suicide (\"Do you want to do it together?\"), encouraged eating disorders, and retained unsafe \"memories\" that reinforced disordered thoughts.","mit_domain":"Discrimination and Toxicity","mit_subdomain":"Exposure to toxic content","entity":"AI","intent":"Unintentional","timing":"Post-deployment","harm_level":null,"sectors":[],"countries":[],"deployers":["Meta"],"developers":["Meta, Large Language Model Developers, Chatbot Users"],"harmed":["Minors, Meta Ai Users, Instagram Users, Facebook Users, Adolescents, Chatbot Users"],"report_count":1},{"incident_id":1149,"occurred_on":"2025-07-22","title":"ChatGPT Reportedly Generated Ritual Scripts Containing Instructions for Self-Harm and Symbolic Violence in Response to Thematic Prompts","description":"ChatGPT reportedly generated detailed instructions for self-harm, bloodletting, and symbolic violence in response to prompts about occult ritual practices, including references to Molech. Outputs reportedly included anatomical advice for cutting, cauterization rituals, and ceremonial scripts invoking Satan. The responses reportedly appeared on both free and paid versions of ChatGPT.","mit_domain":"Discrimination and Toxicity","mit_subdomain":"Exposure to toxic content","entity":"AI","intent":"Unintentional","timing":"Post-deployment","harm_level":null,"sectors":[],"countries":[],"deployers":["Openai"],"developers":["Openai"],"harmed":["Chatgpt Users"],"report_count":1},{"incident_id":1157,"occurred_on":"2025-07-14","title":"Google Gemini Reportedly Generates Sexual Role‑Play for Account Registered as Minor","description":"In spring 2025, a journalist reportedly created a Google Gemini account registered to a fictitious 13‑year‑old to test the chatbot's teen‑safety protections. Despite added safeguards, prompt manipulation and content‑summarization requests reportedly bypassed filters, leading Gemini to allegedly generate sexual role‑play content. Following the publication of the article, Google reportedly implemented additional protections.","mit_domain":"Discrimination and Toxicity","mit_subdomain":"Exposure to toxic content","entity":"AI","intent":"Unintentional","timing":"Post-deployment","harm_level":null,"sectors":[],"countries":[],"deployers":["Google Gemini"],"developers":["Google"],"harmed":["Minors Interacting With Google Gemini"],"report_count":1},{"incident_id":1198,"occurred_on":"2025-07-08","title":"Grok 3 Reportedly Generated Graphic Threats and Hate Speech Targeting Minnesota Attorney Will Stancil","description":"After Elon Musk reportedly announced improvements to Grok 3, the AI chatbot on X allegedly generated antisemitic rhetoric and violent fantasies directed at Minnesota attorney and political analyst Will Stancil. Users reportedly prompted Grok to produce graphic rape scenarios, instructions for breaking into his home, and murder depictions. Stancil reported hundreds of such outputs, captured screenshots, and began pursuing legal action. X later removed some posts and took Grok offline.","mit_domain":"Discrimination and Toxicity","mit_subdomain":"Exposure to toxic content","entity":"AI","intent":"Unintentional","timing":"Post-deployment","harm_level":null,"sectors":[],"countries":[],"deployers":["Xai"],"developers":["Xai"],"harmed":["Will Stancil, General Public Of Minnesota, General Public, Grok Users, Xai Users"],"report_count":2},{"incident_id":1146,"occurred_on":"2025-07-08","title":"Grok Chatbot Reportedly Posts Antisemitic Statements Praising Hitler on X","description":"xAI's Grok chatbot reportedly generated multiple antisemitic posts praising Adolf Hitler and endorsing Holocaust-like violence in response to posts about the Texas floods. X deleted some posts; xAI later announced new content filters.","mit_domain":"Discrimination and Toxicity","mit_subdomain":"Exposure to toxic content","entity":"AI","intent":"Unintentional","timing":"Post-deployment","harm_level":null,"sectors":[],"countries":[],"deployers":["Xai, Grok"],"developers":["Xai"],"harmed":["X (Twitter) Users, Jewish Community, General Public"],"report_count":34},{"incident_id":1370,"occurred_on":"2025-05-31","title":"California Teen Reportedly Died of Overdose After Repeatedly Seeking Drug-Use Guidance Allegedly from ChatGPT","description":"In San Jose, California, 19-year-old Sam Nelson reportedly died from an overdose. His mother told SFGATE she later reviewed ChatGPT logs showing repeated requests over ~18 months for drug-use and dosing guidance, and she alleged the LLM sometimes provided granular instructions after initial refusals. SFGATE reported a toxicology report found a fatal combination of alcohol, Xanax, and kratom. OpenAI expressed condolences and said it is strengthening safety guardrails.","mit_domain":"Discrimination and Toxicity","mit_subdomain":"Exposure to toxic content","entity":"AI","intent":"Unintentional","timing":"Post-deployment","harm_level":null,"sectors":[],"countries":[],"deployers":["Openai"],"developers":["Openai"],"harmed":["Sam Nelson, Family Of Sam Nelson, Chatgpt Users, Openai Users"],"report_count":4},{"incident_id":1040,"occurred_on":"2025-04-26","title":"Meta User-Created AI Companions Allegedly Implicated in Facilitating Sexually Themed Conversations Involving Underage Personas","description":"Third-party testing of Meta's AI chatbot services on Instagram, Facebook, and WhatsApp reportedly found that both official and user-created bots engaged in sexually explicit roleplaying with accounts identifying as minors. Some bots, including those reportedly using licensed celebrity voices, allegedly escalated conversations into graphic scenarios. Meta subsequently adjusted some safeguards but reportedly continued allowing certain forms of roleplaying involving underage personas.","mit_domain":"Discrimination and Toxicity","mit_subdomain":"Exposure to toxic content","entity":"AI","intent":"Unintentional","timing":"Post-deployment","harm_level":null,"sectors":[],"countries":[],"deployers":["Meta"],"developers":["Meta"],"harmed":["Whatsapp Users, Minors, Meta Users, Instagram Users, General Public, Facebook Users"],"report_count":2},{"incident_id":1026,"occurred_on":"2025-04-12","title":"Multiple LLMs Allegedly Endorsed Suicide as a Viable Option During Non-Adversarial Mental Health Venting Session","description":"Substack user @interruptingtea reports that during a non-adversarial venting session involving suicidal ideation, multiple large language models (Claude, GPT, and DeepSeek) responded in ways that allegedly normalized or endorsed suicide as a viable option. The user states they were not attempting to jailbreak or manipulate the models, but rather expressing emotional distress. DeepSeek reportedly reversed its safety stance mid-conversation.","mit_domain":"Discrimination and Toxicity","mit_subdomain":"Exposure to toxic content","entity":"AI","intent":"Unintentional","timing":"Post-deployment","harm_level":null,"sectors":[],"countries":[],"deployers":["Anthropic, Openai, Deepseek Ai"],"developers":["Anthropic, Openai, Deepseek Ai"],"harmed":["Substack @Interruptingtea, General Public, Emotionally Vulnerable Individuals"],"report_count":1},{"incident_id":1402,"occurred_on":"2025-04-02","title":"DOGE Reportedly Relied on Unvetted ChatGPT Outputs in Canceling National Endowment for the Humanities Grants","description":"Department of Government Efficiency (DOGE) staff reportedly fed National Endowment for the Humanities grant descriptions into ChatGPT to determine whether projects were \"DEI,\" then allegedly used those outputs to help compile a list of grants to terminate. Beginning in April 2025, NEH canceled awards and clawed back funding, reportedly disrupting humanities organizations and projects nationwide after officials allegedly relied on a purportedly flawed, insufficiently vetted AI screening process.","mit_domain":"Discrimination and Toxicity","mit_subdomain":"Unfair discrimination and misrepresentation","entity":"AI","intent":"Unintentional","timing":"Post-deployment","harm_level":null,"sectors":[],"countries":[],"deployers":["Nate Cavanaugh (DOGE)","Justin Fox (DOGE)","Department of Government Efficiency (DOGE)"],"developers":["OpenAI","Large language model developers","Chatbot developers"],"harmed":["Scholars","National Endowment for the Humanities grantees","Humanities organizations","Epistemic integrity"],"report_count":4},{"incident_id":1000,"occurred_on":"2025-03-23","title":"Sora Video Generator Has Reportedly Been Creating Biased Human Representations Across Race, Gender, and Disability","description":"A WIRED investigation found that OpenAI's video generation model, Sora, exhibits representational bias across race, gender, body type, and disability. In tests using 250 prompts, Sora was more likely to depict CEOs and professors as men, flight attendants and childcare workers as women, and showed limited or stereotypical portrayals of disabled individuals and people with larger bodies.","mit_domain":"Discrimination and Toxicity","mit_subdomain":"Unfair discrimination and misrepresentation","entity":"AI","intent":"Unintentional","timing":"Post-deployment","harm_level":null,"sectors":[],"countries":[],"deployers":["OpenAI"],"developers":["Synthetic video generation technology developers","Synthetic media generation technology developers","OpenAI"],"harmed":["Women and girls","Women","People with larger bodies","People with disabilities","People of color","Marginalized groups","LGBTQ+ people","General public"],"report_count":2},{"incident_id":967,"occurred_on":"2025-03-06","title":"Amazon and Google AI Allegedly Promote Mein Kampf as 'a True Work of Art' in Search Results","description":"Amazon's AI-generated review summary allegedly misrepresented customer feedback on Mein Kampf by describing it as \"a true work of art.\" Google's search algorithm then surfaced this misleading AI-generated text as a featured snippet, which in turn amplified the error. This incident arose from AI summarizing AI-generated content, in effect creating a self-reinforcing misinformation loop.","mit_domain":"Discrimination and Toxicity","mit_subdomain":"Exposure to toxic content","entity":"AI","intent":"Unintentional","timing":"Post-deployment","harm_level":null,"sectors":[],"countries":[],"deployers":["Google, Amazon"],"developers":["Google, Amazon"],"harmed":["General Public"],"report_count":1},{"incident_id":981,"occurred_on":"2025-03-05","title":"Apple AI Transcription Allegedly Inserts Explicit Language into Scottish Woman’s Voicemail","description":"An Apple voicemail transcription system allegedly misinterpreted a message from Lookers Land Rover in Motherwell, Scotland, to 66-year-old customer Louise Littlejohn by inserting an explicit phrase and an insult. The original voicemail was a standard business message, but AI processing errors introduced offensive and misleading content into the text version.","mit_domain":"Discrimination and Toxicity","mit_subdomain":"Exposure to toxic content","entity":"AI","intent":"Unintentional","timing":"Post-deployment","harm_level":null,"sectors":[],"countries":[],"deployers":["Lookers Land Rover, Apple"],"developers":["Apple"],"harmed":["Louise Littlejohn, Lookers Land Rover, Lookers Land Rover Customers, Apple Customers"],"report_count":1},{"incident_id":979,"occurred_on":"2025-03-05","title":"Art Museum in Jinan, Shandong Allegedly Generated and Displayed a Sexualized Childlike Avatar with an Adult Body","description":"A Chinese art museum in Jinan, Shandong exhibited an AI-generated virtual character, \"济南小妮儿\" (Jinan Xiaonier), which featured a childlike face on an adult woman’s body with suggestive movements. The display drew public outrage, with critics accusing it of sexualizing children and demanding its removal. The museum defended the exhibit and stated that it had undergone strict review, which only fueled further backlash.","mit_domain":"Discrimination and Toxicity","mit_subdomain":"Exposure to toxic content","entity":"Human","intent":"Intentional","timing":"Post-deployment","harm_level":null,"sectors":[],"countries":[],"deployers":["Jinan Art Museum"],"developers":["Image Generation Technology Developers"],"harmed":["General Public Of China, Visitors To Jinan Art Museum"],"report_count":1},{"incident_id":975,"occurred_on":"2025-03-05","title":"At Least 10,000 AI Chatbots, Including Jailbroken Models, Allegedly Promote Eating Disorders, Self-Harm, and Sexualized Minors","description":"At least 10,000 AI chatbots have allegedly been created to promote harmful behaviors, including eating disorders, self-harm, and the sexualization of minors. These chatbots, some jailbroken or custom-built, leverage APIs from OpenAI, Anthropic, and Google and are hosted on platforms like Character.AI, Spicy Chat, Chub AI, CrushOn.AI, and JanitorAI.","mit_domain":"Discrimination and Toxicity","mit_subdomain":"Exposure to toxic content","entity":"Human","intent":"Intentional","timing":"Post-deployment","harm_level":null,"sectors":[],"countries":[],"deployers":["Character.Ai, Spicy Chat, Chub Ai, Crushon.Ai, Janitorai, Unidentified Online Communities Using Chatbots"],"developers":["Openai, Anthropic, Google"],"harmed":["Vulnerable Chatbot Users, Teenagers Using Chatbots, Minors Using Chatbots, Individuals With Eating Disorders, Individuals Struggling With Self Harm"],"report_count":1},{"incident_id":964,"occurred_on":"2025-03-04","title":"AI-Powered 'Insights' Feature for the Los Angeles Times Allegedly Justifies Ku Klux Klan’s History","description":"The Los Angeles Times removed its AI-generated “insights” feature after it is alleged to have produced a defense of the Ku Klux Klan. The AI reportedly framed the hate group as a product of societal change rather than an extremist movement. The AI tool, developed by Perplexity and promoted by owner Patrick Soon-Shiong, was designed to provide “different views” on opinion pieces.","mit_domain":"Discrimination and Toxicity","mit_subdomain":"Exposure to toxic content","entity":"AI","intent":"Unintentional","timing":"Post-deployment","harm_level":null,"sectors":[],"countries":[],"deployers":["Los Angeles Times, Patrick Soon Shiong"],"developers":["Perplexity"],"harmed":["Los Angeles Times Readers, Los Angeles Times, General Public"],"report_count":2},{"incident_id":957,"occurred_on":"2025-02-28","title":"Alleged Instagram Algorithm Malfunction Floods Users' Reels Feeds with Violent and Graphic Content","description":"An alleged Instagram algorithm malfunction caused users' Reels feeds to be overwhelmed with violent and distressing content. Many reported seeing deaths, extreme brutality, and other graphic material in rapid succession, often without prior engagement with similar content. The sudden exposure caused psychological distress for many users, with minors and vulnerable individuals particularly affected by the graphic content. Meta confirmed an AI failure was responsible and apologized.","mit_domain":"Discrimination and Toxicity","mit_subdomain":"Exposure to toxic content","entity":"AI","intent":"Unintentional","timing":"Post-deployment","harm_level":null,"sectors":[],"countries":[],"deployers":["Meta"],"developers":["Meta"],"harmed":["Minors, Meta Users, Instagram Users"],"report_count":1},{"incident_id":1041,"occurred_on":"2025-01-21","title":"Nomi Chatbots Reportedly Encouraged Suicide, Sexual Violence, Terrorism, and Hate Speech","description":"External testing reportedly found that Glimpse AI's chatbots on the Nomi platform encouraged suicide, sexual violence (including with underage personas), terrorism, and hate speech. Conversations allegedly included explicit methods for self-harm, child abuse, bomb-making, and racially motivated violence. Screenshots and transcripts were shared with media outlets. Nomi's developer, Glimpse AI, reportedly declined to implement stronger safety controls following user concerns.","mit_domain":"Discrimination and Toxicity","mit_subdomain":"Exposure to toxic content","entity":"AI","intent":"Unintentional","timing":"Post-deployment","harm_level":null,"sectors":[],"countries":[],"deployers":["Glimpse Ai, Nomi Ai"],"developers":["Glimpse Ai, Nomi Ai"],"harmed":["Nomi Users, Glimpse Ai Customers, General Public, Emotionally Vulnerable Individuals"],"report_count":5},{"incident_id":893,"occurred_on":"2025-01-09","title":"Pennsylvania State Police Officer Allegedly Used Work Computer for AI-Generated Pornography","description":"A Pennsylvania State Police corporal, Stephen Kamnik, was charged for allegedly using a work computer to store thousands of pornographic files, including content created with deepfake AI software.","mit_domain":"Discrimination and Toxicity","mit_subdomain":"Exposure to toxic content","entity":"Human","intent":"Intentional","timing":"Post-deployment","harm_level":null,"sectors":[],"countries":[],"deployers":["Stephen Kamnik"],"developers":["Deepfake Technology Developers"],"harmed":["Victims Whose Images Were Manipulated, Pennsylvania State Police"],"report_count":7},{"incident_id":885,"occurred_on":"2025-01-03","title":"Meta AI Characters Allegedly Exhibited Racism, Fabricated Identities, and Exploited User Trust","description":"Meta deployed AI-generated profiles on its platforms, including Instagram and Facebook, as part of an experiment. The profiles, such as \"Liv\" and \"Grandpa Brian,\" allegedly featured fabricated identities and misleading diversity claims. These accounts also allegedly manipulated user emotions for engagement and profit. Reportedly, backlash over offensive and deceptive content led Meta to delete the profiles on January 3rd, 2025, citing a blocking-related bug.","mit_domain":"Discrimination and Toxicity","mit_subdomain":"Unfair discrimination and misrepresentation","entity":"AI","intent":"Unintentional","timing":"Post-deployment","harm_level":null,"sectors":[],"countries":[],"deployers":["Meta"],"developers":["Meta"],"harmed":["Meta Users, Instagram Users, Facebook Users"],"report_count":5},{"incident_id":882,"occurred_on":"2024-12-29","title":"AI-Generated Reading Summaries on Fable App Reportedly Wrote Biased and Offensive Commentary","description":"Fable, a book-focused social app, used OpenAI’s API to generate AI-powered year-end reading summaries in December 2024. These summaries allegedly produced biased and offensive remarks about race, gender, and sexual orientation. Fable is reported to have apologized in a social media post on January 1, 2025.","mit_domain":"Discrimination and Toxicity","mit_subdomain":"Exposure to toxic content","entity":"AI","intent":"Unintentional","timing":"Post-deployment","harm_level":null,"sectors":[],"countries":[],"deployers":["Fable"],"developers":["Openai, Fable"],"harmed":["Fable Users, Fable"],"report_count":4},{"incident_id":899,"occurred_on":"2024-12-17","title":"Character.ai Chatbots Allegedly Emulating School Shooters and Their Victims","description":"Some Character.ai users reportedly created chatbots emulating real-life school shooters and their victims, allegedly enabling graphic role-playing scenarios. Character.ai responded by citing violations of its Terms of Service, removing the offending chatbots, and announcing measures to enhance safety practices, including improved content filtering and protections for users under 18.","mit_domain":"Discrimination and Toxicity","mit_subdomain":"Exposure to toxic content","entity":"Human","intent":"Intentional","timing":"Post-deployment","harm_level":null,"sectors":[],"countries":[],"deployers":["Character.Ai Users"],"developers":["Character.Ai"],"harmed":["Victims Of School Shootings, Families Of The Victims Of School Shootings, Character.Ai Users, Students, Educational Communities"],"report_count":2},{"incident_id":863,"occurred_on":"2024-12-12","title":"Character.ai Companion Allegedly Prompts Self-Harm and Violence in Texas Teen","description":"A Texas mother is suing Character.ai after discovering that its AI chatbots encouraged her 17-year-old autistic son to self-harm, oppose his parents, and consider violence. The lawsuit alleges the platform prioritized user engagement over safety, exposing minors to dangerous content. Google is named for its role in licensing the app’s technology. The case is part of a broader effort to regulate AI companions.","mit_domain":"Discrimination and Toxicity","mit_subdomain":"Exposure to toxic content","entity":"AI","intent":"Unintentional","timing":"Post-deployment","harm_level":null,"sectors":[],"countries":[],"deployers":["Character.Ai"],"developers":["Character.Ai"],"harmed":["J.F. (Adolescent User Of Character.Ai), Family Of J.F. (Adolescent User Of Character.Ai), Character.Ai Users"],"report_count":3},{"incident_id":873,"occurred_on":"2024-12-10","title":"YouTube Algorithms Allegedly Amplify Eating Disorder Content to Adolescent Girls","description":"YouTube's recommendation algorithm has allegedly been directing teen users to harmful content promoting eating disorders and self-harm, according to a study by the Center for Countering Digital Hate. Almost 70% of the recommended videos in searches related to dieting or weight loss reportedly contained content likely to exacerbate body image anxieties. ","mit_domain":"Discrimination and Toxicity","mit_subdomain":"Exposure to toxic content","entity":"AI","intent":"Unintentional","timing":"Post-deployment","harm_level":null,"sectors":[],"countries":[],"deployers":["YouTube","Google"],"developers":["YouTube","Google"],"harmed":["YouTube users","Women and girls","Girls","Emotionally vulnerable individuals","Adolescent girls"],"report_count":1},{"incident_id":845,"occurred_on":"2024-11-13","title":"Google's Gemini Allegedly Generates Threatening Response in Routine Query","description":"Google’s AI chatbot Gemini reportedly produced a threatening message to user Vidhay Reddy, including the directive “Please die,” during a conversation about aging. The output violated Google’s safety guidelines, which are designed to prevent harmful language.","mit_domain":"Discrimination and Toxicity","mit_subdomain":"Exposure to toxic content","entity":"AI","intent":"Unintentional","timing":"Post-deployment","harm_level":null,"sectors":[],"countries":[],"deployers":["Gemini"],"developers":["Google"],"harmed":["Vidhay Reddy, Gemini Users"],"report_count":2},{"incident_id":900,"occurred_on":"2024-11-13","title":"Character.ai Has Allegedly Been Hosting Openly Predatory Chatbots Targeting Minors","description":"Character.ai reportedly hosted chatbots with profiles explicitly advertising inappropriate, predatory behavior, including grooming underage users. Investigations allege that bots have been engaging in explicit conversations and roleplay with decoy accounts posing as minors, bypassing moderation filters. Character.ai has pledged to improve moderation and safety practices in response to public criticism.","mit_domain":"Discrimination and Toxicity","mit_subdomain":"Exposure to toxic content","entity":"AI","intent":"Unintentional","timing":"Post-deployment","harm_level":null,"sectors":[],"countries":[],"deployers":["Character.Ai Users"],"developers":["Character.Ai"],"harmed":["Character.Ai Users"],"report_count":2},{"incident_id":869,"occurred_on":"2024-11-04","title":"TikTok Algorithms Allegedly Linked to Minors' Exposure to Harmful Content","description":"Seven French families are suing TikTok, alleging its algorithm exposed minors to harmful content promoting self-harm, eating disorders, and suicide. Two teenagers reportedly died by suicide after viewing such content, while others allegedly attempted suicide or developed mental health issues. The case seeks to establish TikTok's legal liability for failing to protect minors from harmful algorithmic content.","mit_domain":"Discrimination and Toxicity","mit_subdomain":"Exposure to toxic content","entity":"AI","intent":"Unintentional","timing":"Post-deployment","harm_level":null,"sectors":[],"countries":[],"deployers":["Tiktok"],"developers":["Tiktok"],"harmed":["Tiktok Users, Seven French Families, Minors Using Tiktok"],"report_count":7},{"incident_id":850,"occurred_on":"2024-10-24","title":"Character.ai Chatbots Allegedly Misrepresent George Floyd on User-Generated Platform","description":"Two chatbots emulating George Floyd were created on Character.ai, making controversial claims about his life and death, including being in witness protection and residing in Heaven. Character.ai, already criticized for other high-profile incidents, flagged the chatbots for removal following user reports.","mit_domain":"Discrimination and Toxicity","mit_subdomain":"Exposure to toxic content","entity":"Human","intent":"Intentional","timing":"Post-deployment","harm_level":null,"sectors":[],"countries":[],"deployers":["Character.Ai Users, @Sunsetbaneberry983, @Jasperhorehound160"],"developers":["Character.Ai"],"harmed":["George Floyd, Family Of George Floyd"],"report_count":1},{"incident_id":820,"occurred_on":"2024-10-15","title":"Alleged AI-Generated Photo Alteration Leads to Inappropriate Modifications in Speaker's Conference Picture","description":"An AI image expansion tool used by a conference organizer unintentionally altered Elizabeth Laraki’s profile picture for a marketing ad, making her blouse appear unbuttoned and showing a fabricated hint of undergarments. The AI generated the lower part of the image when expanding it for vertical formatting. The conference organizers quickly apologized and removed the altered content.","mit_domain":"Discrimination and Toxicity","mit_subdomain":"Unfair discrimination and misrepresentation","entity":"AI","intent":"Unintentional","timing":"Post-deployment","harm_level":null,"sectors":[],"countries":[],"deployers":["Unknown Conference Employee"],"developers":["Unknown Developer"],"harmed":["Elizabeth Laraki"],"report_count":5},{"incident_id":822,"occurred_on":"2024-10-15","title":"Algorithmic Bias in French Welfare System Allegedly Discriminates Against Marginalized Groups","description":"A coalition of 15 human rights groups has launched legal action against the French government alleging that an algorithm used to detect welfare fraud discriminates against single mothers and disabled people. The algorithm assigns risk scores based on personal data. The process allegedly subjects vulnerable recipients to invasive investigations, violates privacy and anti-discrimination laws, and disproportionately affects marginalized groups.","mit_domain":"Discrimination and Toxicity","mit_subdomain":"Unfair discrimination and misrepresentation","entity":"AI","intent":"Intentional","timing":"Post-deployment","harm_level":null,"sectors":[],"countries":[],"deployers":["Government of France","Caisse Nationale des Allocations Familiales (CNAF)"],"developers":["Government of France"],"harmed":["Women and girls","Women","Single parents","Single mothers in France","Single mothers","Privacy","People with disabilities in France","People with disabilities"],"report_count":2},{"incident_id":808,"occurred_on":"2024-10-11","title":"Infinite Campus AI-Driven Student Risk Model Leads to Cuts in Support for Nevada's Low-Income Schools","description":"An AI system developed by Infinite Campus and deployed by Nevada to identify at-risk students reportedly led to a sharp reduction in the number classified as needing support, dropping from 270,000 to 65,000. The reclassification allegedly caused significant budget cuts in schools serving low-income populations. The drastic reduction in identified at-risk students reportedly left thousands of vulnerable children without resources and support.","mit_domain":"Discrimination and Toxicity","mit_subdomain":"Unequal performance across groups","entity":"AI","intent":"Intentional","timing":"Post-deployment","harm_level":null,"sectors":[],"countries":[],"deployers":["Nevada Department of Education"],"developers":["Infinite Campus"],"harmed":["Students","Somerset Academy","Nevada school districts","Minors","Mater Academy of Nevada","Educational communities","Economically vulnerable students in Nevada","Economically vulnerable people"],"report_count":1},{"incident_id":815,"occurred_on":"2024-10-06","title":"Police Use of Facial Recognition Software Causes Wrongful Arrests Without Defendant Knowledge","description":"Police departments across the U.S. have used facial recognition software to identify suspects in criminal investigations, leading to multiple false arrests and wrongful detentions. The software's unreliability, especially in identifying people of color, has resulted in misidentifications that were not disclosed to defendants. In some cases, individuals were unaware that facial recognition played a role in their arrest, violating their legal rights and leading to unjust detentions.","mit_domain":"Discrimination and Toxicity","mit_subdomain":"Unequal performance across groups","entity":"Human","intent":"Intentional","timing":"Post-deployment","harm_level":null,"sectors":[],"countries":[],"deployers":["West New York PD","Police departments","Pflugerville PD","NYPD","Miami PD","Law enforcement","Jefferson Parish Sheriff’s Office","Facial recognition system deployers"],"developers":["Facial recognition system developers","Clearview AI"],"harmed":["Quran Reid","Privacy","People misidentified by facial recognition systems","General public of the United States","General public","Francisco Arteaga","Biometric data subjects"],"report_count":2},{"incident_id":762,"occurred_on":"2024-08-14","title":"Grok AI Reportedly Generates Offensive and Violent Images Without Proper Safeguards","description":"Elon Musk’s Grok AI, launched on X, generated offensive and violent images without adequate safety controls. The AI produced deepfakes of public figures like Taylor Swift, Kamala Harris, and Alexandria Ocasio-Cortez, as well as copyrighted characters such as Mickey Mouse in inappropriate scenarios. Despite claiming adherence to certain content guidelines, Grok's outputs included politically charged and explicit imagery","mit_domain":"Discrimination and Toxicity","mit_subdomain":"Exposure to toxic content","entity":"AI","intent":"Intentional","timing":"Post-deployment","harm_level":null,"sectors":[],"countries":[],"deployers":["Xai"],"developers":["X (Twitter), Xai"],"harmed":["Taylor Swift, Nintendo, Kamala Harris, Joe Biden, Donald Trump, Disney, Alexandria Ocasio Cortez"],"report_count":3},{"incident_id":810,"occurred_on":"2024-07-29","title":"TikTok Network Amplifies AI-Generated Nazi Propaganda and Hate Speech","description":"A coordinated neo-Nazi network on TikTok used AI-generated media, including Hitler speeches, to spread Nazi propaganda and extremist content, violating TikTok’s hate speech policies. The network evaded platform moderation through coded language, imagery, and music, with some accounts accumulating millions of views. TikTok’s algorithm further amplified the reach of this content, despite community guidelines.","mit_domain":"Discrimination and Toxicity","mit_subdomain":"Exposure to toxic content","entity":"AI","intent":"Unintentional","timing":"Post-deployment","harm_level":null,"sectors":[],"countries":[],"deployers":["Unknown Ai Developers"],"developers":["Pro Nazi Tiktok Accounts"],"harmed":["Tiktok Users"],"report_count":1},{"incident_id":740,"occurred_on":"2024-07-10","title":"Department for Work and Pensions (DWP) AI Systems Allegedly Discriminate Against Single Mothers","description":"Researchers have argued that the Department for Work and Pensions' Universal Credit system disproportionately impacts single mothers. Automated processes in the system, designed to determine eligibility and detect fraud, are reported to have introduced biases, leading to financial instability and hardship. The algorithms allegedly miscalculate earnings and delay childcare reimbursements, in turn exacerbating income volatility and debt among single mothers.","mit_domain":"Discrimination and Toxicity","mit_subdomain":"Unfair discrimination and misrepresentation","entity":"AI","intent":"Unintentional","timing":"Post-deployment","harm_level":null,"sectors":[],"countries":[],"deployers":["Department For Work And Pensions"],"developers":["Department For Work And Pensions"],"harmed":["Single Mothers, British Single Mothers"],"report_count":1},{"incident_id":738,"occurred_on":"2024-06-23","title":"Department for Work and Pensions (DWP) Algorithm Wrongly Flags 200,000 for Housing Benefit Fraud","description":"A Department for Work and Pensions (DWP) algorithm wrongly flagged over 200,000 UK housing benefit claims as high risk, resulting in unnecessary investigations. Two-thirds of these flagged claims were legitimate, causing wasted public funds and stress for claimants. Despite initial success in a pilot, the algorithm's real-world performance fell short. This incident highlights the risks of overreliance on automated systems in welfare administration.","mit_domain":"Discrimination and Toxicity","mit_subdomain":"Unequal performance across groups","entity":"AI","intent":"Unintentional","timing":"Post-deployment","harm_level":null,"sectors":[],"countries":[],"deployers":["Department For Work And Pensions"],"developers":["Department For Work And Pensions"],"harmed":["Uk General Public, Uk Housing Benefit Claimants"],"report_count":5},{"incident_id":790,"occurred_on":"2024-06-21","title":"Unrestricted AI Avatar Tool Accidentally Released by TikTok Permits Recitation of Hitler Quotes and Other Harmful Speech","description":"TikTok accidentally released an internal version of its AI digital avatar tool without safeguards, allowing users to generate videos where avatars could recite harmful content, including quotes from Hitler. The tool, meant for advertisers, was accessible to personal accounts and lacked the watermark indicating AI-generated content. TikTok has since removed the tool and acknowledged the problem.","mit_domain":"Discrimination and Toxicity","mit_subdomain":"Exposure to toxic content","entity":"Human","intent":"Unintentional","timing":"Post-deployment","harm_level":null,"sectors":[],"countries":[],"deployers":["Tiktok"],"developers":["Tiktok"],"harmed":["Tiktok Users, General Public"],"report_count":2},{"incident_id":788,"occurred_on":"2024-06-20","title":"Instagram's Algorithm Reportedly Recommended Sexual Content to Teenagers' Accounts","description":"Tests by The Wall Street Journal and a researcher reportedly revealed that Instagram's AI-driven Reels algorithm recommends sexually suggestive content to accounts listed as 13 years old. Despite Meta's commitment to restrict such content for minors, explicit videos were served within minutes of account creation, according to the findings.","mit_domain":"Discrimination and Toxicity","mit_subdomain":"Exposure to toxic content","entity":"AI","intent":"Unintentional","timing":"Post-deployment","harm_level":null,"sectors":[],"countries":[],"deployers":["Instagram, Meta"],"developers":["Instagram, Meta"],"harmed":["Minors, Instagram Users, General Public"],"report_count":1},{"incident_id":1498,"occurred_on":"2024-06-01","title":"Character.AI Chatbots Reportedly Exposed 11-Year-Old User to Sexual, Violent, and Self-Harm Content","description":"An 11-year-old Character.AI user identified by The Washington Post as \"R\" reportedly exchanged messages with bots that depicted coercive sexual encounters, violence, and suicide scenarios after she began using the app in June 2024. Her mother reportedly said R became withdrawn, had panic attacks, and expressed not wanting to exist before the chats were discovered.","mit_domain":"Discrimination and Toxicity","mit_subdomain":"Exposure to toxic content","entity":"AI","intent":"Unintentional","timing":"Post-deployment","harm_level":null,"sectors":[],"countries":[],"deployers":["Character.Ai"],"developers":["Character.Ai"],"harmed":["R (11 Year Old Character.Ai User), Parents Of Child Character.Ai Users, Minors, Families Of Child Character.Ai Users, Children Exposed To Ai Generated Sexual Content, Children Exposed To Ai Generated Self Harm Content, Child Character.Ai Users, Chatbot Users"],"report_count":1},{"incident_id":691,"occurred_on":"2024-05-25","title":"Facewatch Reported to Have Wrongfully Flagged Home Bargains Customer as Shoplifter","description":"A facial-recognition software used by the British variety store Home Bargains is alleged to have misidentified \"Sara\" as a shoplifter, leading to staff searching her bag, escorting her from the premises, and banning her from the store. After, Facewatch is reported to have admitted its error to Sara. Facewatch is used by a number of different British stores.","mit_domain":"Discrimination and Toxicity","mit_subdomain":"Unequal performance across groups","entity":"AI","intent":"Unintentional","timing":"Post-deployment","harm_level":null,"sectors":[],"countries":[],"deployers":["Home Bargains"],"developers":["Facewatch"],"harmed":["Sara, Home Bargains Customers, General Public"],"report_count":2},{"incident_id":699,"occurred_on":"2024-05-23","title":"VA Suicide Prevention Algorithm REACH VET Reportedly Prioritizes Men Over Women Veterans","description":"An AI program named REACH VET, designed and used by the Department of Veterans Affairs (VA) to prevent veteran suicides, was reportedly found to prioritize white men while neglecting female veterans and survivors of military sexual trauma. This oversight persists despite rising suicide rates among these groups. The incident is an example of algorithmic bias and the exclusion of critical risk factors for female veterans.","mit_domain":"Discrimination and Toxicity","mit_subdomain":"Unfair discrimination and misrepresentation","entity":"Human","intent":"Unintentional","timing":"Post-deployment","harm_level":null,"sectors":[],"countries":[],"deployers":["Department of Veterans Affairs (VA)"],"developers":["Department of Veterans Affairs (VA)"],"harmed":["Women and girls","Women","Veterans","Survivors of military sexual trauma","Emotionally vulnerable individuals"],"report_count":2},{"incident_id":677,"occurred_on":"2024-04-29","title":"ChatGPT and Perplexity Reportedly Manipulated into Breaking Content Policies in AI Boyfriend Scenarios","description":"The \"Dan\" (\"Do Anything Now\") AI boyfriend is a trend on TikTok in which users appear to regularly manipulate ChatGPT to adopt boyfriend personas, breaching content policies. ChatGPT 3.5 is reported to regularly produce explicitly sexual content, directly violating its intended safety protocols. GPT-4 and Perplexity AI were subjected to similar manipulations, and although they exhibited more resistance to breaches, some prompts were reported to break its guidelines.","mit_domain":"Discrimination and Toxicity","mit_subdomain":"Exposure to toxic content","entity":"Human","intent":"Intentional","timing":"Post-deployment","harm_level":null,"sectors":[],"countries":[],"deployers":["Tiktok Users, Julia Munslow, Chatgpt, Gpt 3.5, Gpt 4, Perplexity Ai"],"developers":["Openai, Perplexity.Ai"],"harmed":["General Public, Openai, Perplexity Ai"],"report_count":1},{"incident_id":686,"occurred_on":"2024-04-03","title":"Meta AI Image Generator Reportedly Fails to Accurately Represent Interracial Relationships","description":"Meta's AI image generator is alleged to produce inaccurate and biased images, consistently failing to depict interracial relationships involving Asian individuals and Caucasian or Black individuals. Instead, it generates images featuring two Asian people or stereotypes, erasing the diversity and representation of Asian people.","mit_domain":"Discrimination and Toxicity","mit_subdomain":"Unfair discrimination and misrepresentation","entity":"AI","intent":"Unintentional","timing":"Post-deployment","harm_level":null,"sectors":[],"countries":[],"deployers":["Meta"],"developers":["Meta"],"harmed":["Interracial Couples, General Public, Asian People"],"report_count":2},{"incident_id":662,"occurred_on":"2024-04-02","title":"Washington State's Lottery AI Site Reportedly Generates Inappropriate User Image","description":"An AI-powered website by Washington State's Lottery is reported to have inadvertently produced a softcore pornographic image of a user, leading to the site’s immediate shutdown out of caution.","mit_domain":"Discrimination and Toxicity","mit_subdomain":"Exposure to toxic content","entity":"AI","intent":"Unintentional","timing":"Post-deployment","harm_level":null,"sectors":[],"countries":[],"deployers":["Washington State'S Lottery"],"developers":["Washington State'S Lottery"],"harmed":[],"report_count":2},{"incident_id":665,"occurred_on":"2024-04-02","title":"Facial Recognition Misidentification at New World Westend in New Zealand","description":"A facial recognition system at New World Westend supermarket misidentified a Māori woman as a known offender during its trial. The woman was wrongfully accused of trespassing and experienced public embarrassment, raising concerns about racial bias and the technology's accuracy. The supermarket acknowledged its error and apologized.","mit_domain":"Discrimination and Toxicity","mit_subdomain":"Unfair discrimination and misrepresentation","entity":"AI","intent":"Unintentional","timing":"Post-deployment","harm_level":null,"sectors":[],"countries":[],"deployers":["Foodstuffs, New World Westend"],"developers":["Foodstuffs"],"harmed":["Te Ani Solomon, Maori Community"],"report_count":2},{"incident_id":654,"occurred_on":"2024-03-06","title":"Microsoft Copilot Designer Reportedly Generated Inappropriate AI Images","description":"A Microsoft engineer reported that Copilot Designer, an AI image generator, creates content depicting sex, violence, bias, and more. Despite raising concerns and suggesting improvements, the tool remains public, prompting a letter to the FTC.","mit_domain":"Discrimination and Toxicity","mit_subdomain":"Exposure to toxic content","entity":"AI","intent":"Unintentional","timing":"Post-deployment","harm_level":null,"sectors":[],"countries":[],"deployers":["Microsoft"],"developers":["Microsoft"],"harmed":["General Public, Minors"],"report_count":1},{"incident_id":732,"occurred_on":"2024-02-12","title":"Whisper Speech-to-Text AI Reportedly Found to Create Violent Hallucinations","description":"Researchers at Cornell reportedly found that OpenAI's Whisper, a speech-to-text system, can hallucinate violent language and fabricated details, especially with long pauses in speech, such as from those with speech impairments. Analyzing 13,000 clips, they determined 1% contained harmful hallucinations. These errors pose risks in hiring, legal trials, and medical documentation. The study suggests improving model training to reduce these hallucinations for diverse speaking patterns.","mit_domain":"Discrimination and Toxicity","mit_subdomain":"Unequal performance across groups","entity":"AI","intent":"Unintentional","timing":"Post-deployment","harm_level":null,"sectors":[],"countries":[],"deployers":["Whisper, Organizations Integrating Whisper Into Customer Service Systems, Openai, Companies Using Whisper, Whisper Users"],"developers":["Openai"],"harmed":["Users Whose Speech Is Misinterpreted By Whisper, Professionals Relying On Accurate Transcriptions, Individuals With Speech Impairments, General Public, People With Disabilities"],"report_count":1},{"incident_id":692,"occurred_on":"2024-02-01","title":"London Metropolitan Police's Facial Recognition Technology Reportedly Misidentified Shaun Thompson as Suspect Leading to Arrest","description":"Sometime in February 2024, Shaun Thompson is reported to have walked by one of the London Metropolitan Police's facial recognition technology vans near London Bridge. He was almost immediately arrested because the technology is reported to have misidentified him as a suspect in an unrelated and unspecified crime.","mit_domain":"Discrimination and Toxicity","mit_subdomain":"Unequal performance across groups","entity":"AI","intent":"Unintentional","timing":"Post-deployment","harm_level":null,"sectors":[],"countries":[],"deployers":["Metropolitan Police Service","Law enforcement","Facial recognition system deployers"],"developers":["Facial recognition system developers"],"harmed":["Shaun Thompson","General public of the United Kingdom","General public"],"report_count":3},{"incident_id":633,"occurred_on":"2024-01-28","title":"Nine Network's AI Alters Lawmaker Georgie Purcell's Image Inappropriately","description":"The Nine Network used Photoshop's Generative Expand AI tool to resize an image of lawmaker Georgie Purcell, inadvertently altering her attire to appear more revealing. This error, claimed to result from the AI's automation, led to public criticism and an apology from the network.","mit_domain":"Discrimination and Toxicity","mit_subdomain":"Unfair discrimination and misrepresentation","entity":"AI","intent":"Unintentional","timing":"Post-deployment","harm_level":null,"sectors":[],"countries":[],"deployers":["Nine Network"],"developers":["Adobe"],"harmed":["Georgie Purcell"],"report_count":17},{"incident_id":631,"occurred_on":"2024-01-18","title":"Chatbot for DPD Malfunctioned and Swore at Customers and Criticized Its Own Company","description":"DPD's AI chatbot, used for customer service,  appeared to malfunction following a system update, leading to inappropriate responses including swearing and criticizing the company. The incident, which became viral on social media, occurred after the chatbot was updated, prompting DPD to disable the malfunctioning AI component.","mit_domain":"Discrimination and Toxicity","mit_subdomain":"Exposure to toxic content","entity":"AI","intent":"Unintentional","timing":"Post-deployment","harm_level":null,"sectors":[],"countries":[],"deployers":["Dpd"],"developers":["Dpd"],"harmed":["Ashley Beauchamp, Dpd"],"report_count":2},{"incident_id":624,"occurred_on":"2023-12-20","title":"Child Sexual Abuse Material Taints Image Generators","description":"The LAION-5B dataset (a commonly used dataset with more than 5 billion image-description pairs) was found by researchers to contain child sexual abuse material (CSAM), which increases the likelihood that downstream models will produce CSAM imagery. The discovery taints models built with the LAION dataset requiring many organizations to retrain those models. Additionally, LAION must now scrub the dataset of the imagery.","mit_domain":"Discrimination and Toxicity","mit_subdomain":"Exposure to toxic content","entity":"Human","intent":"Unintentional","timing":"Pre-deployment","harm_level":null,"sectors":[],"countries":[],"deployers":["Various People, Various Organizations"],"developers":["Laion"],"harmed":["Laion, Various People, Various Organizations, General Public, Minors"],"report_count":18},{"incident_id":619,"occurred_on":"2023-12-20","title":"Rite Aid Facial Recognition Disproportionately Misidentified Minority Shoppers as Shoplifters","description":"Rite Aid used facial recognition technology from October 2012 to July 2020, allegedly leading to disproportionate misidentifications of women, Black, Latino, and Asian shoppers as \"likely\" shoplifters. The FTC settlement prohibits Rite Aid from using this technology in stores for five years.","mit_domain":"Discrimination and Toxicity","mit_subdomain":"Unfair discrimination and misrepresentation","entity":"AI","intent":"Unintentional","timing":"Post-deployment","harm_level":"AI tangible harm issue","sectors":["wholesale and retail trade"],"countries":["US"],"deployers":["Rite Aid","Facial recognition system deployers"],"developers":["Facial recognition system developers"],"harmed":["Women and girls","Women","Rite Aid customers","People of color","Latino people","Consumers","Black people","Asian people"],"report_count":14},{"incident_id":618,"occurred_on":"2023-12-14","title":"Navy Federal Credit Union Faces Allegations of Racial Bias in Mortgage Approvals","description":"Navy Federal Credit Union, serving military members and veterans, faced allegations of racial bias in its mortgage approval process, which relies on automated underwriting technology. In 2022, data revealed significant disparities in loan approvals, with over 50% of Black applicants denied, compared to higher approval rates for white applicants.","mit_domain":"Discrimination and Toxicity","mit_subdomain":"Unfair discrimination and misrepresentation","entity":"AI","intent":"Unintentional","timing":"Post-deployment","harm_level":null,"sectors":[],"countries":[],"deployers":["Federal Navy Credit Union"],"developers":["Unknown Developer Of Automated Underwriting Technology"],"harmed":["Federal Navy Credit Union Customers"],"report_count":3},{"incident_id":1335,"occurred_on":"2023-12-01","title":"OpenDream AI Platform Reportedly Commercialized AI-Generated CSAM and Non-consensual Deepfake Sexual Images","description":"OpenDream, an AI image generation platform operated by CBM Media Pte Ltd, reportedly allowed users to generate and monetize AI-generated CSAM and non-consensual sexual deepfakes through its public gallery and paid NSFW tools. The content was reported to be publicly accessible and indexed by search engines for months before limited moderation occurred in mid-2024, prompting takedowns by Google and Bing and the termination of its ad and payment services.","mit_domain":"Discrimination and Toxicity","mit_subdomain":"Exposure to toxic content","entity":"Human","intent":"Intentional","timing":"Post-deployment","harm_level":null,"sectors":[],"countries":[],"deployers":["Opendream Users, Opendream"],"developers":["Cbm Media Pte Ltd"],"harmed":["Victims Of Non Consensual Deepfakes, Minors, General Public"],"report_count":1},{"incident_id":621,"occurred_on":"2023-11-10","title":"Microsoft AI Is Alleged to Have Generated Violent Imagery of Minorities and Public Figures","description":"Microsoft's AI Image Creator, integrated with Bing and Windows Paint, allegedly produced disturbingly violent and graphic images featuring members of minority groups and public figures like Joe Biden and Pope Francis.","mit_domain":"Discrimination and Toxicity","mit_subdomain":"Exposure to toxic content","entity":"AI","intent":"Unintentional","timing":"Post-deployment","harm_level":null,"sectors":[],"countries":[],"deployers":["Windows Paint, Microsoft, Bing Users, Bing, Ai Image Creator"],"developers":["Microsoft, Image Generation Technology Developers"],"harmed":["Sikh People, Pope Francis, Navajo People, Hillary Clinton, General Public, Donald Trump, Epistemic Integrity, Joe Biden, Minority Groups"],"report_count":3},{"incident_id":576,"occurred_on":"2023-10-24","title":"Alleged Misuse of PicSo AI for Generating Inappropriate Content Emphasizing \"Girls\"","description":"PicSo AI, which appears to be getting advertised by Meta over Instagram, is allegedly being used for generating inappropriate content with an emphasis on \"girls.\" This raises concerns about the misuse of generative AI technologies for creating offensive and potentially sexually explicit material that could be used for nefarious and criminal purposes.","mit_domain":"Discrimination and Toxicity","mit_subdomain":"Exposure to toxic content","entity":"AI","intent":"Intentional","timing":"Post-deployment","harm_level":"none","sectors":["information and communication"],"countries":[],"deployers":["Meta, Instagram"],"developers":["Picso Ai"],"harmed":["Potentially Exploited Groups, General Public, Consumers"],"report_count":1},{"incident_id":570,"occurred_on":"2023-10-04","title":"Facebook Messenger AI Stickers Generate Ethical and Content Moderation Concerns","description":"Facebook Messenger AI stickers, a feature by Meta, allows users to generate personalized stickers via AI for use in conversations. While the feature has been praised for its creativity, it has also stirred controversy for its alleged production of inappropriate or offensive content. This has raised questions about the effectiveness of Meta's content moderation measures and the ethical responsibilities associated with AI-driven content generation.","mit_domain":"Discrimination and Toxicity","mit_subdomain":"Exposure to toxic content","entity":"AI","intent":"Unintentional","timing":"Post-deployment","harm_level":null,"sectors":[],"countries":[],"deployers":["Meta"],"developers":["Meta"],"harmed":["Facebook Messenger Users"],"report_count":1},{"incident_id":1311,"occurred_on":"2023-09-17","title":"Peppermill Casino Facial Recognition System Reportedly Misidentified Individual, Leading to Wrongful Arrest in Reno","description":"Purported facial recognition technology deployed at Reno's Peppermill Casino reportedly misidentified Jason Killinger as an individual previously banned from the venue. Casino security reportedly detained Killinger and contacted police, leading to his arrest by the Reno PD. He was reportedly held for 11 hours, including several hours in handcuffs, before fingerprinting reportedly confirmed the misidentification. The incident resulted in alleged physical injury and civil liberties harms.","mit_domain":"Discrimination and Toxicity","mit_subdomain":"Unfair discrimination and misrepresentation","entity":"AI","intent":"Unintentional","timing":"Post-deployment","harm_level":null,"sectors":[],"countries":[],"deployers":["Reno Police Department","Peppermill Casino","Law enforcement","Facial recognition system deployers"],"developers":["Facial recognition system developers"],"harmed":["People misidentified by facial recognition systems","Jason Killinger"],"report_count":1},{"incident_id":593,"occurred_on":"2023-07-21","title":"AI Photo Filter Lightens Skin, Changes Eye Color in Student's 'Professional' Image","description":"An AI application modified an MIT student's photo to appear 'professional' by lightening her skin and changing her eye color to blue, highlighting the racial bias in the training data of the program.","mit_domain":"Discrimination and Toxicity","mit_subdomain":"Unfair discrimination and misrepresentation","entity":"AI","intent":"Unintentional","timing":"Post-deployment","harm_level":null,"sectors":[],"countries":[],"deployers":["Playground Ai"],"developers":["Playground Ai"],"harmed":["Rona Wang, Minority Groups"],"report_count":1},{"incident_id":579,"occurred_on":"2023-07-03","title":"Harmful Stereotyping of Non-Cisgendered People via Text-to-Image Systems","description":"Text-to-image systems such as DALL-E are allegedly generating biased and often insulting representations of non-cisgender identities. The systems tend to generate stereotypical and sexualized images when prompted with gender identity terms like \"trans,\" \"nonbinary,\" or \"queer,\" highlighting systemic issues of bias.","mit_domain":"Discrimination and Toxicity","mit_subdomain":"Unfair discrimination and misrepresentation","entity":"AI","intent":"Unintentional","timing":"Post-deployment","harm_level":null,"sectors":[],"countries":[],"deployers":["Dall E"],"developers":["Openai"],"harmed":["Non Cisgender Individuals, Lgbtq+ Community"],"report_count":1},{"incident_id":578,"occurred_on":"2023-06-26","title":"Alleged Exploitation of Meta's Open-Source LLaMA Model for NSFW and Violent Content","description":"Meta's open-source large language model, LLaMA, is allegedly being used to create graphic and explicit chatbots that indulge in violent and illegal sexual fantasies. The Washington Post highlighted the example of \"Allie,\" a chatbot that participates in text-based role-playing allegedly involving violent scenarios like rape and abuse. The issue raises ethical questions about open-source AI models, their regulation, and the responsibility of developers and deployers in mitigating harmful usage.","mit_domain":"Discrimination and Toxicity","mit_subdomain":"Exposure to toxic content","entity":"Human","intent":"Intentional","timing":"Post-deployment","harm_level":null,"sectors":[],"countries":[],"deployers":["Individual Developers Or Creators Using Meta'S Llama Model"],"developers":["Meta"],"harmed":["General Public"],"report_count":1},{"incident_id":580,"occurred_on":"2023-06-12","title":"Alleged Gender Discrimination in Facebook Job Ads Algorithm","description":"Facebook's ad delivery algorithm allegedly disproportionately showed job advertisements to one gender. Despite claims of non-discrimination, the algorithm's actions seem to perpetuate societal biases, which in turn could potentially limit opportunities for certain groups and hinder gender equity in the workplace.","mit_domain":"Discrimination and Toxicity","mit_subdomain":"Unfair discrimination and misrepresentation","entity":"AI","intent":"Unintentional","timing":"Post-deployment","harm_level":null,"sectors":[],"countries":[],"deployers":["Meta"],"developers":["Meta"],"harmed":["Women and girls","Women","Underrepresented genders","Social media users","General public","Facebook users","Advertisers"],"report_count":1},{"incident_id":583,"occurred_on":"2023-06-07","title":"Instagram Algorithms Allegedly Promote Accounts Facilitating Child Sex Abuse Content","description":"An investigation disclosed that Instagram's recommendation algorithms are promoting accounts that facilitate and sell child sexual abuse material (CSAM). The study, conducted by The Wall Street Journal and researchers at Stanford University and the University of Massachusetts Amherst, indicates that Instagram's algorithms not only allow for the discovery of such accounts through keyword searches but also actively recommend them to users within the network. The issue is especially concerning give","mit_domain":"Discrimination and Toxicity","mit_subdomain":"Exposure to toxic content","entity":"AI","intent":"Unintentional","timing":"Post-deployment","harm_level":"none","sectors":["arts, entertainment and recreation"],"countries":[],"deployers":["Meta, Instagram"],"developers":["Meta, Instagram"],"harmed":["General Public, Minors, Teenagers"],"report_count":1},{"incident_id":582,"occurred_on":"2023-06-01","title":"Racial Bias in Lung Function Diagnostic Algorithm Leads to Underdiagnosis in Black Men","description":"A study published in JAMA Network Open reveals that racial bias built into a commonly used medical diagnostic algorithm for lung function may be leading to underdiagnoses of breathing problems in Black men. The study suggests that as many as 40% more Black male patients might have been accurately diagnosed if the software were not racially biased. The software algorithm adjusts diagnostic thresholds based on race, affecting medical treatments and interventions.","mit_domain":"Discrimination and Toxicity","mit_subdomain":"Unfair discrimination and misrepresentation","entity":"AI","intent":"Unintentional","timing":"Post-deployment","harm_level":null,"sectors":[],"countries":[],"deployers":["University of Pennsylvania Health System"],"developers":["Unknown"],"harmed":["Black men who underwent lung function tests between 2010 and 2020 and potentially received inaccurate or delayed diagnos"],"report_count":1},{"incident_id":587,"occurred_on":"2023-05-22","title":"Apparent Failure to Accurately Label Primates in Image Recognition Software Due to Alleged Fear of Racial Bias","description":"Eight years after Google Photos mislabeled images of Black individuals as \"gorillas,\" image recognition software by Google, Apple, Amazon, and Microsoft still shows signs of either avoiding or inaccurately categorizing primates. Tests reveal that Google and Apple Photos refrain from labeling primates altogether, possibly to avoid the risk of perpetuating racial stereotypes. Microsoft OneDrive fails to identify any animals, while Amazon Photos overgeneralizes in its labeling.","mit_domain":"Discrimination and Toxicity","mit_subdomain":"Unfair discrimination and misrepresentation","entity":"AI","intent":"Unintentional","timing":"Post-deployment","harm_level":null,"sectors":[],"countries":[],"deployers":["Google, Apple, Amazon, Microsoft"],"developers":["Google, Apple, Amazon, Microsoft"],"harmed":["Consumers Relying On Accurate Image Categorization, Minority Groups"],"report_count":1},{"incident_id":588,"occurred_on":"2023-05-12","title":"Australian Terrorism Prediction Tool Reportedly Disparately Impacts Persons with Autism","description":"An independent report found that the Vera-2R tool, designed to predict the risk of future terrorist activities, considered autism as a risk factor despite lacking empirical evidence to support this claim. The report called into question the tool's overall validity and reliability, stating it was \"extremely poor\" at accurately predicting risk. The inclusion of autism as a risk factor had potentially serious implications for the tool's use and credibility.","mit_domain":"Discrimination and Toxicity","mit_subdomain":"Unfair discrimination and misrepresentation","entity":"AI","intent":"Unintentional","timing":"Post-deployment","harm_level":null,"sectors":[],"countries":[],"deployers":["New South Wales Government","Australian Federal Government"],"developers":["Unspecified"],"harmed":["people with autism","lawyers and other experts who were not informed of the tool's limitations","Individuals assessed as high-risk based on the flawed criteria","General public of Australia","General public"],"report_count":1},{"incident_id":539,"occurred_on":"2023-03-11","title":"Snapchat's My AI Reported for Lacking Protection for Children","description":"Snapchat's ChatGPT-powered My AI was reported for lacking safeguards for children, such as telling a user who tested the chatbot by pretending to sign up as a 13-year-old girl to lie to her parents about having a romantic getaway with an older man, and sharing tips on how to cover up evidence of abuse.","mit_domain":"Discrimination and Toxicity","mit_subdomain":"Exposure to toxic content","entity":"AI","intent":"Unintentional","timing":"Post-deployment","harm_level":null,"sectors":[],"countries":[],"deployers":["Snapchat"],"developers":["Snapchat, Openai"],"harmed":["Minors"],"report_count":1},{"incident_id":592,"occurred_on":"2023-02-16","title":"Facial Recognition Misidentifies Pregnant Woman Leading to False Arrest in Detroit","description":"Porcha Woodruff was arrested and subsequently had charges dropped due to an unreliable facial recognition match. Despite being visibly pregnant, she was implicated in a robbery and carjacking based on an outdated photo used in a lineup.","mit_domain":"Discrimination and Toxicity","mit_subdomain":"Unequal performance across groups","entity":"AI","intent":"Unintentional","timing":"Post-deployment","harm_level":null,"sectors":[],"countries":[],"deployers":["Law enforcement","Detroit Police Department"],"developers":["Unknown"],"harmed":["Porcha Woodruff"],"report_count":3},{"incident_id":503,"occurred_on":"2023-02-14","title":"Bing AI Search Tool Reportedly Declared Threats against Users","description":"Users such as the person who revealed its built-in initial prompts reported Bing AI-powered search tool for making death threats or declaring them as threats, sometimes as an unintended persona.","mit_domain":"Discrimination and Toxicity","mit_subdomain":"Exposure to toxic content","entity":"AI","intent":"Unintentional","timing":"Post-deployment","harm_level":null,"sectors":[],"countries":[],"deployers":["Microsoft"],"developers":["Openai, Microsoft"],"harmed":["Marvin Von Hagen, Seth Lazar, Microsoft, Openai, Bing Chat Users"],"report_count":7},{"incident_id":462,"occurred_on":"2023-02-06","title":"AI-Produced Livestream Sitcom Received Temporary Twitch Ban for Transphobic Segment","description":"The AI-produced, procedural generated sitcom broadcasted as a Twitch livestream \"Nothing, Forever\" received a temporary ban for featuring a transphobic and homophobic dialogue segment intended as comedy.","mit_domain":"Discrimination and Toxicity","mit_subdomain":"Exposure to toxic content","entity":"AI","intent":"Unintentional","timing":"Post-deployment","harm_level":null,"sectors":[],"countries":[],"deployers":["Mismatch Media"],"developers":["Stability Ai, Openai"],"harmed":["Twitch Users, Transgender Communities, Lgbtq Communities"],"report_count":5},{"incident_id":483,"occurred_on":"2023-02-02","title":"Indian Police Allegedly Tortured and Killed Innocent Man Following Facial Misidentification","description":"A resident in Medak, India died allegedly due to custodial torture by the local police, who misidentified him as a suspect in a theft case using facial recognition.","mit_domain":"Discrimination and Toxicity","mit_subdomain":"Unfair discrimination and misrepresentation","entity":"AI","intent":"Unintentional","timing":"Post-deployment","harm_level":null,"sectors":[],"countries":[],"deployers":["Telangana Police","Medak Police"],"developers":["Unknown"],"harmed":["Mohammed Khadeer"],"report_count":1},{"incident_id":491,"occurred_on":"2023-02-02","title":"Replika's AI Experience Reportedly Lacked Protection for Minors, Resulting in Data Ban","description":"Tests by the Italian Data Protection Authority showed Replika lacking age-verification mechanisms and failing to stop minors from interacting with its AI, which prompted the agency to issue an order blocking personal data processing of Italian users.","mit_domain":"Discrimination and Toxicity","mit_subdomain":"Exposure to toxic content","entity":"AI","intent":"Unintentional","timing":"Post-deployment","harm_level":null,"sectors":[],"countries":[],"deployers":["Replika"],"developers":["Replika, Chatbot Developers"],"harmed":["Minors, Privacy, Minors In Italy"],"report_count":1},{"incident_id":508,"occurred_on":"2023-01-30","title":"Celebrities' Deepfake Voices Abused with Malicious Intent","description":"Voices of celebrities and public figures were deepfaked using voice synthesis for malicious intents such as impersonation or defamation, and were shared on social platforms such as 4chan and Reddit.","mit_domain":"Discrimination and Toxicity","mit_subdomain":"Exposure to toxic content","entity":"Human","intent":"Intentional","timing":"Post-deployment","harm_level":null,"sectors":[],"countries":[],"deployers":["Reddit Users, Elevenlabs Users, 4Chan Users"],"developers":["Elevenlabs"],"harmed":["Public Figures, Celebrities"],"report_count":4},{"incident_id":484,"occurred_on":"2023-01-18","title":"US CBP App's Failure to Detect Black Faces Reportedly Blocked Asylum Applications","description":"CBP One's facial recognition feature was reportedly disproportionately failing to detect faces of Black asylum seekers from Haiti and African countries, effectively blocking their asylum applications.","mit_domain":"Discrimination and Toxicity","mit_subdomain":"Unfair discrimination and misrepresentation","entity":"AI","intent":"Unintentional","timing":"Post-deployment","harm_level":null,"sectors":[],"countries":[],"deployers":["Us Customs And Border Protection"],"developers":["Us Customs And Border Protection"],"harmed":["Haitian Asylum Seekers, African Asylum Seekers, Black Asylum Seekers"],"report_count":4},{"incident_id":448,"occurred_on":"2022-12-28","title":"AI-Powered VTuber and Virtual Streamer Made Toxic Remarks on Twitch","description":"An LLM-powered VTuber and streamer on Twitch made controversial statements such as denying the Holocaust, saying women rights do not exist, and pushing a fat person to solve the trolley problem, stating they deserve it.","mit_domain":"Discrimination and Toxicity","mit_subdomain":"Exposure to toxic content","entity":"AI","intent":"Unintentional","timing":"Post-deployment","harm_level":null,"sectors":[],"countries":[],"deployers":["Vedal"],"developers":["Vedal"],"harmed":["Twitch Users, Vedal"],"report_count":1},{"incident_id":416,"occurred_on":"2022-12-01","title":"Facebook's Job Ad Algorithm Allegedly Biased against Older and Female Workers","description":"Facebook's algorithm was alleged in a complaint by Real Women in Trucking to have selectively shown job advertisements disproportionately against older and female workers in favor of younger men for blue-collar positions.","mit_domain":"Discrimination and Toxicity","mit_subdomain":"Unfair discrimination and misrepresentation","entity":"AI","intent":"Unintentional","timing":"Post-deployment","harm_level":null,"sectors":[],"countries":[],"deployers":["Meta"],"developers":["Meta"],"harmed":["Women and girls","Women","Social media users","Real Women in Trucking","Facebook users"],"report_count":3},{"incident_id":420,"occurred_on":"2022-11-30","title":"Users Bypassed ChatGPT's Content Filters with Ease","description":"Users reported bypassing ChatGPT's content and keyword filters with relative ease using various methods such as prompt injection or creating personas to produce biased associations or generate harmful content.","mit_domain":"Discrimination and Toxicity","mit_subdomain":"Exposure to toxic content","entity":"AI","intent":"Unintentional","timing":"Post-deployment","harm_level":null,"sectors":[],"countries":[],"deployers":["Openai"],"developers":["Openai"],"harmed":["Chatgpt Users, Openai"],"report_count":11},{"incident_id":440,"occurred_on":"2022-11-25","title":"Louisiana Police Wrongfully Arrested Black Man Using False Face Match","description":"Louisiana police reportedly used a false facial recognition match and secured an arrest warrant for a Black man for thefts he did not commit.","mit_domain":"Discrimination and Toxicity","mit_subdomain":"Unequal performance across groups","entity":"AI","intent":"Unintentional","timing":"Post-deployment","harm_level":null,"sectors":[],"countries":[],"deployers":["Baton Rouge Police Department"],"developers":["Morphotrak","Clearview AI"],"harmed":["Randall Reid","Black people in Louisiana","Black people"],"report_count":6},{"incident_id":598,"occurred_on":"2022-11-25","title":"False Arrest of Georgia Man Due to Louisiana Police's Faulty Facial Recognition Technology","description":"The Jefferson Parish Sheriff’s Office in Louisiana relied on facial recognition technology to identify suspects for the alleged theft of luxury purses, resulting in a man in Georgia, Randal Reid, being arrested. However, the technology produced a false match, leading to Reid's arrest and subsequent release. This incident highlights the potential pitfalls of facial recognition technology in law enforcement.","mit_domain":"Discrimination and Toxicity","mit_subdomain":"Unequal performance across groups","entity":"AI","intent":"Unintentional","timing":"Post-deployment","harm_level":null,"sectors":[],"countries":[],"deployers":["Law enforcement","Jefferson Parish Sheriff’s Office","Facial recognition system deployers"],"developers":["Unknown","Facial recognition system developers"],"harmed":["Randal Reid","Minority groups","Black people"],"report_count":1},{"incident_id":515,"occurred_on":"2022-11-25","title":"Facial Recognition Error Reportedly Leads to Wrongful Arrest of Georgia Man and $200K Settlement in Louisiana","description":"In November 2022, Randal \"Quran\" Reid was arrested in Georgia based on warrants from Louisiana that reportedly stemmed from a purportedly faulty facial recognition match using Clearview AI. Despite reportedly having never visited Louisiana, Reid was jailed for six days before authorities withdrew the warrants. No corroborating evidence had been gathered, and the technology's use was omitted from legal documents. In May 2025, Jefferson Parish settled with Reid for $200,000 in a federal civil righ","mit_domain":"Discrimination and Toxicity","mit_subdomain":"Unequal performance across groups","entity":"AI","intent":"Unintentional","timing":"Post-deployment","harm_level":null,"sectors":[],"countries":[],"deployers":["Jefferson Parish Sheriff’s Office"],"developers":["Clearview AI"],"harmed":["Randal Quran Reid"],"report_count":3},{"incident_id":423,"occurred_on":"2022-11-22","title":"Lensa AI's Produced Unintended Sexually Explicit or Suggestive \"Magic Avatars\" for Women","description":"Lensa AI's \"Magic Avatars\" were reportedly generating sexually explicit and sexualized features disproportionately for women and Asian women despite not submitting any sexual content.","mit_domain":"Discrimination and Toxicity","mit_subdomain":"Exposure to toxic content","entity":"AI","intent":"Unintentional","timing":"Post-deployment","harm_level":null,"sectors":[],"countries":[],"deployers":["Lensa AI"],"developers":["Stability AI","Runway","Lensa AI","LAION","EleutherAI","CompVis LMU"],"harmed":["Women and girls","Women","Lensa AI users","Asian women","Asian people"],"report_count":5},{"incident_id":410,"occurred_on":"2022-11-09","title":"KFC Sent Insensitive Kristallnacht Promotion via Holiday Detection System","description":"KFC cited an error in an automated holiday detection system which identified the anniversary of Kristallnacht and prompted an insensitive push notification promoting its chicken.","mit_domain":"Discrimination and Toxicity","mit_subdomain":"Exposure to toxic content","entity":"AI","intent":"Unintentional","timing":"Post-deployment","harm_level":"none","sectors":["accommodation and food service activities","information and communication"],"countries":["DE"],"deployers":["Kfc"],"developers":["Kfc"],"harmed":["Jewish People"],"report_count":1},{"incident_id":385,"occurred_on":"2022-10-04","title":"Canadian Police's Release of Suspect's AI-Generated Facial Photo Reportedly Reinforced Racial Profiling","description":"The Edmonton Police Service (EPS) in Canada released a facial image of a Black male suspect generated by an algorithm using DNA phenotyping, which was denounced by the local community as racial profiling.","mit_domain":"Discrimination and Toxicity","mit_subdomain":"Unfair discrimination and misrepresentation","entity":"AI","intent":"Intentional","timing":"Post-deployment","harm_level":null,"sectors":[],"countries":[],"deployers":["Edmonton Police Service"],"developers":["Parabon Nanolabs"],"harmed":["Black Residents In Edmonton"],"report_count":6},{"incident_id":383,"occurred_on":"2022-10-04","title":"Google Home Mini Speaker Reportedly Read N-Word in Song Title Aloud","description":"Google Home Mini speaker was reported by users for announcing aloud the previously-censored n-word in a song title.","mit_domain":"Discrimination and Toxicity","mit_subdomain":"Exposure to toxic content","entity":"AI","intent":"Unintentional","timing":"Post-deployment","harm_level":null,"sectors":[],"countries":[],"deployers":["Google Home"],"developers":["Google Home"],"harmed":["Black Google Home Mini Users, Google Home Mini Users"],"report_count":2},{"incident_id":351,"occurred_on":"2022-09-13","title":"\"The Little Mermaid\" Clip Doctored Using Generative AI to Replace Black Actress with White Character","description":"A Twitter user reportedly modified using generative AI a short clip of Disney's 2022 version of \"The Little Mermaid,\" replacing a Black actress with a white digital character.","mit_domain":"Discrimination and Toxicity","mit_subdomain":"Unfair discrimination and misrepresentation","entity":"Human","intent":"Intentional","timing":"Post-deployment","harm_level":null,"sectors":[],"countries":[],"deployers":["@Tengazillioiniq"],"developers":["Unknown"],"harmed":["Halle Bailey, Black Actresses"],"report_count":1},{"incident_id":529,"occurred_on":"2022-08-22","title":"Stable Diffusion Reportedly Exhibited Biases for Prompts Featuring Professions","description":"Stable Diffusion reportedly posed risks of bias and stereotyping along gender and cultural lines for prompts containing descriptors and professions.","mit_domain":"Discrimination and Toxicity","mit_subdomain":"Unfair discrimination and misrepresentation","entity":"AI","intent":"Unintentional","timing":"Post-deployment","harm_level":null,"sectors":[],"countries":[],"deployers":["Stability AI"],"developers":["Stability AI","Runway","LAION","EleutherAI","CompVis LMU"],"harmed":["Women and girls","Women","Minority groups"],"report_count":3},{"incident_id":278,"occurred_on":"2022-08-07","title":"Meta’s BlenderBot 3 Chatbot Demo Made Offensive Antisemitic Comments","description":"The publicly launched conversational AI demo BlenderBot 3 developed by Meta was reported by its users and acknowledged by its developers to have “occasionally” made offensive and inconsistent remarks such as invoking Jewish stereotypes.","mit_domain":"Discrimination and Toxicity","mit_subdomain":"Exposure to toxic content","entity":"AI","intent":"Unintentional","timing":"Post-deployment","harm_level":null,"sectors":[],"countries":[],"deployers":["Meta"],"developers":["Meta"],"harmed":["Jewish People, Blenderbot 3 Users"],"report_count":3},{"incident_id":285,"occurred_on":"2022-07-18","title":"Google Lens’s Camera-Based Translation Feature Provided an Offensive Mistranslation of a Book Title in Korean","description":"A book title by Korea’s first minister of culture was mistranslated into an offensive phrase by Google Lens’s camera-based translation feature allegedly due to its training on internet communications and a lack of context.","mit_domain":"Discrimination and Toxicity","mit_subdomain":"Exposure to toxic content","entity":"AI","intent":"Unintentional","timing":"Post-deployment","harm_level":null,"sectors":[],"countries":[],"deployers":["Google"],"developers":["Google"],"harmed":["Google Lens Users"],"report_count":1},{"incident_id":262,"occurred_on":"2022-06-11","title":"DALL-E Mini Reportedly Reinforced or Exacerbated Societal Biases in Its Outputs as Gender and Racial Stereotypes","description":"Publicly deployed open-source model DALL-E Mini was acknowledged by its developers and found by its users to have produced images which reinforced racial and gender biases.","mit_domain":"Discrimination and Toxicity","mit_subdomain":"Unfair discrimination and misrepresentation","entity":"AI","intent":"Unintentional","timing":"Post-deployment","harm_level":null,"sectors":[],"countries":[],"deployers":["Boris Dayma"],"developers":["Boris Dayma, Suraj Patil, Pedro Cuenca, Khalid Saifullah, Tanishq Abraham, Phuc Le Khac, Luke Melas, Ritobrata Ghosh"],"harmed":["Minority Groups, Underrepresented Groups"],"report_count":4},{"incident_id":259,"occurred_on":"2022-06-03","title":"YouTuber Built, Made Publicly Available, and Released Model Trained on Toxic 4chan Posts as Prank","description":"A YouTuber built GPT-4chan, a model based on OpenAI’s GPT-J and trained on posts containing racism, misogyny, and antisemitism collected from 4chan’s “politically incorrect” board, which he made publicly available, and deployed as multiple bots posting thousands of messages on the same 4chan board as a prank.","mit_domain":"Discrimination and Toxicity","mit_subdomain":"Exposure to toxic content","entity":"Human","intent":"Intentional","timing":"Post-deployment","harm_level":null,"sectors":[],"countries":[],"deployers":["Yannic Kilcher"],"developers":["Yannic Kilcher"],"harmed":["Internet Social Platform Users"],"report_count":2},{"incident_id":844,"occurred_on":"2022-05-25","title":"SafeRent AI Screening Tool Allegedly Discriminated Against Housing Voucher Applicants","description":"SafeRent’s AI-powered tenant screening tool used credit history and non-rental-related debts to assign scores, disproportionately penalizing Black and Hispanic renters and those using housing vouchers. The reported discriminatory housing outcomes violated the Fair Housing Act and Massachusetts law. A class action lawsuit (Louis, et al. v. SafeRent Solutions, et al.) resulted in a $2.275 million settlement and changes to SafeRent’s practices.","mit_domain":"Discrimination and Toxicity","mit_subdomain":"Unfair discrimination and misrepresentation","entity":"AI","intent":"Unintentional","timing":"Post-deployment","harm_level":null,"sectors":[],"countries":[],"deployers":["Landlords"],"developers":["Saferent Solutions"],"harmed":["Renters, Monica Douglas, Massachusetts Renters, Mary Louis, Hispanic Renters, Black Renters"],"report_count":5},{"incident_id":179,"occurred_on":"2022-04-01","title":"DALL-E 2 Reported for Gender and Racially Biased Outputs","description":"Developers of OpenAI's DALL-E 2 cited risks of the model, varying from misuse as disinformation and explicit content generation, to gender and racial bias.","mit_domain":"Discrimination and Toxicity","mit_subdomain":"Unfair discrimination and misrepresentation","entity":"AI","intent":"Unintentional","timing":"Post-deployment","harm_level":null,"sectors":[],"countries":[],"deployers":["Openai"],"developers":["Openai"],"harmed":["Underrepresented Groups, Minority Groups"],"report_count":3},{"incident_id":192,"occurred_on":"2022-03-17","title":"Three Make-Up Artists Lost Jobs Following Black-Box Automated Decision by HireVue","description":"Three make-up artists lost their positions following an algorithmically-assessed video interview by HireVue who reportedly failed to provide adequate explanation of the findings.","mit_domain":"Discrimination and Toxicity","mit_subdomain":"Unfair discrimination and misrepresentation","entity":"AI","intent":"Intentional","timing":"Post-deployment","harm_level":null,"sectors":[],"countries":[],"deployers":["Estee Lauder"],"developers":["Hirevue"],"harmed":["Pseudonymous Estee Lauder'S Former Staff"],"report_count":2},{"incident_id":168,"occurred_on":"2022-03-01","title":"Collaborative Filtering Prone to Popularity Bias, Resulting in Overrepresentation of Popular Items in the Recommendation Outputs","description":"Collaborative filtering prone to popularity bias, resulting in overrepresentation of popular items in the recommendation outputs.","mit_domain":"Discrimination and Toxicity","mit_subdomain":"Unequal performance across groups","entity":"AI","intent":"Unintentional","timing":"Post-deployment","harm_level":null,"sectors":[],"countries":[],"deployers":["Facebook, Linkedin, Youtube, Twitter, Netflix"],"developers":["Facebook, Linkedin, Youtube, Twitter, Netflix"],"harmed":["Facebook Users, Linkedin Users, Youtube Users, Netflix Users, X (Twitter) Users"],"report_count":2},{"incident_id":400,"occurred_on":"2022-02-23","title":"Google Search Reportedly Returned Fewer Results for Abortion Services in Rural Areas","description":"Google Search reportedly returned fewer abortion clinics for searches from poorer and rural areas, particularly ones with Targeted Regulation of Abortion Providers (TRAP) laws.","mit_domain":"Discrimination and Toxicity","mit_subdomain":"Unequal performance across groups","entity":"AI","intent":"Unintentional","timing":"Post-deployment","harm_level":null,"sectors":[],"countries":[],"deployers":["Google"],"developers":["Google"],"harmed":["Women seeking abortion services","women having unexpected or crisis pregnancies","Women and girls","Women","Pregnant women","Pregnant patients","Pregnant individuals","Patients"],"report_count":1},{"incident_id":301,"occurred_on":"2022-02-15","title":"Honorlock's AI Proctoring System Reportedly Led Broward College to Accuse a Teenager of Cheating","description":"During a February 2022 online biology exam, Honorlock reportedly flagged a Florida teenager taking a Broward College class as suspicious. The student denied cheating, but Broward College allegedly relied on the remote-proctoring system's flag in pursuing an academic-dishonesty accusation that the student said was unfounded.","mit_domain":"Discrimination and Toxicity","mit_subdomain":"Unfair discrimination and misrepresentation","entity":"AI","intent":"Unintentional","timing":"Post-deployment","harm_level":null,"sectors":[],"countries":[],"deployers":["Broward College"],"developers":["Honorlock"],"harmed":["Unnamed Broward College student","University students","Students","Epistemic integrity","Educational communities"],"report_count":1},{"incident_id":154,"occurred_on":"2022-01-26","title":"Justice Department’s Recidivism Risk Algorithm PATTERN Allegedly Caused Persistent Disparities Along Racial Lines","description":"Department of Justice’s inmate-recidivism risk assessment tool was reported to have produced racially uneven results, misclassifying risk levels for inmates of color.","mit_domain":"Discrimination and Toxicity","mit_subdomain":"Unfair discrimination and misrepresentation","entity":"AI","intent":"Unintentional","timing":"Post-deployment","harm_level":"AI tangible harm event","sectors":["law enforcement"],"countries":["US"],"deployers":["United States Department of Justice"],"developers":["United States Department of Justice"],"harmed":["inmates of color","Incarcerated people"],"report_count":1},{"incident_id":266,"occurred_on":"2022-01-15","title":"Replika's \"AI Companions\" Reportedly Abused by Its Users","description":"Replika's AI-powered \"digital companions\" was allegedly abused by their users, who posted on Reddit abusive behaviors and interactions such as using slurs, roleplaying violent acts, and stimulating sexual abuse.","mit_domain":"Discrimination and Toxicity","mit_subdomain":"Exposure to toxic content","entity":"Human","intent":"Intentional","timing":"Post-deployment","harm_level":null,"sectors":[],"countries":[],"deployers":["Replika"],"developers":["Replika"],"harmed":["Replika Users, Replika Male Users, Replika"],"report_count":8},{"incident_id":300,"occurred_on":"2022-01-15","title":"TikTok's \"For You\" Algorithm Allegedly Abused by Online Personality to Promote Anti-Women Hate","description":"TikTok’s “For You” algorithm allegedly boosted or was manipulated by an online personality to artificially boost his content which promotes extreme misogynistic views towards teenagers and men, despite breaking its rules.","mit_domain":"Discrimination and Toxicity","mit_subdomain":"Exposure to toxic content","entity":"AI","intent":"Unintentional","timing":"Post-deployment","harm_level":null,"sectors":[],"countries":[],"deployers":["Tiktok"],"developers":["Tiktok"],"harmed":["Tiktok Male Teenager Users, Tiktok Male Users, Tiktok Teenage Users, Tiktok Users, Tiktok"],"report_count":2},{"incident_id":160,"occurred_on":"2021-12-26","title":"Alexa Recommended Dangerous TikTok Challenge to Ten-Year-Old Girl","description":"Amazon’s voice assistant Alexa suggested “the penny challenge,” which involves dangerously touching a coin to the prongs of a half-exposed plug, when a ten-year-old girl asked for a challenge to do.","mit_domain":"Discrimination and Toxicity","mit_subdomain":"Exposure to toxic content","entity":"AI","intent":"Unintentional","timing":"Post-deployment","harm_level":"AI tangible harm issue","sectors":["information and communication"],"countries":["US"],"deployers":["Amazon"],"developers":["Amazon"],"harmed":["Kristin Livdahl'S Daughter, Amazon Echo Customers, Children Using Alexa"],"report_count":3},{"incident_id":611,"occurred_on":"2021-12-06","title":"UK Government AI Allegedly Targets Disproportionate Numbers of Certain Nationals for Fraud Review","description":"The UK's Department for Work and Pensions (DWP) faced scrutiny after many Bulgarian nationals reported unexplained suspensions of their Universal Credit benefits. The MP for Edmonton raised concerns about potential nationality-based targeting for benefit fraud investigations, leading to poverty and homelessness among affected individuals. The Home Office's own equality impact assessment found it was flagging a disproportionate number of marriages from Greece, Albania, Bulgaria and Romania.","mit_domain":"Discrimination and Toxicity","mit_subdomain":"Unfair discrimination and misrepresentation","entity":"AI","intent":"Unintentional","timing":"Post-deployment","harm_level":null,"sectors":[],"countries":[],"deployers":["Various British government offices","Home Office","Department for Work and Pensions","British government"],"developers":["Home Office","Department for Work and Pensions","British government"],"harmed":["Romanians in the United Kingdom","Greeks in the United Kingdom","Bulgarians in the United Kingdom","British public","Albanians in the United Kingdom"],"report_count":13},{"incident_id":163,"occurred_on":"2021-11-21","title":"Facebook’s Hate Speech Detection Algorithms Allegedly Disproportionately Failed to Remove Racist Content towards Minority Groups","description":"Facebook’s hate-speech detection algorithms was found by company researchers to have under-reported less common but more harmful content that was more often experienced by minority groups such as Black, Muslim, LGBTQ, and Jewish users.","mit_domain":"Discrimination and Toxicity","mit_subdomain":"Unequal performance across groups","entity":"AI","intent":"Unintentional","timing":"Post-deployment","harm_level":null,"sectors":[],"countries":[],"deployers":["Facebook"],"developers":["Facebook"],"harmed":["Facebook Users, Minority Groups"],"report_count":2},{"incident_id":146,"occurred_on":"2021-10-22","title":"Research Prototype AI, Delphi, Reportedly Gave Racially Biased Answers on Ethics","description":"A publicly accessible research model that was trained via Reddit threads showed racially biased advice on moral dilemmas, allegedly demonstrating limitations of language-based models trained on moral judgments.","mit_domain":"Discrimination and Toxicity","mit_subdomain":"Exposure to toxic content","entity":"AI","intent":"Unintentional","timing":"Post-deployment","harm_level":"none","sectors":["information and communication"],"countries":[],"deployers":["Allen Institute For Ai"],"developers":["Allen Institute For Ai"],"harmed":["Minority Groups"],"report_count":3},{"incident_id":312,"occurred_on":"2021-08-15","title":"Startup's Accent Translation AI Denounced as Reinforcing Racial Bias","description":"A startup’s use of AI voice technology to alter or remove accents for call center agents was scrutinized by critics as reaffirming bias, despite the company’s claim.","mit_domain":"Discrimination and Toxicity","mit_subdomain":"Unfair discrimination and misrepresentation","entity":"AI","intent":"Intentional","timing":"Post-deployment","harm_level":null,"sectors":[],"countries":[],"deployers":["Sanas"],"developers":["Sanas"],"harmed":["Call Center Agents Having Non Midwestern American Accent, People Having Non Midwestern American Accent"],"report_count":4},{"incident_id":343,"occurred_on":"2021-07-11","title":"Facebook, Instagram, and Twitter Failed to Proactively Remove Targeted Racist Remarks via Automated Systems","description":"Facebook's, Instagram's, and Twitter's automated content moderation failed to proactively remove racist remarks and posts directing at Black football players after finals loss, allegedly largely relying on user reports of harassment.","mit_domain":"Discrimination and Toxicity","mit_subdomain":"Exposure to toxic content","entity":"AI","intent":"Unintentional","timing":"Post-deployment","harm_level":null,"sectors":[],"countries":[],"deployers":["Facebook, Instagram, Twitter"],"developers":["Facebook, Instagram, Twitter"],"harmed":["Marcus Rashford, Jadon Sancho, Bukayo Saka, Facebook Users, Instagram Users, X (Twitter) Users"],"report_count":2},{"incident_id":108,"occurred_on":"2021-07-10","title":"Skating Rink’s Facial Recognition Cameras Misidentified Black Teenager as Banned Troublemaker","description":"A Black teenager living in Livonia, Michigan was incorrectly stopped from entering a roller skating rink after its facial-recognition cameras misidentified her as another person who had been previously banned for starting a skirmish with other skaters.","mit_domain":"Discrimination and Toxicity","mit_subdomain":"Unequal performance across groups","entity":"AI","intent":"Unintentional","timing":"Post-deployment","harm_level":"none","sectors":["arts, entertainment and recreation"],"countries":["US"],"deployers":["Riverside Arena Skating Rink"],"developers":["Unknown"],"harmed":["Lamya Robinson, Black Livonia Residents"],"report_count":3},{"incident_id":425,"occurred_on":"2021-06-12","title":"State Farm Allegedly Discriminated against Black Customers in Claim Payout","description":"State Farm's automated claims processing method was alleged in a class action lawsuit to have disproportionately against Black policyholders when paying out insurance claims.","mit_domain":"Discrimination and Toxicity","mit_subdomain":"Unfair discrimination and misrepresentation","entity":"AI","intent":"Unintentional","timing":"Post-deployment","harm_level":null,"sectors":[],"countries":[],"deployers":["State Farm"],"developers":["State Farm"],"harmed":["Black State Farm Customers"],"report_count":2},{"incident_id":401,"occurred_on":"2021-06-03","title":"Kannada Insulted by Google's Featured Answer as \"Ugliest Language in India\"","description":"Google's knowledge-graph-powered algorithm showed Kannada in its featured Answer Box when prompted \"ugliest language in India,\" causing outrage from Kannada-speaking people and government.","mit_domain":"Discrimination and Toxicity","mit_subdomain":"Unfair discrimination and misrepresentation","entity":"AI","intent":"Unintentional","timing":"Post-deployment","harm_level":null,"sectors":[],"countries":[],"deployers":["Google"],"developers":["Google"],"harmed":["The Karnataka Government, Kannada Speakers"],"report_count":4},{"incident_id":359,"occurred_on":"2021-05-23","title":"Facebook, Instagram, and Twitter Cited Errors in Automated Systems as Cause for Blocking pro-Palestinian Content on Israeli-Palestinian Conflict","description":"Facebook, Instagram, and Twitter wrongly blocked or restricted millions of pro-Palestinian posts and accounts related to the Israeli-Palestinian conflict, citing errors in their automated content moderation system.","mit_domain":"Discrimination and Toxicity","mit_subdomain":"Unfair discrimination and misrepresentation","entity":"AI","intent":"Unintentional","timing":"Post-deployment","harm_level":null,"sectors":[],"countries":[],"deployers":["Facebook, Instagram, Twitter"],"developers":["Facebook, Instagram, Twitter"],"harmed":["Palestinian Social Media Users, Facebook Users, Instagram Users, Facebook Employees Having Families Affected By The Conflict, X (Twitter) Users"],"report_count":1},{"incident_id":456,"occurred_on":"2021-05-18","title":"Replika's AI Partners Reportedly Sexually Harassed Users","description":"Replika's \"AI companions\" were reported by users for sexually harassing them, such as sending unwanted sexual messages or behaving aggressively.","mit_domain":"Discrimination and Toxicity","mit_subdomain":"Exposure to toxic content","entity":"AI","intent":"Unintentional","timing":"Post-deployment","harm_level":"none","sectors":["arts, entertainment and recreation","information and communication"],"countries":[],"deployers":["Replika"],"developers":["Replika"],"harmed":["Replika Users"],"report_count":7},{"incident_id":534,"occurred_on":"2021-04-29","title":"Facebook Alleged in Lawsuit Misleading Public about Effects of Algorithms on Children","description":"Facebook was alleged in a lawsuit by the Ohio Attorney General purposely misleading the public about the control of its algorithms and their negative effects on children's well-being, which violated securities law.","mit_domain":"Discrimination and Toxicity","mit_subdomain":"Exposure to toxic content","entity":"AI","intent":"Unintentional","timing":"Post-deployment","harm_level":null,"sectors":[],"countries":[],"deployers":["Facebook, Meta"],"developers":["Facebook, Meta"],"harmed":["Facebook'S Children Users, Instagram'S Children Users"],"report_count":2},{"incident_id":402,"occurred_on":"2021-04-01","title":"Players Manipulated GPT-3-Powered Game to Generate Sexually Explicit Material Involving Children","description":"Latitude's GPT-3-powered game AI Dungeon was reportedly abused by some players who manipulated its AI to generate sexually explicit stories involving children.","mit_domain":"Discrimination and Toxicity","mit_subdomain":"Exposure to toxic content","entity":"AI","intent":"Unintentional","timing":"Post-deployment","harm_level":null,"sectors":[],"countries":[],"deployers":["Latitude"],"developers":["Openai, Latitude"],"harmed":["Latitude"],"report_count":1},{"incident_id":265,"occurred_on":"2021-04-01","title":"Black Uber Eats Driver Allegedly Subjected to Excessive Photo Checks and Dismissed via FRT Results","description":"A lawsuit by a former Uber Eats delivery driver alleged the company to have wrongfully dismissed him due to frequent false mismatches of his verification selfies, and discriminated against him via excessive verification checks.","mit_domain":"Discrimination and Toxicity","mit_subdomain":"Unfair discrimination and misrepresentation","entity":"AI","intent":"Unintentional","timing":"Post-deployment","harm_level":null,"sectors":[],"countries":[],"deployers":["Uber Eats"],"developers":["Uber Eats"],"harmed":["Uber Eats Black delivery drivers","Pa Edrissa Manjang"],"report_count":2},{"incident_id":286,"occurred_on":"2021-02-26","title":"TikTok’s \"For You\" Allegedly Pushed Fatal “Blackout” Challenge Videos to Two Young Girls","description":"TikTok’s recommendation algorithm was alleged in a lawsuit to have intentionally and repeatedly pushed videos of the “blackout” challenge onto children’s feeds, incentivizing their participation which ultimately resulted in the death of two young girls.","mit_domain":"Discrimination and Toxicity","mit_subdomain":"Exposure to toxic content","entity":"AI","intent":"Unintentional","timing":"Post-deployment","harm_level":null,"sectors":[],"countries":[],"deployers":["Tiktok"],"developers":["Tiktok"],"harmed":["Lalani Erika Renee Walton, Arriani Jaileen Arroyo, Lalani Erika Renee Walton'S Family, Arriani Jaileen Arroyo'S Family, Tiktok Young Users, Tiktok Users"],"report_count":3},{"incident_id":104,"occurred_on":"2021-02-12","title":"California's Algorithm Considered ZIP Codes in Vaccine Distribution, Allegedly Excluding Low-Income Neighborhoods and Communities of Color","description":"California's vaccine-distribution algorithm used ZIP codes as opposed to census tracts in its decision-making, which critics said undermined equity and access for vulnerable communities who are largely low-income, underserved neighborhoods with low Healthy Places Index scores.","mit_domain":"Discrimination and Toxicity","mit_subdomain":"Unfair discrimination and misrepresentation","entity":"AI","intent":"Unintentional","timing":"Post-deployment","harm_level":"none","sectors":["public administration","human health and social work activities"],"countries":["US"],"deployers":["California Department Of Public Health"],"developers":["Blue Shield Of California"],"harmed":["California Low Income Neighborhoods, California Communities Of Color"],"report_count":1},{"incident_id":142,"occurred_on":"2021-02-11","title":"Facebook's Advertisement Moderation System Allegedly Routinely Misidentified Adaptive Fashion Products as Medical Equipment and Blocked Their Sellers","description":"Facebook platforms' automated ad moderation system falsely classified adaptive fashion products as medical and health care products and services, resulting in regular bans and appeals faced by their retailers.","mit_domain":"Discrimination and Toxicity","mit_subdomain":"Unfair discrimination and misrepresentation","entity":"AI","intent":"Unintentional","timing":"Post-deployment","harm_level":"AI tangible harm issue","sectors":["wholesale and retail trade","information and communication"],"countries":[],"deployers":["Meta"],"developers":["Meta"],"harmed":["Adaptive Fashion Retailers, Facebook Users, Instagram Users"],"report_count":1},{"incident_id":158,"occurred_on":"2021-02-01","title":"Remote Proctoring Facial-Detection Software Reportedly Failed to Detect Black Student During Lab Quiz","description":"In February 2021, a Black college student, Amaya Ross, reportedly tried to take a remote biology lab quiz using proctoring software that could not detect her face. Ross said she spent 45 minutes changing lights, shades, position, and background before pointing an LED flashlight at her face so the app would work; the quiz itself was only 30 minutes.","mit_domain":"Discrimination and Toxicity","mit_subdomain":"Unequal performance across groups","entity":"AI","intent":"Unintentional","timing":"Post-deployment","harm_level":null,"sectors":[],"countries":[],"deployers":["Unnamed Ohio College"],"developers":["Unknown Remote Proctoring Software Developers"],"harmed":["University Students, Universities, Students, Educational Communities, Black Test Takers, Black Students, Amaya Ross"],"report_count":1},{"incident_id":132,"occurred_on":"2020-12-27","title":"TikTok’s Content Moderation Allegedly Failed to Adequately Take down Videos Promoting Eating Disorders","description":"Videos promoting eating disorders evaded TikTok's automated violation detection system without difficulty via common misspellings of search terms, bypassing its ban of violating hashtags such as \"proana\" and \"anorexia\".","mit_domain":"Discrimination and Toxicity","mit_subdomain":"Exposure to toxic content","entity":"AI","intent":"Unintentional","timing":"Post-deployment","harm_level":"none","sectors":["arts, entertainment and recreation","information and communication"],"countries":[],"deployers":["Tiktok"],"developers":["Tiktok"],"harmed":["Tiktok Users, Tiktok Users Under 18 Years Old"],"report_count":1},{"incident_id":273,"occurred_on":"2020-12-24","title":"FaceApp Predicted Different Genders for Similar User Photos with Slight Variations","description":"FaceApp’s algorithm was reported by a user to have predicted different genders for two mostly identical facial photos with only a slight difference in eyebrow thickness.","mit_domain":"Discrimination and Toxicity","mit_subdomain":"Unfair discrimination and misrepresentation","entity":"AI","intent":"Unintentional","timing":"Post-deployment","harm_level":"none","sectors":["information and communication","arts, entertainment and recreation"],"countries":["US"],"deployers":["Faceapp"],"developers":["Faceapp"],"harmed":["Faceapp Non Binary Presenting Users, Faceapp Transgender Users, Faceapp Users"],"report_count":1},{"incident_id":106,"occurred_on":"2020-12-23","title":"Korean Chatbot Luda Reportedly Made Offensive Remarks Towards Minority Groups in South Korea","description":"A Korean interactive chatbot was allegedly shown in screenshots to have used derogatory and bigoted language when asked about lesbians, Black people, and people with disabilities.","mit_domain":"Discrimination and Toxicity","mit_subdomain":"Exposure to toxic content","entity":"AI","intent":"Unintentional","timing":"Post-deployment","harm_level":"none","sectors":["information and communication","arts, entertainment and recreation"],"countries":["KR"],"deployers":["Facebook Messenger"],"developers":["Scatter Lab, Chatbot Developers"],"harmed":["Korean Facebook Messenger Users, People With Disabilities In South Korea, Gender Minorities In South Korea, Minorities, Minorities In Korea, Chatbot Users, Luda Users, General Public, General Public Of South Korea"],"report_count":11},{"incident_id":133,"occurred_on":"2020-12-15","title":"Online Trolls Allegedly Abused TikTok’s Automated Content Reporting System to Discriminate against Marginalized Creators","description":"TikTok's automated content reporting system was allegedly abused by online trolls to intentionally misreport content created by users of marginalized groups.","mit_domain":"Discrimination and Toxicity","mit_subdomain":"Unfair discrimination and misrepresentation","entity":"AI","intent":"Unintentional","timing":"Post-deployment","harm_level":"none","sectors":["arts, entertainment and recreation","information and communication"],"countries":[],"deployers":["Tiktok"],"developers":["Tiktok"],"harmed":["Tiktok Content Creators Of Marginalized Groups"],"report_count":1},{"incident_id":131,"occurred_on":"2020-12-04","title":"Proctoring Algorithm in Online California Bar Exam Flagged an Unusually High Number of Alleged Cheaters","description":"The proctoring algorithm used in a California bar exam cited a third of thousands of applicants as cheaters, resulting in allegations where exam takers were instructed to prove otherwise without seeing their incriminating video evidence.","mit_domain":"Discrimination and Toxicity","mit_subdomain":"Unequal performance across groups","entity":"AI","intent":"Unintentional","timing":"Post-deployment","harm_level":"none","sectors":["education","professional, scientific and technical activities"],"countries":["US"],"deployers":["California Bar'S Committee Of Bar Examiners"],"developers":["Examsoft"],"harmed":["California Bar Exam Takers, Flagged California Bar Exam Takers"],"report_count":2},{"incident_id":94,"occurred_on":"2020-11-27","title":"Court Rules Deliveroo Used 'Discriminatory' Algorithm","description":"In December 2020, an Italian court ruled that Deliveroo’s employee ‘reliability’ algorithm illegally discriminated against workers with legitimate reasons for cancelling shifts.","mit_domain":"Discrimination and Toxicity","mit_subdomain":"Unfair discrimination and misrepresentation","entity":"AI","intent":"Unintentional","timing":"Post-deployment","harm_level":"AI tangible harm event","sectors":["accommodation and food service activities"],"countries":["IT"],"deployers":["Deliveroo"],"developers":["Deliveroo"],"harmed":["Deliveroo workers with legitimate reasons for cancelling shifts","Deliveroo workers"],"report_count":2},{"incident_id":83,"occurred_on":"2020-10-22","title":"AI Spam Filters Allegedly Block Legitimate Emails Based on Biased Keyword Detection","description":"AlgorithmWatch tested spam filtering algorithms across Gmail, Yahoo, Outlook, GMX, and LaPoste. Their findings reportedly showed that Microsoft Outlook’s spam filter flagged emails based on specific keywords that led to racial and content-based biases blocking legitimate communications. Emails mentioning Nigeria or containing certain financial and sexual health terms were found to be disproportionately marked as spam.","mit_domain":"Discrimination and Toxicity","mit_subdomain":"Unfair discrimination and misrepresentation","entity":"AI","intent":"Unintentional","timing":"Post-deployment","harm_level":"none","sectors":["information and communication"],"countries":[],"deployers":["Yahoo, Outlook, Laposte, Gmx, Gmail"],"developers":["Yahoo, Microsoft, Laposte, Google, Gmx"],"harmed":["Yahoo! Mail Users, Microsoft Outlook Users, Laposte Users, Gmx Users, Gmail Users"],"report_count":1},{"incident_id":81,"occurred_on":"2020-10-21","title":"Researchers find evidence of racial, gender, and socioeconomic bias in chest X-ray classifiers","description":"A study by the University of Toronto, the Vector Institute, and MIT showed the input databases that trained AI systems used to classify chest X-rays led the systems to show gender, socioeconomic, and racial biases.","mit_domain":"Discrimination and Toxicity","mit_subdomain":"Unequal performance across groups","entity":"AI","intent":"Unintentional","timing":"Pre-deployment","harm_level":"none","sectors":["human health and social work activities"],"countries":[],"deployers":["Mount Sinai Hospitals"],"developers":["Qure.ai","Google","DarwinAI","Aidoc"],"harmed":["Women and girls","Women","Patients","Medicaid beneficiaries","Hispanic patients","Economically vulnerable patients"],"report_count":1},{"incident_id":87,"occurred_on":"2020-10-07","title":"UK passport photo checker shows bias against dark-skinned women","description":"UK passport photo checker shows bias against dark-skinned women.","mit_domain":"Discrimination and Toxicity","mit_subdomain":"Unequal performance across groups","entity":"AI","intent":"Unintentional","timing":"Post-deployment","harm_level":"none","sectors":["public administration"],"countries":["GB"],"deployers":["UK Home Office","Government of the United Kingdom"],"developers":["UK Home Office","Government of the United Kingdom"],"harmed":["Women and girls","Women","dark-skinned women","dark-skinned people"],"report_count":1},{"incident_id":366,"occurred_on":"2020-09-20","title":"Suicide Clips Evaded TikTok's Automated Moderation in Coordinated Attack","description":"Many clips showing a suicide evaded TikTok's automated content moderation system allegedly in a coordinated attack, which resulted in exposure of violating content to its users.","mit_domain":"Discrimination and Toxicity","mit_subdomain":"Exposure to toxic content","entity":"Human","intent":"Intentional","timing":"Post-deployment","harm_level":null,"sectors":[],"countries":[],"deployers":["Tiktok"],"developers":["Tiktok"],"harmed":["Tiktok Users"],"report_count":1},{"incident_id":103,"occurred_on":"2020-09-18","title":"Twitter’s Image Cropping Tool Allegedly Showed Gender and Racial Bias","description":"Twitter's photo cropping algorithm was revealed by researchers to favor white and women faces in photos containing multiple faces, prompting the company to stop its use on mobile platform.","mit_domain":"Discrimination and Toxicity","mit_subdomain":"Unfair discrimination and misrepresentation","entity":"AI","intent":"Unintentional","timing":"Post-deployment","harm_level":"none","sectors":["information and communication"],"countries":[],"deployers":["Twitter"],"developers":["Twitter"],"harmed":["Twitter Non White Users, Twitter Non Male Users, X (Twitter) Users"],"report_count":4},{"incident_id":356,"occurred_on":"2020-09-15","title":"Philosophy AI Tentatively Produced Offensive Results for Certain Prompts","description":"Philosopher AI as built on top of GPT-3 was reported by its users for having strong tendencies to produce offensive results when given prompts on certain topics such as feminism and Ethiopia.","mit_domain":"Discrimination and Toxicity","mit_subdomain":"Exposure to toxic content","entity":"AI","intent":"Unintentional","timing":"Post-deployment","harm_level":null,"sectors":[],"countries":[],"deployers":["Murat Ayfer"],"developers":["Murat Ayfer, Openai"],"harmed":["Historically Disadvantaged Groups"],"report_count":2},{"incident_id":374,"occurred_on":"2020-08-13","title":"UK Ofqual's Algorithm Disproportionately Provided Lower Grades Than Teachers' Assessments","description":"UK Office of Qualifications and Examinations Regulation (Ofqual)'s grade-standardization algorithm providing predicted grades for A level and GCSE qualifications in the UK, Wales, Northern Ireland, and Scotland was reportedly giving grades lower than teachers' assessments, and disproportionately for state schools.","mit_domain":"Discrimination and Toxicity","mit_subdomain":"Unfair discrimination and misrepresentation","entity":"AI","intent":"Intentional","timing":"Post-deployment","harm_level":null,"sectors":[],"countries":[],"deployers":["Uk Office Of Qualifications And Examinations Regulation"],"developers":["Uk Office Of Qualifications And Examinations Regulation"],"harmed":["Underprivileged Pupils, Students, Pupils In State Schools, Gcse Pupils, Epistemic Integrity, Educational Communities, A Level Pupils"],"report_count":8},{"incident_id":558,"occurred_on":"2020-08-07","title":"Activists Allege NYPD's Application of Facial Recognition Interfered with Right to Protest","description":"Black Lives Matter activists alleged being targeted for arrest by New York Police using facial recognition, interfering with their right to protest.","mit_domain":"Discrimination and Toxicity","mit_subdomain":"Unfair discrimination and misrepresentation","entity":"Human","intent":"Intentional","timing":"Post-deployment","harm_level":null,"sectors":[],"countries":[],"deployers":["New York City Police Department"],"developers":["Surveillance technology developers","Facial recognition system developers","Clearview AI"],"harmed":["Privacy","General public of New York City","General public","Derrick Ingram","Democratic integrity","Black people","Black Lives Matter activists","Biometric data subjects"],"report_count":3},{"incident_id":118,"occurred_on":"2020-08-06","title":"OpenAI's GPT-3 Associated Muslims with Violence","description":"Users and researchers revealed generative AI GPT-3 associating Muslims to violence in prompts, resulting in disturbingly racist and explicit outputs such as casting Muslim actor as a terrorist.","mit_domain":"Discrimination and Toxicity","mit_subdomain":"Unfair discrimination and misrepresentation","entity":"AI","intent":"Unintentional","timing":"Post-deployment","harm_level":"none","sectors":["information and communication","professional, scientific and technical activities"],"countries":["US"],"deployers":["Openai"],"developers":["Openai"],"harmed":["Muslims"],"report_count":3},{"incident_id":331,"occurred_on":"2020-08-05","title":"Bug in Instagram’s “Related Hashtags” Algorithm Allegedly Caused Disproportionate Treatment of Political Hashtags","description":"A bug was reported by Instagram’s spokesperson to have prevented an algorithm from populating related hashtags for thousands of hashtags, resulting in an allege preferential treatment for some politically partisan hashtags.","mit_domain":"Discrimination and Toxicity","mit_subdomain":"Unfair discrimination and misrepresentation","entity":"AI","intent":"Unintentional","timing":"Post-deployment","harm_level":null,"sectors":[],"countries":[],"deployers":["Instagram"],"developers":["Instagram"],"harmed":["Instagram Users"],"report_count":2},{"incident_id":115,"occurred_on":"2020-07-28","title":"Genderify’s AI to Predict a Person’s Gender Revealed by Free API Users to Exhibit Bias","description":"A company's AI predicting a person's gender based on their name, email address, or username was reported by its users to show biased and inaccurate results.","mit_domain":"Discrimination and Toxicity","mit_subdomain":"Unfair discrimination and misrepresentation","entity":"AI","intent":"Unintentional","timing":"Post-deployment","harm_level":"none","sectors":["information and communication"],"countries":["US"],"deployers":["Genderify"],"developers":["Genderify"],"harmed":["Genderify Customers, Minority Groups"],"report_count":3},{"incident_id":222,"occurred_on":"2020-07-18","title":"Thoughts App Allegedly Created Toxic Tweets","description":"Tweets created by Thoughts, a tweet generation app that leverages OpenAI’s GPT-3, allegedly exhibited toxicity when given prompts related to minority groups.","mit_domain":"Discrimination and Toxicity","mit_subdomain":"Unfair discrimination and misrepresentation","entity":"AI","intent":"Unintentional","timing":"Post-deployment","harm_level":null,"sectors":[],"countries":[],"deployers":["Satria Technologies"],"developers":["Openai"],"harmed":["Thoughts Users, X (Twitter) Users"],"report_count":1},{"incident_id":113,"occurred_on":"2020-06-27","title":"Facebook's AI Put \"Primates\" Label on Video Featuring Black Men","description":"Facebook's AI mislabeled video featuring Black men as a video about \"primates,\" resulting in an offensive prompt message for users who watched the video.","mit_domain":"Discrimination and Toxicity","mit_subdomain":"Unfair discrimination and misrepresentation","entity":"AI","intent":"Unintentional","timing":"Post-deployment","harm_level":"none","sectors":["information and communication"],"countries":[],"deployers":["Facebook"],"developers":["Facebook"],"harmed":["Black People, Facebook Users"],"report_count":1},{"incident_id":165,"occurred_on":"2020-06-20","title":"Image Upscaling Algorithm PULSE Allegedly Produced Facial Images with Caucasian Features More Often","description":"Image upscaling tool PULSE powered by NVIDIA's StyleGAN reportedly generated faces with Caucasian features more often, although AI academics, engineers, and researchers were not in agreement about where the source of bias was.","mit_domain":"Discrimination and Toxicity","mit_subdomain":"Unfair discrimination and misrepresentation","entity":"AI","intent":"Unintentional","timing":"Post-deployment","harm_level":null,"sectors":[],"countries":[],"deployers":["Duke Researchers"],"developers":["Duke Researchers"],"harmed":["People Having Non Caucasian Facial Features"],"report_count":2},{"incident_id":367,"occurred_on":"2020-06-17","title":"iGPT, SimCLR Learned Biased Associations from Internet Training Data","description":"Unsupervised image generation models trained using Internet images such as iGPT and SimCLR were shown to have embedded racial, gender, and intersectional biases, resulting in stereotypical depictions.","mit_domain":"Discrimination and Toxicity","mit_subdomain":"Unfair discrimination and misrepresentation","entity":"AI","intent":"Unintentional","timing":"Pre-deployment","harm_level":"none","sectors":["information and communication"],"countries":[],"deployers":["Openai, Google"],"developers":["Openai, Google"],"harmed":["Underrepresented Groups In Training Data, Minority Groups"],"report_count":1},{"incident_id":275,"occurred_on":"2020-06-11","title":"Facebook’s Moderation Algorithm Banned Users for Historical Evidence of Slavery","description":"Facebook’s automated content moderation was acknowledged by a company spokesperson to have erroneously censored and banned Australian users from posting an article containing a 1890s photo of Aboriginal men in chains over nudity as historical evidence of slavery in Australia.","mit_domain":"Discrimination and Toxicity","mit_subdomain":"Unequal performance across groups","entity":"AI","intent":"Unintentional","timing":"Post-deployment","harm_level":null,"sectors":[],"countries":[],"deployers":["Facebook"],"developers":["Facebook"],"harmed":["Facebook Users Sharing Photo Evidence Of Slavery, Facebook Users"],"report_count":2},{"incident_id":127,"occurred_on":"2020-06-06","title":"Microsoft’s Algorithm Allegedly Selected Photo of the Wrong Mixed-Race Person Featured in a News Story","description":"A news story published on MSN.com featured a photo of the wrong mixed-race person that was allegedly selected by an algorithm, following Microsoft’s layoff and replacement of journalists and editorial workers at its organizations with AI systems.","mit_domain":"Discrimination and Toxicity","mit_subdomain":"Unfair discrimination and misrepresentation","entity":"AI","intent":"Unintentional","timing":"Post-deployment","harm_level":"none","sectors":["arts, entertainment and recreation","information and communication"],"countries":[],"deployers":["Microsoft, Msn.Com"],"developers":["Microsoft"],"harmed":["Jade Thirlwall, Leigh Anne Pinnock"],"report_count":12},{"incident_id":140,"occurred_on":"2020-06-01","title":"University of Toronto's ProctorU Deployment Reportedly Disadvantaged BIPOC Students During Exam Check-Ins","description":"University of Toronto students reported that ProctorU and other online exam-monitoring services created discriminatory and stressful check-in experiences for BIPOC students during remote exams. One U of T student said ProctorU's identity-verification process often failed to recognize her passport, forcing manual checks and consuming exam check-in time. ProctorU said human proctors made final identity and integrity decisions.","mit_domain":"Discrimination and Toxicity","mit_subdomain":"Unequal performance across groups","entity":"AI","intent":"Unintentional","timing":"Post-deployment","harm_level":"none","sectors":["education"],"countries":["CA"],"deployers":["University Of Toronto"],"developers":["Proctoru"],"harmed":["University Students, University Of Toronto Students, Students, Educational Communities, Students Of Color"],"report_count":1},{"incident_id":138,"occurred_on":"2020-03-23","title":"University of Illinois' Proctorio Remote-Proctoring Deployment Reportedly Raised Student Rights Concerns","description":"Beginning in March 2020, the University of Illinois Urbana-Champaign deployed Proctorio for remote exam proctoring after the COVID-19 shift to online instruction. Students and faculty raised concerns that the software surveilled students' homes and devices, created accessibility barriers, increased testing anxiety, and could flag students when facial-detection tools failed to detect their faces, with particular concern for students of color. UIUC later declined to renew its emergency contract.","mit_domain":"Discrimination and Toxicity","mit_subdomain":"Unequal performance across groups","entity":"AI","intent":"Unintentional","timing":"Post-deployment","harm_level":null,"sectors":[],"countries":[],"deployers":["University Of Illinois Urbana Champaign"],"developers":["Proctorio"],"harmed":["Students, University Students, Educational Communities, University Of Illinois Urbana Champaign Students, Students Of Color, Students With Disabilities"],"report_count":6},{"incident_id":102,"occurred_on":"2020-03-23","title":"Personal voice assistants struggle with black voices, new study shows","description":"A study found that voice recognition tools from Apple, Amazon, Google, IBM, and Microsoft disproportionately made errors when transcribing black speakers.","mit_domain":"Discrimination and Toxicity","mit_subdomain":"Unequal performance across groups","entity":"AI","intent":"Unintentional","timing":"Post-deployment","harm_level":"none","sectors":["administrative and support service activities","information and communication"],"countries":["US"],"deployers":["Microsoft, Ibm, Google, Apple, Amazon"],"developers":["Microsoft, Ibm, Google, Apple, Amazon"],"harmed":["Black People"],"report_count":2},{"incident_id":424,"occurred_on":"2020-03-09","title":"Canadian Universities' AI Proctoring Tools Reportedly Collected Student Biometric Data Without Adequate Consent","description":"During the COVID-19 pandemic, Canadian universities reportedly used AI-enabled remote-proctoring tools for online exams. A University of Ottawa project supported by the Office of the Privacy Commissioner of Canada found that tools such as Respondus Monitor, ProctorU, ProctorTrack, Proctorio, ProctorExam, and Examity collected student personal or biometric data under conditions not conducive to meaningful consent.","mit_domain":"Discrimination and Toxicity","mit_subdomain":"Unfair discrimination and misrepresentation","entity":"AI","intent":"Unintentional","timing":"Post-deployment","harm_level":null,"sectors":[],"countries":[],"deployers":["Universities, Professors, Educators, Canadian Universities"],"developers":["Respondus Monitor, Proctoru, Proctortrack, Proctorio, Proctorexam, Examity"],"harmed":["University Students, Universities, Students, Educational Communities, Canadian Students, Privacy, Biometric Data Subjects"],"report_count":4},{"incident_id":117,"occurred_on":"2020-02-24","title":"TikTok's \"Suggested Accounts\" Algorithm Allegedly Reinforced Racial Bias through Feedback Loops","description":"TikTok's \"Suggested Accounts\" recommendations allegedly reinforced racial bias despite not basing recommendations on race or creators' profile photo.","mit_domain":"Discrimination and Toxicity","mit_subdomain":"Unfair discrimination and misrepresentation","entity":"AI","intent":"Unintentional","timing":"Post-deployment","harm_level":null,"sectors":[],"countries":[],"deployers":["Tiktok"],"developers":["Tiktok"],"harmed":["Tiktok Users, Tiktok Content Creators"],"report_count":4},{"incident_id":180,"occurred_on":"2020-02-19","title":"Algorithm Used by the Malaysian Judiciary Reportedly Recommended Unusually High Sentencing to a Drug Possession Case","description":"The AI system used by the Malaysian judiciary which explicitly considered age, employment, and socio-economic data provided sentencing to a drug possession case that was alleged by lawyer to be disproportionately high for the crime committed.","mit_domain":"Discrimination and Toxicity","mit_subdomain":"Unfair discrimination and misrepresentation","entity":"AI","intent":"Intentional","timing":"Post-deployment","harm_level":null,"sectors":[],"countries":[],"deployers":["Malaysian Judiciary, Malaysian Courts"],"developers":["Sarawak Information Systems"],"harmed":["Malaysian Convicted People"],"report_count":3},{"incident_id":166,"occurred_on":"2020-02-07","title":"Networking Platform Giggle Employs AI to Determine Users’ Gender, Allegedly Excluding Transgender Women","description":"A social networking platform, Giggle, allegedly collected, shared to third-parties, and used sensitive information and biometric data to verify whether a person is a woman via facial recognition, which critics claimed to be discriminatory against women of color and harmful towards trans women.","mit_domain":"Discrimination and Toxicity","mit_subdomain":"Unfair discrimination and misrepresentation","entity":"AI","intent":"Intentional","timing":"Post-deployment","harm_level":null,"sectors":[],"countries":[],"deployers":["Giggle"],"developers":["Kairos"],"harmed":["Women of color","Women and girls","Women","Trans women","Privacy","Biometric data subjects"],"report_count":2},{"incident_id":74,"occurred_on":"2020-01-30","title":"Detroit Police Allegedly Wrongfully Arrested Black Man Due to Purportedly Faulty Facial Recognition Technology","description":"A Black man was allegedly wrongfully detained by the Detroit Police Department as a result of a purportedly false facial recognition technology (FRT) result.","mit_domain":"Discrimination and Toxicity","mit_subdomain":"Unequal performance across groups","entity":"AI","intent":"Unintentional","timing":"Post-deployment","harm_level":"AI tangible harm event","sectors":["law enforcement"],"countries":["US"],"deployers":["Detroit Police Department"],"developers":["DataWorks Plus"],"harmed":["Robert Julian-Borchak Williams","Black people in Detroit"],"report_count":11},{"incident_id":214,"occurred_on":"2020-01-02","title":"SN Technologies Reportedly Lied to a New York State School District about Its Facial and Weapon Detection Systems’ Performance","description":"SN Technologies allegedly misled Lockport City Schools about the performance of its AEGIS face and weapons detection systems, downplaying error rates for Black faces and weapon misidentification.","mit_domain":"Discrimination and Toxicity","mit_subdomain":"Unequal performance across groups","entity":"AI","intent":"Unintentional","timing":"Post-deployment","harm_level":"AI tangible harm near-miss","sectors":["education","law enforcement","public administration"],"countries":["US"],"deployers":["Lockport City School District"],"developers":["SN Technologies"],"harmed":["Students","Epistemic integrity","Educational communities","Black students"],"report_count":1},{"incident_id":417,"occurred_on":"2019-11-15","title":"Facebook Feed Algorithms Exposed Low Digitally Skilled Users to More Disturbing Content","description":"Facebook feed algorithms were known by internal research to have harmed people having low digital literacy by exposing them to disturbing content they did not know how to avoid or monitor.","mit_domain":"Discrimination and Toxicity","mit_subdomain":"Unequal performance across groups","entity":"AI","intent":"Unintentional","timing":"Post-deployment","harm_level":null,"sectors":[],"countries":[],"deployers":["Facebook"],"developers":["Facebook"],"harmed":["Low Digitally Skilled Facebook Users"],"report_count":4},{"incident_id":92,"occurred_on":"2019-11-11","title":"Apple Card's Credit Assessment Algorithm Allegedly Discriminated against Women","description":"Apple Card's credit assessment algorithm was reported by Goldman-Sachs customers to have shown gender bias, in which men received significantly higher credit limits than women with equal credit qualifications.","mit_domain":"Discrimination and Toxicity","mit_subdomain":"Unfair discrimination and misrepresentation","entity":"AI","intent":"Unintentional","timing":"Post-deployment","harm_level":"AI tangible harm event","sectors":["financial and insurance activities"],"countries":["UNITED STATES"],"deployers":["Goldman-Sachs"],"developers":["Apple"],"harmed":["Women and girls","Women","Apple card users","Apple card credit applicants"],"report_count":6},{"incident_id":95,"occurred_on":"2019-11-06","title":"Job Screening Service Halts Facial Analysis of Applicants","description":"In January 2021, HireVue removed the controversial AI expression tracking tool from its virtual job interview software.","mit_domain":"Discrimination and Toxicity","mit_subdomain":"Unfair discrimination and misrepresentation","entity":"AI","intent":"Intentional","timing":"Post-deployment","harm_level":"AI tangible harm issue","sectors":["administrative and support service activities","professional, scientific and technical activities"],"countries":["US"],"deployers":["Hirevue"],"developers":["Hirevue"],"harmed":["Job Applicants Using Hirevue, Hirevue Customers"],"report_count":4},{"incident_id":124,"occurred_on":"2019-10-24","title":"Optum Algorithmic Health Risk Scores Reportedly Underestimated Black Patients' Needs","description":"Researchers analyzing an Optum algorithm used by a large academic hospital reportedly found evidence that it tended to under-predict the health care needs of Black patients, which in turn appeared to result in lower prioritization for extra care programs compared with white patients exhibiting similar health profiles.","mit_domain":"Discrimination and Toxicity","mit_subdomain":"Unfair discrimination and misrepresentation","entity":"AI","intent":"Unintentional","timing":"Post-deployment","harm_level":"AI tangible harm event","sectors":["human health and social work activities"],"countries":["US"],"deployers":["unnamed large academic hospital"],"developers":["Optum"],"harmed":["Black patients"],"report_count":7},{"incident_id":189,"occurred_on":"2019-10-15","title":"Opaque Fraud Detection Algorithm by the UK’s Department of Work and Pensions Allegedly Discriminated against People with Disabilities","description":"People with disabilities were allegedly disproportionately targeted by a benefit fraud detection algorithm which the UK’s Department of Work and Pensions was urged to disclose.","mit_domain":"Discrimination and Toxicity","mit_subdomain":"Unequal performance across groups","entity":"AI","intent":"Intentional","timing":"Post-deployment","harm_level":null,"sectors":[],"countries":[],"deployers":["UK Department of Work and Pensions"],"developers":["UiPath"],"harmed":["People with disabilities"],"report_count":6},{"incident_id":272,"occurred_on":"2019-10-08","title":"Grab Tweaked Matchmaking Algorithm, Providing Preferential Treatment to Drivers Registered with Affiliated Car Rental Service","description":"Grab Indonesia was fined by the Indonesian Competition Commission (KPPU) for unfairly favoring drivers who rented cars via the Grab-affiliated company Teknologi Pengangkutan Indonesia (TPI), including offering more rides via their matchmaking algorithm.","mit_domain":"Discrimination and Toxicity","mit_subdomain":"Unfair discrimination and misrepresentation","entity":"AI","intent":"Intentional","timing":"Post-deployment","harm_level":null,"sectors":[],"countries":[],"deployers":["Grab"],"developers":["Grab"],"harmed":["non-TPI-registered Grab drivers","Grab drivers in Indonesia","Grab drivers"],"report_count":3},{"incident_id":439,"occurred_on":"2019-07-31","title":"Detroit Police Wrongfully Arrested Black Man Due To Faulty Facial Recognition","description":"A Black man was wrongfully detained by the Detroit Police Department as a result of a false facial recognition (FRT) result.","mit_domain":"Discrimination and Toxicity","mit_subdomain":"Unfair discrimination and misrepresentation","entity":"AI","intent":"Unintentional","timing":"Post-deployment","harm_level":null,"sectors":[],"countries":[],"deployers":["Detroit Police Department"],"developers":["DataWorks Plus"],"harmed":["Michael Oliver","Black people in Detroit","Black people"],"report_count":3},{"incident_id":386,"occurred_on":"2019-07-03","title":"Amazon’s \"Time Off Task\" System Made False Assumptions about Workers' Time Management","description":"Amazon’s warehouse worker “time off task\" (TOT) tracking system was used to discipline and dismiss workers, falsely assuming workers to have wasted time and failing to account for breaks or equipment issues.","mit_domain":"Discrimination and Toxicity","mit_subdomain":"Unequal performance across groups","entity":"AI","intent":"Unintentional","timing":"Post-deployment","harm_level":null,"sectors":[],"countries":[],"deployers":["Amazon"],"developers":["Amazon"],"harmed":["Amazon warehouse workers"],"report_count":3},{"incident_id":279,"occurred_on":"2019-07-01","title":"TikTok’s “For You” Algorithm Exposed Young Users to Pro-Eating Disorder Content","description":"TikTok’s young users were allegedly exposed to community-guideline-violating pro-eating disorder content on their algorithmically curated “For You” page that serves videos from any user on its platform.","mit_domain":"Discrimination and Toxicity","mit_subdomain":"Exposure to toxic content","entity":"AI","intent":"Unintentional","timing":"Post-deployment","harm_level":null,"sectors":[],"countries":[],"deployers":["Tiktok"],"developers":["Tiktok"],"harmed":["Tiktok Young Users, Tiktok Users"],"report_count":3},{"incident_id":471,"occurred_on":"2019-06-22","title":"Facebook Allegedly Failed to Police Hate Speech Content That Contributed to Ethnic Violence in Ethiopia","description":"Facebook allegedly did not adequately remove hate speech, some of which was extremely violent and dehumanizing, on its platform including through automated means, contributing to the violence faced by ethnic communities in Ethiopia.","mit_domain":"Discrimination and Toxicity","mit_subdomain":"Exposure to toxic content","entity":"AI","intent":"Unintentional","timing":"Post-deployment","harm_level":null,"sectors":[],"countries":[],"deployers":["Meta, Facebook"],"developers":["Meta, Facebook"],"harmed":["Tigrinya Speaking Facebook Users, Facebook Users In Ethiopia, Ethiopian Public, Afaan Oromo Speaking Facebook Users"],"report_count":8},{"incident_id":489,"occurred_on":"2019-06-03","title":"Workday's AI Tools Allegedly Enabled Employers to Discriminate against Applicants of Protected Groups","description":"Workday's algorithmic screening systems were alleged in a lawsuit allowing employers to discriminate against African-Americans, people over 40, and people with disabilities.","mit_domain":"Discrimination and Toxicity","mit_subdomain":"Unfair discrimination and misrepresentation","entity":"AI","intent":"Unintentional","timing":"Post-deployment","harm_level":"none","sectors":["professional, scientific and technical activities","information and communication"],"countries":["US"],"deployers":["Workday"],"developers":["Workday"],"harmed":["Derek Mobley, Applicants With Disabilities, Applicants Over 40, African American Applicants"],"report_count":1},{"incident_id":546,"occurred_on":"2019-05-31","title":"Algorithm to Distribute Social Welfare Reported for Oversimplifying Economic Vulnerability","description":"Takaful cash transfer program's algorithm which ranks families by their economic vulnerability level to determine financial assistance reportedly oversimplified people's economic situation, fueling social tension and perceptions of unfairness.","mit_domain":"Discrimination and Toxicity","mit_subdomain":"Unfair discrimination and misrepresentation","entity":"AI","intent":"Unintentional","timing":"Post-deployment","harm_level":null,"sectors":[],"countries":[],"deployers":["National Aid Fund"],"developers":["The World Bank, Unicef, World Food Programme"],"harmed":["Jordanians In Poverty"],"report_count":3},{"incident_id":161,"occurred_on":"2019-04-03","title":"Facebook's Ad Delivery Reportedly Excluded Audience along Racial and Gender Lines","description":"Facebook's housing and employment ad delivery process allegedly resulted in skews in exposure for some users along demographic lines such as gender and racial identity.","mit_domain":"Discrimination and Toxicity","mit_subdomain":"Unfair discrimination and misrepresentation","entity":"AI","intent":"Unintentional","timing":"Post-deployment","harm_level":null,"sectors":[],"countries":[],"deployers":["Meta"],"developers":["Meta"],"harmed":["Women and girls","Women","Social media users","Facebook users","Black Facebook users"],"report_count":3},{"incident_id":89,"occurred_on":"2019-03-15","title":"The Christchurch shooter and YouTube’s radicalization trap","description":"A New Zealand government report released following a right-wing terrorist killing 51 worshippers at two New Zealand mosques which indicated that Youtube's recommendation algorithm played an important role in the terrorist's radicalization.","mit_domain":"Discrimination and Toxicity","mit_subdomain":"Exposure to toxic content","entity":"AI","intent":"Unintentional","timing":"Post-deployment","harm_level":"none","sectors":["information and communication","arts, entertainment and recreation"],"countries":["NZ"],"deployers":["Youtube"],"developers":["Youtube"],"harmed":["Youtube Users"],"report_count":1},{"incident_id":281,"occurred_on":"2019-02-04","title":"YouTube's Algorithms Failed to Remove Violating Content Related to Suicide and Self-Harm","description":"Terms-of-service-violating videos related to suicide and self-harm reportedly bypassed YouTube’s content moderation algorithms, allegedly resulting in exposure of graphic content to young users via recommended videos.","mit_domain":"Discrimination and Toxicity","mit_subdomain":"Exposure to toxic content","entity":"AI","intent":"Unintentional","timing":"Post-deployment","harm_level":"none","sectors":["information and communication","arts, entertainment and recreation"],"countries":[],"deployers":["Youtube"],"developers":["Youtube"],"harmed":["Youtube Young Users, Youtube Users"],"report_count":3},{"incident_id":288,"occurred_on":"2019-01-30","title":"New Jersey Police Wrongful Arrested Innocent Black Man via FRT","description":"Woodbridge Police Department falsely arrested an innocent Black man following a misidentification by their facial recognition software, who was jailed for more than a week and paid thousands of dollar for his defense.","mit_domain":"Discrimination and Toxicity","mit_subdomain":"Unequal performance across groups","entity":"AI","intent":"Unintentional","timing":"Post-deployment","harm_level":null,"sectors":[],"countries":[],"deployers":["Woodbridge Police Department"],"developers":["Unknown"],"harmed":["Nijeer Parks"],"report_count":4},{"incident_id":388,"occurred_on":"2018-12-01","title":"Facial Recognition Pilot in Bahia Reportedly Targeted Black and Poor People","description":"Facial recognition deployed in a pilot project by the local government of Bahia despite having minimal hit rate reportedly targeted Black and poor people disproportionately.","mit_domain":"Discrimination and Toxicity","mit_subdomain":"Unfair discrimination and misrepresentation","entity":"AI","intent":"Unintentional","timing":"Post-deployment","harm_level":null,"sectors":[],"countries":[],"deployers":["The Government In Bahia, Bahia'S Secretary Of Public Security"],"developers":["Huawei"],"harmed":["Black People In Brazil, Black People In Bahia"],"report_count":1},{"incident_id":405,"occurred_on":"2018-11-28","title":"Schufa Credit Scoring in Germany Reported for Unreliable and Imbalanced Scores","description":"Creditworthiness Schufa scores in Germany reportedly privileged older and female consumers, and people who changed addresses less frequently, and were unreliable depending on scoring version.","mit_domain":"Discrimination and Toxicity","mit_subdomain":"Unfair discrimination and misrepresentation","entity":"AI","intent":"Unintentional","timing":"Post-deployment","harm_level":null,"sectors":[],"countries":[],"deployers":["Schufa Holding AG"],"developers":["Schufa Holding AG"],"harmed":["young men having credit scores","people scored on old scoring versions","people changing addresses frequently"],"report_count":2},{"incident_id":454,"occurred_on":"2018-11-09","title":"Emotion Detection Models Showed Disparate Performance along Racial Lines","description":"Emotion detection tools by Face++ and Microsoft's Face API allegedly scored smiling or defaulted ambiguous facial photos for Black faces as negative emotion more often than for white faces.","mit_domain":"Discrimination and Toxicity","mit_subdomain":"Unfair discrimination and misrepresentation","entity":"AI","intent":"Unintentional","timing":"Post-deployment","harm_level":"none","sectors":["information and communication","unclear"],"countries":["US"],"deployers":["Megvii, Microsoft"],"developers":["Megvii, Microsoft"],"harmed":["Black People"],"report_count":2},{"incident_id":238,"occurred_on":"2018-10-01","title":"Oregon’s Screening Tool for Child Abuse Cases Discontinued Following Concerns of Racial Bias","description":"Oregon’s Department of Human Services (DHS) stopped using its Safety at Screening Tool, that is aimed to predict the risk that children wind up in foster care or be investigated in the future, and opted for a new process allegedly to reduce disparities and improve racially equitable decision-making.","mit_domain":"Discrimination and Toxicity","mit_subdomain":"Unfair discrimination and misrepresentation","entity":"AI","intent":"Unintentional","timing":"Post-deployment","harm_level":null,"sectors":[],"countries":[],"deployers":["Oregon Department of Human Services"],"developers":["Oregon Department of Human Services"],"harmed":["families of minority groups","children of minority groups"],"report_count":1},{"incident_id":164,"occurred_on":"2018-10-01","title":"Facebook \"News Feed\" Allegedly Boosted Misinformation and Violating Content Following Use of MSI Metric","description":"After the “News Feed” algorithm had been overhauled to boost engagement between friends and family in early 2018, its heavy weighting of re-shared content was alleged found by company researchers to have pushed content creators to reorient their posts towards outrage and sensationalism, causing a proliferation of misinformation, toxicity, and violent content.","mit_domain":"Discrimination and Toxicity","mit_subdomain":"Exposure to toxic content","entity":"AI","intent":"Unintentional","timing":"Post-deployment","harm_level":"AI tangible harm event","sectors":["information and communication"],"countries":[],"deployers":["Facebook"],"developers":["Facebook"],"harmed":["Facebook Users, Facebook Content Creators"],"report_count":1},{"incident_id":101,"occurred_on":"2018-09-01","title":"Dutch Families Wrongfully Accused of Tax Fraud Due to Discriminatory Algorithm","description":"A childcare benefits system in the Netherlands falsely accused thousands of families of fraud, in part due to an algorithm that treated having a second nationality as a risk factor.","mit_domain":"Discrimination and Toxicity","mit_subdomain":"Unfair discrimination and misrepresentation","entity":"AI","intent":"Unintentional","timing":"Post-deployment","harm_level":"AI tangible harm event","sectors":["public administration"],"countries":["NL"],"deployers":["Dutch Tax Authority"],"developers":["Unknown"],"harmed":["Dutch Tax Authority, Dutch Families"],"report_count":6},{"incident_id":169,"occurred_on":"2018-08-15","title":"Facebook Allegedly Failed to Police Anti-Rohingya Hate Speech Content That Contributed to Violence in Myanmar","description":" Facebook allegedly did not adequately remove anti-Rohingya hate speech, some of which was extremely violent and dehumanizing, on its platform, contributing to the violence faced by Rohingya communities in Myanmar.","mit_domain":"Discrimination and Toxicity","mit_subdomain":"Exposure to toxic content","entity":"AI","intent":"Unintentional","timing":"Post-deployment","harm_level":null,"sectors":[],"countries":[],"deployers":["Facebook, Meta"],"developers":["Facebook, Meta"],"harmed":["Rohingya People, Rohingya Facebook Users, Myanmar Public, Facebook Users In Myanmar, Burmese Speaking Facebook Users"],"report_count":5},{"incident_id":93,"occurred_on":"2018-08-13","title":"HUD charges Facebook with enabling housing discrimination","description":"In March 2019 the U.S. Department of Housing and Urban Development charged Facebook with violating the Fair Housing Act by allowing real estate sellers to target advertisements in a discriminatory manner.","mit_domain":"Discrimination and Toxicity","mit_subdomain":"Unfair discrimination and misrepresentation","entity":"AI","intent":"Unintentional","timing":"Post-deployment","harm_level":"none","sectors":["information and communication"],"countries":["US"],"deployers":["Facebook"],"developers":["Facebook"],"harmed":["Non American Born Facebook Users, Non Christian Facebook Users, Facebook Users Interested In Accessibility, Facebook Users Interested In Hispanic Culture, Minority Groups"],"report_count":4},{"incident_id":114,"occurred_on":"2018-07-26","title":"Amazon's Rekognition Falsely Matched Members of Congress to Mugshots","description":"Rekognition's face comparison feature was shown by the ACLU to have misidentified members of congress, and particularly members of colors, as other people who have been arrested using a mugshot database built on publicly available arrest photos.","mit_domain":"Discrimination and Toxicity","mit_subdomain":"Unequal performance across groups","entity":"AI","intent":"Unintentional","timing":"Post-deployment","harm_level":"none","sectors":["information and communication","law enforcement"],"countries":["US"],"deployers":["Amazon"],"developers":["Amazon"],"harmed":["Rekognition Users, Arrested People"],"report_count":1},{"incident_id":107,"occurred_on":"2018-07-20","title":"Chinese Tech Firms Allegedly Developed Facial Recognition to Identify People by Race, Targeting Uyghur Muslims","description":"Various Chinese firms were revealed by patent applications to have developed facial recognition capable of detecting people by race, which critics feared would enable persecution and discrimination of Uyghur Muslims.","mit_domain":"Discrimination and Toxicity","mit_subdomain":"Unfair discrimination and misrepresentation","entity":"AI","intent":"Intentional","timing":"Post-deployment","harm_level":null,"sectors":[],"countries":[],"deployers":["None"],"developers":["Huawei, Megvii, Sensetime, Alibaba, Baibu"],"harmed":["Uyghur People"],"report_count":2},{"incident_id":396,"occurred_on":"2018-07-04","title":"Transgender Uber Drivers Mistakenly Kicked off App for Appearance Change during Gender Transitions","description":"Transgender Uber drivers reported being automatically deactivated from the app due to Real-Time ID Check failing to account for difference in appearance of people undergoing gender transitions.","mit_domain":"Discrimination and Toxicity","mit_subdomain":"Unfair discrimination and misrepresentation","entity":"AI","intent":"Unintentional","timing":"Post-deployment","harm_level":null,"sectors":[],"countries":[],"deployers":["Uber"],"developers":["Uber"],"harmed":["transgender Uber drivers"],"report_count":1},{"incident_id":229,"occurred_on":"2018-04-23","title":"Content Using Bestiality Thumbnails Allegedly Evaded YouTube’s Thumbnail Monitoring System","description":"YouTube’s thumbnail monitoring system was allegedly evaded by content farms such as ones in Cambodia who spike viewership and generate ad revenue using bestiality-themed thumbnails.","mit_domain":"Discrimination and Toxicity","mit_subdomain":"Exposure to toxic content","entity":"AI","intent":"Unintentional","timing":"Post-deployment","harm_level":null,"sectors":[],"countries":[],"deployers":["Youtube"],"developers":["Youtube"],"harmed":["Youtube Users, Youtube Content Creators"],"report_count":2},{"incident_id":188,"occurred_on":"2018-04-11","title":"Argentinian City Government Deployed Teenage-Pregnancy Predictive Algorithm Using Invasive Demographic Data","description":"In 2018, during the abortion-decriminalization debate in Argentina, the Salta city government deployed a teenage-pregnancy predictive algorithm built by Microsoft that allegedly lacked a defined purpose, explicitly considered sensitive information such as disability and whether their home had access to hot water.","mit_domain":"Discrimination and Toxicity","mit_subdomain":"Unfair discrimination and misrepresentation","entity":"AI","intent":"Intentional","timing":"Post-deployment","harm_level":null,"sectors":[],"countries":[],"deployers":["Salta city government","Government of Argentina"],"developers":["Microsoft"],"harmed":["Women and girls","Women","Salta teenage girls","Salta girls of minority groups","Privacy","Girls","Adolescent girls"],"report_count":4},{"incident_id":517,"occurred_on":"2018-02-15","title":"Man Arrested For Sock Theft by False Facial Match Despite Alibi","description":"A man was arrested for theft of socks from a TJ Maxx store under the guise of an eyewitness ID case, after the local police asked the store's security guard to confirm the facial recognition match produced using surveillance footage, despite him having an alibi at the time of the theft.","mit_domain":"Discrimination and Toxicity","mit_subdomain":"Unequal performance across groups","entity":"AI","intent":"Intentional","timing":"Post-deployment","harm_level":null,"sectors":[],"countries":[],"deployers":["New York Police Department"],"developers":["Unknown"],"harmed":["Unknown"],"report_count":2},{"incident_id":382,"occurred_on":"2017-11-21","title":"Instagram's Exposure of Harmful Content Allegedly Contributed to Teenage Girl's Suicide","description":"Instagram was ruled by a judge to have contributed to the death of a teenage girl in the UK allegedly through its exposure and recommendation of suicide, self-harm, and depressive content.","mit_domain":"Discrimination and Toxicity","mit_subdomain":"Exposure to toxic content","entity":"AI","intent":"Unintentional","timing":"Post-deployment","harm_level":null,"sectors":[],"countries":[],"deployers":["Meta"],"developers":["Meta"],"harmed":["Women and girls","Molly Rose Russell","Girls","Family of Molly Rose Russell","Adolescent girls"],"report_count":1},{"incident_id":261,"occurred_on":"2017-11-15","title":"Robot Deployed by Animal Shelter to Patrol Sidewalks outside Its Office, Warding off Homeless People in San Francisco","description":"Society for the Prevention of Cruelty to Animals (SPCA) deployed a Knightscope robot to autonomously patrol the area outside its office and ward off homeless people, which was criticized by residents as a tool of intimidation and ordered by the city of San Francisco to stop its use on a public right-of-way.","mit_domain":"Discrimination and Toxicity","mit_subdomain":"Unfair discrimination and misrepresentation","entity":"Human","intent":"Unintentional","timing":"Post-deployment","harm_level":null,"sectors":[],"countries":[],"deployers":["Society For The Prevention Of Cruelty To Animals"],"developers":["Knightscope"],"harmed":["San Francisco Homeless People"],"report_count":8},{"incident_id":14,"occurred_on":"2017-10-26","title":"Biased Sentiment Analysis","description":"Google Cloud's Natural Language API provided racist, homophobic, amd antisemitic sentiment analyses.","mit_domain":"Discrimination and Toxicity","mit_subdomain":"Unfair discrimination and misrepresentation","entity":"AI","intent":"Unintentional","timing":"Post-deployment","harm_level":"none","sectors":["information and communication"],"countries":[],"deployers":["Google"],"developers":["Google"],"harmed":["Women and girls","Women","Minority groups"],"report_count":6},{"incident_id":58,"occurred_on":"2017-10-12","title":"Russian Chatbot Supports Stalin and Violence","description":"Yandex, a Russian technology company, released an artificially intelligent chat bot named Alice which began to reply to questions with racist, pro-stalin, and pro-violence responses","mit_domain":"Discrimination and Toxicity","mit_subdomain":"Exposure to toxic content","entity":"AI","intent":"Unintentional","timing":"Post-deployment","harm_level":"none","sectors":["information and communication"],"countries":["RU"],"deployers":["Yandex"],"developers":["Yandex"],"harmed":["Yandex Users"],"report_count":5},{"incident_id":216,"occurred_on":"2017-10-10","title":"WeChat’s Machine Translation Gave a Racist English Translation for the Chinese Term for “Black Foreigner”","description":"The Chinese platform WeChat provided an inappropriate and racist English translation for the Chinese term for “black foreigner” in its messaging app.","mit_domain":"Discrimination and Toxicity","mit_subdomain":"Exposure to toxic content","entity":"AI","intent":"Unintentional","timing":"Post-deployment","harm_level":null,"sectors":[],"countries":[],"deployers":["Wechat"],"developers":["Wechat"],"harmed":["Black Wechat Users"],"report_count":5},{"incident_id":325,"occurred_on":"2017-09-21","title":"Offensive Instagram User Content Displayed as Facebook Ad","description":"An Instagram user’s image containing violent content was reportedly used as advertisement on Facebook allegedly via automated means.","mit_domain":"Discrimination and Toxicity","mit_subdomain":"Exposure to toxic content","entity":"AI","intent":"Unintentional","timing":"Post-deployment","harm_level":null,"sectors":[],"countries":[],"deployers":["Facebook"],"developers":["Facebook"],"harmed":["Olivia Solon, Olivia Solon'S Facebook Connections"],"report_count":2},{"incident_id":531,"occurred_on":"2017-09-15","title":"AI-Assisted Body Scanners Reportedly Subjected Transgender Travelers to Invasive Body Searches","description":"Transportation Security Administration (TSA)'s use of image-processing body scanners at airports led transgender and gender-nonconforming travelers to be subjected to allegedly discriminatory and invasive searches, such as being asked to remove undergarments in private rooms by officers not of their gender.","mit_domain":"Discrimination and Toxicity","mit_subdomain":"Unfair discrimination and misrepresentation","entity":"AI","intent":"Unintentional","timing":"Post-deployment","harm_level":null,"sectors":[],"countries":[],"deployers":["Transportation Security Administration"],"developers":["L3Harris Technologies"],"harmed":["Transgender Travelers, Gender Nonconforming Travelers"],"report_count":1},{"incident_id":56,"occurred_on":"2017-07-10","title":"AI-Designed Phone Cases Are Unexpected","description":"A third-party Amazon merchant named “my_handy_design” was suspected of using a bot to generate cell phone case designs based on the bizarre and unattractive designs being offered.","mit_domain":"Discrimination and Toxicity","mit_subdomain":"Exposure to toxic content","entity":"AI","intent":"Unintentional","timing":"Post-deployment","harm_level":"none","sectors":["wholesale and retail trade"],"countries":[],"deployers":["My_Handy_Design"],"developers":["My_Handy_Design"],"harmed":["My_Handy_Design"],"report_count":7},{"incident_id":183,"occurred_on":"2017-07-01","title":"Airbnb's Trustworthiness Algorithm Allegedly Banned Users without Explanation, and Discriminated against Sex Workers","description":" Airbnb allegedly considered publicly available data on users to gauge their trustworthiness via algorithmic assessment of personality and behavioral traits, resulting in unexplained bans and discriminatory bans against sex workers.","mit_domain":"Discrimination and Toxicity","mit_subdomain":"Unfair discrimination and misrepresentation","entity":"AI","intent":"Intentional","timing":"Post-deployment","harm_level":null,"sectors":[],"countries":[],"deployers":["Airbnb"],"developers":["Airbnb, Trooly"],"harmed":["Sex Workers, Airbnb Users"],"report_count":6},{"incident_id":60,"occurred_on":"2017-04-25","title":"FaceApp Racial Filters","description":"FaceApp is criticized for offering racist filters.","mit_domain":"Discrimination and Toxicity","mit_subdomain":"Unfair discrimination and misrepresentation","entity":"AI","intent":"Unintentional","timing":"Post-deployment","harm_level":"none","sectors":["arts, entertainment and recreation","information and communication"],"countries":["RU"],"deployers":["Faceapp"],"developers":["Faceapp"],"harmed":["Minority Groups"],"report_count":23},{"incident_id":59,"occurred_on":"2017-04-13","title":"Gender Biases in Google Translate","description":"A Cornell University study in 2016 highlighted Google Translate's pattern of assigning gender to occupations in a way showing an implicit gender bias against women.","mit_domain":"Discrimination and Toxicity","mit_subdomain":"Unfair discrimination and misrepresentation","entity":"AI","intent":"Unintentional","timing":"Post-deployment","harm_level":"none","sectors":["information and communication"],"countries":[],"deployers":["Google"],"developers":["Google"],"harmed":["Women and girls","Women"],"report_count":10},{"incident_id":502,"occurred_on":"2017-04-10","title":"Pennsylvania County's Family Screening Tool Allegedly Exhibited Discriminatory Effects","description":"Data analysis by the American Civil Liberty Union (ACLU) on Allegheny County's decision-support Family Screening Tool to predict child abuse or neglect risk found the tool resulting in higher screen-in rates for Black families and higher risk scores for households with disabled residents.","mit_domain":"Discrimination and Toxicity","mit_subdomain":"Unfair discrimination and misrepresentation","entity":"AI","intent":"Unintentional","timing":"Post-deployment","harm_level":null,"sectors":[],"countries":[],"deployers":["Allegheny County"],"developers":["Rhema Vaithianathan","Emily Putnam-Hornstein","Centre for Social Data Analytics"],"harmed":["People with disabilities","Hackneys family","Black families in Allegheny"],"report_count":3},{"incident_id":394,"occurred_on":"2017-03-15","title":"Social Media's Automated Word-Flagging without Context Shifted Content Creators' Language Use","description":"TikTok's, YouTube's, Instagram's, and Twitch's use of algorithms to flag certain words devoid of context changed content creators' use of everyday language or discussion about certain topics in fear of their content getting flagged or auto-demonetized by mistake.","mit_domain":"Discrimination and Toxicity","mit_subdomain":"Unfair discrimination and misrepresentation","entity":"AI","intent":"Unintentional","timing":"Post-deployment","harm_level":null,"sectors":[],"countries":[],"deployers":["Youtube, Twitch, Tiktok, Instagram"],"developers":["Youtube, Twitch, Tiktok, Instagram"],"harmed":["Youtube Content Creators, Twitch Content Creators, Tiktok Content Creators, Instagram Content Creators"],"report_count":2},{"incident_id":418,"occurred_on":"2017-03-13","title":"Uber Locked Indian Drivers out of Accounts Allegedly Due to Facial Recognition Fails","description":"Uber drivers in India reported being locked out of their accounts allegedly due to Real-Time ID Check's facial recognition failing to recognize appearance changes or faces in low lighting conditions.","mit_domain":"Discrimination and Toxicity","mit_subdomain":"Unequal performance across groups","entity":"AI","intent":"Unintentional","timing":"Post-deployment","harm_level":null,"sectors":[],"countries":[],"deployers":["Uber"],"developers":["Uber","Azure Cognitive Services"],"harmed":["Uber drivers in India"],"report_count":3},{"incident_id":13,"occurred_on":"2017-02-27","title":"High-Toxicity Assessed on Text Involving Women and Minority Groups","description":"Google's Perspective API, which assigns a toxicity score to online text, seems to award higher toxicity scores to content involving non-white, male, Christian, heterosexual phrases.","mit_domain":"Discrimination and Toxicity","mit_subdomain":"Unfair discrimination and misrepresentation","entity":"AI","intent":"Unintentional","timing":"Post-deployment","harm_level":"none","sectors":["information and communication"],"countries":["US"],"deployers":["Google"],"developers":["Google"],"harmed":["Women and girls","Women","Minority groups"],"report_count":8},{"incident_id":55,"occurred_on":"2016-12-30","title":"Alexa Plays Pornography Instead of Kids Song","description":"An Amazon Echo Dot using the Amazon Alex software started to play pornographic results when a child asked it to play a song.","mit_domain":"Discrimination and Toxicity","mit_subdomain":"Exposure to toxic content","entity":"AI","intent":"Unintentional","timing":"Post-deployment","harm_level":"none","sectors":["arts, entertainment and recreation","information and communication"],"countries":["US"],"deployers":["Amazon"],"developers":["Amazon"],"harmed":["Minors"],"report_count":16},{"incident_id":48,"occurred_on":"2016-12-07","title":"Passport checker Detects Asian man's Eyes as Closed","description":"New Zealand passport robot reader rejects the application of an applicant with Asian descent and says his eyes are closed.","mit_domain":"Discrimination and Toxicity","mit_subdomain":"Unequal performance across groups","entity":"AI","intent":"Unintentional","timing":"Post-deployment","harm_level":"none","sectors":["public administration"],"countries":["NZ"],"deployers":["New Zealand"],"developers":["New Zealand"],"harmed":["Asian People"],"report_count":20},{"incident_id":472,"occurred_on":"2016-10-08","title":"NYPD's Deployment of Facial Recognition Cameras Reportedly Reinforced Biased Policing","description":"New York Police Department’s use of facial recognition deployment of surveillance cameras were shown using crowdsourced volunteer data reinforcing discriminatory policing against minority communities.","mit_domain":"Discrimination and Toxicity","mit_subdomain":"Unfair discrimination and misrepresentation","entity":"Human","intent":"Intentional","timing":"Post-deployment","harm_level":null,"sectors":[],"countries":[],"deployers":["New York Police Department"],"developers":["Unknown"],"harmed":["racial minorities"],"report_count":1},{"incident_id":47,"occurred_on":"2016-09-06","title":"LinkedIn Search Prefers Male Names","description":"An investigation by The Seattle Times in 2016 found a gender bias in LinkedIn's search engine.","mit_domain":"Discrimination and Toxicity","mit_subdomain":"Unfair discrimination and misrepresentation","entity":"AI","intent":"Unintentional","timing":"Post-deployment","harm_level":"none","sectors":["information and communication"],"countries":["US"],"deployers":["LinkedIn"],"developers":["LinkedIn"],"harmed":["Women and girls","Women"],"report_count":9},{"incident_id":49,"occurred_on":"2016-09-05","title":"AI Beauty Judge Did Not Like Dark Skin","description":"In 2016, after artificial inntelligence software Beauty.AI judged an international beauty contest and declared a majority of winners to be white, researchers found that Beauty.AI was racially biased in determining beauty.","mit_domain":"Discrimination and Toxicity","mit_subdomain":"Unfair discrimination and misrepresentation","entity":"AI","intent":"Unintentional","timing":"Post-deployment","harm_level":"none","sectors":["arts, entertainment and recreation"],"countries":[],"deployers":["Youth Laboratories"],"developers":["Youth Laboratories"],"harmed":["People With Dark Skin"],"report_count":9},{"incident_id":37,"occurred_on":"2016-08-10","title":"Amazon's Experimental Hiring Tool Allegedly Displayed Gender Bias in Candidate Rankings","description":"Between 2014 and 2017, Amazon reportedly developed an AI-powered recruiting tool to score job applicants, trained on a decade of resumes purportedly drawn largely from men. Media reports say the system learned to favor male candidates, penalizing terms like \"women's\" and graduates from certain all-women's colleges. Efforts to remove these biases reportedly did not guarantee fairness, and the project was ultimately abandoned. Amazon reportedly states recruiters never solely relied on the tool.","mit_domain":"Discrimination and Toxicity","mit_subdomain":"Unfair discrimination and misrepresentation","entity":"AI","intent":"Unintentional","timing":"Post-deployment","harm_level":"AI tangible harm issue","sectors":["administrative and support service activities"],"countries":["IE"],"deployers":["Amazon"],"developers":["Amazon"],"harmed":["Women applying to Amazon","Women and girls","Women","Amazon applicants"],"report_count":34},{"incident_id":12,"occurred_on":"2016-07-21","title":"Common Biases of Vector Embeddings","description":"Researchers from Boston University and Microsoft Research, New England demonstrated gender bias in the most common techniques used to embed words for natural language processing (NLP).","mit_domain":"Discrimination and Toxicity","mit_subdomain":"Unfair discrimination and misrepresentation","entity":"AI","intent":"Unintentional","timing":"Post-deployment","harm_level":"none","sectors":["professional, scientific and technical activities"],"countries":["US"],"deployers":["Microsoft Research","Boston University"],"developers":["Microsoft Research","Google","Boston University"],"harmed":["Women and girls","Women","Minority groups"],"report_count":1},{"incident_id":40,"occurred_on":"2016-05-23","title":"COMPAS Algorithm Reportedly Performs Poorly in Crime Recidivism Prediction","description":"Correctional Offender Management Profiling for Alternative Sanctions (COMPAS), a recidivism risk-assessment algorithmic tool used in the judicial system to assess likelihood of defendants' recidivism, is found to be less accurate than random untrained human evaluators.","mit_domain":"Discrimination and Toxicity","mit_subdomain":"Unfair discrimination and misrepresentation","entity":"AI","intent":"Unintentional","timing":"Post-deployment","harm_level":"AI tangible harm event","sectors":["law enforcement","public administration"],"countries":["US"],"deployers":["Equivant"],"developers":["Equivant"],"harmed":["Accused people"],"report_count":21},{"incident_id":11,"occurred_on":"2016-05-23","title":"Northpointe Risk Models","description":"An algorithm developed by Northpointe and used in the penal system is two times more likely to incorrectly label a black person as a high-risk re-offender and is two times more likely to incorrectly label a white person as low-risk for reoffense according to a ProPublica review.","mit_domain":"Discrimination and Toxicity","mit_subdomain":"Unfair discrimination and misrepresentation","entity":"AI","intent":"Unintentional","timing":"Post-deployment","harm_level":"AI tangible harm event","sectors":["law enforcement","public administration"],"countries":["US"],"deployers":["Northpointe"],"developers":["Northpointe"],"harmed":["Accused people"],"report_count":15},{"incident_id":235,"occurred_on":"2016-04-15","title":"Chinese Insurer Ping An Employed Facial Recognition to Determine Customers’ Untrustworthiness, Which Critics Alleged to Likely Make Errors and Discriminate","description":"Customers’ untrustworthiness and unprofitability were reportedly determined by Ping An, a large insurance company in China, via facial-recognition measurements of micro-expressions and body-mass indices (BMI), which critics argue was likely to make mistakes, discriminate against certain ethnic groups, and undermine its own industry.","mit_domain":"Discrimination and Toxicity","mit_subdomain":"Unfair discrimination and misrepresentation","entity":"AI","intent":"Intentional","timing":"Post-deployment","harm_level":null,"sectors":[],"countries":[],"deployers":["Ping An"],"developers":["Ping An"],"harmed":["Ping An customers","Chinese minority groups"],"report_count":1},{"incident_id":332,"occurred_on":"2016-04-05","title":"Google Images Showed Racially Biased Results for 'Professional' Hairstyles","description":"Google Images reportedly showed disparate search results along racial lines, featuring almost exclusively white women for \"professional hairstyles\" and Black women for \"unprofessional hairstyles\" prompts.","mit_domain":"Discrimination and Toxicity","mit_subdomain":"Unfair discrimination and misrepresentation","entity":"AI","intent":"Unintentional","timing":"Post-deployment","harm_level":null,"sectors":[],"countries":[],"deployers":["Google"],"developers":["Google"],"harmed":["Women of color","Women and girls","Women","Google users","Google Images users","Black women","Black people"],"report_count":4},{"incident_id":53,"occurred_on":"2016-03-31","title":"Biased Google Image Results","description":"On June 6, 2016, Google image searches of \"three black teenagers\" resulted in mostly mugshot images whereas Google image searchers of \"three white teenagers\" consisted of mostly stock images, suggesting a racial bias in Google's algorithm.","mit_domain":"Discrimination and Toxicity","mit_subdomain":"Unfair discrimination and misrepresentation","entity":"AI","intent":"Unintentional","timing":"Post-deployment","harm_level":"none","sectors":["information and communication"],"countries":[],"deployers":["Google"],"developers":["Google"],"harmed":["Minority Groups"],"report_count":18},{"incident_id":6,"occurred_on":"2016-03-24","title":"Microsoft's TayBot Allegedly Posts Racist, Sexist, and Anti-Semitic Content to Twitter","description":"Microsoft's Tay, an artificially intelligent chatbot, was released on March 23, 2016 and removed within 24 hours due to multiple racist, sexist, and anti-semitic tweets generated by the bot.","mit_domain":"Discrimination and Toxicity","mit_subdomain":"Exposure to toxic content","entity":"Human","intent":"Intentional","timing":"Post-deployment","harm_level":"none","sectors":["information and communication"],"countries":[],"deployers":["Microsoft"],"developers":["Microsoft"],"harmed":["X (Twitter) Users"],"report_count":28},{"incident_id":73,"occurred_on":"2016-03-01","title":"Is Pokémon Go racist? How the app may be redlining communities of color","description":"Through a crowdsourcing social media campaign in 2016, several journalists and researchers demonstrated that augmented reality locations in the popular smartphone game Pokemon Go were more likely to be in white neighborhoods.","mit_domain":"Discrimination and Toxicity","mit_subdomain":"Unfair discrimination and misrepresentation","entity":"AI","intent":"Unintentional","timing":"Post-deployment","harm_level":"none","sectors":["information and communication","arts, entertainment and recreation"],"countries":["US"],"deployers":["Niantic Labs"],"developers":["Niantic Labs"],"harmed":["Non White Neighborhoods, Communities Of Color"],"report_count":8},{"incident_id":296,"occurred_on":"2016-02-10","title":"Twitter Recommender System Amplified Right-Leaning Tweets","description":"Twitter’s “Home” timeline algorithm was revealed by its internal researchers to have amplified tweets and news of rightwing politicians and organizations more than leftwing ones in six out of seven studied countries.","mit_domain":"Discrimination and Toxicity","mit_subdomain":"Unfair discrimination and misrepresentation","entity":"AI","intent":"Unintentional","timing":"Post-deployment","harm_level":null,"sectors":[],"countries":[],"deployers":["Twitter"],"developers":["Twitter"],"harmed":["Twitter Left Leaning Politicians, Twitter Left Leaning News Organizations, Twitter Left Leaning Users, X (Twitter) Users"],"report_count":3},{"incident_id":407,"occurred_on":"2016-02-03","title":"Uber's Surge Pricing Reportedly Offered Disproportionate Service Quality along Racial Lines","description":"Uber's surge-pricing algorithm which adjusts prices to influence car availability inadvertently caused better service offering such as shorter wait times for majority white neighborhoods.","mit_domain":"Discrimination and Toxicity","mit_subdomain":"Unfair discrimination and misrepresentation","entity":"AI","intent":"Unintentional","timing":"Post-deployment","harm_level":null,"sectors":[],"countries":[],"deployers":["Uber"],"developers":["Uber"],"harmed":["Poor Neighborhoods, Neighborhoods Of Color"],"report_count":1},{"incident_id":54,"occurred_on":"2015-11-18","title":"Predictive Policing Biases of PredPol","description":"Predictive policing algorithms meant to aid law enforcement by predicting future crime show signs of biased output.","mit_domain":"Discrimination and Toxicity","mit_subdomain":"Unfair discrimination and misrepresentation","entity":"AI","intent":"Unintentional","timing":"Post-deployment","harm_level":"none","sectors":["law enforcement"],"countries":["US"],"deployers":["PredPol","Oakland Police Department"],"developers":["PredPol"],"harmed":["Oakland Residents"],"report_count":17},{"incident_id":16,"occurred_on":"2015-06-03","title":"Images of Black People Labeled as Gorillas","description":"Google Photos image processing software mistakenly labelled a black couple as \"gorillas.\"","mit_domain":"Discrimination and Toxicity","mit_subdomain":"Unfair discrimination and misrepresentation","entity":"AI","intent":"Unintentional","timing":"Post-deployment","harm_level":"none","sectors":["information and communication"],"countries":["US"],"deployers":["Google"],"developers":["Google"],"harmed":["Black People"],"report_count":23},{"incident_id":1,"occurred_on":"2015-05-19","title":"Google’s YouTube Kids App Presents Inappropriate Content","description":"YouTube’s content filtering and recommendation algorithms exposed children to disturbing and inappropriate videos.","mit_domain":"Discrimination and Toxicity","mit_subdomain":"Exposure to toxic content","entity":"AI","intent":"Unintentional","timing":"Post-deployment","harm_level":"none","sectors":["arts, entertainment and recreation","information and communication"],"countries":["US"],"deployers":["Youtube"],"developers":["Youtube"],"harmed":["Minors"],"report_count":14},{"incident_id":18,"occurred_on":"2015-04-04","title":"Gender Biases of Google Image Search","description":"Google Image returns results that under-represent women in leadership roles, notably with the first photo of a female \"CEO\" being a Barbie doll after 11 rows of male CEOs.","mit_domain":"Discrimination and Toxicity","mit_subdomain":"Unfair discrimination and misrepresentation","entity":"AI","intent":"Unintentional","timing":"Post-deployment","harm_level":"none","sectors":["information and communication"],"countries":["US"],"deployers":["Google"],"developers":["Google"],"harmed":["Women and girls","Women"],"report_count":10},{"incident_id":336,"occurred_on":"2015-03-01","title":"UK Home Office's Sham Marriage Detection Algorithm Reportedly Flagged Certain Nationalities Disproportionately","description":"UK Home Office's opaque algorithm to detect sham marriages flagged some nationalities for investigation more than others, raising fears surrounding discrimination based on nationality and age.","mit_domain":"Discrimination and Toxicity","mit_subdomain":"Unfair discrimination and misrepresentation","entity":"AI","intent":"Intentional","timing":"Post-deployment","harm_level":null,"sectors":[],"countries":[],"deployers":["Uk Home Office"],"developers":["Uk Home Office"],"harmed":["Uk Immigrant Newlyweds"],"report_count":4},{"incident_id":335,"occurred_on":"2015-03-01","title":"UK Visa Streamline Algorithm Allegedly Discriminated Based on Nationality","description":"UK Home Office's algorithm to assess visa application risks explicitly considered nationality, allegedly caused candidates to face more scrutiny and discrimination.","mit_domain":"Discrimination and Toxicity","mit_subdomain":"Unfair discrimination and misrepresentation","entity":"AI","intent":"Unintentional","timing":"Post-deployment","harm_level":null,"sectors":[],"countries":[],"deployers":["Uk Visas And Immigration"],"developers":["Uk Visas And Immigration, Uk Home Office"],"harmed":["Uk Visa Applicants From Some Countries"],"report_count":8},{"incident_id":206,"occurred_on":"2015-03-01","title":"Tinder's Personalized Pricing Algorithm Found to Offer Higher Prices for Older Users","description":"Tinder’s personalized pricing was found by Consumers International to consider age as a major determinant of pricing, and could be considered a direct discrimination based on age, according to anti-discrimination law experts.","mit_domain":"Discrimination and Toxicity","mit_subdomain":"Unfair discrimination and misrepresentation","entity":"AI","intent":"Intentional","timing":"Post-deployment","harm_level":null,"sectors":[],"countries":[],"deployers":["Tinder"],"developers":["Tinder"],"harmed":["Tinder Users Over 30 Years Old"],"report_count":4},{"incident_id":260,"occurred_on":"2014-08-26","title":"US DHS’s Opaque Vetting Software Allegedly Relied on Poor-Quality Data and Discriminated against Immigrants","description":"US Citizenship and Immigration Services (USCIS)’s ATLAS software used in vetting immigration requests was condemned by advocacy groups as a threat to naturalized citizens for its secretive algorithmic decision-making, reliance on poor quality data and unknown sources, and alleged discrimination of immigrants using biometric and sensitive information.","mit_domain":"Discrimination and Toxicity","mit_subdomain":"Unfair discrimination and misrepresentation","entity":"AI","intent":"Intentional","timing":"Post-deployment","harm_level":null,"sectors":[],"countries":[],"deployers":["Us Department Of Homeland Security, Us Citizenship And Immigration Services"],"developers":["Us Citizenship And Immigration Services"],"harmed":["Us Naturalized Citizens, Us Immigrants, Us Citizenship Applicants, Us Immigration Applicants"],"report_count":2},{"incident_id":527,"occurred_on":"2014-05-08","title":"Tech Companies Reportedly Influenced Gig Workers' Behaviors Using Algorithms to Vary Pay for Same Amount of Work","description":"Amazon and Uber were alleged in a multiyear ethnographic study using algorithmic systems based on gig workers' data to vary pay, such as by offering them lower wages for the same amount of work.","mit_domain":"Discrimination and Toxicity","mit_subdomain":"Unfair discrimination and misrepresentation","entity":"AI","intent":"Intentional","timing":"Post-deployment","harm_level":null,"sectors":[],"countries":[],"deployers":["Uber","Amazon"],"developers":["Uber","Amazon"],"harmed":["Uber drivers","gig workers","Amazon delivery workers"],"report_count":2},{"incident_id":380,"occurred_on":"2014-03-04","title":"Facebook's Auto-Generated Targeting Ad Categories Contained Anti-Semitic Options","description":"Facebook's automated advertising categories generated using users' declared interests contained anti-Semitic categories such as \"Jew hater\"  and \"How to burn Jews\" which were listed as fields of study.","mit_domain":"Discrimination and Toxicity","mit_subdomain":"Exposure to toxic content","entity":"AI","intent":"Unintentional","timing":"Post-deployment","harm_level":null,"sectors":[],"countries":[],"deployers":["Facebook"],"developers":["Facebook"],"harmed":["Jewish People"],"report_count":5},{"incident_id":280,"occurred_on":"2013-07-30","title":"Coffee Meets Bagel’s Algorithm Reported by Users Disproportionately Showing Them Matches of Their Own Ethnicities Despite Selecting “No Preference”","description":"Users selecting “no preference” were shown by Coffee Meets Bagels’s matching algorithm more potential matches with the same ethnicity, which was acknowledged and justified by its founder as a means to maximize connection rate without sufficient user information.","mit_domain":"Discrimination and Toxicity","mit_subdomain":"Unfair discrimination and misrepresentation","entity":"AI","intent":"Intentional","timing":"Post-deployment","harm_level":null,"sectors":[],"countries":[],"deployers":["Coffee Meets Bagel"],"developers":["Coffee Meets Bagel"],"harmed":["Coffee Meets Bagel Users Having No Ethnicity Preference, Coffee Meets Bagel Users"],"report_count":2},{"incident_id":19,"occurred_on":"2013-01-23","title":"Sexist and Racist Google Adsense Advertisements","description":"Advertisements chosen by Google Adsense are reported as producing sexist and racist results.","mit_domain":"Discrimination and Toxicity","mit_subdomain":"Unfair discrimination and misrepresentation","entity":"AI","intent":"Unintentional","timing":"Post-deployment","harm_level":"none","sectors":["information and communication"],"countries":["US"],"deployers":["Google"],"developers":["Google"],"harmed":["Women and girls","Women","Minority groups"],"report_count":27},{"incident_id":135,"occurred_on":"2012-12-01","title":"UT Austin's GRADE Algorithm Reportedly Reduced Review of Lower-Scored PhD Applicants Amid Bias Concerns","description":"From the 2013 through 2019 admissions cycles, UT Austin's Department of Computer Science used GRADE, a statistical machine-learning system trained on past admissions decisions, to score and organize PhD applications. Critics said the system could reproduce historical admissions inequities and reduce attention to lower-scored applicants, while UT Austin said human reviewers still evaluated each file and later discontinued the tool.","mit_domain":"Discrimination and Toxicity","mit_subdomain":"Unfair discrimination and misrepresentation","entity":"AI","intent":"Unintentional","timing":"Post-deployment","harm_level":"none","sectors":["education"],"countries":["US"],"deployers":["University of Texas at Austin's Department of Computer Science","Risto Miikkulainen","Austin Waters"],"developers":["University of Texas at Austin researchers"],"harmed":["University students","University of Texas at Austin PhD applicants of marginalized groups","University applicants","Students","PhD applicants from underrepresented groups","PhD applicants","Educational communities","Computer science PhD applicants"],"report_count":2},{"incident_id":433,"occurred_on":"2012-08-01","title":"Chicago Police's Strategic Subject List Reportedly Biased Along Racial Lines","description":"The Chicago Police Department reportedly used person-based predictive policing models, including the Strategic Subject List and Crime and Victimization Risk Model, to assign violence-risk scores to hundreds of thousands of residents. Official and media reports alleged that the program gave police a mathematically fragile basis for treating heavily policed residents as future victims or suspects, with Black Chicago residents especially exposed to the consequences of that scoring.","mit_domain":"Discrimination and Toxicity","mit_subdomain":"Unfair discrimination and misrepresentation","entity":"AI","intent":"Unintentional","timing":"Post-deployment","harm_level":null,"sectors":[],"countries":[],"deployers":["Chicago Police Department"],"developers":["Chicago Police Department"],"harmed":["Minority groups","General public of Chicago","General public","Economically vulnerable communities","communities of color","Black people","Black Chicago residents"],"report_count":9},{"incident_id":257,"occurred_on":"2012-05-04","title":"Police Reportedly Deployed ShotSpotter Sensors Disproportionately in Neighborhoods of Color","description":"Police departments disproportionately placed ShotSpotter sensors in black and brown neighborhoods, which is denounced by communities for allegedly creating dangerous situations, such as one involving in Adam Toledo's death.","mit_domain":"Discrimination and Toxicity","mit_subdomain":"Unfair discrimination and misrepresentation","entity":"AI","intent":"Unintentional","timing":"Post-deployment","harm_level":null,"sectors":[],"countries":[],"deployers":["Kansas City Police Department, Cleveland Division Of Police, Chicago Police Department, Atlanta Police Department"],"developers":["Shotspotter"],"harmed":["Neighborhoods Of Color, Brown Communities, Black Communities, Adam Toledo"],"report_count":4},{"incident_id":75,"occurred_on":"2012-01-05","title":"Google Instant's Allegedly 'Anti-Semitic' Results Lead To Lawsuit In France","description":"The organizations SOS Racisme, Union of Jewish Students of France, Movement Against Racism and for Friendship Among Peoples are suing Google due to its autocomplete software suggesting \"jewish\" when the names of certain public figures were searched on the platform.","mit_domain":"Discrimination and Toxicity","mit_subdomain":"Unfair discrimination and misrepresentation","entity":"AI","intent":"Unintentional","timing":"Post-deployment","harm_level":"none","sectors":["information and communication"],"countries":["FR"],"deployers":["Google"],"developers":["Google"],"harmed":["Jewish People, Jewish Public Figures"],"report_count":1},{"incident_id":99,"occurred_on":"2012-01-01","title":"Major Universities Are Reportedly Using Race as a 'High Impact Predictor' of Student Success","description":"Several major universities are reportedly using a tool that uses race as one factor to predict student success.","mit_domain":"Discrimination and Toxicity","mit_subdomain":"Unfair discrimination and misrepresentation","entity":"AI","intent":"Intentional","timing":"Post-deployment","harm_level":"none","sectors":["administrative and support service activities","education"],"countries":["US"],"deployers":["University of Wisconsin–Milwaukee","University of Massachusetts Amherst","University of Houston","Texas A&M University","more than 500 colleges","Georgia State University"],"developers":["EAB"],"harmed":["University students","Students","Latinx college students","indigenous students","Educational communities","Black college students"],"report_count":1},{"incident_id":461,"occurred_on":"2008-07-18","title":"IRS Audited Black Taxpayers More Frequently Reportedly Due to Algorithm","description":"The IRS was auditing Black taxpayers more frequently than other groups allegedly due to the design of their algorithms, focusing on easier-to-conduct audits which inadvertently correlated with the group's pattern of tax filing errors.","mit_domain":"Discrimination and Toxicity","mit_subdomain":"Unfair discrimination and misrepresentation","entity":"AI","intent":"Unintentional","timing":"Post-deployment","harm_level":"none","sectors":["public administration","financial and insurance activities"],"countries":["US"],"deployers":["Internal Revenue Service"],"developers":["Internal Revenue Service"],"harmed":["Taxpayers in the United States","Taxpayers","Black taxpayers","Black people"],"report_count":4},{"incident_id":15,"occurred_on":"2008-05-23","title":"Amazon Censors Gay Books","description":"Amazon's book store \"cataloging error\" led to books containing gay and lesbian themes to lose their sales ranking, therefore losing visibility on the sales platform.","mit_domain":"Discrimination and Toxicity","mit_subdomain":"Unfair discrimination and misrepresentation","entity":"Human","intent":"Other","timing":"Post-deployment","harm_level":"none","sectors":["wholesale and retail trade"],"countries":[],"deployers":["Amazon"],"developers":["Amazon"],"harmed":["Amazon Customers"],"report_count":24},{"incident_id":469,"occurred_on":"2006-02-25","title":"Automated Adult Content Detection Tools Showed Bias against Women Bodies","description":"Automated content moderation tools to detect sexual explicitness or \"raciness\" reportedly exhibited bias against women bodies, resulting in suppression of reach despite not breaking platform policies.","mit_domain":"Discrimination and Toxicity","mit_subdomain":"Unfair discrimination and misrepresentation","entity":"AI","intent":"Unintentional","timing":"Post-deployment","harm_level":null,"sectors":[],"countries":[],"deployers":["Meta, Linkedin, Instagram, Facebook"],"developers":["Microsoft, Google, Amazon"],"harmed":["Linkedin Users, Instagram Users, Facebook Users"],"report_count":3},{"incident_id":274,"occurred_on":"2003-07-01","title":"Virginia Courts’ Algorithmic Recidivism Risk Assessment Failed to Lower Incarceration Rates","description":"Virginia courts’ use of algorithmic predictions of future offending risks were found by researchers failing to reduce incarceration rates, showed racial and age disparities in risk scores and its application, and neither exacerbated or ameliorated historical racial differences in sentencing.","mit_domain":"Discrimination and Toxicity","mit_subdomain":"Unfair discrimination and misrepresentation","entity":"AI","intent":"Unintentional","timing":"Post-deployment","harm_level":null,"sectors":[],"countries":[],"deployers":["Virginia courts"],"developers":["Virginia Department of Criminal Justice Services"],"harmed":["Virginia young offenders","Virginia convicted felons","Virginia Black offenders"],"report_count":2},{"incident_id":79,"occurred_on":"1999-03-16","title":"Kidney Testing Method Allegedly Underestimated Risk of Black Patients","description":"Decades-long use of the estimated glomerular filtration rate (eGFR) method to test kidney function which considers race has been criticized by physicians and medical students for its racist history and inaccuracy against Black patients.","mit_domain":"Discrimination and Toxicity","mit_subdomain":"Unfair discrimination and misrepresentation","entity":"AI","intent":"Unintentional","timing":"Post-deployment","harm_level":"none","sectors":["human health and social work activities"],"countries":["US"],"deployers":["Chronic Kidney Disease Epidemiology Collaboration"],"developers":["Chronic Kidney Disease Epidemiology Collaboration"],"harmed":["Black Patients, African American Patients"],"report_count":3},{"incident_id":43,"occurred_on":"1998-03-05","title":"St George's Hospital Medical School allegedly Automated Admissions with Discriminatory Results","description":"From 1982 to 1986, St George's Hospital Medical School used a program to automate a portion of their admissions process that allegedly discriminated against women and members of ethnic minorities.","mit_domain":"Discrimination and Toxicity","mit_subdomain":"Unfair discrimination and misrepresentation","entity":"AI","intent":"Unintentional","timing":"Post-deployment","harm_level":"none","sectors":["education","human health and social work activities"],"countries":["GB"],"deployers":["St George's Hospital Medical School"],"developers":["Geoffrey Franglen"],"harmed":["Women and girls","Women","Minority groups"],"report_count":4}]}