{"topic":{"slug":"alignment-safety","name":"AI Alignment & Safety","domain":"ai","description":"RLHF and its successors, constitutional and preference optimization methods, scalable oversight, dangerous-capability evaluations, reward hacking and deceptive behavior.","keywords":["AI alignment","RLHF","human feedback","constitutional AI","scalable oversight","reward hacking","AI safety","deceptive alignment","red teaming","direct preference optimization"],"paper_count":238,"latest_published":"2026-10-27","url":"https://space-kitty.com/topics/alignment-safety","api_url":"https://space-kitty.com/api/v1/topics/alignment-safety"},"dossier":null,"top_papers":[{"id":"arxiv:2203.02155","kind":"preprint","title":"Training language models to follow instructions with human feedback","authors":["Long Ouyang","Jeff Wu","Xu Jiang","Diogo Almeida","Carroll L. Wainwright","Pamela Mishkin"],"author_count":20,"year":2022,"published":"2022-03-04","venue":null,"cited_by":4206,"doi":null,"arxiv_id":"2203.02155","nct_id":null,"source_url":"http://arxiv.org/abs/2203.02155","archive_url":"https://space-kitty.com/papers/arxiv:2203.02155"},{"id":"arxiv:2304.01852","kind":"article","title":"Summary of ChatGPT-Related research and perspective towards the future of large language models","authors":["Yiheng Liu","Tianle Han","Siyuan Ma","Jiayue Zhang","Yuanyuan Yang","Jiaming Tian"],"author_count":18,"year":2023,"published":"2023-08-18","venue":"Meta-Radiology","cited_by":851,"doi":"10.1016/j.metrad.2023.100017","arxiv_id":"2304.01852","nct_id":null,"source_url":"https://doi.org/10.1016/j.metrad.2023.100017","archive_url":"https://space-kitty.com/papers/arxiv:2304.01852"},{"id":"arxiv:1706.03741","kind":"preprint","title":"Deep reinforcement learning from human preferences","authors":["Paul F. Christiano","Jan Leike","Brown, Tom B.","Miljan Martic","Shane Legg","Dario Amodei"],"author_count":6,"year":2017,"published":"2017-06-12","venue":null,"cited_by":1058,"doi":null,"arxiv_id":"1706.03741","nct_id":null,"source_url":"http://arxiv.org/abs/1706.03741","archive_url":"https://space-kitty.com/papers/arxiv:1706.03741"},{"id":"doi:10.1093/jamia/ocad072","kind":"article","title":"Using AI-generated suggestions from ChatGPT to optimize clinical decision support","authors":["Siru Liu","Aileen P. Wright","Barron L. Patterson","Jonathan P. Wanderer","Robert W. Turer","Scott D. Nelson"],"author_count":9,"year":2023,"published":"2023-04-22","venue":"Journal of the American Medical Informatics Association","cited_by":396,"doi":"10.1093/jamia/ocad072","arxiv_id":null,"nct_id":null,"source_url":"https://doi.org/10.1093/jamia/ocad072","archive_url":"https://space-kitty.com/papers/doi:10.1093/jamia/ocad072"},{"id":"arxiv:2204.05862","kind":"preprint","title":"Training a Helpful and Harmless Assistant with Reinforcement Learning from Human Feedback","authors":["Yuntao Bai","Jones, Andy","Kamal Ndousse","Amanda Askell","Chen, Anna","Nova DasSarma"],"author_count":31,"year":2022,"published":"2022-04-12","venue":null,"cited_by":442,"doi":null,"arxiv_id":"2204.05862","nct_id":null,"source_url":"http://arxiv.org/abs/2204.05862","archive_url":"https://space-kitty.com/papers/arxiv:2204.05862"},{"id":"arxiv:2009.01325","kind":"preprint","title":"Learning to summarize from human feedback","authors":["Nisan Stiennon","Long Ouyang","Jeff Wu","Ziegler, Daniel M.","Ryan Lowe","Chelsea Voss"],"author_count":9,"year":2020,"published":"2020-09-02","venue":null,"cited_by":550,"doi":null,"arxiv_id":"2009.01325","nct_id":null,"source_url":"http://arxiv.org/abs/2009.01325","archive_url":"https://space-kitty.com/papers/arxiv:2009.01325"},{"id":"arxiv:2305.18290","kind":"preprint","title":"Direct Preference Optimization: Your Language Model is Secretly a Reward Model","authors":["Rafael Rafailov","Archit Sharma","Eric Mitchell","Stefano Ermon","Christopher D. Manning","Chelsea Finn"],"author_count":6,"year":2023,"published":"2023-05-29","venue":null,"cited_by":293,"doi":null,"arxiv_id":"2305.18290","nct_id":null,"source_url":"https://arxiv.org/abs/2305.18290","archive_url":"https://space-kitty.com/papers/arxiv:2305.18290"},{"id":"arxiv:2212.08073","kind":"preprint","title":"Constitutional AI: Harmlessness from AI Feedback","authors":["Bai, Yuntao","Saurav Kadavath","Sandipan Kundu","Amanda Askell","Jackson Kernion","Jones, Andy"],"author_count":51,"year":2022,"published":"2022-12-15","venue":null,"cited_by":324,"doi":null,"arxiv_id":"2212.08073","nct_id":null,"source_url":"http://arxiv.org/abs/2212.08073","archive_url":"https://space-kitty.com/papers/arxiv:2212.08073"},{"id":"doi:10.3390/healthcare11202776","kind":"review","title":"Leveraging Generative AI and Large Language Models: A Comprehensive Roadmap for Healthcare Integration","authors":["Ping Yu","Hua Xu","Xia Hu","Chao Deng"],"author_count":4,"year":2023,"published":"2023-10-20","venue":"Healthcare","cited_by":261,"doi":"10.3390/healthcare11202776","arxiv_id":null,"nct_id":null,"source_url":"https://doi.org/10.3390/healthcare11202776","archive_url":"https://space-kitty.com/papers/doi:10.3390/healthcare11202776"},{"id":"openalex:W2098441518","kind":"article","title":"Policy Shaping: Integrating Human Feedback with Reinforcement Learning","authors":["Shane Griffith","K.A. Subramanian","Jonathan Scholz","Charles Lee Isbell","Andrea L. Thomaz"],"author_count":5,"year":2013,"published":"2013-12-05","venue":null,"cited_by":307,"doi":null,"arxiv_id":null,"nct_id":null,"source_url":"http://citeseerx.ist.psu.edu/viewdoc/summary?doi=10.1.1.646.510","archive_url":"https://space-kitty.com/papers/openalex:W2098441518"},{"id":"doi:10.1613/jair.1.15348","kind":"article","title":"Human-in-the-Loop Reinforcement Learning: A Survey and Position on Requirements, Challenges, and Opportunities","authors":["Carl Orge Retzlaff","Srijita Das","Christabel Wayllace","Payam Mousavi","Mohammad Afshari","Tianpei Yang"],"author_count":10,"year":2024,"published":"2024-01-30","venue":"Journal of Artificial Intelligence Research","cited_by":181,"doi":"10.1613/jair.1.15348","arxiv_id":null,"nct_id":null,"source_url":"https://doi.org/10.1613/jair.1.15348","archive_url":"https://space-kitty.com/papers/doi:10.1613/jair.1.15348"},{"id":"doi:10.1371/journal.pcbi.1005684","kind":"article","title":"Confirmation bias in human reinforcement learning: Evidence from counterfactual feedback processing","authors":["Stefano Palminteri","Germain Lefebvre","Emma Jayne Kilford","Sarah‐Jayne Blakemore"],"author_count":4,"year":2017,"published":"2017-08-11","venue":"PLoS Computational Biology","cited_by":237,"doi":"10.1371/journal.pcbi.1005684","arxiv_id":null,"nct_id":null,"source_url":"https://doi.org/10.1371/journal.pcbi.1005684","archive_url":"https://space-kitty.com/papers/doi:10.1371/journal.pcbi.1005684"},{"id":"arxiv:1802.05267","kind":"article","title":"Reinforcement Learning with Neural Networks for Quantum Feedback","authors":["Thomas Fösel","Petru Tighineanu","Talitha Weiss","Florian Marquardt"],"author_count":4,"year":2018,"published":"2018-09-27","venue":"Physical Review X","cited_by":231,"doi":"10.1103/physrevx.8.031084","arxiv_id":"1802.05267","nct_id":null,"source_url":"https://doi.org/10.1103/physrevx.8.031084","archive_url":"https://space-kitty.com/papers/arxiv:1802.05267"},{"id":"arxiv:2312.00849","kind":"article","title":"RLHF-V: Towards Trustworthy MLLMs via Behavior Alignment from Fine-grained Correctional Human Feedback","authors":["Tianyu Yu","Yuan Yao","Haoye Zhang","Taiwen He","Yifeng Han","Ganqu Cui"],"author_count":11,"year":2023,"published":"2023-12-01","venue":"IEEE Computer Society Conference on Computer Vision and Pattern Recognition/Proceedings - IEEE Computer Society Conference on Computer Vision and Pattern Recognition/Proceedings","cited_by":117,"doi":"10.1109/cvpr52733.2024.01310","arxiv_id":"2312.00849","nct_id":null,"source_url":"https://arxiv.org/abs/2312.00849","archive_url":"https://space-kitty.com/papers/arxiv:2312.00849"},{"id":"doi:10.1162/jocn.2010.21456","kind":"article","title":"Feedback-related Negativity Codes Prediction Error but Not Behavioral Adjustment during Probabilistic Reversal Learning","authors":["Henry W. Chase","Rachel Swainson","Lucy E. Durham","Laura Benham","Roshan Cools"],"author_count":5,"year":2010,"published":"2010-02-10","venue":"Journal of Cognitive Neuroscience","cited_by":224,"doi":"10.1162/jocn.2010.21456","arxiv_id":null,"nct_id":null,"source_url":"https://doi.org/10.1162/jocn.2010.21456","archive_url":"https://space-kitty.com/papers/doi:10.1162/jocn.2010.21456"}],"newest_papers":[{"id":"arxiv:2610.36445","kind":"preprint","title":"From Solver Trajectories to Teaching Trajectories: Cognition-Aligned Reasoning Distillation","authors":["Yashuo Luo","Qianren Mao","Ziqi Qin","Junnan Liu","Haitao Li","Qili Zhang"],"author_count":9,"year":2026,"published":"2026-10-27","venue":null,"cited_by":0,"doi":null,"arxiv_id":"2610.36445","nct_id":null,"source_url":"https://arxiv.org/abs/2610.36445","archive_url":"https://space-kitty.com/papers/arxiv:2610.36445"},{"id":"arxiv:2610.36759","kind":"preprint","title":"DocSkill: Skill-Integrated Agentic Reasoning for Multilingual Multi-Hop Document Question Answering","authors":["Jing Huang","Zhiya Tan","Changtao Miao","Shutao Gong","Yang Wang","Joey Tianyi Zhou"],"author_count":8,"year":2026,"published":"2026-10-27","venue":null,"cited_by":0,"doi":null,"arxiv_id":"2610.36759","nct_id":null,"source_url":"https://arxiv.org/abs/2610.36759","archive_url":"https://space-kitty.com/papers/arxiv:2610.36759"},{"id":"arxiv:2610.37058","kind":"preprint","title":"Exploring the Limits of the Delta Learning Hypothesis: Single-Pair Generator Biases in Direct Preference Optimization","authors":["Giuseppe Curci","Stefano Simonazzi","Andrea Molinari","Andrea Zugarini"],"author_count":4,"year":2026,"published":"2026-10-27","venue":null,"cited_by":0,"doi":null,"arxiv_id":"2610.37058","nct_id":null,"source_url":"https://arxiv.org/abs/2610.37058","archive_url":"https://space-kitty.com/papers/arxiv:2610.37058"},{"id":"arxiv:2610.37085","kind":"preprint","title":"On the Sparsity of Direct Preference Optimization: Weight Disentanglement in the NTK Regime","authors":["Kensuke Sasaki","Issei Sato"],"author_count":2,"year":2026,"published":"2026-10-27","venue":null,"cited_by":0,"doi":null,"arxiv_id":"2610.37085","nct_id":null,"source_url":"https://arxiv.org/abs/2610.37085","archive_url":"https://space-kitty.com/papers/arxiv:2610.37085"},{"id":"arxiv:2610.37245","kind":"preprint","title":"Aligning Large Language Models for Safety in African Languages: A Case Study of Hausa","authors":["Valdini Douglace Lemofouet","Blessing Ngozi Uzor","Paula Chikaodinaka Anyanwu","Danielle Blanche Kapsa","Saminu Mohammad Aliyu","Seid Muhie Yimam"],"author_count":7,"year":2026,"published":"2026-10-27","venue":null,"cited_by":0,"doi":null,"arxiv_id":"2610.37245","nct_id":null,"source_url":"https://arxiv.org/abs/2610.37245","archive_url":"https://space-kitty.com/papers/arxiv:2610.37245"},{"id":"arxiv:2610.37451","kind":"preprint","title":"Interpretable Meta-Policies for Neural Decision Making","authors":["Raban Emunds","Jannis Blüml","Quentin Delfosse","Kristian Kersting"],"author_count":4,"year":2026,"published":"2026-10-27","venue":null,"cited_by":0,"doi":null,"arxiv_id":"2610.37451","nct_id":null,"source_url":"https://arxiv.org/abs/2610.37451","archive_url":"https://space-kitty.com/papers/arxiv:2610.37451"},{"id":"arxiv:2610.34705","kind":"preprint","title":"Safe by Benchmark, Unsafe in Practice: Lessons from 1331 Child-Related AI Incidents","authors":["Junfeng Jiao","Kevin Chen","Shrestha Mishra","Abhejay Murali","David Atkinson","Shriti Raj"],"author_count":7,"year":2026,"published":"2026-10-26","venue":null,"cited_by":0,"doi":null,"arxiv_id":"2610.34705","nct_id":null,"source_url":"https://arxiv.org/abs/2610.34705","archive_url":"https://space-kitty.com/papers/arxiv:2610.34705"},{"id":"arxiv:2610.34965","kind":"preprint","title":"The Target Agent Is Its Own Best Critic but a Poor Proposer: Self-Improved Red Teaming","authors":["Mintong Kang","Stephen Rawls","Melissa Kazemi Rad","Jian Ni","Bo Li"],"author_count":5,"year":2026,"published":"2026-10-26","venue":null,"cited_by":0,"doi":null,"arxiv_id":"2610.34965","nct_id":null,"source_url":"https://arxiv.org/abs/2610.34965","archive_url":"https://space-kitty.com/papers/arxiv:2610.34965"},{"id":"arxiv:2610.35497","kind":"preprint","title":"When Fine-Grained Credit Assignment Helps and Hurts: Multimodal Agentic Reinforcement Learning through Video Editing","authors":["Lei Zhao","Zongheng Cao","Xinyu Hu"],"author_count":3,"year":2026,"published":"2026-10-26","venue":null,"cited_by":0,"doi":null,"arxiv_id":"2610.35497","nct_id":null,"source_url":"https://arxiv.org/abs/2610.35497","archive_url":"https://space-kitty.com/papers/arxiv:2610.35497"},{"id":"arxiv:2610.35716","kind":"preprint","title":"Audit Where Hacks Pay: Budgeted Strong-Verifier Audits Against Reward Hacking in GRPO","authors":["Shurun Li","Michael Hua Wang","Haibo Zeng","Long Wang"],"author_count":4,"year":2026,"published":"2026-10-26","venue":null,"cited_by":0,"doi":null,"arxiv_id":"2610.35716","nct_id":null,"source_url":"https://arxiv.org/abs/2610.35716","archive_url":"https://space-kitty.com/papers/arxiv:2610.35716"}],"related_topics":[{"slug":"llm-architectures","name":"LLM Architectures & Pretraining","api_url":"https://space-kitty.com/api/v1/topics/llm-architectures"},{"slug":"scaling-laws","name":"Scaling Laws & Compute-Optimal Training","api_url":"https://space-kitty.com/api/v1/topics/scaling-laws"},{"slug":"reasoning-models","name":"Reasoning Models & Test-Time Compute","api_url":"https://space-kitty.com/api/v1/topics/reasoning-models"},{"slug":"interpretability","name":"Mechanistic Interpretability","api_url":"https://space-kitty.com/api/v1/topics/interpretability"},{"slug":"ai-agents","name":"LLM Agents & Tool Use","api_url":"https://space-kitty.com/api/v1/topics/ai-agents"},{"slug":"generative-multimodal","name":"Generative & Multimodal Models","api_url":"https://space-kitty.com/api/v1/topics/generative-multimodal"},{"slug":"efficient-ai","name":"Efficient Training & Inference","api_url":"https://space-kitty.com/api/v1/topics/efficient-ai"},{"slug":"ai-for-science","name":"AI for Science","api_url":"https://space-kitty.com/api/v1/topics/ai-for-science"},{"slug":"robotics-embodied","name":"Robot Learning & Embodied AI","api_url":"https://space-kitty.com/api/v1/topics/robotics-embodied"}],"premium":{"dossier":null,"research_pack":{"url":"https://space-kitty.com/api/v1/topics/alignment-safety/pack","price_usd":"0.20"}},"more":"https://space-kitty.com/api/v1/topics/alignment-safety/papers?sort=top&limit=100"}