@article{10.1145/3832100,
author = {Xiong, Yunpeng and Zhang, Ting},
title = {Sifting the Noise: A Comparative Study of LLM Agents in Vulnerability False Positive Filtering},
year = {2026},
issue_date = {October 2026},
publisher = {Association for Computing Machinery},
address = {New York, NY, USA},
volume = {3},
number = {ISSTA},
url = {https://doi.org/10.1145/3832100},
doi = {10.1145/3832100},
abstract = {Static Application Security Testing (SAST) tools are essential for identifying software vulnerabilities, but they often produce a high volume of False Positives (FPs), imposing a substantial manual triage burden on developers. Recent advances in Large Language Model (LLM) agents offer a promising direction by enabling iterative reasoning, tool use, and environment interaction to refine SAST alerts. However, the comparative effectiveness of different LLM-based agent architectures for FP filtering remains poorly understood. In this paper, we present a comparative study of three state-of-the-art LLM-based agent frameworks, i.e., Aider, OpenHands, and SWE-agent, for vulnerability FP filtering. We evaluate these frameworks using the vulnerabilities from the OWASP Benchmark and real-world open-source Java projects. We further conduct a focused post-cutoff C/C++ study using the strongest configuration to test contamination-free generalization and isolate key agentic capabilities. The experimental results show that LLM-based agents can remove the majority of SAST noise, reducing an initial FP detection rate of over 92\% on the OWASP Benchmark to as low as 6.3\% in the best configuration. On a real-world Java dataset, the best configuration of LLM-based agents can achieve an FP identification rate of up to 93.3\% involving CodeQL alerts. However, the benefits of agents are strongly backbone- and CWE-dependent: agentic frameworks significantly outperform vanilla prompting for stronger models such as Claude Sonnet 4 and GPT-5, but yield limited or inconsistent gains for weaker backbones. On the post-cutoff OSS-Fuzz dataset, SWE-agent with Claude Sonnet 4 identifies 95.5\% of FPs while maintaining 95.5\% precision, compared with a 36.4\% FP identification rate for vanilla prompting. Moreover, aggressive FP reduction can come at the cost of suppressing true vulnerabilities, highlighting important trade-offs. Finally, we observe large disparities in computational cost across agent frameworks. Overall, our study demonstrates that LLM-based agents are a powerful but non-uniform solution for SAST FP filtering, and that their practical deployment requires careful consideration of agent design, backbone model choice, vulnerability category, and operational cost.},
journal = {Proc. ACM Softw. Eng.},
month = oct,
articleno = {ISSTA009},
numpages = {24},
keywords = {False Positives, LLM Agents, Static Application Security Testing}
}

@inproceedings{xiong2026afterparty,
  author    = {Xiong, Yunpeng and Zhang, Ting},
  title     = {After the Party: Growth, Governance, and Security Scanning
               in the OpenClaw Agent Skill Ecosystem},
  booktitle = {Proceedings of the 33rd Asia-Pacific Software Engineering
               Conference (APSEC)},
  year      = {2026},
  note      = {To appear}
}