@inproceedings{6fa8363a128b40bab488ea8f578a03a6,
title = "UIED: A hybrid tool for GUI element detection",
abstract = "Graphical User Interface (GUI) elements detection is critical for many GUI automation and GUI testing tasks. Acquiring the accurate positions and classes of GUI elements is also the very first step to conduct GUI reverse engineering or perform GUI testing. In this paper, we implement a User Iterface Element Detection (UIED), a toolkit designed to provide user with a simple and easy-to-use platform to achieve accurate GUI element detection. UIED integrates multiple detection methods including old-fashioned computer vision (CV) approaches and deep learning models to handle diverse and complicated GUI images. Besides, it equips with a novel customized GUI element detection methods to produce state-of-the-art detection results. Our tool enables the user to change and edit the detection result in an interactive dashboard. Finally, it exports the detected UI elements in the GUI image to design files that can be further edited in popular UI design tools such as Sketch and Photoshop. UIED is evaluated to be capable of accurate detection and useful for downstream works. Tool URL: http://uied.online Github Link: https://github.com/MulongXie/UIED",
keywords = "Computer Vision, Deep Learning, Object Detection, User Interface",
author = "Mulong Xie and Sidong Feng and Zhenchang Xing and Jieshan Chen and Chunyang Chen",
note = "Publisher Copyright: {\textcopyright} 2020 ACM.; 28th ACM Joint Meeting European Software Engineering Conference and Symposium on the Foundations of Software Engineering, ESEC/FSE 2020 ; Conference date: 08-11-2020 Through 13-11-2020",
year = "2020",
month = nov,
day = "8",
doi = "10.1145/3368089.3417940",
language = "English",
series = "ESEC/FSE 2020 - Proceedings of the 28th ACM Joint Meeting European Software Engineering Conference and Symposium on the Foundations of Software Engineering",
publisher = "Association for Computing Machinery, Inc",
pages = "1655--1659",
editor = "Prem Devanbu and Myra Cohen and Thomas Zimmermann",
booktitle = "ESEC/FSE 2020 - Proceedings of the 28th ACM Joint Meeting European Software Engineering Conference and Symposium on the Foundations of Software Engineering",
}