@inproceedings{bf89eea9ce1c4dfc9e6a57ef541167c2,
title = "Neural Bokeh: Learning Lens Blur for Computational Videography and Out-of-Focus Mixed Reality",
abstract = "We present Neural Bokeh, a deep learning approach for synthesizing convincing out-of-focus effects with applications in Mixed Reality (MR) image and video compositing. Unlike existing approaches that solely learn the amount of blur for out-of-focus areas, our approach captures the overall characteristic of the bokeh to enable the seamless integration of rendered scene content into real images, ensuring a consistent lens blur over the resulting MR composition. Our method learns spatially varying blur shapes, i.e., bokeh, from a dataset of real images acquired using the physical camera that is used to capture the photograph or video of the MR composition. Accordingly, those learned blur shapes mimic the characteristics of the physical lens. As the run-time and the resulting quality of Neural Bokeh increase with the resolution of input images, we employ low-resolution images for the MR view finding at runtime and high-resolution renderings for compositing with high-resolution photographs or videos in an offline process. We envision a variety of applications, including visual enhancement of image and video compositing containing creative utilization of out-of-focus effects.",
keywords = "Bokeh, Computational Videography, Deep Learning, Depth of Field, Mixed Reality",
author = "David Mandl and Shohei Mori and Peter Mohr and Yifan Peng and Tobias Langlotz and Dieter Schmalstieg and Denis Kalkofen",
year = "2024",
month = apr,
day = "15",
doi = "10.1109/VR58804.2024.00106",
language = "English",
series = "Proceedings - 2024 IEEE Conference on Virtual Reality and 3D User Interfaces, VR 2024",
publisher = "Institute of Electrical and Electronics Engineers",
pages = "870--880",
booktitle = "Proceedings - 2024 IEEE Conference on Virtual Reality and 3D User Interfaces, VR 2024",
address = "United States",
note = "31st IEEE Conference on Virtual Reality and 3D User Interfaces, VR 2024 ; Conference date: 16-03-2024 Through 21-03-2024",
}