
本研究では、混合現実(MR)における視覚推論のリアルタイム処理を実現するため、ヘッドマウントディスプレイの計算制約を補完する協調処理フレームワークを提案する。近年のMRデバイスは空間トラッキングには対応しているものの、高解像度のセマンティックなシーン解析をオンボードで実行する計算能力は十分ではない。これに対し、本研究では、セマンティックセグメンテーション、物体検出、シーンクラス分類といった認識処理を外部のエッジAIデバイスにオフロードするアーキテクチャを構築する。
本システムは、Magic Leap 2やMeta Questといった市販のMRヘッドセットと、Hailo-8などのエッジAIアクセラレータを搭載した小型プロセッサノードを接続する。映像入力(高解像度フレームまたは中間特徴量)をWi-Fiなどの低遅延通信で送信し、外部デバイス上で推論処理を実行する。処理結果として得られる物体マスク、バウンディングボックス、シーンラベルなどの構造化データは、UnityやUnreal Engine上でレンダリングされ、ヘッドセット内に統合される。
これにより、アプリケーションは物理環境と意味的に連動する表現を可能とする。たとえば、仮想コンテンツを検出された物体に動的にアンカーしたり、ナビゲーションシステムをシーンコンテキストに応じて変化させたり、環境の認識結果に基づいてアプリケーション挙動を制御することが可能となる。本手法では推論処理を外部化することで、ヘッドセット単体では不可能なモデルの複雑性や更新頻度にも対応できる。
初期の実装では、屋外でのモバイル利用や、室内環境におけるセマンティックアノテーションの応用において、往復100ms未満の応答性能が確認された。さらに、複数のヘッドセットから単一の推論ノードを共有するユースケースも検証済みである。
本研究は、幾何情報、セマンティック解釈、環境インタラクションの統合を通じて、混合現実におけるSpatial AIの基盤技術としての貢献を目指す。
@inproceedings{Orsholits2025,
title = {Context-Rich Interactions in Mixed Reality through Edge AI Co-Processing},
author = {Alex Orsholits and Manabu Tsukada},
url = {https://link.springer.com/chapter/10.1007/978-3-031-87772-8_3},
doi = {10.1007/978-3-031-87772-8_3},
isbn = {978-3-031-87771-1},
year = {2025},
date = {2025-04-09},
urldate = {2025-04-09},
booktitle = {The 39-th International Conference on Advanced Information Networking and Applications (AINA 2025)},
address = {Barcelona, Spain},
abstract = {Spatial computing is evolving towards leveraging data streaming for computationally demanding applications, facilitating a shift to lightweight, untethered, and standalone devices. These devices are therefore ideal candidates for co-processing, where real-time context understanding and low-latency data streaming are fundamental for seamless, general-purpose Mixed Reality (MR) experiences. This paper demonstrates and evaluates a scalable approach to augmented contextual understanding in MR by implementing multi-modal edge AI co-processing through a Hailo-8 AI accelerator, a low-power ARM-based single board computer (SBC), and the Magic Leap 2 AR headset. The proposed system utilises the native WebRTC streaming capabilities of the Magic Leap 2 to continuously stream camera data to the edge co-processor, where a collection of vision AI models-object detection, pose estimation, face recognition, and depth estimation-are executed. The resulting inferences are then streamed back to the headset for spatial re-projection and transmitted to cloud-based systems for further integration with large-scale AI models, such as LLMs and VLMs. This seamless integration enhances real-time contextual understanding in MR while facilitating advanced multi-modal, multi-device collaboration, supporting richer, scalable spatial cognition across distributed systems.},
keywords = {},
pubstate = {published},
tppubtype = {inproceedings}
}
@misc{Orsholits2025b,
title = {Edge Vision AI Co-Processing for Dynamic Context Awareness in Mixed Reality},
author = {Alex Orsholits and Manabu Tsukada},
url = {https://www.youtube.com/watch?v=xxahKZl4K9w
https://ieeevr.org/2025/awards/conference-awards/#poster-honorable},
doi = {10.1109/VRW66409.2025.00293},
year = {2025},
date = {2025-03-08},
urldate = {2025-03-08},
booktitle = {2025 IEEE Conference on Virtual Reality and 3D User Interfaces Abstracts and Workshops (VRW)},
address = {Saint-Malo, France},
abstract = {Spatial computing is evolving towards leveraging data streaming for computationally demanding applications, facilitating a shift to lightweight, untethered, and standalone devices. These devices are ideal candidates for co-processing, where real-time scene context understanding and low-latency data streaming are fundamental for general-purpose Mixed Reality (MR) experiences. This poster demonstrates and evaluates a scalable approach to augmented contextual understanding in MR by implementing edge AI co-processing through a Hailo-8 AI accelerator, a low-power ARM-based single board computer (SBC), and the Magic Leap 2 AR headset. The resulting inferences are streamed back to the headset for spatial reprojection into the user’s vision.},
howpublished = {IEEE VR 2025, Poster},
note = {Honorable mention},
keywords = {},
pubstate = {published},
tppubtype = {misc}
}
@inproceedings{Zhu2025,
title = {A Distributed Content Subscription Mechanism with Revision Discovery to Decouple Content Sharing Platform and Creator ID},
author = {Zhihai Zhu and Ye Tao and Manabu Tsukada and Hiroshi Esaki},
year = {2025},
date = {2025-02-18},
urldate = {2025-02-18},
booktitle = {International Conference on Artificial Intelligence in Information and Communication (ICAIIC 2025) },
address = {Fukuoka, Japan},
abstract = {Only the chairs can edit This paper proposes a distributed content subscription mechanism that enables content creators to share updates with their audience while maintaining platform independence and anonymity. The mechanism extends the Kademlia distributed hash table (DHT) protocol by incorporating revision numbers and republication timestamps into the DHT key computation, allowing subscribers to discover content updates through heuristic revision queries. It leverages public key cryptography for creator identification and content authenticity, while integrating with established peer-to-peer protocols like BitTorrent for efficient content distribution. Preliminary testing with 200 simulated nodes demonstrates the mechanism's ability to maintain content availability and update discovery even when content creators are offline. This approach particularly benefits creators operating under strict content controls or surveillance, offering them greater creative freedom and distribution autonomy compared to existing centralized and decentralized solutions.},
keywords = {},
pubstate = {published},
tppubtype = {inproceedings}
}
autonomous driving v2x
digital twins extended reality
digital twins
autonomous driving machine learning
machine learning v2x
autonomous driving v2x
extended reality