
As internet-native audiovisual services continue to expand, new expectations are emerging for immersive experiences that go beyond traditional one-way broadcasting. At Tsukada Lab, we have been leading the Software Defined Media (SDM) consortium since 2014, focusing on object-based content delivery and real-time, software-defined control of viewing environments.
The SDM architecture virtualizes AV equipment—such as microphones, speakers, and lighting—as IoT-controllable resources. By exposing them via APIs, developers can build applications that dynamically alter sound images or visual perspectives based on user location or preferences. This approach turns conventional live venues into programmable platforms, where user-specific experiences can be generated. The architecture has been evaluated and demonstrated in multiple hackathons and exhibitions.
Based on this foundation, we developed “LiVRation,” an interactive remote live-streaming platform. LiVRation captures live music performances using 360-degree video and 3D audio, allowing viewers with head-mounted displays (HMDs) to explore the event space from any point of view. Through integration with event metadata, users can freely switch perspectives and focus on specific sound or visual sources. The system was awarded First Prize at the Billboard Live Music Hackasong 2017, hosted by Billboard Japan.
This project aims to make media experiences programmable by merging networking and spatial computing. The platform holds potential for diverse applications including smart buildings, remote education, virtual tourism, and next-generation entertainment experiences that blur the boundary between physical and digital realities.
@article{粕谷貴司2019,
title = {インタラクティブな遠隔ライブVR配信プラットフォーム},
author = {粕谷貴司 and 塚田学 and 菰原裕 and 高坂茂樹 and 水野拓宏 and 野村譲誉 and 上田 雄太 and 江崎 浩},
url = {https://tlab.hongo.wide.ad.jp/papers/kasuya_DCON2019.pdf
https://youtu.be/YoUcCLDtvwM},
isbn = {2187-8897},
year = {2019},
date = {2019-08-20},
journal = {情報処理学会論文誌デジタルコンテンツ(DCON)},
volume = {7},
number = {2},
pages = {1-14},
abstract = {インターネットを前提とした視聴サービスが登場し,中でも空間に存在する視聴対象を解釈し,コンテンツとして活用するオブジェクトベースの視聴サービスの重要性が増している.2014年より,Software Defined Media(SDM)コンソーシアムでは,オブジェクトベースのメディアとインターネットを前提とした視聴空間の研究を行っている.近年,4Kや360度動画による高品質な動画配信サービスが普及しているが,自由視点で視聴可能なコンテンツの配信手法については一般化していない.本研究では,音楽ライブのイベントを収録するとともに,イベント情報のメタデータ定義を行い,HMDを用いてインタラクティブに自由視聴点での三次元映像音声を再生するアプリケーション「LiVRation」を設計,実装し,評価を行なった.さらにビルボードジャパンが開催した「Billboard LIVE MUSIC HACKASONG 2017」にてデモンストレーションを行い,審査員と一般の来場者の投票により最優秀賞および会場賞を受賞した.},
keywords = {},
pubstate = {published},
tppubtype = {article}
}
@inproceedings{Kasuya2019,
title = {LiVRation: Remote VR live platform with interactive 3D audio-visual service},
author = {Takashi Kasuya and Manabu Tsukada and Yu Komohara and Shigeki Takasaka and Takuhiro Mizuno and Yoshitaka Nomura and Yuta Ueda and Hiroshi Esaki},
url = {https://hal.archives-ouvertes.fr/hal-02136247/document?.pdf
https://youtu.be/4MElM4_t2Io},
doi = {10.1109/GEM.2019.8811549},
year = {2019},
date = {2019-06-19},
booktitle = {IEEE Games Entertainment & Media Conference (IEEE GEM) 2019},
pages = {1-7},
address = {Yale University, New Haven, CT, U.S.},
abstract = {Of late, various audio-visual services based on the internet are being deployed extensively. Among these, object- based audio-visual services are attracting more attention. In 2014, we had established the software defined media (SDM) consortium to investigate object-based and internet-based audio- visual services. Despite the increasing demand and popularity of live concert events, the placement of the microphone and camera limit the free-viewpoint watching of the contents of package media, such as DVDs. In this study, we design and implement an interactive 3D audio-visual service system called LiVRation, with a free-view-listen point. For subjective evaluation, 211 people were made to experience LiVRation and answer a questionnaire, subsequently. In addition, we demonstrated the system in the ”Billboard Live Hackasong 2017” hosted by Billboard Japan and received the first prize, based on the votes of the judges as well as the audience.},
keywords = {},
pubstate = {published},
tppubtype = {inproceedings}
}
@article{tsukada2017,
title = {Software Defined Media: 視聴空間サービスのソフトウェア制御},
author = {塚田学 and 小川景子 and 池田雅弘 and 曽根卓朗 and 丹羽健太 and 齊藤翔一郎 and 粕谷貴司 and 砂原秀樹 and 江崎浩},
url = {https://www.jstage.jst.go.jp/article/jssst/34/3/34_3_37/_pdf/-char/ja?.pdf},
doi = {10.11309/jssst.34.3_37},
isbn = {0289-6540},
year = {2017},
date = {2017-09-20},
urldate = {2017-09-20},
journal = {日本ソフトウェア科学会学会誌『コンピュータソフトウェア』「ネットワーク技術」特集},
volume = {34},
number = {3},
pages = {37-58},
abstract = {インターネットを前提とした視聴サービスが登場し,中でも空間に存在する視聴対象を三次元的に解釈するオブジェクトベースの視聴サービスの重要性が増している.そこで著者らは2014年からSoftware Defined Media (SDM)コンソーシアムを設立し,オブジェクトベースのディジタルメディアと,インターネットを前提とした視聴空間の研究を続けてきた.その中で,Internet of Things (IoT)を利用したスマートビルディングと相互運用を目指して,視聴サービスのためのビル設備を仮想化するSDMアーキテクチャを提案している.SDMアーキテクチャに基づき,試作システムを構築し,デモンストレーション展示ハッカソン開催を行った.さらに,試作システムを利用したアプリケーション開発と収録データのデータベース化を行った.プラットフォーム上で動作するアプリケーションの遅延を測定し,多くのインタラクティブなアプリケーションを支援しうるリアルタイム性を確認した.また,試作システムを利用したインタラクティブなアプリケーションを被験者に体験してもらい,主観実験によって開発者の演出意図を高水準で満たせたことを示した.},
keywords = {},
pubstate = {published},
tppubtype = {article}
}
@inproceedings{Tsukada2017c,
title = {Software Defined Media: Virtualization of Audio-Visual Services},
author = {Manabu Tsukada and Keiko Ogawa and Masahiro Ikeda and Takuro Sone and Kenta Niwa and Shoichiro Saito and Takashi Kasuya and Hideki Sunahara and Hiroshi Esaki},
url = {https://arxiv.org/pdf/1702.07452.pdf},
doi = {10.1109/ICC.2017.7996610},
isbn = {1938-1883},
year = {2017},
date = {2017-05-21},
booktitle = {IEEE International Conference on Communications (ICC2017)},
pages = {1-7},
address = {Paris, France},
abstract = {Internet-native audio-visual services are witnessing rapid development. Among these services, object-based audio-visual services are gaining importance. In 2014, we established the Software Defined Media (SDM) consortium to target new research areas and markets involving object-based digital media and Internet-by-design audio-visual environments. In this paper, we introduce the SDM architecture that virtualizes networked audio-visual services along with the development of smart buildings and smart cities using Internet of Things (IoT) devices and smart building facilities. Moreover, we design the SDM architecture as a layered architecture to promote the development of innovative applications on the basis of rapid advancements in software-defined networking (SDN). Then, we implement a prototype system based on the architecture, present the system at an exhibition, and provide it as an SDM API to application developers at hackathons. Various types of applications are developed using the API at these events. An evaluation of SDM API access shows that the prototype SDM platform effectively provides 3D audio reproducibility and interactiveness for SDM applications.},
keywords = {},
pubstate = {published},
tppubtype = {inproceedings}
}
v2x
digital twins extended reality
digital twins
autonomous driving machine learning
machine learning v2x
autonomous driving v2x
extended reality
v2x
We are part of the University of Tokyo’s Graduate School of Information Science and Technology, Department of Creative Informatics and focuses on computer networks and cyber-physical systems
Address
4F, I-REF building, Graduate School of Information Science and Technology, The University of Tokyo, 1-1-1, Yayoi, Bunkyo-ku, Tokyo, 113-8657 Japan
Room 91B1, Bld 2 of Engineering Department, The University of Tokyo, 7-3-1 Hongo, Bunkyo-ku, Tokyo 113-8656, Japan
Mail: