NtKinect: C++ Class Library for Kinect V2 1,a) Kinect for Windows V2 C++ NtKinect NtKinect DLL Kinect V2 Kinect V2, C++, DLL, Unity NtKinect: C++ Class Library for Kinect V2 Nitta Yoshihisa 1,a) Abstract: Kinect for Windows V2 is a device with many functions such as skeleton recognition, face recognition and voice direction acquisition. In order to make its programming easier, we developed NtKinect class library for C++ and made it open source. Programs using NtKinect are multi-threaded, DLLable, and Kinect V2 functions can be used from other programming languages and environments. We will discuss its design policy. Keywords: Kinect V2, C++, DLL, Unity 1. Microsoft Kinect for Windows V2 *1 [1] Kinect V2 C++ API 2. Kinect for Windows V2 Microsoft Kinect V2 Kinect for Windows version 2.0 SDK C++ [2] native code API 1 Tsuda College, 2 1 1, Tsuda, Koadira, Tokyo 187 8577, Japan a) nitta@tsuda.ac.jp *1 Kinect V2 1 Kinect SDK Inteface IAudio 10 50 IBody 11 61 IColor 6 31 ICoordinateMapper 2 17 IDepth 5 26 IInfrared 5 22 IKinectSensor 2 19 IMultiSource 4 16 9 35 54 277 Interface *2 1 Interface 9 Interface 54, 277 SDK [3] *2 C++ c 2017 Information Processing Society of Japan 1
2 ColorSpace DepthSpace CameraSpace Kinect V2 Kinect V2 BOdyIndex 3. NtKinect Kinect V2 C++ NtKinect [4] NtKinect = Kinect SDK + + STL(C++) + OpenCV + mutex Collection STL OpenCV DLL 1 cv::mat mutex 3.1 Kinect 3.2 C++ STL Kinect V2 Collection STL DLL 3.3 OpenCV OpenCV version 2 C++ Kinect V2 OpenCV OpenCV ( ) BodyIndex OpenCV cv::mat ( 1) 3.4 3.5 DLL NtKinect Kinect DLL DLL 4. NtKinect API 3 C++ API 3 4 Kinect V2 (HDFace) 5. NtKinect C++ 2 3 [5] kinect.setskeleton() c 2017 Information Processing Society of Japan 2
3 NtKinect Table 3 Methods of NtKinect. ( ) RGB void setrgb (rgbimage) void setdepth (depthimage) BodyIndex void setbodyindex (bodyindeximage) void setinfrared (infraredimage) void setskeleton (skeleton, skeletonid, skeletontrackingid) pair<int,int> handstate () void setface (facepoint, facerect, facedirection, faceproperty) void sethdface (hdfacevertices, hdfacetrackingid, hdfacestatus) void setaudio (beamangle, beamangleconfidence, audiotrackingid) void drawaudiodirection () bool isopendaudio () void opendaudio () void closeaudio () void setspeechlang () void startspeech () void stopspeech () bool setspeech (recognizedspeech, speechtag, speechitem, speechconfidence) void setgesturefile void setgesture (discretegesture, discretegesturetrackingid, continuousgesture, continuousgesturetrackingid) 4 NtKinect Table 4 Member variables of NtKinect. RGB cv::mat rgbimage cv::mat depthimage BodyIndex cv::mat bodyindeximage BodyIndex cv::mat infraredimage vector<vector<joint>> skeleton vector<int> skeletonid BodyIndex vector<int64> skeletontrackingid TrackingID vector<vector<pointf>> facepoint vector<cv::rect> facerect vector<cv::vec3f> facedirection vector<vector<detectionresult>> faceproperty vector<uint64> facetrackingid skeletontrackingid vector<vector<cameraspacepoitn>> hdfacevertices vector<uint64> hdfacetrackingid skeletontrackingid vector<pair<int,int>> hdfacestatus float beamangle float beamangleconfidence beamangle UINT64 audiotrackingid skeletontrackingid bool recognizedspeech wstring speechtag wstring speechitem float speechconfidence float confidencethreshold vector<pair<...,float>> discretegesture vector<uint64> discretegesturetrackingid skeletontrackingid vector<pair<...,float>> continuousgesture vector<uint64> continuousgesturetrackingid skeletontrackingid c 2017 Information Processing Society of Japan 3
情報処理学会研究報告 #define USE FACE #include NtKinect. h using namespace s t d ; void dojob ( ) { NtKinect k i n e c t ; while ( 1 ) { k i n e c t. setrgb ( ) ; kinect. setskeleton () ; kinect. setface () ; f o r ( auto& r : k i n e c t. f a c e R e c t ) cv : : r e c t a n g l e ( k i n e c t. rgbimage, r, cv : : S c a l a r ( 0, 0, 0 ), 1) ; f o r ( auto& p e r s o n : k i n e c t. s k e l e t o n ) f o r ( auto& j o i n t : p e r s o n ) { i f ( j o i n t. T r a c k i n g S t a t e == T r a c k i n g S t a t e N o t T r a c k e d ) continue ; C o l o r S p a c e P o i n t cp ; k i n e c t. c oordin atemap per >MapCameraPointToColorSpace ( j o i n t. P o s i t i o n,& cp ) ; cv : : r e c t a n g l e ( k i n e c t. rgbimage, cv : : Rect ( ( i n t ) cp. X 5,( i n t ) cp. Y 5, 1 0, 1 0 ), cv : : S c a l a r ( 0, 0, 2 5 5 ),2) ; cv : : imshow ( rgb, k i n e c t. rgbimage ) ; auto key = cv : : waitkey ( 1 ) ; i f ( key == q ) break ; cv : : destroyallwindows ( ) ; i n t main ( i n t ar gc, char a r g v ) { try { dojob ( ) ; catch ( e x c e p t i o n &ex ) { c o u t << ex. what ( ) << e n d l ; return 0 ; 図 2 骨格認識と顔認識の C++コード ので ColorSpace 座標系の座標に変換してから RGB 画像上に四角形を描画する Joint 型のデータは その TrackingState メンバ変数の 値が TrackingState NotTracked である場合は意味を 持たないのでその場合は関節の表示処理を省略する 6. NtKinect を用いた DLL ファイルの開発 顔認識を行い 最大 6 人までの顔の方向 (Euler 角, すな わち pitch, yaw, roll の角度) を返す DLL のヘッダファイ ルを図 4 に C++のコードを図 5 に示す DLL プログラムを開発する場合は NtKinect 型のデー 図 3 骨格認識と顔認識の例 タをヒープ上に確保し void *型のデータとして返す ま た センサーから取得した Collections データは DLL 関 数を呼び出す側が確保したメモリ領域にコピーすることで kinect.setface() メソッドを呼び出して顔認識する 値を返す [6] 顔の矩形領域を黒で塗り潰す 関節の位置は CameraSpace 座標系で表現されている c 2017 Information Processing Society of Japan 4
#i f d e f NTKINECTDLL EXPORTS #define NTKINECTDLL API d e c l s p e c ( d l l e x p o r t ) #else #define NTKINECTDLL API d e c l s p e c ( d l l i m p o r t ) #endif extern C { NTKINECTDLL API void getkinect ( void ) ; NTKINECTDLL API int f a c e D i r e c t i o n ( void ptr, f l o a t d i r ) ; 4 DLL C++ #include s t d a f x. h #include NtKinectDll. h #define USE FACE #include NtKinect. h using namespace std ; NTKINECTDLL API void getkinect ( void ) { NtKinect k i n e c t = new NtKinect ; return static cast <void >( k i n e c t ) ; NTKINECTDLL API int f a c e D i r e c t i o n ( void ptr, f l o a t d i r ) { NtKinect k i n e c t = static cast <NtKinect >( ptr ) ; ( k i n e c t ). s e t S k e l e t o n ( ) ; ( k i n e c t ). setface ( ) ; int idx =0; for ( auto d : ( k i n e c t ). f a c e D i r e c t i o n ) { d i r [ idx++] = d [ 0 ] ; d i r [ idx++] = d [ 1 ] ; d i r [ idx++] = d [ 2 ] ; return ( k i n e c t ). f a c e D i r e c t i o n. s i z e ( ) ; 5 DLL C++ 7. DLL 6 DLL Unity 6 Unity Assets/x86 64/ DLL [6] ( 7) 8. Kinect V2 C++ NtKinect Kinect V2 DLL Unity Kinect V2 Microsoft NtKinect [7] Kinect V2 DLL Unity NtKinect NtKinect 2016 7 [8], [9], [10] NtKinect 2017 2 6 Google NtKinect 5 Unity Web Unreal Engine c 2017 Information Processing Society of Japan 5
using UnityEngine ; using System. C o l l e c t i o n s ; using System. Runtime. I n t e r o p S e r v i c e s ; public c la ss NtKinectBehaviour : MonoBehaviour { [ DllImport ( NtKinectDll ) ] private s t a t i c extern System. I n t P t r g e t K i n e c t ( ) ; [ DllImport ( NtKinectDll ) ] private s t a t i c extern int f a c e D i r e c t i o n ( System. I n t P t r k i n e c t, System. I n t P t r data ) ; private System. I n t P t r k i n e c t ; void S t a r t ( ) { k i n e c t = g etkinect ( ) ;... void Update ( ) { f l o a t [ ] data = new f l o a t [ bodycount 3 ] ; GCHandle gch = GCHandle. A l l o c ( data, GCHandleType. Pinned ) ; int n = f a c e D i r e c t i o n ( k i n e c t, gch. AddrOfPinnedObject ( ) ) ; gch. Free ( ) ; for ( int i =0; i <n / 3 ; i ++) { // data [ i 3 ], data [ i 3+1], data [ i 3+2] = p i t c h, yaw, r o l l 6 DLL Unity C# ( ) 7 Unity DLL 5 Google NtKinect 1 kinect 1 kinect 4 kinect 3 (2017/02/03 access). [6] Yoshihisa Nitta: NtKinect- Kinect V2 DLL Unity, http://nw.tsuda.ac.jp/lec/kinect2/kinectv2 dll2/ (2017/02/03 access). [7] Yoshihisa Nitta: NtKinect- Kinect V2 DLL Unity, http://nw.tsuda.ac.jp/lec/kinect2/kinectv2 dll3/ (2017/02/08 access). [8], :, 2016, P-131 (2016). [9],, :, NICOGRAPH 2016, P-4 (2016). [10],, :, WISS 2016, P-213 (2016). [1] Microsoft: Kinect for Windows, https://developer.microsoft.com/ja-jp/windows/kinect/, (2017/02/03 access). [2] Microsoft: Kinect for Windows SDK C++ Reference, https://msdn.microsoft.com/jajp/library/dn791993.aspx, (2017/02/03). [3],,, KINECT for Windows SDK v2 (2015). [4] Yoshihisa Nitta: NtKinect - Kinect V2 C++ Programming with OpenCV on Windows10, http://nw.tsuda.ac.jp/lec/kinect2/ (2017/02/03 access). [5] Yoshihisa Nitta: NtKinect- Kinect V2 (ColorSpace ), http://nw.tsuda.ac.jp/lec/kinect2/kinectv2 face/ c 2017 Information Processing Society of Japan 6