Differences

This shows you the differences between two versions of the page.

--- sensors:kinect_library [2017/03/01 16:49]
m16devan [Initialisation]
+++ sensors:kinect_library [2019/04/25 14:08] (current)
@@ Line 18: / Line 18: @@
 ==== Initializing the Kinect ====
-The first thing to do is to include the corresponding header <code> #include "kinect.h" </code>
+The first thing to do is to include the corresponding header <code cpp> #include "kinect.h" </code>
 To manage data from kinect you need to create and initialize the IKinectSensor object
-<code>
+<code cpp>
 IKinectSensor* kinect;
 HRESULT hr;
@@ Line 27: / Line 27: @@
 If the kinect is correctly initialized, you can now open it and initialize the desired streams as color, depth and skeleton. Note that the coordinate mapper is also initialized. It will be used to map the 3D skeleton to 2D images for display purpose.
-<code>
+<code cpp>
 hr = kinect->Open();
@@ Line 70: / Line 70: @@
 For getting the current data captured by the kinect, you need to call the 'AcquireLatestFrame' function for each of the reader.
-<code>
+<code cpp>
 // Color data
 IColorFrame* pColorFrame = NULL;
@@ Line 91: / Line 91: @@
 Here we assume that the color frame has been captured in 'pColorFrame'.
-<code>
+<code cpp>
 cv::Mat tmp;
 RGBQUAD *pColorBuffer = new RGBQUAD[1920 * 1080];
@@ Line 111: / Line 111: @@
 Here we assume that the depth frame has been captured in 'pDepthFrame'. In this case, the received data are depth values in milimeter. For display purpose, the conversion in pixel value is needed. This is done using minimum and maximum depth values.
-<code>
+<code cpp>
 USHORT nDepthMaxDistance = 0;
 USHORT nDepthMinDistance = 0;
@@ Line 145: / Line 145: @@
 Here we assume that the skeleton frame has been captured in 'pBodyFrame'. For better clarity, two external functions are employed for drawing the skeleton ('drawSkeleton' and 'drawLine').
-<code>
+<code cpp>
 cv::namedWindow("Skeleton");
 cv::Mat skeletonFrame(424, 512, CV_8UC3, cv::Scalar(0, 0, 0));
@@ Line 185: / Line 185: @@
 The two external functions are
-<code>
+<code cpp>
 void drawLine(cv::Mat src, Joint j1, Joint j2, ICoordinateMapper* mapper){
 	if (j1.TrackingState > 1 && j2.TrackingState > 1){
@@ Line 227: / Line 227: @@
 === Safe Release ===
 Note that each frame is safely released after being processed. This is done using the following function
-<code>
+<code cpp>
 // Safe release for interfaces
 template<class Interface>
@@ Line 240: / Line 240: @@
 At the end of the code, we correctly close the kinect sensor
-<code>
+<code cpp>
 kinect->Close();
 </code>
@@ Line 262: / Line 262: @@
 ==== Getting data ====
-=== Color data ===
 <code python>
+#color data
 if kinect.has_new_color_frame():
     frame = kinect.get_last_color_frame()
-</code>
-=== Depth data ===
+#depth data
-<code python>
 if kinect.has_new_depth_frame():
     frameD = kinect.get_last_depth_frame()
-</code>
-=== Skeleton data ===
+#skeleton data
-<code python>
 if kinect.has_new_body_frame():
     bodies= kinect.get_last_color_frame()
@@ Line 281: / Line 277: @@
 ==== Processing data ====
-Here is some examples how to proceed data in order to display it using pygame.
+Here is some examples how to proceed data in order to display it using pygame. Some packages must be first imported for this example
+<code python>
+import ctypes
+import _ctypes
+import pymedia.video.vcodec as vcodec
+import pygame
+from pygame.locals import *
+import numpy as np
+</code>
 === Color data ===
+This example shows how to display the color frame using pygame. The color frame has been captured in 'frame'.
 <code python>
 pygame.init()
-screen = pygame.display.set_mode((self._infoObject.current_w >> 1, self._infoObject.current_h >> 1),
+infoObject = pygame.display.Info()
+screen = pygame.display.set_mode((infoObject.current_w >> 1, infoObject.current_h >> 1),
                                                pygame.HWSURFACE|pygame.DOUBLEBUF|pygame.RESIZABLE, 32)
 # back buffer surface for getting Kinect color frames, 32bit color, width and height equal to the Kinect color frame size
-frame_surface = pygame.Surface((self._kinect.color_frame_desc.Width, self._kinect.color_frame_desc.Height), 0, 32)
+frame_surface = pygame.Surface((kinect.color_frame_desc.Width, kinect.color_frame_desc.Height), 0, 32)
+frame_surface.lock()
+address = kinect.surface_as_array(frame_surface.get_buffer())
+ctypes.memmove(address, frame.ctypes.data, frame.size)
+del address
+frame_surface.unlock()
+# --- copy back buffer surface pixels to the screen, resize it if needed and keep aspect ratio
+# --- (screen size may be different from Kinect's color frame size)
+h_to_w = float(frame_surface.get_height()) / frame_surface.get_width()
+target_height = int(h_to_w * screen.get_width())
+surface_to_draw = pygame.transform.scale(frame_surface, (screen.get_width(), target_height));
+screen.blit(surface_to_draw, (0,0))
+surface_to_draw = None
+pygame.display.update()
+# --- Go ahead and update the screen with what we've drawn.
+pygame.display.flip()
 </code>
+=== Skeleton data ===
+This code shows how to add skeleton data to the color image. Skeletons have been captured in 'bodies'. It must be implemented before copying back buffer surface pixels to the screen (see above).
+<code python>
+# --- draw skeletons to _frame_surface
+if bodies is not None:
+      for i in range(0, kinect.max_body_count):
+           body = bodies.bodies[i]
+           if not body.is_tracked:
+               continue
+           joints = body.joints
+           ori = body.joint_orientations
+           # convert joint coordinates to color space
+           joint_points_color = kinect.body_joints_to_color_space(joints)
+           draw_body(joints, joint_points_color, pygame.color.THECOLORS["red"])
+</code>
+Similarly to C library, two external functions are used to display the skeleton
+<code python>
+def draw_body_bone(joints, jointColorPoints, color, joint0, joint1):
+    joint0State = joints[joint0].TrackingState;
+    joint1State = joints[joint1].TrackingState;
+    # both joints are not tracked
+    if (joint0State == PyKinectV2.TrackingState_NotTracked) or (joint1State == PyKinectV2.TrackingState_NotTracked):
+        return
+    # both joints are not *really* tracked
+    if (joint0State == PyKinectV2.TrackingState_Inferred) and (joint1State == PyKinectV2.TrackingState_Inferred):
+       return
+    # at least one is good
+    start = (jointColorPoints[joint0].x, jointColorPoints[joint0].y)
+    end = (jointColorPoints[joint1].x, jointColorPoints[joint1].y)
+    try:
+       pygame.draw.line(frame_surface, color, start, end, 8)
+    except: # need to catch it due to possible invalid positions (with inf)
+       pass
+def draw_body(joints, jointColorPoints, color):
+    # Torso
+    draw_body_bone(joints, jointColorPoints, color, PyKinectV2.JointType_Head, PyKinectV2.JointType_Neck);
+    draw_body_bone(joints, jointColorPoints, color, PyKinectV2.JointType_Neck, PyKinectV2.JointType_SpineShoulder);
+    draw_body_bone(joints, jointColorPoints, color, PyKinectV2.JointType_SpineShoulder, PyKinectV2.JointType_SpineMid);
+    draw_body_bone(joints, jointColorPoints, color, PyKinectV2.JointType_SpineMid, PyKinectV2.JointType_SpineBase);
+    draw_body_bone(joints, jointColorPoints, color, PyKinectV2.JointType_SpineShoulder, PyKinectV2.JointType_ShoulderRight);
+    draw_body_bone(joints, jointColorPoints, color, PyKinectV2.JointType_SpineShoulder, PyKinectV2.JointType_ShoulderLeft);
+    draw_body_bone(joints, jointColorPoints, color, PyKinectV2.JointType_SpineBase, PyKinectV2.JointType_HipRight);
+    draw_body_bone(joints, jointColorPoints, color, PyKinectV2.JointType_SpineBase, PyKinectV2.JointType_HipLeft);
+    # Right Arm
+    draw_body_bone(joints, jointColorPoints, color, PyKinectV2.JointType_ShoulderRight, PyKinectV2.JointType_ElbowRight);
+    draw_body_bone(joints, jointColorPoints, color, PyKinectV2.JointType_ElbowRight, PyKinectV2.JointType_WristRight);
+    draw_body_bone(joints, jointColorPoints, color, PyKinectV2.JointType_WristRight, PyKinectV2.JointType_HandRight);
+    draw_body_bone(joints, jointColorPoints, color, PyKinectV2.JointType_HandRight, PyKinectV2.JointType_HandTipRight);
+    draw_body_bone(joints, jointColorPoints, color, PyKinectV2.JointType_WristRight, PyKinectV2.JointType_ThumbRight);
+    # Left Arm
+    draw_body_bone(joints, jointColorPoints, color, PyKinectV2.JointType_ShoulderLeft, PyKinectV2.JointType_ElbowLeft);
+    draw_body_bone(joints, jointColorPoints, color, PyKinectV2.JointType_ElbowLeft, PyKinectV2.JointType_WristLeft);
+    draw_body_bone(joints, jointColorPoints, color, PyKinectV2.JointType_WristLeft, PyKinectV2.JointType_HandLeft);
+    draw_body_bone(joints, jointColorPoints, color, PyKinectV2.JointType_HandLeft, PyKinectV2.JointType_HandTipLeft);
+    draw_body_bone(joints, jointColorPoints, color, PyKinectV2.JointType_WristLeft, PyKinectV2.JointType_ThumbLeft);
+    # Right Leg
+    draw_body_bone(joints, jointColorPoints, color, PyKinectV2.JointType_HipRight, PyKinectV2.JointType_KneeRight);
+    draw_body_bone(joints, jointColorPoints, color, PyKinectV2.JointType_KneeRight, PyKinectV2.JointType_AnkleRight);
+    draw_body_bone(joints, jointColorPoints, color, PyKinectV2.JointType_AnkleRight, PyKinectV2.JointType_FootRight);
+    # Left Leg
+    draw_body_bone(joints, jointColorPoints, color, PyKinectV2.JointType_HipLeft, PyKinectV2.JointType_KneeLeft);
+    draw_body_bone(joints, jointColorPoints, color, PyKinectV2.JointType_KneeLeft, PyKinectV2.JointType_AnkleLeft);
+    draw_body_bone(joints, jointColorPoints, color, PyKinectV2.JointType_AnkleLeft, PyKinectV2.JointType_FootLeft);
+</code>
+=== Depth data ===
+This example shows how to display the depth frame using pygame. The depth frame has been captured in 'frameD'.
+<code python>
+pygame.init()
+infoObject = pygame.display.Info()
+screen = pygame.display.set_mode((infoObject.current_w >> 1, infoObject.current_h >> 1),
+                                               pygame.HWSURFACE|pygame.DOUBLEBUF|pygame.RESIZABLE, 32)
+# back buffer surface for getting Kinect depth frames, 16 bit depth, width and height equal to the Kinect depth frame size
+frame_surface = pygame.Surface((kinect.depth_frame_desc.Width, kinect.depth_frame_desc.Height), 0, 24)
+frame_surface.lock()
+f8=np.uint8(frameD.clip(1,4000)/16.)
+frame8bit=np.dstack((f8,f8,f8))
+address = kinect.surface_as_array(target_surface.get_buffer())
+ctypes.memmove(address, frame8bit.ctypes.data, frame8bit.size)
+del address
+frame_surface.unlock()
+# --- copy back buffer surface pixels to the screen, resize it if needed and keep aspect ratio
+h_to_w = float(frame_surface.get_height()) / frame_surface.get_width()
+target_height = int(h_to_w * screen.get_width())
+surface_to_draw = pygame.transform.scale(frame_surface, (screen.get_width(), target_height));
+screen.blit(surface_to_draw, (0,0))
+surface_to_draw = None
+pygame.display.update()
+# --- Go ahead and update the screen with what we've drawn.
+pygame.display.flip()
+</code>
+==== Closing the Kinect ====
+<code python>
+kinect.close()
+</code>