Kinect in Windows for Processing 5

Posted 2012/08/31 by bryan

This is a more or less finished version of the Kinect for Processing library. It includes the basic skeleton tracking, the RGB image, depth image and a mask image. I shall move the related posts to the research page with more documentation on the data structure and method description. Stay tune.

The sample Processing code:

import pKinect.PKinect;
import pKinect.SkeletonData;
 
PKinect kinect;
PFont font;
ArrayList<SkeletonData> bodies;
PImage img;
 
void setup()
{
  size(640, 480);
  background(0);
  kinect = new PKinect(this);
  bodies = new ArrayList<SkeletonData>();
  smooth();
  font = loadFont("LucidaSans-18.vlw");
  textFont(font, 18);
  textAlign(CENTER);
  img = loadImage("background.png");
}
 
void draw()
{
  background(0);
  image(kinect.GetImage(), 320, 0, 320, 240);
  image(kinect.GetDepth(), 320, 240, 320, 240);
  image(img, 0, 240, 320, 240);
  image(kinect.GetMask(), 0, 240, 320, 240);
  for (int i=0; i<bodies.size(); i++) 
  {
    drawSkeleton(bodies.get(i));
    drawPosition(bodies.get(i));
  }
}
 
void mousePressed() 
{
  println(frameRate);
}
 
void drawPosition(SkeletonData _s) 
{
  noStroke();
  fill(0, 100, 255);
  String s1 = str(_s.dwTrackingID);
  text(s1, _s.position.x*width/2, _s.position.y*height/2);
}
 
void drawSkeleton(SkeletonData _s) 
{
  // Body
  DrawBone(_s, 
  PKinect.NUI_SKELETON_POSITION_HEAD, 
  PKinect.NUI_SKELETON_POSITION_SHOULDER_CENTER);
  DrawBone(_s, 
  PKinect.NUI_SKELETON_POSITION_SHOULDER_CENTER, 
  PKinect.NUI_SKELETON_POSITION_SHOULDER_LEFT);
  DrawBone(_s, 
  PKinect.NUI_SKELETON_POSITION_SHOULDER_CENTER, 
  PKinect.NUI_SKELETON_POSITION_SHOULDER_RIGHT);
  DrawBone(_s, 
  PKinect.NUI_SKELETON_POSITION_SHOULDER_CENTER, 
  PKinect.NUI_SKELETON_POSITION_SPINE);
  DrawBone(_s, 
  PKinect.NUI_SKELETON_POSITION_SHOULDER_LEFT, 
  PKinect.NUI_SKELETON_POSITION_SPINE);
  DrawBone(_s, 
  PKinect.NUI_SKELETON_POSITION_SHOULDER_RIGHT, 
  PKinect.NUI_SKELETON_POSITION_SPINE);
  DrawBone(_s, 
  PKinect.NUI_SKELETON_POSITION_SPINE, 
  PKinect.NUI_SKELETON_POSITION_HIP_CENTER);
  DrawBone(_s, 
  PKinect.NUI_SKELETON_POSITION_HIP_CENTER, 
  PKinect.NUI_SKELETON_POSITION_HIP_LEFT);
  DrawBone(_s, 
  PKinect.NUI_SKELETON_POSITION_HIP_CENTER, 
  PKinect.NUI_SKELETON_POSITION_HIP_RIGHT);
  DrawBone(_s, 
  PKinect.NUI_SKELETON_POSITION_HIP_LEFT, 
  PKinect.NUI_SKELETON_POSITION_HIP_RIGHT);
 
  // Left Arm
  DrawBone(_s, 
  PKinect.NUI_SKELETON_POSITION_SHOULDER_LEFT, 
  PKinect.NUI_SKELETON_POSITION_ELBOW_LEFT);
  DrawBone(_s, 
  PKinect.NUI_SKELETON_POSITION_ELBOW_LEFT, 
  PKinect.NUI_SKELETON_POSITION_WRIST_LEFT);
  DrawBone(_s, 
  PKinect.NUI_SKELETON_POSITION_WRIST_LEFT, 
  PKinect.NUI_SKELETON_POSITION_HAND_LEFT);
 
  // Right Arm
  DrawBone(_s, 
  PKinect.NUI_SKELETON_POSITION_SHOULDER_RIGHT, 
  PKinect.NUI_SKELETON_POSITION_ELBOW_RIGHT);
  DrawBone(_s, 
  PKinect.NUI_SKELETON_POSITION_ELBOW_RIGHT, 
  PKinect.NUI_SKELETON_POSITION_WRIST_RIGHT);
  DrawBone(_s, 
  PKinect.NUI_SKELETON_POSITION_WRIST_RIGHT, 
  PKinect.NUI_SKELETON_POSITION_HAND_RIGHT);
 
  // Left Leg
  DrawBone(_s, 
  PKinect.NUI_SKELETON_POSITION_HIP_LEFT, 
  PKinect.NUI_SKELETON_POSITION_KNEE_LEFT);
  DrawBone(_s, 
  PKinect.NUI_SKELETON_POSITION_KNEE_LEFT, 
  PKinect.NUI_SKELETON_POSITION_ANKLE_LEFT);
  DrawBone(_s, 
  PKinect.NUI_SKELETON_POSITION_ANKLE_LEFT, 
  PKinect.NUI_SKELETON_POSITION_FOOT_LEFT);
 
  // Right Leg
  DrawBone(_s, 
  PKinect.NUI_SKELETON_POSITION_HIP_RIGHT, 
  PKinect.NUI_SKELETON_POSITION_KNEE_RIGHT);
  DrawBone(_s, 
  PKinect.NUI_SKELETON_POSITION_KNEE_RIGHT, 
  PKinect.NUI_SKELETON_POSITION_ANKLE_RIGHT);
  DrawBone(_s, 
  PKinect.NUI_SKELETON_POSITION_ANKLE_RIGHT, 
  PKinect.NUI_SKELETON_POSITION_FOOT_RIGHT);
}
 
void DrawBone(SkeletonData _s, int _j1, int _j2) 
{
  noFill();
  stroke(255, 255, 0);
  if (_s.skeletonPositionTrackingState[_j1] != PKinect.NUI_SKELETON_POSITION_NOT_TRACKED &&
    _s.skeletonPositionTrackingState[_j2] != PKinect.NUI_SKELETON_POSITION_NOT_TRACKED) {
    line(_s.skeletonPositions[_j1].x*width/2, 
    _s.skeletonPositions[_j1].y*height/2, 
    _s.skeletonPositions[_j2].x*width/2, 
    _s.skeletonPositions[_j2].y*height/2);
  }
}
 
void appearEvent(SkeletonData _s) 
{
  if (_s.trackingState == PKinect.NUI_SKELETON_NOT_TRACKED) 
  {
    return;
  }
  synchronized(bodies) {
    bodies.add(_s);
  }
}
 
void disappearEvent(SkeletonData _s) 
{
  synchronized(bodies) {
    for (int i=bodies.size()-1; i>=0; i--) 
    {
      if (_s.dwTrackingID == bodies.get(i).dwTrackingID) 
      {
        bodies.remove(i);
      }
    }
  }
}
 
void moveEvent(SkeletonData _b, SkeletonData _a) 
{
  if (_a.trackingState == PKinect.NUI_SKELETON_NOT_TRACKED) 
  {
    return;
  }
  synchronized(bodies) {
    for (int i=bodies.size()-1; i>=0; i--) 
    {
      if (_b.dwTrackingID == bodies.get(i).dwTrackingID) 
      {
        bodies.get(i).copy(_a);
        break;
      }
    }
  }
}

import pKinect.PKinect; import pKinect.SkeletonData; PKinect kinect; PFont font; ArrayList<SkeletonData> bodies; PImage img; void setup() { size(640, 480); background(0); kinect = new PKinect(this); bodies = new ArrayList<SkeletonData>(); smooth(); font = loadFont("LucidaSans-18.vlw"); textFont(font, 18); textAlign(CENTER); img = loadImage("background.png"); } void draw() { background(0); image(kinect.GetImage(), 320, 0, 320, 240); image(kinect.GetDepth(), 320, 240, 320, 240); image(img, 0, 240, 320, 240); image(kinect.GetMask(), 0, 240, 320, 240); for (int i=0; i<bodies.size(); i++) { drawSkeleton(bodies.get(i)); drawPosition(bodies.get(i)); } } void mousePressed() { println(frameRate); } void drawPosition(SkeletonData _s) { noStroke(); fill(0, 100, 255); String s1 = str(_s.dwTrackingID); text(s1, _s.position.x*width/2, _s.position.y*height/2); } void drawSkeleton(SkeletonData _s) { // Body DrawBone(_s, PKinect.NUI_SKELETON_POSITION_HEAD, PKinect.NUI_SKELETON_POSITION_SHOULDER_CENTER); DrawBone(_s, PKinect.NUI_SKELETON_POSITION_SHOULDER_CENTER, PKinect.NUI_SKELETON_POSITION_SHOULDER_LEFT); DrawBone(_s, PKinect.NUI_SKELETON_POSITION_SHOULDER_CENTER, PKinect.NUI_SKELETON_POSITION_SHOULDER_RIGHT); DrawBone(_s, PKinect.NUI_SKELETON_POSITION_SHOULDER_CENTER, PKinect.NUI_SKELETON_POSITION_SPINE); DrawBone(_s, PKinect.NUI_SKELETON_POSITION_SHOULDER_LEFT, PKinect.NUI_SKELETON_POSITION_SPINE); DrawBone(_s, PKinect.NUI_SKELETON_POSITION_SHOULDER_RIGHT, PKinect.NUI_SKELETON_POSITION_SPINE); DrawBone(_s, PKinect.NUI_SKELETON_POSITION_SPINE, PKinect.NUI_SKELETON_POSITION_HIP_CENTER); DrawBone(_s, PKinect.NUI_SKELETON_POSITION_HIP_CENTER, PKinect.NUI_SKELETON_POSITION_HIP_LEFT); DrawBone(_s, PKinect.NUI_SKELETON_POSITION_HIP_CENTER, PKinect.NUI_SKELETON_POSITION_HIP_RIGHT); DrawBone(_s, PKinect.NUI_SKELETON_POSITION_HIP_LEFT, PKinect.NUI_SKELETON_POSITION_HIP_RIGHT); // Left Arm DrawBone(_s, PKinect.NUI_SKELETON_POSITION_SHOULDER_LEFT, PKinect.NUI_SKELETON_POSITION_ELBOW_LEFT); DrawBone(_s, PKinect.NUI_SKELETON_POSITION_ELBOW_LEFT, PKinect.NUI_SKELETON_POSITION_WRIST_LEFT); DrawBone(_s, PKinect.NUI_SKELETON_POSITION_WRIST_LEFT, PKinect.NUI_SKELETON_POSITION_HAND_LEFT); // Right Arm DrawBone(_s, PKinect.NUI_SKELETON_POSITION_SHOULDER_RIGHT, PKinect.NUI_SKELETON_POSITION_ELBOW_RIGHT); DrawBone(_s, PKinect.NUI_SKELETON_POSITION_ELBOW_RIGHT, PKinect.NUI_SKELETON_POSITION_WRIST_RIGHT); DrawBone(_s, PKinect.NUI_SKELETON_POSITION_WRIST_RIGHT, PKinect.NUI_SKELETON_POSITION_HAND_RIGHT); // Left Leg DrawBone(_s, PKinect.NUI_SKELETON_POSITION_HIP_LEFT, PKinect.NUI_SKELETON_POSITION_KNEE_LEFT); DrawBone(_s, PKinect.NUI_SKELETON_POSITION_KNEE_LEFT, PKinect.NUI_SKELETON_POSITION_ANKLE_LEFT); DrawBone(_s, PKinect.NUI_SKELETON_POSITION_ANKLE_LEFT, PKinect.NUI_SKELETON_POSITION_FOOT_LEFT); // Right Leg DrawBone(_s, PKinect.NUI_SKELETON_POSITION_HIP_RIGHT, PKinect.NUI_SKELETON_POSITION_KNEE_RIGHT); DrawBone(_s, PKinect.NUI_SKELETON_POSITION_KNEE_RIGHT, PKinect.NUI_SKELETON_POSITION_ANKLE_RIGHT); DrawBone(_s, PKinect.NUI_SKELETON_POSITION_ANKLE_RIGHT, PKinect.NUI_SKELETON_POSITION_FOOT_RIGHT); } void DrawBone(SkeletonData _s, int _j1, int _j2) { noFill(); stroke(255, 255, 0); if (_s.skeletonPositionTrackingState[_j1] != PKinect.NUI_SKELETON_POSITION_NOT_TRACKED && _s.skeletonPositionTrackingState[_j2] != PKinect.NUI_SKELETON_POSITION_NOT_TRACKED) { line(_s.skeletonPositions[_j1].x*width/2, _s.skeletonPositions[_j1].y*height/2, _s.skeletonPositions[_j2].x*width/2, _s.skeletonPositions[_j2].y*height/2); } } void appearEvent(SkeletonData _s) { if (_s.trackingState == PKinect.NUI_SKELETON_NOT_TRACKED) { return; } synchronized(bodies) { bodies.add(_s); } } void disappearEvent(SkeletonData _s) { synchronized(bodies) { for (int i=bodies.size()-1; i>=0; i--) { if (_s.dwTrackingID == bodies.get(i).dwTrackingID) { bodies.remove(i); } } } } void moveEvent(SkeletonData _b, SkeletonData _a) { if (_a.trackingState == PKinect.NUI_SKELETON_NOT_TRACKED) { return; } synchronized(bodies) { for (int i=bodies.size()-1; i>=0; i--) { if (_b.dwTrackingID == bodies.get(i).dwTrackingID) { bodies.get(i).copy(_a); break; } } } }

You can download the example and the library here.

Kinect for Windows in Processing 4

Posted 2012/08/29 by bryan

This is the new version of the Kinect for Processing library using the Kinect for Windows 1.5 SDK. In this version, I only implement the skeleton tracking, without the previous RGB image and depth image yet. For all the field, method and constant names, I try to use the same ones as in the SDK. It can have multiple skeletons and comes with three events:

appear (new skeleton enters the screen)
disappear (existing skeleton leaves the screen)
move (existing skeleton stay in the screen)

Here is the example Processing code.

import pKinect.PKinect;
 
PKinect kinect;
PFont font;
ArrayList<SkeletonData> bodies;
 
void setup()
{
  size(640, 480);
  background(0);
  kinect = new PKinect(this);
  bodies = new ArrayList<SkeletonData>();
  smooth();
  font = loadFont("LucidaSans-18.vlw");
  textFont(font, 18);
  textAlign(CENTER);
}
 
void draw()
{
  fill(0, 0, 0, 16);
  rect(0, 0, width, height);
  for (int i=0; i<bodies.size(); i++) 
  {
    drawSkeleton(bodies.get(i));
    drawPosition(bodies.get(i));
  }
}
 
void drawPosition(SkeletonData _s) 
{
  noStroke();
  fill(0, 255, 255);
  String s1 = str(_s.dwTrackingID);
  text(s1, _s.position.x*width, _s.position.y*height);
}
 
void drawSkeleton(SkeletonData _s) 
{
  // Body
  DrawBone(_s, 
  PKinect.NUI_SKELETON_POSITION_HEAD, 
  PKinect.NUI_SKELETON_POSITION_SHOULDER_CENTER);
  DrawBone(_s, 
  PKinect.NUI_SKELETON_POSITION_SHOULDER_CENTER, 
  PKinect.NUI_SKELETON_POSITION_SHOULDER_LEFT);
  DrawBone(_s, 
  PKinect.NUI_SKELETON_POSITION_SHOULDER_CENTER, 
  PKinect.NUI_SKELETON_POSITION_SHOULDER_RIGHT);
  DrawBone(_s, 
  PKinect.NUI_SKELETON_POSITION_SHOULDER_CENTER, 
  PKinect.NUI_SKELETON_POSITION_SPINE);
  DrawBone(_s, 
  PKinect.NUI_SKELETON_POSITION_SHOULDER_LEFT, 
  PKinect.NUI_SKELETON_POSITION_SPINE);
  DrawBone(_s, 
  PKinect.NUI_SKELETON_POSITION_SHOULDER_RIGHT, 
  PKinect.NUI_SKELETON_POSITION_SPINE);
  DrawBone(_s, 
  PKinect.NUI_SKELETON_POSITION_SPINE, 
  PKinect.NUI_SKELETON_POSITION_HIP_CENTER);
  DrawBone(_s, 
  PKinect.NUI_SKELETON_POSITION_HIP_CENTER, 
  PKinect.NUI_SKELETON_POSITION_HIP_LEFT);
  DrawBone(_s, 
  PKinect.NUI_SKELETON_POSITION_HIP_CENTER, 
  PKinect.NUI_SKELETON_POSITION_HIP_RIGHT);
  DrawBone(_s, 
  PKinect.NUI_SKELETON_POSITION_HIP_LEFT, 
  PKinect.NUI_SKELETON_POSITION_HIP_RIGHT);
 
  // Left Arm
  DrawBone(_s, 
  PKinect.NUI_SKELETON_POSITION_SHOULDER_LEFT, 
  PKinect.NUI_SKELETON_POSITION_ELBOW_LEFT);
  DrawBone(_s, 
  PKinect.NUI_SKELETON_POSITION_ELBOW_LEFT, 
  PKinect.NUI_SKELETON_POSITION_WRIST_LEFT);
  DrawBone(_s, 
  PKinect.NUI_SKELETON_POSITION_WRIST_LEFT, 
  PKinect.NUI_SKELETON_POSITION_HAND_LEFT);
 
  // Right Arm
  DrawBone(_s, 
  PKinect.NUI_SKELETON_POSITION_SHOULDER_RIGHT, 
  PKinect.NUI_SKELETON_POSITION_ELBOW_RIGHT);
  DrawBone(_s, 
  PKinect.NUI_SKELETON_POSITION_ELBOW_RIGHT, 
  PKinect.NUI_SKELETON_POSITION_WRIST_RIGHT);
  DrawBone(_s, 
  PKinect.NUI_SKELETON_POSITION_WRIST_RIGHT, 
  PKinect.NUI_SKELETON_POSITION_HAND_RIGHT);
 
  // Left Leg
  DrawBone(_s, 
  PKinect.NUI_SKELETON_POSITION_HIP_LEFT, 
  PKinect.NUI_SKELETON_POSITION_KNEE_LEFT);
  DrawBone(_s, 
  PKinect.NUI_SKELETON_POSITION_KNEE_LEFT, 
  PKinect.NUI_SKELETON_POSITION_ANKLE_LEFT);
  DrawBone(_s, 
  PKinect.NUI_SKELETON_POSITION_ANKLE_LEFT, 
  PKinect.NUI_SKELETON_POSITION_FOOT_LEFT);
 
  // Right Leg
  DrawBone(_s, 
  PKinect.NUI_SKELETON_POSITION_HIP_RIGHT, 
  PKinect.NUI_SKELETON_POSITION_KNEE_RIGHT);
  DrawBone(_s, 
  PKinect.NUI_SKELETON_POSITION_KNEE_RIGHT, 
  PKinect.NUI_SKELETON_POSITION_ANKLE_RIGHT);
  DrawBone(_s, 
  PKinect.NUI_SKELETON_POSITION_ANKLE_RIGHT, 
  PKinect.NUI_SKELETON_POSITION_FOOT_RIGHT);
}
 
void DrawBone(SkeletonData _s, int _j1, int _j2) 
{
  noFill();
  stroke(255, 200, 0);
  if (_s.skeletonPositionTrackingState[_j1] != PKinect.NUI_SKELETON_POSITION_NOT_TRACKED &&
    _s.skeletonPositionTrackingState[_j2] != PKinect.NUI_SKELETON_POSITION_NOT_TRACKED) {
    line(_s.skeletonPositions[_j1].x*width, 
    _s.skeletonPositions[_j1].y*height, 
    _s.skeletonPositions[_j2].x*width, 
    _s.skeletonPositions[_j2].y*height);
  }
}
 
void appearEvent(SkeletonData _s) 
{
  if (_s.trackingState == PKinect.NUI_SKELETON_NOT_TRACKED) 
  {
    return;
  }
  synchronized(bodies) {
    bodies.add(_s);
  }
  println("appearing ..." + _s.dwTrackingID);
}
 
void disappearEvent(SkeletonData _s) 
{
  synchronized(bodies) {
    for (int i=bodies.size()-1; i>=0; i--) 
    {
      if (_s.dwTrackingID == bodies.get(i).dwTrackingID) 
      {
        bodies.remove(i);
      }
    }
  }
  println("Disappearing ... " + _s.dwTrackingID);
}
 
void moveEvent(SkeletonData _b, SkeletonData _a) 
{
  if (_a.trackingState == PKinect.NUI_SKELETON_NOT_TRACKED) 
  {
    return;
  }
  synchronized(bodies) {
    for (int i=bodies.size()-1; i>=0; i--) 
    {
      if (_b.dwTrackingID == bodies.get(i).dwTrackingID) 
      {
        bodies.get(i).copy(_a);
        break;
      }
    }
  }
}

import pKinect.PKinect; PKinect kinect; PFont font; ArrayList<SkeletonData> bodies; void setup() { size(640, 480); background(0); kinect = new PKinect(this); bodies = new ArrayList<SkeletonData>(); smooth(); font = loadFont("LucidaSans-18.vlw"); textFont(font, 18); textAlign(CENTER); } void draw() { fill(0, 0, 0, 16); rect(0, 0, width, height); for (int i=0; i<bodies.size(); i++) { drawSkeleton(bodies.get(i)); drawPosition(bodies.get(i)); } } void drawPosition(SkeletonData _s) { noStroke(); fill(0, 255, 255); String s1 = str(_s.dwTrackingID); text(s1, _s.position.x*width, _s.position.y*height); } void drawSkeleton(SkeletonData _s) { // Body DrawBone(_s, PKinect.NUI_SKELETON_POSITION_HEAD, PKinect.NUI_SKELETON_POSITION_SHOULDER_CENTER); DrawBone(_s, PKinect.NUI_SKELETON_POSITION_SHOULDER_CENTER, PKinect.NUI_SKELETON_POSITION_SHOULDER_LEFT); DrawBone(_s, PKinect.NUI_SKELETON_POSITION_SHOULDER_CENTER, PKinect.NUI_SKELETON_POSITION_SHOULDER_RIGHT); DrawBone(_s, PKinect.NUI_SKELETON_POSITION_SHOULDER_CENTER, PKinect.NUI_SKELETON_POSITION_SPINE); DrawBone(_s, PKinect.NUI_SKELETON_POSITION_SHOULDER_LEFT, PKinect.NUI_SKELETON_POSITION_SPINE); DrawBone(_s, PKinect.NUI_SKELETON_POSITION_SHOULDER_RIGHT, PKinect.NUI_SKELETON_POSITION_SPINE); DrawBone(_s, PKinect.NUI_SKELETON_POSITION_SPINE, PKinect.NUI_SKELETON_POSITION_HIP_CENTER); DrawBone(_s, PKinect.NUI_SKELETON_POSITION_HIP_CENTER, PKinect.NUI_SKELETON_POSITION_HIP_LEFT); DrawBone(_s, PKinect.NUI_SKELETON_POSITION_HIP_CENTER, PKinect.NUI_SKELETON_POSITION_HIP_RIGHT); DrawBone(_s, PKinect.NUI_SKELETON_POSITION_HIP_LEFT, PKinect.NUI_SKELETON_POSITION_HIP_RIGHT); // Left Arm DrawBone(_s, PKinect.NUI_SKELETON_POSITION_SHOULDER_LEFT, PKinect.NUI_SKELETON_POSITION_ELBOW_LEFT); DrawBone(_s, PKinect.NUI_SKELETON_POSITION_ELBOW_LEFT, PKinect.NUI_SKELETON_POSITION_WRIST_LEFT); DrawBone(_s, PKinect.NUI_SKELETON_POSITION_WRIST_LEFT, PKinect.NUI_SKELETON_POSITION_HAND_LEFT); // Right Arm DrawBone(_s, PKinect.NUI_SKELETON_POSITION_SHOULDER_RIGHT, PKinect.NUI_SKELETON_POSITION_ELBOW_RIGHT); DrawBone(_s, PKinect.NUI_SKELETON_POSITION_ELBOW_RIGHT, PKinect.NUI_SKELETON_POSITION_WRIST_RIGHT); DrawBone(_s, PKinect.NUI_SKELETON_POSITION_WRIST_RIGHT, PKinect.NUI_SKELETON_POSITION_HAND_RIGHT); // Left Leg DrawBone(_s, PKinect.NUI_SKELETON_POSITION_HIP_LEFT, PKinect.NUI_SKELETON_POSITION_KNEE_LEFT); DrawBone(_s, PKinect.NUI_SKELETON_POSITION_KNEE_LEFT, PKinect.NUI_SKELETON_POSITION_ANKLE_LEFT); DrawBone(_s, PKinect.NUI_SKELETON_POSITION_ANKLE_LEFT, PKinect.NUI_SKELETON_POSITION_FOOT_LEFT); // Right Leg DrawBone(_s, PKinect.NUI_SKELETON_POSITION_HIP_RIGHT, PKinect.NUI_SKELETON_POSITION_KNEE_RIGHT); DrawBone(_s, PKinect.NUI_SKELETON_POSITION_KNEE_RIGHT, PKinect.NUI_SKELETON_POSITION_ANKLE_RIGHT); DrawBone(_s, PKinect.NUI_SKELETON_POSITION_ANKLE_RIGHT, PKinect.NUI_SKELETON_POSITION_FOOT_RIGHT); } void DrawBone(SkeletonData _s, int _j1, int _j2) { noFill(); stroke(255, 200, 0); if (_s.skeletonPositionTrackingState[_j1] != PKinect.NUI_SKELETON_POSITION_NOT_TRACKED && _s.skeletonPositionTrackingState[_j2] != PKinect.NUI_SKELETON_POSITION_NOT_TRACKED) { line(_s.skeletonPositions[_j1].x*width, _s.skeletonPositions[_j1].y*height, _s.skeletonPositions[_j2].x*width, _s.skeletonPositions[_j2].y*height); } } void appearEvent(SkeletonData _s) { if (_s.trackingState == PKinect.NUI_SKELETON_NOT_TRACKED) { return; } synchronized(bodies) { bodies.add(_s); } println("appearing ..." + _s.dwTrackingID); } void disappearEvent(SkeletonData _s) { synchronized(bodies) { for (int i=bodies.size()-1; i>=0; i--) { if (_s.dwTrackingID == bodies.get(i).dwTrackingID) { bodies.remove(i); } } } println("Disappearing ... " + _s.dwTrackingID); } void moveEvent(SkeletonData _b, SkeletonData _a) { if (_a.trackingState == PKinect.NUI_SKELETON_NOT_TRACKED) { return; } synchronized(bodies) { for (int i=bodies.size()-1; i>=0; i--) { if (_b.dwTrackingID == bodies.get(i).dwTrackingID) { bodies.get(i).copy(_a); break; } } } }

The Processing code and library can be downloaded here.

Telling 32-bit or 64-bit in Java

Posted 2012/08/27 by bryan

Here is a short tip to check whether the current machine (JVM) is running 32 bit or 64 bit in Java and thus Processing.

int arch = Integer.parseInt(System.getProperty("sun.arch.data.model"));

The integer variable arch will give either 32 or 64 depending on the JVM.

Kinect for Java

Posted 2012/08/26 by bryan

I found this Java wrapper Jnet for Eclipse for the Kinect for Windows driver.

Kinect for Windows in Processing 3

Posted 2012/08/25 by bryan

Finally, the skeleton part of the library is done. In this very experimental version, I extract only one skeleton and store the joints information in an array (size 20) of PVector type in Processing. The tracking state is not implemented yet. I use the z-depth value to indicate if that joint is validly tracked or not. The x and y values are normalized to the range from 0 to 1 in the screen space. In the next version, I would like to implement the async event of tracking. It is time to integrate the all three components:

Individual RGB and depth images
Aligned RGB and depth image
Skeleton tracking

Here is a copy of the sample Processing code.

import pKinect.PKinect;
 
PKinect kinect;
PVector [] loc;
 
void setup()
{
  size(640, 480);
  kinect = new PKinect(this);
  smooth();
  noFill();
  stroke(255, 255, 0);
}
 
void draw()
{
  background(0);
  loc = kinect.getSkeleton();
  drawSkeleton();
}
 
void drawSkeleton()
{
  // Body
  DrawBone(kinect.NUI_SKELETON_POSITION_HEAD,
  kinect.NUI_SKELETON_POSITION_SHOULDER_CENTER);
  DrawBone(kinect.NUI_SKELETON_POSITION_SHOULDER_CENTER,
  kinect.NUI_SKELETON_POSITION_SHOULDER_LEFT);
  DrawBone(kinect.NUI_SKELETON_POSITION_SHOULDER_CENTER,
  kinect.NUI_SKELETON_POSITION_SHOULDER_RIGHT);
  DrawBone(kinect.NUI_SKELETON_POSITION_SHOULDER_CENTER,
  kinect.NUI_SKELETON_POSITION_SPINE);
  DrawBone(kinect.NUI_SKELETON_POSITION_SPINE,
  kinect.NUI_SKELETON_POSITION_HIP_CENTER);
  DrawBone(kinect.NUI_SKELETON_POSITION_HIP_CENTER,
  kinect.NUI_SKELETON_POSITION_HIP_LEFT);
  DrawBone(kinect.NUI_SKELETON_POSITION_HIP_CENTER,
  kinect.NUI_SKELETON_POSITION_HIP_RIGHT);
 
  // Left Arm
  DrawBone(kinect.NUI_SKELETON_POSITION_SHOULDER_LEFT,
  kinect.NUI_SKELETON_POSITION_ELBOW_LEFT);
  DrawBone(kinect.NUI_SKELETON_POSITION_ELBOW_LEFT,
  kinect.NUI_SKELETON_POSITION_WRIST_LEFT);
  DrawBone(kinect.NUI_SKELETON_POSITION_WRIST_LEFT,
  kinect.NUI_SKELETON_POSITION_HAND_LEFT);
 
  // Right Arm
  DrawBone(kinect.NUI_SKELETON_POSITION_SHOULDER_RIGHT,
  kinect.NUI_SKELETON_POSITION_ELBOW_RIGHT);
  DrawBone(kinect.NUI_SKELETON_POSITION_ELBOW_RIGHT,
  kinect.NUI_SKELETON_POSITION_WRIST_RIGHT);
  DrawBone(kinect.NUI_SKELETON_POSITION_WRIST_RIGHT,
  kinect.NUI_SKELETON_POSITION_HAND_RIGHT);
 
  // Left Leg
  DrawBone(kinect.NUI_SKELETON_POSITION_HIP_LEFT,
  kinect.NUI_SKELETON_POSITION_KNEE_LEFT);
  DrawBone(kinect.NUI_SKELETON_POSITION_KNEE_LEFT,
  kinect.NUI_SKELETON_POSITION_ANKLE_LEFT);
  DrawBone(kinect.NUI_SKELETON_POSITION_ANKLE_LEFT,
  kinect.NUI_SKELETON_POSITION_FOOT_LEFT);
 
  // Right Leg
  DrawBone(kinect.NUI_SKELETON_POSITION_HIP_RIGHT,
  kinect.NUI_SKELETON_POSITION_KNEE_RIGHT);
  DrawBone(kinect.NUI_SKELETON_POSITION_KNEE_RIGHT,
  kinect.NUI_SKELETON_POSITION_ANKLE_RIGHT);
  DrawBone(kinect.NUI_SKELETON_POSITION_ANKLE_RIGHT,
  kinect.NUI_SKELETON_POSITION_FOOT_RIGHT);
}
 
void DrawBone(int _s, int _e)
{
  if (loc == null)
  {
    return;
  }
  PVector p1 = loc[_s];
  PVector p2 = loc[_e];
  if (p1.z == 0.0 || p2.z == 0.0)
  {
    return;
  }
  line(p1.x*width, p1.y*height, p2.x*width, p2.y*height);
}

import pKinect.PKinect; PKinect kinect; PVector [] loc; void setup() { size(640, 480); kinect = new PKinect(this); smooth(); noFill(); stroke(255, 255, 0); } void draw() { background(0); loc = kinect.getSkeleton(); drawSkeleton(); } void drawSkeleton() { // Body DrawBone(kinect.NUI_SKELETON_POSITION_HEAD, kinect.NUI_SKELETON_POSITION_SHOULDER_CENTER); DrawBone(kinect.NUI_SKELETON_POSITION_SHOULDER_CENTER, kinect.NUI_SKELETON_POSITION_SHOULDER_LEFT); DrawBone(kinect.NUI_SKELETON_POSITION_SHOULDER_CENTER, kinect.NUI_SKELETON_POSITION_SHOULDER_RIGHT); DrawBone(kinect.NUI_SKELETON_POSITION_SHOULDER_CENTER, kinect.NUI_SKELETON_POSITION_SPINE); DrawBone(kinect.NUI_SKELETON_POSITION_SPINE, kinect.NUI_SKELETON_POSITION_HIP_CENTER); DrawBone(kinect.NUI_SKELETON_POSITION_HIP_CENTER, kinect.NUI_SKELETON_POSITION_HIP_LEFT); DrawBone(kinect.NUI_SKELETON_POSITION_HIP_CENTER, kinect.NUI_SKELETON_POSITION_HIP_RIGHT); // Left Arm DrawBone(kinect.NUI_SKELETON_POSITION_SHOULDER_LEFT, kinect.NUI_SKELETON_POSITION_ELBOW_LEFT); DrawBone(kinect.NUI_SKELETON_POSITION_ELBOW_LEFT, kinect.NUI_SKELETON_POSITION_WRIST_LEFT); DrawBone(kinect.NUI_SKELETON_POSITION_WRIST_LEFT, kinect.NUI_SKELETON_POSITION_HAND_LEFT); // Right Arm DrawBone(kinect.NUI_SKELETON_POSITION_SHOULDER_RIGHT, kinect.NUI_SKELETON_POSITION_ELBOW_RIGHT); DrawBone(kinect.NUI_SKELETON_POSITION_ELBOW_RIGHT, kinect.NUI_SKELETON_POSITION_WRIST_RIGHT); DrawBone(kinect.NUI_SKELETON_POSITION_WRIST_RIGHT, kinect.NUI_SKELETON_POSITION_HAND_RIGHT); // Left Leg DrawBone(kinect.NUI_SKELETON_POSITION_HIP_LEFT, kinect.NUI_SKELETON_POSITION_KNEE_LEFT); DrawBone(kinect.NUI_SKELETON_POSITION_KNEE_LEFT, kinect.NUI_SKELETON_POSITION_ANKLE_LEFT); DrawBone(kinect.NUI_SKELETON_POSITION_ANKLE_LEFT, kinect.NUI_SKELETON_POSITION_FOOT_LEFT); // Right Leg DrawBone(kinect.NUI_SKELETON_POSITION_HIP_RIGHT, kinect.NUI_SKELETON_POSITION_KNEE_RIGHT); DrawBone(kinect.NUI_SKELETON_POSITION_KNEE_RIGHT, kinect.NUI_SKELETON_POSITION_ANKLE_RIGHT); DrawBone(kinect.NUI_SKELETON_POSITION_ANKLE_RIGHT, kinect.NUI_SKELETON_POSITION_FOOT_RIGHT); } void DrawBone(int _s, int _e) { if (loc == null) { return; } PVector p1 = loc[_s]; PVector p2 = loc[_e]; if (p1.z == 0.0 || p2.z == 0.0) { return; } line(p1.x*width, p1.y*height, p2.x*width, p2.y*height); }

This version of the code (for Windows 7, both 32-bit and 64-bit) is available here.

More on GridFlow for Pure Data

Posted 2012/08/13 by bryan

Further to the previous post on GridFlow in Pure Data, I include here the installation steps for GridFlow. The Pd Extended package does not include GridFlow. After I download the file from the GridFlow website, I unzip it into the gridflow-9.13 folder. To install, I copy the inside gridflow folder to the following location:

/Library/Pd/

Within Pure Data preference, I add the path and startup option for GridFlow as shown in the following diagrams.

To continue from the previous post with the integration of GEM and GridFlow, I redo one of my Processing work of slit-scan image in Pure Data. The GridFlow package comes with a similar example.

The source patch of the slit-scan example is:

Kinect for Windows in Processing 2

Posted 2012/08/07 by bryan

After the first post of the Kinect for Windows library, this is the second version of the Kinect for Windows in Processing. I modified the pixels array to contain only the colour pixels of players. The rest is painted green as background. In this version, I align the colour and depth pixels in one output frame buffer, with the use of the NuiImageGetColorPixelCoordinateFrameFromDepthPixelFrameAtResolution method.

The Processing code is here. The DLL is in the code folder of the sketch. In this version, I used Windows 7 32-bit and Java 7. I’ll consolidate to build both the 32-bit and 64-bit once the interface is more stable.

I use the Kinect for Windows 1.5 SDK.