Kinect in Windows for Processing 5

This is a more or less finished version of the Kinect for Processing library. It includes the basic skeleton tracking, the RGB image, depth image and a mask image. I shall move the related posts to the research page with more documentation on the data structure and method description. Stay tune.
 

 
The sample Processing code:

import pKinect.PKinect;
import pKinect.SkeletonData;
 
PKinect kinect;
PFont font;
ArrayList<SkeletonData> bodies;
PImage img;
 
void setup()
{
  size(640, 480);
  background(0);
  kinect = new PKinect(this);
  bodies = new ArrayList<SkeletonData>();
  smooth();
  font = loadFont("LucidaSans-18.vlw");
  textFont(font, 18);
  textAlign(CENTER);
  img = loadImage("background.png");
}
 
void draw()
{
  background(0);
  image(kinect.GetImage(), 320, 0, 320, 240);
  image(kinect.GetDepth(), 320, 240, 320, 240);
  image(img, 0, 240, 320, 240);
  image(kinect.GetMask(), 0, 240, 320, 240);
  for (int i=0; i<bodies.size(); i++) 
  {
    drawSkeleton(bodies.get(i));
    drawPosition(bodies.get(i));
  }
}
 
void mousePressed() 
{
  println(frameRate);
}
 
void drawPosition(SkeletonData _s) 
{
  noStroke();
  fill(0, 100, 255);
  String s1 = str(_s.dwTrackingID);
  text(s1, _s.position.x*width/2, _s.position.y*height/2);
}
 
void drawSkeleton(SkeletonData _s) 
{
  // Body
  DrawBone(_s, 
  PKinect.NUI_SKELETON_POSITION_HEAD, 
  PKinect.NUI_SKELETON_POSITION_SHOULDER_CENTER);
  DrawBone(_s, 
  PKinect.NUI_SKELETON_POSITION_SHOULDER_CENTER, 
  PKinect.NUI_SKELETON_POSITION_SHOULDER_LEFT);
  DrawBone(_s, 
  PKinect.NUI_SKELETON_POSITION_SHOULDER_CENTER, 
  PKinect.NUI_SKELETON_POSITION_SHOULDER_RIGHT);
  DrawBone(_s, 
  PKinect.NUI_SKELETON_POSITION_SHOULDER_CENTER, 
  PKinect.NUI_SKELETON_POSITION_SPINE);
  DrawBone(_s, 
  PKinect.NUI_SKELETON_POSITION_SHOULDER_LEFT, 
  PKinect.NUI_SKELETON_POSITION_SPINE);
  DrawBone(_s, 
  PKinect.NUI_SKELETON_POSITION_SHOULDER_RIGHT, 
  PKinect.NUI_SKELETON_POSITION_SPINE);
  DrawBone(_s, 
  PKinect.NUI_SKELETON_POSITION_SPINE, 
  PKinect.NUI_SKELETON_POSITION_HIP_CENTER);
  DrawBone(_s, 
  PKinect.NUI_SKELETON_POSITION_HIP_CENTER, 
  PKinect.NUI_SKELETON_POSITION_HIP_LEFT);
  DrawBone(_s, 
  PKinect.NUI_SKELETON_POSITION_HIP_CENTER, 
  PKinect.NUI_SKELETON_POSITION_HIP_RIGHT);
  DrawBone(_s, 
  PKinect.NUI_SKELETON_POSITION_HIP_LEFT, 
  PKinect.NUI_SKELETON_POSITION_HIP_RIGHT);
 
  // Left Arm
  DrawBone(_s, 
  PKinect.NUI_SKELETON_POSITION_SHOULDER_LEFT, 
  PKinect.NUI_SKELETON_POSITION_ELBOW_LEFT);
  DrawBone(_s, 
  PKinect.NUI_SKELETON_POSITION_ELBOW_LEFT, 
  PKinect.NUI_SKELETON_POSITION_WRIST_LEFT);
  DrawBone(_s, 
  PKinect.NUI_SKELETON_POSITION_WRIST_LEFT, 
  PKinect.NUI_SKELETON_POSITION_HAND_LEFT);
 
  // Right Arm
  DrawBone(_s, 
  PKinect.NUI_SKELETON_POSITION_SHOULDER_RIGHT, 
  PKinect.NUI_SKELETON_POSITION_ELBOW_RIGHT);
  DrawBone(_s, 
  PKinect.NUI_SKELETON_POSITION_ELBOW_RIGHT, 
  PKinect.NUI_SKELETON_POSITION_WRIST_RIGHT);
  DrawBone(_s, 
  PKinect.NUI_SKELETON_POSITION_WRIST_RIGHT, 
  PKinect.NUI_SKELETON_POSITION_HAND_RIGHT);
 
  // Left Leg
  DrawBone(_s, 
  PKinect.NUI_SKELETON_POSITION_HIP_LEFT, 
  PKinect.NUI_SKELETON_POSITION_KNEE_LEFT);
  DrawBone(_s, 
  PKinect.NUI_SKELETON_POSITION_KNEE_LEFT, 
  PKinect.NUI_SKELETON_POSITION_ANKLE_LEFT);
  DrawBone(_s, 
  PKinect.NUI_SKELETON_POSITION_ANKLE_LEFT, 
  PKinect.NUI_SKELETON_POSITION_FOOT_LEFT);
 
  // Right Leg
  DrawBone(_s, 
  PKinect.NUI_SKELETON_POSITION_HIP_RIGHT, 
  PKinect.NUI_SKELETON_POSITION_KNEE_RIGHT);
  DrawBone(_s, 
  PKinect.NUI_SKELETON_POSITION_KNEE_RIGHT, 
  PKinect.NUI_SKELETON_POSITION_ANKLE_RIGHT);
  DrawBone(_s, 
  PKinect.NUI_SKELETON_POSITION_ANKLE_RIGHT, 
  PKinect.NUI_SKELETON_POSITION_FOOT_RIGHT);
}
 
void DrawBone(SkeletonData _s, int _j1, int _j2) 
{
  noFill();
  stroke(255, 255, 0);
  if (_s.skeletonPositionTrackingState[_j1] != PKinect.NUI_SKELETON_POSITION_NOT_TRACKED &&
    _s.skeletonPositionTrackingState[_j2] != PKinect.NUI_SKELETON_POSITION_NOT_TRACKED) {
    line(_s.skeletonPositions[_j1].x*width/2, 
    _s.skeletonPositions[_j1].y*height/2, 
    _s.skeletonPositions[_j2].x*width/2, 
    _s.skeletonPositions[_j2].y*height/2);
  }
}
 
void appearEvent(SkeletonData _s) 
{
  if (_s.trackingState == PKinect.NUI_SKELETON_NOT_TRACKED) 
  {
    return;
  }
  synchronized(bodies) {
    bodies.add(_s);
  }
}
 
void disappearEvent(SkeletonData _s) 
{
  synchronized(bodies) {
    for (int i=bodies.size()-1; i>=0; i--) 
    {
      if (_s.dwTrackingID == bodies.get(i).dwTrackingID) 
      {
        bodies.remove(i);
      }
    }
  }
}
 
void moveEvent(SkeletonData _b, SkeletonData _a) 
{
  if (_a.trackingState == PKinect.NUI_SKELETON_NOT_TRACKED) 
  {
    return;
  }
  synchronized(bodies) {
    for (int i=bodies.size()-1; i>=0; i--) 
    {
      if (_b.dwTrackingID == bodies.get(i).dwTrackingID) 
      {
        bodies.get(i).copy(_a);
        break;
      }
    }
  }
}

You can download the example and the library here.

10 Replies to “Kinect in Windows for Processing 5”

  1. Hi,Bryan
    I am a student in China, my study direction is Interactive Art, I want to get your permission to use your code in my final project. Is that OK? thanks~~
    Paul

  2. Great job!

    I only have a couple of questions..

    When I try to run your example I get this error code:

    This version of Processing only supports libraries and JAR files compiled for Java 1.6 or earlier.
    A library used by this sketch was compiled for Java 1.7 or later,
    and needs to be recompiled to be compatible with Java 1.6.
    Exception in thread “Animation Thread”…

    I am workin on Processing 2.0b8 with embeddedjava 1.6

    Any suggestions?

    Thank you in advance!

  3. hi, nice job, I used to work with simpleOpenNI,
    but I have to make an application with MS SDK installed,
    so your solution is exactly what I need 🙂

    otherwise, you should declare ur draw method as :

    if( bodies.size() > 0)
    {
    for (int i = 0; i < bodies.size(); i++)
    {
    drawSkeleton(bodies.get(i));
    drawPosition(bodies.get(i));
    }
    }

    if not, if you ve got delay on calibration, it does close the sketch,
    well, on my computer it does it :)…
    thank you again!

    1. wow…

      it did this on Processing 2.++, so I tested with 1.5.1,
      and as I had similar problems on similar code on AS3, I thought it was this, but not at all 😉 shame on me, ur code works perfectly!
      but I do have some troubles with P 2.++, I think it comes from java &/or GL drivers, I made a mess a few days ago to use ReconstructMe ;)….

      right now attempting to make a simple right ad left swipe gesture manager
      will keep in touche

      thanx again

  4. Hi,
    I am beginning to like your port. There are some things I want to ask you.
    1) Are u going to open source your port?
    2) How can I access the joint orientation information from Kinect in this? Does the library add this as well.
    3) Can u document the library’s interface (API docs)? So we know which functions /objects are available?
    4) Can u elaborate the amount of coverage of the original v1.5 of Kinect sdk you have covered in this library? Is it possible to contribute to your code by adding in more examples like the point cloud rendering?

    Thanks,
    Mobeen

    1. Dear Mobeen,
      Yes, I am going to post. It is now at https://github.com/chungbwc
      The joint orientation is not ready yet. Sure, I am going to document the data structure for SkeletonData and the related methods in a separate page soon.
      At the moment, the library is more related to 2D measurement. Depth information is in the z dimension of PVector. I have not designed the point cloud usage yet.
      Hope you enjoy the library and happy coding.

      Bryan

      1. Thanks for the quick response Bryan. Another question, I peeked into the code but I want to know how the data flows from the native dll to java and then through the jar file to processing. Can you identify a web link/document which might help me in understanding this?

  5. OK I know the reason. The latest library needs three functions which are to be defined as follows:
    Adding the global
    ArrayList bodies;

    and then define these three methods.

    void appearEvent(pKinect.SkeletonData _s)
    {
    if (_s.trackingState == PKinect.NUI_SKELETON_NOT_TRACKED)
    {
    return;
    }
    synchronized(bodies) {
    bodies.add(_s);
    }
    println(“appearing …” + _s.dwTrackingID);
    }

    void disappearEvent(pKinect.SkeletonData _s)
    {
    synchronized(bodies) {
    for (int i=bodies.size()-1; i>=0; i–)
    {
    if (_s.dwTrackingID == bodies.get(i).dwTrackingID)
    {
    bodies.remove(i);
    }
    }
    }
    println(“Disappearing … ” + _s.dwTrackingID);
    }

    void moveEvent(pKinect.SkeletonData _b, pKinect.SkeletonData _a)
    {
    if (_a.trackingState == PKinect.NUI_SKELETON_NOT_TRACKED)
    {
    return;
    }
    synchronized(bodies) {
    for (int i=bodies.size()-1; i>=0; i–)
    {
    if (_b.dwTrackingID == bodies.get(i).dwTrackingID)
    {
    bodies.get(i).copy(_a);
    break;
    }
    }
    }
    }

    This resolves the exception. Thanks for the wonderful library.

  6. Hi,
    I had been looking for this a processing kinect library that worked with kinect for windows 1.5 and your library does the job perfectly. Thanks for sharing.

    By the way, for ease of development, you can copy the PKinect library (i.e. thet contents of the code subfolder) into the c:\users\[your user name]\My Documents\Processing\libraries\pKinect\library\ folder so in the end you have three files,
    c:\users\[your user name]\My Documents\Processing\libraries\pKinect\library\PKinect.jar
    c:\users\[your user name]\My Documents\Processing\libraries\pKinect\library\PKinect32.dll and
    c:\users\[your user name]\My Documents\Processing\libraries\pKinect\library\PKinect64.dll
    Doing so allows you to directly import the library into processing. It works fine for me only that I get an exception on the processing console:

    java.lang.NoSuchMethodException: Test.appearEvent(pKinect.SkeletonData)
    at java.lang.Class.getMethod(Class.java:1605)
    at pKinect.PKinect.(PKinect.java:38)
    at Test.setup(Test.java:30)
    at processing.core.PApplet.handleDraw(Unknown Source)
    at processing.core.PApplet.run(Unknown Source)
    at java.lang.Thread.run(Thread.java:662)
    Any ideas how to solve this?

Comments are closed.