OpenCL Particles System with Processing

I ported the particles demo program in JavaCL to Processing 2.0 alpha. It has reasonable performance in my iMac up to 500,000 particles at twenty something frames per second. The video is captured using the QuickTime screen recording. The performance is much slower than the original screen version.
 

import processing.opengl.*;
import javax.media.opengl.*;
import javax.media.opengl.glu.GLU;
import java.util.Random;
 
import com.nativelibs4java.opencl.*;
import com.nativelibs4java.opencl.CLMem.Usage;
import org.bridj.Pointer;
 
import static org.bridj.Pointer.*;
 
final int particlesCount = 200000;
 
GL2 gl;
PGL pgl;
 
int [] vbo = new int[1];
 
CLContext context;
CLQueue queue;
 
Pointer<Float> velocities;
CLKernel updateParticleKernel;
 
CLBuffer<Float> massesMem, velocitiesMem;
CLBuffer<Byte> interleavedColorAndPositionsMem;
Pointer<Byte> interleavedColorAndPositionsTemp;
 
int elementSize = 4*4;
 
void setup() {
  size(800, 600, OPENGL);
  background(0);
  randomSeed(millis());
 
  PGraphicsOpenGL pg = (PGraphicsOpenGL) g;
  pgl = pg.beginPGL();
  gl = pgl.gl.getGL().getGL2();
  gl.glClearColor(0, 0, 0, 1);
  gl.glClear(GL.GL_COLOR_BUFFER_BIT);
  gl.glEnable(GL.GL_BLEND);
  gl.glEnable(GL2.GL_POINT_SMOOTH);
  gl.glPointSize(1f);
  initOpenCL();
  pg.endPGL();
}
 
void initOpenCL() {
  context = JavaCL.createContextFromCurrentGL();
  queue = context.createDefaultQueue();
 
  Pointer<Float> masses = allocateFloats(particlesCount).order(context.getByteOrder());
  velocities = allocateFloats(2 * particlesCount).order(context.getByteOrder());
  interleavedColorAndPositionsTemp = allocateBytes(elementSize * particlesCount).order(context.getByteOrder());
 
  Pointer<Float> positionsView = interleavedColorAndPositionsTemp.as(Float.class);
  for (int i = 0; i < particlesCount; i++) {
    masses.set(i, 0.5f + 0.5f * random(1));
    velocities.set(i * 2, random(-0.5, 0.5) * 0.2f);
    velocities.set(i * 2 + 1, random(-0.5, 0.5) * 0.2f);
    int colorOffset = i * elementSize;
    int posOffset = i * (elementSize / 4) + 1;
    byte r = (byte) 220, g = r, b = r, a = r;
    interleavedColorAndPositionsTemp.set(colorOffset++, r);
    interleavedColorAndPositionsTemp.set(colorOffset++, g);
    interleavedColorAndPositionsTemp.set(colorOffset++, b);
    interleavedColorAndPositionsTemp.set(colorOffset, a);
    float x = random(-0.5, 0.5) * width/2.0, 
    y = random(-0.5, 0.5) * height/2.0;
    positionsView.set(posOffset, (float) x);
    positionsView.set(posOffset + 1, (float) y);
  }
  velocitiesMem = context.createBuffer(Usage.InputOutput, velocities, false);
  massesMem = context.createBuffer(Usage.Input, masses, true);
 
  gl.glGenBuffers(1, vbo, 0);
  gl.glBindBuffer(GL.GL_ARRAY_BUFFER, vbo[0]);
  gl.glBufferData(GL.GL_ARRAY_BUFFER, (int) interleavedColorAndPositionsTemp.getValidBytes(), interleavedColorAndPositionsTemp.getByteBuffer(), GL2.GL_DYNAMIC_COPY);
  gl.glBindBuffer(GL.GL_ARRAY_BUFFER, 0);
 
  interleavedColorAndPositionsMem = context.createBufferFromGLBuffer(Usage.InputOutput, vbo[0]);
  String pgmSrc = join(loadStrings(dataPath("ParticlesDemoProgram.cl")), "\n");
  CLProgram program = context.createProgram(pgmSrc);
  updateParticleKernel = program.build().createKernel("updateParticle");
  callKernel();
}
 
void draw() {
  queue.finish();
  gl.glClear(GL.GL_COLOR_BUFFER_BIT);
  gl.glBlendFunc(GL.GL_SRC_ALPHA, GL.GL_SRC_COLOR);
  gl.glMatrixMode(GL2.GL_PROJECTION);
  gl.glLoadIdentity();
  pgl.glu.gluOrtho2D(-width/2 - 1, width/2 + 1, -height/2 - 1, height/2 + 1);
  gl.glMatrixMode(GL2.GL_MODELVIEW);
 
  gl.glBindBuffer(GL2.GL_ARRAY_BUFFER, vbo[0]);
  gl.glInterleavedArrays(GL2.GL_C4UB_V2F, elementSize, 0);
  gl.glDrawArrays(GL.GL_POINTS, 0, particlesCount);
 
  gl.glBindBuffer(GL2.GL_ARRAY_BUFFER, 0);
  callKernel();
}
 
void callKernel() {
  CLEvent kernelCompletion;
  synchronized(updateParticleKernel) {
    interleavedColorAndPositionsMem.acquireGLObject(queue);
    updateParticleKernel.setArgs(massesMem, 
    velocitiesMem, 
    interleavedColorAndPositionsMem.as(Float.class), 
    new float[] {
      mouseX-width/2, height/2-mouseY
    }
    , 
    new float[] {
      width, height
    }
    , 
    2.0, 
    2.0, 
    0.9, 
    0.8, 
    (byte) 0);
 
    int [] globalSizes = new int[] {
      particlesCount
    };
    kernelCompletion = updateParticleKernel.enqueueNDRange(queue, globalSizes);
    interleavedColorAndPositionsMem.releaseGLObject(queue);
  }
}

 
Here is the OpenCL source.

// Ported to JavaCL/OpenCL4Java (+ added colors) by Olivier Chafik
 
#define REPULSION_FORCE 4.0f
#define CENTER_FORCE2 0.0005f
 
#define PI 3.1416f
 
//#pragma OpenCL cl_khr_byte_addressable_store : enable
 
uchar4 HSVAtoRGBA(float4 hsva)
{
    float h = hsva.x, s = hsva.y, v = hsva.z, a = hsva.w;
    float r, g, b;
 
        int i;
        float f, p, q, t;
        if (s == 0) {
                // achromatic (grey)
                r = g = b = v;
                return (uchar4)(r * 255, g * 255, b * 255, a * 255);
        }
        h /= 60;                        // sector 0 to 5
        i = floor( h );
        f = h - i;                      // factorial part of h
        p = v * ( 1 - s );
        q = v * ( 1 - s * f );
        t = v * ( 1 - s * ( 1 - f ) );
        switch( i ) {
                case 0:
                        r = v;
                        g = t;
                        b = p;
                        break;
                case 1:
                        r = q;
                        g = v;
                        b = p;
                        break;
                case 2:
                        r = p;
                        g = v;
                        b = t;
                        break;
                case 3:
                        r = p;
                        g = q;
                        b = v;
                        break;
                case 4:
                        r = t;
                        g = p;
                        b = v;
                        break;
                default:                // case 5:
                        r = v;
                        g = p;
                        b = q;
                        break;
        }
    return (uchar4)(r * 255, g * 255, b * 255, a * 255);
}
 
__kernel void updateParticle(
        __global float* masses,
        __global float2* velocities,
        //__global Particle* particles,
        __global float4* particles,
        //__global char* pParticles,
        const float2 mousePos,
        const float2 dimensions,
        float massFactor,
        float speedFactor,
        float slowDownFactor,
        float mouseWeight,
        char limitToScreen
) {
	int id = get_global_id(0);
 
        float4 particle = particles[id];
 
        uchar4 color = as_uchar4(particle.x);
 
        float2 position = particle.yz;
    	float2 diff = mousePos - position;
 
        float invDistSQ = 1.0f / dot(diff, diff);
	float2 halfD = dimensions / 2.0f;
        diff *= (halfD).y * invDistSQ;
 
        float mass = massFactor * masses[id];
        float2 velocity = velocities[id];
        velocity -= mass * position * CENTER_FORCE2 - diff * mass * mouseWeight;
        position += speedFactor * velocities[id];
 
        if (limitToScreen) {
            float2 halfDims = dimensions / 2.0f;
            position = clamp(position, -halfDims, halfDims);
        }
 
        float dirDot = cross((float4)(diff, (float2)0), (float4)(velocity, (float2)0)).z;
        float speed = length(velocity);
 
        float f = speed / 4 / mass;
        float hue = (dirDot < 0 ? f : f + 1) / 2;
        hue = clamp(hue, 0.0f, 1.0f) * 360;
 
        float opacity = clamp(0.1f + f, 0.0f, 1.0f);
        float saturation = mass / 2;
        float brightness = 0.6f + opacity * 0.3f;
 
        uchar4 targetColor = HSVAtoRGBA((float4)(hue, saturation, brightness, opacity));
 
        float colorSpeedFactor = min(0.01f * speedFactor, 1.0f), otherColorSpeedFactor = 1 - colorSpeedFactor;
        color = (uchar4)(
            (uchar)(targetColor.x * colorSpeedFactor + color.x * otherColorSpeedFactor),
            (uchar)(targetColor.y * colorSpeedFactor + color.y * otherColorSpeedFactor),
            (uchar)(targetColor.z * colorSpeedFactor + color.z * otherColorSpeedFactor),
            (uchar)(targetColor.w * colorSpeedFactor + color.w * otherColorSpeedFactor)
        );
 
        particle.x = as_float(color);
        particle.yz = position;
 
    	particles[id] = particle;
 
        velocity *= slowDownFactor;
        velocities[id] = velocity;
}

OpenGL and Processing 2.0

The existing OpenGL codes for Processing do not work in the 2.0 alpha. Here is an example code segment I modify to use the new PGL class.

import processing.opengl.*;
import javax.media.opengl.*;
 
GL2 gl;
float t, s, c;
 
void setup() {
  size(400, 400, OPENGL);
  background(0);
  PGraphicsOpenGL pg = (PGraphicsOpenGL) g;
  PGL pgl = pg.beginPGL();
  gl = pgl.gl.getGL().getGL2();
  pg.endPGL();
  t = 0.0f;
  s = 0.0f;
  c = 0.0f;
}
 
void draw() {
  t += 0.01;
  s = sin(t);
  c = cos(t);
 
  gl.glClear(GL.GL_COLOR_BUFFER_BIT);
  gl.glBegin(GL.GL_TRIANGLES);
  gl.glColor3f(1, 0, 0);
  gl.glVertex3f(-c, -c, s);
  gl.glColor3f(0, 1, 0);
  gl.glVertex3f(0, c, 0);
  gl.glColor3f(0, 0, 1);
  gl.glVertex3f(s, -s, c);
  gl.glEnd();
}

OpenCL and Processing

I have done a number of testings with various Java implementations of OpenCL and Processing. The major Java bindings of OpenCL include

The are two major OpenCL libraries for Processing at the time I do the testing, MSAOpenCL using the JavaCL and openclp5 using the JOCL. I do not use the Processing libraries and call directly the Java binding codes. Each of the implementation contains a Hello World example kernel that performs calculation across a large array. In my test, I modify the kernel program to do a multiplication between two floating point numbers with an array size of a million cells.

The sample kernel program code is

__kernel void sampleKernel(__global const float *a,
        __global const float *b,
        __global float *c,
        int n)
{
	int gid = get_global_id(0);
	if (gid &gt;= n)
		return;
	c[gid] = a[gid] * b[gid];
}

I use a MacBook Pro for the testing. The operating system is OSX Lion. It includes the OpenCL implementation in the default OS. The graphics card is Nvidia GeForce 9400M with 256M graphic memory. The Processing version is alpha build 2.0a5 running in 64-bit mode. Various JAR files have to be copied to the code folder. Here is the summary.

JOCL

  • JOCL-0.1.7.jar

Java OpenCL (jogamp)

  • jocl.jar
  • gluegen-rt.jar
  • gluegen-rt-natives-macosx-universal.jar
  • libjocl.dylib

JavaCL

  • javacl-1.0.0-RC2-shaded.jar

I did ten consecutive runs of each implementation. The final figure is an average of the ten runs. Each measurement is taken just before and after the kernel execution. The first implementation (JOCL) has more fluctuated results. The second implementation has only a single command to invoke the kernel execution and passing back the result. Therefore, it is not easy to single out the execution time of the GPU. The third implementation (JavaCL) is obviously the fastest one and more stable in terms of parallel execution.

Performance with one million cells

Implementation ms
JOCL 3.207
Java OpenCL 17.0008
JavaCL 2.9865
CPU 10.952

Performance with half a million cells

Implementation ms
JOCL 3.9424
Java OpenCL 8.5672
JavaCL 2.9354
CPU 9.2368

The source files of the testings can be downloaded here:

Kinect for Windows

The new Kinect for Windows is available and will be available in Hong Kong in late May according to the blog description. The commercial SDK is also out now for download.

TouchDesigner has a set of new operators for the Kinect for Windows. They are quite easy to integrate with the existing TouchDesigner working environment.

University of Central Florida Interactive Systems and User Experience Lab has also released a Unity3D plugin with the new Kinect SDK.
 

 
There are also a number of openFrameworks addons for the Kinect for Windows, ofxMSKinect and ofxKinectNui.

For library Cinder, here is the Kinect SDK Block.

For Flash ActionScript users, the AIRKinect can be a good choice.

At the time of writing, I am still waiting for the Java binding and thus the Processing community.

 

DirectShow for Processing – OpenGL

I try to work out another version of the DirectShow for Processing classes in the last post. In this version, I write the movie data directly to an OpenGL texture object. Below is the modified version of the DMovie class. The DCapture class can also be modified in the same way.
 
The modified DMovie class

import de.humatic.dsj.*;
import java.awt.image.BufferedImage;
import com.sun.opengl.util.texture.*;
 
class DMovie implements java.beans.PropertyChangeListener {
 
  private DSMovie movie;
  public int width, height;
  public Texture tex;
 
  DMovie(String _s) {
    movie = new DSMovie(dataPath(_s), DSFiltergraph.DD7, this);
    movie.setVolume(1.0);
    movie.setLoop(false);
    movie.play();
    width = movie.getDisplaySize().width;
    height = movie.getDisplaySize().height;
    tex = TextureIO.newTexture(movie.getImage(), false);
  }
 
  public void updateImage() {
    BufferedImage bimg = movie.getImage();
    TextureData td = TextureIO.newTextureData(bimg, false);
    tex.updateImage(td);
  }
 
  public void loop() {
    movie.setLoop(true);
    movie.play();
  }
 
  public void play() {
    movie.play();
  }
 
  public void propertyChange(java.beans.PropertyChangeEvent e) {
    switch (DSJUtils.getEventType(e)) {
    }
  }
}

 
Sample code that uses the new DMovie class

import processing.opengl.*;
import javax.media.opengl.*;
 
DMovie mov;
PGraphicsOpenGL pgl;
 
void setup()
{
  size(1280, 692, OPENGL);
  pgl = (PGraphicsOpenGL) g;
  GL gl = pgl.beginGL();
  background(0);
  mov = new DMovie("Hugo.mp4");
  mov.loop();
  mov.tex.bind();
  pgl.endGL();
  ;
}
 
void draw()
{
  GL gl = pgl.beginGL();
  mov.updateImage();
  mov.tex.enable();
  gl.glBegin(GL.GL_QUADS);
  gl.glTexCoord2f(0, 0); 
  gl.glVertex2f(0, 0);
  gl.glTexCoord2f(1, 0); 
  gl.glVertex2f(width, 0);
  gl.glTexCoord2f(1, 1); 
  gl.glVertex2f(width, height);
  gl.glTexCoord2f(0, 1); 
  gl.glVertex2f(0, height);
  gl.glEnd();  
 
  mov.tex.disable();
  pgl.endGL();
}

DirectShow for Processing

I adopt the DirectShow Java Wrapper to work in Processing with two classes, one for movie playback and one for video capture. At this moment, there are just two Java classes, not an individual library yet. Since it is for DirectShow, it is of course in Windows platform. You have to package the dsj.jar and the dsj.dll (32bit or 64bit according to your platform) into your code folder.
 
The DMovie class for movie playback

import de.humatic.dsj.*;
import java.awt.image.BufferedImage;
 
class DMovie implements java.beans.PropertyChangeListener {
 
  private DSMovie movie;
  public int width, height;
 
  DMovie(String _s) {
    movie = new DSMovie(dataPath(_s), DSFiltergraph.DD7, this);
    movie.setVolume(1.0);
    movie.setLoop(false);
    movie.play();
    width = movie.getDisplaySize().width;
    height = movie.getDisplaySize().height;
  }
 
  public PImage updateImage() {
    PImage img = createImage(width, height, RGB);
    BufferedImage bimg = movie.getImage();
    bimg.getRGB(0, 0, img.width, img.height, img.pixels, 0, img.width);
    img.updatePixels();
    return img;
  }
 
  public void loop() {
    movie.setLoop(true);
    movie.play();
  }
 
  public void play() {
    movie.play();
  }
 
  public void propertyChange(java.beans.PropertyChangeEvent e) {
    switch (DSJUtils.getEventType(e)) {
    }
  }
}

 
Sample code that uses the DMovie class

DMovie mov;
 
void setup()
{
  size(1280, 692);
  background(0);
  mov = new DMovie("Hugo.mp4");
  mov.loop();
  frameRate(25);
}
 
void draw()
{
  image(mov.updateImage(), 0, 0);
}

 
The DCapture class that performs video capture with the available webcam

import de.humatic.dsj.*;
import java.awt.image.BufferedImage;
 
class DCapture implements java.beans.PropertyChangeListener {
 
  private DSCapture capture;
  public int width, height;
 
  DCapture() {
    DSFilterInfo[][] dsi = DSCapture.queryDevices();
    capture = new DSCapture(DSFiltergraph.DD7, dsi[0][0], false, 
    DSFilterInfo.doNotRender(), this);
    width = capture.getDisplaySize().width;
    height = capture.getDisplaySize().height;
  }
 
  public PImage updateImage() {
    PImage img = createImage(width, height, RGB);
    BufferedImage bimg = capture.getImage();
    bimg.getRGB(0, 0, img.width, img.height, img.pixels, 0, img.width);
    img.updatePixels();
    return img;
  }
 
  public void propertyChange(java.beans.PropertyChangeEvent e) {
    switch (DSJUtils.getEventType(e)) {
    }
  }
}

Sample code that uses the DCapture class

DCapture cap;
 
void setup() 
{
  size(640, 480);
  background(0);
  cap = new DCapture();
}
 
void draw()
{
  image(cap.updateImage(), 0, 0, cap.width, cap.height);
}

Video Playback Performance – Processing

I try out different video playback mechanism in the Processing to compare their performance. The digital video is the one I used in the last post. It is the trailer of the film Hugo. The details are: 1280 x 692 H.264 AAC, bitrate 2,093.

The computer I am using is iMac 3.06 GHz Intel Core 2 Duo, 4 GB RAM, ATI Radeon HD 4670 256 MB graphic card. Processing is the latest 1.5.1 version.

For the video playback classes, I tested the default QuickTime Video library, a FasterMovie class, the GSVideo library, and the JMCVideo library with JavaFX 1.2 SDK.

To render the video, I start with the standard image() function, and proceed to test with various OpenGL texturing methods, including the GLGraphics library.

Again, I sample the CPU and Memory usage with the Activity Monitor from the Mac OSX utilities, in an interval of 30 seconds. The results are the average of 5 samples.

Performance with 2D image function

CPU (%) Memory (Mb)
QuickTime 175 221
FasterMovie 137 275
GSVideo 151 118
JMCVideo 147 87

The OpenGL and QuickTime video libraries have problem working together. The program stops at the size() statement. I have to either put the first video related command before the size() or a dummy line

println(Capture.list());
size(1280, 692, OPENGL);

The second batch of tests use the standard OpenGL vertex and texture functions.

Performance with OpenGL texture and vertex functions

CPU (%) Memory (Mb)
QuickTime 158 430
FasterMovie 143 610
GSVideo 147 315
JMCVideo 142 397

The third batch of tests involve custom arrangement in OpenGL. Both GSVideo and JMCVideo come with their own functions to write directly to OpenGL texture. For the FasterMovie test, I combine it with the pixel buffer object I have shown in my previous post.
 
Performance with custom OpenGL texture method

  CPU (%) Memory (Mb)
FasterMovie+PBO 69 275
GSVideo+GLGraphics 58 120
JMCVideo+OpenGL 57 91

 
Sample code for GSVideo and GLGraphics (from codeanticode)

import processing.opengl.*;
import codeanticode.glgraphics.*;
import codeanticode.gsvideo.*;
 
GSMovie mov;
GLTexture tex;
 
void setup() 
{
  size(1280, 692, GLConstants.GLGRAPHICS);
  background(0);   
  mov = new GSMovie(this, "Hugo.mp4");
  tex = new GLTexture(this);
  mov.setPixelDest(tex);  
  mov.loop();
}
 
void draw() 
{
  if (tex.putPixelsIntoTexture()) 
  {
    image(tex, 0, 0);
  }
}
 
void movieEvent(GSMovie _m) {
  _m.read();
}

 
Sample code for JMCVideo (from Angus Forbes)

import jmcvideo.*;
import processing.opengl.*;
import javax.media.opengl.*; 
 
JMCMovieGL mov;
PGraphicsOpenGL pgl;
 
void setup() 
{
  size(1280, 692, OPENGL);
  background(0);
  mov = new JMCMovieGL(this, "Hugo.mp4", ARGB);
  mov.loop();
 
  pgl = (PGraphicsOpenGL) g;
  GL gl = pgl.beginGL();  
  gl.glViewport(0, 0, width, height);
  pgl.endGL();
}
 
void draw() 
{
  GL gl = pgl.beginGL();  
  mov.image(gl, 0, 0, width, height);
  pgl.endGL();
}

Video Playback Performance – OSX

I have done a simple performance comparison with a number of video playback methods in OSX. It gives me some ideas about which development platforms, namely the OpenFrameworks and Library Cinder to work on with high quality video playback. The test is straight forward, and as the same time, not very rigorous. VLC and Real Player are used as benchmark for comparison. The OpenFrameworks and Cinder programs are the standard QuickTime video player samples in the distributions.

I use the trailer for the film Hugo in 720 HD. The exact dimension is 1280 x 692 H.264 AAC and bitrate at 2,093.

 

Software CPU (%) Memory (Mb)
VLC 34 114
Real Player 34 55
OpenFrameworks 52 37
Cinder 37 43

 
The figures are average of 5 samples taken at an interval of 30 seconds.