An IFS is a list of (A,b,p), where, at each iteration, starting from x0, you choose from the list randomly (such that each list tuple is chosen with probability p) and update by setting x = Ax+b, yes?

I would try something like this (and this would allow for non-linear transforms as well):

Code:

// Define your transforms here
var list = new Array();
list.push(
// Choose x = Ax+b 50% of the time, where:
// A = [a,b,c;d,e,f;g,h,i] and b = [j;k;l]
[0.5, function(x) {
return [
a*x[0]+b*x[1]+c*x[2] + j,
d*x[0]+e*x[1]+f*x[2] + k,
g*x[0]+h*x[1]+i*x[2] + l
];
}]);
list.push(
// Choose x = [x_1;x_2;sin(x_3) / (x_1*x_2)] the other 50% of the time
[0.5, function(x) {
return [
x[0],
x[1],
Math.sin(x[2]) / x[0] / x[1]
];
}]);
// construct the CDF of probabilities
var cdf = new Array(list.length);
cdf[0] = list[0][0]
for (var i = 1; i < list.length; i++) {
cdf[i] = cdf[i-1] + list[i][0];
}
// now whenever you need an appropriately distributed transformation, call getTransform()
// it returns a function. For example:
// var t = getTransform(); x = t(x);
function getTransform() {
var r = Math.random();
for (var i = 0; i < cdf.length; i++) {
if (r <= cdf[i])
return list[i][1];
}
}

Should be enough to get you started at least.