I need to check if an image exists in another image using JavaScript, I need to know what are the best approaches (algorithm) and solutions (ex: librarie) to do this operations
I explained what I need to do in this image:
Using the GPU to help in image processing.
Using the 2D API and some simple tricks you can exploit the GPUs power to speed up Javascript.
Difference
To find an image you need to compare the pixels you are looking for (A) against the pixels in the image (B). If the difference between the Math.abs(A-B) === 0 then the pixels are the same.
A function to do this may look like the following
function findDif(imageDataSource, imageDataDest, xx,yy)
const ds = imageDataSource.data;
const dd = imageDataDest.data;
const w = imageDataSource.width;
const h = imageDataSource.height;
var x,y;
var dif = 0;
for(y = 0; y < h; y += 1){
for(x = 0; x < w; x += 1){
var indexS = (x + y * w) * 4;
var indexD = (x + xx + (y + yy) * imageDataDest.width) * 4;
dif += Math.abs(ds[indexS]-dd[indexD]);
dif += Math.abs(ds[indexS + 1]-dd[indexD + 1]);
dif += Math.abs(ds[indexS + 2]-dd[indexD + 2]);
}
}
return dif;
}
var source = sourceCanvas.getContext("2d").getImageData(0,0,sourceCanvas.width,sourceCanvas.height);
var dest = destinationCanvas.getContext("2d").getImageData(0,0,destinationCanvas.width,destinationCanvas.height);
if(findDif(source,dest,100,100)){ // is the image at 100,100?
// Yes image is very similar
}
Where the source is the image we are looking for and the dest is the image we want to find it in. We run the function for every location that the image may be and if the result is under a level then its a good chance we have found it.
But this is very very slow in JS. This is where the GPU can help.
Using the ctx.globalCompositeOperation = "difference"; operation we can speed up the process as it will do the difference calculation for us
When you render with the comp operation "difference" the resulting pixels are the difference between the pixels you are drawing and those that are already on the canvas. Thus if you draw on something that is the same the result is all pixels are black (no difference)
To find a similar image in the image you render the image you are testing for at each location on the canvas that you want to test for. Then you get the sum of all the pixels you just rendered on, if the result is under a threshold that you have set then the image under that area is very similar to the image you are testing for.
But we still need to count all the pixels one by one.
A GPU mean function
The comp op "difference" already does the pixel difference calculation for you, but to get the sum you can use the inbuilt image smoothing.
After you have rendered to find the difference you take that area and render it at a smaller scale with ctx.imageSmoothingEnabled = true the default setting. The GPU will do something similar to an average and can reduce the amount of work JS has to do by several orders of magnitude.
Now instead of 100s or 1000s of pixels you can reduce it down to as little at 4 or 16 depending on the accuracy you need.
An example.
Using these methods you can get a near realtime image in image search with just the basic numerical analysis.
Click to start a test. Results are shown plus the time it took. The image that is being searched for is in the top right.
//------------------------------------------------------------------------
// Some helper functions
var imageTools = (function () {
var tools = {
canvas(width, height) { // create a blank image (canvas)
var c = document.createElement("canvas");
c.width = width;
c.height = height;
return c;
},
createImage : function (width, height) {
var i = this.canvas(width, height);
i.ctx = i.getContext("2d");
return i;
},
image2Canvas(img) {
var i = this.canvas(img.width, img.height);
i.ctx = i.getContext("2d");
i.ctx.drawImage(img, 0, 0);
return i;
},
copyImage(img){ // just a named stub
return this.image2Canvas(img);
},
};
return tools;
})();
const U = undefined;
const doFor = (count, callback) => {var i = 0; while (i < count && callback(i ++) !== true ); };
const setOf = (count, callback) => {var a = [],i = 0; while (i < count) { a.push(callback(i ++)) } return a };
const randI = (min, max = min + (min = 0)) => (Math.random() * (max - min) + min) | 0;
const rand = (min, max = min + (min = 0)) => Math.random() * (max - min) + min;
const randA = (array) => array[(Math.random() * array.length) | 0];
const randG = (min, max = min + (min = 0)) => Math.random() * Math.random() * Math.random() * Math.random() * (max - min) + min;
// end of helper functions
//------------------------------------------------------------------------
function doit(){
document.body.innerHTML = ""; // clear the page;
var canvas = document.createElement("canvas");
document.body.appendChild(canvas);
var ctx = canvas.getContext("2d");
// a grid of 36 images
canvas.width = 6 * 64;
canvas.height = 6 * 64;
console.log("test");
// get a random character to look for
const digit = String.fromCharCode("A".charCodeAt(0) + randI(26));
// get some characters we dont want
const randomDigits = setOf(6,i=>{
return String.fromCharCode("A".charCodeAt(0) + randI(26));
})
randomDigits.push(digit); // add the image we are looking for
var w = canvas.width;
var h = canvas.height;
// create a canvas for the image we are looking for
const imageToFind = imageTools.createImage(64,64);
// and a larger one to cover pixels on the sides
const imageToFindExtend = imageTools.createImage(128,128);
// Draw the character onto the image with a white background and scaled to fit
imageToFindExtend.ctx.fillStyle = imageToFind.ctx.fillStyle = "White";
imageToFind.ctx.fillRect(0,0,64,64);
imageToFindExtend.ctx.fillRect(0,0,128,128);
ctx.font = imageToFind.ctx.font = "64px arial black";
ctx.textAlign = imageToFind.ctx.textAlign = "center";
ctx.textBaseline = imageToFind.ctx.textBaseline = "middle";
const digWidth = imageToFind.ctx.measureText(digit).width+8;
const scale = Math.min(1,64/digWidth);
imageToFind.ctx.fillStyle = "black";
imageToFind.ctx.setTransform(scale,0,0,scale,32,32);
imageToFind.ctx.fillText(digit,0,0);
imageToFind.ctx.setTransform(1,0,0,1,0,0);
imageToFindExtend.ctx.drawImage(imageToFind,32,32);
imageToFind.extendedImage = imageToFindExtend;
// Now fill the canvas with images of other characters
ctx.fillStyle = "white";
ctx.setTransform(1,0,0,1,0,0);
ctx.fillRect(0,0,w,h);
ctx.fillStyle = "black";
ctx.strokeStyle = "white";
ctx.lineJoin = "round";
ctx.lineWidth = 12;
// some characters will be rotated 90,180,-90 deg
const dirs = [
[1,0,0,1,0,0],
[0,1,-1,0,1,0],
[-1,0,0,-1,1,1],
[0,-1,1,0,0,1],
]
// draw random characters at random directions
doFor(h / 64, y => {
doFor(w / 64, x => {
const dir = randA(dirs)
ctx.setTransform(dir[0] * scale,dir[1] * scale,dir[2] * scale,dir[3] * scale,x * 64 + 32, y * 64 + 32);
const d = randA(randomDigits);
ctx.strokeText(d,0,0);
ctx.fillText(d,0,0);
});
});
ctx.setTransform(1,0,0,1,0,0);
// get a copy of the canvas
const saveCan = imageTools.copyImage(ctx.canvas);
// function that finds the images
// image is the image to find
// dir is the matrix direction to find
// smapleSize is the mean sampling size samller numbers are quicker
function checkFor(image,dir,sampleSize){
const can = imageTools.copyImage(saveCan);
const c = can.ctx;
const stepx = 64;
const stepy = 64;
// the image that will contain the reduced means of the differences
const results = imageTools.createImage(Math.ceil(w / stepx) * sampleSize,Math.ceil(h / stepy) * sampleSize);
const e = image.extendedImage;
// for each potencial image location
// set a clip area and draw the source image on it with
// comp mode "difference";
for(var y = 0 ; y < h; y += stepy ){
for(var x = 0 ; x < w; x += stepx ){
c.save();
c.beginPath();
c.rect(x,y,stepx,stepy);
c.clip();
c.globalCompositeOperation = "difference";
c.setTransform(dir[0],dir[1],dir[2],dir[3],x +32 ,y +32 );
c.drawImage(e,-64,-64);
c.restore();
}
}
// Apply the mean (reducing nnumber of pixels to check
results.ctx.drawImage(can,0,0,results.width,results.height);
// get the pixel data
var dat = new Uint32Array(results.ctx.getImageData(0,0,results.width,results.height).data.buffer);
// for each area get the sum of the difference
for(var y = 0; y < results.height; y += sampleSize){
for(var x = 0; x < results.width; x += sampleSize){
var val = 0;
for(var yy = 0; yy < sampleSize && y+yy < results.height; yy += 1){
var i = x + (y+yy)*results.width;
for(var xx = 0; xx < sampleSize && x + xx < results.width ; xx += 1){
val += dat[i++] & 0xFF;
}
}
// if the sum is under the threshold we have found an image
// and we mark it
if(val < sampleSize * sampleSize * 5){
ctx.strokeStyle = "red";
ctx.fillStyle = "rgba(255,0,0,0.5)";
ctx.lineWidth = 2;
ctx.strokeRect(x * (64/sampleSize),y * (64/sampleSize),64,64);
ctx.fillRect(x * (64/sampleSize),y * (64/sampleSize),64,64);
foundCount += 1;
}
}
}
}
var foundCount = 0;
// find the images at different orientations
var now = performance.now();
checkFor(imageToFind,dirs[0],4);
checkFor(imageToFind,dirs[1],6); // rotated images need larger sample size
checkFor(imageToFind,dirs[2],6);
checkFor(imageToFind,dirs[3],6);
var time = performance.now() - now;
var result = document.createElement("div");
result.textContent = "Found "+foundCount +" matching images in "+time.toFixed(3)+"ms. Click to try again.";
document.body.appendChild(result);
// show the image we are looking for
imageToFind.style.left = (64*6 + 16) + "px";
imageToFind.id = "lookingFor";
document.body.appendChild(imageToFind);
}
document.addEventListener("click",doit);
canvas {
border : 2px solid black;
position : absolute;
top : 28px;
left : 2px;
}
#lookingFor {
border : 4px solid red;
}
div {
border : 2px solid black;
position : absolute;
top : 2px;
left : 2px;
}
Click to start test.
Not perfect
The example is not perfect and will sometimes make mistakes. There is a huge amount of room for improving both the accuracy and the speed. This is just something I threw together as an example to show how to use the GPU via the 2D API. Some further maths will be needed to find the statistically good results.
This method can also work for different scales, and rotations, you can even use some of the other comp modes to remove colour and normalize contrast. I have used a very similar approch to stabilize webcam by tracking points from one frame to the next, and a veriaty of other image tracking uses.
I've been trying to implement a FRIM system for my game for the last couple of days.I did some research and came upon this article - it seemed simple enough to implement so I got started.
Everything seems to work fine except I get some temporal aliasing (moving bodies seem to jump ahead a bit) - this happens when more Box2D steps are processed....I think.
private const FIXED_TIMESTEP:Number = 1 / 60;
private const velocityIterations:int = 8;
private const positionIterations:int = 3;
private var fixedTimestepAccumulator:Number = 0;
private var fixedTimestepAccumulatorRatio:Number = 0;
public function Step(dt:Number):void
{
//dt - time between frames - I'm passing the e.passedTime - from the enter frame event; using Starling
fixedTimestepAccumulator += dt;
var nSteps:uint = Math.floor(fixedTimestepAccumulator / FIXED_TIMESTEP);
if (nSteps > 0)
{
fixedTimestepAccumulator = fixedTimestepAccumulator - nSteps * FIXED_TIMESTEP;
}
fixedTimestepAccumulatorRatio = fixedTimestepAccumulator / FIXED_TIMESTEP;
var nStepsClamped:int = Math.min(nSteps, MAX_STEPS);
for (var i:int = 0; i < nStepsClamped; ++i)
{
resetSmoothStates();
singleStep(FIXED_TIMESTEP);
}
world.ClearForces();
smoothStates();
}
private function resetSmoothStates():void
{
for (var bb:b2Body = world.GetBodyList(); bb; bb = bb.GetNext())
{
if (bb.GetUserData() is MyUserData && bb.GetType() != b2Body.b2_staticBody )
{
//each of my bodies have a reference to their sprite (actor) in userData
var _userdata:MyUserData=bb.GetUserData();
_userdata.x = _userdata.bodyPreviousX = bb.GetPosition().x * RATIO;
_userdata.y= _userdata.bodyPreviousY = - bb.GetPosition().y* RATIO;
_userdata.rotation = _userdata.bodypreviousRotation= _userdata.bodypreviousRotation = - bb.GetAngle();
}
}
}
private function smoothStates():void
{
var oneMinusRatio:Number = 1.0 - fixedTimestepAccumulatorRatio;
for (var bb:b2Body = world.GetBodyList(); bb; bb = bb.GetNext())
{
if (bb.GetUserData() is MyUserData && bb.GetType() != b2Body.b2_staticBody )
{
var userdata=bb.GetUserData();
userdata.x = (fixedTimestepAccumulatorRatio * bb.GetPosition().x * RATIO + oneMinusRatio * userdata.bodyPreviousX) ;
userdata.y = (- fixedTimestepAccumulatorRatio * bb.GetPosition().y * RATIO + oneMinusRatio * userdata.bodyPreviousY) ;
userdata.rotation = (- fixedTimestepAccumulatorRatio * bb.GetAngle() + oneMinusRatio * userdata.bodypreviousRotation);
}
}
}
private function singleStep(dt:Number):void
{
Input();
world.Step(dt, velocityIterations, positionIterations);
}
What am I doing wrong?
Any help, suggestions would be highly appreciated.
Thanks
I did this once in a game I was working on so that I could lockstep the game to a certain update rate (this was in iOS). This was based on the code in Daley's book (Learning iOS Game Programming), which was based on some other article in the web (I believe, it may be Allen Bishop's). The code looked like this:
void GameManager::UpdateGame()
{
const uint32 MAXIMUM_FRAME_RATE = Constants::DEFAULT_OBJECT_CYCLES_PER_SECOND();
const uint32 MINIMUM_FRAME_RATE = 10;
const uint32 MAXIMUM_CYCLES_PER_FRAME = (MAXIMUM_FRAME_RATE/MINIMUM_FRAME_RATE);
const double UPDATE_INTERVAL = (1.0/MAXIMUM_FRAME_RATE);
static double lastFrameTime = 0.0;
static double cyclesLeftOver = 0.0;
double currentTime;
double updateIterations;
currentTime = CACurrentMediaTime();
updateIterations = ((currentTime - lastFrameTime) + cyclesLeftOver);
if(updateIterations > (MAXIMUM_CYCLES_PER_FRAME*UPDATE_INTERVAL))
{
updateIterations = MAXIMUM_CYCLES_PER_FRAME*UPDATE_INTERVAL;
}
while (updateIterations >= UPDATE_INTERVAL)
{
// DebugLogCPP("Frame Running");
updateIterations -= UPDATE_INTERVAL;
// Set the random seed for this cycle.
RanNumGen::SetSeed(_cycleManager->GetObjectCycle());
// Dispatch messages.
_messageManager->SendMessages();
// Update all entities.
_entityManager->Update();
// Update the physics
_gameWorldManager->Update(Constants::DEFAULT_OBJECT_CYCLE_SECONDS());
// Advance the cycle clock.
_cycleManager->Update();
}
cyclesLeftOver = updateIterations;
lastFrameTime = currentTime;
}
I can't put my finger on the specific item that is wrong in yours. However, this part is suspicious:
var nStepsClamped:int = Math.min(nSteps, MAX_STEPS);
Before this, you updated your accumulator with:
fixedTimestepAccumulator = fixedTimestepAccumulator - nSteps * FIXED_TIMESTEP;
But now the actual number of steps you are going to execute may be different because of the clamping (nStepsClamped). So your time accumulation is different than the number of steps you actually execute.
Was this helpful?
I decided to go with another approach. I'm using filtered delta times for the physics (I know this can cause some problems).
Here's what I'm doing now:
//Play around with this filter value if things don't look right
var filter:Number=0.4;
filtered_dt= time_between_frames * filter + filtered_dt * (1 - filter);
//Poll imputs and apply forces
// I use velocityIterations =6, positionIterations=3
world.Step(filtered_dt, velocityIterations, positionIterations);
//move sprites here
world.ClearForces();
Another thing you have to do is to scale the forces you apply to your bodies using filtered_dt so things don't "explode" when the frame rate changes a lot.
Hope this helps some one else ... it's not the perfect solution but it works for me.
If you have slow moving bodies the interpolation method above should work fine too.
I know it was asked a thousand times before, but I still can't find a solution.
Searching SO, I indeed found the algorithm for it, but lacking the mathematical knowledge required to truly understand it, I am helplessly lost!
To start with the beginning, my goal is to compute an entire spectrogram and save it to an image in order to use it for a visualizer.
I tried using Sound.computeSpectrum, but this requires to play the sound and wait for it to end, I want to compute the spectrogram in a way shorter time than that will require to listen all the song. And I have 2 hours long mp3s.
What I am doing now is to read the bytes from a Sound object, the separate into two Vectors(.); Then using a timer, at each 100 ms I call a function (step1) where I have the implementation of the algorithm, as follows:
for each vector (each for a channel) I apply the hann function to the elements;
for each vector I nullify the imaginary part (I have a secondary vector for that)
for each vector I apply FFT
for each vector I find the magnitude for the first N / 2 elements
for each vector I convert squared magnitude to dB scale
end.
But I get only negative values, and only 30 percent of the results might be useful (in the way that the rest are identical)
I will post the code for only one channel to get rid off the "for each vector" part.
private var N:Number = 512;
private function step1() : void
{
var xReLeft:Vector.<Number> = new Vector.<Number>(N);
var xImLeft:Vector.<Number> = new Vector.<Number>(N);
var leftA:Vector.<Number> = new Vector.<Number>(N);
// getting sample range
leftA = this.channels.left.slice(step * N, step * (N) + (N));
if (leftA.length < N)
{
stepper.removeEventListener(TimerEvent.TIMER, getFreq100ms);
return;
}
else if (leftA.length == 0)
{
stepper.removeEventListener(TimerEvent.TIMER, getFreq100ms);
return;
}
var i:int;
// hann window function init
m_win = new Vector.<Number>(N);
for ( var i:int = 0; i < N; i++ )
m_win[i] = (4.0 / N) * 0.5 * (1 - Math.cos(2 * Math.PI * i / N));
// applying hann window function
for ( i = 0; i < N; i++ )
{
xReLeft[i] = m_win[i]*leftA[i];
//xReRight[i] = m_win[i]*rightA[i];
}
// nullify the imaginary part
for ( i = 0; i < N; i++ )
{
xImLeft[i] = 0.0;
//xImRight[i] = 0.0;
}
var magnitutel:Vector.<Number> = new Vector.<Number>(N);
fftl.run( xReLeft, xImLeft );
current = xReLeft;
currf = xImLeft;
for ( i = 0; i < N / 2; i++ )
{
var re:Number = xReLeft[i];
var im:Number = xImLeft[i];
magnitutel[i] = Math.sqrt(re * re + im * im);
}
const SCALE:Number = 20 / Math.LN10;
var l:uint = this.total.length;
for ( i = 0; i < N / 2; i++ )
{
magnitutel[i] = SCALE * Math.log( magnitutel[i] + Number.MIN_VALUE );
}
var bufferl:Vector.<Number> = new Vector.<Number>();
for (i = 0; i < N / 2 ; i++)
{
bufferl[i] = magnitutel[i];
}
var complete:Vector.<Vector.<Number>> = new Vector.<Vector.<Number>>();
complete[0] = bufferl;
this.total[step] = complete;
this.step++;
}
This function is executed in the event dispatched by the timer (stepper).
Obviously I do something wrong, as I said I have only negative values and further more values range between 1 and 7000 (at least).
I want to thank you in advance for any help.
With respect,
Paul
Negative dB values are OK. Just add a constant (representing your volume control) until the number of points you want to color become positive. The remaining values that stay negative are usually just displayed or colored as black in a spectrogram. No matter how negative (as they might just be the FFT's numerical noise, which can be a huge negative dB number or even NaN or -Inf for log(0)).
I'm trying to implement a aabb to circle collision.
Here's my code:
//From another file
radius = (sprite.width + sprite.height) / 4;
private function BoxToCircleCollision(box1:BoundingBox, circle1:BoundingCircle):Boolean
{
var nBoxMinX:Number = box1.sprite.x - box1.sprite.width / 2;
//var nBoxMinY:Number = box1.sprite.x + box1.sprite.width / 2;
var nBoxMaxX:Number = box1.sprite.y - box1.sprite.height / 2;
//var nBoxMaxY:Number = box1.sprite.y + box1.sprite.height / 2;
var nCirMinX:Number = circle1.sprite.x - circle1.radius / 2;
//var nCirMinY:Number = circle1.sprite.y - circle1.radius;
var nCirMaxX:Number = circle1.sprite.x + circle1.radius / 2;
//var nCirMaxY:Number = circle1.sprite.y + circle1.radius;
if (nBoxMaxX, 2 > nCirMinX))
{
Main.WriteDebug("Box max: " + nBoxMaxX + " | Circle min: " + nCirMinX);
return true;
}
else
{
Main.WriteDebug("Box max: " + nBoxMaxX + " | Circle min: " + nCirMinX);
return false;
}
}
Somehow the collision does work as expected.
Either they never move at all and "collided" was traced, or they'll continue moving and never collide when I tried swapping values around.
Is there something i'm missing in my logic???
My box-box and circle-circle collision are working fine.
Thanks in advance for your help.
This row doesn't look at all correct:
var nBoxMaxX:Number = box1.sprite.y - box1.sprite.height / 2;
Maybe you meant this:
var nBoxMaxX:Number = box1.sprite.x + box1.sprite.width / 2;
This line won't compile:
if (nBoxMaxX, 2 > nCirMinX))
Edit:
Here's a function to help you get the AABB <-> Circle collision right. It's not a complete solution but you can combine it with the calculations you have for the AABB min and max values, should be trivial:
private function collideAABBandCircle(c : Circle, aabb:AABB) : Boolean {
var sqDist : Number = sqDist(c.centerPoint, aabb);
return sqDist <= c.radius * c.radius:
}
private function sqDist(p : Point, aabb:AABB) : Number {
/* CALCULATE min and max values of aabb bounds */
var sqDist : Number = 0.0;
if(p.x < minX) {
sqDist += (minX - p.x) * (minX - p.x);
}
if(p.x > maxX) {
sqDist += (p.x - maxX) * (p.x - maxX);
}
if(p.y < minY) {
sqDist += (minY - p.y) * (minY - p.y);
}
if(p.y > maxY) {
sqDist += (p.y - maxY) * (p.y - maxY);
}
return sqDist;
}
There's a good article about using Separating Axis Theorem for 2d collision detection. Great read if you want to find out how 2d collision detection works.
I have a simple code that seems to be giving strange results.
var startPoint:Point = new Point(x, y); // a point
var r:Number = path[i].row + (-Math.floor((length * 2 + 2) / 2)); // just some math
trace(r); // the math checks out and gives a 3
var tey = startPoint.y + r; //this gives a really strange return....
trace(startPoint.y + r, tey); // this works, and then gives the strange return.
the return is
3
10 7function Function() {}
as
var tey:Number = startPoint.y + r;
trace(startPoint.y + r, tey);
the return is
10 NaN
IDE is FlashDevelop if you feel that information is important.
Additional Info/Testing:
trace(startPoint.y);
traces as
7
:Number
var tey:Number = startPoint.y + r;
trace(tey);
equals
NaN
Number()
var tey = Number(startPoint.y) + r;
trace(tey);
equals
7function Function() {}
(excuse my poor post layout, I'm still inexperienced with stackoverflow's editing system and am working on making this a little cleaner)
I can not reproduce your issue.
Your problem is with the X and Y values on this line, probably more so the Y value.
var startPoint:Point = new Point(x, y); // a point
Try hard coding the X and Y to 0
Here is some sample code I ran. This worked as expected, so I will assume its the vars X and Y or something related to the "i" when you access path[i].row
var startPoint:Point = new Point(0, 0); // a point
var r:Number = 3; // just some math
trace(r); // the math checks out and gives a 3
var tey = startPoint.y + r; //this gives a really strange return....
trace(startPoint.y + r, tey); // this works, and then gives the strange return.
I discovered the issue, it was with a part of the code that I appended out of my sample above.
var startPoint:Point = new Point(x, y); // a point
for ( /*for stuff*/ ){
var r:Number = path[i].row + (-Math.floor((length * 2 + 2) / 2)); // just some math
}
trace(r); // the math checks out and gives a 3
var tey = startPoint.y + r; //this gives a really strange return....
trace(startPoint.y + r, tey); // this works, and then gives the strange return.
being var'd inside of the for loop made it act oddly when used outside so I had to move the var assignment outside of the loop and use as normal. A strange occurrence, but I should have knew better.