Hear sight 9
Run Experiment
Hear sight 9
Jaap MurreThe goal is to develop an application for blind people so that they can hear the patterns they are focussing on through saccadic movements or through head movements. The demo includes various shapes, colors, filter options, and games to experience the system. The system is also implemented in the Open Eye Tracker by Kexxu Robotics, a wearable affordable eye tracker.
No comments yet
Comments
No comments yetIf you are a registered user and signed in, you can here copy this script and its stimuli to your own account, where you can edit it and change it in any way you want.
It is absolutely free to register (no credit card info asked!). You can then instantly copy this experiment with one click and edit it, change its accompanying texts, its landing page, stimuli, etc. Invite your colleagues, friends, or students to check out your experiment. There is no limit on how many people can do your experiment, even with a free account.
The catch? There is no catch! Just keep in mind that with a free account, you cannot collect data. For teaching that is usually not a problem. For research, prepaid data collection (unlimited subjects) starts as low as €10.00 for a 10-day period.
var resolution = 1024; // Image is converted to this resolution
var title = addblock(10,0,80,10).text("Move (or click) the mouse for shape sound",60);
//var info = addblock(10,92,80,8); // Block to some some info
var icons = [
addblock(10,92,8,8).icon('circle').on('click',function(){draw('circle')}),
addblock(20,92,8,8).icon('circle').style('color','green').on('click',function(){draw('circle-colored')}),
addblock(30,92,8,8).icon('circle-blank').on('click',function(){draw('circle-blank')}),
addblock(40,92,8,8).icon('sign-blank').on('click',function(){draw('sign-blank')}),
addblock(50,92,8,8).icon('unchecked').on('click',function(){draw('unchecked')}),
addblock(60,92,8,8).icon('align-justify').on('click',function(){draw('align-justify')}),
addblock(70,92,8,8).icon('font').on('click',function(){draw('font')}),
addblock(80,92,8,8).icon('picture').on('click',function(){draw('picture')})
];
var picture_index = -1,
pictures = [
// "car.png",
// "car_edges.png",
"away-3408119_1920.jpg",
"portrait-3083402_1920.jpg",
"road-220058_1280.jpg",
"street-4942809_1920.jpg"
]
function draw(what) {
window.last_draw_what = what;
var canvas = document.getElementById('canvas'),
ctx = canvas.getContext('2d'),
name;
ctx.clearRect(0,0,canvas.width,canvas.height); // Clear canvas
ctx.fillStyle = "black"; // ("rgb(0,0,0)");
ctx.fillRect(0,0,canvas.width,canvas.height); // Clear canvas
switch (what) {
case 'circle':
ctx.fillStyle = "white";
ctx.beginPath();
ctx.arc(255,255,100,0,Math.PI*2);
ctx.fill();
ctx.beginPath();
ctx.arc(1024-255,255,75,0,Math.PI*2);
ctx.fill();
ctx.beginPath();
ctx.arc(255,1024-255,50,0,Math.PI*2);
ctx.fill();
ctx.beginPath();
ctx.arc(1024-255,1024-255,25,0,Math.PI*2);
ctx.fill();
break;
case 'circle-colored':
ctx.fillStyle = "white";
ctx.beginPath();
ctx.arc(255,255,100,0,Math.PI*2);
ctx.fill();
ctx.fillStyle = "red";
ctx.beginPath();
ctx.arc(1024-255,255,100,0,Math.PI*2);
ctx.fill();
ctx.fillStyle = "green";
ctx.beginPath();
ctx.arc(255,1024-255,100,0,Math.PI*2);
ctx.fill();
ctx.fillStyle = "blue";
ctx.beginPath();
ctx.arc(1024-255,1024-255,100,0,Math.PI*2);
ctx.fill();
break;
case 'circle-blank':
ctx.strokeStyle = "white";
ctx.lineWidth = 5;
ctx.beginPath();
ctx.arc(255,255,100,0,Math.PI*2);
ctx.stroke();
ctx.beginPath();
ctx.arc(1024-255,255,75,0,Math.PI*2);
ctx.stroke();
ctx.beginPath();
ctx.arc(255,1024-255,50,0,Math.PI*2);
ctx.stroke();
ctx.beginPath();
ctx.arc(1024-255,1024-255,25,0,Math.PI*2);
ctx.stroke();
break;
case 'sign-blank':
ctx.fillStyle = "white";
ctx.fillRect(255,255,200,200);
ctx.fillRect(1024-255,255,75,75);
ctx.fillRect(255,1024-255,50,50);
ctx.fillRect(1024-255,1024-255,25,25);
break;
case 'unchecked':
ctx.strokeStyle = "white";
ctx.lineWidth = 5;
ctx.strokeRect(255,255,200,200);
ctx.strokeRect(1024-255,255,75,75);
ctx.strokeRect(255,1024-255,50,50);
ctx.strokeRect(1024-255,1024-255,25,25);
break;
case 'align-justify':
ctx.fillStyle = "white";
ctx.strokeStyle = "white";
ctx.lineWidth = 3;
ctx.beginPath();
ctx.moveTo(50,50);
ctx.lineTo(800,300);
ctx.moveTo(50,300);
ctx.lineTo(800,50);
ctx.moveTo(50,600);
ctx.lineTo(800,600);
ctx.moveTo(900,900);
ctx.lineTo(900,50);
ctx.stroke();
break;
case 'font':
ctx.fillStyle = "white";
ctx.strokeStyle = "white";
ctx.font = "200px Arial";
ctx.fillText("A B C",50,200);
ctx.fillText("a cat",50,500);
ctx.strokeText("The fox",50,800);
break;
case 'picture':
__js { var img = new Image(); }
img.crossOrigin = 'anonymous';
++picture_index;
name = pictures[picture_index];
picture_index = (picture_index)%pictures.length;
img.src = image.getRemoteFileName(name);
img.onload = function(){
ctx2d.drawImage(img,0,0);
};
break;
}
}
var b = addblock(10,10,80,80); // Canvas block
b.style("border","thin grey solid");
b.text('');
var canvas = document.getElementById('canvas');
var ctx2d = canvas.getContext('2d');
// Show a practice circle with sx slices:
var sx = 12;
for (var a = 0; a < sx; a++) {
ctx2d.beginPath();
ctx2d.moveTo(256,256);
ctx2d.arc(256,256,100,2*Math.PI*a/sx-0.5*Math.PI,2*Math.PI*(a+1)/sx-0.5*Math.PI,false);
ctx2d.closePath();
ctx2d.fillStyle = 'rgba(' + parseInt(255*a/sx) + ',120,20,1)';
ctx2d.fill();
}
function drawPixel(x,y,r,g,b,a) {
a = a === undefined ? 1 : a;
ctx2d.fillStyle = 'rgba(' + parseInt(r*255)
+ ',' + parseInt(g*255)
+ ',' + parseInt(b*255)
+ ',' + parseInt(a*255)
+ ')';
ctx2d.fillRect(x,y,1,1);
}
function readArea(x,y,size) {
var p = canvas.getContext('2d')
.getImageData(x - Math.floor(size/2),y - Math.floor(size/2),size,size)
.data;
return p;
}
function getMousePos(evt) {
var rect = canvas.getBoundingClientRect(), // abs. size of element
scaleX = canvas.width / rect.width, // relationship bitmap vs. element for X
scaleY = canvas.height / rect.height; // relationship bitmap vs. element for Y
return {
x: (evt.clientX - rect.left) * scaleX, // scale mouse coordinates after they have
y: (evt.clientY - rect.top) * scaleY // been adjusted to be relative to element
}
}
function rgbToHex(r, g, b) {
if (r > 255 || g > 255 || b > 255)
throw "Invalid color component";
return ((r << 16) | (g << 8) | b).toString(16);
}
var mouse_timer;
$('#canvas').mousemove(function(e) {
clearTimeout(mouse_timer);
mouse_timer = setTimeout(function(){
if ($("#onmove").is(":checked")) {
var pos = getMousePos(e);
emitMouseMoveEnd(pos.x,pos.y,e);
shapeSound(pos.x,pos.y);
}
},200);
});
require(["dojo/_base/window"], function(win){
window.win = win;
});
function emitMouseMoveEnd(x,y,e) {
on.emit(win.body(),"mousemoveend",{ // Re-emit dojo-normalized events
bubbles: true,
cancelable: true,
x: x,
y: y,
event: e
});
}
$('#canvas').click(function(e) {
var pos = getMousePos(e);
shapeSound(pos.x,pos.y);
});
$(window).blur(function(e) {
fov && fov.stopAllOcsillators();
});
// Fovea class
function Fovea(/* Canvas 2D context */ctx,diameter,segments,threshold) {
this.ctx = ctx;
this.diameter = diameter || 256; // fovea diameter in pixels on image
this.segments = segments || 24; // Number of segments of the circle
this.threshold = 0.5; // Don't include circles below this fraction of segments
this.preCalculateFovea(); // Pre-calculate distances and reserve memory
}
// Pre-calculate distances and reserve memory
Fovea.prototype.preCalculateFovea = function() {
var res = this.diameter,
x, y, d,
cx = Math.floor(res/2),
cy = cx, // Assume square image
i, dsizes = new Array(cx+1);
// res = resolution;
var mapping = this.mapping = [];
var circles = this.circles = { r: [], g: [], b: [], x: []}; // Provides memory to fill later
for (x = 0; x <= cx; x++) { // circle data for each color channel
mapping.push([]);
}
dsizes.fill(0);
for (x = cx+1; x <= res; x++) { // Right upper quadrant
for (y = 0; y < cy; y++) {
// This can be made more efficient but since it is done only once,
// we will leave this for later (cf. drawCircle for method):
d = Math.floor(Math.sqrt((x-cx)*(x-cx)+(y-cy)*(y-cy)));
if (d > cx) {
continue; // Follow only the largest circle, not the corners
}
mapping[d].push([x,y]);
++dsizes[d];
}
}
for (x = res; x > cx; x--) { // Right lower quadrant
for (y = cy; y < res; y++) {
d = Math.floor(Math.sqrt((x-cx)*(x-cx)+(y-cy)*(y-cy)));
if (d > cx) {
continue; // Follow only the largest circle, not the corners
}
mapping[d].push([x,y]);
++dsizes[d];
}
}
for (x = cx; x >= 0; x--) { // Left lower quadrant
for (y = res; y >= cy; y--) {
d = Math.floor(Math.sqrt((x-cx)*(x-cx)+(y-cy)*(y-cy)));
if (d > cx) {
continue;
}
mapping[d].push([x,y]);
++dsizes[d];
}
}
for (x = 0; x <= cx; x++) { // Left upper quadrant
for (y = cy-1; y >= 0; y--) {
d = Math.floor(Math.sqrt((x-cx)*(x-cx)+(y-cy)*(y-cy)));
if (d > cx) {
continue;
}
mapping[d].push([x,y]);
++dsizes[d];
}
}
for (i = 0; i < dsizes.length; i++)
{
__js { // SJS construct, see https://conductance.io/reference/#sjs:%23language/syntax::__js
circles.r.push(new Uint8Array(dsizes[i]));
circles.g.push(new Uint8Array(dsizes[i]));
circles.b.push(new Uint8Array(dsizes[i]));
}
}
}
// Make array `s` have length `size` by duplication and interpolation
Fovea.prototype.resample = function(/* array */s,size) {
var ss = [], sss = [], newlength, t, steps, i, j, r, mult, step, a;
// Up-sample
if (s.length < size) {
newlength = size;
if (size/s.length > 2) {
a = Math.floor(size/s.length);
s2 = [];
for (i = 0; i < s.length; i++) { // Up-sample very small arrays
for (j = 0; j < a; j++) {
s2.push(s[i]);
}
}
s = s2; // Continue as s
}
} else {
newlength = (Math.floor(s.length/size) + 1)*size;
}
t = s.length/(newlength-s.length);
steps = [];
// Locations where to insert padding numbers
for (i = 0; i < newlength-s.length; i++) {
steps.push(Math.round(i*t,0));
}
// Insert padding numbers (average of left and right number)
for (i = 0, step = 0; i < s.length; i++) {
ss.push(s[i]);
if (steps[step] === i) {
ss.push(s[i+1] !== undefined ? (s[i]+s[i+1])/2 : s[i]);
++step;
}
}
if (ss.length > s.length) {
// Down-sample to length size
mult = ss.length/size; // Should be an integer
for (i = 0; i < ss.length; i += mult) {
r = 0;
for (j = i; j < i+mult; j++)
r += ss[j];
sss.push(r/mult);
}
}
return sss;
}
// Expand circles outside foveola exponentially
function grow(x,retina_radius,d,growth,fovea_radius) {
fovea_radius = fovea_radius || retina_radius/2;
var retval;
if ( grow._cache === undefined
|| grow._cache.growth !== growth
|| grow._cache.retina_radius !== retina_radius
|| grow._cache.fovea_radius !== fovea_radius
) {
grow._cache = {
retina_radius: retina_radius,
growth: growth,
fovea_radius: fovea_radius,
d: {}
}
}
retval = grow._cache[d] && grow._cache[d][x];
if (retval !== undefined) {
return retval;
} else {
retval = d < fovea_radius ? x :
retina_radius + Math.floor((x-retina_radius)*Math.pow(1 + growth,d-fovea_radius));
if (grow._cache[d] === undefined) {
grow._cache[d] = {};
}
grow._cache[d][x] = retval; // Cache answer
return retval;
}
}
Fovea.prototype.initSegmentXs = function(growth,fovea_x) {
var i, j, x, xx, d,
retina_radius = this.diameter/2,
fovea_radius = retina_radius/2;
mapping = this.mapping,
growth = parseFloat($("#growth").val());
if (this.growth === growth) { // Used cached value until change in growth
return;
}
// Calculated cached array
this.segmentX = [];
for (i = 0; i < mapping.length; i++) {
this.segmentX.push([]);
for (j = 0; j < this.segments; j++) {
x = Math.cos((j - this.segments/4)*(1/this.segments)*2*Math.PI)*i;
if (i > fovea_radius) {
xx = x*Math.pow(1.0 + growth,i - fovea_radius);
} else {
xx = x;
}
xx = fovea_x - this.diameter/2 + xx;
this.segmentX[i].push(xx);
}
}
}
// Convert image of size diameter centered at (fovea_x,fovea_y) into circles
Fovea.prototype.getSpectrum = function(fovea_x,fovea_y)
{
fovea_x = Math.floor(fovea_x);
fovea_y = Math.floor(fovea_y);
var x, y, i, j, len, mp, offset,
circles = this.circles,
diameter = this.diameter,
mapping = this.mapping,
growth = parseFloat($("#growth").val()),
// image = readArea(fovea_x,fovea_y,diameter),
image = canvas.getContext('2d')
.getImageData(0,0,resolution,resolution)
.data;
for (i = 0; i < mapping.length; i++) {
len = mapping[i].length;
mp = mapping[i];
for (j = 0; j < len; j++) {
x = grow(mp[j][0],diameter/2,i,growth);
y = grow(mp[j][1],diameter/2,i,growth);
x = fovea_x - diameter/2 + x;
y = fovea_y - diameter/2 + y;
if ($("#draw_waves").is(":checked")) {
drawPixel(x,y,0.5,0.5,0,0.2);
}
if (x < 0 || x > resolution || y < 0 || y > resolution) {
circles.r[i][j] = 0;
circles.g[i][j] = 0;
circles.b[i][j] = 0;
} else {
// We can do sharpening or edge detection here
// circles.e[i][j] for edge octave above r
offset = x*4 + y*resolution*4;
circles.r[i][j] = image[offset];
circles.g[i][j] = image[offset + 1];
circles.b[i][j] = image[offset + 2];
// TODO (***): circles.edge[i][j] = // calc edge here
// if (i === 40) {
// console.log(fovea_x,fovea_y,x,y,image[x*4 + y*resolution*4 + 1]);
// }
}
}
}
var spectrum = {r: [], g: [], b: [], x: []}, offset = 0;
// Resample to length `this.segments`
for (i = 0; i < mapping.length; i++) {
if (circles.r[i].length >= this.segments*this.threshold) {
spectrum.r.push(this.resample(circles.r[i],this.segments));
spectrum.g.push(this.resample(circles.g[i],this.segments));
spectrum.b.push(this.resample(circles.b[i],this.segments));
} else {
++offset;
}
}
this.blind_spot = offset;
return spectrum;
}
// Combines `frame_size` consecutive rows and averages these
Fovea.prototype.condense = function(/*array of arrays*/s,frame_size) {
var i, j, counter, current, result = [];
for (i = 0, step = 0; i < s.length; i++) {
if (!(i%frame_size)) {
counter = 1;
current = new Array(s[i].length).fill(0);
result.push(current);
}
for (j = 0; j < s[i].length; j++) {
current[j] += s[i][j];
}
if (!((i+1)%frame_size) || i === s.length-1) {
for (j = 0; j < s[i].length; j++) {
current[j] /= counter;
}
}
++counter;
}
return result;
}
Fovea.prototype.convertSpectra = function(spectra,frame_size,pitch) {
frame_size = frame_size || 8;
pitch = pitch || 220;
return {
r: this.condense(spectra.r,frame_size),
g: this.condense(spectra.g,frame_size),
b: this.condense(spectra.b,frame_size)
};
}
__js {var ctx = new (AudioContext || webkitAudioContext)(); }
Fovea.prototype.stopAllOcsillators = function() { // Silence and stop all oscillators
for (var i = 0; i < all_osc.length; i++) {
all_osc[i].gain_node.gain.setTargetAtTime(0,ctx.currentTime, 0.003);
}
await(15);
for (i = 0; i < all_osc.length; i++) {
all_osc[i].osc_node.stop();
}
}
Fovea.prototype.playspectra = function(spectra,x,/*ms*/duration,pitch) {
pitch = pitch || 220;
function addOscillator() {
var osc = ctx.createOscillator('sine'); // sine (def), sawtooth, square, triangle
var gain = ctx.createGain();
gain.value = 0;
var panner = ctx.createStereoPanner();
osc.connect(panner);
panner.connect(gain);
gain.connect(ctx.destination);
return { osc_node: osc, gain_node: gain, panner_node: panner }
}
if (window.all_osc) {
this.stopAllOcsillators();
}
window.all_osc = []; // all_osc is global in script
var osc, f, i,
value, pan, xx,
color, colors = ['r','g','b'], // maybe add 'edge' in front
no_of_frames = spectra.r.length;
no_of_segments = colors.length, // one for each each color (or edge)
segments = spectra.r[0].length,
frame_size = parseFloat($("#frame_size").val());
this.initSegmentXs(growth,x,no_of_frames);
// (***) Here I can change the pitch location on the circle
for (i = segments*no_of_segments-1; i >= 0; i--) {
osc = addOscillator();
all_osc.push(osc);
tone = pitch * Math.pow( Math.pow(2,1/segments),i);
osc.osc_node.frequency.value = tone;
osc.gain_node.gain.value = 0;
osc.osc_node.start();
}
osc = addOscillator(); // Reference sound at half base pitch
all_osc.push(osc);
tone = (pitch/2) * Math.pow( Math.pow(2,1/segments),0);
osc.osc_node.frequency.value = tone;
osc.gain_node.gain.value = 0.025;
osc.osc_node.start();
for (f = 0; f < no_of_frames; f++) {
for (i = 0; i < segments*no_of_segments; i++) {
color = colors[Math.floor(i/segments)];
value = spectra[color][f][i%segments]/255/segments;
all_osc[i].gain_node.gain.setTargetAtTime(value,ctx.currentTime,0.003);
xx = this.segmentX[(this.blind_spot||0) + f*frame_size][i%segments];
pan = Math.max(Math.min(2*xx/resolution - 1,1),-1);
all_osc[i].panner_node.pan.setValueAtTime(pan,ctx.currentTime);
}
await(duration);
}
this.stopAllOcsillators();
}
// Defaults Canvas 2D context ctx, diameter = 128, segments = 24, threshold = 0.5
var fov = new Fovea(ctx2d,128,24);
function shapeSound(x,y) {
var pitch = $("#pitch").val(),
frame_size = $("#frame_size").val(),
frame_length = $("#frame_length").val(),
ground = $("#ground").val(),
t1, t2, t3;
t1 = +new Date();
var result = fov.getSpectrum(x,y);
t2 = +new Date();
var gains = fov.convertSpectra(result,frame_size,pitch); // Frame size, base pitch
console.log("getSpectrum: ",t2-t1,"ms");
fov.playspectra(gains,x,frame_length,pitch); // frame length in ms, ground pitch of FFT
}
function renderImage() {
var filter = $("#filter").val();
if (filter != 'none') {
console.log("Start filter processing");
var b = addblock("center","center",30,30).style("color","white")
.text(' Processing...');
await(500);
var img_data = LenaJS[filter].apply(null,[ctx2d.getImageData(0,0,canvas.width,canvas.height)]);
b.destroy();
console.log("Completed filter processing: ",img_data);
ctx2d.putImageData(img_data,0,0);
} else {
if (window.last_draw_what) {
if (window.last_draw_what !== 'picture') {
draw(window.last_draw_what);
} else {
if (picture_index >= 0) {
__js { var img = new Image(); }
img.crossOrigin = 'anonymous';
name = pictures[picture_index];
img.src = image.getRemoteFileName(name);
img.onload = function(){
ctx2d.drawImage(img,0,0);
};
}
}
}
}
}
var b_pitch = addblock(91,10,9,10).align("left")
.text('',50),
b_frame_size = addblock(91,20,9,10).align("left")
.text('',50),
b_frame_length = addblock(91,30,9,10).align("left")
.text('',50),
b_growth = addblock(91,40,9,10).align("left")
.text('',50),
b_onmove = addblock(91,55,9,10).align("left")
.text('',50),
b_draw_waves = addblock(91,60,9,10).align("left")
.text('',50)
;
LenaJS.laplacianBlend = function(pixels) {
var operator = [ 0, -1, 0, // Blends with darkened part of the original
-1, 4.1, -1,
0, -1, 0 ]
return LenaJS.convolution(pixels, operator)
}
LenaJS.outline = function(pixels) {
var operator = [-1, -1, -1, // outline filter (not in LenaJS)
-1, 8, -1,
-1, -1, -1 ]
return LenaJS.convolution(pixels, operator)
}
LenaJS.outlineBlend = function(pixels) {
var operator = [-1, -1, -1, // outline blend filter (not in LenaJS)
-1, 8.25, -1,
-1, -1, -1 ]
return LenaJS.convolution(pixels, operator)
}
var filter_select = `
`;
for (var i = 0, s = ""; i < filter_select.parts.length; i++) {
s += filter_select.parts[i];
}
var b_filter = addblock(91,70,9,10).align("left").text(s,50);
$(document).ready(function(){
$("#filter").change(function(){
renderImage();
});
});
setup_lessons();
awaitkey("ESCAPE");
Data inspection is forthcoming!
In the mean time, authors may download their own data and make it available as an Excel file. Check out the 'Stimuli and Files' tab.
Click on a category to view the stimuli and files
You can download the files shown here by clicking on the file names or image. Note that you cannot link directly to the images, sounds, videos, etc. shown here from other web pages; the link will go stale in about one hour and will no longer work after that.
You can download the files shown here by clicking on the file names or image. Note that you cannot link directly to the images, sounds, videos, etc. shown here from other web pages; the link will go stale in about one hour and will no longer work after that.
You can download the files shown here by clicking on the file names or image. Note that you cannot link directly to the images, sounds, videos, etc. shown here from other web pages; the link will go stale in about one hour and will no longer work after that.