Changeset 28548

Timestamp:

2013-10-29T13:27:04+13:00 (10 years ago)

Author:

davidb

Message:

Changes after developing the demo for the SMAM-2013 keynote talk

Location:

gs3-extensions/web-audio/trunk

Files:

• diy-audio-player/script/similarity-matrix-play.js (modified) (1 diff)
• js-dsp/frequency-player.js (modified) (10 diffs)
• js-dsp/frequency-plotter.js (modified) (7 diffs)
• js-mad/script/main-jsmad.js (modified) (7 diffs)

gs3-extensions/web-audio/trunk/diy-audio-player/script/similarity-matrix-play.js

@@ -279 +279 @@
 
     //var selfSimImgSrc="sites/localsite/collect/" + collect +"/bordered/line-interlaced/"+self_sim_png;
-    var selfSimImgSrc="sites/localsite/collect/" + collect +"/images/blank-775x816.png";
+    var selfSimImgSrc="sites/" + site + "/collect/" + collect +"/images/blank-775x816.png";
     //console.log("self sim img = " + selfSimImgSrc);
 

gs3-extensions/web-audio/trunk/js-dsp/frequency-player.js

@@ -151 +151 @@
 function plotSimilarityMatrix()
 {
+
+    if (frequencyPlotter.hasStoredSelfSimilarityMatrix()) {
+    console.log("Using stored self-similarity matrix image");
+    frequencyPlotter.plotStoredSelfSimilarityMatrix();
+    return;
+    }
+
     var $progressElem = $('#processing');
     var displayProgress = ($progressElem.length>0) ? true : false;
@@ -190 +197 @@
 }
 
+
+function acceleratedProcessingCheckbox(elem)
+{
+    // the following could be done by more directly studying 'elem'
+    var speedup = $('input:checkbox[name=acceleratedProcessingInput]').prop("checked");
+    if (speedup) {
+    console.log("frequency-player: set JsMad Processing to be faster than realtime");
+    jsMadProcessing.setSpeedupParams(100,0); // e.g. if Firefox
+    }
+    else {
+    console.log("frequency-player: set JsMad Processing process in batches of 10, with a 1 sec delay");
+    jsMadProcessing.setSpeedupParams(10,1000); 
+    }
+}
+
+
 function plotFrequencySpectrum(fbSpectrum) 
 {
@@ -223 +246 @@
     }
 
+
+    if (typeof selectedMeandreWorkflow !== "undefined") {
+    // Have a meandre workflow embedded in the page
+    if (selectedMeandreWorkflow.meandreDataOutput.length>0) {
+        // play some processed data
+
+        //var rta_audio = document.getElementById('rta-audio-element');
+        
+        console.log("*** freqPlayerAudioAvailable, Meandre workflow setting 'mysong' volume to 0");
+
+        soundManager.setVolume("mysong",0);
+        pannedPlayback.volume = 1.0;
+        
+        var frame = selectedMeandreWorkflow.meandreDataOutput[0];
+        //console.log("*** meanderDataOutput len = " + selectedMeandreWorkflow.meandreDataOutput.length
+        //      + ", frame size of head = " + frame.length);
+        
+
+        if (webAudioContext!=null) {
+        // Working with Web audio
+        console.log("*** processing meandre output as Web audio");
+        
+        var inputLen = frame.length;
+
+        var outputArrayL = event.outputBuffer.getChannelData(0);
+        var outputArrayR = event.outputBuffer.getChannelData(1);
+
+        if (audio.paused) {
+            // not processing/playing any new audio 
+            
+            for (var i=0; i<stereo_frame.length; i+=2) {            
+            outputArrayL[i] = stereo_frame;
+            }
+            for (var i=0; i<outputArrayR.length; i++) {
+            outputArrayR[i] = 0;
+            }
+            return;
+        }
+
+        // And again the fake values to keep the rest of the code happy
+        // **** Are these actually used in this block??
+        channels = 2;
+        frameBufferLength = channels * inputLen;
+        samplerate = 44100;
+
+        // play the result
+        for (var i=0; i<outputArrayL.length; i++) {
+            outputArrayL[i] = frame[i];
+        }
+        for (var i=0; i<outputArrayR.length; i++) {
+            outputArrayR[i] = frame[i];
+        }
+
+        //written = frame.length;
+
+        if (togglePlayMode == "play") {
+            // displaying play button (which counter-intuatively means it must be paused)
+            return;
+        }
+        
+        //console.log("***### popping head of meandreDataOutput queue");
+        selectedMeandreWorkflow.meandreDataOutput.shift();
+        }
+
+        else {
+        
+        var stereo_frame = new Float32Array(frame.length*2);
+        for (var i=0; i<frame.length; i++) {
+            stereo_frame[2*i] = frame[i];
+            stereo_frame[2*i+1] = frame[i];
+        }
+
+
+        // Start playing the audio data returned from the Meandre workflow
+        var written = pannedPlayback.mozWriteAudio(stereo_frame);
+        
+        if (written == 0) {
+            // audio has backed up => give it a chance to clear
+            return;
+        }
+
+        if (togglePlayMode == "play") {
+            // displaying play button (which counter-intuatively means it must be paused)
+            return;
+        }
+        
+        if (written != stereo_frame.length) {
+            //console.log("*** frame size was " + frame.length + " but only wrote out " + written);
+            //selectedMeandreWorkflow.meandreDataOutput[0] = frame.subarray(written);
+            
+            selectedMeandreWorkflow.meandreDataOutput[0] = frame.subarray(written/2);
+        }
+        else {
+            //console.log("***### popping head of meandreDataOutput queue");
+            selectedMeandreWorkflow.meandreDataOutput.shift();
+        }
+
+        }
+        
+        return; // Data output from Meandre workflow has been player => Prevent any subsequent audio from being played
+    }
+
+    }
+
+
+
     var audio = document.getElementById('rta-audio-element');
 
@@ -276 +405 @@
     samplerate = 44100;
 
-    // Apply Graphics Equalizer, and then play the result
     if (_fp_first_aa) {
         console.log("*** !!!!! freqPlayerAudioAvailable, web audio context setting 'mysong' volume to 0");
@@ -283 +411 @@
     }
 
-    outputArrayL.set(grapheqL.process(inputArrayL));
-    outputArrayR.set(grapheqR.process(inputArrayR));
+    // Apply Graphics Equalizer, and then play the result
+
+    if ((typeof grapheqL !== "undefined") && (typeof grapheqR !== "undefined")) {
+        outputArrayL.set(grapheqL.process(inputArrayL));
+        outputArrayR.set(grapheqR.process(inputArrayR));
+    }
+    else {
+        outputArrayL.set(inputArrayL);
+        outputArrayR.set(inputArrayR);
+    }
+
     }
     else {
@@ -294 +431 @@
     fb = event.frameBuffer;
 
+    // Apply Graphics Equalizer, and then play the result
+
+
     if (channels==2) {
         var mono_len = fb.length/2;
@@ -344 +484 @@
     }
 
-    // Apply Graphics Equalizer, and then play the result
 
     if (_fp_first_aa) {
@@ -352 +491 @@
     }
 
+
     var written = pannedPlayback.mozWriteAudio(fb); // could equally make it baseSong ??
     if (written != fb.length) {
@@ -357 +497 @@
         console.warn("input size = " + fb.length + ", output size = " + written);
     }
+
+
     }
 
@@ -381 +523 @@
     // Uncomment the following line to have the audio data being decoded by
     // the 'rta-audio-element' audio element pushed in to the Javascript DSP workflow
+    //
+    // *** Note: The following has been superceeded by the combination of
+    // JsMad and embeeded Meandre as a way of processing audio, which can operate
+    // at faster then realtime
+    // The related Meandre block of code is at the start of this method
+    
     //activeWorkflow.pumpDataHead("rawInput",fb);
-
-    if (typeof selectedMeandreWorkflow !== "undefined") {
-    // Have a meandre workflow embedded in the page
-    if (selectedMeandreWorkflow.meandreDataOutput.length>0) {
-        // play some processed data
-
-        //var rta_audio = document.getElementById('rta-audio-element');
-        
-        console.log("*** freqPlayerAudioAvailable, Meandre workflow setting 'mysong' volume to 0");
-
-        soundManager.setVolume("mysong",0);
-        pannedPlayback.volume = 1.0;
-        
-        var frame = selectedMeandreWorkflow.meandreDataOutput[0];
-        //console.log("*** meanderDataOutput len = " + selectedMeandreWorkflow.meandreDataOutput.length
-        //      + ", frame size of head = " + frame.length);
-
-        var stereo_frame = new Float32Array(frame.length*2);
-        for (var i=0; i<frame.length; i++) {
-        stereo_frame[2*i] = frame[i];
-        stereo_frame[2*i+1] = frame[i];
-        }
-        
-        // Start playing the audio data returned from the Meandre workflow
-        var written = pannedPlayback.mozWriteAudio(stereo_frame);
-        
-        if (written == 0) {
-        // audio has backed up => give it a chance to clear
-        return;
-        }
-        
-        if (togglePlayMode == "play") {
-        // displaying play button (which counter-intuatively means it must be paused)
-        return;
-        }
-        
-        if (written != stereo_frame.length) {
-        //console.log("*** frame size was " + frame.length + " but only wrote out " + written);
-        //selectedMeandreWorkflow.meandreDataOutput[0] = frame.subarray(written);
-        
-        selectedMeandreWorkflow.meandreDataOutput[0] = frame.subarray(written/2);
-        }
-        else {
-        //console.log("***### popping head of meandreDataOutput queue");
-        selectedMeandreWorkflow.meandreDataOutput.shift();
-        }
-    }
-
-    }
-}
-
+    
+}
+

gs3-extensions/web-audio/trunk/js-dsp/frequency-plotter.js

@@ -35 +35 @@
     this.ctx = canvas.getContext('2d');
     this.label_ctx = label_canvas.getContext('2d');
+    this.canvas_saved_img = null;
+
+    var image_server_url = "http://localhost:8181/greenstone3/cgi-bin/image-server.pl?a=load&c="+gs.cgiParams.c
+    +"&d="+gs.cgiParams.d+"&site="+gs.xsltParams.site_name;
+    var canvas_load_data = { a: "load", 
+                 d: gs.cgiParams.d, 
+                 c: gs.cgiParams.c, 
+                 site: gs.xsltParams.site_name };
+
+    var outerThis = this;
+    var jqxhr = $.ajax(image_server_url)
+    .done(function(imagedata) {
+        var check_prefix = imagedata.toString().substring(0, 5);
+        if (check_prefix == "data:") {
+        console.log("Loaded in previously stored self-similarity map");
+        outerThis.canvas_saved_img = document.createElement("img");
+        outerThis.canvas_saved_img.src = imagedata;
+        }
+    })
+    .fail(function() {
+        console.log("No currently stored self-similarity map for this document");
+    });
+
 
     this.plotMapping[activeWorkflow.chromaCpt.transformMode](this);
@@ -41 +64 @@
     this.prevXLineMaxArray = null;
     this.plottedDataBuffer = new Array();
+
+    this.printedOutOfDataWarning = false;
 
 };
@@ -182 +207 @@
     }
     else {
-    console.error("FrequencyPlotter::plotVisualLine(): No data to play for event time: " + eventTime);
+    if (!this.printedOutOfDataWarning) {
+        console.error("FrequencyPlotter::plotVisualLine(): No data to play for event time: " + eventTime + "\n"
+              + "(Supressing subsequent error messages of this type)");
+        this.printedOutOfDataWarning = true;
+    }
+
     }
 
@@ -230 +260 @@
 
 
+FrequencyPlotter.prototype.hasStoredSelfSimilarityMatrix  = function() {
+    return this.canvas_saved_img != null;
+}
+
+FrequencyPlotter.prototype.plotStoredSelfSimilarityMatrix  = function() {
+
+    this.ctx.drawImage(this.canvas_saved_img,0,0);
+    //var img_data    = canvas.toDataURL("image/png");
+}
+
 
 FrequencyPlotter.prototype.plotSelfSimilarityMatrix  = function(ssm,pausePerc) {
@@ -236 +276 @@
     var num_frames = ssm.length;
 
+    console.log("plotSelfSimilarityMatrex: num_frames = " + num_frames + ", pausePerc = " + pausePerc);
     var activeWidth  = this.fudgeWidth * pausePerc;
     var activeHeight = this.height * pausePerc;
@@ -241 +282 @@
     var x_scale = activeWidth / num_frames;
     var y_scale = activeHeight / num_frames;
-
     var x_width  = Math.ceil(x_scale);
     var y_height = Math.ceil(y_scale);
@@ -282 +322 @@
     else {
         outerThis.progress = 100;
+
+        if ((jsMadProcessing == null) ||
+        ((jsMadProcessing != null) && jsMadProcessing.processingFinished())) {
+
+        var canvas = document.getElementById("freq-plot" );
+        var img_data    = canvas.toDataURL("image/png");
+
+        outerThis.canvas_saved_img = document.createElement("img");
+        outerThis.canvas_saved_img.src = img_data;
+
+
+        var image_server_url = "http://localhost:8181/greenstone3/cgi-bin/image-server.pl";
+
+        var canvas_save_data = { a: "save", 
+                     d: gs.cgiParams.d, 
+                     c: gs.cgiParams.c, 
+                     site: gs.xsltParams.site_name, 
+                     imagedata: img_data };
+
+        $.post(image_server_url, canvas_save_data).done(function( data ) {
+            console.log( "Image data saved " );
+        });
+/*
+        $.ajax({
+            type: "POST",
+            url: url,
+            data: { "a" : "save" }
+            success: success,
+            dataType: dataType
+        });
+*/
+
+        console.log("Done save");
+
+        }   
     }
     })(this);

gs3-extensions/web-audio/trunk/js-mad/script/main-jsmad.js

@@ -26 +26 @@
     this.playBufferStore = new Array();
 
+/*
+    // default is to operate as fast as possible
+    if (BrowserDetect.browser == "Firefox") {
+    this.speedup_batch_limit = 100;
+    this.speedup_pause = 0;
+    }
+    else {
+    // e.g. Chrome 
+    // => the following values played more nicely with letting DSP playback operate straightaway 
+    //    i.e. when jsmad is still working its way through the frames
+    
+    this.speedup_batch_limit = 10;
+    this.speedup_pause = 2;
+
+    //speedup_batch_limit = 100;
+    //speedup_pause = 0;
+
+    //console.log("*** Running JsMadProcessing with: batch limit = " + speedup_batch_limit 
+    //      + ", pause = " + this.speedup_pause + " msecs");
+    }
+*/
+
+    // Assume slow, unless the user explicitly activates "speed up"
+    this.speedup_batch_limit = 10;
+    this.speedup_pause = 1000;
+
+    this.processing_finished = false;
+
     stream.requestAbsolute(1 * 1024, function() { that.streamCallback(stream) });
 
 }
 
-
-
+JsMadProcessing.prototype.setSpeedupParams = function(new_batch_limit,new_pause) {
+    this.speedup_batch_limit = new_batch_limit;
+    this.speedup_pause       = new_pause;
+}
+
+
+JsMadProcessing.prototype.processingFinished = function() {
+    return this.processing_finished;
+}
 
 JsMadProcessing.prototype.streamCallback = function(stream) {
@@ -66 +101 @@
     console.log("*** main-jsmad.js::stream_callback(): hardwiring workflow buffer size to 2048");
 
-    var speedup_batch_limit;
-    var speedup_pause;
-    
-    if (BrowserDetect.browser == "Firefox") {
-    speedup_batch_limit = 100;
-    speedup_pause = 0;
-    }
-    else {
-    // e.g. Chrome 
-    // => the following values played more nicely with letting DSP playback operate straightaway 
-    //    i.e. when jsmad is still working its way through the frames
-    
-    speedup_batch_limit = 10;
-    speedup_pause = 2;
-
-    //speedup_batch_limit = 100;
-    //speedup_pause = 0;
-
-    console.log("*** Running JsMadProcessing with: batch limit = " + speedup_batch_limit 
-            + ", pause = " + speedup_pause + " msecs");
-    }
+    var $acc_proc_status = $('#accProcStatus') || null;
 
 
@@ -96 +111 @@
         var speedup_i = 0;
 
-        while (speedup_i < speedup_batch_limit) { // Decode 100 frames at a time, before setTimeout break
+        while (speedup_i < that.speedup_batch_limit) { // Decode 100 frames at a time, before setTimeout break
 
         frame = Mad.Frame.decode(frame, mpeg);
@@ -169 +184 @@
         speedup_i++;
         i++;
+
+
+        if ((i%10)==0) {
+            if ($acc_proc_status != null) {
+            $acc_proc_status.text(" (decoded " + i + " frames)");
+            }
+        }
+        
         }
         
@@ -175 +198 @@
         if (err == "Header decoding failed") {
         console.info("JsMadProcessing::streamCallack() decoded " + i + " frames");
+        $acc_proc_status.text(" (done)");
+        that.processing_finished = true;
         }
         else {
@@ -183 +208 @@
 
     if (more_frames) {
-        setTimeout(arguments.callee,speedup_pause); // even a delay of '0' gives the browser thread a break
+        setTimeout(arguments.callee,that.speedup_pause); // even a delay of '0' gives the browser thread a break
     }
     else {
@@ -194 +219 @@
         //console.log("Computed song duration: " + song_duration + " secs");
         console.info("JsMad processing done in " + speedup_duration + " secs (a speed factor of " + speedup_factor + ")");
+
+        $acc_proc_status.text(" (done)");
+        that.processing_finished = true;
+
     }