Contents tagged with HTML

  • Speech Recognition in ASP.NET

    Speech synthesis and recognition were both introduced in .NET 3.0. They both live in System.Speech.dll. In the past, I already talked about speech synthesis in the context of ASP.NET Web Form applications, this time, I’m going to talk about speech recognition.

    .NET has in fact two APIs for that:

    I am going to demonstrate a technique that makes use of HTML5 features, namely, Data URIs and the getUserMedia API and also ASP.NET Client Callbacks, which, if you have been following my blog, should know that I am a big fan of.

    First, because we have two APIs that we can use, let’s start by creating an abstract base provider class:

       1: public abstract class SpeechRecognitionProvider : IDisposable
       2: {
       3:     protected SpeechRecognitionProvider(SpeechRecognition recognition)
       4:     {
       5:         this.Recognition = recognition;
       6:     }
       7:  
       8:     ~SpeechRecognitionProvider()
       9:     {
      10:         this.Dispose(false);
      11:     }
      12:  
      13:     public abstract SpeechRecognitionResult RecognizeFromWav(String filename);
      14:  
      15:     protected SpeechRecognition Recognition
      16:     {
      17:         get;
      18:         private set;
      19:     }
      20:  
      21:     protected virtual void Dispose(Boolean disposing)
      22:     {
      23:     }
      24:  
      25:     void IDisposable.Dispose()
      26:     {
      27:         GC.SuppressFinalize(this);
      28:         this.Dispose(true);
      29:     }
      30: }

    It basically features one method, RecognizeFromWav, which takes a physical path and returns a SpeechRecognitionResult (code coming next). For completeness, it also implements the Dispose Pattern, because some provider may require it.

    In a moment we will be creating implementations for the built-in .NET provider as well as Microsoft Speech Platform.

    The SpeechRecognition property refers to our Web Forms control, inheriting from HtmlGenericControl, which is the one that knows how to instantiate one provider or the other:

       1: [ConstructorNeedsTag(false)]
       2: public class SpeechRecognition : HtmlGenericControl, ICallbackEventHandler
       3: {
       4:     public SpeechRecognition() : base("span")
       5:     {
       6:         this.OnClientSpeechRecognized = String.Empty;
       7:         this.Mode = SpeechRecognitionMode.Desktop;
       8:         this.Culture = String.Empty;
       9:         this.SampleRate = 44100;
      10:         this.Grammars = new String[0];
      11:         this.Choices = new String[0];
      12:     }
      13:  
      14:     public event EventHandler<SpeechRecognizedEventArgs> SpeechRecognized;
      15:  
      16:     [DefaultValue("")]
      17:     public String Culture
      18:     {
      19:         get;
      20:         set;
      21:     }
      22:  
      23:     [DefaultValue(SpeechRecognitionMode.Desktop)]
      24:     public SpeechRecognitionMode Mode
      25:     {
      26:         get;
      27:         set;
      28:     }
      29:  
      30:     [DefaultValue("")]
      31:     public String OnClientSpeechRecognized
      32:     {
      33:         get;
      34:         set;
      35:     }
      36:  
      37:     [DefaultValue(44100)]
      38:     public UInt32 SampleRate
      39:     {
      40:         get;
      41:         set;
      42:     }
      43:  
      44:     [TypeConverter(typeof(StringArrayConverter))]
      45:     [DefaultValue("")]
      46:     public String [] Grammars
      47:     {
      48:         get;
      49:         private set;
      50:     }
      51:  
      52:     [TypeConverter(typeof(StringArrayConverter))]
      53:     [DefaultValue("")]
      54:     public String[] Choices
      55:     {
      56:         get;
      57:         set;
      58:     }
      59:  
      60:     protected override void OnInit(EventArgs e)
      61:     {
      62:         if (this.Page.Items.Contains(typeof(SpeechRecognition)))
      63:         {
      64:             throw (new InvalidOperationException("There can be only one SpeechRecognition control on a page."));
      65:         }
      66:  
      67:         var sm = ScriptManager.GetCurrent(this.Page);
      68:         var reference = this.Page.ClientScript.GetCallbackEventReference(this, "sound", String.Format("function(result){{ {0}(JSON.parse(result)); }}", String.IsNullOrWhiteSpace(this.OnClientSpeechRecognized) ? "void" : this.OnClientSpeechRecognized), String.Empty, true);
      69:         var script = String.Format("\nvar processor = document.getElementById('{0}'); processor.stopRecording = function(sampleRate) {{ window.stopRecording(processor, sampleRate ? sampleRate : 44100); }}; processor.startRecording = function() {{ window.startRecording(); }}; processor.process = function(sound){{ {1} }};\n", this.ClientID, reference);
      70:  
      71:         if (sm != null)
      72:         {
      73:             this.Page.ClientScript.RegisterStartupScript(this.GetType(), String.Concat("process", this.ClientID), String.Format("Sys.WebForms.PageRequestManager.getInstance().add_pageLoaded(function() {{ {0} }});\n", script), true);
      74:         }
      75:         else
      76:         {
      77:             this.Page.ClientScript.RegisterStartupScript(this.GetType(), String.Concat("process", this.ClientID), script, true);
      78:         }
      79:  
      80:         this.Page.ClientScript.RegisterClientScriptResource(this.GetType(), String.Concat(this.GetType().Namespace, ".Script.js"));
      81:         this.Page.Items[typeof(SpeechRecognition)] = this;
      82:  
      83:         base.OnInit(e);
      84:     }
      85:  
      86:     protected virtual void OnSpeechRecognized(SpeechRecognizedEventArgs e)
      87:     {
      88:         var handler = this.SpeechRecognized;
      89:  
      90:         if (handler != null)
      91:         {
      92:             handler(this, e);
      93:         }
      94:     }
      95:  
      96:     protected SpeechRecognitionProvider GetProvider()
      97:     {
      98:         switch (this.Mode)
      99:         {
     100:             case SpeechRecognitionMode.Desktop:
     101:                 return (new DesktopSpeechRecognitionProvider(this));
     102:  
     103:             case SpeechRecognitionMode.Server:
     104:                 return (new ServerSpeechRecognitionProvider(this));
     105:         }
     106:  
     107:         return (null);
     108:     }
     109:  
     110:     #region ICallbackEventHandler Members
     111:  
     112:     String ICallbackEventHandler.GetCallbackResult()
     113:     {
     114:         AsyncOperationManager.SynchronizationContext = new SynchronizationContext();
     115:  
     116:         var filename = Path.GetTempFileName();
     117:         var result = null as SpeechRecognitionResult;
     118:  
     119:         using (var engine = this.GetProvider())
     120:         {
     121:             var data = this.Context.Items["data"].ToString();
     122:  
     123:             using (var file = File.OpenWrite(filename))
     124:             {
     125:                 var bytes = Convert.FromBase64String(data);
     126:                 file.Write(bytes, 0, bytes.Length);
     127:             }
     128:  
     129:             result = engine.RecognizeFromWav(filename) ?? new SpeechRecognitionResult(String.Empty);
     130:         }
     131:  
     132:         File.Delete(filename);
     133:  
     134:         var args = new SpeechRecognizedEventArgs(result);
     135:  
     136:         this.OnSpeechRecognized(args);
     137:  
     138:         var json = new JavaScriptSerializer().Serialize(result);
     139:  
     140:         return (json);
     141:     }
     142:  
     143:     void ICallbackEventHandler.RaiseCallbackEvent(String eventArgument)
     144:     {
     145:         this.Context.Items["data"] = eventArgument;
     146:     }
     147:  
     148:     #endregion
     149: }

    SpeechRecognition implements ICallbackEventHandler for a self-contained AJAX experience; it registers a couple of JavaScript functions and also an embedded JavaScript file, for some useful sound manipulation and conversion. Only one instance is allowed on a page. On the client-side, this JavaScript uses getUserMedia to access an audio source and uses a clever mechanism to pack them as a .WAV file in a Data URI. I got these functions from http://typedarray.org/from-microphone-to-wav-with-getusermedia-and-web-audio/ and made some changes to them. I like them because they don’t require any external library, which makes all this pretty much self-contained.

    The control exposes some custom properties:

    • Culture: an optional culture name, such as “pt-PT” or “en-US”; if not specified, it defaults to the current culture in the server machine;
    • Mode: one of the two providers: Desktop (for System.Speech) or Server (for Microsoft.Speech, of Microsoft Speech Platform);
    • OnClientSpeechRecognized: the name of a callback JavaScript version that will be called when there are results (more on this later);
    • SampleRate: a sample rate, by default, it is 44100;
    • Grammars: an optional collection of additional grammar files, with extension .grxml (Speech Recognition Grammar Specification), to add to the engine;
    • Choices: an optional collection of choices to recognize, if we want to restrict the scope, such as “yes”/”no”, “red”/”green”, etc.

    The mode enumeration looks like this:

       1: public enum SpeechRecognitionMode
       2: {
       3:     Desktop,
       4:     Server
       5: }

    The SpeechRecognition control also has an event, SpeechRecognized, which allows overriding the detected phrases. Its argument is this simple class that follows the regular .NET event pattern:

       1: [Serializable]
       2: public sealed class SpeechRecognizedEventArgs : EventArgs
       3: {
       4:     public SpeechRecognizedEventArgs(SpeechRecognitionResult result)
       5:     {
       6:         this.Result = result;
       7:     }
       8:  
       9:     public SpeechRecognitionResult Result
      10:     {
      11:         get;
      12:         private set;
      13:     }
      14: }

    Which in turn holds a SpeechRecognitionResult:

       1: public class SpeechRecognitionResult
       2: {
       3:     public SpeechRecognitionResult(String text, params String [] alternates)
       4:     {
       5:         this.Text = text;
       6:         this.Alternates = alternates.ToList();
       7:     }
       8:  
       9:     public String Text
      10:     {
      11:         get;
      12:         set;
      13:     }
      14:  
      15:     public List<String> Alternates
      16:     {
      17:         get;
      18:         private set;
      19:     }
      20: }

    This class receives the phrase that the speech recognition engine understood plus an array of additional alternatives, in descending order.

    The JavaScript file containing the utility functions is embedded in the assembly:

    image

    You need to add an assembly-level attribute, WebResourceAttribute, possibly in AssemblyInfo.cs, of course, replacing MyNamespace for your assembly’s default namespace:

       1: [assembly: WebResource("MyNamespace.Script.js", "text/javascript")]

    This attribute registers a script file with some content-type so that it can be included in a page by the RegisterClientScriptResource method.

    And here is it:

       1: // variables
       2: var leftchannel = [];
       3: var rightchannel = [];
       4: var recorder = null;
       5: var recording = false;
       6: var recordingLength = 0;
       7: var volume = null;
       8: var audioInput = null;
       9: var audioContext = null;
      10: var context = null;
      11:  
      12: // feature detection 
      13: navigator.getUserMedia = navigator.getUserMedia || navigator.webkitGetUserMedia || navigator.mozGetUserMedia || navigator.msGetUserMedia;
      14:  
      15: if (navigator.getUserMedia)
      16: {
      17:     navigator.getUserMedia({ audio: true }, onSuccess, onFailure);
      18: }
      19: else
      20: {
      21:     alert('getUserMedia not supported in this browser.');
      22: }
      23:  
      24: function startRecording()
      25: {
      26:     recording = true;
      27:     // reset the buffers for the new recording
      28:     leftchannel.length = rightchannel.length = 0;
      29:     recordingLength = 0;
      30:     leftchannel = [];
      31:     rightchannel = [];
      32: }
      33:  
      34: function stopRecording(elm, sampleRate)
      35: {
      36:     recording = false;
      37:  
      38:     // we flat the left and right channels down
      39:     var leftBuffer = mergeBuffers(leftchannel, recordingLength);
      40:     var rightBuffer = mergeBuffers(rightchannel, recordingLength);
      41:     // we interleave both channels together
      42:     var interleaved = interleave(leftBuffer, rightBuffer);
      43:  
      44:     // we create our wav file
      45:     var buffer = new ArrayBuffer(44 + interleaved.length * 2);
      46:     var view = new DataView(buffer);
      47:  
      48:     // RIFF chunk descriptor
      49:     writeUTFBytes(view, 0, 'RIFF');
      50:     view.setUint32(4, 44 + interleaved.length * 2, true);
      51:     writeUTFBytes(view, 8, 'WAVE');
      52:     // FMT sub-chunk
      53:     writeUTFBytes(view, 12, 'fmt ');
      54:     view.setUint32(16, 16, true);
      55:     view.setUint16(20, 1, true);
      56:     // stereo (2 channels)
      57:     view.setUint16(22, 2, true);
      58:     view.setUint32(24, sampleRate, true);
      59:     view.setUint32(28, sampleRate * 4, true);
      60:     view.setUint16(32, 4, true);
      61:     view.setUint16(34, 16, true);
      62:     // data sub-chunk
      63:     writeUTFBytes(view, 36, 'data');
      64:     view.setUint32(40, interleaved.length * 2, true);
      65:  
      66:     // write the PCM samples
      67:     var index = 44;
      68:     var volume = 1;
      69:  
      70:     for (var i = 0; i < interleaved.length; i++)
      71:     {
      72:         view.setInt16(index, interleaved[i] * (0x7FFF * volume), true);
      73:         index += 2;
      74:     }
      75:  
      76:     // our final binary blob
      77:     var blob = new Blob([view], { type: 'audio/wav' });
      78:  
      79:     var reader = new FileReader();
      80:     reader.onloadend = function ()
      81:     {
      82:         var url = reader.result.replace('data:audio/wav;base64,', '');
      83:         elm.process(url);
      84:     };
      85:     reader.readAsDataURL(blob);
      86: }
      87:  
      88: function interleave(leftChannel, rightChannel)
      89: {
      90:     var length = leftChannel.length + rightChannel.length;
      91:     var result = new Float32Array(length);
      92:     var inputIndex = 0;
      93:  
      94:     for (var index = 0; index < length;)
      95:     {
      96:         result[index++] = leftChannel[inputIndex];
      97:         result[index++] = rightChannel[inputIndex];
      98:         inputIndex++;
      99:     }
     100:  
     101:     return result;
     102: }
     103:  
     104: function mergeBuffers(channelBuffer, recordingLength)
     105: {
     106:     var result = new Float32Array(recordingLength);
     107:     var offset = 0;
     108:  
     109:     for (var i = 0; i < channelBuffer.length; i++)
     110:     {
     111:         var buffer = channelBuffer[i];
     112:         result.set(buffer, offset);
     113:         offset += buffer.length;
     114:     }
     115:  
     116:     return result;
     117: }
     118:  
     119: function writeUTFBytes(view, offset, string)
     120: {
     121:     for (var i = 0; i < string.length; i++)
     122:     {
     123:         view.setUint8(offset + i, string.charCodeAt(i));
     124:     }
     125: }
     126:  
     127: function onFailure(e)
     128: {
     129:     alert('Error capturing audio.');
     130: }
     131:  
     132: function onSuccess(e)
     133: {
     134:     // creates the audio context
     135:     audioContext = (window.AudioContext || window.webkitAudioContext);
     136:     context = new audioContext();
     137:  
     138:     // creates a gain node
     139:     volume = context.createGain();
     140:  
     141:     // creates an audio node from the microphone incoming stream
     142:     audioInput = context.createMediaStreamSource(e);
     143:  
     144:     // connect the stream to the gain node
     145:     audioInput.connect(volume);
     146:  
     147:     /* From the spec: This value controls how frequently the audioprocess event is 
     148:     dispatched and how many sample-frames need to be processed each call. 
     149:     Lower values for buffer size will result in a lower (better) latency. 
     150:     Higher values will be necessary to avoid audio breakup and glitches */
     151:     var bufferSize = 2048;
     152:  
     153:     recorder = context.createScriptProcessor(bufferSize, 2, 2);
     154:     recorder.onaudioprocess = function (e)
     155:     {
     156:         if (recording == false)
     157:         {
     158:             return;
     159:         }
     160:  
     161:         var left = e.inputBuffer.getChannelData(0);
     162:         var right = e.inputBuffer.getChannelData(1);
     163:  
     164:         // we clone the samples
     165:         leftchannel.push(new Float32Array(left));
     166:         rightchannel.push(new Float32Array(right));
     167:  
     168:         recordingLength += bufferSize;
     169:     }
     170:  
     171:     // we connect the recorder
     172:     volume.connect(recorder);
     173:     recorder.connect(context.destination);
     174: }

    OK, let’s move on the the provider implementations; first, Desktop:

       1: public class DesktopSpeechRecognitionProvider : SpeechRecognitionProvider
       2: {
       3:     public DesktopSpeechRecognitionProvider(SpeechRecognition recognition) : base(recognition)
       4:     {
       5:     }
       6:  
       7:     public override SpeechRecognitionResult RecognizeFromWav(String filename)
       8:     {
       9:         var engine = null as SpeechRecognitionEngine;
      10:  
      11:         if (String.IsNullOrWhiteSpace(this.Recognition.Culture) == true)
      12:         {
      13:             engine = new SpeechRecognitionEngine();
      14:         }
      15:         else
      16:         {
      17:             engine = new SpeechRecognitionEngine(CultureInfo.CreateSpecificCulture(this.Recognition.Culture));
      18:         }
      19:  
      20:         using (engine)
      21:         {
      22:             if ((this.Recognition.Grammars.Any() == false) && (this.Recognition.Choices.Any() == false))
      23:             {
      24:                 engine.LoadGrammar(new DictationGrammar());
      25:             }
      26:  
      27:             foreach (var grammar in this.Recognition.Grammars)
      28:             {
      29:                 var doc = new SrgsDocument(Path.Combine(HttpRuntime.AppDomainAppPath, grammar));
      30:                 engine.LoadGrammar(new Grammar(doc));
      31:             }
      32:  
      33:             if (this.Recognition.Choices.Any() == true)
      34:             {
      35:                 var choices = new Choices(this.Recognition.Choices.ToArray());
      36:                 engine.LoadGrammar(new Grammar(choices));
      37:             }
      38:  
      39:             engine.SetInputToWaveFile(filename);
      40:  
      41:             var result = engine.Recognize();
      42:  
      43:             return ((result != null) ? new SpeechRecognitionResult(result.Text, result.Alternates.Select(x => x.Text).ToArray()) : null);
      44:         }
      45:     }
      46: }

    What this provider does is simply receive the location of a .WAV file and feed it to SpeechRecognitionEngine, together with some parameters of SpeechRecognition (Culture, AudioRate, Grammars and Choices)

    Finally, the code for the Server (Microsoft Speech Platform Software Development Kit) version:

       1: public class ServerSpeechRecognitionProvider : SpeechRecognitionProvider
       2: {
       3:     public ServerSpeechRecognitionProvider(SpeechRecognition recognition) : base(recognition)
       4:     {
       5:     }
       6:  
       7:     public override SpeechRecognitionResult RecognizeFromWav(String filename)
       8:     {
       9:         var engine = null as SpeechRecognitionEngine;
      10:  
      11:         if (String.IsNullOrWhiteSpace(this.Recognition.Culture) == true)
      12:         {
      13:             engine = new SpeechRecognitionEngine();
      14:         }
      15:         else
      16:         {
      17:             engine = new SpeechRecognitionEngine(CultureInfo.CreateSpecificCulture(this.Recognition.Culture));
      18:         }
      19:  
      20:         using (engine)
      21:         {
      22:             foreach (var grammar in this.Recognition.Grammars)
      23:             {
      24:                 var doc = new SrgsDocument(Path.Combine(HttpRuntime.AppDomainAppPath, grammar));
      25:                 engine.LoadGrammar(new Grammar(doc));
      26:             }
      27:  
      28:             if (this.Recognition.Choices.Any() == true)
      29:             {
      30:                 var choices = new Choices(this.Recognition.Choices.ToArray());
      31:                 engine.LoadGrammar(new Grammar(choices));
      32:             }
      33:  
      34:             engine.SetInputToWaveFile(filename);
      35:  
      36:             var result = engine.Recognize();
      37:  
      38:             return ((result != null) ? new SpeechRecognitionResult(result.Text, result.Alternates.Select(x => x.Text).ToArray()) : null);
      39:         }
      40:     }
      41: }

    As you can see, it is very similar to the Desktop one. Keep in mind, however, that for this provider to work you will have to download the Microsoft Speech Platform SDK, the Microsoft Speech Platform Runtime and at least one language from the Language Pack.

    Here is a sample markup declaration:

       1: <web:SpeechRecognition runat="server" ID="processor" ClientIDMode="Static" Mode="Desktop" Culture="en-US" OnSpeechRecognized="OnSpeechRecognized" OnClientSpeechRecognized="onSpeechRecognized" />

    If you want to add specific choices, add the Choices attribute to the control declaration and separate the values by commas:

       1: Choices="One, Two, Three"

    Or add a grammar file:

       1: Grammars="~/MyGrammar.grxml"

    By the way, grammars are not so difficult to create, you can find a good documentation in MSDN: http://msdn.microsoft.com/en-us/library/ee800145.aspx.

    To finalize, a sample JavaScript for starting recognition and receiving the results:

       1: <script type="text/javascript">
       1:  
       2:     
       3:     function onSpeechRecognized(result)
       4:     {
       5:         window.alert('Recognized: ' + result.Text + '\nAlternatives: ' + String.join(', ', result.Alternatives));
       6:     }
       7:  
       8:     function start()
       9:     {
      10:         document.getElementById('processor').startRecording();
      11:     }
      12:  
      13:     function stop()
      14:     {
      15:         document.getElementById('processor').stopRecording();
      16:     }
      17:  
    </script>

    And that’s it. You start recognition by calling startRecording(), get results in onSpeechRecognized() (or any other function set in the OnClientSpeechRecognized property) and stop recording with stopRecording(). The values passed to onSpeechRecognized() are those that may have been filtered by the server-side SpeechRecognized event handler.

    A final word of advisory: because generated sound files may become very large, do keep the recordings as short as possible.

    Of course, this offers several different possibilities, I am looking forward to hearing them from you! Winking smile

    Read more...

  • Video Streaming with ASP.NET SignalR and HTML5

    I have already talked about SignalR in this blog. I think it is one of the most interesting technologies that Microsoft put out recently, not because it is something substantially new – AJAX, long polling and server-sent events have been around for quite some time -, but because of how easy and extensive they made it.

    Most of the examples of SignalR usually are about chat. You know that I have been digging into HTML5 lately, and I already posted on media acquisition using HTML5’s getUserMedia API. This time, however, I’m going to talk about video streaming!

    I wrote a simple ASP.NET Web Forms control that leverages SignalR to build a real-time communication channel, and this channel transmits images as Data URIs. The source of the feed comes from getUserMedia and is therefore available in all modern browsers, except, alas, one that shall remain unnamed. Any browser, however, can be used to display the streaming feed.

    So, the architecture is quite simple:

    • One SignalR Hub for the communication channel;
    • One ASP.NET Web Forms control, that renders an HTML5 VIDEO tag that is used to display the video being acquired, on any compatible browser.

    And that’s it. Of course, the page where we are hosting the control needs to have references to the required JavaScript files (currently jQuery >= 1.6.4 and jQuery-SignalR 2.0.3), these are included automatically when you add the Nuget package.

    Here are some nice screenshots of my now famous home wall, where one of the browser instances, Chrome, is broadcasting to Firefox, Opera and Safari. Unfortunately, IE is not supported.

    image

    So, all we need is a control declaration on an ASP.NET Web Forms page, which could look like this:

       1: <web:VideoStreaming runat="server" ID="video" ClientIDMode="Static" Width="300px" Height="300px" Interval="100" Source="True" ScalingMode="TargetSize" StreamingMode="Target" TargetClientID="received" OnStreamed="onStreamed" Style="border: solid 1px black" />

    The non-ordinary properties that the VideoStreaming control supports are:

    • Interval: the rate in milliseconds that the control broadcasts that the video is being broadcast; do not set it to less than 200, from my experience, it will not work well;
    • OnStreamed: the name of a JavaScript callback function to call which will receive the streamed image as a Data URI when the StreamingMode is set to Event, not used in any of the other modes;
    • ScalingMode: how the image is scaled; possible values are None, OriginalSize, TargetSize (the size of the target CANVAS or IMG element when using StreamingMode Target) and ControlSize (the size of the VideoStreaming control);
    • Source: if the control is set as a streaming source, or just as a receiver (default);
    • StreamingMode: the possible streaming options: None, does not do anything, Target, sends the image directly to a target CANVAS or IMG element; Event, raises a JavaScript event with the streamed image; and Window, opens up a new browser window which displays the images;
    • TargetClientID: the id of a target CANVAS or IMG element that will receive the images as they are streamed, when using StreamingMode Target, not used in any of the other modes.

    Here is an example of a JavaScript function that serves as the callback function for the Event streaming mode:

       1: function onStreamed(imageUrl, imageWidth, imageHeight)
       2: {
       3:     //for StreamingMode="Event", draw an image on an existing canvas
       4:     //the onload event is for cross-browser compatibility
       5:     //in this example, we are using the canvas width and height
       6:     var canvas = document.getElementById('received');
       7:     var ctx = canvas.getContext('2d');
       8:     var img = new Image();
       9:     img.src = imageUrl;
      10:     img.onload = function()
      11:     {
      12:         ctx.drawImage(img, 0, 0, canvas.width, canvas.height);
      13:     }
      14: }

    And a simple CANVAS element for Target:

       1: <canvas id="received" width="300" height="300" style="border: solid 1px black"></canvas>

    The reason for the two main streaming modes, Target and Event are flexibility: with Target, you directly draw the streamed picture on a CANVAS or IMG (it will detect automatically what to use), which needs to be already present, and with Event, you can do your own custom processing.

    The source for the VideoStreaming control is this:

       1: [assembly: OwinStartup(typeof(VideoStreaming))]
       2:  
       3: public class VideoStreaming : WebControl
       4: {
       5:     public const String Url = "/videostreaming";
       6:  
       7:     public VideoStreaming() : base("video")
       8:     {
       9:         this.Interval = 100;
      10:         this.OnStreamed = String.Empty;
      11:         this.ScalingMode = VideoScalingMode.None;
      12:         this.Source = false;
      13:         this.StreamingMode = VideoStreamingMode.Event;
      14:         this.TargetClientID = String.Empty;        
      15:     }
      16:  
      17:     [DefaultValue(false)]
      18:     public Boolean Source
      19:     {
      20:         get;
      21:         set;
      22:     }
      23:  
      24:     [DefaultValue(VideoStreamingMode.Event)]
      25:     public VideoStreamingMode StreamingMode
      26:     {
      27:         get;
      28:         set;
      29:     }
      30:  
      31:     [DefaultValue(100)]
      32:     public Int32 Interval
      33:     {
      34:         get;
      35:         set;
      36:     }
      37:  
      38:     [DefaultValue("")]
      39:     public String OnStreamed
      40:     {
      41:         get;
      42:         set;
      43:     }
      44:  
      45:     [DefaultValue("")]
      46:     public String TargetClientID
      47:     {
      48:         get;
      49:         set;
      50:     }
      51:  
      52:     [DefaultValue(VideoScalingMode.None)]
      53:     public VideoScalingMode ScalingMode
      54:     {
      55:         get;
      56:         set;
      57:     }
      58:  
      59:     public static void Configuration(IAppBuilder app)
      60:     {
      61:         app.MapSignalR(Url, new HubConfiguration());
      62:     }
      63:     
      64:     protected override void OnLoad(EventArgs e)
      65:     {            
      66:         var sm = ScriptManager.GetCurrent(this.Page);
      67:         var streamingAction = String.Empty;
      68:         var size = (this.ScalingMode == VideoScalingMode.OriginalSize) ? ", imageWidth, imageHeight" : (this.ScalingMode == VideoScalingMode.TargetSize) ? ", canvas.width, canvas.height" : (this.ScalingMode == VideoScalingMode.ControlSize) ? String.Format(", {0}, {1}", this.Width.Value, this.Height.Value) : String.Empty;
      69:  
      70:         switch (this.StreamingMode)
      71:         {
      72:             case VideoStreamingMode.Event:
      73:                 if (String.IsNullOrWhiteSpace(this.OnStreamed) == true)
      74:                 {
      75:                     throw (new InvalidOperationException("OnStreamed cannot be empty when using streaming mode Event"));
      76:                 }
      77:                 streamingAction = String.Format("{0}(imageUrl, imageWidth, imageHeight)", this.OnStreamed);
      78:                 break;
      79:  
      80:             case VideoStreamingMode.Target:
      81:                 if (String.IsNullOrWhiteSpace(this.TargetClientID) == true)
      82:                 {
      83:                     throw (new InvalidOperationException("TargetClientID cannot be empty when using streaming mode Target"));
      84:                 }
      85:                 streamingAction = String.Format("var canvas = document.getElementById('{0}'); if (canvas.tagName == 'CANVAS') {{ var ctx = canvas.getContext('2d'); var img = new Image(); }} else if (canvas.tagName == 'IMG') {{ var img = canvas; }}; img.src = imageUrl; img.width = imageWidth; img.height = imageHeight; if (typeof(ctx) != 'undefined') {{ img.onload = function() {{\n ctx.drawImage(img, 0, 0{1}); \n}} }};", this.TargetClientID, size);
      86:                 break;
      87:  
      88:             case VideoStreamingMode.Window:
      89:                 streamingAction = String.Format("if (typeof(window.videoWindow) == 'undefined') {{ window.videoWindow = window.open(imageUrl, '_blank', 'width=imageWidth,height=imageHeight'); }} else {{ window.videoWindow.location.href = imageUrl; }};");
      90:                 break;
      91:         }
      92:  
      93:         var initScript = String.Format("\ndocument.getElementById('{0}').connection = $.hubConnection('{1}', {{ useDefaultPath: false }}); document.getElementById('{0}').proxy = document.getElementById('{0}').connection.createHubProxy('videoStreamingHub');\n", this.ClientID, Url);
      94:         var startStreamScript = String.Format("\ndocument.getElementById('{0}').startStream = function(){{\n var video = document.getElementById('{0}'); video.proxy.on('send', function(imageUrl, imageWidth, imageHeight) {{\n {2} \n}}); video.connection.start().done(function() {{\n if ((true == {3}) && (video.paused == true) && (video.src == '')) {{\n navigator.getUserMedia = (navigator.getUserMedia || navigator.webkitGetUserMedia || navigator.mozGetUserMedia); navigator.getUserMedia({{ video: true, audio: false }}, function (stream) {{\n video.src = window.URL.createObjectURL(stream); \n}}, function (error) {{\n debugger; \n}}); \n}}; if (video.intervalId) {{\n window.cancelAnimationFrame(video.intervalId); \n}}; var fn = function(time) {{\nif (time >= {1} && video.intervalId != 0) {{ var canvas = document.createElement('canvas'); var context = canvas.getContext('2d'); context.drawImage(video, 0, 0, canvas.width, canvas.height); var picture = canvas.toDataURL(); video.proxy.invoke('send', picture, video.videoWidth, video.videoHeight); }}; window.requestAnimationFrame(fn); \n}}; if (true == {3}) {{ video.intervalId = window.requestAnimationFrame(fn); }}; video.play(); \n}}) }}\n", this.ClientID, this.Interval, streamingAction, this.Source.ToString().ToLower());
      95:         var stopStreamScript = String.Format("\ndocument.getElementById('{0}').stopStream = function(){{ var video = document.getElementById('{0}'); if (video.intervalId) {{ window.cancelAnimationFrame(video.intervalId); }}; video.intervalId = 0; video.pause(); video.connection.stop(); }};\n", this.ClientID);
      96:         var script = String.Concat(initScript, startStreamScript, stopStreamScript);
      97:  
      98:         if (sm != null)
      99:         {
     100:             this.Page.ClientScript.RegisterStartupScript(this.GetType(), String.Concat(Url, this.ClientID), String.Format("Sys.WebForms.PageRequestManager.getInstance().add_pageLoaded(function() {{ {0} }});\n", script), true);
     101:         }
     102:         else
     103:         {
     104:             this.Page.ClientScript.RegisterStartupScript(this.GetType(), String.Concat(Url, this.ClientID), script, true);
     105:         }
     106:  
     107:         if (this.Width != Unit.Empty)
     108:         {
     109:             this.Attributes.Add(HtmlTextWriterAttribute.Width.ToString().ToLower(), this.Width.ToString());
     110:         }
     111:  
     112:         if (this.Height != Unit.Empty)
     113:         {
     114:             this.Attributes.Add(HtmlTextWriterAttribute.Height.ToString().ToLower(), this.Height.ToString());
     115:         }
     116:  
     117:         this.Attributes.Remove("autoplay");
     118:         this.Attributes.Remove("controls");
     119:         this.Attributes.Remove("crossorigin");
     120:         this.Attributes.Remove("loop");
     121:         this.Attributes.Remove("mediagroup");
     122:         this.Attributes.Remove("muted");
     123:         this.Attributes.Remove("poster");
     124:         this.Attributes.Remove("preload");
     125:         this.Attributes.Remove("src");
     126:  
     127:         base.OnLoad(e);
     128:     }
     129: }

    In essence, it is very simple, although it has some nasty inline JavaScript. Basically, what it does is:

    • Renders a VIDEO tag element with the specified Width and Height;
    • Register a SignalR hub;
    • Attaches some JavaScript methods to the generated HTML VIDEO tag (startStream, stopStream);
    • Removes any eventual VIDEO-related attribute that may be present on the control declaration;
    • It uses requestAnimationFrame for periodically (every Interval in milliseconds) broadcasting the image obtained from getUserMedia and drawn on the rendered VIDEO tag as a Data URI to the SignalR hub.

    Of course, we now need the hub code, which is as simple as it could be, just a method that takes the image as a Data URI and its original dimensions and broadcasts it to the world:

       1: public class VideoStreamingHub : Hub
       2: {
       3:     public void Send(String imageUrl, Int32 imageWidth, Int32 imageHeight)
       4:     {
       5:         this.Clients.All.Send(imageUrl, imageWidth, imageHeight);
       6:     }
       7: }

    And finally the simple enumerations used by the VideoStreaming control:

       1: public enum VideoScalingMode
       2: {
       3:     /// <summary>
       4:     /// No scaling is performed.
       5:     /// </summary>
       6:     None,
       7:  
       8:     /// <summary>
       9:     /// Use the original size, coming from getUserMedia.
      10:     /// </summary>
      11:     OriginalSize,
      12:  
      13:     /// <summary>
      14:     /// Use the target CANVAS size.
      15:     /// </summary>
      16:     TargetSize,
      17:  
      18:     /// <summary>
      19:     /// Use the VideoStreaming control size.
      20:     /// </summary>
      21:     ControlSize
      22: }
      23:  
      24: public enum VideoStreamingMode
      25:     {
      26:         /// <summary>
      27:         /// No streaming.
      28:         /// </summary>
      29:         None,
      30:  
      31:         /// <summary>
      32:         /// Raises a JavaScript event.
      33:         /// </summary>
      34:         Event,
      35:  
      36:         /// <summary>
      37:         /// Draws directly to a target CANVAS.
      38:         /// </summary>
      39:         Target,
      40:  
      41:         /// <summary>
      42:         /// Draws in a new window.
      43:         /// </summary>
      44:         Window
      45:     }

    And there you have it! Just drop the VideoStreaming control on an ASPX page and you’re done! If you want to have it broadcast, you need to set its Source property to true, for example, on the containing page:

       1: protected override void OnInit(EventArgs e)
       2: {
       3:     var source = false;
       4:  
       5:     if (Boolean.TryParse(this.Request.QueryString["Source"], out source) == true)
       6:     {
       7:         this.video.Source = source;
       8:     }
       9:  
      10:     if (source == false)
      11:     {
      12:         //if we are not broadcasting, just hide the VIDEO tag
      13:         this.video.Style[HtmlTextWriterStyle.Display] = "none";
      14:     }
      15:  
      16:     base.OnInit(e);
      17: }

    For starting and stopping streaming, you just call the JavaScript functions startStream and stopStream:

       1: <script type="text/javascript">
       1:  
       2:  
       3:     function startStreaming()
       4:     {
       5:         document.getElementById('video').startStream();
       6:     }
       7:  
       8:     function stopStreaming()
       9:     {
      10:         document.getElementById('video').stopStream();
      11:     }
      12:  
    </script>
       2: <input type="button" value="Start Streaming" onclick="startStreaming()"/>
       3: <input type="button" value="Stop Streaming" onclick="stopStreaming()" />

     

    That’s it. Very simple video streaming without the need for any plugin or server. Hope you enjoy it!

    Read more...

  • Taking Picture Snapshots with ASP.NET and HTML5

    This is another post on HTML5 and ASP.NET integration. HTML5 brought along with it a number of great JavaScript APIs; one of them is getUserMedia, which is W3C standard and you can get a good description of it in MDN. In a nutshell, it allows access to the PC’s camera and microphone. Unfortunately, but not unexpectedly, it is not supported by any version of Internet Explorer, but Chrome and Firefox have it.

    I wanted to be able to capture a snapshot and to upload it in AJAX style. Of course, if you know me you can guess I wanted this with ASP.NET Web Forms and client callbacks! Smile

    OK, so I came up with this markup:

       1: <web:PictureSnapshot runat="server" ID="picture" Width="400px" Height="400px" ClientIDMode="Static" OnPictureTaken="OnPictureTaken"/>

    The properties and events worth notice are:

    • Width & Height: should be self explanatory;
    • PictureTaken: a handler for a server-side .NET event that is raised when the picture is sent to the server asynchronously.

    The PictureSnapshot control exposes a number of JavaScript methods:

    • startCapture: when called, starts displaying the output of the camera in real time;
    • stopCapture: pauses the update of the camera;
    • takeSnapshot: takes a snapshot of the current camera output (as visible on screen) and sends it asynchronously to the server, raising the PictureTaken event.

    When the startCapture method is called, the PictureSnapshot control (or, better, its underlying VIDEO tag) starts displaying whatever the camera is pointing to (if we so authorize it).

    image

    Here is an example of one of my home walls… yes, I deliberately got out of the way! Smile

    OK, enough talk, the code for PictureSnapshot looks like this:

       1: public class PictureSnapshot : WebControl, ICallbackEventHandler
       2: {
       3:     public PictureSnapshot() : base("video")
       4:     {
       5:     }
       6:  
       7:     public event EventHandler<PictureTakenEventArgs> PictureTaken;
       8:  
       9:     protected override void OnInit(EventArgs e)
      10:     {
      11:         var sm = ScriptManager.GetCurrent(this.Page);
      12:         var reference = this.Page.ClientScript.GetCallbackEventReference(this, "picture", "function(result, context){ debugger; }", String.Empty, "function (error, context) { debugger; }", true);
      13:         var takeSnapshotScript = String.Format("\ndocument.getElementById('{0}').takeSnapshot = function(){{ var video = document.getElementById('{0}'); var canvas = document.createElement('canvas'); canvas.width = video.width; canvas.height = video.height; var context = canvas.getContext('2d'); context.drawImage(video, 0, 0, video.width, video.height); var picture = canvas.toDataURL(); {1} }};\n", this.ClientID, reference);
      14:         var startCaptureScript = String.Format("\ndocument.getElementById('{0}').startCapture = function(){{ var video = document.getElementById('{0}'); if ((video.paused == true) && (video.src == '')) {{ var getMedia = (navigator.getUserMedia || navigator.webkitGetUserMedia || navigator.mozGetUserMedia); debugger; getMedia = getMedia.bind(navigator); getMedia({{ video: true, audio: false }}, function (stream) {{ video.src = window.URL.createObjectURL(stream); }}, function (error) {{ debugger; }}) }}; video.play(); }};\n", this.ClientID);
      15:         var stopCaptureScript = String.Format("\ndocument.getElementById('{0}').stopCapture = function(){{ var video = document.getElementById('{0}'); video.pause(); }};\n", this.ClientID);
      16:         var script = String.Concat(takeSnapshotScript, startCaptureScript, stopCaptureScript);
      17:  
      18:         if (sm != null)
      19:         {
      20:             this.Page.ClientScript.RegisterStartupScript(this.GetType(), String.Concat("snapshot", this.ClientID), String.Format("Sys.WebForms.PageRequestManager.getInstance().add_pageLoaded(function() {{ {0} }});\n", script), true);
      21:         }
      22:         else
      23:         {
      24:             this.Page.ClientScript.RegisterStartupScript(this.GetType(), String.Concat("snapshot", this.ClientID), script, true);
      25:         }
      26:  
      27:         if (this.Width != Unit.Empty)
      28:         {
      29:             this.Attributes.Add(HtmlTextWriterAttribute.Width.ToString().ToLower(), this.Width.ToString());
      30:         }
      31:  
      32:         if (this.Height != Unit.Empty)
      33:         {
      34:             this.Attributes.Add(HtmlTextWriterAttribute.Height.ToString().ToLower(), this.Height.ToString());
      35:         }
      36:  
      37:         this.Attributes.Remove("autoplay");
      38:         this.Attributes.Remove("controls");
      39:         this.Attributes.Remove("crossorigin");
      40:         this.Attributes.Remove("loop");
      41:         this.Attributes.Remove("mediagroup");
      42:         this.Attributes.Remove("muted");
      43:         this.Attributes.Remove("poster");
      44:         this.Attributes.Remove("preload");
      45:         this.Attributes.Remove("src");
      46:  
      47:         base.OnInit(e);
      48:     }
      49:  
      50:     protected virtual void OnPictureTaken(PictureTakenEventArgs e)
      51:     {
      52:         var handler = this.PictureTaken;
      53:  
      54:         if (handler != null)
      55:         {
      56:             handler(this, e);
      57:         }
      58:     }
      59:  
      60:     #region ICallbackEventHandler Members
      61:  
      62:     String ICallbackEventHandler.GetCallbackResult()
      63:     {
      64:         return (String.Empty);
      65:     }
      66:  
      67:     void ICallbackEventHandler.RaiseCallbackEvent(String eventArgument)
      68:     {
      69:         this.OnPictureTaken(new PictureTakenEventArgs(eventArgument));
      70:     }
      71:  
      72:     #endregion
      73: }

    As you can see, it inherits from WebControl. This is the simplest class in ASP.NET that allows me to output the tag I want, and also includes some nice stuff (width, height, style, class, etc). Also, it implements ICallbackEventHandler for the client callback stuff.

    You may find it strange that I am setting explicitly the WIDTH and HEIGHT attributes, that is because VIDEO requires it in the tag, not just on the STYLE. In the way, I am removing any CROSSORIGIN, SRC, MUTED, PRELOAD, LOOP, AUTOPLAY, MEDIAGROUP, POSTER and CONTROLS attributes that may be present in the markup, because we don’t really want them.

    The current picture is translated to a Data URI by using a temporary CANVAS, and this is what is sent to the server.

    The PictureTakenEventArgs class:

       1: [Serializable]
       2: public sealed class PictureTakenEventArgs : EventArgs
       3: {
       4:     public PictureTakenEventArgs(String base64Picture)
       5:     {
       6:         var index = base64Picture.IndexOf(',');
       7:         var bytes = Convert.FromBase64String(base64Picture.Substring(index + 1));
       8:  
       9:         using (var stream = new MemoryStream(bytes))
      10:         {
      11:             this.Picture = Image.FromStream(stream);
      12:         }
      13:     }
      14:  
      15:     public Image Picture
      16:     {
      17:         get;
      18:         private set;
      19:     }
      20: }

    And this is how to use this control in JavaScript:

       1: <script type="text/javascript">
       1:  
       2:     
       3:     function takeSnapshot()
       4:     {
       5:         document.getElementById('picture').takeSnapshot();
       6:     }
       7:  
       8:     function startCapture()
       9:     {
      10:         document.getElementById('picture').startCapture();
      11:     }
      12:  
      13:     function stopCapture()
      14:     {
      15:         document.getElementById('picture').stopCapture();
      16:     }
      17:  
    </script>
       2:  
       3: <input type="button" value="Start Capturing" onclick="startCapture()"/>
       4: <input type="button" value="Take Snapshot" onclick="takeSnapshot()" />
       5: <input type="button" value="Stop Capturing" onclick="stopCapture()" />

    Finally, a simple event handler for PictureTaken:

       1: protected void OnPictureTaken(Object sender, PictureTakenEventArgs e)
       2: {
       3:     //do something with e.Picture
       4: }

    Pictures arrive as instances of the Image class, and you can do whatever you want with it.

    As always, feedback is greatly appreciated!

    Read more...

  • Speech Synthesis with ASP.NET and HTML5

    The .NET framework includes the SpeechSynthesizer class which can be used to access the Windows speech synthesis engine. The problem with web applications is, of course, this class runs on the server. Because I wanted a mechanism to have speech synthesis (text-to-speech) fired by JavaScript, without requiring any plugin, I decided to implement one myself.

    Once again, I will be using client callbacks, my out-of-the-box ASP.NET favorite AJAX technique. I will also be using HTML5’s AUDIO tag and Data URIs. What I’m going to do is:

    • Set up a control that renders an AUDIO tag;
    • Add to it a JavaScript function that takes a string parameter and causes a client callback;
    • Generate a voice sound from the passed text parameter on the server and save it into an in-memory stream;
    • Convert the stream’s contents to a Data URI;
    • Return the generated Data URI to the client as the response to the client callback.

    Of course, all of this in cross-browser style (provided your browser knows the AUDIO tag and Data URIs, which all modern browsers do).

    So, first of all, my markup looks like this:

       1: <web:SpeechSynthesizer runat="server" ID="synthesizer" Ssml="false" VoiceName="Microsoft Anna" Age="Adult" Gender="Male" Culture="en-US" Rate="0" Volume="100" />

    As you can see, the SpeechSynthesizer control features a few optional properties:

    • Age: the age for the generated voice (default is the one of the system’s default language);
    • Gender: gender of the generated voice (same default as per Age);
    • Culture: the culture of the generated voice (system default);
    • Rate: the speaking rate, from –10 (fastest) to 10 (slowest), where the default is 0 (normal rate);
    • Ssml: if the text is to be considered SSML or not (default is false);
    • Volume: the volume %, between 0 and 100 (default);
    • VoiceName: the name of a voice that is installed on the server machine.

    The Age, Gender and Culture properties and the VoiceName are exclusive, you either specify one or the other. If you want to know the voices installed on your machine, have a look at the GetInstalledVoices method. If no property is specified, the speech will be synthesized with the operating system’s default (Microsoft Anna on Windows 7, Microsoft Dave, Hazel and Zira on Windows 8, etc). By the way, you can get additional voices, either commercially or for free, just look them up in Google.

    Without further delay, here is the code:

       1: [ConstructorNeedsTag(false)]
       2: public class SpeechSynthesizer : HtmlGenericControl, ICallbackEventHandler
       3: {
       4:     private readonly System.Speech.Synthesis.SpeechSynthesizer synth = new System.Speech.Synthesis.SpeechSynthesizer();
       5:  
       6:     public SpeechSynthesizer() : base("audio")
       7:     {
       8:         this.Age = VoiceAge.NotSet;
       9:         this.Gender = VoiceGender.NotSet;
      10:         this.Culture = CultureInfo.CurrentCulture;
      11:         this.VoiceName = String.Empty;
      12:         this.Ssml = false;
      13:     }
      14:  
      15:     [DefaultValue("")]
      16:     public String VoiceName { get; set; }
      17:  
      18:     [DefaultValue(100)]
      19:     public Int32 Volume { get; set; }
      20:  
      21:     [DefaultValue(0)]
      22:     public Int32 Rate { get; set; }
      23:  
      24:     [TypeConverter(typeof(CultureInfoConverter))]
      25:     public CultureInfo Culture { get; set; }
      26:  
      27:     [DefaultValue(VoiceGender.NotSet)]
      28:     public VoiceGender Gender { get; set; }
      29:  
      30:     [DefaultValue(VoiceAge.NotSet)]
      31:     public VoiceAge Age { get; set; }
      32:  
      33:     [DefaultValue(false)]
      34:     public Boolean Ssml { get; set; }
      35:  
      36:     protected override void OnInit(EventArgs e)
      37:     {
      38:         AsyncOperationManager.SynchronizationContext = new SynchronizationContext();
      39:  
      40:         var sm = ScriptManager.GetCurrent(this.Page);
      41:         var reference = this.Page.ClientScript.GetCallbackEventReference(this, "text", String.Format("function(result){{ document.getElementById('{0}').src = result; document.getElementById('{0}').play(); }}", this.ClientID), String.Empty, true);
      42:         var script = String.Format("\ndocument.getElementById('{0}').speak = function(text){{ {1} }};\n", this.ClientID, reference);
      43:  
      44:         if (sm != null)
      45:         {
      46:             this.Page.ClientScript.RegisterStartupScript(this.GetType(), String.Concat("speak", this.ClientID), String.Format("Sys.WebForms.PageRequestManager.getInstance().add_pageLoaded(function() {{ {0} }});\n", script), true);
      47:         }
      48:         else
      49:         {
      50:             this.Page.ClientScript.RegisterStartupScript(this.GetType(), String.Concat("speak", this.ClientID), script, true);
      51:         }
      52:  
      53:         base.OnInit(e);
      54:     }
      55:  
      56:     protected override void OnPreRender(EventArgs e)
      57:     {
      58:         this.Attributes.Remove("class");
      59:         this.Attributes.Remove("src");
      60:         this.Attributes.Remove("preload");
      61:         this.Attributes.Remove("loop");
      62:         this.Attributes.Remove("autoplay");
      63:         this.Attributes.Remove("controls");
      64:         
      65:         this.Style[HtmlTextWriterStyle.Display] = "none";
      66:         this.Style[HtmlTextWriterStyle.Visibility] = "hidden";
      67:  
      68:         base.OnPreRender(e);
      69:     }
      70:  
      71:     public override void Dispose()
      72:     {
      73:         this.synth.Dispose();
      74:  
      75:         base.Dispose();
      76:     }
      77:  
      78:     #region ICallbackEventHandler Members
      79:  
      80:     String ICallbackEventHandler.GetCallbackResult()
      81:     {
      82:         using (var stream = new MemoryStream())
      83:         {
      84:             this.synth.Rate = this.Rate;
      85:             this.synth.Volume = this.Volume;
      86:             this.synth.SetOutputToWaveStream(stream);
      87:  
      88:             if (String.IsNullOrWhiteSpace(this.VoiceName) == false)
      89:             {
      90:                 this.synth.SelectVoice(this.VoiceName);
      91:             }
      92:             else
      93:             {
      94:                 this.synth.SelectVoiceByHints(this.Gender, this.Age, 0, this.Culture);                    
      95:             }
      96:  
      97:             if (this.Ssml == false)
      98:             {
      99:                 this.synth.Speak(this.Context.Items["data"] as String);
     100:             }
     101:             else
     102:             {
     103:                 this.synth.SpeakSsml(this.Context.Items["data"] as String);
     104:             }
     105:  
     106:             return (String.Concat("data:audio/wav;base64,", Convert.ToBase64String(stream.ToArray())));
     107:         }
     108:     }
     109:  
     110:     void ICallbackEventHandler.RaiseCallbackEvent(String eventArgument)
     111:     {
     112:         this.Context.Items["data"] = eventArgument;
     113:     }
     114:  
     115:     #endregion
     116: }

    As you can see, the SpeechSynthesizer control inherits from HtmlGenericControl, this is the simplest out-of-the-box class that will allow me to render my tag of choice (in this case, AUDIO); by the way, this class requires that I decorate it with a ConstructorNeedsTagAttribute, but you don’t have to worry about it. It implements ICallbackEventHandler for the client callback mechanism. I make sure that all of AUDIO’s attributes are removed from the output, because I don’t want them around.

    Inside of it, I have an instance of the SpeechSynthesizer class, the one that will be used to do the actual work. Because this class is disposable, I make sure it is disposed at the end of the control’s life cycle. Based on the parameters being supplied, I either call the SelectVoiceByHints or the SelectVoice methods. One thing to note is, we need to set up a synchronization context, because the SpeechSynthesizer works asynchronously, so that we can wait for its result.

    The generated sound will be output to an in-memory buffer and then converted into a WAV Data URI, which is basically a Base64 encoded string with an associated mime-type.

    Finally, on the client-side, all it takes is to set the returned Data URI as the AUDIO SRC property, and that's it.

    A full markup example would be:

       1: <%@ Register Assembly="System.Speech, Version=4.0.0.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35" Namespace="System.Speech" TagPrefix="web" %>
       2: <!DOCTYPE html>
       3: <html xmlns="http://www.w3.org/1999/xhtml">
       4: <head runat="server">
       5:     <script type="text/javascript">
       1:  
       2:         
       3:         function onSpeak(text)
       4:         {
       5:             document.getElementById('synthesizer').speak(text);
       6:         }
       7:  
       8:     
    </script>
       6: </head>
       7: <body>
       8:     <form runat="server">
       9:     <div>
      10:         <web:SpeechSynthesizer runat="server" ID="synthesizer" Age="Adult" Gender="Male" Culture="en-US" Rate="0" Volume="100" />
      11:         <input type="text" id="text" name="text"/>
      12:         <input type="button" value="Speak" onclick="onSpeak(this.form.text.value)"/>
      13:     </div>
      14:     </form>
      15: </body>
      16: </html>

    And that’s it! Have fun with speech on your web apps! Winking smile

    Read more...

  • ASP.NET Upload Panel

    Introduction

    I have been playing recently with HTML5, and one thing that I got to understand really well was the new upload mechanisms available. Specifically, I wanted to understand how SkyOneDrive, Google Drive, Dropbox, etc, all support dropping files from the local machine, and how to use it in an ASP.NET Web Forms (sorry!) project, and I got it!

    So, I want to have a panel (a DIV) that allows dropping files from the local machine; I want to be able to filter these files by their content type, maximum length, or by any custom condition. If the dropped files pass all conditions, they are sent asynchronously (read, AJAX) and raise a server-side event. After that, if they are multimedia files, I can preview them on the client-side.

    This example will use the XMLHttpRequest object and the new FormData and FileReader JavaScript classes and will work on all modern browsers, from IE9 to Firefox, Chrome and Safari.

    image

    Markup

    My markup looks like this:

       1: <web:UploadPanel runat="server" ID="uploadPanel" MaximumFiles="2" MaximumLength="1000000" ContentTypes="image/gif,image/png,image/jpeg" OnPreviewFile="onPreviewFile" OnBeforeUpload="onBeforeUpload" onUploadCanceled="onUploadCanceled" OnUploadComplete="onUploadComplete" OnUploadFailure="onUploadFailure" OnUploadProgress="onUploadProgress" OnUploadSuccess="onUploadSuccess" OnValidationFailure="onValidationFailure" OnUpload="OnUpload" Style="width: 300px; height: 300px; border: solid 1px;" />

    The UploadPanel control inherits from the Panel class, so it can have any of its properties. In this case, I am setting a specific width, height and border, so that it is easier to target.

    Validations

    Out of the box, it supports the following validations:

    • MaximumFiles: The maximum number of files to drop; if not set, any number is accepted;
    • MaximumLength: The maximum length of any individual file, in bytes; if not set, any file size is accepted;
    • ContentTypes: A comma-delimited list of content types; can take a precise content type, such as “image/gif” or a content type part, such as “image/”; if not specified, any content type is accepted.

    There is also an optional JavaScript event, OnBeforeUpload, that can be used to validate the files individually.

    Client-Side Events

    When a file is dropped into the UploadPanel, the following client-side events are raised:

    • OnValidationFailure: Raised whenever any of the built-in validations (maximum files, maximum length and content types) fails, for any of the dropped files;
    • OnBeforeUpload: Raised before the upload starts, and after all built-in validations (maximum files, maximum length and content types) succeed; this gives developers a chance to analyze the files to upload and to optionally cancel the upload, or to add additional custom parameters that will be posted together with the files;
    • OnUploadFailure: Raised if the upload fails for some reason;
    • OnUploadCanceled: Raised if the upload is canceled;
    • OnUploadProgress: Raised possibly several times as the file is being uploaded, providing an indication of the total upload progress;
    • OnUploadSuccess: Raised when the upload terminates successfully;
    • OnUploadComplete: Raised when the upload completes, either successfully or not;
    • OnPreviewFile: Raised for each multimedia file uploaded (images, videos, sound), to allow previewing it on the client-side; the handler function receives the file as a data URL.

    For each you can optionally specify the name of a JavaScript function that handles it. Some examples of all these events:

       1: <script>
       1:  
       2:  
       3:     function onPreviewFile(name, url)
       4:     {
       5:         document.getElementById('preview').src = url;
       6:     }
       7:  
       8:     function onBeforeUpload(event, props)
       9:     {
      10:         //set two custom properties
      11:         props.a = 1;
      12:         props.b = 2;
      13:         return (true);
      14:     }
      15:  
      16:     function onUploadCanceled(event)
      17:     {
      18:         //canceled
      19:     }
      20:  
      21:     function onUploadComplete(event)
      22:     {
      23:         //complete
      24:     }
      25:  
      26:     function onUploadFailure(event)
      27:     {
      28:         //failure
      29:     }
      30:  
      31:     function onUploadProgress(event)
      32:     {
      33:         if (event.lengthComputable)
      34:         {
      35:             var percentComplete = event.loaded / event.total;
      36:         }
      37:     }
      38:  
      39:     function onUploadSuccess(event)
      40:     {
      41:         //success
      42:     }
      43:  
      44:     function onValidationFailure(event, error)
      45:     {
      46:         //error=0: maximum files reached
      47:         //error=1: maximum file length reached
      48:         //error=2: invalid content type
      49:     }
      50:  
    </script>

    Besides these custom events, you can use any of the regular HTML5 events, such as dragdragenter, dragend, dragover, dragstart, dragleave or drop.

    Server-Side Events

    The Upload event takes a parameter of type UploadEventArgs which looks like this:

       1: [Serializable]
       2: public sealed class UploadEventArgs : EventArgs
       3: {
       4:     public UploadEventArgs(HttpFileCollection files, NameValueCollection form)
       5:     {
       6:         this.Files = files;
       7:         this.Response = String.Empty;
       8:         this.Form = form;
       9:     }
      10:  
      11:     public String Response
      12:     {
      13:         get;
      14:         set;
      15:     }
      16:  
      17:     public NameValueCollection Form
      18:     {
      19:         get;
      20:         private set;
      21:     }
      22:  
      23:     public HttpFileCollection Files
      24:     {
      25:         get;
      26:         private set;
      27:     }
      28: }

    This class receives a list of all the files uploaded and also any custom properties assigned on the OnBeforeUpload event. It also allows the return of a response string, that will be received by the OnUploadSuccess or OnUploadComplete client-side events:

       1: protected void OnUpload(Object sender, UploadEventArgs e)
       2: {
       3:     //do something with e.Files
       4:     
       5:     //output the parameters that were received
       6:     e.Response = String.Format("a={0}\nb={1}", e.Form["a"], e.Form["b"]);
       7: }

    Code

    Finally, the code:

       1: public class UploadPanel : Panel, ICallbackEventHandler
       2: {
       3:     private static readonly String [] MultimediaContentTypePrefixes = new String[]{ "image/", "audio/", "video/" };
       4:  
       5:     public UploadPanel()
       6:     {
       7:         this.ContentTypes = new String[0];
       8:         this.OnUploadFailure = String.Empty;
       9:         this.OnUploadSuccess = String.Empty;
      10:         this.OnValidationFailure = String.Empty;
      11:         this.OnBeforeUpload = String.Empty;
      12:         this.OnUploadComplete = String.Empty;
      13:         this.OnUploadProgress = String.Empty;
      14:         this.OnUploadCanceled = String.Empty;
      15:         this.OnPreviewFile = String.Empty;
      16:     }
      17:  
      18:     public event EventHandler<UploadEventArgs> Upload;
      19:  
      20:     [DefaultValue("")]
      21:     public String OnPreviewFile
      22:     {
      23:         get;
      24:         set;
      25:     }
      26:  
      27:     [DefaultValue("")]
      28:     public String OnBeforeUpload
      29:     {
      30:         get;
      31:         set;
      32:     }
      33:  
      34:     [DefaultValue("")]
      35:     public String OnUploadCanceled
      36:     {
      37:         get;
      38:         set;
      39:     }
      40:  
      41:     [DefaultValue("")]
      42:     public String OnUploadProgress
      43:     {
      44:         get;
      45:         set;
      46:     }
      47:  
      48:     [DefaultValue("")]
      49:     public String OnValidationFailure
      50:     {
      51:         get;
      52:         set;
      53:     }
      54:  
      55:     [DefaultValue("")]
      56:     public String OnUploadComplete
      57:     {
      58:         get;
      59:         set;
      60:     }
      61:  
      62:     [DefaultValue("")]
      63:     public String OnUploadSuccess
      64:     {
      65:         get;
      66:         set;
      67:     }
      68:  
      69:     [DefaultValue("")]
      70:     public String OnUploadFailure
      71:     {
      72:         get;
      73:         set;
      74:     }
      75:  
      76:     [DefaultValue(null)]
      77:     public Int32? MaximumLength
      78:     {
      79:         get;
      80:         set;
      81:     }
      82:  
      83:     [DefaultValue(null)]
      84:     public Int32? MaximumFiles
      85:     {
      86:         get;
      87:         set;
      88:     }
      89:  
      90:     [TypeConverter(typeof(StringArrayConverter))]
      91:     public String[] ContentTypes
      92:     {
      93:         get;
      94:         set;
      95:     }
      96:  
      97:     protected override void OnInit(EventArgs e)
      98:     {
      99:         var script = new StringBuilder();            
     100:         script.AppendFormat("document.getElementById('{0}').addEventListener('drop', function(event) {{\n", this.ClientID);
     101:  
     102:         script.Append("if (event.dataTransfer.files.length == 0)\n");
     103:         script.Append("{\n");
     104:         script.Append("event.returnValue = false;\n");
     105:         script.Append("event.preventDefault();\n");
     106:         script.Append("return(false);\n");
     107:         script.Append("}\n");
     108:  
     109:         if (this.MaximumFiles != null)
     110:         {
     111:             script.AppendFormat("if (event.dataTransfer.files.length > {0})\n", this.MaximumFiles.Value);
     112:             script.Append("{\n");
     113:  
     114:             if (String.IsNullOrWhiteSpace(this.OnValidationFailure) == false)
     115:             {
     116:                 script.AppendFormat("{0}(event, 0);\n", this.OnValidationFailure);
     117:             }
     118:  
     119:             script.Append("event.returnValue = false;\n");
     120:             script.Append("event.preventDefault();\n");
     121:             script.Append("return(false);\n");
     122:             script.Append("}\n");
     123:         }
     124:  
     125:         if (this.MaximumLength != null)
     126:         {
     127:             script.Append("var lengthOk = true;\n");
     128:             script.Append("for (var i = 0; i < event.dataTransfer.files.length; ++i)\n");
     129:             script.Append("{\n");
     130:             script.AppendFormat("if (event.dataTransfer.files[i].size > {0})\n", this.MaximumLength.Value);
     131:             script.Append("{\n");
     132:             script.Append("lengthOk = false;\n");
     133:             script.Append("break;\n");
     134:             script.Append("}\n");
     135:             script.Append("}\n");
     136:             script.Append("if (lengthOk == false)\n");
     137:             script.Append("{\n");
     138:  
     139:             if (String.IsNullOrWhiteSpace(this.OnValidationFailure) == false)
     140:             {
     141:                 script.AppendFormat("{0}(event, 1);\n", this.OnValidationFailure);
     142:             }
     143:  
     144:             script.Append("event.returnValue = false;\n");
     145:             script.Append("event.preventDefault();\n");
     146:             script.Append("return(false);\n");
     147:             script.Append("}\n");
     148:         }
     149:  
     150:         if (this.ContentTypes.Any() == true)
     151:         {
     152:             script.Append("for (var i = 0; i < event.dataTransfer.files.length; ++i)\n");
     153:             script.Append("{\n");
     154:             script.Append("var contentTypeOk = false;\n");
     155:  
     156:             script.AppendFormat("if ({0})", String.Join(" || ", this.ContentTypes.Select(x => String.Format("(event.dataTransfer.files[i].type.toLowerCase().indexOf('{0}') == 0)", x.ToLower()))));
     157:             script.Append("{\n");
     158:             script.Append("contentTypeOk = true;\n");
     159:             script.Append("}\n");
     160:  
     161:             script.Append("}\n");
     162:             script.Append("if (contentTypeOk == false)\n");
     163:             script.Append("{\n");
     164:  
     165:             if (String.IsNullOrWhiteSpace(this.OnValidationFailure) == false)
     166:             {
     167:                 script.AppendFormat("{0}(event, 2);\n", this.OnValidationFailure);
     168:             }
     169:  
     170:             script.Append("event.returnValue = false;\n");
     171:             script.Append("event.preventDefault();\n");
     172:             script.Append("return(false);\n");
     173:             script.Append("}\n");
     174:         }
     175:  
     176:         if (String.IsNullOrWhiteSpace(this.OnBeforeUpload) == false)
     177:         {
     178:             script.Append("var props = new Object();\n");
     179:             script.AppendFormat("if ({0}(event, props) === false)\n", this.OnBeforeUpload);
     180:             script.Append("{\n");
     181:             script.Append("event.returnValue = false;\n");
     182:             script.Append("event.preventDefault();\n");
     183:             script.Append("return(false);\n");
     184:             script.Append("}\n");
     185:         }
     186:  
     187:         script.Append("var data = new FormData();\n");
     188:         script.Append("for (var i = 0; i < event.dataTransfer.files.length; ++i)\n");
     189:         script.Append("{\n");
     190:         script.Append("var file = event.dataTransfer.files[i];\n");
     191:         script.Append("data.append('file' + i, file);\n");
     192:  
     193:         if (String.IsNullOrWhiteSpace(this.OnPreviewFile) == false)
     194:         {
     195:             script.AppendFormat("if ({0})", String.Join(" || ", MultimediaContentTypePrefixes.Select(x => String.Format("(file.type.toLowerCase().indexOf('{0}') == 0)", x.ToLower()))));
     196:             script.Append("{\n");
     197:             script.Append("var reader = new FileReader();\n");
     198:             script.Append("reader.onloadend = function(e)\n");
     199:             script.Append("{\n");
     200:             script.AppendFormat("{0}(file.name, reader.result);\n", this.OnPreviewFile);
     201:             script.Append("}\n");
     202:             script.Append("reader.readAsDataURL(file);\n");
     203:             script.Append("}\n");
     204:         }
     205:  
     206:         script.Append("}\n");
     207:         script.AppendFormat("data.append('__CALLBACKID', '{0}');\n", this.UniqueID);
     208:         script.Append("data.append('__CALLBACKPARAM', '');\n");
     209:         script.Append("data.append('__EVENTTARGET', '');\n");
     210:         script.Append("data.append('__EVENTARGUMENT', '');\n");
     211:         script.Append("for (var key in props)\n");
     212:         script.Append("{\n");
     213:         script.Append("data.append(key, props[key]);\n");
     214:         script.Append("}\n");
     215:         script.Append("var xhr = new XMLHttpRequest();\n");
     216:  
     217:         if (String.IsNullOrWhiteSpace(this.OnUploadProgress) == false)
     218:         {
     219:             script.Append("xhr.onprogress = function(e)\n");
     220:             script.Append("{\n");
     221:             script.AppendFormat("{0}(e);\n", this.OnUploadProgress);
     222:             script.Append("}\n");
     223:         }
     224:  
     225:         if (String.IsNullOrWhiteSpace(this.OnUploadCanceled) == false)
     226:         {
     227:             script.Append("xhr.oncancel = function(e)\n");
     228:             script.Append("{\n");
     229:             script.AppendFormat("{0}(e);\n", this.OnUploadCanceled);
     230:             script.Append("}\n");
     231:         }
     232:  
     233:         script.Append("xhr.onreadystatechange = function(e)\n");
     234:         script.Append("{\n");
     235:         script.Append("if ((xhr.readyState == 4) && (xhr.status == 200))\n");
     236:         script.Append("{\n");
     237:         script.AppendFormat("{0}(e);\n", this.OnUploadSuccess);
     238:         script.Append("}\n");
     239:         script.Append("else if ((xhr.readyState == 4) && (xhr.status != 200))\n");
     240:         script.Append("{\n");
     241:         script.AppendFormat("{0}(e);\n", this.OnUploadFailure);
     242:         script.Append("}\n");
     243:         script.Append("if (xhr.readyState == 4)\n");
     244:         script.Append("{\n");
     245:         script.AppendFormat("{0}(e);\n", this.OnUploadComplete);
     246:         script.Append("}\n");
     247:         script.Append("}\n");
     248:         script.AppendFormat("xhr.open('POST', '{0}', true);\n", this.Context.Request.Url.PathAndQuery);
     249:         script.Append("xhr.send(data);\n");
     250:         script.Append("event.returnValue = false;\n");
     251:         script.Append("event.preventDefault();\n");
     252:         script.Append("return (false);\n");
     253:         script.Append("});\n");
     254:  
     255:         if (ScriptManager.GetCurrent(this.Page) == null)
     256:         {
     257:             this.Page.ClientScript.RegisterStartupScript(this.GetType(), String.Concat(this.UniqueID, "drop"), script.ToString(), true);
     258:             this.Page.ClientScript.RegisterStartupScript(this.GetType(), String.Concat(this.UniqueID, "dragenter"), String.Format("document.getElementById('{0}').addEventListener('dragenter', function(event){{ event.returnValue = false; event.preventDefault(); return(false); }});\n", this.ClientID), true);
     259:             this.Page.ClientScript.RegisterStartupScript(this.GetType(), String.Concat(this.UniqueID, "dragover"), String.Format("document.getElementById('{0}').addEventListener('dragover', function(event){{ event.returnValue = false; event.preventDefault(); return(false); }});\n", this.ClientID), true);
     260:         }
     261:         else
     262:         {
     263:             this.Page.ClientScript.RegisterStartupScript(this.GetType(), String.Concat(this.UniqueID, "drop"), String.Format("Sys.WebForms.PageRequestManager.getInstance().add_pageLoaded(function() {{ {0} }});\n", script), true);
     264:             this.Page.ClientScript.RegisterStartupScript(this.GetType(), String.Concat(this.UniqueID, "dragenter"), String.Format("Sys.WebForms.PageRequestManager.getInstance().add_pageLoaded(function() {{ document.getElementById('{0}').addEventListener('dragenter', function(event){{ event.returnValue = false; event.preventDefault(); return(false); }}); }});\n", this.ClientID), true);
     265:             this.Page.ClientScript.RegisterStartupScript(this.GetType(), String.Concat(this.UniqueID, "dragover"), String.Format("Sys.WebForms.PageRequestManager.getInstance().add_pageLoaded(function() {{ document.getElementById('{0}').addEventListener('dragover', function(event){{ event.returnValue = false; event.preventDefault(); return(false); }}); }});\n", this.ClientID), true);
     266:         }
     267:         
     268:         base.OnInit(e);
     269:     }
     270:  
     271:     protected virtual void OnUpload(UploadEventArgs e)
     272:     {
     273:         var handler = this.Upload;
     274:  
     275:         if (handler != null)
     276:         {
     277:             handler(this, e);
     278:         }
     279:     }
     280:  
     281:     #region ICallbackEventHandler Members
     282:  
     283:     String ICallbackEventHandler.GetCallbackResult()
     284:     {
     285:         var args = new UploadEventArgs(this.Context.Request.Files, this.Context.Request.Form);
     286:  
     287:         this.OnUpload(args);
     288:  
     289:         return (args.Response);
     290:     }
     291:  
     292:     void ICallbackEventHandler.RaiseCallbackEvent(String eventArgument)
     293:     {
     294:     }
     295:  
     296:     #endregion
     297: }

    The UploadPanel class inherits from Panel and implements ICallbackEventHandler, for client callbacks. If you are curious, the __CALLBACKID, __CALLBACKPARAM, __EVENTTARGET and __EVENTARGUMENT are required for ASP.NET to detect a request as a callback, but only __CALLBACKID needs to be set with the unique id of the UploadPanel control.

    Conclusion

    HTML5 offers a lot of exciting new features. Stay tuned for some more examples of its integration with ASP.NET!

    Read more...