[28897] | 1 | // -----------------------------------------------------------------------
|
---|
| 2 | // <copyright file="BackgroundRemovalStreamHandler.cs" company="Microsoft">
|
---|
| 3 | // Copyright (c) Microsoft Corporation. All rights reserved.
|
---|
| 4 | // </copyright>
|
---|
| 5 | // -----------------------------------------------------------------------
|
---|
| 6 |
|
---|
| 7 | namespace Microsoft.Samples.Kinect.Webserver.Sensor
|
---|
| 8 | {
|
---|
| 9 | using System;
|
---|
| 10 | using System.Collections.Generic;
|
---|
| 11 | using System.Diagnostics.CodeAnalysis;
|
---|
| 12 | using System.Globalization;
|
---|
| 13 | using System.Linq;
|
---|
| 14 | using System.Text.RegularExpressions;
|
---|
| 15 | using System.Windows;
|
---|
| 16 |
|
---|
| 17 | using Microsoft.Kinect;
|
---|
| 18 | using Microsoft.Kinect.Toolkit.BackgroundRemoval;
|
---|
| 19 | using Microsoft.Samples.Kinect.Webserver.Properties;
|
---|
| 20 | using Microsoft.Samples.Kinect.Webserver.Sensor.Serialization;
|
---|
| 21 |
|
---|
| 22 | /// <summary>
|
---|
| 23 | /// Implementation of ISensorStreamHandler that exposes background removal streams.
|
---|
| 24 | /// </summary>
|
---|
| 25 | [SuppressMessage("Microsoft.Design", "CA1001:TypesThatOwnDisposableFieldsShouldBeDisposable", Justification = "Disposable background removal stream is disposed when sensor is set to null")]
|
---|
| 26 | public class BackgroundRemovalStreamHandler : SensorStreamHandlerBase
|
---|
| 27 | {
|
---|
| 28 | /// <summary>
|
---|
| 29 | /// JSON name of background removal stream.
|
---|
| 30 | /// </summary>
|
---|
| 31 | internal const string BackgroundRemovalStreamName = "backgroundRemoval";
|
---|
| 32 |
|
---|
| 33 | /// <summary>
|
---|
| 34 | /// JSON name for property representing the tracking user id.
|
---|
| 35 | /// </summary>
|
---|
| 36 | internal const string TrackingIdPropertyName = "trackingId";
|
---|
| 37 |
|
---|
| 38 | /// <summary>
|
---|
| 39 | /// JSON name for property representing the background removed color image resolution.
|
---|
| 40 | /// </summary>
|
---|
| 41 | internal const string ResolutionPropertyName = "resolution";
|
---|
| 42 |
|
---|
| 43 | /// <summary>
|
---|
| 44 | /// Regular expression that matches the background removed frame resolution property.
|
---|
| 45 | /// </summary>
|
---|
| 46 | private static readonly Regex BackgroundRemovalResolutionRegex = new Regex(@"^(?i)(\d+)x(\d+)$");
|
---|
| 47 |
|
---|
| 48 | private static readonly KeyValuePair<ColorImageFormat, Size>[] BackgroundRemovalResolutions =
|
---|
| 49 | {
|
---|
| 50 | new KeyValuePair<ColorImageFormat, Size>(ColorImageFormat.RgbResolution640x480Fps30, new Size(640, 480)),
|
---|
| 51 | new KeyValuePair<ColorImageFormat, Size>(ColorImageFormat.RgbResolution1280x960Fps12, new Size(1280, 960))
|
---|
| 52 | };
|
---|
| 53 |
|
---|
| 54 | /// <summary>
|
---|
| 55 | /// Context that allows this stream handler to communicate with its owner.
|
---|
| 56 | /// </summary>
|
---|
| 57 | private readonly SensorStreamHandlerContext ownerContext;
|
---|
| 58 |
|
---|
| 59 | /// <summary>
|
---|
| 60 | /// Serializable background removal stream message, reused as background removed color frames arrive.
|
---|
| 61 | /// </summary>
|
---|
| 62 | private readonly BackgroundRemovalStreamMessage backgroundRemovalStreamMessage = new BackgroundRemovalStreamMessage { stream = BackgroundRemovalStreamName };
|
---|
| 63 |
|
---|
| 64 | /// <summary>
|
---|
| 65 | /// Sensor providing data to background removal stream.
|
---|
| 66 | /// </summary>
|
---|
| 67 | private KinectSensor sensor;
|
---|
| 68 |
|
---|
| 69 | /// <summary>
|
---|
| 70 | /// Entry point for background removal stream functionality.
|
---|
| 71 | /// </summary>
|
---|
| 72 | private BackgroundRemovedColorStream backgroundRemovalStream;
|
---|
| 73 |
|
---|
| 74 | /// <summary>
|
---|
| 75 | /// Id of the user we choose to track.
|
---|
| 76 | /// </summary>
|
---|
| 77 | private int trackingId;
|
---|
| 78 |
|
---|
| 79 | /// <summary>
|
---|
| 80 | /// True if background removal stream is enabled. Background removal stream is disabled by default.
|
---|
| 81 | /// </summary>
|
---|
| 82 | private bool backgroundRemovalStreamIsEnabled;
|
---|
| 83 |
|
---|
| 84 | /// <summary>
|
---|
| 85 | /// The background removed color image format.
|
---|
| 86 | /// </summary>
|
---|
| 87 | private ColorImageFormat colorImageFormat = ColorImageFormat.RgbResolution640x480Fps30;
|
---|
| 88 |
|
---|
| 89 | /// <summary>
|
---|
| 90 | /// Keep track if we're in the middle of processing a background removed color frame.
|
---|
| 91 | /// </summary>
|
---|
| 92 | private bool isProcessingBackgroundRemovedFrame;
|
---|
| 93 |
|
---|
| 94 | /// <summary>
|
---|
| 95 | /// Initializes a new instance of the <see cref="BackgroundRemovalStreamHandler"/> class
|
---|
| 96 | /// and associates it with a context that allows it to communicate with its owner.
|
---|
| 97 | /// </summary>
|
---|
| 98 | /// <param name="ownerContext">
|
---|
| 99 | /// An instance of <see cref="SensorStreamHandlerContext"/> class.
|
---|
| 100 | /// </param>
|
---|
| 101 | internal BackgroundRemovalStreamHandler(SensorStreamHandlerContext ownerContext)
|
---|
| 102 | {
|
---|
| 103 | this.ownerContext = ownerContext;
|
---|
| 104 |
|
---|
| 105 | this.AddStreamConfiguration(BackgroundRemovalStreamName, new StreamConfiguration(this.GetProperties, this.SetProperty));
|
---|
| 106 | }
|
---|
| 107 |
|
---|
| 108 | /// <summary>
|
---|
| 109 | /// Lets ISensorStreamHandler know that Kinect Sensor associated with this stream
|
---|
| 110 | /// handler has changed.
|
---|
| 111 | /// </summary>
|
---|
| 112 | /// <param name="newSensor">
|
---|
| 113 | /// New KinectSensor.
|
---|
| 114 | /// </param>
|
---|
| 115 | public override void OnSensorChanged(KinectSensor newSensor)
|
---|
| 116 | {
|
---|
| 117 | if (this.sensor != null)
|
---|
| 118 | {
|
---|
| 119 | try
|
---|
| 120 | {
|
---|
| 121 | this.backgroundRemovalStream.BackgroundRemovedFrameReady -= this.BackgroundRemovedFrameReadyAsync;
|
---|
| 122 | this.backgroundRemovalStream.Dispose();
|
---|
| 123 | this.backgroundRemovalStream = null;
|
---|
| 124 |
|
---|
| 125 | this.sensor.ColorStream.Disable();
|
---|
| 126 | }
|
---|
| 127 | catch (InvalidOperationException)
|
---|
| 128 | {
|
---|
| 129 | // KinectSensor might enter an invalid state while enabling/disabling streams or stream features.
|
---|
| 130 | // E.g.: sensor might be abruptly unplugged.
|
---|
| 131 | }
|
---|
| 132 | }
|
---|
| 133 |
|
---|
| 134 | this.sensor = newSensor;
|
---|
| 135 |
|
---|
| 136 | if (newSensor != null)
|
---|
| 137 | {
|
---|
| 138 | this.backgroundRemovalStream = new BackgroundRemovedColorStream(newSensor);
|
---|
| 139 | this.backgroundRemovalStream.BackgroundRemovedFrameReady += this.BackgroundRemovedFrameReadyAsync;
|
---|
| 140 |
|
---|
| 141 | // Force enabling the background removal stream because it hasn't been enabled before.
|
---|
| 142 | this.UpdateBackgroundRemovalFrameFormat(this.colorImageFormat, true);
|
---|
| 143 | }
|
---|
| 144 | }
|
---|
| 145 |
|
---|
| 146 | /// <summary>
|
---|
| 147 | /// Process data from one Kinect color frame.
|
---|
| 148 | /// </summary>
|
---|
| 149 | /// <param name="colorData">
|
---|
| 150 | /// Kinect color data.
|
---|
| 151 | /// </param>
|
---|
| 152 | /// <param name="colorFrame">
|
---|
| 153 | /// <see cref="ColorImageFrame"/> from which we obtained color data.
|
---|
| 154 | /// </param>
|
---|
| 155 | public override void ProcessColor(byte[] colorData, ColorImageFrame colorFrame)
|
---|
| 156 | {
|
---|
| 157 | if (colorData == null)
|
---|
| 158 | {
|
---|
| 159 | throw new ArgumentNullException("colorData");
|
---|
| 160 | }
|
---|
| 161 |
|
---|
| 162 | if (colorFrame == null)
|
---|
| 163 | {
|
---|
| 164 | throw new ArgumentNullException("colorFrame");
|
---|
| 165 | }
|
---|
| 166 |
|
---|
| 167 | if (this.backgroundRemovalStreamIsEnabled)
|
---|
| 168 | {
|
---|
| 169 | this.backgroundRemovalStream.ProcessColor(colorData, colorFrame.Timestamp);
|
---|
| 170 | }
|
---|
| 171 | }
|
---|
| 172 |
|
---|
| 173 | /// <summary>
|
---|
| 174 | /// Process data from one Kinect depth frame.
|
---|
| 175 | /// </summary>
|
---|
| 176 | /// <param name="depthData">
|
---|
| 177 | /// Kinect depth data.
|
---|
| 178 | /// </param>
|
---|
| 179 | /// <param name="depthFrame">
|
---|
| 180 | /// <see cref="DepthImageFrame"/> from which we obtained depth data.
|
---|
| 181 | /// </param>
|
---|
| 182 | public override void ProcessDepth(DepthImagePixel[] depthData, DepthImageFrame depthFrame)
|
---|
| 183 | {
|
---|
| 184 | if (depthData == null)
|
---|
| 185 | {
|
---|
| 186 | throw new ArgumentNullException("depthData");
|
---|
| 187 | }
|
---|
| 188 |
|
---|
| 189 | if (depthFrame == null)
|
---|
| 190 | {
|
---|
| 191 | throw new ArgumentNullException("depthFrame");
|
---|
| 192 | }
|
---|
| 193 |
|
---|
| 194 | if (this.backgroundRemovalStreamIsEnabled)
|
---|
| 195 | {
|
---|
| 196 | this.backgroundRemovalStream.ProcessDepth(depthData, depthFrame.Timestamp);
|
---|
| 197 | }
|
---|
| 198 | }
|
---|
| 199 |
|
---|
| 200 | /// <summary>
|
---|
| 201 | /// Process data from one Kinect skeleton frame.
|
---|
| 202 | /// </summary>
|
---|
| 203 | /// <param name="skeletons">
|
---|
| 204 | /// Kinect skeleton data.
|
---|
| 205 | /// </param>
|
---|
| 206 | /// <param name="skeletonFrame">
|
---|
| 207 | /// <see cref="SkeletonFrame"/> from which we obtained skeleton data.
|
---|
| 208 | /// </param>
|
---|
| 209 | public override void ProcessSkeleton(Skeleton[] skeletons, SkeletonFrame skeletonFrame)
|
---|
| 210 | {
|
---|
| 211 | if (skeletons == null)
|
---|
| 212 | {
|
---|
| 213 | throw new ArgumentNullException("skeletons");
|
---|
| 214 | }
|
---|
| 215 |
|
---|
| 216 | if (skeletonFrame == null)
|
---|
| 217 | {
|
---|
| 218 | throw new ArgumentNullException("skeletonFrame");
|
---|
| 219 | }
|
---|
| 220 |
|
---|
| 221 | if (this.backgroundRemovalStreamIsEnabled)
|
---|
| 222 | {
|
---|
| 223 | this.backgroundRemovalStream.ProcessSkeleton(skeletons, skeletonFrame.Timestamp);
|
---|
| 224 | }
|
---|
| 225 | }
|
---|
| 226 |
|
---|
| 227 | /// <summary>
|
---|
| 228 | /// Event handler for BackgroundRemovedColorStream's BackgroundRemovedFrameReady event
|
---|
| 229 | /// </summary>
|
---|
| 230 | /// <param name="sender">object sending the event</param>
|
---|
| 231 | /// <param name="e">event arguments</param>
|
---|
| 232 | internal async void BackgroundRemovedFrameReadyAsync(object sender, BackgroundRemovedColorFrameReadyEventArgs e)
|
---|
| 233 | {
|
---|
| 234 | if (!this.backgroundRemovalStreamIsEnabled)
|
---|
| 235 | {
|
---|
| 236 | // Directly return if the stream is not enabled.
|
---|
| 237 | return;
|
---|
| 238 | }
|
---|
| 239 |
|
---|
| 240 | if (this.isProcessingBackgroundRemovedFrame)
|
---|
| 241 | {
|
---|
| 242 | // Re-entered BackgroundRemovedFrameReadyAsync while a previous frame is already being processed.
|
---|
| 243 | // Just ignore new frames until the current one finishes processing.
|
---|
| 244 | return;
|
---|
| 245 | }
|
---|
| 246 |
|
---|
| 247 | this.isProcessingBackgroundRemovedFrame = true;
|
---|
| 248 |
|
---|
| 249 | try
|
---|
| 250 | {
|
---|
| 251 | bool haveFrameData = false;
|
---|
| 252 |
|
---|
| 253 | using (var backgroundRemovedFrame = e.OpenBackgroundRemovedColorFrame())
|
---|
| 254 | {
|
---|
| 255 | if (backgroundRemovedFrame != null)
|
---|
| 256 | {
|
---|
| 257 | this.backgroundRemovalStreamMessage.UpdateBackgroundRemovedColorFrame(backgroundRemovedFrame);
|
---|
| 258 |
|
---|
| 259 | haveFrameData = true;
|
---|
| 260 | }
|
---|
| 261 | }
|
---|
| 262 |
|
---|
| 263 | if (haveFrameData)
|
---|
| 264 | {
|
---|
| 265 | await this.ownerContext.SendTwoPartStreamMessageAsync(this.backgroundRemovalStreamMessage, this.backgroundRemovalStreamMessage.Buffer);
|
---|
| 266 | }
|
---|
| 267 | }
|
---|
| 268 | finally
|
---|
| 269 | {
|
---|
| 270 | this.isProcessingBackgroundRemovedFrame = false;
|
---|
| 271 | }
|
---|
| 272 | }
|
---|
| 273 |
|
---|
| 274 | /// <summary>
|
---|
| 275 | /// Get the size for the given color image format.
|
---|
| 276 | /// </summary>
|
---|
| 277 | /// <param name="format">The color image format.</param>
|
---|
| 278 | /// <returns>The width and height of the given image.</returns>
|
---|
| 279 | private static Size GetColorImageSize(ColorImageFormat format)
|
---|
| 280 | {
|
---|
| 281 | try
|
---|
| 282 | {
|
---|
| 283 | var q = from item in BackgroundRemovalResolutions
|
---|
| 284 | where item.Key == format
|
---|
| 285 | select item.Value;
|
---|
| 286 |
|
---|
| 287 | return q.Single();
|
---|
| 288 | }
|
---|
| 289 | catch (InvalidOperationException)
|
---|
| 290 | {
|
---|
| 291 | throw new ArgumentException(Resources.UnsupportedColorFormat, "format");
|
---|
| 292 | }
|
---|
| 293 | }
|
---|
| 294 |
|
---|
| 295 | /// <summary>
|
---|
| 296 | /// Get the color image format for the specified width and height.
|
---|
| 297 | /// </summary>
|
---|
| 298 | /// <param name="width">
|
---|
| 299 | /// Image width.
|
---|
| 300 | /// </param>
|
---|
| 301 | /// <param name="height">
|
---|
| 302 | /// Image height.
|
---|
| 303 | /// </param>
|
---|
| 304 | /// <returns>
|
---|
| 305 | /// The color image format enumeration value.
|
---|
| 306 | /// </returns>
|
---|
| 307 | private static ColorImageFormat GetColorImageFormat(int width, int height)
|
---|
| 308 | {
|
---|
| 309 | try
|
---|
| 310 | {
|
---|
| 311 | var q = from item in BackgroundRemovalResolutions
|
---|
| 312 | where (int)item.Value.Width == width && (int)item.Value.Height == height
|
---|
| 313 | select item.Key;
|
---|
| 314 |
|
---|
| 315 | return q.Single();
|
---|
| 316 | }
|
---|
| 317 | catch (InvalidOperationException)
|
---|
| 318 | {
|
---|
| 319 | throw new ArgumentException(Resources.UnsupportedColorFormat);
|
---|
| 320 | }
|
---|
| 321 | }
|
---|
| 322 |
|
---|
| 323 | /// <summary>
|
---|
| 324 | /// Set the background removed color frame format.
|
---|
| 325 | /// </summary>
|
---|
| 326 | /// <param name="format">
|
---|
| 327 | /// The given color image format.
|
---|
| 328 | /// </param>
|
---|
| 329 | /// <param name="forceEnable">
|
---|
| 330 | /// Streams should be enabled even if new color image format is the same as the old one.
|
---|
| 331 | /// This is useful for the initial enabling of the stream.
|
---|
| 332 | /// </param>
|
---|
| 333 | private void UpdateBackgroundRemovalFrameFormat(ColorImageFormat format, bool forceEnable)
|
---|
| 334 | {
|
---|
| 335 | if (!forceEnable && (format == this.colorImageFormat))
|
---|
| 336 | {
|
---|
| 337 | // No work to do
|
---|
| 338 | return;
|
---|
| 339 | }
|
---|
| 340 |
|
---|
| 341 | if (this.sensor != null)
|
---|
| 342 | {
|
---|
| 343 | try
|
---|
| 344 | {
|
---|
| 345 | this.sensor.ColorStream.Enable(format);
|
---|
| 346 | this.backgroundRemovalStream.Enable(format, DepthImageFormat.Resolution640x480Fps30);
|
---|
| 347 | }
|
---|
| 348 | catch (InvalidOperationException)
|
---|
| 349 | {
|
---|
| 350 | // KinectSensor might enter an invalid state while enabling/disabling streams or stream features.
|
---|
| 351 | // E.g.: sensor might be abruptly unplugged.
|
---|
| 352 | }
|
---|
| 353 | }
|
---|
| 354 |
|
---|
| 355 | // Update the image format property if the action succeeded.
|
---|
| 356 | this.colorImageFormat = format;
|
---|
| 357 | }
|
---|
| 358 |
|
---|
| 359 | /// <summary>
|
---|
| 360 | /// Gets a background removal stream property value.
|
---|
| 361 | /// </summary>
|
---|
| 362 | /// <param name="propertyMap">
|
---|
| 363 | /// Property name->value map where property values should be set.
|
---|
| 364 | /// </param>
|
---|
| 365 | private void GetProperties(Dictionary<string, object> propertyMap)
|
---|
| 366 | {
|
---|
| 367 | propertyMap.Add(KinectRequestHandler.EnabledPropertyName, this.backgroundRemovalStreamIsEnabled);
|
---|
| 368 | propertyMap.Add(TrackingIdPropertyName, this.trackingId);
|
---|
| 369 |
|
---|
| 370 | var size = GetColorImageSize(this.colorImageFormat);
|
---|
| 371 | propertyMap.Add(ResolutionPropertyName, string.Format(CultureInfo.InvariantCulture, @"{0}x{1}", (int)size.Width, (int)size.Height));
|
---|
| 372 | }
|
---|
| 373 |
|
---|
| 374 | /// <summary>
|
---|
| 375 | /// Set a background removal stream property value.
|
---|
| 376 | /// </summary>
|
---|
| 377 | /// <param name="propertyName">
|
---|
| 378 | /// Name of property to set.
|
---|
| 379 | /// </param>
|
---|
| 380 | /// <param name="propertyValue">
|
---|
| 381 | /// Property value to set.
|
---|
| 382 | /// </param>
|
---|
| 383 | /// <returns>
|
---|
| 384 | /// null if property setting was successful, error message otherwise.
|
---|
| 385 | /// </returns>
|
---|
| 386 | private string SetProperty(string propertyName, object propertyValue)
|
---|
| 387 | {
|
---|
| 388 | bool recognized = true;
|
---|
| 389 |
|
---|
| 390 | if (propertyValue == null)
|
---|
| 391 | {
|
---|
| 392 | return Resources.PropertyValueInvalidFormat;
|
---|
| 393 | }
|
---|
| 394 |
|
---|
| 395 | try
|
---|
| 396 | {
|
---|
| 397 | switch (propertyName)
|
---|
| 398 | {
|
---|
| 399 | case KinectRequestHandler.EnabledPropertyName:
|
---|
| 400 | this.backgroundRemovalStreamIsEnabled = (bool)propertyValue;
|
---|
| 401 | break;
|
---|
| 402 |
|
---|
| 403 | case TrackingIdPropertyName:
|
---|
| 404 | {
|
---|
| 405 | var oldTrackingId = this.trackingId;
|
---|
| 406 | this.trackingId = (int)propertyValue;
|
---|
| 407 |
|
---|
| 408 | if (this.trackingId != oldTrackingId)
|
---|
| 409 | {
|
---|
| 410 | this.backgroundRemovalStream.SetTrackedPlayer(this.trackingId);
|
---|
| 411 | }
|
---|
| 412 | }
|
---|
| 413 |
|
---|
| 414 | break;
|
---|
| 415 |
|
---|
| 416 | case ResolutionPropertyName:
|
---|
| 417 | var match = BackgroundRemovalResolutionRegex.Match((string)propertyValue);
|
---|
| 418 | if (!match.Success || (match.Groups.Count != 3))
|
---|
| 419 | {
|
---|
| 420 | return Resources.PropertyValueInvalidFormat;
|
---|
| 421 | }
|
---|
| 422 |
|
---|
| 423 | int width = int.Parse(match.Groups[1].Value, CultureInfo.InvariantCulture);
|
---|
| 424 | int height = int.Parse(match.Groups[2].Value, CultureInfo.InvariantCulture);
|
---|
| 425 |
|
---|
| 426 | try
|
---|
| 427 | {
|
---|
| 428 | var format = GetColorImageFormat(width, height);
|
---|
| 429 | this.UpdateBackgroundRemovalFrameFormat(format, false);
|
---|
| 430 | }
|
---|
| 431 | catch (ArgumentException)
|
---|
| 432 | {
|
---|
| 433 | return Resources.PropertyValueUnsupportedResolution;
|
---|
| 434 | }
|
---|
| 435 |
|
---|
| 436 | break;
|
---|
| 437 |
|
---|
| 438 | default:
|
---|
| 439 | recognized = false;
|
---|
| 440 | break;
|
---|
| 441 | }
|
---|
| 442 |
|
---|
| 443 | if (!recognized)
|
---|
| 444 | {
|
---|
| 445 | return Resources.PropertyNameUnrecognized;
|
---|
| 446 | }
|
---|
| 447 | }
|
---|
| 448 | catch (InvalidCastException)
|
---|
| 449 | {
|
---|
| 450 | return Resources.PropertyValueInvalidFormat;
|
---|
| 451 | }
|
---|
| 452 |
|
---|
| 453 | return null;
|
---|
| 454 | }
|
---|
| 455 | }
|
---|
| 456 | }
|
---|