[28897] | 1 | // -----------------------------------------------------------------------
|
---|
| 2 | // <copyright file="InteractionStreamHandler.cs" company="Microsoft">
|
---|
| 3 | // Copyright (c) Microsoft Corporation. All rights reserved.
|
---|
| 4 | // </copyright>
|
---|
| 5 | // -----------------------------------------------------------------------
|
---|
| 6 |
|
---|
| 7 | namespace Microsoft.Samples.Kinect.Webserver.Sensor
|
---|
| 8 | {
|
---|
| 9 | using System;
|
---|
| 10 | using System.Collections.Generic;
|
---|
| 11 | using System.Diagnostics.CodeAnalysis;
|
---|
| 12 | using System.Globalization;
|
---|
| 13 | using System.Linq;
|
---|
| 14 | using System.Net;
|
---|
| 15 | using System.Text.RegularExpressions;
|
---|
| 16 | using System.Threading.Tasks;
|
---|
| 17 | using System.Windows;
|
---|
| 18 | using System.Windows.Media;
|
---|
| 19 |
|
---|
| 20 | using Microsoft.Kinect;
|
---|
| 21 | using Microsoft.Kinect.Toolkit.Interaction;
|
---|
| 22 | using Microsoft.Samples.Kinect.Webserver.Sensor.Serialization;
|
---|
| 23 |
|
---|
| 24 | /// <summary>
|
---|
| 25 | /// Implementation of ISensorStreamHandler that exposes interaction and user viewer
|
---|
| 26 | /// streams.
|
---|
| 27 | /// </summary>
|
---|
| 28 | [SuppressMessage("Microsoft.Design", "CA1001:TypesThatOwnDisposableFieldsShouldBeDisposable", Justification = "Disposable interaction stream is disposed when sensor is set to null")]
|
---|
| 29 | public class InteractionStreamHandler : SensorStreamHandlerBase, IInteractionClient
|
---|
| 30 | {
|
---|
| 31 | /// <summary>
|
---|
| 32 | /// JSON name of interaction stream.
|
---|
| 33 | /// </summary>
|
---|
| 34 | internal const string InteractionStreamName = "interaction";
|
---|
| 35 |
|
---|
| 36 | /// <summary>
|
---|
| 37 | /// JSON name for property representing primary user tracking ID.
|
---|
| 38 | /// </summary>
|
---|
| 39 | internal const string InteractionPrimaryUserPropertyName = "primaryUser";
|
---|
| 40 |
|
---|
| 41 | /// <summary>
|
---|
| 42 | /// JSON name for property representing user states.
|
---|
| 43 | /// </summary>
|
---|
| 44 | internal const string InteractionUserStatesPropertyName = "userStates";
|
---|
| 45 |
|
---|
| 46 | /// <summary>
|
---|
| 47 | /// JSON name of user viewer stream.
|
---|
| 48 | /// </summary>
|
---|
| 49 | internal const string UserViewerStreamName = "userviewer";
|
---|
| 50 |
|
---|
| 51 | /// <summary>
|
---|
| 52 | /// JSON name for property representing user viewer image resolution.
|
---|
| 53 | /// </summary>
|
---|
| 54 | internal const string UserViewerResolutionPropertyName = "resolution";
|
---|
| 55 |
|
---|
| 56 | /// <summary>
|
---|
| 57 | /// Default width for user viewer image.
|
---|
| 58 | /// </summary>
|
---|
| 59 | internal const int UserViewerDefaultWidth = 128;
|
---|
| 60 |
|
---|
| 61 | /// <summary>
|
---|
| 62 | /// Default height for user viewer image.
|
---|
| 63 | /// </summary>
|
---|
| 64 | internal const int UserViewerDefaultHeight = 96;
|
---|
| 65 |
|
---|
| 66 | /// <summary>
|
---|
| 67 | /// JSON name for property representing default color for users in user viewer image.
|
---|
| 68 | /// </summary>
|
---|
| 69 | internal const string UserViewerDefaultUserColorPropertyName = "defaultUserColor";
|
---|
| 70 |
|
---|
| 71 | /// <summary>
|
---|
| 72 | /// JSON name for property representing a map between user states and colors that should
|
---|
| 73 | /// be used to represent those states in user viewer image.
|
---|
| 74 | /// </summary>
|
---|
| 75 | internal const string UserViewerUserColorsPropertyName = "userColors";
|
---|
| 76 |
|
---|
| 77 | /// <summary>
|
---|
| 78 | /// Sub path for interaction client web-socket RPC endpoint owned by this handler.
|
---|
| 79 | /// </summary>
|
---|
| 80 | internal const string ClientUriSubpath = "CLIENT";
|
---|
| 81 |
|
---|
| 82 | /// <summary>
|
---|
| 83 | /// Default value for default color for users in user viewer image (light gray).
|
---|
| 84 | /// </summary>
|
---|
| 85 | internal static readonly Color UserViewerDefaultDefaultUserColor = new Color { R = 0xd3, G = 0xd3, B = 0xd3, A = 0xff };
|
---|
| 86 |
|
---|
| 87 | /// <summary>
|
---|
| 88 | /// Default color for tracked users in user viewer image (Kinect blue).
|
---|
| 89 | /// </summary>
|
---|
| 90 | internal static readonly Color UserViewerDefaultTrackedUserColor = new Color { R = 0x00, G = 0xbc, B = 0xf2, A = 0xff };
|
---|
| 91 |
|
---|
| 92 | /// <summary>
|
---|
| 93 | /// Default color for engaged users in user viewer image (Kinect purple).
|
---|
| 94 | /// </summary>
|
---|
| 95 | internal static readonly Color UserViewerDefaultEngagedUserColor = new Color { R = 0x51, G = 0x1c, B = 0x74, A = 0xff };
|
---|
| 96 |
|
---|
| 97 | /// <summary>
|
---|
| 98 | /// Regular expression that matches the user viewer resolution property.
|
---|
| 99 | /// </summary>
|
---|
| 100 | private static readonly Regex UserViewerResolutionRegex = new Regex(@"^(?i)(\d+)x(\d+)$");
|
---|
| 101 |
|
---|
| 102 | private static readonly Size[] UserViewerSupportedResolutions =
|
---|
| 103 | {
|
---|
| 104 | new Size(640, 480), new Size(320, 240), new Size(160, 120),
|
---|
| 105 | new Size(128, 96), new Size(80, 60)
|
---|
| 106 | };
|
---|
| 107 |
|
---|
| 108 | /// <summary>
|
---|
| 109 | /// Context that allows this stream handler to communicate with its owner.
|
---|
| 110 | /// </summary>
|
---|
| 111 | private readonly SensorStreamHandlerContext ownerContext;
|
---|
| 112 |
|
---|
| 113 | /// <summary>
|
---|
| 114 | /// Serializable interaction stream message, reused as interaction frames arrive.
|
---|
| 115 | /// </summary>
|
---|
| 116 | private readonly InteractionStreamMessage interactionStreamMessage = new InteractionStreamMessage { stream = InteractionStreamName };
|
---|
| 117 |
|
---|
| 118 | /// <summary>
|
---|
| 119 | /// Serializable user viewer stream message header, reused as depth frames arrive
|
---|
| 120 | /// and are colorized into the user viewer image.
|
---|
| 121 | /// </summary>
|
---|
| 122 | private readonly ImageHeaderStreamMessage userViewerStreamMessage = new ImageHeaderStreamMessage { stream = UserViewerStreamName };
|
---|
| 123 |
|
---|
| 124 | /// <summary>
|
---|
| 125 | /// Ids of users we choose to track.
|
---|
| 126 | /// </summary>
|
---|
| 127 | private readonly int[] recommendedUserTrackingIds = new int[2];
|
---|
| 128 |
|
---|
| 129 | /// <summary>
|
---|
| 130 | /// A map between user state names and colors that should be used to represent those
|
---|
| 131 | /// states in user viewer image.
|
---|
| 132 | /// </summary>
|
---|
| 133 | private readonly Dictionary<string, int> userViewerUserColors = new Dictionary<string, int>();
|
---|
| 134 |
|
---|
| 135 | /// <summary>
|
---|
| 136 | /// Width of user viewer image.
|
---|
| 137 | /// </summary>
|
---|
| 138 | private readonly UserViewerColorizer userViewerColorizer = new UserViewerColorizer(UserViewerDefaultWidth, UserViewerDefaultHeight);
|
---|
| 139 |
|
---|
| 140 | /// <summary>
|
---|
| 141 | /// User state manager.
|
---|
| 142 | /// </summary>
|
---|
| 143 | private readonly IUserStateManager userStateManager = new DefaultUserStateManager();
|
---|
| 144 |
|
---|
| 145 | /// <summary>
|
---|
| 146 | /// Sensor providing data to interaction stream.
|
---|
| 147 | /// </summary>
|
---|
| 148 | private KinectSensor sensor;
|
---|
| 149 |
|
---|
| 150 | /// <summary>
|
---|
| 151 | /// Entry point for interaction stream functionality.
|
---|
| 152 | /// </summary>
|
---|
| 153 | private InteractionStream interactionStream;
|
---|
| 154 |
|
---|
| 155 | /// <summary>
|
---|
| 156 | /// Intermediate storage for the user information received from interaction stream.
|
---|
| 157 | /// </summary>
|
---|
| 158 | private UserInfo[] userInfos;
|
---|
| 159 |
|
---|
| 160 | /// <summary>
|
---|
| 161 | /// true if interaction stream is enabled.
|
---|
| 162 | /// </summary>
|
---|
| 163 | private bool interactionIsEnabled;
|
---|
| 164 |
|
---|
| 165 | /// <summary>
|
---|
| 166 | /// true if user viewer stream is enabled.
|
---|
| 167 | /// </summary>
|
---|
| 168 | private bool userViewerIsEnabled;
|
---|
| 169 |
|
---|
| 170 | /// <summary>
|
---|
| 171 | /// Default color for users in user viewer image, in 32-bit RGBA format.
|
---|
| 172 | /// </summary>
|
---|
| 173 | private int userViewerDefaultUserColor;
|
---|
| 174 |
|
---|
| 175 | /// <summary>
|
---|
| 176 | /// Keep track if we're in the middle of processing an interaction frame.
|
---|
| 177 | /// </summary>
|
---|
| 178 | private bool isProcessingInteractionFrame;
|
---|
| 179 |
|
---|
| 180 | /// <summary>
|
---|
| 181 | /// Keep track if we're in the middle of processing a user viewer image.
|
---|
| 182 | /// </summary>
|
---|
| 183 | private bool isProcessingUserViewerImage;
|
---|
| 184 |
|
---|
| 185 | /// <summary>
|
---|
| 186 | /// Channel used to perform remote procedure calls regarding IInteractionClient state.
|
---|
| 187 | /// </summary>
|
---|
| 188 | private WebSocketRpcChannel clientRpcChannel;
|
---|
| 189 |
|
---|
| 190 | /// <summary>
|
---|
| 191 | /// Initializes a new instance of the <see cref="InteractionStreamHandler"/> class
|
---|
| 192 | /// and associates it with a context that allows it to communicate with its owner.
|
---|
| 193 | /// </summary>
|
---|
| 194 | /// <param name="ownerContext">
|
---|
| 195 | /// An instance of <see cref="SensorStreamHandlerContext"/> class.
|
---|
| 196 | /// </param>
|
---|
| 197 | internal InteractionStreamHandler(SensorStreamHandlerContext ownerContext)
|
---|
| 198 | {
|
---|
| 199 | this.userViewerDefaultUserColor = GetRgbaColorInt(UserViewerDefaultDefaultUserColor);
|
---|
| 200 | this.userViewerUserColors[DefaultUserStateManager.TrackedStateName] = GetRgbaColorInt(UserViewerDefaultTrackedUserColor);
|
---|
| 201 | this.userViewerUserColors[DefaultUserStateManager.EngagedStateName] = GetRgbaColorInt(UserViewerDefaultEngagedUserColor);
|
---|
| 202 |
|
---|
| 203 | this.ownerContext = ownerContext;
|
---|
| 204 | this.userStateManager.UserStateChanged += this.OnUserStateChanged;
|
---|
| 205 |
|
---|
| 206 | this.AddStreamConfiguration(InteractionStreamName, new StreamConfiguration(this.GetInteractionStreamProperties, this.SetInteractionStreamProperty));
|
---|
| 207 | this.AddStreamConfiguration(UserViewerStreamName, new StreamConfiguration(this.GetUserViewerStreamProperties, this.SetUserViewerStreamProperty));
|
---|
| 208 | }
|
---|
| 209 |
|
---|
| 210 | /// <summary>
|
---|
| 211 | /// True if we should process interaction data fed into interaction stream and user state manager.
|
---|
| 212 | /// False otherwise.
|
---|
| 213 | /// </summary>
|
---|
| 214 | private bool ShouldProcessInteractionData
|
---|
| 215 | {
|
---|
| 216 | get
|
---|
| 217 | {
|
---|
| 218 | // Check for a null userInfos since we may still get posted events from
|
---|
| 219 | // the stream after we have unregistered our event handler and deleted
|
---|
| 220 | // our buffers.
|
---|
| 221 | // Also we process interaction data even if only user viewer stream is
|
---|
| 222 | // enabled since data is used by IUserStateManager and UserViewerColorizer
|
---|
| 223 | // as well as by clients listening directly to interaction stream.
|
---|
| 224 | return (this.userInfos != null) && (this.interactionIsEnabled || this.userViewerIsEnabled);
|
---|
| 225 | }
|
---|
| 226 | }
|
---|
| 227 |
|
---|
| 228 | /// <summary>
|
---|
| 229 | /// Lets ISensorStreamHandler know that Kinect Sensor associated with this stream
|
---|
| 230 | /// handler has changed.
|
---|
| 231 | /// </summary>
|
---|
| 232 | /// <param name="newSensor">
|
---|
| 233 | /// New KinectSensor.
|
---|
| 234 | /// </param>
|
---|
| 235 | public override void OnSensorChanged(KinectSensor newSensor)
|
---|
| 236 | {
|
---|
| 237 | if (this.sensor != null)
|
---|
| 238 | {
|
---|
| 239 | try
|
---|
| 240 | {
|
---|
| 241 | this.interactionStream.InteractionFrameReady -= this.InteractionFrameReadyAsync;
|
---|
| 242 | this.interactionStream.Dispose();
|
---|
| 243 | this.interactionStream = null;
|
---|
| 244 |
|
---|
| 245 | this.sensor.SkeletonStream.AppChoosesSkeletons = false;
|
---|
| 246 | }
|
---|
| 247 | catch (InvalidOperationException)
|
---|
| 248 | {
|
---|
| 249 | // KinectSensor might enter an invalid state while enabling/disabling streams or stream features.
|
---|
| 250 | // E.g.: sensor might be abruptly unplugged.
|
---|
| 251 | }
|
---|
| 252 |
|
---|
| 253 | this.userInfos = null;
|
---|
| 254 | }
|
---|
| 255 |
|
---|
| 256 | this.sensor = newSensor;
|
---|
| 257 |
|
---|
| 258 | if (newSensor != null)
|
---|
| 259 | {
|
---|
| 260 | try
|
---|
| 261 | {
|
---|
| 262 | this.interactionStream = new InteractionStream(newSensor, this);
|
---|
| 263 | this.interactionStream.InteractionFrameReady += this.InteractionFrameReadyAsync;
|
---|
| 264 |
|
---|
| 265 | this.sensor.SkeletonStream.AppChoosesSkeletons = true;
|
---|
| 266 |
|
---|
| 267 | this.userInfos = new UserInfo[InteractionFrame.UserInfoArrayLength];
|
---|
| 268 | }
|
---|
| 269 | catch (InvalidOperationException)
|
---|
| 270 | {
|
---|
| 271 | // KinectSensor might enter an invalid state while enabling/disabling streams or stream features.
|
---|
| 272 | // E.g.: sensor might be abruptly unplugged.
|
---|
| 273 | }
|
---|
| 274 | }
|
---|
| 275 |
|
---|
| 276 | this.userStateManager.Reset();
|
---|
| 277 | this.userViewerColorizer.ResetColorLookupTable();
|
---|
| 278 | }
|
---|
| 279 |
|
---|
| 280 | /// <summary>
|
---|
| 281 | /// Process data from one Kinect depth frame.
|
---|
| 282 | /// </summary>
|
---|
| 283 | /// <param name="depthData">
|
---|
| 284 | /// Kinect depth data.
|
---|
| 285 | /// </param>
|
---|
| 286 | /// <param name="depthFrame">
|
---|
| 287 | /// <see cref="DepthImageFrame"/> from which we obtained depth data.
|
---|
| 288 | /// </param>
|
---|
| 289 | public override void ProcessDepth(DepthImagePixel[] depthData, DepthImageFrame depthFrame)
|
---|
| 290 | {
|
---|
| 291 | if (depthData == null)
|
---|
| 292 | {
|
---|
| 293 | throw new ArgumentNullException("depthData");
|
---|
| 294 | }
|
---|
| 295 |
|
---|
| 296 | if (depthFrame == null)
|
---|
| 297 | {
|
---|
| 298 | throw new ArgumentNullException("depthFrame");
|
---|
| 299 | }
|
---|
| 300 |
|
---|
| 301 | if (this.ShouldProcessInteractionData)
|
---|
| 302 | {
|
---|
| 303 | this.interactionStream.ProcessDepth(depthData, depthFrame.Timestamp);
|
---|
| 304 | }
|
---|
| 305 |
|
---|
| 306 | this.ProcessUserViewerImageAsync(depthData, depthFrame);
|
---|
| 307 | }
|
---|
| 308 |
|
---|
| 309 | /// <summary>
|
---|
| 310 | /// Process data from one Kinect skeleton frame.
|
---|
| 311 | /// </summary>
|
---|
| 312 | /// <param name="skeletons">
|
---|
| 313 | /// Kinect skeleton data.
|
---|
| 314 | /// </param>
|
---|
| 315 | /// <param name="skeletonFrame">
|
---|
| 316 | /// <see cref="SkeletonFrame"/> from which we obtained skeleton data.
|
---|
| 317 | /// </param>
|
---|
| 318 | public override void ProcessSkeleton(Skeleton[] skeletons, SkeletonFrame skeletonFrame)
|
---|
| 319 | {
|
---|
| 320 | if (skeletons == null)
|
---|
| 321 | {
|
---|
| 322 | throw new ArgumentNullException("skeletons");
|
---|
| 323 | }
|
---|
| 324 |
|
---|
| 325 | if (skeletonFrame == null)
|
---|
| 326 | {
|
---|
| 327 | throw new ArgumentNullException("skeletonFrame");
|
---|
| 328 | }
|
---|
| 329 |
|
---|
| 330 | if (this.ShouldProcessInteractionData)
|
---|
| 331 | {
|
---|
| 332 | this.interactionStream.ProcessSkeleton(skeletons, this.sensor.AccelerometerGetCurrentReading(), skeletonFrame.Timestamp);
|
---|
| 333 | }
|
---|
| 334 |
|
---|
| 335 | this.userStateManager.ChooseTrackedUsers(skeletons, skeletonFrame.Timestamp, this.recommendedUserTrackingIds);
|
---|
| 336 |
|
---|
| 337 | try
|
---|
| 338 | {
|
---|
| 339 | this.sensor.SkeletonStream.ChooseSkeletons(
|
---|
| 340 | this.recommendedUserTrackingIds[0], this.recommendedUserTrackingIds[1]);
|
---|
| 341 | }
|
---|
| 342 | catch (InvalidOperationException)
|
---|
| 343 | {
|
---|
| 344 | // KinectSensor might enter an invalid state while choosing skeletons.
|
---|
| 345 | // E.g.: sensor might be abruptly unplugged.
|
---|
| 346 | }
|
---|
| 347 | }
|
---|
| 348 |
|
---|
| 349 | /// <summary>
|
---|
| 350 | /// Handle an http request.
|
---|
| 351 | /// </summary>
|
---|
| 352 | /// <param name="streamName">
|
---|
| 353 | /// Name of stream for which property values should be set.
|
---|
| 354 | /// </param>
|
---|
| 355 | /// <param name="requestContext">
|
---|
| 356 | /// Context containing HTTP request data, which will also contain associated
|
---|
| 357 | /// response upon return.
|
---|
| 358 | /// </param>
|
---|
| 359 | /// <param name="subpath">
|
---|
| 360 | /// Request URI path relative to the stream name associated with this sensor stream
|
---|
| 361 | /// handler in the stream handler owner.
|
---|
| 362 | /// </param>
|
---|
| 363 | /// <returns>
|
---|
| 364 | /// Await-able task.
|
---|
| 365 | /// </returns>
|
---|
| 366 | /// <remarks>
|
---|
| 367 | /// Return value should never be null. Implementations should use Task.FromResult(0)
|
---|
| 368 | /// if function is implemented synchronously so that callers can await without
|
---|
| 369 | /// needing to check for null.
|
---|
| 370 | /// </remarks>
|
---|
| 371 | public override Task HandleRequestAsync(string streamName, HttpListenerContext requestContext, string subpath)
|
---|
| 372 | {
|
---|
| 373 | if (streamName == null)
|
---|
| 374 | {
|
---|
| 375 | throw new ArgumentNullException("streamName");
|
---|
| 376 | }
|
---|
| 377 |
|
---|
| 378 | if (requestContext == null)
|
---|
| 379 | {
|
---|
| 380 | throw new ArgumentNullException("requestContext");
|
---|
| 381 | }
|
---|
| 382 |
|
---|
| 383 | if (subpath == null)
|
---|
| 384 | {
|
---|
| 385 | throw new ArgumentNullException("subpath");
|
---|
| 386 | }
|
---|
| 387 |
|
---|
| 388 | if (!InteractionStreamName.Equals(streamName))
|
---|
| 389 | {
|
---|
| 390 | // Only supported endpoints are related to interaction stream
|
---|
| 391 | KinectRequestHandler.CloseResponse(requestContext, HttpStatusCode.NotFound);
|
---|
| 392 | return SharedConstants.EmptyCompletedTask;
|
---|
| 393 | }
|
---|
| 394 |
|
---|
| 395 | var splitPath = KinectRequestHandler.SplitUriSubpath(subpath);
|
---|
| 396 |
|
---|
| 397 | if (splitPath == null)
|
---|
| 398 | {
|
---|
| 399 | KinectRequestHandler.CloseResponse(requestContext, HttpStatusCode.NotFound);
|
---|
| 400 | return SharedConstants.EmptyCompletedTask;
|
---|
| 401 | }
|
---|
| 402 |
|
---|
| 403 | var pathComponent = splitPath.Item1;
|
---|
| 404 | switch (pathComponent)
|
---|
| 405 | {
|
---|
| 406 | case ClientUriSubpath:
|
---|
| 407 | // Only support one client at any one time
|
---|
| 408 | if (this.clientRpcChannel != null)
|
---|
| 409 | {
|
---|
| 410 | if (this.clientRpcChannel.CheckConnectionStatus())
|
---|
| 411 | {
|
---|
| 412 | KinectRequestHandler.CloseResponse(requestContext, HttpStatusCode.Conflict);
|
---|
| 413 | return SharedConstants.EmptyCompletedTask;
|
---|
| 414 | }
|
---|
| 415 |
|
---|
| 416 | this.clientRpcChannel = null;
|
---|
| 417 | }
|
---|
| 418 |
|
---|
| 419 | WebSocketRpcChannel.TryOpenAsync(
|
---|
| 420 | requestContext,
|
---|
| 421 | channel =>
|
---|
| 422 | {
|
---|
| 423 | // Check again in case another request came in before connection was established
|
---|
| 424 | if (this.clientRpcChannel != null)
|
---|
| 425 | {
|
---|
| 426 | channel.Dispose();
|
---|
| 427 | }
|
---|
| 428 |
|
---|
| 429 | this.clientRpcChannel = channel;
|
---|
| 430 | },
|
---|
| 431 | channel =>
|
---|
| 432 | {
|
---|
| 433 | // Only forget the current channel if it matches the channel being closed
|
---|
| 434 | if (this.clientRpcChannel == channel)
|
---|
| 435 | {
|
---|
| 436 | this.clientRpcChannel = null;
|
---|
| 437 | }
|
---|
| 438 | });
|
---|
| 439 |
|
---|
| 440 | break;
|
---|
| 441 |
|
---|
| 442 | default:
|
---|
| 443 | KinectRequestHandler.CloseResponse(requestContext, HttpStatusCode.NotFound);
|
---|
| 444 | break;
|
---|
| 445 | }
|
---|
| 446 |
|
---|
| 447 | return SharedConstants.EmptyCompletedTask;
|
---|
| 448 | }
|
---|
| 449 |
|
---|
| 450 | /// <summary>
|
---|
| 451 | /// Cancel all pending operations.
|
---|
| 452 | /// </summary>
|
---|
| 453 | public override void Cancel()
|
---|
| 454 | {
|
---|
| 455 | if (this.clientRpcChannel != null)
|
---|
| 456 | {
|
---|
| 457 | this.clientRpcChannel.Cancel();
|
---|
| 458 | }
|
---|
| 459 | }
|
---|
| 460 |
|
---|
| 461 | /// <summary>
|
---|
| 462 | /// Lets handler know that it should clean up resources associated with sensor stream
|
---|
| 463 | /// handling.
|
---|
| 464 | /// </summary>
|
---|
| 465 | /// <returns>
|
---|
| 466 | /// Await-able task.
|
---|
| 467 | /// </returns>
|
---|
| 468 | /// <remarks>
|
---|
| 469 | /// Return value should never be null. Implementations should use Task.FromResult(0)
|
---|
| 470 | /// if function is implemented synchronously so that callers can await without
|
---|
| 471 | /// needing to check for null.
|
---|
| 472 | /// </remarks>
|
---|
| 473 | public override async Task UninitializeAsync()
|
---|
| 474 | {
|
---|
| 475 | if (this.clientRpcChannel != null)
|
---|
| 476 | {
|
---|
| 477 | await this.clientRpcChannel.CloseAsync();
|
---|
| 478 | }
|
---|
| 479 | }
|
---|
| 480 |
|
---|
| 481 | /// <summary>
|
---|
| 482 | /// Gets interaction information available for a specified location in UI.
|
---|
| 483 | /// </summary>
|
---|
| 484 | /// <param name="skeletonTrackingId">
|
---|
| 485 | /// The skeleton tracking ID for which interaction information is being retrieved.
|
---|
| 486 | /// </param>
|
---|
| 487 | /// <param name="handType">
|
---|
| 488 | /// The hand type for which interaction information is being retrieved.
|
---|
| 489 | /// </param>
|
---|
| 490 | /// <param name="x">
|
---|
| 491 | /// X-coordinate of UI location for which interaction information is being retrieved.
|
---|
| 492 | /// 0.0 corresponds to left edge of interaction region and 1.0 corresponds to right edge
|
---|
| 493 | /// of interaction region.
|
---|
| 494 | /// </param>
|
---|
| 495 | /// <param name="y">
|
---|
| 496 | /// Y-coordinate of UI location for which interaction information is being retrieved.
|
---|
| 497 | /// 0.0 corresponds to top edge of interaction region and 1.0 corresponds to bottom edge
|
---|
| 498 | /// of interaction region.
|
---|
| 499 | /// </param>
|
---|
| 500 | /// <returns>
|
---|
| 501 | /// An <see cref="InteractionInfo"/> object instance.
|
---|
| 502 | /// </returns>
|
---|
| 503 | public InteractionInfo GetInteractionInfoAtLocation(int skeletonTrackingId, InteractionHandType handType, double x, double y)
|
---|
| 504 | {
|
---|
| 505 | var interactionInfo = new InteractionInfo { IsPressTarget = false, IsGripTarget = false };
|
---|
| 506 |
|
---|
| 507 | if (this.interactionIsEnabled && (this.clientRpcChannel != null))
|
---|
| 508 | {
|
---|
| 509 | var result = this.clientRpcChannel.CallFunction<InteractionStreamHitTestInfo>("getInteractionInfoAtLocation", skeletonTrackingId, handType.ToString(), x, y);
|
---|
| 510 | if (result.Success)
|
---|
| 511 | {
|
---|
| 512 | interactionInfo.IsGripTarget = result.Result.isGripTarget;
|
---|
| 513 | interactionInfo.IsPressTarget = result.Result.isPressTarget;
|
---|
| 514 | var elementId = result.Result.pressTargetControlId;
|
---|
| 515 | interactionInfo.PressTargetControlId = (elementId != null) ? elementId.GetHashCode() : 0;
|
---|
| 516 | interactionInfo.PressAttractionPointX = result.Result.pressAttractionPointX;
|
---|
| 517 | interactionInfo.PressAttractionPointY = result.Result.pressAttractionPointY;
|
---|
| 518 | }
|
---|
| 519 | }
|
---|
| 520 |
|
---|
| 521 | return interactionInfo;
|
---|
| 522 | }
|
---|
| 523 |
|
---|
| 524 | /// <summary>
|
---|
| 525 | /// Converts a color into the corresponding 32-bit integer RGBA representation.
|
---|
| 526 | /// </summary>
|
---|
| 527 | /// <param name="color">
|
---|
| 528 | /// Color to convert
|
---|
| 529 | /// </param>
|
---|
| 530 | /// <returns>
|
---|
| 531 | /// 32-bit integer RGBA representation of color.
|
---|
| 532 | /// </returns>
|
---|
| 533 | internal static int GetRgbaColorInt(Color color)
|
---|
| 534 | {
|
---|
| 535 | return (color.A << 24) | (color.B << 16) | (color.G << 8) | color.R;
|
---|
| 536 | }
|
---|
| 537 |
|
---|
| 538 | /// <summary>
|
---|
| 539 | /// Event handler for InteractionStream's InteractionFrameReady event
|
---|
| 540 | /// </summary>
|
---|
| 541 | /// <param name="sender">object sending the event</param>
|
---|
| 542 | /// <param name="e">event arguments</param>
|
---|
| 543 | internal async void InteractionFrameReadyAsync(object sender, InteractionFrameReadyEventArgs e)
|
---|
| 544 | {
|
---|
| 545 | if (!this.ShouldProcessInteractionData)
|
---|
| 546 | {
|
---|
| 547 | return;
|
---|
| 548 | }
|
---|
| 549 |
|
---|
| 550 | if (this.isProcessingInteractionFrame)
|
---|
| 551 | {
|
---|
| 552 | // Re-entered InteractionFrameReadyAsync while a previous frame is already being processed.
|
---|
| 553 | // Just ignore new frames until the current one finishes processing.
|
---|
| 554 | return;
|
---|
| 555 | }
|
---|
| 556 |
|
---|
| 557 | this.isProcessingInteractionFrame = true;
|
---|
| 558 |
|
---|
| 559 | try
|
---|
| 560 | {
|
---|
| 561 | bool haveFrameData = false;
|
---|
| 562 |
|
---|
| 563 | using (var interactionFrame = e.OpenInteractionFrame())
|
---|
| 564 | {
|
---|
| 565 | // Even though we checked value of userInfos above as part of
|
---|
| 566 | // ShouldProcessInteractionData check, callbacks happening while
|
---|
| 567 | // opening an interaction frame might have invalidated it, so we
|
---|
| 568 | // check value again.
|
---|
| 569 | if ((interactionFrame != null) && (this.userInfos != null))
|
---|
| 570 | {
|
---|
| 571 | // Copy interaction frame data so we can dispose interaction frame
|
---|
| 572 | // right away, even if data processing/event handling takes a while.
|
---|
| 573 | interactionFrame.CopyInteractionDataTo(this.userInfos);
|
---|
| 574 | this.interactionStreamMessage.timestamp = interactionFrame.Timestamp;
|
---|
| 575 | haveFrameData = true;
|
---|
| 576 | }
|
---|
| 577 | }
|
---|
| 578 |
|
---|
| 579 | if (haveFrameData)
|
---|
| 580 | {
|
---|
| 581 | this.userStateManager.UpdateUserInformation(this.userInfos, this.interactionStreamMessage.timestamp);
|
---|
| 582 | this.userViewerColorizer.UpdateColorLookupTable(this.userInfos, this.userViewerDefaultUserColor, this.userStateManager.UserStates, this.userViewerUserColors);
|
---|
| 583 |
|
---|
| 584 | if (this.interactionIsEnabled)
|
---|
| 585 | {
|
---|
| 586 | this.interactionStreamMessage.UpdateHandPointers(this.userInfos, this.userStateManager.PrimaryUserTrackingId);
|
---|
| 587 | await this.ownerContext.SendStreamMessageAsync(this.interactionStreamMessage);
|
---|
| 588 | }
|
---|
| 589 | }
|
---|
| 590 | }
|
---|
| 591 | finally
|
---|
| 592 | {
|
---|
| 593 | this.isProcessingInteractionFrame = false;
|
---|
| 594 | }
|
---|
| 595 | }
|
---|
| 596 |
|
---|
| 597 | /// <summary>
|
---|
| 598 | /// Gets all interaction stream property value.
|
---|
| 599 | /// </summary>
|
---|
| 600 | /// <param name="propertyMap">
|
---|
| 601 | /// Property name->value map where property values should be set.
|
---|
| 602 | /// </param>
|
---|
| 603 | internal void GetInteractionStreamProperties(Dictionary<string, object> propertyMap)
|
---|
| 604 | {
|
---|
| 605 | propertyMap.Add(KinectRequestHandler.EnabledPropertyName, this.interactionIsEnabled);
|
---|
| 606 | propertyMap.Add(InteractionPrimaryUserPropertyName, this.userStateManager.PrimaryUserTrackingId);
|
---|
| 607 | propertyMap.Add(InteractionUserStatesPropertyName, DefaultUserStateManager.GetStateMappingEntryArray(this.userStateManager.UserStates));
|
---|
| 608 | }
|
---|
| 609 |
|
---|
| 610 | /// <summary>
|
---|
| 611 | /// Set an interaction stream property value.
|
---|
| 612 | /// </summary>
|
---|
| 613 | /// <param name="propertyName">
|
---|
| 614 | /// Name of property to set.
|
---|
| 615 | /// </param>
|
---|
| 616 | /// <param name="propertyValue">
|
---|
| 617 | /// Property value to set.
|
---|
| 618 | /// </param>
|
---|
| 619 | /// <returns>
|
---|
| 620 | /// null if property setting was successful, error message otherwise.
|
---|
| 621 | /// </returns>
|
---|
| 622 | internal string SetInteractionStreamProperty(string propertyName, object propertyValue)
|
---|
| 623 | {
|
---|
| 624 | bool recognized = true;
|
---|
| 625 |
|
---|
| 626 | if (propertyValue == null)
|
---|
| 627 | {
|
---|
| 628 | // None of the interaction stream properties accept a null value
|
---|
| 629 | return Properties.Resources.PropertyValueInvalidFormat;
|
---|
| 630 | }
|
---|
| 631 |
|
---|
| 632 | try
|
---|
| 633 | {
|
---|
| 634 | switch (propertyName)
|
---|
| 635 | {
|
---|
| 636 | case KinectRequestHandler.EnabledPropertyName:
|
---|
| 637 | this.interactionIsEnabled = (bool)propertyValue;
|
---|
| 638 | break;
|
---|
| 639 |
|
---|
| 640 | default:
|
---|
| 641 | recognized = false;
|
---|
| 642 | break;
|
---|
| 643 | }
|
---|
| 644 |
|
---|
| 645 | if (!recognized)
|
---|
| 646 | {
|
---|
| 647 | return Properties.Resources.PropertyNameUnrecognized;
|
---|
| 648 | }
|
---|
| 649 | }
|
---|
| 650 | catch (InvalidCastException)
|
---|
| 651 | {
|
---|
| 652 | return Properties.Resources.PropertyValueInvalidFormat;
|
---|
| 653 | }
|
---|
| 654 |
|
---|
| 655 | return null;
|
---|
| 656 | }
|
---|
| 657 |
|
---|
| 658 | /// <summary>
|
---|
| 659 | /// Gets all user viewer stream property value.
|
---|
| 660 | /// </summary>
|
---|
| 661 | /// <param name="propertyMap">
|
---|
| 662 | /// Property name->value map where property values should be set.
|
---|
| 663 | /// </param>
|
---|
| 664 | internal void GetUserViewerStreamProperties(Dictionary<string, object> propertyMap)
|
---|
| 665 | {
|
---|
| 666 | propertyMap.Add(KinectRequestHandler.EnabledPropertyName, this.userViewerIsEnabled);
|
---|
| 667 | propertyMap.Add(UserViewerResolutionPropertyName, string.Format(CultureInfo.InvariantCulture, @"{0}x{1}", this.userViewerColorizer.Width, this.userViewerColorizer.Height));
|
---|
| 668 | propertyMap.Add(UserViewerDefaultUserColorPropertyName, this.userViewerDefaultUserColor);
|
---|
| 669 | propertyMap.Add(UserViewerUserColorsPropertyName, this.userViewerUserColors);
|
---|
| 670 | }
|
---|
| 671 |
|
---|
| 672 | /// <summary>
|
---|
| 673 | /// Set a user viewer stream property.
|
---|
| 674 | /// </summary>
|
---|
| 675 | /// <param name="propertyName">
|
---|
| 676 | /// Name of property to set.
|
---|
| 677 | /// </param>
|
---|
| 678 | /// <param name="propertyValue">
|
---|
| 679 | /// Property value to set.
|
---|
| 680 | /// </param>
|
---|
| 681 | /// <returns>
|
---|
| 682 | /// null if property setting was successful, error message otherwise.
|
---|
| 683 | /// </returns>
|
---|
| 684 | internal string SetUserViewerStreamProperty(string propertyName, object propertyValue)
|
---|
| 685 | {
|
---|
| 686 | bool recognized = true;
|
---|
| 687 |
|
---|
| 688 | try
|
---|
| 689 | {
|
---|
| 690 | switch (propertyName)
|
---|
| 691 | {
|
---|
| 692 | case KinectRequestHandler.EnabledPropertyName:
|
---|
| 693 | this.userViewerIsEnabled = (bool)propertyValue;
|
---|
| 694 | break;
|
---|
| 695 |
|
---|
| 696 | case UserViewerResolutionPropertyName:
|
---|
| 697 | if (propertyValue == null)
|
---|
| 698 | {
|
---|
| 699 | return Properties.Resources.PropertyValueInvalidFormat;
|
---|
| 700 | }
|
---|
| 701 |
|
---|
| 702 | var match = UserViewerResolutionRegex.Match((string)propertyValue);
|
---|
| 703 | if (!match.Success || (match.Groups.Count != 3))
|
---|
| 704 | {
|
---|
| 705 | return Properties.Resources.PropertyValueInvalidFormat;
|
---|
| 706 | }
|
---|
| 707 |
|
---|
| 708 | int width = int.Parse(match.Groups[1].Value, CultureInfo.InvariantCulture);
|
---|
| 709 | int height = int.Parse(match.Groups[2].Value, CultureInfo.InvariantCulture);
|
---|
| 710 |
|
---|
| 711 | if (!IsSupportedUserViewerResolution(width, height))
|
---|
| 712 | {
|
---|
| 713 | return Properties.Resources.PropertyValueUnsupportedResolution;
|
---|
| 714 | }
|
---|
| 715 |
|
---|
| 716 | this.userViewerColorizer.SetResolution(width, height);
|
---|
| 717 | break;
|
---|
| 718 |
|
---|
| 719 | case UserViewerDefaultUserColorPropertyName:
|
---|
| 720 | this.userViewerDefaultUserColor = (int)propertyValue;
|
---|
| 721 |
|
---|
| 722 | this.UpdateColorizerLookupTable();
|
---|
| 723 | break;
|
---|
| 724 |
|
---|
| 725 | case UserViewerUserColorsPropertyName:
|
---|
| 726 | if (propertyValue == null)
|
---|
| 727 | {
|
---|
| 728 | // Null values just clear the set of user colors
|
---|
| 729 | this.userViewerUserColors.Clear();
|
---|
| 730 | break;
|
---|
| 731 | }
|
---|
| 732 |
|
---|
| 733 | var userColors = (Dictionary<string, object>)propertyValue;
|
---|
| 734 |
|
---|
| 735 | // Verify that all dictionary values are integers
|
---|
| 736 | bool allIntegers = userColors.Values.Select(color => color as int?).All(colorInt => colorInt != null);
|
---|
| 737 | if (!allIntegers)
|
---|
| 738 | {
|
---|
| 739 | return Properties.Resources.PropertyValueInvalidFormat;
|
---|
| 740 | }
|
---|
| 741 |
|
---|
| 742 | // If property value specified is compatible, copy values over
|
---|
| 743 | this.userViewerUserColors.Clear();
|
---|
| 744 | foreach (var entry in userColors)
|
---|
| 745 | {
|
---|
| 746 | this.userViewerUserColors.Add(entry.Key, (int)entry.Value);
|
---|
| 747 | }
|
---|
| 748 |
|
---|
| 749 | this.UpdateColorizerLookupTable();
|
---|
| 750 | break;
|
---|
| 751 |
|
---|
| 752 | default:
|
---|
| 753 | recognized = false;
|
---|
| 754 | break;
|
---|
| 755 | }
|
---|
| 756 |
|
---|
| 757 | if (!recognized)
|
---|
| 758 | {
|
---|
| 759 | return Properties.Resources.PropertyNameUnrecognized;
|
---|
| 760 | }
|
---|
| 761 | }
|
---|
| 762 | catch (InvalidCastException)
|
---|
| 763 | {
|
---|
| 764 | return Properties.Resources.PropertyValueInvalidFormat;
|
---|
| 765 | }
|
---|
| 766 | catch (NullReferenceException)
|
---|
| 767 | {
|
---|
| 768 | return Properties.Resources.PropertyValueInvalidFormat;
|
---|
| 769 | }
|
---|
| 770 |
|
---|
| 771 | return null;
|
---|
| 772 | }
|
---|
| 773 |
|
---|
| 774 | /// <summary>
|
---|
| 775 | /// Determine if the specified resolution is supported for user viewer images.
|
---|
| 776 | /// </summary>
|
---|
| 777 | /// <param name="width">
|
---|
| 778 | /// Image width.
|
---|
| 779 | /// </param>
|
---|
| 780 | /// <param name="height">
|
---|
| 781 | /// Image height.
|
---|
| 782 | /// </param>
|
---|
| 783 | /// <returns>
|
---|
| 784 | /// True if specified resolution is supported, false otherwise.
|
---|
| 785 | /// </returns>
|
---|
| 786 | private static bool IsSupportedUserViewerResolution(int width, int height)
|
---|
| 787 | {
|
---|
| 788 | return UserViewerSupportedResolutions.Any(resolution => ((int)resolution.Width == width) && ((int)resolution.Height == height));
|
---|
| 789 | }
|
---|
| 790 |
|
---|
| 791 | /// <summary>
|
---|
| 792 | /// Handler for IUserStateManager.UserStateChanged event.
|
---|
| 793 | /// </summary>
|
---|
| 794 | /// <param name="sender">object sending the event</param>
|
---|
| 795 | /// <param name="e">event arguments</param>
|
---|
| 796 | private async void OnUserStateChanged(object sender, UserStateChangedEventArgs e)
|
---|
| 797 | {
|
---|
| 798 | if (this.interactionIsEnabled)
|
---|
| 799 | {
|
---|
| 800 | // If enabled, forward all user state events to client
|
---|
| 801 | await this.ownerContext.SendEventMessageAsync(e.Message);
|
---|
| 802 | }
|
---|
| 803 | }
|
---|
| 804 |
|
---|
| 805 | /// <summary>
|
---|
| 806 | /// Update colorizer lookup table from color-related configuration.
|
---|
| 807 | /// </summary>
|
---|
| 808 | private void UpdateColorizerLookupTable()
|
---|
| 809 | {
|
---|
| 810 | if (this.ShouldProcessInteractionData)
|
---|
| 811 | {
|
---|
| 812 | this.userViewerColorizer.UpdateColorLookupTable(
|
---|
| 813 | this.userInfos, this.userViewerDefaultUserColor, this.userStateManager.UserStates, this.userViewerUserColors);
|
---|
| 814 | }
|
---|
| 815 | }
|
---|
| 816 |
|
---|
| 817 | /// <summary>
|
---|
| 818 | /// Process depth data to obtain user viewer image.
|
---|
| 819 | /// </summary>
|
---|
| 820 | /// <param name="depthData">
|
---|
| 821 | /// Kinect depth data.
|
---|
| 822 | /// </param>
|
---|
| 823 | /// <param name="depthFrame">
|
---|
| 824 | /// <see cref="DepthImageFrame"/> from which we obtained depth data.
|
---|
| 825 | /// </param>
|
---|
| 826 | private async void ProcessUserViewerImageAsync(DepthImagePixel[] depthData, DepthImageFrame depthFrame)
|
---|
| 827 | {
|
---|
| 828 | if (this.userViewerIsEnabled)
|
---|
| 829 | {
|
---|
| 830 | if (this.isProcessingUserViewerImage)
|
---|
| 831 | {
|
---|
| 832 | // Re-entered ProcessUserViewerImageAsync while a previous image is already being processed.
|
---|
| 833 | // Just ignore new depth frames until the current one finishes processing.
|
---|
| 834 | return;
|
---|
| 835 | }
|
---|
| 836 |
|
---|
| 837 | this.isProcessingUserViewerImage = true;
|
---|
| 838 |
|
---|
| 839 | try
|
---|
| 840 | {
|
---|
| 841 | this.userViewerColorizer.ColorizeDepthPixels(depthData, depthFrame.Width, depthFrame.Height);
|
---|
| 842 | this.userViewerStreamMessage.timestamp = depthFrame.Timestamp;
|
---|
| 843 | this.userViewerStreamMessage.width = this.userViewerColorizer.Width;
|
---|
| 844 | this.userViewerStreamMessage.height = this.userViewerColorizer.Height;
|
---|
| 845 | this.userViewerStreamMessage.bufferLength = this.userViewerColorizer.Buffer.Length;
|
---|
| 846 |
|
---|
| 847 | await this.ownerContext.SendTwoPartStreamMessageAsync(this.userViewerStreamMessage, this.userViewerColorizer.Buffer);
|
---|
| 848 | }
|
---|
| 849 | finally
|
---|
| 850 | {
|
---|
| 851 | this.isProcessingUserViewerImage = false;
|
---|
| 852 | }
|
---|
| 853 | }
|
---|
| 854 | }
|
---|
| 855 | }
|
---|
| 856 | }
|
---|