1 | // -----------------------------------------------------------------------
|
---|
2 | // <copyright file="InteractionStreamHandler.cs" company="Microsoft">
|
---|
3 | // Copyright (c) Microsoft Corporation. All rights reserved.
|
---|
4 | // </copyright>
|
---|
5 | // -----------------------------------------------------------------------
|
---|
6 |
|
---|
7 | namespace Microsoft.Samples.Kinect.Webserver.Sensor
|
---|
8 | {
|
---|
9 | using System;
|
---|
10 | using System.Collections.Generic;
|
---|
11 | using System.Diagnostics.CodeAnalysis;
|
---|
12 | using System.Globalization;
|
---|
13 | using System.Linq;
|
---|
14 | using System.Net;
|
---|
15 | using System.Text.RegularExpressions;
|
---|
16 | using System.Threading.Tasks;
|
---|
17 | using System.Windows;
|
---|
18 | using System.Windows.Media;
|
---|
19 |
|
---|
20 | using Microsoft.Kinect;
|
---|
21 | using Microsoft.Kinect.Toolkit.Interaction;
|
---|
22 | using Microsoft.Samples.Kinect.Webserver.Sensor.Serialization;
|
---|
23 |
|
---|
24 | /// <summary>
|
---|
25 | /// Implementation of ISensorStreamHandler that exposes interaction and user viewer
|
---|
26 | /// streams.
|
---|
27 | /// </summary>
|
---|
28 | [SuppressMessage("Microsoft.Design", "CA1001:TypesThatOwnDisposableFieldsShouldBeDisposable", Justification = "Disposable interaction stream is disposed when sensor is set to null")]
|
---|
29 | public class InteractionStreamHandler : SensorStreamHandlerBase, IInteractionClient
|
---|
30 | {
|
---|
31 | /// <summary>
|
---|
32 | /// JSON name of interaction stream.
|
---|
33 | /// </summary>
|
---|
34 | internal const string InteractionStreamName = "interaction";
|
---|
35 |
|
---|
36 | /// <summary>
|
---|
37 | /// JSON name for property representing primary user tracking ID.
|
---|
38 | /// </summary>
|
---|
39 | internal const string InteractionPrimaryUserPropertyName = "primaryUser";
|
---|
40 |
|
---|
41 | /// <summary>
|
---|
42 | /// JSON name for property representing user states.
|
---|
43 | /// </summary>
|
---|
44 | internal const string InteractionUserStatesPropertyName = "userStates";
|
---|
45 |
|
---|
46 | /// <summary>
|
---|
47 | /// JSON name of user viewer stream.
|
---|
48 | /// </summary>
|
---|
49 | internal const string UserViewerStreamName = "userviewer";
|
---|
50 |
|
---|
51 | /// <summary>
|
---|
52 | /// JSON name for property representing user viewer image resolution.
|
---|
53 | /// </summary>
|
---|
54 | internal const string UserViewerResolutionPropertyName = "resolution";
|
---|
55 |
|
---|
56 | /// <summary>
|
---|
57 | /// Default width for user viewer image.
|
---|
58 | /// </summary>
|
---|
59 | internal const int UserViewerDefaultWidth = 128;
|
---|
60 |
|
---|
61 | /// <summary>
|
---|
62 | /// Default height for user viewer image.
|
---|
63 | /// </summary>
|
---|
64 | internal const int UserViewerDefaultHeight = 96;
|
---|
65 |
|
---|
66 | /// <summary>
|
---|
67 | /// JSON name for property representing default color for users in user viewer image.
|
---|
68 | /// </summary>
|
---|
69 | internal const string UserViewerDefaultUserColorPropertyName = "defaultUserColor";
|
---|
70 |
|
---|
71 | /// <summary>
|
---|
72 | /// JSON name for property representing a map between user states and colors that should
|
---|
73 | /// be used to represent those states in user viewer image.
|
---|
74 | /// </summary>
|
---|
75 | internal const string UserViewerUserColorsPropertyName = "userColors";
|
---|
76 |
|
---|
77 | /// <summary>
|
---|
78 | /// Sub path for interaction client web-socket RPC endpoint owned by this handler.
|
---|
79 | /// </summary>
|
---|
80 | internal const string ClientUriSubpath = "CLIENT";
|
---|
81 |
|
---|
82 | /// <summary>
|
---|
83 | /// Default value for default color for users in user viewer image (light gray).
|
---|
84 | /// </summary>
|
---|
85 | internal static readonly Color UserViewerDefaultDefaultUserColor = new Color { R = 0xd3, G = 0xd3, B = 0xd3, A = 0xff };
|
---|
86 |
|
---|
87 | /// <summary>
|
---|
88 | /// Default color for tracked users in user viewer image (Kinect blue).
|
---|
89 | /// </summary>
|
---|
90 | internal static readonly Color UserViewerDefaultTrackedUserColor = new Color { R = 0x00, G = 0xbc, B = 0xf2, A = 0xff };
|
---|
91 |
|
---|
92 | /// <summary>
|
---|
93 | /// Default color for engaged users in user viewer image (Kinect purple).
|
---|
94 | /// </summary>
|
---|
95 | internal static readonly Color UserViewerDefaultEngagedUserColor = new Color { R = 0x51, G = 0x1c, B = 0x74, A = 0xff };
|
---|
96 |
|
---|
97 | /// <summary>
|
---|
98 | /// Regular expression that matches the user viewer resolution property.
|
---|
99 | /// </summary>
|
---|
100 | private static readonly Regex UserViewerResolutionRegex = new Regex(@"^(?i)(\d+)x(\d+)$");
|
---|
101 |
|
---|
102 | private static readonly Size[] UserViewerSupportedResolutions =
|
---|
103 | {
|
---|
104 | new Size(640, 480), new Size(320, 240), new Size(160, 120),
|
---|
105 | new Size(128, 96), new Size(80, 60)
|
---|
106 | };
|
---|
107 |
|
---|
108 | /// <summary>
|
---|
109 | /// Context that allows this stream handler to communicate with its owner.
|
---|
110 | /// </summary>
|
---|
111 | private readonly SensorStreamHandlerContext ownerContext;
|
---|
112 |
|
---|
113 | /// <summary>
|
---|
114 | /// Serializable interaction stream message, reused as interaction frames arrive.
|
---|
115 | /// </summary>
|
---|
116 | private readonly InteractionStreamMessage interactionStreamMessage = new InteractionStreamMessage { stream = InteractionStreamName };
|
---|
117 |
|
---|
118 | /// <summary>
|
---|
119 | /// Serializable user viewer stream message header, reused as depth frames arrive
|
---|
120 | /// and are colorized into the user viewer image.
|
---|
121 | /// </summary>
|
---|
122 | private readonly ImageHeaderStreamMessage userViewerStreamMessage = new ImageHeaderStreamMessage { stream = UserViewerStreamName };
|
---|
123 |
|
---|
124 | /// <summary>
|
---|
125 | /// Ids of users we choose to track.
|
---|
126 | /// </summary>
|
---|
127 | private readonly int[] recommendedUserTrackingIds = new int[2];
|
---|
128 |
|
---|
129 | /// <summary>
|
---|
130 | /// A map between user state names and colors that should be used to represent those
|
---|
131 | /// states in user viewer image.
|
---|
132 | /// </summary>
|
---|
133 | private readonly Dictionary<string, int> userViewerUserColors = new Dictionary<string, int>();
|
---|
134 |
|
---|
135 | /// <summary>
|
---|
136 | /// Width of user viewer image.
|
---|
137 | /// </summary>
|
---|
138 | private readonly UserViewerColorizer userViewerColorizer = new UserViewerColorizer(UserViewerDefaultWidth, UserViewerDefaultHeight);
|
---|
139 |
|
---|
140 | /// <summary>
|
---|
141 | /// User state manager.
|
---|
142 | /// </summary>
|
---|
143 | private readonly IUserStateManager userStateManager = new DefaultUserStateManager();
|
---|
144 |
|
---|
145 | /// <summary>
|
---|
146 | /// Sensor providing data to interaction stream.
|
---|
147 | /// </summary>
|
---|
148 | private KinectSensor sensor;
|
---|
149 |
|
---|
150 | /// <summary>
|
---|
151 | /// Entry point for interaction stream functionality.
|
---|
152 | /// </summary>
|
---|
153 | private InteractionStream interactionStream;
|
---|
154 |
|
---|
155 | /// <summary>
|
---|
156 | /// Intermediate storage for the user information received from interaction stream.
|
---|
157 | /// </summary>
|
---|
158 | private UserInfo[] userInfos;
|
---|
159 |
|
---|
160 | /// <summary>
|
---|
161 | /// true if interaction stream is enabled.
|
---|
162 | /// </summary>
|
---|
163 | private bool interactionIsEnabled;
|
---|
164 |
|
---|
165 | /// <summary>
|
---|
166 | /// true if user viewer stream is enabled.
|
---|
167 | /// </summary>
|
---|
168 | private bool userViewerIsEnabled;
|
---|
169 |
|
---|
170 | /// <summary>
|
---|
171 | /// Default color for users in user viewer image, in 32-bit RGBA format.
|
---|
172 | /// </summary>
|
---|
173 | private int userViewerDefaultUserColor;
|
---|
174 |
|
---|
175 | /// <summary>
|
---|
176 | /// Keep track if we're in the middle of processing an interaction frame.
|
---|
177 | /// </summary>
|
---|
178 | private bool isProcessingInteractionFrame;
|
---|
179 |
|
---|
180 | /// <summary>
|
---|
181 | /// Keep track if we're in the middle of processing a user viewer image.
|
---|
182 | /// </summary>
|
---|
183 | private bool isProcessingUserViewerImage;
|
---|
184 |
|
---|
185 | /// <summary>
|
---|
186 | /// Channel used to perform remote procedure calls regarding IInteractionClient state.
|
---|
187 | /// </summary>
|
---|
188 | private WebSocketRpcChannel clientRpcChannel;
|
---|
189 |
|
---|
190 | /// <summary>
|
---|
191 | /// Initializes a new instance of the <see cref="InteractionStreamHandler"/> class
|
---|
192 | /// and associates it with a context that allows it to communicate with its owner.
|
---|
193 | /// </summary>
|
---|
194 | /// <param name="ownerContext">
|
---|
195 | /// An instance of <see cref="SensorStreamHandlerContext"/> class.
|
---|
196 | /// </param>
|
---|
197 | internal InteractionStreamHandler(SensorStreamHandlerContext ownerContext)
|
---|
198 | {
|
---|
199 | this.userViewerDefaultUserColor = GetRgbaColorInt(UserViewerDefaultDefaultUserColor);
|
---|
200 | this.userViewerUserColors[DefaultUserStateManager.TrackedStateName] = GetRgbaColorInt(UserViewerDefaultTrackedUserColor);
|
---|
201 | this.userViewerUserColors[DefaultUserStateManager.EngagedStateName] = GetRgbaColorInt(UserViewerDefaultEngagedUserColor);
|
---|
202 |
|
---|
203 | this.ownerContext = ownerContext;
|
---|
204 | this.userStateManager.UserStateChanged += this.OnUserStateChanged;
|
---|
205 |
|
---|
206 | this.AddStreamConfiguration(InteractionStreamName, new StreamConfiguration(this.GetInteractionStreamProperties, this.SetInteractionStreamProperty));
|
---|
207 | this.AddStreamConfiguration(UserViewerStreamName, new StreamConfiguration(this.GetUserViewerStreamProperties, this.SetUserViewerStreamProperty));
|
---|
208 | }
|
---|
209 |
|
---|
210 | /// <summary>
|
---|
211 | /// True if we should process interaction data fed into interaction stream and user state manager.
|
---|
212 | /// False otherwise.
|
---|
213 | /// </summary>
|
---|
214 | private bool ShouldProcessInteractionData
|
---|
215 | {
|
---|
216 | get
|
---|
217 | {
|
---|
218 | // Check for a null userInfos since we may still get posted events from
|
---|
219 | // the stream after we have unregistered our event handler and deleted
|
---|
220 | // our buffers.
|
---|
221 | // Also we process interaction data even if only user viewer stream is
|
---|
222 | // enabled since data is used by IUserStateManager and UserViewerColorizer
|
---|
223 | // as well as by clients listening directly to interaction stream.
|
---|
224 | return (this.userInfos != null) && (this.interactionIsEnabled || this.userViewerIsEnabled);
|
---|
225 | }
|
---|
226 | }
|
---|
227 |
|
---|
228 | /// <summary>
|
---|
229 | /// Lets ISensorStreamHandler know that Kinect Sensor associated with this stream
|
---|
230 | /// handler has changed.
|
---|
231 | /// </summary>
|
---|
232 | /// <param name="newSensor">
|
---|
233 | /// New KinectSensor.
|
---|
234 | /// </param>
|
---|
235 | public override void OnSensorChanged(KinectSensor newSensor)
|
---|
236 | {
|
---|
237 | if (this.sensor != null)
|
---|
238 | {
|
---|
239 | try
|
---|
240 | {
|
---|
241 | this.interactionStream.InteractionFrameReady -= this.InteractionFrameReadyAsync;
|
---|
242 | this.interactionStream.Dispose();
|
---|
243 | this.interactionStream = null;
|
---|
244 |
|
---|
245 | this.sensor.SkeletonStream.AppChoosesSkeletons = false;
|
---|
246 | }
|
---|
247 | catch (InvalidOperationException)
|
---|
248 | {
|
---|
249 | // KinectSensor might enter an invalid state while enabling/disabling streams or stream features.
|
---|
250 | // E.g.: sensor might be abruptly unplugged.
|
---|
251 | }
|
---|
252 |
|
---|
253 | this.userInfos = null;
|
---|
254 | }
|
---|
255 |
|
---|
256 | this.sensor = newSensor;
|
---|
257 |
|
---|
258 | if (newSensor != null)
|
---|
259 | {
|
---|
260 | try
|
---|
261 | {
|
---|
262 | this.interactionStream = new InteractionStream(newSensor, this);
|
---|
263 | this.interactionStream.InteractionFrameReady += this.InteractionFrameReadyAsync;
|
---|
264 |
|
---|
265 | this.sensor.SkeletonStream.AppChoosesSkeletons = true;
|
---|
266 |
|
---|
267 | this.userInfos = new UserInfo[InteractionFrame.UserInfoArrayLength];
|
---|
268 | }
|
---|
269 | catch (InvalidOperationException)
|
---|
270 | {
|
---|
271 | // KinectSensor might enter an invalid state while enabling/disabling streams or stream features.
|
---|
272 | // E.g.: sensor might be abruptly unplugged.
|
---|
273 | }
|
---|
274 | }
|
---|
275 |
|
---|
276 | this.userStateManager.Reset();
|
---|
277 | this.userViewerColorizer.ResetColorLookupTable();
|
---|
278 | }
|
---|
279 |
|
---|
280 | /// <summary>
|
---|
281 | /// Process data from one Kinect depth frame.
|
---|
282 | /// </summary>
|
---|
283 | /// <param name="depthData">
|
---|
284 | /// Kinect depth data.
|
---|
285 | /// </param>
|
---|
286 | /// <param name="depthFrame">
|
---|
287 | /// <see cref="DepthImageFrame"/> from which we obtained depth data.
|
---|
288 | /// </param>
|
---|
289 | public override void ProcessDepth(DepthImagePixel[] depthData, DepthImageFrame depthFrame)
|
---|
290 | {
|
---|
291 | if (depthData == null)
|
---|
292 | {
|
---|
293 | throw new ArgumentNullException("depthData");
|
---|
294 | }
|
---|
295 |
|
---|
296 | if (depthFrame == null)
|
---|
297 | {
|
---|
298 | throw new ArgumentNullException("depthFrame");
|
---|
299 | }
|
---|
300 |
|
---|
301 | if (this.ShouldProcessInteractionData)
|
---|
302 | {
|
---|
303 | this.interactionStream.ProcessDepth(depthData, depthFrame.Timestamp);
|
---|
304 | }
|
---|
305 |
|
---|
306 | this.ProcessUserViewerImageAsync(depthData, depthFrame);
|
---|
307 | }
|
---|
308 |
|
---|
309 | /// <summary>
|
---|
310 | /// Process data from one Kinect skeleton frame.
|
---|
311 | /// </summary>
|
---|
312 | /// <param name="skeletons">
|
---|
313 | /// Kinect skeleton data.
|
---|
314 | /// </param>
|
---|
315 | /// <param name="skeletonFrame">
|
---|
316 | /// <see cref="SkeletonFrame"/> from which we obtained skeleton data.
|
---|
317 | /// </param>
|
---|
318 | public override void ProcessSkeleton(Skeleton[] skeletons, SkeletonFrame skeletonFrame)
|
---|
319 | {
|
---|
320 | if (skeletons == null)
|
---|
321 | {
|
---|
322 | throw new ArgumentNullException("skeletons");
|
---|
323 | }
|
---|
324 |
|
---|
325 | if (skeletonFrame == null)
|
---|
326 | {
|
---|
327 | throw new ArgumentNullException("skeletonFrame");
|
---|
328 | }
|
---|
329 |
|
---|
330 | if (this.ShouldProcessInteractionData)
|
---|
331 | {
|
---|
332 | this.interactionStream.ProcessSkeleton(skeletons, this.sensor.AccelerometerGetCurrentReading(), skeletonFrame.Timestamp);
|
---|
333 | }
|
---|
334 |
|
---|
335 | this.userStateManager.ChooseTrackedUsers(skeletons, skeletonFrame.Timestamp, this.recommendedUserTrackingIds);
|
---|
336 |
|
---|
337 | try
|
---|
338 | {
|
---|
339 | this.sensor.SkeletonStream.ChooseSkeletons(
|
---|
340 | this.recommendedUserTrackingIds[0], this.recommendedUserTrackingIds[1]);
|
---|
341 | }
|
---|
342 | catch (InvalidOperationException)
|
---|
343 | {
|
---|
344 | // KinectSensor might enter an invalid state while choosing skeletons.
|
---|
345 | // E.g.: sensor might be abruptly unplugged.
|
---|
346 | }
|
---|
347 | }
|
---|
348 |
|
---|
349 | /// <summary>
|
---|
350 | /// Handle an http request.
|
---|
351 | /// </summary>
|
---|
352 | /// <param name="streamName">
|
---|
353 | /// Name of stream for which property values should be set.
|
---|
354 | /// </param>
|
---|
355 | /// <param name="requestContext">
|
---|
356 | /// Context containing HTTP request data, which will also contain associated
|
---|
357 | /// response upon return.
|
---|
358 | /// </param>
|
---|
359 | /// <param name="subpath">
|
---|
360 | /// Request URI path relative to the stream name associated with this sensor stream
|
---|
361 | /// handler in the stream handler owner.
|
---|
362 | /// </param>
|
---|
363 | /// <returns>
|
---|
364 | /// Await-able task.
|
---|
365 | /// </returns>
|
---|
366 | /// <remarks>
|
---|
367 | /// Return value should never be null. Implementations should use Task.FromResult(0)
|
---|
368 | /// if function is implemented synchronously so that callers can await without
|
---|
369 | /// needing to check for null.
|
---|
370 | /// </remarks>
|
---|
371 | public override Task HandleRequestAsync(string streamName, HttpListenerContext requestContext, string subpath)
|
---|
372 | {
|
---|
373 | if (streamName == null)
|
---|
374 | {
|
---|
375 | throw new ArgumentNullException("streamName");
|
---|
376 | }
|
---|
377 |
|
---|
378 | if (requestContext == null)
|
---|
379 | {
|
---|
380 | throw new ArgumentNullException("requestContext");
|
---|
381 | }
|
---|
382 |
|
---|
383 | if (subpath == null)
|
---|
384 | {
|
---|
385 | throw new ArgumentNullException("subpath");
|
---|
386 | }
|
---|
387 |
|
---|
388 | if (!InteractionStreamName.Equals(streamName))
|
---|
389 | {
|
---|
390 | // Only supported endpoints are related to interaction stream
|
---|
391 | KinectRequestHandler.CloseResponse(requestContext, HttpStatusCode.NotFound);
|
---|
392 | return SharedConstants.EmptyCompletedTask;
|
---|
393 | }
|
---|
394 |
|
---|
395 | var splitPath = KinectRequestHandler.SplitUriSubpath(subpath);
|
---|
396 |
|
---|
397 | if (splitPath == null)
|
---|
398 | {
|
---|
399 | KinectRequestHandler.CloseResponse(requestContext, HttpStatusCode.NotFound);
|
---|
400 | return SharedConstants.EmptyCompletedTask;
|
---|
401 | }
|
---|
402 |
|
---|
403 | var pathComponent = splitPath.Item1;
|
---|
404 | switch (pathComponent)
|
---|
405 | {
|
---|
406 | case ClientUriSubpath:
|
---|
407 | // Only support one client at any one time
|
---|
408 | if (this.clientRpcChannel != null)
|
---|
409 | {
|
---|
410 | if (this.clientRpcChannel.CheckConnectionStatus())
|
---|
411 | {
|
---|
412 | KinectRequestHandler.CloseResponse(requestContext, HttpStatusCode.Conflict);
|
---|
413 | return SharedConstants.EmptyCompletedTask;
|
---|
414 | }
|
---|
415 |
|
---|
416 | this.clientRpcChannel = null;
|
---|
417 | }
|
---|
418 |
|
---|
419 | WebSocketRpcChannel.TryOpenAsync(
|
---|
420 | requestContext,
|
---|
421 | channel =>
|
---|
422 | {
|
---|
423 | // Check again in case another request came in before connection was established
|
---|
424 | if (this.clientRpcChannel != null)
|
---|
425 | {
|
---|
426 | channel.Dispose();
|
---|
427 | }
|
---|
428 |
|
---|
429 | this.clientRpcChannel = channel;
|
---|
430 | },
|
---|
431 | channel =>
|
---|
432 | {
|
---|
433 | // Only forget the current channel if it matches the channel being closed
|
---|
434 | if (this.clientRpcChannel == channel)
|
---|
435 | {
|
---|
436 | this.clientRpcChannel = null;
|
---|
437 | }
|
---|
438 | });
|
---|
439 |
|
---|
440 | break;
|
---|
441 |
|
---|
442 | default:
|
---|
443 | KinectRequestHandler.CloseResponse(requestContext, HttpStatusCode.NotFound);
|
---|
444 | break;
|
---|
445 | }
|
---|
446 |
|
---|
447 | return SharedConstants.EmptyCompletedTask;
|
---|
448 | }
|
---|
449 |
|
---|
450 | /// <summary>
|
---|
451 | /// Cancel all pending operations.
|
---|
452 | /// </summary>
|
---|
453 | public override void Cancel()
|
---|
454 | {
|
---|
455 | if (this.clientRpcChannel != null)
|
---|
456 | {
|
---|
457 | this.clientRpcChannel.Cancel();
|
---|
458 | }
|
---|
459 | }
|
---|
460 |
|
---|
461 | /// <summary>
|
---|
462 | /// Lets handler know that it should clean up resources associated with sensor stream
|
---|
463 | /// handling.
|
---|
464 | /// </summary>
|
---|
465 | /// <returns>
|
---|
466 | /// Await-able task.
|
---|
467 | /// </returns>
|
---|
468 | /// <remarks>
|
---|
469 | /// Return value should never be null. Implementations should use Task.FromResult(0)
|
---|
470 | /// if function is implemented synchronously so that callers can await without
|
---|
471 | /// needing to check for null.
|
---|
472 | /// </remarks>
|
---|
473 | public override async Task UninitializeAsync()
|
---|
474 | {
|
---|
475 | if (this.clientRpcChannel != null)
|
---|
476 | {
|
---|
477 | await this.clientRpcChannel.CloseAsync();
|
---|
478 | }
|
---|
479 | }
|
---|
480 |
|
---|
481 | /// <summary>
|
---|
482 | /// Gets interaction information available for a specified location in UI.
|
---|
483 | /// </summary>
|
---|
484 | /// <param name="skeletonTrackingId">
|
---|
485 | /// The skeleton tracking ID for which interaction information is being retrieved.
|
---|
486 | /// </param>
|
---|
487 | /// <param name="handType">
|
---|
488 | /// The hand type for which interaction information is being retrieved.
|
---|
489 | /// </param>
|
---|
490 | /// <param name="x">
|
---|
491 | /// X-coordinate of UI location for which interaction information is being retrieved.
|
---|
492 | /// 0.0 corresponds to left edge of interaction region and 1.0 corresponds to right edge
|
---|
493 | /// of interaction region.
|
---|
494 | /// </param>
|
---|
495 | /// <param name="y">
|
---|
496 | /// Y-coordinate of UI location for which interaction information is being retrieved.
|
---|
497 | /// 0.0 corresponds to top edge of interaction region and 1.0 corresponds to bottom edge
|
---|
498 | /// of interaction region.
|
---|
499 | /// </param>
|
---|
500 | /// <returns>
|
---|
501 | /// An <see cref="InteractionInfo"/> object instance.
|
---|
502 | /// </returns>
|
---|
503 | public InteractionInfo GetInteractionInfoAtLocation(int skeletonTrackingId, InteractionHandType handType, double x, double y)
|
---|
504 | {
|
---|
505 | var interactionInfo = new InteractionInfo { IsPressTarget = false, IsGripTarget = false };
|
---|
506 |
|
---|
507 | if (this.interactionIsEnabled && (this.clientRpcChannel != null))
|
---|
508 | {
|
---|
509 | var result = this.clientRpcChannel.CallFunction<InteractionStreamHitTestInfo>("getInteractionInfoAtLocation", skeletonTrackingId, handType.ToString(), x, y);
|
---|
510 | if (result.Success)
|
---|
511 | {
|
---|
512 | interactionInfo.IsGripTarget = result.Result.isGripTarget;
|
---|
513 | interactionInfo.IsPressTarget = result.Result.isPressTarget;
|
---|
514 | var elementId = result.Result.pressTargetControlId;
|
---|
515 | interactionInfo.PressTargetControlId = (elementId != null) ? elementId.GetHashCode() : 0;
|
---|
516 | interactionInfo.PressAttractionPointX = result.Result.pressAttractionPointX;
|
---|
517 | interactionInfo.PressAttractionPointY = result.Result.pressAttractionPointY;
|
---|
518 | }
|
---|
519 | }
|
---|
520 |
|
---|
521 | return interactionInfo;
|
---|
522 | }
|
---|
523 |
|
---|
524 | /// <summary>
|
---|
525 | /// Converts a color into the corresponding 32-bit integer RGBA representation.
|
---|
526 | /// </summary>
|
---|
527 | /// <param name="color">
|
---|
528 | /// Color to convert
|
---|
529 | /// </param>
|
---|
530 | /// <returns>
|
---|
531 | /// 32-bit integer RGBA representation of color.
|
---|
532 | /// </returns>
|
---|
533 | internal static int GetRgbaColorInt(Color color)
|
---|
534 | {
|
---|
535 | return (color.A << 24) | (color.B << 16) | (color.G << 8) | color.R;
|
---|
536 | }
|
---|
537 |
|
---|
538 | /// <summary>
|
---|
539 | /// Event handler for InteractionStream's InteractionFrameReady event
|
---|
540 | /// </summary>
|
---|
541 | /// <param name="sender">object sending the event</param>
|
---|
542 | /// <param name="e">event arguments</param>
|
---|
543 | internal async void InteractionFrameReadyAsync(object sender, InteractionFrameReadyEventArgs e)
|
---|
544 | {
|
---|
545 | if (!this.ShouldProcessInteractionData)
|
---|
546 | {
|
---|
547 | return;
|
---|
548 | }
|
---|
549 |
|
---|
550 | if (this.isProcessingInteractionFrame)
|
---|
551 | {
|
---|
552 | // Re-entered InteractionFrameReadyAsync while a previous frame is already being processed.
|
---|
553 | // Just ignore new frames until the current one finishes processing.
|
---|
554 | return;
|
---|
555 | }
|
---|
556 |
|
---|
557 | this.isProcessingInteractionFrame = true;
|
---|
558 |
|
---|
559 | try
|
---|
560 | {
|
---|
561 | bool haveFrameData = false;
|
---|
562 |
|
---|
563 | using (var interactionFrame = e.OpenInteractionFrame())
|
---|
564 | {
|
---|
565 | // Even though we checked value of userInfos above as part of
|
---|
566 | // ShouldProcessInteractionData check, callbacks happening while
|
---|
567 | // opening an interaction frame might have invalidated it, so we
|
---|
568 | // check value again.
|
---|
569 | if ((interactionFrame != null) && (this.userInfos != null))
|
---|
570 | {
|
---|
571 | // Copy interaction frame data so we can dispose interaction frame
|
---|
572 | // right away, even if data processing/event handling takes a while.
|
---|
573 | interactionFrame.CopyInteractionDataTo(this.userInfos);
|
---|
574 | this.interactionStreamMessage.timestamp = interactionFrame.Timestamp;
|
---|
575 | haveFrameData = true;
|
---|
576 | }
|
---|
577 | }
|
---|
578 |
|
---|
579 | if (haveFrameData)
|
---|
580 | {
|
---|
581 | this.userStateManager.UpdateUserInformation(this.userInfos, this.interactionStreamMessage.timestamp);
|
---|
582 | this.userViewerColorizer.UpdateColorLookupTable(this.userInfos, this.userViewerDefaultUserColor, this.userStateManager.UserStates, this.userViewerUserColors);
|
---|
583 |
|
---|
584 | if (this.interactionIsEnabled)
|
---|
585 | {
|
---|
586 | this.interactionStreamMessage.UpdateHandPointers(this.userInfos, this.userStateManager.PrimaryUserTrackingId);
|
---|
587 | await this.ownerContext.SendStreamMessageAsync(this.interactionStreamMessage);
|
---|
588 | }
|
---|
589 | }
|
---|
590 | }
|
---|
591 | finally
|
---|
592 | {
|
---|
593 | this.isProcessingInteractionFrame = false;
|
---|
594 | }
|
---|
595 | }
|
---|
596 |
|
---|
597 | /// <summary>
|
---|
598 | /// Gets all interaction stream property value.
|
---|
599 | /// </summary>
|
---|
600 | /// <param name="propertyMap">
|
---|
601 | /// Property name->value map where property values should be set.
|
---|
602 | /// </param>
|
---|
603 | internal void GetInteractionStreamProperties(Dictionary<string, object> propertyMap)
|
---|
604 | {
|
---|
605 | propertyMap.Add(KinectRequestHandler.EnabledPropertyName, this.interactionIsEnabled);
|
---|
606 | propertyMap.Add(InteractionPrimaryUserPropertyName, this.userStateManager.PrimaryUserTrackingId);
|
---|
607 | propertyMap.Add(InteractionUserStatesPropertyName, DefaultUserStateManager.GetStateMappingEntryArray(this.userStateManager.UserStates));
|
---|
608 | }
|
---|
609 |
|
---|
610 | /// <summary>
|
---|
611 | /// Set an interaction stream property value.
|
---|
612 | /// </summary>
|
---|
613 | /// <param name="propertyName">
|
---|
614 | /// Name of property to set.
|
---|
615 | /// </param>
|
---|
616 | /// <param name="propertyValue">
|
---|
617 | /// Property value to set.
|
---|
618 | /// </param>
|
---|
619 | /// <returns>
|
---|
620 | /// null if property setting was successful, error message otherwise.
|
---|
621 | /// </returns>
|
---|
622 | internal string SetInteractionStreamProperty(string propertyName, object propertyValue)
|
---|
623 | {
|
---|
624 | bool recognized = true;
|
---|
625 |
|
---|
626 | if (propertyValue == null)
|
---|
627 | {
|
---|
628 | // None of the interaction stream properties accept a null value
|
---|
629 | return Properties.Resources.PropertyValueInvalidFormat;
|
---|
630 | }
|
---|
631 |
|
---|
632 | try
|
---|
633 | {
|
---|
634 | switch (propertyName)
|
---|
635 | {
|
---|
636 | case KinectRequestHandler.EnabledPropertyName:
|
---|
637 | this.interactionIsEnabled = (bool)propertyValue;
|
---|
638 | break;
|
---|
639 |
|
---|
640 | default:
|
---|
641 | recognized = false;
|
---|
642 | break;
|
---|
643 | }
|
---|
644 |
|
---|
645 | if (!recognized)
|
---|
646 | {
|
---|
647 | return Properties.Resources.PropertyNameUnrecognized;
|
---|
648 | }
|
---|
649 | }
|
---|
650 | catch (InvalidCastException)
|
---|
651 | {
|
---|
652 | return Properties.Resources.PropertyValueInvalidFormat;
|
---|
653 | }
|
---|
654 |
|
---|
655 | return null;
|
---|
656 | }
|
---|
657 |
|
---|
658 | /// <summary>
|
---|
659 | /// Gets all user viewer stream property value.
|
---|
660 | /// </summary>
|
---|
661 | /// <param name="propertyMap">
|
---|
662 | /// Property name->value map where property values should be set.
|
---|
663 | /// </param>
|
---|
664 | internal void GetUserViewerStreamProperties(Dictionary<string, object> propertyMap)
|
---|
665 | {
|
---|
666 | propertyMap.Add(KinectRequestHandler.EnabledPropertyName, this.userViewerIsEnabled);
|
---|
667 | propertyMap.Add(UserViewerResolutionPropertyName, string.Format(CultureInfo.InvariantCulture, @"{0}x{1}", this.userViewerColorizer.Width, this.userViewerColorizer.Height));
|
---|
668 | propertyMap.Add(UserViewerDefaultUserColorPropertyName, this.userViewerDefaultUserColor);
|
---|
669 | propertyMap.Add(UserViewerUserColorsPropertyName, this.userViewerUserColors);
|
---|
670 | }
|
---|
671 |
|
---|
672 | /// <summary>
|
---|
673 | /// Set a user viewer stream property.
|
---|
674 | /// </summary>
|
---|
675 | /// <param name="propertyName">
|
---|
676 | /// Name of property to set.
|
---|
677 | /// </param>
|
---|
678 | /// <param name="propertyValue">
|
---|
679 | /// Property value to set.
|
---|
680 | /// </param>
|
---|
681 | /// <returns>
|
---|
682 | /// null if property setting was successful, error message otherwise.
|
---|
683 | /// </returns>
|
---|
684 | internal string SetUserViewerStreamProperty(string propertyName, object propertyValue)
|
---|
685 | {
|
---|
686 | bool recognized = true;
|
---|
687 |
|
---|
688 | try
|
---|
689 | {
|
---|
690 | switch (propertyName)
|
---|
691 | {
|
---|
692 | case KinectRequestHandler.EnabledPropertyName:
|
---|
693 | this.userViewerIsEnabled = (bool)propertyValue;
|
---|
694 | break;
|
---|
695 |
|
---|
696 | case UserViewerResolutionPropertyName:
|
---|
697 | if (propertyValue == null)
|
---|
698 | {
|
---|
699 | return Properties.Resources.PropertyValueInvalidFormat;
|
---|
700 | }
|
---|
701 |
|
---|
702 | var match = UserViewerResolutionRegex.Match((string)propertyValue);
|
---|
703 | if (!match.Success || (match.Groups.Count != 3))
|
---|
704 | {
|
---|
705 | return Properties.Resources.PropertyValueInvalidFormat;
|
---|
706 | }
|
---|
707 |
|
---|
708 | int width = int.Parse(match.Groups[1].Value, CultureInfo.InvariantCulture);
|
---|
709 | int height = int.Parse(match.Groups[2].Value, CultureInfo.InvariantCulture);
|
---|
710 |
|
---|
711 | if (!IsSupportedUserViewerResolution(width, height))
|
---|
712 | {
|
---|
713 | return Properties.Resources.PropertyValueUnsupportedResolution;
|
---|
714 | }
|
---|
715 |
|
---|
716 | this.userViewerColorizer.SetResolution(width, height);
|
---|
717 | break;
|
---|
718 |
|
---|
719 | case UserViewerDefaultUserColorPropertyName:
|
---|
720 | this.userViewerDefaultUserColor = (int)propertyValue;
|
---|
721 |
|
---|
722 | this.UpdateColorizerLookupTable();
|
---|
723 | break;
|
---|
724 |
|
---|
725 | case UserViewerUserColorsPropertyName:
|
---|
726 | if (propertyValue == null)
|
---|
727 | {
|
---|
728 | // Null values just clear the set of user colors
|
---|
729 | this.userViewerUserColors.Clear();
|
---|
730 | break;
|
---|
731 | }
|
---|
732 |
|
---|
733 | var userColors = (Dictionary<string, object>)propertyValue;
|
---|
734 |
|
---|
735 | // Verify that all dictionary values are integers
|
---|
736 | bool allIntegers = userColors.Values.Select(color => color as int?).All(colorInt => colorInt != null);
|
---|
737 | if (!allIntegers)
|
---|
738 | {
|
---|
739 | return Properties.Resources.PropertyValueInvalidFormat;
|
---|
740 | }
|
---|
741 |
|
---|
742 | // If property value specified is compatible, copy values over
|
---|
743 | this.userViewerUserColors.Clear();
|
---|
744 | foreach (var entry in userColors)
|
---|
745 | {
|
---|
746 | this.userViewerUserColors.Add(entry.Key, (int)entry.Value);
|
---|
747 | }
|
---|
748 |
|
---|
749 | this.UpdateColorizerLookupTable();
|
---|
750 | break;
|
---|
751 |
|
---|
752 | default:
|
---|
753 | recognized = false;
|
---|
754 | break;
|
---|
755 | }
|
---|
756 |
|
---|
757 | if (!recognized)
|
---|
758 | {
|
---|
759 | return Properties.Resources.PropertyNameUnrecognized;
|
---|
760 | }
|
---|
761 | }
|
---|
762 | catch (InvalidCastException)
|
---|
763 | {
|
---|
764 | return Properties.Resources.PropertyValueInvalidFormat;
|
---|
765 | }
|
---|
766 | catch (NullReferenceException)
|
---|
767 | {
|
---|
768 | return Properties.Resources.PropertyValueInvalidFormat;
|
---|
769 | }
|
---|
770 |
|
---|
771 | return null;
|
---|
772 | }
|
---|
773 |
|
---|
774 | /// <summary>
|
---|
775 | /// Determine if the specified resolution is supported for user viewer images.
|
---|
776 | /// </summary>
|
---|
777 | /// <param name="width">
|
---|
778 | /// Image width.
|
---|
779 | /// </param>
|
---|
780 | /// <param name="height">
|
---|
781 | /// Image height.
|
---|
782 | /// </param>
|
---|
783 | /// <returns>
|
---|
784 | /// True if specified resolution is supported, false otherwise.
|
---|
785 | /// </returns>
|
---|
786 | private static bool IsSupportedUserViewerResolution(int width, int height)
|
---|
787 | {
|
---|
788 | return UserViewerSupportedResolutions.Any(resolution => ((int)resolution.Width == width) && ((int)resolution.Height == height));
|
---|
789 | }
|
---|
790 |
|
---|
791 | /// <summary>
|
---|
792 | /// Handler for IUserStateManager.UserStateChanged event.
|
---|
793 | /// </summary>
|
---|
794 | /// <param name="sender">object sending the event</param>
|
---|
795 | /// <param name="e">event arguments</param>
|
---|
796 | private async void OnUserStateChanged(object sender, UserStateChangedEventArgs e)
|
---|
797 | {
|
---|
798 | if (this.interactionIsEnabled)
|
---|
799 | {
|
---|
800 | // If enabled, forward all user state events to client
|
---|
801 | await this.ownerContext.SendEventMessageAsync(e.Message);
|
---|
802 | }
|
---|
803 | }
|
---|
804 |
|
---|
805 | /// <summary>
|
---|
806 | /// Update colorizer lookup table from color-related configuration.
|
---|
807 | /// </summary>
|
---|
808 | private void UpdateColorizerLookupTable()
|
---|
809 | {
|
---|
810 | if (this.ShouldProcessInteractionData)
|
---|
811 | {
|
---|
812 | this.userViewerColorizer.UpdateColorLookupTable(
|
---|
813 | this.userInfos, this.userViewerDefaultUserColor, this.userStateManager.UserStates, this.userViewerUserColors);
|
---|
814 | }
|
---|
815 | }
|
---|
816 |
|
---|
817 | /// <summary>
|
---|
818 | /// Process depth data to obtain user viewer image.
|
---|
819 | /// </summary>
|
---|
820 | /// <param name="depthData">
|
---|
821 | /// Kinect depth data.
|
---|
822 | /// </param>
|
---|
823 | /// <param name="depthFrame">
|
---|
824 | /// <see cref="DepthImageFrame"/> from which we obtained depth data.
|
---|
825 | /// </param>
|
---|
826 | private async void ProcessUserViewerImageAsync(DepthImagePixel[] depthData, DepthImageFrame depthFrame)
|
---|
827 | {
|
---|
828 | if (this.userViewerIsEnabled)
|
---|
829 | {
|
---|
830 | if (this.isProcessingUserViewerImage)
|
---|
831 | {
|
---|
832 | // Re-entered ProcessUserViewerImageAsync while a previous image is already being processed.
|
---|
833 | // Just ignore new depth frames until the current one finishes processing.
|
---|
834 | return;
|
---|
835 | }
|
---|
836 |
|
---|
837 | this.isProcessingUserViewerImage = true;
|
---|
838 |
|
---|
839 | try
|
---|
840 | {
|
---|
841 | this.userViewerColorizer.ColorizeDepthPixels(depthData, depthFrame.Width, depthFrame.Height);
|
---|
842 | this.userViewerStreamMessage.timestamp = depthFrame.Timestamp;
|
---|
843 | this.userViewerStreamMessage.width = this.userViewerColorizer.Width;
|
---|
844 | this.userViewerStreamMessage.height = this.userViewerColorizer.Height;
|
---|
845 | this.userViewerStreamMessage.bufferLength = this.userViewerColorizer.Buffer.Length;
|
---|
846 |
|
---|
847 | await this.ownerContext.SendTwoPartStreamMessageAsync(this.userViewerStreamMessage, this.userViewerColorizer.Buffer);
|
---|
848 | }
|
---|
849 | finally
|
---|
850 | {
|
---|
851 | this.isProcessingUserViewerImage = false;
|
---|
852 | }
|
---|
853 | }
|
---|
854 | }
|
---|
855 | }
|
---|
856 | }
|
---|