source: other-projects/playing-in-the-street/summer-2013/trunk/Playing-in-the-Street-WPF/Microsoft.Samples.Kinect.Webserver/Sensor/DefaultUserStateManager.cs@ 28897

Last change on this file since 28897 was 28897, checked in by davidb, 10 years ago

GUI front-end to server base plus web page content
File size: 25.7 KB
Line 
1// -----------------------------------------------------------------------
2// <copyright file="DefaultUserStateManager.cs" company="Microsoft">
3// Copyright (c) Microsoft Corporation. All rights reserved.
4// </copyright>
5// -----------------------------------------------------------------------
6
7namespace Microsoft.Samples.Kinect.Webserver.Sensor
8{
9 using System;
10 using System.Collections.Generic;
11 using System.Linq;
12
13 using Microsoft.Kinect;
14 using Microsoft.Kinect.Toolkit;
15 using Microsoft.Kinect.Toolkit.Interaction;
16 using Microsoft.Samples.Kinect.Webserver.Sensor.Serialization;
17
18 /// <summary>
19 /// Default implementation of <see cref="IUserStateManager"/> interface.
20 /// </summary>
21 public class DefaultUserStateManager : IUserStateManager
22 {
23 /// <summary>
24 /// Name of state representing a tracked user.
25 /// </summary>
26 public const string TrackedStateName = "tracked";
27
28 /// <summary>
29 /// Name of state representing an engaged user.
30 /// </summary>
31 public const string EngagedStateName = "engaged";
32
33 /// <summary>
34 /// Category of all events originating from this class.
35 /// </summary>
36 public const string EventCategory = "userState";
37
38 /// <summary>
39 /// Event type for primary user changed event.
40 /// </summary>
41 public const string PrimaryUserChangedEventType = "primaryUserChanged";
42
43 /// <summary>
44 /// Event type for user state changed event.
45 /// </summary>
46 public const string UserStatesChangedEventType = "userStatesChanged";
47
48 /// <summary>
49 /// Length (in milliseconds) of period of inactivity required
50 /// before users become candidates for tracking.
51 /// </summary>
52 internal const long MinimumInactivityBeforeTrackingMilliseconds = 500;
53
54 /// <summary>
55 /// Object used to keep track of user activity.
56 /// </summary>
57 private readonly UserActivityMeter activityMeter = new UserActivityMeter();
58
59 /// <summary>
60 /// Object used to synchronize modifications to engagement state.
61 /// </summary>
62 private readonly object lockObject = new object();
63
64 /// <summary>
65 /// Helper object used to keep track of previous set of tracked user ids while avoiding
66 /// object allocation while processing each frame.
67 /// </summary>
68 private HashSet<int> previousTrackedUserTrackingIds = new HashSet<int>();
69
70 /// <summary>
71 /// Map between user tracking IDs and user states exposed to clients.
72 /// </summary>
73 private Dictionary<int, string> publicUserStates = new Dictionary<int, string>();
74
75 /// <summary>
76 /// Dictionary used to accumulate mappings between user tracking IDs and user states
77 /// before they're ready to be exposed to clients.
78 /// </summary>
79 private Dictionary<int, string> userStatesAccumulator = new Dictionary<int, string>();
80
81 /// <summary>
82 /// Initializes a new instance of the <see cref="DefaultUserStateManager"/> class.
83 /// </summary>
84 public DefaultUserStateManager()
85 {
86 this.TrackedUserTrackingIds = new HashSet<int>();
87 }
88
89 /// <summary>
90 /// Event triggered whenever user state changes.
91 /// </summary>
92 public event EventHandler<UserStateChangedEventArgs> UserStateChanged;
93
94 /// <summary>
95 /// Dictionary mapping user tracking Ids to names used for states corresponding to
96 /// those users.
97 /// </summary>
98 public IDictionary<int, string> UserStates
99 {
100 get
101 {
102 return this.publicUserStates;
103 }
104 }
105
106 /// <summary>
107 /// Tracking ID corresponding to primary user.
108 /// </summary>
109 /// <remarks>
110 /// May be an invalid tracking id to represent that no user is currently primary.
111 /// </remarks>
112 public int PrimaryUserTrackingId { get; set; }
113
114 /// <summary>
115 /// Tracking ID corresponding to engaged user.
116 /// </summary>
117 /// <remarks>
118 /// May be an invalid tracking id to represent that no user is currently engaged.
119 /// </remarks>
120 private int EngagedUserTrackingId { get; set; }
121
122 /// <summary>
123 /// Set of tracking Ids corresponding to users currently considered to be tracked.
124 /// </summary>
125 private HashSet<int> TrackedUserTrackingIds { get; set; }
126
127 /// <summary>
128 /// Resets all state to the initial state, with no users remembered as engaged or tracked.
129 /// </summary>
130 public void Reset()
131 {
132 using (var callbackLock = new CallbackLock(this.lockObject))
133 {
134 this.activityMeter.Clear();
135 this.TrackedUserTrackingIds.Clear();
136 this.EngagedUserTrackingId = SharedConstants.InvalidUserTrackingId;
137 this.SetPrimaryUserTrackingId(SharedConstants.InvalidUserTrackingId, callbackLock);
138 this.UpdateUserStates(callbackLock);
139 }
140 }
141
142 /// <summary>
143 /// Determines which users should be tracked in the future, based on selection
144 /// metrics and engagement state.
145 /// </summary>
146 /// <param name="frameSkeletons">
147 /// Array of skeletons from which the appropriate user tracking Ids will be selected.
148 /// </param>
149 /// <param name="timestamp">
150 /// Timestamp from skeleton frame.
151 /// </param>
152 /// <param name="chosenTrackingIds">
153 /// Array that will contain the tracking Ids of users to track, sorted from most
154 /// important to least important user to track.
155 /// </param>
156 public void ChooseTrackedUsers(Skeleton[] frameSkeletons, long timestamp, int[] chosenTrackingIds)
157 {
158 if (frameSkeletons == null)
159 {
160 throw new ArgumentNullException("frameSkeletons");
161 }
162
163 if (chosenTrackingIds == null)
164 {
165 throw new ArgumentNullException("chosenTrackingIds");
166 }
167
168 var availableSkeletons = new List<Skeleton>(
169 from skeleton in frameSkeletons
170 where
171 (skeleton.TrackingId != SharedConstants.InvalidUserTrackingId)
172 &&
173 ((skeleton.TrackingState == SkeletonTrackingState.Tracked)
174 || (skeleton.TrackingState == SkeletonTrackingState.PositionOnly))
175 select skeleton);
176 var trackingCandidateSkeletons = new List<Skeleton>();
177
178 // Update user activity metrics
179 this.activityMeter.Update(availableSkeletons, timestamp);
180
181 foreach (var skeleton in availableSkeletons)
182 {
183 UserActivityRecord record;
184 if (this.activityMeter.TryGetActivityRecord(skeleton.TrackingId, out record))
185 {
186 // The tracked skeletons become candidate skeletons for tracking if we have an activity record for them.
187 trackingCandidateSkeletons.Add(skeleton);
188 }
189 }
190
191 // sort the currently tracked skeletons according to our tracking choice criteria
192 trackingCandidateSkeletons.Sort((left, right) => this.ComputeTrackingMetric(right).CompareTo(this.ComputeTrackingMetric(left)));
193
194 for (int i = 0; i < chosenTrackingIds.Length; ++i)
195 {
196 chosenTrackingIds[i] = (i < trackingCandidateSkeletons.Count) ? trackingCandidateSkeletons[i].TrackingId : SharedConstants.InvalidUserTrackingId;
197 }
198 }
199
200 /// <summary>
201 /// Called whenever the set of tracked users has changed.
202 /// </summary>
203 /// <param name="trackedUserInfo">
204 /// User information from which we'll update the set of tracked users and the primary user.
205 /// </param>
206 /// <param name="timestamp">
207 /// Interaction frame timestamp corresponding to given user information.
208 /// </param>
209 public void UpdateUserInformation(IEnumerable<UserInfo> trackedUserInfo, long timestamp)
210 {
211 bool foundEngagedUser = false;
212 int firstTrackedUser = SharedConstants.InvalidUserTrackingId;
213
214 using (var callbackLock = new CallbackLock(this.lockObject))
215 {
216 this.previousTrackedUserTrackingIds.Clear();
217 var nextTrackedIds = this.previousTrackedUserTrackingIds;
218 this.previousTrackedUserTrackingIds = this.TrackedUserTrackingIds;
219 this.TrackedUserTrackingIds = nextTrackedIds;
220
221 var trackedUserInfoArray = trackedUserInfo as UserInfo[] ?? trackedUserInfo.ToArray();
222
223 foreach (var userInfo in trackedUserInfoArray)
224 {
225 if (userInfo.SkeletonTrackingId == SharedConstants.InvalidUserTrackingId)
226 {
227 continue;
228 }
229
230 if (this.EngagedUserTrackingId == userInfo.SkeletonTrackingId)
231 {
232 this.TrackedUserTrackingIds.Add(userInfo.SkeletonTrackingId);
233
234 foundEngagedUser = true;
235 }
236 else if (HasTrackedHands(userInfo)
237 && (this.previousTrackedUserTrackingIds.Contains(userInfo.SkeletonTrackingId)
238 || this.IsInactive(userInfo, timestamp)))
239 {
240 // Keep track of the non-engaged users we find that have at least one
241 // tracked hand pointer and also either (1) were previously tracked or
242 // (2) are not moving too much
243 this.TrackedUserTrackingIds.Add(userInfo.SkeletonTrackingId);
244
245 if (firstTrackedUser == SharedConstants.InvalidUserTrackingId)
246 {
247 // Consider the first non-engaged, stationary user as a candidate for engagement
248 firstTrackedUser = userInfo.SkeletonTrackingId;
249 }
250 }
251 }
252
253 // If engaged user was not found in list of candidate users, engaged user has become invalid.
254 if (!foundEngagedUser)
255 {
256 this.EngagedUserTrackingId = SharedConstants.InvalidUserTrackingId;
257 }
258
259 // Decide who should be the primary user, if anyone
260 this.UpdatePrimaryUser(trackedUserInfoArray, callbackLock);
261
262 // If there's a primary user, it is the preferred candidate for engagement.
263 // Otherwise, the first tracked user seen is the preferred candidate.
264 int candidateUserTrackingId = (this.PrimaryUserTrackingId != SharedConstants.InvalidUserTrackingId)
265 ? this.PrimaryUserTrackingId
266 : firstTrackedUser;
267
268 // If there is a valid candidate user that is not already the engaged user
269 if ((candidateUserTrackingId != SharedConstants.InvalidUserTrackingId)
270 && (candidateUserTrackingId != this.EngagedUserTrackingId))
271 {
272 // If there is currently no engaged user, or if candidate user is the
273 // primary user controlling interactions while the currently engaged user
274 // is not interacting
275 if ((this.EngagedUserTrackingId == SharedConstants.InvalidUserTrackingId)
276 || (candidateUserTrackingId == this.PrimaryUserTrackingId))
277 {
278 this.PromoteCandidateToEngaged(candidateUserTrackingId);
279 }
280 }
281
282 // Update user states as the very last action, to include results from updates
283 // performed so far
284 this.UpdateUserStates(callbackLock);
285 }
286 }
287
288 /// <summary>
289 /// Promote candidate user to be the engaged user.
290 /// </summary>
291 /// <param name="candidateTrackingId">
292 /// Tracking Id of user to be promoted to engaged user.
293 /// If tracking Id does not match the Id of one of the currently tracked users,
294 /// no action is taken.
295 /// </param>
296 /// <returns>
297 /// True if specified candidate could be confirmed as the new engaged user,
298 /// false otherwise.
299 /// </returns>
300 public bool PromoteCandidateToEngaged(int candidateTrackingId)
301 {
302 bool isConfirmed = false;
303
304 if ((candidateTrackingId != SharedConstants.InvalidUserTrackingId) && this.TrackedUserTrackingIds.Contains(candidateTrackingId))
305 {
306 using (var callbackLock = new CallbackLock(this.lockObject))
307 {
308 this.EngagedUserTrackingId = candidateTrackingId;
309 this.UpdateUserStates(callbackLock);
310 }
311
312 isConfirmed = true;
313 }
314
315 return isConfirmed;
316 }
317
318 /// <summary>
319 /// Tries to get the last position observed for the specified user tracking Id.
320 /// </summary>
321 /// <param name="trackingId">
322 /// User tracking Id for which we're finding the last position observed.
323 /// </param>
324 /// <returns>
325 /// Skeleton point, if last position is being tracked for specified
326 /// tracking Id, null otherwise.
327 /// </returns>
328 public SkeletonPoint? TryGetLastPositionForId(int trackingId)
329 {
330 if (SharedConstants.InvalidUserTrackingId == trackingId)
331 {
332 return null;
333 }
334
335 UserActivityRecord record;
336 if (this.activityMeter.TryGetActivityRecord(trackingId, out record))
337 {
338 return record.LastPosition;
339 }
340
341 return null;
342 }
343
344 /// <summary>
345 /// Get a JSON friendly array of user-tracking-id-to-state mapping entries
346 /// representing the specified user state map.
347 /// </summary>
348 /// <param name="userStates">
349 /// Dictionary mapping user tracking ids to user state names.
350 /// </param>
351 /// <returns>
352 /// Array of <see cref="StateMappingEntry"/> objects.
353 /// </returns>
354 internal static StateMappingEntry[] GetStateMappingEntryArray(IDictionary<int, string> userStates)
355 {
356 var mappingEntries = new StateMappingEntry[userStates.Count];
357 int entryIndex = 0;
358 foreach (var userStateEntry in userStates)
359 {
360 mappingEntries[entryIndex] = new StateMappingEntry { id = userStateEntry.Key, userState = userStateEntry.Value };
361 ++entryIndex;
362 }
363
364 return mappingEntries;
365 }
366
367 internal void SetPrimaryUserTrackingId(int newId, CallbackLock callbackLock)
368 {
369 int oldId = this.PrimaryUserTrackingId;
370 this.PrimaryUserTrackingId = newId;
371
372 if (oldId != newId)
373 {
374 callbackLock.LockExit +=
375 () =>
376 this.SendUserStateChanged(
377 new UserTrackingIdChangedEventMessage
378 {
379 category = EventCategory,
380 eventType = PrimaryUserChangedEventType,
381 oldValue = oldId,
382 newValue = newId
383 });
384 }
385 }
386
387 /// <summary>
388 /// Determine if any of the specified user's hands is tracked.
389 /// </summary>
390 /// <param name="userInfo">
391 /// User information from which to determine hand tracking status.
392 /// </param>
393 /// <returns>
394 /// True if user has at least one tracked hand pointer. False otherwise.
395 /// </returns>
396 private static bool HasTrackedHands(UserInfo userInfo)
397 {
398 return userInfo.HandPointers.Any(handPointer => handPointer.IsTracked);
399 }
400
401 /// <summary>
402 /// Update the primary user being tracked.
403 /// </summary>
404 /// <param name="candidateUserInfo">
405 /// User information collection from which we will choose a primary user.
406 /// </param>
407 /// <param name="callbackLock">
408 /// Lock used to delay all events until after we exit lock section.
409 /// </param>
410 private void UpdatePrimaryUser(IEnumerable<UserInfo> candidateUserInfo, CallbackLock callbackLock)
411 {
412 int firstPrimaryUserCandidate = SharedConstants.InvalidUserTrackingId;
413 bool currentPrimaryUserStillPrimary = false;
414 bool engagedUserIsPrimary = false;
415
416 var trackingIdsAvailable = new HashSet<int>();
417
418 foreach (var userInfo in candidateUserInfo)
419 {
420 if (userInfo.SkeletonTrackingId == SharedConstants.InvalidUserTrackingId)
421 {
422 continue;
423 }
424
425 trackingIdsAvailable.Add(userInfo.SkeletonTrackingId);
426
427 foreach (var handPointer in userInfo.HandPointers)
428 {
429 if (handPointer.IsPrimaryForUser)
430 {
431 if (this.PrimaryUserTrackingId == userInfo.SkeletonTrackingId)
432 {
433 // If the current primary user still has an active hand, we should continue to consider them the primary user.
434 currentPrimaryUserStillPrimary = true;
435 }
436 else if (SharedConstants.InvalidUserTrackingId == firstPrimaryUserCandidate)
437 {
438 // Else if this is the first user with an active hand, they are the alternative candidate for primary user.
439 firstPrimaryUserCandidate = userInfo.SkeletonTrackingId;
440 }
441
442 if (this.EngagedUserTrackingId == userInfo.SkeletonTrackingId)
443 {
444 engagedUserIsPrimary = true;
445 }
446 }
447 }
448 }
449
450 // If engaged user has a primary hand, always pick that user as primary user.
451 // If current primary user still has a primary hand, let them remain primary.
452 // Otherwise default to first primary user candidate seen.
453 int primaryUserTrackingId = engagedUserIsPrimary
454 ? this.EngagedUserTrackingId
455 : (currentPrimaryUserStillPrimary ? this.PrimaryUserTrackingId : firstPrimaryUserCandidate);
456 this.SetPrimaryUserTrackingId(primaryUserTrackingId, callbackLock);
457 }
458
459 /// <summary>
460 /// Calculate how valuable it will be to keep tracking the specified skeleton.
461 /// </summary>
462 /// <param name="skeleton">
463 /// Skeleton that is one of several candidates for tracking.
464 /// </param>
465 /// <returns>
466 /// A non-negative metric that estimates how valuable it is to keep tracking
467 /// the specified skeleton. The higher the value, the more valuable the skeleton
468 /// is estimated to be.
469 /// </returns>
470 private double ComputeTrackingMetric(Skeleton skeleton)
471 {
472 const double MaxCameraDistance = 4.0;
473
474 // Give preference to engaged users, then to tracked users, then to users
475 // near the center of the Kinect Sensor's field of view that are also
476 // closer (distance) to the KinectSensor and not moving around too much.
477 const double EngagedWeight = 100.0;
478 const double TrackedWeight = 50.0;
479 const double AngleFromCenterWeight = 1.30;
480 const double DistanceFromCameraWeight = 1.15;
481 const double BodyMovementWeight = 0.05;
482
483 double engagedMetric = (skeleton.TrackingId == this.EngagedUserTrackingId) ? 1.0 : 0.0;
484 double trackedMetric = this.TrackedUserTrackingIds.Contains(skeleton.TrackingId) ? 1.0 : 0.0;
485 double angleFromCenterMetric = (skeleton.Position.Z > 0.0) ? (1.0 - Math.Abs(2 * Math.Atan(skeleton.Position.X / skeleton.Position.Z) / Math.PI)) : 0.0;
486 double distanceFromCameraMetric = (MaxCameraDistance - skeleton.Position.Z) / MaxCameraDistance;
487 UserActivityRecord activityRecord;
488 double bodyMovementMetric = this.activityMeter.TryGetActivityRecord(skeleton.TrackingId, out activityRecord)
489 ? 1.0 - activityRecord.ActivityLevel
490 : 0.0;
491 return (EngagedWeight * engagedMetric) +
492 (TrackedWeight * trackedMetric) +
493 (AngleFromCenterWeight * angleFromCenterMetric) +
494 (DistanceFromCameraWeight * distanceFromCameraMetric) +
495 (BodyMovementWeight * bodyMovementMetric);
496 }
497
498 /// <summary>
499 /// Determine if the specified user information represents a user that has been
500 /// relatively inactive for at least a minimum period of time required for tracking.
501 /// </summary>
502 /// <param name="userInfo">
503 /// User information from which to determine inactivity.
504 /// </param>
505 /// <param name="timestamp">
506 /// Current timestamp used to determine how long user has been inactive.
507 /// </param>
508 /// <returns>
509 /// True if user is present in scene and has been inactive for a minimum threshold
510 /// period of time.
511 /// </returns>
512 private bool IsInactive(UserInfo userInfo, long timestamp)
513 {
514 UserActivityRecord record;
515 return this.activityMeter.TryGetActivityRecord(userInfo.SkeletonTrackingId, out record) && !record.IsActive
516 && (record.StateTransitionTimestamp + MinimumInactivityBeforeTrackingMilliseconds <= timestamp);
517 }
518
519 /// <summary>
520 /// Determines if user states have changed.
521 /// </summary>
522 /// <returns>
523 /// true if accumulated user states are different from the ones currently visible
524 /// to clients.
525 /// </returns>
526 private bool HaveUserStatesChanged()
527 {
528 if (this.publicUserStates.Count != this.userStatesAccumulator.Count)
529 {
530 return true;
531 }
532
533 foreach (var stateEntry in this.publicUserStates)
534 {
535 string accumulatorState;
536 if (!this.userStatesAccumulator.TryGetValue(stateEntry.Key, out accumulatorState))
537 {
538 // Key is absent from accumulator but present in current state map
539 return true;
540 }
541
542 if (!stateEntry.Value.Equals(accumulatorState))
543 {
544 // state names are present in both maps, but they're different
545 return true;
546 }
547 }
548
549 return false;
550 }
551
552 /// <summary>
553 /// Update user states exposed to clients, if necessary.
554 /// </summary>
555 /// <param name="callbackLock">
556 /// Lock used to delay all events until after we exit lock section.
557 /// </param>
558 private void UpdateUserStates(CallbackLock callbackLock)
559 {
560 this.userStatesAccumulator.Clear();
561
562 // Add states for tracked users
563 foreach (var trackingId in this.TrackedUserTrackingIds)
564 {
565 this.userStatesAccumulator.Add(trackingId, TrackedStateName);
566 }
567
568 if (this.EngagedUserTrackingId != SharedConstants.InvalidUserTrackingId)
569 {
570 // Engaged state supersedes all other states
571 this.userStatesAccumulator[this.EngagedUserTrackingId] = EngagedStateName;
572 }
573
574 if (this.HaveUserStatesChanged())
575 {
576 var temporaryMap = this.publicUserStates;
577 this.publicUserStates = this.userStatesAccumulator;
578 this.userStatesAccumulator = temporaryMap;
579
580 var userStatesToSend = GetStateMappingEntryArray(this.publicUserStates);
581
582 callbackLock.LockExit +=
583 () =>
584 this.SendUserStateChanged(
585 new UserStatesChangedEventMessage
586 {
587 category = EventCategory,
588 eventType = UserStatesChangedEventType,
589 userStates = userStatesToSend
590 });
591 }
592 }
593
594 /// <summary>
595 /// Send UserStateChanged event if there are any subscribers.
596 /// </summary>
597 /// <param name="message">
598 /// Message to send.
599 /// </param>
600 private void SendUserStateChanged(EventMessage message)
601 {
602 if (this.UserStateChanged != null)
603 {
604 this.UserStateChanged(this, new UserStateChangedEventArgs(message));
605 }
606 }
607 }
608}
Note: See TracBrowser for help on using the repository browser.