From c3b7a0b814b391c2851e11832427ac76f0efef45 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ha=C3=AF=7E?= Date: Thu, 11 Aug 2022 20:20:26 +0200 Subject: [PATCH] Expose several convergence distances: - Expose two different interpretations of convergence distance. This is more relevant for objects reaching into the intimate space. In both cases, the convergence distance is not factored by looking up, straight, or down. - Distance to the convergence plane (triangle height) This convergence distance corresponds to the distance between the convergence point and an infinite line passing through the two gaze origins. - Distance to the convergence point (length of median line segment) This convergence distance corresponds to the distance between the convergence point and the point between the two gaze origin. - For both interpretations, the distance is exposed as float values from 0 to 1 where the maximum distance saturates at 20 metres, 10 metres, 5 metres, 2 metres, and 1 metre. A float value of 1 would be the maximum distance or above. - For both interpretations, bool parameters indicate if the distance is under 20 metres, 10 metres, 5 metres, 2 metres, 1 metre, 50 cm, 20 cm, and 10 cm. - When the user's eyes happen to be parallel or diverging, then an infinite distance is assumed, causing all floats to saturate at 1. - Eye tracking data is ignored when at least one of the eyes reports as invalid (i.e. when it is closed). In this case, the distance is not updated. --- .../Params/Eye/EyeTrackingParams.cs | 31 ++++++ VRCFaceTracking/UnifiedTrackingData.cs | 102 ++++++++++++++++++ 2 files changed, 133 insertions(+) diff --git a/VRCFaceTracking/Params/Eye/EyeTrackingParams.cs b/VRCFaceTracking/Params/Eye/EyeTrackingParams.cs index daf06370..d1acf916 100644 --- a/VRCFaceTracking/Params/Eye/EyeTrackingParams.cs +++ b/VRCFaceTracking/Params/Eye/EyeTrackingParams.cs @@ -14,6 +14,37 @@ public static class EyeTrackingParams #endregion + #region Convergence + + new EParam(v2 => v2.ConvergencePlaneDistance20M, "ConvergencePlaneDistance20M"), + new EParam(v2 => v2.ConvergencePlaneDistance10M, "ConvergencePlaneDistance10M"), + new EParam(v2 => v2.ConvergencePlaneDistance5M, "ConvergencePlaneDistance5M"), + new EParam(v2 => v2.ConvergencePlaneDistance2M, "ConvergencePlaneDistance2M"), + new EParam(v2 => v2.ConvergencePlaneDistance1M, "ConvergencePlaneDistance1M"), + new BoolParameter(v2 => v2.ConvergencePlaneDistanceRawM < 0.1f, "ConvergencePlaneDistanceUnder10CM"), + new BoolParameter(v2 => v2.ConvergencePlaneDistanceRawM < 0.2f, "ConvergencePlaneDistanceUnder20CM"), + new BoolParameter(v2 => v2.ConvergencePlaneDistanceRawM < 0.5f, "ConvergencePlaneDistanceUnder50CM"), + new BoolParameter(v2 => v2.ConvergencePlaneDistanceRawM < 1f, "ConvergencePlaneDistanceUnder1M"), + new BoolParameter(v2 => v2.ConvergencePlaneDistanceRawM < 2f, "ConvergencePlaneDistanceUnder2M"), + new BoolParameter(v2 => v2.ConvergencePlaneDistanceRawM < 5f, "ConvergencePlaneDistanceUnder5M"), + new BoolParameter(v2 => v2.ConvergencePlaneDistanceRawM < 10f, "ConvergencePlaneDistanceUnder10M"), + new BoolParameter(v2 => v2.ConvergencePlaneDistanceRawM < 20f, "ConvergencePlaneDistanceUnder20M"), + new EParam(v2 => v2.ConvergencePointDistance20M, "ConvergencePointDistance20M"), + new EParam(v2 => v2.ConvergencePointDistance10M, "ConvergencePointDistance10M"), + new EParam(v2 => v2.ConvergencePointDistance5M, "ConvergencePointDistance5M"), + new EParam(v2 => v2.ConvergencePointDistance2M, "ConvergencePointDistance2M"), + new EParam(v2 => v2.ConvergencePointDistance1M, "ConvergencePointDistance1M"), + new BoolParameter(v2 => v2.ConvergencePointDistanceRawM < 0.1f, "ConvergencePointDistanceUnder10CM"), + new BoolParameter(v2 => v2.ConvergencePointDistanceRawM < 0.2f, "ConvergencePointDistanceUnder20CM"), + new BoolParameter(v2 => v2.ConvergencePointDistanceRawM < 0.5f, "ConvergencePointDistanceUnder50CM"), + new BoolParameter(v2 => v2.ConvergencePointDistanceRawM < 1f, "ConvergencePointDistanceUnder1M"), + new BoolParameter(v2 => v2.ConvergencePointDistanceRawM < 2f, "ConvergencePointDistanceUnder2M"), + new BoolParameter(v2 => v2.ConvergencePointDistanceRawM < 5f, "ConvergencePointDistanceUnder5M"), + new BoolParameter(v2 => v2.ConvergencePointDistanceRawM < 10f, "ConvergencePointDistanceUnder10M"), + new BoolParameter(v2 => v2.ConvergencePointDistanceRawM < 20f, "ConvergencePointDistanceUnder20M"), + + #endregion + #region Widen new EParam(v2 => v2.Left.Widen > v2.Right.Widen ? v2.Left.Widen : v2.Right.Widen, "EyesWiden"), diff --git a/VRCFaceTracking/UnifiedTrackingData.cs b/VRCFaceTracking/UnifiedTrackingData.cs index 80caf7f1..2dd0e398 100644 --- a/VRCFaceTracking/UnifiedTrackingData.cs +++ b/VRCFaceTracking/UnifiedTrackingData.cs @@ -1,5 +1,6 @@ using System; using System.Collections.Generic; +using System.Diagnostics; using System.Linq; using ViveSR.anipal.Eye; using ViveSR.anipal.Lip; @@ -47,6 +48,18 @@ public class EyeTrackingData // Custom parameter public float EyesPupilDiameter; + public float ConvergencePlaneDistance20M; + public float ConvergencePlaneDistance10M; + public float ConvergencePlaneDistance5M; + public float ConvergencePlaneDistance2M; + public float ConvergencePlaneDistance1M; + public float ConvergencePlaneDistanceRawM; + public float ConvergencePointDistance20M; + public float ConvergencePointDistance10M; + public float ConvergencePointDistance5M; + public float ConvergencePointDistance2M; + public float ConvergencePointDistance1M; + public float ConvergencePointDistanceRawM; public void UpdateData(EyeData_v2 eyeData) { @@ -78,6 +91,95 @@ public void UpdateData(EyeData_v2 eyeData) EyesDilation = (dilation - _minDilation) / (_maxDilation - _minDilation); EyesPupilDiameter = dilation > 10 ? 1 : dilation / 10; } + + if ( + eyeData.verbose_data.left.GetValidity(SingleEyeDataValidity.SINGLE_EYE_DATA_GAZE_DIRECTION_VALIDITY) + && eyeData.verbose_data.right.GetValidity(SingleEyeDataValidity.SINGLE_EYE_DATA_GAZE_DIRECTION_VALIDITY) + ) + { + UpdateEyeConvergence(eyeData); + } + } + + private void UpdateEyeConvergence(EyeData_v2 eyeData) + { + // In radians + var leftGazeAngle = -Math.Asin(eyeData.verbose_data.left.gaze_direction_normalized.x); + var rightGazeAngle = Math.Asin(eyeData.verbose_data.right.gaze_direction_normalized.x); + + // Form the base of the triangle + var leftComp = Math.PI / 2 - leftGazeAngle; + var rightComp = Math.PI / 2 - rightGazeAngle; + + // The gaze angles are based on the gaze origin, located at the center of the cornea sphere. + // These cornea spheres move, and both get closer to each other when the eyes converge. + // This is effectively the dynamic IPD needed for these calculations, rather than the HMD lenses IPD. + var dynamicIpdMillimeters = eyeData.verbose_data.left.gaze_origin_mm.x - eyeData.verbose_data.right.gaze_origin_mm.x; + + if (leftComp + rightComp >= Math.PI) + { + // Either gazing at infinite distance, or diverging (strabismus) + ConvergencePlaneDistance20M = 1f; + ConvergencePlaneDistance10M = 1f; + ConvergencePlaneDistance5M = 1f; + ConvergencePlaneDistance2M = 1f; + ConvergencePlaneDistance1M = 1f; + ConvergencePlaneDistanceRawM = 10000f; + ConvergencePointDistance20M = 1f; + ConvergencePointDistance10M = 1f; + ConvergencePointDistance5M = 1f; + ConvergencePointDistance2M = 1f; + ConvergencePointDistance1M = 1f; + ConvergencePointDistanceRawM = 10000f; + } + else + { + // Basic trigonometry + // - Calculate the left side of the triangle where leftComp, rightComp, and IPD form the base. + var leftSideMillimetres = Math.Sin(rightComp) * dynamicIpdMillimeters / Math.Sin(Math.PI - leftComp - rightComp); + + // Handle two different interpretations of convergence distance. This is more relevant for objects reaching into the intimate space. + // (In both cases, the convergence distance is not factored by looking up, straight, or down) + { + // # Distance to the convergence plane (triangle height) + // This convergence distance corresponds to the distance between the convergence point + // and an infinite line passing through the two gaze origins (note: gaze origin is not the center of eyeballs). + + // - Calculate the height of the right triangle where leftComp is the angle opposite to the convergence distance + var convergenceDistanceMillimetres = leftSideMillimetres * Math.Sin(leftComp); + + var convergenceDistanceMetres = (float)convergenceDistanceMillimetres / 1000f; + ConvergencePlaneDistance20M = Saturate(convergenceDistanceMetres / 20f); + ConvergencePlaneDistance10M = Saturate(convergenceDistanceMetres / 10f); + ConvergencePlaneDistance5M = Saturate(convergenceDistanceMetres / 5f); + ConvergencePlaneDistance2M = Saturate(convergenceDistanceMetres / 2f); + ConvergencePlaneDistance1M = Saturate(convergenceDistanceMetres / 1f); + ConvergencePlaneDistanceRawM = convergenceDistanceMetres; + } + { + // # Distance to the convergence point (length of median line segment) + // This convergence distance corresponds to the distance between the convergence point + // and the point between the two gaze origins (also known as combined gaze origin). + + // - Calculate the right side of the triangle where leftComp, rightComp, and IPD form the base. + var rightSideMillimetres = Math.Sin(leftComp) * dynamicIpdMillimeters / Math.Sin(Math.PI - leftComp - rightComp); + // - The median length is the average of the two triangle sides + var convergenceDistanceMillimetres = (leftSideMillimetres + rightSideMillimetres) / 2; + + var convergenceDistanceMetres = (float)convergenceDistanceMillimetres / 1000f; + ConvergencePointDistance20M = Saturate(convergenceDistanceMetres / 20f); + ConvergencePointDistance10M = Saturate(convergenceDistanceMetres / 10f); + ConvergencePointDistance5M = Saturate(convergenceDistanceMetres / 5f); + ConvergencePointDistance2M = Saturate(convergenceDistanceMetres / 2f); + ConvergencePointDistance1M = Saturate(convergenceDistanceMetres / 1f); + ConvergencePointDistanceRawM = convergenceDistanceMetres; + } + } + } + + private float Saturate(float convergenceDistance) + { + return convergenceDistance > 1f ? 1f : convergenceDistance; } private void UpdateMinMaxDilation(float readDilation)