You cannot select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
185 lines
7.3 KiB
Python
185 lines
7.3 KiB
Python
import numpy as np
|
|
|
|
|
|
class SpecialGestureDetector:
|
|
"""
|
|
Detects special static gestures using either simple geometric rules or an ML classifier.
|
|
|
|
Supported gesture labels:
|
|
- 'cross' : forearms crossed near the chest
|
|
- 'light' : right arm pose approximating a 'light' toggle
|
|
- 'dome' : arms forming a dome above the head
|
|
- 'none' : no special gesture detected
|
|
"""
|
|
|
|
def __init__(self, mode='geometric', model_path=None, class_names=None, debug=False, thresholds=None):
|
|
self.mode = mode
|
|
self.debug = debug
|
|
|
|
# Defaults for geometric detection
|
|
self.th = {
|
|
'min_conf': 0.5,
|
|
'shoulder_width_min': 30.0,
|
|
'torso_height_min': 10.0,
|
|
'chest_band': 0.25, # widened to be more forgiving
|
|
'wrists_near_factor': 0.6, # relaxed for dome
|
|
'elbow_far_factor': 0.9, # relaxed for dome
|
|
'light_elbow_min': 45.0,
|
|
'light_elbow_max': 120.0,
|
|
'light_shoulder_min': -5.0,
|
|
'light_shoulder_max': 20.0
|
|
}
|
|
if thresholds:
|
|
self.th.update(thresholds)
|
|
|
|
if mode == 'ml':
|
|
from ml_gestures.predict import MLGesturePredictor
|
|
if model_path is None or class_names is None:
|
|
raise ValueError("For ML mode, provide model_path and class_names")
|
|
self.ml_predictor = MLGesturePredictor(model_path, class_names)
|
|
if self.debug:
|
|
print("SpecialGestureDetector: Using ML classifier")
|
|
else:
|
|
self.ml_predictor = None
|
|
if self.debug:
|
|
print("SpecialGestureDetector: Using geometric rules")
|
|
|
|
def predict(self, landmarks):
|
|
if landmarks is None:
|
|
return 'none'
|
|
if self.mode == 'geometric':
|
|
return self._geometric_predict(landmarks)
|
|
else:
|
|
return self.ml_predictor.predict(landmarks)
|
|
|
|
def _geometric_predict(self, landmarks):
|
|
idx = {
|
|
'nose': 0,
|
|
'left_shoulder': 11, 'right_shoulder': 12,
|
|
'left_elbow': 13, 'right_elbow': 14,
|
|
'left_wrist': 15, 'right_wrist': 16,
|
|
'left_hip': 23, 'right_hip': 24,
|
|
}
|
|
|
|
min_conf = self.th['min_conf']
|
|
# Only upper-body required (hips optional)
|
|
required = [
|
|
'left_shoulder', 'right_shoulder',
|
|
'left_elbow', 'right_elbow',
|
|
'left_wrist', 'right_wrist',
|
|
'nose'
|
|
]
|
|
for p in required:
|
|
if landmarks[idx[p]][3] < min_conf:
|
|
if self.debug:
|
|
print(f"[SG] Low confidence for {p}: {landmarks[idx[p]][3]:.2f}")
|
|
return 'none'
|
|
|
|
l_sh = np.array(landmarks[idx['left_shoulder']][:2], dtype=float)
|
|
r_sh = np.array(landmarks[idx['right_shoulder']][:2], dtype=float)
|
|
l_el = np.array(landmarks[idx['left_elbow']][:2], dtype=float)
|
|
r_el = np.array(landmarks[idx['right_elbow']][:2], dtype=float)
|
|
l_wr = np.array(landmarks[idx['left_wrist']][:2], dtype=float)
|
|
r_wr = np.array(landmarks[idx['right_wrist']][:2], dtype=float)
|
|
nose = np.array(landmarks[idx['nose']][:2], dtype=float)
|
|
|
|
l_hip = np.array(landmarks[idx['left_hip']][:2], dtype=float)
|
|
r_hip = np.array(landmarks[idx['right_hip']][:2], dtype=float)
|
|
c_lhip = landmarks[idx['left_hip']][3]
|
|
c_rhip = landmarks[idx['right_hip']][3]
|
|
|
|
shoulder_center_y = (l_sh[1] + r_sh[1]) / 2.0
|
|
shoulder_width = np.linalg.norm(r_sh - l_sh)
|
|
if shoulder_width < self.th['shoulder_width_min']:
|
|
if self.debug:
|
|
print(f"[SG] Shoulder width too small: {shoulder_width:.1f}")
|
|
return 'none'
|
|
|
|
# Torso height: prefer hips if visible, otherwise fallback using nose/shoulders
|
|
if c_lhip >= min_conf and c_rhip >= min_conf:
|
|
hip_center_y = (l_hip[1] + r_hip[1]) / 2.0
|
|
torso_height = hip_center_y - shoulder_center_y
|
|
else:
|
|
nose_to_shoulder = abs(nose[1] - shoulder_center_y)
|
|
torso_height = max(1.6 * nose_to_shoulder, 0.9 * shoulder_width)
|
|
hip_center_y = shoulder_center_y + torso_height
|
|
|
|
if torso_height < self.th['torso_height_min']:
|
|
if self.debug:
|
|
print(f"[SG] Torso height too small: {torso_height:.1f}")
|
|
return 'none'
|
|
|
|
def angle_between_vectors(v1, v2):
|
|
n1 = np.linalg.norm(v1)
|
|
n2 = np.linalg.norm(v2)
|
|
if n1 < 1e-6 or n2 < 1e-6:
|
|
return 0.0
|
|
cos_a = np.dot(v1, v2) / (n1 * n2)
|
|
cos_a = float(np.clip(cos_a, -1.0, 1.0))
|
|
return np.degrees(np.arccos(cos_a))
|
|
|
|
def joint_angle(p_prev, p_joint, p_next):
|
|
v1 = p_prev - p_joint
|
|
v2 = p_next - p_joint
|
|
return angle_between_vectors(v1, v2)
|
|
|
|
def segments_intersect(p1, p2, p3, p4):
|
|
def cross(o, a, b):
|
|
return (a[0] - o[0]) * (b[1] - o[1]) - (a[1] - o[1]) * (b[0] - o[0])
|
|
d1 = cross(p3, p4, p1)
|
|
d2 = cross(p3, p4, p2)
|
|
d3 = cross(p1, p2, p3)
|
|
d4 = cross(p1, p2, p4)
|
|
return (d1 * d2 < 0) and (d3 * d4 < 0)
|
|
|
|
def line_intersection(p1, p2, p3, p4):
|
|
d1 = p2 - p1
|
|
d2 = p4 - p3
|
|
denom = d1[0] * d2[1] - d1[1] * d2[0]
|
|
if abs(denom) < 1e-6:
|
|
return None
|
|
t = ((p3[0] - p1[0]) * d2[1] - (p3[1] - p1[1]) * d2[0]) / denom
|
|
return p1 + t * d1
|
|
|
|
# Angles (geometric cues)
|
|
l_elbow_angle = joint_angle(l_sh, l_el, l_wr)
|
|
r_elbow_angle = joint_angle(r_sh, r_el, r_wr)
|
|
r_shoulder_like_angle = joint_angle(r_hip, r_sh, r_el)
|
|
|
|
# 1) CROSS
|
|
forearms_cross = segments_intersect(l_el, l_wr, r_el, r_wr)
|
|
intersection = line_intersection(l_el, l_wr, r_el, r_wr)
|
|
intersection_on_chest = False
|
|
if intersection is not None:
|
|
band = self.th['chest_band'] * torso_height
|
|
intersection_on_chest = (shoulder_center_y - band) < intersection[1] < (hip_center_y + band)
|
|
|
|
if self.debug:
|
|
print(f"[SG] cross_check: intersect={forearms_cross}, chest={intersection_on_chest}")
|
|
|
|
if forearms_cross and intersection_on_chest:
|
|
return 'cross'
|
|
|
|
# 2) LIGHT
|
|
if (self.th['light_elbow_min'] < r_elbow_angle < self.th['light_elbow_max'] and
|
|
self.th['light_shoulder_min'] < r_shoulder_like_angle < self.th['light_shoulder_max']):
|
|
return 'light'
|
|
|
|
# 3) DOME
|
|
wrists_above_nose = (l_wr[1] < nose[1] and r_wr[1] < nose[1])
|
|
shoulders_top_y = min(l_sh[1], r_sh[1])
|
|
elbows_above_shoulders = (l_el[1] < shoulders_top_y and r_el[1] < shoulders_top_y)
|
|
elbow_distance = np.linalg.norm(l_el - r_el)
|
|
elbows_far_apart = elbow_distance > (self.th['elbow_far_factor'] * shoulder_width)
|
|
wrist_distance = np.linalg.norm(l_wr - r_wr)
|
|
wrists_near = wrist_distance < (self.th['wrists_near_factor'] * shoulder_width)
|
|
|
|
if self.debug:
|
|
print(f"[SG] dome_check: wrists_above={wrists_above_nose}, elbows_above={elbows_above_shoulders}, "
|
|
f"elbow_d={elbow_distance:.1f}, wrist_d={wrist_distance:.1f}")
|
|
|
|
if wrists_above_nose and elbows_above_shoulders and elbows_far_apart and wrists_near:
|
|
return 'dome'
|
|
|
|
return 'none'
|