import numpy as np class SpecialGestureDetector: """ Detects special static gestures using either simple geometric rules or an ML classifier. Supported gesture labels: - 'cross' : forearms crossed near the chest - 'light' : right arm pose approximating a 'light' toggle - 'dome' : arms forming a dome above the head - 'none' : no special gesture detected """ def __init__(self, mode='geometric', model_path=None, class_names=None, debug=False, thresholds=None): self.mode = mode self.debug = debug # Defaults for geometric detection self.th = { 'min_conf': 0.5, 'shoulder_width_min': 30.0, 'torso_height_min': 10.0, 'chest_band': 0.25, # widened to be more forgiving 'wrists_near_factor': 0.6, # relaxed for dome 'elbow_far_factor': 0.9, # relaxed for dome 'light_elbow_min': 45.0, 'light_elbow_max': 120.0, 'light_shoulder_min': -5.0, 'light_shoulder_max': 20.0 } if thresholds: self.th.update(thresholds) if mode == 'ml': from ml_gestures.predict import MLGesturePredictor if model_path is None or class_names is None: raise ValueError("For ML mode, provide model_path and class_names") self.ml_predictor = MLGesturePredictor(model_path, class_names) if self.debug: print("SpecialGestureDetector: Using ML classifier") else: self.ml_predictor = None if self.debug: print("SpecialGestureDetector: Using geometric rules") def predict(self, landmarks): if landmarks is None: return 'none' if self.mode == 'geometric': return self._geometric_predict(landmarks) else: return self.ml_predictor.predict(landmarks) def _geometric_predict(self, landmarks): idx = { 'nose': 0, 'left_shoulder': 11, 'right_shoulder': 12, 'left_elbow': 13, 'right_elbow': 14, 'left_wrist': 15, 'right_wrist': 16, 'left_hip': 23, 'right_hip': 24, } min_conf = self.th['min_conf'] # Only upper-body required (hips optional) required = [ 'left_shoulder', 'right_shoulder', 'left_elbow', 'right_elbow', 'left_wrist', 'right_wrist', 'nose' ] for p in required: if landmarks[idx[p]][3] < min_conf: if self.debug: print(f"[SG] Low confidence for {p}: {landmarks[idx[p]][3]:.2f}") return 'none' l_sh = np.array(landmarks[idx['left_shoulder']][:2], dtype=float) r_sh = np.array(landmarks[idx['right_shoulder']][:2], dtype=float) l_el = np.array(landmarks[idx['left_elbow']][:2], dtype=float) r_el = np.array(landmarks[idx['right_elbow']][:2], dtype=float) l_wr = np.array(landmarks[idx['left_wrist']][:2], dtype=float) r_wr = np.array(landmarks[idx['right_wrist']][:2], dtype=float) nose = np.array(landmarks[idx['nose']][:2], dtype=float) l_hip = np.array(landmarks[idx['left_hip']][:2], dtype=float) r_hip = np.array(landmarks[idx['right_hip']][:2], dtype=float) c_lhip = landmarks[idx['left_hip']][3] c_rhip = landmarks[idx['right_hip']][3] shoulder_center_y = (l_sh[1] + r_sh[1]) / 2.0 shoulder_width = np.linalg.norm(r_sh - l_sh) if shoulder_width < self.th['shoulder_width_min']: if self.debug: print(f"[SG] Shoulder width too small: {shoulder_width:.1f}") return 'none' # Torso height: prefer hips if visible, otherwise fallback using nose/shoulders if c_lhip >= min_conf and c_rhip >= min_conf: hip_center_y = (l_hip[1] + r_hip[1]) / 2.0 torso_height = hip_center_y - shoulder_center_y else: nose_to_shoulder = abs(nose[1] - shoulder_center_y) torso_height = max(1.6 * nose_to_shoulder, 0.9 * shoulder_width) hip_center_y = shoulder_center_y + torso_height if torso_height < self.th['torso_height_min']: if self.debug: print(f"[SG] Torso height too small: {torso_height:.1f}") return 'none' def angle_between_vectors(v1, v2): n1 = np.linalg.norm(v1) n2 = np.linalg.norm(v2) if n1 < 1e-6 or n2 < 1e-6: return 0.0 cos_a = np.dot(v1, v2) / (n1 * n2) cos_a = float(np.clip(cos_a, -1.0, 1.0)) return np.degrees(np.arccos(cos_a)) def joint_angle(p_prev, p_joint, p_next): v1 = p_prev - p_joint v2 = p_next - p_joint return angle_between_vectors(v1, v2) def segments_intersect(p1, p2, p3, p4): def cross(o, a, b): return (a[0] - o[0]) * (b[1] - o[1]) - (a[1] - o[1]) * (b[0] - o[0]) d1 = cross(p3, p4, p1) d2 = cross(p3, p4, p2) d3 = cross(p1, p2, p3) d4 = cross(p1, p2, p4) return (d1 * d2 < 0) and (d3 * d4 < 0) def line_intersection(p1, p2, p3, p4): d1 = p2 - p1 d2 = p4 - p3 denom = d1[0] * d2[1] - d1[1] * d2[0] if abs(denom) < 1e-6: return None t = ((p3[0] - p1[0]) * d2[1] - (p3[1] - p1[1]) * d2[0]) / denom return p1 + t * d1 # Angles (geometric cues) l_elbow_angle = joint_angle(l_sh, l_el, l_wr) r_elbow_angle = joint_angle(r_sh, r_el, r_wr) r_shoulder_like_angle = joint_angle(r_hip, r_sh, r_el) # 1) CROSS forearms_cross = segments_intersect(l_el, l_wr, r_el, r_wr) intersection = line_intersection(l_el, l_wr, r_el, r_wr) intersection_on_chest = False if intersection is not None: band = self.th['chest_band'] * torso_height intersection_on_chest = (shoulder_center_y - band) < intersection[1] < (hip_center_y + band) if self.debug: print(f"[SG] cross_check: intersect={forearms_cross}, chest={intersection_on_chest}") if forearms_cross and intersection_on_chest: return 'cross' # 2) LIGHT if (self.th['light_elbow_min'] < r_elbow_angle < self.th['light_elbow_max'] and self.th['light_shoulder_min'] < r_shoulder_like_angle < self.th['light_shoulder_max']): return 'light' # 3) DOME wrists_above_nose = (l_wr[1] < nose[1] and r_wr[1] < nose[1]) shoulders_top_y = min(l_sh[1], r_sh[1]) elbows_above_shoulders = (l_el[1] < shoulders_top_y and r_el[1] < shoulders_top_y) elbow_distance = np.linalg.norm(l_el - r_el) elbows_far_apart = elbow_distance > (self.th['elbow_far_factor'] * shoulder_width) wrist_distance = np.linalg.norm(l_wr - r_wr) wrists_near = wrist_distance < (self.th['wrists_near_factor'] * shoulder_width) if self.debug: print(f"[SG] dome_check: wrists_above={wrists_above_nose}, elbows_above={elbows_above_shoulders}, " f"elbow_d={elbow_distance:.1f}, wrist_d={wrist_distance:.1f}") if wrists_above_nose and elbows_above_shoulders and elbows_far_apart and wrists_near: return 'dome' return 'none'