530 lines
22 KiB
Python
530 lines
22 KiB
Python
import time
|
|
from pymouse import PyMouse
|
|
import cv2
|
|
import mediapipe as mp
|
|
import math
|
|
import util
|
|
import detect
|
|
import win32gui
|
|
import win32con
|
|
import sys
|
|
|
|
|
|
def hand_angle(hand_):
|
|
'''
|
|
获取对应手相关向量的二维角度,根据角度确定手势
|
|
'''
|
|
angle_list = []
|
|
# ---------------------------- thumb 大拇指角度
|
|
angle_ = util.Util.vector_2d_angle(
|
|
((int(hand_[2][0]) - int(hand_[3][0])), (int(hand_[2][1]) - int(hand_[3][1]))),
|
|
((int(hand_[3][0]) - int(hand_[4][0])), (int(hand_[3][1]) - int(hand_[4][1])))
|
|
)
|
|
angle_list.append(angle_)
|
|
# ---------------------------- index 食指角度
|
|
angle_ = util.Util.vector_2d_angle(
|
|
((int(hand_[5][0]) - int(hand_[6][0])), (int(hand_[5][1]) - int(hand_[6][1]))),
|
|
((int(hand_[7][0]) - int(hand_[8][0])), (int(hand_[7][1]) - int(hand_[8][1])))
|
|
)
|
|
angle_list.append(angle_)
|
|
# ---------------------------- middle 中指角度
|
|
angle_ = util.Util.vector_2d_angle(
|
|
((int(hand_[9][0]) - int(hand_[10][0])), (int(hand_[9][1]) - int(hand_[10][1]))),
|
|
((int(hand_[11][0]) - int(hand_[12][0])), (int(hand_[11][1]) - int(hand_[12][1])))
|
|
)
|
|
angle_list.append(angle_)
|
|
# ---------------------------- ring 无名指角度
|
|
angle_ = util.Util.vector_2d_angle(
|
|
((int(hand_[13][0]) - int(hand_[14][0])), (int(hand_[13][1]) - int(hand_[14][1]))),
|
|
((int(hand_[15][0]) - int(hand_[16][0])), (int(hand_[15][1]) - int(hand_[16][1])))
|
|
)
|
|
angle_list.append(angle_)
|
|
# ---------------------------- pink 小拇指角度
|
|
angle_ = util.Util.vector_2d_angle(
|
|
((int(hand_[17][0]) - int(hand_[18][0])), (int(hand_[17][1]) - int(hand_[18][1]))),
|
|
((int(hand_[19][0]) - int(hand_[20][0])), (int(hand_[19][1]) - int(hand_[20][1])))
|
|
)
|
|
angle_list.append(angle_)
|
|
return angle_list
|
|
|
|
|
|
def judge_zero(is_finger_straight):
|
|
return not is_finger_straight[1] and not is_finger_straight[2] and \
|
|
not is_finger_straight[3] and not is_finger_straight[4]
|
|
|
|
|
|
def judge_two(is_finger_straight):
|
|
return is_finger_straight[1] and is_finger_straight[2] and \
|
|
not is_finger_straight[3] and not is_finger_straight[4]
|
|
|
|
|
|
def judge_three(is_finger_straight):
|
|
return is_finger_straight[1] and is_finger_straight[2] and \
|
|
is_finger_straight[3] and not is_finger_straight[4]
|
|
|
|
|
|
def judge_four(is_finger_straight):
|
|
return is_finger_straight[1] and is_finger_straight[2] and \
|
|
is_finger_straight[3] and is_finger_straight[4]
|
|
|
|
|
|
def judge_five(is_finger_straight):
|
|
return is_finger_straight[1] and is_finger_straight[2] and \
|
|
is_finger_straight[3] and is_finger_straight[4]
|
|
|
|
|
|
class Identify:
|
|
def __init__(self, control_flag, array):
|
|
self.rgb_image = None
|
|
self.control_flag = control_flag
|
|
self.array = array
|
|
self.mouse = PyMouse()
|
|
self.left_hand_flag = False
|
|
self.right_hand_flag = False
|
|
self.catch_person_flag = False
|
|
self.position_x = 0
|
|
self.position_y = 0
|
|
self.image = []
|
|
self.image_height = 0
|
|
self.image_width = 0
|
|
self.person_results = []
|
|
self.identify_results = []
|
|
self.left_hand_points = []
|
|
self.right_hand_points = []
|
|
self.left_hands = []
|
|
self.right_hands = []
|
|
self.left_angle_list = []
|
|
self.right_angle_list = []
|
|
self.is_left_finger_straight = [False, False, False, False, False]
|
|
self.is_right_finger_straight = [False, False, False, False, False]
|
|
self.is_identify = False
|
|
self.last_control_flag = 0
|
|
self.page_up_count = 0
|
|
self.page_down_count = 0
|
|
self.end_count = 0
|
|
self.pencil_count = 0
|
|
self.step_up = 0
|
|
self.step_down = 0
|
|
self.last_wrist_point = (0, 0)
|
|
self.now_time = 0
|
|
self.lase_time = 0
|
|
self.flag = 0
|
|
self.mp_drawing = mp.solutions.drawing_utils
|
|
self.mp_hands = mp.solutions.hands
|
|
self.mp_poses = mp.solutions.pose
|
|
self.ml_identify = []
|
|
self.hands = self.mp_hands.Hands(
|
|
static_image_mode=False,
|
|
max_num_hands=2,
|
|
min_detection_confidence=0.75,
|
|
min_tracking_confidence=0.75)
|
|
self.poses = self.mp_poses.Pose(
|
|
min_detection_confidence=0.5,
|
|
min_tracking_confidence=0.5)
|
|
self.mp_face = mp.solutions.face_detection
|
|
self.face_detection = self.mp_face.FaceDetection(min_detection_confidence=0.5)
|
|
|
|
def begin(self, over_flag):
|
|
capture = cv2.VideoCapture(0)
|
|
last_time = 0
|
|
self.ml_identify = detect.YOLOV5()
|
|
screen_width = self.mouse.screen_size()[0]
|
|
time1 = time.time()
|
|
x1, y1, x2, y2 = 0, 0, 0, 0
|
|
count = 0
|
|
while 1:
|
|
count += 1
|
|
if over_flag.value == 1:
|
|
sys.exit(0)
|
|
ret, self.image = capture.read()
|
|
self.image = cv2.resize(self.image, (320, 240), interpolation=cv2.INTER_LINEAR)
|
|
# fps = 1 / (self.now_time - self.lase_time)
|
|
# self.lase_time = self.now_time
|
|
# print("fps = " + str(fps))
|
|
# self.ml_identify.infer(image=self.image)
|
|
# continue
|
|
# self.catch_person_flag = True
|
|
# if not self.catch_person_flag:
|
|
# self.catch_person_flag = True
|
|
if not self.catch_person_flag:
|
|
self.person_results.clear()
|
|
self.left_hands.clear()
|
|
self.right_hands.clear()
|
|
self.deal_with_image()
|
|
self.find_points()
|
|
x1, y1, x2, y2 = self.find_target_person()
|
|
# print("no no no no no no no no no no no no no")
|
|
else:
|
|
# print("in True")
|
|
# print(x1, y1, x2, y2)
|
|
self.deal_with_image()
|
|
self.image = self.image[y1:y2, x1:x2]
|
|
|
|
# self.identify_results = self.hands.process(self.image)
|
|
|
|
self.is_identify = False
|
|
self.left_hand_flag = False
|
|
self.right_hand_flag = False
|
|
for i in range(5):
|
|
self.is_left_finger_straight[i] = False
|
|
self.is_right_finger_straight[i] = False
|
|
self.left_hand_points.clear()
|
|
self.right_hand_points.clear()
|
|
self.get_hand_points()
|
|
# print("---------------------")
|
|
# print(self.left_hand_flag)
|
|
# print(self.left_hand_points)
|
|
# print(self.right_hand_flag)
|
|
# print(self.right_hand_points)
|
|
self.judge_finger_straight()
|
|
flag = self.judge_control()
|
|
if flag:
|
|
self.flag = flag
|
|
# print("this flag = " + str(flag))
|
|
now_time = time.time()
|
|
self.array[0] = self.position_x
|
|
self.array[1] = self.position_y
|
|
self.array[2] = x2 - x1
|
|
self.array[3] = y2 - y1
|
|
if now_time - last_time < 1:
|
|
continue
|
|
last_time = now_time
|
|
self.catch_person_flag = not self.judge_catch_over()
|
|
# print("**************")
|
|
# for i in range(5):
|
|
# print(self.is_finger_straight[i])
|
|
# x = position[0]
|
|
# y = position[1]
|
|
|
|
# control_flag = flag
|
|
self.control_flag.value = self.flag
|
|
# print("self.v.value = " + str(self.flag))
|
|
# print("final_control_flag = " + str(self.flag))
|
|
self.flag = 0
|
|
|
|
cv2.namedWindow("Video")
|
|
WindowName = "Video"
|
|
cv2.imshow(WindowName, self.image)
|
|
hwnd = win32gui.FindWindow(None, WindowName) # 获取句柄，然后置顶
|
|
CVRECT = cv2.getWindowImageRect(WindowName)
|
|
win32gui.SetWindowPos(hwnd, win32con.HWND_TOPMOST, screen_width - 320, 0, CVRECT[2], CVRECT[3],
|
|
win32con.SWP_SHOWWINDOW)
|
|
# cv2.imshow("Video", self.image)
|
|
if cv2.waitKey(1) & 0xFF == ord('q'):
|
|
break
|
|
capture.release()
|
|
cv2.destroyAllWindows()
|
|
|
|
def judge_catch_over(self):
|
|
if not self.left_hand_flag:
|
|
return False
|
|
if not self.right_hand_flag:
|
|
return False
|
|
if not judge_zero(self.is_left_finger_straight):
|
|
return False
|
|
if not judge_zero(self.is_right_finger_straight):
|
|
return False
|
|
if util.Util.get_distance(self.left_hand_points[6], self.right_hand_points[6]) / \
|
|
util.Util.get_distance(self.left_hand_points[5], self.left_hand_points[9]) > 2.5:
|
|
return False
|
|
return True
|
|
|
|
def find_points(self):
|
|
if self.identify_results.multi_handedness:
|
|
for i in range(len(self.identify_results.multi_handedness)):
|
|
if self.identify_results.multi_handedness[i].classification[0].label == "Left":
|
|
left_hand_points = []
|
|
for hand_landmarks in self.identify_results.multi_hand_landmarks[i].landmark:
|
|
if self.image_height == 0:
|
|
self.image_height, self.image_width, c = self.image.shape
|
|
cx, cy = int(hand_landmarks.x * self.image_width), int(hand_landmarks.y * self.image_height)
|
|
left_hand_points.append((cx, cy))
|
|
# self.mp_drawing.draw_landmarks(
|
|
# self.image, self.identify_results.multi_hand_landmarks[i], self.mp_hands.HAND_CONNECTIONS)
|
|
# if self.identify_results.multi_handedness[i].classification[0].score > 0.5:
|
|
# self.left_hand_flag = True
|
|
# self.is_identify = True
|
|
self.left_hands.append(left_hand_points)
|
|
else:
|
|
right_hand_points = []
|
|
for hand_landmarks in self.identify_results.multi_hand_landmarks[i].landmark:
|
|
if self.image_height == 0:
|
|
self.image_height, self.image_width, c = self.image.shape
|
|
cx, cy = int(hand_landmarks.x * self.image_width), int(hand_landmarks.y * self.image_height)
|
|
right_hand_points.append((cx, cy))
|
|
# self.is_identify = True
|
|
# self.mp_drawing.draw_landmarks(
|
|
# self.image, self.identify_results.multi_hand_landmarks[i], self.mp_hands.HAND_CONNECTIONS)
|
|
# if self.identify_results.multi_handedness[i].classification[0].score > 0.5:
|
|
# self.right_hand_flag = True
|
|
# self.is_identify = True
|
|
self.right_hands.append(right_hand_points)
|
|
|
|
def find_target_person(self):
|
|
target_scopes = self.ml_identify.infer(image=self.image)
|
|
for result in target_scopes:
|
|
# cv2.rectangle(self.image, (result[0], result[1]), (result[2], result[3]), (0, 0, 255), 2)
|
|
one_person_result = [result, [], []]
|
|
for left_hand in self.left_hands:
|
|
if util.Util.is_in_rectangle(left_hand[0], result):
|
|
one_person_result[1] = left_hand
|
|
break
|
|
for right_hand in self.right_hands:
|
|
if util.Util.is_in_rectangle(right_hand[0], result):
|
|
one_person_result[2] = right_hand
|
|
break
|
|
self.person_results.append(one_person_result)
|
|
if self.person_results:
|
|
person_result = self.find_catch_pose()
|
|
if self.catch_person_flag:
|
|
return person_result[0]
|
|
else:
|
|
return 0, 0, 0, 0
|
|
else:
|
|
return 0, 0, 0, 0
|
|
|
|
def find_catch_pose(self):
|
|
max_distance = 0
|
|
max_index = 0
|
|
for i in range(len(self.person_results)):
|
|
person_result = self.person_results[i]
|
|
if not person_result[1]:
|
|
continue
|
|
if not person_result[2]:
|
|
continue
|
|
standard_distance = util.Util.get_distance(person_result[1][7], person_result[1][8])
|
|
if standard_distance == 0:
|
|
continue
|
|
if util.Util.get_distance(person_result[1][4], person_result[2][4]) / standard_distance > 2:
|
|
continue
|
|
if util.Util.get_distance(person_result[1][8], person_result[2][8]) / standard_distance > 1:
|
|
continue
|
|
distance = util.Util.get_distance(person_result[1][0], person_result[1][8])
|
|
if distance > max_distance:
|
|
max_distance = distance
|
|
max_index = i
|
|
self.catch_person_flag = True
|
|
return self.person_results[max_index]
|
|
|
|
def deal_with_image(self):
|
|
self.image = cv2.flip(self.image, 1)
|
|
self.rgb_image = cv2.cvtColor(self.image, cv2.COLOR_BGR2RGB)
|
|
self.identify_results = self.hands.process(self.rgb_image)
|
|
|
|
def get_hand_points(self):
|
|
if not self.identify_results.multi_handedness:
|
|
return 0
|
|
for i in range(len(self.identify_results.multi_handedness)):
|
|
if self.identify_results.multi_handedness[i].classification[0].label == "Left":
|
|
for hand_landmarks in self.identify_results.multi_hand_landmarks[i].landmark:
|
|
if self.image_height == 0:
|
|
self.image_height, self.image_width, c = self.image.shape
|
|
cx, cy = int(hand_landmarks.x * self.image_width), int(hand_landmarks.y * self.image_height)
|
|
self.left_hand_points.append((cx, cy))
|
|
# self.mp_drawing.draw_landmarks(
|
|
# self.image, self.identify_results.multi_hand_landmarks[i], self.mp_hands.HAND_CONNECTIONS)
|
|
if self.identify_results.multi_handedness[i].classification[0].score > 0.5:
|
|
self.left_hand_flag = True
|
|
self.is_identify = True
|
|
|
|
else:
|
|
for hand_landmarks in self.identify_results.multi_hand_landmarks[i].landmark:
|
|
if self.image_height == 0:
|
|
self.image_height, self.image_width, c = self.image.shape
|
|
cx, cy = int(hand_landmarks.x * self.image_width), int(hand_landmarks.y * self.image_height)
|
|
self.right_hand_points.append((cx, cy))
|
|
# self.is_identify = True
|
|
# self.mp_drawing.draw_landmarks(
|
|
# self.image, self.identify_results.multi_hand_landmarks[i], self.mp_hands.HAND_CONNECTIONS)
|
|
if self.identify_results.multi_handedness[i].classification[0].score > 0.5:
|
|
self.right_hand_flag = True
|
|
self.is_identify = True
|
|
|
|
def judge_finger_straight(self):
|
|
if self.is_identify:
|
|
if self.left_hand_flag:
|
|
self.left_angle_list = hand_angle(hand_=self.left_hand_points)
|
|
for i in range(5):
|
|
self.is_left_finger_straight[i] = util.Util.is_straight(self.left_angle_list[i])
|
|
if self.right_hand_flag:
|
|
self.right_angle_list = hand_angle(hand_=self.right_hand_points)
|
|
for i in range(5):
|
|
self.is_right_finger_straight[i] = util.Util.is_straight(self.right_angle_list[i])
|
|
|
|
def judge_one(self, is_finger_straight):
|
|
if self.left_hand_flag:
|
|
self.position_x = self.left_hand_points[8][0]
|
|
self.position_y = self.left_hand_points[8][1]
|
|
# elif self.right_hand_flag:
|
|
# self.position_x = self.right_hand_points[8][0]
|
|
# self.position_y = self.right_hand_points[8][1]
|
|
return is_finger_straight[1] and not is_finger_straight[2] and \
|
|
not is_finger_straight[3] and not is_finger_straight[4]
|
|
|
|
def judge_step_one(self, is_left):
|
|
if is_left:
|
|
if judge_five(self.is_left_finger_straight) and self.left_hand_points[8][0] < self.left_hand_points[0][
|
|
0] and \
|
|
self.left_hand_points[12][0] < self.left_hand_points[0][0] and \
|
|
self.left_hand_points[16][0] < self.left_hand_points[0][0] and self.left_hand_points[20][0] < \
|
|
self.left_hand_points[0][0]:
|
|
return True
|
|
else:
|
|
if judge_five(self.is_right_finger_straight) and self.right_hand_points[8][0] > self.right_hand_points[0][
|
|
0] and \
|
|
self.right_hand_points[12][0] > self.right_hand_points[0][0] and \
|
|
self.right_hand_points[16][0] > self.right_hand_points[0][0] and self.right_hand_points[20][0] > \
|
|
self.right_hand_points[0][0]:
|
|
return True
|
|
return False
|
|
|
|
def judge_step_two(self, is_left):
|
|
if is_left:
|
|
if judge_five(self.is_left_finger_straight) and self.left_hand_points[8][0] > self.left_hand_points[0][
|
|
0] and \
|
|
self.left_hand_points[12][0] > self.left_hand_points[0][0] and \
|
|
self.left_hand_points[16][0] > self.left_hand_points[0][0] and self.left_hand_points[20][0] > \
|
|
self.left_hand_points[0][0]:
|
|
return True
|
|
else:
|
|
if judge_five(self.is_right_finger_straight) and self.right_hand_points[8][0] < self.right_hand_points[0][
|
|
0] and \
|
|
self.right_hand_points[12][0] < self.right_hand_points[0][0] and \
|
|
self.right_hand_points[16][0] < self.right_hand_points[0][0] and self.right_hand_points[20][0] < \
|
|
self.right_hand_points[0][0]:
|
|
return True
|
|
return False
|
|
|
|
def judge_step_three(self):
|
|
if self.left_hand_flag:
|
|
if self.left_hand_points[20][1] < self.left_hand_points[0][1]:
|
|
return True
|
|
else:
|
|
if self.right_hand_points[20][1] < self.right_hand_points[0][1]:
|
|
return True
|
|
return False
|
|
|
|
def judge_page_up(self):
|
|
if not self.right_hand_flag:
|
|
return False
|
|
if self.step_up == 0:
|
|
self.lase_time = time.time()
|
|
if self.step_up == 0 and self.judge_step_three():
|
|
self.step_up = 1
|
|
if self.step_up == 1 and self.judge_step_one(False):
|
|
self.step_up = 2
|
|
elif self.step_up == 2 and self.judge_step_two(False):
|
|
self.step_up = 3
|
|
# elif self.step_up == 2 and judge_zero(self.is_right_finger_straight):
|
|
# self.step_up = 3
|
|
# elif self.step_up == 3:
|
|
# self.step_up = 0
|
|
now_time = time.time()
|
|
if now_time - self.lase_time > 3:
|
|
self.lase_time = now_time
|
|
self.step_up = 0
|
|
return False
|
|
else:
|
|
if self.step_up == 3:
|
|
self.lase_time = now_time
|
|
self.step_up = 0
|
|
return True
|
|
return False
|
|
|
|
def judge_page_down(self):
|
|
if not self.left_hand_flag:
|
|
return False
|
|
if self.step_down == 0:
|
|
self.lase_time = time.time()
|
|
if self.step_down == 0 and self.judge_step_three():
|
|
self.step_down = 1
|
|
# print("step = 1")
|
|
if self.step_down == 1 and self.judge_step_one(True):
|
|
self.step_down = 2
|
|
# print("step = 2")
|
|
elif self.step_down == 2 and self.judge_step_two(True):
|
|
self.step_down = 3
|
|
# print("step = 3")
|
|
now_time = time.time()
|
|
if now_time - self.lase_time > 3:
|
|
self.lase_time = now_time
|
|
self.step_down = 0
|
|
return False
|
|
else:
|
|
if self.step_down == 3:
|
|
self.lase_time = now_time
|
|
self.step_down = 0
|
|
return True
|
|
return False
|
|
|
|
def judge_end(self):
|
|
if not self.left_hand_flag:
|
|
self.end_count = 0
|
|
return False
|
|
if not judge_zero(self.is_left_finger_straight):
|
|
self.end_count = 0
|
|
return False
|
|
self.end_count += 1
|
|
if self.end_count > 10:
|
|
self.end_count = 0
|
|
return True
|
|
return False
|
|
|
|
def judge_system_over(self):
|
|
if not self.judge_one(self.is_left_finger_straight):
|
|
return False
|
|
if not self.judge_one(self.is_right_finger_straight):
|
|
return False
|
|
standard_distance = util.Util.get_distance(self.left_hand_points[7], self.right_hand_points[8])
|
|
if standard_distance == 0:
|
|
return False
|
|
# print(util.Util.get_distance(self.left_hand_points[8], self.right_hand_points[8]) / standard_distance)
|
|
if util.Util.get_distance(self.left_hand_points[8], self.right_hand_points[8]) / standard_distance < 1:
|
|
return True
|
|
return False
|
|
|
|
# def judge_pencil(self):
|
|
# if not self.judge_one(self.is_left_finger_straight):
|
|
# return False
|
|
# if self.right_hand_flag:
|
|
# return False
|
|
# self.pencil_count += 1
|
|
# if self.pencil_count > 5:
|
|
# self.pencil_count = 0
|
|
|
|
def judge_control(self):
|
|
if self.is_identify:
|
|
# print("len = " + str(len(self.identify_results.multi_handedness)))
|
|
# print("left is " + str(self.left_hand_flag))
|
|
# print("right is " + str(self.right_hand_flag))
|
|
if judge_two(self.is_left_finger_straight):
|
|
# print("1")
|
|
return 1
|
|
elif self.judge_page_up():
|
|
# print("2")
|
|
return 2
|
|
elif self.judge_page_down():
|
|
# print("3")
|
|
return 3
|
|
elif self.judge_one(self.is_left_finger_straight) and not self.right_hand_flag:
|
|
# print("4")
|
|
return 4
|
|
elif self.judge_one(self.is_left_finger_straight) and judge_zero(self.is_right_finger_straight):
|
|
# print("5")
|
|
return 5
|
|
elif self.judge_end():
|
|
# print("6")
|
|
return 6
|
|
elif self.judge_system_over():
|
|
# print("7")
|
|
return 7
|
|
# else:
|
|
# # print("other")
|
|
# else:
|
|
# print("no_hand_points")
|
|
return 0
|
|
|
|
#
|
|
# identify = Identify()
|
|
# identify.begin()
|