基本完成
This commit is contained in:
Dinger
parent
dab76027c6
commit
aea0832dbf
Binary file not shown.
Binary file not shown.
Binary file not shown.
446
identify.py
446
identify.py
|
|
@ -4,6 +4,9 @@ import cv2
|
|||
import mediapipe as mp
|
||||
import math
|
||||
import util
|
||||
import detect
|
||||
import win32gui
|
||||
import win32con
|
||||
|
||||
|
||||
# page_up_count = 0
|
||||
|
|
@ -11,24 +14,96 @@ import util
|
|||
# last_wrist_point = (0, 0)
|
||||
|
||||
|
||||
def hand_angle(hand_):
|
||||
'''
|
||||
获取对应手相关向量的二维角度,根据角度确定手势
|
||||
'''
|
||||
angle_list = []
|
||||
# if self.left_hand_flag:
|
||||
# hand_ = self.left_hand_points
|
||||
# else:
|
||||
# hand_ = self.right_hand_points
|
||||
# ---------------------------- thumb 大拇指角度
|
||||
angle_ = util.Util.vector_2d_angle(
|
||||
((int(hand_[2][0]) - int(hand_[3][0])), (int(hand_[2][1]) - int(hand_[3][1]))),
|
||||
((int(hand_[3][0]) - int(hand_[4][0])), (int(hand_[3][1]) - int(hand_[4][1])))
|
||||
)
|
||||
angle_list.append(angle_)
|
||||
# ---------------------------- index 食指角度
|
||||
angle_ = util.Util.vector_2d_angle(
|
||||
((int(hand_[5][0]) - int(hand_[6][0])), (int(hand_[5][1]) - int(hand_[6][1]))),
|
||||
((int(hand_[7][0]) - int(hand_[8][0])), (int(hand_[7][1]) - int(hand_[8][1])))
|
||||
)
|
||||
angle_list.append(angle_)
|
||||
# ---------------------------- middle 中指角度
|
||||
angle_ = util.Util.vector_2d_angle(
|
||||
((int(hand_[9][0]) - int(hand_[10][0])), (int(hand_[9][1]) - int(hand_[10][1]))),
|
||||
((int(hand_[11][0]) - int(hand_[12][0])), (int(hand_[11][1]) - int(hand_[12][1])))
|
||||
)
|
||||
angle_list.append(angle_)
|
||||
# ---------------------------- ring 无名指角度
|
||||
angle_ = util.Util.vector_2d_angle(
|
||||
((int(hand_[13][0]) - int(hand_[14][0])), (int(hand_[13][1]) - int(hand_[14][1]))),
|
||||
((int(hand_[15][0]) - int(hand_[16][0])), (int(hand_[15][1]) - int(hand_[16][1])))
|
||||
)
|
||||
angle_list.append(angle_)
|
||||
# ---------------------------- pink 小拇指角度
|
||||
angle_ = util.Util.vector_2d_angle(
|
||||
((int(hand_[17][0]) - int(hand_[18][0])), (int(hand_[17][1]) - int(hand_[18][1]))),
|
||||
((int(hand_[19][0]) - int(hand_[20][0])), (int(hand_[19][1]) - int(hand_[20][1])))
|
||||
)
|
||||
angle_list.append(angle_)
|
||||
return angle_list
|
||||
|
||||
|
||||
def judge_zero(is_finger_straight):
|
||||
return not is_finger_straight[1] and not is_finger_straight[2] and \
|
||||
not is_finger_straight[3] and not is_finger_straight[4]
|
||||
|
||||
|
||||
def judge_two(is_finger_straight):
|
||||
return is_finger_straight[1] and is_finger_straight[2] and \
|
||||
not is_finger_straight[3] and not is_finger_straight[4]
|
||||
|
||||
|
||||
def judge_three(is_finger_straight):
|
||||
return is_finger_straight[1] and is_finger_straight[2] and \
|
||||
is_finger_straight[3] and not is_finger_straight[4]
|
||||
|
||||
|
||||
def judge_four(is_finger_straight):
|
||||
return is_finger_straight[1] and is_finger_straight[2] and \
|
||||
is_finger_straight[3] and is_finger_straight[4]
|
||||
|
||||
|
||||
def judge_five(is_finger_straight):
|
||||
return is_finger_straight[1] and is_finger_straight[2] and \
|
||||
is_finger_straight[3] and is_finger_straight[4]
|
||||
|
||||
|
||||
class Identify:
|
||||
def __init__(self, value, array):
|
||||
self.value = value
|
||||
self.array = array
|
||||
self.mouse = PyMouse()
|
||||
self.left_hand_flag = False
|
||||
self.right_hand_flag = False
|
||||
self.result = 0
|
||||
self.catch_person_flag = False
|
||||
self.position_x = 0
|
||||
self.position_y = 0
|
||||
self.image = []
|
||||
self.image_height = 0
|
||||
self.image_width = 0
|
||||
self.rgb_image = []
|
||||
self.person_results = []
|
||||
self.identify_results = []
|
||||
self.left_hand_points = []
|
||||
self.right_hand_points = []
|
||||
self.angle_list = []
|
||||
self.is_finger_straight = [False, False, False, False, False]
|
||||
self.left_hands = []
|
||||
self.right_hands = []
|
||||
self.left_angle_list = []
|
||||
self.right_angle_list = []
|
||||
self.is_left_finger_straight = [False, False, False, False, False]
|
||||
self.is_right_finger_straight = [False, False, False, False, False]
|
||||
self.is_identify = False
|
||||
self.last_control_flag = 0
|
||||
self.page_up_count = 0
|
||||
|
|
@ -41,184 +116,246 @@ class Identify:
|
|||
self.flag = 0
|
||||
self.mp_drawing = mp.solutions.drawing_utils
|
||||
self.mp_hands = mp.solutions.hands
|
||||
self.mp_poses = mp.solutions.pose
|
||||
self.ml_identify = []
|
||||
self.hands = self.mp_hands.Hands(
|
||||
static_image_mode=False,
|
||||
max_num_hands=2,
|
||||
min_detection_confidence=0.75,
|
||||
min_tracking_confidence=0.75)
|
||||
self.poses = self.mp_poses.Pose(
|
||||
min_detection_confidence=0.5,
|
||||
min_tracking_confidence=0.5)
|
||||
self.mp_face = mp.solutions.face_detection
|
||||
self.face_detection = self.mp_face.FaceDetection(min_detection_confidence=0.5)
|
||||
|
||||
def begin(self):
|
||||
capture = cv2.VideoCapture(0)
|
||||
last_time = 0
|
||||
self.ml_identify = detect.YOLOV5()
|
||||
screen_width = self.mouse.screen_size()[0]
|
||||
x1, y1, x2, y2 = 0, 0, 0, 0
|
||||
while 1:
|
||||
ret, self.image = capture.read()
|
||||
self.deal_with_image()
|
||||
self.image = cv2.resize(self.image, (320, 240), interpolation=cv2.INTER_LINEAR)
|
||||
if not self.catch_person_flag:
|
||||
self.person_results.clear()
|
||||
self.left_hands.clear()
|
||||
self.right_hands.clear()
|
||||
self.deal_with_image()
|
||||
self.find_points()
|
||||
x1, y1, x2, y2 = self.find_target_person()
|
||||
else:
|
||||
self.image = self.image[y1:y2, x1:x2]
|
||||
self.deal_with_image()
|
||||
self.identify_results = self.hands.process(self.rgb_image)
|
||||
# fps = 1 / (self.now_time - self.lase_time)
|
||||
# self.lase_time = self.now_time
|
||||
# print("fps = " + str(fps))
|
||||
self.is_identify = False
|
||||
self.left_hand_flag = False
|
||||
self.right_hand_flag = False
|
||||
for i in range(5):
|
||||
self.is_left_finger_straight[i] = False
|
||||
self.is_right_finger_straight[i] = False
|
||||
self.left_hand_points.clear()
|
||||
self.right_hand_points.clear()
|
||||
self.get_hand_points()
|
||||
self.judge_finger_straight()
|
||||
flag = self.judge_control()
|
||||
if flag:
|
||||
self.flag = flag
|
||||
print("this flag = " + str(flag))
|
||||
now_time = time.time()
|
||||
self.array[0] = self.position_x
|
||||
self.array[1] = self.position_y
|
||||
self.array[2] = self.image_width
|
||||
self.array[3] = self.image_height
|
||||
if now_time - last_time < 1:
|
||||
continue
|
||||
last_time = now_time
|
||||
self.catch_person_flag = not self.judge_over()
|
||||
# print("**************")
|
||||
# for i in range(5):
|
||||
# print(self.is_finger_straight[i])
|
||||
# x = position[0]
|
||||
# y = position[1]
|
||||
|
||||
# fps = 1 / (self.now_time - self.lase_time)
|
||||
# self.lase_time = self.now_time
|
||||
# print("fps = " + str(fps))
|
||||
self.is_identify = False
|
||||
self.left_hand_flag = False
|
||||
self.right_hand_flag = False
|
||||
for i in range(5):
|
||||
self.is_finger_straight[i] = False
|
||||
self.left_hand_points.clear()
|
||||
self.right_hand_points.clear()
|
||||
self.get_hand_points()
|
||||
self.judge_finger_straight()
|
||||
flag = self.judge_control()
|
||||
if flag:
|
||||
self.flag = flag
|
||||
now_time = time.time()
|
||||
self.array[0] = self.position_x
|
||||
self.array[1] = self.position_y
|
||||
self.array[2] = self.image_width
|
||||
self.array[3] = self.image_height
|
||||
if now_time - last_time < 1:
|
||||
continue
|
||||
last_time = now_time
|
||||
# print("**************")
|
||||
# for i in range(5):
|
||||
# print(self.is_finger_straight[i])
|
||||
# x = position[0]
|
||||
# y = position[1]
|
||||
# cv2.namedWindow("Video")
|
||||
# control_flag = flag
|
||||
self.value.value = self.flag
|
||||
# print("self.v.value = " + str(self.flag))
|
||||
print("final_control_flag = " + str(self.flag))
|
||||
self.flag = 0
|
||||
|
||||
cv2.namedWindow("Video")
|
||||
WindowName = "Video"
|
||||
cv2.imshow(WindowName, self.image)
|
||||
hwnd = win32gui.FindWindow(None, WindowName) # 获取句柄，然后置顶
|
||||
CVRECT = cv2.getWindowImageRect(WindowName)
|
||||
win32gui.SetWindowPos(hwnd, win32con.HWND_TOPMOST, screen_width - 320, 0, CVRECT[2], CVRECT[3],
|
||||
win32con.SWP_SHOWWINDOW)
|
||||
# cv2.imshow("Video", self.image)
|
||||
# if cv2.waitKey(1) & 0xFF == ord('q'):
|
||||
# break
|
||||
# control_flag = flag
|
||||
self.value.value = self.flag
|
||||
# print("self.v.value = " + str(self.flag))
|
||||
print("final_control_flag = " + str(self.flag))
|
||||
self.flag = 0
|
||||
if cv2.waitKey(1) & 0xFF == ord('q'):
|
||||
break
|
||||
capture.release()
|
||||
cv2.destroyAllWindows()
|
||||
|
||||
# def face_detect(self):
|
||||
# results = self.face_detection.process(self.rgb_image)
|
||||
def judge_over(self):
|
||||
if not self.left_hand_flag:
|
||||
return False
|
||||
if not self.right_hand_flag:
|
||||
return False
|
||||
if not judge_zero(self.is_left_finger_straight):
|
||||
return False
|
||||
if not judge_zero(self.is_right_finger_straight):
|
||||
return False
|
||||
if util.Util.get_distance(self.left_hand_points[6], self.right_hand_points[6]) / \
|
||||
util.Util.get_distance(self.left_hand_points[5], self.left_hand_points[9]) > 2:
|
||||
return False
|
||||
return True
|
||||
|
||||
def find_points(self):
|
||||
if self.identify_results.multi_handedness:
|
||||
for i in range(len(self.identify_results.multi_handedness)):
|
||||
if self.identify_results.multi_handedness[i].classification[0].label == "Left":
|
||||
left_hand_points = []
|
||||
for hand_landmarks in self.identify_results.multi_hand_landmarks[i].landmark:
|
||||
if self.image_height == 0:
|
||||
self.image_height, self.image_width, c = self.image.shape
|
||||
cx, cy = int(hand_landmarks.x * self.image_width), int(hand_landmarks.y * self.image_height)
|
||||
left_hand_points.append((cx, cy))
|
||||
# self.mp_drawing.draw_landmarks(
|
||||
# self.image, self.identify_results.multi_hand_landmarks[i], self.mp_hands.HAND_CONNECTIONS)
|
||||
# if self.identify_results.multi_handedness[i].classification[0].score > 0.5:
|
||||
# self.left_hand_flag = True
|
||||
# self.is_identify = True
|
||||
self.left_hands.append(left_hand_points)
|
||||
else:
|
||||
right_hand_points = []
|
||||
for hand_landmarks in self.identify_results.multi_hand_landmarks[i].landmark:
|
||||
if self.image_height == 0:
|
||||
self.image_height, self.image_width, c = self.image.shape
|
||||
cx, cy = int(hand_landmarks.x * self.image_width), int(hand_landmarks.y * self.image_height)
|
||||
right_hand_points.append((cx, cy))
|
||||
# self.is_identify = True
|
||||
# self.mp_drawing.draw_landmarks(
|
||||
# self.image, self.identify_results.multi_hand_landmarks[i], self.mp_hands.HAND_CONNECTIONS)
|
||||
# if self.identify_results.multi_handedness[i].classification[0].score > 0.5:
|
||||
# self.right_hand_flag = True
|
||||
# self.is_identify = True
|
||||
self.right_hands.append(right_hand_points)
|
||||
|
||||
def find_target_person(self):
|
||||
results = self.ml_identify.infer(image=self.image)
|
||||
for result in results:
|
||||
one_person_result = [result, [], []]
|
||||
for left_hand in self.left_hands:
|
||||
if util.Util.is_in_rectangle(left_hand[0], result):
|
||||
one_person_result[1] = left_hand
|
||||
break
|
||||
for right_hand in self.right_hands:
|
||||
if util.Util.is_in_rectangle(right_hand[0], result):
|
||||
one_person_result[2] = right_hand
|
||||
break
|
||||
self.person_results.append(one_person_result)
|
||||
if self.person_results:
|
||||
person_result = self.find_catch_pose()
|
||||
return person_result[0]
|
||||
else:
|
||||
return 0, 0, 0, 0
|
||||
|
||||
def find_catch_pose(self):
|
||||
max_distance = 0
|
||||
max_index = 0
|
||||
for i in range(len(self.person_results)):
|
||||
person_result = self.person_results[i]
|
||||
if not person_result[1]:
|
||||
continue
|
||||
if not person_result[2]:
|
||||
continue
|
||||
standard_distance = util.Util.get_distance(person_result[1][7], person_result[1][8])
|
||||
if util.Util.get_distance(person_result[1][4], person_result[2][4]) / standard_distance > 2:
|
||||
continue
|
||||
if util.Util.get_distance(person_result[1][8], person_result[2][8]) / standard_distance > 1:
|
||||
continue
|
||||
distance = util.Util.get_distance(person_result[1][0], person_result[1][8])
|
||||
if distance > max_distance:
|
||||
max_distance = distance
|
||||
max_index = i
|
||||
self.catch_person_flag = True
|
||||
return self.person_results[max_index]
|
||||
|
||||
def deal_with_image(self):
|
||||
self.image = cv2.flip(self.image, 1)
|
||||
self.rgb_image = cv2.cvtColor(self.image, cv2.COLOR_BGR2RGB)
|
||||
self.identify_results = self.hands.process(self.rgb_image)
|
||||
self.identify_results = self.hands.process(self.image)
|
||||
|
||||
def get_hand_points(self):
|
||||
if self.identify_results.multi_handedness:
|
||||
for i in range(len(self.identify_results.multi_handedness)):
|
||||
# print(self.identify_results.multi_handedness[i].classification[0].label)
|
||||
if self.identify_results.multi_handedness[i].classification[0].label == "Left":
|
||||
for hand_landmarks in self.identify_results.multi_hand_landmarks[i].landmark:
|
||||
if self.image_height == 0:
|
||||
self.image_height, self.image_width, c = self.image.shape
|
||||
cx, cy = int(hand_landmarks.x * self.image_width), int(hand_landmarks.y * self.image_height)
|
||||
self.left_hand_points.append((cx, cy))
|
||||
# self.mp_drawing.draw_landmarks(
|
||||
# self.image, self.identify_results.multi_hand_landmarks[i], self.mp_hands.HAND_CONNECTIONS)
|
||||
if self.identify_results.multi_handedness[i].classification[0].score > 0.5:
|
||||
self.left_hand_flag = True
|
||||
self.is_identify = True
|
||||
|
||||
else:
|
||||
for hand_landmarks in self.identify_results.multi_hand_landmarks[i].landmark:
|
||||
if self.image_height == 0:
|
||||
self.image_height, self.image_width, c = self.image.shape
|
||||
cx, cy = int(hand_landmarks.x * self.image_width), int(hand_landmarks.y * self.image_height)
|
||||
self.right_hand_points.append((cx, cy))
|
||||
if not self.identify_results.multi_handedness:
|
||||
return 0
|
||||
for i in range(len(self.identify_results.multi_handedness)):
|
||||
# print(self.identify_results.multi_handedness[i].classification[0].label)
|
||||
if self.identify_results.multi_handedness[i].classification[0].label == "Left":
|
||||
for hand_landmarks in self.identify_results.multi_hand_landmarks[i].landmark:
|
||||
if self.image_height == 0:
|
||||
self.image_height, self.image_width, c = self.image.shape
|
||||
cx, cy = int(hand_landmarks.x * self.image_width), int(hand_landmarks.y * self.image_height)
|
||||
self.left_hand_points.append((cx, cy))
|
||||
# self.mp_drawing.draw_landmarks(
|
||||
# self.image, self.identify_results.multi_hand_landmarks[i], self.mp_hands.HAND_CONNECTIONS)
|
||||
if self.identify_results.multi_handedness[i].classification[0].score > 0.5:
|
||||
print("identify is left")
|
||||
self.left_hand_flag = True
|
||||
self.is_identify = True
|
||||
# self.mp_drawing.draw_landmarks(
|
||||
# self.image, self.identify_results.multi_hand_landmarks[i], self.mp_hands.HAND_CONNECTIONS)
|
||||
if self.identify_results.multi_handedness[i].classification[0].score > 0.5:
|
||||
self.right_hand_flag = True
|
||||
self.is_identify = True
|
||||
|
||||
def hand_angle(self):
|
||||
'''
|
||||
获取对应手相关向量的二维角度,根据角度确定手势
|
||||
'''
|
||||
angle_list = []
|
||||
if self.left_hand_flag:
|
||||
hand_ = self.left_hand_points
|
||||
else:
|
||||
hand_ = self.right_hand_points
|
||||
# ---------------------------- thumb 大拇指角度
|
||||
angle_ = util.Util.vector_2d_angle(
|
||||
((int(hand_[2][0]) - int(hand_[3][0])), (int(hand_[2][1]) - int(hand_[3][1]))),
|
||||
((int(hand_[3][0]) - int(hand_[4][0])), (int(hand_[3][1]) - int(hand_[4][1])))
|
||||
)
|
||||
angle_list.append(angle_)
|
||||
# ---------------------------- index 食指角度
|
||||
angle_ = util.Util.vector_2d_angle(
|
||||
((int(hand_[5][0]) - int(hand_[6][0])), (int(hand_[5][1]) - int(hand_[6][1]))),
|
||||
((int(hand_[7][0]) - int(hand_[8][0])), (int(hand_[7][1]) - int(hand_[8][1])))
|
||||
)
|
||||
angle_list.append(angle_)
|
||||
# ---------------------------- middle 中指角度
|
||||
angle_ = util.Util.vector_2d_angle(
|
||||
((int(hand_[9][0]) - int(hand_[10][0])), (int(hand_[9][1]) - int(hand_[10][1]))),
|
||||
((int(hand_[11][0]) - int(hand_[12][0])), (int(hand_[11][1]) - int(hand_[12][1])))
|
||||
)
|
||||
angle_list.append(angle_)
|
||||
# ---------------------------- ring 无名指角度
|
||||
angle_ = util.Util.vector_2d_angle(
|
||||
((int(hand_[13][0]) - int(hand_[14][0])), (int(hand_[13][1]) - int(hand_[14][1]))),
|
||||
((int(hand_[15][0]) - int(hand_[16][0])), (int(hand_[15][1]) - int(hand_[16][1])))
|
||||
)
|
||||
angle_list.append(angle_)
|
||||
# ---------------------------- pink 小拇指角度
|
||||
angle_ = util.Util.vector_2d_angle(
|
||||
((int(hand_[17][0]) - int(hand_[18][0])), (int(hand_[17][1]) - int(hand_[18][1]))),
|
||||
((int(hand_[19][0]) - int(hand_[20][0])), (int(hand_[19][1]) - int(hand_[20][1])))
|
||||
)
|
||||
angle_list.append(angle_)
|
||||
self.angle_list = angle_list
|
||||
else:
|
||||
for hand_landmarks in self.identify_results.multi_hand_landmarks[i].landmark:
|
||||
if self.image_height == 0:
|
||||
self.image_height, self.image_width, c = self.image.shape
|
||||
cx, cy = int(hand_landmarks.x * self.image_width), int(hand_landmarks.y * self.image_height)
|
||||
self.right_hand_points.append((cx, cy))
|
||||
# self.is_identify = True
|
||||
# self.mp_drawing.draw_landmarks(
|
||||
# self.image, self.identify_results.multi_hand_landmarks[i], self.mp_hands.HAND_CONNECTIONS)
|
||||
if self.identify_results.multi_handedness[i].classification[0].score > 0.5:
|
||||
print("identify is right")
|
||||
self.right_hand_flag = True
|
||||
self.is_identify = True
|
||||
|
||||
def judge_finger_straight(self):
|
||||
if self.is_identify:
|
||||
self.hand_angle()
|
||||
for i in range(5):
|
||||
self.is_finger_straight[i] = util.Util.is_straight(self.angle_list[i])
|
||||
if self.left_hand_flag:
|
||||
self.left_angle_list = hand_angle(hand_=self.left_hand_points)
|
||||
# print("(((((((((((((((((((")
|
||||
for i in range(5):
|
||||
self.is_left_finger_straight[i] = util.Util.is_straight(self.left_angle_list[i])
|
||||
# print(self.is_left_finger_straight[i])
|
||||
# print(")))))))))))))))))))")
|
||||
if self.right_hand_flag:
|
||||
self.right_angle_list = hand_angle(hand_=self.right_hand_points)
|
||||
for i in range(5):
|
||||
self.is_right_finger_straight[i] = util.Util.is_straight(self.right_angle_list[i])
|
||||
|
||||
def judge_zero(self):
|
||||
return not self.is_finger_straight[1] and not self.is_finger_straight[2] and \
|
||||
not self.is_finger_straight[3] and not self.is_finger_straight[4]
|
||||
|
||||
def judge_one(self):
|
||||
def judge_one(self, is_finger_straight):
|
||||
if self.left_hand_flag:
|
||||
self.position_x = self.left_hand_points[8][0]
|
||||
self.position_y = self.left_hand_points[8][1]
|
||||
elif self.right_hand_flag:
|
||||
self.position_x = self.right_hand_points[8][0]
|
||||
self.position_y = self.right_hand_points[8][1]
|
||||
return self.is_finger_straight[1] and not self.is_finger_straight[2] and \
|
||||
not self.is_finger_straight[3] and not self.is_finger_straight[4]
|
||||
|
||||
def judge_two(self):
|
||||
return self.is_finger_straight[1] and self.is_finger_straight[2] and \
|
||||
not self.is_finger_straight[3] and not self.is_finger_straight[4]
|
||||
|
||||
def judge_three(self):
|
||||
return self.is_finger_straight[1] and self.is_finger_straight[2] and \
|
||||
self.is_finger_straight[3] and not self.is_finger_straight[4]
|
||||
|
||||
def judge_four(self):
|
||||
return self.is_finger_straight[1] and self.is_finger_straight[2] and \
|
||||
self.is_finger_straight[3] and self.is_finger_straight[4]
|
||||
|
||||
def judge_five(self):
|
||||
return self.is_finger_straight[1] and self.is_finger_straight[2] and \
|
||||
self.is_finger_straight[3] and self.is_finger_straight[4]
|
||||
return is_finger_straight[1] and not is_finger_straight[2] and \
|
||||
not is_finger_straight[3] and not is_finger_straight[4]
|
||||
|
||||
def judge_step_one(self, is_left):
|
||||
if is_left:
|
||||
if self.judge_five() and self.left_hand_points[8][0] < self.left_hand_points[0][0] and \
|
||||
if judge_five(self.is_left_finger_straight) and self.left_hand_points[8][0] < self.left_hand_points[0][
|
||||
0] and \
|
||||
self.left_hand_points[12][0] < self.left_hand_points[0][0] and \
|
||||
self.left_hand_points[16][0] < self.left_hand_points[0][0] and self.left_hand_points[20][0] < \
|
||||
self.left_hand_points[0][0]:
|
||||
return True
|
||||
else:
|
||||
if self.judge_five() and self.right_hand_points[8][0] > self.right_hand_points[0][0] and \
|
||||
if judge_five(self.is_right_finger_straight) and self.right_hand_points[8][0] > self.right_hand_points[0][
|
||||
0] and \
|
||||
self.right_hand_points[12][0] > self.right_hand_points[0][0] and \
|
||||
self.right_hand_points[16][0] > self.right_hand_points[0][0] and self.right_hand_points[20][0] > \
|
||||
self.right_hand_points[0][0]:
|
||||
|
|
@ -227,13 +364,15 @@ class Identify:
|
|||
|
||||
def judge_step_two(self, is_left):
|
||||
if is_left:
|
||||
if self.judge_five() and self.left_hand_points[8][0] > self.left_hand_points[0][0] and \
|
||||
if judge_five(self.is_left_finger_straight) and self.left_hand_points[8][0] > self.left_hand_points[0][
|
||||
0] and \
|
||||
self.left_hand_points[12][0] > self.left_hand_points[0][0] and \
|
||||
self.left_hand_points[16][0] > self.left_hand_points[0][0] and self.left_hand_points[20][0] > \
|
||||
self.left_hand_points[0][0]:
|
||||
return True
|
||||
else:
|
||||
if self.judge_five() and self.right_hand_points[8][0] < self.right_hand_points[0][0] and \
|
||||
if judge_five(self.is_right_finger_straight) and self.right_hand_points[8][0] < self.right_hand_points[0][
|
||||
0] and \
|
||||
self.right_hand_points[12][0] < self.right_hand_points[0][0] and \
|
||||
self.right_hand_points[16][0] < self.right_hand_points[0][0] and self.right_hand_points[20][0] < \
|
||||
self.right_hand_points[0][0]:
|
||||
|
|
@ -260,7 +399,7 @@ class Identify:
|
|||
self.step_up = 1
|
||||
elif self.step_up == 1 and self.judge_step_two(False):
|
||||
self.step_up = 3
|
||||
elif self.step_up == 2 and self.judge_zero():
|
||||
elif self.step_up == 2 and judge_zero(self.is_right_finger_straight):
|
||||
self.step_up = 3
|
||||
elif self.step_up == 3:
|
||||
self.step_up = 0
|
||||
|
|
@ -287,7 +426,7 @@ class Identify:
|
|||
elif self.step_down == 1 and self.judge_step_two(True):
|
||||
self.step_down = 3
|
||||
print("step = 3")
|
||||
elif self.step_down == 2 and self.judge_zero():
|
||||
elif self.step_down == 2 and judge_zero(self.is_left_finger_straight):
|
||||
self.step_down = 3
|
||||
elif self.step_down == 3:
|
||||
self.step_down = 0
|
||||
|
|
@ -301,22 +440,35 @@ class Identify:
|
|||
return False
|
||||
|
||||
def judge_end(self):
|
||||
if self.left_hand_flag and self.right_hand_flag and self.judge_zero():
|
||||
if self.left_hand_flag and self.right_hand_flag and judge_zero():
|
||||
return True
|
||||
return False
|
||||
|
||||
def judge_control(self):
|
||||
# print("left_flag = " + str(self.left_hand_flag))
|
||||
# print("right_flag = " + str(self.right_hand_flag))
|
||||
# print("1111111111111111111111")
|
||||
# for i in range(5):
|
||||
# print(self.is_left_finger_straight[i])
|
||||
# print("2222222222222222222222")
|
||||
if self.is_identify:
|
||||
if self.judge_two():
|
||||
if judge_two(self.is_left_finger_straight):
|
||||
print("1")
|
||||
return 1
|
||||
elif self.judge_page_up():
|
||||
print("2")
|
||||
return 2
|
||||
elif self.judge_page_down():
|
||||
print("3")
|
||||
# print("down!down!down!down!down!down!down!down!down!down!down!down!down!down!down!down!down!down!")
|
||||
return 3
|
||||
elif self.judge_one():
|
||||
elif self.judge_one(self.is_left_finger_straight):
|
||||
print("4")
|
||||
return 4
|
||||
elif self.judge_zero():
|
||||
elif self.left_hand_flag and judge_zero(self.is_left_finger_straight):
|
||||
print("5")
|
||||
# for i in range(5):
|
||||
# print(self.is_left_finger_straight[i])
|
||||
return 5
|
||||
else:
|
||||
print("other")
|
||||
|
|
|
|||
7
main.py
7
main.py
|
|
@ -8,10 +8,6 @@ import win32api
|
|||
import time
|
||||
import os
|
||||
|
||||
window = tkinter.Tk()
|
||||
window.title('窗口标题')
|
||||
window.geometry('200x50')
|
||||
|
||||
control_flag = 0
|
||||
x = 0
|
||||
y = 0
|
||||
|
|
@ -182,6 +178,9 @@ def open_ppt(file_path):
|
|||
|
||||
|
||||
if __name__ == '__main__':
|
||||
window = tkinter.Tk()
|
||||
window.title("会议PPT选择")
|
||||
window.geometry("200x100")
|
||||
bt1 = tkinter.Button(window, text='打开文件', width=15, height=15, command=open_file)
|
||||
bt1.pack()
|
||||
window.mainloop()
|
||||
|
|
|
|||
160
test.py
160
test.py
|
|
@ -1,8 +1,35 @@
|
|||
import cv2
|
||||
import mediapipe as mp
|
||||
import detect
|
||||
import numpy as np
|
||||
# !/usr/bin/env python
|
||||
# -*- coding: utf-8 -*-
|
||||
# @Time : 2021/04/08
|
||||
# @Author : Devil_Xiao
|
||||
# Purpose: This script is used to change the video_speed
|
||||
# You can choose to specify the fps,
|
||||
# or you can choose to change the multiple of the original playback speed
|
||||
|
||||
if __name__ == '__main__':
|
||||
import cv2
|
||||
from cv2 import VideoWriter, VideoWriter_fourcc
|
||||
import argparse
|
||||
|
||||
|
||||
def video_speed(video_root, out_root, fps=None, scale=1):
|
||||
"""When fps and scale are specified at the same time, fps is the dominant"""
|
||||
# cap = cv2.VideoCapture(video_root)
|
||||
cap = cv2.VideoCapture(0)
|
||||
video_width = int(cap.get(3))
|
||||
video_height = int(cap.get(4))
|
||||
print(video_width)
|
||||
print(video_height)
|
||||
# fourcc = cv2.VideoWriter_fourcc('m', 'p', '4', 'v')
|
||||
# if fps:
|
||||
# videoWriter = cv2.VideoWriter(out_root, fourcc, fps, (video_width, video_height))
|
||||
# else:
|
||||
# fps = int(cap.get(cv2.CAP_PROP_FPS) * scale)
|
||||
# videoWriter = cv2.VideoWriter(out_root, fourcc, fps, (video_width, video_height))
|
||||
flag = cap.isOpened()
|
||||
mp_drawing = mp.solutions.drawing_utils
|
||||
mp_drawing_styles = mp.solutions.drawing_styles
|
||||
mp_hands = mp.solutions.hands
|
||||
|
|
@ -15,38 +42,127 @@ if __name__ == '__main__':
|
|||
poses = mp_poses.Pose(
|
||||
min_detection_confidence=0.5,
|
||||
min_tracking_confidence=0.5)
|
||||
cap = cv2.VideoCapture(0)
|
||||
yolov5 = detect.YOLOV5()
|
||||
|
||||
# yolov5 = detect.YOLOV5()
|
||||
# blank_image = np.zeros((1000, 1000, 3), np.uint8)
|
||||
# blank_image.fill(255)
|
||||
fourcc = cv2.VideoWriter_fourcc(*'XVID')
|
||||
videoWriter = cv2.VideoWriter('output.avi', fourcc, 60.0, (1280, 720))
|
||||
count = 100
|
||||
while True:
|
||||
ret, frame = cap.read()
|
||||
# blank_image = frame.copy()
|
||||
# blank_image.fill(255)
|
||||
# image = detect.run(image_input=frame)
|
||||
# result = yolov5.infer(image=frame)
|
||||
# print("")
|
||||
# frame = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)
|
||||
# frame = cv2.flip(frame, 1)
|
||||
image = cv2.cvtColor(frame, cv2.COLOR_RGB2BGR)
|
||||
# results = hands.process(image)
|
||||
results = poses.process(image)
|
||||
print("*********************************")
|
||||
|
||||
frame = cv2.flip(frame, 1)
|
||||
frame = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)
|
||||
# frame = cv2.cvtColor(frame, cv2.COLOR_RGB2BGR)
|
||||
results = hands.process(frame)
|
||||
# results = poses.process(image)
|
||||
# print("*********************************")
|
||||
# if results.pose_landmarks:
|
||||
# for
|
||||
# print(results.pose_landmarks.landmark[17])
|
||||
mp_drawing.draw_landmarks(
|
||||
frame,
|
||||
results.pose_landmarks,
|
||||
mp_poses.POSE_CONNECTIONS,
|
||||
landmark_drawing_spec=mp_drawing_styles.get_default_pose_landmarks_style())
|
||||
# mp_drawing.draw_landmarks(
|
||||
# blank_image,
|
||||
# results.pose_landmarks,
|
||||
# mp_poses.POSE_CONNECTIONS,
|
||||
# landmark_drawing_spec=mp_drawing_styles.get_default_pose_landmarks_style())
|
||||
# if results.
|
||||
# print(results.multi_hand_landmarks)
|
||||
# if results.multi_handedness:
|
||||
# # if results.multi_handedness[0].classification[0].label == "Left":
|
||||
# for hand_landmarks in results.multi_hand_landmarks:
|
||||
# # 关键点可视化
|
||||
# mp_drawing.draw_landmarks(
|
||||
# frame, hand_landmarks, mp_hands.HAND_CONNECTIONS)
|
||||
# if results.multi_handedness[0].classification[0].label == "Right":
|
||||
# print(results.multi_handedness[0].classification[0])
|
||||
if results.multi_handedness:
|
||||
if results.multi_handedness[0].classification[0].label == "Left":
|
||||
for hand_landmarks in results.multi_hand_landmarks:
|
||||
# 关键点可视化
|
||||
mp_drawing.draw_landmarks(
|
||||
frame, hand_landmarks, mp_hands.HAND_CONNECTIONS)
|
||||
# if results.multi_handedness[0].classification[0].label == "Right":
|
||||
# print(results.multi_handedness[0].classification[0])
|
||||
cv2.imshow('MediaPipe Hands', frame)
|
||||
# cv2.imshow('black_image', blank_image)
|
||||
# videoWriter.write(blank_image)
|
||||
# count = count - 1
|
||||
# cv2.imwrite("picture3.jpg", frame)
|
||||
if cv2.waitKey(1) & 0xFF == 27:
|
||||
break
|
||||
cap.release()
|
||||
# videoWriter.release()
|
||||
# while (flag):
|
||||
# flag, frame = cap.read()
|
||||
# videoWriter.write(frame)
|
||||
# videoWriter.release()
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument('--video_name', type=str, default=r'video.mp4', help='original video name')
|
||||
parser.add_argument('--result_name', type=str, default=r'result.mp4', help='result name')
|
||||
parser.add_argument('--fps', type=int, default=None, help='Specify the playback frame rate')
|
||||
parser.add_argument('--scale', type=float, default='0.5', help='Change the original video speed')
|
||||
opt = parser.parse_args()
|
||||
print(opt)
|
||||
video_speed(opt.video_name, opt.result_name, opt.fps, 2)
|
||||
|
||||
# if __name__ == '__main__':
|
||||
# parser = argparse.ArgumentParser()
|
||||
# parser.add_argument('--video_name', type=str, default=r'video.mp4', help='original video name')
|
||||
# parser.add_argument('--result_name', type=str, default=r'result.mp4', help='result name')
|
||||
# parser.add_argument('--fps', type=int, default=None, help='Specify the playback frame rate')
|
||||
# parser.add_argument('--scale', type=float, default='0.5', help='Change the original video speed')
|
||||
# opt = parser.parse_args()
|
||||
# mp_drawing = mp.solutions.drawing_utils
|
||||
# mp_drawing_styles = mp.solutions.drawing_styles
|
||||
# mp_hands = mp.solutions.hands
|
||||
# mp_poses = mp.solutions.pose
|
||||
# hands = mp_hands.Hands(
|
||||
# static_image_mode=False,
|
||||
# max_num_hands=2,
|
||||
# min_detection_confidence=0.75,
|
||||
# min_tracking_confidence=0.75)
|
||||
# poses = mp_poses.Pose(
|
||||
# min_detection_confidence=0.5,
|
||||
# min_tracking_confidence=0.5)
|
||||
# cap = cv2.VideoCapture("D:\\Program Files\\JiJiDown\\Download\\Thank_you.mp4")
|
||||
# yolov5 = detect.YOLOV5()
|
||||
# blank_image = np.zeros((1000, 1000, 3), np.uint8)
|
||||
# blank_image.fill(255)
|
||||
# while True:
|
||||
# ret, frame = cap.read()
|
||||
# blank_image = np.zeros((1000, 1000, 3), np.uint8)
|
||||
# blank_image.fill(255)
|
||||
# # image = detect.run(image_input=frame)
|
||||
# # result = yolov5.infer(image=frame)
|
||||
# # print("")
|
||||
# # frame = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)
|
||||
# # frame = cv2.flip(frame, 1)
|
||||
# image = cv2.cvtColor(frame, cv2.COLOR_RGB2BGR)
|
||||
# # results = hands.process(image)
|
||||
# results = poses.process(image)
|
||||
# # print("*********************************")
|
||||
# # if results.pose_landmarks:
|
||||
# # for
|
||||
# # print(results.pose_landmarks.landmark[17])
|
||||
# mp_drawing.draw_landmarks(
|
||||
# blank_image,
|
||||
# results.pose_landmarks,
|
||||
# mp_poses.POSE_CONNECTIONS,
|
||||
# landmark_drawing_spec=mp_drawing_styles.get_default_pose_landmarks_style())
|
||||
# # if results.
|
||||
# # print(results.multi_hand_landmarks)
|
||||
# # if results.multi_handedness:
|
||||
# # # if results.multi_handedness[0].classification[0].label == "Left":
|
||||
# # for hand_landmarks in results.multi_hand_landmarks:
|
||||
# # # 关键点可视化
|
||||
# # mp_drawing.draw_landmarks(
|
||||
# # frame, hand_landmarks, mp_hands.HAND_CONNECTIONS)
|
||||
# # if results.multi_handedness[0].classification[0].label == "Right":
|
||||
# # print(results.multi_handedness[0].classification[0])
|
||||
# cv2.imshow('MediaPipe Hands', frame)
|
||||
# cv2.imshow('black_image', blank_image)
|
||||
# # cv2.imwrite("picture3.jpg", frame)
|
||||
# if cv2.waitKey(1) & 0xFF == 27:
|
||||
# break
|
||||
# cap.release()
|
||||
|
|
|
|||
349
test2.py
349
test2.py
|
|
@ -1,349 +0,0 @@
|
|||
# coding=utf-8
|
||||
import torch
|
||||
import torchvision
|
||||
import torch.nn as nn
|
||||
import os
|
||||
import time
|
||||
import numpy as np
|
||||
import math
|
||||
import random
|
||||
import cv2.cv2 as cv2
|
||||
|
||||
|
||||
def autopad(k, p=None): # kernel, padding
|
||||
# Pad to 'same'
|
||||
if p is None:
|
||||
p = k // 2 if isinstance(k, int) else [x // 2 for x in k] # auto-pad
|
||||
return p
|
||||
|
||||
|
||||
class Conv(nn.Module):
|
||||
# Standard convolution
|
||||
def __init__(self, c1, c2, k=1, s=1, p=None, g=1, act=True): # ch_in, ch_out, kernel, stride, padding, groups
|
||||
super(Conv, self).__init__()
|
||||
self.conv = nn.Conv2d(c1, c2, k, s, autopad(k, p), groups=g, bias=False)
|
||||
self.bn = nn.BatchNorm2d(c2)
|
||||
self.act = nn.SiLU() if act is True else (act if isinstance(act, nn.Module) else nn.Identity())
|
||||
|
||||
def forward(self, x):
|
||||
return self.act(self.bn(self.conv(x)))
|
||||
|
||||
def fuseforward(self, x):
|
||||
return self.act(self.conv(x))
|
||||
|
||||
|
||||
class Ensemble(torch.nn.ModuleList):
|
||||
'''模型集成'''
|
||||
|
||||
def __init__(self):
|
||||
super(Ensemble, self).__init__()
|
||||
|
||||
def forward(self, x, augment=False):
|
||||
y = []
|
||||
for module in self:
|
||||
y.append(module(x, augment)[0])
|
||||
# y = torch.stack(y).max(0)[0] # max ensemble
|
||||
# y = torch.stack(y).mean(0) # mean ensemble
|
||||
y = torch.cat(y, 1) # nms ensemble
|
||||
return y, None # inference, train output
|
||||
|
||||
|
||||
class YOLOV5(object):
|
||||
def __init__(self, conf_thres=0.25,
|
||||
iou_thres=0.45,
|
||||
classes=None,
|
||||
imgsz=640,
|
||||
weights="./yolov5s.pt"):
|
||||
# 超参数设置
|
||||
self.conf_thres = conf_thres # 置信度阈值
|
||||
self.iou_thres = iou_thres # iou阈值
|
||||
self.classes = classes # 分类个数
|
||||
self.imgsz = imgsz # 归一化大小
|
||||
# Load model
|
||||
self.device = torch.device('cpu')
|
||||
self.model = self.attempt_load(weights, map_location=self.device) # load FP32 model
|
||||
self.stride = int(self.model.stride.max()) # model stride
|
||||
self.imgsz = self.check_img_size(imgsz, s=self.stride) # check img_size
|
||||
|
||||
def attempt_load(self, weights, map_location=None):
|
||||
# Loads an ensemble of models weights=[a,b,c] or a single model weights=[a] or weights=a
|
||||
model = Ensemble()
|
||||
for w in weights if isinstance(weights, list) else [weights]:
|
||||
ckpt = torch.load(w, map_location=map_location) # load
|
||||
model.append(ckpt['ema' if ckpt.get('ema') else 'model'].float().fuse().eval()) # FP32 model
|
||||
|
||||
# Compatibility updates
|
||||
for m in model.modules():
|
||||
if type(m) in [nn.Hardswish, nn.LeakyReLU, nn.ReLU, nn.ReLU6, nn.SiLU]:
|
||||
m.inplace = True # pytorch 1.7.0 compatibility
|
||||
elif type(m) is Conv:
|
||||
m._non_persistent_buffers_set = set() # pytorch 1.6.0 compatibility
|
||||
|
||||
if len(model) == 1:
|
||||
return model[-1] # return model
|
||||
else:
|
||||
print('Ensemble created with %s\n' % weights)
|
||||
for k in ['names', 'stride']:
|
||||
setattr(model, k, getattr(model[-1], k))
|
||||
return model # return ensemble
|
||||
|
||||
def make_divisible(self, x, divisor):
|
||||
# Returns x evenly divisible by divisor
|
||||
return math.ceil(x / divisor) * divisor
|
||||
|
||||
def check_img_size(self, img_size, s=32):
|
||||
# Verify img_size is a multiple of stride s
|
||||
new_size = self.make_divisible(img_size, int(s)) # ceil gs-multiple
|
||||
if new_size != img_size:
|
||||
print('WARNING: --img-size %g must be multiple of max stride %g, updating to %g' % (img_size, s, new_size))
|
||||
return new_size
|
||||
|
||||
def letterbox(self, img, new_shape=(640, 640), color=(114, 114, 114), auto=True, scaleFill=False, scaleup=True,
|
||||
stride=32):
|
||||
# Resize and pad image while meeting stride-multiple constraints
|
||||
shape = img.shape[:2] # current shape [height, width]
|
||||
if isinstance(new_shape, int):
|
||||
new_shape = (new_shape, new_shape)
|
||||
|
||||
# Scale ratio (new / old)
|
||||
r = min(new_shape[0] / shape[0], new_shape[1] / shape[1])
|
||||
if not scaleup: # only scale down, do not scale up (for better test mAP)
|
||||
r = min(r, 1.0)
|
||||
|
||||
# Compute padding
|
||||
ratio = r, r # width, height ratios
|
||||
new_unpad = int(round(shape[1] * r)), int(round(shape[0] * r))
|
||||
dw, dh = new_shape[1] - new_unpad[0], new_shape[0] - new_unpad[1] # wh padding
|
||||
if auto: # minimum rectangle
|
||||
dw, dh = np.mod(dw, stride), np.mod(dh, stride) # wh padding
|
||||
elif scaleFill: # stretch
|
||||
dw, dh = 0.0, 0.0
|
||||
new_unpad = (new_shape[1], new_shape[0])
|
||||
ratio = new_shape[1] / shape[1], new_shape[0] / shape[0] # width, height ratios
|
||||
|
||||
dw /= 2 # divide padding into 2 sides
|
||||
dh /= 2
|
||||
|
||||
if shape[::-1] != new_unpad: # resize
|
||||
img = cv2.resize(img, new_unpad, interpolation=cv2.INTER_LINEAR)
|
||||
top, bottom = int(round(dh - 0.1)), int(round(dh + 0.1))
|
||||
left, right = int(round(dw - 0.1)), int(round(dw + 0.1))
|
||||
img = cv2.copyMakeBorder(img, top, bottom, left, right, cv2.BORDER_CONSTANT, value=color) # add border
|
||||
return img, ratio, (dw, dh)
|
||||
|
||||
def box_iou(self, box1, box2):
|
||||
# https://github.com/pytorch/vision/blob/master/torchvision/ops/boxes.py
|
||||
"""
|
||||
Return intersection-over-union (Jaccard index) of boxes.
|
||||
Both sets of boxes are expected to be in (x1, y1, x2, y2) format.
|
||||
Arguments:
|
||||
box1 (Tensor[N, 4])
|
||||
box2 (Tensor[M, 4])
|
||||
Returns:
|
||||
iou (Tensor[N, M]): the NxM matrix containing the pairwise
|
||||
IoU values for every element in boxes1 and boxes2
|
||||
"""
|
||||
|
||||
def box_area(box):
|
||||
# box = 4xn
|
||||
return (box[2] - box[0]) * (box[3] - box[1])
|
||||
|
||||
area1 = box_area(box1.T)
|
||||
area2 = box_area(box2.T)
|
||||
|
||||
# inter(N,M) = (rb(N,M,2) - lt(N,M,2)).clamp(0).prod(2)
|
||||
inter = (torch.min(box1[:, None, 2:], box2[:, 2:]) - torch.max(box1[:, None, :2], box2[:, :2])).clamp(0).prod(2)
|
||||
return inter / (area1[:, None] + area2 - inter) # iou = inter / (area1 + area2 - inter)
|
||||
|
||||
def xywh2xyxy(self, x):
|
||||
# Convert nx4 boxes from [x, y, w, h] to [x1, y1, x2, y2] where xy1=top-left, xy2=bottom-right
|
||||
y = x.clone() if isinstance(x, torch.Tensor) else np.copy(x)
|
||||
y[:, 0] = x[:, 0] - x[:, 2] / 2 # top left x
|
||||
y[:, 1] = x[:, 1] - x[:, 3] / 2 # top left y
|
||||
y[:, 2] = x[:, 0] + x[:, 2] / 2 # bottom right x
|
||||
y[:, 3] = x[:, 1] + x[:, 3] / 2 # bottom right y
|
||||
return y
|
||||
|
||||
def non_max_suppression(self, prediction, conf_thres=0.25, iou_thres=0.45, classes=None, agnostic=False,
|
||||
multi_label=False,
|
||||
labels=()):
|
||||
"""Runs Non-Maximum Suppression (NMS) on inference results
|
||||
|
||||
Returns:
|
||||
list of detections, on (n,6) tensor per image [xyxy, conf, cls]
|
||||
"""
|
||||
|
||||
nc = prediction.shape[2] - 5 # number of classes
|
||||
xc = prediction[..., 4] > conf_thres # candidates
|
||||
|
||||
# Settings
|
||||
min_wh, max_wh = 2, 4096 # (pixels) minimum and maximum box width and height
|
||||
max_det = 300 # maximum number of detections per image
|
||||
max_nms = 30000 # maximum number of boxes into torchvision.ops.nms()
|
||||
time_limit = 10.0 # seconds to quit after
|
||||
redundant = True # require redundant detections
|
||||
multi_label &= nc > 1 # multiple labels per box (adds 0.5ms/img)
|
||||
merge = False # use merge-NMS
|
||||
|
||||
t = time.time()
|
||||
output = [torch.zeros((0, 6), device=prediction.device)] * prediction.shape[0]
|
||||
for xi, x in enumerate(prediction): # image index, image inference
|
||||
# Apply constraints
|
||||
# x[((x[..., 2:4] < min_wh) | (x[..., 2:4] > max_wh)).any(1), 4] = 0 # width-height
|
||||
x = x[xc[xi]] # confidence
|
||||
|
||||
# Cat apriori labels if autolabelling
|
||||
if labels and len(labels[xi]):
|
||||
l = labels[xi]
|
||||
v = torch.zeros((len(l), nc + 5), device=x.device)
|
||||
v[:, :4] = l[:, 1:5] # box
|
||||
v[:, 4] = 1.0 # conf
|
||||
v[range(len(l)), l[:, 0].long() + 5] = 1.0 # cls
|
||||
x = torch.cat((x, v), 0)
|
||||
|
||||
# If none remain process next image
|
||||
if not x.shape[0]:
|
||||
continue
|
||||
|
||||
# Compute conf
|
||||
x[:, 5:] *= x[:, 4:5] # conf = obj_conf * cls_conf
|
||||
|
||||
# Box (center x, center y, width, height) to (x1, y1, x2, y2)
|
||||
box = self.xywh2xyxy(x[:, :4])
|
||||
|
||||
# Detections matrix nx6 (xyxy, conf, cls)
|
||||
if multi_label:
|
||||
i, j = (x[:, 5:] > conf_thres).nonzero(as_tuple=False).T
|
||||
x = torch.cat((box[i], x[i, j + 5, None], j[:, None].float()), 1)
|
||||
else: # best class only
|
||||
conf, j = x[:, 5:].max(1, keepdim=True)
|
||||
x = torch.cat((box, conf, j.float()), 1)[conf.view(-1) > conf_thres]
|
||||
|
||||
# Filter by class
|
||||
if classes is not None:
|
||||
x = x[(x[:, 5:6] == torch.tensor(classes, device=x.device)).any(1)]
|
||||
|
||||
# Apply finite constraint
|
||||
# if not torch.isfinite(x).all():
|
||||
# x = x[torch.isfinite(x).all(1)]
|
||||
|
||||
# Check shape
|
||||
n = x.shape[0] # number of boxes
|
||||
if not n: # no boxes
|
||||
continue
|
||||
elif n > max_nms: # excess boxes
|
||||
x = x[x[:, 4].argsort(descending=True)[:max_nms]] # sort by confidence
|
||||
|
||||
# Batched NMS
|
||||
c = x[:, 5:6] * (0 if agnostic else max_wh) # classes
|
||||
boxes, scores = x[:, :4] + c, x[:, 4] # boxes (offset by class), scores
|
||||
i = torchvision.ops.nms(boxes, scores, iou_thres) # NMS
|
||||
if i.shape[0] > max_det: # limit detections
|
||||
i = i[:max_det]
|
||||
if merge and (1 < n < 3E3): # Merge NMS (boxes merged using weighted mean)
|
||||
# update boxes as boxes(i,4) = weights(i,n) * boxes(n,4)
|
||||
iou = self.box_iou(boxes[i], boxes) > iou_thres # iou matrix
|
||||
weights = iou * scores[None] # box weights
|
||||
x[i, :4] = torch.mm(weights, x[:, :4]).float() / weights.sum(1, keepdim=True) # merged boxes
|
||||
if redundant:
|
||||
i = i[iou.sum(1) > 1] # require redundancy
|
||||
|
||||
output[xi] = x[i]
|
||||
if (time.time() - t) > time_limit:
|
||||
print(f'WARNING: NMS time limit {time_limit}s exceeded')
|
||||
break # time limit exceeded
|
||||
|
||||
return output
|
||||
|
||||
def clip_coords(self, boxes, img_shape):
|
||||
# Clip bounding xyxy bounding boxes to image shape (height, width)
|
||||
boxes[:, 0].clamp_(0, img_shape[1]) # x1
|
||||
boxes[:, 1].clamp_(0, img_shape[0]) # y1
|
||||
boxes[:, 2].clamp_(0, img_shape[1]) # x2
|
||||
boxes[:, 3].clamp_(0, img_shape[0]) # y2
|
||||
|
||||
def scale_coords(self, img1_shape, coords, img0_shape, ratio_pad=None):
|
||||
# Rescale coords (xyxy) from img1_shape to img0_shape
|
||||
if ratio_pad is None: # calculate from img0_shape
|
||||
gain = min(img1_shape[0] / img0_shape[0], img1_shape[1] / img0_shape[1]) # gain = old / new
|
||||
pad = (img1_shape[1] - img0_shape[1] * gain) / 2, (img1_shape[0] - img0_shape[0] * gain) / 2 # wh padding
|
||||
else:
|
||||
gain = ratio_pad[0][0]
|
||||
pad = ratio_pad[1]
|
||||
|
||||
coords[:, [0, 2]] -= pad[0] # x padding
|
||||
coords[:, [1, 3]] -= pad[1] # y padding
|
||||
coords[:, :4] /= gain
|
||||
self.clip_coords(coords, img0_shape)
|
||||
return coords
|
||||
|
||||
def plot_one_box(self, x, img, color=None, label=None, line_thickness=3):
|
||||
# Plots one bounding box on image img
|
||||
tl = line_thickness or round(0.002 * (img.shape[0] + img.shape[1]) / 2) + 1 # line/font thickness
|
||||
color = color or [random.randint(0, 255) for _ in range(3)]
|
||||
c1, c2 = (int(x[0]), int(x[1])), (int(x[2]), int(x[3]))
|
||||
cv2.rectangle(img, c1, c2, color, thickness=tl, lineType=cv2.LINE_AA)
|
||||
if label:
|
||||
tf = max(tl - 1, 1) # font thickness
|
||||
t_size = cv2.getTextSize(label, 0, fontScale=tl / 3, thickness=tf)[0]
|
||||
c2 = c1[0] + t_size[0], c1[1] - t_size[1] - 3
|
||||
cv2.rectangle(img, c1, c2, color, -1, cv2.LINE_AA) # filled
|
||||
cv2.putText(img, label, (c1[0], c1[1] - 2), 0, tl / 3, [225, 255, 255], thickness=tf, lineType=cv2.LINE_AA)
|
||||
|
||||
def infer(self, image, agnostic_nms=False, draw_flag=False):
|
||||
# read image
|
||||
# image=cv2.imread(img_path)
|
||||
|
||||
# Padded resize
|
||||
img = self.letterbox(image, self.imgsz, stride=self.stride)[0]
|
||||
|
||||
# Convert
|
||||
img = img[:, :, ::-1].transpose(2, 0, 1) # BGR to RGB, to 3x416x416
|
||||
img = np.ascontiguousarray(img)
|
||||
|
||||
img = torch.from_numpy(img).to(self.device)
|
||||
img = img.float() # uint8 to fp16/32
|
||||
img /= 255.0 # 0 - 255 to 0.0 - 1.0
|
||||
if img.ndimension() == 3:
|
||||
img = img.unsqueeze(0)
|
||||
|
||||
# Inference
|
||||
pred = self.model(img, augment=False)[0]
|
||||
|
||||
# Apply NMS
|
||||
pred = self.non_max_suppression(pred, self.conf_thres, self.iou_thres, classes=None, agnostic=agnostic_nms)
|
||||
|
||||
# Process detections
|
||||
s = ""
|
||||
s += '%gx%g ' % img.shape[2:] # print string
|
||||
result = []
|
||||
for i, det in enumerate(pred): # detections per image
|
||||
# Rescale boxes from img_size to im0 size
|
||||
det[:, :4] = self.scale_coords(img.shape[2:], det[:, :4], image.shape).round()
|
||||
|
||||
for *xyxy, conf, cls in reversed(det):
|
||||
x1, y1, x2, y2 = int(xyxy[0]), int(xyxy[1]), int(xyxy[2]), int(xyxy[3])
|
||||
result.append([x1, y1, x2, y2])
|
||||
if draw_flag:
|
||||
names = self.model.module.names if hasattr(self.model, 'module') else self.model.names
|
||||
colors = [[random.randint(0, 255) for _ in range(3)] for _ in names]
|
||||
for *xyxy, conf, cls in reversed(det):
|
||||
label = f'{names[int(cls)]} {conf:.2f}'
|
||||
self.plot_one_box(xyxy, image, label=label, color=colors[int(cls)], line_thickness=3)
|
||||
# Print results
|
||||
# for c in det[:, -1].unique():
|
||||
# n = (det[:, -1] == c).sum() # detections per class
|
||||
# s += f"{n}{'s' * (n > 1)}, " # add to string
|
||||
|
||||
# Write results
|
||||
# Get names and colors
|
||||
|
||||
# 显示预测结果
|
||||
# print(s)
|
||||
# print(result)
|
||||
# cv2.namedWindow("result",0)
|
||||
# cv2.imshow("result", image)
|
||||
# cv2.waitKey(0) # 1 millisecond
|
||||
# return image
|
||||
# 后处理
|
||||
return result
|
||||
259
test_identify.py
259
test_identify.py
|
|
@ -1,259 +0,0 @@
|
|||
import time
|
||||
from mem_top import mem_top
|
||||
import cv2
|
||||
import mediapipe as mp
|
||||
import math
|
||||
import util
|
||||
|
||||
|
||||
# page_up_count = 0
|
||||
# page_down_count = 0
|
||||
# last_wrist_point = (0, 0)
|
||||
|
||||
|
||||
class Identify:
|
||||
def __init__(self):
|
||||
self.result = 0
|
||||
self.position_x = 0
|
||||
self.position_y = 0
|
||||
self.image = []
|
||||
self.image_height = 0
|
||||
self.image_width = 0
|
||||
self.rgb_image = []
|
||||
self.identify_results = []
|
||||
self.hand_points = []
|
||||
self.angle_list = []
|
||||
self.is_finger_straight = [False, False, False, False, False]
|
||||
self.is_identify = False
|
||||
self.last_control_flag = 0
|
||||
self.page_up_count = 0
|
||||
self.page_down_count = 0
|
||||
self.last_wrist_point = (0, 0)
|
||||
self.now_time = 0
|
||||
self.lase_time = 0
|
||||
self.mp_drawing = mp.solutions.drawing_utils
|
||||
self.mp_hands = mp.solutions.hands
|
||||
self.hands = self.mp_hands.Hands(
|
||||
static_image_mode=False,
|
||||
max_num_hands=2,
|
||||
min_detection_confidence=0.75,
|
||||
min_tracking_confidence=0.75)
|
||||
|
||||
def begin(self):
|
||||
capture = cv2.VideoCapture(0)
|
||||
while 1:
|
||||
ret, self.image = capture.read()
|
||||
self.deal_with_image()
|
||||
self.now_time = time.time()
|
||||
fps = 1 / (self.now_time - self.lase_time)
|
||||
self.lase_time = self.now_time
|
||||
print("fps = " + str(fps))
|
||||
self.is_identify = False
|
||||
for i in range(5):
|
||||
self.is_finger_straight[i] = False
|
||||
self.hand_points.clear()
|
||||
self.get_hand_points()
|
||||
self.judge_finger_straight()
|
||||
flag = self.judge_control()
|
||||
# x = position[0]
|
||||
# y = position[1]
|
||||
cv2.namedWindow("Video")
|
||||
cv2.imshow("Video", self.image)
|
||||
if cv2.waitKey(1) & 0xFF == ord('q'):
|
||||
break
|
||||
control_flag = flag
|
||||
print("final_control_flag = " + str(control_flag))
|
||||
capture.release()
|
||||
cv2.destroyAllWindows()
|
||||
|
||||
def deal_with_image(self):
|
||||
self.rgb_image = cv2.cvtColor(self.image, cv2.COLOR_BGR2RGB)
|
||||
self.rgb_image = cv2.flip(self.rgb_image, 1)
|
||||
self.identify_results = self.hands.process(self.rgb_image)
|
||||
|
||||
def get_hand_points(self):
|
||||
if self.identify_results.multi_handedness:
|
||||
for i in range(len(self.identify_results.multi_handedness)):
|
||||
if self.identify_results.multi_handedness[i].classification[0].label != "Left":
|
||||
continue
|
||||
for hand_landmarks in self.identify_results.multi_hand_landmarks[i].landmark:
|
||||
if self.image_height == 0:
|
||||
self.image_height, self.image_width, c = self.image.shape
|
||||
cx, cy = int(hand_landmarks.x * self.image_width), int(hand_landmarks.y * self.image_height)
|
||||
self.hand_points.append((cx, cy))
|
||||
self.is_identify = True
|
||||
|
||||
def hand_angle(self):
|
||||
'''
|
||||
获取对应手相关向量的二维角度,根据角度确定手势
|
||||
'''
|
||||
angle_list = []
|
||||
hand_ = self.hand_points
|
||||
# ---------------------------- thumb 大拇指角度
|
||||
angle_ = util.Util.vector_2d_angle(
|
||||
((int(hand_[2][0]) - int(hand_[3][0])), (int(hand_[2][1]) - int(hand_[3][1]))),
|
||||
((int(hand_[3][0]) - int(hand_[4][0])), (int(hand_[3][1]) - int(hand_[4][1])))
|
||||
)
|
||||
angle_list.append(angle_)
|
||||
# ---------------------------- index 食指角度
|
||||
angle_ = util.Util.vector_2d_angle(
|
||||
((int(hand_[5][0]) - int(hand_[6][0])), (int(hand_[5][1]) - int(hand_[6][1]))),
|
||||
((int(hand_[7][0]) - int(hand_[8][0])), (int(hand_[7][1]) - int(hand_[8][1])))
|
||||
)
|
||||
angle_list.append(angle_)
|
||||
# ---------------------------- middle 中指角度
|
||||
angle_ = util.Util.vector_2d_angle(
|
||||
((int(hand_[9][0]) - int(hand_[10][0])), (int(hand_[9][1]) - int(hand_[10][1]))),
|
||||
((int(hand_[11][0]) - int(hand_[12][0])), (int(hand_[11][1]) - int(hand_[12][1])))
|
||||
)
|
||||
angle_list.append(angle_)
|
||||
# ---------------------------- ring 无名指角度
|
||||
angle_ = util.Util.vector_2d_angle(
|
||||
((int(hand_[13][0]) - int(hand_[14][0])), (int(hand_[13][1]) - int(hand_[14][1]))),
|
||||
((int(hand_[15][0]) - int(hand_[16][0])), (int(hand_[15][1]) - int(hand_[16][1])))
|
||||
)
|
||||
angle_list.append(angle_)
|
||||
# ---------------------------- pink 小拇指角度
|
||||
angle_ = util.Util.vector_2d_angle(
|
||||
((int(hand_[17][0]) - int(hand_[18][0])), (int(hand_[17][1]) - int(hand_[18][1]))),
|
||||
((int(hand_[19][0]) - int(hand_[20][0])), (int(hand_[19][1]) - int(hand_[20][1])))
|
||||
)
|
||||
angle_list.append(angle_)
|
||||
self.angle_list = angle_list
|
||||
|
||||
def judge_finger_straight(self):
|
||||
if self.is_identify:
|
||||
self.hand_angle()
|
||||
for i in range(5):
|
||||
self.is_finger_straight[i] = util.Util.is_straight(self.angle_list[i])
|
||||
|
||||
def judge_zero(self):
|
||||
return not self.is_finger_straight[1] and not self.is_finger_straight[2] and \
|
||||
not self.is_finger_straight[3] and not self.is_finger_straight[4]
|
||||
|
||||
def judge_one(self):
|
||||
return self.is_finger_straight[1] and not self.is_finger_straight[2] and \
|
||||
not self.is_finger_straight[3] and not self.is_finger_straight[4]
|
||||
|
||||
def judge_two(self):
|
||||
return self.is_finger_straight[1] and self.is_finger_straight[2] and \
|
||||
not self.is_finger_straight[3] and not self.is_finger_straight[4]
|
||||
|
||||
def judge_three(self):
|
||||
return self.is_finger_straight[1] and self.is_finger_straight[2] and \
|
||||
self.is_finger_straight[3] and not self.is_finger_straight[4]
|
||||
|
||||
def judge_four(self):
|
||||
return self.is_finger_straight[1] and self.is_finger_straight[2] and \
|
||||
self.is_finger_straight[3] and self.is_finger_straight[4]
|
||||
|
||||
def judge_five(self):
|
||||
return self.is_finger_straight[1] and self.is_finger_straight[2] and \
|
||||
self.is_finger_straight[3] and self.is_finger_straight[4]
|
||||
|
||||
# def judge_five(self):
|
||||
# self.hand_angle()
|
||||
# return util.Util.is_straight(self.angle_list[1]) and util.Util.is_straight(
|
||||
# self.angle_list[2]) and util.Util.is_straight(self.angle_list[3]) and util.Util.is_straight(
|
||||
# self.angle_list[4])
|
||||
|
||||
# def judge_open(self):
|
||||
# self.hand_angle()
|
||||
# # angle_ = vector_2d_angle(
|
||||
# # ((int(points[0][0]) - int(points[5][0])), (int(points[0][1]) - int(points[5][1]))),
|
||||
# # ((int(points[5][0]) - int(points[8][0])), (int(points[5][1]) - int(points[8][1])))
|
||||
# # )
|
||||
# return not util.Util.is_straight(self.angle_list[1]) and util.Util.is_straight(
|
||||
# self.angle_list[2]) and util.Util.is_straight(self.angle_list[3]) and util.Util.is_straight(
|
||||
# self.angle_list[4])
|
||||
#
|
||||
# def judge_up(self):
|
||||
# self.hand_angle()
|
||||
# angle_ = util.Util.vector_2d_angle(
|
||||
# ((int(self.hand_points[0][0]) - int(self.hand_points[5][0])),
|
||||
# (int(self.hand_points[0][1]) - int(self.hand_points[5][1]))),
|
||||
# ((int(self.hand_points[5][0]) - int(self.hand_points[8][0])),
|
||||
# (int(self.hand_points[5][1]) - int(self.hand_points[8][1])))
|
||||
# )
|
||||
# return util.Util.is_straight(
|
||||
# self.angle_list[1] and not util.Util.is_straight(self.angle_list[2]) and not util.Util.is_straight(
|
||||
# self.angle_list[3]) and not util.Util.is_straight(self.angle_list[4])) and angle_ <= 40
|
||||
#
|
||||
# def judge_down(self):
|
||||
# self.hand_angle()
|
||||
# return util.Util.is_straight(self.angle_list[1]) and util.Util.is_straight(
|
||||
# self.angle_list[2]) and not util.Util.is_straight(self.angle_list[3]) and not util.Util.is_straight(
|
||||
# self.angle_list[4])
|
||||
#
|
||||
# def judge_end(self):
|
||||
# self.hand_angle()
|
||||
# return not util.Util.is_straight(self.angle_list[1]) and not util.Util.is_straight(
|
||||
# self.angle_list[2]) and not util.Util.is_straight(self.angle_list[3]) and not util.Util.is_straight(
|
||||
# self.angle_list[4])
|
||||
|
||||
# def judge_one(self):
|
||||
# self.hand_angle()
|
||||
# return util.Util.is_straight(self.angle_list[1]) and not util.Util.is_straight(
|
||||
# self.angle_list[2]) and not util.Util.is_straight(self.angle_list[3]) and not util.Util.is_straight(
|
||||
# self.angle_list[4])
|
||||
|
||||
def judge_page_up(self):
|
||||
if self.page_up_count == 0:
|
||||
if self.judge_five():
|
||||
self.last_wrist_point = self.hand_points[0]
|
||||
self.page_up_count += 1
|
||||
return False
|
||||
if self.judge_five() and self.hand_points[0][0] < self.last_wrist_point[0]:
|
||||
if self.hand_points[0][0] > self.last_wrist_point[0]:
|
||||
self.page_up_count = 0
|
||||
return False
|
||||
if self.page_up_count >= 2:
|
||||
self.page_up_count = 0
|
||||
self.last_wrist_point = (0, 0)
|
||||
return True
|
||||
self.page_up_count += 1
|
||||
self.last_wrist_point = self.hand_points[0]
|
||||
return False
|
||||
|
||||
def judge_page_down(self):
|
||||
# global last_wrist_point
|
||||
# global page_down_count
|
||||
# print("page_down_count = " + str(page_down_count))
|
||||
# print("last_wrist_point = " + str(last_wrist_point))
|
||||
# print("points[0] = " + str(points[0]))
|
||||
if self.page_down_count == 0:
|
||||
if self.judge_five():
|
||||
self.last_wrist_point = self.hand_points[0]
|
||||
self.page_down_count += 1
|
||||
return False
|
||||
if self.judge_five() and self.hand_points[0][0] > self.last_wrist_point[0]:
|
||||
if self.hand_points[0][0] < self.last_wrist_point[0]:
|
||||
self.page_down_count = 0
|
||||
return False
|
||||
if self.page_down_count >= 2:
|
||||
self.page_down_count = 0
|
||||
self.last_wrist_point = (0, 0)
|
||||
return True
|
||||
self.page_down_count += 1
|
||||
self.last_wrist_point = self.hand_points[0]
|
||||
return False
|
||||
|
||||
def judge_control(self):
|
||||
if self.is_identify:
|
||||
if self.judge_one():
|
||||
# print("open_ppt")
|
||||
return 1
|
||||
elif self.judge_five():
|
||||
# print("ppt_up")
|
||||
return 2
|
||||
elif self.judge_three():
|
||||
# print("ppt_down")
|
||||
return 3
|
||||
elif self.judge_zero():
|
||||
# print("ppt_end")
|
||||
return 4
|
||||
else:
|
||||
print("other")
|
||||
else:
|
||||
print("no_hand_points")
|
||||
return 0
|
||||
6
util.py
6
util.py
|
|
@ -26,3 +26,9 @@ class Util:
|
|||
|
||||
def get_distance(point1, point2):
|
||||
return math.sqrt((point1[0] - point2[0]) ** 2 + (point1[1] - point2[1]) ** 2)
|
||||
|
||||
def is_in_rectangle(target_point, rectangle):
|
||||
if target_point[0] > rectangle[0] and target_point[0] < rectangle[2] and target_point[1] > rectangle[1] and \
|
||||
target_point[1] < rectangle[3]:
|
||||
return True
|
||||
return False
|
||||
|
|
|
|||
Loading…
Reference in New Issue