最终版本

This commit is contained in:
Dinger 2022-05-14 14:29:25 +08:00
parent aea0832dbf
commit 78ade849df
10 changed files with 181 additions and 164 deletions

Binary file not shown.

Binary file not shown.

Binary file not shown.

Binary file not shown.

Binary file not shown.

View File

View File

@ -49,7 +49,7 @@ class Ensemble(torch.nn.ModuleList):
class YOLOV5(object):
def __init__(self, conf_thres=0.25,
def __init__(self, conf_thres=0.5,
iou_thres=0.45,
classes=None,
imgsz=640,
@ -312,7 +312,7 @@ class YOLOV5(object):
# Apply NMS
pred = self.non_max_suppression(pred, self.conf_thres, self.iou_thres, classes=None, agnostic=agnostic_nms)
draw_flag = False
# Process detections
s = ""
s += '%gx%g ' % img.shape[2:] # print string
@ -341,9 +341,10 @@ class YOLOV5(object):
# 显示预测结果
# print(s)
# print(result)
# cv2.namedWindow("result",0)
# cv2.rectangle(image, (result[0], result[1]), (result[2], result[3]), (0, 0, 255), 2)
# cv2.namedWindow("result", 0)
# cv2.imshow("result", image)
# cv2.waitKey(0) # 1 millisecond
# cv2.waitKey(1) # 1 millisecond
# return image
# 后处理
return result

View File

@ -7,11 +7,7 @@ import util
import detect
import win32gui
import win32con
# page_up_count = 0
# page_down_count = 0
# last_wrist_point = (0, 0)
import sys
def hand_angle(hand_):
@ -19,10 +15,6 @@ def hand_angle(hand_):
获取对应手相关向量的二维角度,根据角度确定手势
'''
angle_list = []
# if self.left_hand_flag:
# hand_ = self.left_hand_points
# else:
# hand_ = self.right_hand_points
# ---------------------------- thumb 大拇指角度
angle_ = util.Util.vector_2d_angle(
((int(hand_[2][0]) - int(hand_[3][0])), (int(hand_[2][1]) - int(hand_[3][1]))),
@ -82,8 +74,9 @@ def judge_five(is_finger_straight):
class Identify:
def __init__(self, value, array):
self.value = value
def __init__(self, control_flag, array):
self.rgb_image = None
self.control_flag = control_flag
self.array = array
self.mouse = PyMouse()
self.left_hand_flag = False
@ -129,29 +122,44 @@ class Identify:
self.mp_face = mp.solutions.face_detection
self.face_detection = self.mp_face.FaceDetection(min_detection_confidence=0.5)
def begin(self):
def begin(self, over_flag):
capture = cv2.VideoCapture(0)
last_time = 0
self.ml_identify = detect.YOLOV5()
screen_width = self.mouse.screen_size()[0]
time1 = time.time()
x1, y1, x2, y2 = 0, 0, 0, 0
count = 0
while 1:
count += 1
if over_flag.value == 1:
sys.exit(0)
ret, self.image = capture.read()
self.image = cv2.resize(self.image, (320, 240), interpolation=cv2.INTER_LINEAR)
# fps = 1 / (self.now_time - self.lase_time)
# self.lase_time = self.now_time
# print("fps = " + str(fps))
# self.ml_identify.infer(image=self.image)
# continue
# self.catch_person_flag = True
if not self.catch_person_flag:
# self.catch_person_flag = True
self.person_results.clear()
self.left_hands.clear()
self.right_hands.clear()
self.deal_with_image()
self.find_points()
x1, y1, x2, y2 = self.find_target_person()
# print("no no no no no no no no no no no no no")
else:
self.image = self.image[y1:y2, x1:x2]
# print("in True")
# print(x1, y1, x2, y2)
self.deal_with_image()
self.identify_results = self.hands.process(self.rgb_image)
# fps = 1 / (self.now_time - self.lase_time)
# self.lase_time = self.now_time
# print("fps = " + str(fps))
self.image = self.image[y1:y2, x1:x2]
# self.identify_results = self.hands.process(self.image)
self.is_identify = False
self.left_hand_flag = False
self.right_hand_flag = False
@ -161,20 +169,25 @@ class Identify:
self.left_hand_points.clear()
self.right_hand_points.clear()
self.get_hand_points()
# print("---------------------")
# print(self.left_hand_flag)
# print(self.left_hand_points)
# print(self.right_hand_flag)
# print(self.right_hand_points)
self.judge_finger_straight()
flag = self.judge_control()
if flag:
self.flag = flag
print("this flag = " + str(flag))
# print("this flag = " + str(flag))
now_time = time.time()
self.array[0] = self.position_x
self.array[1] = self.position_y
self.array[2] = self.image_width
self.array[3] = self.image_height
self.array[2] = x2 - x1
self.array[3] = y2 - y1
if now_time - last_time < 1:
continue
last_time = now_time
self.catch_person_flag = not self.judge_over()
self.catch_person_flag = not self.judge_catch_over()
# print("**************")
# for i in range(5):
# print(self.is_finger_straight[i])
@ -182,9 +195,9 @@ class Identify:
# y = position[1]
# control_flag = flag
self.value.value = self.flag
self.control_flag.value = self.flag
# print("self.v.value = " + str(self.flag))
print("final_control_flag = " + str(self.flag))
# print("final_control_flag = " + str(self.flag))
self.flag = 0
cv2.namedWindow("Video")
@ -200,7 +213,7 @@ class Identify:
capture.release()
cv2.destroyAllWindows()
def judge_over(self):
def judge_catch_over(self):
if not self.left_hand_flag:
return False
if not self.right_hand_flag:
@ -246,8 +259,9 @@ class Identify:
self.right_hands.append(right_hand_points)
def find_target_person(self):
results = self.ml_identify.infer(image=self.image)
for result in results:
target_scopes = self.ml_identify.infer(image=self.image)
for result in target_scopes:
# cv2.rectangle(self.image, (result[0], result[1]), (result[2], result[3]), (0, 0, 255), 2)
one_person_result = [result, [], []]
for left_hand in self.left_hands:
if util.Util.is_in_rectangle(left_hand[0], result):
@ -260,7 +274,10 @@ class Identify:
self.person_results.append(one_person_result)
if self.person_results:
person_result = self.find_catch_pose()
return person_result[0]
if self.catch_person_flag:
return person_result[0]
else:
return 0, 0, 0, 0
else:
return 0, 0, 0, 0
@ -287,13 +304,13 @@ class Identify:
def deal_with_image(self):
self.image = cv2.flip(self.image, 1)
self.identify_results = self.hands.process(self.image)
self.rgb_image = cv2.cvtColor(self.image, cv2.COLOR_BGR2RGB)
self.identify_results = self.hands.process(self.rgb_image)
def get_hand_points(self):
if not self.identify_results.multi_handedness:
return 0
for i in range(len(self.identify_results.multi_handedness)):
# print(self.identify_results.multi_handedness[i].classification[0].label)
if self.identify_results.multi_handedness[i].classification[0].label == "Left":
for hand_landmarks in self.identify_results.multi_hand_landmarks[i].landmark:
if self.image_height == 0:
@ -303,7 +320,6 @@ class Identify:
# self.mp_drawing.draw_landmarks(
# self.image, self.identify_results.multi_hand_landmarks[i], self.mp_hands.HAND_CONNECTIONS)
if self.identify_results.multi_handedness[i].classification[0].score > 0.5:
print("identify is left")
self.left_hand_flag = True
self.is_identify = True
@ -317,7 +333,6 @@ class Identify:
# self.mp_drawing.draw_landmarks(
# self.image, self.identify_results.multi_hand_landmarks[i], self.mp_hands.HAND_CONNECTIONS)
if self.identify_results.multi_handedness[i].classification[0].score > 0.5:
print("identify is right")
self.right_hand_flag = True
self.is_identify = True
@ -325,11 +340,8 @@ class Identify:
if self.is_identify:
if self.left_hand_flag:
self.left_angle_list = hand_angle(hand_=self.left_hand_points)
# print("(((((((((((((((((((")
for i in range(5):
self.is_left_finger_straight[i] = util.Util.is_straight(self.left_angle_list[i])
# print(self.is_left_finger_straight[i])
# print(")))))))))))))))))))")
if self.right_hand_flag:
self.right_angle_list = hand_angle(hand_=self.right_hand_points)
for i in range(5):
@ -339,9 +351,9 @@ class Identify:
if self.left_hand_flag:
self.position_x = self.left_hand_points[8][0]
self.position_y = self.left_hand_points[8][1]
elif self.right_hand_flag:
self.position_x = self.right_hand_points[8][0]
self.position_y = self.right_hand_points[8][1]
# elif self.right_hand_flag:
# self.position_x = self.right_hand_points[8][0]
# self.position_y = self.right_hand_points[8][1]
return is_finger_straight[1] and not is_finger_straight[2] and \
not is_finger_straight[3] and not is_finger_straight[4]
@ -394,22 +406,25 @@ class Identify:
if self.step_up == 0:
self.lase_time = time.time()
if self.step_up == 0 and self.judge_step_three():
self.step_up = 4
if self.step_up == 4 and self.judge_step_one(False):
self.step_up = 1
elif self.step_up == 1 and self.judge_step_two(False):
if self.step_up == 1 and self.judge_step_one(False):
self.step_up = 2
elif self.step_up == 2 and self.judge_step_two(False):
self.step_up = 3
elif self.step_up == 2 and judge_zero(self.is_right_finger_straight):
self.step_up = 3
elif self.step_up == 3:
# elif self.step_up == 2 and judge_zero(self.is_right_finger_straight):
# self.step_up = 3
# elif self.step_up == 3:
# self.step_up = 0
now_time = time.time()
if now_time - self.lase_time > 3:
self.lase_time = now_time
self.step_up = 0
now_time = time.time()
if now_time - self.lase_time < 3:
return False
else:
if self.step_up == 3:
self.lase_time = now_time
self.step_up = 0
return True
else:
self.lase_time = now_time
return False
return False
def judge_page_down(self):
@ -418,25 +433,24 @@ class Identify:
if self.step_down == 0:
self.lase_time = time.time()
if self.step_down == 0 and self.judge_step_three():
self.step_down = 4
print("step = 1")
if self.step_down == 4 and self.judge_step_one(True):
self.step_down = 1
print("step = 2")
elif self.step_down == 1 and self.judge_step_two(True):
# print("step = 1")
if self.step_down == 1 and self.judge_step_one(True):
self.step_down = 2
# print("step = 2")
elif self.step_down == 2 and self.judge_step_two(True):
self.step_down = 3
print("step = 3")
elif self.step_down == 2 and judge_zero(self.is_left_finger_straight):
self.step_down = 3
elif self.step_down == 3:
# print("step = 3")
now_time = time.time()
if now_time - self.lase_time > 3:
self.lase_time = now_time
self.step_down = 0
now_time = time.time()
if now_time - self.lase_time < 3:
return False
else:
if self.step_down == 3:
self.lase_time = now_time
self.step_down = 0
return True
else:
self.lase_time = now_time
return False
return False
def judge_end(self):
@ -444,36 +458,47 @@ class Identify:
return True
return False
def judge_system_over(self):
if not self.judge_one(self.is_left_finger_straight):
return False
if not self.judge_one(self.is_right_finger_straight):
return False
standard_distance = util.Util.get_distance(self.left_hand_points[7], self.right_hand_points[8])
# print(util.Util.get_distance(self.left_hand_points[8], self.right_hand_points[8]) / standard_distance)
if util.Util.get_distance(self.left_hand_points[8], self.right_hand_points[8]) / standard_distance < 1:
return True
return False
def judge_control(self):
# print("left_flag = " + str(self.left_hand_flag))
# print("right_flag = " + str(self.right_hand_flag))
# print("1111111111111111111111")
# for i in range(5):
# print(self.is_left_finger_straight[i])
# print("2222222222222222222222")
if self.is_identify:
# print("len = " + str(len(self.identify_results.multi_handedness)))
# print("left is " + str(self.left_hand_flag))
# print("right is " + str(self.right_hand_flag))
if judge_two(self.is_left_finger_straight):
print("1")
# print("1")
return 1
elif self.judge_page_up():
print("2")
# print("2")
return 2
elif self.judge_page_down():
print("3")
# print("down!down!down!down!down!down!down!down!down!down!down!down!down!down!down!down!down!down!")
# print("3")
return 3
elif self.judge_one(self.is_left_finger_straight):
print("4")
elif self.judge_one(self.is_left_finger_straight) and not self.right_hand_flag:
# print("4")
return 4
elif self.left_hand_flag and judge_zero(self.is_left_finger_straight):
print("5")
# for i in range(5):
# print(self.is_left_finger_straight[i])
elif self.judge_one(self.is_left_finger_straight) and judge_zero(self.is_right_finger_straight):
# print("5")
return 5
else:
print("other")
else:
print("no_hand_points")
elif self.left_hand_flag and judge_zero(self.is_left_finger_straight):
# print("6")
return 6
elif self.judge_system_over():
# print("7")
return 7
# else:
# # print("other")
# else:
# print("no_hand_points")
return 0
#

122
main.py
View File

@ -1,27 +1,15 @@
import multiprocessing
import sys
import tkinter
from identify import *
from tkinter.filedialog import *
from tkinter import messagebox
from win32com.client import Dispatch
import win32con
import win32api
import time
import os
control_flag = 0
x = 0
y = 0
def show():
win32api.keybd_event(116, 0, 0, 0) # 代表按下f键
time.sleep(0.02)
win32api.keybd_event(116, 0, win32con.KEYEVENTF_KEYUP, 0) # 释放f键
time.sleep(2)
win32api.keybd_event(34, 0, 0, 0) # 代表按下f键
time.sleep(0.02)
win32api.keybd_event(34, 0, win32con.KEYEVENTF_KEYUP, 0) # 释放f键
def control_page_up():
win32api.keybd_event(38, 0, 0, 0) # 代表按下f键
@ -61,39 +49,29 @@ def control_writing(m, position_x, position_y):
m.move(position_x, position_y)
def control_draw():
global x, y
# m = pymouse.PyMouse()
# m.move(x, y)
# win32api.keybd_event(40, 0, 0, 0) # 代表按下f键
# time.sleep(0.02)
# win32api.keybd_event(40, 0, win32con.KEYEVENTF_KEYUP, 0) # 释放f键
# class Control:
# def __init__(self, control_flag):
# self.control_flag = control_flag
#
def control_thread(value, array):
def control_thread(control_number, array, over_flag):
last_time = 0.0
step = 0
mouse = PyMouse()
test_time1 = time.time()
count = 0
while 1:
count += 1
# print("step = " + str(step))
# print("control_thread")
now_time = time.time()
# time.sleep(0.1)
# print(end - start)
# print("***********array************")
# print(array)
# print("***********array************")
control_flag = value.value
control_flag = control_number.value
# os._exit(0)
if control_flag == 7:
# test_time2 = time.time()
# fps = count / (test_time2 - test_time1)
# print("count2 = " + str(count))
# print("time = " + str(test_time2 - test_time1))
# print("fps_control = " + str(fps))
over_flag.value = 1
sys.exit(0)
if step == 2:
if control_flag == 4:
# print("inininininininininininininininininininininininin")
position_x = int(mouse.screen_size()[0] / array[2] * array[0])
position_y = int(mouse.screen_size()[1] / array[3] * array[1])
control_writing(mouse, position_x, position_y)
@ -101,22 +79,16 @@ def control_thread(value, array):
step = 1
control_ppt_end()
last_time = now_time
# print("step to 11111111111111111")
continue
if now_time - last_time < 1:
continue
last_time = now_time
# if control_flag in range(1, 2, 3, 4, 5):
# time.sleep(2)
# elif control_flag == 6:
# time.sleep(0.05)
# start = time.time()
print("control_flag = " + str(control_flag))
# print("main.step = " + str(step))
if step == 0 and control_flag == 1:
control_ppt_begin()
# print("control_flag == 1")
step = 1
# print("control_flag2 = " + str(control_flag))
# # print("main.step = " + str(step))
if step == 0:
if control_flag == 1:
control_ppt_begin()
step = 1
elif step == 1:
if control_flag == 2:
control_page_up()
@ -127,38 +99,37 @@ def control_thread(value, array):
elif control_flag == 4:
control_open_pencil()
step = 2
elif control_flag == 5:
elif control_flag == 6:
control_ppt_end()
# print("end!end!end!end!end!end!end!end!end!end!end!")
# print("control_flag == 4")
step = 0
# if control_flag == 5:
# control_open_pencil()
# if control_flag == 6:
# control_draw()
def identify_thread(value, array):
identify = Identify(value, array)
identify.begin()
def show_thread(value, array):
return 0
def identify_thread(control_number, array, over_flag):
identify = Identify(control_number, array)
identify.begin(over_flag)
def open_file():
file_path = askopenfilename(title=u'选择文件')
# file_path = askopenfilename(title=u'选择文件', initialdir=(os.path.expanduser('H:/')))
open_ppt(file_path)
value = multiprocessing.Value('i', 0)
if not open_ppt(file_path):
return False
control_number = multiprocessing.Value('i', 0)
array = multiprocessing.Array('i', 4)
over_flag = multiprocessing.Value('i', 0)
# array2 = multiprocessing.Array()
p1 = multiprocessing.Process(target=identify_thread, args=(value, array))
p2 = multiprocessing.Process(target=control_thread, args=(value, array))
p1 = multiprocessing.Process(target=identify_thread, args=(control_number, array, over_flag))
p2 = multiprocessing.Process(target=control_thread, args=(control_number, array, over_flag))
# p3 = multiprocessing.Process(target=show_thread, args=(value, array))
p1.start()
# p1.terminate()
p2.start()
return True
# p3.start()
# identify_t = threading.Thread(target=identify_thread)
# print("control_flag1 = " + str(control_flag))
@ -170,11 +141,32 @@ def open_file():
# control_t.start()
def judge_ppt(file_path):
string = file_path
string = string.split('.')
if len(string) <= 0:
return False
string = string[len(string) - 1]
if string.find('ppt') != -1:
return True
return False
def open_ppt(file_path):
if not len(file_path):
messagebox.showinfo("选择失败", "您尚未选择ppt文件请重新选择")
return False
if not judge_ppt(file_path):
messagebox.showinfo("选择失败", "该文件非ppt文件请重新选择")
return False
ppt = Dispatch('PowerPoint.Application')
ppt.Visible = 1 # 后台运行
ppt.DisplayAlerts = 0 # 不显示,不警告
ppt.Presentations.Open(file_path)
return True
# def make_button(window):
if __name__ == '__main__':
@ -183,4 +175,6 @@ if __name__ == '__main__':
window.geometry("200x100")
bt1 = tkinter.Button(window, text='打开文件', width=15, height=15, command=open_file)
bt1.pack()
# show_text = tkinter.Text(window, height=2)
# show_text.pack()
window.mainloop()

25
test.py
View File

@ -46,9 +46,6 @@ def video_speed(video_root, out_root, fps=None, scale=1):
# yolov5 = detect.YOLOV5()
# blank_image = np.zeros((1000, 1000, 3), np.uint8)
# blank_image.fill(255)
fourcc = cv2.VideoWriter_fourcc(*'XVID')
videoWriter = cv2.VideoWriter('output.avi', fourcc, 60.0, (1280, 720))
count = 100
while True:
ret, frame = cap.read()
# blank_image = frame.copy()
@ -58,9 +55,9 @@ def video_speed(video_root, out_root, fps=None, scale=1):
# print("")
frame = cv2.flip(frame, 1)
frame = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)
frame_rgb = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)
# frame = cv2.cvtColor(frame, cv2.COLOR_RGB2BGR)
results = hands.process(frame)
results = hands.process(frame_rgb)
# results = poses.process(image)
# print("*********************************")
# if results.pose_landmarks:
@ -74,19 +71,19 @@ def video_speed(video_root, out_root, fps=None, scale=1):
# if results.
# print(results.multi_hand_landmarks)
if results.multi_handedness:
if results.multi_handedness[0].classification[0].label == "Left":
for hand_landmarks in results.multi_hand_landmarks:
# 关键点可视化
mp_drawing.draw_landmarks(
frame, hand_landmarks, mp_hands.HAND_CONNECTIONS)
# if results.multi_handedness[0].classification[0].label == "Right":
# print(results.multi_handedness[0].classification[0])
# if results.multi_handedness[0].classification[0].label == "Left":
for hand_landmarks in results.multi_hand_landmarks:
# 关键点可视化
mp_drawing.draw_landmarks(
frame, hand_landmarks, mp_hands.HAND_CONNECTIONS)
# if results.multi_handedness[0].classification[0].label == "Right":
# print(results.multi_handedness[0].classification[0])
cv2.imshow('MediaPipe Hands', frame)
# cv2.imshow('black_image', blank_image)
# videoWriter.write(blank_image)
# count = count - 1
# cv2.imwrite("picture3.jpg", frame)
if cv2.waitKey(1) & 0xFF == 27:
cv2.imwrite("picture3.jpg", frame)
if cv2.waitKey(3000) & 0xFF == 27:
break
cap.release()
# videoWriter.release()