最终版本

detect.py

@@ -49,7 +49,7 @@ class Ensemble(torch.nn.ModuleList):
 
 
 class YOLOV5(object):
-    def __init__(self, conf_thres=0.25,
+    def __init__(self, conf_thres=0.5,
                  iou_thres=0.45,
                  classes=None,
                  imgsz=640,
@@ -312,7 +312,7 @@ class YOLOV5(object):
 
         # Apply NMS
         pred = self.non_max_suppression(pred, self.conf_thres, self.iou_thres, classes=None, agnostic=agnostic_nms)
-
+        draw_flag = False
         # Process detections
         s = ""
         s += '%gx%g ' % img.shape[2:]  # print string
@@ -341,9 +341,10 @@ class YOLOV5(object):
         # 显示预测结果
         # print(s)
         # print(result)
-        # cv2.namedWindow("result",0)
+        # cv2.rectangle(image, (result[0], result[1]), (result[2], result[3]), (0, 0, 255), 2)
+        # cv2.namedWindow("result", 0)
         # cv2.imshow("result", image)
-        # cv2.waitKey(0)  # 1 millisecond
+        # cv2.waitKey(1)  # 1 millisecond
         # return image
         # 后处理
         return result

identify.py

@@ -7,11 +7,7 @@ import util
 import detect
 import win32gui
 import win32con
-
+import sys
-
-# page_up_count = 0
-# page_down_count = 0
-# last_wrist_point = (0, 0)
 
 
 def hand_angle(hand_):
@@ -19,10 +15,6 @@ def hand_angle(hand_):
         获取对应手相关向量的二维角度,根据角度确定手势
     '''
     angle_list = []
-    # if self.left_hand_flag:
-    #     hand_ = self.left_hand_points
-    # else:
-    #     hand_ = self.right_hand_points
     # ---------------------------- thumb 大拇指角度
     angle_ = util.Util.vector_2d_angle(
         ((int(hand_[2][0]) - int(hand_[3][0])), (int(hand_[2][1]) - int(hand_[3][1]))),
@@ -82,8 +74,9 @@ def judge_five(is_finger_straight):
 
 
 class Identify:
-    def __init__(self, value, array):
+    def __init__(self, control_flag, array):
-        self.value = value
+        self.rgb_image = None
+        self.control_flag = control_flag
         self.array = array
         self.mouse = PyMouse()
         self.left_hand_flag = False
@@ -129,29 +122,44 @@ class Identify:
         self.mp_face = mp.solutions.face_detection
         self.face_detection = self.mp_face.FaceDetection(min_detection_confidence=0.5)
 
-    def begin(self):
+    def begin(self, over_flag):
         capture = cv2.VideoCapture(0)
         last_time = 0
         self.ml_identify = detect.YOLOV5()
         screen_width = self.mouse.screen_size()[0]
+        time1 = time.time()
         x1, y1, x2, y2 = 0, 0, 0, 0
+        count = 0
         while 1:
+            count += 1
+            if over_flag.value == 1:
+                sys.exit(0)
             ret, self.image = capture.read()
             self.image = cv2.resize(self.image, (320, 240), interpolation=cv2.INTER_LINEAR)
+            # fps = 1 / (self.now_time - self.lase_time)
+            # self.lase_time = self.now_time
+            # print("fps = " + str(fps))
+            # self.ml_identify.infer(image=self.image)
+            # continue
+            # self.catch_person_flag = True
             if not self.catch_person_flag:
+                # self.catch_person_flag = True
                 self.person_results.clear()
                 self.left_hands.clear()
                 self.right_hands.clear()
                 self.deal_with_image()
                 self.find_points()
                 x1, y1, x2, y2 = self.find_target_person()
+
+                # print("no no no no no no no no no no no no no")
             else:
-                self.image = self.image[y1:y2, x1:x2]
+                # print("in True")
+                # print(x1, y1, x2, y2)
                 self.deal_with_image()
-                self.identify_results = self.hands.process(self.rgb_image)
+                self.image = self.image[y1:y2, x1:x2]
-                # fps = 1 / (self.now_time - self.lase_time)
+
-                # self.lase_time = self.now_time
+                # self.identify_results = self.hands.process(self.image)
-                # print("fps = " + str(fps))
+
                 self.is_identify = False
                 self.left_hand_flag = False
                 self.right_hand_flag = False
@@ -161,20 +169,25 @@ class Identify:
                 self.left_hand_points.clear()
                 self.right_hand_points.clear()
                 self.get_hand_points()
+                # print("---------------------")
+                # print(self.left_hand_flag)
+                # print(self.left_hand_points)
+                # print(self.right_hand_flag)
+                # print(self.right_hand_points)
                 self.judge_finger_straight()
                 flag = self.judge_control()
                 if flag:
                     self.flag = flag
-                    print("this flag = " + str(flag))
+                    # print("this flag = " + str(flag))
                 now_time = time.time()
                 self.array[0] = self.position_x
                 self.array[1] = self.position_y
-                self.array[2] = self.image_width
+                self.array[2] = x2 - x1
-                self.array[3] = self.image_height
+                self.array[3] = y2 - y1
                 if now_time - last_time < 1:
                     continue
                 last_time = now_time
-                self.catch_person_flag = not self.judge_over()
+                self.catch_person_flag = not self.judge_catch_over()
                 # print("**************")
                 # for i in range(5):
                 #     print(self.is_finger_straight[i])
@@ -182,9 +195,9 @@ class Identify:
                 # y = position[1]
 
                 # control_flag = flag
-                self.value.value = self.flag
+                self.control_flag.value = self.flag
                 # print("self.v.value = " + str(self.flag))
-                print("final_control_flag = " + str(self.flag))
+                # print("final_control_flag = " + str(self.flag))
                 self.flag = 0
 
             cv2.namedWindow("Video")
@@ -200,7 +213,7 @@ class Identify:
         capture.release()
         cv2.destroyAllWindows()
 
-    def judge_over(self):
+    def judge_catch_over(self):
         if not self.left_hand_flag:
             return False
         if not self.right_hand_flag:
@@ -246,8 +259,9 @@ class Identify:
                     self.right_hands.append(right_hand_points)
 
     def find_target_person(self):
-        results = self.ml_identify.infer(image=self.image)
+        target_scopes = self.ml_identify.infer(image=self.image)
-        for result in results:
+        for result in target_scopes:
+            # cv2.rectangle(self.image, (result[0], result[1]), (result[2], result[3]), (0, 0, 255), 2)
             one_person_result = [result, [], []]
             for left_hand in self.left_hands:
                 if util.Util.is_in_rectangle(left_hand[0], result):
@@ -260,7 +274,10 @@ class Identify:
             self.person_results.append(one_person_result)
         if self.person_results:
             person_result = self.find_catch_pose()
-            return person_result[0]
+            if self.catch_person_flag:
+                return person_result[0]
+            else:
+                return 0, 0, 0, 0
         else:
             return 0, 0, 0, 0
 
@@ -287,13 +304,13 @@ class Identify:
 
     def deal_with_image(self):
         self.image = cv2.flip(self.image, 1)
-        self.identify_results = self.hands.process(self.image)
+        self.rgb_image = cv2.cvtColor(self.image, cv2.COLOR_BGR2RGB)
+        self.identify_results = self.hands.process(self.rgb_image)
 
     def get_hand_points(self):
         if not self.identify_results.multi_handedness:
             return 0
         for i in range(len(self.identify_results.multi_handedness)):
-            # print(self.identify_results.multi_handedness[i].classification[0].label)
             if self.identify_results.multi_handedness[i].classification[0].label == "Left":
                 for hand_landmarks in self.identify_results.multi_hand_landmarks[i].landmark:
                     if self.image_height == 0:
@@ -303,7 +320,6 @@ class Identify:
                     # self.mp_drawing.draw_landmarks(
                     #     self.image, self.identify_results.multi_hand_landmarks[i], self.mp_hands.HAND_CONNECTIONS)
                     if self.identify_results.multi_handedness[i].classification[0].score > 0.5:
-                        print("identify is left")
                         self.left_hand_flag = True
                         self.is_identify = True
 
@@ -317,7 +333,6 @@ class Identify:
                     # self.mp_drawing.draw_landmarks(
                     #     self.image, self.identify_results.multi_hand_landmarks[i], self.mp_hands.HAND_CONNECTIONS)
                     if self.identify_results.multi_handedness[i].classification[0].score > 0.5:
-                        print("identify is right")
                         self.right_hand_flag = True
                         self.is_identify = True
 
@@ -325,11 +340,8 @@ class Identify:
         if self.is_identify:
             if self.left_hand_flag:
                 self.left_angle_list = hand_angle(hand_=self.left_hand_points)
-                # print("(((((((((((((((((((")
                 for i in range(5):
                     self.is_left_finger_straight[i] = util.Util.is_straight(self.left_angle_list[i])
-                    # print(self.is_left_finger_straight[i])
-                # print(")))))))))))))))))))")
             if self.right_hand_flag:
                 self.right_angle_list = hand_angle(hand_=self.right_hand_points)
                 for i in range(5):
@@ -339,9 +351,9 @@ class Identify:
         if self.left_hand_flag:
             self.position_x = self.left_hand_points[8][0]
             self.position_y = self.left_hand_points[8][1]
-        elif self.right_hand_flag:
+        # elif self.right_hand_flag:
-            self.position_x = self.right_hand_points[8][0]
+        #     self.position_x = self.right_hand_points[8][0]
-            self.position_y = self.right_hand_points[8][1]
+        #     self.position_y = self.right_hand_points[8][1]
         return is_finger_straight[1] and not is_finger_straight[2] and \
                not is_finger_straight[3] and not is_finger_straight[4]
 
@@ -394,22 +406,25 @@ class Identify:
         if self.step_up == 0:
             self.lase_time = time.time()
         if self.step_up == 0 and self.judge_step_three():
-            self.step_up = 4
-        if self.step_up == 4 and self.judge_step_one(False):
             self.step_up = 1
-        elif self.step_up == 1 and self.judge_step_two(False):
+        if self.step_up == 1 and self.judge_step_one(False):
+            self.step_up = 2
+        elif self.step_up == 2 and self.judge_step_two(False):
             self.step_up = 3
-        elif self.step_up == 2 and judge_zero(self.is_right_finger_straight):
+        # elif self.step_up == 2 and judge_zero(self.is_right_finger_straight):
-            self.step_up = 3
+        #     self.step_up = 3
-        elif self.step_up == 3:
+        # elif self.step_up == 3:
+        #     self.step_up = 0
+        now_time = time.time()
+        if now_time - self.lase_time > 3:
+            self.lase_time = now_time
             self.step_up = 0
-            now_time = time.time()
+            return False
-            if now_time - self.lase_time < 3:
+        else:
+            if self.step_up == 3:
                 self.lase_time = now_time
+                self.step_up = 0
                 return True
-            else:
-                self.lase_time = now_time
-                return False
         return False
 
     def judge_page_down(self):
@@ -418,25 +433,24 @@ class Identify:
         if self.step_down == 0:
             self.lase_time = time.time()
         if self.step_down == 0 and self.judge_step_three():
-            self.step_down = 4
-            print("step = 1")
-        if self.step_down == 4 and self.judge_step_one(True):
             self.step_down = 1
-            print("step = 2")
+            # print("step = 1")
-        elif self.step_down == 1 and self.judge_step_two(True):
+        if self.step_down == 1 and self.judge_step_one(True):
+            self.step_down = 2
+            # print("step = 2")
+        elif self.step_down == 2 and self.judge_step_two(True):
             self.step_down = 3
-            print("step = 3")
+            # print("step = 3")
-        elif self.step_down == 2 and judge_zero(self.is_left_finger_straight):
+        now_time = time.time()
-            self.step_down = 3
+        if now_time - self.lase_time > 3:
-        elif self.step_down == 3:
+            self.lase_time = now_time
             self.step_down = 0
-            now_time = time.time()
+            return False
-            if now_time - self.lase_time < 3:
+        else:
+            if self.step_down == 3:
                 self.lase_time = now_time
+                self.step_down = 0
                 return True
-            else:
-                self.lase_time = now_time
-                return False
         return False
 
     def judge_end(self):
@@ -444,36 +458,47 @@ class Identify:
             return True
         return False
 
+    def judge_system_over(self):
+        if not self.judge_one(self.is_left_finger_straight):
+            return False
+        if not self.judge_one(self.is_right_finger_straight):
+            return False
+        standard_distance = util.Util.get_distance(self.left_hand_points[7], self.right_hand_points[8])
+        # print(util.Util.get_distance(self.left_hand_points[8], self.right_hand_points[8]) / standard_distance)
+        if util.Util.get_distance(self.left_hand_points[8], self.right_hand_points[8]) / standard_distance < 1:
+            return True
+        return False
+
     def judge_control(self):
-        # print("left_flag = " + str(self.left_hand_flag))
-        # print("right_flag = " + str(self.right_hand_flag))
-        # print("1111111111111111111111")
-        # for i in range(5):
-        #     print(self.is_left_finger_straight[i])
-        # print("2222222222222222222222")
         if self.is_identify:
+            # print("len = " + str(len(self.identify_results.multi_handedness)))
+            # print("left is " + str(self.left_hand_flag))
+            # print("right is " + str(self.right_hand_flag))
             if judge_two(self.is_left_finger_straight):
-                print("1")
+                # print("1")
                 return 1
             elif self.judge_page_up():
-                print("2")
+                # print("2")
                 return 2
             elif self.judge_page_down():
-                print("3")
+                # print("3")
-                # print("down!down!down!down!down!down!down!down!down!down!down!down!down!down!down!down!down!down!")
                 return 3
-            elif self.judge_one(self.is_left_finger_straight):
+            elif self.judge_one(self.is_left_finger_straight) and not self.right_hand_flag:
-                print("4")
+                # print("4")
                 return 4
-            elif self.left_hand_flag and judge_zero(self.is_left_finger_straight):
+            elif self.judge_one(self.is_left_finger_straight) and judge_zero(self.is_right_finger_straight):
-                print("5")
+                # print("5")
-                # for i in range(5):
-                #     print(self.is_left_finger_straight[i])
                 return 5
-            else:
+            elif self.left_hand_flag and judge_zero(self.is_left_finger_straight):
-                print("other")
+                # print("6")
-        else:
+                return 6
-            print("no_hand_points")
+            elif self.judge_system_over():
+                # print("7")
+                return 7
+            # else:
+            #     # print("other")
+        # else:
+        #     print("no_hand_points")
         return 0
 
 #

main.py

@@ -1,27 +1,15 @@
 import multiprocessing
+import sys
 import tkinter
 from identify import *
 from tkinter.filedialog import *
+from tkinter import messagebox
 from win32com.client import Dispatch
 import win32con
 import win32api
 import time
 import os
 
-control_flag = 0
-x = 0
-y = 0
-
-
-def show():
-    win32api.keybd_event(116, 0, 0, 0)  # 代表按下f键
-    time.sleep(0.02)
-    win32api.keybd_event(116, 0, win32con.KEYEVENTF_KEYUP, 0)  # 释放f键
-    time.sleep(2)
-    win32api.keybd_event(34, 0, 0, 0)  # 代表按下f键
-    time.sleep(0.02)
-    win32api.keybd_event(34, 0, win32con.KEYEVENTF_KEYUP, 0)  # 释放f键
-
 
 def control_page_up():
     win32api.keybd_event(38, 0, 0, 0)  # 代表按下f键
@@ -61,39 +49,29 @@ def control_writing(m, position_x, position_y):
     m.move(position_x, position_y)
 
 
-def control_draw():
+def control_thread(control_number, array, over_flag):
-    global x, y
-
-    # m = pymouse.PyMouse()
-    # m.move(x, y)
-    # win32api.keybd_event(40, 0, 0, 0)  # 代表按下f键
-    # time.sleep(0.02)
-    # win32api.keybd_event(40, 0, win32con.KEYEVENTF_KEYUP, 0)  # 释放f键
-
-    # class Control:
-    #     def __init__(self, control_flag):
-    #         self.control_flag = control_flag
-    #
-
-
-def control_thread(value, array):
     last_time = 0.0
     step = 0
     mouse = PyMouse()
-
+    test_time1 = time.time()
+    count = 0
     while 1:
+        count += 1
         # print("step = " + str(step))
         # print("control_thread")
         now_time = time.time()
-        # time.sleep(0.1)
+        control_flag = control_number.value
-        # print(end - start)
+        # os._exit(0)
-        # print("***********array************")
+        if control_flag == 7:
-        # print(array)
+            # test_time2 = time.time()
-        # print("***********array************")
+            # fps = count / (test_time2 - test_time1)
-        control_flag = value.value
+            # print("count2 = " + str(count))
+            # print("time = " + str(test_time2 - test_time1))
+            # print("fps_control = " + str(fps))
+            over_flag.value = 1
+            sys.exit(0)
         if step == 2:
             if control_flag == 4:
-                # print("inininininininininininininininininininininininin")
                 position_x = int(mouse.screen_size()[0] / array[2] * array[0])
                 position_y = int(mouse.screen_size()[1] / array[3] * array[1])
                 control_writing(mouse, position_x, position_y)
@@ -101,22 +79,16 @@ def control_thread(value, array):
                 step = 1
                 control_ppt_end()
                 last_time = now_time
-                # print("step to 11111111111111111")
                 continue
         if now_time - last_time < 1:
             continue
         last_time = now_time
-        # if control_flag in range(1, 2, 3, 4, 5):
+        # print("control_flag2 = " + str(control_flag))
-        #     time.sleep(2)
+        # # print("main.step = " + str(step))
-        # elif control_flag == 6:
+        if step == 0:
-        #     time.sleep(0.05)
+            if control_flag == 1:
-        # start = time.time()
+                control_ppt_begin()
-        print("control_flag = " + str(control_flag))
+                step = 1
-        # print("main.step = " + str(step))
-        if step == 0 and control_flag == 1:
-            control_ppt_begin()
-            # print("control_flag == 1")
-            step = 1
         elif step == 1:
             if control_flag == 2:
                 control_page_up()
@@ -127,38 +99,37 @@ def control_thread(value, array):
             elif control_flag == 4:
                 control_open_pencil()
                 step = 2
-            elif control_flag == 5:
+            elif control_flag == 6:
                 control_ppt_end()
-                # print("end!end!end!end!end!end!end!end!end!end!end!")
-                # print("control_flag == 4")
                 step = 0
+
         # if control_flag == 5:
         #     control_open_pencil()
         # if control_flag == 6:
         #     control_draw()
 
 
-def identify_thread(value, array):
+def identify_thread(control_number, array, over_flag):
-    identify = Identify(value, array)
+    identify = Identify(control_number, array)
-    identify.begin()
+    identify.begin(over_flag)
-
-
-def show_thread(value, array):
-    return 0
 
 
 def open_file():
     file_path = askopenfilename(title=u'选择文件')
     # file_path = askopenfilename(title=u'选择文件', initialdir=(os.path.expanduser('H:/')))
-    open_ppt(file_path)
+    if not open_ppt(file_path):
-    value = multiprocessing.Value('i', 0)
+        return False
+    control_number = multiprocessing.Value('i', 0)
     array = multiprocessing.Array('i', 4)
+    over_flag = multiprocessing.Value('i', 0)
     # array2 = multiprocessing.Array()
-    p1 = multiprocessing.Process(target=identify_thread, args=(value, array))
+    p1 = multiprocessing.Process(target=identify_thread, args=(control_number, array, over_flag))
-    p2 = multiprocessing.Process(target=control_thread, args=(value, array))
+    p2 = multiprocessing.Process(target=control_thread, args=(control_number, array, over_flag))
     # p3 = multiprocessing.Process(target=show_thread, args=(value, array))
     p1.start()
+    # p1.terminate()
     p2.start()
+    return True
     # p3.start()
     # identify_t = threading.Thread(target=identify_thread)
     # print("control_flag1 = " + str(control_flag))
@@ -170,11 +141,32 @@ def open_file():
     # control_t.start()
 
 
+def judge_ppt(file_path):
+    string = file_path
+    string = string.split('.')
+    if len(string) <= 0:
+        return False
+    string = string[len(string) - 1]
+    if string.find('ppt') != -1:
+        return True
+    return False
+
+
 def open_ppt(file_path):
+    if not len(file_path):
+        messagebox.showinfo("选择失败", "您尚未选择ppt文件，请重新选择")
+        return False
+    if not judge_ppt(file_path):
+        messagebox.showinfo("选择失败", "该文件非ppt文件，请重新选择")
+        return False
     ppt = Dispatch('PowerPoint.Application')
     ppt.Visible = 1  # 后台运行
     ppt.DisplayAlerts = 0  # 不显示，不警告
     ppt.Presentations.Open(file_path)
+    return True
+
+
+# def make_button(window):
 
 
 if __name__ == '__main__':
@@ -183,4 +175,6 @@ if __name__ == '__main__':
     window.geometry("200x100")
     bt1 = tkinter.Button(window, text='打开文件', width=15, height=15, command=open_file)
     bt1.pack()
+    # show_text = tkinter.Text(window, height=2)
+    # show_text.pack()
     window.mainloop()

test.py

@@ -46,9 +46,6 @@ def video_speed(video_root, out_root, fps=None, scale=1):
     # yolov5 = detect.YOLOV5()
     # blank_image = np.zeros((1000, 1000, 3), np.uint8)
     # blank_image.fill(255)
-    fourcc = cv2.VideoWriter_fourcc(*'XVID')
-    videoWriter = cv2.VideoWriter('output.avi', fourcc, 60.0, (1280, 720))
-    count = 100
     while True:
         ret, frame = cap.read()
         # blank_image = frame.copy()
@@ -58,9 +55,9 @@ def video_speed(video_root, out_root, fps=None, scale=1):
         # print("")
 
         frame = cv2.flip(frame, 1)
-        frame = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)
+        frame_rgb = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)
         # frame = cv2.cvtColor(frame, cv2.COLOR_RGB2BGR)
-        results = hands.process(frame)
+        results = hands.process(frame_rgb)
         # results = poses.process(image)
         # print("*********************************")
         # if results.pose_landmarks:
@@ -74,19 +71,19 @@ def video_speed(video_root, out_root, fps=None, scale=1):
         # if results.
         # print(results.multi_hand_landmarks)
         if results.multi_handedness:
-            if results.multi_handedness[0].classification[0].label == "Left":
+            # if results.multi_handedness[0].classification[0].label == "Left":
-                for hand_landmarks in results.multi_hand_landmarks:
+            for hand_landmarks in results.multi_hand_landmarks:
-                    # 关键点可视化
+                # 关键点可视化
-                    mp_drawing.draw_landmarks(
+                mp_drawing.draw_landmarks(
-                        frame, hand_landmarks, mp_hands.HAND_CONNECTIONS)
+                    frame, hand_landmarks, mp_hands.HAND_CONNECTIONS)
-            # if results.multi_handedness[0].classification[0].label == "Right":
+        # if results.multi_handedness[0].classification[0].label == "Right":
-            #     print(results.multi_handedness[0].classification[0])
+        #     print(results.multi_handedness[0].classification[0])
         cv2.imshow('MediaPipe Hands', frame)
         # cv2.imshow('black_image', blank_image)
         # videoWriter.write(blank_image)
         # count = count - 1
-        # cv2.imwrite("picture3.jpg", frame)
+        cv2.imwrite("picture3.jpg", frame)
-        if cv2.waitKey(1) & 0xFF == 27:
+        if cv2.waitKey(3000) & 0xFF == 27:
             break
     cap.release()
     # videoWriter.release()