graduation-project/identify.py

import time
from pymouse import PyMouse
import cv2
import mediapipe as mp
import math
import util


# page_up_count = 0
# page_down_count = 0
# last_wrist_point = (0, 0)


class Identify:
    def __init__(self, value, array):
        self.value = value
        self.array = array
        self.left_hand_flag = False
        self.right_hand_flag = False
        self.result = 0
        self.position_x = 0
        self.position_y = 0
        self.image = []
        self.image_height = 0
        self.image_width = 0
        self.rgb_image = []
        self.identify_results = []
        self.left_hand_points = []
        self.right_hand_points = []
        self.angle_list = []
        self.is_finger_straight = [False, False, False, False, False]
        self.is_identify = False
        self.last_control_flag = 0
        self.page_up_count = 0
        self.page_down_count = 0
        self.step_up = 0
        self.step_down = 0
        self.last_wrist_point = (0, 0)
        self.now_time = 0
        self.lase_time = 0
        self.flag = 0
        self.mp_drawing = mp.solutions.drawing_utils
        self.mp_hands = mp.solutions.hands
        self.hands = self.mp_hands.Hands(
            static_image_mode=False,
            max_num_hands=2,
            min_detection_confidence=0.75,
            min_tracking_confidence=0.75)
        self.mp_face = mp.solutions.face_detection
        self.face_detection = self.mp_face.FaceDetection(min_detection_confidence=0.5)

    def begin(self):
        capture = cv2.VideoCapture(0)
        last_time = 0
        while 1:
            ret, self.image = capture.read()
            self.deal_with_image()

            # fps = 1 / (self.now_time - self.lase_time)
            # self.lase_time = self.now_time
            # print("fps = " + str(fps))
            self.is_identify = False
            self.left_hand_flag = False
            self.right_hand_flag = False
            for i in range(5):
                self.is_finger_straight[i] = False
            self.left_hand_points.clear()
            self.right_hand_points.clear()
            self.get_hand_points()
            self.judge_finger_straight()
            flag = self.judge_control()
            if flag:
                self.flag = flag
            now_time = time.time()
            self.array[0] = self.position_x
            self.array[1] = self.position_y
            self.array[2] = self.image_width
            self.array[3] = self.image_height
            if now_time - last_time < 1:
                continue
            last_time = now_time
            # print("**************")
            # for i in range(5):
            #     print(self.is_finger_straight[i])
            # x = position[0]
            # y = position[1]
            # cv2.namedWindow("Video")
            # cv2.imshow("Video", self.image)
            # if cv2.waitKey(1) & 0xFF == ord('q'):
            #     break
            # control_flag = flag
            self.value.value = self.flag
            # print("self.v.value = " + str(self.flag))
            print("final_control_flag = " + str(self.flag))
            self.flag = 0
        capture.release()
        cv2.destroyAllWindows()

    # def face_detect(self):
    #     results = self.face_detection.process(self.rgb_image)

    def deal_with_image(self):
        self.image = cv2.flip(self.image, 1)
        self.rgb_image = cv2.cvtColor(self.image, cv2.COLOR_BGR2RGB)
        self.identify_results = self.hands.process(self.rgb_image)

    def get_hand_points(self):
        if self.identify_results.multi_handedness:
            for i in range(len(self.identify_results.multi_handedness)):
                # print(self.identify_results.multi_handedness[i].classification[0].label)
                if self.identify_results.multi_handedness[i].classification[0].label == "Left":
                    for hand_landmarks in self.identify_results.multi_hand_landmarks[i].landmark:
                        if self.image_height == 0:
                            self.image_height, self.image_width, c = self.image.shape
                        cx, cy = int(hand_landmarks.x * self.image_width), int(hand_landmarks.y * self.image_height)
                        self.left_hand_points.append((cx, cy))
                        # self.mp_drawing.draw_landmarks(
                        #     self.image, self.identify_results.multi_hand_landmarks[i], self.mp_hands.HAND_CONNECTIONS)
                        if self.identify_results.multi_handedness[i].classification[0].score > 0.5:
                            self.left_hand_flag = True
                            self.is_identify = True

                else:
                    for hand_landmarks in self.identify_results.multi_hand_landmarks[i].landmark:
                        if self.image_height == 0:
                            self.image_height, self.image_width, c = self.image.shape
                        cx, cy = int(hand_landmarks.x * self.image_width), int(hand_landmarks.y * self.image_height)
                        self.right_hand_points.append((cx, cy))
                        self.is_identify = True
                        # self.mp_drawing.draw_landmarks(
                        #     self.image, self.identify_results.multi_hand_landmarks[i], self.mp_hands.HAND_CONNECTIONS)
                        if self.identify_results.multi_handedness[i].classification[0].score > 0.5:
                            self.right_hand_flag = True
                            self.is_identify = True

    def hand_angle(self):
        '''
            获取对应手相关向量的二维角度,根据角度确定手势
        '''
        angle_list = []
        if self.left_hand_flag:
            hand_ = self.left_hand_points
        else:
            hand_ = self.right_hand_points
        # ---------------------------- thumb 大拇指角度
        angle_ = util.Util.vector_2d_angle(
            ((int(hand_[2][0]) - int(hand_[3][0])), (int(hand_[2][1]) - int(hand_[3][1]))),
            ((int(hand_[3][0]) - int(hand_[4][0])), (int(hand_[3][1]) - int(hand_[4][1])))
        )
        angle_list.append(angle_)
        # ---------------------------- index 食指角度
        angle_ = util.Util.vector_2d_angle(
            ((int(hand_[5][0]) - int(hand_[6][0])), (int(hand_[5][1]) - int(hand_[6][1]))),
            ((int(hand_[7][0]) - int(hand_[8][0])), (int(hand_[7][1]) - int(hand_[8][1])))
        )
        angle_list.append(angle_)
        # ---------------------------- middle 中指角度
        angle_ = util.Util.vector_2d_angle(
            ((int(hand_[9][0]) - int(hand_[10][0])), (int(hand_[9][1]) - int(hand_[10][1]))),
            ((int(hand_[11][0]) - int(hand_[12][0])), (int(hand_[11][1]) - int(hand_[12][1])))
        )
        angle_list.append(angle_)
        # ---------------------------- ring 无名指角度
        angle_ = util.Util.vector_2d_angle(
            ((int(hand_[13][0]) - int(hand_[14][0])), (int(hand_[13][1]) - int(hand_[14][1]))),
            ((int(hand_[15][0]) - int(hand_[16][0])), (int(hand_[15][1]) - int(hand_[16][1])))
        )
        angle_list.append(angle_)
        # ---------------------------- pink 小拇指角度
        angle_ = util.Util.vector_2d_angle(
            ((int(hand_[17][0]) - int(hand_[18][0])), (int(hand_[17][1]) - int(hand_[18][1]))),
            ((int(hand_[19][0]) - int(hand_[20][0])), (int(hand_[19][1]) - int(hand_[20][1])))
        )
        angle_list.append(angle_)
        self.angle_list = angle_list

    def judge_finger_straight(self):
        if self.is_identify:
            self.hand_angle()
            for i in range(5):
                self.is_finger_straight[i] = util.Util.is_straight(self.angle_list[i])

    def judge_zero(self):
        return not self.is_finger_straight[1] and not self.is_finger_straight[2] and \
               not self.is_finger_straight[3] and not self.is_finger_straight[4]

    def judge_one(self):
        if self.left_hand_flag:
            self.position_x = self.left_hand_points[8][0]
            self.position_y = self.left_hand_points[8][1]
        elif self.right_hand_flag:
            self.position_x = self.right_hand_points[8][0]
            self.position_y = self.right_hand_points[8][1]
        return self.is_finger_straight[1] and not self.is_finger_straight[2] and \
               not self.is_finger_straight[3] and not self.is_finger_straight[4]

    def judge_two(self):
        return self.is_finger_straight[1] and self.is_finger_straight[2] and \
               not self.is_finger_straight[3] and not self.is_finger_straight[4]

    def judge_three(self):
        return self.is_finger_straight[1] and self.is_finger_straight[2] and \
               self.is_finger_straight[3] and not self.is_finger_straight[4]

    def judge_four(self):
        return self.is_finger_straight[1] and self.is_finger_straight[2] and \
               self.is_finger_straight[3] and self.is_finger_straight[4]

    def judge_five(self):
        return self.is_finger_straight[1] and self.is_finger_straight[2] and \
               self.is_finger_straight[3] and self.is_finger_straight[4]

    def judge_step_one(self, is_left):
        if is_left:
            if self.judge_five() and self.left_hand_points[8][0] < self.left_hand_points[0][0] and \
                    self.left_hand_points[12][0] < self.left_hand_points[0][0] and \
                    self.left_hand_points[16][0] < self.left_hand_points[0][0] and self.left_hand_points[20][0] < \
                    self.left_hand_points[0][0]:
                return True
        else:
            if self.judge_five() and self.right_hand_points[8][0] > self.right_hand_points[0][0] and \
                    self.right_hand_points[12][0] > self.right_hand_points[0][0] and \
                    self.right_hand_points[16][0] > self.right_hand_points[0][0] and self.right_hand_points[20][0] > \
                    self.right_hand_points[0][0]:
                return True
        return False

    def judge_step_two(self, is_left):
        if is_left:
            if self.judge_five() and self.left_hand_points[8][0] > self.left_hand_points[0][0] and \
                    self.left_hand_points[12][0] > self.left_hand_points[0][0] and \
                    self.left_hand_points[16][0] > self.left_hand_points[0][0] and self.left_hand_points[20][0] > \
                    self.left_hand_points[0][0]:
                return True
        else:
            if self.judge_five() and self.right_hand_points[8][0] < self.right_hand_points[0][0] and \
                    self.right_hand_points[12][0] < self.right_hand_points[0][0] and \
                    self.right_hand_points[16][0] < self.right_hand_points[0][0] and self.right_hand_points[20][0] < \
                    self.right_hand_points[0][0]:
                return True
        return False

    def judge_step_three(self):
        if self.left_hand_flag:
            if self.left_hand_points[20][1] < self.left_hand_points[0][1]:
                return True
        else:
            if self.right_hand_points[20][1] < self.right_hand_points[0][1]:
                return True
        return False

    def judge_page_up(self):
        if not self.right_hand_flag:
            return False
        if self.step_up == 0:
            self.lase_time = time.time()
        if self.step_up == 0 and self.judge_step_three():
            self.step_up = 4
        if self.step_up == 4 and self.judge_step_one(False):
            self.step_up = 1
        elif self.step_up == 1 and self.judge_step_two(False):
            self.step_up = 3
        elif self.step_up == 2 and self.judge_zero():
            self.step_up = 3
        elif self.step_up == 3:
            self.step_up = 0
            now_time = time.time()
            if now_time - self.lase_time < 3:
                self.lase_time = now_time
                return True
            else:
                self.lase_time = now_time
                return False
        return False

    def judge_page_down(self):
        if not self.left_hand_flag:
            return False
        if self.step_down == 0:
            self.lase_time = time.time()
        if self.step_down == 0 and self.judge_step_three():
            self.step_down = 4
            print("step = 1")
        if self.step_down == 4 and self.judge_step_one(True):
            self.step_down = 1
            print("step = 2")
        elif self.step_down == 1 and self.judge_step_two(True):
            self.step_down = 3
            print("step = 3")
        elif self.step_down == 2 and self.judge_zero():
            self.step_down = 3
        elif self.step_down == 3:
            self.step_down = 0
            now_time = time.time()
            if now_time - self.lase_time < 3:
                self.lase_time = now_time
                return True
            else:
                self.lase_time = now_time
                return False
        return False

    def judge_end(self):
        if self.left_hand_flag and self.right_hand_flag and self.judge_zero():
            return True
        return False

    def judge_control(self):
        if self.is_identify:
            if self.judge_two():
                return 1
            elif self.judge_page_up():
                return 2
            elif self.judge_page_down():
                # print("down!down!down!down!down!down!down!down!down!down!down!down!down!down!down!down!down!down!")
                return 3
            elif self.judge_one():
                return 4
            elif self.judge_zero():
                return 5
            else:
                print("other")
        else:
            print("no_hand_points")
        return 0

#
# identify = Identify()
# identify.begin()