手势识别控制pygame精灵
McKay 人气:5步骤:
- 编写简易pygame精灵游戏(只实现键盘上下左右控制)
- 解决opencv手势识别核心问题
- 上述2部分对接上
pygame部分我们只加载个背景,然后里面放1只乌龟精灵,用键盘的上下左右键来控制,直接给出代码:
乌龟精灵代码(DemoSpirit.py):
import pygame class DemoSpirit(pygame.sprite.Sprite): def __init__(self, target, screen_size, position): pygame.sprite.Sprite.__init__(self) self.target_surface = target self.screen_size = screen_size self.position = position self.image = pygame.image.load("resources\\wugui.png").convert_alpha() self.image = pygame.transform.smoothscale(self.image, (50, 50)) def draw(self): # random_text = font_200.render('***', True, white_color) self.target_surface.blit(self.image, self.position) def move_left(self): if self.position[0]-10 > 0: self.position=(self.position[0]-10, self.position[1]) def move_right(self): if self.position[0]+10 < self.screen_size[0]: self.position=(self.position[0]+10, self.position[1]) def move_up(self): if self.position[1] - 10 > 0: self.position=(self.position[0], self.position[1]-10) def move_down(self): if self.position[1] + 10 < self.screen_size[1]: self.position=(self.position[0], self.position[1]+10)
游戏主循环代码(game-main.py):
import pygame from pygame.locals import * background_image_filename = 'resources/back.jpg' pygame.init() # 2、初始化init() 及设置 screen_list = pygame.display.list_modes() screen_size = screen_list[16] screen = pygame.display.set_mode(screen_size) background = pygame.image.load(background_image_filename).convert() background = pygame.transform.scale(background, screen_size) clock = pygame.time.Clock() pos = (screen_size[0] * 0.6, screen_size[1] * 0.3) from cvgame.DemoSpirit import DemoSpirit s1 = DemoSpirit(screen, screen_size, pos) # 开始游戏循环 while True: for event in pygame.event.get(): if event.type == KEYDOWN: if event.key == K_UP: s1.move_up() if event.key == K_DOWN: s1.move_down() if event.key == K_LEFT: s1.move_left() if event.key == K_RIGHT: s1.move_right() elif event.key == K_q: exit() screen.blit(background, (0, 0)) s1.draw() pygame.display.update() # 6、update 更新屏幕显示 clock.tick(100)
效果图:
接下来,进入手势识别领域
我们是做了个小技巧来侧面绕过手跟踪问题,如下图(直接指定了左手右手监控区域,这样就不需要动态跟踪手的rect了):
while success: success, img = cap.read() frame = imutils.resize(img, width=700) cv2.rectangle(frame, (50, 0), (264, 250), (170, 170, 0)) #左手区域 cv2.rectangle(frame, (426, 0), (640, 250), (170, 170, 0)) #右手区域 cv2.imshow("Frame_Original", frame) #显示 rightHand = frame[0:250, 50:264] leftHand = frame[0:210, 426:640] left_hand_event = grdetect(leftHand, fgbg_left, verbose=True) #检测左手手势识别事件 right_hand_event = grdetect(rightHand, fgbg_right, verbose=True) #检测右手手势识别事件 print('left hand: ', left_hand_event, 'right hand: ', right_hand_event) #打印出来检测结果
主要看看grdetect和fgbg_left/fgbg_right:
看上图,除了手之外,还有个大背景,首先得把背景去掉,才能识别出前景色-手,fgbg_left/fgbg_right其实就是用来干着活的,分别为左手、右手的背景减噪用的
fgbg_left = cv2.createBackgroundSubtractorMOG2() fgbg_right = cv2.createBackgroundSubtractorMOG2() def train_bg(fgbg, roi): fgbg.apply(roi) def start(): global fgbg_left global fgbg_right trainingBackgroundCount = 200 #200次来训练背景减噪训练 while trainingBackgroundCount>0: success, img = cap.read() frame = imutils.resize(img, width=700) cv2.imshow("Frame_Original", frame) rightHand = frame[0:250, 50:264] leftHand = frame[0:210, 426:640] train_bg(fgbg_left, leftHand) #训练左手区域 train_bg(fgbg_right, rightHand) #训练右手区域 key = cv2.waitKey(1) & 0xFF trainingBackgroundCount -= 1
再来看看核心函数
def grdetect(array, fgbg, verbose=False): event = {'type': 'none'} copy = array.copy() array = _remove_background(array, fgbg) #移除背景,会用到背景减噪(上述提到) thresh = _bodyskin_detetc(array) #高斯+二值化 contours = _get_contours(thresh.copy()) #计算图像的轮廓点,可能会返回多个轮廓 largecont = max(contours, key=lambda contour: cv2.contourArea(contour)) #选择面积最大的轮廓 hull = cv2.convexHull(largecont, returnPoints=False) #根据轮廓点计算凸点 defects = cv2.convexityDefects(largecont, hull) #计算轮廓的凹点(凸缺陷) if defects is not None: # 利用凹陷点坐标, 根据余弦定理计算图像中锐角个数 copy, ndefects = _get_defects_count(copy, largecont, defects, verbose=verbose) # 根据锐角个数判断手势, 会有一定的误差 if ndefects == 0: event['type'] = '0' elif ndefects == 1: event['type'] = '2' elif ndefects == 2: event['type'] = '3' elif ndefects == 3: event['type'] = '4' elif ndefects == 4: event['type'] = '5' return event
剩下的就是上述的支持函数了
def _remove_background(frame, fgbg): fgmask = fgbg.apply(frame, learningRate=0) #learningRate=0代表不更新背景噪声,也就是不学习 kernel = np.ones((3, 3), np.uint8) fgmask = cv2.erode(fgmask, kernel, iterations=1) res = cv2.bitwise_and(frame, frame, mask=fgmask) return res def _bodyskin_detetc(frame): # 肤色检测: YCrCb之Cr分量 + OTSU二值化 ycrcb = cv2.cvtColor(frame, cv2.COLOR_BGR2YCrCb) # 分解为YUV图像,得到CR分量 (_, cr, _) = cv2.split(ycrcb) cr1 = cv2.GaussianBlur(cr, (5, 5), 0) # 高斯滤波 _, skin = cv2.threshold(cr1, 0, 255, cv2.THRESH_BINARY + cv2.THRESH_OTSU) # OTSU图像二值化 return skin # 检测图像中的凸点(手指)个数 def _get_contours(array): # 利用findContours检测图像中的轮廓, 其中返回值contours包含了图像中所有轮廓的坐标点 contours, _ = cv2.findContours(array, cv2.RETR_TREE, cv2.CHAIN_APPROX_NONE) return contours _COLOR_RED = (0, 0, 255) def _get_eucledian_distance(beg, end): # 计算两点之间的坐标 i = str(beg).split(',') j = i[0].split('(') x1 = int(j[1]) k = i[1].split(')') y1 = int(k[0]) i = str(end).split(',') j = i[0].split('(') x2 = int(j[1]) k = i[1].split(')') y2 = int(k[0]) d = math.sqrt((x1 - x2) * (x1 - x2) + (y1 - y2) * (y1 - y2)) return d # 根据图像中凹凸点中的 (开始点, 结束点, 远点)的坐标, 利用余弦定理计算两根手指之间的夹角, 其必为锐角, 根据锐角的个数判别手势. def _get_defects_count(array, contour, defects, verbose=False): ndefects = 0 for i in range(defects.shape[0]): s, e, f, _ = defects[i, 0] beg = tuple(contour[s][0]) end = tuple(contour[e][0]) far = tuple(contour[f][0]) a = _get_eucledian_distance(beg, end) b = _get_eucledian_distance(beg, far) c = _get_eucledian_distance(end, far) angle = math.acos((b ** 2 + c ** 2 - a ** 2) / (2 * b * c)) # * 57 if angle <= math.pi / 2: # 90: ndefects = ndefects + 1 if verbose: cv2.circle(array, far, 3, _COLOR_RED, -1) if verbose: cv2.line(array, beg, end, _COLOR_RED, 1) return array, ndefects
加载全部内容