Gesture Recognition with gloves

2 次查看（过去 30 天）
Priyana Kumar
Priyana Kumar 2021-4-21
Hello!! I have this code that currently detects hand gestures 1-6 and completes a mouse action with it. We now have to get it to work with any color latex gloves and this is seeming to be a problem. I'm guessing it has to do with "ReturnedColorSpace" but I'm not completely sure. Currently it works well with any skin tone against a dark background with good lighting.
%% Import Java Robot
import java.awt.Robot;
import java.awt.event.*
mouse = Robot;
numFrame = 3000;
%% Create video input object.
cam = imaqhwinfo; % Get Camera information
cameraName = char(cam.InstalledAdaptors(end));
cameraInfo = imaqhwinfo(cameraName);
%cameraId = cameraInfo.DeviceInfo.DeviceID(end);
cameraFormat = char(cameraInfo.DeviceInfo(1).SupportedFormats(end)); %change 1 to 2
vidDevice = imaq.VideoDevice(cameraName, 1, cameraFormat, ... % Input Video from current adapter %change 1 to 2
'ReturnedColorSpace', 'RGB');
%vidDevice = imaq.VideoDevice('macvideo',3);
vidInfo = imaqhwinfo(vidDevice); % Acquire video information
screenSize = get(0,'ScreenSize'); % Acquire system screensize
hblob = vision.BlobAnalysis('AreaOutputPort', false, ... % Setup blob analysis handling
'CentroidOutputPort', true, ...
'BoundingBoxOutputPort', true, ...
'MinimumBlobArea', 3000, ...
'MaximumCount', 6);
palmblob = vision.BlobAnalysis('AreaOutputPort', false, ...
'CentroidOutputPort', true, ...
'BoundingBoxOutputPort', true, ...
'MinimumBlobArea', 3000, ...
'MaximumCount', 1);
hshape = vision.ShapeInserter('BorderColor','Custom', ... % Setup colored box handling
'CustomBorderColor', [1 0 0],...
'Fill', true, ...
'FillColor','Custom', 'CustomFillColor',[1 0 0], ...
'Opacity', 0.4);
palmshape = vision.ShapeInserter('BorderColor','Custom', ... % Setup colored box handling
'CustomBorderColor', [1 1 0],...
'Fill', true, ...
'FillColor','Custom', 'CustomFillColor',[1 1 0], ...
'Opacity', 0.4);
hVideoIn = vision.VideoPlayer('Name', 'Gesture Recognition', ... % Setup output video stream handling
'Position', [100 100 vidInfo.MaxWidth+20 vidInfo.MaxHeight+30]); %%%%%%%%%%%%%%%%%%%
%initializing variables
nFrame = 0;
lCount = 0; rCount = 0; dCount = 0;
sureEvent = 5;
iPos = vidInfo.MaxWidth/2;
while (nFrame < numFrame)
img1 = step(vidDevice);
%img1 = flip(img1,2);
img2 = rgb2gray(img1);
lvl = graythresh(img2);
img3 = imbinarize(img2, lvl);
img4 = bwareaopen(img3, 500);
img5 = medfilt2(img4, [5 5]);
img6 = imfill(img5,'holes');
img7 = imtophat(img6,strel('disk',35));
img8 = imopen(img7, strel('disk',35));
[cc_1,bb_1] = step(palmblob,img8); %palm
[cc_2,bb_2] = step(hblob,img7); %fingers
[B,L] = bwboundaries(img7, 'noholes');
nof = num2str(size(bb_2,1));
nof1 = size(bb_2,1);
CHAR_STR = ' not identified';
if nof1 == 1
CHAR_STR = ' MOUSE MOVE';
elseif nof1 == 2
CHAR_STR = ' LEFT CLICK';
elseif nof1 == 3
CHAR_STR = ' RIGHT CLICK';
elseif nof1 == 4
CHAR_STR = ' DOUBLE CLICK';
elseif nof1 == 5
CHAR_STR = ' SCROLL';
elseif nof1 == 6
CHAR_STR = ' QUIT VERIFICATION';
elseif nof1 == 7
CHAR_STR = ' QUIT';
end
if(~isempty(bb_2))
img1 = step(hshape,img1,bb_2);
img1 = insertText(img1,[15,5],strcat('count: ', nof,' mouse action: ',CHAR_STR),...
'FontSize', 30, 'BoxColor', [0 0 1], 'TextColor', 'white');
end
if (~isempty(bb_1))
img1 = step(palmshape,img1,bb_1);
end
imshow(img1);
movegui('northeast')
WinOnTop();
% Begin Mouse Movements
if nof1==1 %move
mouse.mouseMove(1.5*(cc_2(:,1))*screenSize(3)/vidInfo.MaxWidth, 1.5*(cc_2(:,2))*screenSize(4)/vidInfo.MaxHeight);
elseif nof1==2 %leftclick
mouse.mousePress(16);
pause(0.1);
mouse.mouseRelease(16);
pause(0.2);
mouse.mousePress(16);
pause(0.1);
mouse.mouseRelease(16);
elseif nof1==3 %rightclick brings up menu
mouse.mousePress(16);
pause(0.1);
mouse.mouseRelease(16);
pause(0.2)
mouse.mousePress(4);
pause(0.1);
mouse.mouseRelease(4);
elseif nof1==4 %doubleclick
mouse.mousePress(16);
pause(0.1);
mouse.mouseRelease(16);
pause(0.9);
mouse.mousePress(16);
pause(0.1);
mouse.mouseRelease(16);
pause(0.09);
mouse.mousePress(16);
pause(0.1);
mouse.mouseRelease(16);
lCount = 0; rCount = 0; dCount = 0;
elseif nof1==5 %scroll
mouse.mousePress(16);
pause(0.1);
mouse.mouseRelease(16);
mouse.mouseWheel(10);
mouse.mouseWheel(-10);
elseif nof1==6 %endcode, see function at the bottom
GUI_1()
pause(5);
end
end
nFrame = nFrame + 1;
%% Clearing Memory
release(hVideoIn); % Release all memory and buffer used
release(vidDevice);
clc;
function wasOnTop = WinOnTop( figureHandle, isOnTop )
%WINONTOP allows to trigger figure's "Always On Top" state
%
%% INPUT ARGUMENTS:
%
% # figureHandle - Matlab's figure handle, scalar
% # isOnTop - logical scalar or empty array
%
%
%% USAGE:
%
% * WinOnTop( hfigure, true ); - switch on "always on top"
% * WinOnTop( hfigure, false ); - switch off "always on top"
% * WinOnTop( hfigure ); - equal to WinOnTop( hfigure,true);
% * WinOnTop(); - equal to WinOnTop( gcf, true);
% * WasOnTop = WinOnTop(...); - returns boolean value "if figure WAS on top"
% * isOnTop = WinOnTop(hfigure,[]) - get "if figure is on top" property
%
% For Matlab windows, created via `hf=uifigure()` use `uifigureOnTop()`, see:
% https://www.mathworks.com/matlabcentral/fileexchange/73134-uifigureontop
%
%% LIMITATIONS:
%
% * java enabled
% * figure must be visible
% * figure's "WindowStyle" should be "normal"
% * figureHandle should not be casted to double, if using HG2 (R2014b+)
%
% Written by Igor
% i3v@mail.ru
%% Parse Inputs
if ~exist('figureHandle','var'); figureHandle = gcf; end
assert(...
isscalar( figureHandle ) &&...
ishandle( figureHandle ) &&...
strcmp(get(figureHandle,'Type'),'figure'),...
...
'WinOnTop:Bad_figureHandle_input',...
'%s','Provided figureHandle input is not a figure handle'...
);
assert(...
strcmp('on',get(figureHandle,'Visible')),...
'WinOnTop:FigInisible',...
'%s','Figure Must be Visible'...
);
assert(...
strcmp('normal',get(figureHandle,'WindowStyle')),...
'WinOnTop:FigWrongWindowStyle',...
'%s','WindowStyle Must be Normal'...
);
if ~exist('isOnTop','var'); isOnTop=true; end
assert(...
islogical( isOnTop ) && ...
isscalar( isOnTop ) || ...
isempty( isOnTop ), ...
...
'WinOnTop:Bad_isOnTop_input',...
'%s','Provided isOnTop input is neither boolean, nor empty'...
);
%% Pre-checks
error(javachk('swing',mfilename)) % Swing components must be available.
%% Action
% Flush the Event Queue of Graphic Objects and Update the Figure Window.
drawnow expose
warnStruct=warning('off','MATLAB:HandleGraphics:ObsoletedProperty:JavaFrame');
jFrame = get(handle(figureHandle),'JavaFrame');
warning(warnStruct.state,'MATLAB:HandleGraphics:ObsoletedProperty:JavaFrame');
drawnow
if ishg2(figureHandle)
jFrame_fHGxClient = jFrame.fHG2Client;
else
jFrame_fHGxClient = jFrame.fHG1Client;
end
wasOnTop = jFrame_fHGxClient.getWindow.isAlwaysOnTop;
if ~isempty(isOnTop)
jFrame_fHGxClient.getWindow.setAlwaysOnTop(isOnTop);
end
end
function tf = ishg2(figureHandle)
tf = isa(figureHandle,'matlab.ui.Figure');
end
function [] = GUI_1()
S.fh = figure('units','pixels',...
'position',[1000 25 400 200],...
'menubar','none',...
'name','Warning',...
'numbertitle','off',...
'resize','off');
S.pb = uicontrol('style','text',...
'unit','pix',...
'position',[100 60 200 100],...
'fontsize',14,...
'string',{'Are you sure you want to quit?'});
S.pb1 = uicontrol('style','push',...
'unit','pix',...
'position',[10 10 180 40],...
'fontsize',14,...
'string','Yes Quit',...
'callback',{@pb1_call,S});
S.pb2 = uicontrol('style','push',...
'units','pix',...
'position',[200 10 180 40],...
'fontsize',14,...
'string','No, Continue',...
'callback',{@pb2_call,S});
function [] = pb1_call(varargin)
quit
end
function [] = pb2_call(varargin)
close(S.fh)
end
end
Thank you in advance!

请先登录，再回答此问题。

回答（0 个）

类别

在 Help Center 和 File Exchange 中查找有关 Migrate GUIDE Apps 的更多信息

标签

产品


版本

R2020b

Community Treasure Hunt

Find the treasures in MATLAB Central and discover how the community can help you!

Start Hunting!

Translated by