Upload
peteohanlon
View
449
Download
0
Embed Size (px)
DESCRIPTION
NDC London 2014 Presentation on Developing Natural User Interface Applications with RealSense.
Citation preview
Developing Natural User
Interface Applications With
RealSenseTM DevicesPete O’Hanlon @peteohanlon
Evolution of Perceptual Computing
Allows computer to perceive depth in a similar way we do
Uses 2D and 3D depth camera
Allows us to interact with computers in a completely new way
Different types of camera
Front facing camera
Track hands and fingers
Capture face movements
Detect and differentiate between foreground and background
Rear facing camera
Scan, measure rooms and objects
Snapshot camera
The SDK
Runs on Windows 8.1 as well as in the browser and Android
Available for different languages
C++
C#
Java
JavaScript
Processing (OS framework for developing visual oriented apps – Java like)
Runs on popular game engines
Unity supported
What you need
Hardware 4th generation Intel® Core™ Haswell
Minimum 8GB free hard disk
USB 3 port
RealSenseTM 3D Camera
Software Windows 8.1 64 bit
Visual Studio 2010+
.NET 4+
Unity PRO 4.1.0+
Web Browser
Processing 2.1.2+
JDK 1.7.0_11+
IR CameraFull HD 1080p Image Sensor
IR Laser Projector
SDK Based Application
Language specific interfaces
SDK interfaces
SDK Core I/O Algorithm Algorithm
Natural User Interfaces
Framework features
Emotion detection
Face tracking
Facial recognition
Hand tracking
Gesture recognition
Object tracking
Speech recognition
Speech synthesis
Touchless controller
Segmentation
The potential
Touch free kiosks
Games
Interactive books
Collaboration
Enhanced tutoring
Many more…
DevelopingSetting up
Initialisation
senseManager = PXCMSenseManager.CreateInstance();
private PXCMSenseManager senseManager;private PXCMHandModule handModule;private PXCMHandConfiguration handConfiguration;private PXCMSpeechSynthesis speechSynthesis;private PXCMAudio.AudioInfo audioInfo;private PXCMTouchlessController touchlessController;private PXCMFaceModule faceModule;private PXCMFaceConfiguration faceConfiguration;
if (status >= pxcmStatus.PXCM_STATUS_NO_ERROR){PXCMCapture.Device device =senseManager.captureManager.device;
device.SetDepthConfidenceThreshold(10);device.SetIVCAMLaserPower(16);device.SetIVCAMAccuracy(PXCMCapture.Device.IVCAMAccuracy.IVCAM_ACCURACY_COARSE);device.SetIVCAMMotionRangeTradeOff(21);device.SetIVCAMFilterOption(6);
if (senseManager.StreamFrames(false) >= pxcmStatus.PXCM_STATUS_NO_ERROR)
{// We are streaming data,,,
}}
Emotion detection
senseManager.EnableEmotion();
private pxcmStatus ProcessFrame(…{switch (moduleId){case PXCMEmotion.CUID:ProcessEmotionData(module);break;
}return pxcmStatus.PXCM_STATUS_NO_ERROR;
}
private void ProcessEmotionData(PXCMBase module)
{PXCMEmotion emotion = module.QueryInstance<PXCMEmotion>();
int numberOfFaces = emotion.QueryNumFaces();for (int face = 0; face < numberOfFaces;
face++){PXCMEmotion.EmotionData[] emotionData =
new PXCMEmotion.EmotionData[10];emotion.QueryAllEmotionData(face,
out emotionData);FindStrongestEmotion(emotionData);
}}
private void FindStrongestEmotion(PXCMEmotion.EmotionData[] emotionData)
{PXCMEmotion.EmotionData strongestEmotion = emotionData[0];
int index = 0;for (int emotionIndex = 1; emotionIndex < 7;emotionIndex++){if (emotionData[emotionIndex].evidence >
strongestEmotion.evidence){
index = emotionIndex;strongestEmotion = emotionData[emotionIndex];
}}
}
Segmentation
senseManager.Enable3DSeg();
private pxcmStatus ProcessFrame(int moduleId, PXCMBase module, PXCMCapture.Sample captureSample)
{switch (moduleId){case PXCM3DSeg.CUID:ProcessSegmentData(module);break;
}return pxcmStatus.PXCM_STATUS_NO_ERROR;
}
private void ProcessSegmentData(PXCMBase module)
{PXCM3DSeg segment = module.QueryInstance<PXCM3DSeg>();
if (segment != null){using (PXCMImage image =
segment.AcquireSegmentedImage()){
if (image != null){
ExtractForeground (image);}
}}
}
private void ExtractForeground(PXCMImage image){PXCMImage.ImageData segmentImage;
image.AcquireAccess(PXCMImage.Access.ACCESS_READ_WRITE,PXCMImage.PixelFormat.PIXEL_FORMAT_RGB32,out segmentImage);
try{ProcessImage(image, segmentImage);
}finally{image.ReleaseAccess(segmentImage);
}WriteableBitmap bmp = segmentImage.ToWritableBitmap(0, image.info.width, image.info.height, 96, 96);
}
PXCMImage.ImageInfo info = image.QueryInfo();
for (int y = 0; y < height; y++){unsafe{byte* pixel = (byte*)segmentImage.planes[0] + y * segmentImage.pitches[0];
for (int x = 0; x < width; x++){if (pixel[3] == 0){
for (int ch = 0; ch < 3; ch++){pixel[ch] = (byte)0;
}}pixel += 4;
}}
}
HandsTouchless Control
senseManager.EnableTouchlessController(null);
touchlessController = senseManager.QueryTouchlessController();touchlessController.SubscribeEvent(new PXCMTouchlessController.OnFiredUXEventDelegate(OnTouchlessControllerUXEvent));
PXCMTouchlessController.ProfileInfoprofileInfo;touchlessController.QueryProfile(outprofileInfo);profileInfo.config = PXCMTouchlessController.ProfileInfo.Configuration.Configuration_Scroll_Horizontally;
touchlessController.SetProfile(profileInfo);
private void OnTouchlessControllerUXEvent(PXCMTouchlessController.UXEventData data)
{switch (data.type){case PXCMTouchlessController.UXEventData.UXEventType.UXEvent_StartScroll:break;
case PXCMTouchlessController.UXEventData.UXEventType.UXEvent_Scroll:break;
case PXCMTouchlessController.UXEventData.UXEventType.UXEvent_Select:break;
}}
Speech Synthesis
private void SetupSpeechSynthesis(){if (senseManager.session.CreateImpl<PXCMSpeechSynthesis>(out speechSynthesis) >= pxcmStatus.PXCM_STATUS_NO_ERROR)
{PXCMSpeechSynthesis.ProfileInfo profileInfo;if (speechSynthesis.QueryProfile(0,
out profileInfo) >= pxcmStatus.PXCM_STATUS_NO_ERROR)
{profileInfo.voice =
PXCMSpeechSynthesis.VoiceType.VOICE_ANY;
audioInfo = profileInfo.outputs;speechSynthesis
.SetProfile(profileInfo);}
}}
public void Say(string sentence){if (speechSynthesis != null){speechSynthesis.BuildSentence(1, sentence);int buffer =
speechSynthesis.QueryBufferNum(1);VoiceOut voiceOut =
new VoiceOut(audioInfo);for (int i = 0; i < buffer; i++){
PXCMAudio audio = speechSynthesis.QueryBuffer(1, i);
voiceOut.RenderAudio(audio);}voiceOut.Close();speechSynthesis.ReleaseSentence(1);
}}
Face tracking
senseManager.EnableFace();
PXCMFaceModule faceModule = senseManager.QueryFace();PXCMFaceConfiguration config = faceModule.CreateActiveConfiguration();
config.detection.isEnabled = true;config.detection.maxTrackedFaces = 2;config.landmarks.isEnabled = true;config.ApplyChanges();
private pxcmStatus ProcessFrame(int moduleId,PXCMBase module,PXCMCapture.Sample captureSample)
{switch (moduleId){case PXCMFaceModule.CUID:
ProcessFace(module);break;
}return pxcmStatus.PXCM_STATUS_NO_ERROR;
}
private void ProcessFace(PXCMBase module){using (PXCMFaceModule faceModule =
module.QueryInstance<PXCMFaceModule>()){using (PXCMFaceData faceData = faceModule.CreateOutput()){
int faceCount = faceData.QueryNumberOfDetectedFaces();
for (int faceIndex = 0; faceIndex < faceCount;faceIndex++)
{PXCMFaceData.Face face = faceData.QueryFaceByIndex(faceIndex);
GetFaceBounds(face);GetLandmarkData(face);
}}
}}
private static void GetLandmarkData(PXCMFaceData.Face face)
{PXCMFaceData.LandmarkPoint[] landmarks;PXCMFaceData.LandmarksData landmarkData =face.QueryLandmarks();
landmarkData.QueryPoints(out landmarks);}
private static void GetFaceBounds(PXCMFaceData.Face face)
{PXCMFaceData.DetectionData detectionData =face.QueryDetection();
PXCMRectI32 boundingRect;detectionData.QueryBoundingRect(out boundingRect);
}
Starting the loop
PXCMSenseManager.Handler handler = newPXCMSenseManager.Handler{onModuleProcessedFrame = ProcessFrame
};
private pxcmStatus ProcessFrame(int moduleId, PXCMBase module, PXCMCapture.Sample captureSample)
{return pxcmStatus.PXCM_STATUS_NO_ERROR;
}
pxcmStatus status = senseManager.Init(handler);
What next?
Order your camera
http://click.intel.com/intel-realsense-developer-kit.html
Download the SDK
https://software.intel.com/en-us/realsense/intel-realsense-sdk-for-windows
Code