1 // Video support with XAML 2 3 // Copyright (c) Microsoft Open Technologies, Inc. 4 // All rights reserved. 5 // 6 // (3 - clause BSD License) 7 // 8 // Redistribution and use in source and binary forms, with or without modification, are permitted provided that 9 // the following conditions are met: 10 // 11 // 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the 12 // following disclaimer. 13 // 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the 14 // following disclaimer in the documentation and/or other materials provided with the distribution. 15 // 3. Neither the name of the copyright holder nor the names of its contributors may be used to endorse or 16 // promote products derived from this software without specific prior written permission. 17 // 18 // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED 19 // WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A 20 // PARTICULAR PURPOSE ARE DISCLAIMED.IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY 21 // DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES(INCLUDING, BUT NOT LIMITED TO, 22 // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 23 // HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT(INCLUDING 24 // NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 25 // POSSIBILITY OF SUCH DAMAGE. 26 27 #include "cap_winrt_video.hpp" 28 29 #include <ppl.h> 30 #include <ppltasks.h> 31 #include <concrt.h> 32 #include <agile.h> 33 34 #include <atomic> 35 #include <future> 36 #include <vector> 37 38 39 using namespace ::concurrency; 40 using namespace ::Windows::Foundation; 41 using namespace ::std; 42 43 using namespace Microsoft::WRL; 44 using namespace Windows::Media::Devices; 45 using namespace Windows::Media::MediaProperties; 46 using namespace Windows::Media::Capture; 47 using namespace Windows::UI::Xaml::Media::Imaging; 48 using namespace Windows::Devices::Enumeration; 49 50 #include "cap_winrt/CaptureFrameGrabber.hpp" 51 52 // pull in Media Foundation libs 53 #pragma comment(lib, "mfplat") 54 #pragma comment(lib, "mf") 55 #pragma comment(lib, "mfuuid") 56 57 #if (WINAPI_FAMILY!=WINAPI_FAMILY_PHONE_APP) && !defined(_M_ARM) 58 #pragma comment(lib, "Shlwapi") 59 #endif 60 61 #include "cap_winrt_bridge.hpp" 62 63 Video::Video() {} 64 65 Video &Video::getInstance() { 66 static Video v; 67 return v; 68 } 69 70 bool Video::isStarted() { 71 return bGrabberInited.load(); 72 } 73 74 void Video::closeGrabber() { 75 // assigning nullptr causes deref of grabber and thus closes the device 76 m_frameGrabber = nullptr; 77 bGrabberInited = false; 78 bGrabberInitInProgress = false; 79 } 80 81 // non-blocking 82 bool Video::initGrabber(int device, int w, int h) { 83 // already started? 84 if (bGrabberInited || bGrabberInitInProgress) return false; 85 86 width = w; 87 height = h; 88 89 bGrabberInited = false; 90 bGrabberInitInProgress = true; 91 92 m_deviceID = device; 93 94 create_task(DeviceInformation::FindAllAsync(DeviceClass::VideoCapture)) 95 .then([this](task<DeviceInformationCollection^> findTask) 96 { 97 m_devices = findTask.get(); 98 99 // got selected device? 100 if ((unsigned)m_deviceID >= m_devices.Get()->Size) 101 { 102 OutputDebugStringA("Video::initGrabber - no video device found\n"); 103 return false; 104 } 105 106 auto devInfo = m_devices.Get()->GetAt(m_deviceID); 107 108 auto settings = ref new MediaCaptureInitializationSettings(); 109 settings->StreamingCaptureMode = StreamingCaptureMode::Video; // Video-only capture 110 settings->VideoDeviceId = devInfo->Id; 111 112 auto location = devInfo->EnclosureLocation; 113 bFlipImageX = true; 114 if (location != nullptr && location->Panel == Windows::Devices::Enumeration::Panel::Back) 115 { 116 bFlipImageX = false; 117 } 118 119 m_capture = ref new MediaCapture(); 120 create_task(m_capture->InitializeAsync(settings)).then([this](){ 121 122 auto props = safe_cast<VideoEncodingProperties^>(m_capture->VideoDeviceController->GetMediaStreamProperties(MediaStreamType::VideoPreview)); 123 124 // for 24 bpp 125 props->Subtype = MediaEncodingSubtypes::Rgb24; bytesPerPixel = 3; 126 127 // XAML & WBM use BGRA8, so it would look like 128 // props->Subtype = MediaEncodingSubtypes::Bgra8; bytesPerPixel = 4; 129 130 props->Width = width; 131 props->Height = height; 132 133 return ::Media::CaptureFrameGrabber::CreateAsync(m_capture.Get(), props); 134 135 }).then([this](::Media::CaptureFrameGrabber^ frameGrabber) 136 { 137 m_frameGrabber = frameGrabber; 138 bGrabberInited = true; 139 bGrabberInitInProgress = false; 140 //ready = true; 141 _GrabFrameAsync(frameGrabber); 142 }); 143 144 return true; 145 }); 146 147 // nb. cannot block here - this will lock the UI thread: 148 149 return true; 150 } 151 152 153 void Video::_GrabFrameAsync(::Media::CaptureFrameGrabber^ frameGrabber) { 154 // use rgb24 layout 155 create_task(frameGrabber->GetFrameAsync()).then([this, frameGrabber](const ComPtr<IMF2DBuffer2>& buffer) 156 { 157 // do the RGB swizzle while copying the pixels from the IMF2DBuffer2 158 BYTE *pbScanline; 159 LONG plPitch; 160 unsigned int colBytes = width * bytesPerPixel; 161 CHK(buffer->Lock2D(&pbScanline, &plPitch)); 162 163 // flip 164 if (bFlipImageX) 165 { 166 std::lock_guard<std::mutex> lock(VideoioBridge::getInstance().inputBufferMutex); 167 168 // ptr to input Mat data array 169 auto buf = VideoioBridge::getInstance().backInputPtr; 170 171 for (unsigned int row = 0; row < height; row++) 172 { 173 unsigned int i = 0; 174 unsigned int j = colBytes - 1; 175 176 while (i < colBytes) 177 { 178 // reverse the scan line 179 // as a side effect this also swizzles R and B channels 180 buf[j--] = pbScanline[i++]; 181 buf[j--] = pbScanline[i++]; 182 buf[j--] = pbScanline[i++]; 183 } 184 pbScanline += plPitch; 185 buf += colBytes; 186 } 187 VideoioBridge::getInstance().bIsFrameNew = true; 188 } else 189 { 190 std::lock_guard<std::mutex> lock(VideoioBridge::getInstance().inputBufferMutex); 191 192 // ptr to input Mat data array 193 auto buf = VideoioBridge::getInstance().backInputPtr; 194 195 for (unsigned int row = 0; row < height; row++) 196 { 197 // used for Bgr8: 198 //for (unsigned int i = 0; i < colBytes; i++ ) 199 // buf[i] = pbScanline[i]; 200 201 // used for RGB24: 202 for (unsigned int i = 0; i < colBytes; i += bytesPerPixel) 203 { 204 // swizzle the R and B values (BGR to RGB) 205 buf[i] = pbScanline[i + 2]; 206 buf[i + 1] = pbScanline[i + 1]; 207 buf[i + 2] = pbScanline[i]; 208 209 // no swizzle 210 //buf[i] = pbScanline[i]; 211 //buf[i + 1] = pbScanline[i + 1]; 212 //buf[i + 2] = pbScanline[i + 2]; 213 } 214 215 pbScanline += plPitch; 216 buf += colBytes; 217 } 218 VideoioBridge::getInstance().bIsFrameNew = true; 219 } 220 CHK(buffer->Unlock2D()); 221 222 VideoioBridge::getInstance().frameCounter++; 223 224 if (bGrabberInited) 225 { 226 _GrabFrameAsync(frameGrabber); 227 } 228 }, task_continuation_context::use_current()); 229 } 230 231 232 // copy from input Mat to output WBM 233 // must be on UI thread 234 void Video::CopyOutput() { 235 { 236 std::lock_guard<std::mutex> lock(VideoioBridge::getInstance().outputBufferMutex); 237 238 auto inAr = VideoioBridge::getInstance().frontInputPtr; 239 auto outAr = GetData(VideoioBridge::getInstance().frontOutputBuffer->PixelBuffer); 240 241 const unsigned int bytesPerPixel = 3; 242 auto pbScanline = inAr; 243 auto plPitch = width * bytesPerPixel; 244 245 auto buf = outAr; 246 unsigned int colBytes = width * 4; 247 248 // copy RGB24 to bgra8 249 for (unsigned int row = 0; row < height; row++) 250 { 251 // used for Bgr8: 252 // nb. no alpha 253 // for (unsigned int i = 0; i < colBytes; i++ ) buf[i] = pbScanline[i]; 254 255 // used for RGB24: 256 // nb. alpha is set to full opaque 257 for (unsigned int i = 0, j = 0; i < plPitch; i += bytesPerPixel, j += 4) 258 { 259 // swizzle the R and B values (RGB24 to Bgr8) 260 buf[j] = pbScanline[i + 2]; 261 buf[j + 1] = pbScanline[i + 1]; 262 buf[j + 2] = pbScanline[i]; 263 buf[j + 3] = 0xff; 264 265 // if no swizzle is desired: 266 //buf[i] = pbScanline[i]; 267 //buf[i + 1] = pbScanline[i + 1]; 268 //buf[i + 2] = pbScanline[i + 2]; 269 //buf[i + 3] = 0xff; 270 } 271 272 pbScanline += plPitch; 273 buf += colBytes; 274 } 275 VideoioBridge::getInstance().frontOutputBuffer->PixelBuffer->Length = width * height * 4; 276 } 277 } 278 279 280 bool Video::listDevicesTask() { 281 std::atomic<bool> ready(false); 282 283 auto settings = ref new MediaCaptureInitializationSettings(); 284 285 create_task(DeviceInformation::FindAllAsync(DeviceClass::VideoCapture)) 286 .then([this, &ready](task<DeviceInformationCollection^> findTask) 287 { 288 m_devices = findTask.get(); 289 290 // TODO: collect device data 291 // for (size_t i = 0; i < m_devices->Size; i++) 292 // { 293 // .. deviceInfo; 294 // auto d = m_devices->GetAt(i); 295 // deviceInfo.bAvailable = true; 296 // deviceInfo.deviceName = PlatformStringToString(d->Name); 297 // deviceInfo.hardwareName = deviceInfo.deviceName; 298 // } 299 300 ready = true; 301 }); 302 303 // wait for async task to complete 304 int count = 0; 305 while (!ready) 306 { 307 count++; 308 } 309 310 return true; 311 } 312 313 314 bool Video::listDevices() { 315 // synchronous version of listing video devices on WinRT 316 std::future<bool> result = std::async(std::launch::async, &Video::listDevicesTask, this); 317 return result.get(); 318 } 319 320 // end