microsoft · argo-1 · Oct 17, 2019
diff --git a/HoloLensForCV.sln b/HoloLensForCV.sln
diff --git a/Shared/Audio/Audio.props b/Shared/Audio/Audio.props
diff --git a/Shared/Audio/Audio.vcxproj b/Shared/Audio/Audio.vcxproj
diff --git a/Shared/Audio/Audio.vcxproj.filters b/Shared/Audio/Audio.vcxproj.filters
diff --git a/Shared/Audio/AudioFileReader.cpp b/Shared/Audio/AudioFileReader.cpp
diff --git a/Shared/Audio/AudioStreamReader.cpp b/Shared/Audio/AudioStreamReader.cpp
diff --git a/Shared/Audio/BasicListeningEarcon.wav b/Shared/Audio/BasicListeningEarcon.wav
diff --git a/Shared/Audio/BasicResultsEarcon.wav b/Shared/Audio/BasicResultsEarcon.wav
diff --git a/Shared/Audio/Include/Audio/All.h b/Shared/Audio/Include/Audio/All.h
diff --git a/Shared/Audio/Include/Audio/AudioFileReader.h b/Shared/Audio/Include/Audio/AudioFileReader.h
diff --git a/Shared/Audio/Include/Audio/AudioStreamReader.h b/Shared/Audio/Include/Audio/AudioStreamReader.h
diff --git a/Shared/Audio/Include/Audio/OmnidirectionalSound.h b/Shared/Audio/Include/Audio/OmnidirectionalSound.h
diff --git a/Shared/Audio/Include/Audio/XAudio2Helpers.h b/Shared/Audio/Include/Audio/XAudio2Helpers.h
diff --git a/Shared/Audio/OmnidirectionalSound.cpp b/Shared/Audio/OmnidirectionalSound.cpp
diff --git a/Shared/Audio/README.md b/Shared/Audio/README.md
diff --git a/Shared/Audio/pch.cpp b/Shared/Audio/pch.cpp
diff --git a/Shared/Audio/pch.h b/Shared/Audio/pch.h
diff --git a/Shared/Audio/targetver.h b/Shared/Audio/targetver.h
diff --git a/Shared/Debugging/Debugging.props b/Shared/Debugging/Debugging.props
diff --git a/Shared/Debugging/Debugging.vcxproj b/Shared/Debugging/Debugging.vcxproj
diff --git a/Shared/Debugging/Debugging.vcxproj.filters b/Shared/Debugging/Debugging.vcxproj.filters
diff --git a/Shared/Debugging/Include/Debugging/All.h b/Shared/Debugging/Include/Debugging/All.h
diff --git a/Shared/Debugging/Include/Debugging/CodeContracts.h b/Shared/Debugging/Include/Debugging/CodeContracts.h
diff --git a/Shared/Debugging/Include/Debugging/Timer.h b/Shared/Debugging/Include/Debugging/Timer.h
diff --git a/Shared/Debugging/Include/Debugging/TimerGuard.h b/Shared/Debugging/Include/Debugging/TimerGuard.h
diff --git a/Shared/Debugging/Include/Debugging/Trace.h b/Shared/Debugging/Include/Debugging/Trace.h
diff --git a/Shared/Debugging/README.md b/Shared/Debugging/README.md
diff --git a/Shared/Debugging/Timer.cpp b/Shared/Debugging/Timer.cpp
diff --git a/Shared/Debugging/TimerGuard.cpp b/Shared/Debugging/TimerGuard.cpp
diff --git a/Shared/Debugging/Trace.cpp b/Shared/Debugging/Trace.cpp
diff --git a/Shared/Debugging/pch.cpp b/Shared/Debugging/pch.cpp
diff --git a/Shared/Debugging/pch.h b/Shared/Debugging/pch.h
diff --git a/Shared/Debugging/targetver.h b/Shared/Debugging/targetver.h
diff --git a/Shared/Graphics/CameraResources.cpp b/Shared/Graphics/CameraResources.cpp
diff --git a/Shared/Graphics/DeviceResources.cpp b/Shared/Graphics/DeviceResources.cpp
diff --git a/Shared/Graphics/Graphics.props b/Shared/Graphics/Graphics.props
diff --git a/Shared/Graphics/Graphics.vcxproj b/Shared/Graphics/Graphics.vcxproj
diff --git a/Shared/Graphics/Graphics.vcxproj.filters b/Shared/Graphics/Graphics.vcxproj.filters
diff --git a/Shared/Graphics/Include/Graphics/All.h b/Shared/Graphics/Include/Graphics/All.h
diff --git a/Shared/Graphics/Include/Graphics/CameraResources.h b/Shared/Graphics/Include/Graphics/CameraResources.h
diff --git a/Shared/Graphics/Include/Graphics/DeviceResources.h b/Shared/Graphics/Include/Graphics/DeviceResources.h
diff --git a/Shared/Graphics/Include/Graphics/DirectXHelper.h b/Shared/Graphics/Include/Graphics/DirectXHelper.h
diff --git a/Shared/Graphics/Include/Graphics/StepTimer.h b/Shared/Graphics/Include/Graphics/StepTimer.h
diff --git a/Shared/Graphics/README.md b/Shared/Graphics/README.md
diff --git a/Shared/Graphics/packages.config b/Shared/Graphics/packages.config
diff --git a/Shared/Graphics/pch.cpp b/Shared/Graphics/pch.cpp
diff --git a/Shared/Graphics/pch.h b/Shared/Graphics/pch.h
diff --git a/Shared/Graphics/targetver.h b/Shared/Graphics/targetver.h
diff --git a/Shared/HoloLensForCV/CameraIntrinsics.cpp b/Shared/HoloLensForCV/CameraIntrinsics.cpp
diff --git a/Shared/HoloLensForCV/CameraIntrinsics.h b/Shared/HoloLensForCV/CameraIntrinsics.h
diff --git a/Shared/HoloLensForCV/CsvWriter.cpp b/Shared/HoloLensForCV/CsvWriter.cpp
diff --git a/Shared/HoloLensForCV/CsvWriter.h b/Shared/HoloLensForCV/CsvWriter.h
diff --git a/Shared/HoloLensForCV/HoloLensForCV.vcxproj b/Shared/HoloLensForCV/HoloLensForCV.vcxproj
@@ -155,6 +155,7 @@
       <DisableSpecificWarnings>28204</DisableSpecificWarnings>
       <WarningLevel>Level4</WarningLevel>
       <TreatWarningAsError>true</TreatWarningAsError>
+      <AdditionalIncludeDirectories>$(SolutionDir)packages\Eigen.3.3.3\build\native\include;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
     </ClCompile>
     <Link>
       <SubSystem>Console</SubSystem>

diff --git a/Shared/HoloLensForCV/HoloLensForCV.vcxproj.filters b/Shared/HoloLensForCV/HoloLensForCV.vcxproj.filters
diff --git a/Shared/HoloLensForCV/ICameraIntrinsics.h b/Shared/HoloLensForCV/ICameraIntrinsics.h
diff --git a/Shared/HoloLensForCV/ISensorFrameSink.h b/Shared/HoloLensForCV/ISensorFrameSink.h
diff --git a/Shared/HoloLensForCV/ISensorFrameSinkGroup.h b/Shared/HoloLensForCV/ISensorFrameSinkGroup.h
diff --git a/Shared/HoloLensForCV/MediaFrameReaderContext.cpp b/Shared/HoloLensForCV/MediaFrameReaderContext.cpp
@@ -112,13 +112,23 @@ namespace HoloLensForCV
         Windows::Graphics::Imaging::SoftwareBitmap^ softwareBitmap =
             frame->VideoMediaFrame->SoftwareBitmap;
 
+		Windows::Media::Devices::Core::CameraIntrinsics^ cameraIntrinsics = frame->VideoMediaFrame->CameraIntrinsics;
+
         //
         // Finally, wrap all of the above information in a SensorFrame object and pass it
         // down to the sensor frame sink. We'll also retain a reference to the latest sensor
         // frame on this object for immediate consumption by the app.
         //
-        SensorFrame^ sensorFrame =
-            ref new SensorFrame(_sensorType, timestamp, softwareBitmap);
+		SensorFrame^ sensorFrame =
+			//   ref new SensorFrame(_sensorType, timestamp, softwareBitmap);
+		//	ref new SensorFrame(_sensorType, timestamp, softwareBitmap, frame);
+			ref new SensorFrame(
+				_sensorType
+				,timestamp
+				,softwareBitmap
+			//	,cameraIntrinsics
+			);
+
 
         //
         // Extract the frame-to-origin transform, if the MFT exposed it:
@@ -178,6 +188,8 @@ namespace HoloLensForCV
                 zero;
         }
 
+
+		//frame->Properties->Lookup()
         //
         // Extract camera view transform, if the MFT exposed it:
         //

diff --git a/Shared/HoloLensForCV/MediaFrameReaderContext.h b/Shared/HoloLensForCV/MediaFrameReaderContext.h
diff --git a/Shared/HoloLensForCV/MediaFrameSourceGroup.cpp b/Shared/HoloLensForCV/MediaFrameSourceGroup.cpp
@@ -389,6 +389,87 @@ namespace HoloLensForCV
         }, Concurrency::task_continuation_context::get_current_winrt_context());
     }
 
+	//CameraIntrinsics^ 
+
+
+
+	// ADDED : ARJUN
+
+	Windows::Media::Devices::Core::CameraIntrinsics^ MediaFrameSourceGroup::GetCameraIntrinsics(SensorType inputSensorType) { //Async??
+
+		Windows::Media::Capture::Frames::MediaFrameSource^ selectedMediaFrameSource;
+		Windows::Media::Devices::Core::CameraIntrinsics^ output;
+
+		for (Windows::Foundation::Collections::IKeyValuePair<Platform::String^, Windows::Media::Capture::Frames::MediaFrameSource^>^ kvp : _mediaCapture->FrameSources)
+		{
+			Windows::Media::Capture::Frames::MediaFrameSource^ source =
+				kvp->Value;
+
+			SensorType sensorType =
+				GetSensorType(
+					source);
+
+			if (IsEnabled(sensorType) && sensorType == inputSensorType) {
+
+				selectedMediaFrameSource = source;
+
+				break;
+
+			}
+
+		}
+
+
+		/*Platform::String^ requestedSubtype =
+			nullptr;
+
+		auto found =
+			std::find_if(
+				begin(selectedMediaFrameSource->SupportedFormats),
+				end(selectedMediaFrameSource->SupportedFormats),
+				[&](Windows::Media::Capture::Frames::MediaFrameFormat^ format)
+		{
+			requestedSubtype =
+				GetSubtypeForFrameReader(
+					selectedMediaFrameSource->Info->SourceKind,
+					format);
+
+			return requestedSubtype != nullptr;
+
+		});
+
+		output = selectedMediaFrameSource->TryGetCameraIntrinsics(*found);
+		*/
+
+		output = selectedMediaFrameSource->TryGetCameraIntrinsics(selectedMediaFrameSource->CurrentFormat);
+		if (output == nullptr) {
+			auto found = std::find_if(begin(selectedMediaFrameSource->SupportedFormats),
+				end(selectedMediaFrameSource->SupportedFormats),
+				[&](Windows::Media::Capture::Frames::MediaFrameFormat^ format)
+			{
+				output = selectedMediaFrameSource->TryGetCameraIntrinsics(format);
+
+				if (output == nullptr) {
+					dbg::trace(
+						L"MediaFrameSourceGroup::GetCameraIntrinsics: format %s-%s @%i/%iHz intrinsics nullptr",
+						format->MajorType->Data(),
+						format->Subtype->Data(),
+						format->FrameRate->Numerator,
+						format->FrameRate->Denominator);
+				}
+				return output != nullptr;
+
+			});
+
+		}
+
+		return output;
+
+	}
+
+
+
+
     SensorType MediaFrameSourceGroup::GetSensorType(
         Windows::Media::Capture::Frames::MediaFrameSource^ source)
     {
@@ -709,4 +790,7 @@ namespace HoloLensForCV
 
         return cleanupTask;
     }
+
+
+
 }
diff --git a/Shared/HoloLensForCV/MediaFrameSourceGroup.h b/Shared/HoloLensForCV/MediaFrameSourceGroup.h
@@ -36,6 +36,9 @@ namespace HoloLensForCV
         SensorFrame^ GetLatestSensorFrame(
             SensorType sensorType);
 
+		Windows::Media::Devices::Core::CameraIntrinsics^ GetCameraIntrinsics(SensorType sensorType);
+
+
     private:
         /// <summary>
         /// Returns true if the sensor was explicitly enabled by the user.
@@ -75,6 +78,8 @@ namespace HoloLensForCV
         concurrency::task<bool> TryInitializeMediaCaptureAsync(
             Windows::Media::Capture::Frames::MediaFrameSourceGroup^ group);
 
+
+
     private:
         MediaFrameSourceGroupType _mediaFrameSourceGroupType;
         SpatialPerception^ _spatialPerception;

diff --git a/Shared/HoloLensForCV/MediaFrameSourceGroupType.h b/Shared/HoloLensForCV/MediaFrameSourceGroupType.h
diff --git a/Shared/HoloLensForCV/MultiFrameBuffer.cpp b/Shared/HoloLensForCV/MultiFrameBuffer.cpp
diff --git a/Shared/HoloLensForCV/MultiFrameBuffer.h b/Shared/HoloLensForCV/MultiFrameBuffer.h
diff --git a/Shared/HoloLensForCV/README.md b/Shared/HoloLensForCV/README.md
diff --git a/Shared/HoloLensForCV/SensorFrame.cpp b/Shared/HoloLensForCV/SensorFrame.cpp
@@ -13,13 +13,189 @@
 
 namespace HoloLensForCV
 {
-    SensorFrame::SensorFrame(
-        _In_ SensorType frameType,
-        _In_ Windows::Foundation::DateTime timestamp,
-        _In_ Windows::Graphics::Imaging::SoftwareBitmap^ softwareBitmap)
-    {
-        FrameType = frameType;
-        Timestamp = timestamp;
-        SoftwareBitmap = softwareBitmap;
-    }
+	SensorFrame::SensorFrame(
+		_In_ SensorType frameType,
+		_In_ Windows::Foundation::DateTime timestamp,
+		_In_ Windows::Graphics::Imaging::SoftwareBitmap^ softwareBitmap)
+	{
+		FrameType = frameType;
+		Timestamp = timestamp;
+		SoftwareBitmap = softwareBitmap;
+
+	}
+
+	/*SensorFrame::SensorFrame(
+		_In_ SensorType frameType,
+		_In_ Windows::Foundation::DateTime timestamp,
+		_In_ Windows::Graphics::Imaging::SoftwareBitmap^ softwareBitmap,
+		_In_ Windows::Media::Capture::Frames::MediaFrameReference^ frame)
+	{
+		FrameType = frameType;
+		Timestamp = timestamp;
+		SoftwareBitmap = softwareBitmap;
+
+		CoreCameraIntrinsics = frame->VideoMediaFrame->CameraIntrinsics;
+		if (CoreCameraIntrinsics == nullptr) {
+			dbg::trace(L"SensorFrame:VideoMediaFrame CameraIntrinsics null.");
+		}
+		else {
+			dbg::trace(L"SensorFrame:VideoMediaFrame CameraIntrinsics not null!");
+		}
+
+	}*/
+
+	SensorFrame::SensorFrame(
+		_In_ SensorType frameType,
+		_In_ Windows::Foundation::DateTime timestamp,
+		_In_ Windows::Graphics::Imaging::SoftwareBitmap^ softwareBitmap,
+		_In_ Windows::Media::Devices::Core::CameraIntrinsics^ cameraIntrinsics)
+		{
+
+		FrameType = frameType;
+		Timestamp = timestamp;
+		SoftwareBitmap = softwareBitmap;
+		CoreCameraIntrinsics = cameraIntrinsics;
+		if (CoreCameraIntrinsics == nullptr) {
+			dbg::trace(L"SensorFrame:VideoMediaFrame CameraIntrinsics null.");
+		}
+		else {
+			dbg::trace(L"SensorFrame:VideoMediaFrame CameraIntrinsics not null!");
+		}
+
+
+}
+
+	Platform::Array<uint8_t>^
+	/*void*/ SensorFrame::sensorFrameToImageBuffer() {
+
+		Windows::Graphics::Imaging::SoftwareBitmap^ bitmap;
+		Windows::Graphics::Imaging::BitmapBuffer^ bitmapBuffer;
+		Windows::Foundation::IMemoryBufferReference^ bitmapBufferReference;
+
+		int32_t imageWidth = 0;
+		int32_t imageHeight = 0;
+		int32_t pixelStride = 1;
+		int32_t rowStride = 0;
+
+		Platform::Array<uint8_t>^ imageBufferAsPlatformArray;
+		int32_t imageBufferSize = 0;
+
+		{
+#if DBG_ENABLE_INFORMATIONAL_LOGGING
+			dbg::TimerGuard timerGuard(
+				L"AppMain::sensorFrameToImageBuffer: buffer preparation",
+				4.0 /* minimum_time_elapsed_in_milliseconds */);
+#endif /* DBG_ENABLE_INFORMATIONAL_LOGGING */
+			imageWidth = SoftwareBitmap->PixelWidth;
+			imageHeight = SoftwareBitmap->PixelHeight;
+
+			bitmap =
+				SoftwareBitmap;
+
+			imageWidth = bitmap->PixelWidth;
+			imageHeight = bitmap->PixelHeight;
+
+			bitmapBuffer =
+				SoftwareBitmap->LockBuffer(
+					Windows::Graphics::Imaging::BitmapBufferAccessMode::Read);
+
+			bitmapBufferReference =
+				bitmapBuffer->CreateReference();
+
+			uint32_t bitmapBufferDataSize = 0;
+
+			uint8_t* bitmapBufferData =
+				Io::GetTypedPointerToMemoryBuffer<uint8_t>(
+					bitmapBufferReference,
+					bitmapBufferDataSize);
+
+			switch (SoftwareBitmap->BitmapPixelFormat)
+			{
+			case Windows::Graphics::Imaging::BitmapPixelFormat::Bgra8:
+				pixelStride = 4;
+				break;
+
+			case Windows::Graphics::Imaging::BitmapPixelFormat::Gray16:
+				pixelStride = 2;
+				break;
+
+			case Windows::Graphics::Imaging::BitmapPixelFormat::Gray8:
+				pixelStride = 1;
+				break;
+
+			default:
+#if DBG_ENABLE_INFORMATIONAL_LOGGING
+				dbg::trace(
+					L"SensorFrame::sensorFrameToImageBuffer: unrecognized bitmap pixel format, assuming 1 byte per pixel");
+#endif /* DBG_ENABLE_INFORMATIONAL_LOGGING */
+
+				break;
+			}
+
+			rowStride =
+				imageWidth * pixelStride;
+
+			imageBufferSize =
+				imageHeight * rowStride;
+
+			ASSERT(
+				imageBufferSize == (int32_t)bitmapBufferDataSize);
+
+			imageBufferAsPlatformArray =
+				ref new Platform::Array<uint8_t>(
+					bitmapBufferData,
+					imageBufferSize);
+
+			return imageBufferAsPlatformArray;
+
+		}
+
+	}
+	/*
+	Windows::Foundation::Numerics::float4x4 SensorFrame::GetAbsoluteCameraPose() {
+
+		Windows::Foundation::Numerics::float4x4^ output = 
+			ref new Windows::Foundation::Numerics::float4x4(
+				1, 0, 0, 0,
+				0, -1, 0, 0,
+				0, 0, -1, 0,
+				0, 0, 0, 1);
+
+		Windows::Foundation::Numerics::float4x4 InvFrameToOrigin;
+		Windows::Foundation::Numerics::invert(FrameToOrigin, &InvFrameToOrigin);
+
+		output = output * (CameraViewTransform*InvFrameToOrigin);
+
+		return output;
+
+	}
+
+	Windows::Foundation::Numerics::float4x4 SensorFrame::GetCamToOrigin() {
+
+		Windows::Foundation::Numerics::float4x4 camToRef;
+		Windows::Foundation::Numerics::invert(CameraViewTransform, &camToRef);
+
+		Windows::Foundation::Numerics::float4x4 camToOrigin = camToRef * FrameToOrigin;
+
+
+		Eigen::Vector3f camPinhole(camToOrigin.m41, camToOrigin.m42, camToOrigin.m43);
+
+		Eigen::Matrix3f camToOriginR;
+
+		camToOriginR(0, 0) = camToOrigin.m11;
+		camToOriginR(0, 1) = camToOrigin.m12;
+		camToOriginR(0, 2) = camToOrigin.m13;
+		camToOriginR(1, 0) = camToOrigin.m21;
+		camToOriginR(1, 1) = camToOrigin.m22;
+		camToOriginR(1, 2) = camToOrigin.m23;
+		camToOriginR(2, 0) = camToOrigin.m31;
+		camToOriginR(2, 1) = camToOrigin.m32;
+		camToOriginR(2, 2) = camToOrigin.m33;
+
+
+
+		return camToOrigin;
+
+	}
+	*/
 }