Dev/diff2stat

.github/workflows/p4vfs-verify.yml

@@ -28,7 +28,7 @@ jobs:
     runs-on: ${{matrix.os}}
     strategy:
       matrix:
-        os: [windows-latest]
+        os: [windows-2022]
         configuration: [DebugDev, ReleaseDev]
 
     steps:

external/OpenSSL/OpenSSL.Module.cs

@@ -12,7 +12,7 @@ namespace Microsoft.P4VFS.External
 {
 	public class OpensslModule : Module
 	{
-		private const string OPENSSL_VERSION = "3.3.3";
+		private const string OPENSSL_VERSION = "3.0.18";
 		private const string PERL_PACKAGE_NAME = "StrawberryPerl";
 		private const string PERL_VERSION = "5.28.0.1";
 
@@ -75,7 +75,7 @@ private void BuildOpensslLibrary(string opensslArchiveFolder, string opensslTarg
 				$"@ECHO ON",
 				$"CALL \"{vcvarsScriptPath}\"",
 				$"CD /D \"{opensslArchiveFolder}\"",
-				$"\"{perlExe}\" Configure VC-WIN64A no-asm no-apps no-tests \"--prefix={opensslConfigurationFolder}\" \"--openssldir={opensslConfigurationFolder}-ssl\" --{configuration}",
+				$"\"{perlExe}\" Configure VC-WIN64A no-asm no-tests \"--prefix={opensslConfigurationFolder}\" \"--openssldir={opensslConfigurationFolder}-ssl\" --{configuration}",
 				$"IF %ERRORLEVEL% NEQ 0 EXIT /B 1",
 				$"nmake clean",
 				$"IF %ERRORLEVEL% NEQ 0 EXIT /B 1",

external/P4API/P4API.Module.cs

@@ -12,7 +12,7 @@ namespace Microsoft.P4VFS.External
 {
 	public class P4apiModule : Module
 	{
-		private const string P4API_VERSION = "r24.1";
+		private const string P4API_VERSION = "r25.2";
 		private const string P4API_VISUAL_STUDIO_EDITION = "2022";
 
 		public override string Name

source/P4VFS.Console/P4VFS.Notes.txt

@@ -1,5 +1,28 @@
 Microsoft P4VFS Release Notes
 
+Version [1.29.4.0]
+* Updating to latest P4API 25.2
+* Migrating to OpenSSL 3.0 LTS as recommended by P4API 25.2 with latest 3.0.18
+* Addition to default ExcludedProcessNames of MpDlpService.exe which is common Windows
+  Defender application
+* Optimization to use C++ transparent comparators on std::map<string,*,compare> to enable 
+  heterogeneous lookup from either c_str or string
+* Addition of default property values to SettingManager. It is now used to test
+  against combinations of default and overridden settings values
+* Fixing bug where p4vfs.exe common option '-v' would not actually enable verbose logging 
+  for the command
+* Addition of experimental configuration setting MaxDiff2StatFileCount for specifying
+  the maximum number of files for when fstat commands should be used instead of diff2
+  for determining the have & head filetype of files to sync. Using fstat can be 
+  faster than diff2 in the rare case where a very few number files are updated under a 
+  very large file spec. For that reason, we'll keep this feature disabled by default indefinitly. 
+  A MaxDiff2StatFileCount of zero (default) will use legacy behavior to only use 
+  diff2. A MaxDiff2StatFileCount of less than zero will always use fstat. A 
+  MaxDiff2StatFileCount of greater than zero will use fstat if the number of modifications
+  are less than the value.
+* Addition of a new verbose mode "Preview Time" profile time to the sync command which
+  indicates the time spend gathering virtual sync file modifications to be applied.
+
 Version [1.29.3.0]
 * Perforce SSO login from URL now uses impersonated p4vfs login command and ShellExecute 
   instead of using cmd.exe. This fixes rare case lingering cmd.exe processes from failed 

source/P4VFS.Console/Source/Program.cs

@@ -264,6 +264,8 @@ public static int Main(string[] args)
 			try
 			{
 				VirtualFileSystemLog.Intitialize();
+				VirtualFileSystemLog.Info("P4VFS version {0}", VirtualFileSystem.CurrentVersion);
+
 				SettingManager.RemoteLogging = SettingManager.ConsoleRemoteLogging;
 				SettingManager.ImmediateLogging = SettingManager.ConsoleImmediateLogging;
 				_P4Directory = Environment.CurrentDirectory;
@@ -317,7 +319,6 @@ public static int Main(string[] args)
 					}
 				}
 
-				VirtualFileSystemLog.Info("P4VFS version {0}", VirtualFileSystem.CurrentVersion);
 				if (argIndex >= args.Length)
 				{
 					argIndex = 0;

source/P4VFS.Core/Include/DepotClient.h

@@ -114,7 +114,7 @@ namespace P4 {
 	template <typename Result>
 	Result FDepotClient::Run(const DepotCommand& cmd)
 	{
-		Result result = std::make_shared<Result::element_type>();
+		Result result = MakeResult<Result>();
 		Run(cmd, *result.get());
 		return result;
 	}

source/P4VFS.Core/Include/DepotOperations.h

@@ -147,6 +147,22 @@ namespace P4 {
 			CreateFileSpecFlags::Enum flags = CreateFileSpecFlags::None
 			);
 
+		static DepotStringArray
+		DepotOperations::CreateFileSpecs(
+			DepotClient& depotClient,
+			const DepotSyncActionInfoArray& fileModifications,
+			const DepotRevision& revision, 
+			CreateFileSpecFlags::Enum flags
+			);
+
+		static DepotResultDiff2
+		DepotOperations::Diff2Stat(
+			DepotClient& depotClient,
+			DepotSyncFlags::Enum syncFlags,
+			const DepotStringArray& fileSpecs,
+			const DepotSyncActionInfoArray& fileModifications
+			);
+
 		static DepotResultDiff2
 		Diff2(
 			DepotClient& depotClient,
@@ -175,6 +191,11 @@ namespace P4 {
 			SizesFlags::Enum flags = SizesFlags::None
 			);
 
+		static bool
+		IsFullDepotFileSpec(
+			const DepotString& fileSpec
+			);
+
 		static bool
 		IsFileTypeAlwaysResident(
 			const DepotString& syncResident, 

source/P4VFS.Core/Include/DepotResult.h

@@ -29,13 +29,13 @@ namespace P4 {
 
 	struct FDepotResultTag
 	{
-		bool ContainsKey(const DepotString& tagKey) const;
-		void RemoveKey(const DepotString& tagKey);
-		void SetValue(const DepotString& tagKey, const DepotString& tagValue);
+		bool ContainsKey(const char* tagKey) const;
+		void RemoveKey(const char* tagKey);
+		void SetValue(const char* tagKey, const DepotString& tagValue);
 
-		bool TryGetValue(const DepotString& tagKey, DepotString& value) const;
-		const DepotString& GetValue(const DepotString& tagKey) const;
-		const DepotString* GetValuePtr(const DepotString& tagKey) const;
+		bool TryGetValue(const char* tagKey, DepotString& value) const;
+		const DepotString& GetValue(const char* tagKey) const;
+		const DepotString* GetValuePtr(const char* tagKey) const;
 
 		int32_t GetValueInt32(const char* tagKey, int32_t defaultValue = 0) const;
 		int64_t GetValueInt64(const char* tagKey, int64_t defaultValue = 0) const;
@@ -79,7 +79,12 @@ namespace P4 {
 
 		P4VFS_CORE_API const Array<DepotResultTag>& TagList() const;
 		P4VFS_CORE_API const Array<DepotResultText>& TextList() const;
-		const DepotString& GetTagValue(const DepotString& tagKey) const;
+		const DepotString& GetTagValue(const char* tagKey) const;
+
+		void Append(const FDepotResult& src);
+		void Append(const Array<DepotResultTag>& srcTagList);
+		void Append(const Array<DepotResultText>& srcTextList);
+		void Append(const DepotResultTag& srcTag);
 
 	protected:
 		friend class DepotClientCommand;	
@@ -95,20 +100,20 @@ namespace P4 {
 			return m_Tag.get() ? *m_Tag : Empty;
 		}
 
-		bool ContainsTagKey(const DepotString& tagKey) const
+		bool ContainsTagKey(const char* tagKey) const
 		{
 			return Tag().ContainsKey(tagKey);
 		}
 
-		void RemoveTagKey(const DepotString& tagKey)
+		void RemoveTagKey(const char* tagKey)
 		{
 			if (m_Tag.get())
 			{
 				m_Tag->RemoveKey(tagKey);
 			}
 		}
 
-		void SetTagValue(const DepotString& tagKey, const DepotString& tagValue)
+		void SetTagValue(const char* tagKey, const DepotString& tagValue)
 		{
 			if (m_Tag.get() == nullptr)
 			{
@@ -117,7 +122,7 @@ namespace P4 {
 			m_Tag->SetValue(tagKey, tagValue);
 		}
 
-		const DepotString& GetTagValue(const DepotString& tagKey) const
+		const DepotString& GetTagValue(const char* tagKey) const
 		{
 			return Tag().GetValue(tagKey);
 		}
@@ -162,6 +167,20 @@ namespace P4 {
 			return FDepotResultNode::Create<NodeType>(index < m_TagList.size() ? m_TagList[index] : nullptr);
 		}
 	};
+
+	template <typename Result>
+	Result MakeResult()
+	{
+		return std::make_shared<Result::element_type>();
+	}
+
+	template <typename Result>
+	Result MakeErrorResult(const DepotString& errorText)
+	{
+		Result result = MakeResult<Result>();
+		result->SetError(errorText.c_str());
+		return result;
+	}
 }}}
 
 #pragma managed(pop)

source/P4VFS.Core/Include/FileCore.h

@@ -288,36 +288,42 @@ namespace FileCore {
 		{
 			template <typename TL, typename TR>
 			bool operator()(const TL& a, const TR& b) const { return StringInfo::Strcmp(StringInfo::CStr(a), StringInfo::CStr(b)) < 0; }
+			using is_transparent = int;
 		};
 
 		struct LessInsensitive
 		{
 			template <typename TL, typename TR>
 			bool operator()(const TL& a, const TR& b) const { return StringInfo::Stricmp(StringInfo::CStr(a), StringInfo::CStr(b)) < 0; }
+			using is_transparent = int;
 		};
 
 		struct Greater 
 		{
 			template <typename TL, typename TR>
 			bool operator()(const TL& a, const TR& b) const { return StringInfo::Strcmp(StringInfo::CStr(a), StringInfo::CStr(b)) > 0; }
+			using is_transparent = int;
 		};
 
 		struct GreaterInsensitive
 		{
 			template <typename TL, typename TR>
 			bool operator()(const TL& a, const TR& b) const { return StringInfo::Stricmp(StringInfo::CStr(a), StringInfo::CStr(b)) > 0; }
+			using is_transparent = int;
 		};
 
 		struct Equal
 		{
 			template <typename TL, typename TR>
 			bool operator()(const TL& a, const TR& b) const { return StringInfo::Strcmp(StringInfo::CStr(a), StringInfo::CStr(b)) == 0; }
+			using is_transparent = int;
 		};
 
 		struct EqualInsensitive
 		{
 			template <typename TL, typename TR>
 			bool operator()(const TL& a, const TR& b) const { return StringInfo::Stricmp(StringInfo::CStr(a), StringInfo::CStr(b)) == 0; }
+			using is_transparent = int;
 		};
 
 		class WtoA
@@ -377,6 +383,7 @@ namespace FileCore {
 		};
 
 		#define LITERAL(CharType, str) ::Microsoft::P4VFS::FileCore::StringInfo::Traits::Type<CharType>::Literal(str, L##str)
+		#define LITERAL_STRING(StringType, str) LITERAL(typename std::decay<decltype(StringType)>::type::value_type, str)
 	};
 
 	struct P4VFS_CORE_API FileInfo
@@ -603,6 +610,12 @@ namespace FileCore {
 			return std::any_of(elements.begin(), elements.end(), predicate);
 		}
 
+		template <typename ArrayType, typename Predicate>
+		static bool All(const ArrayType& elements, Predicate predicate)
+		{
+			return std::all_of(elements.begin(), elements.end(), predicate);
+		}
+
 		template <typename ArrayType, typename Predicate>
 		static void RemoveIf(ArrayType& elements, Predicate predicate)
 		{
@@ -739,18 +752,20 @@ namespace FileCore {
 		do { if (((v) == (ns::ev)) || ((ns::ev) != 0 && ((v) & (ns::ev)))) \
 		{ \
 			if (s.empty() == false) \
-				s += "|"; \
-			s += #ev; \
+			{ \
+				s += LITERAL_STRING(s, "|"); \
+			} \
+			s += LITERAL_STRING(s, #ev); \
 		} } while(0)
 
-	#define P4VFS_ENUM_TO_STRING_RETURN(v, ns, ev) \
+	#define P4VFS_ENUM_TO_STRING_RETURN(rt, v, ns, ev) \
 		do { if ((v) == (ns::ev)) \
 		{ \
-			return #ev; \
+			return LITERAL(rt::value_type, #ev); \
 		} } while (0)
 
 	#define P4VFS_STRING_TO_ENUM_RETURN(v, ns, ev) \
-		do { if (Microsoft::P4VFS::FileCore::StringInfo::Stricmp(#ev, Microsoft::P4VFS::FileCore::StringInfo::CStr(v)) == 0) \
+		do { if (Microsoft::P4VFS::FileCore::StringInfo::Stricmp(LITERAL_STRING(v, #ev), Microsoft::P4VFS::FileCore::StringInfo::CStr(v)) == 0) \
 		{ \
 			return ns::ev; \
 		} } while (0)

source/P4VFS.Core/Include/LogDevice.h

@@ -21,8 +21,8 @@ namespace FileCore {
 			Error
 		};
 
-		P4VFS_CORE_API static AString ToString(Enum value);
-		P4VFS_CORE_API static LogChannel::Enum FromString(const AString& value);
+		P4VFS_CORE_API static String ToString(Enum value);
+		P4VFS_CORE_API static LogChannel::Enum FromString(const String& value);
 	};
 
 	struct LogElement

source/P4VFS.Core/Include/SettingManager.h

@@ -24,13 +24,14 @@ namespace FileCore {
 		_N( bool,     SyncDefaultQuiet,                false ) \
 		_N( String,   SyncResidentPattern,             L"" ) \
 		_N( bool,     Unattended,                      false ) \
-		_N( String,   Verbosity,                       CSTR_ATOW(FileCore::LogChannel::ToString(FileCore::LogChannel::Info)) ) \
-		_N( String,   ExcludedProcessNames,            L"MsSense.exe;MsMpEng.exe;SenseCE.exe;SenseIR.exe;SearchProtocolHost.exe" ) \
+		_N( String,   Verbosity,                       FileCore::LogChannel::ToString(FileCore::LogChannel::Info).c_str() ) \
+		_N( String,   ExcludedProcessNames,            L"MsSense.exe;MsMpEng.exe;SenseCE.exe;SenseIR.exe;SearchProtocolHost.exe;MpDlpService.exe" ) \
 		_N( int32_t,  CreateFileRetryCount,            8 ) \
 		_N( int32_t,  CreateFileRetryWaitMs,           250 ) \
 		_N( int32_t,  PoolDefaultNumberOfThreads,      8 ) \
 		_N( int32_t,  GarbageCollectPeriodMs,          5*60*1000 ) \
 		_N( int32_t,  DepotClientCacheIdleTimeoutMs,   5*60*1000 ) \
+		_N( int32_t,  MaxDiff2StatFileCount,           0 ) \
 
 
 	class SettingManager;
@@ -82,7 +83,7 @@ namespace FileCore {
 		P4VFS_CORE_API void SetInt32(int32_t value);
 		P4VFS_CORE_API void SetString(const String& value);
 
-		template <typename T> T Get(const T& defaultValue) const = 0;
+		template <typename T> T Get(const T& defaultValue) const;
 		template <> bool Get<bool>(const bool& defaultValue) const { return GetBool(defaultValue); }
 		template <> int32_t Get<int32_t>(const int32_t& defaultValue) const { return GetInt32(defaultValue); }
 		template <> String Get<String>(const String& defaultValue) const { return GetString(defaultValue); }
@@ -154,6 +155,13 @@ namespace FileCore {
 		SETTING_MANAGER_PROPERTIES(SETTING_MANAGER_DECLARE_PROP)
 		#undef SETTING_MANAGER_DECLARE_PROP
 
+		struct Default
+		{
+			#define SETTING_MANAGER_DECLARE_PROP(type, name, value)  static type name();
+			SETTING_MANAGER_PROPERTIES(SETTING_MANAGER_DECLARE_PROP)
+			#undef SETTING_MANAGER_DECLARE_PROP
+		};
+
 	private:
 		PropertyMap m_PropertMap;
 		HANDLE m_PropertMapMutex;