From 27515692fa0ebd3b99e17b211c0e0e595d529909 Mon Sep 17 00:00:00 2001 From: Nir Date: Wed, 3 Apr 2019 14:14:02 +0300 Subject: [PATCH 1/7] Redis List Operations. --- IntegrationTests/RedisTests.cs | 59 ++++++++++++++++++++++++++++++++++ RedisRepo/IRedisContext.cs | 28 ++++++++++++++++ RedisRepo/RedisContext.cs | 39 ++++++++++++++++++++++ 3 files changed, 126 insertions(+) diff --git a/IntegrationTests/RedisTests.cs b/IntegrationTests/RedisTests.cs index 1c4b777..9d7eeba 100644 --- a/IntegrationTests/RedisTests.cs +++ b/IntegrationTests/RedisTests.cs @@ -480,6 +480,65 @@ public void TestDistributedLockSuccessAfterLockTimePasses() } #endregion + #region Redis Lists + + [TestMethod] + public void TestAddToRedisList() + { + const string key = "TestAddToList"; + var values = new[] { "bar", "bar", "a", "b", "c" }; + + for (int i = 0; i < values.Length; i++) + { + var length = redisContext.AddToList(key, values[i]); + Assert.AreEqual(i + 1, length); + } + + ValidateListResults(key, values); + } + + [TestMethod] + public void TestAddRangeToRedisList() + { + const string key = "TestAddRangeToList"; + var values = new[] { "bar", "bar", "a", "b", "c" }; + + var length = redisContext.AddRangeToList(key, values); + + Assert.AreEqual(values.Length, length); + ValidateListResults(key, values); + } + + [TestMethod] + public void TestGetRedisList() + { + const string key = "TestAddRangeToList"; + var values = new[] { "bar", "bar", "a", "b", "c" }; + + redisContext.AddRangeToList(key, values); + + ValidateListResults(key, values); + ValidateSubListResults(key, -100, 100, values); + + var valuesSubArray = values.Skip(1).Take(3).ToArray(); + ValidateSubListResults(key, 1, 3, valuesSubArray); + ValidateSubListResults(key, -4, -2, valuesSubArray); + } + + private void ValidateListResults(string key, string[] expected) + { + var valuesFromRedis = redisContext.GetList(key); + CollectionAssert.AreEqual(expected, valuesFromRedis); + } + + private void ValidateSubListResults(string key, long start, long end, string[] expected) + { + var valuesFromRedis = redisContext.GetList(key, start, end); + CollectionAssert.AreEqual(expected, valuesFromRedis); + } + + #endregion + #region Redis Sets [TestMethod] diff --git a/RedisRepo/IRedisContext.cs b/RedisRepo/IRedisContext.cs index a641a3b..93d6f12 100644 --- a/RedisRepo/IRedisContext.cs +++ b/RedisRepo/IRedisContext.cs @@ -45,6 +45,34 @@ public interface IRedisContext bool TryGet(string key, out long? value); bool TryGet(string key, out string value); + #region Redis Lists + + /// + /// Adds to the end of a list that is at . + /// If the list doesn't exist then it is created. + /// + /// The length of the list after the addition + long AddToList(string key, string value); + + /// + /// Adds to the end of a list that is at . + /// If the list doesn't exist then it is created. + /// + /// The length of the list after the addition + long AddRangeToList(string key, string[] values); + + /// + /// Returns the list that is at . + /// If and are not given then the whole list will be returned. + /// Else, a sub-list is returned that starts at the index and stops at the index . + /// Please note that the list is zero-based indexed (so 0 is is the first element). + /// If or is negative then it means it's counted from the end of the list (-1 is the last element, -2 is the element before the last element and so on). + /// If the index is out-of-bounds then instead of throwing an exception the index is initialized to the nearest boundary (start or end of the list), and only then the operation will be done. + /// + string[] GetList(string key, long start = 0, long stop = -1); + + #endregion + #region Redis Sets void AddToSet(string key, string[] values); diff --git a/RedisRepo/RedisContext.cs b/RedisRepo/RedisContext.cs index ad33f1c..bd403e0 100644 --- a/RedisRepo/RedisContext.cs +++ b/RedisRepo/RedisContext.cs @@ -430,6 +430,45 @@ public void SetOrAppend(string key, string value) #endregion + #region Redis Lists + + /// + /// Adds to the end of a list that is at . + /// If the list doesn't exist then it is created. + /// + /// The length of the list after the addition + public long AddToList(string key, string value) + { + return AddRangeToList(key, new[] { value }); + } + + /// + /// Adds to the end of a list that is at . + /// If the list doesn't exist then it is created. + /// + /// The length of the list after the addition + public long AddRangeToList(string key, string[] values) + { + var results = Retry(() => this.Database.ListRightPush(Key(key), values.ToRedisValueArray(), flags: commandFlags), defaultRetries); + return results; + } + + /// + /// Returns the list that is at . + /// If and are not given then the whole list will be returned. + /// Else, a sub-list is returned that starts at the index and stops at the index . + /// Please note that the list is zero-based indexed (so 0 is is the first element). + /// If or is negative then it means it's counted from the end of the list (-1 is the last element, -2 is the element before the last element and so on). + /// If the index is out-of-bounds then instead of throwing an exception the index is initialized to the nearest boundary (start or end of the list), and only then the operation will be done. + /// + public string[] GetList(string key, long start = 0, long stop = -1) + { + var results = Retry(() => this.Database.ListRange(Key(key), start, stop, flags: commandFlags), defaultRetries); + return results.ToStringArray(); + } + + #endregion + #region Redis Sets public void AddToSet(string key, string[] values) { From 965360468ceb0692f620d616c0b9c373c8a5c224 Mon Sep 17 00:00:00 2001 From: Nir Date: Wed, 3 Apr 2019 15:32:54 +0300 Subject: [PATCH 2/7] Added Script Operations. --- IntegrationTests/LuaScriptingTests.cs | 214 ++++++++++++++++++++ RedisRepo/IRedisContext.cs | 62 ++++++ RedisRepo/RedisContext.cs | 133 ++++++++++++ RedisRepo/RedisScriptKeysAndArguments.cs | 245 +++++++++++++++++++++++ 4 files changed, 654 insertions(+) create mode 100644 IntegrationTests/LuaScriptingTests.cs create mode 100644 RedisRepo/RedisScriptKeysAndArguments.cs diff --git a/IntegrationTests/LuaScriptingTests.cs b/IntegrationTests/LuaScriptingTests.cs new file mode 100644 index 0000000..664d400 --- /dev/null +++ b/IntegrationTests/LuaScriptingTests.cs @@ -0,0 +1,214 @@ +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.Text; +using System.Threading; +using Microsoft.VisualStudio.TestTools.UnitTesting; +using PubComp.RedisRepo.IntegrationTests; + +namespace PubComp.RedisRepo.IntegrationTests +{ + [TestClass] + public class LuaScriptingTests : RedisTests + { + //private RedisConnectionBuilder connectionBuilder; + + //[TestInitialize] + //public void TestInit() + //{ + // IDecrypt crypt = new PlainEncryptor(); + + // this.connectionBuilder = + // new RedisConnectionBuilder( + // decrypt: crypt /* no need, no password in local */, + // //host: "redis-12349.rediscluster.payoneer.com", + // host: "localhost", + // //port: 12349, + // port: 6379, + // //dbId: 0, + // dbId: 13, + // ctxNamespace: "th", + // connectionPoolSize: 1, + // encryptedPassword: "9vvDVxZthgY2z8q9+oSwsKSsdxu8inj41LmDDrJKAIE="); + //} + + [TestInitialize] + public void TestInitialize() + { + redisContext = RedisTestContext.Retry(() + => new RedisTestContext(nameof(RedisTestsNamedContext), db: 1), 5); + ClearDb(redisContext, TestContext); + + redisContext.Delete(TestContext.TestName); + } + + [TestCleanup] + public void TestCleanup() + { + (redisContext as RedisTestContext)?.Connection?.Dispose(); + } + + [TestMethod] + public void TestSimpleScript() + { + const string script = "return 1"; + + var result = redisContext.RunScriptInt(script, null); + + Assert.AreEqual(1, result); + } + + [TestMethod] + public void TestSimpleScriptStringArray() + { + const string script = "return {'one', 'two'}"; + + var result = redisContext.RunScriptStringArray(script, null); + + CollectionAssert.AreEqual(new[] { "one", "two" }, result); + } + + [TestMethod] + public void TestSimpleScriptThatCallsRedis() + { + const string script = "redis.call('set', @Key1, @IntArg1)"; + + var keysAndArgs = redisContext.CreateScriptKeyAndArguments() + .Apply(x => + { + x.Key1 = "myTest"; + x.IntArg1 = 7878; + }); + + redisContext.RunScript(script, keysAndArgs); + + var getResult = redisContext.TryGet("myTest", out int result); + Assert.IsTrue(getResult); + Assert.AreEqual(7878, result); + } + + // key1 - sl window key + // LongArg1 arg 1 - now in miliseconds + // int arg 2 - sliding window size in miliseconds + // string arg 1 - member value + // int arg 3 - limit + + [TestMethod] + public void TestSlidingWindowScript() + { + var r = new Random(); + var baseline = new DateTime(2018, 01, 01); + long toMiliseconds(DateTime dt) + { + return (long)(dt - baseline).TotalMilliseconds; + } + + + const string script = @"redis.call('ZREMRANGEBYSCORE', @Key1, -1, (@LongArg1 - @LongArg2)) +local windowContent = redis.call('ZRANGE', @Key1, 0, -1) +redis.call('ZADD', @Key1, @LongArg1, @StringArg1) +redis.call('EXPIRE', @Key1, @LongArg2 / 1000) +if (tonumber(@IntArg3) >= #windowContent) + then return 0 + else return 1 +end"; + + var keysAndArgs = redisContext.CreateScriptKeyAndArguments() + .Apply(x => + { + //x.Key1 = $"window{r.Next(0, int.MaxValue)}"; + x.Key1 = "EmailSenderProcessCount1"; + x.LongArg2 = 600_000; + x.IntArg3 = 8_000_000; + }); + + + // call the script 4 times + var runs = 3000; + var results = new int[runs]; + var timeCounters = new long[runs]; + var sw = Stopwatch.StartNew(); + for (var i = 0; i < runs; i++) + { + redisContext.RunScriptInt(script, keysAndArgs.Apply(x => + { + x.LongArg1 = toMiliseconds(DateTime.Now); + x.StringArg1 = $"{{'QueueId':'{Guid.NewGuid()}','__rand':'{Guid.NewGuid()}'}}"; + })); + + timeCounters[i] = sw.ElapsedMilliseconds; + + } + + sw.Stop(); + + //assert not more than 5 ms in average + Assert.IsTrue(sw.Elapsed < TimeSpan.FromMilliseconds(runs * 5)); + } + + + + // key1 - sl window key + // LongArg1 arg 1 - now in miliseconds + // int arg 2 - sliding window size in miliseconds + // string arg 1 - member value + // int arg 3 - limit + + [TestMethod] + public void TestSlidingWindowFunctionality() + { + var r = new Random(); + var baseline = new DateTime(2018, 01, 01); + long toMiliseconds(DateTime dt) + { + return (long)(dt - baseline).TotalMilliseconds; + } + + //redis.call('EXPIRE', @Key1, @LongArg2 / 1000) + + const string script = @"redis.call('ZREMRANGEBYSCORE', @Key1, -1, (@LongArg1 - @LongArg2)) +local windowContent = redis.call('ZRANGE', @Key1, 0, -1) +redis.call('ZADD', @Key1, @LongArg1, @StringArg1) +return #windowContent"; + + var windowSizeInSeconds = 5; + var keysAndArgs = redisContext.CreateScriptKeyAndArguments() + .Apply(x => + { + //x.Key1 = $"window{r.Next(0, int.MaxValue)}"; + x.Key1 = $"testB-{r.Next(0, 900_000)}"; + x.LongArg2 = windowSizeInSeconds * 1000; // sliding window size in miliseconds + x.IntArg3 = 8_000_000; // limit + }); + + + const int runs = 10; + var results = new int[runs]; + var expected = new int[runs]; + for (var i = 0; i < runs; i++) + { + expected[i] = i < windowSizeInSeconds ? i + 1 : windowSizeInSeconds; + } + + + for (var i = 0; i < runs; i++) + { + results[i] = redisContext.RunScriptInt(script, keysAndArgs.Apply(x => + { + x.LongArg1 = toMiliseconds(DateTime.Now); + x.StringArg1 = $"{{ 'a': '{Guid.NewGuid()}' }}"; + })); + + Thread.Sleep(1000); + } + + // assert results + for (var i = 0; i < runs; i++) + { + Assert.AreEqual(expected[i], results[i]); + } + + } + } +} + diff --git a/RedisRepo/IRedisContext.cs b/RedisRepo/IRedisContext.cs index 93d6f12..2ca165e 100644 --- a/RedisRepo/IRedisContext.cs +++ b/RedisRepo/IRedisContext.cs @@ -1,5 +1,6 @@ using System; using System.Collections.Generic; +using PubComp.RedisRepo.Payoneer.Labs.Throttling.Common.Redis; using StackExchange.Redis; namespace PubComp.RedisRepo @@ -119,5 +120,66 @@ public interface IRedisContext bool TryGetDistributedLock(string lockObjectName, string lockerName, TimeSpan lockTtl); #endregion + + #region Lua Scripting + + RedisScriptKeysAndArguments CreateScriptKeyAndArguments(); + + /// + /// Run a lua script against the connected redis instance + /// + /// the script to run. Keys should be @Key1, @Key2 ... @Key10. Int arguments: @IntArg1 .. @IntArg20. String arguments: @StringArg1 .. @StringArg20 + /// an instance of RedisScriptKeysAndArguments + void RunScript(string script, RedisScriptKeysAndArguments keysAndParameters); + + /// + /// Run a lua script against the connected redis instance + /// + /// the script to run. Keys should be @Key1, @Key2 ... @Key10. Int arguments: @IntArg1 .. @IntArg20. String arguments: @StringArg1 .. @StringArg20 + /// an instance of RedisScriptKeysAndArguments + /// result as string + string RunScriptString(string script, RedisScriptKeysAndArguments keysAndParameters); + + /// + /// Run a lua script against the connected redis instance + /// + /// the script to run. Keys should be @Key1, @Key2 ... @Key10. Int arguments: @IntArg1 .. @IntArg20. String arguments: @StringArg1 .. @StringArg20 + /// an instance of RedisScriptKeysAndArguments + /// result as int + int RunScriptInt(string script, RedisScriptKeysAndArguments keysAndParameters); + + /// + /// Run a lua script against the connected redis instance + /// + /// the script to run. Keys should be @Key1, @Key2 ... @Key10. Int arguments: @IntArg1 .. @IntArg20. String arguments: @StringArg1 .. @StringArg20 + /// an instance of RedisScriptKeysAndArguments + /// result as string + long RunScriptLong(string script, RedisScriptKeysAndArguments keysAndParameters); + + /// + /// Run a lua script against the connected redis instance + /// + /// the script to run. Keys should be @Key1, @Key2 ... @Key10. Int arguments: @IntArg1 .. @IntArg20. String arguments: @StringArg1 .. @StringArg20 + /// an instance of RedisScriptKeysAndArguments + /// result as double + double RunScriptDouble(string script, RedisScriptKeysAndArguments keysAndParameters); + + /// + /// Run a lua script against the connected redis instance + /// + /// the script to run. Keys should be @Key1, @Key2 ... @Key10. Int arguments: @IntArg1 .. @IntArg20. String arguments: @StringArg1 .. @StringArg20 + /// an instance of RedisScriptKeysAndArguments + /// result as byte array + byte[] RunScriptByteArray(string script, RedisScriptKeysAndArguments keysAndParameters); + + /// + /// Run a lua script against the connected redis instance + /// + /// the script to run. Keys should be @Key1, @Key2 ... @Key10. Int arguments: @IntArg1 .. @IntArg20. String arguments: @StringArg1 .. @StringArg20 + /// an instance of RedisScriptKeysAndArguments + /// result as string[] + string[] RunScriptStringArray(string script, RedisScriptKeysAndArguments keysAndParameters); + + #endregion } } \ No newline at end of file diff --git a/RedisRepo/RedisContext.cs b/RedisRepo/RedisContext.cs index bd403e0..c15b174 100644 --- a/RedisRepo/RedisContext.cs +++ b/RedisRepo/RedisContext.cs @@ -1,8 +1,10 @@ using System; +using System.Collections.Concurrent; using System.Collections.Generic; using System.Linq; using System.Threading; using NLog; +using PubComp.RedisRepo.Payoneer.Labs.Throttling.Common.Redis; using StackExchange.Redis; namespace PubComp.RedisRepo @@ -23,6 +25,8 @@ public class RedisContext : IRedisContext internal readonly CommandFlags commandFlags; private readonly int totalConnections; + private ConcurrentDictionary loadedScripts = new ConcurrentDictionary(); + public RedisContext( string contextNamespace, string host = "localhost", int port = 6379, string password = null, int db = 0) @@ -732,6 +736,135 @@ public IEnumerable GetKeys(string pattern = null) #endregion + #region Lua Scripting + + public RedisScriptKeysAndArguments CreateScriptKeyAndArguments() + { + return new RedisScriptKeysAndArguments(Key); + } + + /// + /// Run a lua script against the connected redis instance + /// + /// the script to run. Keys should be @Key1, @Key2 ... @Key10. Int arguments: @IntArg1 .. @IntArg20. String arguments: @StringArg1 .. @StringArg20 + /// an instance of RedisScriptKeysAndArguments + public void RunScript(string script, RedisScriptKeysAndArguments keysAndParameters) + { + Retry(() => this.RunScriptInternal(script, keysAndParameters), defaultRetries); + } + + /// + /// Run a lua script against the connected redis instance + /// + /// the script to run. Keys should be @Key1, @Key2 ... @Key10. Int arguments: @IntArg1 .. @IntArg20. String arguments: @StringArg1 .. @StringArg20 + /// an instance of RedisScriptKeysAndArguments + /// result as string + public string RunScriptString(string script, RedisScriptKeysAndArguments keysAndParameters) + { + var result = Retry(() => this.RunScriptInternal(script, keysAndParameters), defaultRetries); + return (string)result; + } + + /// + /// Run a lua script against the connected redis instance + /// + /// the script to run. Keys should be @Key1, @Key2 ... @Key10. Int arguments: @IntArg1 .. @IntArg20. String arguments: @StringArg1 .. @StringArg20 + /// an instance of RedisScriptKeysAndArguments + /// result as int + public int RunScriptInt(string script, RedisScriptKeysAndArguments keysAndParameters) + { + var result = Retry(() => this.RunScriptInternal(script, keysAndParameters), defaultRetries); + + return (int)result; + } + + /// + /// Run a lua script against the connected redis instance + /// + /// the script to run. Keys should be @Key1, @Key2 ... @Key10. Int arguments: @IntArg1 .. @IntArg20. String arguments: @StringArg1 .. @StringArg20 + /// an instance of RedisScriptKeysAndArguments + /// result as string + public long RunScriptLong(string script, RedisScriptKeysAndArguments keysAndParameters) + { + var result = Retry(() => this.RunScriptInternal(script, keysAndParameters), defaultRetries); + + return (long)result; + } + + /// + /// Run a lua script against the connected redis instance + /// + /// the script to run. Keys should be @Key1, @Key2 ... @Key10. Int arguments: @IntArg1 .. @IntArg20. String arguments: @StringArg1 .. @StringArg20 + /// an instance of RedisScriptKeysAndArguments + /// result as double + public double RunScriptDouble(string script, RedisScriptKeysAndArguments keysAndParameters) + { + var result = Retry(() => this.RunScriptInternal(script, keysAndParameters), defaultRetries); + + return (double)result; + } + + /// + /// Run a lua script against the connected redis instance + /// + /// the script to run. Keys should be @Key1, @Key2 ... @Key10. Int arguments: @IntArg1 .. @IntArg20. String arguments: @StringArg1 .. @StringArg20 + /// an instance of RedisScriptKeysAndArguments + /// result as byte array + public byte[] RunScriptByteArray(string script, RedisScriptKeysAndArguments keysAndParameters) + { + var result = Retry(() => this.RunScriptInternal(script, keysAndParameters), defaultRetries); + + return (byte[])result; + } + + /// + /// Run a lua script against the connected redis instance + /// + /// the script to run. Keys should be @Key1, @Key2 ... @Key10. Int arguments: @IntArg1 .. @IntArg20. String arguments: @StringArg1 .. @StringArg20 + /// an instance of RedisScriptKeysAndArguments + /// result as string[] + public string[] RunScriptStringArray(string script, RedisScriptKeysAndArguments keysAndParameters) + { + var result = Retry(() => this.RunScriptInternal(script, keysAndParameters), defaultRetries); + + return (string[])result; + } + + /// + /// Run a lua script against the connected redis instance + /// + /// the script to run. Keys should be @Key1, @Key2 ... @Key10. Int arguments: @IntArg1 .. @IntArg20. String arguments: @StringArg1 .. @StringArg20 + /// an instance of RedisScriptKeysAndArguments + /// + private RedisResult RunScriptInternal(string script, RedisScriptKeysAndArguments keysAndParameters) + { + var conn = this.Connection; + var db = conn.GetDatabase(this.DatabaseNumber); + + if (!this.loadedScripts.TryGetValue(script, out var loadedLuaScript)) + { + var server = conn.GetServer(hosts.First()); + var prepared = LuaScript.Prepare(script); + this.loadedScripts[script] = loadedLuaScript = prepared.Load(server); + } + + try + { + return loadedLuaScript.Evaluate(db, keysAndParameters); + } + catch (RedisServerException) + { + // TODO: validate that the message is NOSCRIPT + var server = conn.GetServer(hosts.First()); + var prepared = LuaScript.Prepare(script); + this.loadedScripts[script] = loadedLuaScript = prepared.Load(server); + + // run + return loadedLuaScript.Evaluate(db, keysAndParameters); + } + } + + #endregion public void CloseConnections() { diff --git a/RedisRepo/RedisScriptKeysAndArguments.cs b/RedisRepo/RedisScriptKeysAndArguments.cs new file mode 100644 index 0000000..f3644e6 --- /dev/null +++ b/RedisRepo/RedisScriptKeysAndArguments.cs @@ -0,0 +1,245 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using StackExchange.Redis; + +namespace PubComp.RedisRepo +{ + namespace Payoneer.Labs.Throttling.Common.Redis + { + public class RedisScriptKeysAndArguments + { + private readonly Func keyConverter; + + internal RedisScriptKeysAndArguments(Func keyConverter = null) + { + this.keyConverter = keyConverter ?? (x => x); + + } + + #region Keys + + public RedisScriptKeysAndArguments Apply(Action action) + { + action(this); + return this; + } + + private RedisKey _key1; + + public RedisKey Key1 + { + get => this._key1; + set => this._key1 = this.keyConverter(value); + } + + private RedisKey _key2; + + public RedisKey Key2 + { + get => this._key2; + set => this._key2 = this.keyConverter(value); + } + + private RedisKey _key3; + + public RedisKey Key3 + { + get => this._key3; + set => this._key3 = this.keyConverter(value); + } + + private RedisKey _key4; + + public RedisKey Key4 + { + get => this._key4; + set => this._key4 = this.keyConverter(value); + } + + private RedisKey _key5; + + public RedisKey Key5 + { + get => this._key5; + set => this._key5 = this.keyConverter(value); + } + + private RedisKey _key6; + + public RedisKey Key6 + { + get => this._key6; + set => this._key6 = this.keyConverter(value); + } + + private RedisKey _key7; + + public RedisKey Key7 + { + get => this._key7; + set => this._key7 = this.keyConverter(value); + } + + private RedisKey _key8; + + public RedisKey Key8 + { + get => this._key8; + set => this._key8 = this.keyConverter(value); + } + + private RedisKey _key9; + + public RedisKey Key9 + { + get => this._key9; + set => this._key9 = this.keyConverter(value); + } + + private RedisKey _key10; + + public RedisKey Key10 + { + get => this._key10; + set => this._key10 = this.keyConverter(value); + } + + #endregion + + #region Int Arguments + + public int IntArg1 { get; set; } + public int IntArg2 { get; set; } + public int IntArg3 { get; set; } + public int IntArg4 { get; set; } + public int IntArg5 { get; set; } + public int IntArg6 { get; set; } + public int IntArg7 { get; set; } + public int IntArg8 { get; set; } + public int IntArg9 { get; set; } + public int IntArg10 { get; set; } + public int IntArg11 { get; set; } + public int IntArg12 { get; set; } + public int IntArg13 { get; set; } + public int IntArg14 { get; set; } + public int IntArg15 { get; set; } + public int IntArg16 { get; set; } + public int IntArg17 { get; set; } + public int IntArg18 { get; set; } + public int IntArg19 { get; set; } + public int IntArg20 { get; set; } + + #endregion + + #region Long Arguments + + public long LongArg1 { get; set; } + public long LongArg2 { get; set; } + public long LongArg3 { get; set; } + public long LongArg4 { get; set; } + public long LongArg5 { get; set; } + public long LongArg6 { get; set; } + public long LongArg7 { get; set; } + public long LongArg8 { get; set; } + public long LongArg9 { get; set; } + public long LongArg10 { get; set; } + public long LongArg11 { get; set; } + public long LongArg12 { get; set; } + public long LongArg13 { get; set; } + public long LongArg14 { get; set; } + public long LongArg15 { get; set; } + public long LongArg16 { get; set; } + public long LongArg17 { get; set; } + public long LongArg18 { get; set; } + public long LongArg19 { get; set; } + public long LongArg20 { get; set; } + + #endregion + + #region String Arguments + + public string StringArg1 { get; set; } + public string StringArg2 { get; set; } + public string StringArg3 { get; set; } + public string StringArg4 { get; set; } + public string StringArg5 { get; set; } + public string StringArg6 { get; set; } + public string StringArg7 { get; set; } + public string StringArg8 { get; set; } + public string StringArg9 { get; set; } + public string StringArg10 { get; set; } + public string StringArg11 { get; set; } + public string StringArg12 { get; set; } + public string StringArg13 { get; set; } + public string StringArg14 { get; set; } + public string StringArg15 { get; set; } + public string StringArg16 { get; set; } + public string StringArg17 { get; set; } + public string StringArg18 { get; set; } + public string StringArg19 { get; set; } + public string StringArg20 { get; set; } + + #endregion + + public void SetKeys(IList keysInOrder) + { + if (keysInOrder == null || !keysInOrder.Any()) + { + return; + } + + var setters = this.GetType().GetProperties().Where(p => p.Name.StartsWith("Key")) + .Select(p => p.SetMethod).ToList(); + + for (var i = 1; i <= 20; i++) + { + if (keysInOrder.Count < i) + { + break; + } + + var keyValue = keysInOrder[i - 1]; + var i1 = i; + setters.FirstOrDefault(x => x.Name == $"set_Key{i1}") + ?.Invoke(this, new object[] { (RedisKey)$"ns=amit:k={keyValue}" }); + } + } + + public void SetStringArguments(IList argumentsInOrder) + { + SetArguments(argumentsInOrder, "String"); + } + + public void SetIntArguments(IList argumentsInOrder) + { + SetArguments(argumentsInOrder, "Int"); + } + + private void SetArguments(IList argumentsInOrder, string typePrefix) + { + if (argumentsInOrder == null || !argumentsInOrder.Any()) + { + return; + } + + var prefix = $"{typePrefix}Arg"; + var setters = this.GetType().GetProperties().Where(p => p.Name.StartsWith(prefix)) + .Select(p => p.SetMethod).ToList(); + + for (var i = 1; i <= 20; i++) + { + if (argumentsInOrder.Count < i) + { + break; + } + + var arg = argumentsInOrder[i - 1]; + var i1 = i; + setters.FirstOrDefault(x => x.Name == $"set_{prefix}{i1}")?.Invoke(this, new[] { arg }); + } + } + } + } +} \ No newline at end of file From c7f2cd1382d6615862fc440216078543b8a248d5 Mon Sep 17 00:00:00 2001 From: Nir Date: Tue, 16 Apr 2019 15:30:36 +0300 Subject: [PATCH 3/7] Moved the lua tests to the redis tests class. --- IntegrationTests/LuaScriptingTests.cs | 214 -------------------------- IntegrationTests/RedisTests.cs | 169 +++++++++++++++++++- IntegrationTests/UnitTest1.cs | 13 -- RedisRepo/RedisRepo.csproj | 5 +- 4 files changed, 170 insertions(+), 231 deletions(-) delete mode 100644 IntegrationTests/LuaScriptingTests.cs delete mode 100644 IntegrationTests/UnitTest1.cs diff --git a/IntegrationTests/LuaScriptingTests.cs b/IntegrationTests/LuaScriptingTests.cs deleted file mode 100644 index 664d400..0000000 --- a/IntegrationTests/LuaScriptingTests.cs +++ /dev/null @@ -1,214 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Diagnostics; -using System.Text; -using System.Threading; -using Microsoft.VisualStudio.TestTools.UnitTesting; -using PubComp.RedisRepo.IntegrationTests; - -namespace PubComp.RedisRepo.IntegrationTests -{ - [TestClass] - public class LuaScriptingTests : RedisTests - { - //private RedisConnectionBuilder connectionBuilder; - - //[TestInitialize] - //public void TestInit() - //{ - // IDecrypt crypt = new PlainEncryptor(); - - // this.connectionBuilder = - // new RedisConnectionBuilder( - // decrypt: crypt /* no need, no password in local */, - // //host: "redis-12349.rediscluster.payoneer.com", - // host: "localhost", - // //port: 12349, - // port: 6379, - // //dbId: 0, - // dbId: 13, - // ctxNamespace: "th", - // connectionPoolSize: 1, - // encryptedPassword: "9vvDVxZthgY2z8q9+oSwsKSsdxu8inj41LmDDrJKAIE="); - //} - - [TestInitialize] - public void TestInitialize() - { - redisContext = RedisTestContext.Retry(() - => new RedisTestContext(nameof(RedisTestsNamedContext), db: 1), 5); - ClearDb(redisContext, TestContext); - - redisContext.Delete(TestContext.TestName); - } - - [TestCleanup] - public void TestCleanup() - { - (redisContext as RedisTestContext)?.Connection?.Dispose(); - } - - [TestMethod] - public void TestSimpleScript() - { - const string script = "return 1"; - - var result = redisContext.RunScriptInt(script, null); - - Assert.AreEqual(1, result); - } - - [TestMethod] - public void TestSimpleScriptStringArray() - { - const string script = "return {'one', 'two'}"; - - var result = redisContext.RunScriptStringArray(script, null); - - CollectionAssert.AreEqual(new[] { "one", "two" }, result); - } - - [TestMethod] - public void TestSimpleScriptThatCallsRedis() - { - const string script = "redis.call('set', @Key1, @IntArg1)"; - - var keysAndArgs = redisContext.CreateScriptKeyAndArguments() - .Apply(x => - { - x.Key1 = "myTest"; - x.IntArg1 = 7878; - }); - - redisContext.RunScript(script, keysAndArgs); - - var getResult = redisContext.TryGet("myTest", out int result); - Assert.IsTrue(getResult); - Assert.AreEqual(7878, result); - } - - // key1 - sl window key - // LongArg1 arg 1 - now in miliseconds - // int arg 2 - sliding window size in miliseconds - // string arg 1 - member value - // int arg 3 - limit - - [TestMethod] - public void TestSlidingWindowScript() - { - var r = new Random(); - var baseline = new DateTime(2018, 01, 01); - long toMiliseconds(DateTime dt) - { - return (long)(dt - baseline).TotalMilliseconds; - } - - - const string script = @"redis.call('ZREMRANGEBYSCORE', @Key1, -1, (@LongArg1 - @LongArg2)) -local windowContent = redis.call('ZRANGE', @Key1, 0, -1) -redis.call('ZADD', @Key1, @LongArg1, @StringArg1) -redis.call('EXPIRE', @Key1, @LongArg2 / 1000) -if (tonumber(@IntArg3) >= #windowContent) - then return 0 - else return 1 -end"; - - var keysAndArgs = redisContext.CreateScriptKeyAndArguments() - .Apply(x => - { - //x.Key1 = $"window{r.Next(0, int.MaxValue)}"; - x.Key1 = "EmailSenderProcessCount1"; - x.LongArg2 = 600_000; - x.IntArg3 = 8_000_000; - }); - - - // call the script 4 times - var runs = 3000; - var results = new int[runs]; - var timeCounters = new long[runs]; - var sw = Stopwatch.StartNew(); - for (var i = 0; i < runs; i++) - { - redisContext.RunScriptInt(script, keysAndArgs.Apply(x => - { - x.LongArg1 = toMiliseconds(DateTime.Now); - x.StringArg1 = $"{{'QueueId':'{Guid.NewGuid()}','__rand':'{Guid.NewGuid()}'}}"; - })); - - timeCounters[i] = sw.ElapsedMilliseconds; - - } - - sw.Stop(); - - //assert not more than 5 ms in average - Assert.IsTrue(sw.Elapsed < TimeSpan.FromMilliseconds(runs * 5)); - } - - - - // key1 - sl window key - // LongArg1 arg 1 - now in miliseconds - // int arg 2 - sliding window size in miliseconds - // string arg 1 - member value - // int arg 3 - limit - - [TestMethod] - public void TestSlidingWindowFunctionality() - { - var r = new Random(); - var baseline = new DateTime(2018, 01, 01); - long toMiliseconds(DateTime dt) - { - return (long)(dt - baseline).TotalMilliseconds; - } - - //redis.call('EXPIRE', @Key1, @LongArg2 / 1000) - - const string script = @"redis.call('ZREMRANGEBYSCORE', @Key1, -1, (@LongArg1 - @LongArg2)) -local windowContent = redis.call('ZRANGE', @Key1, 0, -1) -redis.call('ZADD', @Key1, @LongArg1, @StringArg1) -return #windowContent"; - - var windowSizeInSeconds = 5; - var keysAndArgs = redisContext.CreateScriptKeyAndArguments() - .Apply(x => - { - //x.Key1 = $"window{r.Next(0, int.MaxValue)}"; - x.Key1 = $"testB-{r.Next(0, 900_000)}"; - x.LongArg2 = windowSizeInSeconds * 1000; // sliding window size in miliseconds - x.IntArg3 = 8_000_000; // limit - }); - - - const int runs = 10; - var results = new int[runs]; - var expected = new int[runs]; - for (var i = 0; i < runs; i++) - { - expected[i] = i < windowSizeInSeconds ? i + 1 : windowSizeInSeconds; - } - - - for (var i = 0; i < runs; i++) - { - results[i] = redisContext.RunScriptInt(script, keysAndArgs.Apply(x => - { - x.LongArg1 = toMiliseconds(DateTime.Now); - x.StringArg1 = $"{{ 'a': '{Guid.NewGuid()}' }}"; - })); - - Thread.Sleep(1000); - } - - // assert results - for (var i = 0; i < runs; i++) - { - Assert.AreEqual(expected[i], results[i]); - } - - } - } -} - diff --git a/IntegrationTests/RedisTests.cs b/IntegrationTests/RedisTests.cs index 9d7eeba..776d082 100644 --- a/IntegrationTests/RedisTests.cs +++ b/IntegrationTests/RedisTests.cs @@ -1,4 +1,5 @@ using System; +using System.Diagnostics; using System.Linq; using System.Net; @@ -487,8 +488,8 @@ public void TestAddToRedisList() { const string key = "TestAddToList"; var values = new[] { "bar", "bar", "a", "b", "c" }; - - for (int i = 0; i < values.Length; i++) + + for (var i = 0; i < values.Length; i++) { var length = redisContext.AddToList(key, values[i]); Assert.AreEqual(i + 1, length); @@ -644,6 +645,170 @@ public void TestSetContains(string valueToAdd, string searchForValue, bool expec #endregion + #region scripting tests + + [TestMethod] + public void TestSimpleScript() + { + const string script = "return 1"; + + var result = redisContext.RunScriptInt(script, null); + + Assert.AreEqual(1, result); + } + + [TestMethod] + public void TestSimpleScriptStringArray() + { + const string script = "return {'one', 'two'}"; + + var result = redisContext.RunScriptStringArray(script, null); + + CollectionAssert.AreEqual(new[] { "one", "two" }, result); + } + + [TestMethod] + public void TestSimpleScriptThatCallsRedis() + { + const string script = "redis.call('set', @Key1, @IntArg1)"; + + var keysAndArgs = redisContext.CreateScriptKeyAndArguments() + .Apply(x => + { + x.Key1 = "myTest"; + x.IntArg1 = 7878; + }); + + redisContext.RunScript(script, keysAndArgs); + + var getResult = redisContext.TryGet("myTest", out int result); + Assert.IsTrue(getResult); + Assert.AreEqual(7878, result); + } + + // key1 - sl window key + // LongArg1 arg 1 - now in miliseconds + // int arg 2 - sliding window size in miliseconds + // string arg 1 - member value + // int arg 3 - limit + + [TestMethod] + public void TestSlidingWindowScript() + { + var r = new Random(); + var baseline = new DateTime(2018, 01, 01); + long toMiliseconds(DateTime dt) + { + return (long)(dt - baseline).TotalMilliseconds; + } + + + const string script = @"redis.call('ZREMRANGEBYSCORE', @Key1, -1, (@LongArg1 - @LongArg2)) +local windowContent = redis.call('ZRANGE', @Key1, 0, -1) +redis.call('ZADD', @Key1, @LongArg1, @StringArg1) +redis.call('EXPIRE', @Key1, @LongArg2 / 1000) +if (tonumber(@IntArg3) >= #windowContent) + then return 0 + else return 1 +end"; + + var keysAndArgs = redisContext.CreateScriptKeyAndArguments() + .Apply(x => + { + //x.Key1 = $"window{r.Next(0, int.MaxValue)}"; + x.Key1 = "EmailSenderProcessCount1"; + x.LongArg2 = 600_000; + x.IntArg3 = 8_000_000; + }); + + + // call the script 4 times + var runs = 3000; + var results = new int[runs]; + var timeCounters = new long[runs]; + var sw = Stopwatch.StartNew(); + for (var i = 0; i < runs; i++) + { + redisContext.RunScriptInt(script, keysAndArgs.Apply(x => + { + x.LongArg1 = toMiliseconds(DateTime.Now); + x.StringArg1 = $"{{'QueueId':'{Guid.NewGuid()}','__rand':'{Guid.NewGuid()}'}}"; + })); + + timeCounters[i] = sw.ElapsedMilliseconds; + + } + + sw.Stop(); + + //assert not more than 5 ms in average + Assert.IsTrue(sw.Elapsed < TimeSpan.FromMilliseconds(runs * 5)); + } + + + + // key1 - sl window key + // LongArg1 arg 1 - now in miliseconds + // int arg 2 - sliding window size in miliseconds + // string arg 1 - member value + // int arg 3 - limit + + [TestMethod] + public void TestSlidingWindowFunctionality() + { + var r = new Random(); + var baseline = new DateTime(2018, 01, 01); + long toMiliseconds(DateTime dt) + { + return (long)(dt - baseline).TotalMilliseconds; + } + + //redis.call('EXPIRE', @Key1, @LongArg2 / 1000) + + const string script = @"redis.call('ZREMRANGEBYSCORE', @Key1, -1, (@LongArg1 - @LongArg2)) +local windowContent = redis.call('ZRANGE', @Key1, 0, -1) +redis.call('ZADD', @Key1, @LongArg1, @StringArg1) +return (#windowContent + 1)"; + + var windowSizeInSeconds = 5; + var keysAndArgs = redisContext.CreateScriptKeyAndArguments() + .Apply(x => + { + //x.Key1 = $"window{r.Next(0, int.MaxValue)}"; + x.Key1 = $"testB-{r.Next(0, 900_000)}"; + x.LongArg2 = windowSizeInSeconds * 1000; // sliding window size in miliseconds + }); + + + const int runs = 10; + var results = new int[runs]; + var expected = new int[runs]; + for (var i = 0; i < runs; i++) + { + expected[i] = i < windowSizeInSeconds ? i + 1 : windowSizeInSeconds; + } + + + for (var i = 0; i < runs; i++) + { + results[i] = redisContext.RunScriptInt(script, keysAndArgs.Apply(x => + { + x.LongArg1 = toMiliseconds(DateTime.Now); + x.StringArg1 = $"{{ 'a': '{Guid.NewGuid()}' }}"; + })); + + Thread.Sleep(1000); + } + + // assert results + for (var i = 0; i < runs; i++) + { + Assert.AreEqual(expected[i], results[i]); + } + + } + #endregion + #endregion private void Set(string key, TData value, TimeSpan? ttl = null) diff --git a/IntegrationTests/UnitTest1.cs b/IntegrationTests/UnitTest1.cs deleted file mode 100644 index f5a2efd..0000000 --- a/IntegrationTests/UnitTest1.cs +++ /dev/null @@ -1,13 +0,0 @@ -using Microsoft.VisualStudio.TestTools.UnitTesting; - -namespace IntegrationTests -{ - [TestClass] - public class UnitTest1 - { - [TestMethod] - public void TestMethod1() - { - } - } -} diff --git a/RedisRepo/RedisRepo.csproj b/RedisRepo/RedisRepo.csproj index e525750..c18e536 100644 --- a/RedisRepo/RedisRepo.csproj +++ b/RedisRepo/RedisRepo.csproj @@ -6,8 +6,9 @@ PubComp.RedisRepo true true - 4.0.0.0 - 4.0.0 + 4.0.0.0 + 4.1.0.0 + 4.1.0 true true true From be1c1d4858aca258ca25dfcdde182cbc0b8ac483 Mon Sep 17 00:00:00 2001 From: Nir Date: Wed, 24 Apr 2019 20:09:02 +0300 Subject: [PATCH 4/7] Added a summary for a function, and removed accidental namespace. --- RedisRepo/IRedisContext.cs | 4 +- RedisRepo/RedisContext.cs | 4 +- RedisRepo/RedisRepo.csproj | 4 +- RedisRepo/RedisScriptKeysAndArguments.cs | 377 +++++++++++------------ 4 files changed, 195 insertions(+), 194 deletions(-) diff --git a/RedisRepo/IRedisContext.cs b/RedisRepo/IRedisContext.cs index 2ca165e..0f5c448 100644 --- a/RedisRepo/IRedisContext.cs +++ b/RedisRepo/IRedisContext.cs @@ -1,6 +1,5 @@ using System; using System.Collections.Generic; -using PubComp.RedisRepo.Payoneer.Labs.Throttling.Common.Redis; using StackExchange.Redis; namespace PubComp.RedisRepo @@ -123,6 +122,9 @@ public interface IRedisContext #region Lua Scripting + /// + /// Return a RedisScriptKeysAndArguments instance that can be passed later alongside a script + /// RedisScriptKeysAndArguments CreateScriptKeyAndArguments(); /// diff --git a/RedisRepo/RedisContext.cs b/RedisRepo/RedisContext.cs index c15b174..2b97472 100644 --- a/RedisRepo/RedisContext.cs +++ b/RedisRepo/RedisContext.cs @@ -4,7 +4,6 @@ using System.Linq; using System.Threading; using NLog; -using PubComp.RedisRepo.Payoneer.Labs.Throttling.Common.Redis; using StackExchange.Redis; namespace PubComp.RedisRepo @@ -738,6 +737,9 @@ public IEnumerable GetKeys(string pattern = null) #region Lua Scripting + /// + /// Return a RedisScriptKeysAndArguments instance that can be passed later alongside a script + /// public RedisScriptKeysAndArguments CreateScriptKeyAndArguments() { return new RedisScriptKeysAndArguments(Key); diff --git a/RedisRepo/RedisRepo.csproj b/RedisRepo/RedisRepo.csproj index c18e536..07395ef 100644 --- a/RedisRepo/RedisRepo.csproj +++ b/RedisRepo/RedisRepo.csproj @@ -7,8 +7,8 @@ true true 4.0.0.0 - 4.1.0.0 - 4.1.0 + 4.1.1.0 + 4.1.1 true true true diff --git a/RedisRepo/RedisScriptKeysAndArguments.cs b/RedisRepo/RedisScriptKeysAndArguments.cs index f3644e6..7899de7 100644 --- a/RedisRepo/RedisScriptKeysAndArguments.cs +++ b/RedisRepo/RedisScriptKeysAndArguments.cs @@ -6,239 +6,236 @@ namespace PubComp.RedisRepo { - namespace Payoneer.Labs.Throttling.Common.Redis + public class RedisScriptKeysAndArguments { - public class RedisScriptKeysAndArguments + private readonly Func keyConverter; + + internal RedisScriptKeysAndArguments(Func keyConverter = null) { - private readonly Func keyConverter; + this.keyConverter = keyConverter ?? (x => x); - internal RedisScriptKeysAndArguments(Func keyConverter = null) - { - this.keyConverter = keyConverter ?? (x => x); + } - } + #region Keys - #region Keys + public RedisScriptKeysAndArguments Apply(Action action) + { + action(this); + return this; + } - public RedisScriptKeysAndArguments Apply(Action action) - { - action(this); - return this; - } + private RedisKey _key1; - private RedisKey _key1; + public RedisKey Key1 + { + get => this._key1; + set => this._key1 = this.keyConverter(value); + } - public RedisKey Key1 - { - get => this._key1; - set => this._key1 = this.keyConverter(value); - } + private RedisKey _key2; - private RedisKey _key2; + public RedisKey Key2 + { + get => this._key2; + set => this._key2 = this.keyConverter(value); + } - public RedisKey Key2 - { - get => this._key2; - set => this._key2 = this.keyConverter(value); - } + private RedisKey _key3; - private RedisKey _key3; + public RedisKey Key3 + { + get => this._key3; + set => this._key3 = this.keyConverter(value); + } - public RedisKey Key3 - { - get => this._key3; - set => this._key3 = this.keyConverter(value); - } + private RedisKey _key4; - private RedisKey _key4; + public RedisKey Key4 + { + get => this._key4; + set => this._key4 = this.keyConverter(value); + } - public RedisKey Key4 - { - get => this._key4; - set => this._key4 = this.keyConverter(value); - } + private RedisKey _key5; - private RedisKey _key5; + public RedisKey Key5 + { + get => this._key5; + set => this._key5 = this.keyConverter(value); + } - public RedisKey Key5 - { - get => this._key5; - set => this._key5 = this.keyConverter(value); - } + private RedisKey _key6; - private RedisKey _key6; + public RedisKey Key6 + { + get => this._key6; + set => this._key6 = this.keyConverter(value); + } - public RedisKey Key6 - { - get => this._key6; - set => this._key6 = this.keyConverter(value); - } + private RedisKey _key7; - private RedisKey _key7; + public RedisKey Key7 + { + get => this._key7; + set => this._key7 = this.keyConverter(value); + } - public RedisKey Key7 - { - get => this._key7; - set => this._key7 = this.keyConverter(value); - } + private RedisKey _key8; - private RedisKey _key8; + public RedisKey Key8 + { + get => this._key8; + set => this._key8 = this.keyConverter(value); + } - public RedisKey Key8 - { - get => this._key8; - set => this._key8 = this.keyConverter(value); - } + private RedisKey _key9; - private RedisKey _key9; + public RedisKey Key9 + { + get => this._key9; + set => this._key9 = this.keyConverter(value); + } - public RedisKey Key9 - { - get => this._key9; - set => this._key9 = this.keyConverter(value); - } + private RedisKey _key10; - private RedisKey _key10; + public RedisKey Key10 + { + get => this._key10; + set => this._key10 = this.keyConverter(value); + } - public RedisKey Key10 + #endregion + + #region Int Arguments + + public int IntArg1 { get; set; } + public int IntArg2 { get; set; } + public int IntArg3 { get; set; } + public int IntArg4 { get; set; } + public int IntArg5 { get; set; } + public int IntArg6 { get; set; } + public int IntArg7 { get; set; } + public int IntArg8 { get; set; } + public int IntArg9 { get; set; } + public int IntArg10 { get; set; } + public int IntArg11 { get; set; } + public int IntArg12 { get; set; } + public int IntArg13 { get; set; } + public int IntArg14 { get; set; } + public int IntArg15 { get; set; } + public int IntArg16 { get; set; } + public int IntArg17 { get; set; } + public int IntArg18 { get; set; } + public int IntArg19 { get; set; } + public int IntArg20 { get; set; } + + #endregion + + #region Long Arguments + + public long LongArg1 { get; set; } + public long LongArg2 { get; set; } + public long LongArg3 { get; set; } + public long LongArg4 { get; set; } + public long LongArg5 { get; set; } + public long LongArg6 { get; set; } + public long LongArg7 { get; set; } + public long LongArg8 { get; set; } + public long LongArg9 { get; set; } + public long LongArg10 { get; set; } + public long LongArg11 { get; set; } + public long LongArg12 { get; set; } + public long LongArg13 { get; set; } + public long LongArg14 { get; set; } + public long LongArg15 { get; set; } + public long LongArg16 { get; set; } + public long LongArg17 { get; set; } + public long LongArg18 { get; set; } + public long LongArg19 { get; set; } + public long LongArg20 { get; set; } + + #endregion + + #region String Arguments + + public string StringArg1 { get; set; } + public string StringArg2 { get; set; } + public string StringArg3 { get; set; } + public string StringArg4 { get; set; } + public string StringArg5 { get; set; } + public string StringArg6 { get; set; } + public string StringArg7 { get; set; } + public string StringArg8 { get; set; } + public string StringArg9 { get; set; } + public string StringArg10 { get; set; } + public string StringArg11 { get; set; } + public string StringArg12 { get; set; } + public string StringArg13 { get; set; } + public string StringArg14 { get; set; } + public string StringArg15 { get; set; } + public string StringArg16 { get; set; } + public string StringArg17 { get; set; } + public string StringArg18 { get; set; } + public string StringArg19 { get; set; } + public string StringArg20 { get; set; } + + #endregion + + public void SetKeys(IList keysInOrder) + { + if (keysInOrder == null || !keysInOrder.Any()) { - get => this._key10; - set => this._key10 = this.keyConverter(value); + return; } - #endregion - - #region Int Arguments - - public int IntArg1 { get; set; } - public int IntArg2 { get; set; } - public int IntArg3 { get; set; } - public int IntArg4 { get; set; } - public int IntArg5 { get; set; } - public int IntArg6 { get; set; } - public int IntArg7 { get; set; } - public int IntArg8 { get; set; } - public int IntArg9 { get; set; } - public int IntArg10 { get; set; } - public int IntArg11 { get; set; } - public int IntArg12 { get; set; } - public int IntArg13 { get; set; } - public int IntArg14 { get; set; } - public int IntArg15 { get; set; } - public int IntArg16 { get; set; } - public int IntArg17 { get; set; } - public int IntArg18 { get; set; } - public int IntArg19 { get; set; } - public int IntArg20 { get; set; } - - #endregion - - #region Long Arguments - - public long LongArg1 { get; set; } - public long LongArg2 { get; set; } - public long LongArg3 { get; set; } - public long LongArg4 { get; set; } - public long LongArg5 { get; set; } - public long LongArg6 { get; set; } - public long LongArg7 { get; set; } - public long LongArg8 { get; set; } - public long LongArg9 { get; set; } - public long LongArg10 { get; set; } - public long LongArg11 { get; set; } - public long LongArg12 { get; set; } - public long LongArg13 { get; set; } - public long LongArg14 { get; set; } - public long LongArg15 { get; set; } - public long LongArg16 { get; set; } - public long LongArg17 { get; set; } - public long LongArg18 { get; set; } - public long LongArg19 { get; set; } - public long LongArg20 { get; set; } - - #endregion - - #region String Arguments - - public string StringArg1 { get; set; } - public string StringArg2 { get; set; } - public string StringArg3 { get; set; } - public string StringArg4 { get; set; } - public string StringArg5 { get; set; } - public string StringArg6 { get; set; } - public string StringArg7 { get; set; } - public string StringArg8 { get; set; } - public string StringArg9 { get; set; } - public string StringArg10 { get; set; } - public string StringArg11 { get; set; } - public string StringArg12 { get; set; } - public string StringArg13 { get; set; } - public string StringArg14 { get; set; } - public string StringArg15 { get; set; } - public string StringArg16 { get; set; } - public string StringArg17 { get; set; } - public string StringArg18 { get; set; } - public string StringArg19 { get; set; } - public string StringArg20 { get; set; } - - #endregion - - public void SetKeys(IList keysInOrder) + var setters = this.GetType().GetProperties().Where(p => p.Name.StartsWith("Key")) + .Select(p => p.SetMethod).ToList(); + + for (var i = 1; i <= 20; i++) { - if (keysInOrder == null || !keysInOrder.Any()) + if (keysInOrder.Count < i) { - return; + break; } - var setters = this.GetType().GetProperties().Where(p => p.Name.StartsWith("Key")) - .Select(p => p.SetMethod).ToList(); - - for (var i = 1; i <= 20; i++) - { - if (keysInOrder.Count < i) - { - break; - } - - var keyValue = keysInOrder[i - 1]; - var i1 = i; - setters.FirstOrDefault(x => x.Name == $"set_Key{i1}") - ?.Invoke(this, new object[] { (RedisKey)$"ns=amit:k={keyValue}" }); - } + var keyValue = keysInOrder[i - 1]; + var i1 = i; + setters.FirstOrDefault(x => x.Name == $"set_Key{i1}") + ?.Invoke(this, new object[] { (RedisKey)$"ns=amit:k={keyValue}" }); } + } - public void SetStringArguments(IList argumentsInOrder) - { - SetArguments(argumentsInOrder, "String"); - } + public void SetStringArguments(IList argumentsInOrder) + { + SetArguments(argumentsInOrder, "String"); + } - public void SetIntArguments(IList argumentsInOrder) + public void SetIntArguments(IList argumentsInOrder) + { + SetArguments(argumentsInOrder, "Int"); + } + + private void SetArguments(IList argumentsInOrder, string typePrefix) + { + if (argumentsInOrder == null || !argumentsInOrder.Any()) { - SetArguments(argumentsInOrder, "Int"); + return; } - private void SetArguments(IList argumentsInOrder, string typePrefix) + var prefix = $"{typePrefix}Arg"; + var setters = this.GetType().GetProperties().Where(p => p.Name.StartsWith(prefix)) + .Select(p => p.SetMethod).ToList(); + + for (var i = 1; i <= 20; i++) { - if (argumentsInOrder == null || !argumentsInOrder.Any()) + if (argumentsInOrder.Count < i) { - return; + break; } - var prefix = $"{typePrefix}Arg"; - var setters = this.GetType().GetProperties().Where(p => p.Name.StartsWith(prefix)) - .Select(p => p.SetMethod).ToList(); - - for (var i = 1; i <= 20; i++) - { - if (argumentsInOrder.Count < i) - { - break; - } - - var arg = argumentsInOrder[i - 1]; - var i1 = i; - setters.FirstOrDefault(x => x.Name == $"set_{prefix}{i1}")?.Invoke(this, new[] { arg }); - } + var arg = argumentsInOrder[i - 1]; + var i1 = i; + setters.FirstOrDefault(x => x.Name == $"set_{prefix}{i1}")?.Invoke(this, new[] { arg }); } } } From 5d58fe97d69384e3e00bc2ae7aad40054d375821 Mon Sep 17 00:00:00 2001 From: Nir Agai Date: Thu, 25 Jun 2020 18:22:47 +0300 Subject: [PATCH 5/7] Added async set operations. --- IntegrationTests/RedisTests.cs | 187 +++++++++++++++++++++++++++++- RedisRepo/IRedisContext.cs | 47 ++++++++ RedisRepo/RedisContext.cs | 200 +++++++++++++++++++++++++++++++++ RedisRepo/RetryUtil.cs | 68 +++++++++++ 4 files changed, 501 insertions(+), 1 deletion(-) diff --git a/IntegrationTests/RedisTests.cs b/IntegrationTests/RedisTests.cs index 0eeceb1..fae3494 100644 --- a/IntegrationTests/RedisTests.cs +++ b/IntegrationTests/RedisTests.cs @@ -7,6 +7,7 @@ using System.Linq; using System.Net; using System.Threading; +using System.Threading.Tasks; namespace PubComp.RedisRepo.IntegrationTests { @@ -76,7 +77,7 @@ public static void Retry(Action action, int maxAttempts) return RetryUtil.Retry(() => this.Database.KeyTimeToLive(key), 3); } } - + #endregion #region Test Cases @@ -306,6 +307,91 @@ public void SetOperations_Double() .OrderBy(x => x).ToList()); } + [TestMethod] + public async Task AsyncSetOperations_Parallel() + { + var key1 = TestContext.TestName + ".1"; + + redisContext.Delete(key1); + + var range = Enumerable.Range(1, 10).ToList(); + + var tasks = range.Select(i => redisContext.SetAddAsync(key1, i)); + await Task.WhenAll(tasks); + + var set = (await redisContext.SetGetItemsAsync(key1, RedisValueConverter.ToInt)) + .OrderBy(x => x).ToList(); + + CollectionAssert.AreEquivalent(range, set); + } + + [TestMethod] + public async Task AsyncSetOperations_Double() + { + var key1 = TestContext.TestName + ".1"; + var key2 = TestContext.TestName + ".2"; + var key3 = TestContext.TestName + ".3"; + + redisContext.Delete(key1); + redisContext.Delete(key2); + redisContext.Delete(key3); + + await redisContext.SetAddAsync(key1, new[] { 5.0, 2.0, 1.5 }); + await redisContext.SetAddAsync(key1, 3.5); + + CollectionAssert.AreEquivalent( + new[] { 1.5, 2.0, 3.5, 5.0 }, + (await redisContext.SetGetItemsAsync(key1, RedisValueConverter.ToDouble)) + .OrderBy(x => x).ToList()); + + await redisContext.SetAddAsync(key2, new[] { 7.0, 4.0, 1.5 }); + await redisContext.SetAddAsync(key3, new[] { 1.5, 7.0, 3.5, 8.5 }); + + var actualIntersect123 = await redisContext.SetsIntersectAsync( + new[] { key1, key2, key3 }, RedisValueConverter.ToDouble); + CollectionAssert.AreEquivalent( + new[] { 1.5 }, + actualIntersect123.OrderBy(x => x).ToList()); + + var actualUnion123 = await redisContext.SetsUnionAsync( + new[] { key1, key2, key3 }, RedisValueConverter.ToDouble); + CollectionAssert.AreEquivalent( + new[] { 1.5, 2.0, 3.5, 4.0, 5.0, 7.0, 8.5 }, + actualUnion123.OrderBy(x => x).ToList()); + + var actualMinus123 = await redisContext.SetsDiffAsync( + new[] { key1, key2, key3 }, RedisValueConverter.ToDouble); + CollectionAssert.AreEquivalent( + new[] { 2.0, 5.0 }, + actualMinus123.OrderBy(x => x).ToList()); + + Assert.AreEqual(4, await redisContext.SetLengthAsync(key1)); + Assert.AreEqual(3, await redisContext.SetLengthAsync(key2)); + Assert.AreEqual(4, await redisContext.SetLengthAsync(key3)); + + await redisContext.SetRemoveAsync(key1, 2.0); + Assert.AreEqual(3, await redisContext.SetLengthAsync(key1)); + CollectionAssert.AreEquivalent( + new[] { 1.5, 3.5, 5.0 }, + (await redisContext.SetGetItemsAsync(key1, RedisValueConverter.ToDouble)) + .OrderBy(x => x).ToList()); + + await redisContext.SetRemoveAsync(key3, new[] { 2.0, 8.5, 7.0 }); + Assert.AreEqual(2, await redisContext.SetLengthAsync(key3)); + CollectionAssert.AreEquivalent( + new[] { 1.5, 3.5 }, + (await redisContext.SetGetItemsAsync(key3, RedisValueConverter.ToDouble)) + .OrderBy(x => x).ToList()); + + redisContext.Delete(key2); + await redisContext.SetAddAsync(key2, 9.0); + Assert.AreEqual(1, await redisContext.SetLengthAsync(key2)); + CollectionAssert.AreEquivalent( + new[] { 9.0 }, + (await redisContext.SetGetItemsAsync(key2, RedisValueConverter.ToDouble)) + .OrderBy(x => x).ToList()); + } + [TestMethod] public void SetDeleteMany() { @@ -694,18 +780,40 @@ public void TestAddToRedisSet() ValidateSetResults(key, new[] { "a", "b", "c", "bar" }); } + [TestMethod] + public async Task TestAddToRedisSetAsync() + { + const string key = "k1"; + var values = new[] { "bar", "bar", "a", "b", "c" }; + + await redisContext.AddToSetAsync(key, values); + + await ValidateSetResultsAsync(key, new[] { "a", "b", "c", "bar" }); + } + [TestMethod] public void TestSetContainsTrue() { TestSetContains("a", "a", true); } + [TestMethod] + public async Task TestSetContainsTrueAsync() + { + await TestSetContainsAsync("a", "a", true); + } + [TestMethod] public void TestSetContainsFalse() { TestSetContains("a", "b", false); } + [TestMethod] + public async Task TestSetContainsFalseAsync() + { + await TestSetContainsAsync("a", "b", false); + } [TestMethod] public void TestCountSetMembers() @@ -718,6 +826,17 @@ public void TestCountSetMembers() Assert.AreEqual(4, redisContext.CountSetMembers(key)); } + [TestMethod] + public async Task TestCountSetMembersAsync() + { + const string key = "k2"; + var values = new[] { "bar", "bar", "a", "b", "c", "a", "b" }; + + await redisContext.AddToSetAsync(key, values); + + Assert.AreEqual(4, await redisContext.CountSetMembersAsync(key)); + } + [TestMethod] public void TestSetsDiff() { @@ -734,6 +853,22 @@ public void TestSetsDiff() CollectionAssert.AreEquivalent(new[] { "c", "d", "e" }, results); } + [TestMethod] + public async Task TestSetsDiffAsync() + { + const string key1 = "testSetDiff1"; + var values1 = new[] { "a", "b", "c", "d", "e" }; + + const string key2 = "testSetDiff2"; + var values2 = new[] { "a", "b", }; + + await redisContext.AddToSetAsync(key1, values1); + await redisContext.AddToSetAsync(key2, values2); + + var results = await redisContext.GetSetsDifferenceAsync(new[] { key1, key2 }); + CollectionAssert.AreEquivalent(new[] { "c", "d", "e" }, results); + } + [TestMethod] public void TestSetsUnion() { @@ -750,6 +885,22 @@ public void TestSetsUnion() CollectionAssert.AreEquivalent(new[] { "a", "b", "c" }, results); } + [TestMethod] + public async Task TestSetsUnionAsync() + { + const string key1 = "TestSetsUnion1"; + var values1 = new[] { "a", "c" }; + + const string key2 = "TestSetsUnion2"; + var values2 = new[] { "a", "b" }; + + await redisContext.AddToSetAsync(key1, values1); + await redisContext.AddToSetAsync(key2, values2); + + var results = await redisContext.UnionSetsAsync(new[] { key1, key2 }); + CollectionAssert.AreEquivalent(new[] { "a", "b", "c" }, results); + } + [TestMethod] public void TestSetsIntersect() { @@ -766,12 +917,34 @@ public void TestSetsIntersect() CollectionAssert.AreEquivalent(new[] { "a" }, results); } + [TestMethod] + public async Task TestSetsIntersectAsync() + { + const string key1 = "TestSetsIntersect1"; + var values1 = new[] { "a", "c" }; + + const string key2 = "TestSetsIntersect2"; + var values2 = new[] { "a", "b" }; + + await redisContext.AddToSetAsync(key1, values1); + await redisContext.AddToSetAsync(key2, values2); + + var results = await redisContext.IntersectSetsAsync(new[] { key1, key2 }); + CollectionAssert.AreEquivalent(new[] { "a" }, results); + } + private void ValidateSetResults(string key, string[] expected) { var valuesFromRedis = redisContext.GetSetMembers(key); CollectionAssert.AreEquivalent(expected, valuesFromRedis); } + private async Task ValidateSetResultsAsync(string key, string[] expected) + { + var valuesFromRedis = await redisContext.GetSetMembersAsync(key); + CollectionAssert.AreEquivalent(expected, valuesFromRedis); + } + public void TestSetContains(string valueToAdd, string searchForValue, bool expected) { const string key = "testSetContains"; @@ -784,6 +957,18 @@ public void TestSetContains(string valueToAdd, string searchForValue, bool expec Assert.AreEqual(expected, setContains); } + public async Task TestSetContainsAsync(string valueToAdd, string searchForValue, bool expected) + { + const string key = "testSetContains"; + var values = new[] { "foo", "bar" }; + + await redisContext.AddToSetAsync(key, values); + await redisContext.AddToSetAsync(key, new[] { valueToAdd }); + + var setContains = await redisContext.SetContainsAsync(key, searchForValue); + Assert.AreEqual(expected, setContains); + } + #endregion #region Redis Hashes diff --git a/RedisRepo/IRedisContext.cs b/RedisRepo/IRedisContext.cs index 3d919ff..c7afe0f 100644 --- a/RedisRepo/IRedisContext.cs +++ b/RedisRepo/IRedisContext.cs @@ -1,6 +1,7 @@ using System; using System.Collections; using System.Collections.Generic; +using System.Threading.Tasks; using StackExchange.Redis; namespace PubComp.RedisRepo @@ -78,63 +79,109 @@ public interface IRedisContext #region Redis Sets bool SetAdd(string key, T value); + Task SetAddAsync(string key, T value); long SetAdd(string key, T[] values); + Task SetAddAsync(string key, T[] values); bool SetRemove(string key, T value); + Task SetRemoveAsync(string key, T value); long SetRemove(string key, T[] values); + Task SetRemoveAsync(string key, T[] values); long SetLength(string key); + Task SetLengthAsync(string key); T[] SetGetItems(string key, Func redisValueConverter); + Task SetGetItemsAsync(string key, Func redisValueConverter); T[] SetsUnion(string[] keys, Func redisValueConverter); + Task SetsUnionAsync(string[] keys, Func redisValueConverter); T[] SetsIntersect(string[] keys, Func redisValueConverter); + Task SetsIntersectAsync(string[] keys, Func redisValueConverter); T[] SetsDiff(string[] keys, Func redisValueConverter); + Task SetsDiffAsync(string[] keys, Func redisValueConverter); void AddToSet(string key, string[] values); + Task AddToSetAsync(string key, string[] values); long CountSetMembers(string key); + Task CountSetMembersAsync(string key); string[] GetSetMembers(string key); + Task GetSetMembersAsync(string key); /// /// Get the diff between the set at index 0 of and all other sets in /// string[] GetSetsDifference(string[] keys); + /// + /// Get the diff between the set at index 0 of and all other sets in + /// + Task GetSetsDifferenceAsync(string[] keys); + /// /// Union sets at keys /// string[] UnionSets(string[] keys); + /// + /// Union sets at keys + /// + Task UnionSetsAsync(string[] keys); + /// /// Intersect sets at keys /// string[] IntersectSets(string[] keys); + /// + /// Intersect sets at keys + /// + Task IntersectSetsAsync(string[] keys); + /// /// Get the diff between the set at index 0 of and all other sets in /// store the result at /// void StoreSetsDifference(string destinationKey, string[] keys); + /// + /// Get the diff between the set at index 0 of and all other sets in + /// store the result at + /// + Task StoreSetsDifferenceAsync(string destinationKey, string[] keys); + /// /// Union sets at keys /// store the result at /// void UnionSetsAndStore(string destinationKey, string[] keys); + /// + /// Union sets at keys + /// store the result at + /// + Task UnionSetsAndStoreAsync(string destinationKey, string[] keys); + /// /// Intersect sets at keys /// store the result at /// void IntersectSetsAndStore(string destinationKey, string[] keys); + /// + /// Intersect sets at keys + /// store the result at + /// + Task IntersectSetsAndStoreAsync(string destinationKey, string[] keys); + bool SetContains(string key, string member); + Task SetContainsAsync(string key, string member); bool TryGetDistributedLock(string lockObjectName, string lockerName, TimeSpan lockTtl); diff --git a/RedisRepo/RedisContext.cs b/RedisRepo/RedisContext.cs index f84b674..9a43f8f 100644 --- a/RedisRepo/RedisContext.cs +++ b/RedisRepo/RedisContext.cs @@ -6,6 +6,7 @@ using System.Collections.Concurrent; using System.Collections.Generic; using System.Linq; +using System.Threading.Tasks; namespace PubComp.RedisRepo { @@ -265,10 +266,18 @@ public static TResult Retry(Func func, int maxAttempts) { return RetryUtil.Retry(func, maxAttempts); } + public async static Task RetryAsync(Func> func, int maxAttempts) + { + return await RetryUtil.RetryAsync(func, maxAttempts).ConfigureAwait(false); + } public static void Retry(Action action, int maxAttempts) { RetryUtil.Retry(action, maxAttempts); } + public async static Task RetryAsync(Func action, int maxAttempts) + { + await RetryUtil.RetryAsync(action, maxAttempts).ConfigureAwait(false); + } #endregion public virtual string Key(string key) @@ -489,6 +498,17 @@ public bool SetAdd(string key, T value) return result; } + public async Task SetAddAsync(string key, T value) + { + var redisValue = value.ToRedis(); + + var result = await RetryAsync(async () => + await this.Database.SetAddAsync( + Key(key), redisValue, commandFlags).ConfigureAwait(false), defaultRetries).ConfigureAwait(false); + + return result; + } + public long SetAdd(string key, T[] values) { var redisValues = values?.Select(val => val.ToRedis()).ToArray(); @@ -500,6 +520,17 @@ public long SetAdd(string key, T[] values) return result; } + public async Task SetAddAsync(string key, T[] values) + { + var redisValues = values?.Select(val => val.ToRedis()).ToArray(); + + var result = await RetryAsync(async () => + await this.Database.SetAddAsync( + Key(key), redisValues, commandFlags).ConfigureAwait(false), defaultRetries).ConfigureAwait(false); + + return result; + } + public T[] SetGetItems(string key, Func redisValueConverter) { var results = Retry(() => @@ -508,6 +539,14 @@ public T[] SetGetItems(string key, Func redisValueConverter) return results.Select(r => redisValueConverter(r)).ToArray(); } + public async Task SetGetItemsAsync(string key, Func redisValueConverter) + { + var results = await RetryAsync(async () => + await this.Database.SetMembersAsync(Key(key), commandFlags).ConfigureAwait(false), defaultRetries).ConfigureAwait(false); + + return results.Select(r => redisValueConverter(r)).ToArray(); + } + public T[] SetsUnion(string[] keys, Func redisValueConverter) { var redisKeys = keys.Select(k => (RedisKey)Key(k)).ToArray(); @@ -519,6 +558,17 @@ public T[] SetsUnion(string[] keys, Func redisValueConverter) return results.Select(r => redisValueConverter(r)).ToArray(); } + public async Task SetsUnionAsync(string[] keys, Func redisValueConverter) + { + var redisKeys = keys.Select(k => (RedisKey)Key(k)).ToArray(); + + var results = await RetryAsync(async () => + await this.Database.SetCombineAsync( + SetOperation.Union, redisKeys, commandFlags).ConfigureAwait(false), defaultRetries).ConfigureAwait(false); + + return results.Select(r => redisValueConverter(r)).ToArray(); + } + public T[] SetsIntersect(string[] keys, Func redisValueConverter) { var redisKeys = keys.Select(k => (RedisKey)Key(k)).ToArray(); @@ -530,6 +580,17 @@ public T[] SetsIntersect(string[] keys, Func redisValueConverter) return results.Select(r => redisValueConverter(r)).ToArray(); } + public async Task SetsIntersectAsync(string[] keys, Func redisValueConverter) + { + var redisKeys = keys.Select(k => (RedisKey)Key(k)).ToArray(); + + var results = await RetryAsync(async () => + await this.Database.SetCombineAsync( + SetOperation.Intersect, redisKeys, commandFlags).ConfigureAwait(false), defaultRetries).ConfigureAwait(false); + + return results.Select(r => redisValueConverter(r)).ToArray(); + } + public T[] SetsDiff(string[] keys, Func redisValueConverter) { var redisKeys = keys.Select(k => (RedisKey)Key(k)).ToArray(); @@ -541,6 +602,17 @@ public T[] SetsDiff(string[] keys, Func redisValueConverter) return results.Select(r => redisValueConverter(r)).ToArray(); } + public async Task SetsDiffAsync(string[] keys, Func redisValueConverter) + { + var redisKeys = keys.Select(k => (RedisKey)Key(k)).ToArray(); + + var results = await RetryAsync(async () => + await this.Database.SetCombineAsync( + SetOperation.Difference, redisKeys, commandFlags).ConfigureAwait(false), defaultRetries).ConfigureAwait(false); + + return results.Select(r => redisValueConverter(r)).ToArray(); + } + public bool SetRemove(string key, T value) { var redisValue = value.ToRedis(); @@ -552,6 +624,17 @@ public bool SetRemove(string key, T value) return result; } + public async Task SetRemoveAsync(string key, T value) + { + var redisValue = value.ToRedis(); + + var result = await RetryAsync(async () => + await this.Database.SetRemoveAsync( + Key(key), redisValue, commandFlags).ConfigureAwait(false), defaultRetries).ConfigureAwait(false); + + return result; + } + public long SetRemove(string key, T[] values) { var redisValues = values?.Select(val => val.ToRedis()).ToArray(); @@ -563,6 +646,17 @@ public long SetRemove(string key, T[] values) return result; } + public async Task SetRemoveAsync(string key, T[] values) + { + var redisValues = values?.Select(val => val.ToRedis()).ToArray(); + + var result = await RetryAsync(async () => + await this.Database.SetRemoveAsync( + Key(key), redisValues, commandFlags).ConfigureAwait(false), defaultRetries).ConfigureAwait(false); + + return result; + } + public long SetLength(string key) { var result = Retry(() => @@ -571,22 +665,46 @@ public long SetLength(string key) return result; } + public async Task SetLengthAsync(string key) + { + var result = await RetryAsync(async () => + await this.Database.SetLengthAsync(Key(key), commandFlags).ConfigureAwait(false), defaultRetries).ConfigureAwait(false); + + return result; + } + public void AddToSet(string key, string[] values) { Retry(() => this.Database.SetAdd(Key(key), values.ToRedisValueArray(), flags: commandFlags), defaultRetries); } + public async Task AddToSetAsync(string key, string[] values) + { + await RetryAsync(async () => await this.Database.SetAddAsync(Key(key), values.ToRedisValueArray(), flags: commandFlags).ConfigureAwait(false), defaultRetries).ConfigureAwait(false); + } + public long CountSetMembers(string key) { return Retry(() => this.Database.SetLength(Key(key), flags: commandFlags), defaultRetries); } + public async Task CountSetMembersAsync(string key) + { + return await RetryAsync(async () => await this.Database.SetLengthAsync(Key(key), flags: commandFlags).ConfigureAwait(false), defaultRetries).ConfigureAwait(false); + } + public string[] GetSetMembers(string key) { var results = Retry(() => this.Database.SetMembers(Key(key), flags: commandFlags), defaultRetries); return results.ToStringArray(); } + public async Task GetSetMembersAsync(string key) + { + var results = await RetryAsync(async () => await this.Database.SetMembersAsync(Key(key), flags: commandFlags).ConfigureAwait(false), defaultRetries).ConfigureAwait(false); + return results.ToStringArray(); + } + /// /// Get the diff between the set at index 0 of and all other sets in /// @@ -597,6 +715,16 @@ public string[] GetSetsDifference(string[] keys) keys); } + /// + /// Get the diff between the set at index 0 of and all other sets in + /// + public async Task GetSetsDifferenceAsync(string[] keys) + { + return await OperateOnSetAsync( + SetOperation.Difference, + keys).ConfigureAwait(false); + } + /// /// Union sets at keys /// @@ -605,6 +733,14 @@ public string[] UnionSets(string[] keys) return OperateOnSet(SetOperation.Union, keys); } + /// + /// Union sets at keys + /// + public async Task UnionSetsAsync(string[] keys) + { + return await OperateOnSetAsync(SetOperation.Union, keys).ConfigureAwait(false); + } + /// /// Intersect sets at keys /// @@ -613,6 +749,14 @@ public string[] IntersectSets(string[] keys) return OperateOnSet(SetOperation.Intersect, keys); } + /// + /// Intersect sets at keys + /// + public async Task IntersectSetsAsync(string[] keys) + { + return await OperateOnSetAsync(SetOperation.Intersect, keys).ConfigureAwait(false); + } + /// /// Get the diff between the set at index 0 of and all other sets in /// store the result at @@ -625,6 +769,18 @@ public void StoreSetsDifference(string destinationKey, string[] keys) keys); } + /// + /// Get the diff between the set at index 0 of and all other sets in + /// store the result at + /// + public async Task StoreSetsDifferenceAsync(string destinationKey, string[] keys) + { + await OperateOnSetAndStoreAsync( + SetOperation.Difference, + destinationKey, + keys).ConfigureAwait(false); + } + /// /// Union sets at keys /// store the result at @@ -634,6 +790,15 @@ public void UnionSetsAndStore(string destinationKey, string[] keys) OperateOnSetAndStore(SetOperation.Union, destinationKey, keys); } + /// + /// Union sets at keys + /// store the result at + /// + public async Task UnionSetsAndStoreAsync(string destinationKey, string[] keys) + { + await OperateOnSetAndStoreAsync(SetOperation.Union, destinationKey, keys).ConfigureAwait(false); + } + /// /// Intersect sets at keys /// store the result at @@ -643,11 +808,25 @@ public void IntersectSetsAndStore(string destinationKey, string[] keys) OperateOnSetAndStore(SetOperation.Intersect, destinationKey, keys); } + /// + /// Intersect sets at keys + /// store the result at + /// + public async Task IntersectSetsAndStoreAsync(string destinationKey, string[] keys) + { + await OperateOnSetAndStoreAsync(SetOperation.Intersect, destinationKey, keys).ConfigureAwait(false); + } + public bool SetContains(string key, string member) { return Retry(() => this.Database.SetContains(Key(key), member, commandFlags), defaultRetries); } + public async Task SetContainsAsync(string key, string member) + { + return await RetryAsync(async () => await this.Database.SetContainsAsync(Key(key), member, commandFlags).ConfigureAwait(false), defaultRetries).ConfigureAwait(false); + } + #region set helpers private string[] OperateOnSet(SetOperation op, string[] keys) { @@ -659,6 +838,16 @@ private string[] OperateOnSet(SetOperation op, string[] keys) return results?.ToStringArray(); } + private async Task OperateOnSetAsync(SetOperation op, string[] keys) + { + if (keys == null || keys.Length == 0) return null; + + var redisKeys = keys.Select(c => (RedisKey)Key(c)).ToArray(); + var results = + await RetryAsync(async () => await this.Database.SetCombineAsync(op, redisKeys, commandFlags).ConfigureAwait(false), defaultRetries).ConfigureAwait(false); + + return results?.ToStringArray(); + } private void OperateOnSetAndStore(SetOperation op, string destinationKey, string[] keys) { @@ -669,6 +858,17 @@ private void OperateOnSetAndStore(SetOperation op, string destinationKey, string Retry(() => this.Database.SetCombineAndStore(op, Key(destinationKey), redisKeys, commandFlags), defaultRetries); + } + + private async Task OperateOnSetAndStoreAsync(SetOperation op, string destinationKey, string[] keys) + { + if (keys == null || keys.Length == 0) + return; + + var redisKeys = keys.Select(c => (RedisKey)Key(c)).ToArray(); + + await RetryAsync(async () => await this.Database.SetCombineAndStoreAsync(op, Key(destinationKey), redisKeys, commandFlags).ConfigureAwait(false), defaultRetries).ConfigureAwait(false); + } #endregion diff --git a/RedisRepo/RetryUtil.cs b/RedisRepo/RetryUtil.cs index e8968c0..5b99d9b 100644 --- a/RedisRepo/RetryUtil.cs +++ b/RedisRepo/RetryUtil.cs @@ -52,6 +52,42 @@ public static TResult Retry(Func func, int maxAttempts) return result; } + public async static Task RetryAsync(Func> func, int maxAttempts) + { + TResult result = default(TResult); + + for (int attempts = 0; attempts < maxAttempts; attempts++) + { + try + { + result = await func().ConfigureAwait(false); + break; + } + catch (Exception ex) + { + if (!TestExceptionForRetry(ex)) + { + throw; + } + + if (attempts < maxAttempts - 1) + { + LogManager.GetLogger(typeof(RedisContext).FullName).Warn( + ex, $"Retrying, attempt #{attempts}"); + await Task.Delay(RetryDelay * (attempts + 1)).ConfigureAwait(false); + } + else + { + LogManager.GetLogger(typeof(RedisContext).FullName).Error( + ex, $"Failed, attempt #{attempts}"); + throw; + } + } + } + + return result; + } + public static void Retry(Action action, int maxAttempts) { for (int attempts = 0; attempts < maxAttempts; attempts++) @@ -84,6 +120,38 @@ public static void Retry(Action action, int maxAttempts) } } + public async static Task RetryAsync(Func action, int maxAttempts) + { + for (int attempts = 0; attempts < maxAttempts; attempts++) + { + try + { + await action().ConfigureAwait(false); + break; + } + catch (Exception ex) + { + if (!TestExceptionForRetry(ex)) + { + throw; + } + + if (attempts < maxAttempts - 1) + { + LogManager.GetLogger(typeof(RedisContext).FullName).Warn( + ex, $"Retrying, attempt #{attempts}"); + await Task.Delay(RetryDelay * (attempts + 1)).ConfigureAwait(false); + } + else + { + LogManager.GetLogger(typeof(RedisContext).FullName).Error( + ex, $"Failed, attempt #{attempts}"); + throw; + } + } + } + } + public static bool TestExceptionForRetry(Exception ex) { return From ce1336015ab5301fd4ccf4c529a131c7f1b20e7f Mon Sep 17 00:00:00 2001 From: Nir Agai Date: Sun, 28 Jun 2020 19:05:22 +0300 Subject: [PATCH 6/7] Improved the passing of async methods to the Retry function. Added null and empty values checks in set operations that receive an array of values. --- IntegrationTests/RedisTests.cs | 60 ++++++++++++------ RedisRepo/RedisContext.cs | 110 ++++++++++++++++++++++----------- 2 files changed, 117 insertions(+), 53 deletions(-) diff --git a/IntegrationTests/RedisTests.cs b/IntegrationTests/RedisTests.cs index fae3494..fbddd84 100644 --- a/IntegrationTests/RedisTests.cs +++ b/IntegrationTests/RedisTests.cs @@ -307,24 +307,6 @@ public void SetOperations_Double() .OrderBy(x => x).ToList()); } - [TestMethod] - public async Task AsyncSetOperations_Parallel() - { - var key1 = TestContext.TestName + ".1"; - - redisContext.Delete(key1); - - var range = Enumerable.Range(1, 10).ToList(); - - var tasks = range.Select(i => redisContext.SetAddAsync(key1, i)); - await Task.WhenAll(tasks); - - var set = (await redisContext.SetGetItemsAsync(key1, RedisValueConverter.ToInt)) - .OrderBy(x => x).ToList(); - - CollectionAssert.AreEquivalent(range, set); - } - [TestMethod] public async Task AsyncSetOperations_Double() { @@ -392,6 +374,48 @@ public async Task AsyncSetOperations_Double() .OrderBy(x => x).ToList()); } + [TestMethod] + public async Task AsyncSetOperations_Parallel() + { + var key1 = TestContext.TestName + ".1"; + + redisContext.Delete(key1); + + var range = Enumerable.Range(1, 10).ToList(); + + var tasks = range.Select(i => redisContext.SetAddAsync(key1, i)); + await Task.WhenAll(tasks); + + var set = (await redisContext.SetGetItemsAsync(key1, RedisValueConverter.ToInt)) + .OrderBy(x => x).ToList(); + + CollectionAssert.AreEquivalent(range, set); + } + + + [TestMethod] + public async Task SetOperations_NullEmptyValuesParam() + { + var key1 = TestContext.TestName + ".1"; + + redisContext.Delete(key1); + + int[] nullValues = null; + int[] emptyValues = new int[0]; + + Assert.ThrowsException(() => redisContext.SetAdd(key1, nullValues)); + await Assert.ThrowsExceptionAsync(() => redisContext.SetAddAsync(key1, nullValues)); + + Assert.AreEqual(0, redisContext.SetAdd(key1, emptyValues)); + Assert.AreEqual(0, await redisContext.SetAddAsync(key1, emptyValues)); + + Assert.ThrowsException(() => redisContext.SetRemove(key1, nullValues)); + await Assert.ThrowsExceptionAsync(() => redisContext.SetRemoveAsync(key1, nullValues)); + + Assert.AreEqual(0, redisContext.SetRemove(key1, emptyValues)); + Assert.AreEqual(0, await redisContext.SetRemoveAsync(key1, emptyValues)); + } + [TestMethod] public void SetDeleteMany() { diff --git a/RedisRepo/RedisContext.cs b/RedisRepo/RedisContext.cs index 9a43f8f..b157b19 100644 --- a/RedisRepo/RedisContext.cs +++ b/RedisRepo/RedisContext.cs @@ -502,16 +502,26 @@ public async Task SetAddAsync(string key, T value) { var redisValue = value.ToRedis(); - var result = await RetryAsync(async () => - await this.Database.SetAddAsync( - Key(key), redisValue, commandFlags).ConfigureAwait(false), defaultRetries).ConfigureAwait(false); + var result = await RetryAsync(() => + this.Database.SetAddAsync( + Key(key), redisValue, commandFlags), defaultRetries).ConfigureAwait(false); return result; } public long SetAdd(string key, T[] values) { - var redisValues = values?.Select(val => val.ToRedis()).ToArray(); + if (values == null) + { + throw new ArgumentNullException(nameof(values)); + } + + if (values.Length == 0) + { + return 0; + } + + var redisValues = values.Select(val => val.ToRedis()).ToArray(); var result = Retry(() => this.Database.SetAdd( @@ -522,11 +532,21 @@ public long SetAdd(string key, T[] values) public async Task SetAddAsync(string key, T[] values) { - var redisValues = values?.Select(val => val.ToRedis()).ToArray(); + if (values == null) + { + throw new ArgumentNullException(nameof(values)); + } + + if (values.Length == 0) + { + return 0; + } + + var redisValues = values.Select(val => val.ToRedis()).ToArray(); - var result = await RetryAsync(async () => - await this.Database.SetAddAsync( - Key(key), redisValues, commandFlags).ConfigureAwait(false), defaultRetries).ConfigureAwait(false); + var result = await RetryAsync(() => + this.Database.SetAddAsync( + Key(key), redisValues, commandFlags), defaultRetries).ConfigureAwait(false); return result; } @@ -541,8 +561,8 @@ public T[] SetGetItems(string key, Func redisValueConverter) public async Task SetGetItemsAsync(string key, Func redisValueConverter) { - var results = await RetryAsync(async () => - await this.Database.SetMembersAsync(Key(key), commandFlags).ConfigureAwait(false), defaultRetries).ConfigureAwait(false); + var results = await RetryAsync(() => + this.Database.SetMembersAsync(Key(key), commandFlags), defaultRetries).ConfigureAwait(false); return results.Select(r => redisValueConverter(r)).ToArray(); } @@ -562,9 +582,9 @@ public async Task SetsUnionAsync(string[] keys, Func redisVal { var redisKeys = keys.Select(k => (RedisKey)Key(k)).ToArray(); - var results = await RetryAsync(async () => - await this.Database.SetCombineAsync( - SetOperation.Union, redisKeys, commandFlags).ConfigureAwait(false), defaultRetries).ConfigureAwait(false); + var results = await RetryAsync(() => + this.Database.SetCombineAsync( + SetOperation.Union, redisKeys, commandFlags), defaultRetries).ConfigureAwait(false); return results.Select(r => redisValueConverter(r)).ToArray(); } @@ -584,9 +604,9 @@ public async Task SetsIntersectAsync(string[] keys, Func redi { var redisKeys = keys.Select(k => (RedisKey)Key(k)).ToArray(); - var results = await RetryAsync(async () => - await this.Database.SetCombineAsync( - SetOperation.Intersect, redisKeys, commandFlags).ConfigureAwait(false), defaultRetries).ConfigureAwait(false); + var results = await RetryAsync(() => + this.Database.SetCombineAsync( + SetOperation.Intersect, redisKeys, commandFlags), defaultRetries).ConfigureAwait(false); return results.Select(r => redisValueConverter(r)).ToArray(); } @@ -606,9 +626,9 @@ public async Task SetsDiffAsync(string[] keys, Func redisValu { var redisKeys = keys.Select(k => (RedisKey)Key(k)).ToArray(); - var results = await RetryAsync(async () => - await this.Database.SetCombineAsync( - SetOperation.Difference, redisKeys, commandFlags).ConfigureAwait(false), defaultRetries).ConfigureAwait(false); + var results = await RetryAsync(() => + this.Database.SetCombineAsync( + SetOperation.Difference, redisKeys, commandFlags), defaultRetries).ConfigureAwait(false); return results.Select(r => redisValueConverter(r)).ToArray(); } @@ -628,16 +648,26 @@ public async Task SetRemoveAsync(string key, T value) { var redisValue = value.ToRedis(); - var result = await RetryAsync(async () => - await this.Database.SetRemoveAsync( - Key(key), redisValue, commandFlags).ConfigureAwait(false), defaultRetries).ConfigureAwait(false); + var result = await RetryAsync(() => + this.Database.SetRemoveAsync( + Key(key), redisValue, commandFlags), defaultRetries).ConfigureAwait(false); return result; } public long SetRemove(string key, T[] values) { - var redisValues = values?.Select(val => val.ToRedis()).ToArray(); + if (values == null) + { + throw new ArgumentNullException(nameof(values)); + } + + if (values.Length == 0) + { + return 0; + } + + var redisValues = values.Select(val => val.ToRedis()).ToArray(); var result = Retry(() => this.Database.SetRemove( @@ -648,11 +678,21 @@ public long SetRemove(string key, T[] values) public async Task SetRemoveAsync(string key, T[] values) { - var redisValues = values?.Select(val => val.ToRedis()).ToArray(); + if (values == null) + { + throw new ArgumentNullException(nameof(values)); + } + + if (values.Length == 0) + { + return 0; + } + + var redisValues = values.Select(val => val.ToRedis()).ToArray(); - var result = await RetryAsync(async () => - await this.Database.SetRemoveAsync( - Key(key), redisValues, commandFlags).ConfigureAwait(false), defaultRetries).ConfigureAwait(false); + var result = await RetryAsync(() => + this.Database.SetRemoveAsync( + Key(key), redisValues, commandFlags), defaultRetries).ConfigureAwait(false); return result; } @@ -667,8 +707,8 @@ public long SetLength(string key) public async Task SetLengthAsync(string key) { - var result = await RetryAsync(async () => - await this.Database.SetLengthAsync(Key(key), commandFlags).ConfigureAwait(false), defaultRetries).ConfigureAwait(false); + var result = await RetryAsync(() => + this.Database.SetLengthAsync(Key(key), commandFlags), defaultRetries).ConfigureAwait(false); return result; } @@ -680,7 +720,7 @@ public void AddToSet(string key, string[] values) public async Task AddToSetAsync(string key, string[] values) { - await RetryAsync(async () => await this.Database.SetAddAsync(Key(key), values.ToRedisValueArray(), flags: commandFlags).ConfigureAwait(false), defaultRetries).ConfigureAwait(false); + await RetryAsync(() => this.Database.SetAddAsync(Key(key), values.ToRedisValueArray(), flags: commandFlags), defaultRetries).ConfigureAwait(false); } public long CountSetMembers(string key) @@ -690,7 +730,7 @@ public long CountSetMembers(string key) public async Task CountSetMembersAsync(string key) { - return await RetryAsync(async () => await this.Database.SetLengthAsync(Key(key), flags: commandFlags).ConfigureAwait(false), defaultRetries).ConfigureAwait(false); + return await RetryAsync(() => this.Database.SetLengthAsync(Key(key), flags: commandFlags), defaultRetries).ConfigureAwait(false); } public string[] GetSetMembers(string key) @@ -701,7 +741,7 @@ public string[] GetSetMembers(string key) public async Task GetSetMembersAsync(string key) { - var results = await RetryAsync(async () => await this.Database.SetMembersAsync(Key(key), flags: commandFlags).ConfigureAwait(false), defaultRetries).ConfigureAwait(false); + var results = await RetryAsync(() => this.Database.SetMembersAsync(Key(key), flags: commandFlags), defaultRetries).ConfigureAwait(false); return results.ToStringArray(); } @@ -824,7 +864,7 @@ public bool SetContains(string key, string member) public async Task SetContainsAsync(string key, string member) { - return await RetryAsync(async () => await this.Database.SetContainsAsync(Key(key), member, commandFlags).ConfigureAwait(false), defaultRetries).ConfigureAwait(false); + return await RetryAsync(() => this.Database.SetContainsAsync(Key(key), member, commandFlags), defaultRetries).ConfigureAwait(false); } #region set helpers @@ -844,7 +884,7 @@ private async Task OperateOnSetAsync(SetOperation op, string[] keys) var redisKeys = keys.Select(c => (RedisKey)Key(c)).ToArray(); var results = - await RetryAsync(async () => await this.Database.SetCombineAsync(op, redisKeys, commandFlags).ConfigureAwait(false), defaultRetries).ConfigureAwait(false); + await RetryAsync(() => this.Database.SetCombineAsync(op, redisKeys, commandFlags), defaultRetries).ConfigureAwait(false); return results?.ToStringArray(); } @@ -867,7 +907,7 @@ private async Task OperateOnSetAndStoreAsync(SetOperation op, string destination var redisKeys = keys.Select(c => (RedisKey)Key(c)).ToArray(); - await RetryAsync(async () => await this.Database.SetCombineAndStoreAsync(op, Key(destinationKey), redisKeys, commandFlags).ConfigureAwait(false), defaultRetries).ConfigureAwait(false); + await RetryAsync(() => this.Database.SetCombineAndStoreAsync(op, Key(destinationKey), redisKeys, commandFlags), defaultRetries).ConfigureAwait(false); } #endregion From 11bac5e7e184d4ad33bc66a3a01573ffa25e9ad7 Mon Sep 17 00:00:00 2001 From: Nir Agai Date: Sun, 28 Jun 2020 19:11:43 +0300 Subject: [PATCH 7/7] Updated version. --- RedisRepo/RedisRepo.csproj | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/RedisRepo/RedisRepo.csproj b/RedisRepo/RedisRepo.csproj index d16b1cd..4d55bd1 100644 --- a/RedisRepo/RedisRepo.csproj +++ b/RedisRepo/RedisRepo.csproj @@ -10,8 +10,8 @@ true true - 5.3.0.0 - 5.3.0 + 5.4.0.0 + 5.4.0 5.0.0.0