support sqlite

Changedb.sln

@@ -73,6 +73,14 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "subc", "subc", "{541BF71F-7
 EndProject
 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "ChangeDB.SubcTest", "test\ChangeDB.SubcTest\ChangeDB.SubcTest.csproj", "{DBF6DC1D-C81F-4ECD-B7BA-E9B6D6C10AF5}"
 EndProject
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "ChangeDB.Agent.Sqlite.UnitTest", "test\ChangeDB.Agent.Sqlite.UnitTest\ChangeDB.Agent.Sqlite.UnitTest.csproj", "{50DFF3BD-8D1E-4F34-B7AE-115015289CC7}"
+EndProject
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "ChangeDB.Agent.Sqlite", "src\ChangeDB.Agent.Sqlite\ChangeDB.Agent.Sqlite.csproj", "{69A3060B-BB12-4FD2-9581-36EFD60E4EBA}"
+EndProject
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "ChangeDB.Agent.Sqlite.IntegrationTest", "test\ChangeDB.Agent.Sqlite.IntegrationTest\ChangeDB.Agent.Sqlite.IntegrationTest.csproj", "{EF399B45-AAD6-4684-8AF5-8099D20E73B8}"
+EndProject
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "TestDB.Sqlite", "testdb\TestDB.Sqlite\TestDB.Sqlite.csproj", "{FA22E063-49A2-4DD5-8A50-5D3B3F5C0E6D}"
+EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|Any CPU = Debug|Any CPU
@@ -171,6 +179,22 @@ Global
 		{DBF6DC1D-C81F-4ECD-B7BA-E9B6D6C10AF5}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{DBF6DC1D-C81F-4ECD-B7BA-E9B6D6C10AF5}.Release|Any CPU.ActiveCfg = Release|Any CPU
 		{DBF6DC1D-C81F-4ECD-B7BA-E9B6D6C10AF5}.Release|Any CPU.Build.0 = Release|Any CPU
+		{50DFF3BD-8D1E-4F34-B7AE-115015289CC7}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{50DFF3BD-8D1E-4F34-B7AE-115015289CC7}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{50DFF3BD-8D1E-4F34-B7AE-115015289CC7}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{50DFF3BD-8D1E-4F34-B7AE-115015289CC7}.Release|Any CPU.Build.0 = Release|Any CPU
+		{69A3060B-BB12-4FD2-9581-36EFD60E4EBA}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{69A3060B-BB12-4FD2-9581-36EFD60E4EBA}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{69A3060B-BB12-4FD2-9581-36EFD60E4EBA}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{69A3060B-BB12-4FD2-9581-36EFD60E4EBA}.Release|Any CPU.Build.0 = Release|Any CPU
+		{EF399B45-AAD6-4684-8AF5-8099D20E73B8}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{EF399B45-AAD6-4684-8AF5-8099D20E73B8}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{EF399B45-AAD6-4684-8AF5-8099D20E73B8}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{EF399B45-AAD6-4684-8AF5-8099D20E73B8}.Release|Any CPU.Build.0 = Release|Any CPU
+		{FA22E063-49A2-4DD5-8A50-5D3B3F5C0E6D}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{FA22E063-49A2-4DD5-8A50-5D3B3F5C0E6D}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{FA22E063-49A2-4DD5-8A50-5D3B3F5C0E6D}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{FA22E063-49A2-4DD5-8A50-5D3B3F5C0E6D}.Release|Any CPU.Build.0 = Release|Any CPU
 	EndGlobalSection
 	GlobalSection(SolutionProperties) = preSolution
 		HideSolutionNode = FALSE
@@ -203,6 +227,10 @@ Global
 		{51DAC443-6A8B-4400-A372-9616A153B523} = {7C2E7B66-E684-48B6-BA0C-2BCED0DE69CF}
 		{541BF71F-7E84-4E1C-8D4A-CF708525625C} = {9D50B35B-1EAA-4620-B0A6-245456051247}
 		{DBF6DC1D-C81F-4ECD-B7BA-E9B6D6C10AF5} = {541BF71F-7E84-4E1C-8D4A-CF708525625C}
+		{50DFF3BD-8D1E-4F34-B7AE-115015289CC7} = {7C2E7B66-E684-48B6-BA0C-2BCED0DE69CF}
+		{69A3060B-BB12-4FD2-9581-36EFD60E4EBA} = {6C7E5914-CD6B-40F1-BFD7-8FB17E3EAE11}
+		{EF399B45-AAD6-4684-8AF5-8099D20E73B8} = {4FA4511C-AAA5-4248-ACB4-9F328470BE8F}
+		{FA22E063-49A2-4DD5-8A50-5D3B3F5C0E6D} = {5909CDE9-ADA9-4344-B454-226E5C8A4C12}
 	EndGlobalSection
 	GlobalSection(ExtensibilityGlobals) = postSolution
 		SolutionGuid = {DB11BB21-3F5D-42D6-8737-F19C0666F1F8}

src/ChangeDB.Agent.Sqlite/ChangeDB.Agent.Sqlite.csproj

@@ -0,0 +1,21 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+
+
+    <ItemGroup>
+      <!--<PackageReference Include="Microsoft.EntityFrameworkCore.Sqlite" Version="5.0.17" />-->
+        <PackageReference Include="Microsoft.Data.Sqlite" Version="6.0.5" />
+        <PackageReference Include="Microsoft.EntityFrameworkCore.Sqlite" Version="5.0.17" />
+    </ItemGroup>
+
+    <ItemGroup>
+      <ProjectReference Include="..\ChangeDB.Core\ChangeDB.Core.csproj" />
+    </ItemGroup>
+
+    <ItemGroup Condition="'$(TargetFramework)' == 'net6'">
+      <PackageReference Include="Microsoft.EntityFrameworkCore.Sqlite">
+        <Version>6.0.5</Version>
+      </PackageReference>
+    </ItemGroup>
+
+</Project>

src/ChangeDB.Agent.Sqlite/SqliteAgent.cs

@@ -0,0 +1,15 @@
+namespace ChangeDB.Agent.Sqlite
+{
+
+    public class SqliteAgent : BaseAgent
+    {
+        public override AgentSetting AgentSetting => new()
+        {
+            // there is no a limit itself for the object name.
+            ObjectNameMaxLength = 1024,
+            IdentityName = (_, table) => SqliteUtils.IdentityName(table),
+            DatabaseType = "sqlite"
+        };
+
+    }
+}

src/ChangeDB.Agent.Sqlite/SqliteConnectionProvider.cs

@@ -0,0 +1,13 @@
+using System.Data;
+using System.Data.Common;
+using Microsoft.Data.Sqlite;
+
+namespace ChangeDB.Agent.Sqlite
+{
+    public class SqliteConnectionProvider : IConnectionProvider
+    {
+        public static readonly IConnectionProvider Default = new SqliteConnectionProvider();
+
+        public DbConnection CreateConnection(string connectionString) => new SqliteConnection(connectionString);
+    }
+}

src/ChangeDB.Agent.Sqlite/SqliteDataDumper.cs

@@ -0,0 +1,21 @@
+using ChangeDB.Dump;
+using ChangeDB.Migration;
+
+namespace ChangeDB.Agent.Sqlite
+{
+    public class SqliteDataDumper : BaseDataDumper
+    {
+        public static readonly IDataDumper Default = new SqliteDataDumper();
+        protected override string IdentityName(string schema, string name)
+        {
+            return SqliteUtils.IdentityName(name);
+        }
+
+        protected override string ReprValue(ColumnDescriptor column, object val)
+        {
+            var dataType = SqliteDataTypeMapper.Default.ToDatabaseStoreType(column.DataType);
+
+            return SqliteRepr.ReprConstant(val, dataType);
+        }
+    }
+}

src/ChangeDB.Agent.Sqlite/SqliteDataMigrator.cs

@@ -0,0 +1,97 @@
+using System.Collections.Generic;
+using System.Data;
+using System.Linq;
+using System.Threading.Tasks;
+using ChangeDB.Migration;
+using Microsoft.Data.Sqlite;
+using static ChangeDB.Agent.Sqlite.SqliteUtils;
+
+namespace ChangeDB.Agent.Sqlite
+{
+    public class SqliteDataMigrator : BaseDataMigrator, IDataMigrator
+    {
+        public static readonly IDataMigrator Default = new SqliteDataMigrator();
+        private static readonly HashSet<CommonDataType> canNotOrderByTypes = new HashSet<CommonDataType>()
+        {
+            CommonDataType.Blob,
+            CommonDataType.Text,
+            CommonDataType.NText
+        };
+        private static string BuildColumnNames(IEnumerable<string> names) => string.Join(", ", names.Select(p => $"[{p}]"));
+
+        private static string BuildOrderByColumnNames(TableDescriptor table)
+        {
+            if (table.PrimaryKey?.Columns?.Count > 0)
+            {
+                return BuildColumnNames(table.PrimaryKey?.Columns.ToArray());
+            }
+
+            var names = table.Columns.Where(p => !canNotOrderByTypes.Contains(p.DataType.DbType)).Select(p => p.Name);
+
+            return BuildColumnNames(names);
+        }
+
+        public override Task<long> CountSourceTable(TableDescriptor table, AgentContext agentContext)
+        {
+            var sql = $"select count_big(1) from {IdentityName(table)}";
+            var val = agentContext.Connection.ExecuteScalar<long>(sql);
+            return Task.FromResult(val);
+        }
+
+        public override Task<DataTable> ReadSourceTable(TableDescriptor table, PageInfo pageInfo, AgentContext agentContext)
+        {
+            var sql =
+                $"select * from {IdentityName(table)} order by {BuildOrderByColumnNames(table)} offset {pageInfo.Offset} row fetch next {pageInfo.Limit} row only";
+            return Task.FromResult(agentContext.Connection.ExecuteReaderAsTable(sql));
+        }
+
+        public override Task BeforeWriteTable(TableDescriptor tableDescriptor, AgentContext agentContext)
+        {
+            if (tableDescriptor.Columns.Any(p => p.IdentityInfo != null))
+            {
+                agentContext.Connection.ExecuteNonQuery($"SET IDENTITY_INSERT {tableDescriptor.Name} ON");
+
+            }
+
+            return Task.CompletedTask;
+        }
+
+        public override Task AfterWriteTable(TableDescriptor tableDescriptor, AgentContext agentContext)
+        {
+            if (tableDescriptor.Columns.Any(p => p.IdentityInfo != null))
+            {
+                agentContext.Connection.ExecuteNonQuery($"SET IDENTITY_INSERT {tableDescriptor.Name} OFF");
+
+                tableDescriptor.Columns.Where(p => p.IdentityInfo?.CurrentValue != null)
+                    .Each((column) =>
+                    {
+                        agentContext.Connection.ExecuteNonQuery($"DBCC CHECKIDENT ('{tableDescriptor.Name}', RESEED, {column.IdentityInfo.CurrentValue})");
+                    });
+            }
+
+            return Task.CompletedTask;
+        }
+
+        protected override Task WriteTargetTableInDefaultMode(IAsyncEnumerable<DataTable> datas, TableDescriptor table, AgentContext agentContext)
+        {
+            return WriteTargetTableInBlockCopyMode(datas, table, agentContext);
+        }
+
+        protected override Task WriteTargetTableInBlockCopyMode(IAsyncEnumerable<DataTable> datas, TableDescriptor table, AgentContext agentContext)
+        {
+            //agentContext.Connection.TryOpen();
+            //var options = SqlBulkCopyOptions.Default | SqlBulkCopyOptions.KeepIdentity | SqlBulkCopyOptions.KeepNulls;
+            //await foreach (var datatable in datas)
+            //{
+            //    if (datatable.Rows.Count == 0) continue;
+            //    using var bulkCopy = new SqlBulkCopy(agentContext.Connection as SqlConnection, options, null)
+            //    {
+            //        DestinationTableName = IdentityName(table),
+            //        BatchSize = datatable.Rows.Count,
+            //    };
+            //    bulkCopy.WriteToServer(datatable);
+            //}
+            throw new System.NotImplementedException();
+        }
+    }
+}

src/ChangeDB.Agent.Sqlite/SqliteDatabaseManager.cs

@@ -0,0 +1,56 @@
+using System.Data;
+using System.Data.Common;
+using System.IO;
+using System.Threading.Tasks;
+using ChangeDB.Migration;
+using Microsoft.Data.Sqlite;
+using static ChangeDB.Agent.Sqlite.SqliteUtils;
+namespace ChangeDB.Agent.Sqlite
+{
+    public class SqliteDatabaseManager : IDatabaseManager
+    {
+        public static readonly IDatabaseManager Default = new SqliteDatabaseManager();
+
+        public async Task CreateDatabase(string connectionString, MigrationSetting migrationSetting)
+        {
+            await CreateDatabase(connectionString);
+        }
+
+        public Task DropTargetDatabaseIfExists(string connectionString, MigrationSetting migrationSetting)
+        {
+            DropDatabaseIfExists(connectionString);
+            return Task.CompletedTask;
+        }
+
+        public Task DropDatabaseIfExists(string connectionString)
+        {
+            var fileName = GetDatabaseName(connectionString);
+            lock (fileName)
+            {
+                if (File.Exists(fileName))
+                {
+                    SqliteConnection.ClearAllPools();
+                    File.Delete(fileName);
+                }
+            }
+            return Task.CompletedTask;
+        }
+
+
+        public async Task CreateDatabase(string connection)
+        {
+            using var conn = CreateNoDatabaseConnection(connection);
+            await conn.OpenAsync();
+        }
+
+        private static DbConnection CreateNoDatabaseConnection(string connection)
+        {
+            var builder = new SqliteConnectionStringBuilder(connection);
+            return new SqliteConnection(builder.ConnectionString);
+        }
+        private static string GetDatabaseName(string connectionString)
+        {
+            return new SqliteConnectionStringBuilder(connectionString).DataSource;
+        }
+    }
+}