diff --git a/common/ASC.Data.Storage/DataOperators/IDataOperator.cs b/common/ASC.Data.Storage/DataOperators/IDataOperator.cs index 1f3245caeb3..bd97df1af01 100644 --- a/common/ASC.Data.Storage/DataOperators/IDataOperator.cs +++ b/common/ASC.Data.Storage/DataOperators/IDataOperator.cs @@ -38,6 +38,7 @@ public interface IDataWriteOperator : IAsyncDisposable public interface IDataReadOperator : IDisposable { string GetFolder(); + void SetFolder(string folder); Stream GetEntry(string key); IEnumerable GetEntries(string key); IEnumerable GetDirectories(string key); diff --git a/common/ASC.Data.Storage/DataOperators/ReadOperators/BaseReadOperator.cs b/common/ASC.Data.Storage/DataOperators/ReadOperators/BaseReadOperator.cs index be6158f929f..0eb80db24f0 100644 --- a/common/ASC.Data.Storage/DataOperators/ReadOperators/BaseReadOperator.cs +++ b/common/ASC.Data.Storage/DataOperators/ReadOperators/BaseReadOperator.cs @@ -34,6 +34,11 @@ public string GetFolder() return _tmpdir; } + public void SetFolder(string folder) + { + _tmpdir = folder; + } + public Stream GetEntry(string key) { var filePath = Path.Combine(_tmpdir, key); diff --git a/common/ASC.Migration/Migrators/Model/MigrationStorage.cs b/common/ASC.Migration/Migrators/Model/MigrationStorage.cs index feedcbb0d27..0cd96b0c7e1 100644 --- a/common/ASC.Migration/Migrators/Model/MigrationStorage.cs +++ b/common/ASC.Migration/Migrators/Model/MigrationStorage.cs @@ -27,9 +27,9 @@ namespace ASC.Migration.Core.Migrators.Model; public class MigrationStorage { - public List Folders { get; } = new List(); - public List Files { get; } = new List(); - public List Securities { get; } = new List(); + public List Folders { get; set; } = new List(); + public List Files { get; set; } = new List(); + public List Securities { get; set; } = new List(); public long BytesTotal { get; set; } public FolderType Type { get; set; } = FolderType.USER; public string RootKey { get; set; } diff --git a/common/ASC.Migration/Migrators/Provider/WorkspaceMigrator.cs b/common/ASC.Migration/Migrators/Provider/WorkspaceMigrator.cs index c7b626a28cf..959969a9f02 100644 --- a/common/ASC.Migration/Migrators/Provider/WorkspaceMigrator.cs +++ b/common/ASC.Migration/Migrators/Provider/WorkspaceMigrator.cs @@ -82,65 +82,144 @@ public override async Task ParseAsync(bool reportProgress = tr { var currentUser = SecurityContext.CurrentAccount; _dataReader = DataOperatorFactory.GetReadOperator(_backup, reportProgress ? _cancellationToken : CancellationToken.None, false); + if (_cancellationToken.IsCancellationRequested && reportProgress) { return null; } - if (reportProgress) + var folders = _dataReader.GetDirectories(""); + if (folders.Any(f=> Path.GetFileNameWithoutExtension(f).Contains("databases"))) { - await ReportProgressAsync(30, MigrationResource.UnzippingFinished); + await InnerParseAsync(reportProgress); } - await ParseUsersAsync(); - - if (reportProgress) + else { - await ReportProgressAsync(70, MigrationResource.DataProcessing); + foreach (var folder in folders) + { + _dataReader.SetFolder(folder); + await InnerParseAsync(reportProgress, folders.Count()); + } } - ParseGroup(); - if (reportProgress) - { - await ReportProgressAsync(80, MigrationResource.DataProcessing); - } - MigrationInfo.CommonStorage = new() + } + catch(Exception e) + { + MigrationInfo.FailedArchives.Add(Path.GetFileName(_backup)); + var error = string.Format(MigrationResource.CanNotParseArchive, Path.GetFileNameWithoutExtension(_backup)); + await ReportProgressAsync(_lastProgressUpdate, error); + throw new Exception(error, e); + } + if (reportProgress) + { + await ReportProgressAsync(100, MigrationResource.DataProcessingCompleted); + } + return MigrationInfo.ToApiInfo(); + } + + private async Task InnerParseAsync(bool reportProgress, int count = 1) + { + if (reportProgress) + { + await ReportProgressAsync(_lastProgressUpdate + (double)25 / count, MigrationResource.UnzippingFinished); + } + await ParseUsersAsync(reportProgress, count); + + if (reportProgress) + { + await ReportProgressAsync(_lastProgressUpdate + (double)10 / count, MigrationResource.DataProcessing); + } + ParseGroup(); + + if (reportProgress) + { + await ReportProgressAsync(_lastProgressUpdate + (double)10 / count, MigrationResource.DataProcessing); + } + + if (MigrationInfo.CommonStorage == null) + { + MigrationInfo.CommonStorage = new MigrationStorage() { Type = FolderType.COMMON }; ParseStorage(MigrationInfo.CommonStorage); - - if (reportProgress) + } + else + { + var commonStorage = new MigrationStorage() { - await ReportProgressAsync(90, MigrationResource.DataProcessing); - } - MigrationInfo.ProjectStorage = new() + Type = FolderType.COMMON + }; + ParseAndUnionStorage(commonStorage, MigrationInfo.CommonStorage); + } + + if (reportProgress) + { + await ReportProgressAsync(_lastProgressUpdate + (double)10 / count, MigrationResource.DataProcessing); + } + if (MigrationInfo.ProjectStorage == null) + { + MigrationInfo.ProjectStorage = new MigrationStorage() { Type = FolderType.BUNCH }; ParseStorage(MigrationInfo.ProjectStorage); } - catch(Exception e) + else { - MigrationInfo.FailedArchives.Add(Path.GetFileName(_backup)); - var error = string.Format(MigrationResource.CanNotParseArchive, Path.GetFileNameWithoutExtension(_backup)); - await ReportProgressAsync(_lastProgressUpdate, error); - throw new Exception(error, e); + var projectStorage = new MigrationStorage() + { + Type = FolderType.BUNCH + }; + ParseAndUnionStorage(projectStorage, MigrationInfo.ProjectStorage); } - if (reportProgress) + } + + private void ParseAndUnionStorage(MigrationStorage newStorage, MigrationStorage destinationStorage, string key = "") + { + if (destinationStorage == null) { - await ReportProgressAsync(100, MigrationResource.DataProcessingCompleted); + throw new ArgumentNullException("destinationStorage is null"); } - return MigrationInfo.ToApiInfo(); + + ParseStorage(newStorage, key); + + newStorage.Folders = newStorage.Folders.Select(f => + { + if (f.ParentId.ToString() == newStorage.RootKey) + { + f.ParentId = int.Parse(destinationStorage.RootKey); + } + return f; + }).ToList(); + + newStorage.Files = newStorage.Files.Select(f => + { + if (f.Folder.ToString() == newStorage.RootKey) + { + f.Folder = int.Parse(destinationStorage.RootKey); + } + return f; + }).ToList(); + + destinationStorage.BytesTotal += newStorage.BytesTotal; + destinationStorage.Securities = destinationStorage.Securities.Union(newStorage.Securities).ToList(); + destinationStorage.Files = destinationStorage.Files.Union(newStorage.Files).ToList(); + destinationStorage.Folders = destinationStorage.Folders.Union(newStorage.Folders).ToList(); } - public async Task ParseUsersAsync() + public async Task ParseUsersAsync(bool reportProgress, int count) { await using var stream = _dataReader.GetEntry("databases/core/core_user"); var data = new DataTable(); data.ReadXml(stream); - var progressStep = 50 / data.Rows.Count; + var progressStep = (double)30 / count / data.Rows.Count; foreach (var row in data.Rows.Cast()) { + if (reportProgress) + { + await ReportProgressAsync(_lastProgressUpdate + progressStep, MigrationResource.DataProcessing); + } if (row["removed"].ToString() == "1" || row["removed"].ToString() == "True") { continue; @@ -176,15 +255,31 @@ public async Task ParseUsersAsync() u.Storage = new MigrationStorage { Type = FolderType.USER }; - ParseStorage(u.Storage, key); - if (!(await UserManager.GetUserByEmailAsync(u.Info.Email)).Equals(Constants.LostUser)) { - MigrationInfo.ExistUsers.Add(key, u); + var user = MigrationInfo.ExistUsers.SingleOrDefault(eu => eu.Value.Info.Email == u.Info.Email); + if (user.Value == null) + { + ParseStorage(u.Storage, key); + MigrationInfo.ExistUsers.Add(key, u); + } + else + { + ParseAndUnionStorage(u.Storage, user.Value.Storage, key); + } } else { - MigrationInfo.Users.Add(key, u); + var user = MigrationInfo.Users.SingleOrDefault(eu => eu.Value.Info.Email == u.Info.Email); + if (user.Value == null) + { + ParseStorage(u.Storage, key); + MigrationInfo.Users.Add(key, u); + } + else + { + ParseAndUnionStorage(u.Storage, user.Value.Storage, key); + } } } }