Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Feature/migration from server cs #480

Merged
merged 3 commits into from
Aug 20, 2024
Merged

Feature/migration from server cs #480

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
16df415
migration from server cs
SuhorukovAnton Aug 19, 2024
5dba9c4
fix
SuhorukovAnton Aug 19, 2024
1d4ac6b
refactoring
SuhorukovAnton Aug 19, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions common/ASC.Data.Storage/DataOperators/IDataOperator.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,7 @@ public interface IDataWriteOperator : IAsyncDisposable
public interface IDataReadOperator : IDisposable
{
string GetFolder();
void SetFolder(string folder);
Stream GetEntry(string key);
IEnumerable<string> GetEntries(string key);
IEnumerable<string> GetDirectories(string key);
Expand Down
5 changes: 5 additions & 0 deletions common/ASC.Data.Storage/DataOperators/ReadOperators/BaseReadOperator.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,11 @@ public string GetFolder()
return _tmpdir;
}

public void SetFolder(string folder)
{
_tmpdir = folder;
}

public Stream GetEntry(string key)
{
var filePath = Path.Combine(_tmpdir, key);
Expand Down
6 changes: 3 additions & 3 deletions common/ASC.Migration/Migrators/Model/MigrationStorage.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,9 +27,9 @@
namespace ASC.Migration.Core.Migrators.Model;
public class MigrationStorage
{
public List<MigrationFolder> Folders { get; } = new List<MigrationFolder>();
public List<MigrationFile> Files { get; } = new List<MigrationFile>();
public List<MigrationSecurity> Securities { get; } = new List<MigrationSecurity>();
public List<MigrationFolder> Folders { get; set; } = new List<MigrationFolder>();
public List<MigrationFile> Files { get; set; } = new List<MigrationFile>();
public List<MigrationSecurity> Securities { get; set; } = new List<MigrationSecurity>();
public long BytesTotal { get; set; }
public FolderType Type { get; set; } = FolderType.USER;
public string RootKey { get; set; }
Expand Down
157 changes: 126 additions & 31 deletions common/ASC.Migration/Migrators/Provider/WorkspaceMigrator.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -82,65 +82,144 @@ public override async Task<MigrationApiInfo> ParseAsync(bool reportProgress = tr
{
var currentUser = SecurityContext.CurrentAccount;
_dataReader = DataOperatorFactory.GetReadOperator(_backup, reportProgress ? _cancellationToken : CancellationToken.None, false);

if (_cancellationToken.IsCancellationRequested && reportProgress)
{
return null;
}

if (reportProgress)
var folders = _dataReader.GetDirectories("");
if (folders.Any(f=> Path.GetFileNameWithoutExtension(f).Contains("databases")))
{
await ReportProgressAsync(30, MigrationResource.UnzippingFinished);
await InnerParseAsync(reportProgress);
}
await ParseUsersAsync();

if (reportProgress)
else
{
await ReportProgressAsync(70, MigrationResource.DataProcessing);
foreach (var folder in folders)
{
_dataReader.SetFolder(folder);
await InnerParseAsync(reportProgress, folders.Count());
}
}
ParseGroup();

if (reportProgress)
{
await ReportProgressAsync(80, MigrationResource.DataProcessing);
}
MigrationInfo.CommonStorage = new()
}
catch(Exception e)
{
MigrationInfo.FailedArchives.Add(Path.GetFileName(_backup));
var error = string.Format(MigrationResource.CanNotParseArchive, Path.GetFileNameWithoutExtension(_backup));
await ReportProgressAsync(_lastProgressUpdate, error);
throw new Exception(error, e);
}
if (reportProgress)
{
await ReportProgressAsync(100, MigrationResource.DataProcessingCompleted);
}
return MigrationInfo.ToApiInfo();
}

private async Task InnerParseAsync(bool reportProgress, int count = 1)
{
if (reportProgress)
{
await ReportProgressAsync(_lastProgressUpdate + (double)25 / count, MigrationResource.UnzippingFinished);
}
await ParseUsersAsync(reportProgress, count);

if (reportProgress)
{
await ReportProgressAsync(_lastProgressUpdate + (double)10 / count, MigrationResource.DataProcessing);
}
ParseGroup();

if (reportProgress)
{
await ReportProgressAsync(_lastProgressUpdate + (double)10 / count, MigrationResource.DataProcessing);
}

if (MigrationInfo.CommonStorage == null)
{
MigrationInfo.CommonStorage = new MigrationStorage()
{
Type = FolderType.COMMON
};
ParseStorage(MigrationInfo.CommonStorage);

if (reportProgress)
}
else
{
var commonStorage = new MigrationStorage()
{
await ReportProgressAsync(90, MigrationResource.DataProcessing);
}
MigrationInfo.ProjectStorage = new()
Type = FolderType.COMMON
};
ParseAndUnionStorage(commonStorage, MigrationInfo.CommonStorage);
}

if (reportProgress)
{
await ReportProgressAsync(_lastProgressUpdate + (double)10 / count, MigrationResource.DataProcessing);
}
if (MigrationInfo.ProjectStorage == null)
{
MigrationInfo.ProjectStorage = new MigrationStorage()
{
Type = FolderType.BUNCH
};
ParseStorage(MigrationInfo.ProjectStorage);
}
catch(Exception e)
else
{
MigrationInfo.FailedArchives.Add(Path.GetFileName(_backup));
var error = string.Format(MigrationResource.CanNotParseArchive, Path.GetFileNameWithoutExtension(_backup));
await ReportProgressAsync(_lastProgressUpdate, error);
throw new Exception(error, e);
var projectStorage = new MigrationStorage()
{
Type = FolderType.BUNCH
};
ParseAndUnionStorage(projectStorage, MigrationInfo.ProjectStorage);
}
if (reportProgress)
}

private void ParseAndUnionStorage(MigrationStorage newStorage, MigrationStorage destinationStorage, string key = "")
{
if (destinationStorage == null)
{
await ReportProgressAsync(100, MigrationResource.DataProcessingCompleted);
throw new ArgumentNullException("destinationStorage is null");
}
return MigrationInfo.ToApiInfo();

ParseStorage(newStorage, key);

newStorage.Folders = newStorage.Folders.Select(f =>
{
if (f.ParentId.ToString() == newStorage.RootKey)
{
f.ParentId = int.Parse(destinationStorage.RootKey);
}
return f;
}).ToList();

newStorage.Files = newStorage.Files.Select(f =>
{
if (f.Folder.ToString() == newStorage.RootKey)
{
f.Folder = int.Parse(destinationStorage.RootKey);
}
return f;
}).ToList();

destinationStorage.BytesTotal += newStorage.BytesTotal;
destinationStorage.Securities = destinationStorage.Securities.Union(newStorage.Securities).ToList();
destinationStorage.Files = destinationStorage.Files.Union(newStorage.Files).ToList();
destinationStorage.Folders = destinationStorage.Folders.Union(newStorage.Folders).ToList();
}

public async Task ParseUsersAsync()
public async Task ParseUsersAsync(bool reportProgress, int count)
{
await using var stream = _dataReader.GetEntry("databases/core/core_user");
var data = new DataTable();
data.ReadXml(stream);
var progressStep = 50 / data.Rows.Count;
var progressStep = (double)30 / count / data.Rows.Count;
foreach (var row in data.Rows.Cast<DataRow>())
{
if (reportProgress)
{
await ReportProgressAsync(_lastProgressUpdate + progressStep, MigrationResource.DataProcessing);
}
if (row["removed"].ToString() == "1" || row["removed"].ToString() == "True")
{
continue;
Expand Down Expand Up @@ -176,15 +255,31 @@ public async Task ParseUsersAsync()

u.Storage = new MigrationStorage { Type = FolderType.USER };

ParseStorage(u.Storage, key);

if (!(await UserManager.GetUserByEmailAsync(u.Info.Email)).Equals(Constants.LostUser))
{
MigrationInfo.ExistUsers.Add(key, u);
var user = MigrationInfo.ExistUsers.SingleOrDefault(eu => eu.Value.Info.Email == u.Info.Email);
if (user.Value == null)
{
ParseStorage(u.Storage, key);
MigrationInfo.ExistUsers.Add(key, u);
}
else
{
ParseAndUnionStorage(u.Storage, user.Value.Storage, key);
}
}
else
{
MigrationInfo.Users.Add(key, u);
var user = MigrationInfo.Users.SingleOrDefault(eu => eu.Value.Info.Email == u.Info.Email);
if (user.Value == null)
{
ParseStorage(u.Storage, key);
MigrationInfo.Users.Add(key, u);
}
else
{
ParseAndUnionStorage(u.Storage, user.Value.Storage, key);
}
}
}
}
Expand Down
Loading