Open Closed

Importing/Exporting Users timeout #9361

User avatar
0
shodgson created

Hello,

We're using the microservice template with a blazor web app for our application. All are running with ABP v9.1.0

When trying to import a **small ** users list in the user management page of the administration section it works fine but when trying to import (or even export) a "large" list of users (5000+) we get timeouts.

5000 users is not a lot and we expect the framework to be able to import them without any issues.

Is there any known limitations in terms of how many users should we import at a time?

Thanks

Go to accepted answer
6 Answer(s)
  • User Avatar
    0
    maliming created
    Support Team Fullstack Developer

    hi

    I will share a solution, Wait a sec.

    Thanks.

  • User Avatar
    0
    maliming created
    Support Team Fullstack Developer

    hi

    Can you try to override the ImportUsersFromFileAsync method of IdentityUserAppService to support batch import?

    We will change it in the next version.

    Thanks.

    [Authorize(IdentityPermissions.Users.Import)]
public virtual async Task<ImportUsersFromFileOutput> ImportUsersFromFileAsync(ImportUsersFromFileInputWithStream input)
{
    await IdentityOptions.SetAsync();

    var stream = new MemoryStream();
    await input.File.GetStream().CopyToAsync(stream);

    var invalidUsers = new List<InvalidImportUsersFromFileDto>();
    List<InvalidImportUsersFromFileDto> waitingImportUsers;
    try
    {
        IConfiguration configuration = null;
        if (input.FileType == ImportUsersFromFileType.Csv)
        {
            configuration = new CsvConfiguration { Seperator = ';' };
        }

        waitingImportUsers = (await stream.QueryAsync<InvalidImportUsersFromFileDto>(excelType: input.FileType == ImportUsersFromFileType.Excel ? ExcelType.XLSX : ExcelType.CSV, configuration: configuration)).ToList();
    }
    catch (Exception)
    {
        throw new BusinessException(IdentityProErrorCodes.InvalidImportFileFormat);
    }

    if (!waitingImportUsers.Any())
    {
        throw new BusinessException(IdentityProErrorCodes.NoUserFoundInFile);
    }

    var resultDto = new ImportUsersFromFileOutput
    {
        AllCount = waitingImportUsers.Count
    };

    const int batchSize = 3;
    var totalUsers = waitingImportUsers.Count;
    var batchCount = (int)Math.Ceiling((double)totalUsers / batchSize);
    for (var batchIndex = 0; batchIndex < batchCount; batchIndex++)
    {
        var currentBatch = waitingImportUsers
            .Skip(batchIndex * batchSize)
            .Take(batchSize)
            .ToList();
        var (invalidBatchUsers, successBatchUsers) = await ImportUsersAsync(currentBatch);
        invalidUsers.AddRange(invalidBatchUsers);
        if (invalidBatchUsers.Any() && successBatchUsers.Any())
        {
            var (invalidBatchUsers2, successBatchUsers2) = await ImportUsersAsync(successBatchUsers);
            if (invalidBatchUsers2.Any())
            {
                invalidUsers.AddRange(invalidBatchUsers2);
                invalidUsers.AddRange(successBatchUsers2);
            }
        }
    }

    if (invalidUsers.Any())
    {
        var token = Guid.NewGuid().ToString("N");

        await ImportInvalidUsersCache.SetAsync(
            token,
            new ImportInvalidUsersCacheItem
            {
                Token = token,
                InvalidUsers = invalidUsers,
                FileType = input.FileType
            },
            new DistributedCacheEntryOptions
            {
                AbsoluteExpirationRelativeToNow = TimeSpan.FromMinutes(1)
            });

        resultDto.InvalidUsersDownloadToken = token;
    }

    resultDto.SucceededCount = resultDto.AllCount - invalidUsers.Count;
    resultDto.FailedCount = invalidUsers.Count;

    return resultDto;
}

protected virtual async Task<(List<InvalidImportUsersFromFileDto>, List<InvalidImportUsersFromFileDto>)> ImportUsersAsync(List<InvalidImportUsersFromFileDto> users)
{
    var hasException = false;
    var invalidUsers = new List<InvalidImportUsersFromFileDto>();
    var successUsers = new List<InvalidImportUsersFromFileDto>();
    using (var uow = UnitOfWorkManager.Begin(requiresNew: true, isTransactional: true))
    {
        foreach (var waitingImportUser in users)
        {
            try
            {
                var user = new IdentityUser(
                    GuidGenerator.Create(),
                    waitingImportUser.UserName,
                    waitingImportUser.EmailAddress,
                    CurrentTenant.Id
                )
                {
                    Surname = waitingImportUser.Surname,
                    Name = waitingImportUser.Name
                };

                if (!waitingImportUser.PhoneNumber.IsNullOrWhiteSpace())
                {
                    user.SetPhoneNumber(waitingImportUser.PhoneNumber, false);
                }

                if (!waitingImportUser.Password.IsNullOrWhiteSpace())
                {
                    (await UserManager.CreateAsync(user, waitingImportUser.Password)).CheckErrors();
                }
                else
                {
                    (await UserManager.CreateAsync(user)).CheckErrors();
                }

                if (!waitingImportUser.AssignedRoleNames.IsNullOrWhiteSpace())
                {
                    (await UserManager.SetRolesAsync(user,
                        waitingImportUser.AssignedRoleNames.Split(new[] { ",", ";" },
                            StringSplitOptions.TrimEntries | StringSplitOptions.RemoveEmptyEntries))).CheckErrors();
                }

                if (!waitingImportUser.AssignedOrganizationUnitNames.IsNullOrWhiteSpace())
                {
                    var ouNames = waitingImportUser.AssignedOrganizationUnitNames.Split(new[] { ",", ";" },
                            StringSplitOptions.TrimEntries | StringSplitOptions.RemoveEmptyEntries).ToArray();
                    var ous = await OrganizationUnitRepository.GetListByDisplayNamesAsync(ouNames);
                    if (ous.Any())
                    {
                        await UserManager.SetOrganizationUnitsAsync(user, ous.Select(x => x.Id).ToArray());
                    }
                }

                successUsers.Add(waitingImportUser);
            }
            catch (Exception e)
            {
                hasException = true;

                waitingImportUser.ErrorReason = e is UserFriendlyException ? e.Message : e.ToString();
                invalidUsers.Add(waitingImportUser);
                Logger.LogWarning(e, $"Import user failed: {waitingImportUser}");
            }
        }

        await (hasException ? uow.RollbackAsync() : uow.CompleteAsync());
    }

    return (invalidUsers, successUsers);
}
  • User Avatar
    0
    shodgson created

    hi

    Can you try to override the ImportUsersFromFileAsync method of IdentityUserAppService to support batch import?

    We will change it in the next version.

    Thanks.

    [Authorize(IdentityPermissions.Users.Import)] 
public virtual async Task<ImportUsersFromFileOutput> ImportUsersFromFileAsync(ImportUsersFromFileInputWithStream input) 
{ 
    await IdentityOptions.SetAsync(); 
 
    var stream = new MemoryStream(); 
    await input.File.GetStream().CopyToAsync(stream); 
 
    var invalidUsers = new List<InvalidImportUsersFromFileDto>(); 
    List<InvalidImportUsersFromFileDto> waitingImportUsers; 
    try 
    { 
        IConfiguration configuration = null; 
        if (input.FileType == ImportUsersFromFileType.Csv) 
        { 
            configuration = new CsvConfiguration { Seperator = ';' }; 
        } 
 
        waitingImportUsers = (await stream.QueryAsync<InvalidImportUsersFromFileDto>(excelType: input.FileType == ImportUsersFromFileType.Excel ? ExcelType.XLSX : ExcelType.CSV, configuration: configuration)).ToList(); 
    } 
    catch (Exception) 
    { 
        throw new BusinessException(IdentityProErrorCodes.InvalidImportFileFormat); 
    } 
 
    if (!waitingImportUsers.Any()) 
    { 
        throw new BusinessException(IdentityProErrorCodes.NoUserFoundInFile); 
    } 
 
    var resultDto = new ImportUsersFromFileOutput 
    { 
        AllCount = waitingImportUsers.Count 
    }; 
 
    const int batchSize = 3; 
    var totalUsers = waitingImportUsers.Count; 
    var batchCount = (int)Math.Ceiling((double)totalUsers / batchSize); 
    for (var batchIndex = 0; batchIndex < batchCount; batchIndex++) 
    { 
        var currentBatch = waitingImportUsers 
            .Skip(batchIndex * batchSize) 
            .Take(batchSize) 
            .ToList(); 
        var (invalidBatchUsers, successBatchUsers) = await ImportUsersAsync(currentBatch); 
        invalidUsers.AddRange(invalidBatchUsers); 
        if (invalidBatchUsers.Any() && successBatchUsers.Any()) 
        { 
            var (invalidBatchUsers2, successBatchUsers2) = await ImportUsersAsync(successBatchUsers); 
            if (invalidBatchUsers2.Any()) 
            { 
                invalidUsers.AddRange(invalidBatchUsers2); 
                invalidUsers.AddRange(successBatchUsers2); 
            } 
        } 
    } 
 
    if (invalidUsers.Any()) 
    { 
        var token = Guid.NewGuid().ToString("N"); 
 
        await ImportInvalidUsersCache.SetAsync( 
            token, 
            new ImportInvalidUsersCacheItem 
            { 
                Token = token, 
                InvalidUsers = invalidUsers, 
                FileType = input.FileType 
            }, 
            new DistributedCacheEntryOptions 
            { 
                AbsoluteExpirationRelativeToNow = TimeSpan.FromMinutes(1) 
            }); 
 
        resultDto.InvalidUsersDownloadToken = token; 
    } 
 
    resultDto.SucceededCount = resultDto.AllCount - invalidUsers.Count; 
    resultDto.FailedCount = invalidUsers.Count; 
 
    return resultDto; 
} 
 
protected virtual async Task<(List<InvalidImportUsersFromFileDto>, List<InvalidImportUsersFromFileDto>)> ImportUsersAsync(List<InvalidImportUsersFromFileDto> users) 
{ 
    var hasException = false; 
    var invalidUsers = new List<InvalidImportUsersFromFileDto>(); 
    var successUsers = new List<InvalidImportUsersFromFileDto>(); 
    using (var uow = UnitOfWorkManager.Begin(requiresNew: true, isTransactional: true)) 
    { 
        foreach (var waitingImportUser in users) 
        { 
            try 
            { 
                var user = new IdentityUser( 
                    GuidGenerator.Create(), 
                    waitingImportUser.UserName, 
                    waitingImportUser.EmailAddress, 
                    CurrentTenant.Id 
                ) 
                { 
                    Surname = waitingImportUser.Surname, 
                    Name = waitingImportUser.Name 
                }; 
 
                if (!waitingImportUser.PhoneNumber.IsNullOrWhiteSpace()) 
                { 
                    user.SetPhoneNumber(waitingImportUser.PhoneNumber, false); 
                } 
 
                if (!waitingImportUser.Password.IsNullOrWhiteSpace()) 
                { 
                    (await UserManager.CreateAsync(user, waitingImportUser.Password)).CheckErrors(); 
                } 
                else 
                { 
                    (await UserManager.CreateAsync(user)).CheckErrors(); 
                } 
 
                if (!waitingImportUser.AssignedRoleNames.IsNullOrWhiteSpace()) 
                { 
                    (await UserManager.SetRolesAsync(user, 
                        waitingImportUser.AssignedRoleNames.Split(new[] { ",", ";" }, 
                            StringSplitOptions.TrimEntries | StringSplitOptions.RemoveEmptyEntries))).CheckErrors(); 
                } 
 
                if (!waitingImportUser.AssignedOrganizationUnitNames.IsNullOrWhiteSpace()) 
                { 
                    var ouNames = waitingImportUser.AssignedOrganizationUnitNames.Split(new[] { ",", ";" }, 
                            StringSplitOptions.TrimEntries | StringSplitOptions.RemoveEmptyEntries).ToArray(); 
                    var ous = await OrganizationUnitRepository.GetListByDisplayNamesAsync(ouNames); 
                    if (ous.Any()) 
                    { 
                        await UserManager.SetOrganizationUnitsAsync(user, ous.Select(x => x.Id).ToArray()); 
                    } 
                } 
 
                successUsers.Add(waitingImportUser); 
            } 
            catch (Exception e) 
            { 
                hasException = true; 
 
                waitingImportUser.ErrorReason = e is UserFriendlyException ? e.Message : e.ToString(); 
                invalidUsers.Add(waitingImportUser); 
                Logger.LogWarning(e, $"Import user failed: {waitingImportUser}"); 
            } 
        } 
 
        await (hasException ? uow.RollbackAsync() : uow.CompleteAsync()); 
    } 
 
    return (invalidUsers, successUsers); 
}

    Thanks for the update. I'll try that.

    Any workaround for the Export as well?

  • User Avatar
    0
    maliming created
    Support Team Fullstack Developer

    hi

    You can try to set command timeout to 60 seconds or more

    public override void ConfigureServices(ServiceConfigurationContext context)
{
    //...

    Configure<AbpDbContextOptions>(options =>
    {
        options.UseSqlServer(x => x.CommandTimeout(60));
    });
    
    //...
}
  • User Avatar
    0
    shodgson created

    [maliming] said: hi

    You can try to set command timeout to 60 seconds or more

    public override void ConfigureServices(ServiceConfigurationContext context) 
{ 
    //... 
 
    Configure<AbpDbContextOptions>(options => 
    { 
        options.UseSqlServer(x => x.CommandTimeout(60)); 
    }); 
     
    //... 
}

    Hello,

    I tried to set the command timeout to 2 minutes and I face the same issue regardless after a minute. After a minute the export-as-csv call fails and returns a 401 Unauthorized. Here's the stack trace I get:

    [14:51:15 INF] Executing action method Volo.Abp.Identity.IdentityUserController.GetListAsCsvFileAsync (Volo.Abp.Identity.Pro.HttpApi) - Validation state: Valid
[14:51:15 WRN] ---------- RemoteServiceErrorInfo ----------
{
  "code": null,
  "message": "An internal error occurred during your request!",
  "details": null,
  "data": null,
  "validationErrors": null
}

[14:51:15 WRN] Invalid download token: 1d7c6dcbc0854168a1c9abc4a2323f23
Volo.Abp.Authorization.AbpAuthorizationException: Invalid download token: 1d7c6dcbc0854168a1c9abc4a2323f23
   at Volo.Abp.Identity.IdentityUserAppService.CheckDownloadTokenAsync(String token, Boolean isInvalidUsersToken)
   at Volo.Abp.Identity.IdentityUserAppService.GetExportUsersAsync(GetIdentityUserListAsFileInput input)
   at Volo.Abp.Identity.IdentityUserAppService.GetListAsCsvFileAsync(GetIdentityUserListAsFileInput input)
   at Castle.DynamicProxy.AsyncInterceptorBase.ProceedAsynchronous[TResult](IInvocation invocation, IInvocationProceedInfo proceedInfo)
   at Volo.Abp.Castle.DynamicProxy.CastleAbpMethodInvocationAdapterWithReturnValue`1.ProceedAsync()
   at Volo.Abp.GlobalFeatures.GlobalFeatureInterceptor.InterceptAsync(IAbpMethodInvocation invocation)
   at Volo.Abp.Castle.DynamicProxy.CastleAsyncAbpInterceptorAdapter`1.InterceptAsync[TResult](IInvocation invocation, IInvocationProceedInfo proceedInfo, Func`3 proceed)
   at Castle.DynamicProxy.AsyncInterceptorBase.ProceedAsynchronous[TResult](IInvocation invocation, IInvocationProceedInfo proceedInfo)
   at Volo.Abp.Castle.DynamicProxy.CastleAbpMethodInvocationAdapterWithReturnValue`1.ProceedAsync()
   at Volo.Abp.Authorization.AuthorizationInterceptor.InterceptAsync(IAbpMethodInvocation invocation)
   at Volo.Abp.Castle.DynamicProxy.CastleAsyncAbpInterceptorAdapter`1.InterceptAsync[TResult](IInvocation invocation, IInvocationProceedInfo proceedInfo, Func`3 proceed)
   at Castle.DynamicProxy.AsyncInterceptorBase.ProceedAsynchronous[TResult](IInvocation invocation, IInvocationProceedInfo proceedInfo)
   at Volo.Abp.Castle.DynamicProxy.CastleAbpMethodInvocationAdapterWithReturnValue`1.ProceedAsync()
   at Volo.Abp.Auditing.AuditingInterceptor.ProceedByLoggingAsync(IAbpMethodInvocation invocation, AbpAuditingOptions options, IAuditingHelper auditingHelper, IAuditLogScope auditLogScope)
   at Volo.Abp.Auditing.AuditingInterceptor.InterceptAsync(IAbpMethodInvocation invocation)
   at Volo.Abp.Castle.DynamicProxy.CastleAsyncAbpInterceptorAdapter`1.InterceptAsync[TResult](IInvocation invocation, IInvocationProceedInfo proceedInfo, Func`3 proceed)
   at Castle.DynamicProxy.AsyncInterceptorBase.ProceedAsynchronous[TResult](IInvocation invocation, IInvocationProceedInfo proceedInfo)
   at Volo.Abp.Castle.DynamicProxy.CastleAbpMethodInvocationAdapterWithReturnValue`1.ProceedAsync()
   at Volo.Abp.Validation.ValidationInterceptor.InterceptAsync(IAbpMethodInvocation invocation)
   at Volo.Abp.Castle.DynamicProxy.CastleAsyncAbpInterceptorAdapter`1.InterceptAsync[TResult](IInvocation invocation, IInvocationProceedInfo proceedInfo, Func`3 proceed)
   at Castle.DynamicProxy.AsyncInterceptorBase.ProceedAsynchronous[TResult](IInvocation invocation, IInvocationProceedInfo proceedInfo)
   at Volo.Abp.Castle.DynamicProxy.CastleAbpMethodInvocationAdapterWithReturnValue`1.ProceedAsync()
   at Volo.Abp.Uow.UnitOfWorkInterceptor.InterceptAsync(IAbpMethodInvocation invocation)
   at Volo.Abp.Castle.DynamicProxy.CastleAsyncAbpInterceptorAdapter`1.InterceptAsync[TResult](IInvocation invocation, IInvocationProceedInfo proceedInfo, Func`3 proceed)
   at lambda_method4261(Closure, Object)
   at Microsoft.AspNetCore.Mvc.Infrastructure.ActionMethodExecutor.AwaitableObjectResultExecutor.Execute(ActionContext actionContext, IActionResultTypeMapper mapper, ObjectMethodExecutor executor, Object controller, Object[] arguments)
   at Microsoft.AspNetCore.Mvc.Infrastructure.ControllerActionInvoker.&lt;InvokeActionMethodAsync&gt;g__Logged|12_1(ControllerActionInvoker invoker)
   at Microsoft.AspNetCore.Mvc.Infrastructure.ControllerActionInvoker.&lt;InvokeNextActionFilterAsync&gt;g__Awaited|10_0(ControllerActionInvoker invoker, Task lastTask, State next, Scope scope, Object state, Boolean isCompleted)
   at Microsoft.AspNetCore.Mvc.Infrastructure.ControllerActionInvoker.Rethrow(ActionExecutedContextSealed context)
   at Microsoft.AspNetCore.Mvc.Infrastructure.ControllerActionInvoker.Next(State& next, Scope& scope, Object& state, Boolean& isCompleted)
   at Microsoft.AspNetCore.Mvc.Infrastructure.ControllerActionInvoker.&lt;InvokeInnerFilterAsync&gt;g__Awaited|13_0(ControllerActionInvoker invoker, Task lastTask, State next, Scope scope, Object state, Boolean isCompleted)
   at Microsoft.AspNetCore.Mvc.Infrastructure.ResourceInvoker.&lt;InvokeNextExceptionFilterAsync&gt;g__Awaited|26_0(ResourceInvoker invoker, Task lastTask, State next, Scope scope, Object state, Boolean isCompleted)
[14:51:15 WRN] Code:
[14:51:15 INF] AuthenticationScheme: Bearer was challenged.
[14:51:15 INF] Executed action Volo.Abp.Identity.IdentityUserController.GetListAsCsvFileAsync (Volo.Abp.Identity.Pro.HttpApi) in 68.2107ms
[14:51:15 INF] Executed endpoint 'Volo.Abp.Identity.IdentityUserController.GetListAsCsvFileAsync (Volo.Abp.Identity.Pro.HttpApi)'
[14:51:15 INF] AUDIT LOG: [401: GET    ] /api/identity/users/export-as-csv
  • User Avatar
    0
    maliming created
    Support Team Fullstack Developer

    hi

    The download token AbsoluteExpirationRelativeToNow is short. You can increase the time

    public virtual async Task<DownloadTokenResultDto> GetDownloadTokenAsync()
{
    var token = Guid.NewGuid().ToString("N");

    await DownloadTokenCache.SetAsync(
        token,
        new IdentityUserDownloadTokenCacheItem { Token = token, TenantId = CurrentTenant.Id },
        new DistributedCacheEntryOptions
        {
            AbsoluteExpirationRelativeToNow = TimeSpan.FromSeconds(30)
        });

    return new DownloadTokenResultDto
    {
        Token = token
    };
}

     if (invalidUsers.Any())
{
    var token = Guid.NewGuid().ToString("N");

    await ImportInvalidUsersCache.SetAsync(
        token,
        new ImportInvalidUsersCacheItem
        {
            Token = token,
            InvalidUsers = invalidUsers,
            FileType = input.FileType
        },
        new DistributedCacheEntryOptions
        {
            AbsoluteExpirationRelativeToNow = TimeSpan.FromMinutes(1)
        });

    resultDto.InvalidUsersDownloadToken = token;
}
Assignee
User avatar maliming
Question details
Labels
None yet
Boost Your Development
ABP Live Training
Packages
See Trainings
Mastering ABP Framework Book
The Official Guide
Mastering
ABP Framework
Learn More
Mastering ABP Framework Book
Made with ❤️ on ABP v10.0.0-preview. Updated on September 01, 2025, 08:37