Веб-API Azure, число потоков, растущих под нагрузкой, большинство из них ожидают SqlConnection.Close для завершения
Мы запускаем приложение веб-API в Azure. Иногда под нагрузкой приложение начинает увеличивать количество рабочих потоков.
Мы не можем воспроизвести проблему в среде, отличной от производства, и только тогда, когда это происходит (все еще ищем причину).
Используя Azure DaaS, мы смогли получить дампы процесса w3wp в этих ситуациях, и большинство потоков ожидает SqlConnection.Close завершить.
Точно, все эти темы застряли в этой строке в SqlInternalConnectionTds.ObtainAdditionalLocksForClose, В разных случаях у нас от 300 до 1000+ потоков, как это.
Хотите знать, что может привести к ситуации, когда SqlConnection.Close заблокирован или застрял или занимает много времени?
Завершите стек вызовов этих потоков ниже или в виде сущности:
[Managed to Native Transition]
mscorlib.dll!System.Threading.SemaphoreSlim.WaitUntilCountOrTimeout(int millisecondsTimeout, uint startTime, System.Threading.CancellationToken cancellationToken) Unknown
mscorlib.dll!System.Threading.SemaphoreSlim.Wait(int millisecondsTimeout, System.Threading.CancellationToken cancellationToken) Unknown
mscorlib.dll!System.Threading.SemaphoreSlim.Wait() Unknown
System.Data.dll!System.Data.SqlClient.SqlInternalConnectionTds.SyncAsyncLock.Wait(bool canReleaseFromAnyThread) Unknown
System.Data.dll!System.Data.SqlClient.SqlInternalConnectionTds.ObtainAdditionalLocksForClose() Unknown
System.Data.dll!System.Data.ProviderBase.DbConnectionInternal.CloseConnection(System.Data.Common.DbConnection owningObject, System.Data.ProviderBase.DbConnectionFactory connectionFactory) Unknown
System.Data.dll!System.Data.SqlClient.SqlConnection.Close() Unknown
EntityFramework.dll!System.Data.Entity.Infrastructure.Interception.InternalDispatcher<System.Data.Entity.Infrastructure.Interception.IDbConnectionInterceptor>.Dispatch<System.Data.Common.DbConnection, System.Data.Entity.Infrastructure.Interception.DbConnectionInterceptionContext>(System.Data.Common.DbConnection target, System.Action<System.Data.Common.DbConnection, System.Data.Entity.Infrastructure.Interception.DbConnectionInterceptionContext> operation, System.Data.Entity.Infrastructure.Interception.DbConnectionInterceptionContext interceptionContext, System.Action<System.Data.Entity.Infrastructure.Interception.IDbConnectionInterceptor, System.Data.Common.DbConnection, System.Data.Entity.Infrastructure.Interception.DbConnectionInterceptionContext> executing, System.Action<System.Data.Entity.Infrastructure.Interception.IDbConnectionInterceptor, System.Data.Common.DbConnection, System.Data.Entity.Infrastructure.Interception.DbConnectionInterceptionContext> executed) Unknown
EntityFramework.dll!System.Data.Entity.Infrastructure.Interception.DbConnectionDispatcher.Close(System.Data.Common.DbConnection connection, System.Data.Entity.Infrastructure.Interception.DbInterceptionContext interceptionContext) Unknown
EntityFramework.dll!System.Data.Entity.Core.EntityClient.EntityConnection.StoreCloseHelper() Unknown
EntityFramework.dll!System.Data.Entity.Core.Objects.ObjectContext.ReleaseConnection() Unknown
EntityFramework.dll!System.Data.Entity.Core.Common.Internal.Materialization.Shaper<*.WebApi.Models.Database.T_Table>.Finally() Unknown
EntityFramework.dll!System.Data.Entity.Core.Common.Internal.Materialization.Shaper<System.__Canon>.ObjectQueryNestedEnumerator.Dispose() Unknown
EntityFramework.dll!System.Data.Entity.Internal.LazyAsyncEnumerator<System.__Canon>.Dispose() Unknown
EntityFramework.dll!System.Data.Entity.Infrastructure.IDbAsyncEnumerableExtensions.FirstOrDefaultAsync<*.WebApi.Models.Database.T_Table>(System.Data.Entity.Infrastructure.IDbAsyncEnumerable<*.WebApi.Models.Database.T_Table> source, System.Threading.CancellationToken cancellationToken) Unknown
mscorlib.dll!System.Threading.ExecutionContext.RunInternal(System.Threading.ExecutionContext executionContext, System.Threading.ContextCallback callback, object state, bool preserveSyncCtx) Unknown
mscorlib.dll!System.Threading.ExecutionContext.Run(System.Threading.ExecutionContext executionContext, System.Threading.ContextCallback callback, object state, bool preserveSyncCtx) Unknown
mscorlib.dll!System.Runtime.CompilerServices.AsyncMethodBuilderCore.MoveNextRunner.Run() Unknown
EntityFramework.dll!System.Data.Entity.Utilities.TaskExtensions.CultureAwaiter<bool>.UnsafeOnCompleted.AnonymousMethod__0() Unknown
mscorlib.dll!System.Threading.Tasks.AwaitTaskContinuation.RunOrScheduleAction(System.Action action, bool allowInlining, ref System.Threading.Tasks.Task currentTask) Unknown
mscorlib.dll!System.Threading.Tasks.Task.FinishContinuations() Unknown
mscorlib.dll!System.Threading.Tasks.Task<bool>.TrySetResult(bool result) Unknown
mscorlib.dll!System.Runtime.CompilerServices.AsyncTaskMethodBuilder<bool>.SetResult(bool result) Unknown
EntityFramework.dll!System.Data.Entity.Internal.LazyAsyncEnumerator<*.WebApi.Models.Database.T_Table>.FirstMoveNextAsync(System.Threading.CancellationToken cancellationToken) Unknown
mscorlib.dll!System.Threading.ExecutionContext.RunInternal(System.Threading.ExecutionContext executionContext, System.Threading.ContextCallback callback, object state, bool preserveSyncCtx) Unknown
mscorlib.dll!System.Threading.ExecutionContext.Run(System.Threading.ExecutionContext executionContext, System.Threading.ContextCallback callback, object state, bool preserveSyncCtx) Unknown
mscorlib.dll!System.Runtime.CompilerServices.AsyncMethodBuilderCore.MoveNextRunner.Run() Unknown
EntityFramework.dll!System.Data.Entity.Utilities.TaskExtensions.CultureAwaiter<System.__Canon>.UnsafeOnCompleted.AnonymousMethod__0() Unknown
mscorlib.dll!System.Threading.Tasks.AwaitTaskContinuation.RunOrScheduleAction(System.Action action, bool allowInlining, ref System.Threading.Tasks.Task currentTask) Unknown
mscorlib.dll!System.Threading.Tasks.Task.FinishContinuations() Unknown
mscorlib.dll!System.Threading.Tasks.Task<System.__Canon>.TrySetResult(System.__Canon result) Unknown
mscorlib.dll!System.Runtime.CompilerServices.AsyncTaskMethodBuilder<System.Data.Entity.Core.Objects.ObjectResult<*.WebApi.Models.Database.T_Table>>.SetResult(System.Data.Entity.Core.Objects.ObjectResult<*.WebApi.Models.Database.T_Table> result) Unknown
EntityFramework.dll!System.Data.Entity.Core.Objects.ObjectQuery<*.WebApi.Models.Database.T_Table>.GetResultsAsync(System.Data.Entity.Core.Objects.MergeOption? forMergeOption, System.Data.Entity.Infrastructure.IDbExecutionStrategy executionStrategy, System.Threading.CancellationToken cancellationToken) Unknown
mscorlib.dll!System.Threading.ExecutionContext.RunInternal(System.Threading.ExecutionContext executionContext, System.Threading.ContextCallback callback, object state, bool preserveSyncCtx) Unknown
mscorlib.dll!System.Threading.ExecutionContext.Run(System.Threading.ExecutionContext executionContext, System.Threading.ContextCallback callback, object state, bool preserveSyncCtx) Unknown
mscorlib.dll!System.Runtime.CompilerServices.AsyncMethodBuilderCore.MoveNextRunner.Run() Unknown
EntityFramework.dll!System.Data.Entity.Utilities.TaskExtensions.CultureAwaiter<System.__Canon>.UnsafeOnCompleted.AnonymousMethod__0() Unknown
mscorlib.dll!System.Threading.Tasks.AwaitTaskContinuation.RunOrScheduleAction(System.Action action, bool allowInlining, ref System.Threading.Tasks.Task currentTask) Unknown
mscorlib.dll!System.Threading.Tasks.Task.FinishContinuations() Unknown
mscorlib.dll!System.Threading.Tasks.Task<System.__Canon>.TrySetResult(System.__Canon result) Unknown
mscorlib.dll!System.Runtime.CompilerServices.AsyncTaskMethodBuilder<System.Data.Entity.Core.Objects.ObjectResult<*.WebApi.Models.Database.T_Table>>.SetResult(System.Data.Entity.Core.Objects.ObjectResult<*.WebApi.Models.Database.T_Table> result) Unknown
EntityFramework.SqlServer.dll!System.Data.Entity.SqlServer.DefaultSqlExecutionStrategy.ExecuteAsyncImplementation<System.Data.Entity.Core.Objects.ObjectResult<*.WebApi.Models.Database.T_Table>>(System.Func<System.Threading.Tasks.Task<System.Data.Entity.Core.Objects.ObjectResult<*.WebApi.Models.Database.T_Table>>> func) Unknown
mscorlib.dll!System.Threading.ExecutionContext.RunInternal(System.Threading.ExecutionContext executionContext, System.Threading.ContextCallback callback, object state, bool preserveSyncCtx) Unknown
mscorlib.dll!System.Threading.ExecutionContext.Run(System.Threading.ExecutionContext executionContext, System.Threading.ContextCallback callback, object state, bool preserveSyncCtx) Unknown
mscorlib.dll!System.Runtime.CompilerServices.AsyncMethodBuilderCore.MoveNextRunner.Run() Unknown
mscorlib.dll!System.Threading.Tasks.AwaitTaskContinuation.RunOrScheduleAction(System.Action action, bool allowInlining, ref System.Threading.Tasks.Task currentTask) Unknown
mscorlib.dll!System.Threading.Tasks.Task.FinishContinuations() Unknown
mscorlib.dll!System.Threading.Tasks.Task<System.__Canon>.TrySetResult(System.__Canon result) Unknown
mscorlib.dll!System.Runtime.CompilerServices.AsyncTaskMethodBuilder<System.Data.Entity.Core.Objects.ObjectResult<*.WebApi.Models.Database.T_Table>>.SetResult(System.Data.Entity.Core.Objects.ObjectResult<*.WebApi.Models.Database.T_Table> result) Unknown
EntityFramework.dll!System.Data.Entity.Core.Objects.ObjectContext.ExecuteInTransactionAsync<System.Data.Entity.Core.Objects.ObjectResult<*.WebApi.Models.Database.T_Table>>(System.Func<System.Threading.Tasks.Task<System.Data.Entity.Core.Objects.ObjectResult<*.WebApi.Models.Database.T_Table>>> func, System.Data.Entity.Infrastructure.IDbExecutionStrategy executionStrategy, bool startLocalTransaction, bool releaseConnectionOnSuccess, System.Threading.CancellationToken cancellationToken) Unknown
mscorlib.dll!System.Threading.ExecutionContext.RunInternal(System.Threading.ExecutionContext executionContext, System.Threading.ContextCallback callback, object state, bool preserveSyncCtx) Unknown
mscorlib.dll!System.Threading.ExecutionContext.Run(System.Threading.ExecutionContext executionContext, System.Threading.ContextCallback callback, object state, bool preserveSyncCtx) Unknown
mscorlib.dll!System.Runtime.CompilerServices.AsyncMethodBuilderCore.MoveNextRunner.Run() Unknown
EntityFramework.dll!System.Data.Entity.Utilities.TaskExtensions.CultureAwaiter<System.__Canon>.UnsafeOnCompleted.AnonymousMethod__0() Unknown
mscorlib.dll!System.Threading.Tasks.AwaitTaskContinuation.RunOrScheduleAction(System.Action action, bool allowInlining, ref System.Threading.Tasks.Task currentTask) Unknown
mscorlib.dll!System.Threading.Tasks.Task.FinishContinuations() Unknown
mscorlib.dll!System.Threading.Tasks.Task<System.__Canon>.TrySetResult(System.__Canon result) Unknown
mscorlib.dll!System.Runtime.CompilerServices.AsyncTaskMethodBuilder<System.Data.Entity.Core.Objects.ObjectResult<*.WebApi.Models.Database.T_Table>>.SetResult(System.Data.Entity.Core.Objects.ObjectResult<*.WebApi.Models.Database.T_Table> result) Unknown
EntityFramework.dll!System.Data.Entity.Core.Objects.Internal.ObjectQueryExecutionPlan.ExecuteAsync<*.WebApi.Models.Database.T_Table>(System.Data.Entity.Core.Objects.ObjectContext context, System.Data.Entity.Core.Objects.ObjectParameterCollection parameterValues, System.Threading.CancellationToken cancellationToken) Unknown
mscorlib.dll!System.Threading.ExecutionContext.RunInternal(System.Threading.ExecutionContext executionContext, System.Threading.ContextCallback callback, object state, bool preserveSyncCtx) Unknown
mscorlib.dll!System.Threading.ExecutionContext.Run(System.Threading.ExecutionContext executionContext, System.Threading.ContextCallback callback, object state, bool preserveSyncCtx) Unknown
mscorlib.dll!System.Runtime.CompilerServices.AsyncMethodBuilderCore.MoveNextRunner.Run() Unknown
EntityFramework.dll!System.Data.Entity.Utilities.TaskExtensions.CultureAwaiter<System.__Canon>.UnsafeOnCompleted.AnonymousMethod__0() Unknown
mscorlib.dll!System.Threading.Tasks.AwaitTaskContinuation.RunOrScheduleAction(System.Action action, bool allowInlining, ref System.Threading.Tasks.Task currentTask) Unknown
mscorlib.dll!System.Threading.Tasks.Task.FinishContinuations() Unknown
mscorlib.dll!System.Threading.Tasks.Task<System.__Canon>.TrySetResult(System.__Canon result) Unknown
mscorlib.dll!System.Runtime.CompilerServices.AsyncTaskMethodBuilder<System.Data.Common.DbDataReader>.SetResult(System.Data.Common.DbDataReader result) Unknown
EntityFramework.dll!System.Data.Entity.Core.EntityClient.Internal.EntityCommandDefinition.ExecuteStoreCommandsAsync(System.Data.Entity.Core.EntityClient.EntityCommand entityCommand, System.Data.CommandBehavior behavior, System.Threading.CancellationToken cancellationToken) Unknown
mscorlib.dll!System.Threading.ExecutionContext.RunInternal(System.Threading.ExecutionContext executionContext, System.Threading.ContextCallback callback, object state, bool preserveSyncCtx) Unknown
mscorlib.dll!System.Threading.ExecutionContext.Run(System.Threading.ExecutionContext executionContext, System.Threading.ContextCallback callback, object state, bool preserveSyncCtx) Unknown
mscorlib.dll!System.Runtime.CompilerServices.AsyncMethodBuilderCore.MoveNextRunner.Run() Unknown
EntityFramework.dll!System.Data.Entity.Utilities.TaskExtensions.CultureAwaiter<System.__Canon>.UnsafeOnCompleted.AnonymousMethod__0() Unknown
mscorlib.dll!System.Threading.Tasks.AwaitTaskContinuation.RunOrScheduleAction(System.Action action, bool allowInlining, ref System.Threading.Tasks.Task currentTask) Unknown
mscorlib.dll!System.Threading.Tasks.Task.FinishContinuations() Unknown
mscorlib.dll!System.Threading.Tasks.Task.Finish(bool bUserDelegateExecuted) Unknown
mscorlib.dll!System.Threading.Tasks.Task.ExecuteWithThreadLocal(ref System.Threading.Tasks.Task currentTaskSlot) Unknown
mscorlib.dll!System.Threading.Tasks.Task.ExecuteEntry(bool bPreventDoubleExecution) Unknown
mscorlib.dll!System.Threading.Tasks.ThreadPoolTaskScheduler.TryExecuteTaskInline(System.Threading.Tasks.Task task, bool taskWasPreviouslyQueued) Unknown
mscorlib.dll!System.Threading.Tasks.TaskScheduler.TryRunInline(System.Threading.Tasks.Task task, bool taskWasPreviouslyQueued) Unknown
mscorlib.dll!System.Threading.Tasks.TaskContinuation.InlineIfPossibleOrElseQueue(System.Threading.Tasks.Task task, bool needsProtection) Unknown
mscorlib.dll!System.Threading.Tasks.Task.FinishContinuations() Unknown
mscorlib.dll!System.Threading.Tasks.Task<System.__Canon>.TrySetResult(System.__Canon result) Unknown
mscorlib.dll!System.Threading.Tasks.UnwrapPromise<System.Data.SqlClient.SqlDataReader>.TrySetFromTask(System.Threading.Tasks.Task task, bool lookForOce) Unknown
mscorlib.dll!System.Threading.Tasks.UnwrapPromise<System.__Canon>.ProcessInnerTask(System.Threading.Tasks.Task task) Unknown
mscorlib.dll!System.Threading.Tasks.UnwrapPromise<System.__Canon>.Invoke(System.Threading.Tasks.Task completingTask) Unknown
mscorlib.dll!System.Threading.Tasks.Task.FinishContinuations() Unknown
mscorlib.dll!System.Threading.Tasks.Task.Finish(bool bUserDelegateExecuted) Unknown
mscorlib.dll!System.Threading.Tasks.Task.ExecuteWithThreadLocal(ref System.Threading.Tasks.Task currentTaskSlot) Unknown
mscorlib.dll!System.Threading.Tasks.Task.ExecuteEntry(bool bPreventDoubleExecution) Unknown
mscorlib.dll!System.Threading.ThreadPoolWorkQueue.Dispatch() Unknown
[Native to Managed Transition]
PS: Ссылка на файл исходного кода ведет к SqlClient для.NET Core, хотя похоже, что.NET 4.5.2 (мы используем) имеет точно такую же или похожую реализацию. По крайней мере, все имена методов и классы одинаковы в стеке вызовов.