Поток Python Daemon не завершается
Я исследую проблему с зависшим процессом Python. Я использовал gdb для подключения к программе и получения их следов (см. Ниже).
Из обратных следов кажется, что все остальные потоки являются демоническими (мы можем найти _Thread__daemonic = True).
Поток может быть помечен как "поток демона". Значение этого флага заключается в том, что вся программа Python завершается, когда остаются только потоки демона. ” https://docs.python.org/3/library/threading.html
Но программа никогда не останавливается даже после нескольких часов. Мы можем восстановить, если программа может остановиться. Что здесь происходит, что программа не выходит?
(gdb) info thread
Id Target Id Frame
3 Thread 0x7fd1a4bc8700 (LWP 57711) "Task Coordinato" 0x00007fd1b3e5a893 in select () at ../sysdeps/unix/syscall-template.S:81
2 Thread 0x7fd19ffff700 (LWP 58424) "Task Coordinato" 0x00007fd1b3e5a893 in select () at ../sysdeps/unix/syscall-template.S:81
* 1 Thread 0x7fd1b4e74700 (LWP 57704) "Task Coordinato" sem_wait () at ../nptl/sysdeps/unix/sysv/linux/x86_64/sem_wait.S:85
Тема 1
Thread 1 (Thread 0x7fd1b4e74700 (LWP 57704)):
Python Exception <type 'exceptions.ImportError'> No module named itertools:
#0 sem_wait () at ../nptl/sysdeps/unix/sysv/linux/x86_64/sem_wait.S:85
#1 0x0000000000573cfc in PyThread_acquire_lock (lock=0x2293ab0, waitflag=1) at ../Python/thread_pthread.h:324
#2 0x000000000062637b in lock_PyThread_acquire_lock (self=0x7fd1a6aa45e0, args=()) at ../Modules/threadmodule.c:52
#3 0x0000000000489909 in PyCFunction_Call (func=<built-in method acquire of thread.lock object at remote 0x7fd1a6aa45e0>, arg=(), kw=0x0) at ../Objects/methodobject.c:81
#4 0x000000000052b7f6 in call_function (pp_stack=0x7fff62112f70, oparg=0) at ../Python/ceval.c:4033
#5 0x00000000005266ca in PyEval_EvalFrameEx (
f=Frame 0x23bff40, for file /usr/lib/python2.7/threading.py, line 340, in wait (self=<_Condition(_Verbose__verbose=False, _Condition__lock=<thread.lock at remote 0x7fd1a70f6d10>, acquire=<built-in method acquire of thread.lock object at remote 0x7fd1a70f6d10>, _Condition__waiters=[<thread.lock at remote 0x7fd1a6aa45e0>], release=<built-in method release of thread.lock object at remote 0x7fd1a70f6d10>) at remote 0x7fd1a710a1b0>, timeout=None, waiter=<thread.lock at remote 0x7fd1a6aa45e0>, saved_state=None), throwflag=0) at ../Python/ceval.c:2679
#6 0x0000000000528f7e in PyEval_EvalCodeEx (co=0x7fd1b3d78ca0,
globals={'current_thread': <function at remote 0x7fd1b3ce03a8>, '_BoundedSemaphore': <type at remote 0x1390980>, 'currentThread': <function at remote 0x7fd1b3ce03a8>, '_Timer': <type at remote 0x1392ae0>, '_format_exc': <function at remote 0x7fd1b4cee1b0>, 'Semaphore': <function at remote 0x7fd1b3cdad80>, '_deque': <type at remote 0x8e4d40>, 'activeCount': <function at remote 0x7fd1b3ce0648>, '_profile_hook': None, '_sleep': <built-in function sleep>, '_trace_hook': None, 'ThreadError': <type at remote 0x137ce30>, '_enumerate': <function at remote 0x7fd1b3ce06f0>, '_start_new_thread': <built-in function start_new_thread>, 'BoundedSemaphore': <function at remote 0x7fd1b3cdd4f8>, '_shutdown': <instancemethod at remote 0x7fd1b4defd60>, '__all__': ['activeCount', 'active_count', 'Condition', 'currentThread', 'current_thread', 'enumerate', 'Event', 'Lock', 'RLock', 'Semaphore', 'BoundedSemaphore', 'Thread', 'Timer', 'setprofile', 'settrace', 'local', 'stack_size'], '_Event': <type at remote 0x1390d70>, 'ac---Type <return> to continue, or q <return> to quit---
tive_count': <fu...(truncated), locals=0x0, args=0x23f5818, argcount=1, kws=0x23f5820, kwcount=0, defs=0x7fd1b3cd4948, defcount=1, closure=0x0) at ../Python/ceval.c:3265
#7 0x000000000052bdc9 in fast_function (func=<function at remote 0x7fd1b3cdd300>, pp_stack=0x7fff62113410, n=1, na=1, nk=0) at ../Python/ceval.c:4129
#8 0x000000000052b9ca in call_function (pp_stack=0x7fff62113410, oparg=0) at ../Python/ceval.c:4054
#9 0x00000000005266ca in PyEval_EvalFrameEx (
f=Frame 0x23f5670, for file /usr/lib/python2.7/threading.py, line 949, in join (self=<Thread(_Thread__ident=140538388711168, _Thread__block=<_Condition(_Verbose__verbose=False, _Condition__lock=<thread.lock at remote 0x7fd1a70f6d10>, acquire=<built-in method acquire of thread.lock object at remote 0x7fd1a70f6d10>, _Condition__waiters=[<thread.lock at remote 0x7fd1a6aa45e0>], release=<built-in method release of thread.lock object at remote 0x7fd1a70f6d10>) at remote 0x7fd1a710a1b0>, _Thread__name='Thread-6', _Thread__daemonic=True, _Thread__started=<_Event(_Verbose__verbose=False, _Event__flag=True, _Event__cond=<_Condition(_Verbose__verbose=False, _Condition__lock=<thread.lock at remote 0x7fd1a70f6c70>, acquire=<built-in method acquire of thread.lock object at remote 0x7fd1a70f6c70>, _Condition__waiters=[], release=<built-in method release of thread.lock object at remote 0x7fd1a70f6c70>) at remote 0x7fd1a710a0d0>) at remote 0x7fd1a710a060>, _Thread__stderr=<file at remote 0x7fd1b4e62280>, _Thread__target=<insta...(truncated), throwflag=0) at ../Python/ceval.c:2679
^^^^^^^^^^^^^^^^^^^^^^
#10 ...
Тема 2
Thread 2 (Thread 0x7fd19ffff700 (LWP 58424)):
Python Exception <type 'exceptions.ImportError'> No module named itertools:
#0 0x00007fd1b3e5a893 in select () at ../sysdeps/unix/syscall-template.S:81
#1 0x0000000000629483 in floatsleep (secs=1) at ../Modules/timemodule.c:948
#2 0x00000000006280f1 in time_sleep (self=0x0, args=(1,)) at ../Modules/timemodule.c:206
#3 0x0000000000489909 in PyCFunction_Call (func=<built-in function sleep>, arg=(1,), kw=0x0) at ../Objects/methodobject.c:81
#4 0x000000000052b7f6 in call_function (pp_stack=0x7fd19fffdd00, oparg=1) at ../Python/ceval.c:4033
#5 0x00000000005266ca in PyEval_EvalFrameEx (
f=Frame 0x7fd194000b70, for file /dependencies/debugenv/local/lib/python2.7/site-packages/ddtrace/writer.py, line 170, in _target (self=<AsyncWorker(_service_queue=<Q(_max_size=1000, _closed=False, _things=[], _lock=<thread.lock at remote 0x7fd1a5686d60>) at remote 0x7fd1a707c7d0>, _priority_sampler=None, _thread=<Thread(_Thread__ident=140538309244672, _Thread__block=<_Condition(_Verbose__verbose=False, _Condition__lock=<thread.lock at remote 0x7fd1a5686a90>, acquire=<built-in method acquire of thread.lock object at remote 0x7fd1a5686a90>, _Condition__waiters=[], release=<built-in method release of thread.lock object at remote 0x7fd1a5686a90>) at remote 0x7fd1a707cbc0>, _Thread__name='Thread-7', _Thread__daemonic=True, _Thread__started=<_Event(_Verbose__verbose=False, _Event__flag=True, _Event__cond=<_Condition(_Verbose__verbose=False, _Condition__lock=<thread.lock at remote 0x7fd1a5686b80>, acquire=<built-in method acquire of thread.lock object at remote 0x7fd1a5686b80>, _Condition__waiters=[], release=<built-...(truncated), throwflag=0) at ../Python/ceval.c:2679
^^^^^^^^^^^^^^^^^^^^^^
#6 0x0000000000528f7e in PyEval_EvalCodeEx (co=0x7fd1b2d40250,
globals={'DEFAULT_TIMEOUT': 5, 'MAX_TRACES': 1000, 'LOG_ERR_INTERVAL': 60, 'logging': <module at remote 0x7fd1b4d31678>, 'log': <Logger(name='ddtrace.writer', parent=<RootLogger(name='root', parent=None, handlers=[<RotatingFileHandler(backupCount=3, stream=<file at remote 0x7fd1a56bd4c0>, encoding=None, lock=<_RLock(_Verbose__verbose=False, _RLock__owner=None, _RLock__block=<thread.lock at remote 0x7fd1a56864a0>, _RLock__count=0) at remote 0x7fd1a7635370>, level=20, _name=None, delay=0, baseFilename='/var/log/pocketgems/task/dd-tasking-daemon.log', mode='a', filters=[<RootOnlyLogFilter(name='', nlen=0) at remote 0x7fd1a56d9840>], formatter=<PGLDJFormatter(info={'runner': 'daemon'}, thread_info={140538659948288: {}, 140538309244672: {}, 140538388711168: {}}, datefmt=None, _fmt='%(message)s') at remote 0x7fd1a578eed0>, maxBytes=10485760) at remote 0x7fd1a56e5bc0>, <RotatingFileHandler(backupCount=3, stream=<file at remote 0x7fd1a56bd640>, encoding=None, lock=<_RLock(_Verbose__verbose=False, _RLock__owner=None, _RLock__...(truncated), locals=0x0, args=0x7fd1a707c788, argcount=1, kws=0x7fd1b4e33088, kwcount=0, defs=0x0, defcount=0, closure=0x0) at ../Python/ceval.c:3265
#7 0x00000000004665d1 in function_call (func=<function at remote 0x7fd1b2d62648>,
arg=(<AsyncWorker(_service_queue=<Q(_max_size=1000, _closed=False, _things=[], _lock=<thread.lock at remote 0x7fd1a5686d60>) at remote 0x7fd1a707c7d0>, _priority_sampler=None, _thread=<Thread(_Thread__ident=140538309244672, _Thread__block=<_Condition(_Verbose__verbose=False, _Condition__lock=<thread.lock at remote 0x7fd1a5686a90>, acquire=<built-in method acquire of thread.lock object at remote 0x7fd1a5686a90>, _Condition__waiters=[], release=<built-in method release of thread.lock object at remote 0x7fd1a5686a90>) at remote 0x7fd1a707cbc0>, _Thread__name='Thread-7', _Thread__daemonic=True, _Thread__started=<_Event(_Verbose__verbose=False, _Event__flag=True, _Event__cond=<_Condition(_Verbose__verbose=False, _Condition__lock=<thread.lock at remote 0x7fd1a5686b80>, acquire=<built-in method acquire of thread.lock object at remote 0x7fd1a5686b80>, _Condition__waiters=[], release=<built-in method release of thread.lock object at remote 0x7fd1a5686b80>) at remote 0x7fd1a707ca70>) at remote 0x7fd1a707c990>, _Thread__std...(truncated), kw={}) at ../Objects/funcobject.c:526
#8 ...
Тема 3
Thread 3 (Thread 0x7fd1a4bc8700 (LWP 57711)):
Python Exception <type 'exceptions.ImportError'> No module named itertools:
#0 0x00007fd1b3e5a893 in select () at ../sysdeps/unix/syscall-template.S:81
#1 0x00000000005fb67f in select_select (self=0x0, args=([<_socketobject at remote 0x7fd1a70df240>, <_socket.socket at remote 0x7fd1a56837f8>], [], [], <float at remote 0x249cb58>))
at ../Modules/selectmodule.c:267
#2 0x0000000000489909 in PyCFunction_Call (func=<built-in function select>,
arg=([<_socketobject at remote 0x7fd1a70df240>, <_socket.socket at remote 0x7fd1a56837f8>], [], [], <float at remote 0x249cb58>), kw={}) at ../Objects/methodobject.c:81
#3 0x000000000052cc79 in ext_do_call (func=<built-in function select>, pp_stack=0x7fd1a4bc5310, flags=3, na=0, nk=0) at ../Python/ceval.c:4343
#4 0x0000000000526982 in PyEval_EvalFrameEx (
f=Frame 0x7fd198003a20, for file /dependencies/debugenv/local/lib/python2.7/site-packages/kazoo/handlers/threading.py, line 192, in _select (self=<SequentialThreadingHandler(_state_change=<thread.lock at remote 0x7fd1a70f68b0>, _workers=[], _running=False, completion_queue=<Queue(unfinished_tasks=0, queue=<collections.deque at remote 0x7fd1a56a67e0>, maxsize=0, all_tasks_done=<_Condition(_Verbose__verbose=False, _Condition__lock=<thread.lock at remote 0x7fd1a5686360>, acquire=<built-in method acquire of thread.lock object at remote 0x7fd1a5686360>, _Condition__waiters=[], release=<built-in method release of thread.lock object at remote 0x7fd1a5686360>) at remote 0x7fd1a56aa0d0>, mutex=<thread.lock at remote 0x7fd1a5686360>, not_full=<_Condition(_Verbose__verbose=False, _Condition__lock=<thread.lock at remote 0x7fd1a5686360>, acquire=<built-in method acquire of thread.lock object at remote 0x7fd1a5686360>, _Condition__waiters=[], release=<built-in method release of thread.lock object at remote 0x7fd1a5686360>) a...(truncated), throwflag=0) at ../Python/ceval.c:2718
#5 0x0000000000528f7e in PyEval_EvalCodeEx (co=0x7fd1a56209e0,
globals={'chain': <type at remote 0x8fde00>, 'utils': <module at remote 0x7fd1a5633268>, 'absolute_import': <_Feature(mandatory=(3, 0, 0, 'alpha', 0), optional=(2, 5, 0, 'alpha', 1), compiler_flag=16384) at remote 0x7fd1b3c74df8>, 'select': <module at remote 0x7fd1b1f39748>, 'AsyncResult': <type at remote 0x207d780>, 'log': <Logger(name='kazoo.handlers.threading', parent=<Logger(name='kazoo', parent=<RootLogger(name='root', parent=None, handlers=[<RotatingFileHandler(backupCount=3, stream=<file at remote 0x7fd1a56bd4c0>, encoding=None, lock=<_RLock(_Verbose__verbose=False, _RLock__owner=None, _RLock__block=<thread.lock at remote 0x7fd1a56864a0>, _RLock__count=0) at remote 0x7fd1a7635370>, level=20, _name=None, delay=0, baseFilename='/var/log/pocketgems/task/dd-tasking-daemon.log', mode='a', filters=[<RootOnlyLogFilter(name='', nlen=0) at remote 0x7fd1a56d9840>], formatter=<PGLDJFormatter(info={'runner': 'daemon'}, thread_info={140538659948288: {}, 140538309244672: {}, 140538388711168: {}}, datefmt=None, _fmt='%(messa...(truncated), locals=0x0, args=0x7fd1a5663598, argcount=5, kws=0x7fd1b4e33088, kwcount=0, defs=0x0, defcount=0, closure=0x0) at ../Python/ceval.c:3265
#6 0x00000000004665d1 in function_call (func=<function at remote 0x7fd1a56305a0>,
arg=(<SequentialThreadingHandler(_state_change=<thread.lock at remote 0x7fd1a70f68b0>, _workers=[], _running=False, completion_queue=<Queue(unfinished_tasks=0, queue=<collections.deque at remote 0x7fd1a56a67e0>, maxsize=0, all_tasks_done=<_Condition(_Verbose__verbose=False, _Condition__lock=<thread.lock at remote 0x7fd1a5686360>, acquire=<built-in method acquire of thread.lock object at remote 0x7fd1a5686360>, _Condition__waiters=[], release=<built-in method release of thread.lock object at remote 0x7fd1a5686360>) at remote 0x7fd1a56aa0d0>, mutex=<thread.lock at remote 0x7fd1a5686360>, not_full=<_Condition(_Verbose__verbose=False, _Condition__lock=<thread.lock at remote 0x7fd1a5686360>, acquire=<built-in method acquire of thread.lock object at remote 0x7fd1a5686360>, _Condition__waiters=[], release=<built-in method release of thread.lock object at remote 0x7fd1a5686360>) at remote 0x7fd1a56aa300>, not_empty=<_Condition(_Verbose__verbose=False, _Condition__lock=<thread.lock at remote 0x7fd1a5686360>, acquire=<buil...(truncated), kw={}) at ../Objects/funcobject.c:526
...
#34 0x0000000000427c3a in PyObject_Call (func=<function at remote 0x7fd1a55a43a8>,
arg=(<ConnectionHandler(retry_sleeper=<KazooRetry(_cur_stoptime=None, max_jitter=80, retry_exceptions=(<type at remote 0x2081990>, <type at remote 0x2083190>, <type at remote 0x20b70a0>, <type at remote 0x2085ce0>), sleep_func=<built-in function sleep>, _attempts=0, max_delay=<float at remote 0x20bb1d8>, delay=<float at remote 0x1dcb600>, deadline=None, _cur_delay=<float at remote 0x1dcb600>, interrupt=<function at remote 0x7fd1a5462a38>, max_tries=None, backoff=2) at remote 0x7fd1a71080d0>, _read_sock=<_socket.socket at remote 0x7fd1a56837f8>, _rw_server=None, _socket=<_socketobject at remote 0x7fd1a70df240>, client=<ZkClient(MultiNonBlockingLease=<functools.partial at remote 0x7fd1a564b280>, SetPartitioner=<functools.partial at remote 0x7fd1a564b858>, _state='CONNECTED', _session_timeout=10000, Semaphore=<functools.partial at remote 0x7fd1a564b8e0>, Party=<functools.partial at remote 0x7fd1a564b6c0>, state_listeners=set([]), ChildrenWatch=<functools.partial at remote 0x7fd1a564b060>, read_only=None, chroot='', ...(truncated), kw={}) at ../Objects/abstract.c:2529
#35 0x000000000052cc98 in ext_do_call (func=<function at remote 0x7fd1a55a43a8>, pp_stack=0x7fd1a4bc71e0, flags=3, na=1, nk=0) at ../Python/ceval.c:4346
#36 0x0000000000526982 in PyEval_EvalFrameEx (
f=Frame 0x7fd1a710b060, for file /usr/lib/python2.7/threading.py, line 763, in run (self=<Thread(_Thread__ident=140538388711168, _Thread__block=<_Condition(_Verbose__verbose=False, _Condition__lock=<thread.lock at remote 0x7fd1a70f6d10>, acquire=<built-in method acquire of thread.lock object at remote 0x7fd1a70f6d10>, _Condition__waiters=[<thread.lock at remote 0x7fd1a6aa45e0>], release=<built-in method release of thread.lock object at remote 0x7fd1a70f6d10>) at remote 0x7fd1a710a1b0>, _Thread__name='Thread-6', _Thread__daemonic=True, _Thread__started=<_Event(_Verbose__verbose=False, _Event__flag=True, _Event__cond=<_Condition(_Verbose__verbose=False, _Condition__lock=<thread.lock at remote 0x7fd1a70f6c70>, acquire=<built-in method acquire of thread.lock object at remote 0x7fd1a70f6c70>, _Condition__waiters=[], release=<built-in method release of thread.lock object at remote 0x7fd1a70f6c70>) at remote 0x7fd1a710a0d0>) at remote 0x7fd1a710a060>, _Thread__stderr=<file at remote 0x7fd1b4e62280>, _Thread__target=<i...(truncated), throwflag=0) at ../Python/ceval.c:2718
^^^^^^^^^^^^^^^^^^^^^^
#37 0x000000000052bcb7 in fast_function (func=<function at remote 0x7fd1b3cde3a8>, pp_stack=0x7fd1a4bc74f0, n=1, na=1, nk=0) at ../Python/ceval.c:4119
#38 ...
Я буду рад дать больше информации, если это необходимо. Переполнение стека не позволяет мне вставить всю трассировку из-за ограничения текста.