CTCBeamSearchDecoder - меньше запросов на поиск луча, чем запрошено
Я столкнулся с проблемой после успешного обучения на нескольких пакетах данных (каждый раз по-разному). Интересно то, что если я тренируюсь только на том пакете, на котором произошла ошибка, все в порядке - поэтому в данных нет ничего плохого (я использую набор данных Tensorflow speech_commands). Я очень смущен относительно того, что вызывает это поведение. Любая помощь очень ценится.
Трассировки стека:
Traceback (most recent call last):
File "/lyceum/chk1g16/.conda/envs/py3venv/lib/python3.6/site-packages/tensorflow/python/client/session.py", line 1334, in _do_call
return fn(*args)
File "/lyceum/chk1g16/.conda/envs/py3venv/lib/python3.6/site-packages/tensorflow/python/client/session.py", line 1319, in _run_fn
options, feed_dict, fetch_list, target_list, run_metadata)
File "/lyceum/chk1g16/.conda/envs/py3venv/lib/python3.6/site-packages/tensorflow/python/client/session.py", line 1407, in _call_tf_sessionrun
run_metadata)
tensorflow.python.framework.errors_impl.InvalidArgumentError: Less leaves in the beam search than requested.
[[{{node loss/CTCBeamSearchDecoder}} = CTCBeamSearchDecoder[beam_width=100, merge_repeated=true, top_paths=2, _device="/job:localhost/replica:0/task:0/device:CPU:0"](loss/transpose_1/_91, Fill/_93)]]
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "train.py", line 180, in <module>
train_and_eval()
File "train.py", line 107, in train_and_eval
feed_dict=feed_dict)
File "/lyceum/chk1g16/.conda/envs/py3venv/lib/python3.6/site-packages/tensorflow/python/client/session.py", line 929, in run
run_metadata_ptr)
File "/lyceum/chk1g16/.conda/envs/py3venv/lib/python3.6/site-packages/tensorflow/python/client/session.py", line 1152, in _run
feed_dict_tensor, options, run_metadata)
File "/lyceum/chk1g16/.conda/envs/py3venv/lib/python3.6/site-packages/tensorflow/python/client/session.py", line 1328, in _do_run
run_metadata)
File "/lyceum/chk1g16/.conda/envs/py3venv/lib/python3.6/site-packages/tensorflow/python/client/session.py", line 1348, in _do_call
raise type(e)(node_def, op, message)
tensorflow.python.framework.errors_impl.InvalidArgumentError: Less leaves in the beam search than requested.
[[node loss/CTCBeamSearchDecoder (defined at /mainfs/lyceum/chk1g16/Speech-Recognition/Conv_LSTM_CTC/conv_lstm_ctc_net.py:256) = CTCBeamSearchDecoder[beam_width=100, merge_repeated=true, top_paths=2, _device="/job:localhost/replica:0/task:0/device:CPU:0"](loss/transpose_1/_91, Fill/_93)]]
Caused by op 'loss/CTCBeamSearchDecoder', defined at:
File "train.py", line 180, in <module>
train_and_eval()
File "train.py", line 55, in train_and_eval
data_gen._num_frames, data_gen._num_mel_spec_bins, init_lr, lr_decay_steps, lr_decay_rate)
File "/mainfs/lyceum/chk1g16/Speech-Recognition/Conv_LSTM_CTC/conv_lstm_ctc_net.py", line 330, in create_train_graph
predictions, loss, acc_greedy, edit_dist_greedy, acc_beam, edit_dist_beam, scores = get_ctc_loss(logits, label_batch_plh)
File "/mainfs/lyceum/chk1g16/Speech-Recognition/Conv_LSTM_CTC/conv_lstm_ctc_net.py", line 256, in get_ctc_loss
merge_repeated=True)
File "/lyceum/chk1g16/.conda/envs/py3venv/lib/python3.6/site-packages/tensorflow/python/ops/ctc_ops.py", line 277, in ctc_beam_search_decoder
merge_repeated=merge_repeated))
File "/lyceum/chk1g16/.conda/envs/py3venv/lib/python3.6/site-packages/tensorflow/python/ops/gen_ctc_ops.py", line 74, in ctc_beam_search_decoder
top_paths=top_paths, merge_repeated=merge_repeated, name=name)
File "/lyceum/chk1g16/.conda/envs/py3venv/lib/python3.6/site-packages/tensorflow/python/framework/op_def_library.py", line 787, in _apply_op_helper
op_def=op_def)
File "/lyceum/chk1g16/.conda/envs/py3venv/lib/python3.6/site-packages/tensorflow/python/util/deprecation.py", line 488, in new_func
return func(*args, **kwargs)
File "/lyceum/chk1g16/.conda/envs/py3venv/lib/python3.6/site-packages/tensorflow/python/framework/ops.py", line 3274, in create_op
op_def=op_def)
File "/lyceum/chk1g16/.conda/envs/py3venv/lib/python3.6/site-packages/tensorflow/python/framework/ops.py", line 1770, in __init__
self._traceback = tf_stack.extract_stack()
InvalidArgumentError (see above for traceback): Less leaves in the beam search than requested.
[[node loss/CTCBeamSearchDecoder (defined at /mainfs/lyceum/chk1g16/Speech-Recognition/Conv_LSTM_CTC/conv_lstm_ctc_net.py:256) = CTCBeamSearchDecoder[beam_width=100, merge_repeated=true, top_paths=2, _device="/job:localhost/replica:0/task:0/device:CPU:0"](loss/transpose_1/_91, Fill/_93)]]
функция потерь:
def get_ctc_loss(logits, label_batch):
# logits: [batch_size, max_time, num_classes]
# 1-D tensor showing the length for each label in the batch
batch_labels_lengths = tf.fill([tf.shape(label_batch)[0]], tf.shape(label_batch)[1])
with tf.name_scope('loss'):
# get sparse represenattion of the labels
non_zero_elems_coords = tf.where(tf.not_equal(label_batch, 0))
non_zero_elems = tf.gather_nd(label_batch, non_zero_elems_coords)
sparse_label_batch = tf.SparseTensor(indices=non_zero_elems_coords,
values=non_zero_elems,
dense_shape=tf.shape(label_batch, out_type=tf.int64))
# calculate ctc loss
ctc_loss_op = tf.nn.ctc_loss(labels=sparse_label_batch,
inputs=logits,
sequence_length=batch_labels_lengths,
preprocess_collapse_repeated=True,
time_major=False,
ctc_merge_repeated=True,
ignore_longer_outputs_than_inputs=False)
loss = tf.reduce_mean(ctc_loss_op)
prediction_probabilities = tf.nn.softmax(logits)
max_probabilities = tf.reduce_max(prediction_probabilities, axis=2)
raw_predictions = tf.argmax(prediction_probabilities, axis=2)
# greedy decode logits
# greedy decoder - beeam decoder with beam_width=1 and top_paths=1
logits_T = tf.transpose(logits, perm=[1, 0, 2])
greedy_predictions, neg_sum_logits = tf.nn.ctc_greedy_decoder(inputs=logits_T,
sequence_length=batch_labels_lengths,
merge_repeated=True)
# get greedy performance metrics
edit_dist_greedy = tf.edit_distance(tf.cast(greedy_predictions[0], tf.int32),
sparse_label_batch,
normalize=False)
acc_greedy = tf.reduce_mean(tf.cast(tf.equal(edit_dist_greedy, 0), tf.float32))
edit_dist_greedy = tf.reduce_mean(edit_dist_greedy)
# beam decode logits
beam_predictions, log_probabilities = tf.nn.ctc_beam_search_decoder(inputs=logits_T,
sequence_length=batch_labels_lengths,
beam_width=100,
top_paths=2,
merge_repeated=True)
# get beam performance metrics
edit_dist_beam = tf.edit_distance(tf.cast(beam_predictions[0], tf.int32),
sparse_label_batch,
normalize=False)
acc_beam = tf.reduce_mean(tf.cast(tf.equal(edit_dist_beam, 0), tf.float32))
predictions = tf.cast(tf.sparse.to_dense(beam_predictions[0]), tf.int32)
scores = log_probabilities[:, 0] - log_probabilities[:, 1]
tf.summary.scalar('ctc_loss', loss)
tf.summary.scalar('acc_greedy', acc_greedy)
tf.summary.scalar('edit_dist_greedy', edit_dist_greedy)
tf.summary.scalar('confidence_score', tf.reduce_mean(scores))
# tf.summary.scalar('edit_dist_beam', edit_dist_beam)
tf.summary.scalar('acc_beam', acc_beam)
return predictions, loss, acc_greedy, edit_dist_greedy, acc_beam, edit_dist_beam, scores