uisrnn.utils

  1# Copyright 2018 Google LLC
  2#
  3# Licensed under the Apache License, Version 2.0 (the "License");
  4# you may not use this file except in compliance with the License.
  5# You may obtain a copy of the License at
  6#
  7#     https://www.apache.org/licenses/LICENSE-2.0
  8#
  9# Unless required by applicable law or agreed to in writing, software
 10# distributed under the License is distributed on an "AS IS" BASIS,
 11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 12# See the License for the specific language governing permissions and
 13# limitations under the License.
 14"""Utils for UIS-RNN."""
 15
 16import random
 17import string
 18
 19import numpy as np
 20import torch
 21from torch import autograd
 22
 23
 24def generate_random_string(length=6):
 25  """Generate a random string of upper case letters and digits.
 26
 27  Args:
 28    length: length of the generated string
 29
 30  Returns:
 31    the generated string
 32  """
 33  return ''.join([
 34      random.choice(string.ascii_uppercase + string.digits)
 35      for _ in range(length)])
 36
 37
 38def enforce_cluster_id_uniqueness(cluster_ids):
 39  """Enforce uniqueness of cluster id across sequences.
 40
 41  Args:
 42    cluster_ids: a list of 1-dim list/numpy.ndarray of strings
 43
 44  Returns:
 45    a new list with same length of cluster_ids
 46
 47  Raises:
 48    TypeError: if cluster_ids or its element has wrong type
 49  """
 50  if not isinstance(cluster_ids, list):
 51    raise TypeError('cluster_ids must be a list')
 52  new_cluster_ids = []
 53  for cluster_id in cluster_ids:
 54    sequence_id = generate_random_string()
 55    if isinstance(cluster_id, np.ndarray):
 56      cluster_id = cluster_id.tolist()
 57    if not isinstance(cluster_id, list):
 58      raise TypeError('Elements of cluster_ids must be list or numpy.ndarray')
 59    new_cluster_id = ['_'.join([sequence_id, s]) for s in cluster_id]
 60    new_cluster_ids.append(new_cluster_id)
 61  return new_cluster_ids
 62
 63
 64def concatenate_training_data(train_sequences, train_cluster_ids,
 65                              enforce_uniqueness=True, shuffle=True):
 66  """Concatenate training data.
 67
 68  Args:
 69    train_sequences: a list of 2-dim numpy arrays to be concatenated
 70    train_cluster_ids: a list of 1-dim list/numpy.ndarray of strings
 71    enforce_uniqueness: a boolean indicated whether we should enfore uniqueness
 72      to train_cluster_ids
 73    shuffle: whether to randomly shuffle input order
 74
 75  Returns:
 76    concatenated_train_sequence: a 2-dim numpy array
 77    concatenated_train_cluster_id: a list of strings
 78
 79  Raises:
 80    TypeError: if input has wrong type
 81    ValueError: if sizes/dimensions of input or their elements are incorrect
 82  """
 83  # check input
 84  if not isinstance(train_sequences, list) or not isinstance(
 85      train_cluster_ids, list):
 86    raise TypeError('train_sequences and train_cluster_ids must be lists')
 87  if len(train_sequences) != len(train_cluster_ids):
 88    raise ValueError(
 89        'train_sequences and train_cluster_ids must have same size')
 90  train_cluster_ids = [
 91      x.tolist() if isinstance(x, np.ndarray) else x
 92      for x in train_cluster_ids]
 93  global_observation_dim = None
 94  for i, (train_sequence, train_cluster_id) in enumerate(
 95      zip(train_sequences, train_cluster_ids)):
 96    train_length, observation_dim = train_sequence.shape
 97    if i == 0:
 98      global_observation_dim = observation_dim
 99    elif global_observation_dim != observation_dim:
100      raise ValueError(
101          'train_sequences must have consistent observation dimension')
102    if not isinstance(train_cluster_id, list):
103      raise TypeError(
104          'Elements of train_cluster_ids must be list or numpy.ndarray')
105    if len(train_cluster_id) != train_length:
106      raise ValueError(
107          'Each train_sequence and its train_cluster_id must have same length')
108
109  # enforce uniqueness
110  if enforce_uniqueness:
111    train_cluster_ids = enforce_cluster_id_uniqueness(train_cluster_ids)
112
113  # random shuffle
114  if shuffle:
115    zipped_input = list(zip(train_sequences, train_cluster_ids))
116    random.shuffle(zipped_input)
117    train_sequences, train_cluster_ids = zip(*zipped_input)
118
119  # concatenate
120  concatenated_train_sequence = np.concatenate(train_sequences, axis=0)
121  concatenated_train_cluster_id = [x for train_cluster_id in train_cluster_ids
122                                   for x in train_cluster_id]
123  return concatenated_train_sequence, concatenated_train_cluster_id
124
125
126def sample_permuted_segments(index_sequence, number_samples):
127  """Sample sequences with permuted blocks.
128
129  Args:
130    index_sequence: (integer array, size: L)
131      - subsequence index
132      For example, index_sequence = [1,2,6,10,11,12].
133    number_samples: (integer)
134      - number of subsampled block-preserving permuted sequences.
135      For example, number_samples = 5
136
137  Returns:
138    sampled_index_sequences: (a list of numpy arrays) - a list of subsampled
139      block-preserving permuted sequences. For example,
140    ```
141    sampled_index_sequences =
142    [[10,11,12,1,2,6],
143     [6,1,2,10,11,12],
144     [1,2,10,11,12,6],
145     [6,1,2,10,11,12],
146     [1,2,6,10,11,12]]
147    ```
148      The length of "sampled_index_sequences" is "number_samples".
149  """
150  segments = []
151  if len(index_sequence) == 1:
152    segments.append(index_sequence)
153  else:
154    prev = 0
155    for i in range(len(index_sequence) - 1):
156      if index_sequence[i + 1] != index_sequence[i] + 1:
157        segments.append(index_sequence[prev:(i + 1)])
158        prev = i + 1
159      if i + 1 == len(index_sequence) - 1:
160        segments.append(index_sequence[prev:])
161  # sample permutations
162  sampled_index_sequences = []
163  for _ in range(number_samples):
164    segments_array = []
165    permutation = np.random.permutation(len(segments))
166    for permutation_item in permutation:
167      segments_array.append(segments[permutation_item])
168    sampled_index_sequences.append(np.concatenate(segments_array))
169  return sampled_index_sequences
170
171
172def resize_sequence(sequence, cluster_id, num_permutations=None):
173  """Resize sequences for packing and batching.
174
175  Args:
176    sequence: (real numpy matrix, size: seq_len*obs_size) - observed sequence
177    cluster_id: (numpy vector, size: seq_len) - cluster indicator sequence
178    num_permutations: int - Number of permutations per utterance sampled.
179
180  Returns:
181    sub_sequences: A list of numpy array, with obsevation vector from the same
182      cluster in the same list.
183    seq_lengths: The length of each cluster (+1).
184  """
185  # merge sub-sequences that belong to a single cluster to a single sequence
186  unique_id = np.unique(cluster_id)
187  sub_sequences = []
188  seq_lengths = []
189  if num_permutations and num_permutations > 1:
190    for i in unique_id:
191      idx_set = np.where(cluster_id == i)[0]
192      sampled_idx_sets = sample_permuted_segments(idx_set, num_permutations)
193      for j in range(num_permutations):
194        sub_sequences.append(sequence[sampled_idx_sets[j], :])
195        seq_lengths.append(len(idx_set) + 1)
196  else:
197    for i in unique_id:
198      idx_set = np.where(cluster_id == i)
199      sub_sequences.append(sequence[idx_set, :][0])
200      seq_lengths.append(len(idx_set[0]) + 1)
201  return sub_sequences, seq_lengths
202
203
204def pack_sequence(
205    sub_sequences, seq_lengths, batch_size, observation_dim, device):
206  """Pack sequences for training.
207
208  Args:
209    sub_sequences: A list of numpy array, with obsevation vector from the same
210      cluster in the same list.
211    seq_lengths: The length of each cluster (+1).
212    batch_size: int or None - Run batch learning if batch_size is None. Else,
213      run online learning with specified batch size.
214    observation_dim: int - dimension for observation vectors
215    device: str - Your device. E.g., `cuda:0` or `cpu`.
216
217  Returns:
218    packed_rnn_input: (PackedSequence object) packed rnn input
219    rnn_truth: ground truth
220  """
221  num_clusters = len(seq_lengths)
222  sorted_seq_lengths = np.sort(seq_lengths)[::-1]
223  permute_index = np.argsort(seq_lengths)[::-1]
224
225  if batch_size is None:
226    rnn_input = np.zeros((sorted_seq_lengths[0],
227                          num_clusters,
228                          observation_dim))
229    for i in range(num_clusters):
230      rnn_input[1:sorted_seq_lengths[i], i,
231                :] = sub_sequences[permute_index[i]]
232    rnn_input = autograd.Variable(
233        torch.from_numpy(rnn_input).float()).to(device)
234    packed_rnn_input = torch.nn.utils.rnn.pack_padded_sequence(
235        rnn_input, sorted_seq_lengths, batch_first=False)
236  else:
237    mini_batch = np.sort(np.random.choice(num_clusters, batch_size))
238    rnn_input = np.zeros((sorted_seq_lengths[mini_batch[0]],
239                          batch_size,
240                          observation_dim))
241    for i in range(batch_size):
242      rnn_input[1:sorted_seq_lengths[mini_batch[i]],
243                i, :] = sub_sequences[permute_index[mini_batch[i]]]
244    rnn_input = autograd.Variable(
245        torch.from_numpy(rnn_input).float()).to(device)
246    packed_rnn_input = torch.nn.utils.rnn.pack_padded_sequence(
247        rnn_input, sorted_seq_lengths[mini_batch], batch_first=False)
248  # ground truth is the shifted input
249  rnn_truth = rnn_input[1:, :, :]
250  return packed_rnn_input, rnn_truth
251
252
253def output_result(model_args, training_args, test_record):
254  """Produce a string to summarize the experiment."""
255  accuracy_array, _ = zip(*test_record)
256  total_accuracy = np.mean(accuracy_array)
257  output_string = """
258Config:
259  sigma_alpha: {}
260  sigma_beta: {}
261  crp_alpha: {}
262  learning rate: {}
263  regularization: {}
264  batch size: {}
265
266Performance:
267  averaged accuracy: {:.6f}
268  accuracy numbers for all testing sequences:
269  """.strip().format(
270      training_args.sigma_alpha,
271      training_args.sigma_beta,
272      model_args.crp_alpha,
273      training_args.learning_rate,
274      training_args.regularization_weight,
275      training_args.batch_size,
276      total_accuracy)
277  for accuracy in accuracy_array:
278    output_string += '\n    {:.6f}'.format(accuracy)
279  output_string += '\n' + '=' * 80 + '\n'
280  filename = 'layer_{}_{}_{:.1f}_result.txt'.format(
281      model_args.rnn_hidden_size,
282      model_args.rnn_depth, model_args.rnn_dropout)
283  with open(filename, 'a') as file_object:
284    file_object.write(output_string)
285  return output_string
286
287
288def estimate_transition_bias(cluster_ids, smooth=1):
289  """Estimate the transition bias.
290
291  Args:
292    cluster_id: Either a list of cluster indicator sequences, or a single
293      concatenated sequence. The former is strongly preferred, since the
294      transition_bias estimated from the latter will be inaccurate.
295    smooth: int or float - Smoothing coefficient, avoids -inf value in np.log
296      in the case of a sequence with a single speaker and division by 0 in the
297      case of empty sequences. Using a small value for smooth decreases the
298      bias in the calculation of transition_bias but can also lead to underflow
299      in some remote cases, larger values are safer but less accurate.
300
301  Returns:
302    bias: Flipping coin head probability.
303    bias_denominator: The denominator of the bias, used for multiple calls to
304      fit().
305  """
306  transit_num = smooth
307  bias_denominator = 2 * smooth
308  for cluster_id_seq in cluster_ids:
309    for entry in range(len(cluster_id_seq) - 1):
310      transit_num += (cluster_id_seq[entry] != cluster_id_seq[entry + 1])
311      bias_denominator += 1
312  bias = transit_num / bias_denominator
313  return bias, bias_denominator