tera_kaldi_voxceleb_mfccBase.yaml

transformer:
  input_dim: 24                                         # `int`, 24 for voxceleb mfcc, 39 for mfcc, 40 for fmllr, 80 for fbank, 160 for mel
  downsample_rate: 1                                    # stacked consecutive features vectors to reduce the length of input sequences by this factor.
  hidden_size: 768                                      # Size of the encoder layers and the pooler layer.
  num_hidden_layers: 3                                  # Number of hidden layers in the Transformer encoder.
  num_attention_heads: 12                               # Number of attention heads for each attention layer in the Transformer encoder.
  intermediate_size: 3072                               # The size of the "intermediate" (i.e., feed-forward) layer in the Transformer encoder.
  hidden_act: "gelu"                                    # The non-linear activation function (function or string) in the encoder and pooler. If string, "gelu", "relu" and "swish" are supported.
  hidden_dropout_prob: 0.1                              # The dropout probabilitiy for all fully connected layers in the embeddings, encoder, and pooler.
  attention_probs_dropout_prob: 0.1                     # The dropout ratio for the attention probabilities.
  initializer_range: 0.02                               # The sttdev of the truncated_normal_initializer for initializing all weight matrices.
  layer_norm_eps: "1e-12"                               # The epsilon used by LayerNorm.
  mask_proportion: 0.15                                 # mask this percentage of all spectrogram frames in each sequence at random during MAM training                        
  mask_consecutive_min: 7                               # mask this amount of consecutive frames
  mask_consecutive_max: 7                               # mask this amount of consecutive frames
  mask_allow_overlap: True                              # allow overlap masking
  mask_bucket_ratio: 1.2                                # only used when overlap is not allowed. sample a mask from each bucket in size of [sampled mask_consecutive * mask_bucket_ratio]
  mask_frequency: 5                                     # mask maximum this amount of frequency bands, set to 0 for no frequency mask
  noise_proportion: 0.0                                 # for this percentage of the time, Gaussian noise will be applied on all frames during MAM training, set to 0 for no noise
  prune_headids: None                                   # Usage: 0,1,2,12-15 will prune headids [0,1,2,12,13,14]. headids = layerid * head_num + headid_in_layer
  share_layer: False                                    # Share layer weights
  max_input_length: 1500                                # maximum input length (0 for no restriction)
  pre_layer_norm: False                                 # apply the pre layer normalization technique introduced in: https://arxiv.org/abs/2002.04745

  
optimizer: 
  type: 'adam'                                          # modes: ['adam', 'adamW', 'lamb']
  learning_rate: "2e-4"                                 # Learning rate for opt. "4e-4" for 'data/libri_mel160_subword5000', "2e-4" for 'data/libri_fmllr_cmvn'
  loss_scale: 0                                         # Loss scale to improve fp16 numeric stability. Only used when apex is set to True. 0: dynamic loss scaling. positive power of 2: static loss scaling.
  warmup_proportion: 0.07                               # Proportion of training to perform linear rate warmup.
  gradient_accumulation_steps: 8                        # Number of updates steps to accumulate before performing a backward/update pass
  gradient_clipping: 1.0                                # Maximum gradient norm


dataloader:
  n_jobs: 6                                             # Subprocess used for torch Dataloader
  batch_size: 16                                        # training batch size, 12 for pre-train, 6 for cpc exp
  max_timestep: 0                                       # Max length for audio feature (0 for no restriction), 1500 for pre-train, 3000 for downstream tasks
  
  # LIBRISEECH SETTINGS
  data_path: ../kaldi/egs/voxceleb/v1/data/             # Source data path, path to Kaldi extracted data root, for example the part before 'train': ../kaldi/egs/voxceleb/v1/data/train/feats.scp
  target_path: ''                                       # Target data path, not used when `duo_deature:False`. For reconstruction to a different feature type, for example set dataset to 'libri_linear1025_subword5000'.
  phone_path: ''                                        # phone boundary label data path for the phone classification task. set to 'data/libri_phone' or 'data/cpc_phone'
  train_set: ['train']                                  # can be a set like: ['train', 'dev', 'test'], this will be joined with `data_path`, for example: '../kaldi/egs/voxceleb/v1/data/' + 'train/'


runner:
  # Training options
  apex: False                                           # Use APEX (see https://github.com/NVIDIA/apex for more details)
  total_steps: 30000                                    # total steps for training, a step is a batch of update
  log_step: 250                                         # log training status every this amount of training steps
  save_step: 5000                                       # save model every this amount of training steps
  duo_feature: False                                    # Use different input / output features during training
  max_keep: 2                                           # maximum number of model ckpt to keep during training