repo_description: A Pytorch Implementation of "Neural Speech Synthesis with Transformer Network"
repo_topics: attention-mechanism attention-plots deep-learning pytorch pytorch-implementation text-to-speech transformer tts
code: import numpy as np
  import librosa
  import os, copy
  from scipy import signal
  import hyperparams as hp
  import torch as t
  
  def get_spectrograms(fpath):
      '''Parse the wave file in `fpath` and
      Returns normalized melspectrogram and linear spectrogram.
      Args:
        fpath: A string. The full path of a sound file.
      Returns:
        mel: A 2d array of shape (T, n_mels) and dtype of float32.
        mag: A 2d array of shape (T, 1+n_fft/2) and dtype of float32.
      '''
      # Loading sound file
      y, sr = librosa.load(fpath, sr=hp.sr)
  
      # Trimming
      y, _ = librosa.effects.trim(y)
  
      # Preemphasis
      y = np.append(y[0], y[1:] - hp.preemphasis * y[:-1])
  
      # stft
      linear = librosa.stft(y=y,
                            n_fft=hp.n_fft,
                            hop_length=hp.hop_length,
                            win_length=hp.win_length)
  
      # magnitude spectrogram
      mag = np.abs(linear)  # (1+n_fft//2, T)
  
      # mel spectrogram
      mel_basis = librosa.filters.mel(hp.sr, hp.n_fft, hp.n_mels)  # (n_mels, 1+n_fft//2)
      mel = np.dot(mel_basis, mag)  # (n_mels, t)
  
      # to decibel
      mel = 20 * np.log10(np.maximum(1e-5, mel))
      mag = 20 * np.log10(np.maximum(1e-5, mag))
  
      # normalize
      mel = np.clip((mel - hp.ref_db + hp.max_db) / hp.max_db, 1e-8, 1)
      mag = np.clip((mag - hp.ref_db + hp.max_db) / hp.max_db, 1e-8, 1)
  
      # Transpose
      mel = mel.T.astype(np.float32)  # (T, n_mels)
      mag = mag.T.astype(np.float32)  # (T, 1+n_fft//2)
  
      return mel, mag
  
  def spectrogram2wav(mag):
      '''# Generate wave file from linear magnitude spectrogram
      Args:
        mag: A numpy array of (T, 1+n_fft//2)
      Returns:
        wav: A 1-D numpy array.
      '''
      # transpose
      mag = mag.T
  
      # de-noramlize
      mag = (np.clip(mag, 0, 1) * hp.max_db) - hp.max_db + hp.ref_db
  
      # to amplitude
      mag = np.power(10.0, mag * 0.05)
  
      # wav reconstruction
      wav = griffin_lim(mag**hp.power)
  
      # de-preemphasis
      wav = signal.lfilter([1], [1, -hp.preemphasis], wav)
  
      # trim
      wav, _ = librosa.effects.trim(wav)
  
      return wav.astype(np.float32)
  
  def griffin_lim(spectrogram):
      '''Applies Griffin-Lim's raw.'''
      X_best = copy.deepcopy(spectrogram)
      for i in range(hp.n_iter):
          X_t = invert_spectrogram(X_best)
          est = librosa.stft(X_t, hp.n_fft, hp.hop_length, win_length=hp.win_length)
          phase = est / np.maximum(1e-8, np.abs(est))
          X_best = spectrogram * phase
      X_t = invert_spectrogram(X_best)
      y = np.real(X_t)
  
      return y
  
  def invert_spectrogram(spectrogram):
      '''Applies inverse fft.
      Args:
        spectrogram: [1+n_fft//2, t]
      '''
      return librosa.istft(spectrogram, hp.hop_length, win_length=hp.win_length, window="hann")
  
  def get_positional_table(d_pos_vec, n_position=1024):
      position_enc = np.array([
          [pos / np.power(10000, 2*i/d_pos_vec) for i in range(d_pos_vec)]
          if pos != 0 else np.zeros(d_pos_vec) for pos in range(n_position)])
  
      position_enc[1:, 0::2] = np.sin(position_enc[1:, 0::2]) # dim 2i
      position_enc[1:, 1::2] = np.cos(position_enc[1:, 1::2]) # dim 2i+1
      return t.from_numpy(position_enc).type(t.FloatTensor)
  
  def get_sinusoid_encoding_table(n_position, d_hid, padding_idx=None):
      ''' Sinusoid position encoding table '''
  
      def cal_angle(position, hid_idx):
          return position / np.power(10000, 2 * (hid_idx // 2) / d_hid)
  
      def get_posi_angle_vec(position):
          return [cal_angle(position, hid_j) for hid_j in range(d_hid)]
  
      sinusoid_table = np.array([get_posi_angle_vec(pos_i) for pos_i in range(n_position)])
  
      sinusoid_table[:, 0::2] = np.sin(sinusoid_table[:, 0::2])  # dim 2i
      sinusoid_table[:, 1::2] = np.cos(sinusoid_table[:, 1::2])  # dim 2i+1
  
      if padding_idx is not None:
          # zero vector for padding dimension
          sinusoid_table[padding_idx] = 0.
  
      return t.FloatTensor(sinusoid_table)
  
  def guided_attention(N, T, g=0.2):
      '''Guided attention. Refer to page 3 on the paper.'''
      W = np.zeros((N, T), dtype=np.float32)
      for n_pos in range(W.shape[0]):
          for t_pos in range(W.shape[1]):
              W[n_pos, t_pos] = 1 - np.exp(-(t_pos / float(T) - n_pos / float(N)) ** 2 / (2 * g * g))
      return W
  