diff --git a/examples/fastspeech/alignments/get_alignments.py b/examples/fastspeech/alignments/get_alignments.py
index 3e0eec9..d31bafc 100644
--- a/examples/fastspeech/alignments/get_alignments.py
+++ b/examples/fastspeech/alignments/get_alignments.py
@@ -115,15 +115,10 @@ def alignments(args):
mel_input = fluid.layers.unsqueeze(dg.to_variable(mel_input), [0])
mel_lens = mel_input.shape[1]
- dec_slf_mask = get_triu_tensor(mel_input,
- mel_input).astype(np.float32)
- dec_slf_mask = np.expand_dims(dec_slf_mask, axis=0)
- dec_slf_mask = fluid.layers.cast(
- dg.to_variable(dec_slf_mask != 0), np.float32) * (-2**32 + 1)
pos_mel = np.arange(1, mel_input.shape[1] + 1)
pos_mel = fluid.layers.unsqueeze(dg.to_variable(pos_mel), [0])
mel_pred, postnet_pred, attn_probs, stop_preds, attn_enc, attn_dec = model(
- text, mel_input, pos_text, pos_mel, dec_slf_mask)
+ text, mel_input, pos_text, pos_mel)
mel_input = fluid.layers.concat(
[mel_input, postnet_pred[:, -1:, :]], axis=1)
diff --git a/examples/fastspeech/configs/ljspeech.yaml b/examples/fastspeech/configs/ljspeech.yaml
index 88b335a..96b0d54 100644
--- a/examples/fastspeech/configs/ljspeech.yaml
+++ b/examples/fastspeech/configs/ljspeech.yaml
@@ -29,5 +29,5 @@ train:
grad_clip_thresh: 0.1 #the threshold of grad clip.
checkpoint_interval: 1000
- max_epochs: 10000
+ max_iteration: 500000
diff --git a/examples/fastspeech/train.py b/examples/fastspeech/train.py
index 4ad8dca..e575d0f 100644
--- a/examples/fastspeech/train.py
+++ b/examples/fastspeech/train.py
@@ -62,7 +62,8 @@ def main(args):
cfg = yaml.load(f, Loader=yaml.Loader)
global_step = 0
- place = fluid.CUDAPlace(local_rank) if args.use_gpu else fluid.CPUPlace()
+ place = fluid.CUDAPlace(dg.parallel.Env()
+ .dev_id) if args.use_gpu else fluid.CPUPlace()
fluid.enable_dygraph(place)
if not os.path.exists(args.output):
@@ -88,7 +89,8 @@ def main(args):
cfg['train']['batch_size'],
nranks,
local_rank,
- shuffle=True).reader()
+ shuffle=True).reader
+ iterator = iter(tqdm(reader))
# Load parameters.
global_step = io.load_parameters(
@@ -103,52 +105,53 @@ def main(args):
strategy = dg.parallel.prepare_context()
model = fluid.dygraph.parallel.DataParallel(model, strategy)
- for epoch in range(cfg['train']['max_epochs']):
- pbar = tqdm(reader)
+ while global_step <= cfg['train']['max_iteration']:
+ try:
+ batch = next(iterator)
+ except StopIteration as e:
+ iterator = iter(tqdm(reader))
+ batch = next(iterator)
- for i, data in enumerate(pbar):
- pbar.set_description('Processing at epoch %d' % epoch)
- (character, mel, pos_text, pos_mel, alignment) = data
+ (character, mel, pos_text, pos_mel, alignment) = batch
- global_step += 1
+ global_step += 1
- #Forward
- result = model(
- character, pos_text, mel_pos=pos_mel, length_target=alignment)
- mel_output, mel_output_postnet, duration_predictor_output, _, _ = result
- mel_loss = layers.mse_loss(mel_output, mel)
- mel_postnet_loss = layers.mse_loss(mel_output_postnet, mel)
- duration_loss = layers.mean(
- layers.abs(
- layers.elementwise_sub(duration_predictor_output,
- alignment)))
- total_loss = mel_loss + mel_postnet_loss + duration_loss
+ #Forward
+ result = model(
+ character, pos_text, mel_pos=pos_mel, length_target=alignment)
+ mel_output, mel_output_postnet, duration_predictor_output, _, _ = result
+ mel_loss = layers.mse_loss(mel_output, mel)
+ mel_postnet_loss = layers.mse_loss(mel_output_postnet, mel)
+ duration_loss = layers.mean(
+ layers.abs(
+ layers.elementwise_sub(duration_predictor_output, alignment)))
+ total_loss = mel_loss + mel_postnet_loss + duration_loss
- if local_rank == 0:
- writer.add_scalar('mel_loss', mel_loss.numpy(), global_step)
- writer.add_scalar('post_mel_loss',
- mel_postnet_loss.numpy(), global_step)
- writer.add_scalar('duration_loss',
- duration_loss.numpy(), global_step)
- writer.add_scalar('learning_rate',
- optimizer._learning_rate.step().numpy(),
- global_step)
+ if local_rank == 0:
+ writer.add_scalar('mel_loss', mel_loss.numpy(), global_step)
+ writer.add_scalar('post_mel_loss',
+ mel_postnet_loss.numpy(), global_step)
+ writer.add_scalar('duration_loss',
+ duration_loss.numpy(), global_step)
+ writer.add_scalar('learning_rate',
+ optimizer._learning_rate.step().numpy(),
+ global_step)
- if parallel:
- total_loss = model.scale_loss(total_loss)
- total_loss.backward()
- model.apply_collective_grads()
- else:
- total_loss.backward()
- optimizer.minimize(total_loss)
- model.clear_gradients()
+ if parallel:
+ total_loss = model.scale_loss(total_loss)
+ total_loss.backward()
+ model.apply_collective_grads()
+ else:
+ total_loss.backward()
+ optimizer.minimize(total_loss)
+ model.clear_gradients()
- # save checkpoint
- if local_rank == 0 and global_step % cfg['train'][
- 'checkpoint_interval'] == 0:
- io.save_parameters(
- os.path.join(args.output, 'checkpoints'), global_step,
- model, optimizer)
+ # save checkpoint
+ if local_rank == 0 and global_step % cfg['train'][
+ 'checkpoint_interval'] == 0:
+ io.save_parameters(
+ os.path.join(args.output, 'checkpoints'), global_step, model,
+ optimizer)
if local_rank == 0:
writer.close()
diff --git a/examples/transformer_tts/README.md b/examples/transformer_tts/README.md
index 1ff74a9..0be870c 100644
--- a/examples/transformer_tts/README.md
+++ b/examples/transformer_tts/README.md
@@ -53,7 +53,7 @@ During synthesis, results are saved in `${output}/samples` and tensorboard log i
TransformerTTS model can be trained by running ``train_transformer.py``.
```bash
-python train_trasformer.py \
+python train_transformer.py \
--use_gpu=1 \
--data=${DATAPATH} \
--output='./experiment' \
diff --git a/examples/transformer_tts/configs/ljspeech.yaml b/examples/transformer_tts/configs/ljspeech.yaml
index 33300ab..f5aabf9 100644
--- a/examples/transformer_tts/configs/ljspeech.yaml
+++ b/examples/transformer_tts/configs/ljspeech.yaml
@@ -31,7 +31,7 @@ train:
checkpoint_interval: 1000
image_interval: 2000
- max_epochs: 10000
+ max_iteration: 500000
diff --git a/examples/transformer_tts/train_transformer.py b/examples/transformer_tts/train_transformer.py
index 8cd766b..646176f 100644
--- a/examples/transformer_tts/train_transformer.py
+++ b/examples/transformer_tts/train_transformer.py
@@ -102,105 +102,110 @@ def main(args):
cfg['train']['batch_size'],
nranks,
local_rank,
- shuffle=True).reader()
+ shuffle=True).reader
- for epoch in range(cfg['train']['max_epochs']):
- pbar = tqdm(reader)
- for i, data in enumerate(pbar):
- pbar.set_description('Processing at epoch %d' % epoch)
- character, mel, mel_input, pos_text, pos_mel = data
+ iterator = iter(tqdm(reader))
- global_step += 1
+ global_step += 1
- mel_pred, postnet_pred, attn_probs, stop_preds, attn_enc, attn_dec = model(
- character, mel_input, pos_text, pos_mel)
+ while global_step <= cfg['train']['max_iteration']:
+ try:
+ batch = next(iterator)
+ except StopIteration as e:
+ iterator = iter(tqdm(reader))
+ batch = next(iterator)
- mel_loss = layers.mean(
- layers.abs(layers.elementwise_sub(mel_pred, mel)))
- post_mel_loss = layers.mean(
- layers.abs(layers.elementwise_sub(postnet_pred, mel)))
- loss = mel_loss + post_mel_loss
+ character, mel, mel_input, pos_text, pos_mel = batch
+
+ mel_pred, postnet_pred, attn_probs, stop_preds, attn_enc, attn_dec = model(
+ character, mel_input, pos_text, pos_mel)
+
+ mel_loss = layers.mean(
+ layers.abs(layers.elementwise_sub(mel_pred, mel)))
+ post_mel_loss = layers.mean(
+ layers.abs(layers.elementwise_sub(postnet_pred, mel)))
+ loss = mel_loss + post_mel_loss
+
+ # Note: When used stop token loss the learning did not work.
+ if cfg['network']['stop_token']:
+ label = (pos_mel == 0).astype(np.float32)
+ stop_loss = cross_entropy(stop_preds, label)
+ loss = loss + stop_loss
+
+ if local_rank == 0:
+ writer.add_scalars('training_loss', {
+ 'mel_loss': mel_loss.numpy(),
+ 'post_mel_loss': post_mel_loss.numpy()
+ }, global_step)
- # Note: When used stop token loss the learning did not work.
if cfg['network']['stop_token']:
- label = (pos_mel == 0).astype(np.float32)
- stop_loss = cross_entropy(stop_preds, label)
- loss = loss + stop_loss
-
- if local_rank == 0:
- writer.add_scalars('training_loss', {
- 'mel_loss': mel_loss.numpy(),
- 'post_mel_loss': post_mel_loss.numpy()
- }, global_step)
-
- if cfg['network']['stop_token']:
- writer.add_scalar('stop_loss',
- stop_loss.numpy(), global_step)
-
- if parallel:
- writer.add_scalars('alphas', {
- 'encoder_alpha': model._layers.encoder.alpha.numpy(),
- 'decoder_alpha': model._layers.decoder.alpha.numpy(),
- }, global_step)
- else:
- writer.add_scalars('alphas', {
- 'encoder_alpha': model.encoder.alpha.numpy(),
- 'decoder_alpha': model.decoder.alpha.numpy(),
- }, global_step)
-
- writer.add_scalar('learning_rate',
- optimizer._learning_rate.step().numpy(),
- global_step)
-
- if global_step % cfg['train']['image_interval'] == 1:
- for i, prob in enumerate(attn_probs):
- for j in range(cfg['network']['decoder_num_head']):
- x = np.uint8(
- cm.viridis(prob.numpy()[j * cfg['train'][
- 'batch_size'] // 2]) * 255)
- writer.add_image(
- 'Attention_%d_0' % global_step,
- x,
- i * 4 + j,
- dataformats="HWC")
-
- for i, prob in enumerate(attn_enc):
- for j in range(cfg['network']['encoder_num_head']):
- x = np.uint8(
- cm.viridis(prob.numpy()[j * cfg['train'][
- 'batch_size'] // 2]) * 255)
- writer.add_image(
- 'Attention_enc_%d_0' % global_step,
- x,
- i * 4 + j,
- dataformats="HWC")
-
- for i, prob in enumerate(attn_dec):
- for j in range(cfg['network']['decoder_num_head']):
- x = np.uint8(
- cm.viridis(prob.numpy()[j * cfg['train'][
- 'batch_size'] // 2]) * 255)
- writer.add_image(
- 'Attention_dec_%d_0' % global_step,
- x,
- i * 4 + j,
- dataformats="HWC")
+ writer.add_scalar('stop_loss', stop_loss.numpy(), global_step)
if parallel:
- loss = model.scale_loss(loss)
- loss.backward()
- model.apply_collective_grads()
+ writer.add_scalars('alphas', {
+ 'encoder_alpha': model._layers.encoder.alpha.numpy(),
+ 'decoder_alpha': model._layers.decoder.alpha.numpy(),
+ }, global_step)
else:
- loss.backward()
- optimizer.minimize(loss)
- model.clear_gradients()
+ writer.add_scalars('alphas', {
+ 'encoder_alpha': model.encoder.alpha.numpy(),
+ 'decoder_alpha': model.decoder.alpha.numpy(),
+ }, global_step)
- # save checkpoint
- if local_rank == 0 and global_step % cfg['train'][
- 'checkpoint_interval'] == 0:
- io.save_parameters(
- os.path.join(args.output, 'checkpoints'), global_step,
- model, optimizer)
+ writer.add_scalar('learning_rate',
+ optimizer._learning_rate.step().numpy(),
+ global_step)
+
+ if global_step % cfg['train']['image_interval'] == 1:
+ for i, prob in enumerate(attn_probs):
+ for j in range(cfg['network']['decoder_num_head']):
+ x = np.uint8(
+ cm.viridis(prob.numpy()[j * cfg['train'][
+ 'batch_size'] // nranks]) * 255)
+ writer.add_image(
+ 'Attention_%d_0' % global_step,
+ x,
+ i * 4 + j,
+ dataformats="HWC")
+
+ for i, prob in enumerate(attn_enc):
+ for j in range(cfg['network']['encoder_num_head']):
+ x = np.uint8(
+ cm.viridis(prob.numpy()[j * cfg['train'][
+ 'batch_size'] // nranks]) * 255)
+ writer.add_image(
+ 'Attention_enc_%d_0' % global_step,
+ x,
+ i * 4 + j,
+ dataformats="HWC")
+
+ for i, prob in enumerate(attn_dec):
+ for j in range(cfg['network']['decoder_num_head']):
+ x = np.uint8(
+ cm.viridis(prob.numpy()[j * cfg['train'][
+ 'batch_size'] // nranks]) * 255)
+ writer.add_image(
+ 'Attention_dec_%d_0' % global_step,
+ x,
+ i * 4 + j,
+ dataformats="HWC")
+
+ if parallel:
+ loss = model.scale_loss(loss)
+ loss.backward()
+ model.apply_collective_grads()
+ else:
+ loss.backward()
+ optimizer.minimize(loss)
+ model.clear_gradients()
+
+ # save checkpoint
+ if local_rank == 0 and global_step % cfg['train'][
+ 'checkpoint_interval'] == 0:
+ io.save_parameters(
+ os.path.join(args.output, 'checkpoints'), global_step, model,
+ optimizer)
+ global_step += 1
if local_rank == 0:
writer.close()
diff --git a/parakeet/models/fastspeech/length_regulator.py b/parakeet/models/fastspeech/length_regulator.py
index 478d780..ecf0327 100644
--- a/parakeet/models/fastspeech/length_regulator.py
+++ b/parakeet/models/fastspeech/length_regulator.py
@@ -94,7 +94,8 @@ class LengthRegulator(dg.Layer):
else:
duration_predictor_output = layers.round(duration_predictor_output)
output = self.LR(x, duration_predictor_output, alpha)
- mel_pos = dg.to_variable(np.arange(1, output.shape[1] + 1))
+ mel_pos = dg.to_variable(np.arange(1, output.shape[1] + 1)).astype(
+ np.int64)
mel_pos = layers.unsqueeze(mel_pos, [0])
return output, mel_pos