From 47ec051136b717c324472d2b7f55e925c19a0623 Mon Sep 17 00:00:00 2001 From: TianYuan Date: Wed, 21 Jul 2021 03:39:28 +0000 Subject: [PATCH] add fastspeech2 example data preprocess --- .../fastspeech2/baker/compute_statistics.py | 104 + examples/fastspeech2/baker/conf/default.yaml | 4 +- examples/fastspeech2/baker/config.py | 2 +- .../fastspeech2/baker/fastspeech2_updater.py | 25 +- .../baker/gen_duration_from_textgrid.py | 88 + examples/fastspeech2/baker/get_feats.py | 221 + examples/fastspeech2/baker/normalize.py | 184 + examples/fastspeech2/baker/preprocess.py | 351 ++ examples/fastspeech2/baker/preprocess.sh | 18 + examples/fastspeech2/baker/run.sh | 9 + examples/fastspeech2/baker/simple.lexicon | 4590 +++++++++++++++++ examples/fastspeech2/baker/train.py | 81 +- parakeet/models/__init__.py | 2 +- parakeet/models/fastspeech2.py | 1283 +++-- parakeet/models/fastspeech2_new.py | 660 --- 15 files changed, 6219 insertions(+), 1403 deletions(-) create mode 100644 examples/fastspeech2/baker/compute_statistics.py create mode 100644 examples/fastspeech2/baker/gen_duration_from_textgrid.py create mode 100644 examples/fastspeech2/baker/get_feats.py create mode 100644 examples/fastspeech2/baker/normalize.py create mode 100644 examples/fastspeech2/baker/preprocess.py create mode 100755 examples/fastspeech2/baker/preprocess.sh create mode 100644 examples/fastspeech2/baker/run.sh create mode 100644 examples/fastspeech2/baker/simple.lexicon delete mode 100644 parakeet/models/fastspeech2_new.py diff --git a/examples/fastspeech2/baker/compute_statistics.py b/examples/fastspeech2/baker/compute_statistics.py new file mode 100644 index 0000000..aa4bf4f --- /dev/null +++ b/examples/fastspeech2/baker/compute_statistics.py @@ -0,0 +1,104 @@ +# Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Calculate statistics of feature files.""" + +import argparse +import logging +from pathlib import Path + +import jsonlines +import numpy as np +from parakeet.datasets.data_table import DataTable +from sklearn.preprocessing import StandardScaler +from tqdm import tqdm + +from config import get_cfg_default + + +def main(): + """Run preprocessing process.""" + parser = argparse.ArgumentParser( + description="Compute mean and variance of dumped raw features.") + parser.add_argument( + "--metadata", type=str, help="json file with id and file paths ") + parser.add_argument( + "--field-name", + type=str, + help="name of the field to compute statistics for.") + parser.add_argument( + "--config", type=str, help="yaml format configuration file.") + parser.add_argument( + "--output", + type=str, + help="path to save statistics. if not provided, " + "stats will be saved in the above root directory with name stats.npy") + parser.add_argument( + "--verbose", + type=int, + default=1, + help="logging level. higher is more logging. (default=1)") + args = parser.parse_args() + + # set logger + if args.verbose > 1: + logging.basicConfig( + level=logging.DEBUG, + format="%(asctime)s (%(module)s:%(lineno)d) %(levelname)s: %(message)s" + ) + elif args.verbose > 0: + logging.basicConfig( + level=logging.INFO, + format="%(asctime)s (%(module)s:%(lineno)d) %(levelname)s: %(message)s" + ) + else: + logging.basicConfig( + level=logging.WARN, + format="%(asctime)s (%(module)s:%(lineno)d) %(levelname)s: %(message)s" + ) + logging.warning('Skip DEBUG/INFO messages') + + config = get_cfg_default() + # load config + if args.config: + config.merge_from_file(args.config) + + # check directory existence + if args.output is None: + args.output = Path(args.metadata).parent.with_name(args.field_name + + "_stats.npy") + else: + args.output = Path(args.output) + args.output.parent.mkdir(parents=True, exist_ok=True) + + with jsonlines.open(args.metadata, 'r') as reader: + metadata = list(reader) + dataset = DataTable( + metadata, + fields=[args.field_name], + converters={args.field_name: np.load}, ) + logging.info(f"The number of files = {len(dataset)}.") + + # calculate statistics + scaler = StandardScaler() + for datum in tqdm(dataset): + # StandardScalar supports (*, num_features) by default + scaler.partial_fit(datum[args.field_name]) + + stats = np.stack([scaler.mean_, scaler.scale_], axis=0) + + np.save(str(args.output), stats.astype(np.float32), allow_pickle=False) + + +if __name__ == "__main__": + main() diff --git a/examples/fastspeech2/baker/conf/default.yaml b/examples/fastspeech2/baker/conf/default.yaml index 388dbe3..6d6ac2b 100644 --- a/examples/fastspeech2/baker/conf/default.yaml +++ b/examples/fastspeech2/baker/conf/default.yaml @@ -88,8 +88,8 @@ updater: # OPTIMIZER SETTING # ########################################################### optimizer: - optim: adam # optimizer type - learning_rate: 0.001 # learning rate + optim: adam # optimizer type + learning_rate: 0.0001 # learning rate ########################################################### # TRAINING SETTING # diff --git a/examples/fastspeech2/baker/config.py b/examples/fastspeech2/baker/config.py index ef22188..cc937b9 100644 --- a/examples/fastspeech2/baker/config.py +++ b/examples/fastspeech2/baker/config.py @@ -12,8 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. -import yaml from yacs.config import CfgNode as Configuration +import yaml with open("conf/default.yaml", 'rt') as f: _C = yaml.safe_load(f) diff --git a/examples/fastspeech2/baker/fastspeech2_updater.py b/examples/fastspeech2/baker/fastspeech2_updater.py index f87400f..884efda 100644 --- a/examples/fastspeech2/baker/fastspeech2_updater.py +++ b/examples/fastspeech2/baker/fastspeech2_updater.py @@ -12,15 +12,10 @@ # See the License for the specific language governing permissions and # limitations under the License. -import paddle -from paddle.nn import functional as F -from paddle.fluid.layers import huber_loss - -from parakeet.modules.losses import masked_l1_loss, weighted_mean +from parakeet.models.fastspeech2 import FastSpeech2, FastSpeech2Loss +from parakeet.training.extensions.evaluator import StandardEvaluator from parakeet.training.reporter import report from parakeet.training.updaters.standard_updater import StandardUpdater -from parakeet.training.extensions.evaluator import StandardEvaluator -from parakeet.models.fastspeech2_new import FastSpeech2, FastSpeech2Loss class FastSpeech2Updater(StandardUpdater): @@ -36,7 +31,7 @@ class FastSpeech2Updater(StandardUpdater): self.use_weighted_masking = use_weighted_masking def update_core(self, batch): - before_outs, after_outs, d_outs, p_outs, e_outs, ys, ilens, olens = self.model( + before_outs, after_outs, d_outs, p_outs, e_outs, ys, olens = self.model( text=batch["text"], text_lengths=batch["text_lengths"], speech=batch["speech"], @@ -48,6 +43,7 @@ class FastSpeech2Updater(StandardUpdater): criterion = FastSpeech2Loss( use_masking=self.use_masking, use_weighted_masking=self.use_weighted_masking) + l1_loss, duration_loss, pitch_loss, energy_loss = criterion( after_outs=after_outs, before_outs=before_outs, @@ -58,8 +54,9 @@ class FastSpeech2Updater(StandardUpdater): ds=batch["durations"], ps=batch["pitch"], es=batch["energy"], - ilens=ilens, - olens=olens, ) + ilens=batch["text_lengths"], + olens=olens) + loss = l1_loss + duration_loss + pitch_loss + energy_loss optimizer = self.optimizer @@ -67,7 +64,6 @@ class FastSpeech2Updater(StandardUpdater): loss.backward() optimizer.step() - # import pdb; pdb.set_trace() report("train/loss", float(loss)) report("train/l1_loss", float(l1_loss)) report("train/duration_loss", float(duration_loss)) @@ -86,14 +82,14 @@ class FastSpeech2Evaluator(StandardEvaluator): self.use_weighted_masking = use_weighted_masking def evaluate_core(self, batch): - before_outs, after_outs, d_outs, p_outs, e_outs, ys, ilens, olens = self.model( + before_outs, after_outs, d_outs, p_outs, e_outs, ys, olens = self.model( text=batch["text"], text_lengths=batch["text_lengths"], speech=batch["speech"], speech_lengths=batch["speech_lengths"], durations=batch["durations"], pitch=batch["pitch"], - energy=batch["energy"], ) + energy=batch["energy"]) criterion = FastSpeech2Loss( use_masking=self.use_masking, @@ -108,11 +104,10 @@ class FastSpeech2Evaluator(StandardEvaluator): ds=batch["durations"], ps=batch["pitch"], es=batch["energy"], - ilens=ilens, + ilens=batch["text_lengths"], olens=olens, ) loss = l1_loss + duration_loss + pitch_loss + energy_loss - # import pdb; pdb.set_trace() report("eval/loss", float(loss)) report("eval/l1_loss", float(l1_loss)) report("eval/duration_loss", float(duration_loss)) diff --git a/examples/fastspeech2/baker/gen_duration_from_textgrid.py b/examples/fastspeech2/baker/gen_duration_from_textgrid.py new file mode 100644 index 0000000..3f4d8e7 --- /dev/null +++ b/examples/fastspeech2/baker/gen_duration_from_textgrid.py @@ -0,0 +1,88 @@ +# Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import argparse +import os +from pathlib import Path + +import librosa +import numpy as np +from praatio import tgio + +from config import get_cfg_default + + +def readtg(config, tg_path): + alignment = tgio.openTextgrid(tg_path, readRaw=True) + phones = [] + ends = [] + for interval in alignment.tierDict["phones"].entryList: + phone = interval.label + phones.append(phone) + ends.append(interval.end) + frame_pos = librosa.time_to_frames( + ends, sr=config.fs, hop_length=config.n_shift) + durations = np.diff(frame_pos, prepend=0) + assert len(durations) == len(phones) + results = "" + for (p, d) in zip(phones, durations): + p = "sil" if p == "" else p + results += p + " " + str(d) + " " + return results.strip() + + +# assume that the directory structure of inputdir is inputdir/speaker/*.TextGrid +# in MFA1.x, there are blank labels("") in the end, we replace it with "sil" +def gen_duration_from_textgrid(config, inputdir, output): + durations_dict = {} + + for speaker in os.listdir(inputdir): + subdir = inputdir / speaker + for file in os.listdir(subdir): + if file.endswith(".TextGrid"): + tg_path = subdir / file + name = file.split(".")[0] + durations_dict[name] = readtg(config, tg_path) + with open(output, "w") as wf: + for name in sorted(durations_dict.keys()): + wf.write(name + "|" + durations_dict[name] + "\n") + + +def main(): + # parse config and args + parser = argparse.ArgumentParser( + description="Preprocess audio and then extract features.") + parser.add_argument( + "--inputdir", + default=None, + type=str, + help="directory to alignment files.") + parser.add_argument( + "--output", type=str, required=True, help="output duration file name") + parser.add_argument( + "--config", type=str, help="yaml format configuration file.") + + args = parser.parse_args() + C = get_cfg_default() + if args.config: + C.merge_from_file(args.config) + C.freeze() + + inputdir = Path(args.inputdir).expanduser() + output = Path(args.output).expanduser() + gen_duration_from_textgrid(C, inputdir, output) + + +if __name__ == "__main__": + main() diff --git a/examples/fastspeech2/baker/get_feats.py b/examples/fastspeech2/baker/get_feats.py new file mode 100644 index 0000000..f0f06c1 --- /dev/null +++ b/examples/fastspeech2/baker/get_feats.py @@ -0,0 +1,221 @@ +# Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import librosa +import numpy as np +import pyworld +from scipy.interpolate import interp1d + +from config import get_cfg_default + + +class LogMelFBank(): + def __init__(self, conf): + self.sr = conf.fs + # stft + self.n_fft = conf.n_fft + self.win_length = conf.win_length + self.hop_length = conf.n_shift + self.window = conf.window + self.center = True + self.pad_mode = "reflect" + + # mel + self.n_mels = conf.n_mels + self.fmin = conf.fmin + self.fmax = conf.fmax + + self.mel_filter = self._create_mel_filter() + + def _create_mel_filter(self): + mel_filter = librosa.filters.mel(sr=self.sr, + n_fft=self.n_fft, + n_mels=self.n_mels, + fmin=self.fmin, + fmax=self.fmax) + return mel_filter + + def _stft(self, wav): + D = librosa.core.stft( + wav, + n_fft=self.n_fft, + hop_length=self.hop_length, + win_length=self.win_length, + window=self.window, + center=self.center, + pad_mode=self.pad_mode) + return D + + def _spectrogram(self, wav): + D = self._stft(wav) + return np.abs(D) + + def _mel_spectrogram(self, wav): + S = self._spectrogram(wav) + mel = np.dot(self.mel_filter, S) + return mel + + def get_log_mel_fbank(self, wav): + mel = self._mel_spectrogram(wav) + mel = np.clip(mel, a_min=1e-10, a_max=float("inf")) + mel = np.log10(mel.T) + # (num_frames, n_mels) + return mel + + +class Pitch(): + def __init__(self, conf): + + self.sr = conf.fs + self.hop_length = conf.n_shift + self.f0min = conf.f0min + self.f0max = conf.f0max + + def _convert_to_continuous_f0(self, f0: np.array) -> np.array: + if (f0 == 0).all(): + print("All frames seems to be unvoiced.") + return f0 + + # padding start and end of f0 sequence + start_f0 = f0[f0 != 0][0] + end_f0 = f0[f0 != 0][-1] + start_idx = np.where(f0 == start_f0)[0][0] + end_idx = np.where(f0 == end_f0)[0][-1] + f0[:start_idx] = start_f0 + f0[end_idx:] = end_f0 + + # get non-zero frame index + nonzero_idxs = np.where(f0 != 0)[0] + + # perform linear interpolation + interp_fn = interp1d(nonzero_idxs, f0[nonzero_idxs]) + f0 = interp_fn(np.arange(0, f0.shape[0])) + + return f0 + + def _calculate_f0(self, + input: np.array, + use_continuous_f0=True, + use_log_f0=True) -> np.array: + input = input.astype(np.float) + frame_period = 1000 * self.hop_length / self.sr + f0, timeaxis = pyworld.dio(input, + fs=self.sr, + f0_floor=self.f0min, + f0_ceil=self.f0max, + frame_period=frame_period) + f0 = pyworld.stonemask(input, f0, timeaxis, self.sr) + if use_continuous_f0: + f0 = self._convert_to_continuous_f0(f0) + if use_log_f0: + nonzero_idxs = np.where(f0 != 0)[0] + f0[nonzero_idxs] = np.log(f0[nonzero_idxs]) + return f0.reshape(-1) + + def _average_by_duration(self, input: np.array, d: np.array) -> np.array: + d_cumsum = np.pad(d.cumsum(0), (1, 0), 'constant') + arr_list = [] + for start, end in zip(d_cumsum[:-1], d_cumsum[1:]): + arr = input[start:end] + mask = arr == 0 + arr[mask] = 0 + avg_arr = np.mean(arr, axis=0) if len(arr) != 0 else np.array(0) + arr_list.append(avg_arr) + arr_list = np.expand_dims(np.array(arr_list), 0).T + + return arr_list + + def get_pitch(self, + wav, + use_continuous_f0=True, + use_log_f0=True, + use_token_averaged_f0=True, + duration=None): + f0 = self._calculate_f0(wav, use_continuous_f0, use_log_f0) + if use_token_averaged_f0 and duration is not None: + f0 = self._average_by_duration(f0, duration) + return f0 + + +class Energy(): + def __init__(self, conf): + + self.sr = conf.fs + self.n_fft = conf.n_fft + self.win_length = conf.win_length + self.hop_length = conf.n_shift + self.window = conf.window + self.center = True + self.pad_mode = "reflect" + + def _stft(self, wav): + D = librosa.core.stft( + wav, + n_fft=self.n_fft, + hop_length=self.hop_length, + win_length=self.win_length, + window=self.window, + center=self.center, + pad_mode=self.pad_mode) + return D + + def _calculate_energy(self, input): + input = input.astype(np.float32) + input_stft = self._stft(input) + input_power = np.abs(input_stft)**2 + energy = np.sqrt( + np.clip( + np.sum(input_power, axis=0), a_min=1.0e-10, a_max=float( + 'inf'))) + return energy + + def _average_by_duration(self, input: np.array, d: np.array) -> np.array: + d_cumsum = np.pad(d.cumsum(0), (1, 0), 'constant') + arr_list = [] + for start, end in zip(d_cumsum[:-1], d_cumsum[1:]): + arr = input[start:end] + avg_arr = np.mean(arr, axis=0) if len(arr) != 0 else np.array(0) + arr_list.append(avg_arr) + arr_list = np.expand_dims(np.array(arr_list), 0).T + return arr_list + + def get_energy(self, wav, use_token_averaged_energy=True, duration=None): + energy = self._calculate_energy(wav) + if use_token_averaged_energy and duration is not None: + energy = self._average_by_duration(energy, duration) + return energy + + +if __name__ == "__main__": + C = get_cfg_default() + filename = "../raw_data/data/format.1/000001.flac" + wav, _ = librosa.load(filename, sr=C.fs) + mel_extractor = LogMelFBank(C) + mel = mel_extractor.get_log_mel_fbank(wav) + print(mel) + print(mel.shape) + + pitch_extractor = Pitch(C) + duration = "2 8 8 8 12 11 10 13 11 10 18 9 12 10 12 11 5" + duration = np.array([int(x) for x in duration.split(" ")]) + avg_f0 = pitch_extractor.get_pitch(wav, duration=duration) + print(avg_f0) + print(avg_f0.shape) + + energy_extractor = Energy(C) + duration = "2 8 8 8 12 11 10 13 11 10 18 9 12 10 12 11 5" + duration = np.array([int(x) for x in duration.split(" ")]) + avg_energy = energy_extractor.get_energy(wav, duration=duration) + print(avg_energy) + print(avg_energy.sum()) diff --git a/examples/fastspeech2/baker/normalize.py b/examples/fastspeech2/baker/normalize.py new file mode 100644 index 0000000..2d72ffd --- /dev/null +++ b/examples/fastspeech2/baker/normalize.py @@ -0,0 +1,184 @@ +# Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Normalize feature files and dump them.""" + +import argparse +import logging +from operator import itemgetter +from pathlib import Path + +import jsonlines +import numpy as np +from sklearn.preprocessing import StandardScaler +from tqdm import tqdm +from parakeet.datasets.data_table import DataTable + +from config import get_cfg_default + + +def main(): + """Run preprocessing process.""" + parser = argparse.ArgumentParser( + description="Normalize dumped raw features (See detail in parallel_wavegan/bin/normalize.py)." + ) + parser.add_argument( + "--metadata", + type=str, + required=True, + help="directory including feature files to be normalized. " + "you need to specify either *-scp or rootdir.") + + parser.add_argument( + "--dumpdir", + type=str, + required=True, + help="directory to dump normalized feature files.") + parser.add_argument( + "--speech_stats", + type=str, + required=True, + help="speech statistics file.") + parser.add_argument( + "--pitch_stats", + type=str, + required=True, + help="pitch statistics file.") + parser.add_argument( + "--energy_stats", + type=str, + required=True, + help="energy statistics file.") + parser.add_argument( + "--phones", + type=str, + default="phone_id_map.txt ", + help="phone vocabulary file.") + parser.add_argument( + "--config", type=str, help="yaml format configuration file.") + parser.add_argument( + "--verbose", + type=int, + default=1, + help="logging level. higher is more logging. (default=1)") + args = parser.parse_args() + + # set logger + if args.verbose > 1: + logging.basicConfig( + level=logging.DEBUG, + format="%(asctime)s (%(module)s:%(lineno)d) %(levelname)s: %(message)s" + ) + elif args.verbose > 0: + logging.basicConfig( + level=logging.INFO, + format="%(asctime)s (%(module)s:%(lineno)d) %(levelname)s: %(message)s" + ) + else: + logging.basicConfig( + level=logging.WARN, + format="%(asctime)s (%(module)s:%(lineno)d) %(levelname)s: %(message)s" + ) + logging.warning('Skip DEBUG/INFO messages') + + # load config + config = get_cfg_default() + if args.config: + config.merge_from_file(args.config) + + # check directory existence + dumpdir = Path(args.dumpdir).resolve() + dumpdir.mkdir(parents=True, exist_ok=True) + + # get dataset + with jsonlines.open(args.metadata, 'r') as reader: + metadata = list(reader) + dataset = DataTable( + metadata, + converters={ + "speech": np.load, + "pitch": np.load, + "energy": np.load, + }) + logging.info(f"The number of files = {len(dataset)}.") + + # restore scaler + speech_scaler = StandardScaler() + speech_scaler.mean_ = np.load(args.speech_stats)[0] + speech_scaler.scale_ = np.load(args.speech_stats)[1] + speech_scaler.n_features_in_ = speech_scaler.mean_.shape[0] + + pitch_scaler = StandardScaler() + pitch_scaler.mean_ = np.load(args.pitch_stats)[0] + pitch_scaler.scale_ = np.load(args.pitch_stats)[1] + pitch_scaler.n_features_in_ = pitch_scaler.mean_.shape[0] + + energy_scaler = StandardScaler() + energy_scaler.mean_ = np.load(args.energy_stats)[0] + energy_scaler.scale_ = np.load(args.energy_stats)[1] + energy_scaler.n_features_in_ = energy_scaler.mean_.shape[0] + + voc_phones = {} + with open(args.phones, 'rt') as f: + phn_id = [line.strip().split() for line in f.readlines()] + for phn, id in phn_id: + voc_phones[phn] = int(id) + + # process each file + output_metadata = [] + + for item in tqdm(dataset): + utt_id = item['utt_id'] + speech = item['speech'] + pitch = item['pitch'] + energy = item['energy'] + # normalize + speech = speech_scaler.transform(speech) + speech_dir = dumpdir / "data_speech" + speech_dir.mkdir(parents=True, exist_ok=True) + speech_path = speech_dir / f"{utt_id}_speech.npy" + np.save(speech_path, speech.astype(np.float32), allow_pickle=False) + + pitch = pitch_scaler.transform(pitch) + pitch_dir = dumpdir / "data_pitch" + pitch_dir.mkdir(parents=True, exist_ok=True) + pitch_path = pitch_dir / f"{utt_id}_pitch.npy" + np.save(pitch_path, pitch.astype(np.float32), allow_pickle=False) + + energy = energy_scaler.transform(energy) + energy_dir = dumpdir / "data_energy" + energy_dir.mkdir(parents=True, exist_ok=True) + energy_path = energy_dir / f"{utt_id}_energy.npy" + np.save(energy_path, energy.astype(np.float32), allow_pickle=False) + phone_ids = [voc_phones[p] for p in item['phones']] + record = { + "utt_id": item['utt_id'], + "text": phone_ids, + "text_lengths": item['text_lengths'], + "speech_lengths": item['speech_lengths'], + "durations": item['durations'], + "speech": str(speech_path), + "pitch": str(pitch_path), + "energy": str(energy_path) + } + output_metadata.append(record) + output_metadata.sort(key=itemgetter('utt_id')) + output_metadata_path = Path(args.dumpdir) / "metadata.jsonl" + with jsonlines.open(output_metadata_path, 'w') as writer: + for item in output_metadata: + writer.write(item) + logging.info(f"metadata dumped into {output_metadata_path}") + + +if __name__ == "__main__": + main() diff --git a/examples/fastspeech2/baker/preprocess.py b/examples/fastspeech2/baker/preprocess.py new file mode 100644 index 0000000..853fc01 --- /dev/null +++ b/examples/fastspeech2/baker/preprocess.py @@ -0,0 +1,351 @@ +# Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import argparse +from concurrent.futures import ThreadPoolExecutor +from operator import itemgetter +from pathlib import Path +from typing import List, Dict, Any + +import jsonlines +import librosa +import numpy as np +import tqdm + +from config import get_cfg_default +from get_feats import LogMelFBank, Energy, Pitch + + +def get_phn_dur(file_name): + ''' + read MFA duration.txt + Parameters + ---------- + file_name : str or Path + path of gen_duration_from_textgrid.py's result + Returns + ---------- + Dict + sentence: {'utt': ([char], [int])} + ''' + f = open(file_name, 'r') + sentence = {} + for line in f: + utt = line.strip().split('|')[0] + p_d = line.strip().split('|')[-1] + phn_dur = p_d.split() + phn = phn_dur[::2] + dur = phn_dur[1::2] + assert len(phn) == len(dur) + sentence[utt] = (phn, [int(i) for i in dur]) + f.close() + return sentence + + +def deal_silence(sentence): + ''' + merge silences, set + Parameters + ---------- + sentence : Dict + sentence: {'utt': ([char], [int])} + ''' + for utt in sentence: + cur_phn, cur_dur = sentence[utt] + new_phn = [] + new_dur = [] + + # merge sp and sil + for i, p in enumerate(cur_phn): + if i > 0 and 'sil' == p and cur_phn[i - 1] in {"sil", "sp"}: + new_dur[-1] += cur_dur[i] + new_phn[-1] = 'sil' + else: + new_phn.append(p) + new_dur.append(cur_dur[i]) + + # merge little sil in the begin + if new_phn[0] == 'sil' and new_dur[0] <= 14: + new_phn = new_phn[1:] + new_dur[1] += new_dur[0] + new_dur = new_dur[1:] + + # replace the last sil with if exist + if new_phn[-1] == 'sil': + new_phn[-1] = '' + else: + new_phn.append('') + new_dur.append(0) + + for i, (p, d) in enumerate(zip(new_phn, new_dur)): + if p in {"sil", "sp"}: + if d < 14: + new_phn[i] = 'sp' + else: + new_phn[i] = 'sp1' + + assert len(new_phn) == len(new_dur) + sentence[utt] = (new_phn, new_dur) + + +def get_input_token(sentence, output_path): + ''' + get phone set from training data and save it + Parameters + ---------- + sentence : Dict + sentence: {'utt': ([char], [int])} + output_path : str or path + path to save phone_id_map + ''' + phn_emb = set() + for utt in sentence: + for phn in sentence[utt][0]: + if phn != "": + phn_emb.add(phn) + phn_emb = list(phn_emb) + phn_emb.sort() + phn_emb = ["", ""] + phn_emb + phn_emb += ["，", "。", "？", "！", ""] + + f = open(output_path, 'w') + for i, phn in enumerate(phn_emb): + f.write(phn + ' ' + str(i) + '\n') + f.close() + + +def compare_duration_and_mel_length(sentences, utt, mel): + ''' + check duration error, correct sentences[utt] if possible, else pop sentences[utt] + Parameters + ---------- + sentences : Dict + sentences[utt] = [phones_list ,durations_list] + utt : str + utt_id + mel : np.ndarry + features (num_frames, n_mels) + ''' + + if utt in sentences: + len_diff = mel.shape[0] - sum(sentences[utt][1]) + if len_diff != 0: + if len_diff > 0: + sentences[utt][1][-1] += len_diff + elif sentences[utt][1][-1] + len_diff > 0: + sentences[utt][1][-1] += len_diff + elif sentences[utt][1][0] + len_diff > 0: + sentences[utt][1][0] += len_diff + else: + # 一般不会触发这个 + print("the len_diff is unable to correct:", len_diff) + sentences.pop(utt) + + +def process_sentence( + config: Dict[str, Any], + fp: Path, + sentences: Dict, + output_dir: Path, + mel_extractor=None, + pitch_extractor=None, + energy_extractor=None, ): + utt_id = fp.stem + record = None + if utt_id in sentences: + # reading, resampling may occur + wav, _ = librosa.load(str(fp), sr=config.fs) + assert len(wav.shape) == 1, f"{utt_id} is not a mono-channel audio." + assert np.abs(wav).max( + ) <= 1.0, f"{utt_id} is seems to be different that 16 bit PCM." + # extract mel feats + logmel = mel_extractor.get_log_mel_fbank(wav) + # change duration according to mel_length + compare_duration_and_mel_length(sentences, utt_id, logmel) + phones = sentences[utt_id][0] + duration = sentences[utt_id][1] + num_frames = logmel.shape[0] + assert sum(duration) == num_frames + mel_dir = output_dir / "data_speech" + mel_dir.mkdir(parents=True, exist_ok=True) + mel_path = mel_dir / (utt_id + "_speech.npy") + np.save(mel_path, logmel) + # extract pitch and energy + f0 = pitch_extractor.get_pitch(wav, duration=np.array(duration)) + assert f0.shape[0] == len(duration) + f0_dir = output_dir / "data_pitch" + f0_dir.mkdir(parents=True, exist_ok=True) + f0_path = f0_dir / (utt_id + "_pitch.npy") + np.save(f0_path, f0) + energy = energy_extractor.get_energy(wav, duration=np.array(duration)) + assert energy.shape[0] == len(duration) + energy_dir = output_dir / "data_energy" + energy_dir.mkdir(parents=True, exist_ok=True) + energy_path = energy_dir / (utt_id + "_energy.npy") + np.save(energy_path, energy) + record = { + "utt_id": utt_id, + "phones": phones, + "text_lengths": len(phones), + "speech_lengths": num_frames, + "durations": duration, + # use absolute path + "speech": str(mel_path.resolve()), + "pitch": str(f0_path.resolve()), + "energy": str(energy_path.resolve()) + } + return record + + +def process_sentences(config, + fps: List[Path], + sentences: Dict, + output_dir: Path, + mel_extractor=None, + pitch_extractor=None, + energy_extractor=None, + nprocs: int=1): + if nprocs == 1: + results = [] + for fp in tqdm.tqdm(fps, total=len(fps)): + record = process_sentence(config, fp, sentences, output_dir, + mel_extractor, pitch_extractor, + energy_extractor) + if record: + results.append(record) + else: + with ThreadPoolExecutor(nprocs) as pool: + futures = [] + with tqdm.tqdm(total=len(fps)) as progress: + for fp in fps: + future = pool.submit(process_sentence, config, fp, + sentences, output_dir, mel_extractor, + pitch_extractor, energy_extractor) + future.add_done_callback(lambda p: progress.update()) + futures.append(future) + + results = [] + for ft in futures: + record = ft.result() + if record: + results.append(record) + + results.sort(key=itemgetter("utt_id")) + with jsonlines.open(output_dir / "metadata.jsonl", 'w') as writer: + for item in results: + writer.write(item) + print("Done") + + +def main(): + # parse config and args + parser = argparse.ArgumentParser( + description="Preprocess audio and then extract features.") + parser.add_argument( + "--rootdir", + default=None, + type=str, + help="directory to baker dataset.") + parser.add_argument( + "--dur_path", + default=None, + type=str, + help="path to baker durations.txt.") + parser.add_argument( + "--dumpdir", + type=str, + required=True, + help="directory to dump feature files.") + parser.add_argument( + "--config", type=str, help="yaml format configuration file.") + parser.add_argument( + "--verbose", + type=int, + default=1, + help="logging level. higher is more logging. (default=1)") + parser.add_argument( + "--num_cpu", type=int, default=1, help="number of process.") + args = parser.parse_args() + + C = get_cfg_default() + if args.config: + C.merge_from_file(args.config) + C.freeze() + + if args.verbose > 1: + print(vars(args)) + print(C) + + root_dir = Path(args.rootdir).expanduser() + dumpdir = Path(args.dumpdir).expanduser() + dumpdir.mkdir(parents=True, exist_ok=True) + + sentences = get_phn_dur(args.dur_path) + deal_silence(sentences) + phone_id_map_path = dumpdir / "phone_id_map.txt" + get_input_token(sentences, phone_id_map_path) + wav_files = sorted(list((root_dir / "Wave").rglob("*.wav"))) + + # split data into 3 sections + num_train = 9800 + num_dev = 100 + + train_wav_files = wav_files[:num_train] + dev_wav_files = wav_files[num_train:num_train + num_dev] + test_wav_files = wav_files[num_train + num_dev:] + + train_dump_dir = dumpdir / "train" / "raw" + train_dump_dir.mkdir(parents=True, exist_ok=True) + dev_dump_dir = dumpdir / "dev" / "raw" + dev_dump_dir.mkdir(parents=True, exist_ok=True) + test_dump_dir = dumpdir / "test" / "raw" + test_dump_dir.mkdir(parents=True, exist_ok=True) + + # Extractor + mel_extractor = LogMelFBank(C) + pitch_extractor = Pitch(C) + energy_extractor = Energy(C) + + # process for the 3 sections + process_sentences( + C, + train_wav_files, + sentences, + train_dump_dir, + mel_extractor, + pitch_extractor, + energy_extractor, + nprocs=args.num_cpu) + process_sentences( + C, + dev_wav_files, + sentences, + dev_dump_dir, + mel_extractor, + pitch_extractor, + energy_extractor, + nprocs=args.num_cpu) + process_sentences( + C, + test_wav_files, + sentences, + test_dump_dir, + mel_extractor, + pitch_extractor, + energy_extractor, + nprocs=args.num_cpu) + + +if __name__ == "__main__": + main() diff --git a/examples/fastspeech2/baker/preprocess.sh b/examples/fastspeech2/baker/preprocess.sh new file mode 100755 index 0000000..f0271be --- /dev/null +++ b/examples/fastspeech2/baker/preprocess.sh @@ -0,0 +1,18 @@ +#!/bin/bash + +# get durations from MFA's result +python3 gen_duration_from_textgrid.py --inputdir ./baker_alignment_tone --output durations.txt + +# extract features +python3 preprocess.py --rootdir=~/datasets/BZNSYP/ --dumpdir=dump --dur_path durations.txt --num_cpu 16 + +# # get features' stats(mean and std) +python3 compute_statistics.py --metadata=dump/train/raw/metadata.jsonl --field-name="speech" +python3 compute_statistics.py --metadata=dump/train/raw/metadata.jsonl --field-name="pitch" +python3 compute_statistics.py --metadata=dump/train/raw/metadata.jsonl --field-name="energy" + +# normalize and covert phone to id, dev and test should use train's stats +python3 normalize.py --metadata=dump/train/raw/metadata.jsonl --dumpdir=dump/train/norm --speech_stats=dump/train/speech_stats.npy --pitch_stats=dump/train/pitch_stats.npy --energy_stats=dump/train/energy_stats.npy --phones dump/phone_id_map.txt +python3 normalize.py --metadata=dump/dev/raw/metadata.jsonl --dumpdir=dump/dev/norm --speech_stats=dump/train/speech_stats.npy --pitch_stats=dump/train/pitch_stats.npy --energy_stats=dump/train/energy_stats.npy --phones dump/phone_id_map.txt +python3 normalize.py --metadata=dump/test/raw/metadata.jsonl --dumpdir=dump/test/norm --speech_stats=dump/train/speech_stats.npy --pitch_stats=dump/train/pitch_stats.npy --energy_stats=dump/train/energy_stats.npy --phones dump/phone_id_map.txt + diff --git a/examples/fastspeech2/baker/run.sh b/examples/fastspeech2/baker/run.sh new file mode 100644 index 0000000..f8dcc52 --- /dev/null +++ b/examples/fastspeech2/baker/run.sh @@ -0,0 +1,9 @@ +#!/bin/bash + +python3 train.py \ + --train-metadata=dump/train/norm/metadata.jsonl \ + --dev-metadata=dump/dev/norm/metadata.jsonl \ + --config=conf/default.yaml \ + --output-dir=exp/default \ + --nprocs=1 \ + --phones=dump/phone_id_map.txt \ No newline at end of file diff --git a/examples/fastspeech2/baker/simple.lexicon b/examples/fastspeech2/baker/simple.lexicon new file mode 100644 index 0000000..1a4e0cd --- /dev/null +++ b/examples/fastspeech2/baker/simple.lexicon @@ -0,0 +1,4590 @@ +a1 a1 +a2 a2 +a3 a3 +a4 a4 +a5 a5 +ar1 ar1 +ar2 ar2 +ar3 ar3 +ar4 ar4 +ar5 ar5 +ai1 ai1 +ai2 ai2 +ai3 ai3 +ai4 ai4 +ai5 ai5 +air1 air1 +air2 air2 +air3 air3 +air4 air4 +air5 air5 +ao1 ao1 +ao2 ao2 +ao3 ao3 +ao4 ao4 +ao5 ao5 +aor1 aor1 +aor2 aor2 +aor3 aor3 +aor4 aor4 +aor5 aor5 +an1 an1 +an2 an2 +an3 an3 +an4 an4 +an5 an5 +anr1 anr1 +anr2 anr2 +anr3 anr3 +anr4 anr4 +anr5 anr5 +ang1 ang1 +ang2 ang2 +ang3 ang3 +ang4 ang4 +ang5 ang5 +angr1 angr1 +angr2 angr2 +angr3 angr3 +angr4 angr4 +angr5 angr5 +e1 e1 +e2 e2 +e3 e3 +e4 e4 +e5 e5 +er1 er1 +er2 er2 +er3 er3 +er4 er4 +er5 er5 +ei1 ei1 +ei2 ei2 +ei3 ei3 +ei4 ei4 +ei5 ei5 +eir1 eir1 +eir2 eir2 +eir3 eir3 +eir4 eir4 +eir5 eir5 +en1 en1 +en2 en2 +en3 en3 +en4 en4 +en5 en5 +enr1 enr1 +enr2 enr2 +enr3 enr3 +enr4 enr4 +enr5 enr5 +eng1 eng1 +eng2 eng2 +eng3 eng3 +eng4 eng4 +eng5 eng5 +engr1 engr1 +engr2 engr2 +engr3 engr3 +engr4 engr4 +engr5 engr5 +o1 o1 +o2 o2 +o3 o3 +o4 o4 +o5 o5 +or1 or1 +or2 or2 +or3 or3 +or4 or4 +or5 or5 +ou1 ou1 +ou2 ou2 +ou3 ou3 +ou4 ou4 +ou5 ou5 +our1 our1 +our2 our2 +our3 our3 +our4 our4 +our5 our5 +ong1 ong1 +ong2 ong2 +ong3 ong3 +ong4 ong4 +ong5 ong5 +ongr1 ongr1 +ongr2 ongr2 +ongr3 ongr3 +ongr4 ongr4 +ongr5 ongr5 +yi1 i1 +yi2 i2 +yi3 i3 +yi4 i4 +yi5 i5 +yir1 ir1 +yir2 ir2 +yir3 ir3 +yir4 ir4 +yir5 ir5 +ya1 ia1 +ya2 ia2 +ya3 ia3 +ya4 ia4 +ya5 ia5 +yar1 iar1 +yar2 iar2 +yar3 iar3 +yar4 iar4 +yar5 iar5 +yao1 iao1 +yao2 iao2 +yao3 iao3 +yao4 iao4 +yao5 iao5 +yaor1 iaor1 +yaor2 iaor2 +yaor3 iaor3 +yaor4 iaor4 +yaor5 iaor5 +yan1 ian1 +yan2 ian2 +yan3 ian3 +yan4 ian4 +yan5 ian5 +yanr1 ianr1 +yanr2 ianr2 +yanr3 ianr3 +yanr4 ianr4 +yanr5 ianr5 +yang1 iang1 +yang2 iang2 +yang3 iang3 +yang4 iang4 +yang5 iang5 +yangr1 iangr1 +yangr2 iangr2 +yangr3 iangr3 +yangr4 iangr4 +yangr5 iangr5 +ye1 ie1 +ye2 ie2 +ye3 ie3 +ye4 ie4 +ye5 ie5 +yer1 ier1 +yer2 ier2 +yer3 ier3 +yer4 ier4 +yer5 ier5 +yo1 io1 +yo2 io2 +yo3 io3 +yo4 io4 +yo5 io5 +yor1 ior1 +yor2 ior2 +yor3 ior3 +yor4 ior4 +yor5 ior5 +you1 iou1 +you2 iou2 +you3 iou3 +you4 iou4 +you5 iou5 +your1 iour1 +your2 iour2 +your3 iour3 +your4 iour4 +your5 iour5 +yong1 iong1 +yong2 iong2 +yong3 iong3 +yong4 iong4 +yong5 iong5 +yongr1 iongr1 +yongr2 iongr2 +yongr3 iongr3 +yongr4 iongr4 +yongr5 iongr5 +yin1 in1 +yin2 in2 +yin3 in3 +yin4 in4 +yin5 in5 +yinr1 inr1 +yinr2 inr2 +yinr3 inr3 +yinr4 inr4 +yinr5 inr5 +ying1 ing1 +ying2 ing2 +ying3 ing3 +ying4 ing4 +ying5 ing5 +yingr1 ingr1 +yingr2 ingr2 +yingr3 ingr3 +yingr4 ingr4 +yingr5 ingr5 +wu1 u1 +wu2 u2 +wu3 u3 +wu4 u4 +wu5 u5 +wur1 ur1 +wur2 ur2 +wur3 ur3 +wur4 ur4 +wur5 ur5 +wa1 ua1 +wa2 ua2 +wa3 ua3 +wa4 ua4 +wa5 ua5 +war1 uar1 +war2 uar2 +war3 uar3 +war4 uar4 +war5 uar5 +wai1 uai1 +wai2 uai2 +wai3 uai3 +wai4 uai4 +wai5 uai5 +wair1 uair1 +wair2 uair2 +wair3 uair3 +wair4 uair4 +wair5 uair5 +wan1 uan1 +wan2 uan2 +wan3 uan3 +wan4 uan4 +wan5 uan5 +wanr1 uanr1 +wanr2 uanr2 +wanr3 uanr3 +wanr4 uanr4 +wanr5 uanr5 +wang1 uang1 +wang2 uang2 +wang3 uang3 +wang4 uang4 +wang5 uang5 +wangr1 uangr1 +wangr2 uangr2 +wangr3 uangr3 +wangr4 uangr4 +wangr5 uangr5 +wei1 uei1 +wei2 uei2 +wei3 uei3 +wei4 uei4 +wei5 uei5 +weir1 ueir1 +weir2 ueir2 +weir3 ueir3 +weir4 ueir4 +weir5 ueir5 +wo1 uo1 +wo2 uo2 +wo3 uo3 +wo4 uo4 +wo5 uo5 +wor1 uor1 +wor2 uor2 +wor3 uor3 +wor4 uor4 +wor5 uor5 +wen1 uen1 +wen2 uen2 +wen3 uen3 +wen4 uen4 +wen5 uen5 +wenr1 uenr1 +wenr2 uenr2 +wenr3 uenr3 +wenr4 uenr4 +wenr5 uenr5 +weng1 ueng1 +weng2 ueng2 +weng3 ueng3 +weng4 ueng4 +weng5 ueng5 +wengr1 uengr1 +wengr2 uengr2 +wengr3 uengr3 +wengr4 uengr4 +wengr5 uengr5 +yu1 v1 +yu2 v2 +yu3 v3 +yu4 v4 +yu5 v5 +yur1 vr1 +yur2 vr2 +yur3 vr3 +yur4 vr4 +yur5 vr5 +yue1 ve1 +yue2 ve2 +yue3 ve3 +yue4 ve4 +yue5 ve5 +yuer1 ver1 +yuer2 ver2 +yuer3 ver3 +yuer4 ver4 +yuer5 ver5 +yuan1 van1 +yuan2 van2 +yuan3 van3 +yuan4 van4 +yuan5 van5 +yuanr1 vanr1 +yuanr2 vanr2 +yuanr3 vanr3 +yuanr4 vanr4 +yuanr5 vanr5 +yun1 vn1 +yun2 vn2 +yun3 vn3 +yun4 vn4 +yun5 vn5 +yunr1 vnr1 +yunr2 vnr2 +yunr3 vnr3 +yunr4 vnr4 +yunr5 vnr5 +ba1 b a1 +ba2 b a2 +ba3 b a3 +ba4 b a4 +ba5 b a5 +bar1 b ar1 +bar2 b ar2 +bar3 b ar3 +bar4 b ar4 +bar5 b ar5 +bai1 b ai1 +bai2 b ai2 +bai3 b ai3 +bai4 b ai4 +bai5 b ai5 +bair1 b air1 +bair2 b air2 +bair3 b air3 +bair4 b air4 +bair5 b air5 +bao1 b ao1 +bao2 b ao2 +bao3 b ao3 +bao4 b ao4 +bao5 b ao5 +baor1 b aor1 +baor2 b aor2 +baor3 b aor3 +baor4 b aor4 +baor5 b aor5 +ban1 b an1 +ban2 b an2 +ban3 b an3 +ban4 b an4 +ban5 b an5 +banr1 b anr1 +banr2 b anr2 +banr3 b anr3 +banr4 b anr4 +banr5 b anr5 +bang1 b ang1 +bang2 b ang2 +bang3 b ang3 +bang4 b ang4 +bang5 b ang5 +bangr1 b angr1 +bangr2 b angr2 +bangr3 b angr3 +bangr4 b angr4 +bangr5 b angr5 +be1 b e1 +be2 b e2 +be3 b e3 +be4 b e4 +be5 b e5 +ber1 b er1 +ber2 b er2 +ber3 b er3 +ber4 b er4 +ber5 b er5 +bei1 b ei1 +bei2 b ei2 +bei3 b ei3 +bei4 b ei4 +bei5 b ei5 +beir1 b eir1 +beir2 b eir2 +beir3 b eir3 +beir4 b eir4 +beir5 b eir5 +ben1 b en1 +ben2 b en2 +ben3 b en3 +ben4 b en4 +ben5 b en5 +benr1 b enr1 +benr2 b enr2 +benr3 b enr3 +benr4 b enr4 +benr5 b enr5 +beng1 b eng1 +beng2 b eng2 +beng3 b eng3 +beng4 b eng4 +beng5 b eng5 +bengr1 b engr1 +bengr2 b engr2 +bengr3 b engr3 +bengr4 b engr4 +bengr5 b engr5 +bo1 b o1 +bo2 b o2 +bo3 b o3 +bo4 b o4 +bo5 b o5 +bor1 b or1 +bor2 b or2 +bor3 b or3 +bor4 b or4 +bor5 b or5 +bou1 b ou1 +bou2 b ou2 +bou3 b ou3 +bou4 b ou4 +bou5 b ou5 +bour1 b our1 +bour2 b our2 +bour3 b our3 +bour4 b our4 +bour5 b our5 +bi1 b i1 +bi2 b i2 +bi3 b i3 +bi4 b i4 +bi5 b i5 +bir1 b ir1 +bir2 b ir2 +bir3 b ir3 +bir4 b ir4 +bir5 b ir5 +bia1 b ia1 +bia2 b ia2 +bia3 b ia3 +bia4 b ia4 +bia5 b ia5 +biar1 b iar1 +biar2 b iar2 +biar3 b iar3 +biar4 b iar4 +biar5 b iar5 +biao1 b iao1 +biao2 b iao2 +biao3 b iao3 +biao4 b iao4 +biao5 b iao5 +biaor1 b iaor1 +biaor2 b iaor2 +biaor3 b iaor3 +biaor4 b iaor4 +biaor5 b iaor5 +bian1 b ian1 +bian2 b ian2 +bian3 b ian3 +bian4 b ian4 +bian5 b ian5 +bianr1 b ianr1 +bianr2 b ianr2 +bianr3 b ianr3 +bianr4 b ianr4 +bianr5 b ianr5 +biang1 b iang1 +biang2 b iang2 +biang3 b iang3 +biang4 b iang4 +biang5 b iang5 +biangr1 b iangr1 +biangr2 b iangr2 +biangr3 b iangr3 +biangr4 b iangr4 +biangr5 b iangr5 +bie1 b ie1 +bie2 b ie2 +bie3 b ie3 +bie4 b ie4 +bie5 b ie5 +bier1 b ier1 +bier2 b ier2 +bier3 b ier3 +bier4 b ier4 +bier5 b ier5 +bio1 b io1 +bio2 b io2 +bio3 b io3 +bio4 b io4 +bio5 b io5 +bior1 b ior1 +bior2 b ior2 +bior3 b ior3 +bior4 b ior4 +bior5 b ior5 +biu1 b iou1 +biu2 b iou2 +biu3 b iou3 +biu4 b iou4 +biu5 b iou5 +biur1 b iour1 +biur2 b iour2 +biur3 b iour3 +biur4 b iour4 +biur5 b iour5 +biong1 b iong1 +biong2 b iong2 +biong3 b iong3 +biong4 b iong4 +biong5 b iong5 +biongr1 b iongr1 +biongr2 b iongr2 +biongr3 b iongr3 +biongr4 b iongr4 +biongr5 b iongr5 +bin1 b in1 +bin2 b in2 +bin3 b in3 +bin4 b in4 +bin5 b in5 +binr1 b inr1 +binr2 b inr2 +binr3 b inr3 +binr4 b inr4 +binr5 b inr5 +bing1 b ing1 +bing2 b ing2 +bing3 b ing3 +bing4 b ing4 +bing5 b ing5 +bingr1 b ingr1 +bingr2 b ingr2 +bingr3 b ingr3 +bingr4 b ingr4 +bingr5 b ingr5 +bu1 b u1 +bu2 b u2 +bu3 b u3 +bu4 b u4 +bu5 b u5 +bur1 b ur1 +bur2 b ur2 +bur3 b ur3 +bur4 b ur4 +bur5 b ur5 +pa1 p a1 +pa2 p a2 +pa3 p a3 +pa4 p a4 +pa5 p a5 +par1 p ar1 +par2 p ar2 +par3 p ar3 +par4 p ar4 +par5 p ar5 +pai1 p ai1 +pai2 p ai2 +pai3 p ai3 +pai4 p ai4 +pai5 p ai5 +pair1 p air1 +pair2 p air2 +pair3 p air3 +pair4 p air4 +pair5 p air5 +pao1 p ao1 +pao2 p ao2 +pao3 p ao3 +pao4 p ao4 +pao5 p ao5 +paor1 p aor1 +paor2 p aor2 +paor3 p aor3 +paor4 p aor4 +paor5 p aor5 +pan1 p an1 +pan2 p an2 +pan3 p an3 +pan4 p an4 +pan5 p an5 +panr1 p anr1 +panr2 p anr2 +panr3 p anr3 +panr4 p anr4 +panr5 p anr5 +pang1 p ang1 +pang2 p ang2 +pang3 p ang3 +pang4 p ang4 +pang5 p ang5 +pangr1 p angr1 +pangr2 p angr2 +pangr3 p angr3 +pangr4 p angr4 +pangr5 p angr5 +pe1 p e1 +pe2 p e2 +pe3 p e3 +pe4 p e4 +pe5 p e5 +per1 p er1 +per2 p er2 +per3 p er3 +per4 p er4 +per5 p er5 +pei1 p ei1 +pei2 p ei2 +pei3 p ei3 +pei4 p ei4 +pei5 p ei5 +peir1 p eir1 +peir2 p eir2 +peir3 p eir3 +peir4 p eir4 +peir5 p eir5 +pen1 p en1 +pen2 p en2 +pen3 p en3 +pen4 p en4 +pen5 p en5 +penr1 p enr1 +penr2 p enr2 +penr3 p enr3 +penr4 p enr4 +penr5 p enr5 +peng1 p eng1 +peng2 p eng2 +peng3 p eng3 +peng4 p eng4 +peng5 p eng5 +pengr1 p engr1 +pengr2 p engr2 +pengr3 p engr3 +pengr4 p engr4 +pengr5 p engr5 +po1 p o1 +po2 p o2 +po3 p o3 +po4 p o4 +po5 p o5 +por1 p or1 +por2 p or2 +por3 p or3 +por4 p or4 +por5 p or5 +pou1 p ou1 +pou2 p ou2 +pou3 p ou3 +pou4 p ou4 +pou5 p ou5 +pour1 p our1 +pour2 p our2 +pour3 p our3 +pour4 p our4 +pour5 p our5 +pi1 p i1 +pi2 p i2 +pi3 p i3 +pi4 p i4 +pi5 p i5 +pir1 p ir1 +pir2 p ir2 +pir3 p ir3 +pir4 p ir4 +pir5 p ir5 +pia1 p ia1 +pia2 p ia2 +pia3 p ia3 +pia4 p ia4 +pia5 p ia5 +piar1 p iar1 +piar2 p iar2 +piar3 p iar3 +piar4 p iar4 +piar5 p iar5 +piao1 p iao1 +piao2 p iao2 +piao3 p iao3 +piao4 p iao4 +piao5 p iao5 +piaor1 p iaor1 +piaor2 p iaor2 +piaor3 p iaor3 +piaor4 p iaor4 +piaor5 p iaor5 +pian1 p ian1 +pian2 p ian2 +pian3 p ian3 +pian4 p ian4 +pian5 p ian5 +pianr1 p ianr1 +pianr2 p ianr2 +pianr3 p ianr3 +pianr4 p ianr4 +pianr5 p ianr5 +piang1 p iang1 +piang2 p iang2 +piang3 p iang3 +piang4 p iang4 +piang5 p iang5 +piangr1 p iangr1 +piangr2 p iangr2 +piangr3 p iangr3 +piangr4 p iangr4 +piangr5 p iangr5 +pie1 p ie1 +pie2 p ie2 +pie3 p ie3 +pie4 p ie4 +pie5 p ie5 +pier1 p ier1 +pier2 p ier2 +pier3 p ier3 +pier4 p ier4 +pier5 p ier5 +pio1 p io1 +pio2 p io2 +pio3 p io3 +pio4 p io4 +pio5 p io5 +pior1 p ior1 +pior2 p ior2 +pior3 p ior3 +pior4 p ior4 +pior5 p ior5 +piu1 p iou1 +piu2 p iou2 +piu3 p iou3 +piu4 p iou4 +piu5 p iou5 +piur1 p iour1 +piur2 p iour2 +piur3 p iour3 +piur4 p iour4 +piur5 p iour5 +piong1 p iong1 +piong2 p iong2 +piong3 p iong3 +piong4 p iong4 +piong5 p iong5 +piongr1 p iongr1 +piongr2 p iongr2 +piongr3 p iongr3 +piongr4 p iongr4 +piongr5 p iongr5 +pin1 p in1 +pin2 p in2 +pin3 p in3 +pin4 p in4 +pin5 p in5 +pinr1 p inr1 +pinr2 p inr2 +pinr3 p inr3 +pinr4 p inr4 +pinr5 p inr5 +ping1 p ing1 +ping2 p ing2 +ping3 p ing3 +ping4 p ing4 +ping5 p ing5 +pingr1 p ingr1 +pingr2 p ingr2 +pingr3 p ingr3 +pingr4 p ingr4 +pingr5 p ingr5 +pu1 p u1 +pu2 p u2 +pu3 p u3 +pu4 p u4 +pu5 p u5 +pur1 p ur1 +pur2 p ur2 +pur3 p ur3 +pur4 p ur4 +pur5 p ur5 +ma1 m a1 +ma2 m a2 +ma3 m a3 +ma4 m a4 +ma5 m a5 +mar1 m ar1 +mar2 m ar2 +mar3 m ar3 +mar4 m ar4 +mar5 m ar5 +mai1 m ai1 +mai2 m ai2 +mai3 m ai3 +mai4 m ai4 +mai5 m ai5 +mair1 m air1 +mair2 m air2 +mair3 m air3 +mair4 m air4 +mair5 m air5 +mao1 m ao1 +mao2 m ao2 +mao3 m ao3 +mao4 m ao4 +mao5 m ao5 +maor1 m aor1 +maor2 m aor2 +maor3 m aor3 +maor4 m aor4 +maor5 m aor5 +man1 m an1 +man2 m an2 +man3 m an3 +man4 m an4 +man5 m an5 +manr1 m anr1 +manr2 m anr2 +manr3 m anr3 +manr4 m anr4 +manr5 m anr5 +mang1 m ang1 +mang2 m ang2 +mang3 m ang3 +mang4 m ang4 +mang5 m ang5 +mangr1 m angr1 +mangr2 m angr2 +mangr3 m angr3 +mangr4 m angr4 +mangr5 m angr5 +me1 m e1 +me2 m e2 +me3 m e3 +me4 m e4 +me5 m e5 +mer1 m er1 +mer2 m er2 +mer3 m er3 +mer4 m er4 +mer5 m er5 +mei1 m ei1 +mei2 m ei2 +mei3 m ei3 +mei4 m ei4 +mei5 m ei5 +meir1 m eir1 +meir2 m eir2 +meir3 m eir3 +meir4 m eir4 +meir5 m eir5 +men1 m en1 +men2 m en2 +men3 m en3 +men4 m en4 +men5 m en5 +menr1 m enr1 +menr2 m enr2 +menr3 m enr3 +menr4 m enr4 +menr5 m enr5 +meng1 m eng1 +meng2 m eng2 +meng3 m eng3 +meng4 m eng4 +meng5 m eng5 +mengr1 m engr1 +mengr2 m engr2 +mengr3 m engr3 +mengr4 m engr4 +mengr5 m engr5 +mo1 m o1 +mo2 m o2 +mo3 m o3 +mo4 m o4 +mo5 m o5 +mor1 m or1 +mor2 m or2 +mor3 m or3 +mor4 m or4 +mor5 m or5 +mou1 m ou1 +mou2 m ou2 +mou3 m ou3 +mou4 m ou4 +mou5 m ou5 +mour1 m our1 +mour2 m our2 +mour3 m our3 +mour4 m our4 +mour5 m our5 +mi1 m i1 +mi2 m i2 +mi3 m i3 +mi4 m i4 +mi5 m i5 +mir1 m ir1 +mir2 m ir2 +mir3 m ir3 +mir4 m ir4 +mir5 m ir5 +mia1 m ia1 +mia2 m ia2 +mia3 m ia3 +mia4 m ia4 +mia5 m ia5 +miar1 m iar1 +miar2 m iar2 +miar3 m iar3 +miar4 m iar4 +miar5 m iar5 +miao1 m iao1 +miao2 m iao2 +miao3 m iao3 +miao4 m iao4 +miao5 m iao5 +miaor1 m iaor1 +miaor2 m iaor2 +miaor3 m iaor3 +miaor4 m iaor4 +miaor5 m iaor5 +mian1 m ian1 +mian2 m ian2 +mian3 m ian3 +mian4 m ian4 +mian5 m ian5 +mianr1 m ianr1 +mianr2 m ianr2 +mianr3 m ianr3 +mianr4 m ianr4 +mianr5 m ianr5 +miang1 m iang1 +miang2 m iang2 +miang3 m iang3 +miang4 m iang4 +miang5 m iang5 +miangr1 m iangr1 +miangr2 m iangr2 +miangr3 m iangr3 +miangr4 m iangr4 +miangr5 m iangr5 +mie1 m ie1 +mie2 m ie2 +mie3 m ie3 +mie4 m ie4 +mie5 m ie5 +mier1 m ier1 +mier2 m ier2 +mier3 m ier3 +mier4 m ier4 +mier5 m ier5 +mio1 m io1 +mio2 m io2 +mio3 m io3 +mio4 m io4 +mio5 m io5 +mior1 m ior1 +mior2 m ior2 +mior3 m ior3 +mior4 m ior4 +mior5 m ior5 +miu1 m iou1 +miu2 m iou2 +miu3 m iou3 +miu4 m iou4 +miu5 m iou5 +miur1 m iour1 +miur2 m iour2 +miur3 m iour3 +miur4 m iour4 +miur5 m iour5 +miong1 m iong1 +miong2 m iong2 +miong3 m iong3 +miong4 m iong4 +miong5 m iong5 +miongr1 m iongr1 +miongr2 m iongr2 +miongr3 m iongr3 +miongr4 m iongr4 +miongr5 m iongr5 +min1 m in1 +min2 m in2 +min3 m in3 +min4 m in4 +min5 m in5 +minr1 m inr1 +minr2 m inr2 +minr3 m inr3 +minr4 m inr4 +minr5 m inr5 +ming1 m ing1 +ming2 m ing2 +ming3 m ing3 +ming4 m ing4 +ming5 m ing5 +mingr1 m ingr1 +mingr2 m ingr2 +mingr3 m ingr3 +mingr4 m ingr4 +mingr5 m ingr5 +mu1 m u1 +mu2 m u2 +mu3 m u3 +mu4 m u4 +mu5 m u5 +mur1 m ur1 +mur2 m ur2 +mur3 m ur3 +mur4 m ur4 +mur5 m ur5 +fa1 f a1 +fa2 f a2 +fa3 f a3 +fa4 f a4 +fa5 f a5 +far1 f ar1 +far2 f ar2 +far3 f ar3 +far4 f ar4 +far5 f ar5 +fai1 f ai1 +fai2 f ai2 +fai3 f ai3 +fai4 f ai4 +fai5 f ai5 +fair1 f air1 +fair2 f air2 +fair3 f air3 +fair4 f air4 +fair5 f air5 +fao1 f ao1 +fao2 f ao2 +fao3 f ao3 +fao4 f ao4 +fao5 f ao5 +faor1 f aor1 +faor2 f aor2 +faor3 f aor3 +faor4 f aor4 +faor5 f aor5 +fan1 f an1 +fan2 f an2 +fan3 f an3 +fan4 f an4 +fan5 f an5 +fanr1 f anr1 +fanr2 f anr2 +fanr3 f anr3 +fanr4 f anr4 +fanr5 f anr5 +fang1 f ang1 +fang2 f ang2 +fang3 f ang3 +fang4 f ang4 +fang5 f ang5 +fangr1 f angr1 +fangr2 f angr2 +fangr3 f angr3 +fangr4 f angr4 +fangr5 f angr5 +fe1 f e1 +fe2 f e2 +fe3 f e3 +fe4 f e4 +fe5 f e5 +fer1 f er1 +fer2 f er2 +fer3 f er3 +fer4 f er4 +fer5 f er5 +fei1 f ei1 +fei2 f ei2 +fei3 f ei3 +fei4 f ei4 +fei5 f ei5 +feir1 f eir1 +feir2 f eir2 +feir3 f eir3 +feir4 f eir4 +feir5 f eir5 +fen1 f en1 +fen2 f en2 +fen3 f en3 +fen4 f en4 +fen5 f en5 +fenr1 f enr1 +fenr2 f enr2 +fenr3 f enr3 +fenr4 f enr4 +fenr5 f enr5 +feng1 f eng1 +feng2 f eng2 +feng3 f eng3 +feng4 f eng4 +feng5 f eng5 +fengr1 f engr1 +fengr2 f engr2 +fengr3 f engr3 +fengr4 f engr4 +fengr5 f engr5 +fo1 f o1 +fo2 f o2 +fo3 f o3 +fo4 f o4 +fo5 f o5 +for1 f or1 +for2 f or2 +for3 f or3 +for4 f or4 +for5 f or5 +fou1 f ou1 +fou2 f ou2 +fou3 f ou3 +fou4 f ou4 +fou5 f ou5 +four1 f our1 +four2 f our2 +four3 f our3 +four4 f our4 +four5 f our5 +fu1 f u1 +fu2 f u2 +fu3 f u3 +fu4 f u4 +fu5 f u5 +fur1 f ur1 +fur2 f ur2 +fur3 f ur3 +fur4 f ur4 +fur5 f ur5 +da1 d a1 +da2 d a2 +da3 d a3 +da4 d a4 +da5 d a5 +dar1 d ar1 +dar2 d ar2 +dar3 d ar3 +dar4 d ar4 +dar5 d ar5 +dai1 d ai1 +dai2 d ai2 +dai3 d ai3 +dai4 d ai4 +dai5 d ai5 +dair1 d air1 +dair2 d air2 +dair3 d air3 +dair4 d air4 +dair5 d air5 +dao1 d ao1 +dao2 d ao2 +dao3 d ao3 +dao4 d ao4 +dao5 d ao5 +daor1 d aor1 +daor2 d aor2 +daor3 d aor3 +daor4 d aor4 +daor5 d aor5 +dan1 d an1 +dan2 d an2 +dan3 d an3 +dan4 d an4 +dan5 d an5 +danr1 d anr1 +danr2 d anr2 +danr3 d anr3 +danr4 d anr4 +danr5 d anr5 +dang1 d ang1 +dang2 d ang2 +dang3 d ang3 +dang4 d ang4 +dang5 d ang5 +dangr1 d angr1 +dangr2 d angr2 +dangr3 d angr3 +dangr4 d angr4 +dangr5 d angr5 +de1 d e1 +de2 d e2 +de3 d e3 +de4 d e4 +de5 d e5 +der1 d er1 +der2 d er2 +der3 d er3 +der4 d er4 +der5 d er5 +dei1 d ei1 +dei2 d ei2 +dei3 d ei3 +dei4 d ei4 +dei5 d ei5 +deir1 d eir1 +deir2 d eir2 +deir3 d eir3 +deir4 d eir4 +deir5 d eir5 +den1 d en1 +den2 d en2 +den3 d en3 +den4 d en4 +den5 d en5 +denr1 d enr1 +denr2 d enr2 +denr3 d enr3 +denr4 d enr4 +denr5 d enr5 +deng1 d eng1 +deng2 d eng2 +deng3 d eng3 +deng4 d eng4 +deng5 d eng5 +dengr1 d engr1 +dengr2 d engr2 +dengr3 d engr3 +dengr4 d engr4 +dengr5 d engr5 +dou1 d ou1 +dou2 d ou2 +dou3 d ou3 +dou4 d ou4 +dou5 d ou5 +dour1 d our1 +dour2 d our2 +dour3 d our3 +dour4 d our4 +dour5 d our5 +dong1 d ong1 +dong2 d ong2 +dong3 d ong3 +dong4 d ong4 +dong5 d ong5 +dongr1 d ongr1 +dongr2 d ongr2 +dongr3 d ongr3 +dongr4 d ongr4 +dongr5 d ongr5 +di1 d i1 +di2 d i2 +di3 d i3 +di4 d i4 +di5 d i5 +dir1 d ir1 +dir2 d ir2 +dir3 d ir3 +dir4 d ir4 +dir5 d ir5 +dia1 d ia1 +dia2 d ia2 +dia3 d ia3 +dia4 d ia4 +dia5 d ia5 +diar1 d iar1 +diar2 d iar2 +diar3 d iar3 +diar4 d iar4 +diar5 d iar5 +diao1 d iao1 +diao2 d iao2 +diao3 d iao3 +diao4 d iao4 +diao5 d iao5 +diaor1 d iaor1 +diaor2 d iaor2 +diaor3 d iaor3 +diaor4 d iaor4 +diaor5 d iaor5 +dian1 d ian1 +dian2 d ian2 +dian3 d ian3 +dian4 d ian4 +dian5 d ian5 +dianr1 d ianr1 +dianr2 d ianr2 +dianr3 d ianr3 +dianr4 d ianr4 +dianr5 d ianr5 +diang1 d iang1 +diang2 d iang2 +diang3 d iang3 +diang4 d iang4 +diang5 d iang5 +diangr1 d iangr1 +diangr2 d iangr2 +diangr3 d iangr3 +diangr4 d iangr4 +diangr5 d iangr5 +die1 d ie1 +die2 d ie2 +die3 d ie3 +die4 d ie4 +die5 d ie5 +dier1 d ier1 +dier2 d ier2 +dier3 d ier3 +dier4 d ier4 +dier5 d ier5 +dio1 d io1 +dio2 d io2 +dio3 d io3 +dio4 d io4 +dio5 d io5 +dior1 d ior1 +dior2 d ior2 +dior3 d ior3 +dior4 d ior4 +dior5 d ior5 +diu1 d iou1 +diu2 d iou2 +diu3 d iou3 +diu4 d iou4 +diu5 d iou5 +diur1 d iour1 +diur2 d iour2 +diur3 d iour3 +diur4 d iour4 +diur5 d iour5 +diong1 d iong1 +diong2 d iong2 +diong3 d iong3 +diong4 d iong4 +diong5 d iong5 +diongr1 d iongr1 +diongr2 d iongr2 +diongr3 d iongr3 +diongr4 d iongr4 +diongr5 d iongr5 +din1 d in1 +din2 d in2 +din3 d in3 +din4 d in4 +din5 d in5 +dinr1 d inr1 +dinr2 d inr2 +dinr3 d inr3 +dinr4 d inr4 +dinr5 d inr5 +ding1 d ing1 +ding2 d ing2 +ding3 d ing3 +ding4 d ing4 +ding5 d ing5 +dingr1 d ingr1 +dingr2 d ingr2 +dingr3 d ingr3 +dingr4 d ingr4 +dingr5 d ingr5 +du1 d u1 +du2 d u2 +du3 d u3 +du4 d u4 +du5 d u5 +dur1 d ur1 +dur2 d ur2 +dur3 d ur3 +dur4 d ur4 +dur5 d ur5 +duan1 d uan1 +duan2 d uan2 +duan3 d uan3 +duan4 d uan4 +duan5 d uan5 +duanr1 d uanr1 +duanr2 d uanr2 +duanr3 d uanr3 +duanr4 d uanr4 +duanr5 d uanr5 +dui1 d uei1 +dui2 d uei2 +dui3 d uei3 +dui4 d uei4 +dui5 d uei5 +duir1 d ueir1 +duir2 d ueir2 +duir3 d ueir3 +duir4 d ueir4 +duir5 d ueir5 +duo1 d uo1 +duo2 d uo2 +duo3 d uo3 +duo4 d uo4 +duo5 d uo5 +duor1 d uor1 +duor2 d uor2 +duor3 d uor3 +duor4 d uor4 +duor5 d uor5 +dun1 d uen1 +dun2 d uen2 +dun3 d uen3 +dun4 d uen4 +dun5 d uen5 +dunr1 d uenr1 +dunr2 d uenr2 +dunr3 d uenr3 +dunr4 d uenr4 +dunr5 d uenr5 +ta1 t a1 +ta2 t a2 +ta3 t a3 +ta4 t a4 +ta5 t a5 +tar1 t ar1 +tar2 t ar2 +tar3 t ar3 +tar4 t ar4 +tar5 t ar5 +tai1 t ai1 +tai2 t ai2 +tai3 t ai3 +tai4 t ai4 +tai5 t ai5 +tair1 t air1 +tair2 t air2 +tair3 t air3 +tair4 t air4 +tair5 t air5 +tao1 t ao1 +tao2 t ao2 +tao3 t ao3 +tao4 t ao4 +tao5 t ao5 +taor1 t aor1 +taor2 t aor2 +taor3 t aor3 +taor4 t aor4 +taor5 t aor5 +tan1 t an1 +tan2 t an2 +tan3 t an3 +tan4 t an4 +tan5 t an5 +tanr1 t anr1 +tanr2 t anr2 +tanr3 t anr3 +tanr4 t anr4 +tanr5 t anr5 +tang1 t ang1 +tang2 t ang2 +tang3 t ang3 +tang4 t ang4 +tang5 t ang5 +tangr1 t angr1 +tangr2 t angr2 +tangr3 t angr3 +tangr4 t angr4 +tangr5 t angr5 +te1 t e1 +te2 t e2 +te3 t e3 +te4 t e4 +te5 t e5 +ter1 t er1 +ter2 t er2 +ter3 t er3 +ter4 t er4 +ter5 t er5 +tei1 t ei1 +tei2 t ei2 +tei3 t ei3 +tei4 t ei4 +tei5 t ei5 +teir1 t eir1 +teir2 t eir2 +teir3 t eir3 +teir4 t eir4 +teir5 t eir5 +ten1 t en1 +ten2 t en2 +ten3 t en3 +ten4 t en4 +ten5 t en5 +tenr1 t enr1 +tenr2 t enr2 +tenr3 t enr3 +tenr4 t enr4 +tenr5 t enr5 +teng1 t eng1 +teng2 t eng2 +teng3 t eng3 +teng4 t eng4 +teng5 t eng5 +tengr1 t engr1 +tengr2 t engr2 +tengr3 t engr3 +tengr4 t engr4 +tengr5 t engr5 +tou1 t ou1 +tou2 t ou2 +tou3 t ou3 +tou4 t ou4 +tou5 t ou5 +tour1 t our1 +tour2 t our2 +tour3 t our3 +tour4 t our4 +tour5 t our5 +tong1 t ong1 +tong2 t ong2 +tong3 t ong3 +tong4 t ong4 +tong5 t ong5 +tongr1 t ongr1 +tongr2 t ongr2 +tongr3 t ongr3 +tongr4 t ongr4 +tongr5 t ongr5 +ti1 t i1 +ti2 t i2 +ti3 t i3 +ti4 t i4 +ti5 t i5 +tir1 t ir1 +tir2 t ir2 +tir3 t ir3 +tir4 t ir4 +tir5 t ir5 +tia1 t ia1 +tia2 t ia2 +tia3 t ia3 +tia4 t ia4 +tia5 t ia5 +tiar1 t iar1 +tiar2 t iar2 +tiar3 t iar3 +tiar4 t iar4 +tiar5 t iar5 +tiao1 t iao1 +tiao2 t iao2 +tiao3 t iao3 +tiao4 t iao4 +tiao5 t iao5 +tiaor1 t iaor1 +tiaor2 t iaor2 +tiaor3 t iaor3 +tiaor4 t iaor4 +tiaor5 t iaor5 +tian1 t ian1 +tian2 t ian2 +tian3 t ian3 +tian4 t ian4 +tian5 t ian5 +tianr1 t ianr1 +tianr2 t ianr2 +tianr3 t ianr3 +tianr4 t ianr4 +tianr5 t ianr5 +tiang1 t iang1 +tiang2 t iang2 +tiang3 t iang3 +tiang4 t iang4 +tiang5 t iang5 +tiangr1 t iangr1 +tiangr2 t iangr2 +tiangr3 t iangr3 +tiangr4 t iangr4 +tiangr5 t iangr5 +tie1 t ie1 +tie2 t ie2 +tie3 t ie3 +tie4 t ie4 +tie5 t ie5 +tier1 t ier1 +tier2 t ier2 +tier3 t ier3 +tier4 t ier4 +tier5 t ier5 +tio1 t io1 +tio2 t io2 +tio3 t io3 +tio4 t io4 +tio5 t io5 +tior1 t ior1 +tior2 t ior2 +tior3 t ior3 +tior4 t ior4 +tior5 t ior5 +tiu1 t iou1 +tiu2 t iou2 +tiu3 t iou3 +tiu4 t iou4 +tiu5 t iou5 +tiur1 t iour1 +tiur2 t iour2 +tiur3 t iour3 +tiur4 t iour4 +tiur5 t iour5 +tiong1 t iong1 +tiong2 t iong2 +tiong3 t iong3 +tiong4 t iong4 +tiong5 t iong5 +tiongr1 t iongr1 +tiongr2 t iongr2 +tiongr3 t iongr3 +tiongr4 t iongr4 +tiongr5 t iongr5 +tin1 t in1 +tin2 t in2 +tin3 t in3 +tin4 t in4 +tin5 t in5 +tinr1 t inr1 +tinr2 t inr2 +tinr3 t inr3 +tinr4 t inr4 +tinr5 t inr5 +ting1 t ing1 +ting2 t ing2 +ting3 t ing3 +ting4 t ing4 +ting5 t ing5 +tingr1 t ingr1 +tingr2 t ingr2 +tingr3 t ingr3 +tingr4 t ingr4 +tingr5 t ingr5 +tu1 t u1 +tu2 t u2 +tu3 t u3 +tu4 t u4 +tu5 t u5 +tur1 t ur1 +tur2 t ur2 +tur3 t ur3 +tur4 t ur4 +tur5 t ur5 +tuan1 t uan1 +tuan2 t uan2 +tuan3 t uan3 +tuan4 t uan4 +tuan5 t uan5 +tuanr1 t uanr1 +tuanr2 t uanr2 +tuanr3 t uanr3 +tuanr4 t uanr4 +tuanr5 t uanr5 +tui1 t uei1 +tui2 t uei2 +tui3 t uei3 +tui4 t uei4 +tui5 t uei5 +tuir1 t ueir1 +tuir2 t ueir2 +tuir3 t ueir3 +tuir4 t ueir4 +tuir5 t ueir5 +tuo1 t uo1 +tuo2 t uo2 +tuo3 t uo3 +tuo4 t uo4 +tuo5 t uo5 +tuor1 t uor1 +tuor2 t uor2 +tuor3 t uor3 +tuor4 t uor4 +tuor5 t uor5 +tun1 t uen1 +tun2 t uen2 +tun3 t uen3 +tun4 t uen4 +tun5 t uen5 +tunr1 t uenr1 +tunr2 t uenr2 +tunr3 t uenr3 +tunr4 t uenr4 +tunr5 t uenr5 +na1 n a1 +na2 n a2 +na3 n a3 +na4 n a4 +na5 n a5 +nar1 n ar1 +nar2 n ar2 +nar3 n ar3 +nar4 n ar4 +nar5 n ar5 +nai1 n ai1 +nai2 n ai2 +nai3 n ai3 +nai4 n ai4 +nai5 n ai5 +nair1 n air1 +nair2 n air2 +nair3 n air3 +nair4 n air4 +nair5 n air5 +nao1 n ao1 +nao2 n ao2 +nao3 n ao3 +nao4 n ao4 +nao5 n ao5 +naor1 n aor1 +naor2 n aor2 +naor3 n aor3 +naor4 n aor4 +naor5 n aor5 +nan1 n an1 +nan2 n an2 +nan3 n an3 +nan4 n an4 +nan5 n an5 +nanr1 n anr1 +nanr2 n anr2 +nanr3 n anr3 +nanr4 n anr4 +nanr5 n anr5 +nang1 n ang1 +nang2 n ang2 +nang3 n ang3 +nang4 n ang4 +nang5 n ang5 +nangr1 n angr1 +nangr2 n angr2 +nangr3 n angr3 +nangr4 n angr4 +nangr5 n angr5 +ne1 n e1 +ne2 n e2 +ne3 n e3 +ne4 n e4 +ne5 n e5 +ner1 n er1 +ner2 n er2 +ner3 n er3 +ner4 n er4 +ner5 n er5 +nei1 n ei1 +nei2 n ei2 +nei3 n ei3 +nei4 n ei4 +nei5 n ei5 +neir1 n eir1 +neir2 n eir2 +neir3 n eir3 +neir4 n eir4 +neir5 n eir5 +nen1 n en1 +nen2 n en2 +nen3 n en3 +nen4 n en4 +nen5 n en5 +nenr1 n enr1 +nenr2 n enr2 +nenr3 n enr3 +nenr4 n enr4 +nenr5 n enr5 +neng1 n eng1 +neng2 n eng2 +neng3 n eng3 +neng4 n eng4 +neng5 n eng5 +nengr1 n engr1 +nengr2 n engr2 +nengr3 n engr3 +nengr4 n engr4 +nengr5 n engr5 +nou1 n ou1 +nou2 n ou2 +nou3 n ou3 +nou4 n ou4 +nou5 n ou5 +nour1 n our1 +nour2 n our2 +nour3 n our3 +nour4 n our4 +nour5 n our5 +nong1 n ong1 +nong2 n ong2 +nong3 n ong3 +nong4 n ong4 +nong5 n ong5 +nongr1 n ongr1 +nongr2 n ongr2 +nongr3 n ongr3 +nongr4 n ongr4 +nongr5 n ongr5 +ni1 n i1 +ni2 n i2 +ni3 n i3 +ni4 n i4 +ni5 n i5 +nir1 n ir1 +nir2 n ir2 +nir3 n ir3 +nir4 n ir4 +nir5 n ir5 +nia1 n ia1 +nia2 n ia2 +nia3 n ia3 +nia4 n ia4 +nia5 n ia5 +niar1 n iar1 +niar2 n iar2 +niar3 n iar3 +niar4 n iar4 +niar5 n iar5 +niao1 n iao1 +niao2 n iao2 +niao3 n iao3 +niao4 n iao4 +niao5 n iao5 +niaor1 n iaor1 +niaor2 n iaor2 +niaor3 n iaor3 +niaor4 n iaor4 +niaor5 n iaor5 +nian1 n ian1 +nian2 n ian2 +nian3 n ian3 +nian4 n ian4 +nian5 n ian5 +nianr1 n ianr1 +nianr2 n ianr2 +nianr3 n ianr3 +nianr4 n ianr4 +nianr5 n ianr5 +niang1 n iang1 +niang2 n iang2 +niang3 n iang3 +niang4 n iang4 +niang5 n iang5 +niangr1 n iangr1 +niangr2 n iangr2 +niangr3 n iangr3 +niangr4 n iangr4 +niangr5 n iangr5 +nie1 n ie1 +nie2 n ie2 +nie3 n ie3 +nie4 n ie4 +nie5 n ie5 +nier1 n ier1 +nier2 n ier2 +nier3 n ier3 +nier4 n ier4 +nier5 n ier5 +nio1 n io1 +nio2 n io2 +nio3 n io3 +nio4 n io4 +nio5 n io5 +nior1 n ior1 +nior2 n ior2 +nior3 n ior3 +nior4 n ior4 +nior5 n ior5 +niu1 n iou1 +niu2 n iou2 +niu3 n iou3 +niu4 n iou4 +niu5 n iou5 +niur1 n iour1 +niur2 n iour2 +niur3 n iour3 +niur4 n iour4 +niur5 n iour5 +niong1 n iong1 +niong2 n iong2 +niong3 n iong3 +niong4 n iong4 +niong5 n iong5 +niongr1 n iongr1 +niongr2 n iongr2 +niongr3 n iongr3 +niongr4 n iongr4 +niongr5 n iongr5 +nin1 n in1 +nin2 n in2 +nin3 n in3 +nin4 n in4 +nin5 n in5 +ninr1 n inr1 +ninr2 n inr2 +ninr3 n inr3 +ninr4 n inr4 +ninr5 n inr5 +ning1 n ing1 +ning2 n ing2 +ning3 n ing3 +ning4 n ing4 +ning5 n ing5 +ningr1 n ingr1 +ningr2 n ingr2 +ningr3 n ingr3 +ningr4 n ingr4 +ningr5 n ingr5 +nu1 n u1 +nu2 n u2 +nu3 n u3 +nu4 n u4 +nu5 n u5 +nur1 n ur1 +nur2 n ur2 +nur3 n ur3 +nur4 n ur4 +nur5 n ur5 +nuan1 n uan1 +nuan2 n uan2 +nuan3 n uan3 +nuan4 n uan4 +nuan5 n uan5 +nuanr1 n uanr1 +nuanr2 n uanr2 +nuanr3 n uanr3 +nuanr4 n uanr4 +nuanr5 n uanr5 +nui1 n uei1 +nui2 n uei2 +nui3 n uei3 +nui4 n uei4 +nui5 n uei5 +nuir1 n ueir1 +nuir2 n ueir2 +nuir3 n ueir3 +nuir4 n ueir4 +nuir5 n ueir5 +nuo1 n uo1 +nuo2 n uo2 +nuo3 n uo3 +nuo4 n uo4 +nuo5 n uo5 +nuor1 n uor1 +nuor2 n uor2 +nuor3 n uor3 +nuor4 n uor4 +nuor5 n uor5 +nun1 n uen1 +nun2 n uen2 +nun3 n uen3 +nun4 n uen4 +nun5 n uen5 +nunr1 n uenr1 +nunr2 n uenr2 +nunr3 n uenr3 +nunr4 n uenr4 +nunr5 n uenr5 +nv1 n v1 +nv2 n v2 +nv3 n v3 +nv4 n v4 +nv5 n v5 +nvr1 n vr1 +nvr2 n vr2 +nvr3 n vr3 +nvr4 n vr4 +nvr5 n vr5 +nve1 n ve1 +nve2 n ve2 +nve3 n ve3 +nve4 n ve4 +nve5 n ve5 +nver1 n ver1 +nver2 n ver2 +nver3 n ver3 +nver4 n ver4 +nver5 n ver5 +la1 l a1 +la2 l a2 +la3 l a3 +la4 l a4 +la5 l a5 +lar1 l ar1 +lar2 l ar2 +lar3 l ar3 +lar4 l ar4 +lar5 l ar5 +lai1 l ai1 +lai2 l ai2 +lai3 l ai3 +lai4 l ai4 +lai5 l ai5 +lair1 l air1 +lair2 l air2 +lair3 l air3 +lair4 l air4 +lair5 l air5 +lao1 l ao1 +lao2 l ao2 +lao3 l ao3 +lao4 l ao4 +lao5 l ao5 +laor1 l aor1 +laor2 l aor2 +laor3 l aor3 +laor4 l aor4 +laor5 l aor5 +lan1 l an1 +lan2 l an2 +lan3 l an3 +lan4 l an4 +lan5 l an5 +lanr1 l anr1 +lanr2 l anr2 +lanr3 l anr3 +lanr4 l anr4 +lanr5 l anr5 +lang1 l ang1 +lang2 l ang2 +lang3 l ang3 +lang4 l ang4 +lang5 l ang5 +langr1 l angr1 +langr2 l angr2 +langr3 l angr3 +langr4 l angr4 +langr5 l angr5 +le1 l e1 +le2 l e2 +le3 l e3 +le4 l e4 +le5 l e5 +ler1 l er1 +ler2 l er2 +ler3 l er3 +ler4 l er4 +ler5 l er5 +lei1 l ei1 +lei2 l ei2 +lei3 l ei3 +lei4 l ei4 +lei5 l ei5 +leir1 l eir1 +leir2 l eir2 +leir3 l eir3 +leir4 l eir4 +leir5 l eir5 +len1 l en1 +len2 l en2 +len3 l en3 +len4 l en4 +len5 l en5 +lenr1 l enr1 +lenr2 l enr2 +lenr3 l enr3 +lenr4 l enr4 +lenr5 l enr5 +leng1 l eng1 +leng2 l eng2 +leng3 l eng3 +leng4 l eng4 +leng5 l eng5 +lengr1 l engr1 +lengr2 l engr2 +lengr3 l engr3 +lengr4 l engr4 +lengr5 l engr5 +lo1 l o1 +lo2 l o2 +lo3 l o3 +lo4 l o4 +lo5 l o5 +lor1 l or1 +lor2 l or2 +lor3 l or3 +lor4 l or4 +lor5 l or5 +lou1 l ou1 +lou2 l ou2 +lou3 l ou3 +lou4 l ou4 +lou5 l ou5 +lour1 l our1 +lour2 l our2 +lour3 l our3 +lour4 l our4 +lour5 l our5 +long1 l ong1 +long2 l ong2 +long3 l ong3 +long4 l ong4 +long5 l ong5 +longr1 l ongr1 +longr2 l ongr2 +longr3 l ongr3 +longr4 l ongr4 +longr5 l ongr5 +li1 l i1 +li2 l i2 +li3 l i3 +li4 l i4 +li5 l i5 +lir1 l ir1 +lir2 l ir2 +lir3 l ir3 +lir4 l ir4 +lir5 l ir5 +lia1 l ia1 +lia2 l ia2 +lia3 l ia3 +lia4 l ia4 +lia5 l ia5 +liar1 l iar1 +liar2 l iar2 +liar3 l iar3 +liar4 l iar4 +liar5 l iar5 +liao1 l iao1 +liao2 l iao2 +liao3 l iao3 +liao4 l iao4 +liao5 l iao5 +liaor1 l iaor1 +liaor2 l iaor2 +liaor3 l iaor3 +liaor4 l iaor4 +liaor5 l iaor5 +lian1 l ian1 +lian2 l ian2 +lian3 l ian3 +lian4 l ian4 +lian5 l ian5 +lianr1 l ianr1 +lianr2 l ianr2 +lianr3 l ianr3 +lianr4 l ianr4 +lianr5 l ianr5 +liang1 l iang1 +liang2 l iang2 +liang3 l iang3 +liang4 l iang4 +liang5 l iang5 +liangr1 l iangr1 +liangr2 l iangr2 +liangr3 l iangr3 +liangr4 l iangr4 +liangr5 l iangr5 +lie1 l ie1 +lie2 l ie2 +lie3 l ie3 +lie4 l ie4 +lie5 l ie5 +lier1 l ier1 +lier2 l ier2 +lier3 l ier3 +lier4 l ier4 +lier5 l ier5 +lio1 l io1 +lio2 l io2 +lio3 l io3 +lio4 l io4 +lio5 l io5 +lior1 l ior1 +lior2 l ior2 +lior3 l ior3 +lior4 l ior4 +lior5 l ior5 +liu1 l iou1 +liu2 l iou2 +liu3 l iou3 +liu4 l iou4 +liu5 l iou5 +liur1 l iour1 +liur2 l iour2 +liur3 l iour3 +liur4 l iour4 +liur5 l iour5 +liong1 l iong1 +liong2 l iong2 +liong3 l iong3 +liong4 l iong4 +liong5 l iong5 +liongr1 l iongr1 +liongr2 l iongr2 +liongr3 l iongr3 +liongr4 l iongr4 +liongr5 l iongr5 +lin1 l in1 +lin2 l in2 +lin3 l in3 +lin4 l in4 +lin5 l in5 +linr1 l inr1 +linr2 l inr2 +linr3 l inr3 +linr4 l inr4 +linr5 l inr5 +ling1 l ing1 +ling2 l ing2 +ling3 l ing3 +ling4 l ing4 +ling5 l ing5 +lingr1 l ingr1 +lingr2 l ingr2 +lingr3 l ingr3 +lingr4 l ingr4 +lingr5 l ingr5 +lu1 l u1 +lu2 l u2 +lu3 l u3 +lu4 l u4 +lu5 l u5 +lur1 l ur1 +lur2 l ur2 +lur3 l ur3 +lur4 l ur4 +lur5 l ur5 +luan1 l uan1 +luan2 l uan2 +luan3 l uan3 +luan4 l uan4 +luan5 l uan5 +luanr1 l uanr1 +luanr2 l uanr2 +luanr3 l uanr3 +luanr4 l uanr4 +luanr5 l uanr5 +lui1 l uei1 +lui2 l uei2 +lui3 l uei3 +lui4 l uei4 +lui5 l uei5 +luir1 l ueir1 +luir2 l ueir2 +luir3 l ueir3 +luir4 l ueir4 +luir5 l ueir5 +luo1 l uo1 +luo2 l uo2 +luo3 l uo3 +luo4 l uo4 +luo5 l uo5 +luor1 l uor1 +luor2 l uor2 +luor3 l uor3 +luor4 l uor4 +luor5 l uor5 +lun1 l uen1 +lun2 l uen2 +lun3 l uen3 +lun4 l uen4 +lun5 l uen5 +lunr1 l uenr1 +lunr2 l uenr2 +lunr3 l uenr3 +lunr4 l uenr4 +lunr5 l uenr5 +lv1 l v1 +lv2 l v2 +lv3 l v3 +lv4 l v4 +lv5 l v5 +lvr1 l vr1 +lvr2 l vr2 +lvr3 l vr3 +lvr4 l vr4 +lvr5 l vr5 +lve1 l ve1 +lve2 l ve2 +lve3 l ve3 +lve4 l ve4 +lve5 l ve5 +lver1 l ver1 +lver2 l ver2 +lver3 l ver3 +lver4 l ver4 +lver5 l ver5 +ga1 g a1 +ga2 g a2 +ga3 g a3 +ga4 g a4 +ga5 g a5 +gar1 g ar1 +gar2 g ar2 +gar3 g ar3 +gar4 g ar4 +gar5 g ar5 +gai1 g ai1 +gai2 g ai2 +gai3 g ai3 +gai4 g ai4 +gai5 g ai5 +gair1 g air1 +gair2 g air2 +gair3 g air3 +gair4 g air4 +gair5 g air5 +gao1 g ao1 +gao2 g ao2 +gao3 g ao3 +gao4 g ao4 +gao5 g ao5 +gaor1 g aor1 +gaor2 g aor2 +gaor3 g aor3 +gaor4 g aor4 +gaor5 g aor5 +gan1 g an1 +gan2 g an2 +gan3 g an3 +gan4 g an4 +gan5 g an5 +ganr1 g anr1 +ganr2 g anr2 +ganr3 g anr3 +ganr4 g anr4 +ganr5 g anr5 +gang1 g ang1 +gang2 g ang2 +gang3 g ang3 +gang4 g ang4 +gang5 g ang5 +gangr1 g angr1 +gangr2 g angr2 +gangr3 g angr3 +gangr4 g angr4 +gangr5 g angr5 +ge1 g e1 +ge2 g e2 +ge3 g e3 +ge4 g e4 +ge5 g e5 +ger1 g er1 +ger2 g er2 +ger3 g er3 +ger4 g er4 +ger5 g er5 +gei1 g ei1 +gei2 g ei2 +gei3 g ei3 +gei4 g ei4 +gei5 g ei5 +geir1 g eir1 +geir2 g eir2 +geir3 g eir3 +geir4 g eir4 +geir5 g eir5 +gen1 g en1 +gen2 g en2 +gen3 g en3 +gen4 g en4 +gen5 g en5 +genr1 g enr1 +genr2 g enr2 +genr3 g enr3 +genr4 g enr4 +genr5 g enr5 +geng1 g eng1 +geng2 g eng2 +geng3 g eng3 +geng4 g eng4 +geng5 g eng5 +gengr1 g engr1 +gengr2 g engr2 +gengr3 g engr3 +gengr4 g engr4 +gengr5 g engr5 +gou1 g ou1 +gou2 g ou2 +gou3 g ou3 +gou4 g ou4 +gou5 g ou5 +gour1 g our1 +gour2 g our2 +gour3 g our3 +gour4 g our4 +gour5 g our5 +gong1 g ong1 +gong2 g ong2 +gong3 g ong3 +gong4 g ong4 +gong5 g ong5 +gongr1 g ongr1 +gongr2 g ongr2 +gongr3 g ongr3 +gongr4 g ongr4 +gongr5 g ongr5 +gu1 g u1 +gu2 g u2 +gu3 g u3 +gu4 g u4 +gu5 g u5 +gur1 g ur1 +gur2 g ur2 +gur3 g ur3 +gur4 g ur4 +gur5 g ur5 +gua1 g ua1 +gua2 g ua2 +gua3 g ua3 +gua4 g ua4 +gua5 g ua5 +guar1 g uar1 +guar2 g uar2 +guar3 g uar3 +guar4 g uar4 +guar5 g uar5 +guai1 g uai1 +guai2 g uai2 +guai3 g uai3 +guai4 g uai4 +guai5 g uai5 +guair1 g uair1 +guair2 g uair2 +guair3 g uair3 +guair4 g uair4 +guair5 g uair5 +guan1 g uan1 +guan2 g uan2 +guan3 g uan3 +guan4 g uan4 +guan5 g uan5 +guanr1 g uanr1 +guanr2 g uanr2 +guanr3 g uanr3 +guanr4 g uanr4 +guanr5 g uanr5 +guang1 g uang1 +guang2 g uang2 +guang3 g uang3 +guang4 g uang4 +guang5 g uang5 +guangr1 g uangr1 +guangr2 g uangr2 +guangr3 g uangr3 +guangr4 g uangr4 +guangr5 g uangr5 +gui1 g uei1 +gui2 g uei2 +gui3 g uei3 +gui4 g uei4 +gui5 g uei5 +guir1 g ueir1 +guir2 g ueir2 +guir3 g ueir3 +guir4 g ueir4 +guir5 g ueir5 +guo1 g uo1 +guo2 g uo2 +guo3 g uo3 +guo4 g uo4 +guo5 g uo5 +guor1 g uor1 +guor2 g uor2 +guor3 g uor3 +guor4 g uor4 +guor5 g uor5 +gun1 g uen1 +gun2 g uen2 +gun3 g uen3 +gun4 g uen4 +gun5 g uen5 +gunr1 g uenr1 +gunr2 g uenr2 +gunr3 g uenr3 +gunr4 g uenr4 +gunr5 g uenr5 +ka1 k a1 +ka2 k a2 +ka3 k a3 +ka4 k a4 +ka5 k a5 +kar1 k ar1 +kar2 k ar2 +kar3 k ar3 +kar4 k ar4 +kar5 k ar5 +kai1 k ai1 +kai2 k ai2 +kai3 k ai3 +kai4 k ai4 +kai5 k ai5 +kair1 k air1 +kair2 k air2 +kair3 k air3 +kair4 k air4 +kair5 k air5 +kao1 k ao1 +kao2 k ao2 +kao3 k ao3 +kao4 k ao4 +kao5 k ao5 +kaor1 k aor1 +kaor2 k aor2 +kaor3 k aor3 +kaor4 k aor4 +kaor5 k aor5 +kan1 k an1 +kan2 k an2 +kan3 k an3 +kan4 k an4 +kan5 k an5 +kanr1 k anr1 +kanr2 k anr2 +kanr3 k anr3 +kanr4 k anr4 +kanr5 k anr5 +kang1 k ang1 +kang2 k ang2 +kang3 k ang3 +kang4 k ang4 +kang5 k ang5 +kangr1 k angr1 +kangr2 k angr2 +kangr3 k angr3 +kangr4 k angr4 +kangr5 k angr5 +ke1 k e1 +ke2 k e2 +ke3 k e3 +ke4 k e4 +ke5 k e5 +ker1 k er1 +ker2 k er2 +ker3 k er3 +ker4 k er4 +ker5 k er5 +kei1 k ei1 +kei2 k ei2 +kei3 k ei3 +kei4 k ei4 +kei5 k ei5 +keir1 k eir1 +keir2 k eir2 +keir3 k eir3 +keir4 k eir4 +keir5 k eir5 +ken1 k en1 +ken2 k en2 +ken3 k en3 +ken4 k en4 +ken5 k en5 +kenr1 k enr1 +kenr2 k enr2 +kenr3 k enr3 +kenr4 k enr4 +kenr5 k enr5 +keng1 k eng1 +keng2 k eng2 +keng3 k eng3 +keng4 k eng4 +keng5 k eng5 +kengr1 k engr1 +kengr2 k engr2 +kengr3 k engr3 +kengr4 k engr4 +kengr5 k engr5 +kou1 k ou1 +kou2 k ou2 +kou3 k ou3 +kou4 k ou4 +kou5 k ou5 +kour1 k our1 +kour2 k our2 +kour3 k our3 +kour4 k our4 +kour5 k our5 +kong1 k ong1 +kong2 k ong2 +kong3 k ong3 +kong4 k ong4 +kong5 k ong5 +kongr1 k ongr1 +kongr2 k ongr2 +kongr3 k ongr3 +kongr4 k ongr4 +kongr5 k ongr5 +ku1 k u1 +ku2 k u2 +ku3 k u3 +ku4 k u4 +ku5 k u5 +kur1 k ur1 +kur2 k ur2 +kur3 k ur3 +kur4 k ur4 +kur5 k ur5 +kua1 k ua1 +kua2 k ua2 +kua3 k ua3 +kua4 k ua4 +kua5 k ua5 +kuar1 k uar1 +kuar2 k uar2 +kuar3 k uar3 +kuar4 k uar4 +kuar5 k uar5 +kuai1 k uai1 +kuai2 k uai2 +kuai3 k uai3 +kuai4 k uai4 +kuai5 k uai5 +kuair1 k uair1 +kuair2 k uair2 +kuair3 k uair3 +kuair4 k uair4 +kuair5 k uair5 +kuan1 k uan1 +kuan2 k uan2 +kuan3 k uan3 +kuan4 k uan4 +kuan5 k uan5 +kuanr1 k uanr1 +kuanr2 k uanr2 +kuanr3 k uanr3 +kuanr4 k uanr4 +kuanr5 k uanr5 +kuang1 k uang1 +kuang2 k uang2 +kuang3 k uang3 +kuang4 k uang4 +kuang5 k uang5 +kuangr1 k uangr1 +kuangr2 k uangr2 +kuangr3 k uangr3 +kuangr4 k uangr4 +kuangr5 k uangr5 +kui1 k uei1 +kui2 k uei2 +kui3 k uei3 +kui4 k uei4 +kui5 k uei5 +kuir1 k ueir1 +kuir2 k ueir2 +kuir3 k ueir3 +kuir4 k ueir4 +kuir5 k ueir5 +kuo1 k uo1 +kuo2 k uo2 +kuo3 k uo3 +kuo4 k uo4 +kuo5 k uo5 +kuor1 k uor1 +kuor2 k uor2 +kuor3 k uor3 +kuor4 k uor4 +kuor5 k uor5 +kun1 k uen1 +kun2 k uen2 +kun3 k uen3 +kun4 k uen4 +kun5 k uen5 +kunr1 k uenr1 +kunr2 k uenr2 +kunr3 k uenr3 +kunr4 k uenr4 +kunr5 k uenr5 +ha1 h a1 +ha2 h a2 +ha3 h a3 +ha4 h a4 +ha5 h a5 +har1 h ar1 +har2 h ar2 +har3 h ar3 +har4 h ar4 +har5 h ar5 +hai1 h ai1 +hai2 h ai2 +hai3 h ai3 +hai4 h ai4 +hai5 h ai5 +hair1 h air1 +hair2 h air2 +hair3 h air3 +hair4 h air4 +hair5 h air5 +hao1 h ao1 +hao2 h ao2 +hao3 h ao3 +hao4 h ao4 +hao5 h ao5 +haor1 h aor1 +haor2 h aor2 +haor3 h aor3 +haor4 h aor4 +haor5 h aor5 +han1 h an1 +han2 h an2 +han3 h an3 +han4 h an4 +han5 h an5 +hanr1 h anr1 +hanr2 h anr2 +hanr3 h anr3 +hanr4 h anr4 +hanr5 h anr5 +hang1 h ang1 +hang2 h ang2 +hang3 h ang3 +hang4 h ang4 +hang5 h ang5 +hangr1 h angr1 +hangr2 h angr2 +hangr3 h angr3 +hangr4 h angr4 +hangr5 h angr5 +he1 h e1 +he2 h e2 +he3 h e3 +he4 h e4 +he5 h e5 +her1 h er1 +her2 h er2 +her3 h er3 +her4 h er4 +her5 h er5 +hei1 h ei1 +hei2 h ei2 +hei3 h ei3 +hei4 h ei4 +hei5 h ei5 +heir1 h eir1 +heir2 h eir2 +heir3 h eir3 +heir4 h eir4 +heir5 h eir5 +hen1 h en1 +hen2 h en2 +hen3 h en3 +hen4 h en4 +hen5 h en5 +henr1 h enr1 +henr2 h enr2 +henr3 h enr3 +henr4 h enr4 +henr5 h enr5 +heng1 h eng1 +heng2 h eng2 +heng3 h eng3 +heng4 h eng4 +heng5 h eng5 +hengr1 h engr1 +hengr2 h engr2 +hengr3 h engr3 +hengr4 h engr4 +hengr5 h engr5 +hou1 h ou1 +hou2 h ou2 +hou3 h ou3 +hou4 h ou4 +hou5 h ou5 +hour1 h our1 +hour2 h our2 +hour3 h our3 +hour4 h our4 +hour5 h our5 +hong1 h ong1 +hong2 h ong2 +hong3 h ong3 +hong4 h ong4 +hong5 h ong5 +hongr1 h ongr1 +hongr2 h ongr2 +hongr3 h ongr3 +hongr4 h ongr4 +hongr5 h ongr5 +hu1 h u1 +hu2 h u2 +hu3 h u3 +hu4 h u4 +hu5 h u5 +hur1 h ur1 +hur2 h ur2 +hur3 h ur3 +hur4 h ur4 +hur5 h ur5 +hua1 h ua1 +hua2 h ua2 +hua3 h ua3 +hua4 h ua4 +hua5 h ua5 +huar1 h uar1 +huar2 h uar2 +huar3 h uar3 +huar4 h uar4 +huar5 h uar5 +huai1 h uai1 +huai2 h uai2 +huai3 h uai3 +huai4 h uai4 +huai5 h uai5 +huair1 h uair1 +huair2 h uair2 +huair3 h uair3 +huair4 h uair4 +huair5 h uair5 +huan1 h uan1 +huan2 h uan2 +huan3 h uan3 +huan4 h uan4 +huan5 h uan5 +huanr1 h uanr1 +huanr2 h uanr2 +huanr3 h uanr3 +huanr4 h uanr4 +huanr5 h uanr5 +huang1 h uang1 +huang2 h uang2 +huang3 h uang3 +huang4 h uang4 +huang5 h uang5 +huangr1 h uangr1 +huangr2 h uangr2 +huangr3 h uangr3 +huangr4 h uangr4 +huangr5 h uangr5 +hui1 h uei1 +hui2 h uei2 +hui3 h uei3 +hui4 h uei4 +hui5 h uei5 +huir1 h ueir1 +huir2 h ueir2 +huir3 h ueir3 +huir4 h ueir4 +huir5 h ueir5 +huo1 h uo1 +huo2 h uo2 +huo3 h uo3 +huo4 h uo4 +huo5 h uo5 +huor1 h uor1 +huor2 h uor2 +huor3 h uor3 +huor4 h uor4 +huor5 h uor5 +hun1 h uen1 +hun2 h uen2 +hun3 h uen3 +hun4 h uen4 +hun5 h uen5 +hunr1 h uenr1 +hunr2 h uenr2 +hunr3 h uenr3 +hunr4 h uenr4 +hunr5 h uenr5 +zha1 zh a1 +zha2 zh a2 +zha3 zh a3 +zha4 zh a4 +zha5 zh a5 +zhar1 zh ar1 +zhar2 zh ar2 +zhar3 zh ar3 +zhar4 zh ar4 +zhar5 zh ar5 +zhai1 zh ai1 +zhai2 zh ai2 +zhai3 zh ai3 +zhai4 zh ai4 +zhai5 zh ai5 +zhair1 zh air1 +zhair2 zh air2 +zhair3 zh air3 +zhair4 zh air4 +zhair5 zh air5 +zhao1 zh ao1 +zhao2 zh ao2 +zhao3 zh ao3 +zhao4 zh ao4 +zhao5 zh ao5 +zhaor1 zh aor1 +zhaor2 zh aor2 +zhaor3 zh aor3 +zhaor4 zh aor4 +zhaor5 zh aor5 +zhan1 zh an1 +zhan2 zh an2 +zhan3 zh an3 +zhan4 zh an4 +zhan5 zh an5 +zhanr1 zh anr1 +zhanr2 zh anr2 +zhanr3 zh anr3 +zhanr4 zh anr4 +zhanr5 zh anr5 +zhang1 zh ang1 +zhang2 zh ang2 +zhang3 zh ang3 +zhang4 zh ang4 +zhang5 zh ang5 +zhangr1 zh angr1 +zhangr2 zh angr2 +zhangr3 zh angr3 +zhangr4 zh angr4 +zhangr5 zh angr5 +zhe1 zh e1 +zhe2 zh e2 +zhe3 zh e3 +zhe4 zh e4 +zhe5 zh e5 +zher1 zh er1 +zher2 zh er2 +zher3 zh er3 +zher4 zh er4 +zher5 zh er5 +zhei1 zh ei1 +zhei2 zh ei2 +zhei3 zh ei3 +zhei4 zh ei4 +zhei5 zh ei5 +zheir1 zh eir1 +zheir2 zh eir2 +zheir3 zh eir3 +zheir4 zh eir4 +zheir5 zh eir5 +zhen1 zh en1 +zhen2 zh en2 +zhen3 zh en3 +zhen4 zh en4 +zhen5 zh en5 +zhenr1 zh enr1 +zhenr2 zh enr2 +zhenr3 zh enr3 +zhenr4 zh enr4 +zhenr5 zh enr5 +zheng1 zh eng1 +zheng2 zh eng2 +zheng3 zh eng3 +zheng4 zh eng4 +zheng5 zh eng5 +zhengr1 zh engr1 +zhengr2 zh engr2 +zhengr3 zh engr3 +zhengr4 zh engr4 +zhengr5 zh engr5 +zhou1 zh ou1 +zhou2 zh ou2 +zhou3 zh ou3 +zhou4 zh ou4 +zhou5 zh ou5 +zhour1 zh our1 +zhour2 zh our2 +zhour3 zh our3 +zhour4 zh our4 +zhour5 zh our5 +zhong1 zh ong1 +zhong2 zh ong2 +zhong3 zh ong3 +zhong4 zh ong4 +zhong5 zh ong5 +zhongr1 zh ongr1 +zhongr2 zh ongr2 +zhongr3 zh ongr3 +zhongr4 zh ongr4 +zhongr5 zh ongr5 +zhi1 zh iii1 +zhi2 zh iii2 +zhi3 zh iii3 +zhi4 zh iii4 +zhi5 zh iii5 +zhir1 zh iiir1 +zhir2 zh iiir2 +zhir3 zh iiir3 +zhir4 zh iiir4 +zhir5 zh iiir5 +zhu1 zh u1 +zhu2 zh u2 +zhu3 zh u3 +zhu4 zh u4 +zhu5 zh u5 +zhur1 zh ur1 +zhur2 zh ur2 +zhur3 zh ur3 +zhur4 zh ur4 +zhur5 zh ur5 +zhua1 zh ua1 +zhua2 zh ua2 +zhua3 zh ua3 +zhua4 zh ua4 +zhua5 zh ua5 +zhuar1 zh uar1 +zhuar2 zh uar2 +zhuar3 zh uar3 +zhuar4 zh uar4 +zhuar5 zh uar5 +zhuai1 zh uai1 +zhuai2 zh uai2 +zhuai3 zh uai3 +zhuai4 zh uai4 +zhuai5 zh uai5 +zhuair1 zh uair1 +zhuair2 zh uair2 +zhuair3 zh uair3 +zhuair4 zh uair4 +zhuair5 zh uair5 +zhuan1 zh uan1 +zhuan2 zh uan2 +zhuan3 zh uan3 +zhuan4 zh uan4 +zhuan5 zh uan5 +zhuanr1 zh uanr1 +zhuanr2 zh uanr2 +zhuanr3 zh uanr3 +zhuanr4 zh uanr4 +zhuanr5 zh uanr5 +zhuang1 zh uang1 +zhuang2 zh uang2 +zhuang3 zh uang3 +zhuang4 zh uang4 +zhuang5 zh uang5 +zhuangr1 zh uangr1 +zhuangr2 zh uangr2 +zhuangr3 zh uangr3 +zhuangr4 zh uangr4 +zhuangr5 zh uangr5 +zhui1 zh uei1 +zhui2 zh uei2 +zhui3 zh uei3 +zhui4 zh uei4 +zhui5 zh uei5 +zhuir1 zh ueir1 +zhuir2 zh ueir2 +zhuir3 zh ueir3 +zhuir4 zh ueir4 +zhuir5 zh ueir5 +zhuo1 zh uo1 +zhuo2 zh uo2 +zhuo3 zh uo3 +zhuo4 zh uo4 +zhuo5 zh uo5 +zhuor1 zh uor1 +zhuor2 zh uor2 +zhuor3 zh uor3 +zhuor4 zh uor4 +zhuor5 zh uor5 +zhun1 zh uen1 +zhun2 zh uen2 +zhun3 zh uen3 +zhun4 zh uen4 +zhun5 zh uen5 +zhunr1 zh uenr1 +zhunr2 zh uenr2 +zhunr3 zh uenr3 +zhunr4 zh uenr4 +zhunr5 zh uenr5 +cha1 ch a1 +cha2 ch a2 +cha3 ch a3 +cha4 ch a4 +cha5 ch a5 +char1 ch ar1 +char2 ch ar2 +char3 ch ar3 +char4 ch ar4 +char5 ch ar5 +chai1 ch ai1 +chai2 ch ai2 +chai3 ch ai3 +chai4 ch ai4 +chai5 ch ai5 +chair1 ch air1 +chair2 ch air2 +chair3 ch air3 +chair4 ch air4 +chair5 ch air5 +chao1 ch ao1 +chao2 ch ao2 +chao3 ch ao3 +chao4 ch ao4 +chao5 ch ao5 +chaor1 ch aor1 +chaor2 ch aor2 +chaor3 ch aor3 +chaor4 ch aor4 +chaor5 ch aor5 +chan1 ch an1 +chan2 ch an2 +chan3 ch an3 +chan4 ch an4 +chan5 ch an5 +chanr1 ch anr1 +chanr2 ch anr2 +chanr3 ch anr3 +chanr4 ch anr4 +chanr5 ch anr5 +chang1 ch ang1 +chang2 ch ang2 +chang3 ch ang3 +chang4 ch ang4 +chang5 ch ang5 +changr1 ch angr1 +changr2 ch angr2 +changr3 ch angr3 +changr4 ch angr4 +changr5 ch angr5 +che1 ch e1 +che2 ch e2 +che3 ch e3 +che4 ch e4 +che5 ch e5 +cher1 ch er1 +cher2 ch er2 +cher3 ch er3 +cher4 ch er4 +cher5 ch er5 +chei1 ch ei1 +chei2 ch ei2 +chei3 ch ei3 +chei4 ch ei4 +chei5 ch ei5 +cheir1 ch eir1 +cheir2 ch eir2 +cheir3 ch eir3 +cheir4 ch eir4 +cheir5 ch eir5 +chen1 ch en1 +chen2 ch en2 +chen3 ch en3 +chen4 ch en4 +chen5 ch en5 +chenr1 ch enr1 +chenr2 ch enr2 +chenr3 ch enr3 +chenr4 ch enr4 +chenr5 ch enr5 +cheng1 ch eng1 +cheng2 ch eng2 +cheng3 ch eng3 +cheng4 ch eng4 +cheng5 ch eng5 +chengr1 ch engr1 +chengr2 ch engr2 +chengr3 ch engr3 +chengr4 ch engr4 +chengr5 ch engr5 +chou1 ch ou1 +chou2 ch ou2 +chou3 ch ou3 +chou4 ch ou4 +chou5 ch ou5 +chour1 ch our1 +chour2 ch our2 +chour3 ch our3 +chour4 ch our4 +chour5 ch our5 +chong1 ch ong1 +chong2 ch ong2 +chong3 ch ong3 +chong4 ch ong4 +chong5 ch ong5 +chongr1 ch ongr1 +chongr2 ch ongr2 +chongr3 ch ongr3 +chongr4 ch ongr4 +chongr5 ch ongr5 +chi1 ch iii1 +chi2 ch iii2 +chi3 ch iii3 +chi4 ch iii4 +chi5 ch iii5 +chir1 ch iiir1 +chir2 ch iiir2 +chir3 ch iiir3 +chir4 ch iiir4 +chir5 ch iiir5 +chu1 ch u1 +chu2 ch u2 +chu3 ch u3 +chu4 ch u4 +chu5 ch u5 +chur1 ch ur1 +chur2 ch ur2 +chur3 ch ur3 +chur4 ch ur4 +chur5 ch ur5 +chua1 ch ua1 +chua2 ch ua2 +chua3 ch ua3 +chua4 ch ua4 +chua5 ch ua5 +chuar1 ch uar1 +chuar2 ch uar2 +chuar3 ch uar3 +chuar4 ch uar4 +chuar5 ch uar5 +chuai1 ch uai1 +chuai2 ch uai2 +chuai3 ch uai3 +chuai4 ch uai4 +chuai5 ch uai5 +chuair1 ch uair1 +chuair2 ch uair2 +chuair3 ch uair3 +chuair4 ch uair4 +chuair5 ch uair5 +chuan1 ch uan1 +chuan2 ch uan2 +chuan3 ch uan3 +chuan4 ch uan4 +chuan5 ch uan5 +chuanr1 ch uanr1 +chuanr2 ch uanr2 +chuanr3 ch uanr3 +chuanr4 ch uanr4 +chuanr5 ch uanr5 +chuang1 ch uang1 +chuang2 ch uang2 +chuang3 ch uang3 +chuang4 ch uang4 +chuang5 ch uang5 +chuangr1 ch uangr1 +chuangr2 ch uangr2 +chuangr3 ch uangr3 +chuangr4 ch uangr4 +chuangr5 ch uangr5 +chui1 ch uei1 +chui2 ch uei2 +chui3 ch uei3 +chui4 ch uei4 +chui5 ch uei5 +chuir1 ch ueir1 +chuir2 ch ueir2 +chuir3 ch ueir3 +chuir4 ch ueir4 +chuir5 ch ueir5 +chuo1 ch uo1 +chuo2 ch uo2 +chuo3 ch uo3 +chuo4 ch uo4 +chuo5 ch uo5 +chuor1 ch uor1 +chuor2 ch uor2 +chuor3 ch uor3 +chuor4 ch uor4 +chuor5 ch uor5 +chun1 ch uen1 +chun2 ch uen2 +chun3 ch uen3 +chun4 ch uen4 +chun5 ch uen5 +chunr1 ch uenr1 +chunr2 ch uenr2 +chunr3 ch uenr3 +chunr4 ch uenr4 +chunr5 ch uenr5 +sha1 sh a1 +sha2 sh a2 +sha3 sh a3 +sha4 sh a4 +sha5 sh a5 +shar1 sh ar1 +shar2 sh ar2 +shar3 sh ar3 +shar4 sh ar4 +shar5 sh ar5 +shai1 sh ai1 +shai2 sh ai2 +shai3 sh ai3 +shai4 sh ai4 +shai5 sh ai5 +shair1 sh air1 +shair2 sh air2 +shair3 sh air3 +shair4 sh air4 +shair5 sh air5 +shao1 sh ao1 +shao2 sh ao2 +shao3 sh ao3 +shao4 sh ao4 +shao5 sh ao5 +shaor1 sh aor1 +shaor2 sh aor2 +shaor3 sh aor3 +shaor4 sh aor4 +shaor5 sh aor5 +shan1 sh an1 +shan2 sh an2 +shan3 sh an3 +shan4 sh an4 +shan5 sh an5 +shanr1 sh anr1 +shanr2 sh anr2 +shanr3 sh anr3 +shanr4 sh anr4 +shanr5 sh anr5 +shang1 sh ang1 +shang2 sh ang2 +shang3 sh ang3 +shang4 sh ang4 +shang5 sh ang5 +shangr1 sh angr1 +shangr2 sh angr2 +shangr3 sh angr3 +shangr4 sh angr4 +shangr5 sh angr5 +she1 sh e1 +she2 sh e2 +she3 sh e3 +she4 sh e4 +she5 sh e5 +sher1 sh er1 +sher2 sh er2 +sher3 sh er3 +sher4 sh er4 +sher5 sh er5 +shei1 sh ei1 +shei2 sh ei2 +shei3 sh ei3 +shei4 sh ei4 +shei5 sh ei5 +sheir1 sh eir1 +sheir2 sh eir2 +sheir3 sh eir3 +sheir4 sh eir4 +sheir5 sh eir5 +shen1 sh en1 +shen2 sh en2 +shen3 sh en3 +shen4 sh en4 +shen5 sh en5 +shenr1 sh enr1 +shenr2 sh enr2 +shenr3 sh enr3 +shenr4 sh enr4 +shenr5 sh enr5 +sheng1 sh eng1 +sheng2 sh eng2 +sheng3 sh eng3 +sheng4 sh eng4 +sheng5 sh eng5 +shengr1 sh engr1 +shengr2 sh engr2 +shengr3 sh engr3 +shengr4 sh engr4 +shengr5 sh engr5 +shou1 sh ou1 +shou2 sh ou2 +shou3 sh ou3 +shou4 sh ou4 +shou5 sh ou5 +shour1 sh our1 +shour2 sh our2 +shour3 sh our3 +shour4 sh our4 +shour5 sh our5 +shi1 sh iii1 +shi2 sh iii2 +shi3 sh iii3 +shi4 sh iii4 +shi5 sh iii5 +shir1 sh iiir1 +shir2 sh iiir2 +shir3 sh iiir3 +shir4 sh iiir4 +shir5 sh iiir5 +shu1 sh u1 +shu2 sh u2 +shu3 sh u3 +shu4 sh u4 +shu5 sh u5 +shur1 sh ur1 +shur2 sh ur2 +shur3 sh ur3 +shur4 sh ur4 +shur5 sh ur5 +shua1 sh ua1 +shua2 sh ua2 +shua3 sh ua3 +shua4 sh ua4 +shua5 sh ua5 +shuar1 sh uar1 +shuar2 sh uar2 +shuar3 sh uar3 +shuar4 sh uar4 +shuar5 sh uar5 +shuai1 sh uai1 +shuai2 sh uai2 +shuai3 sh uai3 +shuai4 sh uai4 +shuai5 sh uai5 +shuair1 sh uair1 +shuair2 sh uair2 +shuair3 sh uair3 +shuair4 sh uair4 +shuair5 sh uair5 +shuan1 sh uan1 +shuan2 sh uan2 +shuan3 sh uan3 +shuan4 sh uan4 +shuan5 sh uan5 +shuanr1 sh uanr1 +shuanr2 sh uanr2 +shuanr3 sh uanr3 +shuanr4 sh uanr4 +shuanr5 sh uanr5 +shuang1 sh uang1 +shuang2 sh uang2 +shuang3 sh uang3 +shuang4 sh uang4 +shuang5 sh uang5 +shuangr1 sh uangr1 +shuangr2 sh uangr2 +shuangr3 sh uangr3 +shuangr4 sh uangr4 +shuangr5 sh uangr5 +shui1 sh uei1 +shui2 sh uei2 +shui3 sh uei3 +shui4 sh uei4 +shui5 sh uei5 +shuir1 sh ueir1 +shuir2 sh ueir2 +shuir3 sh ueir3 +shuir4 sh ueir4 +shuir5 sh ueir5 +shuo1 sh uo1 +shuo2 sh uo2 +shuo3 sh uo3 +shuo4 sh uo4 +shuo5 sh uo5 +shuor1 sh uor1 +shuor2 sh uor2 +shuor3 sh uor3 +shuor4 sh uor4 +shuor5 sh uor5 +shun1 sh uen1 +shun2 sh uen2 +shun3 sh uen3 +shun4 sh uen4 +shun5 sh uen5 +shunr1 sh uenr1 +shunr2 sh uenr2 +shunr3 sh uenr3 +shunr4 sh uenr4 +shunr5 sh uenr5 +ra1 r a1 +ra2 r a2 +ra3 r a3 +ra4 r a4 +ra5 r a5 +rar1 r ar1 +rar2 r ar2 +rar3 r ar3 +rar4 r ar4 +rar5 r ar5 +rai1 r ai1 +rai2 r ai2 +rai3 r ai3 +rai4 r ai4 +rai5 r ai5 +rair1 r air1 +rair2 r air2 +rair3 r air3 +rair4 r air4 +rair5 r air5 +rao1 r ao1 +rao2 r ao2 +rao3 r ao3 +rao4 r ao4 +rao5 r ao5 +raor1 r aor1 +raor2 r aor2 +raor3 r aor3 +raor4 r aor4 +raor5 r aor5 +ran1 r an1 +ran2 r an2 +ran3 r an3 +ran4 r an4 +ran5 r an5 +ranr1 r anr1 +ranr2 r anr2 +ranr3 r anr3 +ranr4 r anr4 +ranr5 r anr5 +rang1 r ang1 +rang2 r ang2 +rang3 r ang3 +rang4 r ang4 +rang5 r ang5 +rangr1 r angr1 +rangr2 r angr2 +rangr3 r angr3 +rangr4 r angr4 +rangr5 r angr5 +re1 r e1 +re2 r e2 +re3 r e3 +re4 r e4 +re5 r e5 +rer1 r er1 +rer2 r er2 +rer3 r er3 +rer4 r er4 +rer5 r er5 +rei1 r ei1 +rei2 r ei2 +rei3 r ei3 +rei4 r ei4 +rei5 r ei5 +reir1 r eir1 +reir2 r eir2 +reir3 r eir3 +reir4 r eir4 +reir5 r eir5 +ren1 r en1 +ren2 r en2 +ren3 r en3 +ren4 r en4 +ren5 r en5 +renr1 r enr1 +renr2 r enr2 +renr3 r enr3 +renr4 r enr4 +renr5 r enr5 +reng1 r eng1 +reng2 r eng2 +reng3 r eng3 +reng4 r eng4 +reng5 r eng5 +rengr1 r engr1 +rengr2 r engr2 +rengr3 r engr3 +rengr4 r engr4 +rengr5 r engr5 +rou1 r ou1 +rou2 r ou2 +rou3 r ou3 +rou4 r ou4 +rou5 r ou5 +rour1 r our1 +rour2 r our2 +rour3 r our3 +rour4 r our4 +rour5 r our5 +rong1 r ong1 +rong2 r ong2 +rong3 r ong3 +rong4 r ong4 +rong5 r ong5 +rongr1 r ongr1 +rongr2 r ongr2 +rongr3 r ongr3 +rongr4 r ongr4 +rongr5 r ongr5 +ri1 r iii1 +ri2 r iii2 +ri3 r iii3 +ri4 r iii4 +ri5 r iii5 +rir1 r iiir1 +rir2 r iiir2 +rir3 r iiir3 +rir4 r iiir4 +rir5 r iiir5 +ru1 r u1 +ru2 r u2 +ru3 r u3 +ru4 r u4 +ru5 r u5 +rur1 r ur1 +rur2 r ur2 +rur3 r ur3 +rur4 r ur4 +rur5 r ur5 +ruan1 r uan1 +ruan2 r uan2 +ruan3 r uan3 +ruan4 r uan4 +ruan5 r uan5 +ruanr1 r uanr1 +ruanr2 r uanr2 +ruanr3 r uanr3 +ruanr4 r uanr4 +ruanr5 r uanr5 +rui1 r uei1 +rui2 r uei2 +rui3 r uei3 +rui4 r uei4 +rui5 r uei5 +ruir1 r ueir1 +ruir2 r ueir2 +ruir3 r ueir3 +ruir4 r ueir4 +ruir5 r ueir5 +ruo1 r uo1 +ruo2 r uo2 +ruo3 r uo3 +ruo4 r uo4 +ruo5 r uo5 +ruor1 r uor1 +ruor2 r uor2 +ruor3 r uor3 +ruor4 r uor4 +ruor5 r uor5 +run1 r uen1 +run2 r uen2 +run3 r uen3 +run4 r uen4 +run5 r uen5 +runr1 r uenr1 +runr2 r uenr2 +runr3 r uenr3 +runr4 r uenr4 +runr5 r uenr5 +za1 z a1 +za2 z a2 +za3 z a3 +za4 z a4 +za5 z a5 +zar1 z ar1 +zar2 z ar2 +zar3 z ar3 +zar4 z ar4 +zar5 z ar5 +zai1 z ai1 +zai2 z ai2 +zai3 z ai3 +zai4 z ai4 +zai5 z ai5 +zair1 z air1 +zair2 z air2 +zair3 z air3 +zair4 z air4 +zair5 z air5 +zao1 z ao1 +zao2 z ao2 +zao3 z ao3 +zao4 z ao4 +zao5 z ao5 +zaor1 z aor1 +zaor2 z aor2 +zaor3 z aor3 +zaor4 z aor4 +zaor5 z aor5 +zan1 z an1 +zan2 z an2 +zan3 z an3 +zan4 z an4 +zan5 z an5 +zanr1 z anr1 +zanr2 z anr2 +zanr3 z anr3 +zanr4 z anr4 +zanr5 z anr5 +zang1 z ang1 +zang2 z ang2 +zang3 z ang3 +zang4 z ang4 +zang5 z ang5 +zangr1 z angr1 +zangr2 z angr2 +zangr3 z angr3 +zangr4 z angr4 +zangr5 z angr5 +ze1 z e1 +ze2 z e2 +ze3 z e3 +ze4 z e4 +ze5 z e5 +zer1 z er1 +zer2 z er2 +zer3 z er3 +zer4 z er4 +zer5 z er5 +zei1 z ei1 +zei2 z ei2 +zei3 z ei3 +zei4 z ei4 +zei5 z ei5 +zeir1 z eir1 +zeir2 z eir2 +zeir3 z eir3 +zeir4 z eir4 +zeir5 z eir5 +zen1 z en1 +zen2 z en2 +zen3 z en3 +zen4 z en4 +zen5 z en5 +zenr1 z enr1 +zenr2 z enr2 +zenr3 z enr3 +zenr4 z enr4 +zenr5 z enr5 +zeng1 z eng1 +zeng2 z eng2 +zeng3 z eng3 +zeng4 z eng4 +zeng5 z eng5 +zengr1 z engr1 +zengr2 z engr2 +zengr3 z engr3 +zengr4 z engr4 +zengr5 z engr5 +zou1 z ou1 +zou2 z ou2 +zou3 z ou3 +zou4 z ou4 +zou5 z ou5 +zour1 z our1 +zour2 z our2 +zour3 z our3 +zour4 z our4 +zour5 z our5 +zong1 z ong1 +zong2 z ong2 +zong3 z ong3 +zong4 z ong4 +zong5 z ong5 +zongr1 z ongr1 +zongr2 z ongr2 +zongr3 z ongr3 +zongr4 z ongr4 +zongr5 z ongr5 +zi1 z ii1 +zi2 z ii2 +zi3 z ii3 +zi4 z ii4 +zi5 z ii5 +zir1 z iir1 +zir2 z iir2 +zir3 z iir3 +zir4 z iir4 +zir5 z iir5 +zu1 z u1 +zu2 z u2 +zu3 z u3 +zu4 z u4 +zu5 z u5 +zur1 z ur1 +zur2 z ur2 +zur3 z ur3 +zur4 z ur4 +zur5 z ur5 +zuan1 z uan1 +zuan2 z uan2 +zuan3 z uan3 +zuan4 z uan4 +zuan5 z uan5 +zuanr1 z uanr1 +zuanr2 z uanr2 +zuanr3 z uanr3 +zuanr4 z uanr4 +zuanr5 z uanr5 +zui1 z uei1 +zui2 z uei2 +zui3 z uei3 +zui4 z uei4 +zui5 z uei5 +zuir1 z ueir1 +zuir2 z ueir2 +zuir3 z ueir3 +zuir4 z ueir4 +zuir5 z ueir5 +zuo1 z uo1 +zuo2 z uo2 +zuo3 z uo3 +zuo4 z uo4 +zuo5 z uo5 +zuor1 z uor1 +zuor2 z uor2 +zuor3 z uor3 +zuor4 z uor4 +zuor5 z uor5 +zun1 z uen1 +zun2 z uen2 +zun3 z uen3 +zun4 z uen4 +zun5 z uen5 +zunr1 z uenr1 +zunr2 z uenr2 +zunr3 z uenr3 +zunr4 z uenr4 +zunr5 z uenr5 +ca1 c a1 +ca2 c a2 +ca3 c a3 +ca4 c a4 +ca5 c a5 +car1 c ar1 +car2 c ar2 +car3 c ar3 +car4 c ar4 +car5 c ar5 +cai1 c ai1 +cai2 c ai2 +cai3 c ai3 +cai4 c ai4 +cai5 c ai5 +cair1 c air1 +cair2 c air2 +cair3 c air3 +cair4 c air4 +cair5 c air5 +cao1 c ao1 +cao2 c ao2 +cao3 c ao3 +cao4 c ao4 +cao5 c ao5 +caor1 c aor1 +caor2 c aor2 +caor3 c aor3 +caor4 c aor4 +caor5 c aor5 +can1 c an1 +can2 c an2 +can3 c an3 +can4 c an4 +can5 c an5 +canr1 c anr1 +canr2 c anr2 +canr3 c anr3 +canr4 c anr4 +canr5 c anr5 +cang1 c ang1 +cang2 c ang2 +cang3 c ang3 +cang4 c ang4 +cang5 c ang5 +cangr1 c angr1 +cangr2 c angr2 +cangr3 c angr3 +cangr4 c angr4 +cangr5 c angr5 +ce1 c e1 +ce2 c e2 +ce3 c e3 +ce4 c e4 +ce5 c e5 +cer1 c er1 +cer2 c er2 +cer3 c er3 +cer4 c er4 +cer5 c er5 +cei1 c ei1 +cei2 c ei2 +cei3 c ei3 +cei4 c ei4 +cei5 c ei5 +ceir1 c eir1 +ceir2 c eir2 +ceir3 c eir3 +ceir4 c eir4 +ceir5 c eir5 +cen1 c en1 +cen2 c en2 +cen3 c en3 +cen4 c en4 +cen5 c en5 +cenr1 c enr1 +cenr2 c enr2 +cenr3 c enr3 +cenr4 c enr4 +cenr5 c enr5 +ceng1 c eng1 +ceng2 c eng2 +ceng3 c eng3 +ceng4 c eng4 +ceng5 c eng5 +cengr1 c engr1 +cengr2 c engr2 +cengr3 c engr3 +cengr4 c engr4 +cengr5 c engr5 +cou1 c ou1 +cou2 c ou2 +cou3 c ou3 +cou4 c ou4 +cou5 c ou5 +cour1 c our1 +cour2 c our2 +cour3 c our3 +cour4 c our4 +cour5 c our5 +cong1 c ong1 +cong2 c ong2 +cong3 c ong3 +cong4 c ong4 +cong5 c ong5 +congr1 c ongr1 +congr2 c ongr2 +congr3 c ongr3 +congr4 c ongr4 +congr5 c ongr5 +ci1 c ii1 +ci2 c ii2 +ci3 c ii3 +ci4 c ii4 +ci5 c ii5 +cir1 c iir1 +cir2 c iir2 +cir3 c iir3 +cir4 c iir4 +cir5 c iir5 +cu1 c u1 +cu2 c u2 +cu3 c u3 +cu4 c u4 +cu5 c u5 +cur1 c ur1 +cur2 c ur2 +cur3 c ur3 +cur4 c ur4 +cur5 c ur5 +cuan1 c uan1 +cuan2 c uan2 +cuan3 c uan3 +cuan4 c uan4 +cuan5 c uan5 +cuanr1 c uanr1 +cuanr2 c uanr2 +cuanr3 c uanr3 +cuanr4 c uanr4 +cuanr5 c uanr5 +cui1 c uei1 +cui2 c uei2 +cui3 c uei3 +cui4 c uei4 +cui5 c uei5 +cuir1 c ueir1 +cuir2 c ueir2 +cuir3 c ueir3 +cuir4 c ueir4 +cuir5 c ueir5 +cuo1 c uo1 +cuo2 c uo2 +cuo3 c uo3 +cuo4 c uo4 +cuo5 c uo5 +cuor1 c uor1 +cuor2 c uor2 +cuor3 c uor3 +cuor4 c uor4 +cuor5 c uor5 +cun1 c uen1 +cun2 c uen2 +cun3 c uen3 +cun4 c uen4 +cun5 c uen5 +cunr1 c uenr1 +cunr2 c uenr2 +cunr3 c uenr3 +cunr4 c uenr4 +cunr5 c uenr5 +sa1 s a1 +sa2 s a2 +sa3 s a3 +sa4 s a4 +sa5 s a5 +sar1 s ar1 +sar2 s ar2 +sar3 s ar3 +sar4 s ar4 +sar5 s ar5 +sai1 s ai1 +sai2 s ai2 +sai3 s ai3 +sai4 s ai4 +sai5 s ai5 +sair1 s air1 +sair2 s air2 +sair3 s air3 +sair4 s air4 +sair5 s air5 +sao1 s ao1 +sao2 s ao2 +sao3 s ao3 +sao4 s ao4 +sao5 s ao5 +saor1 s aor1 +saor2 s aor2 +saor3 s aor3 +saor4 s aor4 +saor5 s aor5 +san1 s an1 +san2 s an2 +san3 s an3 +san4 s an4 +san5 s an5 +sanr1 s anr1 +sanr2 s anr2 +sanr3 s anr3 +sanr4 s anr4 +sanr5 s anr5 +sang1 s ang1 +sang2 s ang2 +sang3 s ang3 +sang4 s ang4 +sang5 s ang5 +sangr1 s angr1 +sangr2 s angr2 +sangr3 s angr3 +sangr4 s angr4 +sangr5 s angr5 +se1 s e1 +se2 s e2 +se3 s e3 +se4 s e4 +se5 s e5 +ser1 s er1 +ser2 s er2 +ser3 s er3 +ser4 s er4 +ser5 s er5 +sei1 s ei1 +sei2 s ei2 +sei3 s ei3 +sei4 s ei4 +sei5 s ei5 +seir1 s eir1 +seir2 s eir2 +seir3 s eir3 +seir4 s eir4 +seir5 s eir5 +sen1 s en1 +sen2 s en2 +sen3 s en3 +sen4 s en4 +sen5 s en5 +senr1 s enr1 +senr2 s enr2 +senr3 s enr3 +senr4 s enr4 +senr5 s enr5 +seng1 s eng1 +seng2 s eng2 +seng3 s eng3 +seng4 s eng4 +seng5 s eng5 +sengr1 s engr1 +sengr2 s engr2 +sengr3 s engr3 +sengr4 s engr4 +sengr5 s engr5 +sou1 s ou1 +sou2 s ou2 +sou3 s ou3 +sou4 s ou4 +sou5 s ou5 +sour1 s our1 +sour2 s our2 +sour3 s our3 +sour4 s our4 +sour5 s our5 +song1 s ong1 +song2 s ong2 +song3 s ong3 +song4 s ong4 +song5 s ong5 +songr1 s ongr1 +songr2 s ongr2 +songr3 s ongr3 +songr4 s ongr4 +songr5 s ongr5 +si1 s ii1 +si2 s ii2 +si3 s ii3 +si4 s ii4 +si5 s ii5 +sir1 s iir1 +sir2 s iir2 +sir3 s iir3 +sir4 s iir4 +sir5 s iir5 +su1 s u1 +su2 s u2 +su3 s u3 +su4 s u4 +su5 s u5 +sur1 s ur1 +sur2 s ur2 +sur3 s ur3 +sur4 s ur4 +sur5 s ur5 +suan1 s uan1 +suan2 s uan2 +suan3 s uan3 +suan4 s uan4 +suan5 s uan5 +suanr1 s uanr1 +suanr2 s uanr2 +suanr3 s uanr3 +suanr4 s uanr4 +suanr5 s uanr5 +sui1 s uei1 +sui2 s uei2 +sui3 s uei3 +sui4 s uei4 +sui5 s uei5 +suir1 s ueir1 +suir2 s ueir2 +suir3 s ueir3 +suir4 s ueir4 +suir5 s ueir5 +suo1 s uo1 +suo2 s uo2 +suo3 s uo3 +suo4 s uo4 +suo5 s uo5 +suor1 s uor1 +suor2 s uor2 +suor3 s uor3 +suor4 s uor4 +suor5 s uor5 +sun1 s uen1 +sun2 s uen2 +sun3 s uen3 +sun4 s uen4 +sun5 s uen5 +sunr1 s uenr1 +sunr2 s uenr2 +sunr3 s uenr3 +sunr4 s uenr4 +sunr5 s uenr5 +ji1 j i1 +ji2 j i2 +ji3 j i3 +ji4 j i4 +ji5 j i5 +jir1 j ir1 +jir2 j ir2 +jir3 j ir3 +jir4 j ir4 +jir5 j ir5 +jia1 j ia1 +jia2 j ia2 +jia3 j ia3 +jia4 j ia4 +jia5 j ia5 +jiar1 j iar1 +jiar2 j iar2 +jiar3 j iar3 +jiar4 j iar4 +jiar5 j iar5 +jiao1 j iao1 +jiao2 j iao2 +jiao3 j iao3 +jiao4 j iao4 +jiao5 j iao5 +jiaor1 j iaor1 +jiaor2 j iaor2 +jiaor3 j iaor3 +jiaor4 j iaor4 +jiaor5 j iaor5 +jian1 j ian1 +jian2 j ian2 +jian3 j ian3 +jian4 j ian4 +jian5 j ian5 +jianr1 j ianr1 +jianr2 j ianr2 +jianr3 j ianr3 +jianr4 j ianr4 +jianr5 j ianr5 +jiang1 j iang1 +jiang2 j iang2 +jiang3 j iang3 +jiang4 j iang4 +jiang5 j iang5 +jiangr1 j iangr1 +jiangr2 j iangr2 +jiangr3 j iangr3 +jiangr4 j iangr4 +jiangr5 j iangr5 +jie1 j ie1 +jie2 j ie2 +jie3 j ie3 +jie4 j ie4 +jie5 j ie5 +jier1 j ier1 +jier2 j ier2 +jier3 j ier3 +jier4 j ier4 +jier5 j ier5 +jio1 j io1 +jio2 j io2 +jio3 j io3 +jio4 j io4 +jio5 j io5 +jior1 j ior1 +jior2 j ior2 +jior3 j ior3 +jior4 j ior4 +jior5 j ior5 +jiu1 j iou1 +jiu2 j iou2 +jiu3 j iou3 +jiu4 j iou4 +jiu5 j iou5 +jiur1 j iour1 +jiur2 j iour2 +jiur3 j iour3 +jiur4 j iour4 +jiur5 j iour5 +jiong1 j iong1 +jiong2 j iong2 +jiong3 j iong3 +jiong4 j iong4 +jiong5 j iong5 +jiongr1 j iongr1 +jiongr2 j iongr2 +jiongr3 j iongr3 +jiongr4 j iongr4 +jiongr5 j iongr5 +jin1 j in1 +jin2 j in2 +jin3 j in3 +jin4 j in4 +jin5 j in5 +jinr1 j inr1 +jinr2 j inr2 +jinr3 j inr3 +jinr4 j inr4 +jinr5 j inr5 +jing1 j ing1 +jing2 j ing2 +jing3 j ing3 +jing4 j ing4 +jing5 j ing5 +jingr1 j ingr1 +jingr2 j ingr2 +jingr3 j ingr3 +jingr4 j ingr4 +jingr5 j ingr5 +ju1 j v1 +ju2 j v2 +ju3 j v3 +ju4 j v4 +ju5 j v5 +jur1 j vr1 +jur2 j vr2 +jur3 j vr3 +jur4 j vr4 +jur5 j vr5 +jue1 j ve1 +jue2 j ve2 +jue3 j ve3 +jue4 j ve4 +jue5 j ve5 +juer1 j ver1 +juer2 j ver2 +juer3 j ver3 +juer4 j ver4 +juer5 j ver5 +juan1 j van1 +juan2 j van2 +juan3 j van3 +juan4 j van4 +juan5 j van5 +juanr1 j vanr1 +juanr2 j vanr2 +juanr3 j vanr3 +juanr4 j vanr4 +juanr5 j vanr5 +jun1 j vn1 +jun2 j vn2 +jun3 j vn3 +jun4 j vn4 +jun5 j vn5 +junr1 j vnr1 +junr2 j vnr2 +junr3 j vnr3 +junr4 j vnr4 +junr5 j vnr5 +qi1 q i1 +qi2 q i2 +qi3 q i3 +qi4 q i4 +qi5 q i5 +qir1 q ir1 +qir2 q ir2 +qir3 q ir3 +qir4 q ir4 +qir5 q ir5 +qia1 q ia1 +qia2 q ia2 +qia3 q ia3 +qia4 q ia4 +qia5 q ia5 +qiar1 q iar1 +qiar2 q iar2 +qiar3 q iar3 +qiar4 q iar4 +qiar5 q iar5 +qiao1 q iao1 +qiao2 q iao2 +qiao3 q iao3 +qiao4 q iao4 +qiao5 q iao5 +qiaor1 q iaor1 +qiaor2 q iaor2 +qiaor3 q iaor3 +qiaor4 q iaor4 +qiaor5 q iaor5 +qian1 q ian1 +qian2 q ian2 +qian3 q ian3 +qian4 q ian4 +qian5 q ian5 +qianr1 q ianr1 +qianr2 q ianr2 +qianr3 q ianr3 +qianr4 q ianr4 +qianr5 q ianr5 +qiang1 q iang1 +qiang2 q iang2 +qiang3 q iang3 +qiang4 q iang4 +qiang5 q iang5 +qiangr1 q iangr1 +qiangr2 q iangr2 +qiangr3 q iangr3 +qiangr4 q iangr4 +qiangr5 q iangr5 +qie1 q ie1 +qie2 q ie2 +qie3 q ie3 +qie4 q ie4 +qie5 q ie5 +qier1 q ier1 +qier2 q ier2 +qier3 q ier3 +qier4 q ier4 +qier5 q ier5 +qio1 q io1 +qio2 q io2 +qio3 q io3 +qio4 q io4 +qio5 q io5 +qior1 q ior1 +qior2 q ior2 +qior3 q ior3 +qior4 q ior4 +qior5 q ior5 +qiu1 q iou1 +qiu2 q iou2 +qiu3 q iou3 +qiu4 q iou4 +qiu5 q iou5 +qiur1 q iour1 +qiur2 q iour2 +qiur3 q iour3 +qiur4 q iour4 +qiur5 q iour5 +qiong1 q iong1 +qiong2 q iong2 +qiong3 q iong3 +qiong4 q iong4 +qiong5 q iong5 +qiongr1 q iongr1 +qiongr2 q iongr2 +qiongr3 q iongr3 +qiongr4 q iongr4 +qiongr5 q iongr5 +qin1 q in1 +qin2 q in2 +qin3 q in3 +qin4 q in4 +qin5 q in5 +qinr1 q inr1 +qinr2 q inr2 +qinr3 q inr3 +qinr4 q inr4 +qinr5 q inr5 +qing1 q ing1 +qing2 q ing2 +qing3 q ing3 +qing4 q ing4 +qing5 q ing5 +qingr1 q ingr1 +qingr2 q ingr2 +qingr3 q ingr3 +qingr4 q ingr4 +qingr5 q ingr5 +qu1 q v1 +qu2 q v2 +qu3 q v3 +qu4 q v4 +qu5 q v5 +qur1 q vr1 +qur2 q vr2 +qur3 q vr3 +qur4 q vr4 +qur5 q vr5 +que1 q ve1 +que2 q ve2 +que3 q ve3 +que4 q ve4 +que5 q ve5 +quer1 q ver1 +quer2 q ver2 +quer3 q ver3 +quer4 q ver4 +quer5 q ver5 +quan1 q van1 +quan2 q van2 +quan3 q van3 +quan4 q van4 +quan5 q van5 +quanr1 q vanr1 +quanr2 q vanr2 +quanr3 q vanr3 +quanr4 q vanr4 +quanr5 q vanr5 +qun1 q vn1 +qun2 q vn2 +qun3 q vn3 +qun4 q vn4 +qun5 q vn5 +qunr1 q vnr1 +qunr2 q vnr2 +qunr3 q vnr3 +qunr4 q vnr4 +qunr5 q vnr5 +xi1 x i1 +xi2 x i2 +xi3 x i3 +xi4 x i4 +xi5 x i5 +xir1 x ir1 +xir2 x ir2 +xir3 x ir3 +xir4 x ir4 +xir5 x ir5 +xia1 x ia1 +xia2 x ia2 +xia3 x ia3 +xia4 x ia4 +xia5 x ia5 +xiar1 x iar1 +xiar2 x iar2 +xiar3 x iar3 +xiar4 x iar4 +xiar5 x iar5 +xiao1 x iao1 +xiao2 x iao2 +xiao3 x iao3 +xiao4 x iao4 +xiao5 x iao5 +xiaor1 x iaor1 +xiaor2 x iaor2 +xiaor3 x iaor3 +xiaor4 x iaor4 +xiaor5 x iaor5 +xian1 x ian1 +xian2 x ian2 +xian3 x ian3 +xian4 x ian4 +xian5 x ian5 +xianr1 x ianr1 +xianr2 x ianr2 +xianr3 x ianr3 +xianr4 x ianr4 +xianr5 x ianr5 +xiang1 x iang1 +xiang2 x iang2 +xiang3 x iang3 +xiang4 x iang4 +xiang5 x iang5 +xiangr1 x iangr1 +xiangr2 x iangr2 +xiangr3 x iangr3 +xiangr4 x iangr4 +xiangr5 x iangr5 +xie1 x ie1 +xie2 x ie2 +xie3 x ie3 +xie4 x ie4 +xie5 x ie5 +xier1 x ier1 +xier2 x ier2 +xier3 x ier3 +xier4 x ier4 +xier5 x ier5 +xio1 x io1 +xio2 x io2 +xio3 x io3 +xio4 x io4 +xio5 x io5 +xior1 x ior1 +xior2 x ior2 +xior3 x ior3 +xior4 x ior4 +xior5 x ior5 +xiu1 x iou1 +xiu2 x iou2 +xiu3 x iou3 +xiu4 x iou4 +xiu5 x iou5 +xiur1 x iour1 +xiur2 x iour2 +xiur3 x iour3 +xiur4 x iour4 +xiur5 x iour5 +xiong1 x iong1 +xiong2 x iong2 +xiong3 x iong3 +xiong4 x iong4 +xiong5 x iong5 +xiongr1 x iongr1 +xiongr2 x iongr2 +xiongr3 x iongr3 +xiongr4 x iongr4 +xiongr5 x iongr5 +xin1 x in1 +xin2 x in2 +xin3 x in3 +xin4 x in4 +xin5 x in5 +xinr1 x inr1 +xinr2 x inr2 +xinr3 x inr3 +xinr4 x inr4 +xinr5 x inr5 +xing1 x ing1 +xing2 x ing2 +xing3 x ing3 +xing4 x ing4 +xing5 x ing5 +xingr1 x ingr1 +xingr2 x ingr2 +xingr3 x ingr3 +xingr4 x ingr4 +xingr5 x ingr5 +xu1 x v1 +xu2 x v2 +xu3 x v3 +xu4 x v4 +xu5 x v5 +xur1 x vr1 +xur2 x vr2 +xur3 x vr3 +xur4 x vr4 +xur5 x vr5 +xue1 x ve1 +xue2 x ve2 +xue3 x ve3 +xue4 x ve4 +xue5 x ve5 +xuer1 x ver1 +xuer2 x ver2 +xuer3 x ver3 +xuer4 x ver4 +xuer5 x ver5 +xuan1 x van1 +xuan2 x van2 +xuan3 x van3 +xuan4 x van4 +xuan5 x van5 +xuanr1 x vanr1 +xuanr2 x vanr2 +xuanr3 x vanr3 +xuanr4 x vanr4 +xuanr5 x vanr5 +xun1 x vn1 +xun2 x vn2 +xun3 x vn3 +xun4 x vn4 +xun5 x vn5 +xunr1 x vnr1 +xunr2 x vnr2 +xunr3 x vnr3 +xunr4 x vnr4 +xunr5 x vnr5 diff --git a/examples/fastspeech2/baker/train.py b/examples/fastspeech2/baker/train.py index 993f001..2e52bb7 100644 --- a/examples/fastspeech2/baker/train.py +++ b/examples/fastspeech2/baker/train.py @@ -12,40 +12,30 @@ # See the License for the specific language governing permissions and # limitations under the License. -import os -import sys -import logging import argparse -import dataclasses +import os +import logging from pathlib import Path -import yaml import jsonlines -import paddle import numpy as np -from paddle import nn -from paddle.nn import functional as F -from paddle import distributed as dist -from paddle.io import DataLoader, DistributedBatchSampler -from paddle.optimizer import Adam # No RAdaom -from paddle.optimizer.lr import StepDecay +import paddle from paddle import DataParallel -from visualdl import LogWriter - +from paddle import distributed as dist +from paddle import nn +from paddle.io import DataLoader, DistributedBatchSampler from parakeet.datasets.data_table import DataTable -from parakeet.models.fastspeech2_new import FastSpeech2 - -from parakeet.training.updater import UpdaterBase -from parakeet.training.trainer import Trainer -from parakeet.training.reporter import report -from parakeet.training import extension +from parakeet.models.fastspeech2 import FastSpeech2 from parakeet.training.extensions.snapshot import Snapshot from parakeet.training.extensions.visualizer import VisualDL from parakeet.training.seeding import seed_everything +from parakeet.training.trainer import Trainer +from visualdl import LogWriter +import yaml from batch_fn import collate_baker_examples -from fastspeech2_updater import FastSpeech2Updater, FastSpeech2Evaluator from config import get_cfg_default +from fastspeech2_updater import FastSpeech2Updater, FastSpeech2Evaluator optim_classes = dict( adadelta=paddle.optimizer.Adadelta, @@ -61,7 +51,6 @@ optim_classes = dict( def build_optimizers(model: nn.Layer, optim='adadelta', learning_rate=0.01) -> paddle.optimizer: - optim_class = optim_classes.get(optim) if optim_class is None: raise ValueError(f"must be one of {list(optim_classes)}: {optim}") @@ -100,22 +89,12 @@ def train_sp(args, config): train_dataset = DataTable( data=train_metadata, fields=[ - "text", - "text_lengths", - "speech", - "speech_lengths", - "durations", - "pitch", - "energy", - # "durations_lengths", - # "pitch_lengths", - # "energy_lengths" + "text", "text_lengths", "speech", "speech_lengths", "durations", + "pitch", "energy" ], - converters={ - "speech": np.load, - "pitch": np.load, - "energy": np.load, - }, ) + converters={"speech": np.load, + "pitch": np.load, + "energy": np.load}, ) with jsonlines.open(args.dev_metadata, 'r') as reader: dev_metadata = list(reader) dev_dataset = DataTable( @@ -124,17 +103,15 @@ def train_sp(args, config): "text", "text_lengths", "speech", "speech_lengths", "durations", "pitch", "energy" ], - converters={ - "speech": np.load, - "pitch": np.load, - "energy": np.load, - }, ) + converters={"speech": np.load, + "pitch": np.load, + "energy": np.load}, ) # collate function and dataloader train_sampler = DistributedBatchSampler( train_dataset, batch_size=config.batch_size, - shuffle=False, + shuffle=True, drop_last=True) print("samplers done!") @@ -144,6 +121,7 @@ def train_sp(args, config): batch_sampler=train_sampler, collate_fn=collate_baker_examples, num_workers=config.num_workers) + dev_dataloader = DataLoader( dev_dataset, shuffle=False, @@ -153,16 +131,18 @@ def train_sp(args, config): num_workers=config.num_workers) print("dataloaders done!") - vocab_size = 202 + with open(args.phones, "r") as f: + phn_id = [line.strip().split() for line in f.readlines()] + vocab_size = len(phn_id) + print("vocab_size:", vocab_size) + odim = config.n_mels model = FastSpeech2(idim=vocab_size, odim=odim, **config["model"]) if world_size > 1: model = DataParallel(model) # TODO, do not use vocab size from config - # print(model) print("model done!") optimizer = build_optimizers(model, **config["optimizer"]) - print("optimizer done!") updater = FastSpeech2Updater( @@ -174,8 +154,8 @@ def train_sp(args, config): output_dir = Path(args.output_dir) trainer = Trainer(updater, (config.max_epoch, 'epoch'), output_dir) - evaluator = FastSpeech2Evaluator(model, dev_dataloader, ** - config["updater"]) + evaluator = FastSpeech2Evaluator(model, dev_dataloader, + **config["updater"]) if dist.get_rank() == 0: trainer.extend(evaluator, trigger=(1, "epoch")) @@ -201,6 +181,11 @@ def main(): parser.add_argument( "--nprocs", type=int, default=1, help="number of processes") parser.add_argument("--verbose", type=int, default=1, help="verbose") + parser.add_argument( + "--phones", + type=str, + default="phone_id_map.txt ", + help="phone vocabulary file.") args = parser.parse_args() if args.device == "cpu" and args.nprocs > 1: diff --git a/parakeet/models/__init__.py b/parakeet/models/__init__.py index 38bf4a5..6cf65ec 100644 --- a/parakeet/models/__init__.py +++ b/parakeet/models/__init__.py @@ -20,4 +20,4 @@ from parakeet.models.transformer_tts import * #from parakeet.models.deepvoice3 import * # from parakeet.models.fastspeech import * from parakeet.models.tacotron2 import * -from parakeet.models.fastspeech2_new import * +from parakeet.models.fastspeech2 import * diff --git a/parakeet/models/fastspeech2.py b/parakeet/models/fastspeech2.py index be68f79..3697b52 100644 --- a/parakeet/models/fastspeech2.py +++ b/parakeet/models/fastspeech2.py @@ -11,702 +11,633 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +"""Fastspeech2 related modules for paddle""" +from typing import Dict, Sequence, Tuple + +import numpy as np import paddle from paddle import nn -from paddle.nn import functional as F -from paddle.nn import initializer as I -from paddle.fluid.layers import sequence_mask +from parakeet.modules.fastspeech2_predictor.duration_predictor import DurationPredictor, DurationPredictorLoss +from parakeet.modules.fastspeech2_predictor.length_regulator import LengthRegulator +from parakeet.modules.fastspeech2_predictor.postnet import Postnet +from parakeet.modules.fastspeech2_predictor.variance_predictor import VariancePredictor +from parakeet.modules.fastspeech2_transformer.embedding import PositionalEncoding, ScaledPositionalEncoding +from parakeet.modules.fastspeech2_transformer.encoder import Encoder as TransformerEncoder +from parakeet.modules.nets_utils import initialize, make_non_pad_mask, make_pad_mask +from typeguard import check_argument_types -from parakeet.modules.positioning import position_encoding -from parakeet.modules.attention import (_split_heads, _concat_heads, - scaled_dot_product_attention) -from parakeet.modules import geometry as geo -from parakeet.modules.conv import Conv1dBatchNorm -from typing import Optional - - -class FastSpeechFeedForwardTransformer(nn.Layer): - def __init__(self, - num_layers, - model_dim, - num_heads, - ffn_dim, - ffn_kernel_size, - attention_dropout=0., - residual_dropout=0., - num_speakers=1, - max_position=1000, - input_dim: Optional[int]=None, - epsilon=1e-5, - scheme="post"): - super().__init__() - # optional input layer - input_dim = input_dim or model_dim - self.input_dim = input_dim - self.model_dim = model_dim - if input_dim != model_dim: - self.input_fc = nn.Linear(input_dim, model_dim) - - self.pos_embedding = position_encoding(1 + max_position, model_dim) - - self.num_speakers = num_speakers - if num_speakers > 1: - self.speaker_embedding = nn.Embedding(num_speakers, model_dim) - self.speaker_fc = nn.Linear(model_dim, model_dim) - - self.layers = nn.LayerList([ - FastSpeechFFTBlock(model_dim, num_heads, ffn_dim, ffn_kernel_size, - attention_dropout, residual_dropout, epsilon, - scheme) for _ in range(num_layers) - ]) - - def forward(self, x, mask, speaker_ids=None): - """ - x: [B, T, C] - mask: [B, 1, T] or [B, T, T] - returns: [B, T, C] - """ - if self.input_dim != self.model_dim: - x = self.input_fc(x) - - batch_size, time_steps, _ = x.shape - pos_embed = self.pos_embedding[1:1 + time_steps, :] - x += pos_embed - - if self.num_speakers > 1: - speaker_embedding = self.speaker_embedding(speaker_ids) - speaker_feature = F.softplus(self.speaker_fc(speaker_embedding)) - speaker_feature = paddle.unsqueeze(speaker_feature, 1) # [B, T, C] - x += speaker_feature - - for layer in self.layers: - x, attn = layer(x, mask) - # we do not return attention here - return x - - -class MultiheadAttention(nn.Layer): - def __init__(self, - model_dim: int, - num_heads: int, - k_input_dim: Optional[int]=None, - v_input_dim: Optional[int]=None, - dropout: float=0.): - super().__init__() - if model_dim % num_heads != 0: - raise ValueError("model_dim must be divisible by num_heads") - depth = model_dim // num_heads - k_input_dim = k_input_dim or model_dim - v_input_dim = v_input_dim or model_dim - self.wq = nn.Linear(model_dim, model_dim) - self.wk = nn.Linear(k_input_dim, model_dim) - self.wv = nn.Linear(v_input_dim, model_dim) - self.wo = nn.Linear(model_dim, model_dim) - - self.num_heads = num_heads - self.model_dim = model_dim - self.dropout = dropout - - def forward(self, q, k, v, mask=None): - q = _split_heads(self.wq(q), self.num_heads) # (B, h, T, C) - k = _split_heads(self.wk(k), self.num_heads) - v = _split_heads(self.wv(v), self.num_heads) - if mask is not None: - mask = paddle.unsqueeze(mask, 1) # unsqueeze for the h dim - - context_vectors, attention_weights = scaled_dot_product_attention( - q, k, v, mask, dropout=self.dropout, training=self.training) - context_vectors = _concat_heads(context_vectors) - context_vectors = self.wo(context_vectors) - return context_vectors, attention_weights - - -class FastSpeechSelfAttentionNorm(nn.Layer): - """Self attention & Layer normalization, both schemes are supported.""" - - def __init__(self, - model_dim, - num_heads, - attention_dropout=0., - residual_dropout=0., - epsilon=1e-5, - scheme="post"): - super().__init__() - if scheme not in ["post", "pre"]: - raise ValueError("scheme should be 'pre' or 'post'") - self.scheme = scheme - - self.attention = MultiheadAttention( - model_dim, num_heads, dropout=attention_dropout) - self.layer_norm = nn.LayerNorm([model_dim], epsilon=epsilon) - self.dropout_layer = nn.Dropout(residual_dropout) - - def forward(self, x, mask=None): - # [B, T, C], [B, 1, T] -> [B, T, C], [B, T, T] - if self.scheme is "post": - c, w = self.attention(x, x, x, mask=mask) - out = self.layer_norm(x + self.dropout_layer(c)) - else: - normalized_x = self.layer_norm(x) - c, w = self.attention( - normalized_x, normalized_x, normalized_x, mask=mask) - out = x + self.dropout_layer(c) - - c *= paddle.transpose(mask, [0, 2, 1]) # mask padding positions - return out, w - - -class FastSpeechFFN(nn.Layer): - """FFN, it can either be 2 linear or 2 conv1d.""" - - def __init__(self, model_dim, hidden_dim, kernel_size=1): - super().__init__() - if kernel_size == 1: - self.layer1 = nn.Linear(model_dim, hidden_dim) - self.layer2 = nn.Linear(hidden_dim, model_dim) - else: - self.layer1 = nn.Conv1D( - model_dim, - hidden_dim, - kernel_size, - padding="same", - data_format="NLC") - self.layer2 = nn.Conv1D( - hidden_dim, - model_dim, - kernel_size, - padding="same", - data_format="NLC") - - def forward(self, x, mask=None): - # [B, T, C], [B, T] -> [B, T, C] - h = self.layer1(x) - h = F.relu(h) # TODO: use mish here? - h = self.layer2(h) - h *= paddle.unsqueeze(mask, -1) # mask padding positions - return h - - -class FastSpeechFFNNorm(nn.Layer): - def __init__(self, - model_dim, - hidden_dim, - kernel_size, - residual_dropout=0., - epsilon=1e-5, - scheme="post"): - super().__init__() - if scheme not in ["post", "pre"]: - raise ValueError("scheme should be 'pre' or 'post'") - self.scheme = scheme - - self.ffn = FastSpeechFFN( - model_dim, hidden_dim, kernel_size=kernel_size) - self.layer_norm = nn.LayerNorm([model_dim], epsilon=epsilon) - self.dropout_layer = nn.Dropout(residual_dropout) - - def forward(self, x, mask=None): - if self.scheme == "post": - h = self.ffn(x, mask) - out = self.layer_norm(x + self.dropout_layer(h)) - else: - normalized_x = self.layer_norm(x) - h = self.ffn(normalized_x, mask) - out = x + self.dropout_layer(h) - out *= paddle.unsqueeze(mask, -1) # mask padding positions - return out - - -class FastSpeechFFTBlock(nn.Layer): - def __init__(self, - model_dim, - num_heads, - ffn_dim, - ffn_kernel_size, - attention_dropout=0., - residual_dropout=0., - epsilon=1e-5, - scheme="post"): - super().__init__() - self.attention = FastSpeechSelfAttentionNorm( - model_dim, num_heads, attention_dropout, residual_dropout, epsilon, - scheme) - self.ffn = FastSpeechFFNNorm(model_dim, ffn_dim, ffn_kernel_size, - residual_dropout, epsilon, scheme) - - def forward(self, x, mask): - # [B, T, C] - # [B, 1, T] - c, w = self.attention(x, mask) - c = self.ffn(c, paddle.squeeze(mask)) - return c, w - - -class FastSpeechDurationPredictor(nn.Layer): - def __init__(self, - num_layers: int, - input_dim: int, - hidden_dim: int, - kernel_size: int, - dropout: float=0., - epsilon: float=1e-5): - super().__init__() - convs = [] - for i in range(num_layers): - conv = nn.Conv1D( - input_dim if i == 0 else hidden_dim, - hidden_dim, - kernel_size, - padding="same", - data_format="NLC") - layer_norm = nn.LayerNorm([hidden_dim], epsilon=epsilon) - act = nn.ReLU6() - dropout_layer = nn.Dropout(dropout) - convs.extend([conv, layer_norm, act, dropout_layer]) - self.conv_layers = nn.Sequential(*convs) - self.output_fc = nn.Linear(hidden_dim, 1) - - def forward(self, x, mask): - # [B, T, C], [B, T] -> [B, T] - mask = paddle.unsqueeze(mask, -1) - x *= mask - - h = self.conv_layers(x) - h = self.output_fc(h) - h *= mask - h = F.relu6(h).squeeze(-1) - return h - - -class FastSpeechLengthRegulator(nn.Layer): - def __init__(self): - super().__init__() - - def forward(self, x, durations): - # [B, T, C], [B, T] -> [B, T', C], [B] - output_lens = paddle.sum(durations, axis=-1) - - batch_size = x.shape[0] - expanded_sequences = [] - for i in range(batch_size): - expanded_sequence = geo.repeat(x[i], durations[i], axis=0) - expanded_sequences.append(expanded_sequence) - padded_sequence = geo.pad_sequences(expanded_sequences) - return padded_sequence, output_lens - - -class TacotronPostNet(nn.Layer): - def __init__(self, - num_layers, - input_dim, - hidden_dim, - kernel_size, - dropout=0., - momentum=0.9, - epsilon=1e-5): - super().__init__() - self.conv_bns = nn.LayerList() - self.num_layers = num_layers - for i in range(num_layers): - convbn = Conv1dBatchNorm( - input_dim if i == 0 else hidden_dim, - hidden_dim if i != num_layers - 1 else input_dim, - kernel_size, - padding="same", - data_format="NLC", - momentum=momentum, - epsilon=epsilon) - self.conv_bns.append(convbn) - self.dropout_layer = nn.Dropout(dropout) - - def forward(self, x, mask): - # [B, T, C], [B, T] -> [B, T, C] - mask = paddle.unsqueeze(mask, -1) - for i, convbn in enumerate(self.conv_bns): - x = convbn(x) - if i != self.num_layers - 1: - x = paddle.tanh(x) - x = self.dropout_layer(x) - x *= mask - return x - - -class FastSpeechVariancePredictor(nn.Layer): - def __init__(self, - num_layers: int, - input_dim: int, - hidden_dim: int, - kernel_size: int, - num_speakers: int=1, - speaker_embedding_size: Optional[int]=None, - dropout: float=0., - epsilon: float=1e-5): - super().__init__() - convs = [] - for i in range(num_layers): - conv = nn.Conv1D( - input_dim if i == 0 else hidden_dim, - hidden_dim, - kernel_size, - padding="same", - data_format="NLC") - act = nn.ReLU() - ln = nn.LayerNorm([hidden_dim], epsilon=epsilon) - dropout_layer = nn.Dropout(dropout) - convs.extend([conv, act, ln, dropout_layer]) - self.conv_layers = nn.Sequential(*convs) - self.output_fc = nn.Linear(hidden_dim, 1) - - self.num_speakers = num_speakers - if num_speakers > 1: - self.speaker_embedding = nn.Embedding(num_speakers, - speaker_embedding_size) - self.speaker_fc = nn.Linear(speaker_embedding_size, input_dim) - - def forward(self, x, speaker_ids, mask): - # [B, T, C], [B], [B, T] -> [B, T] - if self.num_speakers > 1: - speaker_embed = self.speaker_embeddings(speaker_ids) - speaker_features = F.softplus(self.speaker_fc(speaker_embed)) - x += paddle.unsqueeze(speaker_features, 1) - - x *= paddle.unsqueeze(mask, -1) - - h = self.conv_layers(x) - out = self.output_fc(h) - out = paddle.squeeze(-1) * mask - return out - - -class FastSpeech(nn.Layer): - def __init__( - self, - vocab_size, - num_speakers, - # encoder params - encoder_num_layers, - encoder_dim, - encoder_num_heads, - encoder_max_position, - encoder_ffn_dim, - encoder_ffn_kernel_size, - # decoder params - decoder_num_layers, - decoder_dim, - decoder_num_heads, - decoder_max_position, - decoder_ffn_dim, - decoder_ffn_kernel_size, - # encoder & decoder common - attention_dropout, - residual_dropout, - # duration predictor - duration_predictor_num_layers, - duration_predictor_dim, - duration_predictor_kernel_size, - duration_predictor_dropout, - # output - mel_dim, - # postnet - postnet_num_layers, - postnet_dim, - postnet_kernel_size, - postnet_dropout, - # other - padding_idx=0, - momentum=0.9, - epsilon=1e-5, - scheme="post"): - super().__init__() - self.embedding = nn.Embedding( - vocab_size, encoder_dim, padding_idx=padding_idx) - self.encoder = FastSpeechFeedForwardTransformer( - encoder_num_layers, - encoder_dim, - encoder_num_heads, - encoder_ffn_dim, - encoder_ffn_kernel_size, - attention_dropout, - residual_dropout, - num_speakers=num_speakers, - max_position=encoder_max_position, - epsilon=epsilon, - scheme=scheme) - self.duration_predictor = FastSpeechDurationPredictor( - duration_predictor_num_layers, - encoder_dim, - duration_predictor_dim, - duration_predictor_kernel_size, - duration_predictor_dropout, - epsilon=epsilon) - self.length_regulator = FastSpeechLengthRegulator() - self.decoder = FastSpeechFeedForwardTransformer( - decoder_num_layers, - decoder_dim, - decoder_num_heads, - decoder_ffn_dim, - decoder_ffn_kernel_size, - attention_dropout, - residual_dropout, - num_speakers=num_speakers, - max_position=decoder_max_position, - input_dim=encoder_dim, - epsilon=epsilon, - scheme=scheme) - self.mel_output_fc = nn.Linear(decoder_dim, mel_dim) - self.postnet = TacotronPostNet( - postnet_num_layers, - mel_dim, - postnet_dim, - postnet_kernel_size, - postnet_dropout, - momentum=momentum, - epsilon=epsilon) - - def forward(self, text_ids, speaker_ids, durations, text_lens): - dtype = paddle.get_default_dtype() - encoder_padding_mask = sequence_mask(text_lens, dtype=dtype) - encoder_attention_mask = encoder_padding_mask.unsqueeze(1) - - embedding = self.embedding(text_ids) - encoder_output = self.encoder(embedding, encoder_attention_mask, - speaker_ids) - - # detach the gradient of duration predictor - # a difference here - predicted_durations = self.duration_predictor(encoder_output.detach(), - encoder_padding_mask) - - expanded_outputs, mel_lens = self.length_regulator(encoder_output, - durations) - decoder_padding_mask = sequence_mask(mel_lens, dtype=dtype) - decoder_attention_mask = decoder_padding_mask.unsqueeze(1) - - decoder_ouputs = self.decoder( - expanded_outputs, - decoder_attention_mask, - speaker_ids, ) - decoder_mel = self.mel_output_fc(decoder_ouputs) - postnet_mel = decoder_mel + self.postnet(decoder_mel, - decoder_padding_mask) - - return decoder_mel, postnet_mel, predicted_durations - - def inference(self, text_ids, speaker_ids, text_lens, speed_ratios): - dtype = paddle.get_default_dtype() - encoder_padding_mask = sequence_mask(text_lens, dtype=dtype) - encoder_attention_mask = encoder_padding_mask.unsqueeze(1) - - embedding = self.embedding(text_ids) - encoder_output = self.encoder(embedding, encoder_attention_mask, - speaker_ids) - - # detach the gradient flow of duration predictor - # a difference here - predicted_log_durations = self.duration_predictor( - encoder_output.detach(), encoder_padding_mask) - predicted_durations = paddle.exp(predicted_log_durations) - 1. - - if speed_ratios is None: - speed_ratios = paddle.ones([1], dtype=dtype) - speed_ratios = paddle.unsqueeze(speed_ratios, -1) - predicted_durations = paddle.round(predicted_durations * - speed_ratios).astype("int32") - - expanded_outputs, mel_lens = self.length_regulator(encoder_output, - predicted_durations) - decoder_padding_mask = sequence_mask(mel_lens, dtype=dtype) - decoder_attention_mask = decoder_padding_mask.unsqueeze(1) - - decoder_ouputs = self.decoder(expanded_outputs, decoder_attention_mask, - speaker_ids) - decoder_mel = self.mel_output_fc(decoder_ouputs) - postnet_mel = decoder_mel + self.postnet(decoder_mel, - decoder_padding_mask) - - return decoder_mel, postnet_mel, predicted_durations - - -# TODO: implement FastSpeech2 class FastSpeech2(nn.Layer): + """FastSpeech2 module. + + This is a module of FastSpeech2 described in `FastSpeech 2: Fast and + High-Quality End-to-End Text to Speech`_. Instead of quantized pitch and + energy, we use token-averaged value introduced in `FastPitch: Parallel + Text-to-speech with Pitch Prediction`_. + + .. _`FastSpeech 2: Fast and High-Quality End-to-End Text to Speech`: + https://arxiv.org/abs/2006.04558 + .. _`FastPitch: Parallel Text-to-speech with Pitch Prediction`: + https://arxiv.org/abs/2006.06873 + + """ + def __init__( self, - vocab_size, - num_speakers, - # encoder params - encoder_num_layers, - encoder_dim, - encoder_num_heads, - encoder_max_position, - encoder_ffn_dim, - encoder_ffn_kernel_size, - # decoder params - decoder_num_layers, - decoder_dim, - decoder_num_heads, - decoder_max_position, - decoder_ffn_dim, - decoder_ffn_kernel_size, - # encoder & decoder common - attention_dropout, - residual_dropout, + # network structure related + idim: int, + odim: int, + adim: int=384, + aheads: int=4, + elayers: int=6, + eunits: int=1536, + dlayers: int=6, + dunits: int=1536, + postnet_layers: int=5, + postnet_chans: int=512, + postnet_filts: int=5, + positionwise_layer_type: str="conv1d", + positionwise_conv_kernel_size: int=1, + use_scaled_pos_enc: bool=True, + use_batch_norm: bool=True, + encoder_normalize_before: bool=True, + decoder_normalize_before: bool=True, + encoder_concat_after: bool=False, + decoder_concat_after: bool=False, + reduction_factor: int=1, + encoder_type: str="transformer", + decoder_type: str="transformer", # duration predictor - duration_predictor_num_layers, - duration_predictor_dim, - duration_predictor_kernel_size, - duration_predictor_dropout, - # output - mel_dim, - # postnet - postnet_num_layers, - postnet_dim, - postnet_kernel_size, - postnet_dropout, - # variance predictor - variance_predictor_num_layers, - variance_predictor_dim, - variance_predictor_kernel_size, - variance_predictor_dropout, - # other - padding_idx=0, - momentum=0.9, - epsilon=1e-5, - scheme="post"): + duration_predictor_layers: int=2, + duration_predictor_chans: int=384, + duration_predictor_kernel_size: int=3, + # energy predictor + energy_predictor_layers: int=2, + energy_predictor_chans: int=384, + energy_predictor_kernel_size: int=3, + energy_predictor_dropout: float=0.5, + energy_embed_kernel_size: int=9, + energy_embed_dropout: float=0.5, + stop_gradient_from_energy_predictor: bool=False, + # pitch predictor + pitch_predictor_layers: int=2, + pitch_predictor_chans: int=384, + pitch_predictor_kernel_size: int=3, + pitch_predictor_dropout: float=0.5, + pitch_embed_kernel_size: int=9, + pitch_embed_dropout: float=0.5, + stop_gradient_from_pitch_predictor: bool=False, + # training related + transformer_enc_dropout_rate: float=0.1, + transformer_enc_positional_dropout_rate: float=0.1, + transformer_enc_attn_dropout_rate: float=0.1, + transformer_dec_dropout_rate: float=0.1, + transformer_dec_positional_dropout_rate: float=0.1, + transformer_dec_attn_dropout_rate: float=0.1, + duration_predictor_dropout_rate: float=0.1, + postnet_dropout_rate: float=0.5, + init_type: str="xavier_uniform", + init_enc_alpha: float=1.0, + init_dec_alpha: float=1.0, + use_masking: bool=False, + use_weighted_masking: bool=False, ): + """Initialize FastSpeech2 module.""" + assert check_argument_types() super().__init__() - self.embedding = nn.Embedding( - vocab_size, encoder_dim, padding_idx=padding_idx) - self.encoder = FastSpeechFeedForwardTransformer( - encoder_num_layers, - encoder_dim, - encoder_num_heads, - encoder_ffn_dim, - encoder_ffn_kernel_size, - attention_dropout, - residual_dropout, - num_speakers=num_speakers, - max_position=encoder_max_position, - epsilon=epsilon, - scheme=scheme) - self.duration_predictor = FastSpeechDurationPredictor( - duration_predictor_num_layers, - encoder_dim, - duration_predictor_dim, - duration_predictor_kernel_size, - duration_predictor_dropout, - epsilon=epsilon) - self.length_regulator = FastSpeechLengthRegulator() - self.decoder = FastSpeechFeedForwardTransformer( - decoder_num_layers, - decoder_dim, - decoder_num_heads, - decoder_ffn_dim, - decoder_ffn_kernel_size, - attention_dropout, - residual_dropout, - num_speakers=num_speakers, - max_position=decoder_max_position, - input_dim=encoder_dim, - epsilon=epsilon, - scheme=scheme) - self.mel_output_fc = nn.Linear(decoder_dim, mel_dim) - self.postnet = TacotronPostNet( - postnet_num_layers, - mel_dim, - postnet_dim, - postnet_kernel_size, - postnet_dropout, - momentum=momentum, - epsilon=epsilon) - # difference here? - self.f0_predictor = FastSpeechVariancePredictor( - variance_predictor_num_layers, - embed_dim, - variance_predictor_dim, - variancce_predictor_kernel_size, - num_speakers, - speaker_embedding_size=embed_dim) - self.energy_predictor = FastSpeechVariancePredictor( - variance_predictor_num_layers, - embed_dim, - variance_predictor_dim, - variancce_predictor_kernel_size, - num_speakers, - speaker_embedding_size=embed_dim) - #self.duration_predictor = FastSpeechVariancePredictor( - #variance_predictor_num_layers, - #embed_dim, - #variance_predictor_dim, - #variancce_predictor_kernel_size, - #num_speakers, - #speaker_embedding_size=embed_dim) - self.f0_embedding = nn.Conv1D( - 1, encoder_dim, kernel_size=9, padding="same", data_format="NLC") - self.f0_dropout_layer = nn.Dropout(0.5) - self.energy_embeddings = nn.Conv1D( - 1, encoder_dim, kernel_size=9, padding="same", data_format="NLC") - self.energy_dropout = nn.Dropout(0.5) - def forward(self, text_ids, speaker_ids, durations, text_lens): - dtype = paddle.get_default_dtype() - encoder_padding_mask = sequence_mask(text_lens, dtype=dtype) - encoder_attention_mask = encoder_padding_mask.unsqueeze(1) + # store hyperparameters + self.idim = idim + self.odim = odim + self.eos = idim - 1 + self.reduction_factor = reduction_factor + self.encoder_type = encoder_type + self.decoder_type = decoder_type + self.stop_gradient_from_pitch_predictor = stop_gradient_from_pitch_predictor + self.stop_gradient_from_energy_predictor = stop_gradient_from_energy_predictor + self.use_scaled_pos_enc = use_scaled_pos_enc - embedding = self.embedding(text_ids) - encoder_output = self.encoder(embedding, encoder_attention_mask, - speaker_ids) + # use idx 0 as padding idx + self.padding_idx = 0 - # detach the gradient of duration predictor - # a difference here - predicted_durations = self.duration_predictor(encoder_output.detach(), - encoder_padding_mask) + # initialize parameters + initialize(self, init_type) - expanded_outputs, mel_lens = self.length_regulator(encoder_output, - durations) - decoder_padding_mask = sequence_mask(mel_lens, dtype=dtype) - decoder_attention_mask = decoder_padding_mask.unsqueeze(1) + # get positional encoding class + pos_enc_class = (ScaledPositionalEncoding + if self.use_scaled_pos_enc else PositionalEncoding) - decoder_ouputs = self.decoder( - expanded_outputs, - decoder_attention_mask, - speaker_ids, ) - decoder_mel = self.mel_output_fc(decoder_ouputs) - postnet_mel = decoder_mel + self.postnet(decoder_mel, - decoder_padding_mask) + # define encoder + encoder_input_layer = nn.Embedding( + num_embeddings=idim, + embedding_dim=adim, + padding_idx=self.padding_idx) - return decoder_mel, postnet_mel, predicted_durations + if encoder_type == "transformer": + self.encoder = TransformerEncoder( + idim=idim, + attention_dim=adim, + attention_heads=aheads, + linear_units=eunits, + num_blocks=elayers, + input_layer=encoder_input_layer, + dropout_rate=transformer_enc_dropout_rate, + positional_dropout_rate=transformer_enc_positional_dropout_rate, + attention_dropout_rate=transformer_enc_attn_dropout_rate, + pos_enc_class=pos_enc_class, + normalize_before=encoder_normalize_before, + concat_after=encoder_concat_after, + positionwise_layer_type=positionwise_layer_type, + positionwise_conv_kernel_size=positionwise_conv_kernel_size, ) + else: + raise ValueError(f"{encoder_type} is not supported.") - def inference(self, text_ids, speaker_ids, text_lens, speed_ratios): - dtype = paddle.get_default_dtype() - encoder_padding_mask = sequence_mask(text_lens, dtype=dtype) - encoder_attention_mask = encoder_padding_mask.unsqueeze(1) + # define duration predictor + self.duration_predictor = DurationPredictor( + idim=adim, + n_layers=duration_predictor_layers, + n_chans=duration_predictor_chans, + kernel_size=duration_predictor_kernel_size, + dropout_rate=duration_predictor_dropout_rate, ) - embedding = self.embedding(text_ids) - encoder_output = self.encoder(embedding, encoder_attention_mask, - speaker_ids) + # define pitch predictor + self.pitch_predictor = VariancePredictor( + idim=adim, + n_layers=pitch_predictor_layers, + n_chans=pitch_predictor_chans, + kernel_size=pitch_predictor_kernel_size, + dropout_rate=pitch_predictor_dropout, ) + # We use continuous pitch + FastPitch style avg + self.pitch_embed = nn.Sequential( + nn.Conv1D( + in_channels=1, + out_channels=adim, + kernel_size=pitch_embed_kernel_size, + padding=(pitch_embed_kernel_size - 1) // 2, ), + nn.Dropout(pitch_embed_dropout), ) - # detach the gradient flow of duration predictor - # a difference here - predicted_log_durations = self.duration_predictor( - encoder_output.detach(), encoder_padding_mask) - predicted_durations = paddle.exp(predicted_log_durations) - 1. + # define energy predictor + self.energy_predictor = VariancePredictor( + idim=adim, + n_layers=energy_predictor_layers, + n_chans=energy_predictor_chans, + kernel_size=energy_predictor_kernel_size, + dropout_rate=energy_predictor_dropout, ) + # We use continuous enegy + FastPitch style avg + self.energy_embed = nn.Sequential( + nn.Conv1D( + in_channels=1, + out_channels=adim, + kernel_size=energy_embed_kernel_size, + padding=(energy_embed_kernel_size - 1) // 2, ), + nn.Dropout(energy_embed_dropout), ) - if speed_ratios is None: - speed_ratios = paddle.ones([1], dtype=dtype) - speed_ratios = paddle.unsqueeze(speed_ratios, -1) - predicted_durations = paddle.round(predicted_durations * - speed_ratios).astype("int32") + # define length regulator + self.length_regulator = LengthRegulator() - expanded_outputs, mel_lens = self.length_regulator(encoder_output, - predicted_durations) - decoder_padding_mask = sequence_mask(mel_lens, dtype=dtype) - decoder_attention_mask = decoder_padding_mask.unsqueeze(1) + # define decoder + # NOTE: we use encoder as decoder + # because fastspeech's decoder is the same as encoder + if decoder_type == "transformer": + self.decoder = TransformerEncoder( + idim=0, + attention_dim=adim, + attention_heads=aheads, + linear_units=dunits, + num_blocks=dlayers, + input_layer=None, + dropout_rate=transformer_dec_dropout_rate, + positional_dropout_rate=transformer_dec_positional_dropout_rate, + attention_dropout_rate=transformer_dec_attn_dropout_rate, + pos_enc_class=pos_enc_class, + normalize_before=decoder_normalize_before, + concat_after=decoder_concat_after, + positionwise_layer_type=positionwise_layer_type, + positionwise_conv_kernel_size=positionwise_conv_kernel_size, ) + else: + raise ValueError(f"{decoder_type} is not supported.") - decoder_ouputs = self.decoder(expanded_outputs, decoder_attention_mask, - speaker_ids) - decoder_mel = self.mel_output_fc(decoder_ouputs) - postnet_mel = decoder_mel + self.postnet(decoder_mel, - decoder_padding_mask) + # define final projection + self.feat_out = nn.Linear(adim, odim * reduction_factor) - return decoder_mel, postnet_mel, predicted_durations + # define postnet + self.postnet = (None if postnet_layers == 0 else Postnet( + idim=idim, + odim=odim, + n_layers=postnet_layers, + n_chans=postnet_chans, + n_filts=postnet_filts, + use_batch_norm=use_batch_norm, + dropout_rate=postnet_dropout_rate, )) + + self._reset_parameters( + init_enc_alpha=init_enc_alpha, + init_dec_alpha=init_dec_alpha, ) + + # define criterions + self.criterion = FastSpeech2Loss( + use_masking=use_masking, use_weighted_masking=use_weighted_masking) + + def forward( + self, + text: paddle.Tensor, + text_lengths: paddle.Tensor, + speech: paddle.Tensor, + speech_lengths: paddle.Tensor, + durations: paddle.Tensor, + pitch: paddle.Tensor, + energy: paddle.Tensor, ) -> Tuple[paddle.Tensor, Dict[ + str, paddle.Tensor], paddle.Tensor]: + """Calculate forward propagation. + + Parameters + ---------- + text : LongTensor + Batch of padded token ids (B, Tmax). + text_lengths : LongTensor) + Batch of lengths of each input (B,). + speech : Tensor + Batch of padded target features (B, Lmax, odim). + speech_lengths : LongTensor + Batch of the lengths of each target (B,). + durations : LongTensor + Batch of padded durations (B, Tmax). + pitch : Tensor + Batch of padded token-averaged pitch (B, Tmax, 1). + energy : Tensor + Batch of padded token-averaged energy (B, Tmax, 1). + Returns + ---------- + Tensor + mel outs before postnet + Tensor + mel outs after postnet + Tensor + duration predictor's output + Tensor + pitch predictor's output + Tensor + energy predictor's output + Tensor + speech + Tensor + real text_lengths + Tensor + speech_lengths, modified if reduction_factor >1 + """ + + xs = paddle.to_tensor(text) + ilens = text_lengths + ys, ds, ps, es = speech, durations, pitch, energy + olens = speech_lengths + + # forward propagation + before_outs, after_outs, d_outs, p_outs, e_outs = self._forward( + xs, ilens, ys, olens, ds, ps, es, is_inference=False) + # modify mod part of groundtruth + if self.reduction_factor > 1: + olens = paddle.to_tensor([ + olen - olen % self.reduction_factor for olen in olens.numpy() + ]) + max_olen = max(olens) + ys = ys[:, :max_olen] + + return before_outs, after_outs, d_outs, p_outs, e_outs, ys, olens + + def _forward( + self, + xs: paddle.Tensor, + ilens: paddle.Tensor, + ys: paddle.Tensor=None, + olens: paddle.Tensor=None, + ds: paddle.Tensor=None, + ps: paddle.Tensor=None, + es: paddle.Tensor=None, + is_inference: bool=False, + alpha: float=1.0, ) -> Sequence[paddle.Tensor]: + # forward encoder + x_masks = self._source_mask(ilens) + + hs, _ = self.encoder(xs, x_masks) # (B, Tmax, adim) + # forward duration predictor and variance predictors + d_masks = make_pad_mask(ilens) + + if self.stop_gradient_from_pitch_predictor: + p_outs = self.pitch_predictor(hs.detach(), d_masks.unsqueeze(-1)) + else: + p_outs = self.pitch_predictor(hs, d_masks.unsqueeze(-1)) + if self.stop_gradient_from_energy_predictor: + e_outs = self.energy_predictor(hs.detach(), d_masks.unsqueeze(-1)) + else: + e_outs = self.energy_predictor(hs, d_masks.unsqueeze(-1)) + + if is_inference: + # (B, Tmax) + d_outs = self.duration_predictor.inference(hs, d_masks) + # use prediction in inference + # (B, Tmax, 1) + p_embs = self.pitch_embed(p_outs.transpose((0, 2, 1))).transpose( + (0, 2, 1)) + e_embs = self.energy_embed(e_outs.transpose((0, 2, 1))).transpose( + (0, 2, 1)) + hs = hs + e_embs + p_embs + # (B, Lmax, adim) + hs = self.length_regulator(hs, d_outs, alpha) + else: + d_outs = self.duration_predictor(hs, d_masks) + # use groundtruth in training + p_embs = self.pitch_embed(ps.transpose((0, 2, 1))).transpose( + (0, 2, 1)) + e_embs = self.energy_embed(es.transpose((0, 2, 1))).transpose( + (0, 2, 1)) + hs = hs + e_embs + p_embs + # (B, Lmax, adim) + hs = self.length_regulator(hs, ds) + + # forward decoder + if olens is not None and not is_inference: + if self.reduction_factor > 1: + olens_in = paddle.to_tensor( + [olen // self.reduction_factor for olen in olens.numpy()]) + else: + olens_in = olens + h_masks = self._source_mask(olens_in) + else: + h_masks = None + # (B, Lmax, adim) + zs, _ = self.decoder(hs, h_masks) + # (B, Lmax, odim) + before_outs = self.feat_out(zs).reshape((zs.shape[0], -1, self.odim)) + + # postnet -> (B, Lmax//r * r, odim) + if self.postnet is None: + after_outs = before_outs + else: + after_outs = before_outs + self.postnet( + before_outs.transpose((0, 2, 1))).transpose((0, 2, 1)) + + return before_outs, after_outs, d_outs, p_outs, e_outs + + def inference( + self, + text: paddle.Tensor, + speech: paddle.Tensor=None, + durations: paddle.Tensor=None, + pitch: paddle.Tensor=None, + energy: paddle.Tensor=None, + alpha: float=1.0, + use_teacher_forcing: bool=False, ) -> Tuple[ + paddle.Tensor, paddle.Tensor, paddle.Tensor]: + """Generate the sequence of features given the sequences of characters. + + Parameters + ---------- + text : LongTensor + Input sequence of characters (T,). + speech : Tensor, optional + Feature sequence to extract style (N, idim). + durations : LongTensor, optional + Groundtruth of duration (T + 1,). + pitch : Tensor, optional + Groundtruth of token-averaged pitch (T + 1, 1). + energy : Tensor, optional + Groundtruth of token-averaged energy (T + 1, 1). + alpha : float, optional + Alpha to control the speed. + use_teacher_forcing : bool, optional + Whether to use teacher forcing. + If true, groundtruth of duration, pitch and energy will be used. + + Returns + ---------- + Tensor + Output sequence of features (L, odim). + None + Dummy for compatibility. + + """ + x, y = text, speech + d, p, e = durations, pitch, energy + + # add eos at the last of sequence + x = np.pad(text.numpy(), + pad_width=((0, 1)), + mode="constant", + constant_values=self.eos) + + x = paddle.to_tensor(x) + + # setup batch axis + ilens = paddle.to_tensor( + [x.shape[0]], dtype=paddle.int64, place=x.place) + xs, ys = x.unsqueeze(0), None + + if y is not None: + ys = y.unsqueeze(0) + + if use_teacher_forcing: + # use groundtruth of duration, pitch, and energy + ds, ps, es = d.unsqueeze(0), p.unsqueeze(0), e.unsqueeze(0) + # (1, L, odim) + _, outs, *_ = self._forward( + xs, + ilens, + ys, + ds=ds, + ps=ps, + es=es, ) + else: + # (1, L, odim) + _, outs, *_ = self._forward( + xs, + ilens, + ys, + is_inference=True, + alpha=alpha, ) + + return outs[0], None, None + + def _source_mask(self, ilens: paddle.Tensor) -> paddle.Tensor: + """Make masks for self-attention. + + Parameters + ---------- + ilens : LongTensor + Batch of lengths (B,). + + Returns + ------- + Tensor + Mask tensor for self-attention. + dtype=paddle.bool + + Examples + ------- + >>> ilens = [5, 3] + >>> self._source_mask(ilens) + tensor([[[1, 1, 1, 1, 1], + [1, 1, 1, 0, 0]]]) bool + + """ + x_masks = make_non_pad_mask(ilens) + return x_masks.unsqueeze(-2) + + def _reset_parameters(self, init_enc_alpha: float, init_dec_alpha: float): + + # initialize alpha in scaled positional encoding + if self.encoder_type == "transformer" and self.use_scaled_pos_enc: + init_enc_alpha = paddle.to_tensor(init_enc_alpha) + self.encoder.embed[-1].alpha = paddle.create_parameter( + shape=init_enc_alpha.shape, + dtype=str(init_enc_alpha.numpy().dtype), + default_initializer=paddle.nn.initializer.Assign( + init_enc_alpha)) + if self.decoder_type == "transformer" and self.use_scaled_pos_enc: + init_dec_alpha = paddle.to_tensor(init_dec_alpha) + self.decoder.embed[-1].alpha = paddle.create_parameter( + shape=init_dec_alpha.shape, + dtype=str(init_dec_alpha.numpy().dtype), + default_initializer=paddle.nn.initializer.Assign( + init_dec_alpha)) + + +class FastSpeech2Loss(nn.Layer): + """Loss function module for FastSpeech2.""" + + def __init__(self, + use_masking: bool=True, + use_weighted_masking: bool=False): + """Initialize feed-forward Transformer loss module. + + Parameters + ---------- + use_masking : bool + Whether to apply masking for padded part in loss calculation. + use_weighted_masking : bool + Whether to weighted masking in loss calculation. + """ + assert check_argument_types() + super().__init__() + + assert (use_masking != use_weighted_masking) or not use_masking + self.use_masking = use_masking + self.use_weighted_masking = use_weighted_masking + + # define criterions + reduction = "none" if self.use_weighted_masking else "mean" + self.l1_criterion = nn.L1Loss(reduction=reduction) + self.mse_criterion = nn.MSELoss(reduction=reduction) + self.duration_criterion = DurationPredictorLoss(reduction=reduction) + + def forward( + self, + after_outs: paddle.Tensor, + before_outs: paddle.Tensor, + d_outs: paddle.Tensor, + p_outs: paddle.Tensor, + e_outs: paddle.Tensor, + ys: paddle.Tensor, + ds: paddle.Tensor, + ps: paddle.Tensor, + es: paddle.Tensor, + ilens: paddle.Tensor, + olens: paddle.Tensor, ) -> Tuple[paddle.Tensor, paddle.Tensor, + paddle.Tensor, paddle.Tensor]: + """Calculate forward propagation. + + Parameters + ---------- + after_outs : Tensor + Batch of outputs after postnets (B, Lmax, odim). + before_outs : Tensor + Batch of outputs before postnets (B, Lmax, odim). + d_outs : LongTensor + Batch of outputs of duration predictor (B, Tmax). + p_outs : Tensor + Batch of outputs of pitch predictor (B, Tmax, 1). + e_outs : Tensor + Batch of outputs of energy predictor (B, Tmax, 1). + ys : Tensor + Batch of target features (B, Lmax, odim). + ds : LongTensor + Batch of durations (B, Tmax). + ps : Tensor + Batch of target token-averaged pitch (B, Tmax, 1). + es : Tensor + Batch of target token-averaged energy (B, Tmax, 1). + ilens : LongTensor + Batch of the lengths of each input (B,). + olens : LongTensor + Batch of the lengths of each target (B,). + + Returns + ---------- + Tensor + L1 loss value. + Tensor + Duration predictor loss value. + Tensor + Pitch predictor loss value. + Tensor + Energy predictor loss value. + + """ + # apply mask to remove padded part + if self.use_masking: + out_masks = make_non_pad_mask(olens).unsqueeze(-1) + before_outs = before_outs.masked_select( + out_masks.broadcast_to(before_outs.shape)) + if after_outs is not None: + after_outs = after_outs.masked_select( + out_masks.broadcast_to(after_outs.shape)) + ys = ys.masked_select(out_masks.broadcast_to(ys.shape)) + duration_masks = make_non_pad_mask(ilens) + d_outs = d_outs.masked_select( + duration_masks.broadcast_to(d_outs.shape)) + ds = ds.masked_select(duration_masks.broadcast_to(ds.shape)) + pitch_masks = make_non_pad_mask(ilens).unsqueeze(-1) + p_outs = p_outs.masked_select( + pitch_masks.broadcast_to(p_outs.shape)) + e_outs = e_outs.masked_select( + pitch_masks.broadcast_to(e_outs.shape)) + ps = ps.masked_select(pitch_masks.broadcast_to(ps.shape)) + es = es.masked_select(pitch_masks.broadcast_to(es.shape)) + + # calculate loss + l1_loss = self.l1_criterion(before_outs, ys) + if after_outs is not None: + l1_loss += self.l1_criterion(after_outs, ys) + duration_loss = self.duration_criterion(d_outs, ds) + pitch_loss = self.mse_criterion(p_outs, ps) + energy_loss = self.mse_criterion(e_outs, es) + + # make weighted mask and apply it + if self.use_weighted_masking: + out_masks = make_non_pad_mask(olens).unsqueeze(-1) + out_weights = out_masks.cast( + dtype=paddle.float32) / out_masks.cast( + dtype=paddle.float32).sum(axis=1, keepdim=True) + out_weights /= ys.shape[0] * ys.shape[2] + duration_masks = make_non_pad_mask(ilens) + duration_weights = (duration_masks.cast(dtype=paddle.float32) / + duration_masks.cast(dtype=paddle.float32).sum( + axis=1, keepdim=True)) + duration_weights /= ds.shape[0] + + # apply weight + + l1_loss = l1_loss.multiply(out_weights) + l1_loss = l1_loss.masked_select( + out_masks.broadcast_to(l1_loss.shape)).sum() + duration_loss = (duration_loss.multiply(duration_weights) + .masked_select(duration_masks).sum()) + pitch_masks = duration_masks.unsqueeze(-1) + pitch_weights = duration_weights.unsqueeze(-1) + pitch_loss = pitch_loss.multiply(pitch_weights) + pitch_loss = pitch_loss.masked_select( + pitch_masks.broadcast_to(pitch_loss.shape)).sum() + energy_loss = energy_loss.multiply(pitch_weights) + energy_loss = energy_loss.masked_select( + pitch_masks.broadcast_to(energy_loss.shape)).sum() + + return l1_loss, duration_loss, pitch_loss, energy_loss diff --git a/parakeet/models/fastspeech2_new.py b/parakeet/models/fastspeech2_new.py deleted file mode 100644 index 80d78fb..0000000 --- a/parakeet/models/fastspeech2_new.py +++ /dev/null @@ -1,660 +0,0 @@ -# Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -"""Fastspeech2 related modules for paddle""" - -from typing import Dict -from typing import Sequence -from typing import Tuple -from typeguard import check_argument_types - -import paddle -import numpy as np -from paddle import nn -from parakeet.modules.fastspeech2_predictor.duration_predictor import DurationPredictor -from parakeet.modules.fastspeech2_predictor.duration_predictor import DurationPredictorLoss -from parakeet.modules.fastspeech2_predictor.length_regulator import LengthRegulator -from parakeet.modules.fastspeech2_predictor.postnet import Postnet -from parakeet.modules.fastspeech2_predictor.variance_predictor import VariancePredictor -from parakeet.modules.fastspeech2_transformer.embedding import PositionalEncoding -from parakeet.modules.fastspeech2_transformer.embedding import ScaledPositionalEncoding -from parakeet.modules.fastspeech2_transformer.encoder import Encoder as TransformerEncoder -from parakeet.modules.nets_utils import initialize -from parakeet.modules.nets_utils import make_non_pad_mask -from parakeet.modules.nets_utils import make_pad_mask - - -class FastSpeech2(nn.Layer): - """FastSpeech2 module. - - This is a module of FastSpeech2 described in `FastSpeech 2: Fast and - High-Quality End-to-End Text to Speech`_. Instead of quantized pitch and - energy, we use token-averaged value introduced in `FastPitch: Parallel - Text-to-speech with Pitch Prediction`_. - - .. _`FastSpeech 2: Fast and High-Quality End-to-End Text to Speech`: - https://arxiv.org/abs/2006.04558 - .. _`FastPitch: Parallel Text-to-speech with Pitch Prediction`: - https://arxiv.org/abs/2006.06873 - - """ - - def __init__( - self, - # network structure related - idim: int, - odim: int, - adim: int=384, - aheads: int=4, - elayers: int=6, - eunits: int=1536, - dlayers: int=6, - dunits: int=1536, - postnet_layers: int=5, - postnet_chans: int=512, - postnet_filts: int=5, - positionwise_layer_type: str="conv1d", - positionwise_conv_kernel_size: int=1, - use_scaled_pos_enc: bool=True, - use_batch_norm: bool=True, - encoder_normalize_before: bool=True, - decoder_normalize_before: bool=True, - encoder_concat_after: bool=False, - decoder_concat_after: bool=False, - reduction_factor: int=1, - encoder_type: str="transformer", - decoder_type: str="transformer", - # duration predictor - duration_predictor_layers: int=2, - duration_predictor_chans: int=384, - duration_predictor_kernel_size: int=3, - # energy predictor - energy_predictor_layers: int=2, - energy_predictor_chans: int=384, - energy_predictor_kernel_size: int=3, - energy_predictor_dropout: float=0.5, - energy_embed_kernel_size: int=9, - energy_embed_dropout: float=0.5, - stop_gradient_from_energy_predictor: bool=False, - # pitch predictor - pitch_predictor_layers: int=2, - pitch_predictor_chans: int=384, - pitch_predictor_kernel_size: int=3, - pitch_predictor_dropout: float=0.5, - pitch_embed_kernel_size: int=9, - pitch_embed_dropout: float=0.5, - stop_gradient_from_pitch_predictor: bool=False, - # training related - transformer_enc_dropout_rate: float=0.1, - transformer_enc_positional_dropout_rate: float=0.1, - transformer_enc_attn_dropout_rate: float=0.1, - transformer_dec_dropout_rate: float=0.1, - transformer_dec_positional_dropout_rate: float=0.1, - transformer_dec_attn_dropout_rate: float=0.1, - duration_predictor_dropout_rate: float=0.1, - postnet_dropout_rate: float=0.5, - init_type: str="xavier_uniform", - init_enc_alpha: float=1.0, - init_dec_alpha: float=1.0, - use_masking: bool=False, - use_weighted_masking: bool=False, ): - """Initialize FastSpeech2 module.""" - assert check_argument_types() - super().__init__() - - # store hyperparameters - self.idim = idim - self.odim = odim - self.eos = idim - 1 - self.reduction_factor = reduction_factor - self.encoder_type = encoder_type - self.decoder_type = decoder_type - self.stop_gradient_from_pitch_predictor = stop_gradient_from_pitch_predictor - self.stop_gradient_from_energy_predictor = stop_gradient_from_energy_predictor - self.use_scaled_pos_enc = use_scaled_pos_enc - - # use idx 0 as padding idx - self.padding_idx = 0 - - # initialize parameters - initialize(self, init_type) - - # get positional encoding class - pos_enc_class = (ScaledPositionalEncoding - if self.use_scaled_pos_enc else PositionalEncoding) - - # define encoder - encoder_input_layer = nn.Embedding( - num_embeddings=idim, - embedding_dim=adim, - padding_idx=self.padding_idx) - - if encoder_type == "transformer": - self.encoder = TransformerEncoder( - idim=idim, - attention_dim=adim, - attention_heads=aheads, - linear_units=eunits, - num_blocks=elayers, - input_layer=encoder_input_layer, - dropout_rate=transformer_enc_dropout_rate, - positional_dropout_rate=transformer_enc_positional_dropout_rate, - attention_dropout_rate=transformer_enc_attn_dropout_rate, - pos_enc_class=pos_enc_class, - normalize_before=encoder_normalize_before, - concat_after=encoder_concat_after, - positionwise_layer_type=positionwise_layer_type, - positionwise_conv_kernel_size=positionwise_conv_kernel_size, ) - else: - raise ValueError(f"{encoder_type} is not supported.") - - # define duration predictor - self.duration_predictor = DurationPredictor( - idim=adim, - n_layers=duration_predictor_layers, - n_chans=duration_predictor_chans, - kernel_size=duration_predictor_kernel_size, - dropout_rate=duration_predictor_dropout_rate, ) - - # define pitch predictor - self.pitch_predictor = VariancePredictor( - idim=adim, - n_layers=pitch_predictor_layers, - n_chans=pitch_predictor_chans, - kernel_size=pitch_predictor_kernel_size, - dropout_rate=pitch_predictor_dropout, ) - # We use continuous pitch + FastPitch style avg - self.pitch_embed = nn.Sequential( - nn.Conv1D( - in_channels=1, - out_channels=adim, - kernel_size=pitch_embed_kernel_size, - padding=(pitch_embed_kernel_size - 1) // 2, ), - nn.Dropout(pitch_embed_dropout), ) - - # define energy predictor - self.energy_predictor = VariancePredictor( - idim=adim, - n_layers=energy_predictor_layers, - n_chans=energy_predictor_chans, - kernel_size=energy_predictor_kernel_size, - dropout_rate=energy_predictor_dropout, ) - # We use continuous enegy + FastPitch style avg - self.energy_embed = nn.Sequential( - nn.Conv1D( - in_channels=1, - out_channels=adim, - kernel_size=energy_embed_kernel_size, - padding=(energy_embed_kernel_size - 1) // 2, ), - nn.Dropout(energy_embed_dropout), ) - - # define length regulator - self.length_regulator = LengthRegulator() - - # define decoder - # NOTE: we use encoder as decoder - # because fastspeech's decoder is the same as encoder - if decoder_type == "transformer": - self.decoder = TransformerEncoder( - idim=0, - attention_dim=adim, - attention_heads=aheads, - linear_units=dunits, - num_blocks=dlayers, - input_layer=None, - dropout_rate=transformer_dec_dropout_rate, - positional_dropout_rate=transformer_dec_positional_dropout_rate, - attention_dropout_rate=transformer_dec_attn_dropout_rate, - pos_enc_class=pos_enc_class, - normalize_before=decoder_normalize_before, - concat_after=decoder_concat_after, - positionwise_layer_type=positionwise_layer_type, - positionwise_conv_kernel_size=positionwise_conv_kernel_size, ) - else: - raise ValueError(f"{decoder_type} is not supported.") - - # define final projection - self.feat_out = nn.Linear(adim, odim * reduction_factor) - - # define postnet - self.postnet = (None if postnet_layers == 0 else Postnet( - idim=idim, - odim=odim, - n_layers=postnet_layers, - n_chans=postnet_chans, - n_filts=postnet_filts, - use_batch_norm=use_batch_norm, - dropout_rate=postnet_dropout_rate, )) - - self._reset_parameters( - init_type=init_type, - init_enc_alpha=init_enc_alpha, - init_dec_alpha=init_dec_alpha, ) - - # define criterions - self.criterion = FastSpeech2Loss( - use_masking=use_masking, use_weighted_masking=use_weighted_masking) - - def forward( - self, - text: paddle.Tensor, - text_lengths: paddle.Tensor, - speech: paddle.Tensor, - speech_lengths: paddle.Tensor, - durations: paddle.Tensor, - pitch: paddle.Tensor, - energy: paddle.Tensor, ) -> Tuple[paddle.Tensor, Dict[ - str, paddle.Tensor], paddle.Tensor]: - """Calculate forward propagation. - - Parameters - ---------- - text : LongTensor - Batch of padded token ids (B, Tmax). - text_lengths : LongTensor) - Batch of lengths of each input (B,). - speech : Tensor - Batch of padded target features (B, Lmax, odim). - speech_lengths : LongTensor - Batch of the lengths of each target (B,). - durations : LongTensor - Batch of padded durations (B, Tmax + 1). - pitch : Tensor - Batch of padded token-averaged pitch (B, Tmax + 1, 1). - energy : Tensor - Batch of padded token-averaged energy (B, Tmax + 1, 1). - Returns - ---------- - Tensor - mel outs before postnet - Tensor - mel outs after postnet - Tensor - duration predictor's output - Tensor - pitch predictor's output - Tensor - energy predictor's output - Tensor - speech - Tensor - real text_lengths - Tensor - speech_lengths, modified if reduction_factor >1 - """ - - batch_size = text.shape[0] - - # Add eos at the last of sequence - xs = np.pad(text.numpy(), - pad_width=((0, 0), (0, 1)), - mode="constant", - constant_values=self.padding_idx) - xs = paddle.to_tensor(xs) - for i, l in enumerate(text_lengths): - xs[i, l] = self.eos - ilens = text_lengths + 1 - - ys, ds, ps, es = speech, durations, pitch, energy - olens = speech_lengths - - # forward propagation - before_outs, after_outs, d_outs, p_outs, e_outs = self._forward( - xs, ilens, ys, olens, ds, ps, es, is_inference=False) - # modify mod part of groundtruth - if self.reduction_factor > 1: - olens = paddle.to_tensor([ - olen - olen % self.reduction_factor for olen in olens.numpy() - ]) - max_olen = max(olens) - ys = ys[:, :max_olen] - - return before_outs, after_outs, d_outs, p_outs, e_outs, ys, ilens, olens - - def _forward( - self, - xs: paddle.Tensor, - ilens: paddle.Tensor, - ys: paddle.Tensor=None, - olens: paddle.Tensor=None, - ds: paddle.Tensor=None, - ps: paddle.Tensor=None, - es: paddle.Tensor=None, - is_inference: bool=False, - alpha: float=1.0, ) -> Sequence[paddle.Tensor]: - # forward encoder - x_masks = self._source_mask(ilens) - - hs, _ = self.encoder(xs, x_masks) # (B, Tmax, adim) - # forward duration predictor and variance predictors - d_masks = make_pad_mask(ilens) - - if self.stop_gradient_from_pitch_predictor: - p_outs = self.pitch_predictor(hs.detach(), d_masks.unsqueeze(-1)) - else: - p_outs = self.pitch_predictor(hs, d_masks.unsqueeze(-1)) - if self.stop_gradient_from_energy_predictor: - e_outs = self.energy_predictor(hs.detach(), d_masks.unsqueeze(-1)) - else: - e_outs = self.energy_predictor(hs, d_masks.unsqueeze(-1)) - - if is_inference: - d_outs = self.duration_predictor.inference(hs, - d_masks) # (B, Tmax) - # use prediction in inference - # (B, Tmax, 1) - - p_embs = self.pitch_embed(p_outs.transpose((0, 2, 1))).transpose( - (0, 2, 1)) - e_embs = self.energy_embed(e_outs.transpose((0, 2, 1))).transpose( - (0, 2, 1)) - hs = hs + e_embs + p_embs - hs = self.length_regulator(hs, d_outs, alpha) # (B, Lmax, adim) - else: - d_outs = self.duration_predictor(hs, d_masks) - # use groundtruth in training - p_embs = self.pitch_embed(ps.transpose((0, 2, 1))).transpose( - (0, 2, 1)) - e_embs = self.energy_embed(es.transpose((0, 2, 1))).transpose( - (0, 2, 1)) - hs = hs + e_embs + p_embs - hs = self.length_regulator(hs, ds) # (B, Lmax, adim) - - # forward decoder - if olens is not None and not is_inference: - if self.reduction_factor > 1: - olens_in = paddle.to_tensor( - [olen // self.reduction_factor for olen in olens.numpy()]) - else: - olens_in = olens - h_masks = self._source_mask(olens_in) - else: - h_masks = None - # (B, Lmax, adim) - zs, _ = self.decoder(hs, h_masks) - # (B, Lmax, odim) - before_outs = self.feat_out(zs).reshape((zs.shape[0], -1, self.odim)) - - # postnet -> (B, Lmax//r * r, odim) - if self.postnet is None: - after_outs = before_outs - else: - after_outs = before_outs + self.postnet( - before_outs.transpose((0, 2, 1))).transpose((0, 2, 1)) - - return before_outs, after_outs, d_outs, p_outs, e_outs - - def inference( - self, - text: paddle.Tensor, - speech: paddle.Tensor=None, - durations: paddle.Tensor=None, - pitch: paddle.Tensor=None, - energy: paddle.Tensor=None, - alpha: float=1.0, - use_teacher_forcing: bool=False, ) -> Tuple[ - paddle.Tensor, paddle.Tensor, paddle.Tensor]: - """Generate the sequence of features given the sequences of characters. - - Parameters - ---------- - text : LongTensor - Input sequence of characters (T,). - speech : Tensor, optional - Feature sequence to extract style (N, idim). - durations : LongTensor, optional - Groundtruth of duration (T + 1,). - pitch : Tensor, optional - Groundtruth of token-averaged pitch (T + 1, 1). - energy : Tensor, optional - Groundtruth of token-averaged energy (T + 1, 1). - alpha : float, optional - Alpha to control the speed. - use_teacher_forcing : bool, optional - Whether to use teacher forcing. - If true, groundtruth of duration, pitch and energy will be used. - - Returns - ---------- - Tensor - Output sequence of features (L, odim). - None - Dummy for compatibility. - - """ - x, y = text, speech - d, p, e = durations, pitch, energy - - # add eos at the last of sequence - x = np.pad(text.numpy(), - pad_width=((0, 1)), - mode="constant", - constant_values=self.eos) - - x = paddle.to_tensor(x) - - # setup batch axis - ilens = paddle.to_tensor( - [x.shape[0]], dtype=paddle.int64, place=x.place) - xs, ys = x.unsqueeze(0), None - - if y is not None: - ys = y.unsqueeze(0) - - if use_teacher_forcing: - # use groundtruth of duration, pitch, and energy - ds, ps, es = d.unsqueeze(0), p.unsqueeze(0), e.unsqueeze(0) - _, outs, *_ = self._forward( - xs, - ilens, - ys, - ds=ds, - ps=ps, - es=es, ) # (1, L, odim) - else: - _, outs, *_ = self._forward( - xs, - ilens, - ys, - is_inference=True, - alpha=alpha, ) # (1, L, odim) - - return outs[0], None, None - - def _source_mask(self, ilens: paddle.Tensor) -> paddle.Tensor: - """Make masks for self-attention. - - Parameters - ---------- - ilens : LongTensor - Batch of lengths (B,). - - Returns - ------- - Tensor - Mask tensor for self-attention. - dtype=paddle.bool - - Examples - ------- - >>> ilens = [5, 3] - >>> self._source_mask(ilens) - tensor([[[1, 1, 1, 1, 1], - [1, 1, 1, 0, 0]]]) bool - - """ - x_masks = make_non_pad_mask(ilens) - return x_masks.unsqueeze(-2) - - def _reset_parameters(self, - init_type: str, - init_enc_alpha: float, - init_dec_alpha: float): - - # initialize alpha in scaled positional encoding - if self.encoder_type == "transformer" and self.use_scaled_pos_enc: - init_enc_alpha = paddle.to_tensor(init_enc_alpha) - self.encoder.embed[-1].alpha = paddle.create_parameter( - shape=init_enc_alpha.shape, - dtype=str(init_enc_alpha.numpy().dtype), - default_initializer=paddle.nn.initializer.Assign( - init_enc_alpha)) - if self.decoder_type == "transformer" and self.use_scaled_pos_enc: - init_dec_alpha = paddle.to_tensor(init_dec_alpha) - self.decoder.embed[-1].alpha = paddle.create_parameter( - shape=init_dec_alpha.shape, - dtype=str(init_dec_alpha.numpy().dtype), - default_initializer=paddle.nn.initializer.Assign( - init_dec_alpha)) - - -class FastSpeech2Loss(nn.Layer): - """Loss function module for FastSpeech2.""" - - def __init__(self, - use_masking: bool=True, - use_weighted_masking: bool=False): - """Initialize feed-forward Transformer loss module. - - Parameters - ---------- - use_masking : bool - Whether to apply masking for padded part in loss calculation. - use_weighted_masking : bool - Whether to weighted masking in loss calculation. - """ - assert check_argument_types() - super().__init__() - - assert (use_masking != use_weighted_masking) or not use_masking - self.use_masking = use_masking - self.use_weighted_masking = use_weighted_masking - - # define criterions - reduction = "none" if self.use_weighted_masking else "mean" - self.l1_criterion = nn.L1Loss(reduction=reduction) - self.mse_criterion = nn.MSELoss(reduction=reduction) - self.duration_criterion = DurationPredictorLoss(reduction=reduction) - - def forward( - self, - after_outs: paddle.Tensor, - before_outs: paddle.Tensor, - d_outs: paddle.Tensor, - p_outs: paddle.Tensor, - e_outs: paddle.Tensor, - ys: paddle.Tensor, - ds: paddle.Tensor, - ps: paddle.Tensor, - es: paddle.Tensor, - ilens: paddle.Tensor, - olens: paddle.Tensor, ) -> Tuple[paddle.Tensor, paddle.Tensor, - paddle.Tensor, paddle.Tensor]: - """Calculate forward propagation. - - Parameters - ---------- - after_outs : Tensor - Batch of outputs after postnets (B, Lmax, odim). - before_outs : Tensor - Batch of outputs before postnets (B, Lmax, odim). - d_outs : LongTensor - Batch of outputs of duration predictor (B, Tmax). - p_outs : Tensor - Batch of outputs of pitch predictor (B, Tmax, 1). - e_outs : Tensor - Batch of outputs of energy predictor (B, Tmax, 1). - ys : Tensor - Batch of target features (B, Lmax, odim). - ds : LongTensor - Batch of durations (B, Tmax). - ps : Tensor - Batch of target token-averaged pitch (B, Tmax, 1). - es : Tensor - Batch of target token-averaged energy (B, Tmax, 1). - ilens : LongTensor - Batch of the lengths of each input (B,). - olens : LongTensor - Batch of the lengths of each target (B,). - - Returns - ---------- - Tensor - L1 loss value. - Tensor - Duration predictor loss value. - Tensor - Pitch predictor loss value. - Tensor - Energy predictor loss value. - - """ - # apply mask to remove padded part - if self.use_masking: - out_masks = make_non_pad_mask(olens).unsqueeze(-1) - before_outs = before_outs.masked_select( - out_masks.broadcast_to(before_outs.shape)) - if after_outs is not None: - after_outs = after_outs.masked_select( - out_masks.broadcast_to(after_outs.shape)) - ys = ys.masked_select(out_masks.broadcast_to(ys.shape)) - duration_masks = make_non_pad_mask(ilens) - d_outs = d_outs.masked_select( - duration_masks.broadcast_to(d_outs.shape)) - ds = ds.masked_select(duration_masks.broadcast_to(ds.shape)) - pitch_masks = make_non_pad_mask(ilens).unsqueeze(-1) - p_outs = p_outs.masked_select( - pitch_masks.broadcast_to(p_outs.shape)) - e_outs = e_outs.masked_select( - pitch_masks.broadcast_to(e_outs.shape)) - ps = ps.masked_select(pitch_masks.broadcast_to(ps.shape)) - es = es.masked_select(pitch_masks.broadcast_to(es.shape)) - - # calculate loss - l1_loss = self.l1_criterion(before_outs, ys) - if after_outs is not None: - l1_loss += self.l1_criterion(after_outs, ys) - duration_loss = self.duration_criterion(d_outs, ds) - pitch_loss = self.mse_criterion(p_outs, ps) - energy_loss = self.mse_criterion(e_outs, es) - - # make weighted mask and apply it - if self.use_weighted_masking: - out_masks = make_non_pad_mask(olens).unsqueeze(-1) - out_weights = out_masks.cast( - dtype=paddle.float32) / out_masks.cast( - dtype=paddle.float32).sum(axis=1, keepdim=True) - out_weights /= ys.shape[0] * ys.shape[2] - duration_masks = make_non_pad_mask(ilens) - duration_weights = (duration_masks.cast(dtype=paddle.float32) / - duration_masks.cast(dtype=paddle.float32).sum( - axis=1, keepdim=True)) - duration_weights /= ds.shape[0] - - # apply weight - - l1_loss = l1_loss.multiply(out_weights) - l1_loss = l1_loss.masked_select( - out_masks.broadcast_to(l1_loss.shape)).sum() - duration_loss = (duration_loss.multiply(duration_weights) - .masked_select(duration_masks).sum()) - pitch_masks = duration_masks.unsqueeze(-1) - pitch_weights = duration_weights.unsqueeze(-1) - pitch_loss = pitch_loss.multiply(pitch_weights) - pitch_loss = pitch_loss.masked_select( - pitch_masks.broadcast_to(pitch_loss.shape)).sum() - energy_loss = energy_loss.multiply(pitch_weights) - energy_loss = energy_loss.masked_select( - pitch_masks.broadcast_to(energy_loss.shape)).sum() - - return l1_loss, duration_loss, pitch_loss, energy_loss