From fd009bb088d664bfc100cad48a0290b8c96783c4 Mon Sep 17 00:00:00 2001 From: chenfeiyu Date: Fri, 6 Mar 2020 06:53:50 +0000 Subject: [PATCH 1/2] use os.path instead of pathlib. --- parakeet/datasets/ljspeech.py | 10 ++++------ parakeet/models/waveflow/data.py | 3 ++- 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/parakeet/datasets/ljspeech.py b/parakeet/datasets/ljspeech.py index 62209e9..3ab8ac9 100644 --- a/parakeet/datasets/ljspeech.py +++ b/parakeet/datasets/ljspeech.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -from pathlib import Path +import os import numpy as np import pandas as pd import librosa @@ -27,13 +27,11 @@ from ..data.batch import TextIDBatcher, SpecBatcher class LJSpeech(DatasetMixin): def __init__(self, root): super(LJSpeech, self).__init__() - assert isinstance(root, ( - str, Path)), "root should be a string or Path object" - self.root = root if isinstance(root, Path) else Path(root) + self.root = root self.metadata = self._prepare_metadata() def _prepare_metadata(self): - csv_path = self.root.joinpath("metadata.csv") + csv_path = os.path.join(self.root, "metadata.csv") metadata = pd.read_csv( csv_path, sep="|", @@ -51,7 +49,7 @@ class LJSpeech(DatasetMixin): """ fname, raw_text, normalized_text = metadatum - wav_path = self.root.joinpath("wavs", fname + ".wav") + wav_path = os.path.join(self.root, "wavs", fname + ".wav") # load -> trim -> preemphasis -> stft -> magnitude -> mel_scale -> logscale -> normalize wav, sample_rate = librosa.load( diff --git a/parakeet/models/waveflow/data.py b/parakeet/models/waveflow/data.py index 0c1e914..83438f7 100644 --- a/parakeet/models/waveflow/data.py +++ b/parakeet/models/waveflow/data.py @@ -12,6 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. +import os import random import librosa @@ -32,7 +33,7 @@ class Dataset(ljspeech.LJSpeech): def _get_example(self, metadatum): fname, _, _ = metadatum - wav_path = self.root.joinpath("wavs", fname + ".wav") + wav_path = os.path.join(self.root, "wavs", fname + ".wav") loaded_sr, audio = read(wav_path) assert loaded_sr == self.config.sample_rate From 0b2c6329809dffc85163ef7802107ad6fdf60ec5 Mon Sep 17 00:00:00 2001 From: chenfeiyu Date: Fri, 6 Mar 2020 14:35:21 +0800 Subject: [PATCH 2/2] refine doc --- doc/data.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/doc/data.md b/doc/data.md index 5f4099a..5584a54 100644 --- a/doc/data.md +++ b/doc/data.md @@ -44,13 +44,13 @@ Note that filter is applied to all the examples in the base dataset when initial ### CacheDataset -By default, we preprocess dataset lazily in `DatasetMixin.get_example`. An example is preprocessed only only requested. But `CacheDataset` caches a base dataset (preprocesses all examples and cache s them eagerly beforehand). When preprocessing the dataset is slow, you can use Cachedataset to speed it up, but caching may consume a lot of RAM if the dataset is large. +By default, we preprocess dataset lazily in `DatasetMixin.get_example`. An example is preprocessed only only requested. But `CacheDataset` caches the base dataset lazily, so each example is processed only once when it is first requested. When preprocessing the dataset is slow, you can use Cachedataset to speed it up, but caching may consume a lot of RAM if the dataset is large. -Finally, if preprocessing the dataset is slow and the processed dataset is too large to cache, you can write your own code to save them into files or databases and defines a Dataset to load them. Dataset is flexible, so you can create your own dataset painlessly. +Finally, if preprocessing the dataset is slow and the processed dataset is too large to cache, you can write your own code to save them into files or databases, and then define a Dataset to load them. `Dataset` is flexible, so you can create your own dataset painlessly. ## DataCargo -`DataCargo`, like `Dataset`, is an iterable of batches. We need datacargo because in deep learning, batching examples into batches exploits the computational resources of modern hardwares. You can iterate over it by `iter(datacargo)` or `for batch in datacargo`. It is an iterable but not an iterator, in that in can be iterated more than once. +`DataCargo`, like `Dataset`, is an iterable of batches. We need datacargo because in deep learning, batching examples into batches exploits the computational resources of modern hardwares. You can iterate it by `iter(datacargo)` or `for batch in datacargo`. `DataCargo` is an iterable but not an iterator, in that in can be iterated more than once. ### batch function @@ -104,7 +104,7 @@ Dataset --> Iterable[Example] | iter(Dataset) -> Iterator[Example] DataCargo --> Iterable[Batch] | iter(DataCargo) -> Iterator[Batch] ``` - In order to construct an iterator of batches from an iterator of examples, we construct a DataCargo from a Dataset. +In order to construct an iterator of batches from an iterator of examples, we construct a DataCargo from a Dataset.