From e53b9a0745794ea5a043c39f3778e015be86690f Mon Sep 17 00:00:00 2001 From: chenfeiyu Date: Mon, 11 Jan 2021 17:14:48 +0800 Subject: [PATCH 01/18] fix: the condition to init DataParallel --- examples/waveflow/train.py | 2 +- examples/wavenet/train.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/examples/waveflow/train.py b/examples/waveflow/train.py index 443cc8b..c64ace6 100644 --- a/examples/waveflow/train.py +++ b/examples/waveflow/train.py @@ -46,7 +46,7 @@ class Experiment(ExperimentBase): n_mels=config.data.n_mels, kernel_size=config.model.kernel_size) - if self.parallel > 1: + if self.parallel: model = paddle.DataParallel(model) optimizer = paddle.optimizer.Adam( config.training.lr, parameters=model.parameters()) diff --git a/examples/wavenet/train.py b/examples/wavenet/train.py index 166e23d..51d000a 100644 --- a/examples/wavenet/train.py +++ b/examples/wavenet/train.py @@ -49,7 +49,7 @@ class Experiment(ExperimentBase): loss_type=config.model.loss_type, log_scale_min=config.model.log_scale_min) - if self.parallel > 1: + if self.parallel: model = paddle.DataParallel(model) lr_scheduler = paddle.optimizer.lr.StepDecay( From 7c5e98dfb3a93cb93a83b77127e1bee9977afae6 Mon Sep 17 00:00:00 2001 From: chenfeiyu Date: Mon, 11 Jan 2021 17:24:46 +0800 Subject: [PATCH 02/18] fix a bug when using a method other than forward with DataParallel --- examples/wavenet/train.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/examples/wavenet/train.py b/examples/wavenet/train.py index 51d000a..b62e4a3 100644 --- a/examples/wavenet/train.py +++ b/examples/wavenet/train.py @@ -119,7 +119,7 @@ class Experiment(ExperimentBase): mel, wav, audio_starts = batch y = self.model(wav, mel, audio_starts) - loss = self.model.loss(y, wav) + loss = self.model_core.loss(y, wav) loss.backward() self.optimizer.step() iteration_time = time.time() - start @@ -141,7 +141,7 @@ class Experiment(ExperimentBase): valid_losses = [] mel, wav, audio_starts = next(valid_iterator) y = self.model(wav, mel, audio_starts) - loss = self.model.loss(y, wav) + loss = self.model_core.loss(y, wav) valid_losses.append(float(loss)) valid_loss = np.mean(valid_losses) self.visualizer.add_scalar( From c2a279c433acff029942f7c42493fb56cc1fe725 Mon Sep 17 00:00:00 2001 From: iclementine Date: Wed, 13 Jan 2021 11:06:15 +0800 Subject: [PATCH 03/18] add documentation sections --- doc/source/advanced.rst | 31 +++++++++ doc/source/conf.py | 3 + doc/source/design.rst | 4 ++ doc/source/index.rst | 35 +++++++++- doc/source/install.rst | 85 +++++++++++++++++++++++ doc/source/tutorials.rst | 65 +++++++++++++++++ docs/{installation_cn.md => install.rst} | 72 ++++++++++--------- images/logo-small.png | Bin 0 -> 33681 bytes parakeet/data/__init__.py | 2 + 9 files changed, 261 insertions(+), 36 deletions(-) create mode 100644 doc/source/advanced.rst create mode 100644 doc/source/design.rst create mode 100644 doc/source/install.rst create mode 100644 doc/source/tutorials.rst rename docs/{installation_cn.md => install.rst} (52%) create mode 100644 images/logo-small.png diff --git a/doc/source/advanced.rst b/doc/source/advanced.rst new file mode 100644 index 0000000..dccb98e --- /dev/null +++ b/doc/source/advanced.rst @@ -0,0 +1,31 @@ +====================== +Advanced Usage +====================== + +This sections covers how to extend parakeet by implementing you own models and +experiments. Guidelines on implementation are also elaborated. + +Model +------------- + +As a common practice with paddlepaddle, models are implemented as subclasse +of ``paddle.nn.Layer``. More complicated models, it is recommended to split +the model into different components. + +For a encoder-decoder model, it is natural to split it into the encoder and +the decoder. For a model composed of several similar layers, it is natural to +extract the sublayer as a seperate layer. + +There are two common ways to define a model which consists of several modules. + +#. + + +Data +------------- + +Config +------------- + +Experiment +-------------- \ No newline at end of file diff --git a/doc/source/conf.py b/doc/source/conf.py index dd4a270..3b652e4 100644 --- a/doc/source/conf.py +++ b/doc/source/conf.py @@ -48,6 +48,7 @@ extensions = [ "sphinx_rtd_theme", 'sphinx.ext.mathjax', 'numpydoc', + 'sphinx.ext.autosummary', ] # Add any paths that contain templates here, relative to this directory. @@ -63,8 +64,10 @@ exclude_patterns = [] # The theme to use for HTML and HTML Help pages. See the documentation for # a list of builtin themes. # + html_theme = "sphinx_rtd_theme" + # Add any paths that contain custom static files (such as style sheets) here, # relative to this directory. They are copied after the builtin static files, # so a file named "default.css" will overwrite the builtin "default.css". diff --git a/doc/source/design.rst b/doc/source/design.rst new file mode 100644 index 0000000..6e7fb11 --- /dev/null +++ b/doc/source/design.rst @@ -0,0 +1,4 @@ +============================== +Design of Parakeet +============================== + diff --git a/doc/source/index.rst b/doc/source/index.rst index 389f291..08eb577 100644 --- a/doc/source/index.rst +++ b/doc/source/index.rst @@ -3,14 +3,43 @@ You can adapt this file completely to your liking, but it should at least contain the root `toctree` directive. -Welcome to parakeet's documentation! +Parakeet ==================================== +``parakeet`` is a deep learning based text-to-speech toolkit built upon ``paddlepaddle`` framework. It aims to provide a flexible, efficient and state-of-the-art text-to-speech toolkit for the open-source community. It includes many influential TTS models proposed by `Baidu Research `_ and other research groups. + +``parakeet`` mainly consists of components below. + +#. Implementation of models and commonly used neural network layers. +#. Dataset abstraction and common data preprocessing pipelines. +#. Ready-to-run experiments. + .. toctree:: - :maxdepth: 2 - :caption: Contents: + :caption: Getting started + :maxdepth: 1 + install + tutorials +.. toctree:: + :caption: Design of Parakeet + :maxdepth: 1 + + advanced + design + +.. toctree:: + :caption: Documentation + :maxdepth: 1 + + parakeet.audio + parakeet.data + parakeet.datasets + parakeet.frontend + parakeet.modules + parakeet.models + parakeet.training + parakeet.utils Indices and tables ================== diff --git a/doc/source/install.rst b/doc/source/install.rst new file mode 100644 index 0000000..3758ef7 --- /dev/null +++ b/doc/source/install.rst @@ -0,0 +1,85 @@ +============= +Installation +============= + + +Install PaddlePaddle +------------------- +Parakeet requires PaddlePaddle as its backend. Not that 2.0rc or newer versions +of paddle is required. + +Since paddlepaddle has multiple packages depending on the device (cpu or gpu) +and the dependency libraries, it is recommended to install a proper package of +paddlepaddle with respect to the device and dependency library versons via +pip. + +Installing paddlepaddle with conda or build paddlepaddle from source is also +supported. Please refer to `PaddlePaddle installation `_ for more details. + +Example instruction to install paddlepaddle via pip is listed below. + +**PaddlePaddle with gpu** + +.. code-block:: bash + + python -m pip install paddlepaddle-gpu==2.0.0rc1.post101 -f https://paddlepaddle.org.cn/whl/stable.html + python -m pip install paddlepaddle-gpu==2.0.0rc1.post100 -f https://paddlepaddle.org.cn/whl/stable.html + + +**PaddlePaddle with cpu** + +.. code-block:: bash + + python -m pip install paddlepaddle==2.0.0rc1 -i https://mirror.baidu.com/pypi/simple + + +Install libsndfile +------------------- + +Experimemts in parakeet often involve audio and spectrum processing, thus +``librosa`` and ``soundfile`` are required. ``soundfile`` requires a extra +C library ``libsndfile``, which is not always handled by pip. + +For windows and mac users, ``libsndfile`` is also installed when Installing +``soundfile`` via pip, but for linux users, installing ``libsndfile`` via +system package manager is required. Example commands for popular distributions +are listed below. + +.. code-block:: + + # ubuntu, debian + sudo apt-get install libsndfile1 + + # centos, fedora, + sudo yum install libsndfile + + # openSUSE + sudo zypper in libsndfile + +For any problem with installtion of soundfile, please refer to +`SoundFile `_. + +Insrall Parakeet +------------------ + +There are two ways to install parakeet according to the purpose of using it. + +#. If you want to run experiments provided by parakeet or add new models and +experiments, it is recommended to clone the project from github +(`Parakeet `_), and install it in +editable mode. + + .. code-block:: bash + + git clone https://github.com/PaddlePaddle/Parakeet + cd Parakeet + pip install -e . + + +#. If you only need to use the models for inference by parakeet, install from +pypi is recommended。 + + .. code-block:: bash + + pip install paddle-parakeet + diff --git a/doc/source/tutorials.rst b/doc/source/tutorials.rst new file mode 100644 index 0000000..99e7fcb --- /dev/null +++ b/doc/source/tutorials.rst @@ -0,0 +1,65 @@ +=========== +Tutorials +=========== + +Basic Usage +------------------- + +Pretrained models are provided in a archive. Extract it to get a folder like this:: + + checkpoint_name/ + ├──config.yaml + └──step-310000.pdparams + +The ``config.yaml`` stores the config used to train the model, the ``step-N.pdparams`` is the parameter file, where N is the steps it has been trained. + +The example code below shows how to use the models for prediction. + +text to spectrogram +^^^^^^^^^^^^^^^^^^^^^^ + +The code below show how to use a transformer_tts model. After loading the pretrained model, use ``model.predict(sentence)`` to generate spectrogram (in numpy.ndarray format), which can be further used to synthesize waveflow. + +>>> import parakeet +>>> from parakeet.frontend import English +>>> from parakeet.models import TransformerTTS +>>> from pathlib import Path +>>> import yacs + +>>> # load the pretrained model +>>> frontend = English() +>>> checkpoint_dir = Path("transformer_tts_pretrained") +>>> config = yacs.config.CfgNode.load_cfg(str(checkpoint_dir / "config.yaml")) +>>> checkpoint_path = str(checkpoint_dir / "step-310000") +>>> model = TransformerTTS.from_pretrained( +>>> frontend, config, checkpoint_path) +>>> model.eval() + +>>> # text to spectrogram +>>> sentence = "Printing, in the only sense with which we are at present concerned, differs from most if not from all the arts and crafts represented in the Exhibition" +>>> outputs = model.predict(sentence, verbose=args.verbose) +>>> mel_output = outputs["mel_output"] + +vocoder +^^^^^^^^^^ + +Like the example above, after loading the pretrained ConditionalWaveFlow model, call ``model.predict(mel)`` to synthesize waveflow (in numpy.ndarray format). + +>>> import soundfile as df +>>> from parakeet.models import ConditionalWaveFlow + +>>> # load the pretrained model +>>> checkpoint_dir = Path("waveflow_pretrained") +>>> config = yacs.config.CfgNode.load_cfg(str(checkpoint_dir / "config.yaml")) +>>> checkpoint_path = str(checkpoint_dir / "step-2000000") +>>> vocoder = ConditionalWaveFlow.from_pretrained(config, checkpoint_path) +>>> vocoder.eval() + +>>> # synthesize +>>> audio = vocoder.predict(mel_output) +>>> sf.write(audio_path, audio, config.data.sample_rate) + +For more details on how to use the model, please refer the documentation. + + + diff --git a/docs/installation_cn.md b/docs/install.rst similarity index 52% rename from docs/installation_cn.md rename to docs/install.rst index 030b721..4f6191e 100644 --- a/docs/installation_cn.md +++ b/docs/install.rst @@ -1,57 +1,63 @@ -# 安装 - -[TOC] +============= +安装 +============= -## 安装 PaddlePaddle - -Parakeet 以 PaddlePaddle 作为其后端，因此依赖 PaddlePaddle，值得说明的是 Parakeet 要求 2.0 及以上版本的 PaddlePaddle。你可以通过 pip 安装。如果需要安装支持 gpu 版本的 PaddlePaddle，需要根据环境中的 cuda 和 cudnn 的版本来选择 wheel 包的版本。使用 conda 安装以及源码编译安装的方式请参考 [PaddlePaddle 快速安装](https://www.paddlepaddle.org.cn/install/quick/zh/2.0rc-linux-pip). +安装 PaddlePaddle +------------------- +Parakeet 以 PaddlePaddle 作为其后端，因此依赖 PaddlePaddle，值得说明的是 Parakeet 要求 2.0 及以上版本的 PaddlePaddle。你可以通过 pip 安装。如果需要安装支持 gpu 版本的 PaddlePaddle，需要根据环境中的 cuda 和 cudnn 的版本来选择 wheel 包的版本。使用 conda 安装以及源码编译安装的方式请参考 `PaddlePaddle 快速安装 `_. **gpu 版 PaddlePaddle** -```bash -python -m pip install paddlepaddle-gpu==2.0.0rc0.post101 -f https://paddlepaddle.org.cn/whl/stable.html -python -m pip install paddlepaddle-gpu==2.0.0rc0.post100 -f https://paddlepaddle.org.cn/whl/stable.html -``` +.. code-block:: bash + + python -m pip install paddlepaddle-gpu==2.0.0rc1.post101 -f https://paddlepaddle.org.cn/whl/stable.html + python -m pip install paddlepaddle-gpu==2.0.0rc1.post100 -f https://paddlepaddle.org.cn/whl/stable.html + **cpu 版 PaddlePaddle** -```bash -python -m pip install paddlepaddle==2.0.0rc0 -i https://mirror.baidu.com/pypi/simple -``` +.. code-block:: bash -## 安装 libsndfile + python -m pip install paddlepaddle==2.0.0rc1 -i https://mirror.baidu.com/pypi/simple -因为 Parakeet 的实验中常常会需要用到和音频处理，以及频谱处理相关的功能，所以我们依赖 librosa 和 soundfile 进行音频处理。而 librosa 和 soundfile 依赖一个 C 的库 libsndfile, 因为这不是 python 的包，对于 windows 用户和 mac 用户，使用 pip 安装 soundfile 的时候，libsndfile 也会被安装。如果遇到问题也可以参考 [SoundFile](https://pypi.org/project/SoundFile). + +安装 libsndfile +------------------- + +因为 Parakeet 的实验中常常会需要用到和音频处理，以及频谱处理相关的功能，所以我们依赖 librosa 和 soundfile 进行音频处理。而 librosa 和 soundfile 依赖一个 C 的库 libsndfile, 因为这不是 python 的包，对于 windows 用户和 mac 用户，使用 pip 安装 soundfile 的时候，libsndfile 也会被安装。如果遇到问题也可以参考 `SoundFile `_. 对于 linux 用户，需要使用系统的包管理器安装这个包，常见发行版上的命令参考如下。 -```bash -# ubuntu, debian -sudo apt-get install libsndfile1 +.. code-block:: -# centos, fedora, -sudo yum install libsndfile + # ubuntu, debian + sudo apt-get install libsndfile1 -# openSUSE -sudo zypper in libsndfile -``` + # centos, fedora, + sudo yum install libsndfile -## 安装 Parakeet + # openSUSE + sudo zypper in libsndfile +安装 Parakeet +------------------ + 我们提供两种方式来使用 Parakeet. -1. 需要运行 Parakeet 自带的实验代码，或者希望进行二次开发的用户，可以先从 github 克隆本工程，cd 仅工程目录，并进行可编辑式安装（不会被复制到 site-packages, 而且对工程的修改会立即生效，不需要重新安装），之后就可以使用了。 +#. 需要运行 Parakeet 自带的实验代码，或者希望进行二次开发的用户，可以先从 github 克隆本工程，cd 仅工程目录，并进行可编辑式安装（不会被复制到 site-packages, 而且对工程的修改会立即生效，不需要重新安装），之后就可以使用了。 - ```bash - # -e 表示可编辑式安装 - pip install -e . - ``` + .. code-block:: bash -2. 仅需要使用我们提供的训练好的模型进行预测，那么也可以直接安装 pypi 上的 wheel 包的版本。 + # -e 表示可编辑式安装 + pip install -e . + + +#. 仅需要使用我们提供的训练好的模型进行预测，那么也可以直接安装 pypi 上的 wheel 包的版本。 + + .. code-block:: bash + + pip install paddle-parakeet - ```bash - pip install paddle-parakeet - ``` diff --git a/images/logo-small.png b/images/logo-small.png new file mode 100644 index 0000000000000000000000000000000000000000..fa2b7f36556abed7cc1dcecc73c0f15600f3e6ef GIT binary patch literal 33681 zcmd2?Ra;z5vqggi3k(E@kl+>w?j9sKGdM$V5AGh^-Q5C%1$PZPxVyXC;Bt7sUvZxO z^u^v6UENi+t5>b64pmZ+#zZGZhl7K|l=&{90tW}r^xr^z_x8ojE6VBZh1OhLT**>g zTHMyk)?W3OfuXUek?l`o6=|`ryu3VcaCEVTV6e(}Cb|KG4`A@X2qQhZy^BgnNQ4U5 zr@OON{gX1rF=w;8A(Pm z;qW`&lnbsx7P0!%MoGD!5|^D&3U}ESlv`LBNE7N!i7KnYC@WIy;4NybY~n5z>QBv9 zpaKxMii0`gN4$@C|M8>92A1+OFAix|#7|S4pQye#>C|1F>m3PxemmG=!v)EH27XwG zh_7nDQ_`pq1AK)Ld)j#i3nh(}3_qqu5Pd|wfae{RH7Ec(*1u^E?bmlr2RJwky#EF~ zTw2CQI5-hF83|D}SNP*Llnf(jClP+W#eyZIeYaiK7@Fjo-E-3 zNlZA^I}R#TY1L;AN^MI^&)QCOWa$z9zdgY$2zBBhr&2c>2}-EWNfv9zF9XO#OHT1u zK32%>NC{@-Y}D&BYefpj>wZzK-M`3_6@ISf(uKvRfVOmQotBdXQ+Ka zs=tvRU#FWC$v;OvTD@E0w>wZI%3be0cGqcJUwgaU-o#mJE-q-fcH;g3obzy>i>?cr zWA+*vyB&@jFR~&3hx)mbFQiK-MZNrG6iTjXS)VmqF_Wch*~F0vRY-d9Kz8%w_%SK41?bC5c zmLG|wVykkJ4SgZ1JTn4XRS`ik5u|RH-^Vq`($q)IpAh>cN?PxEeterU;iPHdf~^t2@gz^f7%hEg%xY%vk< z?WoTpuNEd^iu3{O)MwP^Fza&ya9Hw!fd;hpBmEH-aT6}G^N%}XK0V1JXxQ8AJi3d( zL0kGfJdDao^j=vjnx6Yr;%0zEjja~i)#=q`H@b7bl9}8m>pDtiwhNdG@!|3OkThsM za%AdwUC0s65Iel+EZ)=bQt*v#VT#bgDWZVUghLY8;0mjx7U178_0@5DR+8vl8O$x; zThcyLw(yNDGuI7!wauD&!9r+X_bV(4|7BXgsov?M=VpW6RTwgcH1QdZj$5$RYtoRM zwl1=Uof9%99ZrJCKlT&@{y!F?gZv2`WzT=VNFbI@-BnnwoC}^M*W6QST62~r+Os4@ z3)|Nna~N)whbId@_aipZDcAg`ag%e}EUyzlq!WgR9xX{e1`;^Q;~e)_sPB$nzYDx) zU~a24K@NWWb|f#noaPXI>KOJnaIT-GLi!aI2r+!f6OX~=pDu1>638P^74J~1o z)5-}Rgyul}>wnYl7dO8=`ZyNBjd!5^7U;_6q_0M*LO-HIPJ!b>`WpS6m#ySuy(sJU z%4j7{Q~#OTV=x5H%Jp9e=Q$f;7u-UcM-{-lQMIkWYA;(YA@~=C1Vi6uTU?zxI!%s5*2c zLGe~;)?QM_-uVW2?=MaBe?x}JCd%$kLozE&yhhP%HVixViKU;ol3WsZ)xpS;T?ZW1 z&Nr|VZh@m+<__0g3UrqAM)hXd;T|4@gPu%y3cY%JW3QD$pKG5&`njPaeCx@AoQV=8 zo?jqZR}S~s2#IOFX^HZ`oy@M&w7O^2HOPT|=pzTSiv!7C{^QZHuvt*U>Uo*IP20sW zAt7l&w>S-gxtwc=@DTG{t4Z&G#t}xfu}1}$WP=U+bGH>!hQ8lmIR$LqdqcWYMf*X7 zYJ8bMOm$&GOs(@=#&+c1YgvvbP}{+*mf|Y+arLN=Tc3txQwX+b?bJ)4TA_=65|gF- z<<_a=Duu(xwa42^%ibx3ItHi_e2%E$at|S=2E!c8{ZSw)+V6v0Wf~*m@Hk6RaKd-k zmD%^tEn#xaK*y?mH|$5@3YYsUM-Q`fRheL)rWnm8QkzW1rNeCN=8fYqtCxi)C^m3O zRS<>i&2g}-JX_rLU3wutioYALAOHQC_WNu=d?^2IY9eN$US3|Z*oq@y=!*P?e6D7D z&|l28`$yK$3PW-(n%+9}ZuF&g!ECkuBFMB=vOB%cgM6qbPL^Vw>TZ74>s8%0fyDP! zx>QIEat2p=z&&Jj`mgaPv0^0Wa~O7tiF1n5WV!< zqzU4%b-jP(HqRfp!rkFE9?rCE)J%-x+ zL^DG#E9TAjYyF4=8y#b3Cw&rDgf6TA;SLV!zDn@i8|94p5?le**Vv*(!w z{f_6oJ(j}Cgt6uS4!h3mk!HxtwwrzWNQXnUyq2!!U!ABamRq%q?w>haeQa^D1CiJy zrMm=K?tZnitn?G{Vz%9zp)2$Q0}?jZ5z)!FuzDN(b@x2=AJ@a-Z)-)oOSNj=RF5t- zt*ev)?E1DHsihTsnH`<%D+^fZLLCYk17z7|e`rSi8q>?KD_*O@c_ouO^Xh@_`@e|)3Av6fYkP?%Xt0bt8@T4J z*6B6+I6-yhtWdYc`Vn)gJs*CDTrHW~W_f@0x`?|;TJTW$G{q3uh^Htpk?FRZdMZxw z5JN&7XR3b^{Ozo9uEqV#zp;6eAXr`1%LL}G0thVL^PywZ)|rZ4uKFc$IOYT&Yaf-LnOL6_i^ zWPDu;xP%yVzR}9C#2_LKqtmmbh{(vnOg7HBwY4(>X1>X`J;5_OR#$lMMp{J(u-twx z(h!7=4ox=mxWnxGUdy|M`| zmX)xW$2KcLBm_j zzvC267(slp&w!)rH#|>AgxQv(He6TUIoTuyfL&$3>WJ=BQ{0D;kdiuT*tK0{o;iZ_ z+>8^OJ=XnDmaQ9Bfb{8DHNfnxUGnFjn!^va@9TO^zVwpgxUQ23Kun!JVfL;EQzg-+ zu~alICPILMkjoNcHkpI<|E(tMst=;?p;eysoaxfPp}42QQj_;bc6{(?d%WxVQs!B{ zRH@7s3>Lwje|@NYUCeZxlsTPNr8s42-p|WZDXHyXaY<=7E<7aqVNxTQSy_<-Hoh;! zvM9Ahb^|B7sbsBL%ym2;sqYm9Bktz%@2eq{J>y>{L4~L)p2j*-CKRe`(&7r#u4{(b znNQlBNE;zx6X8am`O+^adHcydtgfrP9-W^}*w~wKph?U}pHj(NsG@kmPB8mBh7zCf?+Sq<5Xf3YvE1H-(42Xi=EDI6TkW_oDCJK-Uk*G1tTX9 zVQUK>s*%--CqV-6aj4A{rj7WkoB;3Fimb;3AgpWO4rT*dJtXg?4lGm zta&ZJJV2__4k$yQrptBNA(zra8y|Y8jrXkZl)N7yv6Vx)2$|@T|4(2oH+kK8Vs)hJ z^lux-0*6MDkq~eK6>a8`WL*X)nk0o#^o@~AO_uy>@99+bOD+G#Z@lL34maU?*Pm|p zd|vi=4TY5=+YNOTUnht;H$AYx?{c0F>7*JE$N9E#k=r8SIeyDHhhb=yFW%ZTGV5Me zQ3&J4#LrhlfKmt7N8rr3t29P0I(t8K_J~@-Z$M`iV}ArRCtn@|zOR!;<1= zC~L>E;0BuFzQt=0cWal|7RlN#*ux8j{j+q|m*jAq3Qs(AuM@cZ*3@b7QNqG{EAtG8 zz{R8i)@JY5U`Q4>uhx|L5QQ4j7|v_qx)3}l(eG~+L} zUa<-LUI=ZIAhyn?QKfl(@M^vG+2GuASr+(9@>l#%wje740t3d5P*)xOa<_ZdGp|ui zMbEp$7O(PyXAwzCPc@XGXYr-82gu7cs7~wVclS&V67}G7L5$0nf@2~TED3tnr-I}x zRkyaMMW%$EFEv4{oZaX7r2yV<0TDAhu&XQOzr^a`&P4*W8Pr2%?zgC=<)`=J!=)WU#_nvgs!4)zq3UX|Egw`bnH zU-`ssj5=+m-bj?S9m~2&snVI}tS!T)3l$Ze;ZN+DO?x*i5Fb=;RlZ;UMeOLm)(zF% zE@}2Ws-Jc`!yxH&42RAG0eZ^eeX3&zw_OjZ?vcSFH9(g(2RlKHRk_09L-g`ZS($?z zOhGL>CrMuW&xha(NnHSHz^gF+yu~Tm+?xC7J%gMr%IE*E4^dH#b#PK|=JT!)`0%)C zX~XA%d!%sx@*kG)3N;o)1Ut!)j^Zanw-sahQ{l+wr%4lu??R*a$o7XmsJT z!N?Q)+PLr-m{MbR)np-pSaA4I(d^vf@*ybr%f4hw6sZWv0o&mjrnuye*XHvdgOO&p zeAjcO7~`|8m>#G3afB>9ZGaB@HV-6Aom@lDK7_>VNe&5}f|R|E)6aPjFo1KVmxp@I z9O0tQTv*1=2y1U-d-M+^h3qRT2J$ODwmp^dkGka_7BoB)yrsrd?~Ti6TysyH19mv23EipD&w+#5;&EScTdSW9RGdmuYKJ7CND zQ(K%aXhtVS^@}z^WcTkM1uyc$JEkX42pJdN$`%IS_abQ1`zO*4zNZBY*%_H$?UJnf zy0V+VN@M#DbJ@o)Z@yB5+jl(KFd943!$HgOH{L*Og+cP=5hxB>KiK5{ea*sN%3eIM z8_Z0+prAJ8S|rdy?tmz?zChUOsM81TWO_iIaY~S!d$1(YhGD+?M_r1SE9%+zM|EJ@ zsN7Iezv-PAetBB>vv94Z_VUW_o;2be_WJ>PLmT5HtV@I#qN_i$!ovYMjNcQl`JN>D z3`C7Id-})P=aoB3R5p%o1RAZ1gp!kho=6+;s^hj98A?A z4rb1>{O5W9=m7Zvr{wuVza_XkET%VF1HR^-rP2*oZ<2i#&==FLb1{(oSFc##>>ueR zDNHE~VVpJ251C0^bX{qJmEF@D7c`&%{xc8lT+bn&f9Z+e{1S2gXe&X^e% zGganHc9%)_qT_=Y29t95k`8hZ=s0Oa^eV6~$@~)aJDnv%dBr{+yPW01mRcA0oi)g# z@Agb!&nZWB@%9ZNrYRg-)b;Nj?&PM)@df`KpwYRM|7qDvR%;$iRYIT%fEn5tr2oF6 z_+o}nLlzz^zG`tAWva3qm%Fm~I-dP898QGp2=cj@d}jh1H6P-x66LN;@WyGTD%RcV z*YN2o4VfLwWs$RT^!{1#S-FR5jT{%89`boUBPoB;KP?pwotHojN`vB4})l9G~KSso?^fhwP=Lf*$p9zyKuec86r-UL3zuNA!pTmn=f*oHt z#M=3@-48XUCEf*l4uW_z&SH|Vm*q?XXGmyLc+4=^BBLXkCj6+ltKV3cvqi>xJ$mex z-SHI5sEC#uR}O!|2sks&ZSSx|%hZXTzZ00=g&(y-KU$E=Q6kzU_>Epb1_+6I^u~h` zWe)kb8KPlGV%27*X7>~qMRI3Cw-bk-+kWsPmF>t>Kzn zyM6uj{R(VxNPwgg5u8s|f0lg#LpPt47yFlyl$`<$hm<3p;#h?EG`ob(_+5nej=@RQ z<`R_URYf0}+?;RG?tc)?NI7=`Es;(O!e-DBL@Q>jnw8;;E|h)apA@n2^3XHp-*K1@ zd{A^FlXM%69^%OONTIu@RZhj*otDo$L;hRgFL!GM73-ryt~@zUJ?Z^s__v+mo;=y@ zb}|0~cWmd{K-q$t4P z;KG=MuNA(bg2RC>qvu@!6%TO0hl}dCZWA;}YVC$IJQ6~(WY%maXs{tw%e8=4ev6FN zBK)_2p^6?)SC{ESNg~aB%=AN8U0{Y8Ln$e_+JQ*=?0i+U!Ql#vTY!yjokrCkVOBL3 zRHAla)f0Z%N(MMJ{fyDD{T}N{YwT#=eb=GpD2?~2+I{EW@v zP=>y*&Qj0;99tXzpge0RlJh=<`Qkqjh^Cj)xG4#{~@U`knwu!p9zh4-MI?FKiqs9D> zs=O!(40dm(VKkG39k?$;UDz%T2~_9G6v#46@jw&TfHELN_(=O!z{LXlcWqxyf`TtB zN7(g!%g@91KMaSP+QK^4?Uqo529G6$LVl2%Wctq^ZiBd<$?&A6p#g=T5L6O^zvo!F$&zT_Nee2EjqAaVD#p6T`dju~)bI zwjUUU@4d7Q%@E*n%UGvmFw~TV_y6|S9?_u)3z+T6mWeBVG*+s0Ql@AkRepKlKIPy0 z43%?qdK0DIUl_)p^(q|LDDYIU+=U)783$QOP@ma7u{j$R^i~UA9L^{$)prB-Pf7;Z zi-%1H%bNEmy(ZlaJ_k%vwU0NHkk;bWPFMVN>P5Zm@{CBW?3d2s)1qDd{e%1|epbL! zBgczZm=!N+`BL2{bDRFhhc}mvM;Zv(*`$phHv4Q|FjK%_lD+2rI-B0NLy*Szf~=TQ zwA(Z!2Vl(><7@=JYjh(oe^dFz^ zL-)LPI8U^0LKPx2IH)unD?hm?e@&1lvdk1_}9mVRheWLh~Yt=yJ>pZWlzVn z#i=8Iy|;7+nS|}Q;nGp4SqosBePD5_o*El|k2|1@kXJNbfGKA`{+n_A=l~)Wz2)4? zwCX{d_J38FaQW1=QtMGo(ox=GHbbVCU7&H~Y`h}x8PuM)UU9GOq2L^LAHj8O`07Q%AG_O&?*S%(7WaQ@ke*S_8 zn1mpktaYBxLm{U9{wJcxJuhq60;2$T8%UB89;mLy;A+2hm4xmLUf4*n9OSz6Xwf+- z!^^&%`N2DPL0J0lf9bK1*1Fu`J-2I>pWPk^ z?_|DZIWt7@77Btb9OyjuUucPpV+VvhJkNrXg--AFNh6rVr-6D)3AT-^@JU2hQu*IY z75hyo-L91lqUTGp z9>}BKcM|C!LbyeRWvarfPPOp3MHMlFZ5I-#x~Wi}Mc>4SDvQ`s;oRc#s={~bW946d zHjf@uFA|k3D6JB!tBag7;Dx*uGZDN?v%alL0%_hy9L^zu4f)FUj6MysTDxvsWFG$$ z{V$Bzk$zi*&EySz)vT>vF1t~rZC%G`wr5~aPu^|0(MCS32g*^9CSQf~w#J(1X$G7q z7It@;)@!vaKOeE!!`!o1`-5o~O%|)4-;GtctM$VL*GWff6tOJn(lx+TWkbXDgp#ZPA??i3?kqw$5KO@BCSx5^)t| zvu=Vc8$wsIe-!-L;#b4#a|gjJ+*bQ4f<6sc7Z(Y~Rz5Sg_?#eL#gi1^UY5W+$clabu*O zb>ZFuqHL-yIr;8jBwDOC%KQg?+ z_s!=P`h;%9HyYO5$OTb9yBEAS$btd$3=)UwXQV!HDsE^- zpRo{x4HrUIiE!d+JK4KZB1kQHbe>Yj0-BfjfDU_tmj`R)DZ|bTmmDwG=uLW%3xtUk z-q`k34$&XQ{Sq|Dn#EqC0@Uc`yI2jU72E@?kU9a41d%{E317Ohc8O$MW!`bQxS6`k z>i5P%k?;`4LwinR=slRt{TC(uVJ>*F@@bv>FomnsdPNY!O}6(x2k0~Go}?dpStf3x zIIqu{PAX<=d)gR#?ruhlETR|KiUh3y$EpcKHC^M@zLABvr;s4`FzAUIyaQ;JuOI+@ z-VpHZK*5-{!Y{LLixmyU2fNpiE$zHN=1RGje?itTqedM9dM=5t@5ko_xF=6tG;PJ5 z6RhXf_;=k&wknh@LqTWd?k#&q5#UdMsYXa=3plYykWQd|!7;HEn`L}{T-sgxszSZe zp2k~O3OTYvUP#07kE-9_U|pGWVn&s))C*Pn)x3kS&q{(-QOWf@_se-Px6i}!)%NbH zT(4u7+Hj92{h!_RoDAzCWg{u=RLLBSczO#mQh!FsgmTlQI2LR?m>ixYqyCgA^Q`_@ zYZqvHXWig7MQ?L8tGeStMA#S`OTFUi5Vc%E9i7${y^@Ex9};mYr3i(M64`l$2d_Nk z-!>M~mbpNa){dftHdz`RN8dH%L`gpxRnWT>H}u5X9CzIP$(v~#K+Vx0jRkrcP!=;g zNQ^^8)uHO^N9BW?m*y%JjS7l&COg^)W#-<;?Cb5UTNk+ff_qo?gD^k2Xpd+0xwY#O z(@a&{IhJ^>q@jDo1w0MXFLz67miRBM-U75zNTTtTwo8}boY=>`=GY$&E3+igb`7@z zy-mGsHm9Mf+K!tFF4J+^%Q6@h4*g86DaV??SLZiJQM$i|y&t)3^DMH@O@&)G=Cb*2OR6Uy z+9S49X(kJYv3-)TvOSIVferVo4UI)*F{camX^#t)dYiztdkIa^5eF|pf1XHrN~7UH zr~FRJ4u7#*vEaln-)LL*`s3#C+}c)?FcgvZow-*ADl?a!(wl6SE@@lpiiM^|G~=2A z9c&!1wI94(E;mV5mfJC7cfb<|&-FmAN|^Yawt8C3b~*4yt#YjIxzU$RdgD zt1_9`BLTf0mUlEEFk|zrrc82r53VUuB2!o>C|DI$&^ZE_2O$LvIxlyDP%?=sbfHE-U&TU!vQS6O-fnW#%%zjh(D4{t8$k zvGJ79iVUKT2O$Oh`kIL&{+UJnmVR>$4 z0)wm3Zo%eIv~_;|gyK1uTpxs`k;z`!!v)k$!ZkLldc??)!;UbQPteONPh)TX?buw!eHi!xB06wo*esf|_7a^QjWQ!DO! zJ}pPKn!EMbSzm%T?1f$1=uMR;NzG|;=J(oX(3vK!NL6QyduME*s<R^ylJ7{T{~3}z7Sry z2tUJLAIkm!zgD#e^>#2n`qZg@AYLhpC3KBkmHjk?$MPDOG^KsNC6AkWfZKS#mgQyB zmgEIA8I;MeF{m(~k`eycIb?+9Fa)n_V0`L#a@z#;cFta^g*E6z>p1=nu=U5|Ls06D zK*q*P10&bZU9p>Ol(l|gELb5Sv3c|od=V{xmp?|$#xMJqSTR{^v-hCrWV6w;^C$uT z7igJK6kmO*MXN0dUx8y~tvbJ|?ibyRae`LSJ4*s#e(1bGH-{D4R{G4Kfjr+a z&>u)=cK_m$T(`4r-7PiA_Gpbb^ht8&?#Y;7iKz#xACnZz#d@&GQD5d4uiX!pp9;56 zIW9RrCFz(@l9YeYaQijX6?QT;sC$SI=BmOz8Qb5mETc%~bQNuJ#9;lY>XJZr^-GLK z$)Ig*^NBEk5`b^D0hiT#5P8S>cBS^s8Cu;7fhyIZe)F(i15!V2<8-?8B#fOcPxj`o@o!^@~zqz%V#aV!N_;&7%6RlsIByq35 zO~n;Wl_@~akG$B)w##~|mUDVkQybr2sru=2V?!uftEibjj6b4lSU0UZxXwi{ub!;g zZFzv2VB#G+-mRx?6;V4fJjG$R`!a22%I)mvc-&<%lr@ZO^kMDt&ZwU?TN;UTt`nTg z<1cyHsj@x`CNffOdCm~^h<19G`X&y3+1~1DXqoU!NBUFYk{K=RsW<|H@>cab?$~WmDYak0&Zh=_NO}gH*TVcBlGz252|Bey% z2RKqS7*ATD!~~LtcHw@2Kmwy7N2vt(wdHY!;jWu@YtD+E$3D$PdcO`&W`Z>}HM4Ud zL#yuqkf&QchfToRP2s|GR)wPvs?W=VIN|69F?s8%!z1meuGSd|uj*;4hi7@Bpq|Mk zwNYC3=W6Pmz>+eVJrgdLb?Q0B`u@*r%=#y1xb`)ck=AyG;tfp~xlR?l?j4>hte{-O zP=9GcXxV8zNhIjRJ*dTPOS{=0z|i%1PXe~S4Ie5ngXW%7BF}68FyQ(N-?fwlecHOg z!e)Y=8Ga^hc`=?EdwT^4S_gfpiygV?WVUYDcQN7*A;M>{_$Iad5XtJ?lP|Y2m356f z*WDJ`)VQzkgu*d~y@L=<09!y zA>o%mHWU&C9c&F}CjdzH599E5hABv;o&4cnu{1GQ3g;uC)rw~g6PtKCvxX+|!%lh5 zgPONlCEd&(U&H=|{Wwdc$)gfcyQEI*oyl!P@l!rZIU97HkaZSY1?m^vPOVHksd@5T z$$NnEn$xZM^2VAaVf)&wtX)D)%acM~*|+&7WaFdaWsTVpJr1`{`%YLLT?Br$kCT^8 z1MyL9ctf`F>RJKA))K+>T13!QuG>UJl~L$3UKM2zvHe_H^O8pgQ|#}0N@Sdg=ACl% zGJDNa62AlfqTStWChJbOq!b3BhN2ihZ7tJ%6Q{4UNNhBh|Zq1vgvt z9+&U?^}U7EOg+Mud>6<=re9BS_fx=1oOq?uKw!uGCA!NVVx@A`Cc%h;2=e={)PFhy z_XdCC_x>RQuSnLm-M1HXt@yY47&L-NWv-N1-Ky1ej|&e3;eo`6cv8M#$wHos z-|)@SG`ou29g{09u$>C!WWnU&PM(S1>p+~FSZpoyK>PI)J)B?0CT?$}>E}O_gk9ZN zvS7e9vAzCPhGcac<_)@l-%Tp78MfzsQdiiGy?VfsMx#i5WPP<{_uf1lV&$5G~8eH)JrUZ^G4Wqax zeO%mI4G>8DPM2D#t~t+Mv2CZK?Gj4&Uk>ff3dNCUxR$?(l{lZj?(upGt zT(($W8c2u{PVqs%)J(Bh3sN!UY|v%V-iVyUzODk?^?wsOG`8ZMheI%*4f1)XD9D_Y zlfP&m8u~jB=$Fk&ZV$V0E%l!RAltt3yL8VlyRk)!puHhTS+E=~@s$;$ryL*ex=Ves z0EV7DAcSOrg&fW%?(fR6l`inDpUJzhtXhl}0hQqJy#SAL`x%WE<(hQS@3#z6b_*R6XhRAqpAzI6G`Ot!%sam zTj>FPtRy(Cb4P}!?fkFfCyRADi@Ske0IoN2aNrmy2FZNFe}()>wtZrF{?xu2A(!4} zjFt2G3HI2Xe_Fp2IBr`#OLCraP!^Benvw*mQP|~YKVU~v@8iaFVpvcn(x5UJB4pX- zzxQktMaz-c$Dv02aNUyuqhbvpG+vLsqj(I(H1es{Q?|N}Xd>}>;C_+*wl6SCf_f>= zvi!Dt0<7!guVC-cCx}Za!I{Ho);YAT0-m zM1?&`ReGsddvSt_wJ+`E%BJ+e)^(`l$Ct7y2qQwOdi3n9g4y!OzxAq2CXdJAlNEq+L30+~ zXfm+=I3c~qxbml5(spEDO3T(<<&^1U{EyB;#mm$su2B;{_*tu$YubJOqPs?c`He|V z3dj8Xe*Dg`_>g!UURE4rOt%ETS$BI829PE8BLB%cBb>tZj$jq}na@vhVo|}mn)dFn zK92#IQau+z>mL$_tscGm3YZq{IsMMXMc`@HQZ}fOE8Jp&7OlxNecNgb4?J>%J6`zO zbMkWhCN0U7e~^nU;}fnt_yenGjOM#e(L7RK+Uz~F^#s=#X+5Qz=GCNX1P!7#k+9^} z(~RKRDnZdxO*aw%!oO}f%wuLcO}>18iLLP0^QAPWUdg`Pt9bis!|{QmDCLK|qY=V^r(a-qp_!c9Rpq zJM?5o)8XrQR^>vpkh&fT>4l>Go&i*oGL#2kbo&9*EEVc_- zELJ_;6c&DcATbwiZ+)nkg2&0l#9Q1HR7`F`>NdVTEW6(Ig4SKrQ~l@OJaiK4hHK4R zwY*!@!X*t4+tpUsK)yD%4S$p2L{pQt0bcstEm*4#Gz<6Ws2))E0AwRZ_J1n_J})sO zBsC0RAaxe=GVtI@`6NuGZf{csza+%(Z#H1sVc;0zW)jiHaBdTpgVtWyal@7W^vtGT zZPSO)o4=8xg?>2H=NodfLd`_9{ zy@vxto+UM-KAg*G;)Dk#_n^3a1$`O_uze80-AsD&{{|1=iUv~ZdgneHWIP)#_xYR5 zd9|IrV7OOg=B*X?w_h1c_4oMnsZd7$h}7>lFW2xSXG2x70V7jcgiEBx%!<~+Ia45&k{8^a(tyCd51(dy#!Ip&>VuXw%;IRzVSlV!hp8o=Tt=``2BZHV?C)j zZfxi>Ua}}VGNWlq^auBAlhO&&%Q?w%65Z4M7x^-W)Z!_uhCc*ggxJfJO(2w9ui`H)t-I zi1xRCwkcA3LtXr=$B0*z&1EVI_XPIcx}?77U-9S&@OT`vHt-H@5sa$;g`!E~4}ypr z8mwI2ZQm7V@kWx5*u=%wyakKJ}yS97eo zFh%@<9WU}K3~%HJAH)*_fIR_Y7&&dJ`P9(AMtdd{<{0OTcg(Wbs0MidGPW9-M6!t_ zKS&aoBa2QJRd;rh95=u#H7dhyZXz$C3}kW?su%n)As&H(wbrouifS%*u6pc0tt#gW z=uU!Tq!6x&dYGR(^Rmldd1xMvc1!WPuf@=2z>B40+}9*g6pv<2ameb^F>x=*pQ`#W z_29WO{`KdXgFlAE4E|f^c>T2>X{A=%uGCIH-JKKO4qMHj4JNa`Ih=|SL4&^$V%7ZE zKtApj<^BPgGs&4(Qim?BFTh%?Jy$KOt`|w8<~slN{I$&6ToLk&dEiY!1JuDJQd(~f z+f(TNvxu%Tra11&<yk} ze{+CE%jxM^ZCYl=+qRD#eAXJ~I;>8=-x&L4y`y=jFZ7-~hDRHNo&Fc#o$f$h7Jh3* z_TTX>fk@uYD0hqZC=-E7ydR1G&V7XlEsiHMAZdT|ujiu0Odwuhu})B?aNG3#c8Q?j zws+qG|5;;#1$2(@IZfQeZYYANz3Z@kffR9ao1G-Q9<4C*!J7_?431kbP;WjlRCrTjH~yUNgwYRZNGP=VW24G8zn}m|fH-_{KRc#z+wMv7 z8rFQ8Ye!zRmEtF)##WIuh{Ig3Kg!a33`ni#g@ZcG4IGXg*e9-Yknm8ZB=rqef6E(o zUQxU&D3jI|hQFG@yH+RUP4vej;7sOD3TUiC2$0t=oB=o*F3tE!qt8mF9wJi|^dpts zolv^I*@L&f1W5$0fDVtLc%vkxV6eKI;S$`*HQA^!x|~IZT|7YY$T!z|^M@jx;sfJ- zrg86EuMRTBT_9nl*JEt{h!6c(m*-?40EG0IwV-U&gT}I_(73BQJHAj&q4kOSzz)Z#t9G zh{pkb(8Tmw7MiKeCbDOn$O5(_C=k_I#EW^k#v-F{0`UJ`eN8;Z8vA}0^43Kbe9ohB z|JJ$%XCM(VErXFt4rz_lK~7Ha0hONKSmmcb6xIRWtw`N>flY*a z9D?=C^CeN5Zn+U^*A&Nwx_Z*E9FnkW!8>0hstD(nG=K7;=>~Qu1EcXgyut3O(1wxoihl&bQ!hG%SPGB59Ni#bz4cTDeIc@&a|r!`-Kw>bS&a~ne4+k*hyC|v&et(Sx3WKAwKBhxAlV<}92~7KCEL7g zT-qaVH~hR`mOk}--`=3`!)R~131AHBf1!z|(CWNiFAy0$VCNkh>ys74RBmtn`7h5_ z+Z2ajNg3($pZR*^_zD3QQ4M`0Q75B^km_ZBj))pYf4Kp3OAtso7!zkjhZO}EFqJ)l z9C`)ei^}z<+S-F|24H=O{`S5wLXJc#hyV@#OG(bIGZbd;BAOBRUw>cl_;-yacE}jN z*9a{^WMmt?P9qL2a*>FqU(AA%o+y2b_*i9a9@G2>u+sKF!;1Jw@4Vbf{rRtw6w|n9 zV|DiWt3G*Q1ah-|;NcDg$Q(_s{_>$*9{moRL z6x)ylvA75y+;^|-D?l?EvU~ESy`8YLEIz@a`fvs-ciDT4V(U;)NIJmPD6#UQgdak z!|8-D+9I;ORekR`;qPeZr#6I=-p(4b$m-5%`|+B?iEK7?6rmDA!1J-X+jd~{kHRXB zpNSfjg$yuHL3pAFW-rR42oN62SkiQ7)qKhfNeK5)s+i+6?Z(Q9^bLDUn-w=irl2F9M*JZ$-UfKHiTs( zp1bdo5uonj@3)f(e`h6+iBw^@6BAGuXI?5x3uy{=-nmng0i8v4IUYlE%-2SkP z{djgG)2rl6I}r6if{a_GBsBQDuNyJ4U)MVB==(%x=;!)bMcXoIpOQ&y{s}hT)&wR~ z6vRm9%9j1RndL6O>mr!Scq{XK0FxE|& zi>kP-_d3wZ6WB=RP|DE8(DoQchCBUH7@rbU8^_#jCx>C^OXdVb!6+){&>y-}Y{+Pg&cjB*~S$)Jh1C~ihsW{k4cJepgB=8>+ext z0JjEoUkHZFs z!1_AoKfD-GRWo#9>rg@9yP|@R82ZCd-Ro-S9@xlAzI%Cn3sXwr@PAE;@@!Zr_B)*E zQHK;Z&%czpD!{#|LA+pnx^pN4T-wk!ChrH@c$DBxl>pyd_tKM{Lsp&;?`CV{87vr5 zr)Z{M@cY{GV1ab)w0b*rR1doED*AC-nxOxq;TrtvaNl;eUCXYM+w!vAs*~*&wrnoj zwrzV^tCoFQcAf0^e1D(M`xiXA@B6uNU6%yr-eD^{ZQK8A0RVxukz$0OE%x+s;dxRB z2hEgTR~NZvOci&8a`oUx)d;F5En2YqGrNqls;$3!ru%%`$5-I1#7i!af2H` zTiY|7V+wi>+-Y8(bIf>1tlfd`^Wro`g^9u$=2#@tHdmGauU=`rJ}~Kvf0|vs2``9e z4hTn;T;C0%hgQ|5a6$#9E{zb?i^tiNadoFaF@r+_96r?G5t(DFszA}c#m=c2^9;G? zTgxKR_e#2nAlP2WU|%{FXz2Vg?u;l?mxWj#smCIc#KC})G=N$;TZI(gh^GFRJ#OGWZLK#towJpg7nZaB9-q6}bUu(|3 z0W-j+aDP#%h~kK~p07&2Q}h_GgD{`i&RKh_`Zi;K}r`I$8TT+ueI!x4-C#RTbXB zAI|F<>>FGym7;xo=}>8MOc)BE99LypCxyj5xnNA1(LW<((K$c)y4q4_@Uue*jiKZ} zmSSb(1lo@8t@kH#5yu~o6Xa>E!`=Pavs&p_*7OYFhu-{;+Li&M5#o9la$**Yvdl-C zkSiIGWe=nz10TtJ^~!?;%aV>0RNG9=<2kJR2EW;Xc%H7y8p2i|H8wAcfI>wBBB)B1 zFfV5Ap9_78qfI#wY0qn$r({^>xaG438{wT5!4CRYnkTdTpapbk|vb|b98?w}K_ z%5|EzaUm2(+aPGyCQ#P0$JeH-mxd37vJ8holF}NnVf7zDgHVW_=7Wdv`Jc~d=86wfJw=ar9M*Do#Dm7Fv z1qkg_jW3x8(Ra98h~9SD!J}4Hp+Nn0LmcI=iT;<>5JH~27`v#sfr48Z50J91c-LW zBwYjh3}ag)1c(d6jw2Ad$nbN3JEI+orp!nmfg)hbp}@3BQ(v*Tq;wuleov?>Z2leM zRpvJi!e;eYQsXabg>1SyP&`CR<^ZyD5otUpMETJgV^EXGf>{Rt6K9$&AxqCPS%Wzw z`?YmX9R5&Y!sDib`MB*_WKS1NHdssYtQXJX7*n_@Qk4!27+55{wZR>bChE*V6`t&U zGsjWoeAhs9Of;U9+#cgc*5P%HUvHh4egZQ6-SWR+M^Wui31Tw?zi{%Iv%fMcS(7a0 zlOA`k4us8I2{UOE>or$B$gX1inSMC~M10wgMIU~b1j>8<4 zMzilY5GlsjLMne9S_xZNeC^^=Cw>ix$Fqks$aGS_$z3F`W<)$%F&8Umfj#((p`jcV zmPnx&O5Tf~O|z^&Tw}};dwu-Pa=p;dJYLk$Jqoz-W@F=6AP1zG6NZFs%YNBMP}@@P zzxKd{s86o2z`tzKhQ1j!l!(GIJkH8q7@ljk55=m6RC8=F;^D!5pNKPwO7`o ztzz5`@WDFCA(1+k=d3xa*Sn&TvE~)R;bS!r&g)Ivp;a`;=U- zB|^kzjZcW4)-WS{MiYtRL=V_d#uDVnGdGin6B&bX%MVpm*56O4wQX{C(7kR47Ly`_ zU?;l08ZJP;tD=MhlU)aS=$~P5~pO#~oGB?`z0vM5mWiVE+=M5U{ zb>*tZH8OeWP7*B^#>Ka6@4Gw|c>Io8M~zxG!MFxw5Ge5CPkzRY2Y>I)EW&fLr#NZW z?<;bD8e%qJU>E|yR8ci7fAlAqfIa#dXME5)ZENsfm!vEJ62oFj!^t}2aMpqrb8#F_ zhSu&J8h)zgS*mc_EqZkIg%aZ?R)N*se=~motX~;z#3Oh>IH87~_yWUS=$UPik84fMKRQKFHhA$Id_}z26K!^xhop%&G8@yz_oLdmJ^sw-E6bbq@F* zTuAy#ir1}Fd-_I1F`%-bZY50)*RfI8oj>wS6$Mspo4BTru(!JC*eZ=gwCWJ4BRj>~ zTScyK6bXKNA>vObg2rXn;X1!Te17GuH zbja9s-WsSp9d4)F;`09w%ZZf@OZPlR_kGKC$m5jaQ_PhApw);7BKHXbI=0c?{x(Wb z(rJQa?Rp30jAy!a6PRtP00J_O>pIA*1 zI?(>n-~=CX&mpzeMW;7Y%b2wUR27=~3d}B0alryR@z+%c722oB3bei!e-imdC11^k z&FJ54uz=(3F9j-uph6xZcXm)8Vawda&Fq?{QgBadV9>4#ia>OswpBtZ!6tRq?9Uj_ zqA+sz`Csb*h}^Amt~;_g>oUYn`)TID%F(ddcA?lVAm7d z^Zhx8$xvyd12Tp~pmVX5c#~r;26;xvhCAbO3(t{OShs<5&zraZ{r08MIZH}e8WZ8sEcd5V*LAQKixzG!O>?KOws*sJ&lmZm*`gt`j_1CitVDt!={v3d>%?a0}(P z`?5Kdoa@BCarh*`bK0a^Jxn#2z}sRZ*tKy=$SuBi)r56Pz!>_3X0denUI zQsUZ&GtU>s?2FPe&WW?^nJ{Thl_kIx<4uJxpV{Cf+L z#gk&2+Zxgx{xjk2xXho<6bzVWD83e%w+WO0S+cI6Mi-8UpORFLsf2R;E)o;``ynL{ z#;EltdemWOGN7_^!MsuObzL_tw!xl#9TI9CgS)%)r#Y+b$Lz~A-NeYG|2TfM&|dYX z9l$cTIlAv6KQ~rDSxupNQUZSaF}p;#(9OM5C(PTq`%c@r_4vg~W17-NKj@jvb4yvS zsCH@14YHq@)d6+U_B6QX>A{yaeTzjZ`%#q3lWjqB(nv!7zKVXgF3X#Zx3nGsj-sP~ zudF7oG-XGWzhep)DR;2IG!t(l{n0rYU_L>hHgpb|pc@uWok+M`^f?r~{qC@(R<$jc zlC1I-2A^VcAtvcy2)iFMn{B1zKz&lL=&)^{7nU=fv-fT|FxCk)_^C|~o(%H3g&Qka zcHMcTpN}MNR57sv@1(j`5Jct+wwWf^>Hi%C_iDTAlbhJA8LiPP^#c|f=@q%lfM zy+CIP_(oJQ(2S(LuXfh81IYNDL^PWly5Yoq*-l!$BK=msH=RmTlXKM&{)Q}6vvIFZ zT)1A_#ed`Oa|rN%Nl$|VPLyK*$3By;FD>c>FI77Z7@YRjcYnDLg9EIb5*<`WpSL4pM*&_5 zoXAwqEneS!wi40(`6DcG+#>>r=?y;Pc(48M8hgLrX{CK0_1AhY7%AS>+v59^Fq#Gp z$1JIH3xcj-(Q06n$g6;orz3XZLpD?HN9H<>5*`TdJqSGZEHKbO^YUz1ao9)IK1&R{ zjJZff6=%@Z*;j5h4A5NGh?kp0*;fYK!Y}Jii-@gR30i2B_2B=46sp|4#w|ZL7k|Hq zyss|zfXI{8GyVQUtBY9T>VgoJ)2b=jJ>;Cv^k4g#EMs!^+=P-;!m_y+cJjl7YI`MR z`5A&AcCD;#ZSITMfwOKGj5eF3@!q8^%Xf&k1&W z9&=*qi{2%xh(JDuQi`Tp0*W5XyTYv+*fciVm^_OCW|DMhdqihA8Z}x9{)IdIGwsm6 z@-hK()kq<0omr;mqHU0_E2L!QIEFAZnxpI)B)+b?c|({ve06I&&Rtq&a&!oE{ee+1 zv&nn7-|VTV~ygK2hd8Y4&fAao^Z7)etqZ3PZ>6Yn>31#-J8F# z&qQUuW7DA)6xB&`(f!*WmdpRP49)@g)xY6(L`NR?Q4vUICQcjio4q>a0bGTcShs|l zFf@lR2nD3;cp-lKC9Mfoz6VtRn$XV2N+@}?Mz7*)B~X87=elnlY3(j$Hsx>aiqS%Q zFdQ9q}K0t>QvCk5lV-dyeAZ|8?FPyaCW20|2;yHB_Od>DN%PX+tNi9YRayQ?(Kg17sco)ericeN&IV&VoU z_bZV;mefWbmkGDf&OL>1lfPq-QHj!|hy`{UhCT4F>fV22%qPO_^qmm7R5?G2@t0Q0<^1C`OY#&)&2u`+6B^Vw=5!N%=&}mse-@ zZrN>}FgPd?ueClOX03V^^eLj5*Fo@|Zm$FU;YII3mH)$-^Y>0n9(#p7VJS$1<(|pz zkMCLIi*gn>mtRf5qLX4epN%=ePcX9xlI)?^mh_WdFg6a}k%DH>#!ck_ikBg)qs(g0 zl8w(?Xi|*pyz)MsFBnEpE5-7BQ$Ovdvc57Cs@jQ00^1x7u!Ly)V%{^i(c$q#R#&q~{w2b4jxD7VqWlDWSaC_|emIFK zo8IR4NO^6HK?D%Tk|t!|+A7jRXf^?&zTvxoqsHokZ%B4{Jq{tcMq>NMCBi|hr~mAL zI#=*vjO%c|i^wo}5sqpTzVNyM75ubmfU!zjpStSG-(x7HOZLT7P3vD3Bq~(df>V;6 zw;P!w4V_`^hj;_O-jqx`IZ}SJa}OCXU)FC(JPOu?H%egNugCY%TSMWJePMu1ho<=q zJ)Sy8t5^74Ecyac9H3EQw`Z8^AA`LAW@b6UtHsKpc&-KqnMmR!1tFf-Yi1dbAxnsi zjTgc8aCR0`yp{WvXZ2DgOB&rOkvr>Gax}vuYBn~;t6AKkROLYl0Wm0K_oxuHmH0J} zCtw^0BEaG4I0+AFmadf})vvI>JgCk~oiPp+-BMUk#$*p`IOd^}!{Y;GCVfcAT9r_DpjIFop}A zctv$Pr7FcTaNgddZ~47)^7d`k<96izr@#;yI8?_;@s-1;BVds_`sYhf)bIQe%Cc=) z4>E!Zipdc~&l5tQXOz$O(DKQih8UR~1?0|G@r5z$BU}~>R<%CP5wN}IgYfxX=9>HL znpiPVz>SfHSXC*i;Ak%_?M{8RNhrgP%YQ&D08_Ts9Yu&$|6o$_Y{Uv7XKeHG*kp$p zJ5$P}o1N@`mujNj{x3Jp=W($v^-!g_xerqBe@DUYIRd-R=r_1OdfzXn^K18bYJ18; zn`Q52H?w<_9qI9-JlQhZjs7BZg$b0Kvj-LJw z7&GCzCaCq|_;H(-79!(r@q?+_4}9Tn_h3CkddH=C%@_q>SAz-vBPjQ$>OfxPCZLkp z4EvKj;z_r*!~(hi~L}c?GA_9qGXMsPWi+M)2_uIH*xwOTQfa(6=@pGvbnn`FJOv+m5n)Eo6rZ#B3Fy<;DqOL0$}mJsSqpwz5I)8vEGAbgU9K;_$LyH> zs`S2GAZmhj)tyHk3c=qN6T7#iyRiurt%@iMBF-yKc(%PF;B02+s_2Ro@QUoVP()JV zO!Y!Uy0$@=HjO2p`#0zi4j0dqpJ*LyzK!xN&GSrkoY3#B)PE=S#Szh{c*3h;rSa;53hbUo>&Vb1yeS_6j(ALS=MIsK6ZCx^5#%20i9A}7b2 z=A6(@*p+yJu{mORWDQ7C4i2HO&rYcMXP(TeB{z83Ax z?cMD28nO}bwl8?s;G={<@HDZP9*lY~S|$;sh3P~F7eSF>wnPVhp~h8Cv*|FPBKnKfEde!FPbS|>rA6f zp^B^;+F~RPN}M$4M7ZcF;m$sNrE#ZlJ?mlDed=v0^({Fur@pMopej}8*g>v$-%n-C@Vjso|lAm|;JteK);n_t!c zTSs%1F;42NS}j6vpxtL$t(8r85bnY$TIC!kyOd7RWi?Hq6UM(tJswVSgvV(__sofD7L+1@Ph0> zy;hL}>Opfv9L&o^nI603sITs0!MF99zcaoBFs}p~3p&$>EPA#%!V5Mdi<7~a3?X<> z0ad?1$QbZtpl9Y|_5hSaO~rzQGqY<62ss{Yf29gW$7$5G)@+KhdQn#}R^M$D_5H4e zy_a1}c_A$B7KlHAi&<+qy*2r z9OpG1O(}y1aFh`0mm+e74I9AN>i;G~oR2aRP2rkwBN4?l*V3$p9jNOvCirN`+B;~} zvqhTT=W-V%{8DS{gx1Af*AjBM{Y}?V;#0LlQapJO>S-KFNfH%osQ~!ggr%D>#%Hv@xe}8 zSI-ex7Z}+JUZ(Zu{wij@A5=M~8NqI@C%&qfEDD1d7Nv~1(9iR2EUy~fj7OEsb&U6I zMG3Pd!k|JbeAK-(nH{)ri&pkH@JZeBjan z)Xc`sqr9+mE3fnz{0MsD;wKKmcw~#KF)e{B3lang_A-oQIo#PJRD^8W+(Td0Kl*nI zU1?@k#fM7&h}TD3g(w|mfoiaNBo3-jwmxt#LQugAEcADw+X^;MhZ=&nG$y^p7a3hoxHs zgPjQ+e7DlJ{0>wg>1gmLXl^@9hhgrDR`p00XqyshZ&MY{`>g9*fc4^!9%s$<^Y>cCOxKL>Q!{(OxT$Hu!t(=GhYom! zHHY4Aufqw20L5OpWQ(Y7alCb|7ijwQPd>+DtEbc@J;MC*sJFyOXL$c#P(s%(&8!jEU z4I-Ra_#=fkcR4b#y(gD3BdH#vFZ^Cw`4!5kEENzmC=|+3?I;|wbrhAh;^ZL}}@IU}Lo)jfok%Z>kRHZxQc-vIrH!3Y#y7n`^8SAc6>lW@4 ze{Kny3Y>6*R@rE zs`(w%FZHp?^%E9`GAY3#HAVrI^q|jH9o?zWXp$=b#OjTOIYsxMKc6}9BBwVD9O052 zNt8@WZ+JgTUl@8U)XaQLofMYt# zZrq=))joYE9(NMSnGb&AP*b|)c;W=z1!ynaCc-h%giIaQ7Z_;gK zT?LNnOK;ysIF1Y5%Aa8)UMSqk&#YB?CId@rK#_(QmT+swKp&_i*8bcDyh z5R-0m*LS+S{>QHU%#?U&)D&vj+oFV<4Zx;jfBu%hv1D92ZXU0RhpPohCnw+-yDUZQ zRc3-mD?tgc@YPX`bff%Trd_*!)1$Hk?OH6})$A~Hc`(^JxIB*xJki=JWb19m3b;&~ z#Cg^l)7)t!S+viC1494iWrc}{M^~saI!e7Y(XJ>%c%PnNCG-7lf2HtITXu1<)wrJH z`u0ZQf79u_g|L}yDGg`&*7i%2VlX65k1nqo_WxZ@^kms^wfRAr7sH=;XB>E|SBtkP*k+rLvblSZ%RZyWH=2k%j6 zZ_8LQe6yaVpF0)&$PRezd6}V7?T77>?svDL=qqB_CeCz0-*ukU3Xz6z24ttJV>rs^ z?WKl{;*V!h{Z{bo8Bj}O9bCPIQhE_S#f0J6Hm;o|7?ox;>9_jGMABrxn1s-AjZA%l zr$ok}1WZV*j>*69cq1f=xD3vfAP7b z>Y4fVpEwg--fx*auiy3IGycmldXA}=?`(;V(XS;X=|Wt@*LU7=`duw&6;A%^+KMAod zbO0@pfTz!vupf#YYuuej)=_{+89v*AEza`Xj4+#_U^Lr*Q`Qp`S#H99-Tdz&TrU){ zu%YV-*lss+Iua1! z#(2zY={yz4dC#ORS|{`VU_pnVZ7YwL;cR&HhU-{5K$ofe+MyR(0FL zC~J$|O}Ku8Dl>cvRdGZ~Lr8A(T6>tSmkS&^(lPivp2|vQ5vaqYPWNaGZ;aw|mn7)08(&-5d}$mHgbwZZXJGzf& zoV3&kgkCm@z*MBfq!+h=Mw26JUZJLQ;KQL=6O(_KDTOnKwCyfNt%Nb!?qwuw-J7d7Sz4ke}dc1 z!af)F-#GXFn)`8-fvw9((dhj_Bmq@vKkd}w3uQcRVva{D+jxs?2NZ;@w6`%7Qq^t& z-wsROKh-yvka0E(ZA}%yr6slE@Ed3p-z2;Tnq;S*q-LW(;?&7>i%+QYoMo7q=t0Nt z)~d4d^|HC}er{yDs~3S$zUxBVps8Cud(L#KkJTiwsP3kz<18dg9OCS!@BLBj6D!6ridHM0!sAK3EU7RfyG60GbtF zdzjg6LwK_&b+ge&Dzy9tm09Pr*|=F9e1+MRN6Xa{C~FSQ7;3(z;~PCwEbd*ucEAP z-Wbb(0wCB~V%aFPQ-~ny*l*HaPj3ayDmXbx2}8qc)asVtX(1)dOwxqfQW9>eFPnHSH%yX7 z_;zZ4n}~eFw{t4GO+Jigbeu6ncCt7AR#w)TwDe7#vhb`q?~)h3J< zm!G;$^syvutz>W#ZTj3wR$!AHI1m&s`U$#n>Di=BAE;4g=-c<*_=Uw->Pz`b8eSK@ zBHB}r*wY`Ll8pb9;1G}IA0IvYzB~_Dk-tUQHNRcyroQF8hRp;{PH`*K+lTHq8KhbP zzp2@Iq;LO>RS}(U)!Uqt(fI5)fNKBks{-PiHHEpYcxJ~n?$m;#!zunvqm9HBbjB7n zomOQV4|>qVu;T}gycuU^7>4# zBo0|V^7VUuRN5)Ni}%X(VQP*rY>FNF~Z+WfBilPq1KaQU>c9pv|&74<^q0QX*uDS8j+<6DqJL|#!rsH5V zmYtX8{XPyF#Erpp+2!z;XKpkVYmhBKlfF2}O&}D1%0gXnXWoC?{v{#`!ykp0Wg`~% zF!DMnaoJp6r&BsOMI%wKbfnIJ?v04H(MPBiaiXD#)dc}1P0nC~zSV$_zjX9u*p0C2 zEjYEcVw#owLe()(cvb)EF~Jg%Dx`*zf;Dt~W=}U|`E@(J4*uvdn9aUvfK2<}F3+u1 z0sEme|4EjxJLuW5p133yZAj_?T^IxN!tN)kHDorqd3QBeb*(=)1A=?gymc>5N_1|- z4OjZ`7PeVm&oE9on;<&1F1tq6OOvyAO8c4Jm_z?{pzp!+m0Ph{-6V37&3Z7)pv$c2 z;9P`nvHLP(-|}hibUMmI`m^IZZSNm-qs7< z&dW~D<6=I}*v}GYtB->dHKZ(=^8khE1octl*y%5l8^6`q*F|nST5xsdEVnIS70VCJ z-Yr;sHqT1(RzDgpCGblU(ctQt`LU8pf4)e}__rIZM)qlv`ocWt4Sv5GF&Q31V97l_ z(=iB=G#3#!TubJpA7;O+U=4m4CA1A7CA(O2yhdwqH|~01>pu~yuD&9}0($g7L>~wb zdxF-VjQBj8b^50ZFo>aXAQOaczm!`fNDv{F1`WHn!W-X~q4~FVxbD0@Y=O`4!|ZX? zG~)wrY19VvO;N1I`VBrjju^rWylMW_8jaX^)BQ0L3TXESGjM>4C(m7LYe6)_w$Fz| zrG>S#-@UoehTEDgknUp;!1;Z&URN#6^&8_qJoOXsU4Sm4h44tFD z$su?|7fxv-JZ6n3{PEuj-aH=o!bjsc)OyX8y(|(OH{jd=r9=UIO~=5g=wU~3TN!(I z$TBzcBAX2P$9(;jXJ<%c#j+a@HAJ(;k+SS12b;A*66qURq(PWHdRmQn^kSK*R zSX{eiGZc5|1l4H6Hmhf68lXB;a{Qm z*HP0=?uPBp!pMld7iGISA1;qy%T(Oc{^|FiRV7XS^!oTbB(%e}^Q4l6BYIW_!@!P_ z^>@jlSlPB{&WG@w6+J(hEPW=qh>f#^@sRT>GoNKYIOdU5mYdppPG) zwy;zgMnQDdvocI3QeR{EXgNkP$|!wpq9nJ4=ug3Px~{C%FG_dkSK-5qEf$mkn(Ns7 zlTSW(NtZh<1ij?fSZ!O|_M1*~QI{mOEN^&z%lE=UEc-asmcjvth6Wq0=TaWw+k!s| zkmAli8H9h*Z5wnHR0dL8Kr0m07{mDeL}oH<4B6#w`~lrPNZ7~YAujnsQcKYjdFZ(_ zUZDqFZ0~Ud0`^DhbwLk=$kMRrArYnGLnp(Gw7J3H1_BY{p(GG^iG4;bx--Ol=$x(* z2wMycl75!NHM@pW2YguV3;O>Uk`?{gd$Qo!|9Ngai9MA`B-|dm|XI6c0dqeDF3V6-^0*(jyoF zWw|mXp688ArmgXllX*lTw%L&+K}S+X?+PG%HB%GGsTwKbf3X;Fu*Et$2w@#nkk742VxRchT9^M0+|ruzb22hsIRl+Hj-@beB7w53JtCp!?@lV0i*Z&$AN7>nO$(CjOpNYd+cW&bbrl#^Y zV);q=o?HqA6{~@t;NyP7HZ!_lG@=0v%eW}#bU>ycaRxCpn5f3IuFr8!$|3M?o6-kn zRQka5*egJIkCaGeia8}vJB2iStt`PrOq8Dmr5yx27Llj1B$~3f1pHAhJV*^Yz8ggb zHS!zv!me{l%V6Pd4KIJMea)SCf0uo^gYI^QV$JmplI}QQGD}%FU=YGL|($ zj`oT3k~m(dFsT5_tAZ(-N<*=|{BJriq>6XqT?44LYw8EX0~$+Qe8F=k@SXB;Azf!U9ywV@Sr#6K%&nky?cO{=*kv zJd>gdotBF(z*uVq6-&i6u^-Yqh9@h4@X;OS#EJiPN&ByyGp9ca)6GBoI)gu%Tw&UY zG-u$}S8AhO}Bp9{4oDmQ~?AlNUgs+?W^?mI_z!8-yUicUX~pQwUmr0+A_F1q6m*| ziG;*x&8J5bWPdYaLhIX5RsOYcJ=8)Fz3m^6GPowj#Qw8fN+*L1tXTm%|mS@XxIM`8NALF8=+N{6xSL0~#3sPNI2^_X*^)shHMO)b;%#Y{na z9lOF>*pIpi$hf+PMvVceDJUZS7|IPWMI0UwwMbNB(_O#!GIUk zZu%`P3Z?Uop6^EkF?3>KHSk88#L*h~!t(Vgv-nZzk$efV? z9e5?0fUBQ{OkXPR3mp{9?2976w4$FZY88O1jihfZlhJ&90K4`W4U6qalCn-Bl)uO| z+R!?M-1sx6?(7@V&@k`ytIUZ^(F7zZs}y3H&%^WQFjp9BChi?}%t#M zrSB-8LJC_gj7OxhvHQ|gsXLVg7aI1`*F@&Kl+bt9NBgceG#|I}7{6FSjOu1CZ zo)$!)9RoHaQA`eK7=W+fc(*S-d?L%CM&a(Rhr(kke(u_a(Tg`<7+l~ow{{(SGD?Xsw z7IQk8A+#!3*D0bUfN-sEBbQ+OTbp_~KyXYp@--aS!BZT;k+}AaVN00=t2tLB3gK#`b#T*@r- zIm+X)QQ$wD!%I=lO3Q(k3*cA~xWil9{w^$_o}HDuLaYnc>&3yr0F2%823R98F35;U6F(zl z-@QI5A&Vl1W3faJGzRIBto(bOAarFBaBTG<_wDkDiQ)P7?!(zjHR`P9OV(fcj-_N3 z%ZjNe>CLDRW&OSirej--`H^p za&U}^^+TGzL{U7Ott&e`RV@-wReXURZm4pN?Re)AT;{QWSmS+ zOzO+X`3p+cbhXS>(KTr$`T3q}xU^@i>%awu*lhownWfxZ<70@Vh>T$Sd(R#5L3s!V3R`_l?Pe)Tt9+qZ~`9TTH`mWk_XUAIh9#6Nm#u z3r!m`SEs7rv3Ev!XcwGKCq>i@(gGI&75;XJp3)HJuG}y7GJ3UGQL=R|`~52A5!nhW zsmDU%=Zj`2EK-Dm2IT|2KTBHrUUNg^!Krr?{?*i2n%AM)`+^G+E;?wQ83cd(J+|(- z<<+N^dT35P)zjyB8@a24Q=>B`T3v6s!>D8yYaAW`YsI6*&qAjUW)3sv{A#}^)o)I` zh{5+I1nL=w1%r;Pi6{$Jfl;^sns76!p;|6V!^P)LMf@|PRl~D4AqAgPpZq`HR4%Ok z%#xy(Kxcd;CTDXE9pVOytAx;Fe8V%4)K{aU>$V{3{;p>oSTbnDMUDKUN`-Z5ZcxK{ zy-sNKEZ9${>D+fDOT#7F(q^T#Z8-Ru-lE)32de7b>~jk-b2Cc&V+uUKr@W@OXwzhW)YfHJ9y`qX z{k!ks6@ss0OAu9AfK7# z8&dA%(K zd-jZmONNo>LSib5F+cTvk9+3I<><^0Dt<`E^Ui8%$)j)!HtO)cl-d1$$m)3WZ5ZXd zVeFpX8$TK!4DIgto=6`wEu1_C{nGQlWH`#wWhO3do74C}#gVzSomMy_!yi_1#*I&W z`Q1E4mo2Deb+2!}(17vL>>l{~gR_8MSBlYrfp9p*&`_8lFPgAS#&SmIe57)`&nQ`> z>Qsn1qgWKNNvQsOam{WGWY0K?TLj*pFbgj9w71+gH@$`ZC}kp+ge)jqf8Drz-u^vP z2!cX}6Tl!z9564pp7&=oGD#DaM4F5H@y{EYuI+f7NbCIRtV#=t#R1V(`1Z0*^Y7Vk z9hT#J@YiJ#)3@@2&yeIXa~}4^QiOodBw#KR3p<`sp?G>RVe5DQ912!}O#cz{1Sy=j zC9KHYe2gCk*8)ZeSm_!N1*d8_vJ!6mtArD)K=do~;l;9_`9TFq5 zGg*HB+IDziYb2;B>hfhoAojX1Fy=^$;6cA@a$n6c7h+yKGYx)&c^W&CgA@IOv#3fmHICC=|dCf!?xTORG_wTA!2yhCDU)*libR(t@< z-?Cr&j$i1EC%iD^^@Q{rQYY<#&5{OkRcQOO}zF%JgE!@8}4e*DqqI2|?YjrT8qt!&fhfwY~y-OAcEDT9NnAFE> zs4*v!SoJJ(AbshuE8slN5nd^h8T)d^&;23g!NRkd8uq&tq!#^tXgU9#RMns0#EtjJy_Cj=e3#;2{ zzrdx`j*PJP0|vn-Ri|rx_Gv1Y-RB?~WryDC5eC1e|9Tr)(;%Uj$)Ji26miPs1~g^x zDtGkW65<%C-LI6oh0jWxX?fee`-FtMhE<;L75rs+tO2gd^MZo|=sYltHoMb6wO-qQ zW$JuP{mm+Z1Zf4^Ulct0Zc-@>?+7cS9>)4tAqAeRn`f3cdku(D%tit^0F7y5Iv=EDlSbgw}Xdxk@UtQ1hd9#T-y|xq$NEt6G?{~wX z!0_q4X^-OEk*=adV^f`qeM;e78B<=p@gjph*D`l6{ z+u)|_M{^$LnYPV+*islA zOF#IyJ>bXXyp`_|6Z1Jk{zR)kGo_2E%QdgS(Cp}L)*)Q+KJFs~f0t8C{~bLYikh+h zj{n<92Zq;oo|Nh>Iu&+4ETXbhHTqy`4fmJB+;z&ro=q4@q|N)f{_6Axth%X`48@Fc_?AB*q;FiJYD@<);T3K0RWx3wow28 literal 0 HcmV?d00001 diff --git a/parakeet/data/__init__.py b/parakeet/data/__init__.py index 26a6c7f..31304ab 100644 --- a/parakeet/data/__init__.py +++ b/parakeet/data/__init__.py @@ -1,3 +1,5 @@ +"""Parakeet's infrastructure for data processing. +""" # Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); From c321fcd09877959f881db4d71bf97cd5df358bec Mon Sep 17 00:00:00 2001 From: iclementine Date: Wed, 13 Jan 2021 14:58:26 +0800 Subject: [PATCH 04/18] polish documentation --- doc/source/advanced.rst | 62 +++++++++++++++++++++++++++++++++++++-- doc/source/demo.rst | 2 ++ doc/source/index.rst | 8 ++++- doc/source/tutorials.rst | 7 +++++ parakeet/data/__init__.py | 4 +-- 5 files changed, 78 insertions(+), 5 deletions(-) create mode 100644 doc/source/demo.rst diff --git a/doc/source/advanced.rst b/doc/source/advanced.rst index dccb98e..c9c1812 100644 --- a/doc/source/advanced.rst +++ b/doc/source/advanced.rst @@ -8,7 +8,7 @@ experiments. Guidelines on implementation are also elaborated. Model ------------- -As a common practice with paddlepaddle, models are implemented as subclasse +As a common practice with paddlepaddle, models are implemented as subclasses of ``paddle.nn.Layer``. More complicated models, it is recommended to split the model into different components. @@ -18,8 +18,66 @@ extract the sublayer as a seperate layer. There are two common ways to define a model which consists of several modules. -#. +#. Define a module given the specifications. + .. code-block:: python + + class MLP(nn.Layer): + def __init__(self, input_size, hidden_size, output_size): + self.linear1 = nn.Linear(input_size, hidden_size) + self.linear2 = nn.Linear(hidden_size, output_size) + + def forward(self, x): + return self.linear2(paddle.tanh(self.linear1(x)) + + module = MLP(16, 32, 4) # intialize a module + + When the module is intended to be a generic reusable layer that can be + integrated into a larger model, we prefer to define it in this way. + + For considerations of readability and usability, we strongly recommend **NOT** to + pack specifications into a single object. Here's an example below. + + .. code-block:: python + + class MLP(nn.Layer): + def __init__(self, hparams): + self.linear1 = nn.Linear(hparams.input_size, hparams.hidden_size) + self.linear2 = nn.Linear(hparams.hidden_size, hparams.output_size) + + def forward(self, x): + return self.linear2(paddle.tanh(self.linear1(x)) + + For a module defined in this way, it's harder for the user to initialize a + instance. The user have to read the code to check what attributes are used. + + Code in this style tend to pass a huge config object to initialize every + module used in an experiment, thought each module may not need the whole + configuration. + + We prefer to be explicit. + +#. Define a module as a combination given its components. + + .. code-block:: python + + class Seq2Seq(nn.Layer): + def __init__(self, encoder, decoder): + self.encoder = encoder + self.decoder = decoder + + def forward(self, x): + encoder_output = self.encoder(x) + output = self.decoder(encoder_output) + return output + + encoder = Encoder(...) + decoder = Decoder(...) + model = Seq2Seq(encoder, decoder) # compose two components + + When a model is a complicated one made up of several components, each of which + has a separate functionality, and can be replaced by other components with the + same functionality, we prefer to define it in this way. Data ------------- diff --git a/doc/source/demo.rst b/doc/source/demo.rst new file mode 100644 index 0000000..f0ca761 --- /dev/null +++ b/doc/source/demo.rst @@ -0,0 +1,2 @@ +Audio Sample +================== \ No newline at end of file diff --git a/doc/source/index.rst b/doc/source/index.rst index 08eb577..c1beca2 100644 --- a/doc/source/index.rst +++ b/doc/source/index.rst @@ -20,12 +20,18 @@ Parakeet install tutorials + advanced + +.. toctree:: + :caption: Demos + :maxdepth: 1 + + demo .. toctree:: :caption: Design of Parakeet :maxdepth: 1 - advanced design .. toctree:: diff --git a/doc/source/tutorials.rst b/doc/source/tutorials.rst index 99e7fcb..6ce1723 100644 --- a/doc/source/tutorials.rst +++ b/doc/source/tutorials.rst @@ -61,5 +61,12 @@ Like the example above, after loading the pretrained ConditionalWaveFlow model, For more details on how to use the model, please refer the documentation. +.. raw:: html + + + diff --git a/parakeet/data/__init__.py b/parakeet/data/__init__.py index 31304ab..007e514 100644 --- a/parakeet/data/__init__.py +++ b/parakeet/data/__init__.py @@ -14,5 +14,5 @@ # See the License for the specific language governing permissions and # limitations under the License. -from .dataset import * -from .batch import * +from parakeet.data.dataset import * +from parakeet.data.batch import * From 4fde5c7e645c7ca72baebe3d5d844ee6f23ee107 Mon Sep 17 00:00:00 2001 From: iclementine Date: Wed, 13 Jan 2021 20:06:20 +0800 Subject: [PATCH 05/18] add demo and tutorials --- doc/source/demo.rst | 143 ++++++++++++++++++++++++++++++++++++++- doc/source/tutorials.rst | 9 +-- 2 files changed, 144 insertions(+), 8 deletions(-) diff --git a/doc/source/demo.rst b/doc/source/demo.rst index f0ca761..4bb1513 100644 --- a/doc/source/demo.rst +++ b/doc/source/demo.rst @@ -1,2 +1,143 @@ Audio Sample -================== \ No newline at end of file +================== + +TTS udio samples +------------------- + +Audio samples generated by a TTS system. Text is first transformed into spectrogram +by a text-to-spectrogram model, then the spectrogram is converted into raw audio by +a vocoder. + +.. raw:: html + + + + + + + + + + + + +
TransformerTTS + WaveFlow Tacotron2 + WaveFlow
+ + + + + + + + + + + + + + + + + + + +
+ + + +Vocoder audio samples +-------------------------- + +Audio samples generated from ground-truth spectrograms with a vocoder. + + diff --git a/doc/source/tutorials.rst b/doc/source/tutorials.rst index 6ce1723..18f0843 100644 --- a/doc/source/tutorials.rst +++ b/doc/source/tutorials.rst @@ -25,7 +25,7 @@ The code below show how to use a transformer_tts model. After loading the pretra >>> from parakeet.models import TransformerTTS >>> from pathlib import Path >>> import yacs - +>>> >>> # load the pretrained model >>> frontend = English() >>> checkpoint_dir = Path("transformer_tts_pretrained") @@ -61,12 +61,7 @@ Like the example above, after loading the pretrained ConditionalWaveFlow model, For more details on how to use the model, please refer the documentation. -.. raw:: html - - + From 8ba7eeb1da8423fca375c4b7e80f53e86953c937 Mon Sep 17 00:00:00 2001 From: iclementine Date: Wed, 13 Jan 2021 20:30:51 +0800 Subject: [PATCH 06/18] rename doc folder --- {doc => docs}/Makefile | 0 {doc => docs}/make.bat | 0 {doc => docs}/source/advanced.rst | 0 {doc => docs}/source/conf.py | 0 {doc => docs}/source/demo.rst | 0 {doc => docs}/source/design.rst | 0 {doc => docs}/source/index.rst | 0 {doc => docs}/source/install.rst | 0 {doc => docs}/source/modules.rst | 0 {doc => docs}/source/parakeet.audio.rst | 0 {doc => docs}/source/parakeet.data.rst | 0 {doc => docs}/source/parakeet.datasets.rst | 0 {doc => docs}/source/parakeet.frontend.normalizer.rst | 0 {doc => docs}/source/parakeet.frontend.rst | 0 {doc => docs}/source/parakeet.models.rst | 0 {doc => docs}/source/parakeet.modules.rst | 0 {doc => docs}/source/parakeet.rst | 0 {doc => docs}/source/parakeet.training.rst | 0 {doc => docs}/source/parakeet.utils.rst | 0 {doc => docs}/source/tutorials.rst | 6 +++--- {docs => docs_cn}/config_cn.md | 0 {docs => docs_cn}/data_cn.md | 0 {docs => docs_cn}/experiment_cn.md | 0 {docs => docs_cn}/experiment_guide_cn.md | 0 {docs => docs_cn}/install.rst | 0 {docs => docs_cn}/overview_cn.md | 0 26 files changed, 3 insertions(+), 3 deletions(-) rename {doc => docs}/Makefile (100%) rename {doc => docs}/make.bat (100%) rename {doc => docs}/source/advanced.rst (100%) rename {doc => docs}/source/conf.py (100%) rename {doc => docs}/source/demo.rst (100%) rename {doc => docs}/source/design.rst (100%) rename {doc => docs}/source/index.rst (100%) rename {doc => docs}/source/install.rst (100%) rename {doc => docs}/source/modules.rst (100%) rename {doc => docs}/source/parakeet.audio.rst (100%) rename {doc => docs}/source/parakeet.data.rst (100%) rename {doc => docs}/source/parakeet.datasets.rst (100%) rename {doc => docs}/source/parakeet.frontend.normalizer.rst (100%) rename {doc => docs}/source/parakeet.frontend.rst (100%) rename {doc => docs}/source/parakeet.models.rst (100%) rename {doc => docs}/source/parakeet.modules.rst (100%) rename {doc => docs}/source/parakeet.rst (100%) rename {doc => docs}/source/parakeet.training.rst (100%) rename {doc => docs}/source/parakeet.utils.rst (100%) rename {doc => docs}/source/tutorials.rst (99%) rename {docs => docs_cn}/config_cn.md (100%) rename {docs => docs_cn}/data_cn.md (100%) rename {docs => docs_cn}/experiment_cn.md (100%) rename {docs => docs_cn}/experiment_guide_cn.md (100%) rename {docs => docs_cn}/install.rst (100%) rename {docs => docs_cn}/overview_cn.md (100%) diff --git a/doc/Makefile b/docs/Makefile similarity index 100% rename from doc/Makefile rename to docs/Makefile diff --git a/doc/make.bat b/docs/make.bat similarity index 100% rename from doc/make.bat rename to docs/make.bat diff --git a/doc/source/advanced.rst b/docs/source/advanced.rst similarity index 100% rename from doc/source/advanced.rst rename to docs/source/advanced.rst diff --git a/doc/source/conf.py b/docs/source/conf.py similarity index 100% rename from doc/source/conf.py rename to docs/source/conf.py diff --git a/doc/source/demo.rst b/docs/source/demo.rst similarity index 100% rename from doc/source/demo.rst rename to docs/source/demo.rst diff --git a/doc/source/design.rst b/docs/source/design.rst similarity index 100% rename from doc/source/design.rst rename to docs/source/design.rst diff --git a/doc/source/index.rst b/docs/source/index.rst similarity index 100% rename from doc/source/index.rst rename to docs/source/index.rst diff --git a/doc/source/install.rst b/docs/source/install.rst similarity index 100% rename from doc/source/install.rst rename to docs/source/install.rst diff --git a/doc/source/modules.rst b/docs/source/modules.rst similarity index 100% rename from doc/source/modules.rst rename to docs/source/modules.rst diff --git a/doc/source/parakeet.audio.rst b/docs/source/parakeet.audio.rst similarity index 100% rename from doc/source/parakeet.audio.rst rename to docs/source/parakeet.audio.rst diff --git a/doc/source/parakeet.data.rst b/docs/source/parakeet.data.rst similarity index 100% rename from doc/source/parakeet.data.rst rename to docs/source/parakeet.data.rst diff --git a/doc/source/parakeet.datasets.rst b/docs/source/parakeet.datasets.rst similarity index 100% rename from doc/source/parakeet.datasets.rst rename to docs/source/parakeet.datasets.rst diff --git a/doc/source/parakeet.frontend.normalizer.rst b/docs/source/parakeet.frontend.normalizer.rst similarity index 100% rename from doc/source/parakeet.frontend.normalizer.rst rename to docs/source/parakeet.frontend.normalizer.rst diff --git a/doc/source/parakeet.frontend.rst b/docs/source/parakeet.frontend.rst similarity index 100% rename from doc/source/parakeet.frontend.rst rename to docs/source/parakeet.frontend.rst diff --git a/doc/source/parakeet.models.rst b/docs/source/parakeet.models.rst similarity index 100% rename from doc/source/parakeet.models.rst rename to docs/source/parakeet.models.rst diff --git a/doc/source/parakeet.modules.rst b/docs/source/parakeet.modules.rst similarity index 100% rename from doc/source/parakeet.modules.rst rename to docs/source/parakeet.modules.rst diff --git a/doc/source/parakeet.rst b/docs/source/parakeet.rst similarity index 100% rename from doc/source/parakeet.rst rename to docs/source/parakeet.rst diff --git a/doc/source/parakeet.training.rst b/docs/source/parakeet.training.rst similarity index 100% rename from doc/source/parakeet.training.rst rename to docs/source/parakeet.training.rst diff --git a/doc/source/parakeet.utils.rst b/docs/source/parakeet.utils.rst similarity index 100% rename from doc/source/parakeet.utils.rst rename to docs/source/parakeet.utils.rst diff --git a/doc/source/tutorials.rst b/docs/source/tutorials.rst similarity index 99% rename from doc/source/tutorials.rst rename to docs/source/tutorials.rst index 18f0843..c4f5b2c 100644 --- a/doc/source/tutorials.rst +++ b/docs/source/tutorials.rst @@ -34,7 +34,7 @@ The code below show how to use a transformer_tts model. After loading the pretra >>> model = TransformerTTS.from_pretrained( >>> frontend, config, checkpoint_path) >>> model.eval() - +>>> >>> # text to spectrogram >>> sentence = "Printing, in the only sense with which we are at present concerned, differs from most if not from all the arts and crafts represented in the Exhibition" >>> outputs = model.predict(sentence, verbose=args.verbose) @@ -47,14 +47,14 @@ Like the example above, after loading the pretrained ConditionalWaveFlow model, >>> import soundfile as df >>> from parakeet.models import ConditionalWaveFlow - +>>> >>> # load the pretrained model >>> checkpoint_dir = Path("waveflow_pretrained") >>> config = yacs.config.CfgNode.load_cfg(str(checkpoint_dir / "config.yaml")) >>> checkpoint_path = str(checkpoint_dir / "step-2000000") >>> vocoder = ConditionalWaveFlow.from_pretrained(config, checkpoint_path) >>> vocoder.eval() - +>>> >>> # synthesize >>> audio = vocoder.predict(mel_output) >>> sf.write(audio_path, audio, config.data.sample_rate) diff --git a/docs/config_cn.md b/docs_cn/config_cn.md similarity index 100% rename from docs/config_cn.md rename to docs_cn/config_cn.md diff --git a/docs/data_cn.md b/docs_cn/data_cn.md similarity index 100% rename from docs/data_cn.md rename to docs_cn/data_cn.md diff --git a/docs/experiment_cn.md b/docs_cn/experiment_cn.md similarity index 100% rename from docs/experiment_cn.md rename to docs_cn/experiment_cn.md diff --git a/docs/experiment_guide_cn.md b/docs_cn/experiment_guide_cn.md similarity index 100% rename from docs/experiment_guide_cn.md rename to docs_cn/experiment_guide_cn.md diff --git a/docs/install.rst b/docs_cn/install.rst similarity index 100% rename from docs/install.rst rename to docs_cn/install.rst diff --git a/docs/overview_cn.md b/docs_cn/overview_cn.md similarity index 100% rename from docs/overview_cn.md rename to docs_cn/overview_cn.md From 9d59de0f3bb9487a1def7422962eff33d3593b95 Mon Sep 17 00:00:00 2001 From: iclementine Date: Wed, 13 Jan 2021 20:36:25 +0800 Subject: [PATCH 07/18] add requirements for build doc --- requirements.txt | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 requirements.txt diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..786ca3b --- /dev/null +++ b/requirements.txt @@ -0,0 +1,3 @@ +numpydoc +sphinx-rtd-theme + From c522e56e86c0e22a115ab616dfa162279b66b1e0 Mon Sep 17 00:00:00 2001 From: iclementine Date: Wed, 13 Jan 2021 23:42:43 +0800 Subject: [PATCH 08/18] fix links to audio samples --- docs/source/demo.rst | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/docs/source/demo.rst b/docs/source/demo.rst index 4bb1513..561b08c 100644 --- a/docs/source/demo.rst +++ b/docs/source/demo.rst @@ -1,7 +1,7 @@ Audio Sample ================== -TTS udio samples +TTS audio samples ------------------- Audio samples generated by a TTS system. Text is first transformed into spectrogram @@ -20,55 +20,55 @@ a vocoder. From 27cba27d1befd98c745d00e190df5035ef78b131 Mon Sep 17 00:00:00 2001 From: iclementine Date: Thu, 14 Jan 2021 12:40:18 +0800 Subject: [PATCH 09/18] update config file for readthedocs --- .readthedocs.yml | 29 +++++++++++++++++++++++++++++ docs/requirements.txt | 1 + requirements.txt | 3 --- 3 files changed, 30 insertions(+), 3 deletions(-) create mode 100644 .readthedocs.yml create mode 100644 docs/requirements.txt delete mode 100644 requirements.txt diff --git a/.readthedocs.yml b/.readthedocs.yml new file mode 100644 index 0000000..cbb9613 --- /dev/null +++ b/.readthedocs.yml @@ -0,0 +1,29 @@ +# .readthedocs.yml +# Read the Docs configuration file +# See https://docs.readthedocs.io/en/stable/config-file/v2.html for details + +# Required +version: 0.2 + +# Build documentation in the docs/ directory with Sphinx +sphinx: + configuration: docs/source/conf.py + +# Build documentation with MkDocs +#mkdocs: +# configuration: mkdocs.yml + +# Optionally build your docs in additional formats such as PDF +formats: [] + +# Optionally set the version of Python and requirements required to build your docs +python: + version: 3.7 + install: + - method: pip + path: . + extra_requirements: + - doc + + - requirements: docs/requirements.txt + diff --git a/docs/requirements.txt b/docs/requirements.txt new file mode 100644 index 0000000..213ff3d --- /dev/null +++ b/docs/requirements.txt @@ -0,0 +1 @@ +paddlepaddle==2.0.0.rc1 \ No newline at end of file diff --git a/requirements.txt b/requirements.txt deleted file mode 100644 index 786ca3b..0000000 --- a/requirements.txt +++ /dev/null @@ -1,3 +0,0 @@ -numpydoc -sphinx-rtd-theme - From f751e3cfb68ca48cbad40047553a07fadf919bbc Mon Sep 17 00:00:00 2001 From: iclementine Date: Thu, 14 Jan 2021 12:42:52 +0800 Subject: [PATCH 10/18] fix config file --- .readthedocs.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.readthedocs.yml b/.readthedocs.yml index cbb9613..2d7374d 100644 --- a/.readthedocs.yml +++ b/.readthedocs.yml @@ -3,7 +3,7 @@ # See https://docs.readthedocs.io/en/stable/config-file/v2.html for details # Required -version: 0.2 +version: 2 # Build documentation in the docs/ directory with Sphinx sphinx: From f78a20c4a0f9e949ce010af42e5a7a9cf41ada2c Mon Sep 17 00:00:00 2001 From: iclementine Date: Thu, 14 Jan 2021 12:53:06 +0800 Subject: [PATCH 11/18] fix apidoc --- docs/source/conf.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/source/conf.py b/docs/source/conf.py index 3b652e4..41a10a0 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -24,9 +24,9 @@ # add these directories to sys.path here. If the directory is relative to the # documentation root, use os.path.abspath to make it absolute, like shown here. # -# import os -# import sys -# sys.path.insert(0, os.path.abspath('.')) +import os +import sys +sys.path.insert(0, os.path.abspath('../..')) # -- Project information ----------------------------------------------------- From a18bb23f300003acb97b9927d10c0718868fc8a7 Mon Sep 17 00:00:00 2001 From: iclementine Date: Thu, 14 Jan 2021 13:01:42 +0800 Subject: [PATCH 12/18] mock librosa and soundfile --- docs/source/conf.py | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/source/conf.py b/docs/source/conf.py index 41a10a0..4e9e583 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -27,6 +27,7 @@ import os import sys sys.path.insert(0, os.path.abspath('../..')) +autodoc_mock_imports = ["soundfile", "librosa"] # -- Project information ----------------------------------------------------- From eed6f9af087f1511b158908063c770fac79d1ded Mon Sep 17 00:00:00 2001 From: iclementine Date: Thu, 14 Jan 2021 13:07:55 +0800 Subject: [PATCH 13/18] mock paddle --- .readthedocs.yml | 1 - docs/requirements.txt | 1 - docs/source/conf.py | 2 +- 3 files changed, 1 insertion(+), 3 deletions(-) delete mode 100644 docs/requirements.txt diff --git a/.readthedocs.yml b/.readthedocs.yml index 2d7374d..aa2bcea 100644 --- a/.readthedocs.yml +++ b/.readthedocs.yml @@ -25,5 +25,4 @@ python: extra_requirements: - doc - - requirements: docs/requirements.txt diff --git a/docs/requirements.txt b/docs/requirements.txt deleted file mode 100644 index 213ff3d..0000000 --- a/docs/requirements.txt +++ /dev/null @@ -1 +0,0 @@ -paddlepaddle==2.0.0.rc1 \ No newline at end of file diff --git a/docs/source/conf.py b/docs/source/conf.py index 4e9e583..0a21475 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -27,7 +27,7 @@ import os import sys sys.path.insert(0, os.path.abspath('../..')) -autodoc_mock_imports = ["soundfile", "librosa"] +autodoc_mock_imports = ["soundfile", "librosa", "paddle"] # -- Project information ----------------------------------------------------- From b017c73100191b2580a4b660db9318bf1b78a4aa Mon Sep 17 00:00:00 2001 From: iclementine Date: Thu, 14 Jan 2021 13:17:29 +0800 Subject: [PATCH 14/18] remove mocking of paddle, fix typos --- docs/source/conf.py | 2 +- docs/source/install.rst | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/source/conf.py b/docs/source/conf.py index 0a21475..4e9e583 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -27,7 +27,7 @@ import os import sys sys.path.insert(0, os.path.abspath('../..')) -autodoc_mock_imports = ["soundfile", "librosa", "paddle"] +autodoc_mock_imports = ["soundfile", "librosa"] # -- Project information ----------------------------------------------------- diff --git a/docs/source/install.rst b/docs/source/install.rst index 3758ef7..b9647d6 100644 --- a/docs/source/install.rst +++ b/docs/source/install.rst @@ -59,12 +59,12 @@ are listed below. For any problem with installtion of soundfile, please refer to `SoundFile `_. -Insrall Parakeet +Install Parakeet ------------------ There are two ways to install parakeet according to the purpose of using it. -#. If you want to run experiments provided by parakeet or add new models and +1. If you want to run experiments provided by parakeet or add new models and experiments, it is recommended to clone the project from github (`Parakeet `_), and install it in editable mode. From 73374528d0ffef80a9323e12cc00a58ebe7bf3ef Mon Sep 17 00:00:00 2001 From: iclementine Date: Thu, 14 Jan 2021 15:12:36 +0800 Subject: [PATCH 15/18] add tutorials into sdvanced --- docs/source/advanced.rst | 96 ++++++++++++++++++++---- docs/source/{tutorials.rst => basic.rst} | 28 +++---- docs/source/index.rst | 2 +- docs/source/install.rst | 30 ++++---- 4 files changed, 111 insertions(+), 45 deletions(-) rename docs/source/{tutorials.rst => basic.rst} (77%) diff --git a/docs/source/advanced.rst b/docs/source/advanced.rst index c9c1812..262efc5 100644 --- a/docs/source/advanced.rst +++ b/docs/source/advanced.rst @@ -9,16 +9,18 @@ Model ------------- As a common practice with paddlepaddle, models are implemented as subclasses -of ``paddle.nn.Layer``. More complicated models, it is recommended to split -the model into different components. +of ``paddle.nn.Layer``. Models could be simple, like a single layer RNN. For +complicated models, it is recommended to split the model into different +components. For a encoder-decoder model, it is natural to split it into the encoder and the decoder. For a model composed of several similar layers, it is natural to -extract the sublayer as a seperate layer. +extract the sublayer as a separate layer. There are two common ways to define a model which consists of several modules. -#. Define a module given the specifications. +#. Define a module given the specifications. Here is an example with multilayer + perceptron. .. code-block:: python @@ -32,11 +34,11 @@ There are two common ways to define a model which consists of several modules. module = MLP(16, 32, 4) # intialize a module - When the module is intended to be a generic reusable layer that can be + When the module is intended to be a generic and reusable layer that can be integrated into a larger model, we prefer to define it in this way. - For considerations of readability and usability, we strongly recommend **NOT** to - pack specifications into a single object. Here's an example below. + For considerations of readability and usability, we strongly recommend + **NOT** to pack specifications into a single object. Here's an example below. .. code-block:: python @@ -48,16 +50,17 @@ There are two common ways to define a model which consists of several modules. def forward(self, x): return self.linear2(paddle.tanh(self.linear1(x)) - For a module defined in this way, it's harder for the user to initialize a - instance. The user have to read the code to check what attributes are used. + For a module defined in this way, it's harder for the user to initialize an + instance. Users have to read the code to check what attributes are used. - Code in this style tend to pass a huge config object to initialize every - module used in an experiment, thought each module may not need the whole - configuration. + Also, code in this style tend to be abused by passing a huge config object + to initialize every module used in an experiment, thought each module may + not need the whole configuration. We prefer to be explicit. -#. Define a module as a combination given its components. +#. Define a module as a combination given its components. Here is an example + for a sequence-to-sequence model. .. code-block:: python @@ -75,15 +78,78 @@ There are two common ways to define a model which consists of several modules. decoder = Decoder(...) model = Seq2Seq(encoder, decoder) # compose two components - When a model is a complicated one made up of several components, each of which + When a model is a complicated and made up of several components, each of which has a separate functionality, and can be replaced by other components with the same functionality, we prefer to define it in this way. Data ------------- +Another critical componnet for a deep learning project is data. As a common +practice, we use the dataset and dataloader abstraction. + +Dataset +^^^^^^^^^^ +Dataset is the representation of a set of examples used for a projet. In most of +the cases, dataset is a collection of examples. Dataset is an object which has +methods below. + +#. ``__len__``, to get the size of the dataset. +#. ``__getitem__``, to get an example by key or index. + +Examples is a record consisting of several fields. In practice, we usually +represent it as a namedtuple for convenience, yet dict and user-defined object +are also supported. + +We define our own dataset by subclassing ``paddle.io.Dataset``. + +DataLoader +^^^^^^^^^^^ +In deep learning practice, models are trained with minibatches. DataLoader +meets the need for iterating the dataset in batches. It is done by providing +a sampler and a batch function in addition to a dataset. + +#. sampler, sample indices or keys used to get examples from the dataset. +#. batch function, transform a list of examples into a batch. + +An commonly used sampler is ``RandomSampler``, it shuffles all the valid +indices and then iterate over them sequentially. ``DistributedBatchSampler`` is +a sampler used for distributed data parallel training, when the sampler handles +data sharding in a dynamic way. + +Batch function is used to transform selected examples into a batch. For a simple +case where an example is composed of several fields, each of which is represented +by an fixed size array, batch function can be simply stacking each field. For +cases where variable size arrays are included in the example, batching could +invlove padding and stacking. While in theory, batch function can do more like +randomly slicing, etc. + +For a custom dataset used for a custom model, it is required to define a batch +function for it. + Config ------------- +It's common to change the running configuration to compare results. To keep track +of running configuration, we use ``yaml`` configuration files. + +Also, we want to interact with command line options. Some options that usually +change according to running environments is provided by command line arguments. +In addition, we wan to override an option in the config file without editing +it. + +Taking these requirements in to consideration, we use `yacs `_ +as a confi management tool. Other tools like `omegaconf `_ +are also powerful and have similar functions. + +In each example provided, there is a ``config.py``, where the default config is +defined. If you want to get the default config, import ``config.py`` and call +``get_cfg_defaults()`` to get the default config. Then it can be updated with +yaml config file or command line arguments if needed. + +For details about how to use yacs in experiments, see `yacs `_. + + Experiment --------------- \ No newline at end of file +-------------- + diff --git a/docs/source/tutorials.rst b/docs/source/basic.rst similarity index 77% rename from docs/source/tutorials.rst rename to docs/source/basic.rst index c4f5b2c..11cee6d 100644 --- a/docs/source/tutorials.rst +++ b/docs/source/basic.rst @@ -1,24 +1,30 @@ =========== -Tutorials -=========== - Basic Usage -------------------- +=========== -Pretrained models are provided in a archive. Extract it to get a folder like this:: +This section shows how to use pretrained models provided by parakeet and make +inference with them. + +Pretrained models are provided in a archive. Extract it to get a folder like +this:: checkpoint_name/ ├──config.yaml └──step-310000.pdparams -The ``config.yaml`` stores the config used to train the model, the ``step-N.pdparams`` is the parameter file, where N is the steps it has been trained. +The ``config.yaml`` stores the config used to train the model, the +``step-N.pdparams`` is the parameter file, where N is the steps it has been +trained. The example code below shows how to use the models for prediction. text to spectrogram ^^^^^^^^^^^^^^^^^^^^^^ -The code below show how to use a transformer_tts model. After loading the pretrained model, use ``model.predict(sentence)`` to generate spectrogram (in numpy.ndarray format), which can be further used to synthesize waveflow. +The code below show how to use a transformer_tts model. After loading the +pretrained model, use ``model.predict(sentence)`` to generate spectrograms +(in numpy.ndarray format), which can be further used to synthesize raw audio +with a vocoder. >>> import parakeet >>> from parakeet.frontend import English @@ -43,7 +49,8 @@ The code below show how to use a transformer_tts model. After loading the pretra vocoder ^^^^^^^^^^ -Like the example above, after loading the pretrained ConditionalWaveFlow model, call ``model.predict(mel)`` to synthesize waveflow (in numpy.ndarray format). +Like the example above, after loading the pretrained ``ConditionalWaveFlow`` +model, call ``model.predict(mel)`` to synthesize raw audio (in wav format). >>> import soundfile as df >>> from parakeet.models import ConditionalWaveFlow @@ -60,8 +67,3 @@ Like the example above, after loading the pretrained ConditionalWaveFlow model, >>> sf.write(audio_path, audio, config.data.sample_rate) For more details on how to use the model, please refer the documentation. - - - - - diff --git a/docs/source/index.rst b/docs/source/index.rst index c1beca2..78f1057 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -19,7 +19,7 @@ Parakeet :maxdepth: 1 install - tutorials + basic advanced .. toctree:: diff --git a/docs/source/install.rst b/docs/source/install.rst index b9647d6..5555672 100644 --- a/docs/source/install.rst +++ b/docs/source/install.rst @@ -4,8 +4,8 @@ Installation Install PaddlePaddle -------------------- -Parakeet requires PaddlePaddle as its backend. Not that 2.0rc or newer versions +------------------------ +Parakeet requires PaddlePaddle as its backend. Not that 2.0.0rc1 or newer versions of paddle is required. Since paddlepaddle has multiple packages depending on the device (cpu or gpu) @@ -50,7 +50,7 @@ are listed below. # ubuntu, debian sudo apt-get install libsndfile1 - # centos, fedora, + # centos, fedora sudo yum install libsndfile # openSUSE @@ -64,22 +64,20 @@ Install Parakeet There are two ways to install parakeet according to the purpose of using it. -1. If you want to run experiments provided by parakeet or add new models and -experiments, it is recommended to clone the project from github -(`Parakeet `_), and install it in -editable mode. +#. If you want to run experiments provided by parakeet or add new models and + experiments, it is recommended to clone the project from github + (`Parakeet `_), and install it in + editable mode. .. code-block:: bash - - git clone https://github.com/PaddlePaddle/Parakeet - cd Parakeet - pip install -e . - + + git clone https://github.com/PaddlePaddle/Parakeet + cd Parakeet + pip install -e . #. If you only need to use the models for inference by parakeet, install from -pypi is recommended。 + pypi is recommended. .. code-block:: bash - - pip install paddle-parakeet - + + pip install paddle-parakeet From cf892c5ed7bd9c31cea6e9c3c38b381642b4d0de Mon Sep 17 00:00:00 2001 From: iclementine Date: Thu, 14 Jan 2021 15:30:31 +0800 Subject: [PATCH 16/18] add paddlepaddle into requirements for readthedocs --- .readthedocs.yml | 2 ++ docs/requirements.txt | 1 + 2 files changed, 3 insertions(+) create mode 100644 docs/requirements.txt diff --git a/.readthedocs.yml b/.readthedocs.yml index aa2bcea..ffefb00 100644 --- a/.readthedocs.yml +++ b/.readthedocs.yml @@ -24,5 +24,7 @@ python: path: . extra_requirements: - doc + + - requirements: docs/requirements.txt diff --git a/docs/requirements.txt b/docs/requirements.txt new file mode 100644 index 0000000..213ff3d --- /dev/null +++ b/docs/requirements.txt @@ -0,0 +1 @@ +paddlepaddle==2.0.0.rc1 \ No newline at end of file From 087d7bf16eff0fa87f56475c66eb0e0abceaa0c8 Mon Sep 17 00:00:00 2001 From: iclementine Date: Mon, 18 Jan 2021 13:08:03 +0800 Subject: [PATCH 17/18] remove dead links --- README.md | 212 ++++-------------------------------------------------- 1 file changed, 13 insertions(+), 199 deletions(-) diff --git a/README.md b/README.md index ca4bdcb..695af22 100644 --- a/README.md +++ b/README.md @@ -3,7 +3,7 @@ Parakeet aims to provide a flexible, efficient and state-of-the-art text-to-speech toolkit for the open-source community. It is built on PaddlePaddle Fluid dynamic graph and includes many influential TTS models proposed by [Baidu Research](http://research.baidu.com) and other research groups.
-
+
In particular, it features the latest [WaveFlow](https://arxiv.org/abs/1912.01219) model proposed by Baidu Research. @@ -18,17 +18,15 @@ In order to facilitate exploiting the existing TTS models directly and developin - Vocoders - [WaveFlow: A Compact Flow-based Model for Raw Audio](https://arxiv.org/abs/1912.01219) - - [ClariNet: Parallel Wave Generation in End-to-End Text-to-Speech](https://arxiv.org/abs/1807.07281) - [WaveNet: A Generative Model for Raw Audio](https://arxiv.org/abs/1609.03499) - TTS models - - [Deep Voice 3: Scaling Text-to-Speech with Convolutional Sequence Learning](https://arxiv.org/abs/1710.07654) - [Neural Speech Synthesis with Transformer Network (Transformer TTS)](https://arxiv.org/abs/1809.08895) - - [FastSpeech: Fast, Robust and Controllable Text to Speech](https://arxiv.org/abs/1905.09263) + - [Natural TTS Synthesis by Conditioning WaveNet on Mel Spectrogram Predictions](arxiv.org/abs/1712.05884) + And more will be added in the future. -See the [guide](docs/experiment_guide.md) for details about how to build your own model and experiment in Parakeet. ## Setup @@ -40,221 +38,37 @@ sudo apt-get install libsndfile1 ### Install PaddlePaddle -See [install](https://www.paddlepaddle.org.cn/install/quick) for more details. This repo requires PaddlePaddle **1.8.2** or above. +See [install](https://www.paddlepaddle.org.cn/install/quick) for more details. This repo requires PaddlePaddle **2.0.0.rc1** or above. ### Install Parakeet +```bash +pip install -U paddle-parakeet +``` +or ```bash git clone https://github.com/PaddlePaddle/Parakeet cd Parakeet pip install -e . ``` -### Install CMUdict for nltk - -CMUdict from nltk is used to transform text into phonemes. - -```python -import nltk -nltk.download("punkt") -nltk.download("cmudict") -``` +See [install](https://paddle-parakeet.readthedocs.io/en/latest/install.html) for more details. ## Examples Entries to the introduction, and the launch of training and synthsis for different example models: - [>>> WaveFlow](./examples/waveflow) -- [>>> Clarinet](./examples/clarinet) - [>>> WaveNet](./examples/wavenet) -- [>>> Deep Voice 3](./examples/deepvoice3) - [>>> Transformer TTS](./examples/transformer_tts) -- [>>> FastSpeech](./examples/fastspeech) +- [>>> Tacotron2](./examples/tacotron2) -## Pre-trained models and audio samples +## Audio samples -Parakeet also releases some well-trained parameters for the example models, which can be accessed in the following tables. Each column of these tables lists resources for one model, including the url link to the pre-trained model, the dataset that the model is trained on, and synthesized audio samples based on the pre-trained model. Click each model name to download, then you can get the compressed package which contains the pre-trained model and the `yaml` config describing how the model is trained. +### TTS models (Acoustic Model + Neural Vocoder) -#### Vocoders - -We provide the model checkpoints of WaveFlow with 64, 96 and 128 residual channels, ClariNet and WaveNet. - -
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
- WaveFlow (res. channels 64) - - WaveFlow (res. channels 96) - - WaveFlow (res. channels 128) -
LJSpeech LJSpeech LJSpeech
- -
- -
- -
- -
- - - 		- -
- -
- -
- -
- - - 		- -
- -
- -
- -
- - -
- ClariNet - - WaveNet -
LJSpeech LJSpeech
- -
- -
- -
- -
- - - 		- -
- -
- -
- -
- - -
-
- - -      **Note:** The input mel spectrogams are from validation dataset, which are not seen during training. - -#### TTS models - -We also provide checkpoints for different end-to-end TTS models, and present the synthesized audio examples for some randomly chosen famous quotes. The corresponding texts are displayed as follows. - -||Text | From | -|:-:|:-- | :--: | -0|*Life was like a box of chocolates, you never know what you're gonna get.* | *Forrest Gump* | -1|*With great power there must come great responsibility.* | *Spider-Man*| -2|*To be or not to be, that’s a question.*|*Hamlet*| -3|*Death is just a part of life, something we're all destined to do.*| *Forrest Gump*| -4|*Don’t argue with the people of strong determination, because they may change the fact!*| *William Shakespeare* | - -Users have the option to use different vocoders to convert the linear/mel spectrogam to the raw audio in TTS models. Taking this into account, we are going to release the checkpoints for TTS models adapted to different vocoders, including the [Griffin-Lim](https://ieeexplore.ieee.org/document/1164317) algorithm and some neural vocoders. - -##### 1) Griffin-Lim - -
- - - - - - - - - - - - - - - - - - -
- Transformer TTS - - FastSpeech -
LJSpeech LJSpeech
- -
- -
- -
- -
- - - 		- -
- -
- -
- -
- - -
-
- -##### 2) Neural vocoders - -under preparation +Check our [website](https://paddle-parakeet.readthedocs.io/en/latest/demo.html) for audio sampels. ## Copyright and License From c5acfbd8eb3ea97588061c5f07cbd7b0e342aeb7 Mon Sep 17 00:00:00 2001 From: iclementine Date: Mon, 18 Jan 2021 15:15:49 +0800 Subject: [PATCH 18/18] fix typos --- README.md | 2 +- docs/source/advanced.rst | 8 ++++---- docs/source/install.rst | 4 ++-- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index 695af22..7f17950 100644 --- a/README.md +++ b/README.md @@ -38,7 +38,7 @@ sudo apt-get install libsndfile1 ### Install PaddlePaddle -See [install](https://www.paddlepaddle.org.cn/install/quick) for more details. This repo requires PaddlePaddle **2.0.0.rc1** or above. +See [install](https://www.paddlepaddle.org.cn/install/quick) for more details. This repo requires PaddlePaddle **2.0.0rc1** or above. ### Install Parakeet ```bash diff --git a/docs/source/advanced.rst b/docs/source/advanced.rst index 262efc5..25b7e48 100644 --- a/docs/source/advanced.rst +++ b/docs/source/advanced.rst @@ -2,7 +2,7 @@ Advanced Usage ====================== -This sections covers how to extend parakeet by implementing you own models and +This sections covers how to extend parakeet by implementing your own models and experiments. Guidelines on implementation are also elaborated. Model @@ -90,7 +90,7 @@ practice, we use the dataset and dataloader abstraction. Dataset ^^^^^^^^^^ -Dataset is the representation of a set of examples used for a projet. In most of +Dataset is the representation of a set of examples used by a project. In most of the cases, dataset is a collection of examples. Dataset is an object which has methods below. @@ -135,11 +135,11 @@ of running configuration, we use ``yaml`` configuration files. Also, we want to interact with command line options. Some options that usually change according to running environments is provided by command line arguments. -In addition, we wan to override an option in the config file without editing +In addition, we want to override an option in the config file without editing it. Taking these requirements in to consideration, we use `yacs `_ -as a confi management tool. Other tools like `omegaconf `_ +as a config management tool. Other tools like `omegaconf `_ are also powerful and have similar functions. In each example provided, there is a ``config.py``, where the default config is diff --git a/docs/source/install.rst b/docs/source/install.rst index 5555672..49f2ea3 100644 --- a/docs/source/install.rst +++ b/docs/source/install.rst @@ -5,7 +5,7 @@ Installation Install PaddlePaddle ------------------------ -Parakeet requires PaddlePaddle as its backend. Not that 2.0.0rc1 or newer versions +Parakeet requires PaddlePaddle as its backend. Note that 2.0.0rc1 or newer versions of paddle is required. Since paddlepaddle has multiple packages depending on the device (cpu or gpu) @@ -40,7 +40,7 @@ Experimemts in parakeet often involve audio and spectrum processing, thus ``librosa`` and ``soundfile`` are required. ``soundfile`` requires a extra C library ``libsndfile``, which is not always handled by pip. -For windows and mac users, ``libsndfile`` is also installed when Installing +For windows and mac users, ``libsndfile`` is also installed when installing ``soundfile`` via pip, but for linux users, installing ``libsndfile`` via system package manager is required. Example commands for popular distributions are listed below.