From c955c4192bec48eefffde23ed17c68fedbd76718 Mon Sep 17 00:00:00 2001
From: Hui Zhang <zhtclz@foxmail.com>
Date: Thu, 18 Feb 2021 19:53:58 +0800
Subject: [PATCH] fix tensorboard error
 https://github.com/pytorch/fairseq/issues/2357

---
 parakeet/training/experiment.py | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/parakeet/training/experiment.py b/parakeet/training/experiment.py
index 94caa66..5f9c588 100644
--- a/parakeet/training/experiment.py
+++ b/parakeet/training/experiment.py
@@ -195,6 +195,8 @@ class ExperimentBase(object):
         except KeyboardInterrupt:
             self.save()
             exit(-1)
+        finally:
+            self.destory()
 
     def setup_output_dir(self):
         """Create a directory used for output.
@@ -218,6 +220,11 @@ class ExperimentBase(object):
 
         self.checkpoint_dir = checkpoint_dir
 
+    @mp_tools.rank_zero_only
+    def destory(self):
+        # https://github.com/pytorch/fairseq/issues/2357
+        self.visualizer.close()
+
     @mp_tools.rank_zero_only
     def setup_visualizer(self):
         """Initialize a visualizer to log the experiment.