diff --git a/parakeet/training/experiment.py b/parakeet/training/experiment.py
index 482e012..94caa66 100644
--- a/parakeet/training/experiment.py
+++ b/parakeet/training/experiment.py
@@ -124,6 +124,7 @@ class ExperimentBase(object):
         """
         dist.init_parallel_env()
 
+    @mp_tools.rank_zero_only
     def save(self):
         """Save checkpoint (model parameters and optimizer states).
         """