diff --git a/egs/librispeech/ASR/zipformer/decode_stream.py b/egs/librispeech/ASR/zipformer/decode_stream.py
index 946db275ce..d6918bf328 100644
--- a/egs/librispeech/ASR/zipformer/decode_stream.py
+++ b/egs/librispeech/ASR/zipformer/decode_stream.py
@@ -79,12 +79,12 @@ def __init__(
         self.pad_length = 7 + 2 * 3
 
         if params.decoding_method == "greedy_search":
-            self.hyp = [params.blank_id] * params.context_size
+            self.hyp = [-1] * (params.context_size - 1) + [params.blank_id]
         elif params.decoding_method == "modified_beam_search":
             self.hyps = HypothesisList()
             self.hyps.add(
                 Hypothesis(
-                    ys=[params.blank_id] * params.context_size,
+                    ys=[-1] * (params.context_size - 1) + [params.blank_id],
                     log_prob=torch.zeros(1, dtype=torch.float32, device=device),
                 )
             )