diff --git a/models/s4/s4.py b/models/s4/s4.py
index deb5341..bf26cab 100644
--- a/models/s4/s4.py
+++ b/models/s4/s4.py
@@ -1918,6 +1918,7 @@ def forward(self, x, lengths=None, **kwargs): # absorbs return_output and transf
 
         y, state = self.layer(x, **kwargs)
 
+        y = self.activation(y)
 
         if self.gate is not None:
             y = self.output_gate(y)