Merge pull request #193 from hlyang1992/master

Add the missing negative sign in the formula.

Author: vpj

labml_nn/transformers/rope/__init__.py

@@ -137,7 +137,7 @@ def _build_cache(self, x: torch.Tensor):
         # Get sequence length
         seq_len = x.shape[0]
 
-        # $\Theta = {\theta_i = 10000^{\frac{2(i-1)}{d}}, i \in [1, 2, ..., \frac{d}{2}]}$
+        # $\Theta = {\theta_i = 10000^{-\frac{2(i-1)}{d}}, i \in [1, 2, ..., \frac{d}{2}]}$
         theta = 1. / (self.base ** (torch.arange(0, self.d, 2).float() / self.d)).to(x.device)
 
         # Create position indexes `[0, 1, ..., seq_len - 1]`