fix key len bug

Author: MarinPostma

libsql-wal/src/registry.rs

@@ -521,7 +521,7 @@ where
         let mut seen = RoaringBitmap::new();
         let replicator = StorageReplicator::new(storage, shared.namespace().clone());
         let stream = replicator
-            .stream(&mut seen, local_current_frame_no, 1)
+            .stream(&mut seen, remote_durable_frame_no, 1)
             .peekable();
         let mut injector = Injector::new(shared.clone(), 10)?;
         // use pin to the heap so that we can drop the stream in the loop, and count `seen`.

libsql-wal/src/storage/backend/s3.rs

@@ -198,12 +198,14 @@ impl<IO: Io> S3Backend<IO> {
         folder_key: &FolderKey<'_>,
         frame_no: u64,
     ) -> Result<Option<SegmentKey>> {
+        let lookup_key_prefix = s3_segment_index_lookup_key_prefix(&folder_key);
         let lookup_key = s3_segment_index_lookup_key(&folder_key, frame_no);
 
         let objects = self
             .client
             .list_objects_v2()
             .bucket(&config.bucket)
+            .prefix(lookup_key_prefix)
             .start_after(lookup_key)
             .send()
             .await
@@ -214,15 +216,10 @@ impl<IO: Io> S3Backend<IO> {
         };
         let key = contents.key().expect("misssing key?");
         let key_path: &Path = key.as_ref();
-        let segment_key: SegmentKey = key_path
-            .file_stem()
-            .expect("invalid key")
-            .to_str()
-            .expect("invalid key")
-            .parse()
-            .expect("invalid key");
-
-        Ok(Some(segment_key))
+
+        let key = SegmentKey::validate_from_path(key_path, &folder_key.namespace);
+
+        Ok(key)
     }
 
     // This method could probably be optimized a lot by using indexes and only downloading useful
@@ -335,8 +332,12 @@ fn s3_segment_index_key(folder_key: &FolderKey, segment_key: &SegmentKey) -> Str
     format!("{folder_key}/indexes/{segment_key}")
 }
 
+fn s3_segment_index_lookup_key_prefix(folder_key: &FolderKey) -> String {
+    format!("{folder_key}/indexes/")
+}
+
 fn s3_segment_index_lookup_key(folder_key: &FolderKey, frame_no: u64) -> String {
-    format!("{folder_key}/indexes/{:019}", u64::MAX - frame_no)
+    format!("{folder_key}/indexes/{:020}", u64::MAX - frame_no)
 }
 
 impl<IO> Backend for S3Backend<IO>

libsql-wal/src/storage/mod.rs

@@ -52,12 +52,12 @@ pub enum RestoreOptions {
 /// let meta = SegmentMeta { start_frame_no: 101, end_frame_no: 1000 };
 /// map.insert(SegmentKey(&meta).to_string(), meta);
 ///
-/// map.range(format!("{:019}", u64::MAX - 50)..).next();
-/// map.range(format!("{:019}", u64::MAX - 0)..).next();
-/// map.range(format!("{:019}", u64::MAX - 1)..).next();
-/// map.range(format!("{:019}", u64::MAX - 100)..).next();
-/// map.range(format!("{:019}", u64::MAX - 101)..).next();
-/// map.range(format!("{:019}", u64::MAX - 5000)..).next();
+/// map.range(format!("{:020}", u64::MAX - 50)..).next();
+/// map.range(format!("{:020}", u64::MAX - 0)..).next();
+/// map.range(format!("{:020}", u64::MAX - 1)..).next();
+/// map.range(format!("{:020}", u64::MAX - 100)..).next();
+/// map.range(format!("{:020}", u64::MAX - 101)..).next();
+/// map.range(format!("{:020}", u64::MAX - 5000)..).next();
 /// ```
 #[derive(Debug, Clone, Copy, PartialEq, Eq)]
 pub struct SegmentKey {
@@ -137,7 +137,7 @@ impl fmt::Display for SegmentKey {
     fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
         write!(
             f,
-            "{:019}-{:019}",
+            "{:020}-{:020}",
             u64::MAX - self.start_frame_no,
             u64::MAX - self.end_frame_no,
         )