New SequentialLabeling::build_dcf method

Author: vigna

algo/src/distances/hyperball.rs

@@ -182,37 +182,13 @@ use webgraph::utils::Granularity;
 /// # Examples
 ///
 /// ```
-/// use webgraph::graphs::vec_graph::VecGraph;
-/// use webgraph::graphs::bvgraph::DCF;
-/// use webgraph::traits::{RandomAccessLabeling, SequentialLabeling};
-/// use webgraph_algo::distances::hyperball::*;
-/// use dsi_progress_logger::no_logging;
-/// use sux::prelude::*;
-/// use rand::SeedableRng;
-/// use lender::prelude::*;
-///
-/// // A small graph: 0 → 1 → 2 → 0, 1 → 3
+/// # use webgraph::graphs::vec_graph::VecGraph;
+/// # use webgraph::traits::SequentialLabeling;
+/// # use webgraph_algo::distances::hyperball::*;
+/// # use dsi_progress_logger::no_logging;
+/// # use rand::SeedableRng;
 /// let graph = VecGraph::from_arcs([(0, 1), (1, 2), (2, 0), (1, 3)]);
-///
-/// // Build the degree cumulative function (DCF)
-/// let mut efb = EliasFanoBuilder::new(
-///     graph.num_nodes() + 1,
-///     graph.num_arcs() as usize,
-/// );
-/// efb.push(0);
-/// let mut cumul = 0;
-/// let mut lender = graph.iter();
-/// while let Some((_, succs)) = lender.next() {
-///     cumul += succs.into_iter().count();
-///     efb.push(cumul);
-/// }
-/// let dcf: DCF = unsafe {
-///     efb.build().map_high_bits(|high_bits| {
-///         SelectZeroAdaptConst::<_, _, 12, 4>::new(
-///             SelectAdaptConst::<_, _, 12, 4>::new(high_bits),
-///         )
-///     })
-/// };
+/// let dcf = graph.build_dcf();
 ///
 /// // Build and run HyperBall (neighborhood function only)
 /// let rng = rand::rngs::SmallRng::seed_from_u64(0);
@@ -230,28 +206,13 @@ use webgraph::utils::Granularity;
 ///
 /// ```
 /// # use webgraph::graphs::vec_graph::VecGraph;
-/// # use webgraph::graphs::bvgraph::DCF;
-/// # use webgraph::traits::{RandomAccessLabeling, SequentialLabeling};
+/// # use webgraph::traits::SequentialLabeling;
 /// # use webgraph_algo::distances::hyperball::*;
 /// # use dsi_progress_logger::no_logging;
-/// # use sux::prelude::*;
 /// # use rand::SeedableRng;
-/// # use lender::prelude::*;
 /// # let graph = VecGraph::from_arcs([(0, 1), (1, 2), (2, 0), (1, 3)]);
-/// # let mut efb = EliasFanoBuilder::new(
-/// #     graph.num_nodes() + 1, graph.num_arcs() as usize);
-/// # efb.push(0);
-/// # let mut cumul = 0;
-/// # let mut lender = graph.iter();
-/// # while let Some((_, succs)) = lender.next() {
-/// #     cumul += succs.into_iter().count();
-/// #     efb.push(cumul);
-/// # }
-/// # let dcf: DCF = unsafe {
-/// #     efb.build().map_high_bits(|high_bits| {
-/// #         SelectZeroAdaptConst::<_, _, 12, 4>::new(
-/// #             SelectAdaptConst::<_, _, 12, 4>::new(high_bits))})};
-/// let rng = rand::rngs::SmallRng::seed_from_u64(0);
+/// # let dcf = graph.build_dcf();
+/// # let rng = rand::rngs::SmallRng::seed_from_u64(0);
 /// let mut hyperball = HyperBallBuilder::with_hyper_log_log(
 ///     &graph, None::<&VecGraph>, &dcf, 6, None,
 /// )?

algo/tests/test_llp.rs

@@ -5,40 +5,12 @@
  */
 
 use anyhow::Result;
-use lender::prelude::*;
 use predicates::prelude::PredicateBooleanExt;
-use sux::prelude::*;
-use webgraph::graphs::bvgraph::DCF;
 use webgraph::graphs::vec_graph::VecGraph;
 use webgraph::traits::SequentialLabeling;
 use webgraph_algo::llp;
 use webgraph_algo::llp::preds::*;
 
-/// Builds a degree cumulative function (DCF) from a graph.
-///
-/// The DCF is an Elias-Fano representation of the sequence
-/// 0, d₀, d₀+d₁, ..., total_arcs where dᵢ is the outdegree of node i.
-fn build_dcf(graph: &VecGraph) -> DCF {
-    let num_nodes = graph.num_nodes();
-    let num_arcs = graph.num_arcs_hint().unwrap_or(0) as usize;
-
-    let mut efb = EliasFanoBuilder::new(num_nodes + 1, num_arcs);
-    efb.push(0);
-    let mut cumul = 0usize;
-    let mut lender = graph.iter();
-    while let Some((_node, succs)) = lender.next() {
-        cumul += succs.into_iter().count();
-        efb.push(cumul);
-    }
-
-    let ef = efb.build();
-    unsafe {
-        ef.map_high_bits(|bits| {
-            SelectZeroAdaptConst::<_, _, 12, 4>::new(SelectAdaptConst::<_, _, 12, 4>::new(bits))
-        })
-    }
-}
-
 #[test]
 fn test_llp_small_symmetric_graph() -> Result<()> {
     use webgraph::utils::Granularity;
@@ -60,7 +32,7 @@ fn test_llp_small_symmetric_graph() -> Result<()> {
     let num_nodes = graph.num_nodes();
     assert_eq!(num_nodes, 4);
 
-    let deg_cumul = build_dcf(&graph);
+    let deg_cumul = graph.build_dcf();
 
     let dir = tempfile::tempdir()?;
     let gammas = vec![0.0, 1.0];
@@ -101,7 +73,7 @@ fn test_llp_labels_only_and_combine() -> Result<()> {
         (4, 3),
     ]);
     let num_nodes = graph.num_nodes();
-    let deg_cumul = build_dcf(&graph);
+    let deg_cumul = graph.build_dcf();
 
     let dir = tempfile::tempdir()?;
 
@@ -136,7 +108,7 @@ fn test_llp_multiple_gammas() -> Result<()> {
         (0, 4),
         (4, 0),
     ]);
-    let deg_cumul = build_dcf(&graph);
+    let deg_cumul = graph.build_dcf();
 
     let dir = tempfile::tempdir()?;
     let gammas = vec![0.0, 0.5, 1.0, 2.0];
@@ -175,7 +147,7 @@ fn test_llp_complete_graph() -> Result<()> {
         (2, 3),
         (3, 2),
     ]);
-    let deg_cumul = build_dcf(&graph);
+    let deg_cumul = graph.build_dcf();
 
     let dir = tempfile::tempdir()?;
 

webgraph/CHANGELOG.md

@@ -13,6 +13,11 @@
 
 - Support for π codes, both in the file format and in the CLI tools.
 
+- `SequentialLabeling::build_dcf` new default method for building a degree
+  cumulative function. It is overridden by more efficient methods, for example,
+  in `BvGraphSeq`. All iterators have fast constant-time implementations of
+  `count` whenever possible to support the method.
+
 ### Changed
 
 - All parallel methods now use the current Rayon global thread pool rather than

webgraph/src/graphs/btree_graph.rs

@@ -397,6 +397,10 @@ impl<L: Clone + 'static> Iterator for LabeledSucc<'_, L> {
     fn next(&mut self) -> Option<Self::Item> {
         self.0.next().map(|(succ, labels)| (*succ, labels.clone()))
     }
+
+    fn count(self) -> usize {
+        self.len()
+    }
 }
 
 impl<L: Clone + 'static> ExactSizeIterator for LabeledSucc<'_, L> {
@@ -418,6 +422,10 @@ impl Iterator for Succ<'_> {
     fn next(&mut self) -> Option<Self::Item> {
         self.0.next()
     }
+
+    fn count(self) -> usize {
+        self.len()
+    }
 }
 
 impl ExactSizeIterator for Succ<'_> {

webgraph/src/graphs/bvgraph/masked_iter.rs

@@ -96,4 +96,8 @@ impl<I: Iterator<Item = usize>> Iterator for MaskedIter<I> {
         self.blocks[self.block_idx] -= 1;
         result
     }
+
+    fn count(self) -> usize {
+        self.len()
+    }
 }

webgraph/src/graphs/bvgraph/mod.rs

@@ -195,7 +195,12 @@ impl<T: for<'a> DeserInner<DeserType<'a>: IndexedSeq<Input = usize, Output<'a> =
 {
 }
 
-/// The default version of EliasFano we use for the cumulative function of degrees.
+/// The default type we use for the cumulative function of degrees.
+///
+/// It provides an indexed dictionary](sux::traits::indexed_dict::IndexedDict) with
+/// [successor](sux::traits::indexed_dict::Succ) and [predecessor](sux::traits::indexed_dict::Pred) support.
+///
+/// This is the type returned by [`crate::traits::labels::SequentialLabeling::build_dcf`].
 pub type DCF = sux::dict::EliasFano<
     sux::rank_sel::SelectZeroAdaptConst<
         sux::rank_sel::SelectAdaptConst<sux::bits::BitVec<Box<[usize]>>, Box<[usize]>, 12, 4>,

webgraph/src/graphs/bvgraph/random_access.rs

@@ -479,6 +479,10 @@ impl<D: Decode> Iterator for Succ<D> {
         Some(min)
     }
 
+    fn count(self) -> usize {
+        self.len()
+    }
+
     fn size_hint(&self) -> (usize, Option<usize>) {
         (self.size, Some(self.size))
     }

webgraph/src/graphs/bvgraph/sequential.rs

@@ -100,6 +100,33 @@ impl<F: SequentialDecoderFactory> SequentialLabeling for BvGraphSeq<F> {
 
         iter
     }
+
+    fn build_dcf(&self) -> DCF {
+        let n = self.num_nodes();
+        let num_arcs = self
+            .num_arcs_hint()
+            .expect("build_dcf requires num_arcs_hint()") as usize;
+        let mut efb = sux::dict::EliasFanoBuilder::new(n + 1, num_arcs);
+        efb.push(0);
+        let mut cumul_deg = 0usize;
+        let mut iter = OffsetDegIter::new(
+            self.factory.new_decoder().unwrap(),
+            n,
+            self.compression_window,
+            self.min_interval_length,
+        );
+        for _ in 0..n {
+            cumul_deg += iter.next_degree().unwrap();
+            efb.push(cumul_deg);
+        }
+        unsafe {
+            efb.build().map_high_bits(|high_bits| {
+                sux::rank_sel::SelectZeroAdaptConst::<_, _, 12, 4>::new(
+                    sux::rank_sel::SelectAdaptConst::<_, _, 12, 4>::new(high_bits),
+                )
+            })
+        }
+    }
 }
 
 impl<F: SequentialDecoderFactory> SequentialGraph for BvGraphSeq<F> {}

webgraph/src/graphs/csr_graph.rs

@@ -10,7 +10,11 @@ use crate::traits::*;
 use common_traits::UnsignedInt;
 use epserde::Epserde;
 use lender::{IntoLender, Lend, Lender, Lending, check_covariance, for_};
-use sux::{bits::BitFieldVec, dict::EliasFanoBuilder, prelude::SelectAdaptConst};
+use sux::{
+    bits::BitFieldVec,
+    dict::EliasFanoBuilder,
+    rank_sel::{SelectAdaptConst, SelectZeroAdaptConst},
+};
 use value_traits::{
     iter::{IterFrom, IterateByValueFrom},
     slices::SliceByValue,
@@ -324,6 +328,22 @@ where
             successors_iter: self.successors.iter_value_from(offset),
         }
     }
+
+    fn build_dcf(&self) -> crate::graphs::bvgraph::DCF {
+        let n = self.num_nodes();
+        let num_arcs = self.num_arcs_hint().unwrap() as usize;
+        let mut efb = EliasFanoBuilder::new(n + 1, num_arcs);
+        for val in self.dcf.iter_value_from(0).take(n + 1) {
+            efb.push(val);
+        }
+        unsafe {
+            efb.build().map_high_bits(|high_bits| {
+                SelectZeroAdaptConst::<_, _, 12, 4>::new(
+                    SelectAdaptConst::<_, _, 12, 4>::new(high_bits),
+                )
+            })
+        }
+    }
 }
 
 impl<DCF, S> SequentialLabeling for CsrSortedGraph<DCF, S>
@@ -351,18 +371,22 @@ where
     fn iter_from(&self, from: usize) -> Self::Lender<'_> {
         LenderSortedImpl(self.0.iter_from(from))
     }
+
+    fn build_dcf(&self) -> crate::graphs::bvgraph::DCF {
+        self.0.build_dcf()
+    }
 }
 
-impl<DCF, S> SequentialGraph for CsrGraph<DCF, S>
+impl<D, S> SequentialGraph for CsrGraph<D, S>
 where
-    DCF: SliceByValue + IterateByValueFrom<Item = usize>,
+    D: SliceByValue + IterateByValueFrom<Item = usize>,
     S: SliceByValue + IterateByValueFrom<Item = usize>,
 {
 }
 
-impl<DCF, S> SequentialGraph for CsrSortedGraph<DCF, S>
+impl<D, S> SequentialGraph for CsrSortedGraph<D, S>
 where
-    DCF: SliceByValue + IterateByValueFrom<Item = usize>,
+    D: SliceByValue + IterateByValueFrom<Item = usize>,
     S: SliceByValue + IterateByValueFrom<Item = usize>,
 {
 }
@@ -577,6 +601,11 @@ impl<D: Iterator<Item = usize>> Iterator for SeqSucc<'_, D> {
         self.succ_iter.next()
     }
 
+    #[inline(always)]
+    fn count(self) -> usize {
+        self.len()
+    }
+
     #[inline(always)]
     fn size_hint(&self) -> (usize, Option<usize>) {
         let len = self.last_offset - *self.current_offset;

webgraph/src/graphs/permuted_graph.rs

@@ -162,6 +162,10 @@ impl<I: Iterator<Item = usize>, P: SliceByValue<Value = usize>> Iterator for Suc
     fn next(&mut self) -> Option<Self::Item> {
         self.iter.next().map(|succ| self.perm.index_value(succ))
     }
+
+    fn count(self) -> usize {
+        self.iter.count()
+    }
 }
 
 impl<I: ExactSizeIterator<Item = usize>, P: SliceByValue<Value = usize>> ExactSizeIterator