15#include <sdsl/bit_vectors.hpp>
129template <data_layout data_layout_mode_ = data_layout::uncompressed>
134 template <data_layout data_layout_mode>
158 13043817825332782213ULL,
159 10650232656628343401ULL,
160 16499269484942379435ULL,
161 4893150838803335377ULL};
175 h *= 11400714819323198485ULL;
177#ifdef __SIZEOF_INT128__
178 h =
static_cast<uint64_t
>((
static_cast<__uint128_t
>(h) *
static_cast<__uint128_t
>(
bin_size_)) >> 64);
192 template <
typename value_t>
230 throw std::logic_error{
"The number of hash functions must be > 0 and <= 5."};
252 std::tie(ibf.bins, ibf.technical_bins, ibf.bin_size_, ibf.hash_shift, ibf.bin_words, ibf.hash_funs);
254 data = sdsl::bit_vector{ibf.data.begin(), ibf.data.end()};
272 std::tie(ibf.bins, ibf.technical_bins, ibf.bin_size_, ibf.hash_shift, ibf.bin_words, ibf.hash_funs);
274 data = sdsl::sd_vector<>{ibf.data};
296 assert(bin.get() <
bins);
301 assert(idx <
data.size());
320 assert(bin.get() <
bins);
338 template <
typename rng_t>
340 void clear(rng_t && bin_range)
noexcept
342 static_assert(std::ranges::forward_range<rng_t>,
"The range of bins to clear must model a forward_range.");
343 static_assert(std::same_as<std::remove_cvref_t<std::ranges::range_reference_t<rng_t>>,
bin_index>,
344 "The reference type of the range to clear must be seqan3::bin_index.");
346 for (
auto && bin : bin_range)
347 assert(bin.get() <
bins);
351 for (
auto && bin : bin_range)
381 size_t new_bins = new_bins_.get();
387 size_t new_bin_words = (new_bins + 63) >> 6;
394 size_t new_technical_bins = new_bin_words << 6;
395 size_t new_bits =
bin_size_ * new_technical_bins;
397 size_t idx_{new_bits}, idx{
data.size()};
400 data.resize(new_bits);
402 for (
size_t i = idx_, j = idx; j > 0; i -= new_technical_bins, j -=
technical_bins)
404 size_t stop = i - new_technical_bins;
406 for (
size_t ii = i - delta, jj = j - 64; stop && ii >= stop; ii -= 64, jj -= 64)
408 uint64_t old =
data.get_int(jj);
410 data.set_int(ii, old);
449 template <
typename value_t = u
int16_t>
525 return !(lhs == rhs);
558 template <cereal_archive archive_t>
582template <data_layout data_layout_mode>
593 class binning_bitvector;
638 assert(ibf_ptr !=
nullptr);
639 assert(result_buffer.
size() == ibf_ptr->bin_count());
642 std::memcpy(&bloom_filter_indices, &ibf_ptr->hash_seeds,
sizeof(
size_t) * ibf_ptr->hash_funs);
644 for (
size_t i = 0; i < ibf_ptr->hash_funs; ++i)
645 bloom_filter_indices[i] = ibf_ptr->hash_and_fit(value, bloom_filter_indices[i]);
647 for (
size_t batch = 0; batch < ibf_ptr->bin_words; ++batch)
650 for (
size_t i = 0; i < ibf_ptr->hash_funs; ++i)
652 assert(bloom_filter_indices[i] < ibf_ptr->
data.size());
653 tmp &= ibf_ptr->data.get_int(bloom_filter_indices[i]);
654 bloom_filter_indices[i] += 64;
657 result_buffer.
data.set_int(batch << 6, tmp);
660 return result_buffer;
670template <data_layout data_layout_mode>
737 return lhs.data == rhs.data;
743 return !(lhs == rhs);
807template <std::
integral value_t>
815 template <
typename binning_bitvector_t>
817 std::same_as<binning_bitvector_t,
819 || std::same_as<binning_bitvector_t,
833 using base_t::base_t;
848 template <
typename binning_bitvector_t>
849 requires is_binning_bitvector<binning_bitvector_t>
853 [
this](
size_t const bin)
867 template <
typename binning_bitvector_t>
868 requires is_binning_bitvector<binning_bitvector_t>
872 [
this](
size_t const bin)
874 assert((*
this)[bin] > 0);
922 template <
typename binning_bitvector_t,
typename on_bin_fn_t>
925 assert(this->
size() >= binning_bitvector.size());
928 auto jump_to_next_1bit = [](
size_t & x)
936 for (
size_t bit_pos = 0; bit_pos < binning_bitvector.size(); bit_pos += 64)
939 size_t bit_sequence = binning_bitvector.raw_data().get_int(bit_pos);
942 for (
size_t bin = bit_pos; bit_sequence != 0u; ++bin, bit_sequence >>= 1)
945 bin += jump_to_next_1bit(bit_sequence);
962template <data_layout data_layout_mode>
963template <
typename value_t>
967 static_assert(std::integral<value_t>,
"The value type must model std::integral.");
994 ibf_ptr(
std::addressof(ibf)),
1025 template <std::ranges::range value_range_t>
1028 assert(ibf_ptr !=
nullptr);
1029 assert(result_buffer.
size() == ibf_ptr->bin_count());
1031 static_assert(std::ranges::input_range<value_range_t>,
"The values must model input_range.");
1032 static_assert(std::unsigned_integral<std::ranges::range_value_t<value_range_t>>,
1033 "An individual value must be an unsigned integral.");
1037 for (
auto && value : values)
1040 return result_buffer;
1045 template <std::ranges::range value_range_t>
Adaptions of concepts from the Cereal library.
A data structure that behaves like a std::vector and can be used to consolidate the results of multip...
Definition interleaved_bloom_filter.hpp:809
static constexpr bool is_binning_bitvector
Is binning_bitvector_t a seqan3::interleaved_bloom_filter::membership_agent_type::binning_bitvector?
Definition interleaved_bloom_filter.hpp:816
counting_vector & operator+=(counting_vector const &rhs)
Bin-wise addition of two seqan3::counting_vectors.
Definition interleaved_bloom_filter.hpp:890
void for_each_set_bin(binning_bitvector_t &&binning_bitvector, on_bin_fn_t &&on_bin_fn)
Enumerates all bins of a seqan3::interleaved_bloom_filter::membership_agent_type::binning_bitvector.
Definition interleaved_bloom_filter.hpp:923
counting_vector & operator=(counting_vector const &)=default
Defaulted.
counting_vector & operator=(counting_vector &&)=default
Defaulted.
counting_vector & operator+=(binning_bitvector_t const &binning_bitvector)
Bin-wise adds the bits of a seqan3::interleaved_bloom_filter::membership_agent_type::binning_bitvecto...
Definition interleaved_bloom_filter.hpp:850
counting_vector(counting_vector const &)=default
Defaulted.
~counting_vector()=default
Defaulted.
counting_vector(counting_vector &&)=default
Defaulted.
counting_vector & operator-=(binning_bitvector_t const &binning_bitvector)
Bin-wise subtracts the bits of a seqan3::interleaved_bloom_filter::membership_agent_type::binning_bit...
Definition interleaved_bloom_filter.hpp:869
counting_vector()=default
Defaulted.
counting_vector & operator-=(counting_vector const &rhs)
Bin-wise substraction of two seqan3::counting_vectors.
Definition interleaved_bloom_filter.hpp:903
CRTP base class to declare a strong typedef for a regular type to avoid ambiguous parameter settings ...
Definition strong_type.hpp:174
constexpr value_t & get() &noexcept
Returns the underlying value.
Definition strong_type.hpp:201
constexpr strong_type() noexcept=default
Defaulted.
Manages counting ranges of values for the seqan3::interleaved_bloom_filter.
Definition interleaved_bloom_filter.hpp:965
counting_vector< value_t > const & bulk_count(value_range_t &&values) &noexcept
Counts the occurrences in each bin for all values in a range.
Definition interleaved_bloom_filter.hpp:1026
membership_agent_type membership_agent
Store a seqan3::interleaved_bloom_filter::membership_agent to call bulk_contains.
Definition interleaved_bloom_filter.hpp:976
~counting_agent_type()=default
Defaulted.
counting_agent_type()=default
Defaulted.
counting_agent_type(counting_agent_type &&)=default
Defaulted.
counting_agent_type(counting_agent_type const &)=default
Defaulted.
counting_vector< value_t > const & bulk_count(value_range_t &&values) &&noexcept=delete
Counts the occurrences in each bin for all values in a range.
counting_agent_type & operator=(counting_agent_type const &)=default
Defaulted.
counting_vector< value_t > result_buffer
Stores the result of bulk_count().
Definition interleaved_bloom_filter.hpp:1001
counting_agent_type(ibf_t const &ibf)
Construct a counting_agent_type for an existing seqan3::interleaved_bloom_filter.
Definition interleaved_bloom_filter.hpp:993
counting_agent_type & operator=(counting_agent_type &&)=default
Defaulted.
A bitvector representing the result of a call to bulk_contains of the seqan3::interleaved_bloom_filte...
Definition interleaved_bloom_filter.hpp:672
binning_bitvector(binning_bitvector &&)=default
Defaulted.
constexpr data_type & raw_data() noexcept
Provides direct, unsafe access to the underlying data structure.
Definition interleaved_bloom_filter.hpp:771
auto end() noexcept
Returns an iterator to the element following the last element of the container.
Definition interleaved_bloom_filter.hpp:719
constexpr data_type const & raw_data() const noexcept
Provides direct, unsafe access to the underlying data structure.
Definition interleaved_bloom_filter.hpp:777
~binning_bitvector()=default
Defaulted.
binning_bitvector(binning_bitvector const &)=default
Defaulted.
auto end() const noexcept
Returns an iterator to the element following the last element of the container.
Definition interleaved_bloom_filter.hpp:725
data_type data
The bitvector.
Definition interleaved_bloom_filter.hpp:677
auto operator[](size_t const i) const noexcept
Return the i-th element.
Definition interleaved_bloom_filter.hpp:758
binning_bitvector(size_t const size)
Construct with given size.
Definition interleaved_bloom_filter.hpp:693
size_t size() const noexcept
Returns the number of elements.
Definition interleaved_bloom_filter.hpp:698
binning_bitvector & operator=(binning_bitvector &&)=default
Defaulted.
auto begin() noexcept
Returns an iterator to the first element of the container.
Definition interleaved_bloom_filter.hpp:707
auto begin() const noexcept
Returns an iterator to the first element of the container.
Definition interleaved_bloom_filter.hpp:713
auto operator[](size_t const i) noexcept
Return the i-th element.
Definition interleaved_bloom_filter.hpp:751
binning_bitvector & operator=(binning_bitvector const &)=default
Defaulted.
sdsl::bit_vector data_type
The underlying datatype to use.
Definition interleaved_bloom_filter.hpp:675
binning_bitvector()=default
Defaulted.
friend bool operator==(binning_bitvector const &lhs, binning_bitvector const &rhs) noexcept
Test for equality.
Definition interleaved_bloom_filter.hpp:735
friend bool operator!=(binning_bitvector const &lhs, binning_bitvector const &rhs) noexcept
Test for inequality.
Definition interleaved_bloom_filter.hpp:741
Manages membership queries for the seqan3::interleaved_bloom_filter.
Definition interleaved_bloom_filter.hpp:584
binning_bitvector const & bulk_contains(size_t const value) &noexcept
Determines set membership of a given value.
Definition interleaved_bloom_filter.hpp:636
~membership_agent_type()=default
Defaulted.
membership_agent_type(ibf_t const &ibf)
Construct a membership_agent_type from a seqan3::interleaved_bloom_filter.
Definition interleaved_bloom_filter.hpp:609
membership_agent_type & operator=(membership_agent_type const &)=default
Defaulted.
binning_bitvector const & bulk_contains(size_t const value) &&noexcept=delete
Determines set membership of a given value.
membership_agent_type(membership_agent_type &&)=default
Defaulted.
membership_agent_type(membership_agent_type const &)=default
Defaulted.
membership_agent_type & operator=(membership_agent_type &&)=default
Defaulted.
membership_agent_type()=default
Defaulted.
binning_bitvector result_buffer
Stores the result of bulk_contains().
Definition interleaved_bloom_filter.hpp:614
The IBF binning directory. A data structure that efficiently answers set-membership queries for multi...
Definition interleaved_bloom_filter.hpp:131
interleaved_bloom_filter(interleaved_bloom_filter< data_layout::uncompressed > const &ibf)
Construct a compressed Interleaved Bloom Filter.
Definition interleaved_bloom_filter.hpp:268
void emplace(size_t const value, bin_index const bin) noexcept
Inserts a value into a specific bin.
Definition interleaved_bloom_filter.hpp:293
size_t hash_funs
The number of hash functions.
Definition interleaved_bloom_filter.hpp:153
size_t bin_words
The number of 64-bit integers needed to store bins many bits (e.g. bins = 50 -> bin_words = 1).
Definition interleaved_bloom_filter.hpp:151
interleaved_bloom_filter(interleaved_bloom_filter< data_layout::compressed > const &ibf)
Construct an uncompressed Interleaved Bloom Filter from a compressed one.
Definition interleaved_bloom_filter.hpp:248
size_t bin_size_
The size of each bin in bits.
Definition interleaved_bloom_filter.hpp:147
interleaved_bloom_filter & operator=(interleaved_bloom_filter const &)=default
Defaulted.
membership_agent_type membership_agent() const
Returns a seqan3::interleaved_bloom_filter::membership_agent_type to be used for lookup.
Definition interleaved_bloom_filter.hpp:433
size_t hash_function_count() const noexcept
Returns the number of hash functions used in the Interleaved Bloom Filter.
Definition interleaved_bloom_filter.hpp:462
constexpr size_t hash_and_fit(size_t h, size_t const seed) const
Perturbs a value and fits it into the vector.
Definition interleaved_bloom_filter.hpp:170
constexpr data_type const & raw_data() const noexcept
Provides direct, unsafe access to the underlying data structure.
Definition interleaved_bloom_filter.hpp:545
void clear(bin_index const bin) noexcept
Clears a specific bin.
Definition interleaved_bloom_filter.hpp:317
interleaved_bloom_filter(seqan3::bin_count bins_, seqan3::bin_size size, seqan3::hash_function_count funs=seqan3::hash_function_count{2u})
Construct an uncompressed Interleaved Bloom Filter.
Definition interleaved_bloom_filter.hpp:218
interleaved_bloom_filter()=default
Defaulted.
size_t hash_shift
The number of bits to shift the hash value before doing multiplicative hashing.
Definition interleaved_bloom_filter.hpp:149
static constexpr std::array< size_t, 5 > hash_seeds
Precalculated seeds for multiplicative hashing. We use large irrational numbers for a uniform hashing...
Definition interleaved_bloom_filter.hpp:157
counting_agent_type< value_t > counting_agent() const
Returns a seqan3::interleaved_bloom_filter::counting_agent_type to be used for counting.
Definition interleaved_bloom_filter.hpp:450
constexpr data_type & raw_data() noexcept
Provides direct, unsafe access to the underlying data structure.
Definition interleaved_bloom_filter.hpp:539
interleaved_bloom_filter & operator=(interleaved_bloom_filter &&)=default
Defaulted.
friend bool operator!=(interleaved_bloom_filter const &lhs, interleaved_bloom_filter const &rhs) noexcept
Test for inequality.
Definition interleaved_bloom_filter.hpp:523
interleaved_bloom_filter(interleaved_bloom_filter &&)=default
Defaulted.
void increase_bin_number_to(bin_count const new_bins_)
Increases the number of bins stored in the Interleaved Bloom Filter.
Definition interleaved_bloom_filter.hpp:378
size_t bin_count() const noexcept
Returns the number of bins that the Interleaved Bloom Filter manages.
Definition interleaved_bloom_filter.hpp:470
void clear(rng_t &&bin_range) noexcept
Clears a range of bins.
Definition interleaved_bloom_filter.hpp:340
size_t bin_size() const noexcept
Returns the size of a single bin that the Interleaved Bloom Filter manages.
Definition interleaved_bloom_filter.hpp:478
size_t bins
The number of bins specified by the user.
Definition interleaved_bloom_filter.hpp:143
size_t bit_size() const noexcept
Returns the size of the underlying bitvector.
Definition interleaved_bloom_filter.hpp:486
interleaved_bloom_filter(interleaved_bloom_filter const &)=default
Defaulted.
~interleaved_bloom_filter()=default
Defaulted.
size_t technical_bins
The number of bins stored in the IBF (next multiple of 64 of bins).
Definition interleaved_bloom_filter.hpp:145
friend bool operator==(interleaved_bloom_filter const &lhs, interleaved_bloom_filter const &rhs) noexcept
Test for equality.
Definition interleaved_bloom_filter.hpp:500
static constexpr data_layout data_layout_mode
Indicates whether the Interleaved Bloom Filter is compressed.
Definition interleaved_bloom_filter.hpp:188
data_type data
The bitvector.
Definition interleaved_bloom_filter.hpp:155
@ offset
Sequence (seqan3::field::seq) relative start position (0-based), unsigned value.
data_layout
Determines if the Interleaved Bloom Filter is compressed.
Definition interleaved_bloom_filter.hpp:25
@ uncompressed
The Interleaved Bloom Filter is uncompressed.
Definition interleaved_bloom_filter.hpp:26
@ compressed
The Interleaved Bloom Filter is compressed.
Definition interleaved_bloom_filter.hpp:27
The main SeqAn3 namespace.
Definition aligned_sequence_concept.hpp:26
SeqAn specific customisations in the standard namespace.
Provides basic data structure for strong types.
A strong type that represents the number of bins for the seqan3::interleaved_bloom_filter.
Definition interleaved_bloom_filter.hpp:33
A strong type that represents the bin index for the seqan3::interleaved_bloom_filter.
Definition interleaved_bloom_filter.hpp:54
A strong type that represents the number of bits for each bin in the seqan3::interleaved_bloom_filter...
Definition interleaved_bloom_filter.hpp:40
A strong type that represents the number of hash functions for the seqan3::interleaved_bloom_filter.
Definition interleaved_bloom_filter.hpp:47
strong_type for seed.
Definition minimiser_hash.hpp:22