001/* 002 * Copyright (C) 2011 The Guava Authors 003 * 004 * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except 005 * in compliance with the License. You may obtain a copy of the License at 006 * 007 * http://www.apache.org/licenses/LICENSE-2.0 008 * 009 * Unless required by applicable law or agreed to in writing, software distributed under the License 010 * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express 011 * or implied. See the License for the specific language governing permissions and limitations under 012 * the License. 013 */ 014 015package com.google.common.hash; 016 017import static com.google.common.base.Preconditions.checkArgument; 018import static com.google.common.base.Preconditions.checkNotNull; 019import static java.lang.Byte.toUnsignedInt; 020import static java.lang.Math.log; 021import static java.lang.Math.max; 022import static java.lang.Math.round; 023 024import com.google.common.annotations.Beta; 025import com.google.common.annotations.J2ktIncompatible; 026import com.google.common.annotations.VisibleForTesting; 027import com.google.common.base.Predicate; 028import com.google.common.hash.BloomFilterStrategies.LockFreeBitArray; 029import com.google.common.math.DoubleMath; 030import com.google.common.primitives.SignedBytes; 031import com.google.common.primitives.UnsignedBytes; 032import com.google.errorprone.annotations.CanIgnoreReturnValue; 033import com.google.errorprone.annotations.InlineMe; 034import java.io.DataInputStream; 035import java.io.DataOutputStream; 036import java.io.IOException; 037import java.io.InputStream; 038import java.io.InvalidObjectException; 039import java.io.ObjectInputStream; 040import java.io.OutputStream; 041import java.io.Serializable; 042import java.math.RoundingMode; 043import java.util.Objects; 044import java.util.stream.Collector; 045import org.jspecify.annotations.Nullable; 046 047/** 048 * A Bloom filter for instances of {@code T}. A Bloom filter offers an approximate containment test 049 * with one-sided error: if it claims that an element is contained in it, this might be in error, 050 * but if it claims that an element is <i>not</i> contained in it, then this is definitely true. 051 * 052 * <p>If you are unfamiliar with Bloom filters, this nice <a 053 * href="http://llimllib.github.io/bloomfilter-tutorial/">tutorial</a> may help you understand how 054 * they work. 055 * 056 * <p>The false positive probability ({@code FPP}) of a Bloom filter is defined as the probability 057 * that {@link #mightContain(Object)} will erroneously return {@code true} for an object that has 058 * not actually been put in the {@code BloomFilter}. 059 * 060 * <p>Bloom filters are serializable. They also support a more compact serial representation via the 061 * {@link #writeTo} and {@link #readFrom} methods. Both serialized forms will continue to be 062 * supported by future versions of this library. However, serial forms generated by newer versions 063 * of the code may not be readable by older versions of the code (e.g., a serialized Bloom filter 064 * generated today may <i>not</i> be readable by a binary that was compiled 6 months ago). 065 * 066 * <p>As of Guava 23.0, this class is thread-safe and lock-free. It internally uses atomics and 067 * compare-and-swap to ensure correctness when multiple threads are used to access it. 068 * 069 * @param <T> the type of instances that the {@code BloomFilter} accepts 070 * @author Dimitris Andreou 071 * @author Kevin Bourrillion 072 * @since 11.0 (thread-safe since 23.0) 073 */ 074@J2ktIncompatible 075@Beta 076public final class BloomFilter<T extends @Nullable Object> implements Predicate<T>, Serializable { 077 /** 078 * A strategy to translate T instances, to {@code numHashFunctions} bit indexes. 079 * 080 * <p>Implementations should be collections of pure functions (i.e. stateless). 081 */ 082 interface Strategy extends Serializable { 083 084 /** 085 * Sets {@code numHashFunctions} bits of the given bit array, by hashing a user element. 086 * 087 * <p>Returns whether any bits changed as a result of this operation. 088 */ 089 <T extends @Nullable Object> boolean put( 090 @ParametricNullness T object, 091 Funnel<? super T> funnel, 092 int numHashFunctions, 093 LockFreeBitArray bits); 094 095 /** 096 * Queries {@code numHashFunctions} bits of the given bit array, by hashing a user element; 097 * returns {@code true} if and only if all selected bits are set. 098 */ 099 <T extends @Nullable Object> boolean mightContain( 100 @ParametricNullness T object, 101 Funnel<? super T> funnel, 102 int numHashFunctions, 103 LockFreeBitArray bits); 104 105 /** 106 * Identifier used to encode this strategy, when marshalled as part of a BloomFilter. Only 107 * values in the [-128, 127] range are valid for the compact serial form. Non-negative values 108 * are reserved for enums defined in BloomFilterStrategies; negative values are reserved for any 109 * custom, stateful strategy we may define (e.g. any kind of strategy that would depend on user 110 * input). 111 */ 112 int ordinal(); 113 } 114 115 /** The bit set of the BloomFilter (not necessarily power of 2!) */ 116 private final LockFreeBitArray bits; 117 118 /** Number of hashes per element */ 119 private final int numHashFunctions; 120 121 /** The funnel to translate Ts to bytes */ 122 private final Funnel<? super T> funnel; 123 124 /** The strategy we employ to map an element T to {@code numHashFunctions} bit indexes. */ 125 private final Strategy strategy; 126 127 /** Natural logarithm of 2, used to optimize calculations in Bloom filter sizing. */ 128 private static final double LOG_TWO = log(2); 129 130 /** Square of the natural logarithm of 2, reused to optimize the bit size calculation. */ 131 private static final double SQUARED_LOG_TWO = LOG_TWO * LOG_TWO; 132 133 /** Creates a BloomFilter. */ 134 private BloomFilter( 135 LockFreeBitArray bits, int numHashFunctions, Funnel<? super T> funnel, Strategy strategy) { 136 checkArgument(numHashFunctions > 0, "numHashFunctions (%s) must be > 0", numHashFunctions); 137 checkArgument( 138 numHashFunctions <= 255, "numHashFunctions (%s) must be <= 255", numHashFunctions); 139 this.bits = checkNotNull(bits); 140 this.numHashFunctions = numHashFunctions; 141 this.funnel = checkNotNull(funnel); 142 this.strategy = checkNotNull(strategy); 143 } 144 145 /** 146 * Creates a new {@code BloomFilter} that's a copy of this instance. The new instance is equal to 147 * this instance but shares no mutable state. 148 * 149 * @since 12.0 150 */ 151 public BloomFilter<T> copy() { 152 return new BloomFilter<>(bits.copy(), numHashFunctions, funnel, strategy); 153 } 154 155 /** 156 * Returns {@code true} if the element <i>might</i> have been put in this Bloom filter, {@code 157 * false} if this is <i>definitely</i> not the case. 158 */ 159 public boolean mightContain(@ParametricNullness T object) { 160 return strategy.mightContain(object, funnel, numHashFunctions, bits); 161 } 162 163 /** 164 * @deprecated Provided only to satisfy the {@link Predicate} interface; use {@link #mightContain} 165 * instead. 166 */ 167 @InlineMe(replacement = "this.mightContain(input)") 168 @Deprecated 169 @Override 170 public boolean apply(@ParametricNullness T input) { 171 return mightContain(input); 172 } 173 174 /** 175 * @deprecated Provided only to satisfy the {@link java.util.function.Predicate} interface; use 176 * {@link #mightContain} instead. 177 * @since 21.0 178 */ 179 @InlineMe(replacement = "this.mightContain(input)") 180 @Deprecated 181 @Override 182 public boolean test(@ParametricNullness T input) { 183 return mightContain(input); 184 } 185 186 /** 187 * Puts an element into this {@code BloomFilter}. Ensures that subsequent invocations of {@link 188 * #mightContain(Object)} with the same element will always return {@code true}. 189 * 190 * @return true if the Bloom filter's bits changed as a result of this operation. If the bits 191 * changed, this is <i>definitely</i> the first time {@code object} has been added to the 192 * filter. If the bits haven't changed, this <i>might</i> be the first time {@code object} has 193 * been added to the filter. Note that {@code put(t)} always returns the <i>opposite</i> 194 * result to what {@code mightContain(t)} would have returned at the time it is called. 195 * @since 12.0 (present in 11.0 with {@code void} return type}) 196 */ 197 @CanIgnoreReturnValue 198 public boolean put(@ParametricNullness T object) { 199 return strategy.put(object, funnel, numHashFunctions, bits); 200 } 201 202 /** 203 * Returns the probability that {@link #mightContain(Object)} will erroneously return {@code true} 204 * for an object that has not actually been put in the {@code BloomFilter}. 205 * 206 * <p>Ideally, this number should be close to the {@code fpp} parameter passed in {@link 207 * #create(Funnel, int, double)}, or smaller. If it is significantly higher, it is usually the 208 * case that too many elements (more than expected) have been put in the {@code BloomFilter}, 209 * degenerating it. 210 * 211 * @since 14.0 (since 11.0 as expectedFalsePositiveProbability()) 212 */ 213 public double expectedFpp() { 214 return Math.pow((double) bits.bitCount() / bitSize(), numHashFunctions); 215 } 216 217 /** 218 * Returns an estimate for the total number of distinct elements that have been added to this 219 * Bloom filter. This approximation is reasonably accurate if it does not exceed the value of 220 * {@code expectedInsertions} that was used when constructing the filter. 221 * 222 * @since 22.0 223 */ 224 public long approximateElementCount() { 225 long bitSize = bits.bitSize(); 226 long bitCount = bits.bitCount(); 227 228 /* 229 * Each insertion is expected to reduce the # of clear bits by a factor of 230 * `numHashFunctions/bitSize`. So, after n insertions, expected bitCount is `bitSize * (1 - (1 - 231 * numHashFunctions/bitSize)^n)`. Solving that for n, and approximating `ln x` as `x - 1` when x 232 * is close to 1 (why?), gives the following formula. 233 */ 234 double fractionOfBitsSet = (double) bitCount / bitSize; 235 return DoubleMath.roundToLong( 236 -Math.log1p(-fractionOfBitsSet) * bitSize / numHashFunctions, RoundingMode.HALF_UP); 237 } 238 239 /** Returns the number of bits in the underlying bit array. */ 240 @VisibleForTesting 241 long bitSize() { 242 return bits.bitSize(); 243 } 244 245 /** 246 * Returns the number of bytes required to serialize this bloom filter using the {@link 247 * #writeTo(OutputStream)} method. This method can be useful if you need to know the serialization 248 * size before writing to pre-allocated space (e.g., for memory-mapped files or fixed-size 249 * records). 250 * 251 * @since 33.6.0 252 */ 253 public long serializedSize() { 254 // We return a long (and not an int) because the max serialized size is approximately 255 // 17 billion bytes (2^31 * 8 bytes). 256 257 // The serialization header is 6 bytes: 258 // 1 byte for the strategy 259 // 1 byte for the number of hash functions 260 // 4 bytes for the array length 261 return (bits.bitSize() / 8) + 6; 262 } 263 264 /** 265 * Determines whether a given Bloom filter is compatible with this Bloom filter. For two Bloom 266 * filters to be compatible, they must: 267 * 268 * <ul> 269 * <li>not be the same instance 270 * <li>have the same number of hash functions 271 * <li>have the same bit size 272 * <li>have the same strategy 273 * <li>have equal funnels 274 * </ul> 275 * 276 * @param that The Bloom filter to check for compatibility. 277 * @since 15.0 278 */ 279 public boolean isCompatible(BloomFilter<T> that) { 280 checkNotNull(that); 281 return this != that 282 && this.numHashFunctions == that.numHashFunctions 283 && this.bitSize() == that.bitSize() 284 && this.strategy.equals(that.strategy) 285 && this.funnel.equals(that.funnel); 286 } 287 288 /** 289 * Combines this Bloom filter with another Bloom filter by performing a bitwise OR of the 290 * underlying data. The mutations happen to <b>this</b> instance. Callers must ensure the Bloom 291 * filters are appropriately sized to avoid saturating them. 292 * 293 * @param that The Bloom filter to combine this Bloom filter with. It is not mutated. 294 * @throws IllegalArgumentException if {@code isCompatible(that) == false} 295 * @since 15.0 296 */ 297 public void putAll(BloomFilter<T> that) { 298 checkNotNull(that); 299 checkArgument(this != that, "Cannot combine a BloomFilter with itself."); 300 checkArgument( 301 this.numHashFunctions == that.numHashFunctions, 302 "BloomFilters must have the same number of hash functions (%s != %s)", 303 this.numHashFunctions, 304 that.numHashFunctions); 305 checkArgument( 306 this.bitSize() == that.bitSize(), 307 "BloomFilters must have the same size underlying bit arrays (%s != %s)", 308 this.bitSize(), 309 that.bitSize()); 310 checkArgument( 311 this.strategy.equals(that.strategy), 312 "BloomFilters must have equal strategies (%s != %s)", 313 this.strategy, 314 that.strategy); 315 checkArgument( 316 this.funnel.equals(that.funnel), 317 "BloomFilters must have equal funnels (%s != %s)", 318 this.funnel, 319 that.funnel); 320 this.bits.putAll(that.bits); 321 } 322 323 /** 324 * Indicates whether another object is equal to this Bloom filter; <b>discouraged</b>. This method 325 * performs a potentially expensive comparison of all data and configuration of the two Bloom 326 * filters. 327 */ 328 @Override 329 public boolean equals(@Nullable Object object) { 330 if (object == this) { 331 return true; 332 } 333 if (object instanceof BloomFilter) { 334 BloomFilter<?> that = (BloomFilter<?>) object; 335 return this.numHashFunctions == that.numHashFunctions 336 && this.funnel.equals(that.funnel) 337 && this.bits.equals(that.bits) 338 && this.strategy.equals(that.strategy); 339 } 340 return false; 341 } 342 343 /** 344 * Returns a hash code value for this Bloom filter; <b>discouraged</b>. This method performs a 345 * potentially expensive hashing operation on all data and configuration of this Bloom filter. 346 */ 347 @Override 348 public int hashCode() { 349 return Objects.hash(numHashFunctions, funnel, strategy, bits); 350 } 351 352 /** 353 * Returns a {@code Collector} expecting the specified number of insertions, and yielding a {@link 354 * BloomFilter} with false positive probability 3%. 355 * 356 * <p>Note that if the {@code Collector} receives significantly more elements than specified, the 357 * resulting {@code BloomFilter} will suffer a sharp deterioration of its false positive 358 * probability. 359 * 360 * <p>The constructed {@code BloomFilter} will be serializable if the provided {@code Funnel<T>} 361 * is. 362 * 363 * <p>It is recommended that the funnel be implemented as a Java enum. This has the benefit of 364 * ensuring proper serialization and deserialization, which is important since {@link #equals} 365 * also relies on object identity of funnels. 366 * 367 * @param funnel the funnel of T's that the constructed {@code BloomFilter} will use 368 * @param expectedInsertions the number of expected insertions to the constructed {@code 369 * BloomFilter}; must be positive 370 * @return a {@code Collector} generating a {@code BloomFilter} of the received elements 371 * @since 23.0 (but only since 33.4.0 in the Android flavor) 372 */ 373 public static <T extends @Nullable Object> Collector<T, ?, BloomFilter<T>> toBloomFilter( 374 Funnel<? super T> funnel, long expectedInsertions) { 375 return toBloomFilter(funnel, expectedInsertions, 0.03); 376 } 377 378 /** 379 * Returns a {@code Collector} expecting the specified number of insertions, and yielding a {@link 380 * BloomFilter} with the specified expected false positive probability. 381 * 382 * <p>Note that if the {@code Collector} receives significantly more elements than specified, the 383 * resulting {@code BloomFilter} will suffer a sharp deterioration of its false positive 384 * probability. 385 * 386 * <p>The constructed {@code BloomFilter} will be serializable if the provided {@code Funnel<T>} 387 * is. 388 * 389 * <p>It is recommended that the funnel be implemented as a Java enum. This has the benefit of 390 * ensuring proper serialization and deserialization, which is important since {@link #equals} 391 * also relies on object identity of funnels. 392 * 393 * @param funnel the funnel of T's that the constructed {@code BloomFilter} will use 394 * @param expectedInsertions the number of expected insertions to the constructed {@code 395 * BloomFilter}; must be positive 396 * @param fpp the desired false positive probability (must be positive and less than 1.0) 397 * @return a {@code Collector} generating a {@code BloomFilter} of the received elements 398 * @since 23.0 (but only since 33.4.0 in the Android flavor) 399 */ 400 public static <T extends @Nullable Object> Collector<T, ?, BloomFilter<T>> toBloomFilter( 401 Funnel<? super T> funnel, long expectedInsertions, double fpp) { 402 checkNotNull(funnel); 403 checkArgument( 404 expectedInsertions >= 0, "Expected insertions (%s) must be >= 0", expectedInsertions); 405 checkArgument(fpp > 0.0, "False positive probability (%s) must be > 0.0", fpp); 406 checkArgument(fpp < 1.0, "False positive probability (%s) must be < 1.0", fpp); 407 return Collector.of( 408 () -> create(funnel, expectedInsertions, fpp), 409 BloomFilter::put, 410 (bf1, bf2) -> { 411 bf1.putAll(bf2); 412 return bf1; 413 }, 414 Collector.Characteristics.UNORDERED, 415 Collector.Characteristics.CONCURRENT); 416 } 417 418 /** 419 * Creates a {@link BloomFilter} with the expected number of insertions and expected false 420 * positive probability. 421 * 422 * <p>Note that overflowing a {@code BloomFilter} with significantly more elements than specified, 423 * will result in its saturation, and a sharp deterioration of its false positive probability. 424 * 425 * <p>The constructed {@code BloomFilter} will be serializable if the provided {@code Funnel<T>} 426 * is. 427 * 428 * <p>It is recommended that the funnel be implemented as a Java enum. This has the benefit of 429 * ensuring proper serialization and deserialization, which is important since {@link #equals} 430 * also relies on object identity of funnels. 431 * 432 * @param funnel the funnel of T's that the constructed {@code BloomFilter} will use 433 * @param expectedInsertions the number of expected insertions to the constructed {@code 434 * BloomFilter}; must be positive 435 * @param fpp the desired false positive probability (must be positive and less than 1.0) 436 * @return a {@code BloomFilter} 437 */ 438 public static <T extends @Nullable Object> BloomFilter<T> create( 439 Funnel<? super T> funnel, int expectedInsertions, double fpp) { 440 return create(funnel, (long) expectedInsertions, fpp); 441 } 442 443 /** 444 * Creates a {@link BloomFilter} with the expected number of insertions and expected false 445 * positive probability. 446 * 447 * <p>Note that overflowing a {@code BloomFilter} with significantly more elements than specified, 448 * will result in its saturation, and a sharp deterioration of its false positive probability. 449 * 450 * <p>The constructed {@code BloomFilter} will be serializable if the provided {@code Funnel<T>} 451 * is. 452 * 453 * <p>It is recommended that the funnel be implemented as a Java enum. This has the benefit of 454 * ensuring proper serialization and deserialization, which is important since {@link #equals} 455 * also relies on object identity of funnels. 456 * 457 * @param funnel the funnel of T's that the constructed {@code BloomFilter} will use 458 * @param expectedInsertions the number of expected insertions to the constructed {@code 459 * BloomFilter}; must be positive 460 * @param fpp the desired false positive probability (must be positive and less than 1.0) 461 * @return a {@code BloomFilter} 462 * @since 19.0 463 */ 464 public static <T extends @Nullable Object> BloomFilter<T> create( 465 Funnel<? super T> funnel, long expectedInsertions, double fpp) { 466 return create(funnel, expectedInsertions, fpp, BloomFilterStrategies.MURMUR128_MITZ_64); 467 } 468 469 @VisibleForTesting 470 static <T extends @Nullable Object> BloomFilter<T> create( 471 Funnel<? super T> funnel, long expectedInsertions, double fpp, Strategy strategy) { 472 checkNotNull(funnel); 473 checkArgument( 474 expectedInsertions >= 0, "Expected insertions (%s) must be >= 0", expectedInsertions); 475 checkArgument(fpp > 0.0, "False positive probability (%s) must be > 0.0", fpp); 476 checkArgument(fpp < 1.0, "False positive probability (%s) must be < 1.0", fpp); 477 checkNotNull(strategy); 478 479 if (expectedInsertions == 0) { 480 expectedInsertions = 1; 481 } 482 /* 483 * TODO(user): Put a warning in the javadoc about tiny fpp values, since the resulting size 484 * is proportional to -log(p), but there is not much of a point after all, e.g. 485 * optimalM(1000, 0.0000000000000001) = 76680 which is less than 10kb. Who cares! 486 */ 487 long numBits = optimalNumOfBits(expectedInsertions, fpp); 488 int numHashFunctions = optimalNumOfHashFunctions(fpp); 489 try { 490 return new BloomFilter<>(new LockFreeBitArray(numBits), numHashFunctions, funnel, strategy); 491 } catch (IllegalArgumentException e) { 492 throw new IllegalArgumentException("Could not create BloomFilter of " + numBits + " bits", e); 493 } 494 } 495 496 /** 497 * Creates a {@link BloomFilter} with the expected number of insertions and a default expected 498 * false positive probability of 3%. 499 * 500 * <p>Note that overflowing a {@code BloomFilter} with significantly more elements than specified, 501 * will result in its saturation, and a sharp deterioration of its false positive probability. 502 * 503 * <p>The constructed {@code BloomFilter} will be serializable if the provided {@code Funnel<T>} 504 * is. 505 * 506 * <p>It is recommended that the funnel be implemented as a Java enum. This has the benefit of 507 * ensuring proper serialization and deserialization, which is important since {@link #equals} 508 * also relies on object identity of funnels. 509 * 510 * @param funnel the funnel of T's that the constructed {@code BloomFilter} will use 511 * @param expectedInsertions the number of expected insertions to the constructed {@code 512 * BloomFilter}; must be positive 513 * @return a {@code BloomFilter} 514 */ 515 public static <T extends @Nullable Object> BloomFilter<T> create( 516 Funnel<? super T> funnel, int expectedInsertions) { 517 return create(funnel, (long) expectedInsertions); 518 } 519 520 /** 521 * Creates a {@link BloomFilter} with the expected number of insertions and a default expected 522 * false positive probability of 3%. 523 * 524 * <p>Note that overflowing a {@code BloomFilter} with significantly more elements than specified, 525 * will result in its saturation, and a sharp deterioration of its false positive probability. 526 * 527 * <p>The constructed {@code BloomFilter} will be serializable if the provided {@code Funnel<T>} 528 * is. 529 * 530 * <p>It is recommended that the funnel be implemented as a Java enum. This has the benefit of 531 * ensuring proper serialization and deserialization, which is important since {@link #equals} 532 * also relies on object identity of funnels. 533 * 534 * @param funnel the funnel of T's that the constructed {@code BloomFilter} will use 535 * @param expectedInsertions the number of expected insertions to the constructed {@code 536 * BloomFilter}; must be positive 537 * @return a {@code BloomFilter} 538 * @since 19.0 539 */ 540 public static <T extends @Nullable Object> BloomFilter<T> create( 541 Funnel<? super T> funnel, long expectedInsertions) { 542 return create(funnel, expectedInsertions, 0.03); // FYI, for 3%, we always get 5 hash functions 543 } 544 545 // Cheat sheet: 546 // 547 // m: total bits 548 // n: expected insertions 549 // b: m/n, bits per insertion 550 // p: expected false positive probability 551 // 552 // 1) Optimal k = b * ln2 553 // 2) p = (1 - e ^ (-kn/m))^k 554 // 3) For optimal k: p = 2 ^ (-k) ~= 0.6185^b 555 // 4) For optimal k: m = -nlnp / ((ln2) ^ 2) 556 557 /** 558 * Computes the optimal number of hash functions (k) for a given false positive probability (p). 559 * 560 * <p>See http://en.wikipedia.org/wiki/File:Bloom_filter_fp_probability.svg for the formula. 561 * 562 * @param p desired false positive probability (must be between 0 and 1, exclusive) 563 */ 564 @VisibleForTesting 565 static int optimalNumOfHashFunctions(double p) { 566 // -log(p) / log(2), ensuring the result is rounded to avoid truncation. 567 return max(1, (int) round(-log(p) / LOG_TWO)); 568 } 569 570 /** 571 * Computes m (total bits of Bloom filter) which is expected to achieve, for the specified 572 * expected insertions, the required false positive probability. 573 * 574 * <p>See http://en.wikipedia.org/wiki/Bloom_filter#Probability_of_false_positives for the 575 * formula. 576 * 577 * @param n expected insertions (must be positive) 578 * @param p false positive rate (must be 0 < p < 1) 579 */ 580 @VisibleForTesting 581 static long optimalNumOfBits(long n, double p) { 582 if (p == 0) { 583 p = Double.MIN_VALUE; 584 } 585 return (long) (-n * log(p) / SQUARED_LOG_TWO); 586 } 587 588 private Object writeReplace() { 589 return new SerialForm<T>(this); 590 } 591 592 private void readObject(ObjectInputStream stream) throws InvalidObjectException { 593 throw new InvalidObjectException("Use SerializedForm"); 594 } 595 596 private static final class SerialForm<T extends @Nullable Object> implements Serializable { 597 final long[] data; 598 final int numHashFunctions; 599 final Funnel<? super T> funnel; 600 final Strategy strategy; 601 602 SerialForm(BloomFilter<T> bf) { 603 this.data = LockFreeBitArray.toPlainArray(bf.bits.data); 604 this.numHashFunctions = bf.numHashFunctions; 605 this.funnel = bf.funnel; 606 this.strategy = bf.strategy; 607 } 608 609 Object readResolve() { 610 return new BloomFilter<T>(new LockFreeBitArray(data), numHashFunctions, funnel, strategy); 611 } 612 613 private static final long serialVersionUID = 1; 614 } 615 616 /** 617 * Writes this {@code BloomFilter} to an output stream, with a custom format (not Java 618 * serialization). This has been measured to save at least 400 bytes compared to regular 619 * serialization. 620 * 621 * <p>Use {@link #readFrom(InputStream, Funnel)} to reconstruct the written BloomFilter. 622 */ 623 public void writeTo(OutputStream out) throws IOException { 624 // Serial form: 625 // 1 signed byte for the strategy 626 // 1 unsigned byte for the number of hash functions 627 // 1 big endian int, the number of longs in our bitset 628 // N big endian longs of our bitset 629 DataOutputStream dout = new DataOutputStream(out); 630 dout.writeByte(SignedBytes.checkedCast(strategy.ordinal())); 631 dout.writeByte(UnsignedBytes.checkedCast(numHashFunctions)); // note: checked at the c'tor 632 dout.writeInt(bits.data.length()); 633 for (int i = 0; i < bits.data.length(); i++) { 634 dout.writeLong(bits.data.get(i)); 635 } 636 } 637 638 /** 639 * Reads a byte stream, which was written by {@link #writeTo(OutputStream)}, into a {@code 640 * BloomFilter}, allocating as much memory as the byte stream requests. 641 * 642 * <p>The {@code Funnel} to be used is not encoded in the stream, so it must be provided here. 643 * <b>Warning:</b> the funnel provided <b>must</b> behave identically to the one used to populate 644 * the original Bloom filter! 645 * 646 * @throws IOException if the InputStream throws an {@code IOException}, or if its data does not 647 * appear to be a BloomFilter serialized using the {@link #writeTo(OutputStream)} method. 648 */ 649 @SuppressWarnings("CatchingUnchecked") // sneaky checked exception 650 public static <T extends @Nullable Object> BloomFilter<T> readFrom( 651 InputStream in, Funnel<? super T> funnel) throws IOException { 652 return readFrom(in, funnel, Long.MAX_VALUE); 653 } 654 655 /** 656 * Reads a byte stream, which was written by {@link #writeTo(OutputStream)}, into a {@code 657 * BloomFilter}, allocating at most {@code maxAllowedSizeInBits} for the backing array. 658 * 659 * <p>The {@code Funnel} to be used is not encoded in the stream, so it must be provided here. 660 * <b>Warning:</b> the funnel provided <b>must</b> behave identically to the one used to populate 661 * the original Bloom filter! 662 * 663 * @param in the InputStream to read from 664 * @param funnel the funnel to use for the Bloom filter 665 * @param maxAllowedSizeInBits the maximum number of bits that the Bloom filter can have 666 * @throws IOException if the InputStream throws an {@code IOException}, or if its data does not 667 * appear to be a BloomFilter serialized using the {@link #writeTo(OutputStream)} method. 668 * @throws IllegalArgumentException if {@code maxAllowedSizeInBits} is negative, or if the number 669 * of bits in the stream is greater than {@code maxAllowedSizeInBits}. 670 * @since 33.7.0 671 */ 672 @SuppressWarnings("CatchingUnchecked") // sneaky checked exception 673 public static <T extends @Nullable Object> BloomFilter<T> readFrom( 674 InputStream in, Funnel<? super T> funnel, long maxAllowedSizeInBits) throws IOException { 675 checkNotNull(in, "InputStream"); 676 checkNotNull(funnel, "Funnel"); 677 checkArgument( 678 maxAllowedSizeInBits >= 0, "maxAllowedSizeInBits (%s) must be >= 0", maxAllowedSizeInBits); 679 int strategyOrdinal = -1; 680 int numHashFunctions = -1; 681 int longArraySize = -1; 682 try { 683 DataInputStream din = new DataInputStream(in); 684 // currently this assumes there is no negative ordinal; will have to be updated if we 685 // add non-stateless strategies (for which we've reserved negative ordinals; see 686 // Strategy.ordinal()). 687 strategyOrdinal = din.readByte(); 688 numHashFunctions = toUnsignedInt(din.readByte()); 689 longArraySize = din.readInt(); 690 691 checkArgument(longArraySize >= 0, "longArraySize (%s) must be >= 0", longArraySize); 692 checkArgument( 693 longArraySize <= maxAllowedSizeInBits / 64, 694 "longArraySize (%s) must be <= %s", 695 longArraySize, 696 maxAllowedSizeInBits / 64); 697 698 /* 699 * We document in BloomFilterStrategies that we must not change the ordering, and we have a 700 * test that verifies that we don't do so. 701 */ 702 @SuppressWarnings("EnumOrdinal") 703 Strategy strategy = BloomFilterStrategies.values()[strategyOrdinal]; 704 705 LockFreeBitArray dataArray = new LockFreeBitArray(Math.multiplyExact(longArraySize, 64L)); 706 for (int i = 0; i < longArraySize; i++) { 707 dataArray.putData(i, din.readLong()); 708 } 709 710 return new BloomFilter<>(dataArray, numHashFunctions, funnel, strategy); 711 } catch (IOException e) { 712 throw e; 713 } catch (Exception e) { // sneaky checked exception 714 String message = 715 "Unable to deserialize BloomFilter from InputStream." 716 + " strategyOrdinal: " 717 + strategyOrdinal 718 + " numHashFunctions: " 719 + numHashFunctions 720 + " longArraySize: " 721 + longArraySize; 722 throw new IOException(message, e); 723 } 724 } 725 726 private static final long serialVersionUID = 0xcafebabe; 727}