Skip to content

Commit 3d41608

Browse files
squarejessesquarejesse
committed
New APIs: ByteString.toIndex() and ByteString.toFraction()
The first one may be useful with hashing to put byte strings in partitioning buckets for scaling. For example, to divide a dataset into 32 partitions, hash the key then use toIndex(32) to map the key to its partition. The second one may be useful with dynamic experiments and A/B tests. For example, to assign a control group to 5% of customers hash the customer key then check if toFraction() is less than 0.05.
1 parent 3181015 commit 3d41608

File tree

6 files changed

+210
-0
lines changed

6 files changed

+210
-0
lines changed

‎okio/src/commonMain/kotlin/okio/ByteString.kt

+92
Original file line numberDiff line numberDiff line change
@@ -137,6 +137,98 @@ internal constructor(data: ByteArray) : Comparable<ByteString> {
137137

138138
override fun compareTo(other: ByteString): Int
139139

140+
/**
141+
* Projects this value to the range `[0..size)` using linear interpolation. This is equivalent to
142+
* a sorted partitioning of all possible byte strings across [size] equally-sized buckets and
143+
* returning the index of the bucket that this byte string fits in.
144+
*
145+
* For example, the byte string `8000` is the median of all 2-element byte strings, and calling
146+
* `toIndex(100)` on it returns 50. Some other examples:
147+
*
148+
* | Byte String (hex) | `toIndex(100)` | `toIndex(256)` | `toIndex(Int.MAX_VALUE)` |
149+
* | :----------------- | -------------: | -------------: | -----------------------: |
150+
* | (empty) | 0 | 0 | 0 |
151+
* | 00 | 0 | 0 | 0 |
152+
* | 0000 | 0 | 0 | 0 |
153+
* | 000000 | 0 | 0 | 0 |
154+
* | 0000000001 | 0 | 0 | 0 |
155+
* | 00000001 | 0 | 0 | 0 |
156+
* | 00000002 | 0 | 0 | 0 |
157+
* | 00000003 | 0 | 0 | 1 |
158+
* | 01 | 0 | 1 | 8388607 |
159+
* | 02 | 0 | 2 | 16777215 |
160+
* | 03 | 1 | 3 | 25165823 |
161+
* | 80 | 50 | 128 | 1073741823 |
162+
* | 8000 | 50 | 128 | 1073741823 |
163+
* | 80000000 | 50 | 128 | 1073741823 |
164+
* | 81 | 50 | 129 | 1082130431 |
165+
* | 81ffffff | 50 | 129 | 1090519038 |
166+
* | 82 | 50 | 130 | 1090519039 |
167+
* | 83 | 51 | 131 | 1098907647 |
168+
* | ff | 99 | 255 | 2139095039 |
169+
* | ffff | 99 | 255 | 2147450879 |
170+
* | ffffffff | 99 | 255 | 2147483646 |
171+
* | ffffffffffff | 99 | 255 | 2147483646 |
172+
*
173+
* This interprets the bytes in this byte string as **unsigned**. This behavior is consistent with
174+
* [compareTo]. The returned value is also consistent with [compareTo] though the dynamic range
175+
* is compressed. For two byte strings `a` and `b`, if `a < b`, then
176+
* `a.toIndex(n) <= b.toIndex(n)` for all sizes `n`.
177+
*
178+
* This examines at most the first 4 bytes of this byte string. Data beyond the first 4 bytes is
179+
* not used to compute the result.
180+
*
181+
* @param size a positive integer.
182+
* @return a value that is greater than or equal to `0` and less than [size].
183+
*/
184+
fun toIndex(size: Int): Int
185+
186+
/**
187+
* Projects this value to the range `[0.0..1.0)` using linear interpolation. This is equivalent to
188+
* sorting all possible byte strings and returning the fraction that precede this byte string.
189+
*
190+
* For example, the byte string `8000` is the median of all 2-element byte strings, and calling
191+
* `toFraction()` on it returns 0.5. Some other examples:
192+
*
193+
* | Byte String (hex) | `toFraction()` |
194+
* | :----------------- | :----------------- |
195+
* | (empty) | 0.0 |
196+
* | 00 | 0.0 |
197+
* | 0000 | 0.0 |
198+
* | 000000 | 0.0 |
199+
* | 00000000000001 | 0.0 |
200+
* | 00000000000007 | 0.0 |
201+
* | 00000000000008 | 0.0000000000000001 |
202+
* | 0000000001 | 0.0000000000009094 |
203+
* | 00000001 | 0.0000000002328306 |
204+
* | 01 | 0.00390625 |
205+
* | 02 | 0.0078125 |
206+
* | 03 | 0.01171875 |
207+
* | 80 | 0.5 |
208+
* | 8000 | 0.5 |
209+
* | 80000000000000 | 0.5 |
210+
* | 81 | 0.50390625 |
211+
* | 81ffffff | 0.5078124997671694 |
212+
* | 82 | 0.5078125 |
213+
* | 83 | 0.51171875 |
214+
* | ff | 0.99609375 |
215+
* | ffff | 0.9999847412109375 |
216+
* | ffffffff | 0.9999999997671694 |
217+
* | ffffffffffff | 0.9999999999999964 |
218+
* | ffffffffffffff | 0.9999999999999999 |
219+
*
220+
* This interprets the bytes in this byte string as **unsigned**. This behavior is consistent with
221+
* [compareTo]. The returned value is also consistent with [compareTo] though the dynamic range
222+
* is compressed. For two byte strings `a` and `b`, if `a < b`, then
223+
* `a.toFraction() <= b.toFraction()`.
224+
*
225+
* This examines at most the first 7 bytes of this byte string. Data beyond the first 7 bytes is
226+
* not used to compute the result.
227+
*
228+
* @return a value that is greater than or equal to `0.0` and less than `1.0`.
229+
*/
230+
fun toFraction(): Double
231+
140232
/**
141233
* Returns a human-readable string that describes the contents of this byte string. Typically this
142234
* is a string like `[text=Hello]` or `[hex=0000ffff]`.

‎okio/src/commonMain/kotlin/okio/internal/ByteString.kt

+32
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,7 @@ import okio.isIsoControl
3030
import okio.processUtf8CodePoints
3131
import okio.shr
3232
import okio.toUtf8String
33+
import kotlin.math.min
3334

3435
// TODO Kotlin's expect classes can't have default implementations, so platform implementations
3536
// have to call these functions. Remove all this nonsense when expect class allow actual code.
@@ -248,6 +249,37 @@ internal inline fun ByteString.commonCompareTo(other: ByteString): Int {
248249
return if (sizeA < sizeB) -1 else 1
249250
}
250251

252+
@Suppress("NOTHING_TO_INLINE")
253+
internal inline fun ByteString.commonToIndex(size: Int): Int {
254+
require(size > 0)
255+
var numerator = 0L
256+
var denominator = 1L
257+
for (i in 0 until min(4, this.size)) {
258+
numerator = (numerator shl 8) + (get(i) and 0xff)
259+
denominator = (denominator shl 8)
260+
}
261+
return (size * numerator / denominator).toInt()
262+
}
263+
264+
@Suppress("NOTHING_TO_INLINE")
265+
internal inline fun ByteString.commonToFraction(): Double {
266+
var numerator = 0L
267+
var denominator = 1L
268+
for (i in 0 until min(7, size)) {
269+
numerator = (numerator shl 8) + (get(i) and 0xff)
270+
denominator = (denominator shl 8)
271+
}
272+
273+
// Double wants 53 bits of precision but we have 56. Discard 3 bits of precision. Without this
274+
// it's possible that this method returns 1.0 for byte strings like "ffffffffffffff".
275+
if (size >= 7) {
276+
numerator = numerator shr 3
277+
denominator = denominator shr 3
278+
}
279+
280+
return numerator.toDouble() / denominator
281+
}
282+
251283
@Suppress("NOTHING_TO_INLINE")
252284
internal inline fun commonOf(data: ByteArray) = ByteString(data.copyOf())
253285

‎okio/src/commonTest/kotlin/okio/ByteStringTest.kt

+68
Original file line numberDiff line numberDiff line change
@@ -457,4 +457,72 @@ abstract class AbstractByteStringTest internal constructor(
457457
sortedByteStrings.sort()
458458
assertEquals(originalByteStrings, sortedByteStrings)
459459
}
460+
461+
@Test fun toIndex() {
462+
assertEquals(0, factory.decodeHex("").toIndex(1))
463+
assertEquals(0, factory.decodeHex("00").toIndex(1))
464+
assertEquals(0, factory.decodeHex("ff").toIndex(1))
465+
assertEquals(0, factory.decodeHex("ffffffff").toIndex(1))
466+
assertEquals(0, factory.decodeHex("ffffffffffff").toIndex(1))
467+
468+
assertEquals(0, factory.decodeHex("").toIndex(100))
469+
assertEquals(0, factory.decodeHex("00").toIndex(100))
470+
assertEquals(10, factory.decodeHex("1a").toIndex(100))
471+
assertEquals(25, factory.decodeHex("40").toIndex(100))
472+
assertEquals(50, factory.decodeHex("80").toIndex(100))
473+
assertEquals(75, factory.decodeHex("c0").toIndex(100))
474+
assertEquals(99, factory.decodeHex("ff").toIndex(100))
475+
assertEquals(99, factory.decodeHex("ffff").toIndex(100))
476+
assertEquals(99, factory.decodeHex("ffffff").toIndex(100))
477+
assertEquals(99, factory.decodeHex("ffffffff").toIndex(100))
478+
479+
assertEquals(0, factory.decodeHex("").toIndex(Int.MAX_VALUE))
480+
assertEquals(0x7f7fffff, factory.decodeHex("ff").toIndex(Int.MAX_VALUE))
481+
assertEquals(0x7fff7fff, factory.decodeHex("ffff").toIndex(Int.MAX_VALUE))
482+
assertEquals(0x7fffff7f, factory.decodeHex("ffffff").toIndex(Int.MAX_VALUE))
483+
assertEquals(0x7ffffffe, factory.decodeHex("ffffffff").toIndex(Int.MAX_VALUE))
484+
}
485+
486+
/**
487+
* Our math is incorrect for values that round differently depending on data beyond the first 4
488+
* bytes. For example, "aaaaaaaaab".toIndex(3) is 1, but if we did arbitrary-precision math the
489+
* result would be 2.
490+
*/
491+
@Test fun toIndexHonorsFirstFourBytesOnly() {
492+
assertEquals(2, factory.decodeHex("aaaaab").toIndex(3))
493+
assertEquals(2, factory.decodeHex("aaaaaaab").toIndex(3))
494+
495+
// Note: wrong due to truncation!
496+
assertEquals(1, factory.decodeHex("aaaaaaaaab").toIndex(3))
497+
assertEquals(1, factory.decodeHex("aaaaaaaaaaab").toIndex(3))
498+
assertEquals(1, factory.decodeHex("aaaaaaaaaaaaab").toIndex(3))
499+
}
500+
501+
@Test fun toFraction() {
502+
assertEquals(0.0, factory.decodeHex("").toFraction())
503+
assertEquals(0.0, factory.decodeHex("00").toFraction())
504+
assertEquals(0.0, factory.decodeHex("00").toFraction())
505+
assertEquals(0.1015625, factory.decodeHex("1a").toFraction())
506+
assertEquals(0.25, factory.decodeHex("40").toFraction())
507+
assertEquals(0.5, factory.decodeHex("80").toFraction())
508+
assertEquals(0.75, factory.decodeHex("c0").toFraction())
509+
assertEquals(0.7929493631236255, factory.decodeHex("cafebabe").toFraction())
510+
assertEquals(0.99609375, factory.decodeHex("ff").toFraction())
511+
assertEquals(0.9999847412109375, factory.decodeHex("ffff").toFraction())
512+
assertEquals(0.9999999403953552, factory.decodeHex("ffffff").toFraction())
513+
assertEquals(0.9999999997671694, factory.decodeHex("ffffffff").toFraction())
514+
assertEquals(0.9999999999999964, factory.decodeHex("ffffffffffff").toFraction())
515+
assertEquals(0.9999999999999999, factory.decodeHex("ffffffffffffff").toFraction())
516+
assertEquals(0.9999999999999999, factory.decodeHex("ffffffffffffffff").toFraction())
517+
}
518+
519+
/** Only 5 bits of the 7th byte are used. We use 53 bits in total for IEEE 754 doubles. */
520+
@Test fun toFractionLast5BitsOf7thByte() {
521+
assertEquals(0.0000000000000000, factory.decodeHex("00000000000007").toFraction())
522+
assertEquals(1.1102230246251565E-16, factory.decodeHex("00000000000008").toFraction())
523+
assertEquals(1.1102230246251565E-16, factory.decodeHex("0000000000000f").toFraction())
524+
assertEquals(2.220446049250313E-16, factory.decodeHex("00000000000010").toFraction())
525+
assertEquals(0.9999999999999998, factory.decodeHex("fffffffffffff0").toFraction())
526+
assertEquals(0.9999999999999999, factory.decodeHex("fffffffffffff8").toFraction())
527+
}
460528
}

‎okio/src/jsMain/kotlin/okio/ByteString.kt

+6
Original file line numberDiff line numberDiff line change
@@ -38,6 +38,8 @@ import okio.internal.commonToAsciiLowercase
3838
import okio.internal.commonToAsciiUppercase
3939
import okio.internal.commonToByteArray
4040
import okio.internal.commonToByteString
41+
import okio.internal.commonToFraction
42+
import okio.internal.commonToIndex
4143
import okio.internal.commonToString
4244
import okio.internal.commonUtf8
4345
import okio.internal.commonWrite
@@ -119,6 +121,10 @@ internal actual constructor(
119121

120122
actual override fun compareTo(other: ByteString) = commonCompareTo(other)
121123

124+
actual fun toIndex(size: Int) = commonToIndex(size)
125+
126+
actual fun toFraction() = commonToFraction()
127+
122128
/**
123129
* Returns a human-readable string that describes the contents of this byte string. Typically this
124130
* is a string like `[text=Hello]` or `[hex=0000ffff]`.

‎okio/src/jvmMain/kotlin/okio/ByteString.kt

+6
Original file line numberDiff line numberDiff line change
@@ -38,6 +38,8 @@ import okio.internal.commonToAsciiLowercase
3838
import okio.internal.commonToAsciiUppercase
3939
import okio.internal.commonToByteArray
4040
import okio.internal.commonToByteString
41+
import okio.internal.commonToFraction
42+
import okio.internal.commonToIndex
4143
import okio.internal.commonToString
4244
import okio.internal.commonUtf8
4345
import okio.internal.commonWrite
@@ -181,6 +183,10 @@ internal actual constructor(
181183

182184
actual override fun compareTo(other: ByteString) = commonCompareTo(other)
183185

186+
actual fun toIndex(size: Int) = commonToIndex(size)
187+
188+
actual fun toFraction() = commonToFraction()
189+
184190
actual override fun toString() = commonToString()
185191

186192
@Throws(IOException::class)

‎okio/src/nativeMain/kotlin/okio/ByteString.kt

+6
Original file line numberDiff line numberDiff line change
@@ -39,6 +39,8 @@ import okio.internal.commonToAsciiLowercase
3939
import okio.internal.commonToAsciiUppercase
4040
import okio.internal.commonToByteArray
4141
import okio.internal.commonToByteString
42+
import okio.internal.commonToFraction
43+
import okio.internal.commonToIndex
4244
import okio.internal.commonToString
4345
import okio.internal.commonUtf8
4446
import okio.internal.commonWrite
@@ -125,6 +127,10 @@ internal actual constructor(
125127

126128
actual override fun compareTo(other: ByteString) = commonCompareTo(other)
127129

130+
actual fun toIndex(size: Int) = commonToIndex(size)
131+
132+
actual fun toFraction() = commonToFraction()
133+
128134
/**
129135
* Returns a human-readable string that describes the contents of this byte string. Typically this
130136
* is a string like `[text=Hello]` or `[hex=0000ffff]`.

0 commit comments

Comments
 (0)