diff --git a/iabtcf-decoder/src/main/java/com/iabtcf/decoder/TCStringDecoder.java b/iabtcf-decoder/src/main/java/com/iabtcf/decoder/TCStringDecoder.java index 73b7b4bc..40a69de6 100644 --- a/iabtcf-decoder/src/main/java/com/iabtcf/decoder/TCStringDecoder.java +++ b/iabtcf-decoder/src/main/java/com/iabtcf/decoder/TCStringDecoder.java @@ -62,7 +62,7 @@ public static TCString decode(String consentString, DecoderOption... options) case 1: return TCStringV1.fromBitVector(bitVector); case 2: - TCString tcString = null; + TCString tcString; if (split.length > 1) { BitReader[] remaining = new BitReader[split.length - 1]; for (int i = 1; i < split.length; i++) { diff --git a/iabtcf-decoder/src/main/java/com/iabtcf/decoder/TCStringV2.java b/iabtcf-decoder/src/main/java/com/iabtcf/decoder/TCStringV2.java index 20fdd56b..a542b821 100644 --- a/iabtcf-decoder/src/main/java/com/iabtcf/decoder/TCStringV2.java +++ b/iabtcf-decoder/src/main/java/com/iabtcf/decoder/TCStringV2.java @@ -147,20 +147,16 @@ public IntIterable getPubPurposesConsent() { * @throws InvalidRangeFieldException */ static BitSetIntIterable fillVendors(BitReader bbv, FieldDefs maxVendor, FieldDefs vendorField) { - BitSet bs = new BitSet(); - int maxV = bbv.readBits16(maxVendor); - boolean isRangeEncoding = bbv.readBits1(maxVendor.getEnd(bbv)); + final int maxV = bbv.readBits16(maxVendor); + final boolean isRangeEncoding = bbv.readBits1(maxVendor.getEnd(bbv)); + final int vendorFieldOffset = vendorField.getOffset(bbv); + BitSet bs = new BitSet(); if (isRangeEncoding) { vendorIdsFromRange(bbv, bs, vendorField, Optional.of(maxVendor)); } else { - for (int i = 0; i < maxV; i++) { - boolean hasVendorConsent = bbv.readBits1(vendorField.getOffset(bbv) + i); - if (hasVendorConsent) { - bs.set(i + 1); - } - } + bs = bbv.readBitSet(vendorFieldOffset, maxV, 1); } return BitSetIntIterable.from(bs); } @@ -172,22 +168,24 @@ static BitSetIntIterable fillVendors(BitReader bbv, FieldDefs maxVendor, FieldDe */ static int vendorIdsFromRange(BitReader bbv, BitSet bs, int numberOfVendorEntriesOffset, Optional maxVendor) { - int numberOfVendorEntries = bbv.readBits12(numberOfVendorEntriesOffset); + + final int numberOfVendorEntries = bbv.readBits12(numberOfVendorEntriesOffset); + final int maxV = maxVendor.map(maxVF -> bbv.readBits16(maxVF)).orElse(Integer.MAX_VALUE); + final int vendorIdFieldLength = FieldDefs.START_OR_ONLY_VENDOR_ID.getLength(bbv); + final int vendorIdMask = 0xFFFFFFFF >>> (32 - vendorIdFieldLength); int offset = numberOfVendorEntriesOffset + FieldDefs.NUM_ENTRIES.getLength(bbv); - int maxV = maxVendor.map(maxVF -> bbv.readBits16(maxVF)).orElse(Integer.MAX_VALUE); for (int j = 0; j < numberOfVendorEntries; j++) { boolean isRangeEntry = bbv.readBits1(offset++); - int startOrOnlyVendorId = bbv.readBits16(offset); - offset += FieldDefs.START_OR_ONLY_VENDOR_ID.getLength(bbv); if (isRangeEntry) { - int endVendorId = bbv.readBits16(offset); - offset += FieldDefs.START_OR_ONLY_VENDOR_ID.getLength(bbv); + final int content = bbv.readBits32(offset); + final int startVendorId = (content >>> vendorIdFieldLength) & vendorIdMask; + final int endVendorId = content & vendorIdMask; + offset += 2 * vendorIdFieldLength; - if (startOrOnlyVendorId > endVendorId) { + if (startVendorId > endVendorId) { throw new InvalidRangeFieldException(String.format( - "start vendor id (%d) is greater than endVendorId (%d)", startOrOnlyVendorId, - endVendorId)); + "start vendor id (%d) is greater than endVendorId (%d)", startVendorId, endVendorId)); } if (endVendorId > maxV) { @@ -195,9 +193,11 @@ static int vendorIdsFromRange(BitReader bbv, BitSet bs, int numberOfVendorEntrie String.format("end vendor id (%d) is greater than max (%d)", endVendorId, maxV)); } - bs.set(startOrOnlyVendorId, endVendorId + 1); + bs.set(startVendorId, endVendorId + 1); } else { - bs.set(startOrOnlyVendorId); + final int onlyVendorId = bbv.readBits16(offset); + offset += vendorIdFieldLength; + bs.set(onlyVendorId); } } @@ -217,22 +217,22 @@ static void vendorIdsFromRange(BitReader bbv, BitSet bs, FieldDefs vendorField, private int fillPublisherRestrictions( List publisherRestrictions, int currentPointer, BitReader bitVector) { - int numberOfPublisherRestrictions = bitVector.readBits12(currentPointer); + final int numberOfPublisherRestrictions = bitVector.readBits12(currentPointer); + final int purposeIdFieldLength = FieldDefs.PURPOSE_ID.getLength(bitVector); currentPointer += FieldDefs.NUM_ENTRIES.getLength(bitVector); for (int i = 0; i < numberOfPublisherRestrictions; i++) { - int purposeId = bitVector.readBits6(currentPointer); - currentPointer += FieldDefs.PURPOSE_ID.getLength(bitVector); - - int restrictionTypeId = bitVector.readBits2(currentPointer); - currentPointer += 2; - RestrictionType restrictionType = RestrictionType.from(restrictionTypeId); + final byte content = bitVector.readByteBits(currentPointer, 8); + final int purposeId = (content >>> 2) & (0xFF >>> (8 - purposeIdFieldLength)); + final int restrictionTypeId = content & 0b11; + currentPointer += purposeIdFieldLength + 2; - BitSet bs = new BitSet(); + final BitSet bs = new BitSet(); currentPointer = vendorIdsFromRange(bbv, bs, currentPointer, Optional.empty()); - PublisherRestriction publisherRestriction = - new PublisherRestriction(purposeId, restrictionType, BitSetIntIterable.from(bs)); - publisherRestrictions.add(publisherRestriction); + publisherRestrictions.add( + new PublisherRestriction( + purposeId, RestrictionType.from(restrictionTypeId), + BitSetIntIterable.from(bs))); } return currentPointer; } diff --git a/iabtcf-decoder/src/main/java/com/iabtcf/utils/BitReader.java b/iabtcf-decoder/src/main/java/com/iabtcf/utils/BitReader.java index e1e10908..a31cb95e 100644 --- a/iabtcf-decoder/src/main/java/com/iabtcf/utils/BitReader.java +++ b/iabtcf-decoder/src/main/java/com/iabtcf/utils/BitReader.java @@ -185,7 +185,11 @@ public byte readBits6(int offset) { * * @throws ByteParseException */ - private byte readByteBits(int offset, int nbits) { + public byte readByteBits(int offset, int nbits) { + if (nbits < 0 || nbits > 8) { + throw new ByteParseException("Only 0 to 8 bytes can be read into a byte"); + } + int startByte = offset >>> 3; int bitPos = offset % 8; int n = 8 - bitPos; @@ -286,6 +290,30 @@ public int readBits24(int offset) { } } + /** + * @throws ByteParseException + */ + public int readBits32(int offset) { + int startByte = offset >>> 3; + int bitPos = offset % 8; + int n = 8 - bitPos; + + if (n < 8) { + ensureReadable(startByte, 5); + return ((unsafeReadLsb(buffer[startByte], bitPos, n) & 0xFF) << 24) + | (buffer[startByte + 1] & 0xFF) << (16 + bitPos) + | (buffer[startByte + 2] & 0xFF) << (8 + bitPos) + | (buffer[startByte + 3] & 0xFF) << bitPos + | (unsafeReadMsb(buffer[startByte + 4], 0, bitPos) & 0xFF); + } else { + ensureReadable(startByte, 4); + return (buffer[startByte] & 0xFF) << 24 + | (buffer[startByte + 1] & 0xFF) << 16 + | (buffer[startByte + 2] & 0xFF) << 8 + | (buffer[startByte + 3] & 0xFF); + } + } + /** * @throws ByteParseException */ @@ -320,20 +348,83 @@ public long readBits36(int offset) { } } + /** + * @throws ByteParseException + */ + public long readBits64(int offset) { + int startByte = offset >>> 3; + int bitPos = offset % 8; + int n = 8 - bitPos; // # bits to read + + if (n < 8) { + ensureReadable(startByte, 9); + return ((long) unsafeReadLsb(buffer[startByte], bitPos, n) & 0xFF) << 56 + | ((long) buffer[startByte + 1] & 0xFF) << (48 + bitPos) + | ((long) buffer[startByte + 2] & 0xFF) << (40 + bitPos) + | ((long) buffer[startByte + 3] & 0xFF) << (32 + bitPos) + | ((long) buffer[startByte + 4] & 0xFF) << (24 + bitPos) + | ((long) buffer[startByte + 5] & 0xFF) << (16 + bitPos) + | ((long) buffer[startByte + 6] & 0xFF) << (8 + bitPos) + | ((long) buffer[startByte + 7] & 0xFF) << bitPos + | ((long) unsafeReadMsb(buffer[startByte + 8], 0, bitPos) & 0xFF); + } else { + ensureReadable(startByte, 8); + return ((long) buffer[startByte] & 0xFF) << 56 + | ((long) buffer[startByte + 1] & 0xFF) << 48 + | ((long) buffer[startByte + 2] & 0xFF) << 40 + | ((long) buffer[startByte + 3] & 0xFF) << 32 + | ((long) buffer[startByte + 4] & 0xFF) << 24 + | ((long) buffer[startByte + 5] & 0xFF) << 16 + | ((long) buffer[startByte + 6] & 0xFF) << 8 + | ((long) buffer[startByte + 7] & 0xFF); + } + } + /** * @throws ByteParseException */ public BitSet readBitSet(int offset, int length) { - // TODO(mk): can we read larger chunks at a time? - BitSet bs = new BitSet(length); - for (int i = 0; i < length; i++) { - if (readBits1(offset + i)) { - bs.set(i); + return readBitSet(offset, length, 0); + } + + /** + * @throws ByteParseException + */ + public BitSet readBitSet(int offset, int length, int resultShift) { + final BitSet bs = new BitSet(length); + int i = 0; + while (i < length) { + final int remaining = length - i; + final int readIndex = offset + i; + final int writeIndex = resultShift + i; + if (remaining >= 64) { + fillBitSetWithContent(bs, readBits64(readIndex), 64, writeIndex); + i += 64; + } else if (remaining >= 32) { + fillBitSetWithContent(bs, readBits32(readIndex), 32, writeIndex); + i += 32; + } else if (remaining >= 16) { + fillBitSetWithContent(bs, readBits16(readIndex), 16, writeIndex); + i += 16; + } else if (remaining >= 8) { + fillBitSetWithContent(bs, readByteBits(readIndex, 8), 8, writeIndex); + i += 8; + } else { + fillBitSetWithContent(bs, readByteBits(readIndex, remaining), remaining, writeIndex); + i += remaining; } } return bs; } + private void fillBitSetWithContent(BitSet bs, long content, int size, int offset) { + for (int j = 0; j < size; j++) { + if (((content >>> (size - 1 - j)) & 1) == 1) { + bs.set(offset + j); + } + } + } + private byte unsafeReadMsb(byte from, int offset, int length) { return length == 0 ? 0 : (byte) ((from >>> ((8 - length) - offset)) & ((1 << length) - 1)); } diff --git a/iabtcf-decoder/src/test/java/com/iabtcf/decoder/BitReaderTest.java b/iabtcf-decoder/src/test/java/com/iabtcf/decoder/BitReaderTest.java index 97cd1444..9d3811ec 100644 --- a/iabtcf-decoder/src/test/java/com/iabtcf/decoder/BitReaderTest.java +++ b/iabtcf-decoder/src/test/java/com/iabtcf/decoder/BitReaderTest.java @@ -314,6 +314,119 @@ private void checkTestReadBits24N_Random() { } } + @Test + public void testReadBits32_0() { + BitReader bv = new BitReader(new byte[] { + 0b00000001, 0b00000001, 0b00000001, 0b00000001, + }); + assertEquals(0x1010101, bv.readBits32(0)); + } + + @Test + public void testReadBits32_1() { + BitReader bv = new BitReader(new byte[] {(byte) 0x00, + (byte) 0b10000000, (byte) 0b10000000, (byte) 0b10000000, (byte) 0b10000000, + }); + assertEquals(0x1010101, bv.readBits32(1)); + } + + @Test + public void testReadBits32_2() { + byte[] g = new byte[] {(byte) 0x00, + (byte) 0b10000000, (byte) 0b10000000, (byte) 0b10000000, (byte) 0b10000000, + }; + BitReader bv = new BitReader(g); + assertEquals(0x1010101, bv.readBits32(1)); + + shift(g); + + bv = new BitReader(g); + assertEquals(0x1010101, bv.readBits32(2)); + } + + @Test + public void testReadBits32N_Random() { + for (int i = 0; i < 1000; i++) { + checkTestReadBits32N_Random(); + } + } + + private void checkTestReadBits32N_Random() { + byte[] rb = new byte[5]; + r.nextBytes(rb); + + byte[] g = new byte[] {(byte) 0b0000001, + rb[0], rb[1], rb[2], rb[3], rb[4], + (byte) 0x00, (byte) 0x00}; + + BitReader bv = new BitReader(g); + long expect = bv.readBits32(4); + + for (int i = 1; i < 16; i++) { + shift(g); + bv = new BitReader(g); + assertEquals(String.format("%d", i), expect, bv.readBits32(4 + i)); + } + } + + @Test + public void testReadBits64_0() { + BitReader bv = new BitReader(new byte[] { + 0b00000001, 0b00000001, 0b00000001, 0b00000001, + 0b00000001, 0b00000001, 0b00000001, 0b00000001, + }); + assertEquals(0x101010101010101L, bv.readBits64(0)); + } + + @Test + public void testReadBits64_1() { + BitReader bv = new BitReader(new byte[] {(byte) 0x00, + (byte) 0b10000000, (byte) 0b10000000, (byte) 0b10000000, (byte) 0b10000000, + (byte) 0b10000000, (byte) 0b10000000, (byte) 0b10000000, (byte) 0b10000000, + }); + assertEquals(0x101010101010101L, bv.readBits64(1)); + } + + @Test + public void testReadBits64_2() { + byte[] g = new byte[] {(byte) 0x00, + (byte) 0b10000000, (byte) 0b10000000, (byte) 0b10000000, (byte) 0b10000000, + (byte) 0b10000000, (byte) 0b10000000, (byte) 0b10000000, (byte) 0b10000000, + }; + BitReader bv = new BitReader(g); + assertEquals(0x101010101010101L, bv.readBits64(1)); + + shift(g); + + bv = new BitReader(g); + assertEquals(0x101010101010101L, bv.readBits64(2)); + } + + @Test + public void testReadBits64N_Random() { + for (int i = 0; i < 1000; i++) { + checkTestReadBits64N_Random(); + } + } + + private void checkTestReadBits64N_Random() { + byte[] rb = new byte[9]; + r.nextBytes(rb); + + byte[] g = new byte[] {(byte) 0b0000001, + rb[0], rb[1], rb[2], rb[3], rb[4], rb[5], rb[6], rb[7], rb[8], + (byte) 0x00, (byte) 0x00}; + + BitReader bv = new BitReader(g); + long expect = bv.readBits64(4); + + for (int i = 1; i < 16; i++) { + shift(g); + bv = new BitReader(g); + assertEquals(String.format("%d", i), expect, bv.readBits64(4 + i)); + } + } + @Test public void testShift() { BitReader bv; diff --git a/pom.xml b/pom.xml index 0f97187d..d3d371c2 100644 --- a/pom.xml +++ b/pom.xml @@ -112,7 +112,7 @@ - +