Use crc32c algorithm to calculate tags if CRC32 algorothm is specified. (ec574564) · Commits · kinetic-archive / kinetic-java

kinetic-common/src/main/java/com/google/cloud/Crc32c.java

0 → 100644

+160 −0

Original line number	Diff line number	Diff line
		// Copyright 2011 Google Inc. All rights reserved.

		package com.google.cloud;

		import java.util.zip.Checksum;

		/**
		* This class generates a CRC32C checksum, defined by rfc3720 section B.4.
		*
		* license: Apache License Version 2.0, January 2004
		* http://www.apache.org/licenses/
		*
		* https://github.com/GoogleCloudPlatform/crc32c-java/blob/master/LICENSE
		*
		* This class is adopted (no modifications) from here:
		*
		* https://github.com/GoogleCloudPlatform/crc32c-java/blob/master/src/com/google
		* /cloud/Crc32c.java
		*/
		public final class Crc32c implements Checksum {

		private static final long[] CRC_TABLE = {
		0x00000000, 0xf26b8303, 0xe13b70f7, 0x1350f3f4,
		0xc79a971f, 0x35f1141c, 0x26a1e7e8, 0xd4ca64eb,
		0x8ad958cf, 0x78b2dbcc, 0x6be22838, 0x9989ab3b,
		0x4d43cfd0, 0xbf284cd3, 0xac78bf27, 0x5e133c24,
		0x105ec76f, 0xe235446c, 0xf165b798, 0x030e349b,
		0xd7c45070, 0x25afd373, 0x36ff2087, 0xc494a384,
		0x9a879fa0, 0x68ec1ca3, 0x7bbcef57, 0x89d76c54,
		0x5d1d08bf, 0xaf768bbc, 0xbc267848, 0x4e4dfb4b,
		0x20bd8ede, 0xd2d60ddd, 0xc186fe29, 0x33ed7d2a,
		0xe72719c1, 0x154c9ac2, 0x061c6936, 0xf477ea35,
		0xaa64d611, 0x580f5512, 0x4b5fa6e6, 0xb93425e5,
		0x6dfe410e, 0x9f95c20d, 0x8cc531f9, 0x7eaeb2fa,
		0x30e349b1, 0xc288cab2, 0xd1d83946, 0x23b3ba45,
		0xf779deae, 0x05125dad, 0x1642ae59, 0xe4292d5a,
		0xba3a117e, 0x4851927d, 0x5b016189, 0xa96ae28a,
		0x7da08661, 0x8fcb0562, 0x9c9bf696, 0x6ef07595,
		0x417b1dbc, 0xb3109ebf, 0xa0406d4b, 0x522bee48,
		0x86e18aa3, 0x748a09a0, 0x67dafa54, 0x95b17957,
		0xcba24573, 0x39c9c670, 0x2a993584, 0xd8f2b687,
		0x0c38d26c, 0xfe53516f, 0xed03a29b, 0x1f682198,
		0x5125dad3, 0xa34e59d0, 0xb01eaa24, 0x42752927,
		0x96bf4dcc, 0x64d4cecf, 0x77843d3b, 0x85efbe38,
		0xdbfc821c, 0x2997011f, 0x3ac7f2eb, 0xc8ac71e8,
		0x1c661503, 0xee0d9600, 0xfd5d65f4, 0x0f36e6f7,
		0x61c69362, 0x93ad1061, 0x80fde395, 0x72966096,
		0xa65c047d, 0x5437877e, 0x4767748a, 0xb50cf789,
		0xeb1fcbad, 0x197448ae, 0x0a24bb5a, 0xf84f3859,
		0x2c855cb2, 0xdeeedfb1, 0xcdbe2c45, 0x3fd5af46,
		0x7198540d, 0x83f3d70e, 0x90a324fa, 0x62c8a7f9,
		0xb602c312, 0x44694011, 0x5739b3e5, 0xa55230e6,
		0xfb410cc2, 0x092a8fc1, 0x1a7a7c35, 0xe811ff36,
		0x3cdb9bdd, 0xceb018de, 0xdde0eb2a, 0x2f8b6829,
		0x82f63b78, 0x709db87b, 0x63cd4b8f, 0x91a6c88c,
		0x456cac67, 0xb7072f64, 0xa457dc90, 0x563c5f93,
		0x082f63b7, 0xfa44e0b4, 0xe9141340, 0x1b7f9043,
		0xcfb5f4a8, 0x3dde77ab, 0x2e8e845f, 0xdce5075c,
		0x92a8fc17, 0x60c37f14, 0x73938ce0, 0x81f80fe3,
		0x55326b08, 0xa759e80b, 0xb4091bff, 0x466298fc,
		0x1871a4d8, 0xea1a27db, 0xf94ad42f, 0x0b21572c,
		0xdfeb33c7, 0x2d80b0c4, 0x3ed04330, 0xccbbc033,
		0xa24bb5a6, 0x502036a5, 0x4370c551, 0xb11b4652,
		0x65d122b9, 0x97baa1ba, 0x84ea524e, 0x7681d14d,
		0x2892ed69, 0xdaf96e6a, 0xc9a99d9e, 0x3bc21e9d,
		0xef087a76, 0x1d63f975, 0x0e330a81, 0xfc588982,
		0xb21572c9, 0x407ef1ca, 0x532e023e, 0xa145813d,
		0x758fe5d6, 0x87e466d5, 0x94b49521, 0x66df1622,
		0x38cc2a06, 0xcaa7a905, 0xd9f75af1, 0x2b9cd9f2,
		0xff56bd19, 0x0d3d3e1a, 0x1e6dcdee, 0xec064eed,
		0xc38d26c4, 0x31e6a5c7, 0x22b65633, 0xd0ddd530,
		0x0417b1db, 0xf67c32d8, 0xe52cc12c, 0x1747422f,
		0x49547e0b, 0xbb3ffd08, 0xa86f0efc, 0x5a048dff,
		0x8ecee914, 0x7ca56a17, 0x6ff599e3, 0x9d9e1ae0,
		0xd3d3e1ab, 0x21b862a8, 0x32e8915c, 0xc083125f,
		0x144976b4, 0xe622f5b7, 0xf5720643, 0x07198540,
		0x590ab964, 0xab613a67, 0xb831c993, 0x4a5a4a90,
		0x9e902e7b, 0x6cfbad78, 0x7fab5e8c, 0x8dc0dd8f,
		0xe330a81a, 0x115b2b19, 0x020bd8ed, 0xf0605bee,
		0x24aa3f05, 0xd6c1bc06, 0xc5914ff2, 0x37faccf1,
		0x69e9f0d5, 0x9b8273d6, 0x88d28022, 0x7ab90321,
		0xae7367ca, 0x5c18e4c9, 0x4f48173d, 0xbd23943e,
		0xf36e6f75, 0x0105ec76, 0x12551f82, 0xe03e9c81,
		0x34f4f86a, 0xc69f7b69, 0xd5cf889d, 0x27a40b9e,
		0x79b737ba, 0x8bdcb4b9, 0x988c474d, 0x6ae7c44e,
		0xbe2da0a5, 0x4c4623a6, 0x5f16d052, 0xad7d5351
		};

		private static final long LONG_MASK = 0xffffffffL;
		private static final long BYTE_MASK = 0xff;

		private long crc;

		public Crc32c() {
		crc = 0;
		}

		/**
		* Updates the checksum with a new byte.
		* @param b the new byte.
		*/
		@Override
		public void update(int b) {
		long newCrc = crc ^ LONG_MASK;
		newCrc = updateByte((byte) b, newCrc);
		crc = newCrc ^ LONG_MASK;
		}

		/**
		* Updates the checksum with an array of bytes.
		* @param bArray the array of bytes.
		* @param off the offset into the array where the update should begin.
		* @param len the length of data to examine.
		*/
		@Override
		public void update(byte[] bArray, int off, int len) {
		long newCrc = crc ^ LONG_MASK;
		for (int i = off; i < off + len; i++) {
		newCrc = updateByte(bArray[i], newCrc);
		}
		crc = newCrc ^ LONG_MASK;
		}

		/**
		* Returns the value of the checksum.
		* @return the long representation of the checksum (high bits set to zero).
		*/
		@Override
		public long getValue() {
		return crc;
		}

		/**
		* Returns the value of the checksum.
		* @return the 4-byte array representation of the checksum in network byte order (big endian).
		*/
		public byte[] getValueAsBytes() {
		long value = crc;
		byte[] result = new byte[4];
		for (int i = 3; i >= 0; i--) {
		result[i] = (byte) (value & 0xffL);
		value >>= 8;
		}
		return result;
		}

		/**
		* Resets the crc.
		*/
		@Override
		public void reset() {
		crc = 0;
		}

		private long updateByte(byte newByte, long crc) {
		byte b = (byte) (newByte & BYTE_MASK);
		int index = (int) ((crc ^ b) & BYTE_MASK);
		return (CRC_TABLE[index] ^ (crc >> 8)) & LONG_MASK;
		}
		}

kinetic-common/src/main/java/com/seagate/kinetic/common/lib/Crc32cTagCalc.java

+2 −0

Original line number	Diff line number	Diff line
		@@ -55,6 +55,8 @@ public class Crc32cTagCalc implements KineticTagCalc {
		// calculate crc32c checksum
		int cval = Snappy.calculateChecksum(bb);

		logger.info("****** cval = " + cval);

		// convert to byte[]
		byte[] checkSum = ByteBuffer.allocate(4).putInt(cval).array();

kinetic-common/src/main/java/com/seagate/kinetic/common/lib/Crc32cTagCalc2.java

0 → 100644

+66 −0

Original line number	Diff line number	Diff line
		/**
		* Copyright (C) 2014 Seagate Technology.
		*
		* This library is free software; you can redistribute it and/or
		* modify it under the terms of the GNU Lesser General Public
		* License as published by the Free Software Foundation; either
		* version 2.1 of the License, or (at your option) any later version.
		*
		* This library is distributed in the hope that it will be useful,
		* but WITHOUT ANY WARRANTY; without even the implied warranty of
		* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
		* Lesser General Public License for more details.
		*
		* You should have received a copy of the GNU Lesser General Public
		* License along with this library; if not, write to the Free Software
		* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
		*/
		package com.seagate.kinetic.common.lib;

		import java.util.logging.Logger;

		import com.google.cloud.Crc32c;
		import com.google.protobuf.ByteString;

		/**
		* Kinetic tag calculation with CRC32C checksum algorithm util.
		*
		* @author chiaming
		*
		*/
		public class Crc32cTagCalc2 implements KineticTagCalc {

		private final static Logger logger = Logger.getLogger(Crc32cTagCalc2.class
		.getName());

		// crc32c algo
		private static String myName = "CRC32c";

		private Crc32c crc32c = new Crc32c();

		public Crc32cTagCalc2() {
		logger.info(myName + " checksum instance instantiated ...");
		}

		@Override
		public synchronized ByteString calculateTag(byte[] value) {

		try {

		// calculate crc32c checksum
		this.crc32c.update(value, 0, value.length);
		byte[] checkSum = this.crc32c.getValueAsBytes();

		// convert to bytestring and return
		return ByteString.copyFrom(checkSum);
		} finally {
		this.crc32c.reset();
		}
		}

		@Override
		public String getAlgoName() {
		return myName;
		}

		}

kinetic-common/src/main/java/com/seagate/kinetic/common/lib/MessageDigestUtil.java

+2 −3

Original line number	Diff line number	Diff line
		@@ -165,7 +165,7 @@ public class MessageDigestUtil {

		// check if already constructed
		if (crc32 == null) {
		crc32 = new Crc32cTagCalc();
		crc32 = new Crc32cTagCalc2();
		}
		}

		@@ -181,8 +181,7 @@ public class MessageDigestUtil {
		case SHA2:
		return getSha2Instance();
		case CRC32:
		return getCrc32Instance();
		// return getCrc32cInstance();
		return getCrc32cInstance();
		default:
		throw new java.lang.UnsupportedOperationException(
		"unsupported algorithm., name = " + algo.name());

kinetic-test/src/test/java/com/seagate/kinetic/example/client/TagUsageExample.java

0 → 100644

+135 −0

Original line number	Diff line number	Diff line
		/**
		*
		* Copyright (C) 2014 Seagate Technology.
		*
		* This program is free software; you can redistribute it and/or
		* modify it under the terms of the GNU General Public License
		* as published by the Free Software Foundation; either version 2
		* of the License, or (at your option) any later version.
		*
		* This program is distributed in the hope that it will be useful,
		* but WITHOUT ANY WARRANTY; without even the implied warranty of
		* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
		* GNU General Public License for more details.
		*
		* You should have received a copy of the GNU General Public License
		* along with this program; if not, write to the Free Software
		* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
		*
		*/
		package com.seagate.kinetic.example.client;

		import java.io.UnsupportedEncodingException;
		import java.util.Arrays;

		import kinetic.client.ClientConfiguration;
		import kinetic.client.Entry;
		import kinetic.client.KineticClient;
		import kinetic.client.KineticClientFactory;
		import kinetic.client.KineticException;

		import com.google.protobuf.ByteString;
		import com.seagate.kinetic.common.lib.MessageDigestUtil;
		import com.seagate.kinetic.proto.Kinetic.Command.Algorithm;

		/**
		* Kinetic tag field usage example.
		*
		*/
		public class TagUsageExample {

		// String to byte[] encoding
		public static final String UTF8 = "utf8";

		// kinetic client
		private KineticClient client = null;

		public void runExample() throws KineticException, InterruptedException {

		// Client configuration and initialization
		ClientConfiguration clientConfig = new ClientConfiguration();

		// create client instance
		client = KineticClientFactory.createInstance(clientConfig);

		// initialize key and value
		byte[] key = stringToBytes("hello");

		byte[] value = new byte[32];
		for (int i = 0; i < value.length; i++) {
		value[i] = (byte) 0XFF;
		}

		// create two entries
		Entry simpleEntry1 = new Entry(key, value);

		// set tag algorithm
		simpleEntry1.getEntryMetadata()
		.setAlgorithm(Algorithm.CRC32.toString());

		// calculate tag with CRC32C algorithm
		ByteString tag = MessageDigestUtil.calculateTag(Algorithm.CRC32, value);

		// to byte[]
		byte[] crc32c = tag.toByteArray();

		// set tag
		simpleEntry1.getEntryMetadata().setTag(crc32c);

		// put entry to store
		client.putForced(simpleEntry1);

		// get entry from store
		Entry entry2 = client.get(key);

		// calculate tag from value
		byte[] calculatedTag = MessageDigestUtil.calculateTag(Algorithm.CRC32,
		entry2.getValue()).toByteArray();

		// get the tag field
		byte[] tag2 = entry2.getEntryMetadata().getTag();

		// compare tags
		if (Arrays.equals(calculatedTag, tag2) == false) {
		throw new RuntimeException("tag does not compare");
		}

		System.out.println("tag is verified.");

		// delete entry
		client.delete(simpleEntry1);

		// close kinetic client
		this.client.close();
		}

		/**
		* convert string to byte[] using UTF8 encoding.
		*
		* @param string
		* string to be converted to byte[].
		*
		* @return the byte[] representation of the specified string
		*/
		private static byte[] stringToBytes(String string) {

		try {
		return string.getBytes(UTF8);
		} catch (UnsupportedEncodingException e) {
		e.printStackTrace();
		}

		return null;
		}

		/**
		* Simple example to demonstrate Kinetic tag field usage.
		*/
		public static void main(String[] args) throws KineticException,
		InterruptedException {

		TagUsageExample tagUsage = new TagUsageExample();

		tagUsage.runExample();
		}
		}