ccan/crc32c: new module for accelerated CRC32 (on x86-64).

Note: the previous code in ccan/crc is wrong, so I started fresh with
actual test vectors.
Signed-off-by: Rusty Russell <rusty@rustcorp.com.au>

ccan/crc32c/_info

+#include "config.h"
+#include <string.h>
+#include <stdio.h>
+
+/**
+ * crc32c - routine for Castagnoli CRC (crc32c) of bytes
+ *
+ * Cyclic Redundancy Check routine, optimized for x86-64.  Reasonably fast
+ * checksum routines, but not suitable for cryptographic use.
+ *
+ * They are useful for simple error detection, eg. a 32-bit CRC will
+ * detect a single error burst of up to 32 bits.
+ *
+ * Example:
+ *	#include <ccan/crc32c/crc32c.h>
+ *	#include <stdio.h>
+ *	#include <stdlib.h>
+ *
+ *	// Given "123456789" outputs 0xe3069283
+ *	int main(int argc, char *argv[])
+ *	{
+ *		if (argc != 2) {
+ *			fprintf(stderr, "Usage: %s <string>\n"
+ *				"Prints 32 bit CRC of the string\n", argv[0]);
+ *			exit(1);
+ *		}
+ *		printf("0x%08x\n", crc32c(0, argv[1], strlen(argv[1])));
+ *		exit(0);
+ *	}
+ *
+ * License: MIT
+ * Author: Mark Adler
+ * Maintainer: Rusty Russell <rusty@rustcorp.com.au>
+ */
+int main(int argc, char *argv[])
+{
+	if (argc != 2)
+		return 1;
+
+	if (strcmp(argv[1], "depends") == 0) {
+		printf("ccan/compiler\n");
+		return 0;
+	}
+
+	return 1;
+}

ccan/crc32c/benchmark/Makefile

+CCANDIR=../../..
+CFLAGS=-Wall -Werror -O3 -I$(CCANDIR) -flto
+#CFLAGS=-Wall -Werror -g3 -I$(CCANDIR)
+LDFLAGS := -flto -O3
+
+all: bench
+
+CCAN_OBJS:=ccan-tal.o ccan-tal-grab_file.o ccan-noerr.o ccan-take.o ccan-time.o
+
+bench: bench.o $(CCAN_OBJS)
+
+clean:
+	rm -f bench *.o
+
+ccan-time.o: $(CCANDIR)/ccan/time/time.c
+	$(CC) $(CFLAGS) -c -o $@ $<
+ccan-tal.o: $(CCANDIR)/ccan/tal/tal.c
+	$(CC) $(CFLAGS) -c -o $@ $<
+ccan-take.o: $(CCANDIR)/ccan/take/take.c
+	$(CC) $(CFLAGS) -c -o $@ $<
+ccan-noerr.o: $(CCANDIR)/ccan/noerr/noerr.c
+	$(CC) $(CFLAGS) -c -o $@ $<
+ccan-tal-grab_file.o: $(CCANDIR)/ccan/tal/grab_file/grab_file.c
+	$(CC) $(CFLAGS) -c -o $@ $<

ccan/crc32c/benchmark/bench.c

+#include <ccan/tal/grab_file/grab_file.h>
+#include <ccan/tal/tal.h>
+#include <ccan/time/time.h>
+#include <ccan/crc32c/crc32c.c>
+#include <assert.h>
+#include <ccan/err/err.h>
+
+#define RUNS 65536
+
+int main(int argc, char *argv[])
+{
+	void *p;
+	struct timeabs start, end;
+	size_t len, runs;
+	uint64_t sums = 0;
+	bool sw = false, hw = false;
+
+	if (argv[1]) {
+		if (streq(argv[1], "--software")) {
+			sw = true;
+			argv++;
+			argc--;
+
+		} else if (streq(argv[1], "--hardware")) {
+			hw = true;
+			argv++;
+			argc--;
+		}
+	}
+
+	if (argc < 2 || (runs = atol(argv[1])) == 0)
+		errx(1, "Usage: bench <num-runs> [<file>]");
+
+	p = grab_file(NULL, argv[2]);
+	if (!p)
+		err(1, "Reading %s", argv[2] ? argv[2] : "<stdin>");
+	len = tal_count(p) - 1;
+	start = time_now();
+	if (sw) {
+		for (size_t i = 0; i < runs; i++)
+			sums += crc32c_sw(0, p, len);
+	} else if (hw) {
+		for (size_t i = 0; i < runs; i++)
+			sums += crc32c_hw(0, p, len);
+	} else {
+		for (size_t i = 0; i < runs; i++)
+			sums += crc32c(0, p, len);
+	}
+	end = time_now();
+
+	assert(sums % runs == 0);
+	printf("%u usec for %zu bytes, sum=%08x\n",
+	       (int)time_to_usec(time_divide(time_between(end, start), runs)),
+	       len,
+	       (unsigned int)(sums / runs));
+	return 0;
+}

ccan/crc32c/crc32c.h

+/* Licensed under MIT - see LICENSE file for details */
+#ifndef CCAN_CRC32C_H
+#define CCAN_CRC32C_H
+#include <stdint.h>
+#include <stdlib.h>
+
+/**
+ * crc32c - Castagnoli 32 bit crc of string of bytes
+ * @start_crc: the initial crc (usually 0)
+ * @buf: pointer to bytes
+ * @size: length of buffer
+ *
+ * If you don't know what crc32 to use, use this one: it's the best.
+ *
+ * @Article{castagnoli-crc,
+ * author =       { Guy Castagnoli and Stefan Braeuer and Martin Herrman},
+ * title =        {{Optimization of Cyclic Redundancy-Check Codes with 24
+ *                 and 32 Parity Bits}},
+ * journal =      IEEE Transactions on Communication,
+ * year =         {1993},
+ * volume =       {41},
+ * number =       {6},
+ * pages =        {},
+ * month =        {June},
+ *}
+ * 32 bit CRC checksum using polynomial
+ * X^32+X^28+X^27+X^26+X^25+X^23+X^22+X^20+X^19+X^18+X^14+X^13+X^11+X^10+X^9+X^8+X^6+X^0.
+ *
+ * You can calculate the CRC of non-contiguous arrays by passing @start_crc
+ * as 0 the first time, and the current crc result from then on.
+ *
+ * Example:
+ *	#include <sys/uio.h>
+ *	...
+ *	// Check that iovec has the crc we expect (Castagnoli version)
+ *	static bool check_crc(uint32_t expected, const struct iovec *iov, int l)
+ *	{
+ *		uint32_t crc = 0;
+ *		while (l >= 0) {
+ *			crc = crc32c(crc, iov->iov_base, iov->iov_len);
+ *			iov++;
+ *		}
+ *		return crc == expected;
+ *	}
+ */
+uint32_t crc32c(uint32_t start_crc, const void *buf, size_t size);
+
+#endif /* CCAN_CRC32C_H */

ccan/crc32c/test/api-crc32c.c

+/* Test vectors from https://tools.ietf.org/html/rfc3720#appendix-B.4 */
+#include <ccan/crc32c/crc32c.h>
+#include <ccan/tap/tap.h>
+#include <string.h>
+
+#define BSWAP_32(val)					\
+	((((uint32_t)(val) & 0x000000ff) << 24)		\
+	 | (((uint32_t)(val) & 0x0000ff00) << 8)		\
+	 | (((uint32_t)(val) & 0x00ff0000) >> 8)		\
+	 | (((uint32_t)(val) & 0xff000000) >> 24))
+
+#if HAVE_LITTLE_ENDIAN
+#define BE32_TO_CPU(le_val) BSWAP_32((uint32_t)le_val)
+#else
+#define BE32_TO_CPU(le_val) ((uint32_t)(le_val))
+#endif
+
+int main(void)
+{
+	unsigned char m[48];
+
+	plan_tests(5);
+
+	/* 32 bytes of zeroes:
+
+     Byte:        0  1  2  3
+
+        0:       00 00 00 00
+      ...
+       28:       00 00 00 00
+
+      CRC:       aa 36 91 8a
+	*/
+
+	memset(m, 0, 32);
+	ok1(crc32c(0, m, 32) == BE32_TO_CPU(0xaa36918a));
+
+	/* 32 bytes of ones:
+
+     Byte:        0  1  2  3
+
+        0:       ff ff ff ff
+      ...
+       28:       ff ff ff ff
+
+      CRC:       43 ab a8 62
+	*/
+	memset(m, 0xff, 32);
+	ok1(crc32c(0, m, 32) == BE32_TO_CPU(0x43aba862));
+
+	/* 32 bytes of incrementing 00..1f:
+
+     Byte:        0  1  2  3
+
+        0:       00 01 02 03
+      ...
+       28:       1c 1d 1e 1f
+
+      CRC:       4e 79 dd 46
+	*/
+	for (size_t i = 0; i < 32; i++)
+		m[i] = i;
+	ok1(crc32c(0, m, 32) == BE32_TO_CPU(0x4e79dd46));
+
+	/*  32 bytes of decrementing 1f..00:
+
+     Byte:        0  1  2  3
+
+        0:       1f 1e 1d 1c
+      ...
+       28:       03 02 01 00
+
+      CRC:       5c db 3f 11
+	*/
+	for (size_t i = 0; i < 32; i++)
+		m[i] = 31 - i;
+	ok1(crc32c(0, m, 32) == BE32_TO_CPU(0x5cdb3f11));
+
+	/*  An iSCSI - SCSI Read (10) Command PDU
+    Byte:        0  1  2  3
+
+       0:       01 c0 00 00
+       4:       00 00 00 00
+       8:       00 00 00 00
+      12:       00 00 00 00
+      16:       14 00 00 00
+      20:       00 00 04 00
+      24:       00 00 00 14
+      28:       00 00 00 18
+      32:       28 00 00 00
+      36:       00 00 00 00
+      40:       02 00 00 00
+      44:       00 00 00 00
+
+     CRC:       56 3a 96 d9
+	*/
+	memset(m, 0, sizeof(m));
+	m[0] = 0x01;
+	m[1] = 0xc0;
+	m[16] = 0x14;
+	m[22] = 0x04;
+	m[27] = 0x14;
+	m[31] = 0x18;
+	m[32] = 0x28;
+	m[40] = 0x02;
+	ok1(crc32c(0, m, sizeof(m)) == BE32_TO_CPU(0x563a96d9));
+
+	return exit_status();
+}

ccan/crc32c/test/run-crc32c.c

+/* Test vectors from https://tools.ietf.org/html/rfc3720#appendix-B.4 */
+
+/* Get access to sw version explicitly */
+#include <ccan/crc32c/crc32c.c>
+#include <ccan/tap/tap.h>
+#include <string.h>
+
+#define BSWAP_32(val)					\
+	((((uint32_t)(val) & 0x000000ff) << 24)		\
+	 | (((uint32_t)(val) & 0x0000ff00) << 8)		\
+	 | (((uint32_t)(val) & 0x00ff0000) >> 8)		\
+	 | (((uint32_t)(val) & 0xff000000) >> 24))
+
+#if HAVE_LITTLE_ENDIAN
+#define BE32_TO_CPU(le_val) BSWAP_32((uint32_t)le_val)
+#else
+#define BE32_TO_CPU(le_val) ((uint32_t)(le_val))
+#endif
+
+int main(void)
+{
+	unsigned char m[48];
+
+	plan_tests(5);
+
+	/* 32 bytes of zeroes:
+
+     Byte:        0  1  2  3
+
+        0:       00 00 00 00
+      ...
+       28:       00 00 00 00
+
+      CRC:       aa 36 91 8a
+	*/
+
+	memset(m, 0, 32);
+	ok1(crc32c_sw(0, m, 32) == BE32_TO_CPU(0xaa36918a));
+
+	/* 32 bytes of ones:
+
+     Byte:        0  1  2  3
+
+        0:       ff ff ff ff
+      ...
+       28:       ff ff ff ff
+
+      CRC:       43 ab a8 62
+	*/
+	memset(m, 0xff, 32);
+	ok1(crc32c_sw(0, m, 32) == BE32_TO_CPU(0x43aba862));
+
+	/* 32 bytes of incrementing 00..1f:
+
+     Byte:        0  1  2  3
+
+        0:       00 01 02 03
+      ...
+       28:       1c 1d 1e 1f
+
+      CRC:       4e 79 dd 46
+	*/
+	for (size_t i = 0; i < 32; i++)
+		m[i] = i;
+	ok1(crc32c_sw(0, m, 32) == BE32_TO_CPU(0x4e79dd46));
+
+	/*  32 bytes of decrementing 1f..00:
+
+     Byte:        0  1  2  3
+
+        0:       1f 1e 1d 1c
+      ...
+       28:       03 02 01 00
+
+      CRC:       5c db 3f 11
+	*/
+	for (size_t i = 0; i < 32; i++)
+		m[i] = 31 - i;
+	ok1(crc32c_sw(0, m, 32) == BE32_TO_CPU(0x5cdb3f11));
+
+	/*  An iSCSI - SCSI Read (10) Command PDU
+    Byte:        0  1  2  3
+
+       0:       01 c0 00 00
+       4:       00 00 00 00
+       8:       00 00 00 00
+      12:       00 00 00 00
+      16:       14 00 00 00
+      20:       00 00 04 00
+      24:       00 00 00 14
+      28:       00 00 00 18
+      32:       28 00 00 00
+      36:       00 00 00 00
+      40:       02 00 00 00
+      44:       00 00 00 00
+
+     CRC:       56 3a 96 d9
+	*/
+	memset(m, 0, sizeof(m));
+	m[0] = 0x01;
+	m[1] = 0xc0;
+	m[16] = 0x14;
+	m[22] = 0x04;
+	m[27] = 0x14;
+	m[31] = 0x18;
+	m[32] = 0x28;
+	m[40] = 0x02;
+	ok1(crc32c_sw(0, m, sizeof(m)) == BE32_TO_CPU(0x563a96d9));
+
+	return exit_status();
+}