crypto/modes/gcm128.c

145 static void gcm_gmult_8bit(u64 Xi[2], const u128 Htable[256])
148 	const u8 *xi = (const u8 *)Xi+15;
221 		if ((u8 *)Xi==xi)	break;
236 		Xi[0] = BSWAP8(Z.hi);
237 		Xi[1] = BSWAP8(Z.lo);
239 		u8 *p = (u8 *)Xi;
248 		Xi[0] = Z.hi;
249 		Xi[1] = Z.lo;
252 #define GCM_MUL(ctx,Xi)   gcm_gmult_8bit(ctx->Xi.u,ctx->Htable)
335 static void gcm_gmult_4bit(u64 Xi[2], const u128 Htable[16])
342 	nlo  = ((const u8 *)Xi)[15];
363 		nlo  = ((const u8 *)Xi)[cnt];
381 		Xi[0] = BSWAP8(Z.hi);
382 		Xi[1] = BSWAP8(Z.lo);
384 		u8 *p = (u8 *)Xi;
393 		Xi[0] = Z.hi;
394 		Xi[1] = Z.lo;
406 static void gcm_ghash_4bit(u64 Xi[2],const u128 Htable[16],
417 	nlo  = ((const u8 *)Xi)[15];
439 		nlo  = ((const u8 *)Xi)[cnt];
513 		nlo  = ((const u8 *)Xi)[cnt];
531 	nlo  = ((const u8 *)Xi)[0];
551 		Xi[0] = BSWAP8(Z.hi);
552 		Xi[1] = BSWAP8(Z.lo);
554 		u8 *p = (u8 *)Xi;
563 		Xi[0] = Z.hi;
564 		Xi[1] = Z.lo;
570 void gcm_gmult_4bit(u64 Xi[2],const u128 Htable[16]);
571 void gcm_ghash_4bit(u64 Xi[2],const u128 Htable[16],const u8 *inp,size_t len);
574 #define GCM_MUL(ctx,Xi)   gcm_gmult_4bit(ctx->Xi.u,ctx->Htable)
576 #define GHASH(ctx,in,len) gcm_ghash_4bit((ctx)->Xi.u,(ctx)->Htable,in,len)
585 static void gcm_gmult_1bit(u64 Xi[2],const u64 H[2])
590 	const long *xi = (const long *)Xi;
625 		Xi[0] = BSWAP8(Z.hi);
626 		Xi[1] = BSWAP8(Z.lo);
628 		u8 *p = (u8 *)Xi;
637 		Xi[0] = Z.hi;
638 		Xi[1] = Z.lo;
641 #define GCM_MUL(ctx,Xi)	  gcm_gmult_1bit(ctx->Xi.u,ctx->H.u)
654 void gcm_init_clmul(u128 Htable[16],const u64 Xi[2]);
655 void gcm_gmult_clmul(u64 Xi[2],const u128 Htable[16]);
656 void gcm_ghash_clmul(u64 Xi[2],const u128 Htable[16],const u8 *inp,size_t len);
660 void gcm_gmult_4bit_mmx(u64 Xi[2],const u128 Htable[16]);
661 void gcm_ghash_4bit_mmx(u64 Xi[2],const u128 Htable[16],const u8 *inp,size_t len);
663 void gcm_gmult_4bit_x86(u64 Xi[2],const u128 Htable[16]);
664 void gcm_ghash_4bit_x86(u64 Xi[2],const u128 Htable[16],const u8 *inp,size_t len);
671 void gcm_gmult_neon(u64 Xi[2],const u128 Htable[16]);
672 void gcm_ghash_neon(u64 Xi[2],const u128 Htable[16],const u8 *inp,size_t len);
679 # define GCM_MUL(ctx,Xi)	(*gcm_gmult_p)(ctx->Xi.u,ctx->Htable)
682 #  define GHASH(ctx,in,len)	(*gcm_ghash_p)(ctx->Xi.u,ctx->Htable,in,len)
761 	void (*gcm_gmult_p)(u64 Xi[2],const u128 Htable[16])	= ctx->gmult;
766 	ctx->Xi.u[0]  = 0;
767 	ctx->Xi.u[1]  = 0;
840 	void (*gcm_gmult_p)(u64 Xi[2],const u128 Htable[16])	= ctx->gmult;
842 	void (*gcm_ghash_p)(u64 Xi[2],const u128 Htable[16],
857 			ctx->Xi.c[n] ^= *(aad++);
861 		if (n==0) GCM_MUL(ctx,Xi);
876 		for (i=0; i<16; ++i) ctx->Xi.c[i] ^= aad[i];
877 		GCM_MUL(ctx,Xi);
884 		for (i=0; i<len; ++i) ctx->Xi.c[i] ^= aad[i];
902 	void (*gcm_gmult_p)(u64 Xi[2],const u128 Htable[16])	= ctx->gmult;
904 	void (*gcm_ghash_p)(u64 Xi[2],const u128 Htable[16],
919 		GCM_MUL(ctx,Xi);
937 				ctx->Xi.c[n] ^= *(out++) = *(in++)^ctx->EKi.c[n];
941 			if (n==0) GCM_MUL(ctx,Xi);
1019 				ctx->Xi.t[i] ^=
1021 			GCM_MUL(ctx,Xi);
1039 				ctx->Xi.c[n] ^= out[n] = in[n]^ctx->EKi.c[n];
1061 		ctx->Xi.c[n] ^= out[i] = in[i]^ctx->EKi.c[n];
1064 			GCM_MUL(ctx,Xi);
1082 	void (*gcm_gmult_p)(u64 Xi[2],const u128 Htable[16])	= ctx->gmult;
1084 	void (*gcm_ghash_p)(u64 Xi[2],const u128 Htable[16],
1096 		GCM_MUL(ctx,Xi);
1116 				ctx->Xi.c[n] ^= c;
1120 			if (n==0) GCM_MUL (ctx,Xi);
1198 				ctx->Xi.t[i] ^= c;
1200 			GCM_MUL(ctx,Xi);
1219 				ctx->Xi.c[n] ^= c;
1245 		ctx->Xi.c[n] ^= c;
1248 			GCM_MUL(ctx,Xi);
1265 	void (*gcm_gmult_p)(u64 Xi[2],const u128 Htable[16])	= ctx->gmult;
1267 	void (*gcm_ghash_p)(u64 Xi[2],const u128 Htable[16],
1279 		GCM_MUL(ctx,Xi);
1295 			ctx->Xi.c[n] ^= *(out++) = *(in++)^ctx->EKi.c[n];
1299 		if (n==0) GCM_MUL(ctx,Xi);
1343 			for (i=0;i<16;++i) ctx->Xi.c[i] ^= out[i];
1344 			GCM_MUL(ctx,Xi);
1361 			ctx->Xi.c[n] ^= out[n] = in[n]^ctx->EKi.c[n];
1380 	void (*gcm_gmult_p)(u64 Xi[2],const u128 Htable[16])	= ctx->gmult;
1382 	void (*gcm_ghash_p)(u64 Xi[2],const u128 Htable[16],
1394 		GCM_MUL(ctx,Xi);
1412 			ctx->Xi.c[n] ^= c;
1416 		if (n==0) GCM_MUL (ctx,Xi);
1448 			for (k=0;k<16;++k) ctx->Xi.c[k] ^= in[k];
1449 			GCM_MUL(ctx,Xi);
1482 			ctx->Xi.c[n] ^= c;
1499 	void (*gcm_gmult_p)(u64 Xi[2],const u128 Htable[16])	= ctx->gmult;
1503 		GCM_MUL(ctx,Xi);
1520 	ctx->Xi.u[0] ^= alen;
1521 	ctx->Xi.u[1] ^= clen;
1522 	GCM_MUL(ctx,Xi);
1524 	ctx->Xi.u[0] ^= ctx->EK0.u[0];
1525 	ctx->Xi.u[1] ^= ctx->EK0.u[1];
1527 	if (tag && len<=sizeof(ctx->Xi))
1528 		return memcmp(ctx->Xi.c,tag,len);
1536 	memcpy(tag, ctx->Xi.c, len<=sizeof(ctx->Xi.c)?len:sizeof(ctx->Xi.c));
1890 	void (*gcm_ghash_p)(u64 Xi[2],const u128 Htable[16],