
#define _GF16_XORJIT_COPY_ALIGN 16
#include "gf16_xor_common.h"
#undef _GF16_XORJIT_COPY_ALIGN

#include <string.h>
#ifdef __SSE2__
int gf16_xor_available_sse2 = 1;
#else
int gf16_xor_available_sse2 = 0;
#endif

#ifdef __SSE2__

/* modified versions of PXOR/XORPS mem to have fixed sized instructions */
static HEDLEY_ALWAYS_INLINE size_t _jit_pxor_mod(uint8_t* jit, uint8_t xreg, uint8_t mreg, int32_t offs) {
	*(jit++) = 0x66;
	size_t p = _jit_rex_pref(&jit, xreg, 0) +1;
	xreg &= 7;
	write32(jit, 0x40EF0F | (xreg <<19) | (mreg <<16));
	jit[3] = (uint8_t)offs;
	return p+4;
}
static HEDLEY_ALWAYS_INLINE size_t _jit_xorps_mod(uint8_t* jit, uint8_t xreg, uint8_t mreg, int32_t offs) {
	size_t p = _jit_rex_pref(&jit, xreg, 0);
	xreg &= 7;
	write32(jit, 0x40570F | (xreg <<19) | (mreg <<16) | lshift32(offs, 24));
	return p+4;
}

struct gf16_xor_jit_scratch_sse2 {
	/* code lookup tables for XOR-JIT; should align to 64 to maximize cache line usage */
	// TODO: for 32-bit, consider interleaving register sourced items more
	ALIGN_TO(16, __m128i clut_code1[64]);
	ALIGN_TO(16, __m128i clut_code2[64]);
#ifdef PLATFORM_AMD64
	ALIGN_TO(16, __m128i clut_code3[16]);
	ALIGN_TO(16, __m128i clut_code4[64]);
#else
	ALIGN_TO(16, __m128i clut_code3[64]);
	ALIGN_TO(16, __m128i clut_code4[16]);
#endif
	ALIGN_TO(16, __m128i clut_code5[64]);
	ALIGN_TO(16, __m128i clut_code6[16]);
	ALIGN_TO(16, uint16_t clut_info_mem[64]);
	ALIGN_TO(16, uint16_t clut_info_reg[64]);
	
	ALIGN_TO(16, __m128i clut_nocomm[8*16]);
	ALIGN_TO(16, uint16_t clut_ncinfo_mem[15]);
	ALIGN_TO(16, uint16_t clut_ncinfo_rm[15]);
	ALIGN_TO(16, uint16_t clut_ncinfo_reg[15]);
	
	struct gf16_xor_scratch s;
};

// seems like the no-common optimisation isn't worth it, so disable it by default
#define XORDEP_DISABLE_NO_COMMON 1

static void gf16_xor_create_jit_lut_sse2(struct gf16_xor_jit_scratch_sse2* scratch) {
	int i, j;
	
	memset(scratch->clut_code1, 0, sizeof(scratch->clut_code1));
	memset(scratch->clut_code2, 0, sizeof(scratch->clut_code2));
	memset(scratch->clut_code3, 0, sizeof(scratch->clut_code3));
	memset(scratch->clut_code4, 0, sizeof(scratch->clut_code4));
	memset(scratch->clut_code5, 0, sizeof(scratch->clut_code5));
	memset(scratch->clut_code6, 0, sizeof(scratch->clut_code6));
	
	
	/* XOR pairs/triples from memory */
#ifdef PLATFORM_AMD64
	#define MEM_XP 1
	#define MEM_XT 1
#else
	#define MEM_XP 5
	#define MEM_XT 3
#endif
	
	for(i=0; i<64; i++) {
		int m = i;
		int posM[4] = {0, 0, 0, 0};
		int posR[4] = {0, 0, 0, 0};
		uint8_t* pC[6] = {
			(uint8_t*)(scratch->clut_code1 + i),
			(uint8_t*)(scratch->clut_code2 + i),
			i < (int)(sizeof(scratch->clut_code3)/sizeof(*scratch->clut_code3)) ? (uint8_t*)(scratch->clut_code3 + i) : NULL,
			i < (int)(sizeof(scratch->clut_code4)/sizeof(*scratch->clut_code4)) ? (uint8_t*)(scratch->clut_code4 + i) : NULL,
			(uint8_t*)(scratch->clut_code5 + i),
			i < (int)(sizeof(scratch->clut_code6)/sizeof(*scratch->clut_code6)) ? (uint8_t*)(scratch->clut_code6 + i) : NULL
		};
		
		for(j=0; j<3; j++) {
			int msk = m&3;
			int k;
			if(msk == 1) {
				// (XORPS)
				for(k=0; k<MEM_XT; k++)
					pC[k] += _jit_xorps_mod(pC[k], 0, AX, lshift32(j-8 + k*3, 4));
#ifdef PLATFORM_AMD64
				pC[1] += _jit_xorps_r(pC[1], 0, j+3);
				pC[3] += _jit_xorps_r(pC[3], 0, j+8);
				pC[4] += _jit_xorps_r(pC[4], 0, j+11);
				if(i < 16 && pC[2]) { // pC[2] doesn't need to be checked, but GCC may throw a warning without it
					pC[2] += _jit_xorps_r(pC[2], 0, j+6);
					pC[5] += _jit_xorps_r(pC[5], 0, j+14);
				}
#else
				if(i < 16 && pC[3]) {
					pC[3] += _jit_xorps_mod(pC[3], 0, AX, (j+1) <<4);
					pC[5] += _jit_xorps_r(pC[5], 0, j+6);
				}
				pC[4] += _jit_xorps_r(pC[4], 0, j+3);
#endif
				// transformations (XORPS -> MOVAPS)
				if(posM[1] == 0) posM[1] = posM[0] +1;
				if(posR[1] == 0) posR[1] = posR[0] +1;
			} else if(msk) {
				int isCommon = msk == 3;
				int reg = 1 + isCommon;
				
				// (PXOR)
				for(k=0; k<MEM_XT; k++)
					pC[k] += _jit_pxor_mod(pC[k], reg, AX, lshift32(j-8 + k*3, 4));
#ifdef PLATFORM_AMD64
				pC[1] += _jit_pxor_r(pC[1], reg, j+3);
				pC[3] += _jit_pxor_r(pC[3], reg, j+8);
				pC[4] += _jit_pxor_r(pC[4], reg, j+11);
				if(i < 16 && pC[2]) {
					pC[2] += _jit_pxor_r(pC[2], reg, j+6);
					pC[5] += _jit_pxor_r(pC[5], reg, j+14);
				}
#else
				if(i < 16 && pC[3]) {
					pC[3] += _jit_pxor_mod(pC[3], reg, AX, (j+1) <<4);
					pC[5] += _jit_pxor_r(pC[5], reg, j+6);
				}
				pC[4] += _jit_pxor_r(pC[4], reg, j+3);
#endif
				
				// transformations (PXOR -> MOVDQA)
				if(posM[reg+1] == 0) posM[reg+1] = posM[0] +2;
				if(posR[reg+1] == 0) posR[reg+1] = posR[0] +2;
			}
			
			if(msk) { // bit1 || bit2
				int xb = msk != 1; // only bit1 set -> using XORPS (1 less byte)
				
				/* advance pointers */
				posM[0] += 4+xb;
				posR[0] += 3+xb;
			}
			
			m >>= 2;
		}
		
		scratch->clut_info_mem[i] = posM[0] | (posM[1] << 4) | (posM[2] << 8) | (posM[3] << 12);
		scratch->clut_info_reg[i] = posR[0] | (posR[1] << 4) | (posR[2] << 8) | (posR[3] << 12);
	}
	
#ifndef XORDEP_DISABLE_NO_COMMON
	memset(scratch->clut_nocomm, 0, sizeof(scratch->clut_code6));
	// handle cases of no common-mask optimisation
	for(i=0; i<15 /* not 16 */; i++) {
		// since we can only fit 2 pairs in an XMM register, cannot do 6 bit lookups
		int m = i;
		int k;
		int posM[3] = {0, 0, 0};
		int posR[3] = {0, 0, 0};
		int posRM[3] = {0, 0, 0};
		uint8_t* pC[8];
		for(k=0; k<8; k++) {
			pC[k] = (uint8_t*)(scratch->clut_nocomm + i + k*16);
		}
		
		
		for(j=0; j<2; j++) {
			if(m & 1) {
				// (XORPS)
				for(k=0; k<MEM_XP; k++) {
					pC[k] += _jit_xorps_m(pC[k], 0, AX, lshift32(j-8+k*2, 4));
				}
				if(j==0) {
					pC[MEM_XP] += _jit_xorps_m(pC[MEM_XP], 0, AX, lshift32(-8+MEM_XP*2, 4));
				} else {
					pC[MEM_XP] += _jit_xorps_r(pC[MEM_XP], 0, 3);
				}

				for(k=0; k<2; k++) {
					pC[k+MEM_XP+1] += _jit_xorps_r(pC[k+MEM_XP+1], 0, j+4+k*2);
				}
#ifdef PLATFORM_AMD64
				// registers64
				for(k=0; k<4; k++) {
					pC[k+4] += _jit_xorps_r(pC[k+4], 0, j+8+k*2);
				}
#endif
				// transformations (XORPS -> MOVAPS)
				if(posM[1] == 0) posM[1] = posM[0] +1;
				if(posR[1] == 0) posR[1] = posR[0] +1;
				if(posRM[1] == 0) posRM[1] = posRM[0] +1;
				posM[0] += 4;
				posR[0] += 3;
				posRM[0] += 3 + (j==0);
			}
			if(m & 2) {
				// (PXOR)
				for(k=0; k<MEM_XP; k++) {
					pC[k] += _jit_pxor_m(pC[k], 1, AX, lshift32(j-8+k*2, 4));
				}
				if(j==0) {
					pC[MEM_XP] += _jit_pxor_m(pC[MEM_XP], 1, AX, lshift32(-8+MEM_XP*2, 4));
				} else {
					pC[MEM_XP] += _jit_pxor_r(pC[MEM_XP], 1, 3);
				}

				for(k=0; k<2; k++) {
					pC[k+MEM_XP+1] += _jit_pxor_r(pC[k+MEM_XP+1], 1, j+4+k*2);
				}
#ifdef PLATFORM_AMD64
				// registers64
				for(k=0; k<4; k++) {
					pC[k+4] += _jit_pxor_r(pC[k+4], 1, j+8+k*2);
				}
#endif
				
				// transformations (PXOR -> MOVDQA)
				if(posM[2] == 0) posM[2] = posM[0] +2;
				if(posR[2] == 0) posR[2] = posR[0] +2;
				if(posRM[2] == 0) posRM[2] = posRM[0] +2;
				posM[0] += 5;
				posR[0] += 4;
				posRM[0] += 4 + (j==0);
			}
			
			m >>= 2;
		}
		
		scratch->clut_ncinfo_mem[i] = posM[0] | (posM[1] << 8) | (posM[2] << 12);
		scratch->clut_ncinfo_reg[i] = posR[0] | (posR[1] << 8) | (posR[2] << 12);
		scratch->clut_ncinfo_rm[i] = posRM[0] | (posRM[1] << 8) | (posRM[2] << 12);
		
	}
#endif
	#undef MEM_XP
	#undef MEM_XT
}




/* tune flags set by GCC; not ideal, but good enough I guess (note, I don't care about anything older than Core2) */
#if defined(__tune_core2__) || defined(__tune_atom__)
/* on pre-Nehalem Intel CPUs, it is faster to store unaligned XMM registers in halves */
static HEDLEY_ALWAYS_INLINE void STOREU_XMM(void* dest, __m128i xmm) {
	_mm_storel_epi64((__m128i*)(dest), xmm);
	_mm_storeh_pi(((__m64*)(dest) +1), _mm_castsi128_ps(xmm));
}
#else
# define STOREU_XMM(dest, xmm) \
	_mm_storeu_si128((__m128i*)(dest), xmm)
#endif

/* conditional move, because, for whatever reason, no-one thought of making a CMOVcc intrinsic */
#if defined(__GNUC__) || defined(__clang__)
	#define CMOV(cond, dst, src) __asm__( \
		"test %[c], %[c]\n" \
		"cmovnz %[s], %[d]\n" \
		: [d]"+r"(dst): [c]"r"(cond), [s]"r"(src))
#else
	//#define CMOV(c,d,s) (d) = ((c) & (s)) | (~(uintptr_t)(c) & (d));
	#define CMOV(c, d, s) if(c) (d) = (s)
#endif

static HEDLEY_ALWAYS_INLINE uint_fast16_t xor_jit_bitpair3(uint8_t* dest, uint_fast32_t mask, const __m128i* tCode, const uint16_t* tInfo, intptr_t* posC, long* movC, uint_fast8_t isR64) {
	uint_fast16_t info = tInfo[mask>>1];
	intptr_t pC = info >> 12;
	
	// copy code segment
	STOREU_XMM(dest, _mm_load_si128((__m128i*)((const uint64_t*)tCode + mask)));
	
	// handle conditional move for common mask (since it's always done)
	CMOV(*movC, *posC, pC+isR64);
	*posC -= info & 0xF;
	*movC &= -(long)(pC == 0);
	
	return info;
}

static HEDLEY_ALWAYS_INLINE uint_fast16_t xor_jit_bitpair3_noxor(uint8_t* dest, uint_fast16_t info, intptr_t* pos1, long* mov1, intptr_t* pos2, long* mov2, int isR64) {
	UNUSED(dest);
	uintptr_t p1 = (info >> 4) & 0xF;
	uintptr_t p2 = (info >> 8) & 0xF;
	CMOV(*mov1, *pos1, p1+isR64);
	CMOV(*mov2, *pos2, p2+isR64);
	*pos1 -= info & 0xF;
	*pos2 -= info & 0xF;
	*mov1 &= -(long)(p1 == 0);
	*mov2 &= -(long)(p2 == 0);
	return info & 0xF;
}

static HEDLEY_ALWAYS_INLINE uint_fast16_t xor_jit_bitpair3_nc_noxor(uint8_t* dest, uint_fast16_t info, intptr_t* pos1, long* mov1, intptr_t* pos2, long* mov2, int isR64) {
	UNUSED(dest);
	uintptr_t p1 = (info >> 8) & 0xF;
	uintptr_t p2 = info >> 12;
	CMOV(*mov1, *pos1, p1+isR64);
	CMOV(*mov2, *pos2, p2+isR64);
	*pos1 -= info & 0xF;
	*pos2 -= info & 0xF;
	*mov1 &= -(long)(p1 == 0);
	*mov2 &= -(long)(p2 == 0);
	return info & 0xF;
}
#undef CMOV



static inline void* xor_write_jit_sse(const void *HEDLEY_RESTRICT _scratch, uint8_t *HEDLEY_RESTRICT jitptr, uint16_t val, const int mode, const int prefetch) {
	const struct gf16_xor_jit_scratch_sse2 *HEDLEY_RESTRICT scratch = (const struct gf16_xor_jit_scratch_sse2*)_scratch;
	uint_fast32_t bit;
	ALIGN_TO(16, uint32_t lumask[8]);
	
	__m128i depmask1 = _mm_load_si128((__m128i*)(scratch->s.deps + ((val & 0xf) << 7)));
	__m128i depmask2 = _mm_load_si128((__m128i*)(scratch->s.deps + ((val & 0xf) << 7)) +1);
	depmask1 = _mm_xor_si128(depmask1, _mm_load_si128((__m128i*)(scratch->s.deps + ((val << 3) & 0x780)) + 1*2));
	depmask2 = _mm_xor_si128(depmask2, _mm_load_si128((__m128i*)(scratch->s.deps + ((val << 3) & 0x780)) + 1*2 +1));
	depmask1 = _mm_xor_si128(depmask1, _mm_load_si128((__m128i*)(scratch->s.deps + ((val >> 1) & 0x780)) + 2*2));
	depmask2 = _mm_xor_si128(depmask2, _mm_load_si128((__m128i*)(scratch->s.deps + ((val >> 1) & 0x780)) + 2*2 +1));
	depmask1 = _mm_xor_si128(depmask1, _mm_load_si128((__m128i*)(scratch->s.deps + ((val >> 5) & 0x780)) + 3*2));
	depmask2 = _mm_xor_si128(depmask2, _mm_load_si128((__m128i*)(scratch->s.deps + ((val >> 5) & 0x780)) + 3*2 +1));
	
	_mm_store_si128((__m128i*)(lumask), depmask1);
	_mm_store_si128((__m128i*)(lumask + 4), depmask2);
	
#ifndef XORDEP_DISABLE_NO_COMMON
	/* find cases where we don't wish to create the common queue - this is an optimisation to remove a single move operation when the common queue only contains one element */
	/* we have the common elements between pairs, but it doesn't make sense to process a separate queue if there's only one common element (0 XORs), so find those */
	__m128i common_mask1 = _mm_and_si128(depmask1, _mm_add_epi32(depmask1, depmask1));
	common_mask1 = _mm_and_si128(common_mask1, _mm_set1_epi8(0xaa));
	__m128i common_mask2 = _mm_and_si128(depmask2, _mm_add_epi32(depmask2, depmask2));
	common_mask2 = _mm_and_si128(common_mask2, _mm_set1_epi8(0xaa));
	
	__m128i common_mask_packed = _mm_packs_epi32(common_mask1, common_mask2);
	/* "(v & (v-1)) == 0" is true if only zero/one bit is set in each word */
	common_mask1 = _mm_and_si128(common_mask1, _mm_add_epi32(common_mask1, _mm_set1_epi32(-1)));
	common_mask2 = _mm_and_si128(common_mask2, _mm_add_epi32(common_mask2, _mm_set1_epi32(-1)));
	__m128i common_mask = _mm_andnot_si128(
		_mm_cmpeq_epi16(_mm_setzero_si128(), common_mask_packed),
		_mm_cmpeq_epi16(_mm_setzero_si128(), _mm_packs_epi32(common_mask1, common_mask2))
	);
	/* now we have a 8x16 mask of one-bit common masks we wish to remove; pack into an int for easy dealing with */
	// TODO: if this is in a memory source, consider allowing the common mask
	int no_common_mask = _mm_movemask_epi8(common_mask);
#else
	#define no_common_mask 0
#endif
	
	
	if(prefetch) {
		jitptr += _jit_add_i(jitptr, SI, 128);
		jitptr += _jit_prefetch_m(jitptr, prefetch, SI, 0);
		jitptr += _jit_prefetch_m(jitptr, prefetch, SI, 64);
	}
	
	//_jit_movaps_load(jit, reg, xreg, offs)
	// (we just save a conditional by hardcoding this)
	#define _LD_APS(xreg, mreg, offs) \
		write32((jitptr), 0x40280F + ((xreg) <<19) + ((mreg) <<16) + ((uint32_t)((offs)&0xFF) <<24)); \
		jitptr += 4
	#define _ST_APS(mreg, offs, xreg) \
		write32((jitptr), 0x40290F + ((xreg) <<19) + ((mreg) <<16) + ((uint32_t)((offs)&0xFF) <<24)); \
		jitptr += 4
	#define _LD_APS64(xreg, mreg, offs) \
		write64((jitptr), 0x40280F44 + ((xreg-8) <<27) + ((mreg) <<24) + ((int64_t)((offs)&0xFF) <<32)); \
		jitptr += 5
	#define _ST_APS64(mreg, offs, xreg) \
		write64((jitptr), 0x40290F44 + ((xreg-8) <<27) + ((mreg) <<24) + ((int64_t)((offs)&0xFF) <<32)); \
		jitptr += 5

#ifdef PLATFORM_AMD64
	#define _LD_DQA(xreg, mreg, offs) \
		write64((jitptr), 0x406F0F66 + ((xreg) <<27) + ((mreg) <<24) + ((int64_t)((offs)&0xFF) <<32)); \
		jitptr += 5
#else
	#define _LD_DQA(xreg, mreg, offs) \
		write32((jitptr), 0x406F0F66 + ((xreg) <<27) + ((mreg) <<24)); \
		*(jitptr +4) = (uint8_t)((offs)&0xFF); \
		jitptr += 5
#endif
	#define _LD_DQA64(xreg, mreg, offs) \
		write64((jitptr), 0x406F0F4466 + ((int64_t)(xreg-8) <<35) + ((int64_t)(mreg) <<32) + ((int64_t)((offs)&0xFF) <<40)); \
		jitptr += 6
	
	
	//_jit_xorps_m(jit, reg, AX, lshift32(offs, 4));
	#define _XORPS_M_(reg, offs, tr) \
		write32((jitptr), (0x40570F + ((reg) << 19) + (((offs)&0xFF) <<28)) ^ (tr))
	#define _C_XORPS_M(reg, offs, c) \
		_XORPS_M_(reg, offs, 0); \
		jitptr += (c)<<2
	#define _XORPS_M64_(reg, offs, tr) \
		write64((jitptr), (0x40570F44 + (((reg)-8) << 27) + ((int64_t)((offs)&0xFF) <<36)) ^ ((tr)<<8))
	#define _C_XORPS_M64(reg, offs, c) \
		_XORPS_M64_(reg, offs, 0); \
		jitptr += ((c)<<2)+(c)
	
	//_jit_pxor_m(jit, 1, AX, lshift32(offs, 4));
#ifdef PLATFORM_AMD64
	#define _PXOR_M_(reg, offs, tr) \
		write64((jitptr), (0x40EF0F66 + ((reg) << 27) + ((int64_t)((offs)&0xFF) << 36)) ^ (tr))
#else
	#define _PXOR_M_(reg, offs, tr) \
		write32((jitptr), (0x40EF0F66 + ((reg) << 27)) ^ (tr)); \
		*(jitptr +4) = (uint8_t)(((offs)&0xFF) << 4)
#endif
	#define _PXOR_M(reg, offs) \
		_PXOR_M_(reg, offs, 0); \
		jitptr += 5
	#define _C_PXOR_M(reg, offs, c) \
		_PXOR_M_(reg, offs, 0); \
		jitptr += ((c)<<2)+(c)
	#define _PXOR_M64_(reg, offs, tr) \
		write64((jitptr), (0x40EF0F4466 + ((int64_t)((reg)-8) << 35) + ((int64_t)((offs)&0xFF) << 44)) ^ ((tr)<<8))
	#define _C_PXOR_M64(reg, offs, c) \
		_PXOR_M64_(reg, offs, 0); \
		jitptr += ((c)<<2)+((c)<<1)
	
	//_jit_xorps_r(jit, r2, r1)
	#define _XORPS_R_(r2, r1, tr) \
		write32((jitptr), (0xC0570F + ((r2) <<19) + ((r1) <<16)) ^ (tr))
	#define _XORPS_R(r2, r1) \
		_XORPS_R_(r2, r1, 0); \
		jitptr += 3
	#define _C_XORPS_R(r2, r1, c) \
		_XORPS_R_(r2, r1, 0); \
		jitptr += ((c)<<1)+(c)
	// r2 is always < 8, r1 here is >= 8
	#define _XORPS_R64_(r2, r1, tr) \
		write32((jitptr), (0xC0570F41 + ((r2) <<27) + ((r1) <<24)) ^ ((tr)<<8))
	#define _C_XORPS_R64(r2, r1, c) \
		_XORPS_R64_(r2, r1, 0); \
		jitptr += (c)<<2
	
	//_jit_pxor_r(jit, r2, r1)
	#define _PXOR_R_(r2, r1, tr) \
		write32((jitptr), (0xC0EF0F66 + ((r2) <<27) + ((r1) <<24)) ^ (tr))
	#define _PXOR_R(r2, r1) \
		_PXOR_R_(r2, r1, 0); \
		jitptr += 4
	#define _C_PXOR_R(r2, r1, c) \
		_PXOR_R_(r2, r1, 0); \
		jitptr += (c)<<2
	#define _PXOR_R64_(r2, r1, tr) \
		write64((jitptr), (0xC0EF0F4166 + ((int64_t)(r2) <<35) + ((int64_t)(r1) <<32)) ^ (((int64_t)tr)<<8))
	#define _C_PXOR_R64(r2, r1, c) \
		_PXOR_R64_(r2, r1, 0); \
		jitptr += ((c)<<2)+(c)
	
	/* optimised mix of xor/mov operations */
	#define _MOV_OR_XOR_FP_M(reg, offs, flag, c) \
		_XORPS_M_(reg, offs, flag); \
		flag &= (c)-1; \
		jitptr += (c)<<2
	#define _MOV_OR_XOR_FP_M64(reg, offs, flag, c) \
		_XORPS_M64_(reg, offs, flag); \
		flag &= (c)-1; \
		jitptr += ((c)<<2)+(c)
	#define _MOV_OR_XOR_FP_INIT (0x570F ^ 0x280F)
	
	#define _MOV_OR_XOR_INT_M(reg, offs, flag, c) \
		_PXOR_M_(reg, offs, flag); \
		flag &= (c)-1; \
		jitptr += ((c)<<2)+(c)
	#define _MOV_OR_XOR_INT_M64(reg, offs, flag, c) \
		_PXOR_M64_(reg, offs, flag); \
		flag &= (c)-1; \
		jitptr += ((c)<<2)+((c)<<1)
	#define _MOV_OR_XOR_INT_INIT (0xEF0F00 ^ 0x6F0F00)
	
	#define _MOV_OR_XOR_R_FP(r2, r1, flag, c) \
		_XORPS_R_(r2, r1, flag); \
		flag &= (c)-1; \
		jitptr += ((c)<<1)+(c)
	#define _MOV_OR_XOR_R64_FP(r2, r1, flag, c) \
		_XORPS_R64_(r2, r1, flag); \
		flag &= (c)-1; \
		jitptr += (c)<<2
	
	#define _MOV_OR_XOR_R_INT(r2, r1, flag, c) \
		_PXOR_R_(r2, r1, flag); \
		flag &= (c)-1; \
		jitptr += (c)<<2
	#define _MOV_OR_XOR_R64_INT(r2, r1, flag, c) \
		_PXOR_R64_(r2, r1, flag); \
		flag &= (c)-1; \
		jitptr += ((c)<<2)+(c)
	
	/* generate code */
	if(mode == XORDEP_JIT_MODE_MULADD) {
		for(bit=0; bit<8; bit++) {
			int destOffs = (bit<<5)-128;
			int destOffs2 = destOffs+16;
			long movC = 0xFF;
			intptr_t posC = 0;
			uint_fast32_t mask = lumask[bit];
			_LD_APS(0, DX, destOffs);
			_LD_DQA(1, DX, destOffs2);
			
			if(no_common_mask & 1) {
				#define PROC_BITPAIR(n, inf, m) \
					STOREU_XMM(jitptr, _mm_load_si128((__m128i*)((uint64_t*)scratch->clut_nocomm + (n<<5) + ((m) & (0xF<<1))))); \
					jitptr += ((uint8_t*)(scratch->clut_ncinfo_ ##inf))[(m) & (0xF<<1)]; \
					mask >>= 4
				
				PROC_BITPAIR(0, mem, mask<<1);
				mask <<= 1;
#ifdef PLATFORM_AMD64
				PROC_BITPAIR(1, rm, mask);
				PROC_BITPAIR(2, reg, mask);
				PROC_BITPAIR(3, reg, mask);
				PROC_BITPAIR(4, mem, mask);
				PROC_BITPAIR(5, mem, mask);
				PROC_BITPAIR(6, mem, mask);
				PROC_BITPAIR(7, mem, mask);
#else
				PROC_BITPAIR(1, mem, mask);
				PROC_BITPAIR(2, mem, mask);
				PROC_BITPAIR(3, mem, mask);
				PROC_BITPAIR(4, mem, mask);
				PROC_BITPAIR(5, rm, mask);
				PROC_BITPAIR(6, reg, mask);
				PROC_BITPAIR(7, reg, mask);
#endif
				#undef PROC_BITPAIR
			} else {
				#define PROC_BITPAIR(n, bits, inf, m, r64) \
					jitptr += xor_jit_bitpair3(jitptr, (m) & ((2<<bits)-2), scratch->clut_code ##n, scratch->clut_info_ ##inf, &posC, &movC, r64) & 0xF; \
					mask >>= bits
				PROC_BITPAIR(1, 6, mem, mask<<1, 0);
				mask <<= 1;
#ifdef PLATFORM_AMD64
				PROC_BITPAIR(2, 6, reg, mask, 0);
				PROC_BITPAIR(3, 4, reg, mask, 0);
				PROC_BITPAIR(4, 6, mem, mask, 1);
				PROC_BITPAIR(5, 6, mem, mask, 1);
				PROC_BITPAIR(6, 4, mem, mask, 1);
#else
				PROC_BITPAIR(2, 6, mem, mask, 0);
				PROC_BITPAIR(3, 6, mem, mask, 0);
				PROC_BITPAIR(4, 4, mem, mask, 0);
				PROC_BITPAIR(5, 6, reg, mask, 0);
				PROC_BITPAIR(6, 4, reg, mask, 0);
#endif
				#undef PROC_BITPAIR
				
				jitptr[posC + movC] = 0x6F; // PXOR -> MOVDQA
#ifdef PLATFORM_AMD64
				write64(jitptr, (0xC0570F + (2 <<16)) + ((0xC0EF0F66ULL + (1 <<27) + (2 <<24)) <<24));
				jitptr += ((movC==0)<<3) - (movC==0);
#else
				_C_XORPS_R(0, 2, movC==0);
				_C_PXOR_R(1, 2, movC==0); /*penalty?*/
#endif
			}
#ifndef XORDEP_DISABLE_NO_COMMON
			no_common_mask >>= 2;
#endif
			
			_ST_APS(DX, destOffs, 0);
			_ST_APS(DX, destOffs2, 1);
		}
	} else {
		for(bit=0; bit<8; bit++) {
			int destOffs = (bit<<5)-128;
			int destOffs2 = destOffs+16;
			long mov1 = 0xFF, mov2 = 0xFF,
			     movC = 0xFF;
			intptr_t pos1 = 0, pos2 = 0, posC = 0;
			uint_fast32_t mask = lumask[bit];
			
			if(no_common_mask & 1) {
				#define PROC_BITPAIR(n, inf, m, r64) \
					STOREU_XMM(jitptr, _mm_load_si128((__m128i*)((uint64_t*)scratch->clut_nocomm + (n<<5) + ((m) & (0xF<<1))))); \
					jitptr += xor_jit_bitpair3_nc_noxor(jitptr, scratch->clut_ncinfo_ ##inf[((m) & (0xF<<1))>>1], &pos1, &mov1, &pos2, &mov2, r64); \
					mask >>= 4

				PROC_BITPAIR(0, mem, mask<<1, 0);
				mask <<= 1;
#ifdef PLATFORM_AMD64
				PROC_BITPAIR(1, rm, mask, 0);
				PROC_BITPAIR(2, reg, mask, 0);
				PROC_BITPAIR(3, reg, mask, 0);
				PROC_BITPAIR(4, mem, mask, 1);
				PROC_BITPAIR(5, mem, mask, 1);
				PROC_BITPAIR(6, mem, mask, 1);
				PROC_BITPAIR(7, mem, mask, 1);
#else
				PROC_BITPAIR(1, mem, mask, 0);
				PROC_BITPAIR(2, mem, mask, 0);
				PROC_BITPAIR(3, mem, mask, 0);
				PROC_BITPAIR(4, mem, mask, 0);
				PROC_BITPAIR(5, rm, mask, 0);
				PROC_BITPAIR(6, reg, mask, 0);
				PROC_BITPAIR(7, reg, mask, 0);
#endif
				#undef PROC_BITPAIR
				jitptr[pos1 + mov1] = 0x28; // XORPS -> MOVAPS
				jitptr[pos2 + mov2] = 0x6F; // PXOR -> MOVDQA
			} else {
				#define PROC_BITPAIR(n, bits, inf, m, r64) \
					jitptr += xor_jit_bitpair3_noxor(jitptr, xor_jit_bitpair3(jitptr, (m) & ((2<<bits)-2), scratch->clut_code ##n, scratch->clut_info_ ##inf, &posC, &movC, r64), &pos1, &mov1, &pos2, &mov2, r64); \
					mask >>= bits
				PROC_BITPAIR(1, 6, mem, mask<<1, 0);
				mask <<= 1;
#ifdef PLATFORM_AMD64
				PROC_BITPAIR(2, 6, reg, mask, 0);
				PROC_BITPAIR(3, 4, reg, mask, 0);
				PROC_BITPAIR(4, 6, mem, mask, 1);
				PROC_BITPAIR(5, 6, mem, mask, 1);
				PROC_BITPAIR(6, 4, mem, mask, 1);
#else
				PROC_BITPAIR(2, 6, mem, mask, 0);
				PROC_BITPAIR(3, 6, mem, mask, 0);
				PROC_BITPAIR(4, 4, mem, mask, 0);
				PROC_BITPAIR(5, 6, reg, mask, 0);
				PROC_BITPAIR(6, 4, reg, mask, 0);
#endif
				#undef PROC_BITPAIR
			
				jitptr[pos1 + mov1] = 0x28; // XORPS -> MOVAPS
				jitptr[pos2 + mov2] = 0x6F; // PXOR -> MOVDQA
				if(!movC) {
					jitptr[posC] = 0x6F; // PXOR -> MOVDQA
					if(mov1) { /* no additional XORs were made? */
						_ST_APS(DX, destOffs, 2);
					} else {
						_XORPS_R(0, 2);
					}
					if(mov2) {
						_ST_APS(DX, destOffs2, 2);
					} else {
						_PXOR_R(1, 2); /*penalty?*/
					}
				}
			}
#ifndef XORDEP_DISABLE_NO_COMMON
			no_common_mask >>= 2;
#else
			#undef no_common_mask
#endif
			
			if(!mov1) {
				_ST_APS(DX, destOffs, 0);
			}
			if(!mov2) {
				_ST_APS(DX, destOffs2, 1);
			}
		}
	}
	
	/* cmp */
#ifdef PLATFORM_AMD64
	write64(jitptr, 0x800FC03948 | (DX <<16) | (CX <<19) | ((uint64_t)JL <<32));
	jitptr += 5;
#else
	write32(jitptr, 0x800FC039 | (DX <<8) | (CX <<11) | (JL <<24));
	jitptr += 4;
#endif
	
	return jitptr;
}

static HEDLEY_ALWAYS_INLINE void gf16_xor_jit_mul_sse2_base(const void *HEDLEY_RESTRICT scratch, void* dst, const void* src, size_t len, uint16_t coefficient, void *HEDLEY_RESTRICT mutScratch, const int mode, const int doPrefetch, const void *HEDLEY_RESTRICT prefetch) {
	jit_wx_pair* jit = (jit_wx_pair*)mutScratch;
	gf16_xorjit_write_jit(&(((struct gf16_xor_jit_scratch_sse2*)scratch)->s), coefficient, jit, mode, doPrefetch, &xor_write_jit_sse, scratch);
	
	// exec
	/* adding 128 to the destination pointer allows the register offset to be coded in 1 byte
	 * eg: 'movdqa xmm0, [rdx+0x90]' is 8 bytes, whilst 'movdqa xmm0, [rdx-0x60]' is 5 bytes */
	if(mode == XORDEP_JIT_MODE_MUL_INSITU) {
		// need a place to store a copy of the source, that won't fit in registers; these will be used as the memory source
#ifdef PLATFORM_AMD64
		ALIGN_TO(16, __m128i spill[3]);
#else
		ALIGN_TO(16, __m128i spill[11]);
#endif
		gf16_xor_jit_stub(
			(intptr_t)spill + 128,
			(intptr_t)dst + len - 128,
			(intptr_t)dst - 128,
			(intptr_t)prefetch - 128,
			(uint8_t*)jit->x + XORDEP_JIT_SIZE/2
		);
	} else {
		gf16_xor_jit_stub(
			(intptr_t)src - 128,
			(intptr_t)dst + len - 128,
			(intptr_t)dst - 128,
			(intptr_t)prefetch - 128,
			jit->x
		);
	}
}
#endif /* defined(__SSE2__) */

#ifdef PARPAR_INVERT_SUPPORT
void gf16_xor_jit_mul_sse2(const void *HEDLEY_RESTRICT scratch, void* dst, const void* src, size_t len, uint16_t coefficient, void *HEDLEY_RESTRICT mutScratch) {
#ifdef __SSE2__
	if(coefficient == 0) {
		memset(dst, 0, len);
		return;
	}
	gf16_xor_jit_mul_sse2_base(scratch, dst, src, len, coefficient, mutScratch, dst==src ? XORDEP_JIT_MODE_MUL_INSITU : XORDEP_JIT_MODE_MUL, 0, NULL);
#else
	UNUSED(scratch); UNUSED(dst); UNUSED(src); UNUSED(len); UNUSED(coefficient); UNUSED(mutScratch);
#endif
}
#endif

void gf16_xor_jit_muladd_sse2(const void *HEDLEY_RESTRICT scratch, void *HEDLEY_RESTRICT dst, const void *HEDLEY_RESTRICT src, size_t len, uint16_t coefficient, void *HEDLEY_RESTRICT mutScratch) {
#ifdef __SSE2__
	if(coefficient == 0) return;
	gf16_xor_jit_mul_sse2_base(scratch, dst, src, len, coefficient, mutScratch, XORDEP_JIT_MODE_MULADD, 0, NULL);
#else
	UNUSED(scratch); UNUSED(dst); UNUSED(src); UNUSED(len); UNUSED(coefficient); UNUSED(mutScratch);
#endif
}

void gf16_xor_jit_muladd_prefetch_sse2(const void *HEDLEY_RESTRICT scratch, void *HEDLEY_RESTRICT dst, const void *HEDLEY_RESTRICT src, size_t len, uint16_t coefficient, void *HEDLEY_RESTRICT mutScratch, const void *HEDLEY_RESTRICT prefetch) {
#ifdef __SSE2__
	if(coefficient == 0) return;
	gf16_xor_jit_mul_sse2_base(scratch, dst, src, len, coefficient, mutScratch, XORDEP_JIT_MODE_MULADD, _MM_HINT_T1, prefetch);
#else
	UNUSED(scratch); UNUSED(dst); UNUSED(src); UNUSED(len); UNUSED(coefficient); UNUSED(mutScratch); UNUSED(prefetch);
#endif
}

#ifdef __SSE2__
struct gf16_xor_tables {
	ALIGN_TO(16, __m128 deps[8*16]);
	uint64_t BitsIndexTable[256];
};
static const unsigned char BitsSetTable256[256] = 
{
#   define B2(n) n,     n+1,     n+1,     n+2
#   define B4(n) B2(n), B2(n+1), B2(n+1), B2(n+2)
#   define B6(n) B4(n), B4(n+1), B4(n+1), B4(n+2)
    B6(0), B6(1), B6(1), B6(2)
};

static HEDLEY_ALWAYS_INLINE void gf16_xor_write_deptable(intptr_t *HEDLEY_RESTRICT deptable, uint_fast32_t *HEDLEY_RESTRICT counts, const struct gf16_xor_tables *HEDLEY_RESTRICT scratch, uint16_t val, uintptr_t dstSrcOffset) {
	ALIGN_TO(16, uint16_t tmp_depmask[16]);
	const uint8_t* deps = (const uint8_t*)(scratch->deps);
	__m128i depmask1 = _mm_load_si128((__m128i*)(deps + ((val & 0xf) << 7)));
	__m128i depmask2 = _mm_load_si128((__m128i*)(deps + ((val & 0xf) << 7)) +1);
	depmask1 = _mm_xor_si128(depmask1, _mm_load_si128((__m128i*)(deps + ((val << 3) & 0x780)) + 1*2));
	depmask2 = _mm_xor_si128(depmask2, _mm_load_si128((__m128i*)(deps + ((val << 3) & 0x780)) + 1*2 +1));
	depmask1 = _mm_xor_si128(depmask1, _mm_load_si128((__m128i*)(deps + ((val >> 1) & 0x780)) + 2*2));
	depmask2 = _mm_xor_si128(depmask2, _mm_load_si128((__m128i*)(deps + ((val >> 1) & 0x780)) + 2*2 +1));
	depmask1 = _mm_xor_si128(depmask1, _mm_load_si128((__m128i*)(deps + ((val >> 5) & 0x780)) + 3*2));
	depmask2 = _mm_xor_si128(depmask2, _mm_load_si128((__m128i*)(deps + ((val >> 5) & 0x780)) + 3*2 +1));
	
	
	/* generate needed tables */
	_mm_store_si128((__m128i*)(tmp_depmask), depmask1);
	_mm_store_si128((__m128i*)(tmp_depmask + 8), depmask2);
	for(int bit=0; bit<16; bit++) {
		uint_fast32_t cnt = BitsSetTable256[tmp_depmask[bit] & 0xff];
		
		__m128i idx = _mm_loadl_epi64((__m128i*)(scratch->BitsIndexTable + (tmp_depmask[bit] & 0xff)));
		__m128i idx2 = _mm_or_si128(
			_mm_set1_epi8(8U<<4),
			_mm_loadl_epi64((__m128i*)(scratch->BitsIndexTable + (tmp_depmask[bit] >> 8)))
		);
		counts[bit] = cnt + BitsSetTable256[tmp_depmask[bit] >> 8] -1;
		if(sizeof(uintptr_t) == 8) {
			__m128i addr = _mm_set1_epi64x(dstSrcOffset);
			#define STORE_TBL(storetype, offs, data) \
				_mm_##storetype##_si128((__m128i*)(deptable + bit*16 +offs), _mm_add_epi64( \
					data, addr \
			))
			idx = _mm_unpacklo_epi8(idx, _mm_setzero_si128());
			__m128i tmp1 = _mm_unpacklo_epi16(idx, _mm_setzero_si128());
			__m128i tmp2 = _mm_unpackhi_epi16(idx, _mm_setzero_si128());
			STORE_TBL(store, 0, _mm_unpacklo_epi32(tmp1, _mm_setzero_si128()));
			STORE_TBL(store, 2, _mm_unpackhi_epi32(tmp1, _mm_setzero_si128()));
			STORE_TBL(store, 4, _mm_unpacklo_epi32(tmp2, _mm_setzero_si128()));
			STORE_TBL(store, 6, _mm_unpackhi_epi32(tmp2, _mm_setzero_si128()));
			
			idx2 = _mm_unpacklo_epi8(idx2, _mm_setzero_si128());
			tmp1 = _mm_unpacklo_epi16(idx2, _mm_setzero_si128());
			tmp2 = _mm_unpackhi_epi16(idx2, _mm_setzero_si128());
			STORE_TBL(storeu, cnt+0, _mm_unpacklo_epi32(tmp1, _mm_setzero_si128()));
			STORE_TBL(storeu, cnt+2, _mm_unpackhi_epi32(tmp1, _mm_setzero_si128()));
			STORE_TBL(storeu, cnt+4, _mm_unpacklo_epi32(tmp2, _mm_setzero_si128()));
			STORE_TBL(storeu, cnt+6, _mm_unpackhi_epi32(tmp2, _mm_setzero_si128()));
			#undef STORE_TBL
		} else { // 32-bit
			__m128i addr = _mm_set1_epi32((int)dstSrcOffset);
			idx = _mm_unpacklo_epi8(idx, _mm_setzero_si128());
			_mm_store_si128((__m128i*)(deptable + bit*16 +0), _mm_add_epi32(
				_mm_unpacklo_epi16(idx, _mm_setzero_si128()), addr
			));
			_mm_store_si128((__m128i*)(deptable + bit*16 +4), _mm_add_epi32(
				_mm_unpackhi_epi16(idx, _mm_setzero_si128()), addr
			));
			
			idx2 = _mm_unpacklo_epi8(idx2, _mm_setzero_si128());
			_mm_storeu_si128((__m128i*)(deptable + bit*16 +cnt+0), _mm_add_epi32(
				_mm_unpacklo_epi16(idx2, _mm_setzero_si128()), addr
			));
			_mm_storeu_si128((__m128i*)(deptable + bit*16 +cnt+4), _mm_add_epi32(
				_mm_unpackhi_epi16(idx2, _mm_setzero_si128()), addr
			));
		}
		
		/* for storing as byte indicies; we do the full expansion above because it saves a register in the main loop
		_mm_storel_epi64(
			(__m128i*)(deptable + bit*16),
			_mm_loadl_epi64((__m128i*)(scratch->BitsIndexTable + (tmp_depmask[bit] & 0xff)))
		);
		_mm_storel_epi64(
			(__m128i*)(deptable + bit*16 + cnt),
			_mm_or_si128(
				_mm_set1_epi8(8<<4),
				_mm_loadl_epi64((__m128i*)(scratch->BitsIndexTable + (tmp_depmask[bit] >> 8)))
			)
		);
		*/
	}
}

static HEDLEY_ALWAYS_INLINE void gf16_xor_mul_block_sse2(const uint8_t* inP, uint8_t* outP, uint_fast32_t counts[16], intptr_t deptable[256]) {
	/* Note that we assume that all counts are at least 1; I don't think it's possible for that to be false */
	#define STEP(bit, type, typev, typed) { \
		intptr_t* deps = deptable + bit*16; \
		typev tmp = _mm_load_ ## type((typed*)(inP + deps[ 0])); \
		HEDLEY_ASSUME(counts[bit] <= 15); \
		switch(counts[bit]) { \
			case 15: tmp = _mm_xor_ ## type(tmp, *(typev*)(inP + deps[15])); /* FALLTHRU */ \
			case 14: tmp = _mm_xor_ ## type(tmp, *(typev*)(inP + deps[14])); /* FALLTHRU */ \
			case 13: tmp = _mm_xor_ ## type(tmp, *(typev*)(inP + deps[13])); /* FALLTHRU */ \
			case 12: tmp = _mm_xor_ ## type(tmp, *(typev*)(inP + deps[12])); /* FALLTHRU */ \
			case 11: tmp = _mm_xor_ ## type(tmp, *(typev*)(inP + deps[11])); /* FALLTHRU */ \
			case 10: tmp = _mm_xor_ ## type(tmp, *(typev*)(inP + deps[10])); /* FALLTHRU */ \
			case  9: tmp = _mm_xor_ ## type(tmp, *(typev*)(inP + deps[ 9])); /* FALLTHRU */ \
			case  8: tmp = _mm_xor_ ## type(tmp, *(typev*)(inP + deps[ 8])); /* FALLTHRU */ \
			case  7: tmp = _mm_xor_ ## type(tmp, *(typev*)(inP + deps[ 7])); /* FALLTHRU */ \
			case  6: tmp = _mm_xor_ ## type(tmp, *(typev*)(inP + deps[ 6])); /* FALLTHRU */ \
			case  5: tmp = _mm_xor_ ## type(tmp, *(typev*)(inP + deps[ 5])); /* FALLTHRU */ \
			case  4: tmp = _mm_xor_ ## type(tmp, *(typev*)(inP + deps[ 4])); /* FALLTHRU */ \
			case  3: tmp = _mm_xor_ ## type(tmp, *(typev*)(inP + deps[ 3])); /* FALLTHRU */ \
			case  2: tmp = _mm_xor_ ## type(tmp, *(typev*)(inP + deps[ 2])); /* FALLTHRU */ \
			case  1: tmp = _mm_xor_ ## type(tmp, *(typev*)(inP + deps[ 1])); /* FALLTHRU */ \
		} \
		_mm_store_ ## type((typed*)outP + bit, tmp); \
	}
	STEP( 0, si128, __m128i, __m128i)
	STEP( 1, si128, __m128i, __m128i)
	STEP( 2, si128, __m128i, __m128i)
	STEP( 3, si128, __m128i, __m128i)
	STEP( 4, si128, __m128i, __m128i)
	STEP( 5, si128, __m128i, __m128i)
	STEP( 6, si128, __m128i, __m128i)
	STEP( 7, si128, __m128i, __m128i)
	STEP( 8, si128, __m128i, __m128i)
	STEP( 9, si128, __m128i, __m128i)
	STEP(10, si128, __m128i, __m128i)
	STEP(11, si128, __m128i, __m128i)
	STEP(12, si128, __m128i, __m128i)
	STEP(13, si128, __m128i, __m128i)
	STEP(14, si128, __m128i, __m128i)
	STEP(15, si128, __m128i, __m128i)
	#undef STEP
}
#endif

#ifdef PARPAR_INVERT_SUPPORT
void gf16_xor_mul_sse2(const void *HEDLEY_RESTRICT scratch, void* dst, const void* src, size_t len, uint16_t val, void *HEDLEY_RESTRICT mutScratch) {
	UNUSED(mutScratch);
#ifdef __SSE2__
	if(val == 0) {
		memset(dst, 0, len);
		return;
	}
	uint_fast32_t counts[16];
	ALIGN_TO(16, intptr_t deptable[256]);
	uint8_t* _dst = (uint8_t*)dst + len;
	
	if(dst == src) {
		// for in-situ mul, write to a temp block and copy back
		ALIGN_TO(16, uint8_t tmp[256]);
		__m128i* _tmp = (__m128i*)tmp;
		gf16_xor_write_deptable(deptable, counts, (const struct gf16_xor_tables*)scratch, val, 0);
		for(intptr_t ptr = -(intptr_t)len; ptr; ptr += sizeof(__m128i)*16) {
			uint8_t* p = _dst + ptr;
			gf16_xor_mul_block_sse2(p, tmp, counts, deptable);
			for(int i=0; i<16; i++) {
				_mm_store_si128((__m128i*)p + i, _mm_load_si128(_tmp+i));
			}
		}
	} else {
		gf16_xor_write_deptable(deptable, counts, (const struct gf16_xor_tables*)scratch, val, (uintptr_t)src - (uintptr_t)dst);
		
		for(intptr_t ptr = -(intptr_t)len; ptr; ptr += sizeof(__m128i)*16) {
			gf16_xor_mul_block_sse2(_dst + ptr, _dst + ptr, counts, deptable);
		}
	}
#else
	UNUSED(scratch); UNUSED(dst); UNUSED(src); UNUSED(len); UNUSED(val);
#endif
}
#endif


void gf16_xor_muladd_sse2(const void *HEDLEY_RESTRICT scratch, void *HEDLEY_RESTRICT dst, const void *HEDLEY_RESTRICT src, size_t len, uint16_t val, void *HEDLEY_RESTRICT mutScratch) {
	UNUSED(mutScratch);
#ifdef __SSE2__
	if(val == 0) return;
	uint_fast32_t counts[16];
	ALIGN_TO(16, intptr_t deptable[256]);
	uint8_t* _dst = (uint8_t*)dst + len;

	gf16_xor_write_deptable(deptable, counts, (const struct gf16_xor_tables*)scratch, val, (uintptr_t)src - (uintptr_t)dst);
	
	for(intptr_t ptr = -(intptr_t)len; ptr; ptr += sizeof(__m128i)*16) {
		uint8_t* p = _dst + ptr;
		#define STEP(bit, type, typev, typed) { \
			intptr_t* deps = deptable + bit*16; \
			typev tmp = _mm_load_ ## type((typed*)((typed*)p + bit)); \
			HEDLEY_ASSUME(counts[bit] <= 15); \
			switch(counts[bit]) { \
				case 15: tmp = _mm_xor_ ## type(tmp, *(typev*)(p + deps[15])); /* FALLTHRU */ \
				case 14: tmp = _mm_xor_ ## type(tmp, *(typev*)(p + deps[14])); /* FALLTHRU */ \
				case 13: tmp = _mm_xor_ ## type(tmp, *(typev*)(p + deps[13])); /* FALLTHRU */ \
				case 12: tmp = _mm_xor_ ## type(tmp, *(typev*)(p + deps[12])); /* FALLTHRU */ \
				case 11: tmp = _mm_xor_ ## type(tmp, *(typev*)(p + deps[11])); /* FALLTHRU */ \
				case 10: tmp = _mm_xor_ ## type(tmp, *(typev*)(p + deps[10])); /* FALLTHRU */ \
				case  9: tmp = _mm_xor_ ## type(tmp, *(typev*)(p + deps[ 9])); /* FALLTHRU */ \
				case  8: tmp = _mm_xor_ ## type(tmp, *(typev*)(p + deps[ 8])); /* FALLTHRU */ \
				case  7: tmp = _mm_xor_ ## type(tmp, *(typev*)(p + deps[ 7])); /* FALLTHRU */ \
				case  6: tmp = _mm_xor_ ## type(tmp, *(typev*)(p + deps[ 6])); /* FALLTHRU */ \
				case  5: tmp = _mm_xor_ ## type(tmp, *(typev*)(p + deps[ 5])); /* FALLTHRU */ \
				case  4: tmp = _mm_xor_ ## type(tmp, *(typev*)(p + deps[ 4])); /* FALLTHRU */ \
				case  3: tmp = _mm_xor_ ## type(tmp, *(typev*)(p + deps[ 3])); /* FALLTHRU */ \
				case  2: tmp = _mm_xor_ ## type(tmp, *(typev*)(p + deps[ 2])); /* FALLTHRU */ \
				case  1: tmp = _mm_xor_ ## type(tmp, *(typev*)(p + deps[ 1])); /* FALLTHRU */ \
				case  0: tmp = _mm_xor_ ## type(tmp, *(typev*)(p + deps[ 0])); /* FALLTHRU */ \
			} \
			_mm_store_ ## type((typed*)p + bit, tmp); \
		}
		STEP( 0, si128, __m128i, __m128i)
		STEP( 1, si128, __m128i, __m128i)
		STEP( 2, si128, __m128i, __m128i)
		STEP( 3, si128, __m128i, __m128i)
		STEP( 4, si128, __m128i, __m128i)
		STEP( 5, si128, __m128i, __m128i)
		STEP( 6, si128, __m128i, __m128i)
		STEP( 7, si128, __m128i, __m128i)
		STEP( 8, si128, __m128i, __m128i)
		STEP( 9, si128, __m128i, __m128i)
		STEP(10, si128, __m128i, __m128i)
		STEP(11, si128, __m128i, __m128i)
		STEP(12, si128, __m128i, __m128i)
		STEP(13, si128, __m128i, __m128i)
		STEP(14, si128, __m128i, __m128i)
		STEP(15, si128, __m128i, __m128i)
		#undef STEP
	}
#else
	UNUSED(scratch); UNUSED(dst); UNUSED(src); UNUSED(len); UNUSED(val);
#endif
}



#ifdef __SSE2__
#define LOAD_HALVES(a, b, upper) \
	_mm_castps_si128(_mm_loadh_pi( \
		_mm_castsi128_ps(_mm_loadl_epi64((__m128i*)(_src + 120 + upper*4 - (a)*8))), \
		(__m64*)(_src + 120 + upper*4 - (b)*8) \
	))
#define LOAD_X4(offs, dst1, dst2, upper) { \
	__m128i src02 = LOAD_HALVES(offs+0, offs+2, upper); /* 22222222 00000000 */ \
	__m128i src13 = LOAD_HALVES(offs+1, offs+3, upper); /* 33333333 11111111 */ \
	__m128i src01 = _mm_unpacklo_epi8(src02, src13); /* 10101010 10101010 */ \
	__m128i src23 = _mm_unpackhi_epi8(src02, src13); /* 32323232 32323232 */ \
	dst1 = _mm_unpacklo_epi16(src01, src23); /* 32103210 32103210 */ \
	dst2 = _mm_unpackhi_epi16(src01, src23); /* 32103210 32103210 */ \
}

#define UNPACK_VECTS \
	srcQ0a = _mm_unpacklo_epi32(srcD0a, srcD4a); /* 76543210 76543210 */ \
	srcQ0b = _mm_unpackhi_epi32(srcD0a, srcD4a); \
	srcQ0c = _mm_unpacklo_epi32(srcD0b, srcD4b); \
	srcQ0d = _mm_unpackhi_epi32(srcD0b, srcD4b); \
	srcQ8a = _mm_unpacklo_epi32(srcD8a, srcD12a); \
	srcQ8b = _mm_unpackhi_epi32(srcD8a, srcD12a); \
	srcQ8c = _mm_unpacklo_epi32(srcD8b, srcD12b); \
	srcQ8d = _mm_unpackhi_epi32(srcD8b, srcD12b); \
	 \
	srcDQa = _mm_unpacklo_epi64(srcQ0a, srcQ8a); \
	srcDQb = _mm_unpackhi_epi64(srcQ0a, srcQ8a); \
	srcDQc = _mm_unpacklo_epi64(srcQ0b, srcQ8b); \
	srcDQd = _mm_unpackhi_epi64(srcQ0b, srcQ8b); \
	srcDQe = _mm_unpacklo_epi64(srcQ0c, srcQ8c); \
	srcDQf = _mm_unpackhi_epi64(srcQ0c, srcQ8c); \
	srcDQg = _mm_unpacklo_epi64(srcQ0d, srcQ8d); \
	srcDQh = _mm_unpackhi_epi64(srcQ0d, srcQ8d)

#define EXTRACT_BITS(target, srcVec) \
	write16((target) + 7, _mm_movemask_epi8(srcVec)); \
	for(int i=6; i>=0; i--) { \
		srcVec = _mm_add_epi8(srcVec, srcVec); \
		write16((target) + i, _mm_movemask_epi8(srcVec)); \
	}
void gf16_xor_finish_block_sse2(void *HEDLEY_RESTRICT dst) {
	uint16_t* _dst = (uint16_t*)dst;
	const uint16_t* _src = _dst;
	
	__m128i srcD0a, srcD0b, srcD4a, srcD4b, srcD8a, srcD8b, srcD12a, srcD12b;
	__m128i srcQ0a, srcQ0b, srcQ0c, srcQ0d, srcQ8a, srcQ8b, srcQ8c, srcQ8d;
	__m128i srcDQa, srcDQb, srcDQc, srcDQd, srcDQe, srcDQf, srcDQg, srcDQh;
	__m128i dstA, dstB, dstC, dstD;
	// setting to undefined to stop ICC warning about uninitialized values
	#ifndef __APPLE__
	dstA = _mm_undefined_si128();
	dstB = _mm_undefined_si128();
	dstC = _mm_undefined_si128();
	dstD = _mm_undefined_si128();
	#endif
	
	// load 16x 64-bit inputs
	LOAD_X4( 0, srcD0a , srcD0b, 0)
	LOAD_X4( 4, srcD4a , srcD4b, 0)
	LOAD_X4( 8, srcD8a , srcD8b, 0)
	LOAD_X4(12, srcD12a, srcD12b,0)
	
	// interleave bytes in all 8 vectors
	UNPACK_VECTS;
	
	// write half of these vectors, and store the other half for later
	#define EXTRACT_BITS_HALF(target, targVec, vecUpper, srcVec) { \
		uint16_t mskA, mskB, mskC, mskD; \
		mskD = _mm_movemask_epi8(srcVec); \
		srcVec = _mm_add_epi8(srcVec, srcVec); \
		mskC = _mm_movemask_epi8(srcVec); \
		srcVec = _mm_add_epi8(srcVec, srcVec); \
		mskB = _mm_movemask_epi8(srcVec); \
		srcVec = _mm_add_epi8(srcVec, srcVec); \
		mskA = _mm_movemask_epi8(srcVec); \
		targVec = vecUpper ? _mm_insert_epi16(targVec, mskA, 4) : _mm_cvtsi32_si128(mskA); \
		targVec = _mm_insert_epi16(targVec, mskB, 1 + vecUpper*4); \
		targVec = _mm_insert_epi16(targVec, mskC, 2 + vecUpper*4); \
		targVec = _mm_insert_epi16(targVec, mskD, 3 + vecUpper*4); \
		srcVec = _mm_add_epi8(srcVec, srcVec); \
		write16((target)+3, _mm_movemask_epi8(srcVec)); \
		srcVec = _mm_add_epi8(srcVec, srcVec); \
		write16((target)+2, _mm_movemask_epi8(srcVec)); \
		srcVec = _mm_add_epi8(srcVec, srcVec); \
		write16((target)+1, _mm_movemask_epi8(srcVec)); \
		srcVec = _mm_add_epi8(srcVec, srcVec); \
		write16((target)+0, _mm_movemask_epi8(srcVec)); \
	}
	EXTRACT_BITS_HALF(_dst +  0, dstA, 0, srcDQa)
	EXTRACT_BITS_HALF(_dst +  8, dstA, 1, srcDQb)
	EXTRACT_BITS_HALF(_dst + 16, dstB, 0, srcDQc)
	EXTRACT_BITS_HALF(_dst + 24, dstB, 1, srcDQd)
	EXTRACT_BITS_HALF(_dst + 32, dstC, 0, srcDQe)
	EXTRACT_BITS_HALF(_dst + 40, dstC, 1, srcDQf)
	EXTRACT_BITS_HALF(_dst + 48, dstD, 0, srcDQg)
	EXTRACT_BITS_HALF(_dst + 56, dstD, 1, srcDQh)
	#undef EXTRACT_BITS_HALF
	
	
	
	
	// load second half & write saved output once source has been read
	LOAD_X4(12, srcD12a, srcD12b,1)
	_mm_storel_epi64((__m128i*)(_dst +  4), dstA);
	_mm_storeh_pi((__m64*)(_dst + 12), _mm_castsi128_ps(dstA));
	_mm_storel_epi64((__m128i*)(_dst + 20), dstB);
	_mm_storeh_pi((__m64*)(_dst + 28), _mm_castsi128_ps(dstB));
	
	LOAD_X4( 8, srcD8a , srcD8b, 1)
	_mm_storel_epi64((__m128i*)(_dst + 36), dstC);
	_mm_storeh_pi((__m64*)(_dst + 44), _mm_castsi128_ps(dstC));
	_mm_storel_epi64((__m128i*)(_dst + 52), dstD);
	_mm_storeh_pi((__m64*)(_dst + 60), _mm_castsi128_ps(dstD));
	
	LOAD_X4( 4, srcD4a , srcD4b, 1)
	LOAD_X4( 0, srcD0a , srcD0b, 1)
	
	UNPACK_VECTS;
	
	// extract & write all bits
	// TODO: consider saving some to a register to reduce write ops
	EXTRACT_BITS(_dst + 64 +  0, srcDQa)
	EXTRACT_BITS(_dst + 64 +  8, srcDQb)
	EXTRACT_BITS(_dst + 64 + 16, srcDQc)
	EXTRACT_BITS(_dst + 64 + 24, srcDQd)
	EXTRACT_BITS(_dst + 64 + 32, srcDQe)
	EXTRACT_BITS(_dst + 64 + 40, srcDQf)
	EXTRACT_BITS(_dst + 64 + 48, srcDQg)
	EXTRACT_BITS(_dst + 64 + 56, srcDQh)
}
void gf16_xor_finish_copy_block_sse2(void *HEDLEY_RESTRICT dst, const void *HEDLEY_RESTRICT src) {
	uint16_t* _dst = (uint16_t*)dst;
	const uint16_t* _src = (uint16_t*)src;
	
	__m128i srcD0a, srcD0b, srcD4a, srcD4b, srcD8a, srcD8b, srcD12a, srcD12b;
	__m128i srcQ0a, srcQ0b, srcQ0c, srcQ0d, srcQ8a, srcQ8b, srcQ8c, srcQ8d;
	__m128i srcDQa, srcDQb, srcDQc, srcDQd, srcDQe, srcDQf, srcDQg, srcDQh;
	
	// load 16x 64-bit inputs
	LOAD_X4( 0, srcD0a , srcD0b, 0)
	LOAD_X4( 4, srcD4a , srcD4b, 0)
	LOAD_X4( 8, srcD8a , srcD8b, 0)
	LOAD_X4(12, srcD12a, srcD12b,0)
	
	// interleave bytes in all 8 vectors
	UNPACK_VECTS;
	
	// write extracted bits
	EXTRACT_BITS(_dst +  0, srcDQa)
	EXTRACT_BITS(_dst +  8, srcDQb)
	EXTRACT_BITS(_dst + 16, srcDQc)
	EXTRACT_BITS(_dst + 24, srcDQd)
	EXTRACT_BITS(_dst + 32, srcDQe)
	EXTRACT_BITS(_dst + 40, srcDQf)
	EXTRACT_BITS(_dst + 48, srcDQg)
	EXTRACT_BITS(_dst + 56, srcDQh)
	
	
	// load second half
	LOAD_X4( 0, srcD0a , srcD0b, 1)
	LOAD_X4( 4, srcD4a , srcD4b, 1)
	LOAD_X4( 8, srcD8a , srcD8b, 1)
	LOAD_X4(12, srcD12a, srcD12b,1)
	
	UNPACK_VECTS;
	
	EXTRACT_BITS(_dst + 64 +  0, srcDQa)
	EXTRACT_BITS(_dst + 64 +  8, srcDQb)
	EXTRACT_BITS(_dst + 64 + 16, srcDQc)
	EXTRACT_BITS(_dst + 64 + 24, srcDQd)
	EXTRACT_BITS(_dst + 64 + 32, srcDQe)
	EXTRACT_BITS(_dst + 64 + 40, srcDQf)
	EXTRACT_BITS(_dst + 64 + 48, srcDQg)
	EXTRACT_BITS(_dst + 64 + 56, srcDQh)
}
void gf16_xor_finish_copy_blocku_sse2(void *HEDLEY_RESTRICT dst, const void *HEDLEY_RESTRICT src, size_t bytes) {
	uint16_t block[128];
	gf16_xor_finish_copy_block_sse2(block, src);
	memcpy(dst, block, bytes);
}
#undef EXTRACT_BITS
#undef UNPACK_VECTS
#undef LOAD_HALVES
#undef LOAD_X4
#endif



#define MWORD_SIZE 16
#define _mword __m128i
#define _MM(f) _mm_ ## f
#define _MMI(f) _mm_ ## f ## _si128
#define _FNSUFFIX _sse2
#define _MM_END

#if defined(__SSE2__)
# define _AVAILABLE
# include "gf16_checksum_x86.h"
#endif
#include "gf16_xor_common_funcs.h"
#undef _AVAILABLE

#undef MWORD_SIZE
#undef _mword
#undef _MM
#undef _MMI
#undef _FNSUFFIX
#undef _MM_END


#ifdef __SSE2__
GF_FINISH_PACKED_FUNCS(gf16_xor, _sse2, sizeof(__m128i)*16, gf16_xor_finish_copy_block_sse2, gf16_xor_finish_copy_blocku_sse2, 1, (void)0, gf16_checksum_block_sse2, gf16_checksum_blocku_sse2, gf16_checksum_exp_sse2, &gf16_xor_finish_block_sse2, sizeof(__m128i))
#else
GF_FINISH_PACKED_FUNCS_STUB(gf16_xor, _sse2)
#endif


#include "gf16_bitdep_init_sse2.h"

#ifdef PLATFORM_X86
static void xor_write_init_jit(uint8_t *jitCodeNorm, uint8_t *jitCodeInsitu, uint_fast16_t* sizeNorm, uint_fast16_t* sizeInsitu) {
	uint8_t *jitCodeStart = jitCodeNorm;
	jitCodeNorm += _jit_add_i(jitCodeNorm, AX, 256);
	jitCodeNorm += _jit_add_i(jitCodeNorm, DX, 256);
	
# ifdef PLATFORM_AMD64
	/* preload upper 13 inputs into registers */
	for(int i=3; i<16; i++) {
		jitCodeNorm += _jit_movaps_load(jitCodeNorm, i, AX, lshift32(i-8, 4));
	}
# else
	/* can only fit 5 in 32-bit mode :( */
	for(int i=3; i<8; i++) { /* despite appearances, we're actually loading the top 5, not mid 5 */
		jitCodeNorm += _jit_movaps_load(jitCodeNorm, i, AX, i<<4);
	}
# endif
	
	if(sizeNorm) *sizeNorm = (uint_fast16_t)(jitCodeNorm-jitCodeStart);
	
	// in-situ version
	jitCodeStart = jitCodeInsitu;
	jitCodeInsitu += _jit_add_i(jitCodeInsitu, DX, 256);
	
# ifdef PLATFORM_AMD64
	for(int i=0; i<16; i++) {
		jitCodeInsitu += _jit_movaps_load(jitCodeInsitu, i, DX, lshift32(i-8, 4));
	}
	for(int i=0; i<3; i++) {
		jitCodeInsitu += _jit_movaps_store(jitCodeInsitu, AX, lshift32(i-8, 4), i);
	}
# else
	for(int i=0; i<11; i++) {
		jitCodeInsitu += _jit_movaps_load(jitCodeInsitu, 0, DX, lshift32(i-8, 4));
		jitCodeInsitu += _jit_movaps_store(jitCodeInsitu, AX, lshift32(i-8, 4), 0);
	}
	for(int i=3; i<8; i++) { /* despite appearances, we're actually loading the top 5, not mid 5 */
		jitCodeInsitu += _jit_movaps_load(jitCodeInsitu, i, DX, i<<4);
	}
# endif
	
	if(sizeInsitu) *sizeInsitu = (uint_fast16_t)(jitCodeInsitu-jitCodeStart);
}
#endif

void* gf16_xor_jit_init_sse2(int polynomial, int jitOptStrat) {
#ifdef __SSE2__
	struct gf16_xor_jit_scratch_sse2* ret;
	uint8_t tmpCode[XORDEP_JIT_CODE_SIZE];
	
	ALIGN_ALLOC(ret, sizeof(struct gf16_xor_jit_scratch_sse2), 64); // align to 16 required, 64 might better use cachelines
	gf16_bitdep_init128(ret->s.deps, polynomial, GF16_BITDEP_INIT128_GEN_XORJIT);
	
	gf16_xor_create_jit_lut_sse2(ret);
	
	ret->s.jitOptStrat = jitOptStrat;
	xor_write_init_jit(tmpCode, tmpCode, &(ret->s.codeStart), &(ret->s.codeStartInsitu));
	return ret;
#else
	UNUSED(polynomial); UNUSED(jitOptStrat);
	return NULL;
#endif
}

void* gf16_xor_jit_init_mut_sse2() {
#ifdef PLATFORM_X86
	jit_wx_pair *jitCode = jit_alloc(XORDEP_JIT_SIZE);
	if(!jitCode) return NULL;
	xor_write_init_jit(jitCode->w, (uint8_t*)jitCode->w + XORDEP_JIT_SIZE/2, NULL, NULL);
	return jitCode;
#else
	return NULL;
#endif
}

void gf16_xor_jit_uninit(void* scratch) {
#ifdef PLATFORM_X86
	jit_free(scratch);
#else
	UNUSED(scratch);
#endif
}

#ifdef PARPAR_INVERT_SUPPORT
static HEDLEY_ALWAYS_INLINE uint16_t gf16_xorX_replace_word(void* data, size_t index, uint16_t newValue, size_t width) {
	uint8_t* base = (uint8_t*)data + (index & ~(width*8-1)) * 2; // advance pointer to correct group
	base += ((index >> 3) & (width-1)); // advance to correct byte
	
	unsigned bitIndex = index&7;
	uint16_t oldValue = 0;
	unsigned _newValue = newValue << bitIndex;
	uint8_t byteMask = 1 << bitIndex;
	for(int i=0; i<16; i++) {
		uint8_t byte = base[i*width];
		oldValue <<= 1;
		_newValue <<= 1;
		oldValue |= (byte >> bitIndex) & 1;
		
		base[i*width] = (byte & ~byteMask) | ((_newValue >> 16) & byteMask);
	}
	return oldValue;
}
uint16_t gf16_xor16_replace_word(void* data, size_t index, uint16_t newValue) {
	return gf16_xorX_replace_word(data, index, newValue, 16);
}
uint16_t gf16_xor32_replace_word(void* data, size_t index, uint16_t newValue) {
	return gf16_xorX_replace_word(data, index, newValue, 32);
}
uint16_t gf16_xor64_replace_word(void* data, size_t index, uint16_t newValue) {
	return gf16_xorX_replace_word(data, index, newValue, 64);
}
#endif


void* gf16_xor_init_sse2(int polynomial) {
#ifdef __SSE2__
	struct gf16_xor_tables* ret;
	ALIGN_ALLOC(ret, sizeof(struct gf16_xor_tables), 16);
	gf16_bitdep_init128(ret->deps, polynomial, GF16_BITDEP_INIT128_GEN_XOR);
	
	for(int i=0; i<256; i++) {
		uint64_t n=0;
		int j=8;
		while(j--)
			if(i & (1<<j))
				n = (n<<8)|(j<<4);
		ret->BitsIndexTable[i] = n;
	}
	return ret;
#else
	UNUSED(polynomial);
	return NULL;
#endif
}
