import lib.util; [[vk::binding(0, 0)]] RWStructuredBuffer input; [[vk::binding(1, 0)]] RWStructuredBuffer output; [[vk::binding(2, 0)]] StructuredBuffer twiddles; [[vk::binding(3, 0)]] cbuffer Uniforms { uint input_len; uint n_iter; }; groupshared float2 scratch[4096]; void fft_v2_stage(uint which_fft, uint id, uint stride, uint big_n) { const uint k = id / stride; const uint start_idx = id % stride; // Collect inputs. let i0 = scratch[start_idx + 8 * k * stride]; let i1 = scratch[start_idx + (8 * k + 1) * stride]; let i2 = scratch[start_idx + (8 * k + 2) * stride]; let i3 = scratch[start_idx + (8 * k + 3) * stride]; let i4 = scratch[start_idx + (8 * k + 4) * stride]; let i5 = scratch[start_idx + (8 * k + 5) * stride]; let i6 = scratch[start_idx + (8 * k + 6) * stride]; let i7 = scratch[start_idx + (8 * k + 7) * stride]; // Collect relevant twiddles. let ot1 = twiddles[1 * k * stride]; let ot2 = twiddles[2 * k * stride]; let ot3 = twiddles[3 * k * stride]; let ot4 = twiddles[4 * k * stride]; let ot5 = twiddles[5 * k * stride]; let ot6 = twiddles[6 * k * stride]; let ot7 = twiddles[7 * k * stride]; let a = i0; let b = cmul(ot1, i1); let c = cmul(ot2, i2); let d = cmul(ot3, i3); let e = cmul(ot4, i4); let f = cmul(ot5, i5); let g = cmul(ot6, i6); let h = cmul(ot7, i7); let ae_sum = a + e; let ae_diff = a - e; let bf_sum = b + f; let bf_diff = b - f; let cg_sum = c + g; let cg_diff = c - g; let dh_sum = d + h; let dh_diff = d - h; let w00 = ae_sum + cg_sum; let w01 = ae_sum - cg_sum; let w10 = ae_diff + rot_270(cg_diff); let w11 = ae_diff - rot_270(cg_diff); let x00 = bf_sum + dh_sum; let x01 = rot_270(bf_sum) + rot_90(dh_sum); let x10 = rot_45(rot_270(bf_diff) + rot_180(dh_diff)); let x11 = rot_45(rot_180(bf_diff) + rot_270(dh_diff)); GroupMemoryBarrierWithGroupSync(); scratch[start_idx + k * stride] = w00 + x00; scratch[start_idx + (k + big_n / 8) * stride] = w10 + x10; scratch[start_idx + (k + big_n / 4) * stride] = w01 + x01; scratch[start_idx + (k + 3 * big_n / 8) * stride] = w11 + x11; scratch[start_idx + (k + big_n / 2) * stride] = w00 - x00; scratch[start_idx + (k + 5 * big_n / 8) * stride] = w10 - x10; scratch[start_idx + (k + 3 * big_n / 4) * stride] = w01 - x01; scratch[start_idx + (k + 7 * big_n / 8) * stride] = w11 - x11; GroupMemoryBarrierWithGroupSync(); } [shader("compute")] [numthreads(64, 8, 1)] void fft_v4_radix_8(uint3 globalId : SV_DispatchThreadID, uint id: SV_GroupIndex) { const uint which_fft = globalId.z; uint stride = Uniforms.input_len; uint big_n = 1; const uint offset = which_fft * Uniforms.input_len; for (uint i = 0; i < 8; i++) { scratch[id + i * 512] = input[offset + id + i * 512]; } AllMemoryBarrierWithGroupSync(); for (uint i = 0; i < Uniforms.n_iter; i++) { stride /= 8; big_n *= 8; fft_v2_stage(which_fft, id, stride, big_n); } for (uint i = 0; i < 8; i++) { output[offset + id + i * 512] = scratch[id + i * 512]; } AllMemoryBarrierWithGroupSync(); }