Math: FFT: Move twiddle factors data to DRAM

src/include/sof/math/fft.h

@@ -52,6 +52,7 @@ struct fft_plan {
 	struct icomplex32 *outb32;	/* pointer to output integer complex buffer */
 	struct icomplex16 *inb16;	/* pointer to input integer complex buffer */
 	struct icomplex16 *outb16;	/* pointer to output integer complex buffer */
+	void *twiddle;
 };
 
 struct fft_multi_plan {
@@ -60,6 +61,8 @@ struct fft_multi_plan {
 	struct icomplex32 *tmp_o32[FFT_MULTI_COUNT_MAX]; /* pointer to output buffer */
 	struct icomplex32 *inb32;	/* pointer to input integer complex buffer */
 	struct icomplex32 *outb32;	/* pointer to output integer complex buffer */
+	int32_t *multi_twiddle;		/* twiddle factors for combine FFT */
+	int32_t *shared_twiddle;	/* twiddle factors for three FFTs */
 	uint16_t *bit_reverse_idx;
 	uint32_t total_size;
 	uint32_t fft_size;

src/math/Kconfig

@@ -111,6 +111,16 @@ config MATH_FFT_MULTI
 	  this should not be selected directly, please select it from other
 	  audio components where need it.
 
+config MATH_FFT_COLD_TWIDDLE_FACTORS
+	bool "FFT cold twiddle factors"
+	depends on MATH_FFT
+	default y
+	help
+	  Link twiddle factors for the maximum FFT size to
+	  DRAM and copy the needed set to SRAM when the FFT
+	  plan is initialized. The twiddle factors data
+	  size is currently 29 kB.
+
 menu "Supported FFT word lengths"
 	visible if MATH_FFT
 

src/include/sof/audio/coefficients/fft/twiddle_16.h → src/math/fft/coef/twiddle_16.h

@@ -14,7 +14,7 @@
 #define FFT_SIZE_MAX	1024
 
 /* in Q1.15, generated from cos(i * 2 * pi / FFT_SIZE_MAX) */
-const int16_t twiddle_real_16[FFT_SIZE_MAX] = {
+SOF_MATH_FFT_COLD_RODATA static const int16_t twiddle_real_16[FFT_SIZE_MAX] = {
 	32767,
 	32767,
 	32766,
@@ -1042,7 +1042,7 @@ const int16_t twiddle_real_16[FFT_SIZE_MAX] = {
 };
 
 /* in Q1.15, generated from sin(i * 2 * pi / FFT_SIZE_MAX) */
-const int16_t twiddle_imag_16[FFT_SIZE_MAX] = {
+SOF_MATH_FFT_COLD_RODATA static const int16_t twiddle_imag_16[FFT_SIZE_MAX] = {
 	0,
 	-201,
 	-402,

src/include/sof/audio/coefficients/fft/twiddle_3072_32.h → src/math/fft/coef/twiddle_3072_32.h

@@ -14,7 +14,7 @@
 #define FFT_MULTI_TWIDDLE_SIZE	2048
 
 /* in Q1.31, generated from cos(i * 2 * pi / FFT_SIZE_MAX) */
-const int32_t multi_twiddle_real_32[FFT_MULTI_TWIDDLE_SIZE] = {
+SOF_MATH_FFT_COLD_RODATA static const int32_t multi_twiddle_real_32[FFT_MULTI_TWIDDLE_SIZE] = {
 	2147483647,
 	2147479156,
 	2147465681,
@@ -2066,7 +2066,7 @@ const int32_t multi_twiddle_real_32[FFT_MULTI_TWIDDLE_SIZE] = {
 };
 
 /* in Q1.31, generated from sin(i * 2 * pi / FFT_SIZE_MAX) */
-const int32_t multi_twiddle_imag_32[FFT_MULTI_TWIDDLE_SIZE] = {
+SOF_MATH_FFT_COLD_RODATA static const int32_t multi_twiddle_imag_32[FFT_MULTI_TWIDDLE_SIZE] = {
 	0,
 	-4392262,
 	-8784505,

src/include/sof/audio/coefficients/fft/twiddle_32.h → src/math/fft/coef/twiddle_32.h

@@ -14,7 +14,7 @@
 #define FFT_SIZE_MAX	1024
 
 /* in Q1.31, generated from cos(i * 2 * pi / FFT_SIZE_MAX) */
-const int32_t twiddle_real_32[FFT_SIZE_MAX] = {
+SOF_MATH_FFT_COLD_RODATA static const int32_t twiddle_real_32[FFT_SIZE_MAX] = {
 	2147483647,
 	2147443222,
 	2147321946,
@@ -1042,7 +1042,7 @@ const int32_t twiddle_real_32[FFT_SIZE_MAX] = {
 };
 
 /* in Q1.31, generated from sin(i * 2 * pi / FFT_SIZE_MAX) */
-const int32_t twiddle_imag_32[FFT_SIZE_MAX] = {
+SOF_MATH_FFT_COLD_RODATA static const int32_t twiddle_imag_32[FFT_SIZE_MAX] = {
 	0,
 	-13176712,
 	-26352928,

src/math/fft/fft_16.c

@@ -10,7 +10,6 @@
 #include <sof/math/fft.h>
 
 #ifdef FFT_GENERIC
-#include <sof/audio/coefficients/fft/twiddle_16.h>
 
 /*
  * Helpers for 16 bit FFT calculation
@@ -75,6 +74,7 @@ void fft_execute_16(struct fft_plan *plan, bool ifft)
 	struct icomplex16 tmp2;
 	struct icomplex16 *inb;
 	struct icomplex16 *outb;
+	struct icomplex16 *twiddle;
 	int depth;
 	int top;
 	int bottom;
@@ -104,10 +104,11 @@ void fft_execute_16(struct fft_plan *plan, bool ifft)
 		icomplex16_shift(&inb[i], -(plan->len), &outb[plan->bit_reverse_idx[i]]);
 
 	/* step 2: loop to do FFT transform in smaller size */
+	twiddle = plan->twiddle;
 	for (depth = 1; depth <= plan->len; ++depth) {
 		m = 1 << depth;
 		n = m >> 1;
-		i = FFT_SIZE_MAX >> depth;
+		i = plan->size >> depth;
 
 		/* doing FFT transforms in size m */
 		for (k = 0; k < plan->size; k += m) {
@@ -116,8 +117,7 @@ void fft_execute_16(struct fft_plan *plan, bool ifft)
 				index = i * j;
 				top = k + j;
 				bottom = top + n;
-				tmp1.real = twiddle_real_16[index];
-				tmp1.imag = twiddle_imag_16[index];
+				tmp1 = twiddle[index];
 				/* calculate the accumulator: twiddle * bottom */
 				icomplex16_mul(&tmp1, &outb[bottom], &tmp2);
 				tmp1 = outb[top];

src/math/fft/fft_16_hifi3.c

@@ -9,7 +9,6 @@
 #include <sof/math/fft.h>
 
 #ifdef FFT_HIFI3
-#include <sof/audio/coefficients/fft/twiddle_16.h>
 #include <xtensa/tie/xt_hifi3.h>
 
 /**
@@ -31,6 +30,7 @@ void fft_execute_16(struct fft_plan *plan, bool ifft)
 	ae_int16x4 *out16x4;
 	ae_valign inu = AE_ZALIGN64();
 	ae_valign outu = AE_ZALIGN64();
+	int16_t *twiddle;
 	int depth, top, bottom, index;
 	int i, j, k, m, n;
 	int size = plan->size;
@@ -67,22 +67,22 @@ void fft_execute_16(struct fft_plan *plan, bool ifft)
 	}
 
 	/* step 2: loop to do FFT transform in smaller size */
+	twiddle = plan->twiddle;
 	for (depth = 1; depth <= plan->len; ++depth) {
 		m = 1 << depth;
 		n = m >> 1;
-		i = FFT_SIZE_MAX >> depth;
+		i = size >> depth;
 
 		/* doing FFT transforms in size m */
 		for (k = 0; k < plan->size; k += m) {
 			/* doing one FFT transform for size m */
 			for (j = 0; j < n; ++j) {
-				index = i * j;
+				index = 2 * i * j;
 				top = k + j;
 				bottom = top + n;
 				/* store twiddle and bottom as Q9.23*/
 				temp1 = AE_CVTP24A16X2_LL(outb[bottom].real, outb[bottom].imag);
-				temp2 = AE_CVTP24A16X2_LL(twiddle_real_16[index],
-							  twiddle_imag_16[index]);
+				temp2 = AE_CVTP24A16X2_LL(twiddle[index], twiddle[index + 1]);
 				/* calculate the accumulator: twiddle * bottom */
 				res = AE_MULFC24RA(temp1, temp2);
 				/* saturate and round the result to 16bit and put it in

src/math/fft/fft_32.c

@@ -13,11 +13,8 @@
 #include <stdio.h>
 
 #ifdef FFT_GENERIC
-#include <sof/audio/coefficients/fft/twiddle_32.h>
-
 #include "fft_32.h"
 
-
 /**
  * \brief Execute the 32-bits Fast Fourier Transform (FFT) or Inverse FFT (IFFT)
  *	  For the configured fft_pan.
@@ -30,6 +27,7 @@ void fft_execute_32(struct fft_plan *plan, bool ifft)
 	struct icomplex32 tmp2;
 	struct icomplex32 *inb;
 	struct icomplex32 *outb;
+	struct icomplex32 *twiddle;
 	int depth;
 	int top;
 	int bottom;
@@ -59,10 +57,11 @@ void fft_execute_32(struct fft_plan *plan, bool ifft)
 		icomplex32_shift(&inb[i], -(plan->len), &outb[plan->bit_reverse_idx[i]]);
 
 	/* step 2: loop to do FFT transform in smaller size */
+	twiddle = plan->twiddle;
 	for (depth = 1; depth <= plan->len; ++depth) {
 		m = 1 << depth;
 		n = m >> 1;
-		i = FFT_SIZE_MAX >> depth;
+		i = plan->size >> depth;
 
 		/* doing FFT transforms in size m */
 		for (k = 0; k < plan->size; k += m) {
@@ -71,8 +70,7 @@ void fft_execute_32(struct fft_plan *plan, bool ifft)
 				index = i * j;
 				top = k + j;
 				bottom = top + n;
-				tmp1.real = twiddle_real_32[index];
-				tmp1.imag = twiddle_imag_32[index];
+				tmp1 = twiddle[index];
 				/* calculate the accumulator: twiddle * bottom */
 				icomplex32_mul(&tmp1, &outb[bottom], &tmp2);
 				tmp1 = outb[top];

src/math/fft/fft_32_hifi3.c

@@ -10,7 +10,6 @@
 #include <sof/math/fft.h>
 
 #ifdef FFT_HIFI3
-#include <sof/audio/coefficients/fft/twiddle_32.h>
 #include <xtensa/tie/xt_hifi3.h>
 
 void fft_execute_32(struct fft_plan *plan, bool ifft)
@@ -23,6 +22,7 @@ void fft_execute_32(struct fft_plan *plan, bool ifft)
 	ae_int32x2 *outx = (ae_int32x2 *)plan->outb32;
 	ae_int32x2 *outtop;
 	ae_int32x2 *outbottom;
+	ae_int32x2 *twiddle;
 	uint16_t *idx = &plan->bit_reverse_idx[0];
 	int depth, top, bottom, index;
 	int i, j, k, m, n;
@@ -55,23 +55,22 @@ void fft_execute_32(struct fft_plan *plan, bool ifft)
 	}
 
 	/* step 2: loop to do FFT transform in smaller size */
+	twiddle = plan->twiddle;
 	for (depth = 1; depth <= len; ++depth) {
 		m = 1 << depth;
 		n = m >> 1;
-		i = FFT_SIZE_MAX >> depth;
+		i = size >> depth;
 
 		/* doing FFT transforms in size m */
 		for (k = 0; k < size; k += m) {
 			/* doing one FFT transform for size m */
 			for (j = 0; j < n; ++j) {
-				index = i * j;
+				index = i * j * sizeof(ae_int32x2);
 				top = k + j;
 				bottom = top + n;
 
 				/* load twiddle factor to sample1 */
-				sample1 = twiddle_real_32[index];
-				sample2 = twiddle_imag_32[index];
-				sample1 = AE_SEL32_LH(sample1, sample2);
+				sample1 = AE_L32X2_X(twiddle, index);
 
 				/* calculate the accumulator: twiddle * bottom */
 				sample2 = outx[bottom];

src/math/fft/fft_common.c

@@ -13,7 +13,11 @@
 #include <sof/common.h>
 #include <rtos/alloc.h>
 #include <sof/math/fft.h>
+
 #include "fft_common.h"
+#include "fft_32.h"
+#include "coef/twiddle_32.h"
+#include "coef/twiddle_16.h"
 
 LOG_MODULE_REGISTER(math_fft, CONFIG_SOF_LOG_LEVEL);
 SOF_DEFINE_REG_UUID(math_fft);
@@ -78,6 +82,37 @@ void fft_plan_init_bit_reverse(uint16_t *bit_reverse_idx, int size, int len)
 		bit_reverse_idx[i] = (bit_reverse_idx[i >> 1] >> 1) | ((i & 1) << (len - 1));
 }
 
+void *fft_plan_allocate_twiddle(struct processing_module *mod, int size, int bits)
+{
+	int twiddle_size = ((bits == 32) ? sizeof(int32_t) : sizeof(int16_t)) * 2 * size;
+
+	return mod_alloc_align(mod, twiddle_size, 2 * sizeof(int32_t));
+}
+
+void fft_plan_init_twiddle(void *twiddle, int size, int bits)
+{
+	int32_t *twiddle32;
+	int16_t *twiddle16;
+	int k = FFT_SIZE_MAX / size;
+	int i, j;
+
+	if (bits == 32) {
+		twiddle32 = twiddle;
+		for (i = 0; i < size; i++) {
+			j = i * k;
+			*twiddle32++ = twiddle_real_32[j];
+			*twiddle32++ = twiddle_imag_32[j];
+		}
+	} else {
+		twiddle16 = twiddle;
+		for (i = 0; i < size; i++) {
+			j = i * k;
+			*twiddle16++ = twiddle_real_16[j];
+			*twiddle16++ = twiddle_imag_16[j];
+		}
+	}
+}
+
 struct fft_plan *mod_fft_plan_new(struct processing_module *mod, void *inb,
 				  void *outb, uint32_t size, int bits)
 {
@@ -95,12 +130,29 @@ struct fft_plan *mod_fft_plan_new(struct processing_module *mod, void *inb,
 	plan->bit_reverse_idx = mod_zalloc(mod,	plan->size * sizeof(uint16_t));
 	if (!plan->bit_reverse_idx) {
 		comp_cl_err(mod->dev, "Failed to allocate bit reverse table.");
-		mod_free(mod, plan);
-		return NULL;
+		goto err;
 	}
 
 	fft_plan_init_bit_reverse(plan->bit_reverse_idx, plan->size, plan->len);
+
+	/* Allocate memory for packed twiddle factors */
+	plan->twiddle = fft_plan_allocate_twiddle(mod, size, bits);
+	if (!plan->twiddle) {
+		comp_cl_err(mod->dev, "Failed to allocate twiddle factors.");
+		goto err_free_bit_reverse;
+	}
+
+	/* Pack twiddle factors from sparse real and image to complex pairs */
+	fft_plan_init_twiddle(plan->twiddle, size, bits);
+
 	return plan;
+
+err_free_bit_reverse:
+	mod_free(mod, plan->bit_reverse_idx);
+
+err:
+	mod_free(mod, plan);
+	return NULL;
 }
 
 void mod_fft_plan_free(struct processing_module *mod, struct fft_plan *plan)

src/math/fft/fft_common.h

@@ -4,6 +4,15 @@
 //
 // Author: Seppo Ingalsuo <seppo.ingalsuo@linux.intel.com>
 
+#ifndef __SOF_FFT_COMMON_H__
+#define __SOF_FFT_COMMON_H__
+
+#if CONFIG_MATH_FFT_COLD_TWIDDLE_FACTORS
+#define SOF_MATH_FFT_COLD_RODATA __cold_rodata
+#else
+#define SOF_MATH_FFT_COLD_RODATA
+#endif
+
 /**
  * fft_plan_common_new() - Common FFT prepare function
  * @param mod: Pointer to module
@@ -23,3 +32,22 @@ struct fft_plan *fft_plan_common_new(struct processing_module *mod, void *inb,
  * @param len: Power of two value equals FFT size
  */
 void fft_plan_init_bit_reverse(uint16_t *bit_reverse_idx, int size, int len);
+
+/**
+ * fft_plan_allocate_twiddle - Allocates memory for twiddle factors
+ * @param mod: pointer to module
+ * @param size: size of FFT
+ * @param bits: word length of FFT, 32 or 16
+ * @return Pointer to allocated memory
+ */
+void *fft_plan_allocate_twiddle(struct processing_module *mod, int size, int bits);
+
+/**
+ * fft_plan_init_twiddle - sets up a decimated and re-arranged twiddle factors tabled
+ * @param twiddle: pointer to destination twiddle factors
+ * @param size: size of FFT
+ * @param bits: word length of FFT, 32 or 16
+ */
+void fft_plan_init_twiddle(void *twiddle, int size, int bits);
+
+#endif /* __SOF_FFT_COMMON_H__ */