#include "speech_app.h" #include "speech_init.h" #include "LVGL/lvgl_init.h" #include static const char *TAG_LVGL = "sr"; // 音频设备句柄 esp_codec_dev_handle_t record_dev = NULL; const char *cmd_phoneme[] = { "ni hao xiao qi", "xiao qi zai jian", // 小奇,再见 "xiao qi tan ge zi ta", // 小奇,弹个吉他 "xiao qi bi ge xin", // 小奇,比个心 "xiao qi si kao yi xia zhe ge wen ti", // 小奇,思考一下这个问题 }; const uint32_t cmd_phoneme_number = sizeof(cmd_phoneme)/sizeof(cmd_phoneme[0]); typedef struct __attribute__((packed)) { char chunkID[4]; uint32_t chunkSize; char format[4]; char subchunk1ID[4]; uint32_t subchunk1Size; uint16_t audioFormat; uint16_t numChannels; uint32_t sampleRate; uint32_t byteRate; uint16_t blockAlign; uint16_t bitsPerSample; char subchunk2ID[4]; uint32_t subchunk2Size; } wav_header_t; void write_wav_header(FILE* fp, uint32_t total_data_bytes) { wav_header_t wav = { .chunkID = {'R','I','F','F'}, .format = {'W','A','V','E'}, .subchunk1ID = {'f','m','t',' '}, .subchunk1Size = 16, .audioFormat = 1, .numChannels = 2, // 你的参数是resdata .sampleRate = 16000, // 你的参数是16kHz .bitsPerSample = 16, // 你的参数是16-bit .subchunk2ID = {'d','a','t','a'}, .subchunk2Size = total_data_bytes, }; // 计算其他依赖的字段 wav.byteRate = wav.sampleRate * wav.numChannels * wav.bitsPerSample / 8; wav.blockAlign = wav.numChannels * wav.bitsPerSample / 8; wav.chunkSize = 36 + wav.subchunk2Size; // 文件总大小 - 8 // 写入文件头 fwrite(&wav, sizeof(wav), 1, fp); } int slience_count = 0; #define GAIN 1.5f // 例如增益 1.5(约 +3.5 dB) #define MAX_VAL 32767 #define MIN_VAL -32768 /** * @brief 音频数据采集任务 * 从I2S接口读取音频数据并喂给音频前端处理 * * @param arg 音频前端数据句柄 */ void feed_Task(void *arg) { ESP_LOGI(TAG_LVGL, "Starting feed task"); esp_afe_sr_data_t *afe_data_ = arg; // 获取音频前端处理的块大小和通道数 /* int feed_chunksize = afe_handle->get_feed_chunksize(afe_data_); int feed_nch = afe_handle->get_feed_channel_num(afe_data_); ESP_LOGE(TAG_LVGL, "feed_chunksize: %d, feed_nch: %d", feed_chunksize, feed_nch); */ /* FILE* fp = fopen("/extflash/audio1.wav", "wb"); // 或 "/extflash/audio.wav" if (!fp) { ESP_LOGW(TAG_MAIN, "WAV文件打开失败,无法写入"); } uint32_t sampleRate = 16000; uint16_t numChannels = 2; uint16_t bitsPerSample = 16; uint32_t totalSamples = sampleRate * 2; // 2秒总采样数(每声道) uint32_t totalDataBytes = totalSamples * numChannels * (bitsPerSample / 8); write_wav_header(fp, totalDataBytes); int counter = 0; */ while (1) { size_t bytesIn = 0; // 从I2S接口读取音频数据 //int ret=esp_codec_dev_read(record_dev, feed_buff, feed_chunksize * feed_nch * sizeof(int16_t)); // 读取音频数据(录音) // 将左右声道数据求和取半,转为单声道数据 //for (int i = 0; i < feed_chunksize; i++) { // int left = feed_buff[2*i]; // int right = feed_buff[2*i + 1]; // 求和并取平均,防止数值溢出 // mono_buff[i] = (left + right) >> 1; //} //ESP_LOGW(TAG_LVGL, "codec read error: %d", ret); /* if(counter < 33) // 采集100次后关闭文件(约10秒音频) { fwrite(feed_buff, sizeof(int16_t), feed_chunksize * feed_nch, fp); counter++; } if(counter == 33) // 采集100次后关闭文件(约10秒音频) { fclose(fp); counter++; FILE* fp = fopen("/extflash/audio2.wav", "wb"); write_wav_header(fp, totalDataBytes); ESP_LOGW(TAG_MAIN,"WAV 文件已保存:/extflash/audio1.wav\n"); ESP_LOGW(TAG_MAIN,"开始写入第二个文件:/extflash/audio2.wav\n"); } if(counter > 33 && counter < 66) { fwrite(feed_buff, sizeof(int16_t), feed_chunksize * feed_nch, fp); counter++; } if(counter == 66) // 采集100次后关闭文件(约10秒音频) { fclose(fp); ESP_LOGW(TAG_MAIN,"第二个 WAV 文件已保存:/extflash/audio2.wav\n"); counter++; FILE* fp = fopen("/extflash/audio3.wav", "wb"); write_wav_header(fp, totalDataBytes); ESP_LOGW(TAG_MAIN,"开始写入第三个文件:/extflash/audio3.wav\n"); } if(counter > 66 && counter < 99) { fwrite(feed_buff, sizeof(int16_t), feed_chunksize * feed_nch, fp); counter++; } if(counter == 99) // 采集100次后关闭文件(约10秒音频) { fclose(fp); ESP_LOGW(TAG_MAIN,"第三个 WAV 文件已保存:/extflash/audio3.wav\n"); counter++; break; // 结束循环,停止采集 } */ // 将音频数据喂给音频前端处理 //ESP_LOGI("I2S RAW", "L:%d, R:%d, L:%d, R:%d,L:%d, R:%d,L:%d, R:%d", feed_buff[0], feed_buff[1], feed_buff[2], feed_buff[3], feed_buff[4], feed_buff[5], feed_buff[6], feed_buff[7]); //int num_samples = feed_chunksize * feed_nch; /* long long sum_squares = 0; for (int i = 0; i < num_samples; i++) { sum_squares += (long long)feed_buff[i] * feed_buff[i]; } double rms = sqrt((double)sum_squares / num_samples); #define REFERENCE_RMS 750 double db = 20.0 * log10(rms / REFERENCE_RMS); //ESP_LOGI("AUDIO", "RMS: %.2f, dB: %.2f", rms, db); */ //afe_handle->feed(afe_data_, feed_buff); vTaskDelay(pdMS_TO_TICKS(1)); //ESP_LOGW(TAG_LVGL,"feed_task"); } // 任务退出时释放资源 vTaskDelete(NULL); } extern bool is_ai_recording; // AI语音交互状态标志(外部声明) extern bool is_ai_create_pcmfile; // AI语音交互生成pcm文件状态标志(外部声明) extern bool is_ai_end_recording; // AI语音交互结束录音状态标志(外部声明) void get_file_size(const char *path) { struct stat st; if (stat(path, &st) != 0) { ESP_LOGW(TAG_MAIN,"stat failed for %s\n", path); return; } float size_kb = st.st_size / 1024.0; ESP_LOGI(TAG_MAIN, "File %s size: %.2f KB", path, size_kb); } /** * @brief 语音检测任务 * 从音频前端获取处理后的数据并进行语音命令识别 * * @param arg 音频前端数据句柄 */ extern int64_t init_start_us; int silence_count = 0; void detect_Task(void *arg) { ESP_LOGI(TAG_LVGL, "Starting detect task"); int feed_chunksize=512; int feed_nch=2; // 分配音频数据缓冲区 int16_t *feed_buff = (int16_t *)heap_caps_malloc(feed_chunksize * feed_nch * sizeof(int16_t), MALLOC_CAP_SPIRAM); assert(feed_buff); // 确保内存分配成功 int16_t *mono_buff = (int16_t *)malloc(feed_chunksize * sizeof(int16_t)); if (mono_buff == NULL) { ESP_LOGE(TAG_LVGL, "Failed to allocate mono buffer"); return; } record_dev = ES7243E_Init(); //esp_afe_sr_data_t *afe_data_ = arg; // 获取获取数据的块大小并分配缓冲区 //int afe_chunksize = afe_handle->get_fetch_chunksize(afe_data_); //ESP_LOGW(TAG_LVGL, "afe_chunksize: %d", afe_chunksize); /* FILE* fp = fopen("/extflash/audio1res.wav", "wb"); if (!fp) { ESP_LOGW(TAG_MAIN, "WAV文件打开失败,无法写入"); } uint32_t sampleRate = 16000; uint16_t numChannels = 1; uint16_t bitsPerSample = 16; uint32_t totalSamples = sampleRate * 3.2; // 3.2秒总采样数(每声道) uint32_t totalDataBytes = totalSamples * numChannels * (bitsPerSample / 8); write_wav_header(fp, totalDataBytes); int counter = 0; */ ESP_LOGW(TAG_LVGL, "开始识别"); const int64_t init_elapsed_us = esp_timer_get_time() - init_start_us; ESP_LOGI(TAG_MAIN, "语音识别初始化耗时: %" PRId64 " ms (%" PRId64 " us)", init_elapsed_us / 1000, init_elapsed_us); while (1) { //对话开启,启动PCM文件创建 if(is_ai_create_pcmfile == true){ FILE* fp = fopen("/extflash/audio_ai.pcm", "wb"); if (!fp) { ESP_LOGW(TAG_MAIN, "PCM文件打开失败,无法写入,语音对话阶段一失败"); } uint32_t sampleRate = 16000; uint16_t numChannels = 2; uint16_t bitsPerSample = 16; uint32_t totalSamples = sampleRate * 2; // 2秒总采样数(每声道) uint32_t totalDataBytes = 0; // PCM文件不需要头部,数据大小可以动态增长,需要最后填入 write_wav_header(fp, totalDataBytes); is_ai_create_pcmfile = false; ESP_LOGW(TAG_LVGL, "创建PCM基础格式成功"); is_ai_recording = true;//进入录音状态 fclose(fp); } int ret=esp_codec_dev_read(record_dev, feed_buff, feed_chunksize * feed_nch * sizeof(int16_t)); // 读取音频数据(录音) //分贝检测 //音频录音 uint32_t totalDataBytes = 0; if(is_ai_recording == true){ FILE* fp = fopen("/extflash/audio_ai.pcm", "ab"); if (!fp) { ESP_LOGW(TAG_MAIN, "PCM文件录音失败,无法写入,语音对话阶段二失败"); } fwrite(feed_buff, sizeof(int16_t), feed_chunksize * feed_nch, fp); totalDataBytes += feed_chunksize * feed_nch * sizeof(int16_t); //结束录音状态 fclose(fp); //RMS对话结束检测函数 int num_samples = feed_chunksize * feed_nch; long long sum_squares = 0; for (int i = 0; i < num_samples; i++) { sum_squares += (long long)feed_buff[i] * feed_buff[i]; } double rms = sqrt((double)sum_squares / num_samples); #define REFERENCE_RMS 750 double db = 20.0 * log10(rms / REFERENCE_RMS); ESP_LOGI("AUDIO", "RMS: %.2f, dB: %.2f", rms, db); if(rms<400){ silence_count++; } if(silence_count==53){ is_ai_recording = false; is_ai_end_recording = true; silence_count = 0; } } // 录制结束,更新 PCM 文件头中的数据大小 if(is_ai_end_recording == true){ FILE* fp = fopen("/extflash/audio_ai.pcm", "ab"); if (!fp) { ESP_LOGW(TAG_MAIN, "PCM文件录音结束失败,语音对话阶段三失败"); } fseek(fp, 0, SEEK_SET); // 定位到文件开头 write_wav_header(fp, totalDataBytes); // 更新 WAV 头部中的数据 fclose(fp); get_file_size("/extflash/audio_ai.pcm"); ESP_LOGW(TAG_LVGL, "PCM文件录制完成并更新头部成功"); change_task_post_action(0X000C); is_ai_end_recording = false; //重置结束录音状态 } // 将左右声道数据求和取半,转为单声道数据 for (int i = 0; i < feed_chunksize; i++) { int left = feed_buff[2*i]; int right = feed_buff[2*i + 1]; // 求和并取平均,防止数值溢出 int avg = (left + right) >> 1; // 取平均 mono_buff[i] = (int)avg; } ESP_LOGI("I2S RAW", "L:%d, R:%d, L:%d, R:%d,L:%d, R:%d,L:%d, R:%d", feed_buff[0], feed_buff[1], feed_buff[2], feed_buff[3], feed_buff[4], feed_buff[5], feed_buff[6], feed_buff[7]); ESP_LOGI("I2S RAW", "1:%d, 2:%d, 3:%d, 4:%d,5:%d, 6:%d,7:%d, 8:%d", mono_buff[0], mono_buff[1], mono_buff[2], mono_buff[3], mono_buff[4], mono_buff[5], mono_buff[6], mono_buff[7]); // 从音频前端获取处理后的数据 //ESP_LOGW(TAG_LVGL, "detect_task"); //afe_fetch_result_t *res = afe_handle->fetch(afe_data_); //int sample_count = res->data_size / sizeof(int16_t); // 检查获取结果是否有效 //if (!res || res->ret_value == ESP_FAIL) //{ // ESP_LOGE(TAG_LVGL, "获取音频数据失败!"); // res = afe_handle->fetch(afe_data_); // break; //} /* if(counter < 100) // 采集100次后关闭文件(约10秒音频) { fwrite(res->data, sizeof(int16_t), sample_count, fp); counter++; } if(counter == 100) // 采集100次后关闭文件(约10秒音频) { fclose(fp); counter++; FILE* fp = fopen("/extflash/audio2res.wav", "wb"); write_wav_header(fp, totalDataBytes); ESP_LOGW(TAG_MAIN,"WAV 文件已保存:/extflash/audio1res.wav\n"); ESP_LOGW(TAG_MAIN,"开始写入第二个文件:/extflash/audio2res.wav\n"); } if(counter > 100 && counter < 200) { fwrite(res->data, sizeof(int16_t), sample_count, fp); counter++; } if(counter == 200) // 采集100次后关闭文件(约10秒音频) { fclose(fp); ESP_LOGW(TAG_MAIN,"第二个 WAV 文件已保存:/extflash/audio2res.wav\n"); counter++; FILE* fp = fopen("/extflash/audio3res.wav", "wb"); write_wav_header(fp, totalDataBytes); ESP_LOGW(TAG_MAIN,"开始写入第三个文件:/extflash/audio3res.wav\n"); } if(counter > 200 && counter < 300) { fwrite(res->data, sizeof(int16_t), sample_count, fp); counter++; } if(counter == 300) // 采集100次后关闭文件(约10秒音频) { fclose(fp); ESP_LOGW(TAG_MAIN,"第三个 WAV 文件已保存:/extflash/audio3res.wav\n"); counter++; break; // 结束循环,停止采集 } // 获取并处理 VAD 状态 if (res->vad_state == 1) { // 检测到语音活动 ESP_LOGI(TAG_LVGL, "VAD: Speech detected."); } else if (res->vad_state == 0) { // 当前为静音或噪声 ESP_LOGE(TAG_LVGL, "VAD: Silence."); } */ /* int sample_count = res->data_size / sizeof(int16_t); int16_t *samples = res->data; ESP_LOGI(TAG_LVGL, "data[0~5] = %d, %d, %d, %d, %d, %d", samples[0], samples[1], samples[2], samples[3], samples[4], samples[5]); int16_t *samples2 = res->raw_data; ESP_LOGI(TAG_LVGL, "raw_data[0~5] = %d, %d, %d, %d, %d, %d", samples2[0], samples2[1], samples2[2], samples2[3], samples2[4], samples2[5]); */ //ESP_LOGI(TAG_LVGL, "data_volume = %.2f dB, ringbuff_free_pct = %.2f%%", // res->data_volume, res->ringbuff_free_pct * 100);*/ // 直接进行语音命令识别,跳过唤醒词检测 // 使用多网络模型进行语音命令检测 esp_mn_state_t mn_state = ESP_MN_STATE_DETECTING; mn_state = multinet->detect(model_data, mono_buff); // 如果仍在检测中,继续循环 if (ESP_MN_STATE_DETECTING == mn_state) { //ESP_LOGI(TAG_LVGL, "语音识别中..."); continue; } // 处理超时情况 if (ESP_MN_STATE_TIMEOUT == mn_state) { ESP_LOGW(TAG_LVGL, "语音识别超时"); sr_result_t result = { .state = mn_state, .command_id = 0, }; // xQueueSend(g_result_que, &result, 10); continue; } // 处理识别到命令的情况 if (ESP_MN_STATE_DETECTED == mn_state) { // 获取识别结果 esp_mn_results_t *mn_result = multinet->get_results(model_data); for (int i = 0; i < mn_result->num; i++) { int prob_permille = (int)(mn_result->prob[i] * 1000.0f); ESP_LOGI(TAG_LVGL, "TOP %d, command_id: %d, phrase_id: %d, prob: %d.%03d", i + 1, mn_result->command_id[i], mn_result->phrase_id[i], prob_permille / 1000, abs(prob_permille % 1000)); } int sr_command_id = mn_result->command_id[0]; // 在这里添加指令对应的中文打印 switch (sr_command_id) { case 0: // "ni hao xiao qi" ESP_LOGE(TAG_LVGL, "识别到指令:你好小奇"); change_task_post_action(0X000B); is_ai_create_pcmfile = true; //ret =lv_obj_send_event(ui_Screen_Lottie, LV_EVENT_KEY, (void *)LV_KEY_ENTER); ESP_LOGW(TAG_LVGL, "RET: %d", ret); break; case 1: // "huan ge feng ge" ESP_LOGE(TAG_LVGL, "识别到指令:小奇,再见"); change_task_post_action(0X000C); //ret =lv_obj_send_event(ui_Screen_Lottie, LV_EVENT_KEY, (void *)LV_KEY_NEXT); ESP_LOGW(TAG_LVGL, "RET: %d", ret); break; case 2: // "yao tou huang nao" ESP_LOGE(TAG_LVGL, "识别到指令:小奇,弹个吉他"); break; case 3: // "yao tou huang nao" ESP_LOGE(TAG_LVGL, "识别到指令:小奇,比个心"); break; case 4: // "yao tou huang nao" ESP_LOGE(TAG_LVGL, "识别到指令:小奇,思考一下这个问题"); break; default: ESP_LOGE(TAG_LVGL, "识别到未知指令:%d", sr_command_id); break; } // 组织识别结果 sr_result_t result = { .state = mn_state, .command_id = sr_command_id, }; } vTaskDelay(pdMS_TO_TICKS(1)); } if (feed_buff) { free(feed_buff); feed_buff = NULL; } if (mono_buff) { free(mono_buff); mono_buff = NULL; } vTaskDelete(NULL); }