From b7c0201ca318b532b57a77f3c881cbe855b58172 Mon Sep 17 00:00:00 2001 From: marlonz <235833017@qq.com> Date: Tue, 17 Jun 2025 16:40:17 +0930 Subject: [PATCH 01/14] =?UTF-8?q?=E6=8F=90=E4=BA=A4mqtt=E5=90=8E=E7=AB=AF?= =?UTF-8?q?=E4=BF=AE=E6=94=B9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../xiaozhi/common/constant/Constant.java | 9 +++++++ .../java/xiaozhi/common/utils/JsonUtils.java | 2 +- .../device/controller/OTAController.java | 4 ++++ .../device/dto/DeviceReportRespDTO.java | 20 ++++++++++++++++ .../service/impl/DeviceServiceImpl.java | 18 ++++++++++++++ .../resources/db/changelog/202506152342.sql | 3 +++ .../resources/db/changelog/202506152350.sql | 5 ++++ .../db/changelog/db.changelog-master.yaml | 16 ++++++++++++- main/manager-web/package-lock.json | 24 +++++++++++++++---- 9 files changed, 94 insertions(+), 7 deletions(-) create mode 100644 main/manager-api/src/main/resources/db/changelog/202506152342.sql create mode 100644 main/manager-api/src/main/resources/db/changelog/202506152350.sql diff --git a/main/manager-api/src/main/java/xiaozhi/common/constant/Constant.java b/main/manager-api/src/main/java/xiaozhi/common/constant/Constant.java index 44294756d..d6f1867d3 100644 --- a/main/manager-api/src/main/java/xiaozhi/common/constant/Constant.java +++ b/main/manager-api/src/main/java/xiaozhi/common/constant/Constant.java @@ -91,6 +91,15 @@ public interface Constant { */ String SERVER_WEBSOCKET = "server.websocket"; + /** + * mqtt gateway 配置 + */ + String SERVER_MQTT_GATEWAY = "server.mqtt_gateway"; + + /** + * websocket地址 + */ + /** * ota地址 */ diff --git a/main/manager-api/src/main/java/xiaozhi/common/utils/JsonUtils.java b/main/manager-api/src/main/java/xiaozhi/common/utils/JsonUtils.java index 288dae9ab..093f2acf1 100644 --- a/main/manager-api/src/main/java/xiaozhi/common/utils/JsonUtils.java +++ b/main/manager-api/src/main/java/xiaozhi/common/utils/JsonUtils.java @@ -66,5 +66,5 @@ public static List parseArray(String text, Class clazz) { throw new RuntimeException(e); } } - + } diff --git a/main/manager-api/src/main/java/xiaozhi/modules/device/controller/OTAController.java b/main/manager-api/src/main/java/xiaozhi/modules/device/controller/OTAController.java index 05d36e9ac..7a826a336 100644 --- a/main/manager-api/src/main/java/xiaozhi/modules/device/controller/OTAController.java +++ b/main/manager-api/src/main/java/xiaozhi/modules/device/controller/OTAController.java @@ -78,6 +78,10 @@ public ResponseEntity activateDevice( @GetMapping @Hidden public ResponseEntity getOTA() { + String mqttUdpConfig = sysParamsService.getValue(Constant.SERVER_MQTT_GATEWAY, false); + if(StringUtils.isBlank(mqttUdpConfig)) { + return ResponseEntity.ok("OTA接口不正常，缺少websocket地址，请登录智控台，在参数管理找到【server.mqtt_udp】配置"); + } String wsUrl = sysParamsService.getValue(Constant.SERVER_WEBSOCKET, true); if (StringUtils.isBlank(wsUrl) || wsUrl.equals("null")) { return ResponseEntity.ok("OTA接口不正常，缺少websocket地址，请登录智控台，在参数管理找到【server.websocket】配置"); diff --git a/main/manager-api/src/main/java/xiaozhi/modules/device/dto/DeviceReportRespDTO.java b/main/manager-api/src/main/java/xiaozhi/modules/device/dto/DeviceReportRespDTO.java index f938001e0..5c53b7880 100644 --- a/main/manager-api/src/main/java/xiaozhi/modules/device/dto/DeviceReportRespDTO.java +++ b/main/manager-api/src/main/java/xiaozhi/modules/device/dto/DeviceReportRespDTO.java @@ -22,6 +22,9 @@ public class DeviceReportRespDTO { @Schema(description = "WebSocket配置") private Websocket websocket; + + @Schema(description = "MQTT Gateway配置") + private MQTT mqtt; @Getter @Setter @@ -70,4 +73,21 @@ public static class Websocket { @Schema(description = "WebSocket服务器地址") private String url; } + + @Getter + @Setter + public static class MQTT { + @Schema(description = "MQTT 配置网址") + private String endpoint; + @Schema(description = "MQTT 客户端唯一标识符") + private String client_id; + @Schema(description = "MQTT 认证用户名") + private String username; + @Schema(description = "MQTT 认证密码") + private String password; + @Schema(description = "ESP32 发布消息的主题") + private String publish_topic; + @Schema(description = "ESP32 订阅的主题") + private String subscribe_topic; + } } diff --git a/main/manager-api/src/main/java/xiaozhi/modules/device/service/impl/DeviceServiceImpl.java b/main/manager-api/src/main/java/xiaozhi/modules/device/service/impl/DeviceServiceImpl.java index ee31c41f0..26a370f41 100644 --- a/main/manager-api/src/main/java/xiaozhi/modules/device/service/impl/DeviceServiceImpl.java +++ b/main/manager-api/src/main/java/xiaozhi/modules/device/service/impl/DeviceServiceImpl.java @@ -174,6 +174,24 @@ public DeviceReportRespDTO checkDeviceActive(String macAddress, String clientId, } response.setWebsocket(websocket); + + // 添加MQTT UDP配置 + // 从系统参数获取WebSocket URL，如果未配置不使用默认值 + String mqttUdpConfig = sysParamsService.getValue(Constant.SERVER_MQTT_GATEWAY, false); + if(!StringUtils.isBlank(mqttUdpConfig)) { + DeviceReportRespDTO.MQTT mqtt= new DeviceReportRespDTO.MQTT(); + mqtt.setEndpoint(mqttUdpConfig); + mqtt.setClient_id(clientId); + String userNameString = deviceById.getId().replace(":", "_"); + mqtt.setUsername(userNameString); + mqtt.setPassword(deviceById.getBoard()); + String topicString = deviceById.getBoard() + '/' + deviceById.getAgentId(); + mqtt.setPublish_topic(topicString); + String subscribeString = "devices/p2p/"+userNameString; + mqtt.setSubscribe_topic(subscribeString); + response.setMqtt(mqtt); + } + if (deviceById != null) { // 如果设备存在，则异步更新上次连接时间和版本信息 diff --git a/main/manager-api/src/main/resources/db/changelog/202506152342.sql b/main/manager-api/src/main/resources/db/changelog/202506152342.sql new file mode 100644 index 000000000..a18dd8efb --- /dev/null +++ b/main/manager-api/src/main/resources/db/changelog/202506152342.sql @@ -0,0 +1,3 @@ +delete from `sys_params` where id = 108; +INSERT INTO `sys_params` (id, param_code, param_value, value_type, param_type, remark) VALUES (108, 'server.mqtt_udp', 'null', 'string', 1, 'mqtt udp 配置'); + diff --git a/main/manager-api/src/main/resources/db/changelog/202506152350.sql b/main/manager-api/src/main/resources/db/changelog/202506152350.sql new file mode 100644 index 000000000..917d04251 --- /dev/null +++ b/main/manager-api/src/main/resources/db/changelog/202506152350.sql @@ -0,0 +1,5 @@ +delete from `sys_params` where id = 108; +INSERT INTO `sys_params` (id, param_code, param_value, value_type, param_type, remark) VALUES (108, 'server.mqtt_gateway', 'null', 'string', 1, 'mqtt gateway 配置'); + +delete from `sys_params` where id = 109; +INSERT INTO `sys_params` (id, param_code, param_value, value_type, param_type, remark) VALUES (109, 'server.udp_gateway', 'null', 'string', 1, 'udp gateway 配置'); diff --git a/main/manager-api/src/main/resources/db/changelog/db.changelog-master.yaml b/main/manager-api/src/main/resources/db/changelog/db.changelog-master.yaml index f33d28a29..392349688 100755 --- a/main/manager-api/src/main/resources/db/changelog/db.changelog-master.yaml +++ b/main/manager-api/src/main/resources/db/changelog/db.changelog-master.yaml @@ -204,4 +204,18 @@ databaseChangeLog: changes: - sqlFile: encoding: utf8 - path: classpath:db/changelog/202506080955.sql \ No newline at end of file + path: classpath:db/changelog/202506080955.sql + - changeSet: + id: 202506152342 + author: marlonz + changes: + - sqlFile: + encoding: utf8 + path: classpath:db/changelog/202506152342.sql + - changeSet: + id: 202506152350 + author: marlonz + changes: + - sqlFile: + encoding: utf8 + path: classpath:db/changelog/202506152350.sql diff --git a/main/manager-web/package-lock.json b/main/manager-web/package-lock.json index c104b60c2..6d1f51283 100644 --- a/main/manager-web/package-lock.json +++ b/main/manager-web/package-lock.json @@ -10,6 +10,7 @@ "dependencies": { "core-js": "^3.41.0", "cross-env": "^7.0.3", + "dotenv": "^16.5.0", "element-ui": "^2.15.14", "flyio": "^0.6.14", "normalize.css": "^8.0.1", @@ -2540,6 +2541,16 @@ } } }, + "node_modules/@vue/cli-service/node_modules/dotenv": { + "version": "10.0.0", + "resolved": "https://registry.npmmirror.com/dotenv/-/dotenv-10.0.0.tgz", + "integrity": "sha512-rlBi9d8jpv9Sf1klPjNfFAuWDjKLwTIJJ/VxtoTwIR6hnZxcEOQCZg2oIL3MWBYw5GpUDKOEnND7LXTbIpQ03Q==", + "dev": true, + "license": "BSD-2-Clause", + "engines": { + "node": ">=10" + } + }, "node_modules/@vue/cli-shared-utils": { "version": "5.0.8", "resolved": "https://registry.npmmirror.com/@vue/cli-shared-utils/-/cli-shared-utils-5.0.8.tgz", @@ -4776,12 +4787,15 @@ } }, "node_modules/dotenv": { - "version": "10.0.0", - "resolved": "https://registry.npmmirror.com/dotenv/-/dotenv-10.0.0.tgz", - "integrity": "sha512-rlBi9d8jpv9Sf1klPjNfFAuWDjKLwTIJJ/VxtoTwIR6hnZxcEOQCZg2oIL3MWBYw5GpUDKOEnND7LXTbIpQ03Q==", - "dev": true, + "version": "16.5.0", + "resolved": "https://registry.npmmirror.com/dotenv/-/dotenv-16.5.0.tgz", + "integrity": "sha512-m/C+AwOAr9/W1UOIZUo232ejMNnJAJtYQjUbHoNTBNTJSvqzzDh7vnrei3o3r3m9blf6ZoDkvcw0VmozNRFJxg==", + "license": "BSD-2-Clause", "engines": { - "node": ">=10" + "node": ">=12" + }, + "funding": { + "url": "https://dotenvx.com" } }, "node_modules/dotenv-expand": { From 3d3875f14fe3cb862391e9995c4b98f5b76857ce Mon Sep 17 00:00:00 2001 From: marlonz <235833017@qq.com> Date: Sat, 5 Jul 2025 12:34:57 +0930 Subject: [PATCH 02/14] update lq base sql --- .../src/main/resources/db/changelog/202506152342.sql | 3 --- .../db/changelog/{202506152350.sql => 202507051233.sql} | 2 +- 2 files changed, 1 insertion(+), 4 deletions(-) delete mode 100644 main/manager-api/src/main/resources/db/changelog/202506152342.sql rename main/manager-api/src/main/resources/db/changelog/{202506152350.sql => 202507051233.sql} (90%) diff --git a/main/manager-api/src/main/resources/db/changelog/202506152342.sql b/main/manager-api/src/main/resources/db/changelog/202506152342.sql deleted file mode 100644 index a18dd8efb..000000000 --- a/main/manager-api/src/main/resources/db/changelog/202506152342.sql +++ /dev/null @@ -1,3 +0,0 @@ -delete from `sys_params` where id = 108; -INSERT INTO `sys_params` (id, param_code, param_value, value_type, param_type, remark) VALUES (108, 'server.mqtt_udp', 'null', 'string', 1, 'mqtt udp 配置'); - diff --git a/main/manager-api/src/main/resources/db/changelog/202506152350.sql b/main/manager-api/src/main/resources/db/changelog/202507051233.sql similarity index 90% rename from main/manager-api/src/main/resources/db/changelog/202506152350.sql rename to main/manager-api/src/main/resources/db/changelog/202507051233.sql index 917d04251..a85688772 100644 --- a/main/manager-api/src/main/resources/db/changelog/202506152350.sql +++ b/main/manager-api/src/main/resources/db/changelog/202507051233.sql @@ -2,4 +2,4 @@ delete from `sys_params` where id = 108; INSERT INTO `sys_params` (id, param_code, param_value, value_type, param_type, remark) VALUES (108, 'server.mqtt_gateway', 'null', 'string', 1, 'mqtt gateway 配置'); delete from `sys_params` where id = 109; -INSERT INTO `sys_params` (id, param_code, param_value, value_type, param_type, remark) VALUES (109, 'server.udp_gateway', 'null', 'string', 1, 'udp gateway 配置'); +INSERT INTO `sys_params` (id, param_code, param_value, value_type, param_type, remark) VALUES (109, 'server.udp_gateway', 'null', 'string', 1, 'udp gateway 配置'); \ No newline at end of file From 7c0367729575e4edf8a7a8265f3f207da906ea90 Mon Sep 17 00:00:00 2001 From: marlonz <235833017@qq.com> Date: Sat, 5 Jul 2025 13:25:42 +0930 Subject: [PATCH 03/14] update change log --- .../main/resources/db/changelog/db.changelog-master.yaml | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/main/manager-api/src/main/resources/db/changelog/db.changelog-master.yaml b/main/manager-api/src/main/resources/db/changelog/db.changelog-master.yaml index 4d1d3798a..ffc60bdbb 100755 --- a/main/manager-api/src/main/resources/db/changelog/db.changelog-master.yaml +++ b/main/manager-api/src/main/resources/db/changelog/db.changelog-master.yaml @@ -245,11 +245,4 @@ databaseChangeLog: changes: - sqlFile: encoding: utf8 - path: classpath:db/changelog/202506152342.sql - - changeSet: - id: 202506152350 - author: marlonz - changes: - - sqlFile: - encoding: utf8 - path: classpath:db/changelog/202506152350.sql + path: classpath:db/changelog/202507051233.sql From aed54bb3a683095ca1f81a06cbaa3eda90c5108d Mon Sep 17 00:00:00 2001 From: marlonz <235833017@qq.com> Date: Sat, 5 Jul 2025 13:30:14 +0930 Subject: [PATCH 04/14] tech review change --- .../src/main/java/xiaozhi/common/constant/Constant.java | 4 ---- .../src/main/java/xiaozhi/common/utils/JsonUtils.java | 1 - 2 files changed, 5 deletions(-) diff --git a/main/manager-api/src/main/java/xiaozhi/common/constant/Constant.java b/main/manager-api/src/main/java/xiaozhi/common/constant/Constant.java index d9fced7cd..518ebc70c 100644 --- a/main/manager-api/src/main/java/xiaozhi/common/constant/Constant.java +++ b/main/manager-api/src/main/java/xiaozhi/common/constant/Constant.java @@ -96,10 +96,6 @@ public interface Constant { */ String SERVER_MQTT_GATEWAY = "server.mqtt_gateway"; - /** - * websocket地址 - */ - /** * ota地址 */ diff --git a/main/manager-api/src/main/java/xiaozhi/common/utils/JsonUtils.java b/main/manager-api/src/main/java/xiaozhi/common/utils/JsonUtils.java index 093f2acf1..6d628d381 100644 --- a/main/manager-api/src/main/java/xiaozhi/common/utils/JsonUtils.java +++ b/main/manager-api/src/main/java/xiaozhi/common/utils/JsonUtils.java @@ -66,5 +66,4 @@ public static List parseArray(String text, Class clazz) { throw new RuntimeException(e); } } - } From 076aaae944555db4e9c414ae15969cb2c07a16c4 Mon Sep 17 00:00:00 2001 From: marlonz <235833017@qq.com> Date: Sat, 5 Jul 2025 13:47:49 +0930 Subject: [PATCH 05/14] mqtt gateway project --- main/mqtt-gateway/.gitignore | 4 + main/mqtt-gateway/LICENSE | 21 + main/mqtt-gateway/README.md | 215 ++++++ main/mqtt-gateway/app.js | 623 ++++++++++++++++++ main/mqtt-gateway/config/mqtt.json.example | 15 + main/mqtt-gateway/config/mqtt.json.test | 12 + main/mqtt-gateway/ecosystem.config.js | 9 + main/mqtt-gateway/generate_signature_key.js | 81 +++ main/mqtt-gateway/mqtt-protocol.js | 499 ++++++++++++++ main/mqtt-gateway/utils/config-manager.js | 78 +++ main/mqtt-gateway/utils/mqtt_config_v2.js | 103 +++ ...13\350\257\225\350\257\264\346\230\216.md" | 239 +++++++ 12 files changed, 1899 insertions(+) create mode 100644 main/mqtt-gateway/.gitignore create mode 100644 main/mqtt-gateway/LICENSE create mode 100644 main/mqtt-gateway/README.md create mode 100644 main/mqtt-gateway/app.js create mode 100644 main/mqtt-gateway/config/mqtt.json.example create mode 100644 main/mqtt-gateway/config/mqtt.json.test create mode 100644 main/mqtt-gateway/ecosystem.config.js create mode 100644 main/mqtt-gateway/generate_signature_key.js create mode 100644 main/mqtt-gateway/mqtt-protocol.js create mode 100644 main/mqtt-gateway/utils/config-manager.js create mode 100644 main/mqtt-gateway/utils/mqtt_config_v2.js create mode 100644 "main/mqtt-gateway/\346\265\213\350\257\225\350\257\264\346\230\216.md" diff --git a/main/mqtt-gateway/.gitignore b/main/mqtt-gateway/.gitignore new file mode 100644 index 000000000..71b7ebc22 --- /dev/null +++ b/main/mqtt-gateway/.gitignore @@ -0,0 +1,4 @@ +node_modules/ +build/ +dist/ +package-lock.json diff --git a/main/mqtt-gateway/LICENSE b/main/mqtt-gateway/LICENSE new file mode 100644 index 000000000..b75f43388 --- /dev/null +++ b/main/mqtt-gateway/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2025 Xiaoxia + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. \ No newline at end of file diff --git a/main/mqtt-gateway/README.md b/main/mqtt-gateway/README.md new file mode 100644 index 000000000..bb591758c --- /dev/null +++ b/main/mqtt-gateway/README.md @@ -0,0 +1,215 @@ +# MQTT+UDP 到 WebSocket 桥接服务 + +## 项目概述 + +这是一个用于物联网设备通信的桥接服务，实现了MQTT和UDP协议到WebSocket的转换。该服务允许设备通过MQTT协议进行控制消息传输，同时通过UDP协议高效传输音频数据，并将这些数据桥接到WebSocket服务。 + +## 功能特点 + +- **多协议支持**: 同时支持MQTT、UDP和WebSocket协议 +- **音频数据传输**: 专为音频数据流优化的传输机制 +- **加密通信**: 使用AES-128-CTR加密UDP数据传输 +- **会话管理**: 完整的设备会话生命周期管理 +- **自动重连**: 连接断开时自动重连机制 +- **心跳检测**: 定期检查连接活跃状态 +- **开发/生产环境配置**: 支持不同环境的配置切换 + +## 技术架构 + +- **MQTT服务器**: 处理设备控制消息 +- **UDP服务器**: 处理高效的音频数据传输 +- **WebSocket客户端**: 连接到聊天服务器 +- **桥接层**: 在不同协议间转换和路由消息 + +## 项目结构 + +``` +├── app.js # 主应用入口 +├── mqtt-protocol.js # MQTT协议实现 +├── ecosystem.config.js # PM2配置文件 +├── package.json # 项目依赖 +├── .env # 环境变量配置 +├── utils/ +│ ├── config-manager.js # 配置管理工具 +│ ├── mqtt_config_v2.js # MQTT配置验证工具 +│ └── weixinAlert.js # 微信告警工具 +└── config/ # 配置文件目录 +``` + +## 依赖项 + +- **debug**: 调试日志输出 +- **dotenv**: 环境变量管理 +- **ws**: WebSocket客户端 +- **events**: Node.js 事件模块 + +## 安装要求 + +- Node.js 14.x 或更高版本 +- npm 或 yarn 包管理器 +- PM2 (用于生产环境部署) + +## 安装步骤 + +1. 克隆仓库 +```bash +git clone <仓库地址> +cd mqtt-websocket-bridge +``` + +2. 安装依赖 +```bash +npm install +``` + +3. 创建配置文件 +```bash +mkdir -p config +cp config/mqtt.json.example config/mqtt.json +``` + +4. 编辑配置文件 `config/mqtt.json`，设置适当的参数 + +## 配置说明 + +配置文件 `config/mqtt.json` 需要包含以下内容: + +```json +{ + "debug": false, + "development": { + "mac_addresss": ["aa:bb:cc:dd:ee:ff"], + "chat_servers": ["wss://dev-chat-server.example.com/ws"] + }, + "production": { + "chat_servers": ["wss://chat-server.example.com/ws"] + } +} +``` + +## 环境变量 + +创建 `.env` 文件并设置以下环境变量: + +``` +MQTT_PORT=1883 # MQTT服务器端口 +UDP_PORT=8884 # UDP服务器端口 +PUBLIC_IP=your-ip # 服务器公网IP +``` + +## 运行服务 + +### 开发环境 + +```bash +# 直接运行 +node app.js + +# 调试模式运行 +DEBUG=mqtt-server node app.js +``` + +### 生产环境 (使用PM2) + +```bash +# 安装PM2 +npm install -g pm2 + +# 启动服务 +pm2 start ecosystem.config.js + +# 查看日志 +pm2 logs xz-mqtt + +# 监控服务 +pm2 monit +``` + +服务将在以下端口启动: +- MQTT 服务器: 端口 1883 (可通过环境变量修改) +- UDP 服务器: 端口 8884 (可通过环境变量修改) + +## 协议说明 + +### 设备连接流程 + +1. 设备通过MQTT协议连接到服务器 +2. 设备发送 `hello` 消息，包含音频参数和特性 +3. 服务器创建WebSocket连接到聊天服务器 +4. 服务器返回UDP连接参数给设备 +5. 设备通过UDP发送音频数据 +6. 服务器将音频数据转发到WebSocket +7. WebSocket返回的控制消息通过MQTT发送给设备 + +### 消息格式 + +#### Hello 消息 (设备 -> 服务器) +```json +{ + "type": "hello", + "version": 3, + "audio_params": { ... }, + "features": { ... } +} +``` + +#### Hello 响应 (服务器 -> 设备) +```json +{ + "type": "hello", + "version": 3, + "session_id": "uuid", + "transport": "udp", + "udp": { + "server": "server-ip", + "port": 8884, + "encryption": "aes-128-ctr", + "key": "hex-encoded-key", + "nonce": "hex-encoded-nonce" + }, + "audio_params": { ... } +} +``` + +## 安全说明 + +- UDP通信使用AES-128-CTR加密 +- 每个会话使用唯一的加密密钥 +- 使用序列号防止重放攻击 +- 设备通过MAC地址进行身份验证 +- 支持设备分组和UUID验证 + +## 性能优化 + +- 使用预分配的缓冲区减少内存分配 +- UDP协议用于高效传输音频数据 +- 定期清理不活跃的连接 +- 连接数和活跃连接数监控 +- 支持多聊天服务器负载均衡 + +## 故障排除 + +- 检查设备MAC地址格式是否正确 +- 确保UDP端口在防火墙中开放 +- 启用调试模式查看详细日志 +- 检查配置文件中的聊天服务器地址是否正确 +- 验证设备认证信息是否正确 + +## 开发指南 + +### 添加新功能 + +1. 修改 `mqtt-protocol.js` 以支持新的MQTT功能 +2. 在 `MQTTConnection` 类中添加新的消息处理方法 +3. 更新配置管理器以支持新的配置选项 +4. 在 `WebSocketBridge` 类中添加新的WebSocket处理逻辑 + +### 调试技巧 + +```bash +# 启用所有调试输出 +DEBUG=* node app.js + +# 只启用MQTT服务器调试 +DEBUG=mqtt-server node app.js +``` diff --git a/main/mqtt-gateway/app.js b/main/mqtt-gateway/app.js new file mode 100644 index 000000000..aad3bc905 --- /dev/null +++ b/main/mqtt-gateway/app.js @@ -0,0 +1,623 @@ +// Description: MQTT+UDP 到 WebSocket 的桥接 +// Author: terrence@tenclass.com +// Date: 2025-03-12 + +require('dotenv').config(); +const net = require('net'); +const debugModule = require('debug'); +const debug = debugModule('mqtt-server'); +const crypto = require('crypto'); +const dgram = require('dgram'); +const Emitter = require('events'); +const WebSocket = require('ws'); +const { MQTTProtocol } = require('./mqtt-protocol'); +const { ConfigManager } = require('./utils/config-manager'); +const { validateMqttCredentials } = require('./utils/mqtt_config_v2'); + + +function setDebugEnabled(enabled) { + if (enabled) { + debugModule.enable('mqtt-server'); + } else { + debugModule.disable(); + } +} + +const configManager = new ConfigManager('mqtt.json'); +configManager.on('configChanged', (config) => { + setDebugEnabled(config.debug); +}); + +setDebugEnabled(configManager.get('debug')); + +class WebSocketBridge extends Emitter { + constructor(connection, protocolVersion, macAddress, uuid, userData) { + super(); + this.connection = connection; + this.macAddress = macAddress; + this.uuid = uuid; + this.userData = userData; + this.wsClient = null; + this.protocolVersion = protocolVersion; + this.deviceSaidGoodbye = false; + this.initializeChatServer(); + } + + initializeChatServer() { + const devMacAddresss = configManager.get('development')?.mac_addresss || []; + let chatServers; + if (devMacAddresss.includes(this.macAddress)) { + chatServers = configManager.get('development')?.chat_servers; + } else { + chatServers = configManager.get('production')?.chat_servers; + } + if (!chatServers) { + throw new Error(`未找到 ${this.macAddress} 的聊天服务器`); + } + this.chatServer = chatServers[Math.floor(Math.random() * chatServers.length)]; + } + + async connect(audio_params, features) { + return new Promise((resolve, reject) => { + const headers = { + 'device-id': this.macAddress, + 'protocol-version': '2', + 'authorization': `Bearer test-token` + }; + if (this.uuid) { + headers['client-id'] = this.uuid; + } + if (this.userData && this.userData.ip) { + headers['x-forwarded-for'] = this.userData.ip; + } + this.wsClient = new WebSocket(this.chatServer, { headers }); + + this.wsClient.on('open', () => { + this.sendJson({ + type: 'hello', + version: 2, + transport: 'websocket', + audio_params, + features + }); + }); + + this.wsClient.on('message', (data, isBinary) => { + if (isBinary) { + const timestamp = data.readUInt32BE(8); + const opusLength = data.readUInt32BE(12); + const opus = data.subarray(16, 16 + opusLength); + // 二进制数据通过UDP发送 + this.connection.sendUdpMessage(opus, timestamp); + } else { + // JSON数据通过MQTT发送 + const message = JSON.parse(data.toString()); + if (message.type === 'hello') { + resolve(message); + } else { + this.connection.sendMqttMessage(JSON.stringify(message)); + } + } + }); + + this.wsClient.on('error', (error) => { + console.error(`WebSocket error for device ${this.macAddress}:`, error); + this.emit('close'); + reject(error); + }); + + this.wsClient.on('close', () => { + this.emit('close'); + }); + }); + } + + sendJson(message) { + if (this.wsClient && this.wsClient.readyState === WebSocket.OPEN) { + this.wsClient.send(JSON.stringify(message)); + } + } + + sendAudio(opus, timestamp) { + if (this.wsClient && this.wsClient.readyState === WebSocket.OPEN) { + const buffer = Buffer.alloc(16 + opus.length); + buffer.writeUInt32BE(timestamp, 8); + buffer.writeUInt32BE(opus.length, 12); + buffer.set(opus, 16); + this.wsClient.send(buffer, { binary: true }); + } + } + + isAlive() { + return this.wsClient && this.wsClient.readyState === WebSocket.OPEN; + } + + close() { + if (this.wsClient) { + this.wsClient.close(); + this.wsClient = null; + } + } +} + +const MacAddressRegex = /^[0-9a-f]{2}(:[0-9a-f]{2}){5}$/; + +/** + * MQTT连接类 + * 负责应用层逻辑处理 + */ +class MQTTConnection { + constructor(socket, connectionId, server) { + this.server = server; + this.connectionId = connectionId; + this.clientId = null; + this.username = null; + this.password = null; + this.bridge = null; + this.udp = { + remoteAddress: null, + cookie: null, + localSequence: 0, + remoteSequence: 0 + }; + this.headerBuffer = Buffer.alloc(16); + + // 创建协议处理器，并传入socket + this.protocol = new MQTTProtocol(socket); + + this.setupProtocolHandlers(); + } + + setupProtocolHandlers() { + // 设置协议事件处理 + this.protocol.on('connect', (connectData) => { + this.handleConnect(connectData); + }); + + this.protocol.on('publish', (publishData) => { + this.handlePublish(publishData); + }); + + this.protocol.on('subscribe', (subscribeData) => { + this.handleSubscribe(subscribeData); + }); + + this.protocol.on('disconnect', () => { + this.handleDisconnect(); + }); + + this.protocol.on('close', () => { + debug(`${this.clientId} 客户端断开连接`); + this.server.removeConnection(this); + }); + + this.protocol.on('error', (err) => { + debug(`${this.clientId} 连接错误:`, err); + this.close(); + }); + + this.protocol.on('protocolError', (err) => { + debug(`${this.clientId} 协议错误:`, err); + this.close(); + }); + } + + handleConnect(connectData) { + this.clientId = connectData.clientId; + this.username = connectData.username; + this.password = connectData.password; + + debug('客户端连接:', { + clientId: this.clientId, + username: this.username, + password: this.password, + protocol: connectData.protocol, + protocolLevel: connectData.protocolLevel, + keepAlive: connectData.keepAlive + }); + + const parts = this.clientId.split('@@@'); + if (parts.length === 3) { // GID_test@@@mac_address@@@uuid + try { + const validated = validateMqttCredentials(this.clientId, this.username, this.password); + this.groupId = validated.groupId; + this.macAddress = validated.macAddress; + this.uuid = validated.uuid; + this.userData = validated.userData; + } catch (error) { + debug('MQTT凭据验证失败:', error.message); + this.close(); + return; + } + } else if (parts.length === 2) { // GID_test@@@mac_address + this.groupId = parts[0]; + this.macAddress = parts[1].replace(/_/g, ':'); + if (!MacAddressRegex.test(this.macAddress)) { + debug('无效的 macAddress:', this.macAddress); + this.close(); + return; + } + } else { + debug('无效的 clientId:', this.clientId); + this.close(); + return; + } + this.replyTo = `devices/p2p/${parts[1]}`; + + this.server.addConnection(this); + } + + handleSubscribe(subscribeData) { + debug('客户端订阅主题:', { + clientId: this.clientId, + topic: subscribeData.topic, + packetId: subscribeData.packetId + }); + + // 发送 SUBACK + this.protocol.sendSuback(subscribeData.packetId, 0); + } + + handleDisconnect() { + debug('收到断开连接请求:', { clientId: this.clientId }); + // 清理连接 + this.server.removeConnection(this); + } + + close() { + this.closing = true; + if (this.bridge) { + this.bridge.close(); + this.bridge = null; + } else { + this.protocol.close(); + } + } + + checkKeepAlive() { + const now = Date.now(); + const keepAliveInterval = this.protocol.getKeepAliveInterval(); + + // 如果keepAliveInterval为0，表示不需要心跳检查 + if (keepAliveInterval === 0 || !this.protocol.isConnected) return; + + const lastActivity = this.protocol.getLastActivity(); + const timeSinceLastActivity = now - lastActivity; + + // 如果超过心跳间隔，关闭连接 + if (timeSinceLastActivity > keepAliveInterval) { + debug('心跳超时，关闭连接:', this.clientId); + this.close(); + } + } + + handlePublish(publishData) { + debug('收到发布消息:', { + clientId: this.clientId, + topic: publishData.topic, + payload: publishData.payload, + qos: publishData.qos + }); + + if (publishData.qos !== 0) { + debug('不支持的 QoS 级别:', publishData.qos, '关闭连接'); + this.close(); + return; + } + + const json = JSON.parse(publishData.payload); + if (json.type === 'hello') { + if (json.version !== 3) { + debug('不支持的协议版本:', json.version, '关闭连接'); + this.close(); + return; + } + this.parseHelloMessage(json).catch(error => { + debug('处理 hello 消息失败:', error); + this.close(); + }); + } else { + this.parseOtherMessage(json).catch(error => { + debug('处理其他消息失败:', error); + this.close(); + }); + } + } + + sendMqttMessage(payload) { + debug(`发送消息到 ${this.replyTo}: ${payload}`); + this.protocol.sendPublish(this.replyTo, payload, 0, false, false); + } + + sendUdpMessage(payload, timestamp) { + if (!this.udp.remoteAddress) { + debug(`设备 ${this.clientId} 未连接，无法发送 UDP 消息`); + return; + } + this.udp.localSequence++; + const header = this.generateUdpHeader(payload.length, timestamp, this.udp.localSequence); + const cipher = crypto.createCipheriv(this.udp.encryption, this.udp.key, header); + const message = Buffer.concat([header, cipher.update(payload), cipher.final()]); + this.server.sendUdpMessage(message, this.udp.remoteAddress); + } + + generateUdpHeader(length, timestamp, sequence) { + // 重用预分配的缓冲区 + this.headerBuffer.writeUInt8(1, 0); + this.headerBuffer.writeUInt16BE(length, 2); + this.headerBuffer.writeUInt32BE(this.connectionId, 4); + this.headerBuffer.writeUInt32BE(timestamp, 8); + this.headerBuffer.writeUInt32BE(sequence, 12); + return Buffer.from(this.headerBuffer); // 返回副本以避免并发问题 + } + + async parseHelloMessage(json) { + this.udp = { + ...this.udp, + key: crypto.randomBytes(16), + nonce: this.generateUdpHeader(0, 0, 0), + encryption: 'aes-128-ctr', + remoteSequence: 0, + localSequence: 0, + startTime: Date.now() + } + + if (this.bridge) { + debug(`${this.clientId} 收到重复 hello 消息，关闭之前的 bridge`); + this.bridge.close(); + await new Promise(resolve => setTimeout(resolve, 100)); + } + this.bridge = new WebSocketBridge(this, json.version, this.macAddress, this.uuid, this.userData); + this.bridge.on('close', () => { + const seconds = (Date.now() - this.udp.startTime) / 1000; + console.log(`通话结束: ${this.clientId} Session: ${this.udp.session_id} Duration: ${seconds}s`); + this.sendMqttMessage(JSON.stringify({ type: 'goodbye', session_id: this.udp.session_id })); + this.bridge = null; + if (this.closing) { + this.protocol.close(); + } + }); + + try { + console.log(`通话开始: ${this.clientId} Protocol: ${json.version} ${this.bridge.chatServer}`); + const helloReply = await this.bridge.connect(json.audio_params, json.features); + this.udp.session_id = helloReply.session_id; + this.sendMqttMessage(JSON.stringify({ + type: 'hello', + version: json.version, + session_id: this.udp.session_id, + transport: 'udp', + udp: { + server: this.server.publicIp, + port: this.server.udpPort, + encryption: this.udp.encryption, + key: this.udp.key.toString('hex'), + nonce: this.udp.nonce.toString('hex'), + }, + audio_params: helloReply.audio_params + })); + } catch (error) { + this.sendMqttMessage(JSON.stringify({ type: 'error', message: '处理 hello 消息失败' })); + console.error(`${this.clientId} 处理 hello 消息失败: ${error}`); + } + } + + async parseOtherMessage(json) { + if (!this.bridge) { + if (json.type !== 'goodbye') { + this.sendMqttMessage(JSON.stringify({ type: 'goodbye', session_id: json.session_id })); + } + return; + } + + if (json.type === 'goodbye') { + this.bridge.close(); + this.bridge = null; + return; + } + + this.bridge.sendJson(json); + } + + onUdpMessage(rinfo, message, payloadLength, timestamp, sequence) { + if (!this.bridge) { + return; + } + if (this.udp.remoteAddress !== rinfo) { + this.udp.remoteAddress = rinfo; + } + if (sequence < this.udp.remoteSequence) { + return; + } + + // 处理加密数据 + const header = message.slice(0, 16); + const encryptedPayload = message.slice(16, 16 + payloadLength); + const cipher = crypto.createDecipheriv(this.udp.encryption, this.udp.key, header); + const payload = Buffer.concat([cipher.update(encryptedPayload), cipher.final()]); + + this.bridge.sendAudio(payload, timestamp); + this.udp.remoteSequence = sequence; + } + + isAlive() { + return this.bridge && this.bridge.isAlive(); + } +} + +class MQTTServer { + constructor() { + this.mqttPort = parseInt(process.env.MQTT_PORT) || 1883; + this.udpPort = parseInt(process.env.UDP_PORT) || this.mqttPort; + this.publicIp = process.env.PUBLIC_IP || 'mqtt.xiaozhi.me'; + this.connections = new Map(); // clientId -> MQTTConnection + this.keepAliveTimer = null; + this.keepAliveCheckInterval = 1000; // 默认每1秒检查一次 + + this.headerBuffer = Buffer.alloc(16); + } + + generateNewConnectionId() { + // 生成一个32位不重复的整数 + let id; + do { + id = Math.floor(Math.random() * 0xFFFFFFFF); + } while (this.connections.has(id)); + return id; + } + + start() { + this.mqttServer = net.createServer((socket) => { + const connectionId = this.generateNewConnectionId(); + debug(`新客户端连接: ${connectionId}`); + new MQTTConnection(socket, connectionId, this); + }); + + this.mqttServer.listen(this.mqttPort, () => { + console.warn(`MQTT 服务器正在监听端口 ${this.mqttPort}`); + }); + + + this.udpServer = dgram.createSocket('udp4'); + this.udpServer.on('message', this.onUdpMessage.bind(this)); + this.udpServer.on('error', err => { + console.error('UDP 错误', err); + setTimeout(() => { process.exit(1); }, 1000); + }); + this.udpServer.bind(this.udpPort, () => { + console.warn(`UDP 服务器正在监听 ${this.publicIp}:${this.udpPort}`); + }); + + // 启动全局心跳检查定时器 + this.setupKeepAliveTimer(); + } + + /** + * 设置全局心跳检查定时器 + */ + setupKeepAliveTimer() { + // 清除现有定时器 + this.clearKeepAliveTimer(); + this.lastConnectionCount = 0; + this.lastActiveConnectionCount = 0; + + // 设置新的定时器 + this.keepAliveTimer = setInterval(() => { + // 检查所有连接的心跳状态 + for (const connection of this.connections.values()) { + connection.checkKeepAlive(); + } + + const activeCount = Array.from(this.connections.values()).filter(connection => connection.isAlive()).length; + if (activeCount !== this.lastActiveConnectionCount || this.connections.size !== this.lastConnectionCount) { + console.log(`连接数: ${this.connections.size}, 活跃数: ${activeCount}`); + this.lastActiveConnectionCount = activeCount; + this.lastConnectionCount = this.connections.size; + } + }, this.keepAliveCheckInterval); + } + + /** + * 清除心跳检查定时器 + */ + clearKeepAliveTimer() { + if (this.keepAliveTimer) { + clearInterval(this.keepAliveTimer); + this.keepAliveTimer = null; + } + } + + addConnection(connection) { + // 检查是否已存在相同 clientId 的连接 + for (const [key, value] of this.connections.entries()) { + if (value.clientId === connection.clientId) { + debug(`${connection.clientId} 已存在连接，关闭旧连接`); + value.close(); + } + } + this.connections.set(connection.connectionId, connection); + } + + removeConnection(connection) { + debug(`关闭连接: ${connection.connectionId}`); + if (this.connections.has(connection.connectionId)) { + this.connections.delete(connection.connectionId); + } + } + + sendUdpMessage(message, remoteAddress) { + this.udpServer.send(message, remoteAddress.port, remoteAddress.address); + } + + onUdpMessage(message, rinfo) { + // message format: [type: 1u, flag: 1u, payloadLength: 2u, cookie: 4u, timestamp: 4u, sequence: 4u, payload: n] + if (message.length < 16) { + console.warn('收到不完整的 UDP Header', rinfo); + return; + } + + try { + const type = message.readUInt8(0); + if (type !== 1) return; + + const payloadLength = message.readUInt16BE(2); + if (message.length < 16 + payloadLength) return; + + const connectionId = message.readUInt32BE(4); + const connection = this.connections.get(connectionId); + if (!connection) return; + + const timestamp = message.readUInt32BE(8); + const sequence = message.readUInt32BE(12); + + connection.onUdpMessage(rinfo, message, payloadLength, timestamp, sequence); + } catch (error) { + console.error('UDP 消息处理错误:', error); + } + } + + /** + * 停止服务器 + */ + async stop() { + if (this.stopping) { + return; + } + this.stopping = true; + // 清除心跳检查定时器 + this.clearKeepAliveTimer(); + + if (this.connections.size > 0) { + console.warn(`等待 ${this.connections.size} 个连接关闭`); + for (const connection of this.connections.values()) { + connection.close(); + } + await new Promise(resolve => setTimeout(resolve, 300)); + debug('等待连接关闭完成'); + this.connections.clear(); + } + + if (this.udpServer) { + this.udpServer.close(); + this.udpServer = null; + console.warn('UDP 服务器已停止'); + } + + // 关闭MQTT服务器 + if (this.mqttServer) { + this.mqttServer.close(); + this.mqttServer = null; + console.warn('MQTT 服务器已停止'); + } + + process.exit(0); + } +} + +// 创建并启动服务器 +const server = new MQTTServer(); +server.start(); +process.on('SIGINT', () => { + console.warn('收到 SIGINT 信号，开始关闭'); + server.stop(); +}); diff --git a/main/mqtt-gateway/config/mqtt.json.example b/main/mqtt-gateway/config/mqtt.json.example new file mode 100644 index 000000000..05c6b9b58 --- /dev/null +++ b/main/mqtt-gateway/config/mqtt.json.example @@ -0,0 +1,15 @@ +{ + "production": { + "chat_servers": [ + "ws://example:8080" + ] + }, + "development": { + "chat_servers": [ + "ws://example:8180" + ], + "mac_addresss": [ + ] + }, + "debug": false +} diff --git a/main/mqtt-gateway/config/mqtt.json.test b/main/mqtt-gateway/config/mqtt.json.test new file mode 100644 index 000000000..d4680a20a --- /dev/null +++ b/main/mqtt-gateway/config/mqtt.json.test @@ -0,0 +1,12 @@ +{ + "production": { + "chat_servers": [ + "ws://192.168.68.66:8000/xiaozhi/v1/" + ] + }, + "development": { + "chat_servers": ["ws://192.168.68.66:8000/xiaozhi/v1/"], + "mac_addresss": ["d8:43:ae:3e:4b:5a"] + }, + "debug": false +} diff --git a/main/mqtt-gateway/ecosystem.config.js b/main/mqtt-gateway/ecosystem.config.js new file mode 100644 index 000000000..a3d9edb47 --- /dev/null +++ b/main/mqtt-gateway/ecosystem.config.js @@ -0,0 +1,9 @@ +module.exports = { + "apps": [ + { + "name": "xz-mqtt", + "script": "app.js", + "time": true + } + ] +} diff --git a/main/mqtt-gateway/generate_signature_key.js b/main/mqtt-gateway/generate_signature_key.js new file mode 100644 index 000000000..89aa9a266 --- /dev/null +++ b/main/mqtt-gateway/generate_signature_key.js @@ -0,0 +1,81 @@ +#!/usr/bin/env node +/** + * MQTT签名密钥生成器 + * 用于生成MQTT_SIGNATURE_KEY环境变量 + */ + +const crypto = require('crypto'); + +function generateSecureKey(length = 32) { + // 生成随机字节 + const randomBytes = crypto.randomBytes(length); + + // 转换为base64字符串 + const base64Key = randomBytes.toString('base64'); + + return base64Key; +} + +function generateHexKey(length = 32) { + // 生成随机字节 + const randomBytes = crypto.randomBytes(length); + + // 转换为十六进制字符串 + const hexKey = randomBytes.toString('hex'); + + return hexKey; +} + +function generateUUIDKey() { + // 使用UUID v4作为密钥 + const uuid = crypto.randomUUID(); + return uuid; +} + +console.log('='.repeat(60)); +console.log('MQTT签名密钥生成器'); +console.log('='.repeat(60)); + +console.log('\n1. Base64格式密钥 (推荐):'); +const base64Key = generateSecureKey(); +console.log(` ${base64Key}`); + +console.log('\n2. 十六进制格式密钥:'); +const hexKey = generateHexKey(); +console.log(` ${hexKey}`); + +console.log('\n3. UUID格式密钥:'); +const uuidKey = generateUUIDKey(); +console.log(` ${uuidKey}`); + +console.log('\n='.repeat(60)); +console.log('使用方法:'); +console.log('='.repeat(60)); +console.log('\n在Windows PowerShell中设置环境变量:'); +console.log(`$env:MQTT_SIGNATURE_KEY="${base64Key}"`); + +console.log('\n在Windows CMD中设置环境变量:'); +console.log(`set MQTT_SIGNATURE_KEY=${base64Key}`); + +console.log('\n在Linux/macOS中设置环境变量:'); +console.log(`export MQTT_SIGNATURE_KEY="${base64Key}"`); + +console.log('\n在.env文件中设置:'); +console.log(`MQTT_SIGNATURE_KEY=${base64Key}`); + +console.log('\n='.repeat(60)); +console.log('注意事项:'); +console.log('='.repeat(60)); +console.log('1. 请妥善保管生成的密钥，不要泄露给他人'); +console.log('2. 在生产环境中，建议使用更长的密钥 (64字节)'); +console.log('3. 密钥设置后需要重启MQTT服务器才能生效'); +console.log('4. 客户端连接时需要使用相同的密钥生成密码签名'); + +// 如果提供了命令行参数，生成指定长度的密钥 +if (process.argv[2]) { + const customLength = parseInt(process.argv[2]); + if (customLength > 0) { + console.log(`\n自定义长度密钥 (${customLength}字节):`); + console.log(` ${generateSecureKey(customLength)}`); + } +} \ No newline at end of file diff --git a/main/mqtt-gateway/mqtt-protocol.js b/main/mqtt-gateway/mqtt-protocol.js new file mode 100644 index 000000000..09ba8a20f --- /dev/null +++ b/main/mqtt-gateway/mqtt-protocol.js @@ -0,0 +1,499 @@ +const debug = require('debug')('mqtt-server'); +const EventEmitter = require('events'); + +// MQTT 固定头部的类型 +const PacketType = { + CONNECT: 1, + CONNACK: 2, + PUBLISH: 3, + SUBSCRIBE: 8, + SUBACK: 9, + PINGREQ: 12, + PINGRESP: 13, + DISCONNECT: 14 // 添加 DISCONNECT +}; + +/** + * MQTT协议处理类 + * 负责MQTT协议的解析和封装，以及心跳维持 + */ +class MQTTProtocol extends EventEmitter { + constructor(socket) { + super(); + this.socket = socket; + this.buffer = Buffer.alloc(0); + this.isConnected = false; + this.keepAliveInterval = 0; + this.lastActivity = Date.now(); + + this.setupSocketHandlers(); + } + + /** + * 设置Socket事件处理 + */ + setupSocketHandlers() { + this.socket.on('data', (data) => { + this.lastActivity = Date.now(); + this.buffer = Buffer.concat([this.buffer, data]); + this.processBuffer(); + }); + + this.socket.on('close', () => { + this.emit('close'); + }); + + this.socket.on('error', (err) => { + this.emit('error', err); + }); + } + + /** + * 处理缓冲区中的所有完整消息 + */ + processBuffer() { + // 持续处理缓冲区中的数据，直到没有完整的消息可以处理 + while (this.buffer.length > 0) { + // 至少需要2个字节才能开始解析（1字节固定头部 + 至少1字节的剩余长度） + if (this.buffer.length < 2) return; + + try { + // 获取消息类型 + const firstByte = this.buffer[0]; + const type = (firstByte >> 4); + + // 解析剩余长度 + const { value: remainingLength, bytesRead } = this.decodeRemainingLength(this.buffer); + + // 计算整个消息的长度 + const messageLength = 1 + bytesRead + remainingLength; + + // 检查缓冲区中是否有完整的消息 + if (this.buffer.length < messageLength) { + // 消息不完整，等待更多数据 + return; + } + + // 提取完整的消息 + const message = this.buffer.subarray(0, messageLength); + if (!this.isConnected && type !== PacketType.CONNECT) { + debug('未连接时收到非CONNECT消息，关闭连接'); + this.socket.end(); + return; + } + + // 根据消息类型处理 + switch (type) { + case PacketType.CONNECT: + this.parseConnect(message); + break; + case PacketType.PUBLISH: + this.parsePublish(message); + break; + case PacketType.SUBSCRIBE: + this.parseSubscribe(message); + break; + case PacketType.PINGREQ: + this.parsePingReq(message); + break; + case PacketType.DISCONNECT: + this.parseDisconnect(message); + break; + default: + debug('未处理的包类型:', type, message); + this.emit('protocolError', new Error(`未处理的包类型: ${type}`)); + } + + // 从缓冲区中移除已处理的消息 + this.buffer = this.buffer.subarray(messageLength); + + } catch (err) { + // 如果解析出错，可能是数据不完整，等待更多数据 + if (err.message === 'Malformed Remaining Length') { + return; + } + // 其他错误可能是协议错误，清空缓冲区并发出错误事件 + this.buffer = Buffer.alloc(0); + this.emit('protocolError', err); + return; + } + } + } + + /** + * 解析MQTT报文中的Remaining Length字段 + * @param {Buffer} buffer - 消息缓冲区 + * @returns {{value: number, bytesRead: number}} 包含解析的值和读取的字节数 + */ + decodeRemainingLength(buffer) { + let multiplier = 1; + let value = 0; + let bytesRead = 0; + let digit; + + do { + if (bytesRead >= 4 || bytesRead >= buffer.length - 1) { + throw new Error('Malformed Remaining Length'); + } + + digit = buffer[bytesRead + 1]; + bytesRead++; + + value += (digit & 127) * multiplier; + multiplier *= 128; + + } while ((digit & 128) !== 0); + + return { value, bytesRead }; + } + + /** + * 编码MQTT报文中的Remaining Length字段 + * @param {number} length - 要编码的长度值 + * @returns {{bytes: Buffer, bytesLength: number}} 包含编码后的字节和字节长度 + */ + encodeRemainingLength(length) { + let digit; + const bytes = Buffer.alloc(4); // 最多4个字节 + let bytesLength = 0; + + do { + digit = length % 128; + length = Math.floor(length / 128); + // 如果还有更多字节，设置最高位 + if (length > 0) { + digit |= 0x80; + } + bytes[bytesLength++] = digit; + } while (length > 0 && bytesLength < 4); + + return { bytes, bytesLength }; + } + + /** + * 解析CONNECT消息 + * @param {Buffer} message - 完整的CONNECT消息 + */ + parseConnect(message) { + // 解析剩余长度 + const { value: remainingLength, bytesRead } = this.decodeRemainingLength(message); + + // 固定头部之后的位置 (MQTT固定头部第一个字节 + Remaining Length字段的字节) + const headerLength = 1 + bytesRead; + + // 从可变头部开始位置读取协议名长度 + const protocolLength = message.readUInt16BE(headerLength); + const protocol = message.toString('utf8', headerLength + 2, headerLength + 2 + protocolLength); + + // 更新位置指针，跳过协议名 + let pos = headerLength + 2 + protocolLength; + + // 协议级别，4为MQTT 3.1.1 + const protocolLevel = message[pos]; + + // 检查协议版本 + if (protocolLevel !== 4) { // 4 表示 MQTT 3.1.1 + debug('不支持的协议版本:', protocolLevel); + // 发送 CONNACK，使用不支持的协议版本的返回码 (0x01) + this.sendConnack(1, false); + // 关闭连接 + this.socket.end(); + return; + } + + pos += 1; + + // 连接标志 + const connectFlags = message[pos]; + const hasUsername = (connectFlags & 0x80) !== 0; + const hasPassword = (connectFlags & 0x40) !== 0; + const cleanSession = (connectFlags & 0x02) !== 0; + pos += 1; + + // 保持连接时间 + const keepAlive = message.readUInt16BE(pos); + pos += 2; + + // 解析 clientId + const clientIdLength = message.readUInt16BE(pos); + pos += 2; + const clientId = message.toString('utf8', pos, pos + clientIdLength); + pos += clientIdLength; + + // 解析 username（如果存在） + let username = ''; + if (hasUsername) { + const usernameLength = message.readUInt16BE(pos); + pos += 2; + username = message.toString('utf8', pos, pos + usernameLength); + pos += usernameLength; + } + + // 解析 password（如果存在） + let password = ''; + if (hasPassword) { + const passwordLength = message.readUInt16BE(pos); + pos += 2; + password = message.toString('utf8', pos, pos + passwordLength); + pos += passwordLength; + } + + // 设置心跳间隔（客户端指定的keepAlive值的1.5倍，单位为秒） + this.keepAliveInterval = keepAlive * 1000 * 1.5; + + // 发送 CONNACK + this.sendConnack(0, false); + + // 标记为已连接 + this.isConnected = true; + + // 发出连接事件 + this.emit('connect', { + clientId, + protocol, + protocolLevel, + keepAlive, + username, + password, + cleanSession + }); + } + + /** + * 解析PUBLISH消息 + * @param {Buffer} message - 完整的PUBLISH消息 + */ + parsePublish(message) { + // 从第一个字节中提取QoS级别(bits 1-2) + const firstByte = message[0]; + const qos = (firstByte & 0x06) >> 1; // 0x06 是二进制 00000110，用于掩码提取QoS位 + const dup = (firstByte & 0x08) !== 0; // 0x08 是二进制 00001000，用于掩码提取DUP标志 + const retain = (firstByte & 0x01) !== 0; // 0x01 是二进制 00000001，用于掩码提取RETAIN标志 + + // 使用通用方法解析剩余长度 + const { value: remainingLength, bytesRead } = this.decodeRemainingLength(message); + + // 固定头部之后的位置 (MQTT固定头部第一个字节 + Remaining Length字段的字节) + const headerLength = 1 + bytesRead; + + // 解析主题 + const topicLength = message.readUInt16BE(headerLength); + const topic = message.toString('utf8', headerLength + 2, headerLength + 2 + topicLength); + + // 对于QoS > 0，包含消息ID + let packetId = null; + let payloadStart = headerLength + 2 + topicLength; + + if (qos > 0) { + packetId = message.readUInt16BE(payloadStart); + payloadStart += 2; + } + + // 解析有效载荷 + const payload = message.slice(payloadStart).toString('utf8'); + + // 发出发布事件 + this.emit('publish', { + topic, + payload, + qos, + dup, + retain, + packetId + }); + } + + /** + * 解析SUBSCRIBE消息 + * @param {Buffer} message - 完整的SUBSCRIBE消息 + */ + parseSubscribe(message) { + const packetId = message.readUInt16BE(2); + const topicLength = message.readUInt16BE(4); + const topic = message.toString('utf8', 6, 6 + topicLength); + const qos = message[6 + topicLength]; // QoS值 + + // 发出订阅事件 + this.emit('subscribe', { + packetId, + topic, + qos + }); + } + + /** + * 解析PINGREQ消息 + * @param {Buffer} message - 完整的PINGREQ消息 + */ + parsePingReq(message) { + debug('收到心跳请求'); + + // 发送 PINGRESP + this.sendPingResp(); + + debug('已发送心跳响应'); + } + + /** + * 解析DISCONNECT消息 + * @param {Buffer} message - 完整的DISCONNECT消息 + */ + parseDisconnect(message) { + // 标记为未连接 + this.isConnected = false; + + // 发出断开连接事件 + this.emit('disconnect'); + + // 关闭 socket + this.socket.end(); + } + + /** + * 发送CONNACK消息 + * @param {number} returnCode - 返回码 + * @param {boolean} sessionPresent - 会话存在标志 + */ + sendConnack(returnCode = 0, sessionPresent = false) { + if (!this.socket.writable) return; + + const packet = Buffer.from([ + PacketType.CONNACK << 4, + 2, // Remaining length + sessionPresent ? 1 : 0, // Connect acknowledge flags + returnCode // Return code + ]); + + this.socket.write(packet); + } + + /** + * 发送PUBLISH消息 + * @param {string} topic - 主题 + * @param {string} payload - 有效载荷 + * @param {number} qos - QoS级别 + * @param {boolean} dup - 重复标志 + * @param {boolean} retain - 保留标志 + * @param {number} packetId - 包ID（仅QoS > 0时需要） + */ + sendPublish(topic, payload, qos = 0, dup = false, retain = false, packetId = null) { + if (!this.isConnected || !this.socket.writable) return; + + const topicLength = Buffer.byteLength(topic); + const payloadLength = Buffer.byteLength(payload); + + // 计算剩余长度 + let remainingLength = 2 + topicLength + payloadLength; + + // 如果QoS > 0，需要包含包ID + if (qos > 0 && packetId) { + remainingLength += 2; + } + + // 编码可变长度 + const { bytes: remainingLengthBytes, bytesLength: remainingLengthSize } = this.encodeRemainingLength(remainingLength); + + // 分配缓冲区：固定头部(1字节) + 可变长度字段 + 剩余长度值 + const packet = Buffer.alloc(1 + remainingLengthSize + remainingLength); + + // 写入固定头部 + let firstByte = PacketType.PUBLISH << 4; + if (dup) firstByte |= 0x08; + if (qos > 0) firstByte |= (qos << 1); + if (retain) firstByte |= 0x01; + + packet[0] = firstByte; + + // 写入可变长度字段 + remainingLengthBytes.copy(packet, 1, 0, remainingLengthSize); + + // 写入主题长度和主题 + const variableHeaderStart = 1 + remainingLengthSize; + packet.writeUInt16BE(topicLength, variableHeaderStart); + packet.write(topic, variableHeaderStart + 2); + + // 如果QoS > 0，写入包ID + let payloadStart = variableHeaderStart + 2 + topicLength; + if (qos > 0 && packetId) { + packet.writeUInt16BE(packetId, payloadStart); + payloadStart += 2; + } + + // 写入有效载荷 + packet.write(payload, payloadStart); + + this.socket.write(packet); + this.lastActivity = Date.now(); + } + + /** + * 发送SUBACK消息 + * @param {number} packetId - 包ID + * @param {number} returnCode - 返回码 + */ + sendSuback(packetId, returnCode = 0) { + if (!this.isConnected || !this.socket.writable) return; + + const packet = Buffer.from([ + PacketType.SUBACK << 4, + 3, // Remaining length + packetId >> 8, // Packet ID MSB + packetId & 0xFF, // Packet ID LSB + returnCode // Return code + ]); + + this.socket.write(packet); + this.lastActivity = Date.now(); + } + + /** + * 发送PINGRESP消息 + */ + sendPingResp() { + if (!this.isConnected || !this.socket.writable) return; + + const packet = Buffer.from([ + PacketType.PINGRESP << 4, // Fixed header + 0 // Remaining length + ]); + + this.socket.write(packet); + this.lastActivity = Date.now(); + } + + /** + * 获取上次活动时间 + */ + getLastActivity() { + return this.lastActivity; + } + + /** + * 获取心跳间隔 + */ + getKeepAliveInterval() { + return this.keepAliveInterval; + } + + /** + * 清空缓冲区 + */ + clearBuffer() { + this.buffer = Buffer.alloc(0); + } + + /** + * 关闭连接 + */ + close() { + if (this.socket.writable) { + this.socket.end(); + } + } +} + +// 导出 PacketType 和 MQTTProtocol 类 +module.exports = { + PacketType, + MQTTProtocol +}; \ No newline at end of file diff --git a/main/mqtt-gateway/utils/config-manager.js b/main/mqtt-gateway/utils/config-manager.js new file mode 100644 index 000000000..59d3c6614 --- /dev/null +++ b/main/mqtt-gateway/utils/config-manager.js @@ -0,0 +1,78 @@ +const fs = require('fs'); +const path = require('path'); +const EventEmitter = require('events'); + +class ConfigManager extends EventEmitter { + constructor(fileName) { + super(); + this.config = {}; // 移除默认的 apiKeys 配置 + this.configPath = path.join(__dirname, "..", "config", fileName); + this.loadConfig(); + this.watchConfig(); + // 添加防抖计时器变量 + this.watchDebounceTimer = null; + } + + loadConfig() { + try { + const data = fs.readFileSync(this.configPath, 'utf8'); + const newConfig = JSON.parse(data); + + // 检测配置是否发生变化 + if (JSON.stringify(this.config) !== JSON.stringify(newConfig)) { + console.log('配置已更新', this.configPath); + this.config = newConfig; + // 发出配置更新事件 + this.emit('configChanged', this.config); + } + } catch (error) { + console.error('加载配置出错:', error, this.configPath); + if (error.code === 'ENOENT') { + this.createEmptyConfig(); + } + } + } + + createEmptyConfig() { + try { + const dir = path.dirname(this.configPath); + if (!fs.existsSync(dir)) { + fs.mkdirSync(dir, { recursive: true }); + } + const defaultConfig = {}; // 空配置对象 + fs.writeFileSync(this.configPath, JSON.stringify(defaultConfig, null, 2)); + console.log('已创建空配置文件', this.configPath); + } catch (error) { + console.error('创建空配置文件出错:', error, this.configPath); + } + } + + watchConfig() { + fs.watch(path.dirname(this.configPath), (eventType, filename) => { + if (filename === path.basename(this.configPath) && eventType === 'change') { + // 清除之前的计时器 + if (this.watchDebounceTimer) { + clearTimeout(this.watchDebounceTimer); + } + // 设置新的计时器，300ms 后执行 + this.watchDebounceTimer = setTimeout(() => { + this.loadConfig(); + }, 300); + } + }); + } + + // 获取配置的方法 + getConfig() { + return this.config; + } + + // 获取特定配置项的方法 + get(key) { + return this.config[key]; + } +} + +module.exports = { + ConfigManager +}; \ No newline at end of file diff --git a/main/mqtt-gateway/utils/mqtt_config_v2.js b/main/mqtt-gateway/utils/mqtt_config_v2.js new file mode 100644 index 000000000..0fecd459f --- /dev/null +++ b/main/mqtt-gateway/utils/mqtt_config_v2.js @@ -0,0 +1,103 @@ +require('dotenv').config(); +const crypto = require('crypto'); + + +function generatePasswordSignature(content, secretKey) { + // Create an HMAC object using SHA256 and the secretKey + const hmac = crypto.createHmac('sha256', secretKey); + + // Update the HMAC object with the clientId + hmac.update(content); + + // Generate the HMAC digest in binary format + const binarySignature = hmac.digest(); + + // Encode the binary signature to Base64 + const base64Signature = binarySignature.toString('base64'); + + return base64Signature; +} + +function validateMqttCredentials(clientId, username, password) { + // 验证密码签名 + const signatureKey = process.env.MQTT_SIGNATURE_KEY; + if (signatureKey) { + const expectedSignature = generatePasswordSignature(clientId + '|' + username, signatureKey); + if (password !== expectedSignature) { + throw new Error('密码签名验证失败'); + } + } else { + console.warn('缺少MQTT_SIGNATURE_KEY环境变量，跳过密码签名验证'); + } + + // 验证clientId + if (!clientId || typeof clientId !== 'string') { + throw new Error('clientId必须是非空字符串'); + } + + // 验证clientId格式（必须包含@@@分隔符） + const clientIdParts = clientId.split('@@@'); + // 新版本 MQTT 参数 + if (clientIdParts.length !== 3) { + throw new Error('clientId格式错误，必须包含@@@分隔符'); + } + + // 验证username + if (!username || typeof username !== 'string') { + throw new Error('username必须是非空字符串'); + } + + // 尝试解码username（应该是base64编码的JSON） + let userData; + try { + const decodedUsername = Buffer.from(username, 'base64').toString(); + userData = JSON.parse(decodedUsername); + } catch (error) { + throw new Error('username不是有效的base64编码JSON'); + } + + // 解析clientId中的信息 + const [groupId, macAddress, uuid] = clientIdParts; + + // 如果验证成功，返回解析后的有用信息 + return { + groupId, + macAddress: macAddress.replace(/_/g, ':'), + uuid, + userData + }; +} + +function generateMqttConfig(groupId, macAddress, uuid, userData) { + const endpoint = process.env.MQTT_ENDPOINT; + const signatureKey = process.env.MQTT_SIGNATURE_KEY; + if (!signatureKey) { + console.warn('No signature key, skip generating MQTT config'); + return; + } + const deviceIdNoColon = macAddress.replace(/:/g, '_'); + const clientId = `${groupId}@@@${deviceIdNoColon}@@@${uuid}`; + const username = Buffer.from(JSON.stringify(userData)).toString('base64'); + const password = generatePasswordSignature(clientId + '|' + username, signatureKey); + return { + endpoint, + port: 8883, + client_id: clientId, + username, + password, + publish_topic: 'device-server', + subscribe_topic: 'null' // 旧版本固件不返回此字段会出错 + } +} + +module.exports = { + generateMqttConfig, + validateMqttCredentials +} + +if (require.main === module) { + const config = generateMqttConfig('GID_test', '11:22:33:44:55:66', '36c98363-3656-43cb-a00f-8bced2391a90', { ip: '222.222.222.222' }); + console.log('config', config); + const credentials = validateMqttCredentials(config.client_id, config.username, config.password); + console.log('credentials', credentials); +} diff --git "a/main/mqtt-gateway/\346\265\213\350\257\225\350\257\264\346\230\216.md" "b/main/mqtt-gateway/\346\265\213\350\257\225\350\257\264\346\230\216.md" new file mode 100644 index 000000000..202147295 --- /dev/null +++ "b/main/mqtt-gateway/\346\265\213\350\257\225\350\257\264\346\230\216.md" @@ -0,0 +1,239 @@ +# MQTT网关转发功能测试指南 + +本文档说明如何测试MQTT网关的转发功能是否配置成功。 + +## 快速测试 + +### 1. 基础连接测试 + +运行快速测试脚本： + +```bash +node quick-test.js +``` + +这个脚本会： +- ✅ 检查配置文件 +- ✅ 测试MQTT服务器连接（端口1883） +- ✅ 测试WebSocket后端服务连接 +- ✅ 显示测试结果和建议 + +### 2. 完整功能测试 + +运行完整测试脚本： + +```bash +node test-mqtt.js +``` + +这个脚本会： +- 🔗 连接到MQTT服务器 +- 🔐 使用配置的MAC地址进行认证 +- 📝 订阅回复主题 +- 📤 发送hello消息测试WebSocket转发 +- 📥 等待并显示服务器响应 +- 🧪 发送其他测试消息 + +## 手动测试步骤 + +### 1. 启动MQTT网关 + +```bash +node app.js +``` + +应该看到类似输出： +``` +MQTT 服务器启动在端口 1883 +UDP 服务器启动在端口 1883 +连接数: 0, 活跃数: 0 +``` + +### 2. 检查配置文件 + +确认 `config/mqtt.json` 配置正确： + +```json +{ + "production": { + "chat_servers": [ + "ws://192.168.68.66:8000/xiaozhi/v1/" + ] + }, + "development": { + "chat_servers": ["ws://192.168.68.66:8000/xiaozhi/v1/"], + "mac_addresss": ["d8:43:ae:3e:4b:5a"] + }, + "debug": false +} +``` + +### 3. 使用MQTT客户端工具测试 + +#### 使用mosquitto客户端： + +**连接测试：** +```bash +mosquitto_pub -h localhost -p 1883 -t "test/topic" -m "hello" +``` + +**订阅测试：** +```bash +mosquitto_sub -h localhost -p 1883 -t "devices/p2p/d8_43_ae_3e_4b_5a" +``` + +#### 使用MQTT.fx或其他GUI工具： + +1. **连接设置：** + - 服务器：localhost + - 端口：1883 + - 客户端ID：`GID_test@@@d8_43_ae_3e_4b_5a` + - 用户名：test_user + - 密码：test_password + +2. **发布测试消息：** + - 主题：`test/topic` + - 消息： + ```json + { + "type": "hello", + "version": 3, + "transport": "mqtt", + "audio_params": { + "sample_rate": 16000, + "channels": 1, + "format": "opus" + }, + "features": ["tts", "asr"] + } + ``` + +3. **订阅回复主题：** + - 主题：`devices/p2p/d8_43_ae_3e_4b_5a` + +## 测试结果判断 + +### ✅ 成功指标 + +1. **MQTT连接成功：** + ``` + ✓ 成功连接到MQTT服务器 localhost:1883 + ✓ MQTT连接成功 + ``` + +2. **WebSocket转发成功：** + ``` + ✓ WebSocket服务器连接成功 + ✓ 收到PUBLISH消息 + ``` + +3. **服务器日志正常：** + ``` + 客户端连接: { clientId: 'GID_test@@@d8_43_ae_3e_4b_5a', ... } + 收到发布消息: { topic: 'test/topic', payload: '...', qos: 0 } + ``` + +### ❌ 常见问题 + +1. **MQTT服务器连接失败：** + ``` + ✗ 连接错误: connect ECONNREFUSED 127.0.0.1:1883 + ``` + **解决方案：** 确保运行 `node app.js` 启动MQTT网关 + +2. **WebSocket连接失败：** + ``` + ✗ WebSocket服务器连接失败: connect ECONNREFUSED + ``` + **解决方案：** 检查WebSocket后端服务是否运行，网络是否可达 + +3. **认证失败：** + ``` + 无效的 clientId: xxx + ``` + **解决方案：** 检查客户端ID格式是否为 `GID_test@@@mac_address` + +4. **MAC地址不匹配：** + ``` + 未找到 xx:xx:xx:xx:xx:xx 的聊天服务器 + ``` + **解决方案：** 确认MAC地址在配置文件的 `mac_addresss` 列表中 + +## 调试技巧 + +### 1. 开启调试模式 + +修改 `config/mqtt.json`： +```json +{ + "debug": true +} +``` + +或设置环境变量： +```bash +DEBUG=mqtt-server node app.js +``` + +### 2. 查看详细日志 + +调试模式下会显示详细的连接和消息处理信息： +``` +mqtt-server 客户端连接: { clientId: '...', username: '...', ... } +mqtt-server 收到发布消息: { topic: '...', payload: '...', qos: 0 } +mqtt-server 发送消息到 devices/p2p/...: {...} +``` + +### 3. 网络连通性测试 + +**测试WebSocket服务器：** +```bash +curl -I http://192.168.68.66:8000/xiaozhi/v1/ +``` + +**测试MQTT端口：** +```bash +telnet localhost 1883 +``` + +## 性能测试 + +### 并发连接测试 + +可以修改测试脚本创建多个并发连接： + +```javascript +// 创建多个测试客户端 +const clients = []; +for (let i = 0; i < 10; i++) { + const client = new MQTTTestClient(); + clients.push(client); + // 连接和测试... +} +``` + +### 消息吞吐量测试 + +发送大量消息测试转发性能： + +```javascript +// 发送100条消息 +for (let i = 0; i < 100; i++) { + client.publish('test/topic', `测试消息 ${i}`); +} +``` + +## 故障排除清单 + +- [ ] MQTT网关服务是否运行 (`node app.js`) +- [ ] 端口1883是否被占用 +- [ ] 配置文件格式是否正确 +- [ ] WebSocket后端服务是否运行 +- [ ] 网络连接是否正常 +- [ ] MAC地址是否在配置的白名单中 +- [ ] 客户端ID格式是否正确 +- [ ] 防火墙是否阻止连接 + +## 总结 + +通过以上测试方法，你可以全面验证MQTT网关的转发功能是否配置成功。建议先运行快速测试，如果有问题再进行详细的手动测试和调试。 \ No newline at end of file From 5f96f5657476070931c4aeee1111a014213e9217 Mon Sep 17 00:00:00 2001 From: marlonz <235833017@qq.com> Date: Sat, 5 Jul 2025 13:50:47 +0930 Subject: [PATCH 06/14] remove gitignore --- main/mqtt-gateway/.gitignore | 4 ---- 1 file changed, 4 deletions(-) delete mode 100644 main/mqtt-gateway/.gitignore diff --git a/main/mqtt-gateway/.gitignore b/main/mqtt-gateway/.gitignore deleted file mode 100644 index 71b7ebc22..000000000 --- a/main/mqtt-gateway/.gitignore +++ /dev/null @@ -1,4 +0,0 @@ -node_modules/ -build/ -dist/ -package-lock.json From eed95033916756362a9764f6e980100fb50b5500 Mon Sep 17 00:00:00 2001 From: FAN-yeB <1442100690@qq.com> Date: Mon, 8 Sep 2025 11:04:42 +0800 Subject: [PATCH 07/14] mqtt --- .../xiaozhi/common/constant/Constant.java | 6 ++ .../device/controller/OTAController.java | 4 + .../device/dto/DeviceReportRespDTO.java | 22 ++++- .../service/impl/DeviceServiceImpl.java | 92 ++++++++++++++++++- .../resources/db/changelog/202509080921.sql | 7 ++ .../resources/db/changelog/202509080927.sql | 2 + .../db/changelog/db.changelog-master.yaml | 16 ++++ main/xiaozhi-server/core/connection.py | 82 +++++++++++++++++ .../core/handle/receiveAudioHandle.py | 32 +++---- .../core/handle/sendAudioHandle.py | 50 ++++++++-- .../xiaozhi-server/core/providers/tts/base.py | 3 +- 11 files changed, 289 insertions(+), 27 deletions(-) create mode 100644 main/manager-api/src/main/resources/db/changelog/202509080921.sql create mode 100644 main/manager-api/src/main/resources/db/changelog/202509080927.sql diff --git a/main/manager-api/src/main/java/xiaozhi/common/constant/Constant.java b/main/manager-api/src/main/java/xiaozhi/common/constant/Constant.java index afd7addf5..6d4e7ef0d 100644 --- a/main/manager-api/src/main/java/xiaozhi/common/constant/Constant.java +++ b/main/manager-api/src/main/java/xiaozhi/common/constant/Constant.java @@ -91,6 +91,12 @@ public interface Constant { */ String SERVER_WEBSOCKET = "server.websocket"; + /** + * mqtt gateway 配置 + */ + String SERVER_MQTT_GATEWAY = "server.mqtt_gateway"; + + /** * ota地址 */ diff --git a/main/manager-api/src/main/java/xiaozhi/modules/device/controller/OTAController.java b/main/manager-api/src/main/java/xiaozhi/modules/device/controller/OTAController.java index 023510cd1..26e5dbe64 100644 --- a/main/manager-api/src/main/java/xiaozhi/modules/device/controller/OTAController.java +++ b/main/manager-api/src/main/java/xiaozhi/modules/device/controller/OTAController.java @@ -77,6 +77,10 @@ public ResponseEntity activateDevice( @GetMapping @Hidden public ResponseEntity getOTA() { + String mqttUdpConfig = sysParamsService.getValue(Constant.SERVER_MQTT_GATEWAY, false); + if(StringUtils.isBlank(mqttUdpConfig)) { + return ResponseEntity.ok("OTA接口不正常，缺少mqtt_gateway地址，请登录智控台，在参数管理找到【server.mqtt_gateway】配置"); + } String wsUrl = sysParamsService.getValue(Constant.SERVER_WEBSOCKET, true); if (StringUtils.isBlank(wsUrl) || wsUrl.equals("null")) { return ResponseEntity.ok("OTA接口不正常，缺少websocket地址，请登录智控台，在参数管理找到【server.websocket】配置"); diff --git a/main/manager-api/src/main/java/xiaozhi/modules/device/dto/DeviceReportRespDTO.java b/main/manager-api/src/main/java/xiaozhi/modules/device/dto/DeviceReportRespDTO.java index f938001e0..423868ca1 100644 --- a/main/manager-api/src/main/java/xiaozhi/modules/device/dto/DeviceReportRespDTO.java +++ b/main/manager-api/src/main/java/xiaozhi/modules/device/dto/DeviceReportRespDTO.java @@ -23,6 +23,9 @@ public class DeviceReportRespDTO { @Schema(description = "WebSocket配置") private Websocket websocket; + @Schema(description = "MQTT Gateway配置") + private MQTT mqtt; + @Getter @Setter public static class Firmware { @@ -70,4 +73,21 @@ public static class Websocket { @Schema(description = "WebSocket服务器地址") private String url; } -} + + @Getter + @Setter + public static class MQTT { + @Schema(description = "MQTT 配置网址") + private String endpoint; + @Schema(description = "MQTT 客户端唯一标识符") + private String client_id; + @Schema(description = "MQTT 认证用户名") + private String username; + @Schema(description = "MQTT 认证密码") + private String password; + @Schema(description = "ESP32 发布消息的主题") + private String publish_topic; + @Schema(description = "ESP32 订阅的主题") + private String subscribe_topic; + } +} \ No newline at end of file diff --git a/main/manager-api/src/main/java/xiaozhi/modules/device/service/impl/DeviceServiceImpl.java b/main/manager-api/src/main/java/xiaozhi/modules/device/service/impl/DeviceServiceImpl.java index 3cff111db..eeacce9a2 100644 --- a/main/manager-api/src/main/java/xiaozhi/modules/device/service/impl/DeviceServiceImpl.java +++ b/main/manager-api/src/main/java/xiaozhi/modules/device/service/impl/DeviceServiceImpl.java @@ -1,12 +1,16 @@ package xiaozhi.modules.device.service.impl; +import java.nio.charset.StandardCharsets; import java.time.Instant; +import java.util.Base64; import java.util.Date; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.TimeZone; import java.util.UUID; +import javax.crypto.Mac; +import javax.crypto.spec.SecretKeySpec; import org.apache.commons.lang3.StringUtils; import org.springframework.aop.framework.AopContext; @@ -175,7 +179,22 @@ public DeviceReportRespDTO checkDeviceActive(String macAddress, String clientId, } response.setWebsocket(websocket); - + + // 添加MQTT UDP配置 + // 从系统参数获取MQTT Gateway地址，如果未配置不使用默认值 + String mqttUdpConfig = sysParamsService.getValue(Constant.SERVER_MQTT_GATEWAY, false); + if(!StringUtils.isBlank(mqttUdpConfig) && deviceById != null) { + try { + DeviceReportRespDTO.MQTT mqtt = buildMqttConfig(macAddress, clientId, deviceById); + if (mqtt != null) { + mqtt.setEndpoint(mqttUdpConfig); + response.setMqtt(mqtt); + } + } catch (Exception e) { + log.error("生成MQTT配置失败: {}", e.getMessage()); + } + } + if (deviceById != null) { // 如果设备存在，则异步更新上次连接时间和版本信息 String appVersion = deviceReport.getApplication() != null ? deviceReport.getApplication().getVersion() @@ -437,4 +456,75 @@ public void manualAddDevice(Long userId, DeviceManualAddDTO dto) { entity.setAutoUpdate(1); baseDao.insert(entity); } + + /** + * 生成MQTT密码签名 + * @param content 签名内容 (clientId + '|' + username) + * @param secretKey 密钥 + * @return Base64编码的HMAC-SHA256签名 + */ + private String generatePasswordSignature(String content, String secretKey) throws Exception { + Mac hmac = Mac.getInstance("HmacSHA256"); + SecretKeySpec keySpec = new SecretKeySpec(secretKey.getBytes(StandardCharsets.UTF_8), "HmacSHA256"); + hmac.init(keySpec); + byte[] signature = hmac.doFinal(content.getBytes(StandardCharsets.UTF_8)); + return Base64.getEncoder().encodeToString(signature); + } + + /** + * 构建MQTT配置信息 + * @param macAddress MAC地址 + * @param clientId 客户端ID (UUID) + * @param device 设备信息 + * @return MQTT配置对象 + */ + private DeviceReportRespDTO.MQTT buildMqttConfig(String macAddress, String clientId, DeviceEntity device) throws Exception { + // 从环境变量或系统参数获取签名密钥 + String signatureKey = System.getenv("MQTT_SIGNATURE_KEY"); + if (StringUtils.isBlank(signatureKey)) { + // 如果环境变量没有，尝试从系统参数获取 + signatureKey = sysParamsService.getValue("mqtt.signature_key", false); + } + + if (StringUtils.isBlank(signatureKey)) { + log.warn("缺少MQTT_SIGNATURE_KEY，跳过MQTT配置生成"); + return null; + } + + // 构建客户端ID格式：groupId@@@macAddress_without_colon@@@uuid + String groupId = device.getBoard() != null ? device.getBoard() : "GID_default"; + String deviceIdNoColon = macAddress.replace(":", "_"); + String mqttClientId = String.format("%s@@@%s@@@%s", groupId, deviceIdNoColon, clientId); + + // 构建用户数据（包含IP等信息） + Map userData = new HashMap<>(); + // 尝试获取客户端IP + try { + ServletRequestAttributes attributes = (ServletRequestAttributes) RequestContextHolder.getRequestAttributes(); + if (attributes != null) { + HttpServletRequest request = attributes.getRequest(); + String clientIp = request.getRemoteAddr(); + userData.put("ip", clientIp); + } + } catch (Exception e) { + userData.put("ip", "unknown"); + } + + // 将用户数据编码为Base64 JSON + String userDataJson = new com.fasterxml.jackson.databind.ObjectMapper().writeValueAsString(userData); + String username = Base64.getEncoder().encodeToString(userDataJson.getBytes(StandardCharsets.UTF_8)); + + // 生成密码签名 + String password = generatePasswordSignature(mqttClientId + "|" + username, signatureKey); + + // 构建MQTT配置 + DeviceReportRespDTO.MQTT mqtt = new DeviceReportRespDTO.MQTT(); + mqtt.setClient_id(mqttClientId); + mqtt.setUsername(username); + mqtt.setPassword(password); + mqtt.setPublish_topic("device-server"); + mqtt.setSubscribe_topic("devices/p2p/" + deviceIdNoColon); + + return mqtt; + } } diff --git a/main/manager-api/src/main/resources/db/changelog/202509080921.sql b/main/manager-api/src/main/resources/db/changelog/202509080921.sql new file mode 100644 index 000000000..4d89ee57d --- /dev/null +++ b/main/manager-api/src/main/resources/db/changelog/202509080921.sql @@ -0,0 +1,7 @@ +delete from `sys_params` where id = 108; +delete from `sys_params` where param_code = 'server.mqtt_gateway'; +INSERT INTO `sys_params` (id, param_code, param_value, value_type, param_type, remark) VALUES (108, 'server.mqtt_gateway', 'null', 'string', 1, 'mqtt gateway 配置'); + +delete from `sys_params` where param_code = 'server.udp_gateway'; +INSERT INTO `sys_params` (id, param_code, param_value, value_type, param_type, remark) VALUES (109, 'server.udp_gateway', 'null', 'string', 1, 'udp gateway 配置'); + diff --git a/main/manager-api/src/main/resources/db/changelog/202509080927.sql b/main/manager-api/src/main/resources/db/changelog/202509080927.sql new file mode 100644 index 000000000..4741af88e --- /dev/null +++ b/main/manager-api/src/main/resources/db/changelog/202509080927.sql @@ -0,0 +1,2 @@ +delete from `sys_params` where param_code = 'mqtt.signature_key'; +INSERT INTO `sys_params` (id, param_code, param_value, value_type, param_type, remark) VALUES (120, 'mqtt.signature_key', 'null', 'string', 1, 'mqtt 密钥 配置'); diff --git a/main/manager-api/src/main/resources/db/changelog/db.changelog-master.yaml b/main/manager-api/src/main/resources/db/changelog/db.changelog-master.yaml index 0e8d3dc35..ed66ebb78 100755 --- a/main/manager-api/src/main/resources/db/changelog/db.changelog-master.yaml +++ b/main/manager-api/src/main/resources/db/changelog/db.changelog-master.yaml @@ -303,3 +303,19 @@ databaseChangeLog: - sqlFile: encoding: utf8 path: classpath:db/changelog/202508131557.sql + + - changeSet: + id: 202509080921 + author: fan + changes: + - sqlFile: + encoding: utf8 + path: classpath:db/changelog/202509080921.sql + + - changeSet: + id: 202509080927 + author: fan + changes: + - sqlFile: + encoding: utf8 + path: classpath:db/changelog/202509080927.sql \ No newline at end of file diff --git a/main/xiaozhi-server/core/connection.py b/main/xiaozhi-server/core/connection.py index 4b3b9d1cc..3e6e8633b 100644 --- a/main/xiaozhi-server/core/connection.py +++ b/main/xiaozhi-server/core/connection.py @@ -281,8 +281,86 @@ async def _route_message(self, message): return if self.asr is None: return + + + if len(message) >= 16: + try: + timestamp = int.from_bytes(message[8:12], 'big') + audio_length = int.from_bytes(message[12:16], 'big') + + + # 提取音频数据 + if audio_length > 0 and len(message) >= 16 + audio_length: + audio_data = message[16:16 + audio_length] + + # 基于时间戳进行简单排序 + self._process_websocket_audio(audio_data, timestamp) + return + elif len(message) > 16: + # 去掉16字节头部 + audio_data = message[16:] + self.asr_audio_queue.put(audio_data) + return + except Exception as e: + self.logger.bind(tag=TAG).error(f"解析WebSocket音频包失败: {e}") + self.asr_audio_queue.put(message) + def _process_websocket_audio(self, audio_data, timestamp): + """处理WebSocket格式的音频包""" + # 初始化时间戳序列管理 + if not hasattr(self, 'audio_timestamp_buffer'): + self.audio_timestamp_buffer = {} + self.last_processed_timestamp = 0 + self.max_timestamp_buffer_size = 20 + + # 如果时间戳是递增的，直接处理 + if timestamp >= self.last_processed_timestamp: + self.asr_audio_queue.put(audio_data) + self.last_processed_timestamp = timestamp + + # 处理缓冲区中的后续包 + processed_any = True + while processed_any: + processed_any = False + for ts in sorted(self.audio_timestamp_buffer.keys()): + if ts > self.last_processed_timestamp: + buffered_audio = self.audio_timestamp_buffer.pop(ts) + self.asr_audio_queue.put(buffered_audio) + self.last_processed_timestamp = ts + processed_any = True + break + else: + # 乱序包，暂存 + if len(self.audio_timestamp_buffer) < self.max_timestamp_buffer_size: + self.audio_timestamp_buffer[timestamp] = audio_data + else: + self.asr_audio_queue.put(audio_data) + + def _process_sequenced_audio(self, audio_data, sequence, timestamp): + """处理有序的音频包""" + # 初始化音频缓冲区 + if not hasattr(self, 'audio_buffer'): + self.audio_buffer = {} + self.expected_sequence = sequence + self.max_buffer_size = 20 # 最大缓冲20个包 + + # 如果是下一个期望的包，直接处理 + if sequence == self.expected_sequence: + self.asr_audio_queue.put(audio_data) + self.expected_sequence += 1 + + # 检查缓冲区中是否有后续的连续包 + while self.expected_sequence in self.audio_buffer: + buffered_audio = self.audio_buffer.pop(self.expected_sequence) + self.asr_audio_queue.put(buffered_audio) + self.expected_sequence += 1 + + elif sequence > self.expected_sequence: + # 乱序包，暂存到缓冲区 + if len(self.audio_buffer) < self.max_buffer_size: + self.audio_buffer[sequence] = audio_data + async def handle_restart(self, message): """处理服务器重启请求""" try: @@ -921,6 +999,10 @@ def clearSpeakStatus(self): async def close(self, ws=None): """资源清理方法""" try: + # 清理音频缓冲区 + if hasattr(self, 'audio_buffer'): + self.audio_buffer.clear() + # 取消超时任务 if self.timeout_task and not self.timeout_task.done(): self.timeout_task.cancel() diff --git a/main/xiaozhi-server/core/handle/receiveAudioHandle.py b/main/xiaozhi-server/core/handle/receiveAudioHandle.py index 8db506338..e6f396327 100644 --- a/main/xiaozhi-server/core/handle/receiveAudioHandle.py +++ b/main/xiaozhi-server/core/handle/receiveAudioHandle.py @@ -1,11 +1,12 @@ +from core.handle.sendAudioHandle import send_stt_message +from core.handle.intentHandler import handle_user_intent +from core.utils.output_counter import check_device_output_limit +from core.handle.abortHandle import handleAbortMessage import time -import json import asyncio +import json +from core.handle.sendAudioHandle import SentenceType from core.utils.util import audio_to_data -from core.handle.abortHandle import handleAbortMessage -from core.handle.intentHandler import handle_user_intent -from core.utils.output_counter import check_device_output_limit -from core.handle.sendAudioHandle import send_stt_message, SentenceType TAG = __name__ @@ -21,6 +22,7 @@ async def handleAudioMessage(conn, audio): if not hasattr(conn, "vad_resume_task") or conn.vad_resume_task.done(): conn.vad_resume_task = asyncio.create_task(resume_vad_detection(conn)) return + if have_voice: if conn.client_is_speaking: await handleAbortMessage(conn) @@ -29,16 +31,18 @@ async def handleAudioMessage(conn, audio): # 接收音频 await conn.asr.receive_audio(conn, audio, have_voice) + async def resume_vad_detection(conn): # 等待2秒后恢复VAD检测 await asyncio.sleep(1) conn.just_woken_up = False + async def startToChat(conn, text): # 检查输入是否是JSON格式（包含说话人信息） speaker_name = None actual_text = text - + try: # 尝试解析JSON格式的输入 if text.strip().startswith('{') and text.strip().endswith('}'): @@ -47,13 +51,13 @@ async def startToChat(conn, text): speaker_name = data['speaker'] actual_text = data['content'] conn.logger.bind(tag=TAG).info(f"解析到说话人信息: {speaker_name}") - + # 直接使用JSON格式的文本，不解析 actual_text = text except (json.JSONDecodeError, KeyError): # 如果解析失败，继续使用原始文本 pass - + # 保存说话人信息到连接对象 if speaker_name: conn.current_speaker = speaker_name @@ -114,12 +118,10 @@ async def no_voice_close_connect(conn, have_voice): async def max_out_size(conn): - # 播放超出最大输出字数的提示 - conn.client_abort = False text = "不好意思，我现在有点事情要忙，明天这个时候我们再聊，约好了哦！明天不见不散，拜拜！" await send_stt_message(conn, text) file_path = "config/assets/max_output_size.wav" - opus_packets = audio_to_data(file_path) + opus_packets, _ = audio_to_data(file_path) conn.tts.tts_audio_queue.put((SentenceType.LAST, opus_packets, text)) conn.close_after_chat = True @@ -138,7 +140,7 @@ async def check_bind_device(conn): # 播放提示音 music_path = "config/assets/bind_code.wav" - opus_packets = audio_to_data(music_path) + opus_packets, _ = audio_to_data(music_path) conn.tts.tts_audio_queue.put((SentenceType.FIRST, opus_packets, text)) # 逐个播放数字 @@ -146,17 +148,15 @@ async def check_bind_device(conn): try: digit = conn.bind_code[i] num_path = f"config/assets/bind_code/{digit}.wav" - num_packets = audio_to_data(num_path) + num_packets, _ = audio_to_data(num_path) conn.tts.tts_audio_queue.put((SentenceType.MIDDLE, num_packets, None)) except Exception as e: conn.logger.bind(tag=TAG).error(f"播放数字音频失败: {e}") continue conn.tts.tts_audio_queue.put((SentenceType.LAST, [], None)) else: - # 播放未绑定提示 - conn.client_abort = False text = f"没有找到该设备的版本信息，请正确配置 OTA地址，然后重新编译固件。" await send_stt_message(conn, text) music_path = "config/assets/bind_not_found.wav" - opus_packets = audio_to_data(music_path) + opus_packets, _ = audio_to_data(music_path) conn.tts.tts_audio_queue.put((SentenceType.LAST, opus_packets, text)) diff --git a/main/xiaozhi-server/core/handle/sendAudioHandle.py b/main/xiaozhi-server/core/handle/sendAudioHandle.py index 661c4e212..c8a05ded3 100644 --- a/main/xiaozhi-server/core/handle/sendAudioHandle.py +++ b/main/xiaozhi-server/core/handle/sendAudioHandle.py @@ -55,6 +55,7 @@ async def sendAudio(conn, audios, frame_duration=60): "last_send_time": 0, "packet_count": 0, "start_time": time.perf_counter(), + "sequence": 0, # 添加序列号 } flow_control = conn.audio_flow_control @@ -67,11 +68,22 @@ async def sendAudio(conn, audios, frame_duration=60): if delay > 0: await asyncio.sleep(delay) - # 发送数据包 - await conn.websocket.send(audios) + # 为opus数据包添加16字节头部 + timestamp = int((flow_control["start_time"] + flow_control["packet_count"] * frame_duration / 1000) * 1000) % (2**32) + header = bytearray(16) + header[0] = 1 # type + header[2:4] = len(audios).to_bytes(2, 'big') # payload length + header[4:8] = flow_control["sequence"].to_bytes(4, 'big') # connection id/sequence + header[8:12] = timestamp.to_bytes(4, 'big') # 时间戳 + header[12:16] = len(audios).to_bytes(4, 'big') # opus长度 + + # 发送包含头部的完整数据包 + complete_packet = bytes(header) + audios + await conn.websocket.send(complete_packet) # 更新流控状态 flow_control["packet_count"] += 1 + flow_control["sequence"] += 1 flow_control["last_send_time"] = time.perf_counter() else: # 文件型音频走普通播放 @@ -81,11 +93,21 @@ async def sendAudio(conn, audios, frame_duration=60): # 执行预缓冲 pre_buffer_frames = min(3, len(audios)) for i in range(pre_buffer_frames): - await conn.websocket.send(audios[i]) + # 为预缓冲包添加头部 + timestamp = int((start_time + i * frame_duration / 1000) * 1000) % (2**32) + header = bytearray(16) + header[0] = 1 # type + header[2:4] = len(audios[i]).to_bytes(2, 'big') # payload length + header[4:8] = i.to_bytes(4, 'big') # sequence + header[8:12] = timestamp.to_bytes(4, 'big') # 时间戳 + header[12:16] = len(audios[i]).to_bytes(4, 'big') # opus长度 + + complete_packet = bytes(header) + audios[i] + await conn.websocket.send(complete_packet) remaining_audios = audios[pre_buffer_frames:] - + # 播放剩余音频帧 - for opus_packet in remaining_audios: + for i, opus_packet in enumerate(remaining_audios): if conn.client_abort: break @@ -98,9 +120,21 @@ async def sendAudio(conn, audios, frame_duration=60): delay = expected_time - current_time if delay > 0: await asyncio.sleep(delay) - - await conn.websocket.send(opus_packet) - + + # 为opus数据包添加16字节头部 (timestamp at offset 8, length at offset 12) + timestamp = int((start_time + play_position / 1000) * 1000) % (2**32) # 使用播放位置计算时间戳 + sequence = pre_buffer_frames + i # 确保序列号连续 + header = bytearray(16) + header[0] = 1 # type + header[2:4] = len(opus_packet).to_bytes(2, 'big') # payload length + header[4:8] = sequence.to_bytes(4, 'big') # sequence + header[8:12] = timestamp.to_bytes(4, 'big') # 时间戳在第8-11字节 + header[12:16] = len(opus_packet).to_bytes(4, 'big') # opus长度在第12-15字节 + + # 发送包含头部的完整数据包 + complete_packet = bytes(header) + opus_packet + await conn.websocket.send(complete_packet) + play_position += frame_duration diff --git a/main/xiaozhi-server/core/providers/tts/base.py b/main/xiaozhi-server/core/providers/tts/base.py index 04a7fa360..c4b323551 100644 --- a/main/xiaozhi-server/core/providers/tts/base.py +++ b/main/xiaozhi-server/core/providers/tts/base.py @@ -336,7 +336,8 @@ def _audio_play_priority_thread(self): self.conn.audio_flow_control = { 'last_send_time': 0, 'packet_count': 0, - 'start_time': time.perf_counter() + 'start_time': time.perf_counter(), + 'sequence': 0 # 添加序列号 } # 上报TTS数据 From eee907b1a23156d712f2c282cbade8faf65de38e Mon Sep 17 00:00:00 2001 From: FAN-yeB <1442100690@qq.com> Date: Tue, 9 Sep 2025 09:37:31 +0800 Subject: [PATCH 08/14] =?UTF-8?q?=E5=85=BC=E5=AE=B9udp=E5=92=8Cwebsocket?= =?UTF-8?q?=E5=8D=8F=E8=AE=AE=E4=BC=A0=E8=BE=93=E9=9F=B3=E9=A2=91?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../service/impl/DeviceServiceImpl.java | 4 +- main/xiaozhi-server/config.yaml | 2 + main/xiaozhi-server/core/connection.py | 12 ++- .../core/handle/sendAudioHandle.py | 94 ++++++++++++------- 4 files changed, 73 insertions(+), 39 deletions(-) diff --git a/main/manager-api/src/main/java/xiaozhi/modules/device/service/impl/DeviceServiceImpl.java b/main/manager-api/src/main/java/xiaozhi/modules/device/service/impl/DeviceServiceImpl.java index eeacce9a2..b4343276d 100644 --- a/main/manager-api/src/main/java/xiaozhi/modules/device/service/impl/DeviceServiceImpl.java +++ b/main/manager-api/src/main/java/xiaozhi/modules/device/service/impl/DeviceServiceImpl.java @@ -181,9 +181,9 @@ public DeviceReportRespDTO checkDeviceActive(String macAddress, String clientId, response.setWebsocket(websocket); // 添加MQTT UDP配置 - // 从系统参数获取MQTT Gateway地址，如果未配置不使用默认值 + // 从系统参数获取MQTT Gateway地址，仅在配置有效时使用 String mqttUdpConfig = sysParamsService.getValue(Constant.SERVER_MQTT_GATEWAY, false); - if(!StringUtils.isBlank(mqttUdpConfig) && deviceById != null) { + if(mqttUdpConfig != null && !mqttUdpConfig.equals("null") && !mqttUdpConfig.isEmpty() && deviceById != null) { try { DeviceReportRespDTO.MQTT mqtt = buildMqttConfig(macAddress, clientId, deviceById); if (mqtt != null) { diff --git a/main/xiaozhi-server/config.yaml b/main/xiaozhi-server/config.yaml index c0aa3f934..1afc3530b 100644 --- a/main/xiaozhi-server/config.yaml +++ b/main/xiaozhi-server/config.yaml @@ -25,6 +25,8 @@ server: # 所以如果你使用docker部署时，将vision_explain设置成局域网地址 # 如果你使用公网部署时，将vision_explain设置成公网地址 vision_explain: http://你的ip或者域名:端口号/mcp/vision/explain + # mqtt网关地址，当这个值为null时，mqtt网关桥接功能不开启，使用websocket双向通信，不使用mqtt和udp协议 + mqtt_gateway: null # OTA返回信息时区偏移量 timezone_offset: +8 # 认证配置 diff --git a/main/xiaozhi-server/core/connection.py b/main/xiaozhi-server/core/connection.py index 3e6e8633b..e23826a11 100644 --- a/main/xiaozhi-server/core/connection.py +++ b/main/xiaozhi-server/core/connection.py @@ -282,8 +282,17 @@ async def _route_message(self, message): if self.asr is None: return + # 检查是否需要处理头部（只有当mqtt_gateway有实际值时才处理头部） + mqtt_gateway = self.config.get("server", {}).get("mqtt_gateway") + # 当mqtt_gateway为None, "null", "", 或实际的null值时，不处理头部 + need_header_processing = mqtt_gateway and mqtt_gateway not in [None, "null", ""] and str(mqtt_gateway).strip() != "" + + # 调试日志：首次连接时记录配置 + if not hasattr(self, '_logged_mqtt_config'): + self.logger.bind(tag=TAG).info(f"MQTT Gateway配置: '{mqtt_gateway}', 头部处理: {need_header_processing}") + self._logged_mqtt_config = True - if len(message) >= 16: + if need_header_processing and len(message) >= 16: try: timestamp = int.from_bytes(message[8:12], 'big') audio_length = int.from_bytes(message[12:16], 'big') @@ -304,6 +313,7 @@ async def _route_message(self, message): except Exception as e: self.logger.bind(tag=TAG).error(f"解析WebSocket音频包失败: {e}") + # 不需要头部处理或没有头部时，直接处理原始消息 self.asr_audio_queue.put(message) def _process_websocket_audio(self, audio_data, timestamp): diff --git a/main/xiaozhi-server/core/handle/sendAudioHandle.py b/main/xiaozhi-server/core/handle/sendAudioHandle.py index c8a05ded3..56d219322 100644 --- a/main/xiaozhi-server/core/handle/sendAudioHandle.py +++ b/main/xiaozhi-server/core/handle/sendAudioHandle.py @@ -43,6 +43,11 @@ async def sendAudio(conn, audios, frame_duration=60): if audios is None or len(audios) == 0: return + # 检查是否需要添加头部（只有当mqtt_gateway有实际值时才添加头部） + mqtt_gateway = conn.config.get("server", {}).get("mqtt_gateway") + # 当mqtt_gateway为None, "null", "", 或实际的null值时，不添加头部 + need_header = mqtt_gateway and mqtt_gateway not in [None, "null", ""] and str(mqtt_gateway).strip() != "" + if isinstance(audios, bytes): if conn.client_abort: return @@ -68,18 +73,22 @@ async def sendAudio(conn, audios, frame_duration=60): if delay > 0: await asyncio.sleep(delay) - # 为opus数据包添加16字节头部 - timestamp = int((flow_control["start_time"] + flow_control["packet_count"] * frame_duration / 1000) * 1000) % (2**32) - header = bytearray(16) - header[0] = 1 # type - header[2:4] = len(audios).to_bytes(2, 'big') # payload length - header[4:8] = flow_control["sequence"].to_bytes(4, 'big') # connection id/sequence - header[8:12] = timestamp.to_bytes(4, 'big') # 时间戳 - header[12:16] = len(audios).to_bytes(4, 'big') # opus长度 - - # 发送包含头部的完整数据包 - complete_packet = bytes(header) + audios - await conn.websocket.send(complete_packet) + if need_header: + # 为opus数据包添加16字节头部 + timestamp = int((flow_control["start_time"] + flow_control["packet_count"] * frame_duration / 1000) * 1000) % (2**32) + header = bytearray(16) + header[0] = 1 # type + header[2:4] = len(audios).to_bytes(2, 'big') # payload length + header[4:8] = flow_control["sequence"].to_bytes(4, 'big') # connection id/sequence + header[8:12] = timestamp.to_bytes(4, 'big') # 时间戳 + header[12:16] = len(audios).to_bytes(4, 'big') # opus长度 + + # 发送包含头部的完整数据包 + complete_packet = bytes(header) + audios + await conn.websocket.send(complete_packet) + else: + # 直接发送opus数据包，不添加头部 + await conn.websocket.send(audios) # 更新流控状态 flow_control["packet_count"] += 1 @@ -90,20 +99,29 @@ async def sendAudio(conn, audios, frame_duration=60): start_time = time.perf_counter() play_position = 0 + # 检查是否需要添加头部（只有当mqtt_gateway有实际值时才添加头部） + mqtt_gateway = conn.config.get("server", {}).get("mqtt_gateway") + # 当mqtt_gateway为None, "null", "", 或实际的null值时，不添加头部 + need_header = mqtt_gateway and mqtt_gateway not in [None, "null", ""] and str(mqtt_gateway).strip() != "" + # 执行预缓冲 pre_buffer_frames = min(3, len(audios)) for i in range(pre_buffer_frames): - # 为预缓冲包添加头部 - timestamp = int((start_time + i * frame_duration / 1000) * 1000) % (2**32) - header = bytearray(16) - header[0] = 1 # type - header[2:4] = len(audios[i]).to_bytes(2, 'big') # payload length - header[4:8] = i.to_bytes(4, 'big') # sequence - header[8:12] = timestamp.to_bytes(4, 'big') # 时间戳 - header[12:16] = len(audios[i]).to_bytes(4, 'big') # opus长度 - - complete_packet = bytes(header) + audios[i] - await conn.websocket.send(complete_packet) + if need_header: + # 为预缓冲包添加头部 + timestamp = int((start_time + i * frame_duration / 1000) * 1000) % (2**32) + header = bytearray(16) + header[0] = 1 # type + header[2:4] = len(audios[i]).to_bytes(2, 'big') # payload length + header[4:8] = i.to_bytes(4, 'big') # sequence + header[8:12] = timestamp.to_bytes(4, 'big') # 时间戳 + header[12:16] = len(audios[i]).to_bytes(4, 'big') # opus长度 + + complete_packet = bytes(header) + audios[i] + await conn.websocket.send(complete_packet) + else: + # 直接发送预缓冲包，不添加头部 + await conn.websocket.send(audios[i]) remaining_audios = audios[pre_buffer_frames:] # 播放剩余音频帧 @@ -121,19 +139,23 @@ async def sendAudio(conn, audios, frame_duration=60): if delay > 0: await asyncio.sleep(delay) - # 为opus数据包添加16字节头部 (timestamp at offset 8, length at offset 12) - timestamp = int((start_time + play_position / 1000) * 1000) % (2**32) # 使用播放位置计算时间戳 - sequence = pre_buffer_frames + i # 确保序列号连续 - header = bytearray(16) - header[0] = 1 # type - header[2:4] = len(opus_packet).to_bytes(2, 'big') # payload length - header[4:8] = sequence.to_bytes(4, 'big') # sequence - header[8:12] = timestamp.to_bytes(4, 'big') # 时间戳在第8-11字节 - header[12:16] = len(opus_packet).to_bytes(4, 'big') # opus长度在第12-15字节 - - # 发送包含头部的完整数据包 - complete_packet = bytes(header) + opus_packet - await conn.websocket.send(complete_packet) + if need_header: + # 为opus数据包添加16字节头部 (timestamp at offset 8, length at offset 12) + timestamp = int((start_time + play_position / 1000) * 1000) % (2**32) # 使用播放位置计算时间戳 + sequence = pre_buffer_frames + i # 确保序列号连续 + header = bytearray(16) + header[0] = 1 # type + header[2:4] = len(opus_packet).to_bytes(2, 'big') # payload length + header[4:8] = sequence.to_bytes(4, 'big') # sequence + header[8:12] = timestamp.to_bytes(4, 'big') # 时间戳在第8-11字节 + header[12:16] = len(opus_packet).to_bytes(4, 'big') # opus长度在第12-15字节 + + # 发送包含头部的完整数据包 + complete_packet = bytes(header) + opus_packet + await conn.websocket.send(complete_packet) + else: + # 直接发送opus数据包，不添加头部 + await conn.websocket.send(opus_packet) play_position += frame_duration From ef02a0143916ced34a00a697864ccbe9e3a699e3 Mon Sep 17 00:00:00 2001 From: hrz <1710360675@qq.com> Date: Wed, 10 Sep 2025 17:38:29 +0800 Subject: [PATCH 09/14] =?UTF-8?q?update:=E4=BC=98=E5=8C=96=E6=99=BA?= =?UTF-8?q?=E6=8E=A7=E5=8F=B0=E5=8F=82=E6=95=B0=E5=90=8D=E7=A7=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../service/impl/DeviceServiceImpl.java | 30 +++++++++---------- .../resources/db/changelog/202509080921.sql | 7 ----- .../resources/db/changelog/202509080922.sql | 8 +++++ .../resources/db/changelog/202509080927.sql | 2 -- .../db/changelog/db.changelog-master.yaml | 14 ++------- 5 files changed, 26 insertions(+), 35 deletions(-) delete mode 100644 main/manager-api/src/main/resources/db/changelog/202509080921.sql create mode 100644 main/manager-api/src/main/resources/db/changelog/202509080922.sql delete mode 100644 main/manager-api/src/main/resources/db/changelog/202509080927.sql diff --git a/main/manager-api/src/main/java/xiaozhi/modules/device/service/impl/DeviceServiceImpl.java b/main/manager-api/src/main/java/xiaozhi/modules/device/service/impl/DeviceServiceImpl.java index b4343276d..663800417 100644 --- a/main/manager-api/src/main/java/xiaozhi/modules/device/service/impl/DeviceServiceImpl.java +++ b/main/manager-api/src/main/java/xiaozhi/modules/device/service/impl/DeviceServiceImpl.java @@ -9,6 +9,7 @@ import java.util.Map; import java.util.TimeZone; import java.util.UUID; + import javax.crypto.Mac; import javax.crypto.spec.SecretKeySpec; @@ -37,6 +38,7 @@ import xiaozhi.common.utils.ConvertUtils; import xiaozhi.common.utils.DateUtils; import xiaozhi.modules.device.dao.DeviceDao; +import xiaozhi.modules.device.dto.DeviceManualAddDTO; import xiaozhi.modules.device.dto.DevicePageUserDTO; import xiaozhi.modules.device.dto.DeviceReportReqDTO; import xiaozhi.modules.device.dto.DeviceReportRespDTO; @@ -48,7 +50,6 @@ import xiaozhi.modules.security.user.SecurityUser; import xiaozhi.modules.sys.service.SysParamsService; import xiaozhi.modules.sys.service.SysUserUtilService; -import xiaozhi.modules.device.dto.DeviceManualAddDTO; @Slf4j @Service @@ -179,11 +180,11 @@ public DeviceReportRespDTO checkDeviceActive(String macAddress, String clientId, } response.setWebsocket(websocket); - + // 添加MQTT UDP配置 // 从系统参数获取MQTT Gateway地址，仅在配置有效时使用 String mqttUdpConfig = sysParamsService.getValue(Constant.SERVER_MQTT_GATEWAY, false); - if(mqttUdpConfig != null && !mqttUdpConfig.equals("null") && !mqttUdpConfig.isEmpty() && deviceById != null) { + if (mqttUdpConfig != null && !mqttUdpConfig.equals("null") && !mqttUdpConfig.isEmpty() && deviceById != null) { try { DeviceReportRespDTO.MQTT mqtt = buildMqttConfig(macAddress, clientId, deviceById); if (mqtt != null) { @@ -194,7 +195,7 @@ public DeviceReportRespDTO checkDeviceActive(String macAddress, String clientId, log.error("生成MQTT配置失败: {}", e.getMessage()); } } - + if (deviceById != null) { // 如果设备存在，则异步更新上次连接时间和版本信息 String appVersion = deviceReport.getApplication() != null ? deviceReport.getApplication().getVersion() @@ -459,7 +460,8 @@ public void manualAddDevice(Long userId, DeviceManualAddDTO dto) { /** * 生成MQTT密码签名 - * @param content 签名内容 (clientId + '|' + username) + * + * @param content 签名内容 (clientId + '|' + username) * @param secretKey 密钥 * @return Base64编码的HMAC-SHA256签名 */ @@ -473,19 +475,16 @@ private String generatePasswordSignature(String content, String secretKey) throw /** * 构建MQTT配置信息 + * * @param macAddress MAC地址 - * @param clientId 客户端ID (UUID) - * @param device 设备信息 + * @param clientId 客户端ID (UUID) + * @param device 设备信息 * @return MQTT配置对象 */ - private DeviceReportRespDTO.MQTT buildMqttConfig(String macAddress, String clientId, DeviceEntity device) throws Exception { + private DeviceReportRespDTO.MQTT buildMqttConfig(String macAddress, String clientId, DeviceEntity device) + throws Exception { // 从环境变量或系统参数获取签名密钥 - String signatureKey = System.getenv("MQTT_SIGNATURE_KEY"); - if (StringUtils.isBlank(signatureKey)) { - // 如果环境变量没有，尝试从系统参数获取 - signatureKey = sysParamsService.getValue("mqtt.signature_key", false); - } - + String signatureKey = sysParamsService.getValue("server.mqtt_signature_key", false); if (StringUtils.isBlank(signatureKey)) { log.warn("缺少MQTT_SIGNATURE_KEY，跳过MQTT配置生成"); return null; @@ -500,7 +499,8 @@ private DeviceReportRespDTO.MQTT buildMqttConfig(String macAddress, String clien Map userData = new HashMap<>(); // 尝试获取客户端IP try { - ServletRequestAttributes attributes = (ServletRequestAttributes) RequestContextHolder.getRequestAttributes(); + ServletRequestAttributes attributes = (ServletRequestAttributes) RequestContextHolder + .getRequestAttributes(); if (attributes != null) { HttpServletRequest request = attributes.getRequest(); String clientIp = request.getRemoteAddr(); diff --git a/main/manager-api/src/main/resources/db/changelog/202509080921.sql b/main/manager-api/src/main/resources/db/changelog/202509080921.sql deleted file mode 100644 index 4d89ee57d..000000000 --- a/main/manager-api/src/main/resources/db/changelog/202509080921.sql +++ /dev/null @@ -1,7 +0,0 @@ -delete from `sys_params` where id = 108; -delete from `sys_params` where param_code = 'server.mqtt_gateway'; -INSERT INTO `sys_params` (id, param_code, param_value, value_type, param_type, remark) VALUES (108, 'server.mqtt_gateway', 'null', 'string', 1, 'mqtt gateway 配置'); - -delete from `sys_params` where param_code = 'server.udp_gateway'; -INSERT INTO `sys_params` (id, param_code, param_value, value_type, param_type, remark) VALUES (109, 'server.udp_gateway', 'null', 'string', 1, 'udp gateway 配置'); - diff --git a/main/manager-api/src/main/resources/db/changelog/202509080922.sql b/main/manager-api/src/main/resources/db/changelog/202509080922.sql new file mode 100644 index 000000000..424b3cd85 --- /dev/null +++ b/main/manager-api/src/main/resources/db/changelog/202509080922.sql @@ -0,0 +1,8 @@ +delete from `sys_params` where param_code = 'server.mqtt_gateway'; +INSERT INTO `sys_params` (id, param_code, param_value, value_type, param_type, remark) VALUES (116, 'server.mqtt_gateway', 'null', 'string', 1, 'mqtt gateway 配置'); + +delete from `sys_params` where param_code = 'server.mqtt_signature_key'; +INSERT INTO `sys_params` (id, param_code, param_value, value_type, param_type, remark) VALUES (117, 'server.mqtt_signature_key', 'null', 'string', 1, 'mqtt 密钥 配置'); + +delete from `sys_params` where param_code = 'server.udp_gateway'; +INSERT INTO `sys_params` (id, param_code, param_value, value_type, param_type, remark) VALUES (118, 'server.udp_gateway', 'null', 'string', 1, 'udp gateway 配置'); \ No newline at end of file diff --git a/main/manager-api/src/main/resources/db/changelog/202509080927.sql b/main/manager-api/src/main/resources/db/changelog/202509080927.sql deleted file mode 100644 index 4741af88e..000000000 --- a/main/manager-api/src/main/resources/db/changelog/202509080927.sql +++ /dev/null @@ -1,2 +0,0 @@ -delete from `sys_params` where param_code = 'mqtt.signature_key'; -INSERT INTO `sys_params` (id, param_code, param_value, value_type, param_type, remark) VALUES (120, 'mqtt.signature_key', 'null', 'string', 1, 'mqtt 密钥 配置'); diff --git a/main/manager-api/src/main/resources/db/changelog/db.changelog-master.yaml b/main/manager-api/src/main/resources/db/changelog/db.changelog-master.yaml index ed66ebb78..c355a16a4 100755 --- a/main/manager-api/src/main/resources/db/changelog/db.changelog-master.yaml +++ b/main/manager-api/src/main/resources/db/changelog/db.changelog-master.yaml @@ -305,17 +305,9 @@ databaseChangeLog: path: classpath:db/changelog/202508131557.sql - changeSet: - id: 202509080921 - author: fan + id: 202509080922 + author: fyb changes: - sqlFile: encoding: utf8 - path: classpath:db/changelog/202509080921.sql - - - changeSet: - id: 202509080927 - author: fan - changes: - - sqlFile: - encoding: utf8 - path: classpath:db/changelog/202509080927.sql \ No newline at end of file + path: classpath:db/changelog/202509080922.sql \ No newline at end of file From f81be335397b07749bc5bd713fd6590a3245d409 Mon Sep 17 00:00:00 2001 From: FAN-yeB <1442100690@qq.com> Date: Wed, 10 Sep 2025 17:46:55 +0800 Subject: [PATCH 10/14] =?UTF-8?q?=E4=BC=98=E5=8C=96mqtt=E5=90=AF=E7=94=A8?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- main/xiaozhi-server/core/connection.py | 9 ++++----- .../xiaozhi-server/core/handle/sendAudioHandle.py | 15 ++++++--------- 2 files changed, 10 insertions(+), 14 deletions(-) diff --git a/main/xiaozhi-server/core/connection.py b/main/xiaozhi-server/core/connection.py index e23826a11..9f1348b70 100644 --- a/main/xiaozhi-server/core/connection.py +++ b/main/xiaozhi-server/core/connection.py @@ -282,14 +282,13 @@ async def _route_message(self, message): if self.asr is None: return - # 检查是否需要处理头部（只有当mqtt_gateway有实际值时才处理头部） - mqtt_gateway = self.config.get("server", {}).get("mqtt_gateway") - # 当mqtt_gateway为None, "null", "", 或实际的null值时，不处理头部 - need_header_processing = mqtt_gateway and mqtt_gateway not in [None, "null", ""] and str(mqtt_gateway).strip() != "" + # 检查是否需要处理头部（只有当websocket URL以"?from=mqtt"为结尾时才处理头部） + request_path = self.websocket.request.path + need_header_processing = request_path.endswith("?from=mqtt") # 调试日志：首次连接时记录配置 if not hasattr(self, '_logged_mqtt_config'): - self.logger.bind(tag=TAG).info(f"MQTT Gateway配置: '{mqtt_gateway}', 头部处理: {need_header_processing}") + self.logger.bind(tag=TAG).info(f"WebSocket URL路径: '{request_path}', 头部处理: {need_header_processing}") self._logged_mqtt_config = True if need_header_processing and len(message) >= 16: diff --git a/main/xiaozhi-server/core/handle/sendAudioHandle.py b/main/xiaozhi-server/core/handle/sendAudioHandle.py index 56d219322..df18f2a08 100644 --- a/main/xiaozhi-server/core/handle/sendAudioHandle.py +++ b/main/xiaozhi-server/core/handle/sendAudioHandle.py @@ -42,11 +42,9 @@ async def sendAudio(conn, audios, frame_duration=60): """ if audios is None or len(audios) == 0: return - - # 检查是否需要添加头部（只有当mqtt_gateway有实际值时才添加头部） - mqtt_gateway = conn.config.get("server", {}).get("mqtt_gateway") - # 当mqtt_gateway为None, "null", "", 或实际的null值时，不添加头部 - need_header = mqtt_gateway and mqtt_gateway not in [None, "null", ""] and str(mqtt_gateway).strip() != "" + # 检查是否需要处理头部（只有当websocket URL以"?from=mqtt"为结尾时才处理头部） + request_path = conn.websocket.request.path + need_header = request_path.endswith("?from=mqtt") if isinstance(audios, bytes): if conn.client_abort: @@ -99,10 +97,9 @@ async def sendAudio(conn, audios, frame_duration=60): start_time = time.perf_counter() play_position = 0 - # 检查是否需要添加头部（只有当mqtt_gateway有实际值时才添加头部） - mqtt_gateway = conn.config.get("server", {}).get("mqtt_gateway") - # 当mqtt_gateway为None, "null", "", 或实际的null值时，不添加头部 - need_header = mqtt_gateway and mqtt_gateway not in [None, "null", ""] and str(mqtt_gateway).strip() != "" + # 检查是否需要添加头部（只有当websocket URL以"?from=mqtt"为结尾时才添加头部） + request_path = conn.websocket.request.path + need_header = request_path.endswith("?from=mqtt") # 执行预缓冲 pre_buffer_frames = min(3, len(audios)) From eea7689c3dcfd8cd88da5466d62eb38620e8456f Mon Sep 17 00:00:00 2001 From: FAN-yeB <1442100690@qq.com> Date: Wed, 10 Sep 2025 17:48:31 +0800 Subject: [PATCH 11/14] Revert "Merge branch 'mqtt' of https://github.com/xinnan-tech/xiaozhi-esp32-server into mqtt" This reverts commit 7b9e34c3e5247ddd2bab613ff05216d7c7c15b4d, reversing changes made to f81be335397b07749bc5bd713fd6590a3245d409. --- .../service/impl/DeviceServiceImpl.java | 30 +++++++++---------- .../resources/db/changelog/202509080921.sql | 7 +++++ .../resources/db/changelog/202509080922.sql | 8 ----- .../resources/db/changelog/202509080927.sql | 2 ++ .../db/changelog/db.changelog-master.yaml | 14 +++++++-- 5 files changed, 35 insertions(+), 26 deletions(-) create mode 100644 main/manager-api/src/main/resources/db/changelog/202509080921.sql delete mode 100644 main/manager-api/src/main/resources/db/changelog/202509080922.sql create mode 100644 main/manager-api/src/main/resources/db/changelog/202509080927.sql diff --git a/main/manager-api/src/main/java/xiaozhi/modules/device/service/impl/DeviceServiceImpl.java b/main/manager-api/src/main/java/xiaozhi/modules/device/service/impl/DeviceServiceImpl.java index 663800417..b4343276d 100644 --- a/main/manager-api/src/main/java/xiaozhi/modules/device/service/impl/DeviceServiceImpl.java +++ b/main/manager-api/src/main/java/xiaozhi/modules/device/service/impl/DeviceServiceImpl.java @@ -9,7 +9,6 @@ import java.util.Map; import java.util.TimeZone; import java.util.UUID; - import javax.crypto.Mac; import javax.crypto.spec.SecretKeySpec; @@ -38,7 +37,6 @@ import xiaozhi.common.utils.ConvertUtils; import xiaozhi.common.utils.DateUtils; import xiaozhi.modules.device.dao.DeviceDao; -import xiaozhi.modules.device.dto.DeviceManualAddDTO; import xiaozhi.modules.device.dto.DevicePageUserDTO; import xiaozhi.modules.device.dto.DeviceReportReqDTO; import xiaozhi.modules.device.dto.DeviceReportRespDTO; @@ -50,6 +48,7 @@ import xiaozhi.modules.security.user.SecurityUser; import xiaozhi.modules.sys.service.SysParamsService; import xiaozhi.modules.sys.service.SysUserUtilService; +import xiaozhi.modules.device.dto.DeviceManualAddDTO; @Slf4j @Service @@ -180,11 +179,11 @@ public DeviceReportRespDTO checkDeviceActive(String macAddress, String clientId, } response.setWebsocket(websocket); - + // 添加MQTT UDP配置 // 从系统参数获取MQTT Gateway地址，仅在配置有效时使用 String mqttUdpConfig = sysParamsService.getValue(Constant.SERVER_MQTT_GATEWAY, false); - if (mqttUdpConfig != null && !mqttUdpConfig.equals("null") && !mqttUdpConfig.isEmpty() && deviceById != null) { + if(mqttUdpConfig != null && !mqttUdpConfig.equals("null") && !mqttUdpConfig.isEmpty() && deviceById != null) { try { DeviceReportRespDTO.MQTT mqtt = buildMqttConfig(macAddress, clientId, deviceById); if (mqtt != null) { @@ -195,7 +194,7 @@ public DeviceReportRespDTO checkDeviceActive(String macAddress, String clientId, log.error("生成MQTT配置失败: {}", e.getMessage()); } } - + if (deviceById != null) { // 如果设备存在，则异步更新上次连接时间和版本信息 String appVersion = deviceReport.getApplication() != null ? deviceReport.getApplication().getVersion() @@ -460,8 +459,7 @@ public void manualAddDevice(Long userId, DeviceManualAddDTO dto) { /** * 生成MQTT密码签名 - * - * @param content 签名内容 (clientId + '|' + username) + * @param content 签名内容 (clientId + '|' + username) * @param secretKey 密钥 * @return Base64编码的HMAC-SHA256签名 */ @@ -475,16 +473,19 @@ private String generatePasswordSignature(String content, String secretKey) throw /** * 构建MQTT配置信息 - * * @param macAddress MAC地址 - * @param clientId 客户端ID (UUID) - * @param device 设备信息 + * @param clientId 客户端ID (UUID) + * @param device 设备信息 * @return MQTT配置对象 */ - private DeviceReportRespDTO.MQTT buildMqttConfig(String macAddress, String clientId, DeviceEntity device) - throws Exception { + private DeviceReportRespDTO.MQTT buildMqttConfig(String macAddress, String clientId, DeviceEntity device) throws Exception { // 从环境变量或系统参数获取签名密钥 - String signatureKey = sysParamsService.getValue("server.mqtt_signature_key", false); + String signatureKey = System.getenv("MQTT_SIGNATURE_KEY"); + if (StringUtils.isBlank(signatureKey)) { + // 如果环境变量没有，尝试从系统参数获取 + signatureKey = sysParamsService.getValue("mqtt.signature_key", false); + } + if (StringUtils.isBlank(signatureKey)) { log.warn("缺少MQTT_SIGNATURE_KEY，跳过MQTT配置生成"); return null; @@ -499,8 +500,7 @@ private DeviceReportRespDTO.MQTT buildMqttConfig(String macAddress, String clien Map userData = new HashMap<>(); // 尝试获取客户端IP try { - ServletRequestAttributes attributes = (ServletRequestAttributes) RequestContextHolder - .getRequestAttributes(); + ServletRequestAttributes attributes = (ServletRequestAttributes) RequestContextHolder.getRequestAttributes(); if (attributes != null) { HttpServletRequest request = attributes.getRequest(); String clientIp = request.getRemoteAddr(); diff --git a/main/manager-api/src/main/resources/db/changelog/202509080921.sql b/main/manager-api/src/main/resources/db/changelog/202509080921.sql new file mode 100644 index 000000000..4d89ee57d --- /dev/null +++ b/main/manager-api/src/main/resources/db/changelog/202509080921.sql @@ -0,0 +1,7 @@ +delete from `sys_params` where id = 108; +delete from `sys_params` where param_code = 'server.mqtt_gateway'; +INSERT INTO `sys_params` (id, param_code, param_value, value_type, param_type, remark) VALUES (108, 'server.mqtt_gateway', 'null', 'string', 1, 'mqtt gateway 配置'); + +delete from `sys_params` where param_code = 'server.udp_gateway'; +INSERT INTO `sys_params` (id, param_code, param_value, value_type, param_type, remark) VALUES (109, 'server.udp_gateway', 'null', 'string', 1, 'udp gateway 配置'); + diff --git a/main/manager-api/src/main/resources/db/changelog/202509080922.sql b/main/manager-api/src/main/resources/db/changelog/202509080922.sql deleted file mode 100644 index 424b3cd85..000000000 --- a/main/manager-api/src/main/resources/db/changelog/202509080922.sql +++ /dev/null @@ -1,8 +0,0 @@ -delete from `sys_params` where param_code = 'server.mqtt_gateway'; -INSERT INTO `sys_params` (id, param_code, param_value, value_type, param_type, remark) VALUES (116, 'server.mqtt_gateway', 'null', 'string', 1, 'mqtt gateway 配置'); - -delete from `sys_params` where param_code = 'server.mqtt_signature_key'; -INSERT INTO `sys_params` (id, param_code, param_value, value_type, param_type, remark) VALUES (117, 'server.mqtt_signature_key', 'null', 'string', 1, 'mqtt 密钥 配置'); - -delete from `sys_params` where param_code = 'server.udp_gateway'; -INSERT INTO `sys_params` (id, param_code, param_value, value_type, param_type, remark) VALUES (118, 'server.udp_gateway', 'null', 'string', 1, 'udp gateway 配置'); \ No newline at end of file diff --git a/main/manager-api/src/main/resources/db/changelog/202509080927.sql b/main/manager-api/src/main/resources/db/changelog/202509080927.sql new file mode 100644 index 000000000..4741af88e --- /dev/null +++ b/main/manager-api/src/main/resources/db/changelog/202509080927.sql @@ -0,0 +1,2 @@ +delete from `sys_params` where param_code = 'mqtt.signature_key'; +INSERT INTO `sys_params` (id, param_code, param_value, value_type, param_type, remark) VALUES (120, 'mqtt.signature_key', 'null', 'string', 1, 'mqtt 密钥 配置'); diff --git a/main/manager-api/src/main/resources/db/changelog/db.changelog-master.yaml b/main/manager-api/src/main/resources/db/changelog/db.changelog-master.yaml index c355a16a4..ed66ebb78 100755 --- a/main/manager-api/src/main/resources/db/changelog/db.changelog-master.yaml +++ b/main/manager-api/src/main/resources/db/changelog/db.changelog-master.yaml @@ -305,9 +305,17 @@ databaseChangeLog: path: classpath:db/changelog/202508131557.sql - changeSet: - id: 202509080922 - author: fyb + id: 202509080921 + author: fan changes: - sqlFile: encoding: utf8 - path: classpath:db/changelog/202509080922.sql \ No newline at end of file + path: classpath:db/changelog/202509080921.sql + + - changeSet: + id: 202509080927 + author: fan + changes: + - sqlFile: + encoding: utf8 + path: classpath:db/changelog/202509080927.sql \ No newline at end of file From c93cb3bb52e272f25398e40e8628fdbf6f035532 Mon Sep 17 00:00:00 2001 From: FAN-yeB <1442100690@qq.com> Date: Wed, 10 Sep 2025 17:51:45 +0800 Subject: [PATCH 12/14] Reapply "Merge branch 'mqtt' of https://github.com/xinnan-tech/xiaozhi-esp32-server into mqtt" This reverts commit eea7689c3dcfd8cd88da5466d62eb38620e8456f. --- .../service/impl/DeviceServiceImpl.java | 30 +++++++++---------- .../resources/db/changelog/202509080921.sql | 7 ----- .../resources/db/changelog/202509080922.sql | 8 +++++ .../resources/db/changelog/202509080927.sql | 2 -- .../db/changelog/db.changelog-master.yaml | 14 ++------- 5 files changed, 26 insertions(+), 35 deletions(-) delete mode 100644 main/manager-api/src/main/resources/db/changelog/202509080921.sql create mode 100644 main/manager-api/src/main/resources/db/changelog/202509080922.sql delete mode 100644 main/manager-api/src/main/resources/db/changelog/202509080927.sql diff --git a/main/manager-api/src/main/java/xiaozhi/modules/device/service/impl/DeviceServiceImpl.java b/main/manager-api/src/main/java/xiaozhi/modules/device/service/impl/DeviceServiceImpl.java index b4343276d..663800417 100644 --- a/main/manager-api/src/main/java/xiaozhi/modules/device/service/impl/DeviceServiceImpl.java +++ b/main/manager-api/src/main/java/xiaozhi/modules/device/service/impl/DeviceServiceImpl.java @@ -9,6 +9,7 @@ import java.util.Map; import java.util.TimeZone; import java.util.UUID; + import javax.crypto.Mac; import javax.crypto.spec.SecretKeySpec; @@ -37,6 +38,7 @@ import xiaozhi.common.utils.ConvertUtils; import xiaozhi.common.utils.DateUtils; import xiaozhi.modules.device.dao.DeviceDao; +import xiaozhi.modules.device.dto.DeviceManualAddDTO; import xiaozhi.modules.device.dto.DevicePageUserDTO; import xiaozhi.modules.device.dto.DeviceReportReqDTO; import xiaozhi.modules.device.dto.DeviceReportRespDTO; @@ -48,7 +50,6 @@ import xiaozhi.modules.security.user.SecurityUser; import xiaozhi.modules.sys.service.SysParamsService; import xiaozhi.modules.sys.service.SysUserUtilService; -import xiaozhi.modules.device.dto.DeviceManualAddDTO; @Slf4j @Service @@ -179,11 +180,11 @@ public DeviceReportRespDTO checkDeviceActive(String macAddress, String clientId, } response.setWebsocket(websocket); - + // 添加MQTT UDP配置 // 从系统参数获取MQTT Gateway地址，仅在配置有效时使用 String mqttUdpConfig = sysParamsService.getValue(Constant.SERVER_MQTT_GATEWAY, false); - if(mqttUdpConfig != null && !mqttUdpConfig.equals("null") && !mqttUdpConfig.isEmpty() && deviceById != null) { + if (mqttUdpConfig != null && !mqttUdpConfig.equals("null") && !mqttUdpConfig.isEmpty() && deviceById != null) { try { DeviceReportRespDTO.MQTT mqtt = buildMqttConfig(macAddress, clientId, deviceById); if (mqtt != null) { @@ -194,7 +195,7 @@ public DeviceReportRespDTO checkDeviceActive(String macAddress, String clientId, log.error("生成MQTT配置失败: {}", e.getMessage()); } } - + if (deviceById != null) { // 如果设备存在，则异步更新上次连接时间和版本信息 String appVersion = deviceReport.getApplication() != null ? deviceReport.getApplication().getVersion() @@ -459,7 +460,8 @@ public void manualAddDevice(Long userId, DeviceManualAddDTO dto) { /** * 生成MQTT密码签名 - * @param content 签名内容 (clientId + '|' + username) + * + * @param content 签名内容 (clientId + '|' + username) * @param secretKey 密钥 * @return Base64编码的HMAC-SHA256签名 */ @@ -473,19 +475,16 @@ private String generatePasswordSignature(String content, String secretKey) throw /** * 构建MQTT配置信息 + * * @param macAddress MAC地址 - * @param clientId 客户端ID (UUID) - * @param device 设备信息 + * @param clientId 客户端ID (UUID) + * @param device 设备信息 * @return MQTT配置对象 */ - private DeviceReportRespDTO.MQTT buildMqttConfig(String macAddress, String clientId, DeviceEntity device) throws Exception { + private DeviceReportRespDTO.MQTT buildMqttConfig(String macAddress, String clientId, DeviceEntity device) + throws Exception { // 从环境变量或系统参数获取签名密钥 - String signatureKey = System.getenv("MQTT_SIGNATURE_KEY"); - if (StringUtils.isBlank(signatureKey)) { - // 如果环境变量没有，尝试从系统参数获取 - signatureKey = sysParamsService.getValue("mqtt.signature_key", false); - } - + String signatureKey = sysParamsService.getValue("server.mqtt_signature_key", false); if (StringUtils.isBlank(signatureKey)) { log.warn("缺少MQTT_SIGNATURE_KEY，跳过MQTT配置生成"); return null; @@ -500,7 +499,8 @@ private DeviceReportRespDTO.MQTT buildMqttConfig(String macAddress, String clien Map userData = new HashMap<>(); // 尝试获取客户端IP try { - ServletRequestAttributes attributes = (ServletRequestAttributes) RequestContextHolder.getRequestAttributes(); + ServletRequestAttributes attributes = (ServletRequestAttributes) RequestContextHolder + .getRequestAttributes(); if (attributes != null) { HttpServletRequest request = attributes.getRequest(); String clientIp = request.getRemoteAddr(); diff --git a/main/manager-api/src/main/resources/db/changelog/202509080921.sql b/main/manager-api/src/main/resources/db/changelog/202509080921.sql deleted file mode 100644 index 4d89ee57d..000000000 --- a/main/manager-api/src/main/resources/db/changelog/202509080921.sql +++ /dev/null @@ -1,7 +0,0 @@ -delete from `sys_params` where id = 108; -delete from `sys_params` where param_code = 'server.mqtt_gateway'; -INSERT INTO `sys_params` (id, param_code, param_value, value_type, param_type, remark) VALUES (108, 'server.mqtt_gateway', 'null', 'string', 1, 'mqtt gateway 配置'); - -delete from `sys_params` where param_code = 'server.udp_gateway'; -INSERT INTO `sys_params` (id, param_code, param_value, value_type, param_type, remark) VALUES (109, 'server.udp_gateway', 'null', 'string', 1, 'udp gateway 配置'); - diff --git a/main/manager-api/src/main/resources/db/changelog/202509080922.sql b/main/manager-api/src/main/resources/db/changelog/202509080922.sql new file mode 100644 index 000000000..424b3cd85 --- /dev/null +++ b/main/manager-api/src/main/resources/db/changelog/202509080922.sql @@ -0,0 +1,8 @@ +delete from `sys_params` where param_code = 'server.mqtt_gateway'; +INSERT INTO `sys_params` (id, param_code, param_value, value_type, param_type, remark) VALUES (116, 'server.mqtt_gateway', 'null', 'string', 1, 'mqtt gateway 配置'); + +delete from `sys_params` where param_code = 'server.mqtt_signature_key'; +INSERT INTO `sys_params` (id, param_code, param_value, value_type, param_type, remark) VALUES (117, 'server.mqtt_signature_key', 'null', 'string', 1, 'mqtt 密钥 配置'); + +delete from `sys_params` where param_code = 'server.udp_gateway'; +INSERT INTO `sys_params` (id, param_code, param_value, value_type, param_type, remark) VALUES (118, 'server.udp_gateway', 'null', 'string', 1, 'udp gateway 配置'); \ No newline at end of file diff --git a/main/manager-api/src/main/resources/db/changelog/202509080927.sql b/main/manager-api/src/main/resources/db/changelog/202509080927.sql deleted file mode 100644 index 4741af88e..000000000 --- a/main/manager-api/src/main/resources/db/changelog/202509080927.sql +++ /dev/null @@ -1,2 +0,0 @@ -delete from `sys_params` where param_code = 'mqtt.signature_key'; -INSERT INTO `sys_params` (id, param_code, param_value, value_type, param_type, remark) VALUES (120, 'mqtt.signature_key', 'null', 'string', 1, 'mqtt 密钥 配置'); diff --git a/main/manager-api/src/main/resources/db/changelog/db.changelog-master.yaml b/main/manager-api/src/main/resources/db/changelog/db.changelog-master.yaml index ed66ebb78..c355a16a4 100755 --- a/main/manager-api/src/main/resources/db/changelog/db.changelog-master.yaml +++ b/main/manager-api/src/main/resources/db/changelog/db.changelog-master.yaml @@ -305,17 +305,9 @@ databaseChangeLog: path: classpath:db/changelog/202508131557.sql - changeSet: - id: 202509080921 - author: fan + id: 202509080922 + author: fyb changes: - sqlFile: encoding: utf8 - path: classpath:db/changelog/202509080921.sql - - - changeSet: - id: 202509080927 - author: fan - changes: - - sqlFile: - encoding: utf8 - path: classpath:db/changelog/202509080927.sql \ No newline at end of file + path: classpath:db/changelog/202509080922.sql \ No newline at end of file From 5bd78cac1a44fcbdcfadd48dc8ac3b7d09243f8d Mon Sep 17 00:00:00 2001 From: hrz <1710360675@qq.com> Date: Wed, 10 Sep 2025 21:40:25 +0800 Subject: [PATCH 13/14] =?UTF-8?q?update:=E4=BC=98=E5=8C=96=E4=BB=A3?= =?UTF-8?q?=E7=A0=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- main/xiaozhi-server/config.yaml | 2 - main/xiaozhi-server/core/connection.py | 125 +++++++++--------- .../core/handle/receiveAudioHandle.py | 38 +++--- .../core/handle/sendAudioHandle.py | 100 +++++++------- 4 files changed, 133 insertions(+), 132 deletions(-) diff --git a/main/xiaozhi-server/config.yaml b/main/xiaozhi-server/config.yaml index 1afc3530b..c0aa3f934 100644 --- a/main/xiaozhi-server/config.yaml +++ b/main/xiaozhi-server/config.yaml @@ -25,8 +25,6 @@ server: # 所以如果你使用docker部署时，将vision_explain设置成局域网地址 # 如果你使用公网部署时，将vision_explain设置成公网地址 vision_explain: http://你的ip或者域名:端口号/mcp/vision/explain - # mqtt网关地址，当这个值为null时，mqtt网关桥接功能不开启，使用websocket双向通信，不使用mqtt和udp协议 - mqtt_gateway: null # OTA返回信息时区偏移量 timezone_offset: +8 # 认证配置 diff --git a/main/xiaozhi-server/core/connection.py b/main/xiaozhi-server/core/connection.py index 9f1348b70..d80ec8622 100644 --- a/main/xiaozhi-server/core/connection.py +++ b/main/xiaozhi-server/core/connection.py @@ -154,6 +154,9 @@ def __init__( # {"mcp":true} 表示启用MCP功能 self.features = None + # 标记连接是否来自MQTT + self.conn_from_mqtt_gateway = False + # 初始化提示词管理器 self.prompt_manager = PromptManager(config, self.logger) @@ -198,6 +201,13 @@ async def handle_connection(self, ws): self.websocket = ws self.device_id = self.headers.get("device-id", None) + # 检查是否来自MQTT连接 + request_path = ws.request.path + self.conn_from_mqtt_gateway = request_path.endswith("?from=mqtt") + self.logger.bind(tag=TAG).info( + f"WebSocket URL路径: '{request_path}', 来自MQTT连接: {self.conn_from_mqtt_gateway}" + ) + # 初始化活动时间戳 self.last_activity_time = time.time() * 1000 @@ -277,57 +287,64 @@ async def _route_message(self, message): if isinstance(message, str): await handleTextMessage(self, message) elif isinstance(message, bytes): - if self.vad is None: - return - if self.asr is None: + if self.vad is None or self.asr is None: return - - # 检查是否需要处理头部（只有当websocket URL以"?from=mqtt"为结尾时才处理头部） - request_path = self.websocket.request.path - need_header_processing = request_path.endswith("?from=mqtt") - - # 调试日志：首次连接时记录配置 - if not hasattr(self, '_logged_mqtt_config'): - self.logger.bind(tag=TAG).info(f"WebSocket URL路径: '{request_path}', 头部处理: {need_header_processing}") - self._logged_mqtt_config = True - - if need_header_processing and len(message) >= 16: - try: - timestamp = int.from_bytes(message[8:12], 'big') - audio_length = int.from_bytes(message[12:16], 'big') - - - # 提取音频数据 - if audio_length > 0 and len(message) >= 16 + audio_length: - audio_data = message[16:16 + audio_length] - - # 基于时间戳进行简单排序 - self._process_websocket_audio(audio_data, timestamp) - return - elif len(message) > 16: - # 去掉16字节头部 - audio_data = message[16:] - self.asr_audio_queue.put(audio_data) - return - except Exception as e: - self.logger.bind(tag=TAG).error(f"解析WebSocket音频包失败: {e}") - + + # 处理来自MQTT网关的音频包 + if self.conn_from_mqtt_gateway and len(message) >= 16: + handled = await self._process_mqtt_audio_message(message) + if handled: + return + # 不需要头部处理或没有头部时，直接处理原始消息 self.asr_audio_queue.put(message) + async def _process_mqtt_audio_message(self, message): + """ + 处理来自MQTT网关的音频消息，解析16字节头部并提取音频数据 + + Args: + message: 包含头部的音频消息 + + Returns: + bool: 是否成功处理了消息 + """ + try: + # 提取头部信息 + timestamp = int.from_bytes(message[8:12], "big") + audio_length = int.from_bytes(message[12:16], "big") + + # 提取音频数据 + if audio_length > 0 and len(message) >= 16 + audio_length: + # 有指定长度，提取精确的音频数据 + audio_data = message[16 : 16 + audio_length] + # 基于时间戳进行排序处理 + self._process_websocket_audio(audio_data, timestamp) + return True + elif len(message) > 16: + # 没有指定长度或长度无效，去掉头部后处理剩余数据 + audio_data = message[16:] + self.asr_audio_queue.put(audio_data) + return True + except Exception as e: + self.logger.bind(tag=TAG).error(f"解析WebSocket音频包失败: {e}") + + # 处理失败，返回False表示需要继续处理 + return False + def _process_websocket_audio(self, audio_data, timestamp): """处理WebSocket格式的音频包""" # 初始化时间戳序列管理 - if not hasattr(self, 'audio_timestamp_buffer'): + if not hasattr(self, "audio_timestamp_buffer"): self.audio_timestamp_buffer = {} self.last_processed_timestamp = 0 self.max_timestamp_buffer_size = 20 - + # 如果时间戳是递增的，直接处理 if timestamp >= self.last_processed_timestamp: self.asr_audio_queue.put(audio_data) self.last_processed_timestamp = timestamp - + # 处理缓冲区中的后续包 processed_any = True while processed_any: @@ -346,30 +363,6 @@ def _process_websocket_audio(self, audio_data, timestamp): else: self.asr_audio_queue.put(audio_data) - def _process_sequenced_audio(self, audio_data, sequence, timestamp): - """处理有序的音频包""" - # 初始化音频缓冲区 - if not hasattr(self, 'audio_buffer'): - self.audio_buffer = {} - self.expected_sequence = sequence - self.max_buffer_size = 20 # 最大缓冲20个包 - - # 如果是下一个期望的包，直接处理 - if sequence == self.expected_sequence: - self.asr_audio_queue.put(audio_data) - self.expected_sequence += 1 - - # 检查缓冲区中是否有后续的连续包 - while self.expected_sequence in self.audio_buffer: - buffered_audio = self.audio_buffer.pop(self.expected_sequence) - self.asr_audio_queue.put(buffered_audio) - self.expected_sequence += 1 - - elif sequence > self.expected_sequence: - # 乱序包，暂存到缓冲区 - if len(self.audio_buffer) < self.max_buffer_size: - self.audio_buffer[sequence] = audio_data - async def handle_restart(self, message): """处理服务器重启请求""" try: @@ -940,7 +933,11 @@ def _handle_function_result(self, result, function_call_data, depth): { "id": function_id, "function": { - "arguments": "{}" if function_arguments == "" else function_arguments, + "arguments": ( + "{}" + if function_arguments == "" + else function_arguments + ), "name": function_name, }, "type": "function", @@ -1009,9 +1006,9 @@ async def close(self, ws=None): """资源清理方法""" try: # 清理音频缓冲区 - if hasattr(self, 'audio_buffer'): + if hasattr(self, "audio_buffer"): self.audio_buffer.clear() - + # 取消超时任务 if self.timeout_task and not self.timeout_task.done(): self.timeout_task.cancel() diff --git a/main/xiaozhi-server/core/handle/receiveAudioHandle.py b/main/xiaozhi-server/core/handle/receiveAudioHandle.py index e6f396327..8b6359091 100644 --- a/main/xiaozhi-server/core/handle/receiveAudioHandle.py +++ b/main/xiaozhi-server/core/handle/receiveAudioHandle.py @@ -1,12 +1,11 @@ -from core.handle.sendAudioHandle import send_stt_message -from core.handle.intentHandler import handle_user_intent -from core.utils.output_counter import check_device_output_limit -from core.handle.abortHandle import handleAbortMessage import time -import asyncio import json -from core.handle.sendAudioHandle import SentenceType +import asyncio from core.utils.util import audio_to_data +from core.handle.abortHandle import handleAbortMessage +from core.handle.intentHandler import handle_user_intent +from core.utils.output_counter import check_device_output_limit +from core.handle.sendAudioHandle import send_stt_message, SentenceType TAG = __name__ @@ -22,7 +21,6 @@ async def handleAudioMessage(conn, audio): if not hasattr(conn, "vad_resume_task") or conn.vad_resume_task.done(): conn.vad_resume_task = asyncio.create_task(resume_vad_detection(conn)) return - if have_voice: if conn.client_is_speaking: await handleAbortMessage(conn) @@ -42,22 +40,22 @@ async def startToChat(conn, text): # 检查输入是否是JSON格式（包含说话人信息） speaker_name = None actual_text = text - + try: # 尝试解析JSON格式的输入 - if text.strip().startswith('{') and text.strip().endswith('}'): + if text.strip().startswith("{") and text.strip().endswith("}"): data = json.loads(text) - if 'speaker' in data and 'content' in data: - speaker_name = data['speaker'] - actual_text = data['content'] + if "speaker" in data and "content" in data: + speaker_name = data["speaker"] + actual_text = data["content"] conn.logger.bind(tag=TAG).info(f"解析到说话人信息: {speaker_name}") - + # 直接使用JSON格式的文本，不解析 actual_text = text except (json.JSONDecodeError, KeyError): # 如果解析失败，继续使用原始文本 pass - + # 保存说话人信息到连接对象 if speaker_name: conn.current_speaker = speaker_name @@ -118,10 +116,12 @@ async def no_voice_close_connect(conn, have_voice): async def max_out_size(conn): + # 播放超出最大输出字数的提示 + conn.client_abort = False text = "不好意思，我现在有点事情要忙，明天这个时候我们再聊，约好了哦！明天不见不散，拜拜！" await send_stt_message(conn, text) file_path = "config/assets/max_output_size.wav" - opus_packets, _ = audio_to_data(file_path) + opus_packets = audio_to_data(file_path) conn.tts.tts_audio_queue.put((SentenceType.LAST, opus_packets, text)) conn.close_after_chat = True @@ -140,7 +140,7 @@ async def check_bind_device(conn): # 播放提示音 music_path = "config/assets/bind_code.wav" - opus_packets, _ = audio_to_data(music_path) + opus_packets = audio_to_data(music_path) conn.tts.tts_audio_queue.put((SentenceType.FIRST, opus_packets, text)) # 逐个播放数字 @@ -148,15 +148,17 @@ async def check_bind_device(conn): try: digit = conn.bind_code[i] num_path = f"config/assets/bind_code/{digit}.wav" - num_packets, _ = audio_to_data(num_path) + num_packets = audio_to_data(num_path) conn.tts.tts_audio_queue.put((SentenceType.MIDDLE, num_packets, None)) except Exception as e: conn.logger.bind(tag=TAG).error(f"播放数字音频失败: {e}") continue conn.tts.tts_audio_queue.put((SentenceType.LAST, [], None)) else: + # 播放未绑定提示 + conn.client_abort = False text = f"没有找到该设备的版本信息，请正确配置 OTA地址，然后重新编译固件。" await send_stt_message(conn, text) music_path = "config/assets/bind_not_found.wav" - opus_packets, _ = audio_to_data(music_path) + opus_packets = audio_to_data(music_path) conn.tts.tts_audio_queue.put((SentenceType.LAST, opus_packets, text)) diff --git a/main/xiaozhi-server/core/handle/sendAudioHandle.py b/main/xiaozhi-server/core/handle/sendAudioHandle.py index df18f2a08..810e8803f 100644 --- a/main/xiaozhi-server/core/handle/sendAudioHandle.py +++ b/main/xiaozhi-server/core/handle/sendAudioHandle.py @@ -30,6 +30,28 @@ async def sendAudioMessage(conn, sentenceType, audios, text): await conn.close() +async def _send_to_mqtt_gateway(conn, opus_packet, timestamp, sequence): + """ + 发送带16字节头部的opus数据包给mqtt_gateway + Args: + conn: 连接对象 + opus_packet: opus数据包 + timestamp: 时间戳 + sequence: 序列号 + """ + # 为opus数据包添加16字节头部 + header = bytearray(16) + header[0] = 1 # type + header[2:4] = len(opus_packet).to_bytes(2, "big") # payload length + header[4:8] = sequence.to_bytes(4, "big") # sequence + header[8:12] = timestamp.to_bytes(4, "big") # 时间戳 + header[12:16] = len(opus_packet).to_bytes(4, "big") # opus长度 + + # 发送包含头部的完整数据包 + complete_packet = bytes(header) + opus_packet + await conn.websocket.send(complete_packet) + + # 播放音频 async def sendAudio(conn, audios, frame_duration=60): """ @@ -42,9 +64,6 @@ async def sendAudio(conn, audios, frame_duration=60): """ if audios is None or len(audios) == 0: return - # 检查是否需要处理头部（只有当websocket URL以"?from=mqtt"为结尾时才处理头部） - request_path = conn.websocket.request.path - need_header = request_path.endswith("?from=mqtt") if isinstance(audios, bytes): if conn.client_abort: @@ -71,19 +90,19 @@ async def sendAudio(conn, audios, frame_duration=60): if delay > 0: await asyncio.sleep(delay) - if need_header: - # 为opus数据包添加16字节头部 - timestamp = int((flow_control["start_time"] + flow_control["packet_count"] * frame_duration / 1000) * 1000) % (2**32) - header = bytearray(16) - header[0] = 1 # type - header[2:4] = len(audios).to_bytes(2, 'big') # payload length - header[4:8] = flow_control["sequence"].to_bytes(4, 'big') # connection id/sequence - header[8:12] = timestamp.to_bytes(4, 'big') # 时间戳 - header[12:16] = len(audios).to_bytes(4, 'big') # opus长度 - - # 发送包含头部的完整数据包 - complete_packet = bytes(header) + audios - await conn.websocket.send(complete_packet) + if conn.conn_from_mqtt_gateway: + # 计算时间戳 + timestamp = int( + ( + flow_control["start_time"] + + flow_control["packet_count"] * frame_duration / 1000 + ) + * 1000 + ) % (2**32) + # 调用通用函数发送带头部的数据包 + await _send_to_mqtt_gateway( + conn, audios, timestamp, flow_control["sequence"] + ) else: # 直接发送opus数据包，不添加头部 await conn.websocket.send(audios) @@ -97,30 +116,21 @@ async def sendAudio(conn, audios, frame_duration=60): start_time = time.perf_counter() play_position = 0 - # 检查是否需要添加头部（只有当websocket URL以"?from=mqtt"为结尾时才添加头部） - request_path = conn.websocket.request.path - need_header = request_path.endswith("?from=mqtt") - # 执行预缓冲 pre_buffer_frames = min(3, len(audios)) for i in range(pre_buffer_frames): - if need_header: - # 为预缓冲包添加头部 - timestamp = int((start_time + i * frame_duration / 1000) * 1000) % (2**32) - header = bytearray(16) - header[0] = 1 # type - header[2:4] = len(audios[i]).to_bytes(2, 'big') # payload length - header[4:8] = i.to_bytes(4, 'big') # sequence - header[8:12] = timestamp.to_bytes(4, 'big') # 时间戳 - header[12:16] = len(audios[i]).to_bytes(4, 'big') # opus长度 - - complete_packet = bytes(header) + audios[i] - await conn.websocket.send(complete_packet) + if conn.conn_from_mqtt_gateway: + # 计算时间戳 + timestamp = int((start_time + i * frame_duration / 1000) * 1000) % ( + 2**32 + ) + # 调用通用函数发送带头部的数据包 + await _send_to_mqtt_gateway(conn, audios[i], timestamp, i) else: # 直接发送预缓冲包，不添加头部 await conn.websocket.send(audios[i]) remaining_audios = audios[pre_buffer_frames:] - + # 播放剩余音频帧 for i, opus_packet in enumerate(remaining_audios): if conn.client_abort: @@ -135,25 +145,19 @@ async def sendAudio(conn, audios, frame_duration=60): delay = expected_time - current_time if delay > 0: await asyncio.sleep(delay) - - if need_header: - # 为opus数据包添加16字节头部 (timestamp at offset 8, length at offset 12) - timestamp = int((start_time + play_position / 1000) * 1000) % (2**32) # 使用播放位置计算时间戳 + + if conn.conn_from_mqtt_gateway: + # 计算时间戳和序列号 + timestamp = int((start_time + play_position / 1000) * 1000) % ( + 2**32 + ) # 使用播放位置计算时间戳 sequence = pre_buffer_frames + i # 确保序列号连续 - header = bytearray(16) - header[0] = 1 # type - header[2:4] = len(opus_packet).to_bytes(2, 'big') # payload length - header[4:8] = sequence.to_bytes(4, 'big') # sequence - header[8:12] = timestamp.to_bytes(4, 'big') # 时间戳在第8-11字节 - header[12:16] = len(opus_packet).to_bytes(4, 'big') # opus长度在第12-15字节 - - # 发送包含头部的完整数据包 - complete_packet = bytes(header) + opus_packet - await conn.websocket.send(complete_packet) + # 调用通用函数发送带头部的数据包 + await _send_to_mqtt_gateway(conn, opus_packet, timestamp, sequence) else: # 直接发送opus数据包，不添加头部 await conn.websocket.send(opus_packet) - + play_position += frame_duration From 26fd3110b47803c6aefe9b06f3ac15eafbab0154 Mon Sep 17 00:00:00 2001 From: hrz <1710360675@qq.com> Date: Wed, 10 Sep 2025 22:20:43 +0800 Subject: [PATCH 14/14] =?UTF-8?q?update:=E4=BC=98=E5=8C=96=E4=BB=A3?= =?UTF-8?q?=E7=A0=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- main/xiaozhi-server/core/connection.py | 7 +-- .../core/handle/sendAudioHandle.py | 62 +++++++++++++------ 2 files changed, 45 insertions(+), 24 deletions(-) diff --git a/main/xiaozhi-server/core/connection.py b/main/xiaozhi-server/core/connection.py index d80ec8622..3db8f86c2 100644 --- a/main/xiaozhi-server/core/connection.py +++ b/main/xiaozhi-server/core/connection.py @@ -203,10 +203,9 @@ async def handle_connection(self, ws): # 检查是否来自MQTT连接 request_path = ws.request.path - self.conn_from_mqtt_gateway = request_path.endswith("?from=mqtt") - self.logger.bind(tag=TAG).info( - f"WebSocket URL路径: '{request_path}', 来自MQTT连接: {self.conn_from_mqtt_gateway}" - ) + self.conn_from_mqtt_gateway = request_path.endswith("?from=mqtt_gateway") + if self.conn_from_mqtt_gateway: + self.logger.bind(tag=TAG).info("连接来自:MQTT网关") # 初始化活动时间戳 self.last_activity_time = time.time() * 1000 diff --git a/main/xiaozhi-server/core/handle/sendAudioHandle.py b/main/xiaozhi-server/core/handle/sendAudioHandle.py index 810e8803f..d159d3fff 100644 --- a/main/xiaozhi-server/core/handle/sendAudioHandle.py +++ b/main/xiaozhi-server/core/handle/sendAudioHandle.py @@ -30,6 +30,31 @@ async def sendAudioMessage(conn, sentenceType, audios, text): await conn.close() +def calculate_timestamp_and_sequence(conn, start_time, packet_index, frame_duration=60): + """ + 计算音频数据包的时间戳和序列号 + Args: + conn: 连接对象 + start_time: 起始时间（性能计数器值） + packet_index: 数据包索引 + frame_duration: 帧时长（毫秒），匹配 Opus 编码 + Returns: + tuple: (timestamp, sequence) + """ + # 计算时间戳（使用播放位置计算） + timestamp = int((start_time + packet_index * frame_duration / 1000) * 1000) % ( + 2**32 + ) + + # 计算序列号 + if hasattr(conn, "audio_flow_control"): + sequence = conn.audio_flow_control["sequence"] + else: + sequence = packet_index # 如果没有流控状态，直接使用索引 + + return timestamp, sequence + + async def _send_to_mqtt_gateway(conn, opus_packet, timestamp, sequence): """ 发送带16字节头部的opus数据包给mqtt_gateway @@ -91,18 +116,15 @@ async def sendAudio(conn, audios, frame_duration=60): await asyncio.sleep(delay) if conn.conn_from_mqtt_gateway: - # 计算时间戳 - timestamp = int( - ( - flow_control["start_time"] - + flow_control["packet_count"] * frame_duration / 1000 - ) - * 1000 - ) % (2**32) - # 调用通用函数发送带头部的数据包 - await _send_to_mqtt_gateway( - conn, audios, timestamp, flow_control["sequence"] + # 计算时间戳和序列号 + timestamp, sequence = calculate_timestamp_and_sequence( + conn, + flow_control["start_time"], + flow_control["packet_count"], + frame_duration, ) + # 调用通用函数发送带头部的数据包 + await _send_to_mqtt_gateway(conn, audios, timestamp, sequence) else: # 直接发送opus数据包，不添加头部 await conn.websocket.send(audios) @@ -120,12 +142,12 @@ async def sendAudio(conn, audios, frame_duration=60): pre_buffer_frames = min(3, len(audios)) for i in range(pre_buffer_frames): if conn.conn_from_mqtt_gateway: - # 计算时间戳 - timestamp = int((start_time + i * frame_duration / 1000) * 1000) % ( - 2**32 + # 计算时间戳和序列号 + timestamp, sequence = calculate_timestamp_and_sequence( + conn, start_time, i, frame_duration ) # 调用通用函数发送带头部的数据包 - await _send_to_mqtt_gateway(conn, audios[i], timestamp, i) + await _send_to_mqtt_gateway(conn, audios[i], timestamp, sequence) else: # 直接发送预缓冲包，不添加头部 await conn.websocket.send(audios[i]) @@ -147,11 +169,11 @@ async def sendAudio(conn, audios, frame_duration=60): await asyncio.sleep(delay) if conn.conn_from_mqtt_gateway: - # 计算时间戳和序列号 - timestamp = int((start_time + play_position / 1000) * 1000) % ( - 2**32 - ) # 使用播放位置计算时间戳 - sequence = pre_buffer_frames + i # 确保序列号连续 + # 计算时间戳和序列号（使用当前的数据包索引确保连续性） + packet_index = pre_buffer_frames + i + timestamp, sequence = calculate_timestamp_and_sequence( + conn, start_time, packet_index, frame_duration + ) # 调用通用函数发送带头部的数据包 await _send_to_mqtt_gateway(conn, opus_packet, timestamp, sequence) else: