diff --git a/Dockerfile b/Dockerfile index 498e891..a044503 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,8 +1,8 @@ FROM node:22-alpine AS base -ENV ARDUINO_CLI_VERSION=1.1.0 +ENV ARDUINO_CLI_VERSION=1.2.0 ENV SENSEBOXCORE_VERSION=2.0.0 -ENV ARDUINO_SAMD_VERSION=1.8.13 +ENV ARDUINO_SAMD_VERSION=1.8.12 ENV ARDUINO_AVR_VERSION=1.8.5 ENV ESP32_VERSION=2.0.17 ENV SENSEBOXCORE_URL=https://raw.githubusercontent.com/mariopesch/senseBoxMCU-core/master/package_sensebox_index.json @@ -45,6 +45,12 @@ RUN cp /tmp/OTAFiles/boards.txt /root/.arduino15/packages/esp32/hardware/esp32/$ cp /tmp/OTAFiles/APOTA.bin /root/.arduino15/packages/esp32/hardware/esp32/${ESP32_VERSION}/variants/sensebox_mcu_esp32s2/ && \ cp /tmp/OTAFiles/variant.cpp /root/.arduino15/packages/esp32/hardware/esp32/${ESP32_VERSION}/variants/sensebox_mcu_esp32s2/ +COPY ./uf2/ /tmp/uf2/ + +# Add uf2conv.py +RUN cp /tmp/uf2/uf2conv.py /usr/local/bin/uf2conv.py && \ + cp /tmp/uf2/uf2families.json /usr/local/bin/uf2families.json && \ + cp /tmp/uf2/uf2conv.c /usr/local/bin/uf2conv.c # install Libraries with arduino-cli RUN arduino-cli lib install "Ethernet" diff --git a/src/builder.js b/src/builder.js index a9d5733..3d761c1 100644 --- a/src/builder.js +++ b/src/builder.js @@ -19,7 +19,7 @@ export const boardBinaryFileextensions = { "sensebox-esp32s2": "bin", }; -const baseArgs = ["--build-cache-path", `/app/src/build-cache`]; +const baseArgs = ["--build-path", `/app/src/build-cache`]; export const payloadValidator = function payloadValidator(req, res, next) { // reject all non application/json requests @@ -37,7 +37,7 @@ export const payloadValidator = function payloadValidator(req, res, next) { } // check if parameters sketch and board are specified and valid - let { sketch, board } = req.body; + let { sketch, board, uf2 } = req.body; if (!sketch || !board) { return next( @@ -50,6 +50,7 @@ export const payloadValidator = function payloadValidator(req, res, next) { sketch = sketch.toString().trim(); board = board.toString().trim(); + uf2 = uf2 ? uf2.toString().trim() : false; if (!sketch || !board) { return next( @@ -71,11 +72,16 @@ export const payloadValidator = function payloadValidator(req, res, next) { ); } - req._builderParams = { sketch, board }; + req._builderParams = { sketch, board, uf2 }; next(); }; -const execBuilder = async function execBuilder({ board, sketch, buildDir }) { +const execBuilder = async function execBuilder({ + board, + sketch, + buildDir, + uf2, +}) { // const tmpSketchPath = await tempWrite(sketch); const sketchDir = `${temporaryDirectory()}/sketch`; mkdirSync(sketchDir); @@ -93,6 +99,31 @@ const execBuilder = async function execBuilder({ board, sketch, buildDir }) { sketchDir, ]); + const ext = boardBinaryFileextensions[board]; + const binaryFile = `${buildDir}/sketch.ino.${ext}`; + + // Optional: Convert to UF2 if requested + if (uf2 && ext === "bin") { + const uf2File = `${buildDir}/sketch.uf2`; + try { + await spawn("python3", [ + "/usr/local/bin/uf2conv.py", + binaryFile, + "-c", + "-b", + "0x00", + "-f", + "ESP32S2", + "--output", + uf2File, + ]); + console.log(`UF2 file created: ${uf2File}`); + } catch (err) { + console.warn("UF2 conversion failed:", err.message); + // You may choose to throw here depending on how critical UF2 output is + } + } + try { const dirname = _dirname(tmpSketchPath); await rimraf(`${dirname}`); @@ -130,7 +161,7 @@ export const compileHandler = async function compileHandler(req, res, next) { ); } catch (err) { if (process.env.NODE_ENV === "test") { - console.error(err.message) + console.error(err.message); } return next(new HTTPError({ error: err.message })); } diff --git a/src/download.js b/src/download.js index 6f010b3..a7d69a0 100644 --- a/src/download.js +++ b/src/download.js @@ -1,14 +1,26 @@ -import { createReadStream } from "fs"; +import { createReadStream, existsSync } from "fs"; +import { join } from "path"; import { rimraf } from "rimraf"; import { boardBinaryFileextensions } from "./builder.js"; import { HTTPError } from "./utils.js"; -const readFile = async function readFile({ id, board }) { - return Promise.resolve( - createReadStream( - `/tmp/${id}/sketch.ino.${boardBinaryFileextensions[board]}` - ) +const readFile = async function readFile({ id, board, format }) { + const ext = format === "uf2" ? "uf2" : boardBinaryFileextensions[board]; + const filePath = join( + "/tmp", + id, + `sketch.${format !== "uf2" ? "ino." : ""}${ext}` ); + console.log(`Reading file: ${filePath}`); + + if (!existsSync(filePath)) { + throw new HTTPError({ + code: 404, + error: `Compiled file not found: sketch.${ext}`, + }); + } + + return Promise.resolve(createReadStream(filePath)); }; export const downloadHandler = async function downloadHandler(req, res, next) { @@ -21,7 +33,7 @@ export const downloadHandler = async function downloadHandler(req, res, next) { ); } - const { id, board } = req._url.query; + const { id, board, format } = req._url.query; if (!id || !board) { return next( @@ -32,30 +44,27 @@ export const downloadHandler = async function downloadHandler(req, res, next) { ); } - // execute builder with parameters from user try { - const stream = await readFile(req._url.query); + const ext = format === "uf2" ? "uf2" : boardBinaryFileextensions[board]; + const stream = await readFile({ id, board, format }); const filename = req._url.query.filename || "sketch"; + console.log(`Downloading ${filename}.${ext} for board ${board}`); stream.on("error", function (err) { return next(err); }); + stream.on("end", async () => { try { - await rimraf(`/tmp/${req._url.query.id}`); + await rimraf(`/tmp/${id}`); } catch (error) { - console.log( - `Error deleting compile sketch folder with ${req._url.query.id}: `, - error - ); + console.log(`Error deleting compile sketch folder with ${id}: `, error); } }); res.setHeader("Content-Type", "application/octet-stream"); res.setHeader( "Content-Disposition", - `attachment; filename=${filename}.${ - boardBinaryFileextensions[req._url.query.board] - }` + `attachment; filename=${filename}.${ext}` ); stream.pipe(res); } catch (err) { diff --git a/uf2/uf2conv.c b/uf2/uf2conv.c new file mode 100644 index 0000000..0f5f16e --- /dev/null +++ b/uf2/uf2conv.c @@ -0,0 +1,45 @@ +#include +#include +#include "uf2format.h" + +int main(int argc, char **argv) { + if (argc < 2) { + fprintf(stderr, "USAGE: %s file.bin [file.uf2]\n", argv[0]); + return 1; + } + FILE *f = fopen(argv[1], "rb"); + if (!f) { + fprintf(stderr, "No such file: %s\n", argv[1]); + return 1; + } + + fseek(f, 0L, SEEK_END); + uint32_t sz = ftell(f); + fseek(f, 0L, SEEK_SET); + + const char *outname = argc > 2 ? argv[2] : "flash.uf2"; + + FILE *fout = fopen(outname, "wb"); + + UF2_Block bl; + memset(&bl, 0, sizeof(bl)); + + bl.magicStart0 = UF2_MAGIC_START0; + bl.magicStart1 = UF2_MAGIC_START1; + bl.magicEnd = UF2_MAGIC_END; + bl.targetAddr = APP_START_ADDRESS; + bl.numBlocks = (sz + 255) / 256; + bl.payloadSize = 256; + int numbl = 0; + while (fread(bl.data, 1, bl.payloadSize, f)) { + bl.blockNo = numbl++; + fwrite(&bl, 1, sizeof(bl), fout); + bl.targetAddr += bl.payloadSize; + // clear for next iteration, in case we get a short read + memset(bl.data, 0, sizeof(bl.data)); + } + fclose(fout); + fclose(f); + printf("Wrote %d blocks to %s\n", numbl, outname); + return 0; +} \ No newline at end of file diff --git a/uf2/uf2conv.md b/uf2/uf2conv.md new file mode 100644 index 0000000..6c818c2 --- /dev/null +++ b/uf2/uf2conv.md @@ -0,0 +1,68 @@ +# uf2conv -- Packing and unpacking UF2 files + +## SYNOPSIS + +**uf2conv.py** [-h] [-l] + +**uf2conv.py** [-b BASE] [-f FAMILY] [-o FILE] [-d DEVICE_PATH] [-l] [-c] [-D] + [-w] [-C] + [HEX or BIN FILE] + +**uf2conv.py** [-c] [-D] [-w] [-i] [UF2 FILE] + +## DESCRIPTION + +## EXAMPLES + +### Pack a .bin/.hex to .uf2 + +```uf2conv.py cpx/firmware.bin --convert --output cpx/firmware.uf2``` + +```uf2conv.py metro_m4/firmware.bin --base 0x4000 --convert --output metro_m4/firmware.uf2``` + +```uf2conv.py nrf52840_xxaa.hex --family 0xADA52840 --convert --output nrf52840_xxaa.uf2``` + +### Unpack a .uf2 to .bin + +```uf2conv.py current.uf2 --convert --output current.bin``` + +## OPTIONS +`-b` +`--base` +: set base address of application for BIN format (default: 0x2000) + +`-f` +`--family` +: specify familyID - number or name (default: 0x0) + +`-o` +`--output` +: write output to named file (defaults to "flash.uf2" or "flash.bin" where sensible) + +`-d` +`--device` +: select a device path to flash + +`-l` +`--list` +: list connected devices + +`-c` +`--convert` +: do not flash, just convert + +`-D` +`--deploy` +: just flash, do not convert + +`-w` +`--wait` +: wait for device to flash + +`-C` +`--carray` +: convert binary file to a C array, not UF2 + +`-i` +`--info` +: display header information from UF2, do not convert diff --git a/uf2/uf2conv.py b/uf2/uf2conv.py new file mode 100755 index 0000000..c658081 --- /dev/null +++ b/uf2/uf2conv.py @@ -0,0 +1,361 @@ +#!/usr/bin/env python3 +import sys +import struct +import subprocess +import re +import os +import os.path +import argparse +import json +from time import sleep + + +UF2_MAGIC_START0 = 0x0A324655 # "UF2\n" +UF2_MAGIC_START1 = 0x9E5D5157 # Randomly selected +UF2_MAGIC_END = 0x0AB16F30 # Ditto + +INFO_FILE = "/INFO_UF2.TXT" + +appstartaddr = 0x2000 +familyid = 0x0 + + +def is_uf2(buf): + w = struct.unpack(" 476: + assert False, "Invalid UF2 data size at " + ptr + newaddr = hd[3] + if (hd[2] & 0x2000) and (currfamilyid == None): + currfamilyid = hd[7] + if curraddr == None or ((hd[2] & 0x2000) and hd[7] != currfamilyid): + currfamilyid = hd[7] + curraddr = newaddr + if familyid == 0x0 or familyid == hd[7]: + appstartaddr = newaddr + padding = newaddr - curraddr + if padding < 0: + assert False, "Block out of order at " + ptr + if padding > 10*1024*1024: + assert False, "More than 10M of padding needed at " + ptr + if padding % 4 != 0: + assert False, "Non-word padding size at " + ptr + while padding > 0: + padding -= 4 + outp.append(b"\x00\x00\x00\x00") + if familyid == 0x0 or ((hd[2] & 0x2000) and familyid == hd[7]): + outp.append(block[32 : 32 + datalen]) + curraddr = newaddr + datalen + if hd[2] & 0x2000: + if hd[7] in families_found.keys(): + if families_found[hd[7]] > newaddr: + families_found[hd[7]] = newaddr + else: + families_found[hd[7]] = newaddr + if prev_flag == None: + prev_flag = hd[2] + if prev_flag != hd[2]: + all_flags_same = False + if blockno == (numblocks - 1): + print("--- UF2 File Header Info ---") + families = load_families() + for family_hex in families_found.keys(): + family_short_name = "" + for name, value in families.items(): + if value == family_hex: + family_short_name = name + print("Family ID is {:s}, hex value is 0x{:08x}".format(family_short_name,family_hex)) + print("Target Address is 0x{:08x}".format(families_found[family_hex])) + if all_flags_same: + print("All block flag values consistent, 0x{:04x}".format(hd[2])) + else: + print("Flags were not all the same") + print("----------------------------") + if len(families_found) > 1 and familyid == 0x0: + outp = [] + appstartaddr = 0x0 + return b"".join(outp) + +def convert_to_carray(file_content): + outp = "const unsigned long bindata_len = %d;\n" % len(file_content) + outp += "const unsigned char bindata[] __attribute__((aligned(16))) = {" + for i in range(len(file_content)): + if i % 16 == 0: + outp += "\n" + outp += "0x%02x, " % file_content[i] + outp += "\n};\n" + return bytes(outp, "utf-8") + +def convert_to_uf2(file_content): + global familyid + datapadding = b"" + while len(datapadding) < 512 - 256 - 32 - 4: + datapadding += b"\x00\x00\x00\x00" + numblocks = (len(file_content) + 255) // 256 + outp = [] + for blockno in range(numblocks): + ptr = 256 * blockno + chunk = file_content[ptr:ptr + 256] + flags = 0x0 + if familyid: + flags |= 0x2000 + hd = struct.pack(b"= 3 and words[1] == "2" and words[2] == "FAT": + drives.append(words[0]) + else: + searchpaths = ["/media"] + if sys.platform == "darwin": + searchpaths = ["/Volumes"] + elif sys.platform == "linux": + searchpaths += ["/media/" + os.environ["USER"], '/run/media/' + os.environ["USER"]] + + for rootpath in searchpaths: + if os.path.isdir(rootpath): + for d in os.listdir(rootpath): + if os.path.isdir(rootpath): + drives.append(os.path.join(rootpath, d)) + + + def has_info(d): + try: + return os.path.isfile(d + INFO_FILE) + except: + return False + + return list(filter(has_info, drives)) + + +def board_id(path): + with open(path + INFO_FILE, mode='r') as file: + file_content = file.read() + return re.search(r"Board-ID: ([^\r\n]*)", file_content).group(1) + + +def list_drives(): + for d in get_drives(): + print(d, board_id(d)) + + +def write_file(name, buf): + with open(name, "wb") as f: + f.write(buf) + print("Wrote %d bytes to %s" % (len(buf), name)) + + +def load_families(): + # The expectation is that the `uf2families.json` file is in the same + # directory as this script. Make a path that works using `__file__` + # which contains the full path to this script. + filename = "uf2families.json" + pathname = os.path.join(os.path.dirname(os.path.abspath(__file__)), filename) + with open(pathname) as f: + raw_families = json.load(f) + + families = {} + for family in raw_families: + families[family["short_name"]] = int(family["id"], 0) + + return families + + +def main(): + global appstartaddr, familyid + def error(msg): + print(msg, file=sys.stderr) + sys.exit(1) + parser = argparse.ArgumentParser(description='Convert to UF2 or flash directly.') + parser.add_argument('input', metavar='INPUT', type=str, nargs='?', + help='input file (HEX, BIN or UF2)') + parser.add_argument('-b', '--base', dest='base', type=str, + default="0x2000", + help='set base address of application for BIN format (default: 0x2000)') + parser.add_argument('-f', '--family', dest='family', type=str, + default="0x0", + help='specify familyID - number or name (default: 0x0)') + parser.add_argument('-o', '--output', metavar="FILE", dest='output', type=str, + help='write output to named file; defaults to "flash.uf2" or "flash.bin" where sensible') + parser.add_argument('-d', '--device', dest="device_path", + help='select a device path to flash') + parser.add_argument('-l', '--list', action='store_true', + help='list connected devices') + parser.add_argument('-c', '--convert', action='store_true', + help='do not flash, just convert') + parser.add_argument('-D', '--deploy', action='store_true', + help='just flash, do not convert') + parser.add_argument('-w', '--wait', action='store_true', + help='wait for device to flash') + parser.add_argument('-C', '--carray', action='store_true', + help='convert binary file to a C array, not UF2') + parser.add_argument('-i', '--info', action='store_true', + help='display header information from UF2, do not convert') + args = parser.parse_args() + appstartaddr = int(args.base, 0) + + families = load_families() + + if args.family.upper() in families: + familyid = families[args.family.upper()] + else: + try: + familyid = int(args.family, 0) + except ValueError: + error("Family ID needs to be a number or one of: " + ", ".join(families.keys())) + + if args.list: + list_drives() + else: + if not args.input: + error("Need input file") + with open(args.input, mode='rb') as f: + inpbuf = f.read() + from_uf2 = is_uf2(inpbuf) + ext = "uf2" + if args.deploy: + outbuf = inpbuf + elif from_uf2 and not args.info: + outbuf = convert_from_uf2(inpbuf) + ext = "bin" + elif from_uf2 and args.info: + outbuf = "" + convert_from_uf2(inpbuf) + elif is_hex(inpbuf): + outbuf = convert_from_hex_to_uf2(inpbuf.decode("utf-8")) + elif args.carray: + outbuf = convert_to_carray(inpbuf) + ext = "h" + else: + outbuf = convert_to_uf2(inpbuf) + if not args.deploy and not args.info: + print("Converted to %s, output size: %d, start address: 0x%x" % + (ext, len(outbuf), appstartaddr)) + if args.convert or ext != "uf2": + if args.output == None: + args.output = "flash." + ext + if args.output: + write_file(args.output, outbuf) + if ext == "uf2" and not args.convert and not args.info: + drives = get_drives() + if len(drives) == 0: + if args.wait: + print("Waiting for drive to deploy...") + while len(drives) == 0: + sleep(0.1) + drives = get_drives() + elif not args.output: + error("No drive to deploy.") + for d in drives: + print("Flashing %s (%s)" % (d, board_id(d))) + write_file(d + "/NEW.UF2", outbuf) + + +if __name__ == "__main__": + main() diff --git a/uf2/uf2families.json b/uf2/uf2families.json new file mode 100644 index 0000000..14c8e5d --- /dev/null +++ b/uf2/uf2families.json @@ -0,0 +1,247 @@ +[ + { + "id": "0x16573617", + "short_name": "ATMEGA32", + "description": "Microchip (Atmel) ATmega32" + }, + { + "id": "0x1851780a", + "short_name": "SAML21", + "description": "Microchip (Atmel) SAML21" + }, + { + "id": "0x1b57745f", + "short_name": "NRF52", + "description": "Nordic NRF52" + }, + { + "id": "0x1c5f21b0", + "short_name": "ESP32", + "description": "ESP32" + }, + { + "id": "0x1e1f432d", + "short_name": "STM32L1", + "description": "ST STM32L1xx" + }, + { + "id": "0x202e3a91", + "short_name": "STM32L0", + "description": "ST STM32L0xx" + }, + { + "id": "0x21460ff0", + "short_name": "STM32WL", + "description": "ST STM32WLxx" + }, + { + "id": "0x2abc77ec", + "short_name": "LPC55", + "description": "NXP LPC55xx" + }, + { + "id": "0x300f5633", + "short_name": "STM32G0", + "description": "ST STM32G0xx" + }, + { + "id": "0x31d228c6", + "short_name": "GD32F350", + "description": "GD32F350" + }, + { + "id": "0x04240bdf", + "short_name": "STM32L5", + "description": "ST STM32L5xx" + }, + { + "id": "0x4c71240a", + "short_name": "STM32G4", + "description": "ST STM32G4xx" + }, + { + "id": "0x4fb2d5bd", + "short_name": "MIMXRT10XX", + "description": "NXP i.MX RT10XX" + }, + { + "id": "0x53b80f00", + "short_name": "STM32F7", + "description": "ST STM32F7xx" + }, + { + "id": "0x55114460", + "short_name": "SAMD51", + "description": "Microchip (Atmel) SAMD51" + }, + { + "id": "0x57755a57", + "short_name": "STM32F4", + "description": "ST STM32F4xx" + }, + { + "id": "0x5a18069b", + "short_name": "FX2", + "description": "Cypress FX2" + }, + { + "id": "0x5d1a0a2e", + "short_name": "STM32F2", + "description": "ST STM32F2xx" + }, + { + "id": "0x5ee21072", + "short_name": "STM32F1", + "description": "ST STM32F103" + }, + { + "id": "0x621e937a", + "short_name": "NRF52833", + "description": "Nordic NRF52833" + }, + { + "id": "0x647824b6", + "short_name": "STM32F0", + "description": "ST STM32F0xx" + }, + { + "id": "0x68ed2b88", + "short_name": "SAMD21", + "description": "Microchip (Atmel) SAMD21" + }, + { + "id": "0x6b846188", + "short_name": "STM32F3", + "description": "ST STM32F3xx" + }, + { + "id": "0x6d0922fa", + "short_name": "STM32F407", + "description": "ST STM32F407" + }, + { + "id": "0x6db66082", + "short_name": "STM32H7", + "description": "ST STM32H7xx" + }, + { + "id": "0x70d16653", + "short_name": "STM32WB", + "description": "ST STM32WBxx" + }, + { + "id": "0x7eab61ed", + "short_name": "ESP8266", + "description": "ESP8266" + }, + { + "id": "0x7f83e793", + "short_name": "KL32L2", + "description": "NXP KL32L2x" + }, + { + "id": "0x8fb060fe", + "short_name": "STM32F407VG", + "description": "ST STM32F407VG" + }, + { + "id": "0xada52840", + "short_name": "NRF52840", + "description": "Nordic NRF52840" + }, + { + "id": "0xbfdd4eee", + "short_name": "ESP32S2", + "description": "ESP32-S2" + }, + { + "id": "0xc47e5767", + "short_name": "ESP32S3", + "description": "ESP32-S3" + }, + { + "id": "0xd42ba06c", + "short_name": "ESP32C3", + "description": "ESP32-C3" + }, + { + "id": "0x2b88d29c", + "short_name": "ESP32C2", + "description": "ESP32-C2" + }, + { + "id": "0x332726f6", + "short_name": "ESP32H2", + "description": "ESP32-H2" + }, + { + "id": "0x540ddf62", + "short_name": "ESP32C6", + "description": "ESP32-C6" + }, + { + "id": "0x3d308e94", + "short_name": "ESP32P4", + "description": "ESP32-P4" + }, + { + "id": "0xe48bff56", + "short_name": "RP2040", + "description": "Raspberry Pi RP2040" + }, + { + "id": "0x00ff6919", + "short_name": "STM32L4", + "description": "ST STM32L4xx" + }, + { + "id": "0x9af03e33", + "short_name": "GD32VF103", + "description": "GigaDevice GD32VF103" + }, + { + "id": "0x4f6ace52", + "short_name": "CSK4", + "description": "LISTENAI CSK300x/400x" + }, + { + "id": "0x6e7348a8", + "short_name": "CSK6", + "description": "LISTENAI CSK60xx" + }, + { + "id": "0x11de784a", + "short_name": "M0SENSE", + "description": "M0SENSE BL702" + }, + { + "id": "0x4b684d71", + "short_name": "MaixPlay-U4", + "description": "Sipeed MaixPlay-U4(BL618)" + }, + { + "id": "0x9517422f", + "short_name": "RZA1LU", + "description": "Renesas RZ/A1LU (R7S7210xx)" + }, + { + "id": "0x2dc309c5", + "short_name": "STM32F411xE", + "description": "ST STM32F411xE" + }, + { + "id": "0x06d1097b", + "short_name": "STM32F411xC", + "description": "ST STM32F411xC" + }, + { + "id": "0x72721d4e", + "short_name": "NRF52832xxAA", + "description": "Nordic NRF52832xxAA" + }, + { + "id": "0x6f752678", + "short_name": "NRF52832xxAB", + "description": "Nordic NRF52832xxAB" + } +]