From a9692f6ff393e0288b5278c1a62f06e26f960c93 Mon Sep 17 00:00:00 2001 From: sean Date: Tue, 17 Jun 2025 18:57:32 +0200 Subject: [PATCH 1/2] Add Vector Binary Logging Format pattern --- README.md | 1 + patterns/blf.hexpat | 535 ++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 536 insertions(+) create mode 100644 patterns/blf.hexpat diff --git a/README.md b/README.md index 7684376f..2c4c9746 100644 --- a/README.md +++ b/README.md @@ -38,6 +38,7 @@ Everything will immediately show up in ImHex's Content Store and gets bundled wi | Bencode | `application/x-bittorrent` | [`patterns/bencode.hexpat`](patterns/bencode.hexpat) | Bencode encoding, used by Torrent files | | Prusa BGCODE | | [`patterns/bgcode.hexpat`](patterns/bgcode.hexpat) | PrusaSlicer Binary G-Code files | | BLEND | | [`patterns/blend.hexpat`](patterns/blend.hexpat) | Blender Project file | +| BLF | | [`patterns/blf.hexpat`](patterns/blf.hexpat) | Vector BLF Frame Logging Files | | BMP | `image/bmp` | [`patterns/bmp.hexpat`](patterns/bmp.hexpat) | OS2/Windows Bitmap files | | BIN | | [`patterns/selinux.hexpat`](patterns/selinux.pat) | SE Linux modules | | BINKA | | [`patterns/binka.hexpat`](patterns/binka.pat) | RAD Game Tools Bink Audio (BINKA) files | diff --git a/patterns/blf.hexpat b/patterns/blf.hexpat new file mode 100644 index 00000000..7b221cb2 --- /dev/null +++ b/patterns/blf.hexpat @@ -0,0 +1,535 @@ + +// Pattern for binary logging files (Vector BLF) +// References used for writing this: +// https://python-can.readthedocs.io/en/stable/_modules/can/io/blf.html +// https://bitbucket.org/tobylorenz/vector_blf + +#pragma magic [ 4C 4F 47 47 ] @ 0x00 +#pragma description Vector BLF Frame Logging Files +#pragma array_limit 4194304 +#pragma pattern_limit 4294967296 +#pragma endian little + +import hex.dec; +import std.io; +import type.magic; + +enum object_type : u32 { + unknown = 0, //< unknown object + can_message = 1, //< CAN message object + can_error = 2, //< CAN error frame object + can_overload = 3, //< CAN overload frame object + can_statistic = 4, //< CAN driver statistics object + app_trigger = 5, //< application trigger object + env_integer = 6, //< environment integer object + env_double = 7, //< environment double object + env_string = 8, //< environment string object + env_data = 9, //< environment data object + log_container = 10, //< container object + lin_message = 11, //< LIN message object + lin_crc_error = 12, //< LIN CRC error object + lin_dlc_info = 13, //< LIN DLC info object + lin_rcv_error = 14, //< LIN receive error object + lin_snd_error = 15, //< LIN send error object + lin_slv_timeout = 16, //< LIN slave timeout object + lin_sched_modch = 17, //< LIN scheduler mode change object + lin_syn_error = 18, //< LIN sync error object + lin_baudrate = 19, //< LIN baudrate event object + lin_sleep = 20, //< LIN sleep mode event object + lin_wakeup = 21, //< LIN wakeup event object + most_spy = 22, //< MOST spy message object + most_ctrl = 23, //< MOST control message object + most_lightlock = 24, //< MOST light lock object + most_statistic = 25, //< MOST statistic object + + flexray_data = 29, //< FLEXRAY data object + flexray_sync = 30, //< FLEXRAY sync object + can_driver_error = 31, //< CAN driver error object + most_pkt = 32, //< MOST Packet + most_pkt2 = 33, //< MOST Packet including original timestamp + most_hwmode = 34, //< MOST hardware mode event + most_reg = 35, //< MOST register data (various chips) + most_genreg = 36, //< MOST register data (MOST register) + most_netstate = 37, //< MOST NetState event + most_datalost = 38, //< MOST data lost + most_trigger = 39, //< MOST trigger + flexray_cycle = 40, //< FLEXRAY V6 start cycle object + flexray_message = 41, //< FLEXRAY V6 message object + lin_checksum_info = 42, //< LIN checksum info event object + lin_spike_event = 43, //< LIN spike event object + can_driver_sync = 44, //< CAN driver hardware sync + flexray_status = 45, //< FLEXRAY status event object + gps_event = 46, //< GPS event object + flexray_error = 47, //< FLEXRAY error event object + flexray_status = 48, //< FLEXRAY status event object + flexray_startcycle = 49, //< FLEXRAY start cycle event object + flexray_rcv_message = 50, //< FLEXRAY receive message event object + realtime_clock = 51, //< Realtime clock object + + lin_statistic = 54, //< LIN statistic event object + j1708_message = 55, //< J1708 message object + j1708_virtual_message = 56, //< J1708 message object with more than 21 data bytes + lin_message2 = 57, //< LIN frame object - extended + lin_snd_error2 = 58, //< LIN transmission error object - extended + lin_syn_error2 = 59, //< LIN sync error object - extended + lin_crc_error2 = 60, //< LIN checksum error object - extended + lin_crv_error2 = 61, //< LIN receive error object + lin_wakeup2 = 62, //< LIN wakeup event object - extended + lin_spike_event2 = 63, //< LIN spike event object - extended + lin_long_dom_sig = 64, //< LIN long dominant signal object + app_text = 65, //< text object + flexray_rcvmessage_ex = 66, //< FLEXRAY receive message ex event object + most_statistic_ex = 67, //< MOST extended statistic event + most_txlight = 68, //< MOST TxLight event + most_alloctab = 69, //< MOST Allocation table event + most_stress = 70, //< MOST Stress event + ethernet_frame = 71, //< Ethernet frame object + sys_variable = 72, //< system variable object + can_error_ext = 73, //< CAN error frame object (extended) + can_driver_error_ext = 74, //< CAN driver error object (extended) + lin_long_dom_sig2 = 75, //< LIN long dominant signal object - extended + most_150_message = 76, //< MOST150 Control channel message + most_150_pkt = 77, //< MOST150 Asynchronous channel message + most_ethernet_pkt = 78, //< MOST Ethernet channel message + most_150_message_fragment = 79, //< Partial transmitted MOST50/150 Control channel message + most_150_pkt_fragment = 80, //< Partial transmitted MOST50/150 data packet on asynchronous channel + most_ethernet_pkt_fragment = 81, //< Partial transmitted MOST Ethernet packet on asynchronous channel + most_system_event = 82, //< Event for various system states on MOST + most_150_alloctab = 83, //< MOST50/150 Allocation table event + most_50_message = 84, //< MOST50 Control channel message + most_50_pkg = 85, //< MOST50 Asynchronous channel message + can_message2 = 86, //< CAN message object - extended + lin_unexpected_wakeup = 87, + lin_short_or_slow_response = 88, + lin_disturbance_event = 89, + serial_event = 90, + overrun_error = 91, //< driver overrun event + event_comment = 92, + wlan_frame = 93, + wlan_statistic = 94, + most_ecl = 95, //< MOST Electrical Control Line event + global_marker = 96, + afdx_frame = 97, + afdx_statistic = 98, + kline_statusevent = 99, //< E.g. wake-up pattern + can_fd_message = 100, //< CAN FD message object + can_fd_message_64 = 101, //< CAN FD message object + ethernet_rx_error = 102, //< Ethernet RX error object + ethernet_status = 103, //< Ethernet status object + can_fd_error_64 = 104, //< CAN FD Error Frame object + lin_short_or_slow_response2 = 105, + afdx_status = 106, //< AFDX status object + afdx_bus_statistic = 107, //< AFDX line-dependent busstatistic object + + afdx_error_event = 109, //< AFDX asynchronous error event + a429_error = 110, //< A429 error object + a429_status = 111, //< A429 status object + a429_bus_statistic = 112, //< A429 busstatistic object + a429_message = 113, //< A429 Message + ethernet_statistic = 114, //< Ethernet statistic object + restore_point_container = 115, //< Restore point container, use unknown + + test_structure = 118, //< Event for test execution flow + diag_request_information = 119, //< Event for correct interpretation of diagnostic requests + ethernet_frame_ex = 120, //< Ethernet packet extended object + ethernet_frame_forwarded = 121, //< Ethernet packet forwarded object + ethernet_error_ex = 122, //< Ethernet error extended object + ethernet_error_forwarded = 123, //< Ethernet error forwarded object + function_bus = 124, //< FunctionBus object + data_lost_begin = 125, //< Data lost begin + data_lost_end = 126, //< Data lost end + water_mark_event = 127, //< Watermark event + trigger_condition = 128, //< Trigger Condition event + can_setting_changed = 129, //< CAN Settings Changed object + distributed_object_member = 130, //< Distributed object member (communication setup) + attribute_event = 131, //< ATTRIBUTE event (communication setup) +}; + +bitfield can_msg_flags { + is_tx : 1; + padding : 4; + nerr : 1; + wu : 1; + rtr : 1; +}; + +struct can_msg { + u16 channel; + can_msg_flags flags; + u8 dlc; + u32 id; + u8 data[8]; +}; +struct can_msg2 { + u16 channel; + can_msg_flags flags; + u8 dlc; + u32 id; + + auto struct_len = 2 + 1 + 1 + 4 + 4 + 1 + 1 + 2; // TODO: Alternative way of doing this? + u8 data[parent.header.object_size - parent.header.header_size - struct_len]; + + // The frame length in nanoseconds + u32 frame_length; + // Total number of bits of the CAN frame + u8 bit_count; + padding[1]; + padding[2]; +}; + +bitfield can_fd_msg_flags { + edl : 1; //< Extended data length + brs : 1; //< Bit rate switch + esi : 1; //< Error state indicator + padding : 5; +}; + +fn format_can_fd_dlc(u8 dlc) { + if (dlc > 8) + return 8 + (dlc - 8) * 4; + return dlc; +}; + +struct can_fd_msg { + u16 channel; + u8 flags; + u8 dlc [[format("format_can_fd_dlc")]]; + u32 id; + + // The frame length in nanoseconds + u32 frame_length; + u8 arbitration_bit_count; + can_fd_msg_flags fd_flags; + u8 valid_data_bytes; + padding[1]; + padding[4]; + + u8 data[64]; + + padding[4]; +}; + +bitfield can_fd_msg_64_flags { + padding : 2; + nerr : 1; + hv_wake_up : 1; + remote_frame : 1; + padding : 1; + tx_ack : 1; + tx_req : 1; + padding : 1; + srr : 1; + r0 : 1; + r1 : 1; + edl : 1; //< Extended data length + brs : 1; //< Bit rate switch + esi : 1; //< Error state indicator + padding : 2; + burst : 1; + padding : 13; +}; +struct can_bitrate_cfg { + u8 quartz_frequency; + u8 prescaler; + u8 btl_cycles; + u8 sampling_point; +}; +struct can_fd_msg_64 { + u8 channel; + u8 dlc [[format("format_can_fd_dlc")]]; + u8 valid_data_bytes; + u8 tx_count; + u32 id; + u32 frame_length; + can_fd_msg_64_flags flags; + can_bitrate_cfg arbitration_bitrate; + can_bitrate_cfg data_bitrate; + u32 brs_time_offset; + u32 crc_time_offset; + u16 bit_length; + u8 direction; + u8 data_offset; + u32 crc; + + u8 data[valid_data_bytes]; +}; + +fn format_bus_load(u16 bus_load) { + return std::format("{}%", float(bus_load) / 100.f); +}; +struct can_statistic { + u16 channel; + // Bus load in 1/100 percent + u16 bus_load [[format("format_bus_load")]]; + + u32 standard_data_frames; + u32 extended_data_frames; + + u32 standard_remote_frames; + u32 extended_remote_frames; + + u32 error_frames; + u32 overload_frames; + + u32 reserved; +}; + +struct can_driver_error { + u16 channel; + u8 tx_errors; + u8 rx_errors; + u32 error_code; +}; + +enum app_text_source : u32 { + comment = 0, + database_info = 1, + metadata = 2, +}; +enum database_bus_type : u8 { + can = 1, + lin = 5, + most = 6, + flexray = 7, + j1708 = 9, + ethernet = 10, + wlan = 13, + afdx = 14, +}; +bitfield app_text_database_info { + version : 8; + channel_num : 8; + database_bus_type bus_type : 8; + is_can_fd : 1; + padding : 7; +}; +struct app_text { + app_text_source source; + if (source == 1) + app_text_database_info database_info; + else + padding[4]; // TODO: This is not necessarily padding, there's data here + u32 text_length; + padding[4]; + char text[text_length]; +}; + +// No idea what this is or does +struct restore_point_container { + u8 rpc[14]; + u16 data_len; + u8 data[data_len]; +}; + +enum compression_method : u16 { + no_compression = 0, + zlib = 2, +}; + +// The following section contains all of the decompressed data at once +std::mem::Section decompressed_data = std::mem::create_section("Decompressed data"); +// This section is used only for decompressing data +std::mem::Section zlib_decompress_result = std::mem::create_section("zlib decompress result"); + +struct log_container { + u64 container_begin = $; + + compression_method compression_method; + padding[2]; + padding[4]; + u32 uncompressed_size; + padding[4]; + + if (compression_method == compression_method::zlib) { + std::mem::set_section_size(zlib_decompress_result, uncompressed_size); + + // Create a pattern that defines the compressed array data + auto compressed_byte_len = parent.header.object_size - parent.header.header_size - ($ - container_begin); + u8 compressed[compressed_byte_len]; + if (uncompressed_size != 0) + padding[parent.header.object_size % 4]; // Idk, the format wants this... for some reason + + std::assert(hex::dec::zlib_decompress(compressed, zlib_decompress_result) == compressed_byte_len, + "zlib decompress needs to succeed"); + + // Copy the decompressed data to the end of the section + std::mem::copy_section_to_section(zlib_decompress_result, 0, + decompressed_data, std::mem::get_section_size(decompressed_data), + std::mem::get_section_size(zlib_decompress_result)); + } else if (compression_method == compression_method::no_compression) { + u8 data[uncompressed_size]; + std::mem::copy_value_to_section(data, decompressed_data, std::mem::get_section_size(decompressed_data)); + } else { + std::assert(false, "Invalid/unknown compression method"); + } +}; + +enum object_flags : u32 { + // Timestamps are stored with a unit of 10us (10 microseconds) + time_10_us = 1, + // Timestamps are stored with a unit of 1ns (1 nanosecond) + time_1_ns = 2, +}; + +enum timestamp_status : u8 { + // Means original timestamps are valid + orig = 0x01, + // Timestamp is generated by software (1) or by hardware (0) + swhw = 0x02, + user = 0x10, +}; + +struct obj_v1_header { + object_flags flags; + u16 client_index; + u16 object_version; + u64 timestamp; +}; +struct obj_v2_header { + object_flags flags; + timestamp_status timestamp_status; + padding[1]; + u16 object_version; + u64 object_timestamp; + u64 original_timestamp; +}; + +struct obj_header { + type::Magic<"LOBJ"> magic; // 4C 4F 42 4A + u16 header_size; + u16 header_version; + u32 object_size; + object_type object_type; + + std::assert(header_version == 1 || header_version == 2, "Invalid/unknown header version"); +}; + +struct obj_struct { + auto object_begin = $; + obj_header header [[inline]]; + + if (header.object_type == object_type::log_container) { + // Log containers seem to never include additional V1 or V2 headers + log_container log [[inline]]; + } else { + if (header.header_version == 1) + obj_v1_header v1_header [[inline]]; + else if (header.header_version == 2) + obj_v2_header v2_header [[inline]]; + + match(header.object_type) { + (object_type::can_message): { + can_msg message [[inline]]; + } + (object_type::can_statistic): { + can_statistic statistics [[inline]]; + } + (object_type::can_driver_error): { + can_driver_error errors [[inline]]; + } + (object_type::can_message2): { + can_msg2 message [[inline]]; + } + (object_type::can_fd_message): { + can_fd_msg message [[inline]]; + } + (object_type::can_fd_message_64): { + can_fd_msg_64 message [[inline]]; + padding[header.object_size - ($ - object_begin)]; // TODO: This pattern doesn't support the extra data for this object + } + (object_type::app_text): { + app_text text [[inline]]; + padding[header.object_size % 4]; + } + (object_type::restore_point_container): { + restore_point_container rpc [[inline]]; + } + (_): u8 bytes[header.object_size - header.header_size]; + } + } +}; + +enum application_id : u8 { + unknown = 0, + canalyzer = 1, + canoe = 2, + canstress = 3, + canlog = 4, + canape = 5, + cancasexl = 6, + vlconfig = 7, + porsche_logger = 200, + caetec_logger = 201, + vector_net_sim = 202, + ipetronik_logger = 203, + rtpk = 204, + piketec = 205, + sparks = 206, +}; + +fn format_api_version(u32 api_version) { + return std::format("{}.{}.{}", + api_version / 1000000, + (api_version % 1000000) / 1000, + (api_version % 1000) / 100); +}; + +// Mostly just the zlib compression levels, but with some extras +enum compression_level : u8 { + no_compression = 0, + best_speed = 1, + default_compression = 6, + best_compression = 9, + // This means that the file contains only log containers, usually compressed at level 6 + default_container_compression = 10, +}; + +struct timestamp { + u16 year; + u16 month; + u16 day_of_week; + u16 day; + u16 hour; + u16 minute; + u16 second; + u16 millisecond; +} [[format("format_timestamp")]]; +fn format_timestamp(timestamp ts) { + return std::format("{}-{}-{}_{}-{}-{}", + ts.year, ts.month, ts.day, ts.hour, ts.minute, ts.second); +}; + +struct file_header { + type::Magic<"LOGG"> magic; // 4C 4F 47 47 + u32 header_length; + + u32 api_version [[format("format_api_version")]]; + application_id app_id; + compression_level compression_level; + u8 app_major; + u8 app_minor; + + u64 file_length; + u64 uncompressed_length; + u32 object_count; + u32 application_build; + + timestamp start_timestamp; + timestamp stop_timestamp; + + u64 restore_point_offset; // ? +} [[inline]]; + +struct file_layout { + file_header header; + padding[header.header_length - sizeof(header)]; + obj_struct objects[while($ < std::mem::size())]; + + // Decode all objects from the zlib compressed data + if (std::mem::get_section_size(decompressed_data) != 0) + obj_struct decompressed_objects[header.object_count] @ 0x00 in decompressed_data; +} [[inline]]; + +file_layout file @ 0x00; + +std::assert_warn(std::mem::size() == file.header.file_length, "file size mismatch"); From 39de8e4bbfcddaf9d42d1e86df87cd18ac608e60 Mon Sep 17 00:00:00 2001 From: sean Date: Wed, 18 Jun 2025 14:40:31 +0200 Subject: [PATCH 2/2] Combine object header extensions into single type --- patterns/blf.hexpat | 26 ++++++++++++-------------- 1 file changed, 12 insertions(+), 14 deletions(-) diff --git a/patterns/blf.hexpat b/patterns/blf.hexpat index 7b221cb2..b0e49852 100644 --- a/patterns/blf.hexpat +++ b/patterns/blf.hexpat @@ -1,4 +1,5 @@ + // Pattern for binary logging files (Vector BLF) // References used for writing this: // https://python-can.readthedocs.io/en/stable/_modules/can/io/blf.html @@ -379,19 +380,18 @@ enum timestamp_status : u8 { user = 0x10, }; -struct obj_v1_header { - object_flags flags; - u16 client_index; - u16 object_version; - u64 timestamp; -}; -struct obj_v2_header { +struct obj_header_ext { object_flags flags; - timestamp_status timestamp_status; - padding[1]; + if (parent.header.header_version == 1) + u16 client_index; + else if (parent.header.header_version == 2) { + timestamp_status timestamp_status; + padding[1]; + } u16 object_version; u64 object_timestamp; - u64 original_timestamp; + if (parent.header.header_version == 2) + u64 original_timestamp; }; struct obj_header { @@ -412,10 +412,7 @@ struct obj_struct { // Log containers seem to never include additional V1 or V2 headers log_container log [[inline]]; } else { - if (header.header_version == 1) - obj_v1_header v1_header [[inline]]; - else if (header.header_version == 2) - obj_v2_header v2_header [[inline]]; + obj_header_ext ext_header [[inline]]; match(header.object_type) { (object_type::can_message): { @@ -533,3 +530,4 @@ struct file_layout { file_layout file @ 0x00; std::assert_warn(std::mem::size() == file.header.file_length, "file size mismatch"); +