Add: Remove unnecessary alignment of interlaced frames (#1266)

- Remove unnecessary alignment of first fields to even N*frame_time
and second fields to odd N*frame_time. This caused extra latency
in interlaced streams. Problems with user_pacing and is not required by
SMPTE spec.
- expose user_timestamp as json parameter for RxTxApp
- Align timestamps created by RxTxApp to N*frame_time
- Add small fix to the lib: For user_pacing now it aligns to the closest
N*frame_time instead of the last.

---------

Signed-off-by: Kasiewicz, Marek <[email protected]>

Author: Sakoram

config/tx_1v_user_pacing.json

@@ -25,10 +25,11 @@
                     "input_format": "YUV422PLANAR10LE",
                     "transport_format": "YUV_422_10bit",
                     "st20p_url": "./yuv422p10le_1080p.yuv",
+                    "user_timestamp": true,
                     "user_pacing": true
                 }
             ],
-            "user_pacing_offset" : 20000000
+            "user_time_offset" : 20000000
         }
     ]
 }

doc/design.md

@@ -431,7 +431,8 @@ Consequently, the application will receive notifications through `notify_event`
 
 By default, applications simply push frames into MTL TX sessions, where the MTL TX session automatically assigns timestamps and epochs based on timing.
 
-The ST**_TX_FLAG_USER_TIMESTAMP flag allows applications to assign the RTP timestamp for each frame. MTL retrieves the timestamp from st**_tx_frame_meta and delays the frame transmission until the PTP reaches this timestamp. As a result, applications must calculate the timestamp with precision.
+The ST**_TX_FLAG_USER_TIMESTAMP flag allows applications to assign the RTP timestamp for each frame. MTL retrieves the timestamp from st**_tx_frame_meta and calculates RTP timestamp straight from this value.
+Transmission time in unchanged.
 
 The ST**_TX_FLAG_USER_TIMESTAMP flag is provided to enable applications to use their own timestamp values for each frame. Consequently, the RTP timestamp for all packets that belong to that frame will be assigned the customized value.
 

lib/src/st2110/st_tx_ancillary_session.c

@@ -270,7 +270,7 @@ static int tx_ancillary_session_sync_pacing(struct mtl_main_impl* impl,
 
   if (required_tai) {
     uint64_t ptp_epochs = ptp_time / frame_time;
-    epochs = required_tai / frame_time;
+    epochs = (required_tai + frame_time / 2) / frame_time;
     dbg("%s(%d), required tai %" PRIu64 " ptp_epochs %" PRIu64 " epochs %" PRIu64 "\n",
         __func__, s->idx, required_tai, ptp_epochs, epochs);
     if (epochs < ptp_epochs) {
@@ -291,11 +291,9 @@ static int tx_ancillary_session_sync_pacing(struct mtl_main_impl* impl,
   }
 
   if (interlaced) {
-    if (second_field) { /* align to odd epoch */
-      if (!(epochs & 0x1)) epochs++;
+    if (second_field) {
       ST_SESSION_STAT_INC(s, port_user_stats, stat_interlace_second_field);
-    } else { /* align to even epoch */
-      if (epochs & 0x1) epochs++;
+    } else {
       ST_SESSION_STAT_INC(s, port_user_stats, stat_interlace_first_field);
     }
   }

lib/src/st2110/st_tx_audio_session.c

@@ -275,7 +275,7 @@ static int tx_audio_session_sync_pacing(struct mtl_main_impl* impl,
 
   if (required_tai) {
     ptp_epochs = ptp_time / pkt_time;
-    epochs = required_tai / pkt_time;
+    epochs = (required_tai + pkt_time / 2) / pkt_time;
     if (epochs < ptp_epochs) {
       ST_SESSION_STAT_INC(s, port_user_stats.common, stat_error_user_timestamp);
       dbg("%s(%d), required tai %" PRIu64 " ptp_epochs %" PRIu64 " epochs %" PRIu64 "\n",

lib/src/st2110/st_tx_fastmetadata_session.c

@@ -270,7 +270,7 @@ static int tx_fastmetadata_session_sync_pacing(struct mtl_main_impl* impl,
 
   if (required_tai) {
     uint64_t ptp_epochs = ptp_time / frame_time;
-    epochs = required_tai / frame_time;
+    epochs = (required_tai + frame_time / 2) / frame_time;
     dbg("%s(%d), required tai %" PRIu64 " ptp_epochs %" PRIu64 " epochs %" PRIu64 "\n",
         __func__, s->idx, required_tai, ptp_epochs, epochs);
     if (epochs < ptp_epochs) {
@@ -291,11 +291,9 @@ static int tx_fastmetadata_session_sync_pacing(struct mtl_main_impl* impl,
   }
 
   if (interlaced) {
-    if (second_field) { /* align to odd epoch */
-      if (!(epochs & 0x1)) epochs++;
+    if (second_field) {
       ST_SESSION_STAT_INC(s, port_user_stats, stat_interlace_second_field);
-    } else { /* align to even epoch */
-      if (epochs & 0x1) epochs++;
+    } else {
       ST_SESSION_STAT_INC(s, port_user_stats, stat_interlace_first_field);
     }
   }

lib/src/st2110/st_tx_video_session.c

@@ -570,29 +570,15 @@ static int tv_init_pacing_epoch(struct mtl_main_impl* impl,
   return 0;
 }
 
-static inline uint64_t align_for_interlaced(struct st_tx_video_session_impl* s,
-                                            uint64_t frame_count, bool second_field) {
-  if (second_field) { /* align to odd epoch */
-    ST_SESSION_STAT_INC(s, port_user_stats, stat_interlace_second_field);
-    if (!(frame_count & 0x1)) frame_count++;
-  } else { /* align to even epoch */
-    ST_SESSION_STAT_INC(s, port_user_stats, stat_interlace_first_field);
-    if (frame_count & 0x1) frame_count++;
-  }
-  return frame_count;
-}
-
 static inline uint64_t calc_frame_count_since_epoch(struct st_tx_video_session_impl* s,
                                                     uint64_t cur_tai,
-                                                    uint64_t required_tai,
-                                                    bool second_field) {
+                                                    uint64_t required_tai) {
   uint64_t frame_count_tai = cur_tai / s->pacing.frame_time;
   uint64_t next_free_frame_slot = s->pacing.cur_epochs + 1;
-  bool interlaced = s->ops.interlaced;
   uint64_t frame_count;
 
   if (required_tai) {
-    frame_count = required_tai / s->pacing.frame_time;
+    frame_count = (required_tai + s->pacing.frame_time / 2) / s->pacing.frame_time;
   } else {
     if (frame_count_tai <= next_free_frame_slot) {
       /* There is time buffer until the next available frame time window */
@@ -618,11 +604,6 @@ static inline uint64_t calc_frame_count_since_epoch(struct st_tx_video_session_i
       frame_count = frame_count_tai;
     }
   }
-
-  if (interlaced) {
-    frame_count = align_for_interlaced(s, frame_count, second_field);
-  }
-
   return frame_count;
 }
 
@@ -645,7 +626,7 @@ static inline uint64_t validate_and_adjust_user_timestamp(
 }
 
 static int tv_sync_pacing(struct mtl_main_impl* impl, struct st_tx_video_session_impl* s,
-                          uint64_t required_tai, bool second_field) {
+                          uint64_t required_tai) {
   struct st_tx_video_pacing* pacing = &s->pacing;
   uint64_t cur_tai = mt_get_ptp_time(impl, MTL_PORT_P);
   uint64_t cur_tsc = mt_get_tsc(impl);
@@ -656,8 +637,7 @@ static int tv_sync_pacing(struct mtl_main_impl* impl, struct st_tx_video_session
     required_tai = validate_and_adjust_user_timestamp(s, required_tai, cur_tai);
   }
 
-  pacing->cur_epochs =
-      calc_frame_count_since_epoch(s, cur_tai, required_tai, second_field);
+  pacing->cur_epochs = calc_frame_count_since_epoch(s, cur_tai, required_tai);
   if (s->ops.flags & ST20_TX_FLAG_EXACT_USER_PACING) {
     start_time_tai = required_tai;
   } else {
@@ -685,12 +665,12 @@ static int tv_sync_pacing(struct mtl_main_impl* impl, struct st_tx_video_session
 
 static int tv_sync_pacing_st22(struct mtl_main_impl* impl,
                                struct st_tx_video_session_impl* s, uint64_t required_tai,
-                               bool second_field, int pkts_in_frame) {
+                               int pkts_in_frame) {
   struct st_tx_video_pacing* pacing = &s->pacing;
   /* reset trs */
   pacing->trs = pacing->frame_time * pacing->reactive / pkts_in_frame;
   dbg("%s(%d), trs %f\n", __func__, s->idx, pacing->trs);
-  return tv_sync_pacing(impl, s, required_tai, second_field);
+  return tv_sync_pacing(impl, s, required_tai);
 }
 
 static int tv_init_next_meta(struct st_tx_video_session_impl* s,
@@ -1266,14 +1246,17 @@ static int tv_build_rtp(struct mtl_main_impl* impl, struct st_tx_video_session_i
     s->port_user_stats.common.port[MTL_SESSION_PORT_P].frames++;
     if (s->ops.num_port > 1) s->port_user_stats.common.port[MTL_SESSION_PORT_R].frames++;
     s->st20_rtp_time = rtp->tmstamp;
-    bool second_field = false;
     if (s->ops.interlaced) {
       struct st20_rfc4175_rtp_hdr* rfc4175 = rte_pktmbuf_mtod_offset(
           pkt, struct st20_rfc4175_rtp_hdr*, sizeof(struct mt_udp_hdr));
       uint16_t line1_number = ntohs(rfc4175->row_number);
-      second_field = (line1_number & ST20_SECOND_FIELD) ? true : false;
+      if (line1_number & ST20_SECOND_FIELD) {
+        ST_SESSION_STAT_INC(s, port_user_stats, stat_interlace_second_field);
+      } else {
+        ST_SESSION_STAT_INC(s, port_user_stats, stat_interlace_first_field);
+      }
     }
-    tv_sync_pacing(impl, s, 0, second_field);
+    tv_sync_pacing(impl, s, 0);
     if (s->ops.flags & ST20_TX_FLAG_USER_TIMESTAMP) {
       s->pacing.rtp_time_stamp = ntohl(rtp->tmstamp);
     } else {
@@ -1332,14 +1315,17 @@ static int tv_build_rtp_chain(struct mtl_main_impl* impl,
     s->port_user_stats.common.port[MTL_SESSION_PORT_P].frames++;
     if (s->ops.num_port > 1) s->port_user_stats.common.port[MTL_SESSION_PORT_R].frames++;
     s->st20_rtp_time = rtp->tmstamp;
-    bool second_field = false;
     if (s->ops.interlaced) {
       struct st20_rfc4175_rtp_hdr* rfc4175 =
           rte_pktmbuf_mtod(pkt_chain, struct st20_rfc4175_rtp_hdr*);
       uint16_t line1_number = ntohs(rfc4175->row_number);
-      second_field = (line1_number & ST20_SECOND_FIELD) ? true : false;
+      if (line1_number & ST20_SECOND_FIELD) {
+        ST_SESSION_STAT_INC(s, port_user_stats, stat_interlace_second_field);
+      } else {
+        ST_SESSION_STAT_INC(s, port_user_stats, stat_interlace_first_field);
+      }
     }
-    tv_sync_pacing(impl, s, 0, second_field);
+    tv_sync_pacing(impl, s, 0);
     if (s->ops.flags & ST20_TX_FLAG_USER_TIMESTAMP) {
       s->pacing.rtp_time_stamp = ntohl(rtp->tmstamp);
     } else {
@@ -1793,8 +1779,16 @@ static int tv_tasklet_frame(struct mtl_main_impl* impl,
 
       /* user timestamp control if any */
       uint64_t required_tai = tv_pacing_required_tai(s, meta.tfmt, meta.timestamp);
-      bool second_field = frame->tv_meta.second_field;
-      tv_sync_pacing(impl, s, required_tai, second_field);
+      if (s->ops.interlaced) {
+        if (frame->tv_meta.second_field) {
+          ST_SESSION_STAT_INC(s, port_user_stats, stat_interlace_second_field);
+        } else {
+          ST_SESSION_STAT_INC(s, port_user_stats, stat_interlace_first_field);
+        }
+        /* s->second_field is used to init the next frame */
+        s->second_field = !frame->tv_meta.second_field;
+      }
+      tv_sync_pacing(impl, s, required_tai);
       if (ops->flags & ST20_TX_FLAG_USER_TIMESTAMP) {
         pacing->rtp_time_stamp =
             st10_get_media_clk(meta.tfmt, meta.timestamp, s->fps_tm.sampling_clock_rate);
@@ -1815,9 +1809,6 @@ static int tv_tasklet_frame(struct mtl_main_impl* impl,
       frame->tv_meta.rtp_timestamp = pacing->rtp_time_stamp;
       frame->tv_meta.epoch = pacing->cur_epochs;
       /* init to next field */
-      if (ops->interlaced) {
-        s->second_field = second_field ? false : true;
-      }
       MT_USDT_ST20_TX_FRAME_NEXT(s->mgr->idx, s->idx, next_frame_idx, frame->addr,
                                  pacing->rtp_time_stamp);
       /* check if dump USDT enabled */
@@ -2305,9 +2296,16 @@ static int tv_tasklet_st22(struct mtl_main_impl* impl,
 
       /* user timestamp control if any */
       uint64_t required_tai = tv_pacing_required_tai(s, meta.tfmt, meta.timestamp);
-      bool second_field = frame->tx_st22_meta.second_field;
-      tv_sync_pacing_st22(impl, s, required_tai, second_field,
-                          st22_info->st22_total_pkts);
+      if (s->ops.interlaced) {
+        if (frame->tx_st22_meta.second_field) {
+          ST_SESSION_STAT_INC(s, port_user_stats, stat_interlace_second_field);
+        } else {
+          ST_SESSION_STAT_INC(s, port_user_stats, stat_interlace_first_field);
+        }
+        /* s->second_field is used to init the next frame */
+        s->second_field = !frame->tx_st22_meta.second_field;
+      }
+      tv_sync_pacing_st22(impl, s, required_tai, st22_info->st22_total_pkts);
       if (ops->flags & ST20_TX_FLAG_USER_TIMESTAMP) {
         pacing->rtp_time_stamp =
             st10_get_media_clk(meta.tfmt, meta.timestamp, s->fps_tm.sampling_clock_rate);
@@ -2327,10 +2325,6 @@ static int tv_tasklet_st22(struct mtl_main_impl* impl,
       frame->tx_st22_meta.timestamp = pacing->ptp_time_cursor;
       frame->tx_st22_meta.epoch = pacing->cur_epochs;
       frame->tx_st22_meta.rtp_timestamp = pacing->rtp_time_stamp;
-      /* init to next field */
-      if (ops->interlaced) {
-        s->second_field = second_field ? false : true;
-      }
       MT_USDT_ST22_TX_FRAME_NEXT(s->mgr->idx, s->idx, next_frame_idx, frame->addr,
                                  pacing->rtp_time_stamp, codestream_size);
       /* check if dump USDT enabled */

tests/tools/RxTxApp/src/app_base.h

@@ -50,7 +50,7 @@
 
 #define ST_APP_DEFAULT_FB_CNT (3)
 
-#define ST_APP_USER_PACING_DEFAULT_OFFSET (10 * NS_PER_MS) /* 10ms */
+#define ST_APP_USER_CLOCK_DEFAULT_OFFSET (10 * NS_PER_MS) /* 10ms */
 
 #define ST_APP_EXPECT_NEAR(val, expect, delta) \
   ((val > (expect - delta)) && (val < (expect + delta)))
@@ -99,10 +99,10 @@ struct st_display {
   pthread_mutex_t display_frame_mutex;
 };
 
-struct st_user_pacing {
+struct st_user_time {
   uint64_t base_tai_time;
   pthread_mutex_t base_tai_time_mutex;
-  uint64_t user_pacing_offset;
+  uint64_t user_time_offset;
 };
 
 struct st_app_frameinfo {
@@ -521,9 +521,9 @@ struct st_app_tx_st20p_session {
   uint64_t last_stat_time_ns;
   bool sha_check;
   /* for now used only with user pacing to keep track of the frame timestamps */
-  uint64_t frame_time;
+  uint64_t frame_num;
   uint64_t local_tai_base_time;
-  struct st_user_pacing* user_pacing;
+  struct st_user_time* user_time;
 
   char st20p_source_url[ST_APP_URL_MAX_LEN];
   uint8_t* st20p_source_begin;
@@ -581,10 +581,10 @@ struct st_app_tx_st30p_session {
   uint8_t num_port;
   uint64_t last_stat_time_ns;
   /* for now used only with user pacing to keep track of the frame timestamps */
-  uint64_t frame_time;
+  uint64_t frame_num;
   uint64_t packet_time;
   uint64_t local_tai_base_time;
-  struct st_user_pacing* user_pacing;
+  struct st_user_time* user_time;
 
   char st30p_source_url[ST_APP_URL_MAX_LEN];
   uint8_t* st30p_source_begin;
@@ -757,7 +757,7 @@ struct st_app_context {
   char ttf_file[ST_APP_URL_MAX_LEN];
   int utc_offset;
 
-  struct st_user_pacing user_pacing;
+  struct st_user_time user_time;
 };
 
 static inline void* st_app_malloc(size_t sz) {
@@ -801,7 +801,7 @@ int st_set_mtl_log_file(struct st_app_context* ctx, const char* file);
 
 void st_sha_dump(const char* tag, const unsigned char* sha);
 
-uint64_t st_app_user_pacing_time(void* ctx, struct st_user_pacing* user_pacing,
-                                 uint64_t frame_time, bool restart_base_time);
+uint64_t st_app_user_time(void* ctx, struct st_user_time* user_time, uint64_t frame_num,
+                          double frame_time, bool restart_base_time);
 
 #endif

tests/tools/RxTxApp/src/args.c

@@ -152,7 +152,7 @@ enum st_args_cmd {
   ST_ARG_RSS_SCH_NB,
   ST_ARG_ALLOW_ACROSS_NUMA_CORE,
   ST_ARG_NO_MULTICAST,
-  ST_ARG_TX_USER_PACING_OFFSET,
+  ST_ARG_TX_USER_CLOCK_OFFSET,
   ST_ARG_MAX,
 };
 
@@ -301,7 +301,7 @@ static struct option st_app_args_options[] = {
     {"rss_sch_nb", required_argument, 0, ST_ARG_RSS_SCH_NB},
     {"allow_across_numa_core", no_argument, 0, ST_ARG_ALLOW_ACROSS_NUMA_CORE},
     {"no_multicast", no_argument, 0, ST_ARG_NO_MULTICAST},
-    {"tx_user_pacing_offset", required_argument, 0, ST_ARG_TX_USER_PACING_OFFSET},
+    {"tx_user_time_offset", required_argument, 0, ST_ARG_TX_USER_CLOCK_OFFSET},
     {"timestamp_epoch", no_argument, 0, ST_ARG_TIMESTAMP_EPOCH},
 
     {0, 0, 0, 0}};
@@ -930,8 +930,8 @@ int st_app_parse_args(struct st_app_context* ctx, struct mtl_init_params* p, int
       case ST_ARG_NO_MULTICAST:
         p->flags |= MTL_FLAG_NO_MULTICAST;
         break;
-      case ST_ARG_TX_USER_PACING_OFFSET:
-        ctx->user_pacing.user_pacing_offset = atoi(optarg);
+      case ST_ARG_TX_USER_CLOCK_OFFSET:
+        ctx->user_time.user_time_offset = atoi(optarg);
         break;
       case '?':
         break;

tests/tools/RxTxApp/src/parse_json.c

@@ -2041,6 +2041,9 @@ static int st_json_parse_tx_st20p(int idx, json_object* st20p_obj,
   st20p->exact_user_pacing =
       json_object_get_boolean(st_json_object_object_get(st20p_obj, "exact_user_pacing"));
 
+  st20p->user_timestamp =
+      json_object_get_boolean(st_json_object_object_get(st20p_obj, "user_timestamp"));
+
   return ST_JSON_SUCCESS;
 }
 
@@ -2720,12 +2723,12 @@ int st_app_parse_json(st_json_context_t* ctx, const char* filename) {
           }
         }
       }
-      /* parse global pacing offset options*/
-      json_object* pacing_obj = st_json_object_object_get(tx_group, "user_pacing_offset");
+      /* parse global clock offset options*/
+      json_object* pacing_obj = st_json_object_object_get(tx_group, "user_time_offset");
       if (pacing_obj) {
-        ctx->user_pacing_offset = json_object_get_int(pacing_obj);
+        ctx->user_time_offset = json_object_get_int(pacing_obj);
       } else {
-        ctx->user_pacing_offset = ST_APP_USER_PACING_DEFAULT_OFFSET;
+        ctx->user_time_offset = ST_APP_USER_CLOCK_DEFAULT_OFFSET;
       }
     }
   }

tests/tools/RxTxApp/src/parse_json.h

@@ -277,7 +277,8 @@ typedef struct st_json_st20p_session {
   bool enable_rtcp;
   bool user_pacing;
   bool exact_user_pacing;
-  uint64_t user_pacing_offset;
+  bool user_timestamp;
+  uint64_t user_time_offset;
 } st_json_st20p_session_t;
 
 typedef struct st_json_context {
@@ -292,7 +293,7 @@ typedef struct st_json_context {
   bool shared_rx_queues;
   bool tx_no_chain;
   char* log_file;
-  uint64_t user_pacing_offset;
+  uint64_t user_time_offset;
 
   st_json_video_session_t* tx_video_sessions;
   int tx_video_session_cnt;