out_s3: add time offset support for s3 key format

The S3 key format is not time zone aware, which makes it hard to use
when logs need to be stored in a different timezone than the S3
bucket's default.

This patch adds support for a configurable time offset in the S3 key
format. The offset is specified as a string like +0100 or -0100,
following the same convention used in other parse configuration
parameters.

Signed-off-by: Fan Zhuang <[email protected]>

Author: Noahnut

include/fluent-bit/flb_aws_util.h

@@ -194,7 +194,7 @@ int flb_read_file(const char *path, char **out_buf, size_t *out_size);
 
 /* Constructs S3 object key as per the format. */
 flb_sds_t flb_get_s3_key(const char *format, time_t time, const char *tag,
-                         char *tag_delimiter, uint64_t seq_index);
+                         char *tag_delimiter, uint64_t seq_index, const char *time_offset);
 
 /* Constructs S3 object key as per the blob format. */
 flb_sds_t flb_get_s3_blob_key(const char *format, const char *tag,
@@ -208,5 +208,12 @@ flb_sds_t flb_get_s3_blob_key(const char *format, const char *tag,
 size_t flb_aws_strftime_precision(char **out_buf, const char *time_format,
                                   struct flb_time *tms);
 
+
+/*
+ * Parses the time offset and returns the offset in seconds.
+ */
+size_t flb_aws_parse_tz_offset(const char *time_offset);
+
+
 #endif
 #endif /* FLB_HAVE_AWS */

plugins/out_s3/s3.c

@@ -1460,7 +1460,7 @@ static int s3_put_object(struct flb_s3 *ctx, const char *tag, time_t file_first_
     char final_body_md5[25];
 
     s3_key = flb_get_s3_key(ctx->s3_key_format, file_first_log_time, tag,
-                            ctx->tag_delimiters, ctx->seq_index);
+                            ctx->tag_delimiters, ctx->seq_index, ctx->time_offset);
     if (!s3_key) {
         flb_plg_error(ctx->ins, "Failed to construct S3 Object Key for %s", tag);
         return -1;
@@ -1648,7 +1648,7 @@ static struct multipart_upload *create_upload(struct flb_s3 *ctx, const char *ta
         return NULL;
     }
     s3_key = flb_get_s3_key(ctx->s3_key_format, file_first_log_time, tag,
-                            ctx->tag_delimiters, ctx->seq_index);
+                            ctx->tag_delimiters, ctx->seq_index, ctx->time_offset);
     if (!s3_key) {
         flb_plg_error(ctx->ins, "Failed to construct S3 Object Key for %s", tag);
         flb_free(m_upload);
@@ -4034,6 +4034,14 @@ static struct flb_config_map config_map[] = {
     "documentation."
     },
 
+    {
+     FLB_CONFIG_MAP_STR, "time_offset", "+0000",
+     0, FLB_TRUE, offsetof(struct flb_s3, time_offset),
+     "Time offset to apply to the s3 key format. This is useful when the s3 key format is "
+     "in a different timezone than the S3 bucket. The format is +HHMM or -HHMM. "
+     "Default is +0000."
+    },
+
     {
      FLB_CONFIG_MAP_STR, "s3_key_format_tag_delimiters", ".",
      0, FLB_TRUE, offsetof(struct flb_s3, tag_delimiters),

plugins/out_s3/s3.h

@@ -105,6 +105,7 @@ struct flb_s3 {
     char *bucket;
     char *region;
     char *s3_key_format;
+    char *time_offset;
     char *tag_delimiters;
     char *endpoint;
     char *sts_endpoint;
@@ -113,7 +114,7 @@ struct flb_s3 {
     char *storage_class;
     char *log_key;
     char *external_id;
-    char *profile;
+    char *profile;   
     int free_endpoint;
     int retry_requests;
     int use_put_object;

src/aws/flb_aws_util.c

@@ -27,10 +27,12 @@
 #include <fluent-bit/flb_jsmn.h>
 #include <fluent-bit/flb_env.h>
 
+#include <stddef.h>
 #include <stdlib.h>
 #include <sys/types.h>
 #include <sys/stat.h>
 #include <fcntl.h>
+#include <ctype.h>
 
 #define AWS_SERVICE_ENDPOINT_FORMAT            "%s.%s.amazonaws.com"
 #define AWS_SERVICE_ENDPOINT_BASE_LEN          15
@@ -1009,7 +1011,7 @@ flb_sds_t flb_get_s3_blob_key(const char *format,
 
 /* Constructs S3 object key as per the format. */
 flb_sds_t flb_get_s3_key(const char *format, time_t time, const char *tag,
-                         char *tag_delimiter, uint64_t seq_index)
+                         char *tag_delimiter, uint64_t seq_index, const char *time_offset)
 {
     int i = 0;
     int ret = 0;
@@ -1027,6 +1029,13 @@ flb_sds_t flb_get_s3_key(const char *format, time_t time, const char *tag,
     flb_sds_t tmp_key = NULL;
     flb_sds_t tmp_tag = NULL;
     struct tm gmt = {0};
+    size_t tz_offset = 0;
+
+    tz_offset = flb_aws_parse_tz_offset(time_offset);
+
+    if (tz_offset != 0) {
+        time += tz_offset;
+    }
 
     if (strlen(format) > S3_KEY_SIZE){
         flb_warn("[s3_key] Object key length is longer than the 1024 character limit.");
@@ -1324,3 +1333,38 @@ size_t flb_aws_strftime_precision(char **out_buf, const char *time_format,
 
     return out_size;
 }
+
+
+size_t flb_aws_parse_tz_offset(const char *time_offset) {
+    int sign, hh, mm;
+
+    if (!time_offset) {
+        return 0;
+    }
+
+    if (strlen(time_offset) < 5) {
+        return 0;
+    }
+
+    if (time_offset[0] != '+' && time_offset[0] != '-') {
+        return 0;
+    }
+
+    sign = (time_offset[0] == '+') ? 1 : -1;
+
+    if (!isdigit(time_offset[1]) || 
+        !isdigit(time_offset[2]) || 
+        !isdigit(time_offset[3]) || 
+        !isdigit(time_offset[4])) {
+        return 0;
+    }
+
+    hh = ((time_offset[1] - '0') * 10) + (time_offset[2] - '0');
+    mm = ((time_offset[3] - '0') * 10) + (time_offset[4] - '0');
+
+    if (hh < 0 || hh > 23 || mm < 0 || mm > 59) {
+        return 0;
+    }
+
+    return sign * (hh * 3600 + mm * 60);
+}