metaframe (#7)

* updated readme with build and install instructions

* nicer

* fixed repo link

* x86_64 arch detection fix

* Trace container with meta information.

It's pull request must be merget with pull request bpt-trace-PR to bap-pintrace repository.

* Support streams.

This patch adds support for non-seekable devices. It also cleans the
code and simplifies the implementation.

1. Removed any actions from the destructor. A finish member-function
must be called.
2. Removed an overload for a container, there is no need to bloat the
interface.
3. Header is initialized at the creation time.
4. Removed unnecessary instance variables and member functions.

The seek function is no longer called without a purpose. At the finish
procedure, if the device is seekable, we're storing the table of
contents and update an offset in the header. Otherwise, we just skip
it.

* finish the trace in the copytrace.

As now autofinish is disabled.

Author: feseal

libtrace/src/copytrace.cpp

@@ -25,7 +25,8 @@ int main(int argc, char **argv) {
   std::string dstfile(argv[2]);
 
   TraceContainerReader r(srcfile);
-  TraceContainerWriter w(dstfile, r.get_arch(), r.get_machine(), r.get_frames_per_toc_entry(), true);
+  TraceContainerWriter w(dstfile, r.get_arch(), r.get_machine(), r.get_frames_per_toc_entry());
 
   copy_all(r, w);
+  w.finish();
 }

libtrace/src/trace.container.cpp

@@ -23,133 +23,92 @@ typedef off_t traceoff_t;
 
 namespace SerializedTrace {
 
-  TraceContainerWriter::TraceContainerWriter(std::string filename,
-                                             frame_architecture arch,
-                                             uint64_t machine,
-                                             uint64_t frames_per_toc_entry_in,
-                                             bool auto_finish_in) throw (TraceException)
-    : num_frames (0),
-      frames_per_toc_entry (frames_per_toc_entry_in),
-      arch (arch),
-      mach (machine),
-      auto_finish (auto_finish_in),
-      is_finished (false)
-  {
-    ofs = fopen(filename.c_str(), "wb");
-    if (!ofs) { throw (TraceException("Unable to open trace file for writing")); }
-    SEEK(ofs, first_frame_offset);
+  FILE *open_trace(const std::string& filename,
+                  frame_architecture arch,
+                  uint64_t machine,
+                  uint64_t trace_version) {
+    FILE *ofs = fopen(filename.c_str(), "wb");
+    if (!ofs) throw TraceException("Unable to open trace file for writing");
+    int64_t toc_off = 0LL, toc_num_frames = 0LL;
+
+    WRITE(magic_number);
+    WRITE(trace_version);
+    uint64_t archt = (uint64_t) arch;
+    WRITE(archt);
+    WRITE(machine);
+    WRITE(toc_num_frames);
+    WRITE(toc_off);
+    return ofs;
   }
 
-  TraceContainerWriter::~TraceContainerWriter(void) throw () {
 
-    /** Call finish if it has not been called already ANd if
-        auto_finish is set. */
-    if (!is_finished && auto_finish) {
-      try {
-        finish();
-      }
-      catch (std::exception const &e) {
-        std::cerr << "Exception " << e.what() << " occured during TraceContainerWriter's auto-finish" << std::endl;
-      }
+  TraceContainerWriter::TraceContainerWriter(const std::string& filename,
+                                             frame_architecture arch,
+                                             uint64_t machine,
+                                             uint64_t frames_per_toc_entry_in)
+    throw (TraceException)
+    : num_frames (0)
+    , frames_per_toc_entry (frames_per_toc_entry_in)
+    , ofs(open_trace(filename,arch,machine,1LL)) {}
+
+  TraceContainerWriter::TraceContainerWriter(const std::string& filename,
+                                             const meta_frame& meta,
+                                             frame_architecture arch,
+                                             uint64_t machine,
+                                             uint64_t frames_per_toc_entry_in)
+    throw (TraceException)
+    : num_frames (0)
+    , frames_per_toc_entry (frames_per_toc_entry_in)
+    , ofs(open_trace(filename,arch,machine,2LL)) {
+    std::string meta_data;
+    if (!(meta.SerializeToString(&meta_data))) {
+      throw TraceException("Unable to serialize meta frame to ostream");
+    }
+
+    uint64_t meta_size = meta_data.length();
+    WRITE(meta_size);
+    if (fwrite(meta_data.c_str(), 1, meta_size, ofs) != meta_size) {
+      throw (TraceException("Unable to write meta frame to trace file"));
     }
   }
 
-  void TraceContainerWriter::add(frame &f) throw (TraceException) {
-    /* Is is time for a toc entry? */
+  void TraceContainerWriter::add(const frame &f) throw (TraceException) {
     if (num_frames > 0 && (num_frames % frames_per_toc_entry) == 0) {
-      /* Yes.  Add the file offset where we will insert this frame to
-         toc. */
       toc.push_back(TELL(ofs));
     }
-
     num_frames++;
 
-    /* Serialize to string so we can get the length. */
     std::string s;
     if (!(f.SerializeToString(&s))) {
       throw (TraceException("Unable to serialize frame to ostream"));
     }
-
-    /* Write the length before the frame. */
     uint64_t len = s.length();
-    if (len == 0) {
-      throw (TraceException("Attempt to add zero-length frame to trace"));
-    }
     WRITE(len);
-
-    /* Write the frame. */
-    traceoff_t old_offset = TELL(ofs);
-    if (old_offset == -1) {
-      throw (TraceException("Unable to determine the current offset in the trace"));
-    }
-
     if (fwrite(s.c_str(), 1, len, ofs) != len) {
       throw (TraceException("Unable to write frame to trace file"));
     }
-
-    /* Double-check our size. */
-    assert ((uint64_t)old_offset + len == (uint64_t)TELL(ofs));
   }
 
-  void TraceContainerWriter::finish(void) throw (TraceException) {
-    if (is_finished) {
-      throw (TraceException("finish called twice"));
-    }
-
-    /* Save the offset where we will write the toc. */
+  void TraceContainerWriter::finish() {
     uint64_t toc_offset = TELL(ofs);
-    if (toc_offset == -1) {
-      throw (TraceException("Unable to determine the current offset in the trace"));
-    }
-
-    /* Make sure the toc is the right size. */
-    assert ((num_frames == 0) || ((num_frames - 1) / frames_per_toc_entry) == toc.size());
-
-    /* Write frames per toc entry. */
-    WRITE(frames_per_toc_entry);
-
-    /* Write each offset to the file. */
-    for (std::vector<uint64_t>::size_type i = 0; i < toc.size(); i++) {
-      WRITE(toc[i]);
+    // if we have a positive offset, then the device is seekable, so
+    // we will write the TOC, otherwise we will skip it.
+    if (toc_offset > 0) {
+      assert ((num_frames - 1) / frames_per_toc_entry == toc.size());
+      WRITE(frames_per_toc_entry);
+      for (std::vector<uint64_t>::size_type i = 0; i < toc.size(); i++) {
+        WRITE(toc[i]);
+      }
+      SEEK(ofs, num_trace_frames_offset);
+      WRITE(num_frames);
+      SEEK(ofs, toc_offset_offset);
+      WRITE(toc_offset);
     }
 
-    /* Now we need to write the magic number, number of trace frames
-       and the offset of field m at the start of the trace. */
-
-    /* Magic number. */
-    SEEK(ofs, magic_number_offset);
-    WRITE(magic_number);
-
-    /* Trace version. */
-    SEEK(ofs, trace_version_offset);
-    WRITE(out_trace_version);
-
-    /* CPU architecture. */
-    SEEK(ofs, frame_arch_offset);
-    uint64_t archt = (uint64_t) arch;
-    WRITE(archt);
-
-    /* Machine type. */
-    SEEK(ofs, frame_machine_offset);
-    WRITE(mach);
-
-    /* Numer of trace frames */
-    SEEK(ofs, num_trace_frames_offset);
-    WRITE(num_frames);
-
-    /* Offset of toc. */
-    SEEK(ofs, toc_offset_offset);
-    WRITE(toc_offset);
-
-    /* Finally, close the file and mark us as finished. */
-    if (fclose(ofs)) {
-      throw (TraceException("Unable to close trace file"));
+    if (fclose(ofs) != 0) {
+      throw TraceException("Error while closing the trace");
     }
-    is_finished = true;
-  }
-
-  bool TraceContainerWriter::has_finished(void) throw () {
-    return is_finished;
+    ofs = NULL;
   }
 
   TraceContainerReader::TraceContainerReader(std::string filename) throw (TraceException)

libtrace/src/trace.container.hpp

@@ -6,6 +6,8 @@
 #endif
 #include "frame_arch.h"
 
+
+
 /**
  * A container for trace frames.  We do not use protobuffers because
  * protobuffers can not stream output (the whole trace would have to
@@ -50,7 +52,6 @@ namespace SerializedTrace {
   const uint64_t magic_number = 7456879624156307493LL;
 
   const uint64_t default_frames_per_toc_entry = 10000;
-  const uint64_t default_auto_finish = false;
   const frame_architecture default_arch = frame_arch_i386;
   const uint64_t default_machine = frame_mach_i386_i386;
 
@@ -62,9 +63,9 @@ namespace SerializedTrace {
   const uint64_t toc_offset_offset = 40LL;
   const uint64_t first_frame_offset = 48LL;
 
-  const uint64_t out_trace_version = 1LL;
   const uint64_t lowest_supported_version = 1LL;
-  const uint64_t highest_supported_version = out_trace_version;
+  const uint64_t highest_supported_version = 2LL;
+
 
     class TraceException: public std::exception
     {
@@ -93,34 +94,30 @@ namespace SerializedTrace {
     /** Creates a trace container writer that will output to
         [filename]. An entry will be added to the table of contents
         every [frames_per_toc_entry] entries.*/
-    TraceContainerWriter(std::string filename,
+    TraceContainerWriter(const std::string& filename,
                          frame_architecture arch = default_arch,
                          uint64_t machine = default_machine,
-                         uint64_t frames_per_toc_entry = default_frames_per_toc_entry,
-                         bool auto_finish = default_auto_finish) throw (TraceException);
+                         uint64_t frames_per_toc_entry = default_frames_per_toc_entry)
+      throw (TraceException);
 
-    /** Destructor that calls finish if auto_finish is true. */
-    ~TraceContainerWriter(void) throw ();
+    // creates a container for the second version of a protocol.
+    TraceContainerWriter(const std::string& filename,
+                         const meta_frame& meta,
+                         frame_architecture arch = default_arch,
+                         uint64_t machine = default_machine,
+                         uint64_t frames_per_toc_entry = default_frames_per_toc_entry)
+      throw (TraceException);
 
     /** Add [frame] to the trace. */
-    void add(frame &f) throw (TraceException);
-
-    /** Add all frames in container [c] to the trace. */
-    template <typename C>
-    void add(C &c) throw (TraceException) {
-      for (typename C::iterator i = c.begin(); i != c.end(); i++) {
-    	add(*i);
-      }
-    }
+    void add(const frame &f) throw (TraceException);
 
-    /** Finish the trace.  Builds and writes the table of contents to
-     * the file. Closes the file. */
-    void finish(void) throw (TraceException);
+    // closes the trace and underlying file stream. If the stream is
+    // seekable, the output a table of contents and update the header
+    // with an offset to the TOC.
+    void finish();
 
-    /** Returns true iff finish() has not been called on this trace. */
-    bool has_finished(void) throw ();
+    private:
 
-    protected:
 
     /* Output fstream for trace container file.
      *
@@ -137,17 +134,6 @@ namespace SerializedTrace {
     /** Frames per toc entry. */
     const uint64_t frames_per_toc_entry;
 
-    /** Architecture. */
-    const frame_architecture arch;
-
-    /** Machine type. */
-    const uint64_t mach;
-
-    /** Call [finish()] in destructor if not already done. */
-    bool auto_finish;
-
-    /** True if [finish()] been called on this writer. */
-    bool is_finished;
   };
 
   class TraceContainerReader {

test/README.md

@@ -0,0 +1,5 @@
+# Build tracedump utility:
+
+```
+bapbuild tracedump.native
+```

test/_tags

@@ -0,0 +1,2 @@
+true : package(bap-traces), package(cmdliner)
+<*.native> : package(findlib.dynload)

test/tracedump.ml

@@ -1,9 +1,11 @@
 open Core_kernel.Std
 open Bap.Std
 open Bap_traces.Std
+open Bap_plugins.Std
 open Result
 open Cmdliner
 
+
 let uri =
   let doc = "Trace resource identifier RFC3986." in
   let uri =
@@ -18,10 +20,11 @@ let uri =
 let print_error = function
   | `Protocol_error err -> Error.pp Format.err_formatter err
   | `System_error err -> prerr_string @@ Unix.error_message err
-  | `No_provider -> prerr_string "No provider for a given URI"
-  | `Ambiguous_uri -> prerr_string "More than one provider for a given URI"
+  | `No_provider -> prerr_string "No provider for a given URI\n"
+  | `Ambiguous_uri -> prerr_string "More than one provider for a given URI\n"
 
 let main uri =
+  Plugins.run ();
   Trace.load uri >>|
   (fun trace ->
      Trace.meta trace |>
@@ -41,7 +44,6 @@ let cmd =
   Term.info "tracedump" ~doc ~man
 
 let () =
-  let module M = Frame_trace_plugin in
   match Term.eval cmd with
   | `Error _ -> exit 1
   | `Ok result ->