Add new "dry run" CLI option

docs/MMTC_Users_Guide.adoc

@@ -662,6 +662,13 @@ This option and any other *--clkchgrate-** options are mutually exclusive.
 |N/A
 |In a testing environment and in certain operational scenarios, it can be necessary to have MMTC assign the clock change rate of the new time correlation to 1.0, indicating no clock drift, rather than computing it. Also, if this option is selected,  MMTC will not replace the existing clock change rate in the SCLK kernel with an interpolated value. The value 1.00000000000 will appear in the new (latest) time correlation record in the SCLK kernel and SCLK/SCET file. This option and any other *--clkchgrate-** options are mutually exclusive.
 
+|-D
+|--dry-run
+|N/A
+|Runs MMTC in dry run mode. No output products will be modified or created and "what-if" output
+products will instead be printed to the console and written to the log. The run history file will likewise
+remain unappended. Otherwise runs MMTC normally according to other options and config keys.
+
 |-F
 |--disable-contact-filter
 |N/A

mmtc-core/src/main/java/edu/jhuapl/sd/sig/mmtc/app/TimeCorrelationApp.java

@@ -2,6 +2,8 @@
 
 import java.math.BigInteger;
 import java.math.RoundingMode;
+import java.nio.file.Files;
+import java.nio.file.Paths;
 import java.util.*;
 import java.math.BigDecimal;
 
@@ -24,6 +26,7 @@
 import org.apache.commons.lang3.StringUtils;
 import org.apache.logging.log4j.Logger;
 import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.core.appender.RollingFileAppender;
 
 import static edu.jhuapl.sd.sig.mmtc.app.MmtcCli.USER_NOTICE;
 
@@ -94,7 +97,9 @@ private void init() throws Exception {
         runHistoryFile = new RunHistoryFile(config.getRunHistoryFilePath(), config.getAllOutputProductDefs());
         runHistoryFile.updateRowsForNewProducts();
         newRunHistoryFileRecord = new TableRecord(runHistoryFile.getHeaders());
-        recordRunHistoryFilePreRunValues();
+        if (!ctx.config.isDryRun()) {
+            recordRunHistoryFilePreRunValues();
+        }
 
         tk_sclk_fine_tick_modulus = config.getTkSclkFineTickModulus();
         logger.info("tk_sclk_fine_tick_modulus set to: " + tk_sclk_fine_tick_modulus);
@@ -379,6 +384,10 @@ public void run() throws Exception {
         if (config.isTestMode()) {
             logger.warn(String.format("Test mode is enabled! One-way light time will be set to the provided value %f and ancillary positional and velocity calculations will be skipped.", config.getTestModeOwlt()));
         }
+        if (config.isDryRun()) {
+            logger.warn("Dry run mode is enabled! No data products from this run will be kept and will instead be printed to the console and recorded in the log file according to log4j2.xml");
+        }
+
 
         // Select telemetry for this new time correlation run
         final TimeCorrelationTarget tcTarget = selectSampleSetAndTimeCorrelationTarget();
@@ -483,22 +492,31 @@ public void run() throws Exception {
         // Perform all ancillary post-correlation operations
         new TimeCorrelationAncillaryOperations(ctx).perform();
 
-        // Write all output products
+        // Write or log all output products
         ctx.newSclkVersionString.set(getNextSclkKernelVersionString());
         for (OutputProductDefinition<?> prodDef : config.getAllOutputProductDefs()) {
             final String postRunColProdColName = RunHistoryFile.getPostRunProductColNameFor(prodDef);
 
-            if (prodDef.shouldBeWritten(ctx)) {
+            if (prodDef.shouldBeWritten(ctx) && !ctx.config.isDryRun()) {
                 final ProductWriteResult res = prodDef.write(ctx);
                 newRunHistoryFileRecord.setValue(postRunColProdColName, res.newVersion);
+
+            } else if (prodDef.shouldBeWritten(ctx) && ctx.config.isDryRun()) {
+                // Log/print output products instead of writing them to files
+                final String productPrintout = prodDef.getDryRunPrintout(ctx);
+                logger.info(USER_NOTICE, productPrintout);
             } else {
                 newRunHistoryFileRecord.setValue(postRunColProdColName,  runHistoryFile.getLatestNonEmptyValueOfCol(postRunColProdColName, RunHistoryFile.RollbackEntryOption.IGNORE_ROLLBACKS).orElse("-"));
             }
         }
-        runHistoryFile.writeRecord(newRunHistoryFileRecord);
 
-        logger.info(USER_NOTICE, "Appended a new entry to Run History File located at " + runHistoryFile.getPath());
-        logger.info(String.format("Run at %s recorded to %s", ctx.appRunTime, config.getRunHistoryFilePath().toString()));
+        // Update run history file if this isn't a dry run. If it is, delete the previously created temp SCLK kernel.
+        if (!ctx.config.isDryRun()) {
+            runHistoryFile.writeRecord(newRunHistoryFileRecord);
+            logger.info(USER_NOTICE, "Appended a new entry to Run History File located at " + runHistoryFile.getPath());
+            logger.info(String.format("Run at %s recorded to %s", ctx.appRunTime, config.getRunHistoryFilePath().toString()));
+        }
+
         logger.info(USER_NOTICE, "MMTC completed successfully.");
     }
 
@@ -590,4 +608,4 @@ public static String constructNextSclkKernelCounter(String currentName, String s
 
         return nextVersionStr;
     }
-}
+}

mmtc-core/src/main/java/edu/jhuapl/sd/sig/mmtc/cfg/CorrelationCommandLineConfig.java

@@ -92,6 +92,13 @@ public class CorrelationCommandLineConfig implements IConfiguration {
                 "Run in test mode, which allows the user to override one-way-light-time."
         );
 
+        opts.addOption(
+                "D",
+                "dry-run",
+                false,
+                "Enables dry-run mode, resulting in correlation outputs being printed & logged but not written to the filesystem. The run history file likewise won't be modified."
+        );
+
         opts.addOption("h", "help", false, "Print this message.");
 
         for (Option additionalOption : additionalOptions) {
@@ -176,6 +183,10 @@ boolean isTestMode() {
         return cmdLine.hasOption("T") || cmdLine.hasOption("test-mode-owlt");
     }
 
+    boolean isDryRun() {
+        return cmdLine.hasOption("D") || cmdLine.hasOption("dry-run");
+    }
+
     double getTestModeOwlt() {
         return Double.parseDouble(cmdLine.getOptionValue('T'));
     }

mmtc-core/src/main/java/edu/jhuapl/sd/sig/mmtc/cfg/TimeCorrelationAppConfig.java

@@ -280,6 +280,15 @@ public Map<String, TimeCorrelationFilter> getFilters() throws MmtcException {
         return filters;
     }
 
+    /**
+     * Indicates if this is a dry run
+     *
+     * @return true if -D or --dry-run CLI options are invoked
+     */
+    public boolean isDryRun() {
+        return cmdLineConfig.isDryRun();
+    }
+
     /**
      * Indicates if an optional Uplink Command File is to be created. This can be specified in either the command line
      * or in configuration parameters. Command line overrides the configuration parameter.

mmtc-core/src/main/java/edu/jhuapl/sd/sig/mmtc/correlation/TimeCorrelationContext.java

@@ -23,6 +23,7 @@ public class TimeCorrelationContext {
     public final Settable<Integer> runId = new Settable<>();
 
     public final Settable<SclkKernel> currentSclkKernel = new Settable<>();
+    public final Settable<SclkKernel> newSclkKernel = new Settable<>();
     public final Settable<String> newSclkVersionString = new Settable<>();
     public final Settable<Path> newSclkKernelPath = new Settable<>();
     public final Settable<Integer> tk_sclk_fine_tick_modulus = new Settable<>();

mmtc-core/src/main/java/edu/jhuapl/sd/sig/mmtc/products/definition/OutputProductDefinition.java

@@ -6,9 +6,12 @@
 import edu.jhuapl.sd.sig.mmtc.correlation.TimeCorrelationContext;
 import edu.jhuapl.sd.sig.mmtc.products.definition.util.ProductWriteResult;
 import edu.jhuapl.sd.sig.mmtc.products.definition.util.ResolvedProductLocation;
+import edu.jhuapl.sd.sig.mmtc.products.model.TextProductException;
 import edu.jhuapl.sd.sig.mmtc.rollback.TimeCorrelationRollback;
 import edu.jhuapl.sd.sig.mmtc.util.Settable;
+import edu.jhuapl.sd.sig.mmtc.util.TimeConvertException;
 
+import java.io.IOException;
 import java.nio.file.Path;
 import java.util.Map;
 import java.util.Optional;
@@ -59,6 +62,19 @@ public final boolean isBuiltIn() {
      */
     public abstract boolean shouldBeWritten(TimeCorrelationContext context);
 
+    /**
+     * Generates a string summarizing the hypothetical changes to an output product that would have been made had this been a real
+     * run and the dry run flag wasn't passed. This generally involves all the steps usually taken to compute and produce an MMTC
+     * output product except actually writing it to disk (except the SCLK kernel which must be written but will later be deleted).
+     * @param ctx The current TimeCorrelationContext
+     * @return A string representation of any additions that would have been made to an output product
+     * @throws MmtcException
+     * @throws TextProductException
+     * @throws IOException
+     * @throws TimeConvertException
+     */
+    public abstract String getDryRunPrintout(TimeCorrelationContext ctx) throws MmtcException, TextProductException, IOException, TimeConvertException;
+
     /**
      * Write the product to the filesystem.  Called once per successful time correlation run.
      *

mmtc-core/src/main/java/edu/jhuapl/sd/sig/mmtc/products/definition/RawTlmTableProductDefinition.java

@@ -6,10 +6,12 @@
 import edu.jhuapl.sd.sig.mmtc.products.definition.util.ProductWriteResult;
 import edu.jhuapl.sd.sig.mmtc.products.definition.util.ResolvedProductPath;
 import edu.jhuapl.sd.sig.mmtc.products.model.RawTelemetryTable;
+import edu.jhuapl.sd.sig.mmtc.products.model.TableRecord;
 
 import java.nio.file.Path;
-import java.util.HashMap;
-import java.util.Map;
+import java.util.*;
+import java.util.stream.Collectors;
+import java.util.stream.IntStream;
 
 public class RawTlmTableProductDefinition extends AppendedFileOutputProductDefinition {
     public RawTlmTableProductDefinition() {
@@ -26,6 +28,18 @@ public boolean shouldBeWritten(TimeCorrelationContext context) {
         return true;
     }
 
+    @Override
+    public String getDryRunPrintout(TimeCorrelationContext ctx) {
+        TableRecord rawTlmTableRecord = RawTelemetryTable.calculateUpdatedRawTlmTable(ctx);
+        List<String> rtHeaders = new RawTelemetryTable(ctx.config.getRawTelemetryTablePath()).getHeaders();
+        Collection<String> rtValues = rawTlmTableRecord.getValues();
+        String zippedRtRow = IntStream.range(0, rtHeaders.size())
+                .mapToObj(i -> "\t" + rtHeaders.get(i) + "\t:\t"+new ArrayList<>(rtValues)
+                        .get(i))
+                .collect(Collectors.joining("\n"));
+        return String.format("[DRY RUN] Updated Raw TLM table records: \n%s", zippedRtRow);
+    }
+
     @Override
     public ProductWriteResult appendToProduct(TimeCorrelationContext context) throws MmtcException {
         return RawTelemetryTable.appendCorrelationFrameSamplesToRawTelemetryTable(context);

mmtc-core/src/main/java/edu/jhuapl/sd/sig/mmtc/products/definition/SclkKernelProductDefinition.java

@@ -45,6 +45,27 @@ public boolean shouldBeWritten(TimeCorrelationContext context) {
         return true;
     }
 
+    /**
+     * This implementation is distinct from that of other output products in that it does still write the SCLK kernel to
+     * disk. The only difference is that it's written to the /tmp directory and its latest two lines are recorded here.
+     * @param ctx The active run's TimeCorrelationContext
+     * @return A string with details about the changed SCLK kernel(s) ready to be logged.
+     * @throws MmtcException if there are any problems generating the SCLK kernel
+     */
+    @Override
+    public String getDryRunPrintout(TimeCorrelationContext ctx) throws MmtcException {
+        SclkKernel.writeNewProduct(ctx);
+
+        String[] newSclkEntries = ctx.newSclkKernel.get().getLastXRecords(2);
+        // If an interpolated clock change rate has replaced the rate in the existing SCLK kernel record, retrieve the two latest records.
+        // Otherwise, just return the new record
+        if (ctx.newSclkKernel.get().hasNewClkChgRateSet()) {
+            return String.format("[DRY RUN] Updated SCLK entries: \n" + newSclkEntries[0] + "\n" + newSclkEntries[1]);
+        } else {
+            return String.format("[DRY RUN] New SCLK entry: \n" + newSclkEntries[0]);
+        }
+    }
+
     @Override
     public Map<String, String> getSandboxConfigUpdates(MmtcConfig originalConfig, Path newProductOutputDir) {
         final Map<String, String> confUpdates = new HashMap<>();

mmtc-core/src/main/java/edu/jhuapl/sd/sig/mmtc/products/definition/SclkScetProductDefinition.java

@@ -7,6 +7,18 @@
 import edu.jhuapl.sd.sig.mmtc.products.definition.util.ResolvedProductDirPrefixSuffix;
 import edu.jhuapl.sd.sig.mmtc.products.model.SclkKernel;
 import edu.jhuapl.sd.sig.mmtc.products.model.SclkScetFile;
+import edu.jhuapl.sd.sig.mmtc.products.model.TextProductException;
+import edu.jhuapl.sd.sig.mmtc.util.TimeConvertException;
+
+import java.io.IOException;
+import java.util.Arrays;
+
+import java.nio.file.Path;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.stream.Collectors;
+import java.util.stream.IntStream;
 
 import java.nio.file.Path;
 import java.util.HashMap;
@@ -20,6 +32,7 @@ public class SclkScetProductDefinition extends EntireFileOutputProductDefinition
     public SclkScetProductDefinition() {
         super("SCLKSCET File");
     }
+    private final int ENTRIES_TO_PRINT = 4;
 
     @Override
     public ResolvedProductDirPrefixSuffix resolveLocation(MmtcConfig conf) throws MmtcException {
@@ -37,6 +50,22 @@ public boolean shouldBeWritten(TimeCorrelationContext ctx) {
         return ctx.config.createSclkScetFile();
     }
 
+    @Override
+    public String getDryRunPrintout(TimeCorrelationContext ctx) throws MmtcException {
+        SclkScetFile scetFile = SclkScetFile.calculateNewProduct(ctx);
+        try {
+            scetFile.setSourceFilespec(ctx.newSclkKernel.get().getPath());
+            scetFile.updateFile();
+        } catch (TextProductException | TimeConvertException | IOException e) {
+            throw new MmtcException("Failed to generate SCLKSCET file");
+        }
+        List<String> newProductLines = scetFile.getNewProductLines();
+        String newRecs = IntStream.range(Math.max(0, newProductLines.size() - (ENTRIES_TO_PRINT+1)), newProductLines.size()-1) // Ignore footer row
+                .mapToObj(newProductLines::get)
+                .collect(Collectors.joining("\n\t"));
+        return String.format("[DRY RUN] Latest %d SCLKSCET file entries: \n\t%s\n\t%s", ENTRIES_TO_PRINT, scetFile.getSclkscetFields(), newRecs);
+    }
+
     /**
      * Writes a new SCLK-SCET File
      * @param ctx the current time correlation context from which to pull information for the output product

mmtc-core/src/main/java/edu/jhuapl/sd/sig/mmtc/products/definition/TimeHistoryFileProductDefinition.java

@@ -5,11 +5,14 @@
 import edu.jhuapl.sd.sig.mmtc.correlation.TimeCorrelationContext;
 import edu.jhuapl.sd.sig.mmtc.products.definition.util.ProductWriteResult;
 import edu.jhuapl.sd.sig.mmtc.products.definition.util.ResolvedProductPath;
+import edu.jhuapl.sd.sig.mmtc.products.model.TableRecord;
 import edu.jhuapl.sd.sig.mmtc.products.model.TimeHistoryFile;
+import edu.jhuapl.sd.sig.mmtc.util.TimeConvertException;
 
 import java.nio.file.Path;
-import java.util.HashMap;
-import java.util.Map;
+import java.util.*;
+import java.util.stream.Collectors;
+import java.util.stream.IntStream;
 
 public class TimeHistoryFileProductDefinition extends AppendedFileOutputProductDefinition {
     public TimeHistoryFileProductDefinition() {
@@ -26,6 +29,24 @@ public boolean shouldBeWritten(TimeCorrelationContext ctx) {
         return ctx.config.createTimeHistoryFile();
     }
 
+    @Override
+    public String getDryRunPrintout(TimeCorrelationContext ctx) throws MmtcException {
+        TimeHistoryFile timeHistFile = new TimeHistoryFile(ctx.config.getTimeHistoryFilePath(), ctx.config.getTimeHistoryFileExcludeColumns());
+        TableRecord timeHistRecord = new TableRecord(timeHistFile.getHeaders());
+        try {
+            TimeHistoryFile.generateNewTimeHistRec(ctx, timeHistFile, timeHistRecord);
+        } catch (TimeConvertException e) {
+            throw new RuntimeException(e);
+        }
+        List<String> thHeaders = new TimeHistoryFile(ctx.config.getTimeHistoryFilePath()).getHeaders();
+        Collection<String> thValues = timeHistRecord.getValues();
+        String zippedThRow = IntStream.range(0, thHeaders.size())
+                .mapToObj(i -> "\t" + thHeaders.get(i) + "\t:\t"+new ArrayList<>(thValues)
+                .get(i))
+                .collect(Collectors.joining("\n"));
+        return String.format("[DRY RUN] Updated Time History file records: \n%s", zippedThRow);
+    }
+
     @Override
     public ProductWriteResult appendToProduct(TimeCorrelationContext ctx) throws MmtcException {
         return TimeHistoryFile.appendRowFor(ctx);

mmtc-core/src/main/java/edu/jhuapl/sd/sig/mmtc/products/definition/UplinkCommandFileProductDefinition.java

@@ -6,6 +6,7 @@
 import edu.jhuapl.sd.sig.mmtc.products.definition.util.ProductWriteResult;
 import edu.jhuapl.sd.sig.mmtc.products.definition.util.ResolvedProductDirPrefixSuffix;
 import edu.jhuapl.sd.sig.mmtc.products.model.*;
+import edu.jhuapl.sd.sig.mmtc.util.TimeConvertException;
 
 import java.nio.file.Path;
 import java.nio.file.Paths;
@@ -37,6 +38,16 @@ public boolean shouldBeWritten(TimeCorrelationContext context) {
         return context.config.isCreateUplinkCmdFile();
     }
 
+    @Override
+    public String getDryRunPrintout(TimeCorrelationContext ctx) throws MmtcException {
+        try {
+            UplinkCommand uplinkCommand = UplinkCmdFile.generateNewProduct(ctx);
+            return String.format("[DRY RUN] Generated Uplink Command string: \n\t"+ uplinkCommand);
+        } catch (TimeConvertException e) {
+            throw new MmtcException("Unable to generate the Uplink Command File: ", e);
+        }
+    }
+
     /**
      * Writes the Uplink Command File
      * @param ctx the current time correlation context from which to pull information for the output product

mmtc-core/src/main/java/edu/jhuapl/sd/sig/mmtc/products/model/RawTelemetryTable.java

@@ -54,6 +54,24 @@ public List<String> getHeaders() {
         );
     }
 
+    /**
+     * Generate an updated RawTelemetryTable based on the current run, but don't append the new rows to the file.
+     * Intended for use in dry runs where the updated table is desired but not the changes to the file.
+     * @param ctx the current time correlation context from which to pull information for the output product
+     * @return an updated RawTelemetryTable
+     */
+    public static TableRecord calculateUpdatedRawTlmTable(TimeCorrelationContext ctx) {
+        final RawTelemetryTable rawTlmTable = new RawTelemetryTable(ctx.config.getRawTelemetryTablePath());
+        TableRecord rawTlmTableRecord = new TableRecord(rawTlmTable.getHeaders());
+
+        for (FrameSample sample : ctx.correlation.target.get().getSampleSet()) {
+            rawTlmTableRecord = sample.toRawTelemetryTableRecord(rawTlmTable.getHeaders());
+        }
+        rawTlmTableRecord.setValue(RawTelemetryTable.RUN_TIME, ctx.appRunTime.toString());
+
+        return rawTlmTableRecord;
+    }
+
     /**
      * Write the current time correlation's sample set to the raw telemetry table.
      *

mmtc-core/src/main/java/edu/jhuapl/sd/sig/mmtc/products/model/RunHistoryFile.java

@@ -270,4 +270,4 @@ public boolean anyValuesInColumn(String columnName) throws MmtcException {
         List<TableRecord> recs = readRecords(RollbackEntryOption.INCLUDE_ROLLBACKS);
         return recs.stream().anyMatch(r -> ! r.getValue(columnName).equals("-"));
     }
-}
+}