Kawing jaeger

.travis.yml

@@ -5,7 +5,7 @@ services:
 
 script:
   # Build the deb package in the yarn development container
-  - docker run -v $PWD:$PWD -w $PWD yarnpkg/dev:latest /bin/sh -c "yarn && yarn build-dist && scripts/build-deb.sh"
+  - docker run --rm -u $(id -u):$(id -g) -v $PWD:$PWD -w $PWD yarnpkg/dev:latest /bin/sh -c "yarn && yarn build-dist && scripts/build-deb.sh"
 
   # Install gcloud so we can upload the build to our bucket
   - if [ ! -d "$HOME/google-cloud-sdk/bin" ]; then rm -rf $HOME/google-cloud-sdk; export CLOUDSDK_CORE_DISABLE_PROMPTS=1; curl https://sdk.cloud.google.com | bash; fi
@@ -17,4 +17,4 @@ script:
   - gcloud auth activate-service-account --key-file=credentials.json
 
   # Upload the built snap file
-  - gsutil cp artifacts/*.deb $GOOGLE_BUCKET/stemn-yarn.deb
+  - gsutil cp artifacts/*.deb $GOOGLE_BUCKET/stemn-yarn-$TRAVIS_BRANCH.deb

package.json

@@ -1,5 +1,5 @@
 {
-  "name": "yarn",
+  "name": "@stemn/yarn",
   "installationMethod": "unknown",
   "version": "1.15.0-0",
   "license": "BSD-2-Clause",
@@ -28,11 +28,13 @@
     "is-builtin-module": "^2.0.0",
     "is-ci": "^1.0.10",
     "is-webpack-bundle": "^1.0.0",
+    "jaeger-client": "^3.10.0",
     "leven": "^2.0.0",
     "loud-rejection": "^1.2.0",
     "micromatch": "^2.3.11",
     "mkdirp": "^0.5.1",
     "node-emoji": "^1.6.1",
+    "nodemon": "^1.18.9",
     "normalize-url": "^2.0.0",
     "npm-logical-tree": "^1.2.1",
     "object-path": "^0.11.2",
@@ -82,6 +84,7 @@
     "eslint-plugin-yarn-internal": "file:scripts/eslint-rules",
     "execa": "^0.11.0",
     "fancy-log": "^1.3.2",
+    "file-loader": "^3.0.1",
     "flow-bin": "^0.66.0",
     "git-release-notes": "^3.0.0",
     "gulp": "^4.0.0",
@@ -97,11 +100,12 @@
     "prettier": "^1.5.2",
     "string-replace-loader": "^2.1.1",
     "temp": "^0.8.3",
-    "webpack": "^2.1.0-beta.25",
+    "webpack": "3",
     "yargs": "^6.3.0"
   },
   "resolutions": {
-    "sshpk": "^1.14.2"
+    "sshpk": "^1.14.2",
+    "uglifyjs-webpack-plugin": "1"
   },
   "engines": {
     "node": ">=4.0.0"

scripts/build-webpack.js

@@ -81,6 +81,10 @@ const compiler = webpack({
         exclude: /node_modules|Caches/,
         loader: require.resolve('babel-loader')
       },
+      {
+        test: /\.thrift$/,
+        use: 'file-loader'
+      },
       {
         test: /rx\.lite\.aggregates\.js/,
         use: 'imports-loader?define=>false'

src/cli/index.js

@@ -25,6 +25,12 @@ import {version} from '../util/yarn-version.js';
 import handleSignals from '../util/signal-handler.js';
 import {boolify, boolifyWithDefault} from '../util/conversion.js';
 
+// STEMN import
+import {benchmark, debug} from './logging.js';
+var opentracing = require('opentracing');
+import {initTracer} from './jaeger.js';
+
+
 function findProjectRoot(base: string): string {
   let prev = null;
   let dir = base;
@@ -41,6 +47,7 @@ function findProjectRoot(base: string): string {
   return base;
 }
 
+/* [STEMN]: main function here (multiple threads executed) */
 export async function main({
   startArgs,
   args,
@@ -58,6 +65,7 @@ export async function main({
   loudRejection();
   handleSignals();
 
+
   // set global options
   commander.version(version, '-v, --version');
   commander.usage('[command] [flags]');
@@ -126,7 +134,7 @@ export async function main({
 
   // if -v is the first command, then always exit after returning the version
   if (args[0] === '-v') {
-    console.log(version.trim());
+    console.log(version.trim() + " [TRACING]");
     process.exitCode = 0;
     return;
   }
@@ -246,10 +254,32 @@ export async function main({
   const outputWrapperEnabled = boolifyWithDefault(process.env.YARN_WRAP_OUTPUT, true);
   const shouldWrapOutput = outputWrapperEnabled && !commander.json && command.hasWrapper(commander, commander.args);
 
+  /* [STEMN]: Initialize the tracer here */
   if (shouldWrapOutput) {
     reporter.header(commandName, {name: 'yarn', version});
+
+/*
+    const tracer = initTracer("yarn");
+    opentracing.initGlobalTracer(tracer);
+
+    process.on("exit", () => {
+      console.error("Closing tracer");
+      tracer.close();
+    })
+
+    process.on("error", () => {
+      conosle.error("Tracer prematurely closed due to error");
+      tracer.close();
+    })
+
+    process.on("SIGINT", () => {
+      conosle.error("Tracer prematurely closed due to interrupt");
+      tracer.close();
+    })
+*/
   }
 
+
   if (commander.nodeVersionCheck && !semver.satisfies(process.versions.node, constants.SUPPORTED_NODE_VERSIONS)) {
     reporter.warn(reporter.lang('unsupportedNodeVersion', process.versions.node, constants.SUPPORTED_NODE_VERSIONS));
   }
@@ -281,8 +311,10 @@ export async function main({
 
     return command.run(config, reporter, commander, commander.args).then(exitCode => {
       if (shouldWrapOutput) {
-        reporter.footer(false);
+          reporter.footer(false);
       }
+
+      /* [STEMN]: Possible hook here for exitCode conditional code */
       return exitCode;
     });
   };
@@ -478,6 +510,9 @@ export async function main({
     if (errorReportLoc) {
       reporter.info(reporter.lang('bugReport', errorReportLoc));
     }
+
+    /* [STEMN]: Warn in logs about unexpected error -> inconsistencies */
+    debug(">>>>>>>> WARNING: LOGS MAY BE INCONSISTENT DUE TO ERROR <<<<<<<<\n");
   }
 
   function writeErrorReport(log): ?string {

src/cli/logging.js

@@ -0,0 +1,70 @@
+const fs = require('fs');
+const rl = require("readline");
+const spawn = require('child_process').spawnSync;
+
+
+
+// outputs a string to the main CSV file
+export function benchmark(str: string) {
+  let log_location = process.env["YARN_LOG_PATH"] || "/tmp/yarn.csv";
+  fs.appendFileSync(log_location, str, function(err){if (err) throw err;});
+}
+
+// outputs a string to the debugging log file
+export function debug(str: string) {
+  let log_location = process.env["YARN_DEBUG_PATH"] || "/tmp/debug.log";
+  fs.appendFileSync(log_location, str, function(err){if (err) throw err;});
+}
+
+// post processes the debug log information into a more span-like format
+export function post_process() {
+  let log_location = process.env["YARN_DEBUG_PATH"] || "/tmp/debug.log";
+
+  // run $(column) on data and
+  let results = [];
+  let child = spawn("column", ["-s", "," , "-t", log_location]);
+  results = child.stdout.toString().split("\n");
+  if(!results) { console.error("Make sure console is installed and in $PATH !"); }
+
+  results = results.filter(String);   // remove empty string
+
+  let depth = 1; 
+  results.forEach( function(s, index) {
+
+    // change the indenting (conditional)
+    let indent_depth = depth;
+
+    // BEGIN and END of same process should be on same indent
+    if(results[index].match("END") && results[index-1].match("BEGIN")) {
+      indent_depth = depth - 1;
+      depth--;
+    }
+
+    if(results[index].match("BEGIN") && index > 0 && results[index-1].match("END")) {
+      indent_depth = depth + 1;
+      depth++;
+    }
+
+    results[index] = `(${indent_depth-1})\t` + "    ".repeat(indent_depth-1) + s;
+    results[index] = results[index].replace(/\]\t/, "]\t\t");
+
+    // increase/decrease indent for next line
+    if(s.match("BEGIN")) {
+      depth++;
+    } else if (s.match("END")) {
+      depth--;
+    } else { throw new Error('Regex mismatch !'); }
+  });
+
+  // change BEGIN and END to new separators
+  results.forEach( function(s, index) {
+    results[index] = results[index].replace(/ *(BEGIN|END) */,"^"); 
+  });
+
+  // run $(column) a second time
+  child = spawn("column", ["-s", "^", "-t"], {input: results.join("\n") + "\n"});
+
+  // write output to file
+  fs.writeFileSync(log_location, child.stdout.toString(), function(err){if (err) throw err;});
+
+}

src/cli/tracing.js

@@ -0,0 +1,43 @@
+import { initTracer } from 'jaeger-client';
+
+// returns a Tracer instance that will be given to initGlobalTracer
+function initJaegerTracer(serviceName) {
+
+  const config = {
+    serviceName: serviceName,
+    sampler: {
+      type: "const",
+      param: 1,
+    },
+    reporter: {
+      logSpans: true,
+    },
+  };
+
+  const options = {
+    logger: {
+      info(msg) {
+        // console.error(`INFO ${msg} [${serviceName}] [${process.pid}]`);
+      },
+      error(msg) {
+        // console.error("ERROR", msg);
+      },
+    },
+  };
+
+  return initTracer(config, options);
+}
+
+let tracer = undefined;
+export function getTracer() {
+
+  if(!tracer) {
+    //init globalTracer
+    tracer = initJaegerTracer("yarn");
+    //console.error("tracer has been initialized");
+  } else {
+    // console.error("returning already spawned tracer");
+  }
+
+    return tracer;
+}

src/reporters/console/console-reporter.js

@@ -22,6 +22,12 @@ import {sortTrees, recurseTree, getFormattedOutput} from './helpers/tree-helper.
 import inquirer from 'inquirer';
 import Table from 'cli-table3';
 
+// STEMN import
+import {benchmark, debug, post_process} from '../../cli/logging.js';
+import {getTracer} from '../../cli/tracing.js';
+var opentracing = require('opentracing');
+const fs = require('fs');
+
 const {inspect} = require('util');
 const readline = require('readline');
 const chalk = require('chalk');
@@ -159,20 +165,72 @@ export default class ConsoleReporter extends BaseReporter {
     }
   }
 
+ /* [STEMN]: header reporter. Hook to clean logs */
   header(command: string, pkg: Package) {
     this.log(this.format.bold(`${pkg.name} ${command} v${pkg.version}`));
+
+    if(!process.env.YARN_LOG_PATH) {
+      this._logCategory('LOGGING', 'magenta', "YARN_LOG_PATH env var not found.\tDefaulting to \'/tmp/yarn.csv\'");
+    }
+
+    if(!process.env.YARN_DEBUG_PATH) {
+      this._logCategory('LOGGING', 'magenta', "YARN_DEBUG_PATH env var not found.\tDefaulting to \'/tmp/debug.log\'");
+    }
+
+      this._logCategory('LOGGING', 'magenta', "env: YARN_JAEGER_TRACE will be used to store the traceID for the root span, " +  this.format.bold('DO NOT OVERWRITE'));
+		// we perform our own reporting as well
+    this._logCategory('LOGGING', 'magenta', "Cleaning logs of previous run...");
+    this._logCategory('LOGGING', 'magenta', "Truncating and preparing log file...");
+    let log_location = process.env["YARN_LOG_PATH"] || "/tmp/yarn.csv";
+    let debug_location = process.env["YARN_DEBUG_PATH"] || "/tmp/debug.log";
+
+    fs.writeFile(log_location, '', function(){})
+    fs.writeFile(debug_location, '', function(){})
+
+    var csv_header = "PID,Command,Timestamp,Duration,PWD\n";
+    fs.writeFileSync(log_location, csv_header, function (err) {
+        if (err) throw err;
+    });
+
+    // Begin the root span and save to environment variable
+    /*
+    const tracer = getTracer();
+    const tspan = tracer.startSpan("test-span");
+    tspan.setTag("processID", process.pid);
+
+    const carrier = {};
+    tracer.inject(tspan, opentracing.FORMAT_TEXT_MAP, carrier);
+    const env_value = JSON.stringify(carrier);
+    process.env['YARN_JAEGER_TRACE'] = env_value;
+    console.log(`Environment varible stores: ${env_value}`);
+
+    tspan.finish();
+    */
   }
 
+/* [STEMN]: footer reporter. Hook to do log post-processing */
   footer(showPeakMemory?: boolean) {
     this.stopProgress();
 
+    let log_location = process.env["YARN_LOG_PATH"] || "/tmp/yarn.csv";
+
     const totalTime = (this.getTotalTime() / 1000).toFixed(2);
     let msg = `Done in ${totalTime}s.`;
     if (showPeakMemory) {
       const peakMemory = (this.peakMemory / 1024 / 1024).toFixed(2);
       msg += ` Peak memory usage ${peakMemory}MB.`;
     }
     this.log(this._prependEmoji(msg, '✨'));
+
+    this._logCategory('LOGGING', 'magenta', "Post-processing logs into suitable format...");
+    post_process();
+    this._logCategory('LOGGING', 'magenta', "Output file: " + this.format.underline(log_location));
+
+    /*
+    const tracer = getTracer();
+    tracer.close();
+    */
+
   }
 
   log(msg: string, {force = false}: {force?: boolean} = {}) {