1757 tm ci test

.gitignore

@@ -2,6 +2,7 @@ node_modules/
 .idea/
 wwwroot/build/
 wwwroot/doc/
+wwwroot/prerendered/
 npm-debug.log
 version.js
 wwwroot/init/nm.json
@@ -17,6 +18,7 @@ datasources/00_National_Data_Sets.json
 /error.log
 /output.log
 privateserverconfig.json
+wwwroot/sitemap.xml
 wwwroot/privateconfig.json
 deploy/packages/
 deploy/work/

.travis.yml

@@ -1,7 +1,15 @@
 language: node_js
+dist: trusty
+addons:
+  apt:
+    packages:
+      - libnss3
 node_js:
 - '8'
 script:
 - gulp lint release
 env:
 - NODE_OPTIONS=--max_old_space_size=2048
+before_install:
+  # Enable user namespace cloning
+  - "sysctl kernel.unprivileged_userns_clone=1"

buildprocess/generate-init-routes.js

@@ -0,0 +1,51 @@
+var fs = require("fs");
+var path = require("path");
+var URI = require("urijs");
+var CATALOG_ROUTE = require("terriajs/lib/ReactViewModels/TerriaRouting").CATALOG_ROUTE;
+
+const rootGroupName = "Root Group";
+
+const getNameFromItem = item => item.name;
+const getIdFromItem = item => item.id;
+
+
+const getRoutes = (topCatalogItem, currentRoute = rootGroupName) =>
+  topCatalogItem.reduce((acc, catalogItem) => {
+    const nameForCurrentItem = getNameFromItem(catalogItem);
+    // Id is used for uniqueid, so disregard any pathing
+    const idFromItem = getIdFromItem(catalogItem);
+    const finalRoute = idFromItem || `${currentRoute}/${nameForCurrentItem}`;
+    const items = catalogItem.items && getRoutes(catalogItem.items, finalRoute);
+
+    if (items) {
+      return [...acc, ...items, finalRoute];
+    }
+    return [...acc, finalRoute];
+  }, []);
+/**
+ * Given an array of init names for files in wwwroot/init/, traverse the catalog
+ * and generate a list of routes to be prerendered at build time
+ * 
+ * @param  {Array} initUrls
+ */
+function generateRoutes(initUrls) {
+  const resUrl = url =>
+    path.resolve(__dirname, "..", "wwwroot", "init", `${url}.json`);
+
+  const getCatalogFromInitName = initName => resUrl(initName);
+
+  const routesFromInitCatalogs = initUrls.reduce((acc, initName) => {
+    const fsPathForInit = getCatalogFromInitName(initName);
+    const data = fs.readFileSync(fsPathForInit, "utf8");
+    const literal = JSON.parse(data).catalog;
+
+    return [...acc, ...getRoutes(literal, `${rootGroupName}`)];
+  }, []);
+
+  const encodedRoutes = routesFromInitCatalogs.map(route => `${CATALOG_ROUTE}${URI.encode(route)}`);
+
+  // Make sure we also prerender /catalog/ incase user reloads on it? or serve up in terriajs server?
+  return [CATALOG_ROUTE, ...encodedRoutes];
+}
+
+module.exports = generateRoutes;

buildprocess/generate-terria-sitemap.js

@@ -0,0 +1,35 @@
+const generateUrl = (loc, lastmod, changefreq, priority) => (`
+  <url>
+    <loc>${loc}</loc>
+    <lastmod>${lastmod}</lastmod>
+    <changefreq>${changefreq}</changefreq>
+    <priority>${priority}</priority>
+  </url>
+`);
+
+/**
+ * Generate a sitemap to be written out during build
+ * 
+ * @param  {string} baseUrl absolute URL to where your map is going to be deployed to, with a trailing slash
+ * @param  {array} catalogRoutes list of routes, e.g. ['/catalog', '/catalog/someId']
+ */
+const generateTerriaSitemap = (baseUrl, catalogRoutes) => {
+  if (typeof baseUrl !== 'string') {
+    throw 'baseUrl passed to generateTerriaSitemap is not a string?'; 
+  }
+  if (baseUrl.substr(0,4) !== 'http') {
+    throw 'baseUrl does not look absolute, check that it begins with http or https';
+  }
+  const baseUrlWithoutTrailingSlash = (baseUrl.substr(-1) === '/') ? baseUrl.slice(0,-1) : baseUrl;
+  const date = new Date().toISOString().substr(0, 10);
+
+  const start = '<?xml version="1.0" encoding="UTF-8"?>';
+  const urlsetStart = '<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">';
+  const urlsetEnd = `</urlset>`;
+  const homeUrl = generateUrl(baseUrl, date, 'daily', '0.9');
+  const routeUrls = catalogRoutes.map(route => generateUrl(`${baseUrlWithoutTrailingSlash}${route}`, date, 'weekly', '0.5'));
+
+  return `${start}${urlsetStart}${homeUrl}${routeUrls.join('')}${urlsetEnd}`;
+};
+
+module.exports = generateTerriaSitemap;

buildprocess/webpack.config.js

@@ -1,9 +1,15 @@
 'use strict';
 
 /*global require*/
+var fs = require('fs');
 var configureWebpackForTerriaJS = require('terriajs/buildprocess/configureWebpack');
 var MiniCssExtractPlugin = require('mini-css-extract-plugin');
+var generateRoutes = require("./generate-init-routes");
+var generateTerriaSitemap = require("./generate-terria-sitemap");
+var PrerenderSPAPlugin = require("prerender-spa-plugin");
+var Renderer = PrerenderSPAPlugin.PuppeteerRenderer;
 var path = require('path');
+var json5 = require("json5");
 
 module.exports = function(devMode, hot) {
     var config = {
@@ -108,5 +114,51 @@ module.exports = function(devMode, hot) {
         }
     };
     config.resolve.alias['terriajs-variables'] = require.resolve('../lib/Styles/variables.scss');
+
+    if (!devMode) {
+        var configJsonPath = fs.readFileSync(path.resolve(__dirname, '..','wwwroot', 'config.json'), 'utf8');
+        var configJson = json5.parse(configJsonPath);
+        var prerenderRoutes =
+            (configJson &&
+                configJson.initializationUrls &&
+                configJson.initializationUrls.length > 0 &&
+                generateRoutes(configJson.initializationUrls)) ||
+            [];
+        var appBaseUrl =
+            (configJson &&
+                configJson.parameters &&
+                configJson.parameters.appBaseUrl &&
+                configJson.parameters.appBaseUrl.length > 0 &&
+                configJson.parameters.appBaseUrl);
+
+        console.log('The following routes generated from config.json\'s initializationUrls will be prerendered:');
+        console.log(prerenderRoutes);
+
+        if (appBaseUrl) {
+            try {
+                console.log('Attempting to write sitemap with appBaseUrl: ', appBaseUrl);
+                var sitemap = generateTerriaSitemap(appBaseUrl, prerenderRoutes);
+                var sitemapPath = path.resolve(__dirname, '..', 'wwwroot', 'sitemap.xml');
+                fs.writeFileSync(sitemapPath, new Buffer(sitemap));
+                console.log('Wrote out sitemap to: ' + sitemapPath);
+            } catch (e) {
+                console.error("Couldn't generate sitemap?", e);
+            }
+        } else {
+            console.warn("Warning - no appBaseUrl specified, no sitemap will be generated.")
+        }
+        config.plugins = [...config.plugins, new PrerenderSPAPlugin({
+            staticDir: path.resolve(__dirname, '..', 'wwwroot', ),
+            outputDir: path.resolve(__dirname, '..', 'wwwroot', 'prerendered'),
+            indexPath: path.resolve(__dirname, '..', 'wwwroot', 'index.html'),
+            routes: prerenderRoutes,
+            renderer: new Renderer({
+                timeout: 3000,
+                renderAfterDocumentEvent: 'prerender-end', 
+                maxConcurrentRoutes: 12,
+                // headless: false, // set to false for debugging
+            }),
+        })];
+    }
     return configureWebpackForTerriaJS(path.dirname(require.resolve('terriajs/package.json')), config, devMode, hot, MiniCssExtractPlugin);
 };

deploy/docker/Dockerfile

@@ -2,6 +2,8 @@
 FROM node:8
 
 RUN apt-get update && apt-get install -y gdal-bin
+RUN apt-get -qq -y install gconf-service libasound2 libatk1.0-0 libatk-bridge2.0-0 libc6 libcairo2 libcups2 libdbus-1-3 libexpat1 libfontconfig1 libgcc1 libgconf-2-4 libgdk-pixbuf2.0-0 libglib2.0-0 libgtk-3-0 libnspr4 libpango-1.0-0 libpangocairo-1.0-0 libstdc++6 libx11-6 libx11-xcb1 libxcb1 libxcomposite1 libxcursor1 libxdamage1 libxext6 libxfixes3 libxi6 libxrandr2 libxrender1 libxss1 libxtst6 ca-certificates fonts-liberation libappindicator1 libnss3 lsb-release xdg-utils wget
+
 
 RUN mkdir -p /usr/src/app && mkdir -p /etc/config/client
 WORKDIR /usr/src/app/component

devserverconfig.json

@@ -1,6 +1,8 @@
 {
     "port": 3001,
 
+    "baseHref": "/",
+
     "allowProxyFor" : [
         "nicta.com.au",
         "gov.au",

gulpfile.js

@@ -250,6 +250,14 @@ function mergeConfigs(original, override) {
 gulp.task('render-datasource-templates', function(done) {
     var ejs = require('ejs');
     var JSON5 = require('json5');
+    var serverConfig = JSON5.parse(fs.readFileSync('devserverconfig.json', 'utf8'));
+
+    var index = fs.readFileSync('wwwroot/index.ejs', 'utf8');
+    var baseHrefFromConfig = serverConfig.baseHref;
+    var indexResult = ejs.render(index, { baseHref: baseHrefFromConfig || "/"});
+
+    fs.writeFileSync(path.join('wwwroot', 'index.html'), new Buffer(indexResult));
+
     var templateDir = 'datasources';
     try {
         fs.accessSync(templateDir);

lib/Styles/loader.css

@@ -1,3 +1,6 @@
+      .tjs-explorer-window__modal-wrapper {
+        z-index:10000 !important;
+      }
       .loader-ui-right img{
         display: block;
         margin: calc(50vh - 150px) auto;

lib/Views/UserInterface.jsx

@@ -9,7 +9,7 @@ import PropTypes from "prop-types";
 import React from "react";
 import RelatedMaps from "./RelatedMaps";
 import SplitPoint from "terriajs/lib/ReactViews/SplitPoint";
-import StandardUserInterface from "terriajs/lib/ReactViews/StandardUserInterface/StandardUserInterface.jsx";
+import { StandardUserInterface } from "terriajs/lib/ReactViews/StandardUserInterface/StandardUserInterface.jsx";
 import version from "../../version";
 
 import "./global.scss";