Remove obsolete 'experimentalAlignedExceptEdge' admin flag (#2433)

This flag was used to fetch edge runs back when they were not tagged
with the experimental label. It is no longer useful, and is also
misleading (the name implies it might remove Edge from the default
query, which it does not do).  See
#2431 (comment)

This change also makes util/populate_dev_data.go closer to what is
shipped on wpt.fyi - the flags are updated and a new Edge experimental
test run is added for the local testing sha. The data for that run is a copy
of the Edge stable data for the same sha.

Finally, the `webdriver/` README.md file is extensively updated with more
description and a lot of debugging tips I've discovered over the last day!

Author: stephenmcgruer

shared/test_run_filter.go

@@ -119,23 +119,6 @@ func (filter TestRunFilter) OrExperimentalRuns() TestRunFilter {
 	return filter
 }
 
-// OrAlignedExperimentalRunsExceptEdge returns the current filter, or, if it is a default
-// query, returns a query for the latest experimental runs.
-func (filter TestRunFilter) OrAlignedExperimentalRunsExceptEdge() TestRunFilter {
-	if !filter.IsDefaultQuery() {
-		return filter
-	}
-	aligned := true
-	filter.Aligned = &aligned
-	filter.Products = GetDefaultProducts()
-	for i := range filter.Products {
-		if filter.Products[i].BrowserName != "edge" {
-			filter.Products[i].Labels = mapset.NewSetWith("experimental")
-		}
-	}
-	return filter
-}
-
 // MasterOnly returns the current filter, ensuring it has with the master-only
 // restriction (a label of "master").
 func (filter TestRunFilter) MasterOnly() TestRunFilter {

util/populate_dev_data.go

@@ -89,6 +89,11 @@ func main() {
 	edge.ResultsURL = fmt.Sprintf(summaryURLFmtString, *localHost, staticRunSHA[:10], "edge[stable].json")
 	edge.Labels = []string{"edge", shared.StableLabel}
 
+	edgeExp := edge
+	edgeExp.BrowserVersion = "20"
+	edgeExp.ResultsURL = strings.Replace(edge.ResultsURL, "[stable]", "[experimental]", -1)
+	edgeExp.Labels = []string{"edge", shared.ExperimentalLabel}
+
 	firefox := chrome
 	firefox.BrowserName = "firefox"
 	firefox.BrowserVersion = "66"
@@ -115,6 +120,7 @@ func main() {
 		chrome,
 		chromeExp,
 		edge,
+		edgeExp,
 		firefox,
 		firefoxExp,
 		safari,
@@ -160,7 +166,7 @@ func main() {
 	addFlag(store, "diffFilter", enabledFlag)
 	addFlag(store, "diffFromAPI", enabledFlag)
 	addFlag(store, "experimentalByDefault", enabledFlag)
-	addFlag(store, "experimentalAlignedExceptEdge", enabledFlag)
+	addFlag(store, "experimentalAligned", enabledFlag)
 	addFlag(store, "structuredQueries", enabledFlag)
 	addFlag(store, "diffRenames", enabledFlag)
 	addFlag(store, "paginationTokens", enabledFlag)

webapp/components/wpt-flags.js

@@ -42,7 +42,6 @@ Object.defineProperty(wpt, 'ClientSideFeatures', {
       'displayMetadata',
       'experimentalByDefault',
       'experimentalAligned',
-      'experimentalAlignedExceptEdge',
       'fetchManifestForTestList',
       'githubCommitLinks',
       'githubLogin',
@@ -349,11 +348,6 @@ class WPTEnvironmentFlagsEditor extends FlagsEditorClass(/*environmentFlags*/ tr
         Align the default experimental runs
       </paper-checkbox>
     </paper-item>
-    <paper-item sub-item>
-      <paper-checkbox checked="{{experimentalAlignedExceptEdge}}">
-        All experimental, except edge[stable], and aligned
-      </paper-checkbox>
-    </paper-item>
     <paper-item>
       <paper-checkbox checked="{{paginationTokens}}">
         Return "wpt-next-page" pagination token HTTP header in /api/runs

webapp/static/24278ab617/edge[experimental].json

@@ -105,14 +105,10 @@ func populateHomepageData(r *http.Request) (data homepageData, err error) {
 	} else {
 		if pr == nil && testRunFilter.IsDefaultQuery() {
 			if aeAPI.IsFeatureEnabled("experimentalByDefault") {
-				if aeAPI.IsFeatureEnabled("experimentalAlignedExceptEdge") {
-					testRunFilter = testRunFilter.OrAlignedExperimentalRunsExceptEdge()
-				} else {
-					testRunFilter = testRunFilter.OrExperimentalRuns()
-					if aeAPI.IsFeatureEnabled("experimentalAligned") {
-						aligned := true
-						testRunFilter.Aligned = &aligned
-					}
+				testRunFilter = testRunFilter.OrExperimentalRuns()
+				if aeAPI.IsFeatureEnabled("experimentalAligned") || true {
+					aligned := true
+					testRunFilter.Aligned = &aligned
 				}
 			} else {
 				testRunFilter = testRunFilter.OrAlignedStableRuns()

webapp/test_results_handler.go

@@ -1,17 +1,130 @@
 # wpt.fyi Webdriver tests
 
-This directory covers Webdriver tests for the `webapp`, written in a 3rd-party
-Golang Webdriver client, [tebeka/selenium](https://github.com/tebeka/selenium).
+This directory containers integration tests for webapp/. These tests bring up
+the full webserver then use a Golang Webdriver client,
+[tebeka/selenium](https://github.com/tebeka/selenium), to load pages in a
+browser (Chrome or Firefox) and assert that the webapp behaves as expected.
 
 To run the tests, from the root `wpt.fyi` directory, run:
 
-    make go_webdriver_test
+    make go_large_test
 
-If you want to actually see the tests in action, disable the frame buffer.
+You can run just one of Chrome or Firefox via:
 
-    make webdriver_deps
-    go test --frame_buffer=false -tags=large ./webdriver
+    make go_chrome_test
+    make go_firefox_test
 
-If you want to use a custom installed location of selenium / browser / driver
-binaries, the required flags are shown in [the Makefile](../Makefile)
-`go_webdriver_test' rule.
+Note that when running these tests outside of docker (see [Running in
+docker](#running-in-docker)), they will use your locally installed browser and
+webdriver clients, so it is worth making sure they are the versions you expect.
+
+## Debugging
+
+Debugging the webdriver/ tests can be difficult as they are integration tests
+and the problem can occur anywhere from controlling the browser, to the webapp
+frontend, to the backend - and other weird bugs inbetween! This section
+contains some tips on how to effectively debug them.
+
+After running one of the above `make` commands at least once, one can directly
+run the golang tests via:
+
+```
+# You can copy GECKODRIVER_PATH out of the make output; it should be installed
+# locally under webapp/node_modules/selenium-standalone/...
+go test -v -timeout=15m -tags=large ./webdriver -args \
+    -firefox_path=/usr/bin/firefox \
+    -geckodriver_path=$GECKODRIVER_PATH \
+    -chrome_path=/usr/bin/google-chrome \
+    -chromedriver_path=/usr/bin/chromedriver \
+    -frame_buffer=true \
+    -staging=false \
+    -browser=chrome  # Or firefox
+```
+
+It is worth comparing this command-line against the Makefile, in case this
+documentation becomes out of date.
+
+### Running only one test
+
+If you only need to run one test, you can use the golang test [`-run`
+parameter](https://golang.org/pkg/testing/#hdr-Subtests_and_Sub_benchmarks).
+For example:
+
+```
+go test -v -timeout=15m -tags=large ./webdriver \
+    -run TestProductParam_Order/Order \
+    -args \
+    -firefox_path=/usr/bin/firefox \
+    -geckodriver_path=$GECKODRIVER_PATH \
+    -chrome_path=/usr/bin/google-chrome \
+    -chromedriver_path=/usr/bin/chromedriver \
+    -frame_buffer=true \
+    -staging=false \
+    -browser=chrome  # Or firefox
+```
+
+### Visual Output
+
+Many of the tests run some javascript (or click on an element, etc) and expect
+to find some resulting change on the page. When that doesn't occur, they
+usually just timeout and it can be difficult to know why. One very useful trick
+is to enable visual output, so that you can actually see the webpage as the
+test runs.
+
+To do this, set the `frame_buffer` argument to `false`, e.g.:
+
+```
+go test -v -timeout=15m -tags=large ./webdriver -args \
+    -firefox_path=/usr/bin/firefox \
+    -geckodriver_path=$GECKODRIVER_PATH \
+    -chrome_path=/usr/bin/google-chrome \
+    -chromedriver_path=/usr/bin/chromedriver \
+    -frame_buffer=false \
+    -staging=false \
+    -browser=chrome  # Or firefox
+```
+
+### Verbose webdriver output
+
+By default, webdriver output is hidden as it is very noisy. You can re-enable
+it by passing `-debug=true` to the tests, e.g.:
+
+```
+go test -v -timeout=15m -tags=large ./webdriver -args \
+    -firefox_path=/usr/bin/firefox \
+    -geckodriver_path=$GECKODRIVER_PATH \
+    -chrome_path=/usr/bin/google-chrome \
+    -chromedriver_path=/usr/bin/chromedriver \
+    -frame_buffer=true \
+    -staging=false \
+    -browser=chrome \
+    -debug=true
+```
+
+Redirecting stderr to stdout and saving it to a log-file is recommended due to
+the verbosity of webdriver logs (append `2>&1 | tee my-log.txt` to the above
+command).
+
+### Running in docker
+
+Sometimes bugs only occur in a docker-like environment. This can be difficult
+to reproduce, but a first step is to run the tests inside of docker. To do
+this, first start the docker container in one terminal tab:
+
+```
+./util/docker-dev/run.sh
+```
+
+Then, in another tab, we need to get the instance id of the container, exec
+'bash' inside of it, and run our test:
+
+```
+docker container ls
+[ note the CONTAINER ID ]
+docker exec -it $CONTAINER_ID bash
+user@abce84dd426d:~/wpt.fyi$
+[now you can run 'make go_chrome_test', or 'go test ...' directly, etc]
+```
+
+Note that this maps the host machine's wpt.fyi checkout into docker, so any
+code edits you make on the host are reflected in the container and vice-versa.

webdriver/README.md

@@ -15,13 +15,8 @@ import (
 
 func TestLabelParam_Results(t *testing.T) {
 	runWebdriverTest(t, func(t *testing.T, app AppServer, wd selenium.WebDriver) {
-		// Local static data only have 2 experimental browsers, and neither has aligned
-		// experimental runs.
-		if *staging {
-			testLabel(t, wd, app, "/", "experimental", "wpt-results", 4, false)
-		} else {
-			testLabel(t, wd, app, "/", "experimental", "wpt-results", 3, false)
-		}
+		aligned := false
+		testLabel(t, wd, app, "/", "experimental", "wpt-results", 4, aligned)
 	})
 
 }