Add integration test using Cockpit test API

Closes #16

.gitignore

@@ -6,3 +6,7 @@ node_modules/
 dist/
 /.vagrant
 package-lock.json
+Test*FAIL*
+bots/
+test/common/
+test/images/

Makefile

@@ -1,4 +1,9 @@
 PACKAGE_NAME := $(shell python3 -c "import json; print(json.load(open('package.json'))['name'])")
+ifeq ($(TEST_OS),)
+TEST_OS = centos-7
+endif
+export TEST_OS
+VM_IMAGE=$(CURDIR)/test/images/$(TEST_OS)
 
 all: node_modules/react-lite
 	NODE_ENV=$(NODE_ENV) npm run build
@@ -41,7 +46,28 @@ rpm: dist-gzip
 	rm -r "`pwd`/rpmbuild"
 	rm -r "`pwd`/output" "`pwd`/build"
 
+# build a VM with locally built cockpit-starter-kit.rpm installed
+$(VM_IMAGE): rpm bots
+	bots/image-customize -v -r 'rpm -e cockpit-starter-kit || true' -i cockpit -i `pwd`/cockpit-starter-kit-*.noarch.rpm -s $(CURDIR)/test/vm.install $(TEST_OS)
+
+# run the browser integration tests; skip check for SELinux denials
+check: node_modules/react-lite $(VM_IMAGE) test/common
+	TEST_AUDIT_NO_SELINUX=1 test/check-starter-kit
+
+# checkout Cockpit's bots/ directory for standard test VM images and API to launch them
+bots:
+	git fetch --depth=1 https://github.com/cockpit-project/cockpit.git
+	git checkout --force FETCH_HEAD -- bots/
+	git reset bots
+
+# checkout Cockpit's test API
+test/common:
+	# when running in CI, update FETCH_HEAD to cockpit repo
+	if git remote | grep -qF test; then git fetch origin master; fi
+	git checkout --force FETCH_HEAD -- test/common
+	git reset test/common
+
 node_modules/react-lite:
 	npm install
 
-.PHONY: all clean install install-only dist-gzip srpm rpm
+.PHONY: all clean install install-only dist-gzip srpm rpm check

README.md

@@ -14,6 +14,26 @@ to generate the distribution tarball. In `production` mode, source files are
 automatically minified and compressed. Set `NODE_ENV=production` if you want to
 duplicate this behavior.
 
+# Testing
+
+Run `make check` to build an RPM, install it into a standard Cockpit test VM
+(centos-7 by default), and run the test/check-starter-kit integration test on
+it. This uses Cockpit's Chrome DevTools Protocol based browser tests, through a
+Python API abstraction. Note that this API is not guaranteed to be stable, so
+if you run into failures and don't want to adjust tests, consider checking out
+Cockpit's test/common from a tag instead of master (see the `test/common`
+target in `Makefile`).
+
+After the test VM is prepared, you can manually run the test without rebuilding
+the VM, possibly with extra options for tracing and halting on test failures
+(for interactive debugging):
+
+    TEST_OS=centos-7 test/check-starter-kit -tvs
+
+You can also run the test against a different Cockpit image, for example:
+
+    TEST_OS=fedora-27 make check
+
 # Vagrant
 
 This directory contains a Vagrantfile that installs and starts cockpit on a

package.json

@@ -14,6 +14,7 @@
     "babel-loader": "^7.0.0",
     "babel-plugin-transform-react-jsx": "^6.24.1",
     "babel-preset-env": "^1.5.2",
+    "chrome-remote-interface": "^0.25.5",
     "compression-webpack-plugin": "~1.0.0",
     "copy-webpack-plugin": "~3.0.1",
     "eslint": "^3.0.0",
@@ -21,6 +22,7 @@
     "eslint-plugin-react": "~6.9.0",
     "jshint": "~2.9.1",
     "jshint-loader": "~0.8.3",
+    "sizzle": "^2.3.3",
     "webpack": "^2.6.1"
   },
   "dependencies": {

test/check-starter-kit

@@ -0,0 +1,33 @@
+#!/usr/bin/python
+# Run this with --help to see available options for tracing and debugging
+# See https://github.com/cockpit-project/cockpit/blob/master/test/common/testlib.py
+# "class Browser" and "class MachineCase" for the available API.
+
+import os
+import sys
+
+# import Cockpit's machinery for test VMs and its browser test API
+TEST_DIR = os.path.dirname(__file__)
+sys.path.append(os.path.join(TEST_DIR, "common"))
+sys.path.append(os.path.join(os.path.dirname(TEST_DIR), "bots/machine"))
+import testlib
+
+
+class TestStarterKit(testlib.MachineCase):
+    def testBasic(self):
+        b = self.browser
+        m = self.machine
+
+        self.login_and_go("/starter-kit")
+        # verify expected heading
+        b.wait_present(".container-fluid h2")
+        b.wait_text(".container-fluid h2", "Starter Kit")
+
+        # verify expected host name
+        hostname = m.execute("hostname").strip()
+        b.wait_present(".container-fluid span")
+        b.wait_text(".container-fluid span", "Running on " + hostname)
+
+
+if __name__ == '__main__':
+    testlib.test_main()

test/run

@@ -0,0 +1,4 @@
+#! /bin/bash
+# This is the expected entry point for Cockpit CI; will be called without
+# arguments but with an appropriate $TEST_OS
+make check

test/vm.install

@@ -0,0 +1,12 @@
+#!/bin/sh
+# image-customize script to enable cockpit in test VMs
+# The starter-kit RPM will be installed separately
+set -eu
+
+# don't force https:// (self-signed cert)
+printf "[WebService]\\nAllowUnencrypted=true\\n" > /etc/cockpit/cockpit.conf
+
+if type firewall-cmd >/dev/null 2>&1; then
+    firewall-cmd --add-service=cockpit --permanent
+fi
+systemctl enable cockpit.socket