test: Add support for out-of-band storage of test/reference

.github/workflows/pixeltest-review.yml

@@ -0,0 +1,38 @@
+name: pixeltext-review
+on: pull_request_target
+jobs:
+  comment:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Clone
+        uses: actions/checkout@v2
+      - name: Look for changed reference images
+        uses: actions/github-script@v4
+        with:
+          github-token: ${{secrets.GITHUB_TOKEN}}
+          script: |
+            const { execSync } = require('child_process');
+
+            function get_path_sha(root, path) {
+               execSync('git fetch --depth=1 origin ' + root);
+               const val = execSync('git rev-parse --verify --quiet ' + root + ':' + path + '|| true');
+               return val.toString().trim();
+            }
+
+            const head = context.payload.pull_request.head.sha;
+            const base = context.payload.pull_request.base.sha;
+
+            const head_test_reference = get_path_sha(head, 'test/reference');
+            const base_test_reference = get_path_sha(base, 'test/reference');
+
+            core.info("shas: " + head_test_reference + ", " + base_test_reference);
+
+            if (head_test_reference != base_test_reference) {
+              core.info("commenting");
+              github.issues.createComment({
+                issue_number: context.issue.number,
+                owner: context.repo.owner,
+                repo: context.repo.repo,
+                body: 'Pixel test references have changed in ' + head + '.  You can review them [here](/cockpit-project/pixel-test-reference/compare/' + (base_test_reference || "empty") + '..' + (head_test_reference || "empty") + ').'
+              })
+            }

.gitmodules

@@ -0,0 +1,3 @@
+[submodule "test/reference"]
+	path = test/reference
+	url = ../pixel-test-reference

Makefile

@@ -93,6 +93,7 @@ $(TARFILE): $(WEBPACK_TEST) cockpit-$(PACKAGE_NAME).spec
 	touch dist/*
 	tar czf cockpit-$(PACKAGE_NAME)-$(VERSION).tar.gz --transform 's,^,cockpit-$(PACKAGE_NAME)/,' \
 		--exclude cockpit-$(PACKAGE_NAME).spec.in \
+		--exclude test/reference \
 		$$(git ls-files) $(LIB_TEST) src/lib/patternfly/*.scss package-lock.json cockpit-$(PACKAGE_NAME).spec dist/
 	mv node_modules.release node_modules
 
@@ -132,7 +133,7 @@ vm: $(VM_IMAGE)
 
 # run the browser integration tests; skip check for SELinux denials
 # this will run all tests/check-* and format them as TAP
-check: $(NODE_MODULES_TEST) $(VM_IMAGE) test/common
+check: $(NODE_MODULES_TEST) $(VM_IMAGE) test/common test/reference
 	TEST_AUDIT_NO_SELINUX=1 test/common/run-tests
 
 # checkout Cockpit's bots for standard test VM images and API to launch them
@@ -151,6 +152,9 @@ test/common:
 	    git checkout --force FETCH_HEAD -- test/common; \
 	    git reset test/common'
 
+test/reference: test/common
+	test/common/pixel-tests pull
+
 # checkout Cockpit's PF/React/build library; again this has no API stability guarantee, so check out a stable tag
 $(LIB_TEST):
 	flock Makefile sh -ec '\

test/browser/main.fmf

@@ -10,5 +10,6 @@ require:
   - make
   - npm
   - python3
+  - python3-pillow
 test: ./browser.sh
 duration: 60m

test/browser/run-test.sh

@@ -4,7 +4,7 @@ set -eux
 # tests need cockpit's bots/ libraries and test infrastructure
 cd $SOURCE
 git init
-make bots test/common
+make bots test/common test/reference
 
 # only install a subset to save time/space
 rm -f package-lock.json  # otherwise the command below installs *everything*, argh