diff --git a/Makefile b/Makefile
index 43050ab..2082c2b 100644
--- a/Makefile
+++ b/Makefile
@@ -81,11 +81,18 @@ devel-install: $(WEBPACK_TEST)
 	ln -s `pwd`/dist ~/.local/share/cockpit/$(PACKAGE_NAME)
 
 # when building a distribution tarball, call webpack with a 'production' environment
+# ship a stub node_modules/ so that `make` works without re-running `npm install`
 dist-gzip: NODE_ENV=production
-dist-gzip: $(NODE_MODULES_TEST) all cockpit-$(PACKAGE_NAME).spec
+dist-gzip: all cockpit-$(PACKAGE_NAME).spec
+	mv node_modules node_modules.release
+	mkdir -p $(NODE_MODULES_TEST)
+	touch -r package.json $(NODE_MODULES_TEST)
+	touch dist/*
 	tar czf cockpit-$(PACKAGE_NAME)-$(VERSION).tar.gz --transform 's,^,cockpit-$(PACKAGE_NAME)/,' \
 		--exclude cockpit-$(PACKAGE_NAME).spec.in \
-		$$(git ls-files) cockpit-$(PACKAGE_NAME).spec dist/
+		$$(git ls-files) cockpit-$(PACKAGE_NAME).spec dist/ node_modules
+	rm -rf node_modules
+	mv node_modules.release node_modules
 
 srpm: dist-gzip cockpit-$(PACKAGE_NAME).spec
 	rpmbuild -bs \
diff --git a/cockpit-starter-kit.spec.in b/cockpit-starter-kit.spec.in
index eb6c7b2..574bf0d 100644
--- a/cockpit-starter-kit.spec.in
+++ b/cockpit-starter-kit.spec.in
@@ -17,7 +17,7 @@ Cockpit Starter Kit Example Module
 %setup -n cockpit-starter-kit
 
 %build
-make
+# There is nothing to do, release tarballs already have dist/.
 
 %install
 make install DESTDIR=%{buildroot}