starter-kit/bots/images/scripts/openshift.setup

#! /bin/bash

set -eux

# Wait for x for many minutes
function wait() {
    for i in $(seq 1 100); do
        if eval "$@"; then
            return 0
        fi
        sleep 6
    done
    exit 6
}

function docker_images_has() {
    docker images | tr -s ' ' | cut -d ' ' --output-delimiter=: -f1,2 | grep -q "$1"
}

function docker_pull() {
    docker pull $1
    echo "$1" >> /tmp/pulledImages
    docker_images_has $1
}
rm -f /tmp/pulledImages # will be populated by pulled images names

# Cleanup the file system a bit
rm -rf /var/cache/dnf /var/cache/yum
xfs_growfs /

echo foobar | passwd --stdin root

nmcli con add con-name "static-eth1" ifname eth1 type ethernet ip4 "10.111.112.101/20" gw4 10.111.112.1 ipv4.dns "10.111.112.1"
nmcli con up "static-eth1"

echo "10.111.112.101     f1.cockpit.lan" >> /etc/hosts

printf "OPENSHIFT CONSOLE\n  https://10.111.112.101:8443\n  Login: scruffy  Password: scruffy\n\n" >> /etc/issue
printf "OPENSHIFT LISTENING ON LOCALHOST\n  $ ssh -NL 8443:localhost:8443 root@10.111.112.101\n\n" >> /etc/issue

# Disable these things
ln -sf ../selinux/config /etc/sysconfig/selinux
printf 'SELINUX=permissive\nSELINUXTYPE=targeted\n' > /etc/selinux/config
setenforce 0
systemctl stop firewalld
dnf mark install iptables
dnf -y remove firewalld
iptables -F

wait dnf -y install docker python libselinux-python

hostnamectl set-hostname f1.cockpit.lan

# Setup a nfs server
wait dnf install -y nfs-utils
mkdir /nfsexport
echo "/nfsexport *(rw,sync)" > /etc/exports

# This name is put into /etc/hosts later
echo "INSECURE_REGISTRY='--insecure-registry registry:5000'" >> /etc/sysconfig/docker
systemctl enable docker

# HACK: docker falls over regularly, print its log if it does
systemctl start docker || journalctl -u docker

# Can't use latest because release on older versions are done out of order
RELEASES_JSON=$(curl -s https://api.github.com/repos/openshift/origin/releases)
set +x
VERSION=$(echo "$RELEASES_JSON" | LC_ALL=C.UTF-8 python3 -c "import json, sys, distutils.version; obj=json.load(sys.stdin); releases = [x.get('tag_name', '') for x in obj if not x.get('prerelease')]; print(sorted (releases, reverse=True, key=distutils.version.LooseVersion)[0])") || {
    echo "Failed to parse latest release:" >&2
    echo "$RELEASES_JSON" >&2
    echo "------------------------------------" >&2
    exit 1
}
set -x

# origin is too rotund to build in a normal sized VM. The linker
# step runs out of memory. In addition origin has no Fedora packages
docker_pull "openshift/origin:$VERSION"
docker run --rm --entrypoint tar "openshift/origin:$VERSION" -C /usr/bin -c openshift oc kubectl | tar -C /usr/bin -xv

# Runs a master if on the right address, otherwise runs a node
cat > /openshift-prep <<EOF
#!/bin/sh -ex
/usr/bin/hostnamectl set-hostname f1.cockpit.lan
/usr/bin/systemctl enable rpcbind
/usr/bin/systemctl start rpcbind
/usr/bin/systemctl start nfs-server
cmd="/usr/bin/openshift start --master=10.111.112.101 --listen=https://0.0.0.0:8443"
echo "#!/bin/sh -ex
\$cmd" > /openshift-run
EOF

chmod +x /openshift-prep
touch /openshift-run
chmod +x /openshift-run

cat > /etc/systemd/system/openshift.service <<EOF
[Unit]
Description=Openshift
Wants=network-online.target
After=network-online.target docker.service
Requires=docker.service
[Service]
ExecStartPre=/openshift-prep
ExecStart=/openshift-run
Restart=always
RestartSec=60
[Install]
WantedBy=multi-user.target
EOF

systemctl daemon-reload
systemctl enable systemd-networkd-wait-online
systemctl enable openshift
systemctl start openshift || journalctl -u openshift

# Now pull all the images we're going to use with openshift
docker_pull "openshift/origin-deployer:$VERSION"
docker_pull "openshift/origin-docker-registry:$VERSION"
docker_pull "openshift/origin-pod:$VERSION"

# Now pull images used for integration tests
docker_pull registry:2

# HACK: Make openshift registry recognize docker registrys with the OpenShift CA
# (https://github.com/openshift/origin/issues/1753)
mkdir /tmp/registry
cd /tmp/registry
cat << EOF > Dockerfile
FROM openshift/origin-docker-registry:$VERSION
ADD *.crt /etc/pki/ca-trust/source/anchors/
USER 0
RUN update-ca-trust extract
USER 1001
EOF
cp /openshift.local.config/master/ca.crt openshift-ca.crt
docker build --tag openshift/origin-docker-registry:$VERSION .
cd /tmp/
rm -r /tmp/registry
cp /openshift.local.config/master/ca.crt /etc/pki/ca-trust/source/anchors/openshift-ca.crt
update-ca-trust extract

# HACK: Work around GnuTLS (client-side) or Go TLS (server-side) bug with
# multiple O= RDNs; if it's in the "wrong" order, create a new admin
# certificate that swaps it around
# See https://github.com/openshift/origin/issues/18715
dnf install -y openssl
if openssl x509 -in /openshift.local.config/master/admin.crt -text | grep -q 'Subject:.*system:cluster-admins.*system:masters'; then
    echo "Regenerating admin certificate to work around https://github.com/openshift/origin/issues/18715"
    pushd /openshift.local.config/master/
    mv admin.key admin.key.orig
    mv admin.crt admin.crt.orig
    mv admin.kubeconfig admin.kubeconfig.orig
    openssl genrsa -out admin.key 2048
    openssl req -new -nodes -key admin.key -out admin.csr -subj '/O=system:masters/O=system:cluster-admins/CN=system:admin'
    openssl x509 -req -in admin.csr -CA ca.crt -CAkey ca.key -CAcreateserial -days 730 -out admin.crt
    rm admin.csr
    oc adm create-kubeconfig --certificate-authority=ca.crt --client-certificate=admin.crt --client-key=admin.key --master="https://10.111.112.101:8443" --kubeconfig=admin.kubeconfig
    popd
fi

mkdir -p /root/.kube
cp /openshift.local.config/master/admin.kubeconfig /root/.kube/config

# Check if we can connect to openshift
wait oc get namespaces

wait oc get scc/restricted

# Tell openshift to allow root containers by default. Otherwise most
# development examples just plain fail to work
oc patch scc restricted -p '{ "runAsUser": { "type": "RunAsAny" } }'

# Tell openshift to allow logins from the openshift web console on a localhost system
oc patch oauthclient/openshift-web-console -p '{"redirectURIs":["https://10.111.112.101:8443/console/", "https://localhost:9000/"]}'

# Deploy the registry
# --credentials deprecated
rm -rf /usr/share/rhel/secrets
oc adm registry

function endpoint_has_address() {
    oc get endpoints $1 --template='{{.subsets}}' | grep -q addresses
}

function images_has() {
    oc get images | grep -q "$1"
}

# Wait for registry deployment to happen
wait oc get endpoints docker-registry
wait endpoint_has_address docker-registry

# Load in some remote images
echo '{"apiVersion":"v1","kind":"ImageStream","metadata": {"name":"busybox"},"spec":{"dockerImageRepository": "busybox"}}' > /tmp/imagestream.json
oc create -f /tmp/imagestream.json

# Get registry address and configure docker for it
address="$(oc get services docker-registry | grep -Eo '([0-9]{1,3}\.){3}[0-9]{1,3}')"
echo "$address     registry registry.cockpit.lan" >> /etc/hosts
echo "INSECURE_REGISTRY='--insecure-registry registry:5000 --insecure-registry $address'" >> /etc/sysconfig/docker

# Log in as another user
printf "scruffy\r\nscruffy\r\n" | oc login
oc new-project marmalade

token=$(oc whoami -t)
docker login -p "$token" -u unneeded registry:5000

echo '{"apiVersion":"v1","kind":"ImageStream","metadata": {"name":"busybee"}}' > /tmp/imagestream.json
oc create -f /tmp/imagestream.json
echo '{"apiVersion":"v1","kind":"ImageStream","metadata": {"name":"juggs"}}' > /tmp/imagestream.json
oc create -f /tmp/imagestream.json
echo '{"apiVersion":"v1","kind":"ImageStream","metadata": {"name":"origin"}}' > /tmp/imagestream.json
oc create -f /tmp/imagestream.json

# Get ready to push busybox into place
docker_pull busybox
docker tag busybox registry:5000/marmalade/busybee:latest
docker tag busybox registry:5000/marmalade/busybee:0.x
docker push registry:5000/marmalade/busybee

mkdir /tmp/juggs
cd /tmp/juggs
printf '#!/bin/sh\necho hello from container\nsleep 100000\n' > echo-script
printf 'FROM busybox\nMAINTAINER cockpit@example.com\nEXPOSE 8888\nADD echo-script /\nRUN chmod +x /echo-script\nCMD \"/echo-script\"' > Dockerfile
docker build -t registry:5000/marmalade/juggs:latest .
printf "FROM registry:5000/marmalade/juggs:latest\nVOLUME /test\nVOLUME /another\nWORKDIR /tmp" > Dockerfile
docker build -t registry:5000/marmalade/juggs:2.11 .
cp /usr/bin/openshift .
printf "FROM registry:5000/marmalade/juggs:latest\nADD openshift /usr/bin\nUSER nobody:wheel\nENTRYPOINT [\"top\", \"-b\"]\nCMD [\"-c\"]" > Dockerfile
docker build -t registry:5000/marmalade/juggs:2.5 .
printf "FROM registry:5000/marmalade/juggs:2.5\nSTOPSIGNAL SIGKILL\nONBUILD ADD . /app/src\nARG hello=test\nARG simple\nLABEL Test=Value\nLABEL version=\"1.0\"" > Dockerfile
docker build -t registry:5000/marmalade/juggs:2.8 .
printf "FROM registry:5000/marmalade/juggs:2.8\nLABEL description=\"This is a test description of an image. It can be as long as a paragraph, featuring a nice brogrammer sales pitch.\"\nLABEL name=\"Juggs Image\"\nLABEL build-date=2016-03-04\nLABEL url=\"http://hipsum.co/\"" > Dockerfile
docker build -t registry:5000/marmalade/juggs:2.9 .
cd /tmp
rm -r /tmp/juggs

docker push registry:5000/marmalade/juggs

# Tag this image twice
docker tag docker.io/busybox:latest registry:5000/marmalade/origin
docker push registry:5000/marmalade/origin
docker tag "openshift/origin:$VERSION" registry:5000/marmalade/origin
docker push registry:5000/marmalade/origin

oc new-project pizzazz

# Some big image streams
for i in $(seq 1 15); do
    for j in $(seq 1 10); do
        docker tag docker.io/busybox:latest registry:5000/pizzazz/stream$i:tag$j
    done
    docker push registry:5000/pizzazz/stream$i
done

# And a monster sized one
for j in $(seq 1 100); do
    docker tag docker.io/busybox:latest registry:5000/pizzazz/monster:tag$j
done
docker push registry:5000/pizzazz/monster

# Use the admin context by default
oc config use-context default/10-111-112-101:8443/system:admin

# Some roles for testing against
printf '{"kind":"List","apiVersion":"v1","items":[{"kind":"RoleBinding","apiVersion":"v1","metadata":{"name":"registry-editor","namespace":"marmalade","resourceVersion":"1"},"userNames":["scruffy","amanda"],"groupNames":null,"subjects":[{"kind":"User","name":"scruffy"},{"kind":"User","name":"amanda"}],"roleRef":{"name":"registry-editor"}},{"kind":"RoleBinding","apiVersion":"v1","metadata":{"name":"registry-viewer","namespace":"marmalade","resourceVersion":"1"},"userNames":["scruffy","tom","amanda"],"groupNames":["sports"],"subjects":[{"kind":"User","name":"scruffy"},{"kind":"User","name":"tom"},{"kind":"User","name":"amanda"},{"kind":"Group","name":"sports"}],"roleRef":{"name":"registry-viewer"}}]}' | oc create -f -
oc patch rolebinding/admin --namespace=marmalade -p '{"kind": "RoleBinding", "metadata":{"name":"admin","namespace":"marmalade"},"userNames":["scruffy"],"groupNames":null,"subjects":[{"kind":"User","name":"scruffys"}],"roleRef":{"name":"admin"}}' || true

# For testing the Cockpit OAuth client
printf '{"kind":"OAuthClient","apiVersion":"v1","metadata":{"name":"cockpit-oauth-devel"},"respondWithChallenges":false,"secret":"secret","allowAnyScope":true,"redirectURIs":["http://localhost:9001"] }' | oc create -f -

# Wait for it to download
wait images_has busybox

# Setup basics for building images
docker build -t cockpit/base /var/tmp/cockpit-base

# Print out the kubeconfig file for copy paste
echo "---------------------------------------------------------------"
cat /root/.kube/config

# Wait a bit in case an operator wants to copy some info
sleep 20

# Use standard locations for kubelet kubeconfig. f1.cockpit.lan is the master hostname, which
# is its own node and we just copy that for the others
mkdir -p /var/lib/kubelet
cp /openshift.local.config/node-f1.cockpit.lan/node.kubeconfig /var/lib/kubelet/kubeconfig

# Turn this on in sshd_config, not in use until binary is in place
printf 'AuthorizedKeysCommand /usr/local/bin/authorized-kube-keys --kubeconfig=/var/lib/kubelet/kubeconfig\nAuthorizedKeysCommandUser root' >> /etc/ssh/sshd_config

# Pull down remaining images
/var/lib/testvm/docker-images.setup

dnf install -y cockpit-system

docker info

# reduce image size
dnf clean all

systemctl stop docker
# write all changes before filling the disk
sync
/var/lib/testvm/zero-disk.setup
systemctl start docker && sleep 10

# Verify all pulled docker images are really present
echo All present images:
docker images
echo "Total docker images:"
docker images | wc

docker images --format "{{.Repository}}:{{.Tag}}" > /tmp/presentImages

echo
echo All images actually pulled
cat /tmp/presentImages
echo

echo
echo All images expected to be pulled
cat /tmp/pulledImages
echo

# Verify all expected are actually pulled
while read img ; do
  echo Verify "$img"
  grep "$img" /tmp/presentImages || (echo "Error: Image $img is missing" && exit 10)
done < /tmp/pulledImages