diff --git a/bundle/tests/scorecard/kuttl-all-namespaces/configmap/00-assert.yaml b/bundle/tests/scorecard/kuttl-all-namespaces/configmap/00-assert.yaml new file mode 100644 index 00000000..57820b89 --- /dev/null +++ b/bundle/tests/scorecard/kuttl-all-namespaces/configmap/00-assert.yaml @@ -0,0 +1,8 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: configmap-wsliberty-app +status: + replicas: 1 + readyReplicas: 1 + updatedReplicas: 1 diff --git a/bundle/tests/scorecard/kuttl-all-namespaces/configmap/00-dry-run.yaml b/bundle/tests/scorecard/kuttl-all-namespaces/configmap/00-dry-run.yaml new file mode 100644 index 00000000..3c0876f7 --- /dev/null +++ b/bundle/tests/scorecard/kuttl-all-namespaces/configmap/00-dry-run.yaml @@ -0,0 +1,9 @@ +apiVersion: liberty.websphere.ibm.com/v1 +kind: WebSphereLibertyApplication +metadata: + name: configmap-wsliberty-app +spec: + applicationImage: icr.io/appcafe/websphere-liberty:full-java8-openj9-ubi + license: + accept: true + replicas: 1 diff --git a/bundle/tests/scorecard/kuttl-all-namespaces/configmap/01-delete.yaml b/bundle/tests/scorecard/kuttl-all-namespaces/configmap/01-delete.yaml new file mode 100644 index 00000000..755b0d8e --- /dev/null +++ b/bundle/tests/scorecard/kuttl-all-namespaces/configmap/01-delete.yaml @@ -0,0 +1,5 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +commands: + - command: kubectl delete webspherelibertyapplications --all --all-namespaces + - command: kubectl delete configmap websphere-liberty-operator -n openshift-operators diff --git a/bundle/tests/scorecard/kuttl-all-namespaces/configmap/01-errors.yaml b/bundle/tests/scorecard/kuttl-all-namespaces/configmap/01-errors.yaml new file mode 100644 index 00000000..3ef7c21f --- /dev/null +++ b/bundle/tests/scorecard/kuttl-all-namespaces/configmap/01-errors.yaml @@ -0,0 +1,5 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: websphere-liberty-operator + namespace: openshift-operators diff --git a/bundle/tests/scorecard/kuttl-all-namespaces/configmap/02-assert.yaml b/bundle/tests/scorecard/kuttl-all-namespaces/configmap/02-assert.yaml new file mode 100644 index 00000000..3ef7c21f --- /dev/null +++ b/bundle/tests/scorecard/kuttl-all-namespaces/configmap/02-assert.yaml @@ -0,0 +1,5 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: websphere-liberty-operator + namespace: openshift-operators diff --git a/bundle/tests/scorecard/kuttl-all-namespaces/configmap/02-liberty.yaml b/bundle/tests/scorecard/kuttl-all-namespaces/configmap/02-liberty.yaml new file mode 100644 index 00000000..3c0876f7 --- /dev/null +++ b/bundle/tests/scorecard/kuttl-all-namespaces/configmap/02-liberty.yaml @@ -0,0 +1,9 @@ +apiVersion: liberty.websphere.ibm.com/v1 +kind: WebSphereLibertyApplication +metadata: + name: configmap-wsliberty-app +spec: + applicationImage: icr.io/appcafe/websphere-liberty:full-java8-openj9-ubi + license: + accept: true + replicas: 1 diff --git a/bundle/tests/scorecard/kuttl-single-namespace/configmap/00-assert.yaml b/bundle/tests/scorecard/kuttl-single-namespace/configmap/00-assert.yaml new file mode 100644 index 00000000..57820b89 --- /dev/null +++ b/bundle/tests/scorecard/kuttl-single-namespace/configmap/00-assert.yaml @@ -0,0 +1,8 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: configmap-wsliberty-app +status: + replicas: 1 + readyReplicas: 1 + updatedReplicas: 1 diff --git a/bundle/tests/scorecard/kuttl-single-namespace/configmap/00-dry-run.yaml b/bundle/tests/scorecard/kuttl-single-namespace/configmap/00-dry-run.yaml new file mode 100644 index 00000000..3c0876f7 --- /dev/null +++ b/bundle/tests/scorecard/kuttl-single-namespace/configmap/00-dry-run.yaml @@ -0,0 +1,9 @@ +apiVersion: liberty.websphere.ibm.com/v1 +kind: WebSphereLibertyApplication +metadata: + name: configmap-wsliberty-app +spec: + applicationImage: icr.io/appcafe/websphere-liberty:full-java8-openj9-ubi + license: + accept: true + replicas: 1 diff --git a/bundle/tests/scorecard/kuttl-single-namespace/configmap/01-delete.yaml b/bundle/tests/scorecard/kuttl-single-namespace/configmap/01-delete.yaml new file mode 100644 index 00000000..34bdcb03 --- /dev/null +++ b/bundle/tests/scorecard/kuttl-single-namespace/configmap/01-delete.yaml @@ -0,0 +1,6 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +commands: + - command: kubectl delete webspherelibertyapplications --all + namespaced: true + - command: kubectl delete configmap websphere-liberty-operator -n WEBSPHERE_LIBERTY_OPERATOR_NAMESPACE diff --git a/bundle/tests/scorecard/kuttl-single-namespace/configmap/01-errors.yaml b/bundle/tests/scorecard/kuttl-single-namespace/configmap/01-errors.yaml new file mode 100644 index 00000000..f0d57a75 --- /dev/null +++ b/bundle/tests/scorecard/kuttl-single-namespace/configmap/01-errors.yaml @@ -0,0 +1,5 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: websphere-liberty-operator + namespace: WEBSPHERE_LIBERTY_OPERATOR_NAMESPACE diff --git a/bundle/tests/scorecard/kuttl-single-namespace/configmap/02-assert.yaml b/bundle/tests/scorecard/kuttl-single-namespace/configmap/02-assert.yaml new file mode 100644 index 00000000..f0d57a75 --- /dev/null +++ b/bundle/tests/scorecard/kuttl-single-namespace/configmap/02-assert.yaml @@ -0,0 +1,5 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: websphere-liberty-operator + namespace: WEBSPHERE_LIBERTY_OPERATOR_NAMESPACE diff --git a/bundle/tests/scorecard/kuttl-single-namespace/configmap/02-liberty.yaml b/bundle/tests/scorecard/kuttl-single-namespace/configmap/02-liberty.yaml new file mode 100644 index 00000000..3c0876f7 --- /dev/null +++ b/bundle/tests/scorecard/kuttl-single-namespace/configmap/02-liberty.yaml @@ -0,0 +1,9 @@ +apiVersion: liberty.websphere.ibm.com/v1 +kind: WebSphereLibertyApplication +metadata: + name: configmap-wsliberty-app +spec: + applicationImage: icr.io/appcafe/websphere-liberty:full-java8-openj9-ubi + license: + accept: true + replicas: 1 diff --git a/bundle/tests/scorecard/kuttl/configmap/00-assert.yaml b/bundle/tests/scorecard/kuttl/configmap/00-assert.yaml new file mode 100644 index 00000000..57820b89 --- /dev/null +++ b/bundle/tests/scorecard/kuttl/configmap/00-assert.yaml @@ -0,0 +1,8 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: configmap-wsliberty-app +status: + replicas: 1 + readyReplicas: 1 + updatedReplicas: 1 diff --git a/bundle/tests/scorecard/kuttl/configmap/00-dry-run.yaml b/bundle/tests/scorecard/kuttl/configmap/00-dry-run.yaml new file mode 100644 index 00000000..3c0876f7 --- /dev/null +++ b/bundle/tests/scorecard/kuttl/configmap/00-dry-run.yaml @@ -0,0 +1,9 @@ +apiVersion: liberty.websphere.ibm.com/v1 +kind: WebSphereLibertyApplication +metadata: + name: configmap-wsliberty-app +spec: + applicationImage: icr.io/appcafe/websphere-liberty:full-java8-openj9-ubi + license: + accept: true + replicas: 1 diff --git a/bundle/tests/scorecard/kuttl/configmap/01-delete.yaml b/bundle/tests/scorecard/kuttl/configmap/01-delete.yaml new file mode 100644 index 00000000..0d9b0784 --- /dev/null +++ b/bundle/tests/scorecard/kuttl/configmap/01-delete.yaml @@ -0,0 +1,7 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +commands: + - command: kubectl delete webspherelibertyapplications --all + namespaced: true + - command: kubectl delete configmap websphere-liberty-operator + namespaced: true diff --git a/bundle/tests/scorecard/kuttl/configmap/01-errors.yaml b/bundle/tests/scorecard/kuttl/configmap/01-errors.yaml new file mode 100644 index 00000000..db216f7b --- /dev/null +++ b/bundle/tests/scorecard/kuttl/configmap/01-errors.yaml @@ -0,0 +1,4 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: websphere-liberty-operator diff --git a/bundle/tests/scorecard/kuttl/configmap/02-assert.yaml b/bundle/tests/scorecard/kuttl/configmap/02-assert.yaml new file mode 100644 index 00000000..db216f7b --- /dev/null +++ b/bundle/tests/scorecard/kuttl/configmap/02-assert.yaml @@ -0,0 +1,4 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: websphere-liberty-operator diff --git a/bundle/tests/scorecard/kuttl/configmap/02-liberty.yaml b/bundle/tests/scorecard/kuttl/configmap/02-liberty.yaml new file mode 100644 index 00000000..9f4c6d1c --- /dev/null +++ b/bundle/tests/scorecard/kuttl/configmap/02-liberty.yaml @@ -0,0 +1,9 @@ +apiVersion: liberty.websphere.ibm.com/v1 +kind: WebSphereLibertyApplication +metadata: + name: configmap-wsliberty-app +spec: + applicationImage: icr.io/appcafe/websphere-liberty:full-java8-openj9-ubi + license: + accept: true + replicas: 1 \ No newline at end of file diff --git a/config/rbac/kuttl-rbac-watch-all.yaml b/config/rbac/kuttl-rbac-watch-all.yaml new file mode 100644 index 00000000..dfb965fe --- /dev/null +++ b/config/rbac/kuttl-rbac-watch-all.yaml @@ -0,0 +1,53 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: scorecard-kuttl-watch-all + namespace: WEBSPHERE_LIBERTY_WATCH_NAMESPACE +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: scorecard-kuttl-watch-all +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: scorecard-kuttl-watch-all +subjects: +- kind: ServiceAccount + name: scorecard-kuttl-watch-all + namespace: WEBSPHERE_LIBERTY_WATCH_NAMESPACE +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: scorecard-kuttl-watch-all +rules: +- apiGroups: + - "" + resources: + - configmaps + verbs: + - get + - list + - delete +- apiGroups: + - apps + resources: + - deployments + - statefulsets + verbs: + - get + - create + - patch +- apiGroups: + - liberty.websphere.ibm.com + resources: + - webspherelibertyapplications + - webspherelibertydumps + - webspherelibertytraces + verbs: + - get + - list + - create + - patch + - delete diff --git a/config/rbac/kuttl-rbac-watch-another.yaml b/config/rbac/kuttl-rbac-watch-another.yaml new file mode 100644 index 00000000..0d2bb9b4 --- /dev/null +++ b/config/rbac/kuttl-rbac-watch-another.yaml @@ -0,0 +1,83 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: scorecard-kuttl-watch-another + namespace: WEBSPHERE_LIBERTY_WATCH_NAMESPACE +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: watched-role + namespace: WEBSPHERE_LIBERTY_OPERATOR_NAMESPACE +rules: + - apiGroups: + - "" + resources: + - configmaps + verbs: + - get + - list + - delete +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: watcher-role + namespace: WEBSPHERE_LIBERTY_WATCH_NAMESPACE +rules: +- apiGroups: + - "" + resources: + - configmaps + verbs: + - get + - list + - delete +- apiGroups: + - apps + resources: + - deployments + - statefulsets + verbs: + - get + - create + - patch +- apiGroups: + - liberty.websphere.ibm.com + resources: + - webspherelibertyapplications + - webspherelibertydumps + - webspherelibertytraces + verbs: + - get + - list + - create + - patch + - delete +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: watched-rolebinding + namespace: WEBSPHERE_LIBERTY_OPERATOR_NAMESPACE +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: watched-role +subjects: +- kind: ServiceAccount + name: scorecard-kuttl-watch-another + namespace: WEBSPHERE_LIBERTY_WATCH_NAMESPACE +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: watcher-rolebinding + namespace: WEBSPHERE_LIBERTY_WATCH_NAMESPACE +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: watcher-role +subjects: +- kind: ServiceAccount + name: scorecard-kuttl-watch-another diff --git a/config/rbac/kuttl-rbac.yaml b/config/rbac/kuttl-rbac.yaml index b174b710..da38dc5d 100644 --- a/config/rbac/kuttl-rbac.yaml +++ b/config/rbac/kuttl-rbac.yaml @@ -41,6 +41,14 @@ rules: verbs: - get - list +- apiGroups: + - "" + resources: + - configmaps + verbs: + - get + - list + - delete - apiGroups: - "" resources: diff --git a/scripts/pipeline/fyre-e2e.sh b/scripts/pipeline/fyre-e2e.sh index 97e67cde..64a04533 100755 --- a/scripts/pipeline/fyre-e2e.sh +++ b/scripts/pipeline/fyre-e2e.sh @@ -20,9 +20,6 @@ setup_env() { echo "****** Creating test namespace: ${TEST_NAMESPACE} for release ${RELEASE}" oc new-project "${TEST_NAMESPACE}" || oc project "${TEST_NAMESPACE}" - - ## Create service account for Kuttl tests - oc apply -f config/rbac/kuttl-rbac.yaml } ## cleanup_env : Delete generated resources that are not bound to a test TEST_NAMESPACE. @@ -148,39 +145,140 @@ main() { echo "Updating global pull secret" oc set data secret/pull-secret -n openshift-config --from-file=.dockerconfigjson=/tmp/pull-secret-merged.yaml - echo "****** Installing operator from catalog: ${CATALOG_IMAGE}" + # Run Kuttl scorecard tests + run_scorecard "OwnNamespace" + run_scorecard "AllNamespaces" + run_scorecard "SingleNamespace" + + echo "****** Cleaning up test environment..." + cleanup_env + + return $result +} + +run_scorecard() { + INSTALL_MODE=$1 + echo "****** Provisioning the cluster to setup operator in ${INSTALL_MODE} mode..." + if [ "$INSTALL_MODE" == "AllNamespaces" ]; then + KUTTL_TEST_DIR="kuttl-all-namespaces" + CONTROLLER_MANAGER_NAMESPACE="openshift-operators" + SERVICE_ACCOUNT_NAME="scorecard-kuttl-watch-all" + OPERATOR_GROUP_TARGET_NAMESPACE="openshift-operators" # used for CSV cleanup + WATCH_NAMESPACE="${TEST_NAMESPACE}" + elif [ "$INSTALL_MODE" == "SingleNamespace" ]; then + KUTTL_TEST_DIR="kuttl-single-namespace" + CONTROLLER_MANAGER_NAMESPACE="${TEST_NAMESPACE}" + SERVICE_ACCOUNT_NAME="scorecard-kuttl-watch-another" + OPERATOR_GROUP_NAMESPACE="${CONTROLLER_MANAGER_NAMESPACE}" + WATCH_NAMESPACE="wlo-watch-${TEST_TAG}" + OPERATOR_GROUP_TARGET_NAMESPACE="${WATCH_NAMESPACE}" + oc new-project "$WATCH_NAMESPACE" + elif [ "$INSTALL_MODE" == "OwnNamespace" ]; then + KUTTL_TEST_DIR="kuttl" + CONTROLLER_MANAGER_NAMESPACE="${TEST_NAMESPACE}" + SERVICE_ACCOUNT_NAME="scorecard-kuttl" + OPERATOR_GROUP_NAMESPACE="${TEST_NAMESPACE}" + OPERATOR_GROUP_TARGET_NAMESPACE="${TEST_NAMESPACE}" + WATCH_NAMESPACE="${TEST_NAMESPACE}" + fi + + # Delete a subscription or catalog source that may be blocking the install + oc delete subscription.operators.coreos.com websphere-liberty-operator-subscription -n ${CONTROLLER_MANAGER_NAMESPACE} + oc delete catalogsource websphere-liberty-catalog -n ${CONTROLLER_MANAGER_NAMESPACE} + install_operator # Wait for operator deployment to be ready - while [[ $(oc get deploy "${CONTROLLER_MANAGER_NAME}" -o jsonpath='{ .status.readyReplicas }') -ne "1" ]]; do - echo "****** Waiting for ${CONTROLLER_MANAGER_NAME} to be ready..." + while [[ $(oc get deploy "${CONTROLLER_MANAGER_NAME}" -n ${CONTROLLER_MANAGER_NAMESPACE} -o jsonpath='{ .status.readyReplicas }') -ne "1" ]]; do + echo "****** Waiting for ${CONTROLLER_MANAGER_NAME} in namespace ${CONTROLLER_MANAGER_NAMESPACE} to be ready..." sleep 10 done echo "****** ${CONTROLLER_MANAGER_NAME} deployment is ready..." - - echo "****** Starting scorecard tests..." - operator-sdk scorecard --verbose --kubeconfig ${HOME}/.kube/config --selector=suite=kuttlsuite --namespace="${TEST_NAMESPACE}" --service-account="scorecard-kuttl" --wait-time 30m ./bundle || { + + echo "****** Starting scorecard tests in the '${KUTTL_TEST_DIR}' directory..." + oc project "${WATCH_NAMESPACE}" + set_rbac + set_kuttl_test_dir + TESTS_FAILED=false + operator-sdk scorecard --verbose --kubeconfig ${HOME}/.kube/config --selector=suite=kuttlsuite --namespace="${WATCH_NAMESPACE}" --service-account="${SERVICE_ACCOUNT_NAME}" --wait-time 30m ./bundle || { echo "****** Scorecard tests failed..." - exit 1 + TESTS_FAILED=true } - result=$? + echo "****** Starting cluster cleanup..." + unset_kuttl_test_dir + unset_rbac + uninstall_operator + if [ "$INSTALL_MODE" == "SingleNamespace" ]; then + oc delete project "${WATCH_NAMESPACE}" + fi + if $TESTS_FAILED ; then + exit 1 + fi +} - echo "****** Cleaning up test environment..." - cleanup_env +set_rbac() { + if [ "$INSTALL_MODE" == "OwnNamespace" ]; then + oc apply -f config/rbac/kuttl-rbac.yaml + elif [ "$INSTALL_MODE" == "AllNamespaces" ]; then + cp config/rbac/kuttl-rbac-watch-all.yaml ./ + sed -i "s/WEBSPHERE_LIBERTY_WATCH_NAMESPACE/${WATCH_NAMESPACE}/" kuttl-rbac-watch-all.yaml + oc apply -f kuttl-rbac-watch-all.yaml + elif [ "$INSTALL_MODE" == "SingleNamespace" ]; then + cp config/rbac/kuttl-rbac-watch-another.yaml ./ + sed -i "s/WEBSPHERE_LIBERTY_OPERATOR_NAMESPACE/${CONTROLLER_MANAGER_NAMESPACE}/" kuttl-rbac-watch-another.yaml + sed -i "s/WEBSPHERE_LIBERTY_WATCH_NAMESPACE/${WATCH_NAMESPACE}/" kuttl-rbac-watch-another.yaml + oc apply -f kuttl-rbac-watch-another.yaml + fi +} - return $result +unset_rbac() { + if [ "$INSTALL_MODE" == "OwnNamespace" ]; then + oc delete -f config/rbac/kuttl-rbac.yaml + elif [ "$INSTALL_MODE" == "AllNamespaces" ]; then + oc delete -f kuttl-rbac-watch-all.yaml + rm kuttl-rbac-watch-all.yaml + elif [ "$INSTALL_MODE" == "SingleNamespace" ]; then + oc delete -f kuttl-rbac-watch-another.yaml + rm kuttl-rbac-watch-another.yaml + fi +} + +set_kuttl_test_dir() { + if [ "$KUTTL_TEST_DIR" == "kuttl-single-namespace" ]; then + mv bundle/tests/scorecard/kuttl bundle/tests/scorecard/kuttl-temp + cp -r bundle/tests/scorecard/${KUTTL_TEST_DIR} bundle/tests/scorecard/kuttl + find bundle/tests/scorecard/kuttl -type f -name "*.yaml" -print0 | xargs -0 sed -i "s/WEBSPHERE_LIBERTY_OPERATOR_NAMESPACE/${CONTROLLER_MANAGER_NAMESPACE}/" + mv bundle/tests/scorecard/kuttl-temp/kuttl-test.yaml bundle/tests/scorecard/kuttl + elif [ "$KUTTL_TEST_DIR" != "kuttl" ]; then + mv bundle/tests/scorecard/kuttl bundle/tests/scorecard/kuttl-temp + mv bundle/tests/scorecard/${KUTTL_TEST_DIR} bundle/tests/scorecard/kuttl + mv bundle/tests/scorecard/kuttl-temp/kuttl-test.yaml bundle/tests/scorecard/kuttl + fi +} + +unset_kuttl_test_dir() { + if [ "$KUTTL_TEST_DIR" == "kuttl-single-namespace" ]; then + mv bundle/tests/scorecard/kuttl/kuttl-test.yaml bundle/tests/scorecard/kuttl-temp + rm -rf bundle/tests/scorecard/kuttl + mv bundle/tests/scorecard/kuttl-temp bundle/tests/scorecard/kuttl + elif [ "$KUTTL_TEST_DIR" != "kuttl" ]; then + mv bundle/tests/scorecard/kuttl/kuttl-test.yaml bundle/tests/scorecard/kuttl-temp + mv bundle/tests/scorecard/kuttl bundle/tests/scorecard/${KUTTL_TEST_DIR} + mv bundle/tests/scorecard/kuttl-temp bundle/tests/scorecard/kuttl + fi } install_operator() { - # Apply the catalog - echo "****** Applying the catalog source..." + echo "****** Installing the operator in ${INSTALL_MODE} mode..." + + echo "****** Applying catalog source..." cat <