From 0761d462f1e87e5ebded407b3f5d836816ece838 Mon Sep 17 00:00:00 2001 From: kurokobo <2920259+kurokobo@users.noreply.github.com> Date: Sun, 6 Jun 2021 09:56:20 -0400 Subject: [PATCH] feat: support backup and restore using operator --- README.md | 183 ++++++++++++++++++++++++++++++++++++- backup/awxbackup.yaml | 10 ++ backup/kustomization.yaml | 12 +++ backup/namespace.yaml | 5 + backup/pv.yaml | 14 +++ backup/pvc.yaml | 13 +++ restore/awxrestore.yaml | 17 ++++ restore/kustomization.yaml | 19 ++++ restore/namespace.yaml | 5 + restore/pv.yaml | 44 +++++++++ restore/pvc.yaml | 27 ++++++ 11 files changed, 347 insertions(+), 2 deletions(-) create mode 100644 backup/awxbackup.yaml create mode 100644 backup/kustomization.yaml create mode 100644 backup/namespace.yaml create mode 100644 backup/pv.yaml create mode 100644 backup/pvc.yaml create mode 100644 restore/awxrestore.yaml create mode 100644 restore/kustomization.yaml create mode 100644 restore/namespace.yaml create mode 100644 restore/pv.yaml create mode 100644 restore/pvc.yaml diff --git a/README.md b/README.md index 5103868..06ae7e3 100644 --- a/README.md +++ b/README.md @@ -7,6 +7,26 @@ An example implementation of AWX on single node K3s using AWX Operator, with eas - Fixed (configurable) passwords for AWX and PostgreSQL - Fixed (configurable) versions of AWX and PostgreSQL +## Table of Contents + +- [AWX on Single Node K3s](#awx-on-single-node-k3s) + - [Table of Contents](#table-of-contents) + - [Environment](#environment) + - [References](#references) + - [Procedure](#procedure) + - [Prepare CentOS 8 host](#prepare-centos-8-host) + - [Install K3s](#install-k3s) + - [Install AWX Operator](#install-awx-operator) + - [Prepare required files](#prepare-required-files) + - [Deploy AWX](#deploy-awx) + - [Backing up and Restoring using AWX Operator](#backing-up-and-restoring-using-awx-operator) + - [Backing up using AWX Operator](#backing-up-using-awx-operator) + - [Prepare for Backup](#prepare-for-backup) + - [Invoke Manual Backup](#invoke-manual-backup) + - [Restoring using AWX Operator](#restoring-using-awx-operator) + - [Prepare for Restore](#prepare-for-restore) + - [Invoke Manual Restore](#invoke-manual-restore) + ## Environment - Tested on: @@ -64,7 +84,7 @@ AWX_HOST="awx.example.com" openssl req -x509 -nodes -days 3650 -newkey rsa:2048 -out ./base/tls.crt -keyout ./base/tls.key -subj "/CN=${AWX_HOST}/O=${AWX_HOST}" -addext "subjectAltName = DNS:${AWX_HOST}" ``` -Modify `hostname` in `base\awx.yaml`. +Modify `hostname` in `base/awx.yaml`. ```yaml ... @@ -75,7 +95,7 @@ spec: ... ``` -Modify two `password`s in `base\kustomization.yaml`. +Modify two `password`s in `base/kustomization.yaml`. ```yaml ... @@ -146,3 +166,162 @@ statefulset.apps/awx-postgres 1/1 4m30s ``` Now AWX is available at `https:///`. + +## Backing up and Restoring using AWX Operator + +The AWX Operator `0.10.0` or later has the ability to backup and restore AWX in easy way. + +### Backing up using AWX Operator + +#### Prepare for Backup + +Prepare directories for Persistent Volumes to store backup files that defined in `backup/pv.yaml`. + +```bash +sudo mkdir -p /data/backup +``` + +Then deploy Persistent Volume and Persistent Volume Claim. + +```bash +kubectl apply -k backup +``` + +#### Invoke Manual Backup + +Modify the name of the AWXBackup object in `backup/awxbackup.yaml`. + +```yaml +... +kind: AWXBackup +metadata: + name: awxbackup-2021-06-06 👈👈👈 + namespace: awx +... +``` + +Then invoke backup by applying this manifest file. + +```bash +kubectl apply -f backup/awxbackup.yaml +``` + +Once this completed, the logs of `deployment/awx-operator` end with: + +```txt +$ kubectl logs -f deployment/awx-operator +--------------------------- Ansible Task Status Event StdOut ----------------- +PLAY RECAP ********************************************************************* +localhost : ok=4 changed=0 unreachable=0 failed=0 skipped=7 rescued=0 ignored=0 +------------------------------------------------------------------------------- +``` + +This will create AWXBackup object in the namespace and also create backup files in the Persistent Volume. In this example those files are available at `/data/backup`. + +```bash +$ kubectl get awxbackup -n awx +NAME AGE +awxbackup-2021-06-06 6m47s +``` + +```bash +$ ls -l /data/backup/ +total 0 +drwxr-xr-x. 2 root root 59 Jun 5 06:51 tower-openshift-backup-2021-06-06-10:51:49 + +$ ls -l /data/backup/tower-openshift-backup-2021-06-06-10\:51\:49/ +total 736 +-rw-r--r--. 1 root root 749 Jun 6 06:51 awx_object +-rw-r--r--. 1 root root 482 Jun 6 06:51 secrets.yml +-rw-------. 1 systemd-coredump root 745302 Jun 6 06:51 tower.db +``` + +Note that the contents of the Secret that passed through `ingress_tls_secret` parameter will not be included in this backup files. If necessary, get a dump of this Secret, or keep original certificate file and key file. + +```bash +kubectl get secret awx-secret-tls -n awx -o yaml > awx-secret-tls.yaml +``` + +### Restoring using AWX Operator + +#### Prepare for Restore + +If your PV, PVC, and Secret still exist, no preparation is required. + +If you are restoring the entire AWX to a new environment, create the PVs and PVCs first to be restored. + +```bash +sudo mkdir -p /data/postgres +sudo mkdir -p /data/projects +sudo chown 1000:0 /data/projects +``` + +Then deploy Persistent Volume and Persistent Volume Claim. + +```bash +kubectl apply -k restore +``` + +#### Invoke Manual Restore + +Modify the name of the AWXRestore object in `restore/awxrestore.yaml`. + +```yaml +... +kind: AWXRestore +metadata: + name: awxrestore-2021-06-06 👈👈👈 + namespace: awx +... +``` + +If you want to restore from AWXBackup object, specify its name in `restore/awxrestore.yaml`. + +```yaml +... + # Parameters to restore from AWXBackup object + backup_pvc_namespace: awx + backup_name: awxbackup-2021-06-06 👈👈👈 +... +``` + +If the AWXBackup object no longer exists, place the backup files and specify the name of the PVC and directory in `restore/awxrestore.yaml`. + +```yaml +... + # Parameters to restore from existing files on PVC (without AWXBackup object) + backup_pvc_namespace: awx + backup_pvc: awx-backup-claim 👈👈👈 + backup_dir: /backups/tower-openshift-backup-2021-06-06-10:51:49 👈👈👈 +... +``` + +Then invoke restore by applying this manifest file. + +```bash +kubectl apply -f restore/awxrestore.yaml +``` + +Once this completed, the logs of `deployment/awx-operator` end with: + +```txt +$ kubectl logs -f deployment/awx-operator +--------------------------- Ansible Task Status Event StdOut ----------------- +PLAY RECAP ********************************************************************* +localhost : ok=53 changed=2 unreachable=0 failed=0 skipped=30 rescued=0 ignored=0 +------------------------------------------------------------------------------- +``` + +This will create AWXRestore object in the namespace. + +```bash +$ kubectl get awxrestore -n awx +NAME AGE +awxrestore-2021-06-06 137m +``` + +Then restore the Secret for TLS manually (or create newly using original certificate and key file). + +```bash +kubectl apply -f awx-secret-tls.yaml +``` diff --git a/backup/awxbackup.yaml b/backup/awxbackup.yaml new file mode 100644 index 0000000..6a7b0a9 --- /dev/null +++ b/backup/awxbackup.yaml @@ -0,0 +1,10 @@ +--- +apiVersion: awx.ansible.com/v1beta1 +kind: AWXBackup +metadata: + name: awxbackup-2021-06-06 + namespace: awx +spec: + deployment_name: awx + backup_pvc: awx-backup-claim + postgres_label_selector: app.kubernetes.io/instance=postgres-awx diff --git a/backup/kustomization.yaml b/backup/kustomization.yaml new file mode 100644 index 0000000..2088c2d --- /dev/null +++ b/backup/kustomization.yaml @@ -0,0 +1,12 @@ +--- +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +namespace: awx + +generatorOptions: + disableNameSuffixHash: true + +resources: + - namespace.yaml + - pv.yaml + - pvc.yaml diff --git a/backup/namespace.yaml b/backup/namespace.yaml new file mode 100644 index 0000000..e24dd13 --- /dev/null +++ b/backup/namespace.yaml @@ -0,0 +1,5 @@ +--- +apiVersion: v1 +kind: Namespace +metadata: + name: awx diff --git a/backup/pv.yaml b/backup/pv.yaml new file mode 100644 index 0000000..b6c6822 --- /dev/null +++ b/backup/pv.yaml @@ -0,0 +1,14 @@ +--- +apiVersion: v1 +kind: PersistentVolume +metadata: + name: awx-backup-volume +spec: + accessModes: + - ReadWriteOnce + persistentVolumeReclaimPolicy: Retain + capacity: + storage: 2Gi + storageClassName: awx-backup-volume + hostPath: + path: /data/backup diff --git a/backup/pvc.yaml b/backup/pvc.yaml new file mode 100644 index 0000000..0d11dd8 --- /dev/null +++ b/backup/pvc.yaml @@ -0,0 +1,13 @@ +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: awx-backup-claim +spec: + accessModes: + - ReadWriteOnce + volumeMode: Filesystem + resources: + requests: + storage: 2Gi + storageClassName: awx-backup-volume diff --git a/restore/awxrestore.yaml b/restore/awxrestore.yaml new file mode 100644 index 0000000..2498c5d --- /dev/null +++ b/restore/awxrestore.yaml @@ -0,0 +1,17 @@ +--- +apiVersion: awx.ansible.com/v1beta1 +kind: AWXRestore +metadata: + name: awxrestore-2021-06-06 + namespace: awx +spec: + deployment_name: awx + + # Parameters to restore from AWXBackup object + #backup_pvc_namespace: awx + #backup_name: awxbackup-2021-06-06 + + # Parameters to restore from existing files on PVC (without AWXBackup object) + #backup_pvc_namespace: awx + #backup_pvc: awx-backup-claim + #backup_dir: /backups/tower-openshift-backup-2021-06-06-10:51:49 diff --git a/restore/kustomization.yaml b/restore/kustomization.yaml new file mode 100644 index 0000000..bea8462 --- /dev/null +++ b/restore/kustomization.yaml @@ -0,0 +1,19 @@ +--- +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +namespace: awx + +generatorOptions: + disableNameSuffixHash: true + +#secretGenerator: +# - name: awx-secret-tls +# type: kubernetes.io/tls +# files: +# - tls.crt +# - tls.key + +resources: + - namespace.yaml + - pv.yaml + - pvc.yaml diff --git a/restore/namespace.yaml b/restore/namespace.yaml new file mode 100644 index 0000000..e24dd13 --- /dev/null +++ b/restore/namespace.yaml @@ -0,0 +1,5 @@ +--- +apiVersion: v1 +kind: Namespace +metadata: + name: awx diff --git a/restore/pv.yaml b/restore/pv.yaml new file mode 100644 index 0000000..0ac1297 --- /dev/null +++ b/restore/pv.yaml @@ -0,0 +1,44 @@ +--- +apiVersion: v1 +kind: PersistentVolume +metadata: + name: awx-postgres-volume +spec: + accessModes: + - ReadWriteOnce + persistentVolumeReclaimPolicy: Retain + capacity: + storage: 2Gi + storageClassName: awx-postgres-volume + hostPath: + path: /data/postgres + +--- +apiVersion: v1 +kind: PersistentVolume +metadata: + name: awx-projects-volume +spec: + accessModes: + - ReadWriteOnce + persistentVolumeReclaimPolicy: Retain + capacity: + storage: 2Gi + storageClassName: awx-projects-volume + hostPath: + path: /data/projects + +--- +apiVersion: v1 +kind: PersistentVolume +metadata: + name: awx-backup-volume +spec: + accessModes: + - ReadWriteOnce + persistentVolumeReclaimPolicy: Retain + capacity: + storage: 2Gi + storageClassName: awx-backup-volume + hostPath: + path: /data/backup diff --git a/restore/pvc.yaml b/restore/pvc.yaml new file mode 100644 index 0000000..7ef184e --- /dev/null +++ b/restore/pvc.yaml @@ -0,0 +1,27 @@ +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: awx-projects-claim +spec: + accessModes: + - ReadWriteOnce + volumeMode: Filesystem + resources: + requests: + storage: 2Gi + storageClassName: awx-projects-volume + +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: awx-backup-claim +spec: + accessModes: + - ReadWriteOnce + volumeMode: Filesystem + resources: + requests: + storage: 2Gi + storageClassName: awx-backup-volume