1536feaf21
- 2026-06-10 incident: 48G disk hit 100% — pipelines died w. ENOSPC in the UT/IT dependency install + the Woodpecker repo UI 500'd (SQLite couldn't write; the repos LIST still rendered, pure read). Culprits: ~15G stale CI images/build cache + 19G of gitea repo-archive cache (crawlers on the public instance generate zip/tar snapshots faster than gitea's @midnight archive_cleanup reclaims them) - cicd/docker-prune.sh → /etc/cron.weekly/docker-prune (installed live via ssh too — running the full playbook clobbers the droplet's nginx config, known template bug): image/builder/container prune keeping the last 168h + a repo-archive cache wipe - gitea container restarted to unwedge the post-receive queues the disk-full era jammed (pushes landed in the bare repo but the UI + Woodpecker webhook never heard about them) Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
124 lines
3.4 KiB
YAML
124 lines
3.4 KiB
YAML
- hosts: cicd
|
|
|
|
tasks:
|
|
- name: Install Docker
|
|
ansible.builtin.apt:
|
|
name: docker.io
|
|
state: latest
|
|
update_cache: true
|
|
become: true
|
|
|
|
- name: Install Nginx
|
|
ansible.builtin.apt:
|
|
name: nginx
|
|
state: latest
|
|
become: true
|
|
|
|
- name: Add out user to the docker group, so we don't need sudo/become
|
|
ansible.builtin.user:
|
|
name: '{{ ansible_user }}'
|
|
groups: docker
|
|
append: true # don't remove any existing groups
|
|
become: true
|
|
|
|
- name: Reset ssh connection to allow the user/group change to take effect
|
|
ansible.builtin.meta: reset_connection
|
|
|
|
- name: Install docker-compose-plugin & certbot
|
|
ansible.builtin.apt:
|
|
name:
|
|
- docker-compose-v2
|
|
- certbot
|
|
- python3-certbot-nginx
|
|
state: latest
|
|
become: true
|
|
|
|
- name: Create /opt/cicd/ directory tree
|
|
ansible.builtin.file:
|
|
path: "/opt/cicd/nginx"
|
|
state: directory
|
|
become: true
|
|
|
|
- name: Cp docker-compose.yaml to server
|
|
ansible.builtin.copy:
|
|
src: cicd/docker-compose.yaml
|
|
dest: /opt/cicd/docker-compose.yaml
|
|
become: true
|
|
|
|
- name: Template .env to /opt/cicd/
|
|
ansible.builtin.template:
|
|
src: cicd/.env.j2
|
|
dest: /opt/cicd/.env
|
|
mode: "0600"
|
|
become: true
|
|
|
|
- name: Deploy nginx config (Gitea)
|
|
ansible.builtin.copy:
|
|
src: cicd/nginx/gitea.conf
|
|
dest: /etc/nginx/sites-available/gitea
|
|
become: true
|
|
notify: Restart nginx
|
|
|
|
- name: Deploy nginx config (Woodpecker)
|
|
ansible.builtin.copy:
|
|
src: cicd/nginx/woodpecker.conf
|
|
dest: /etc/nginx/sites-available/woodpecker
|
|
become: true
|
|
notify: Restart nginx
|
|
|
|
- name: Enable nginx site (Gitea)
|
|
ansible.builtin.file:
|
|
src: /etc/nginx/sites-available/gitea
|
|
dest: /etc/nginx/sites-enabled/gitea
|
|
state: link
|
|
become: true
|
|
notify: Restart nginx
|
|
|
|
- name: Enable nginx site (Woodpecker)
|
|
ansible.builtin.file:
|
|
src: /etc/nginx/sites-available/woodpecker
|
|
dest: /etc/nginx/sites-enabled/woodpecker
|
|
state: link
|
|
become: true
|
|
notify: Restart nginx
|
|
|
|
- name: Remove default nginx site
|
|
ansible.builtin.file:
|
|
path: /etc/nginx/sites-enabled/default
|
|
state: absent
|
|
become: true
|
|
notify: Restart nginx
|
|
|
|
- name: Obtain SSL certs via certbot
|
|
ansible.builtin.command:
|
|
cmd: >
|
|
certbot --nginx
|
|
-d gitea.earthmanrpg.me
|
|
-d ci.earthmanrpg.me
|
|
--non-interactive
|
|
--agree-tos
|
|
-m discodedisco@outlook.com
|
|
creates: /etc/letsencrypt/live/gitea.earthmanrpg.me/fullchain.pem
|
|
become: true
|
|
|
|
# Weekly docker + gitea-archive-cache prune — the 2026-06-10 disk-full
|
|
# incident (48G/48G: pipelines ENOSPC'd, Woodpecker repo UI 500'd) was
|
|
# ~15G of stale CI images + 19G of crawler-generated repo archives.
|
|
- name: Install weekly docker/archive prune cron
|
|
ansible.builtin.copy:
|
|
src: cicd/docker-prune.sh
|
|
dest: /etc/cron.weekly/docker-prune
|
|
mode: "0755"
|
|
become: true
|
|
|
|
- name: Run docker compose -f /opt/cicd/docker-compose.yaml up -d
|
|
ansible.builtin.command:
|
|
cmd: docker compose -f /opt/cicd/docker-compose.yaml up -d
|
|
become: true
|
|
|
|
handlers:
|
|
- name: Restart nginx
|
|
ansible.builtin.service:
|
|
name: nginx
|
|
state: restarted
|
|
become: true |