From 9bc9a4065dfa17f416fd0296d311f80bffac3b64 Mon Sep 17 00:00:00 2001 From: Kevin Fenzi Date: Thu, 27 Apr 2017 20:25:20 +0000 Subject: add varnish config for buildvm-s390x-07.s390.fedoraproject.org which needs to cache packages for s390x builders in bos --- .../buildvm-s390x-07.s390.fedoraproject.org | 2 + playbooks/groups/buildvm.yml | 15 ++++ roles/varnish/files/varnish.f25 | 32 ------- roles/varnish/tasks/main.yml | 2 +- roles/varnish/templates/kojipkgs.vcl.j2 | 1 + roles/varnish/templates/s390kojipkgs.vcl.j2 | 72 ++++++++++++++++ roles/varnish/templates/varnish.f25.j2 | 36 ++++++++ roles/varnish/templates/varnish.j2 | 99 ++++++++++++++++++++++ 8 files changed, 226 insertions(+), 33 deletions(-) create mode 100644 inventory/host_vars/buildvm-s390x-07.s390.fedoraproject.org delete mode 100644 roles/varnish/files/varnish.f25 create mode 100644 roles/varnish/templates/s390kojipkgs.vcl.j2 create mode 100644 roles/varnish/templates/varnish.f25.j2 create mode 100644 roles/varnish/templates/varnish.j2 diff --git a/inventory/host_vars/buildvm-s390x-07.s390.fedoraproject.org b/inventory/host_vars/buildvm-s390x-07.s390.fedoraproject.org new file mode 100644 index 000000000..0e17cf97d --- /dev/null +++ b/inventory/host_vars/buildvm-s390x-07.s390.fedoraproject.org @@ -0,0 +1,2 @@ +--- +varnish_group: s390kojipkgs diff --git a/playbooks/groups/buildvm.yml b/playbooks/groups/buildvm.yml index 5d93c1440..6e7650916 100644 --- a/playbooks/groups/buildvm.yml +++ b/playbooks/groups/buildvm.yml @@ -139,5 +139,20 @@ } } +- name: configure varnish cache on buildvm-s390x-07.s390.fedoraproject.org + hosts: buildvm-s390x-07.s390.fedoraproject.org + tags: + - varnish + user: root + gather_facts: True + + vars_files: + - /srv/web/infra/ansible/vars/global.yml + - "/srv/private/ansible/vars.yml" + - /srv/web/infra/ansible/vars/{{ ansible_distribution }}.yml + + roles: + - varnish + handlers: - include: "{{ handlers_path }}/restart_services.yml" diff --git a/roles/varnish/files/varnish.f25 b/roles/varnish/files/varnish.f25 deleted file mode 100644 index 4cecb5b01..000000000 --- a/roles/varnish/files/varnish.f25 +++ /dev/null @@ -1,32 +0,0 @@ -# Varnish environment configuration description. This was derived from -# the old style sysconfig/defaults settings - -# Set this to 1 to make systemd reload try to switch VCL without restart. -RELOAD_VCL=1 - -# Main configuration file. You probably want to change it. -VARNISH_VCL_CONF=/etc/varnish/default.vcl - -# Default address and port to bind to. Blank address means all IPv4 -# and IPv6 interfaces, otherwise specify a host name, an IPv4 dotted -# quad, or an IPv6 address in brackets. -# VARNISH_LISTEN_ADDRESS=192.168.1.5 -VARNISH_LISTEN_PORT=80 - -# Admin interface listen address and port -VARNISH_ADMIN_LISTEN_ADDRESS=127.0.0.1 -VARNISH_ADMIN_LISTEN_PORT=6082 - -# Shared secret file for admin interface -VARNISH_SECRET_FILE=/etc/varnish/secret - -# Backend storage specification, see Storage Types in the varnishd(5) -# man page for details. -VARNISH_STORAGE="malloc,72GB" - -# User and group for the varnishd worker processes -VARNISH_USER=varnish -VARNISH_GROUP=varnish - -# Other options, see the man page varnishd(1) -#DAEMON_OPTS="-p thread_pool_min=5 -p thread_pool_max=500 -p thread_pool_timeout=300" diff --git a/roles/varnish/tasks/main.yml b/roles/varnish/tasks/main.yml index 41efe9771..14ce35971 100644 --- a/roles/varnish/tasks/main.yml +++ b/roles/varnish/tasks/main.yml @@ -24,7 +24,7 @@ when: ansible_distribution_major_version|int == 7 - name: install varnish /etc/sysconfig/varnish file (fedora) - copy: src=varnish.f25 dest=/etc/varnish/varnish.params owner=root group=root + template: src=varnish.f25.j2 dest=/etc/varnish/varnish.params owner=root group=root notify: - restart varnish tags: diff --git a/roles/varnish/templates/kojipkgs.vcl.j2 b/roles/varnish/templates/kojipkgs.vcl.j2 index 1a0011a98..c3f7d51d1 100644 --- a/roles/varnish/templates/kojipkgs.vcl.j2 +++ b/roles/varnish/templates/kojipkgs.vcl.j2 @@ -15,6 +15,7 @@ acl repoallowed { "10.5.126.224"/32; "10.5.126.225"/32; "10.5.126.226"/32; + "10.16.0.0"/24; } acl purge { diff --git a/roles/varnish/templates/s390kojipkgs.vcl.j2 b/roles/varnish/templates/s390kojipkgs.vcl.j2 new file mode 100644 index 000000000..1ac1ef9e8 --- /dev/null +++ b/roles/varnish/templates/s390kojipkgs.vcl.j2 @@ -0,0 +1,72 @@ +vcl 4.0; + +import std; +import directors; + +# +# These nets/machines are allowed /repo access +# +acl repoallowed { + "10.16.0.0"/24; +} + +acl purge { + "127.0.0.1"/32; +} +backend proxy01 { + .host = "proxy01.phx2.fedoraproject.org"; + .probe = { + .url = "/"; + .timeout = 1s; + .interval = 5s; + .window = 5; + .threshold = 3; + } +} + +backend proxy10 { + .host = "proxy10.phx2.fedoraproject.org"; + .probe = { + .url = "/"; + .timeout = 1s; + .interval = 5s; + .window = 5; + .threshold = 3; + } +} + +sub vcl_init { + new primarykojipkgs = directors.round_robin(); + primarykojipkgs.add_backend(proxy01); + primarykojipkgs.add_backend(proxy10); +} + +sub vcl_recv { + # This gets arround the silly, ::1 that Apache adds on the proxies (still need to look at that) + set req.http.X-Forwarded-For = regsub(req.http.X-Forwarded-For, "^([a-f0-9:.]+), .+$", "\1"); + + set req.backend_hint = primarykojipkgs.backend(); + unset req.http.cookie; + set req.http.clear-cookies = "yes"; + + if (req.method == "PURGE") { + if (!client.ip ~ purge) { + return (synth(405, "Not allowed")); + } + return(purge); + } + + if (req.url ~ "^/repo/" && !(std.ip(req.http.X-Forwarded-For, "0.0.0.0") ~ repoallowed)) { + return(synth(403, "Access denied.")); + } + if (req.url ~ "^/mash/") { + return (pipe); + } + if (req.url ~ "^/compose/") { + return (pipe); + } + if (req.url ~ "h264") { + return (pipe); + } + return (hash); +} diff --git a/roles/varnish/templates/varnish.f25.j2 b/roles/varnish/templates/varnish.f25.j2 new file mode 100644 index 000000000..a21a5aff5 --- /dev/null +++ b/roles/varnish/templates/varnish.f25.j2 @@ -0,0 +1,36 @@ +# Varnish environment configuration description. This was derived from +# the old style sysconfig/defaults settings + +# Set this to 1 to make systemd reload try to switch VCL without restart. +RELOAD_VCL=1 + +# Main configuration file. You probably want to change it. +VARNISH_VCL_CONF=/etc/varnish/default.vcl + +# Default address and port to bind to. Blank address means all IPv4 +# and IPv6 interfaces, otherwise specify a host name, an IPv4 dotted +# quad, or an IPv6 address in brackets. +# VARNISH_LISTEN_ADDRESS=192.168.1.5 +VARNISH_LISTEN_PORT=80 + +# Admin interface listen address and port +VARNISH_ADMIN_LISTEN_ADDRESS=127.0.0.1 +VARNISH_ADMIN_LISTEN_PORT=6082 + +# Shared secret file for admin interface +VARNISH_SECRET_FILE=/etc/varnish/secret + +# Backend storage specification, see Storage Types in the varnishd(5) +# man page for details. +{% if varnish_group = 'kojipkgs' %} +VARNISH_STORAGE="malloc,72GB" +{% elif varnish_group = 's390kojipkgs' %} +VARNISH_STORAGE="file,/var/lib/varnish/varnish_storage.bin,20G" +{% endif %} + +# User and group for the varnishd worker processes +VARNISH_USER=varnish +VARNISH_GROUP=varnish + +# Other options, see the man page varnishd(1) +#DAEMON_OPTS="-p thread_pool_min=5 -p thread_pool_max=500 -p thread_pool_timeout=300" diff --git a/roles/varnish/templates/varnish.j2 b/roles/varnish/templates/varnish.j2 new file mode 100644 index 000000000..763e5fdbc --- /dev/null +++ b/roles/varnish/templates/varnish.j2 @@ -0,0 +1,99 @@ +# Configuration file for varnish +# +# /etc/init.d/varnish expects the variable $DAEMON_OPTS to be set from this +# shell script fragment. +# + +# Maximum number of open files (for ulimit -n) +NFILES=131072 + +# Locked shared memory (for ulimit -l) +# Default log size is 82MB + header +MEMLOCK=82000 + +# Maximum size of corefile (for ulimit -c). Default in Fedora is 0 +# DAEMON_COREFILE_LIMIT="unlimited" + +# This file contains 4 alternatives, please use only one. + +## Alternative 1, Minimal configuration, no VCL +# +# Listen on port 6081, administration on localhost:6082, and forward to +# content server on localhost:8080. Use a fixed-size cache file. +# +#DAEMON_OPTS="-a :6081 \ +# -T localhost:6082 \ +# -b localhost:8080 \ +# -u varnish -g varnish \ +# -s file,/var/lib/varnish/varnish_storage.bin,1G" + + +## Alternative 2, Configuration with VCL +# +# Listen on port 6081, administration on localhost:6082, and forward to +# one content server selected by the vcl file, based on the request. Use a +# fixed-size cache file. +# +DAEMON_OPTS="-a :6081 \ + -T localhost:6082 \ + -f /etc/varnish/default.vcl \ + -u varnish -g varnish \ + -s file,/var/lib/varnish/varnish_storage.bin,1G" + +VARNISH_ADMIN_LISTEN_ADDRESS=127.0.0.1 +VARNISH_ADMIN_LISTEN_PORT=6082 + +## Alternative 3, Advanced configuration +# +# See varnishd(1) for more information. +# +# # Main configuration file. You probably want to change it :) +# VARNISH_VCL_CONF=/etc/varnish/default.vcl +# +# # Default address and port to bind to +# # Blank address means all IPv4 and IPv6 interfaces, otherwise specify +# # a host name, an IPv4 dotted quad, or an IPv6 address in brackets. +# VARNISH_LISTEN_ADDRESS= +# VARNISH_LISTEN_PORT=6081 +# +# # Telnet admin interface listen address and port +# VARNISH_ADMIN_LISTEN_ADDRESS=127.0.0.1 +# VARNISH_ADMIN_LISTEN_PORT=6082 +# +# # The minimum number of worker threads to start +# VARNISH_MIN_THREADS=1 +# +# # The Maximum number of worker threads to start +# VARNISH_MAX_THREADS=1000 +# +# # Idle timeout for worker threads +# VARNISH_THREAD_TIMEOUT=120 +# +# # Cache file location +# VARNISH_STORAGE_FILE=/var/lib/varnish/varnish_storage.bin +# +# # Cache file size: in bytes, optionally using k / M / G / T suffix, +# # or in percentage of available disk space using the % suffix. +# VARNISH_STORAGE_SIZE=1G +# +# # Backend storage specification +# VARNISH_STORAGE="file,${VARNISH_STORAGE_FILE},${VARNISH_STORAGE_SIZE}" +# +# # Default TTL used when the backend does not specify one +# VARNISH_TTL=120 +# +# # DAEMON_OPTS is used by the init script. If you add or remove options, make +# # sure you update this section, too. +# DAEMON_OPTS="-a ${VARNISH_LISTEN_ADDRESS}:${VARNISH_LISTEN_PORT} \ +# -f ${VARNISH_VCL_CONF} \ +# -T ${VARNISH_ADMIN_LISTEN_ADDRESS}:${VARNISH_ADMIN_LISTEN_PORT} \ +# -t ${VARNISH_TTL} \ +# -w ${VARNISH_MIN_THREADS},${VARNISH_MAX_THREADS},${VARNISH_THREAD_TIMEOUT} \ +# -u varnish -g varnish \ +# -s ${VARNISH_STORAGE}" +# + + +## Alternative 4, Do It Yourself. See varnishd(1) for more information. +# +# DAEMON_OPTS="" -- cgit