From adaf582b80f43b786cca23fe0307c6389f84a3ac Mon Sep 17 00:00:00 2001 From: "Antonio J. Delgado" Date: Sat, 19 Nov 2022 22:55:50 +0200 Subject: [PATCH] use alternate setup --- defaults/main.yml | 7 +- tasks/docker/docker.yml | 160 ++++++++++++++-------- templates/application.env.j2 | 36 +++++ templates/database.env.j2 | 34 +++++ templates/docker-compose.yml.j2 | 227 ++++++++++++++++++-------------- templates/env.j2 | 69 ---------- 6 files changed, 312 insertions(+), 221 deletions(-) create mode 100644 templates/application.env.j2 create mode 100644 templates/database.env.j2 delete mode 100644 templates/env.j2 diff --git a/defaults/main.yml b/defaults/main.yml index 236753e..996708f 100644 --- a/defaults/main.yml +++ b/defaults/main.yml @@ -1,4 +1,9 @@ --- +mastodon_initial_setup: false + +mastodon_owner: MyUser +mastodon_owner_email: myemail@example.org + redis_pass: "{{ vault_mastodon_redis_password }}" mastodon_secret_key_base: "{{ vault_mastodon_secret_key_base }}" mastodon_otp_secret: "{{ vault_mastodon_otp_secret }}" @@ -60,7 +65,7 @@ redis_port: 6379 db_host: 127.0.0.1 # /var/run/postgresql self_signed_cert_location: /etc/ssl/certs self_signed_key_location: /etc/ssl/private -mastodon_composer_folder: /usr/src/mastodon +mastodon_composer_folder: /opt/mastodon es_enabled: 'false' es_host: localhost diff --git a/tasks/docker/docker.yml b/tasks/docker/docker.yml index 5fa66f6..b901ec4 100644 --- a/tasks/docker/docker.yml +++ b/tasks/docker/docker.yml @@ -1,15 +1,50 @@ --- -- name: Asses state of local repo - stat: +- name: Create Mastodon folder + file: path: "{{ mastodon_composer_folder }}" - register: repo_folder + state: directory -- name: Clone Mastodon repo - git: - repo: https://github.com/mastodon/mastodon.git - dest: "{{ mastodon_composer_folder }}" - depth: 1 - when: not repo_folder.stat.exists +- name: Create Mastodon database folder + file: + path: "{{ mastodon_composer_folder }}/database" + state: directory + +- name: Create Mastodon postgresql folder + file: + path: "{{ mastodon_composer_folder }}/database/postgresql" + state: directory + +- name: Create Mastodon redis folder + file: + path: "{{ mastodon_composer_folder }}/database/redis" + state: directory + +- name: Create Mastodon elasticsearch folder + file: + path: "{{ mastodon_composer_folder }}/database/elasticsearch" + owner: 1000 + state: directory + +- name: Create Mastodon web folder + file: + path: "{{ mastodon_composer_folder }}/web" + owner: 991 + group: 991 + state: directory + +- name: Create Mastodon web public folder + file: + path: "{{ mastodon_composer_folder }}/web/public" + owner: 991 + group: 991 + state: directory + +- name: Create Mastodon web system folder + file: + path: "{{ mastodon_composer_folder }}/web/system" + owner: 991 + group: 991 + state: directory - name: Copy Dockerfile from Mastodon repo copy: @@ -26,43 +61,17 @@ src: templates/docker-compose.yml.j2 dest: "{{ mastodon_composer_folder }}/docker-compose.yml" -- name: Ensure mastodon configuration is present +- name: Ensure mastodon application configuration is present template: - src: templates/env.j2 - dest: "{{ mastodon_composer_folder }}/.env.production" + src: templates/application.env.j2 + dest: "{{ mastodon_composer_folder }}/application.env.production" + mode: 0600 -- name: Build Mastodon container - community.docker.docker_compose: - project_name: mastodon - project_src: "{{ mastodon_composer_folder }}/" - build: true - state: present - stopped: true - -- name: Stop mastodon_web_1 container - docker_container: - name: mastodon_web_1 - state: stopped - -- name: Stop mastodon_streaming_1 container - docker_container: - name: mastodon_streaming_1 - state: stopped - -- name: Stop mastodon_sidekiq_1 container - docker_container: - name: mastodon_sidekiq_1 - state: stopped - -- name: Stop mastodon_redis_1 container - docker_container: - name: mastodon_redis_1 - state: stopped - -- name: Stop mastodon_db_1 container - docker_container: - name: mastodon_db_1 - state: stopped +- name: Ensure mastodon database configuration is present + template: + src: templates/database.env.j2 + dest: "{{ mastodon_composer_folder }}/database.env.production" + mode: 0600 - name: Generate secret key shell: docker-compose run --rm web bundle exec rake secret @@ -78,10 +87,11 @@ - name: Add secret key to Mastodon config lineinfile: - path: "{{ mastodon_composer_folder }}/.env.production" + path: "{{ mastodon_composer_folder }}/application.env.production" line: "SECRET_KEY_BASE={{ mastodon_secret_key_base }}" regexp: "^SECRET_KEY_BASE=" backup: yes + mode: 0600 - name: Generate OTP secret key shell: docker-compose run --rm web bundle exec rake secret @@ -97,10 +107,11 @@ - name: Add OTP key to Mastodon config lineinfile: - path: "{{ mastodon_composer_folder }}/.env.production" + path: "{{ mastodon_composer_folder }}/application.env.production" line: "OTP_SECRET={{ mastodon_otp_secret }}" regexp: "^OTP_SECRET=" backup: yes + mode: 0600 - name: Generate Paperclip secret key shell: docker-compose run --rm web bundle exec rake secret @@ -116,10 +127,11 @@ - name: Add Paperclip key to Mastodon config lineinfile: - path: "{{ mastodon_composer_folder }}/.env.production" + path: "{{ mastodon_composer_folder }}/application.env.production" line: "PAPERCLIP_SECRET={{ mastodon_paperclip_secret }}" regexp: "^PAPERCLIP_SECRET=" backup: yes + mode: 0600 - name: Generate VAPID keypair shell: 'docker-compose run --rm web bundle exec rake mastodon:webpush:generate_vapid_key' @@ -140,19 +152,21 @@ - name: Add Paperclip private key to Mastodon config lineinfile: - path: "{{ mastodon_composer_folder }}/.env.production" + path: "{{ mastodon_composer_folder }}/application.env.production" line: "{{ vapid_private_key }}" regexp: "^VAPID_PRIVATE_KEY=" + mode: 0600 backup: yes - name: Add Paperclip public key to Mastodon config lineinfile: - path: "{{ mastodon_composer_folder }}/.env.production" + path: "{{ mastodon_composer_folder }}/application.env.production" line: "{{ vapid_public_key }}" regexp: "^VAPID_PUBLIC_KEY=" + mode: 0600 backup: yes -- name: Build (again) Mastodon container to include secrets +- name: Build Mastodon container to include secrets community.docker.docker_compose: project_name: mastodon project_src: "{{ mastodon_composer_folder }}/" @@ -175,18 +189,54 @@ name: mastodon_sidekiq_1 state: stopped -- name: Stop mastodon_redis_1 container +- name: Start mastodon_redis_1 container docker_container: name: mastodon_redis_1 - state: stopped + state: started -- name: Stop mastodon_db_1 container +- name: Start mastodon_db_1 container docker_container: name: mastodon_db_1 - state: stopped + state: started - name: Set up database - shell: docker-compose run --rm web rails db:migrate && echo "Database set up." > /var/lib/mastodon_db_setup + shell: "docker-compose -f '{{ mastodon_composer_folder }}' run --rm shell bundle exec rake db:setup && echo 'Database ready' > /var/lib/mastodon_db_setup" args: creates: /var/lib/mastodon_db_setup chdir: "{{ mastodon_composer_folder }}" + when: mastodon_initial_setup + +- name: Migrate database + shell: "docker-compose -f '{{ mastodon_composer_folder }}' run --rm shell bundle exec rake db:migrate && echo 'Database migrated' > /var/lib/mastodon_db_migrated" + args: + creates: /var/lib/mastodon_db_migrated + chdir: "{{ mastodon_composer_folder }}" + when: not mastodon_initial_setup + +- name: Start mastodon_web_1 container + docker_container: + name: mastodon_web_1 + state: started + +- name: Start mastodon_streaming_1 container + docker_container: + name: mastodon_streaming_1 + state: started + +- name: Start mastodon_sidekiq_1 container + docker_container: + name: mastodon_sidekiq_1 + state: started + +- name: Create initial user + shell: "docker-compose -f /opt/mastodon/docker-compose.yml run --rm shell bin/tootctl accounts create '{{ mastodon_owner }}' --email '{{ mastodon_owner_email }}' --confirmed --role Owner && echo 'Owner account created' > /var/lib/mastodon_owner_created" + args: + creates: /var/lib/mastodon_owner_created + chdir: "{{ mastodon_composer_folder }}" + register: owner_result + when: mastodon_initial_setup + +- name: Show owner password + debug: + var: owner_result + when: mastodon_initial_setup diff --git a/templates/application.env.j2 b/templates/application.env.j2 new file mode 100644 index 0000000..2c4e60f --- /dev/null +++ b/templates/application.env.j2 @@ -0,0 +1,36 @@ +# environment +RAILS_ENV=production +NODE_ENV=production + +# domain +LOCAL_DOMAIN={{ mastodon_host }} + +# redirect to the first profile +SINGLE_USER_MODE=true + +# do not serve static files +RAILS_SERVE_STATIC_FILES=false + +# concurrency +WEB_CONCURRENCY=2 +MAX_THREADS=5 + +# pgbouncer +#PREPARED_STATEMENTS=false + +# locale +DEFAULT_LOCALE=en + +# email, not used +SMTP_SERVER={{ smtp_server }} +SMTP_PORT={{ smtp_port | default(587) }} +SMTP_LOGIN={{ smtp_login | default('') }} +SMTP_PASSWORD={{ smtp_password | default('') }} +SMTP_FROM_ADDRESS={{ smtp_from_address }} + +# secrets +SECRET_KEY_BASE= +OTP_SECRET= + +VAPID_PRIVATE_KEY= +VAPID_PUBLIC_KEY= diff --git a/templates/database.env.j2 b/templates/database.env.j2 new file mode 100644 index 0000000..ddbcf74 --- /dev/null +++ b/templates/database.env.j2 @@ -0,0 +1,34 @@ +# postgresql configuration +POSTGRES_USER=mastodon +POSTGRES_DB=mastodon_production +POSTGRES_PASSWORD=O6lOD6nF2LbhhJs1e7QL + +# pgbouncer configuration +#POOL_MODE=transaction +#ADMIN_USERS=postgres,mastodon +#DATABASE_URL="postgres://mastodon:O6lOD6nF2LbhhJs1e7QL@postgresql:5432/mastodon_production" + +# elasticsearch +ES_JAVA_OPTS=-Xms512m -Xmx512m +ELASTIC_PASSWORD=gpwETw6U875pbhnPxbo4 + +# mastodon database configuration +#DB_HOST=pgbouncer +DB_HOST={{ mastodon_db_login_unix_socket }} +DB_USER={{ mastodon_db_user }} +DB_NAME={{ mastodon_db }} +DB_PASS={{ mastodon_db_password }} +DB_PORT=5432 + +REDIS_HOST={{ redis_host }} +REDIS_PORT={{ redis_port }} + +CACHE_REDIS_HOST=redis-volatile +CACHE_REDIS_PORT=6379 + +ES_ENABLED={{ es_enabled }} +ES_HOST={{ es_host }} +ES_PORT={{ es_port }} +# Authentication for ES (optional) +ES_USER={{ es_user }} +ES_PASS={{ es_pass }} diff --git a/templates/docker-compose.yml.j2 b/templates/docker-compose.yml.j2 index c534286..089ba2c 100644 --- a/templates/docker-compose.yml.j2 +++ b/templates/docker-compose.yml.j2 @@ -1,133 +1,168 @@ version: '3' + services: - db: - restart: always + postgresql: image: postgres:14-alpine + env_file: database.env.production + restart: always shm_size: 256mb - networks: - - internal_network healthcheck: test: ['CMD', 'pg_isready', '-U', 'postgres'] volumes: - - ./postgres14:/var/lib/postgresql/data - environment: - - 'POSTGRES_HOST_AUTH_METHOD=trust' - - redis: - restart: always - image: redis:7-alpine + - postgresql:/var/lib/postgresql/data networks: - internal_network + +# pgbouncer: +# image: edoburu/pgbouncer:1.12.0 +# env_file: database.env.production +# depends_on: +# - postgresql +# healthcheck: +# test: ['CMD', 'pg_isready', '-h', 'localhost'] +# networks: +# - internal_network + + redis: + image: redis:7-alpine + restart: always healthcheck: test: ['CMD', 'redis-cli', 'ping'] volumes: - - ./redis:/data - - # es: - # restart: always - # image: docker.elastic.co/elasticsearch/elasticsearch:7.17.4 - # environment: - # - "ES_JAVA_OPTS=-Xms512m -Xmx512m -Des.enforce.bootstrap.checks=true" - # - "xpack.license.self_generated.type=basic" - # - "xpack.security.enabled=false" - # - "xpack.watcher.enabled=false" - # - "xpack.graph.enabled=false" - # - "xpack.ml.enabled=false" - # - "bootstrap.memory_lock=true" - # - "cluster.name=es-mastodon" - # - "discovery.type=single-node" - # - "thread_pool.write.queue_size=1000" - # networks: - # - external_network - # - internal_network - # healthcheck: - # test: ["CMD-SHELL", "curl --silent --fail localhost:9200/_cluster/health || exit 1"] - # volumes: - # - ./elasticsearch:/usr/share/elasticsearch/data - # ulimits: - # memlock: - # soft: -1 - # hard: -1 - # nofile: - # soft: 65536 - # hard: 65536 - # ports: - # - '127.0.0.1:9200:9200' - - web: - build: . - image: tootsuite/mastodon - restart: always - env_file: .env.production - command: bash -c "rm -f /mastodon/tmp/pids/server.pid; bundle exec rails s -p 3000" + - redis:/data networks: - - external_network - internal_network + + redis-volatile: + image: redis:7-alpine + restart: always healthcheck: - # prettier-ignore - test: ['CMD-SHELL', 'wget -q --spider --proxy=off localhost:3000/health || exit 1'] + test: ['CMD', 'redis-cli', 'ping'] + networks: + - internal_network +{% if es_enabled %} + elasticsearch: + image: elasticsearch:latest + restart: always + env_file: database.env.production + environment: + - cluster.name=elasticsearch-mastodon + - discovery.type=single-node + - bootstrap.memory_lock=true + - xpack.security.enabled=true + - ingest.geoip.downloader.enabled=false + ulimits: + memlock: + soft: -1 + hard: -1 + healthcheck: + test: ["CMD-SHELL", "nc -z elasticsearch 9200"] + volumes: + - elasticsearch:/usr/share/elasticsearch/data + networks: + - internal_network +{% endif %} + website: + image: tootsuite/mastodon:latest + env_file: + - application.env.production + - database.env.production + command: bash -c "bundle exec rails s -p 3000" + restart: always + depends_on: + - postgresql +# - pgbouncer + - redis + - redis-volatile + - elasticsearch ports: - '127.0.0.1:3000:3000' - depends_on: - - db - - redis - # - es + networks: + - internal_network + - external_network + healthcheck: + test: ['CMD-SHELL', 'wget -q --spider --proxy=off localhost:3000/health || exit 1'] volumes: - - ./public/system:/mastodon/public/system + - uploads:/mastodon/public/system + + shell: + image: tootsuite/mastodon:latest + env_file: + - application.env.production + - database.env.production + command: /bin/bash + restart: "no" + networks: + - internal_network + - external_network + volumes: + - uploads:/mastodon/public/system streaming: - build: . - image: tootsuite/mastodon - restart: always - env_file: .env.production + image: tootsuite/mastodon:latest + env_file: + - application.env.production + - database.env.production command: node ./streaming - networks: - - external_network - - internal_network - healthcheck: - # prettier-ignore - test: ['CMD-SHELL', 'wget -q --spider --proxy=off localhost:4000/api/v1/streaming/health || exit 1'] + restart: always + depends_on: + - postgresql +# - pgbouncer + - redis + - redis-volatile + - elasticsearch ports: - '127.0.0.1:4000:4000' - depends_on: - - db - - redis + networks: + - internal_network + - external_network + healthcheck: + test: ['CMD-SHELL', 'wget -q --spider --proxy=off localhost:4000/api/v1/streaming/health || exit 1'] sidekiq: - build: . - image: tootsuite/mastodon - restart: always - env_file: .env.production + image: tootsuite/mastodon:latest + env_file: + - application.env.production + - database.env.production command: bundle exec sidekiq + restart: always depends_on: - - db + - postgresql +# - pgbouncer - redis + - redis-volatile + - website networks: - - external_network - internal_network - volumes: - - ./public/system:/mastodon/public/system + - external_network healthcheck: test: ['CMD-SHELL', "ps aux | grep '[s]idekiq\ 6' || false"] - - ## Uncomment to enable federation with tor instances along with adding the following ENV variables - ## http_proxy=http://privoxy:8118 - ## ALLOW_ACCESS_TO_HIDDEN_SERVICE=true - # tor: - # image: sirboops/tor - # networks: - # - external_network - # - internal_network - # - # privoxy: - # image: sirboops/privoxy - # volumes: - # - ./priv-config:/opt/config - # networks: - # - external_network - # - internal_network + volumes: + - uploads:/mastodon/public/system networks: external_network: internal_network: internal: true + +volumes: + postgresql: + driver_opts: + type: none + device: {{ mastodon_composer_folder }}/database/postgresql + o: bind + redis: + driver_opts: + type: none + device: {{ mastodon_composer_folder }}/database/redis + o: bind +{% if es_enabled %} elasticsearch: + driver_opts: + type: none + device: {{ mastodon_composer_folder }}/database/elasticsearch + o: bind +{% endif %} uploads: + driver_opts: + type: none + device: {{ mastodon_composer_folder }}/web/system + o: bind diff --git a/templates/env.j2 b/templates/env.j2 deleted file mode 100644 index 0d16f3b..0000000 --- a/templates/env.j2 +++ /dev/null @@ -1,69 +0,0 @@ -# This is a sample configuration file. You can generate your configuration -# with the `rake mastodon:setup` interactive setup wizard, but to customize -# your setup even further, you'll need to edit it manually. This sample does -# not demonstrate all available configuration options. Please look at -# https://docs.joinmastodon.org/admin/config/ for the full documentation. - -# Note that this file accepts slightly different syntax depending on whether -# you are using `docker-compose` or not. In particular, if you use -# `docker-compose`, the value of each declared variable will be taken verbatim, -# including surrounding quotes. -# See: https://github.com/mastodon/mastodon/issues/16895 - -# Federation -# ---------- -# This identifies your server and cannot be changed safely later -# ---------- -LOCAL_DOMAIN={{ mastodon_host }} - -# Redis -# ----- -REDIS_HOST={{ redis_host }} -REDIS_PORT={{ redis_port }} - -# PostgreSQL -# ---------- -DB_HOST={{ mastodon_db_login_unix_socket }} -DB_USER={{ mastodon_db_user }} -DB_NAME={{ mastodon_db }} -DB_PASS={{ mastodon_db_password }} -DB_PORT={{ mastodon_db_port }} - -# Elasticsearch (optional) -# ------------------------ -ES_ENABLED={{ es_enabled }} -ES_HOST={{ es_host }} -ES_PORT={{ es_port }} -# Authentication for ES (optional) -ES_USER={{ es_user }} -ES_PASS={{ es_pass }} - -# Secrets -# ------- -# Make sure to use `rake secret` to generate secrets -# ------- -SECRET_KEY_BASE= -OTP_SECRET= - -# Web Push -# -------- -# Generate with `rake mastodon:webpush:generate_vapid_key` -# -------- -VAPID_PRIVATE_KEY= -VAPID_PUBLIC_KEY= - -# Sending mail -# ------------ -SMTP_SERVER={{ smtp_server }} -SMTP_PORT={{ smtp_port | default(587) }} -SMTP_LOGIN={{ smtp_login | default('') }} -SMTP_PASSWORD={{ smtp_password | default('') }} -SMTP_FROM_ADDRESS={{ smtp_from_address }} - -# File storage (optional) -# ----------------------- -S3_ENABLED={{ s3_enabled }} -S3_BUCKET={{ s3_bucket }} -AWS_ACCESS_KEY_ID={{ aws_access_key_id }} -AWS_SECRET_ACCESS_KEY={{ aws_secret_access_key }} -S3_ALIAS_HOST={{ s3_alias_host}}