use alternate setup

defaults/main.yml

@@ -1,4 +1,9 @@
 ---
+mastodon_initial_setup: false
+
+mastodon_owner: MyUser
+mastodon_owner_email: myemail@example.org
+
 redis_pass: "{{ vault_mastodon_redis_password }}"
 mastodon_secret_key_base: "{{ vault_mastodon_secret_key_base }}"
 mastodon_otp_secret: "{{ vault_mastodon_otp_secret }}"
@@ -60,7 +65,7 @@ redis_port: 6379
 db_host: 127.0.0.1 # /var/run/postgresql
 self_signed_cert_location: /etc/ssl/certs
 self_signed_key_location: /etc/ssl/private
-mastodon_composer_folder: /usr/src/mastodon
+mastodon_composer_folder: /opt/mastodon
 
 es_enabled: 'false'
 es_host: localhost

tasks/docker/docker.yml

@@ -1,15 +1,50 @@
 ---
-- name: Asses state of local repo
-  stat:
+- name: Create Mastodon folder
+  file:
     path: "{{ mastodon_composer_folder }}"
-  register: repo_folder
+    state: directory
 
-- name: Clone Mastodon repo
-  git:
-    repo: https://github.com/mastodon/mastodon.git
-    dest: "{{ mastodon_composer_folder }}"
-    depth: 1
-  when: not repo_folder.stat.exists
+- name: Create Mastodon database folder
+  file:
+    path: "{{ mastodon_composer_folder }}/database"
+    state: directory
+
+- name: Create Mastodon postgresql folder
+  file:
+    path: "{{ mastodon_composer_folder }}/database/postgresql"
+    state: directory
+
+- name: Create Mastodon redis folder
+  file:
+    path: "{{ mastodon_composer_folder }}/database/redis"
+    state: directory
+
+- name: Create Mastodon elasticsearch folder
+  file:
+    path: "{{ mastodon_composer_folder }}/database/elasticsearch"
+    owner: 1000
+    state: directory
+
+- name: Create Mastodon web folder
+  file:
+    path: "{{ mastodon_composer_folder }}/web"
+    owner: 991
+    group: 991
+    state: directory
+
+- name: Create Mastodon web public folder
+  file:
+    path: "{{ mastodon_composer_folder }}/web/public"
+    owner: 991
+    group: 991
+    state: directory
+
+- name: Create Mastodon web system folder
+  file:
+    path: "{{ mastodon_composer_folder }}/web/system"
+    owner: 991
+    group: 991
+    state: directory
 
 - name: Copy Dockerfile from Mastodon repo
   copy:
@@ -26,43 +61,17 @@
     src: templates/docker-compose.yml.j2
     dest: "{{ mastodon_composer_folder }}/docker-compose.yml"
 
-- name: Ensure mastodon configuration is present
+- name: Ensure mastodon application configuration is present
   template:
-    src: templates/env.j2
-    dest: "{{ mastodon_composer_folder }}/.env.production"
+    src: templates/application.env.j2
+    dest: "{{ mastodon_composer_folder }}/application.env.production"
+    mode: 0600
 
-- name: Build Mastodon container
-  community.docker.docker_compose:
-    project_name: mastodon
-    project_src: "{{ mastodon_composer_folder }}/"
-    build: true
-    state: present
-    stopped: true
-
-- name: Stop mastodon_web_1 container
-  docker_container:
-    name: mastodon_web_1
-    state: stopped
-
-- name: Stop mastodon_streaming_1 container
-  docker_container:
-    name: mastodon_streaming_1
-    state: stopped
-
-- name: Stop mastodon_sidekiq_1 container
-  docker_container:
-    name: mastodon_sidekiq_1
-    state: stopped
-
-- name: Stop mastodon_redis_1 container
-  docker_container:
-    name: mastodon_redis_1
-    state: stopped
-
-- name: Stop mastodon_db_1 container
-  docker_container:
-    name: mastodon_db_1
-    state: stopped
+- name: Ensure mastodon database configuration is present
+  template:
+    src: templates/database.env.j2
+    dest: "{{ mastodon_composer_folder }}/database.env.production"
+    mode: 0600
 
 - name: Generate secret key
   shell: docker-compose run --rm web bundle exec rake secret
@@ -78,10 +87,11 @@
 
 - name: Add secret key to Mastodon config
   lineinfile:
-    path: "{{ mastodon_composer_folder }}/.env.production"
+    path: "{{ mastodon_composer_folder }}/application.env.production"
     line: "SECRET_KEY_BASE={{ mastodon_secret_key_base }}"
     regexp: "^SECRET_KEY_BASE="
     backup: yes
+    mode: 0600
 
 - name: Generate OTP secret key
   shell: docker-compose run --rm web bundle exec rake secret
@@ -97,10 +107,11 @@
 
 - name: Add OTP key to Mastodon config
   lineinfile:
-    path: "{{ mastodon_composer_folder }}/.env.production"
+    path: "{{ mastodon_composer_folder }}/application.env.production"
     line: "OTP_SECRET={{ mastodon_otp_secret }}"
     regexp: "^OTP_SECRET="
     backup: yes
+    mode: 0600
 
 - name: Generate Paperclip secret key
   shell: docker-compose run --rm web bundle exec rake secret
@@ -116,10 +127,11 @@
 
 - name: Add Paperclip key to Mastodon config
   lineinfile:
-    path: "{{ mastodon_composer_folder }}/.env.production"
+    path: "{{ mastodon_composer_folder }}/application.env.production"
     line: "PAPERCLIP_SECRET={{ mastodon_paperclip_secret }}"
     regexp: "^PAPERCLIP_SECRET="
     backup: yes
+    mode: 0600
 
 - name: Generate VAPID keypair
   shell: 'docker-compose run --rm web bundle exec rake mastodon:webpush:generate_vapid_key'
@@ -140,19 +152,21 @@
 
 - name: Add Paperclip private key to Mastodon config
   lineinfile:
-    path: "{{ mastodon_composer_folder }}/.env.production"
+    path: "{{ mastodon_composer_folder }}/application.env.production"
     line: "{{ vapid_private_key }}"
     regexp: "^VAPID_PRIVATE_KEY="
+    mode: 0600
     backup: yes
 
 - name: Add Paperclip public key to Mastodon config
   lineinfile:
-    path: "{{ mastodon_composer_folder }}/.env.production"
+    path: "{{ mastodon_composer_folder }}/application.env.production"
     line: "{{ vapid_public_key }}"
     regexp: "^VAPID_PUBLIC_KEY="
+    mode: 0600
     backup: yes
 
-- name: Build (again) Mastodon container to include secrets
+- name: Build Mastodon container to include secrets
   community.docker.docker_compose:
     project_name: mastodon
     project_src: "{{ mastodon_composer_folder }}/"
@@ -175,18 +189,54 @@
     name: mastodon_sidekiq_1
     state: stopped
 
-- name: Stop mastodon_redis_1 container
+- name: Start mastodon_redis_1 container
   docker_container:
     name: mastodon_redis_1
-    state: stopped
+    state: started
 
-- name: Stop mastodon_db_1 container
+- name: Start mastodon_db_1 container
   docker_container:
     name: mastodon_db_1
-    state: stopped
+    state: started
 
 - name: Set up database
-  shell: docker-compose run --rm web rails db:migrate && echo "Database set up." > /var/lib/mastodon_db_setup
+  shell: "docker-compose -f '{{ mastodon_composer_folder }}' run --rm shell bundle exec rake db:setup && echo 'Database ready' > /var/lib/mastodon_db_setup"
   args:
     creates: /var/lib/mastodon_db_setup
     chdir: "{{ mastodon_composer_folder }}"
+  when: mastodon_initial_setup
+
+- name: Migrate database
+  shell: "docker-compose -f '{{ mastodon_composer_folder }}' run --rm shell bundle exec rake db:migrate && echo 'Database migrated' > /var/lib/mastodon_db_migrated"
+  args:
+    creates: /var/lib/mastodon_db_migrated
+    chdir: "{{ mastodon_composer_folder }}"
+  when: not mastodon_initial_setup
+
+- name: Start mastodon_web_1 container
+  docker_container:
+    name: mastodon_web_1
+    state: started
+
+- name: Start mastodon_streaming_1 container
+  docker_container:
+    name: mastodon_streaming_1
+    state: started
+
+- name: Start mastodon_sidekiq_1 container
+  docker_container:
+    name: mastodon_sidekiq_1
+    state: started
+
+- name: Create initial user
+  shell: "docker-compose -f /opt/mastodon/docker-compose.yml run --rm shell bin/tootctl accounts create '{{ mastodon_owner }}' --email '{{ mastodon_owner_email }}' --confirmed --role Owner && echo 'Owner account created' > /var/lib/mastodon_owner_created"
+  args:
+    creates: /var/lib/mastodon_owner_created
+    chdir: "{{ mastodon_composer_folder }}"
+  register: owner_result
+  when: mastodon_initial_setup
+
+- name: Show owner password
+  debug:
+    var: owner_result
+  when: mastodon_initial_setup

templates/application.env.j2

@@ -0,0 +1,36 @@
+# environment
+RAILS_ENV=production
+NODE_ENV=production
+
+# domain
+LOCAL_DOMAIN={{ mastodon_host }}
+
+# redirect to the first profile
+SINGLE_USER_MODE=true
+
+# do not serve static files
+RAILS_SERVE_STATIC_FILES=false
+
+# concurrency
+WEB_CONCURRENCY=2
+MAX_THREADS=5
+
+# pgbouncer
+#PREPARED_STATEMENTS=false
+
+# locale
+DEFAULT_LOCALE=en
+
+# email, not used
+SMTP_SERVER={{ smtp_server }}
+SMTP_PORT={{ smtp_port | default(587) }}
+SMTP_LOGIN={{ smtp_login | default('') }}
+SMTP_PASSWORD={{ smtp_password | default('') }}
+SMTP_FROM_ADDRESS={{ smtp_from_address }}
+
+# secrets
+SECRET_KEY_BASE=
+OTP_SECRET=
+
+VAPID_PRIVATE_KEY=
+VAPID_PUBLIC_KEY=

templates/database.env.j2

@@ -0,0 +1,34 @@
+# postgresql configuration
+POSTGRES_USER=mastodon
+POSTGRES_DB=mastodon_production
+POSTGRES_PASSWORD=O6lOD6nF2LbhhJs1e7QL
+
+# pgbouncer configuration
+#POOL_MODE=transaction
+#ADMIN_USERS=postgres,mastodon
+#DATABASE_URL="postgres://mastodon:O6lOD6nF2LbhhJs1e7QL@postgresql:5432/mastodon_production"
+
+# elasticsearch
+ES_JAVA_OPTS=-Xms512m -Xmx512m
+ELASTIC_PASSWORD=gpwETw6U875pbhnPxbo4
+
+# mastodon database configuration
+#DB_HOST=pgbouncer
+DB_HOST={{ mastodon_db_login_unix_socket }}
+DB_USER={{ mastodon_db_user }}
+DB_NAME={{ mastodon_db }}
+DB_PASS={{ mastodon_db_password }}
+DB_PORT=5432
+
+REDIS_HOST={{ redis_host }}
+REDIS_PORT={{ redis_port }}
+
+CACHE_REDIS_HOST=redis-volatile
+CACHE_REDIS_PORT=6379
+
+ES_ENABLED={{ es_enabled }}
+ES_HOST={{ es_host }}
+ES_PORT={{ es_port }}
+# Authentication for ES (optional)
+ES_USER={{ es_user }}
+ES_PASS={{ es_pass }}

templates/docker-compose.yml.j2

@@ -1,133 +1,168 @@
 version: '3'
+
 services:
-  db:
-    restart: always
+  postgresql:
     image: postgres:14-alpine
+    env_file: database.env.production
+    restart: always
     shm_size: 256mb
-    networks:
-      - internal_network
     healthcheck:
       test: ['CMD', 'pg_isready', '-U', 'postgres']
     volumes:
-      - ./postgres14:/var/lib/postgresql/data
-    environment:
-      - 'POSTGRES_HOST_AUTH_METHOD=trust'
-
-  redis:
-    restart: always
-    image: redis:7-alpine
+      - postgresql:/var/lib/postgresql/data
     networks:
       - internal_network
+
+#  pgbouncer:
+#    image: edoburu/pgbouncer:1.12.0
+#    env_file: database.env.production
+#    depends_on:
+#      - postgresql
+#    healthcheck:
+#      test: ['CMD', 'pg_isready', '-h', 'localhost']
+#    networks:
+#      - internal_network
+
+  redis:
+    image: redis:7-alpine
+    restart: always
     healthcheck:
       test: ['CMD', 'redis-cli', 'ping']
     volumes:
-      - ./redis:/data
-
-  # es:
-  #   restart: always
-  #   image: docker.elastic.co/elasticsearch/elasticsearch:7.17.4
-  #   environment:
-  #     - "ES_JAVA_OPTS=-Xms512m -Xmx512m -Des.enforce.bootstrap.checks=true"
-  #     - "xpack.license.self_generated.type=basic"
-  #     - "xpack.security.enabled=false"
-  #     - "xpack.watcher.enabled=false"
-  #     - "xpack.graph.enabled=false"
-  #     - "xpack.ml.enabled=false"
-  #     - "bootstrap.memory_lock=true"
-  #     - "cluster.name=es-mastodon"
-  #     - "discovery.type=single-node"
-  #     - "thread_pool.write.queue_size=1000"
-  #   networks:
-  #      - external_network
-  #      - internal_network
-  #   healthcheck:
-  #      test: ["CMD-SHELL", "curl --silent --fail localhost:9200/_cluster/health || exit 1"]
-  #   volumes:
-  #      - ./elasticsearch:/usr/share/elasticsearch/data
-  #   ulimits:
-  #     memlock:
-  #       soft: -1
-  #       hard: -1
-  #     nofile:
-  #       soft: 65536
-  #       hard: 65536
-  #   ports:
-  #     - '127.0.0.1:9200:9200'
-
-  web:
-    build: .
-    image: tootsuite/mastodon
-    restart: always
-    env_file: .env.production
-    command: bash -c "rm -f /mastodon/tmp/pids/server.pid; bundle exec rails s -p 3000"
+      - redis:/data
     networks:
-      - external_network
       - internal_network
+
+  redis-volatile:
+    image: redis:7-alpine
+    restart: always
     healthcheck:
-      # prettier-ignore
-      test: ['CMD-SHELL', 'wget -q --spider --proxy=off localhost:3000/health || exit 1']
+      test: ['CMD', 'redis-cli', 'ping']
+    networks:
+      - internal_network
+{% if es_enabled %}
+  elasticsearch:
+    image: elasticsearch:latest
+    restart: always
+    env_file: database.env.production
+    environment:
+      - cluster.name=elasticsearch-mastodon
+      - discovery.type=single-node
+      - bootstrap.memory_lock=true
+      - xpack.security.enabled=true
+      - ingest.geoip.downloader.enabled=false
+    ulimits:
+      memlock:
+        soft: -1
+        hard: -1
+    healthcheck:
+      test: ["CMD-SHELL", "nc -z elasticsearch 9200"]
+    volumes:
+      - elasticsearch:/usr/share/elasticsearch/data
+    networks:
+      - internal_network
+{% endif %}
+  website:
+    image: tootsuite/mastodon:latest
+    env_file: 
+      - application.env.production
+      - database.env.production
+    command: bash -c "bundle exec rails s -p 3000"
+    restart: always    
+    depends_on:
+      - postgresql
+#      - pgbouncer
+      - redis
+      - redis-volatile
+      - elasticsearch
     ports:
       - '127.0.0.1:3000:3000'
-    depends_on:
-      - db
-      - redis
-      # - es
+    networks:
+      - internal_network
+      - external_network
+    healthcheck:
+      test: ['CMD-SHELL', 'wget -q --spider --proxy=off localhost:3000/health || exit 1']
     volumes:
-      - ./public/system:/mastodon/public/system
+      - uploads:/mastodon/public/system
+
+  shell:
+    image: tootsuite/mastodon:latest
+    env_file: 
+      - application.env.production
+      - database.env.production
+    command: /bin/bash 
+    restart: "no"
+    networks:
+      - internal_network
+      - external_network
+    volumes:
+      - uploads:/mastodon/public/system
 
   streaming:
-    build: .
-    image: tootsuite/mastodon
-    restart: always
-    env_file: .env.production
+    image: tootsuite/mastodon:latest
+    env_file: 
+      - application.env.production
+      - database.env.production
     command: node ./streaming
-    networks:
-      - external_network
-      - internal_network
-    healthcheck:
-      # prettier-ignore
-      test: ['CMD-SHELL', 'wget -q --spider --proxy=off localhost:4000/api/v1/streaming/health || exit 1']
+    restart: always
+    depends_on:
+      - postgresql
+#      - pgbouncer
+      - redis
+      - redis-volatile
+      - elasticsearch
     ports:
       - '127.0.0.1:4000:4000'
-    depends_on:
-      - db
-      - redis
+    networks:
+      - internal_network
+      - external_network
+    healthcheck:
+      test: ['CMD-SHELL', 'wget -q --spider --proxy=off localhost:4000/api/v1/streaming/health || exit 1']
 
   sidekiq:
-    build: .
-    image: tootsuite/mastodon
-    restart: always
-    env_file: .env.production
+    image: tootsuite/mastodon:latest
+    env_file: 
+      - application.env.production
+      - database.env.production
     command: bundle exec sidekiq
+    restart: always
     depends_on:
-      - db
+      - postgresql
+#      - pgbouncer
       - redis
+      - redis-volatile
+      - website
     networks:
-      - external_network
       - internal_network
-    volumes:
-      - ./public/system:/mastodon/public/system
+      - external_network
     healthcheck:
       test: ['CMD-SHELL', "ps aux | grep '[s]idekiq\ 6' || false"]
-
-  ## Uncomment to enable federation with tor instances along with adding the following ENV variables
-  ## http_proxy=http://privoxy:8118
-  ## ALLOW_ACCESS_TO_HIDDEN_SERVICE=true
-  # tor:
-  #   image: sirboops/tor
-  #   networks:
-  #      - external_network
-  #      - internal_network
-  #
-  # privoxy:
-  #   image: sirboops/privoxy
-  #   volumes:
-  #     - ./priv-config:/opt/config
-  #   networks:
-  #     - external_network
-  #     - internal_network
+    volumes:
+      - uploads:/mastodon/public/system
 
 networks:
   external_network:
   internal_network:
     internal: true
+
+volumes:
+  postgresql:
+    driver_opts:
+      type: none
+      device: {{ mastodon_composer_folder }}/database/postgresql
+      o: bind    
+  redis:
+    driver_opts:
+      type: none
+      device: {{ mastodon_composer_folder }}/database/redis
+      o: bind    
+{% if es_enabled %}  elasticsearch:
+    driver_opts:
+      type: none
+      device: {{ mastodon_composer_folder }}/database/elasticsearch
+      o: bind
+{% endif %}  uploads:
+    driver_opts:
+      type: none
+      device: {{ mastodon_composer_folder }}/web/system
+      o: bind   

templates/env.j2

@@ -1,69 +0,0 @@
-# This is a sample configuration file. You can generate your configuration
-# with the `rake mastodon:setup` interactive setup wizard, but to customize
-# your setup even further, you'll need to edit it manually. This sample does
-# not demonstrate all available configuration options. Please look at
-# https://docs.joinmastodon.org/admin/config/ for the full documentation.
-
-# Note that this file accepts slightly different syntax depending on whether
-# you are using `docker-compose` or not. In particular, if you use
-# `docker-compose`, the value of each declared variable will be taken verbatim,
-# including surrounding quotes.
-# See: https://github.com/mastodon/mastodon/issues/16895
-
-# Federation
-# ----------
-# This identifies your server and cannot be changed safely later
-# ----------
-LOCAL_DOMAIN={{ mastodon_host }}
-
-# Redis
-# -----
-REDIS_HOST={{ redis_host }}
-REDIS_PORT={{ redis_port }}
-
-# PostgreSQL
-# ----------
-DB_HOST={{ mastodon_db_login_unix_socket }}
-DB_USER={{ mastodon_db_user }}
-DB_NAME={{ mastodon_db }}
-DB_PASS={{ mastodon_db_password }}
-DB_PORT={{ mastodon_db_port }}
-
-# Elasticsearch (optional)
-# ------------------------
-ES_ENABLED={{ es_enabled }}
-ES_HOST={{ es_host }}
-ES_PORT={{ es_port }}
-# Authentication for ES (optional)
-ES_USER={{ es_user }}
-ES_PASS={{ es_pass }}
-
-# Secrets
-# -------
-# Make sure to use `rake secret` to generate secrets
-# -------
-SECRET_KEY_BASE=
-OTP_SECRET=
-
-# Web Push
-# --------
-# Generate with `rake mastodon:webpush:generate_vapid_key`
-# --------
-VAPID_PRIVATE_KEY=
-VAPID_PUBLIC_KEY=
-
-# Sending mail
-# ------------
-SMTP_SERVER={{ smtp_server }}
-SMTP_PORT={{ smtp_port | default(587) }}
-SMTP_LOGIN={{ smtp_login | default('') }}
-SMTP_PASSWORD={{ smtp_password | default('') }}
-SMTP_FROM_ADDRESS={{ smtp_from_address }}
-
-# File storage (optional)
-# -----------------------
-S3_ENABLED={{ s3_enabled }}
-S3_BUCKET={{ s3_bucket }}
-AWS_ACCESS_KEY_ID={{ aws_access_key_id }}
-AWS_SECRET_ACCESS_KEY={{ aws_secret_access_key }}
-S3_ALIAS_HOST={{ s3_alias_host}}