Merge branch 'main' into feature/menuandfonts

UPDATING.md

@@ -0,0 +1,47 @@
+# Updating
+
+It is quite important for operators to check this file before updating their
+production instances, as it may contain information that is critical for good
+continuity of the services.
+
+
+# Kinds of breaking changes
+
+## Nextcloud upgrades
+
+Nextcloud upgrades must be carried out in a very specific order, if you follow
+DD closely you should not have issues with this on 2023 and beyond.
+
+However, if you run an older instance, please find the
+**2022-12 - Nextcloud 24** section on this document, and follow the commands
+closely.
+
+Further major upgrades will be documented here too, but they should have
+little to no impact.
+
+
+# Important changes in reverse chronological order
+
+## 2022
+
+### 2022-12 - Nextcloud 24
+
+Starting with commits in december, DD's blessed Nextcloud version is 24.0.8.
+
+Work is already being carried out to upgrade this to major version 25.
+
+In order to safely upgrade to major version 24, the upgrade must be performed
+step-wise:
+
+    # Latest commit with NC major version 21 by default
+    ./dd-ctl update db4a6d14e9a05ee1b0334aaf87af31703a4ae4d0
+    # Upgrade to NC version 22
+    ./dd-ctl update cbb4e06e896da40430d66435589ab5ea8cf71420
+    # Upgrade to NC version 23
+    ./dd-ctl update 57c87d7172579b8fbe0d8249f47ee0966621e252
+    # Upgrade to NC version 24.0.8  (latest minor on december 2022)
+    ./dd-ctl update f55e830e2cd3071aceb604642fcfb5fdd8d5df17
+    #
+    # Upgrade to latest state of DD
+    # (read more recent notices of important changes before running this)
+    #./dd-ctl update

dd-apps/docker/nextcloud/dd-patch

@@ -1,9 +1,7 @@
 # Generate .orig and .patch files with ./dd-ctl genpatches
 # file	license	author	source
-nginx.conf	AGPL-3.0-or-later	https://github.com/nextcloud/	https://raw.githubusercontent.com/nextcloud/docker/522559eefdd56d2e49259c3b0f4a0e92882cdb87/.examples/docker-compose/with-nginx-proxy/postgres/fpm/web/nginx.conf
+nginx.conf	AGPL-3.0-or-later	https://github.com/nextcloud/	https://raw.githubusercontent.com/nextcloud/docker/04ab3157e4f1197a9f8f4b91cf2430586356dd9a/.examples/docker-compose/with-nginx-proxy/postgres/fpm/web/nginx.conf
-#nc_mail/appinfo.xml	AGPL-3.0-or-later	https://github.com/nextcloud/	https://raw.githubusercontent.com/MaadixNet/mail/feature/occ-account-update-command/appinfo/info.xml
+supervisord.conf	AGPL-3.0-or-later	https://github.com/nextcloud/	https://raw.githubusercontent.com/nextcloud/docker/04ab3157e4f1197a9f8f4b91cf2430586356dd9a/.examples/dockerfiles/cron/fpm-alpine/supervisord.conf
-#nc_mail/lib/Command/UpdateAccount.php	AGPL-3.0-or-later	https://github.com/nextcloud/	https://raw.githubusercontent.com/MaadixNet/mail/feature/occ-account-update-command/lib/Command/UpdateAccount.php
+nc_mail/appinfo/info.xml	AGPL-3.0-or-later	https://github.com/nextcloud/	https://raw.githubusercontent.com/nextcloud/mail/v1.15.1/appinfo/info.xml
-#nc_mail/lib/Db/MailAccountMapper.php	AGPL-3.0-or-later	https://github.com/nextcloud/	https://raw.githubusercontent.com/MaadixNet/mail/feature/occ-account-update-command/lib/Db/MailAccountMapper.php
+nc_mail/lib/Command/UpdateAccount.php	AGPL-3.0-or-later	https://github.com/nextcloud/	https://raw.githubusercontent.com/nextcloud/mail/1e777a1783254bd4b7f69f39a6c5123323f8b701/lib/Command/UpdateAccount.php
-nc_mail/appinfo/info.xml	AGPL-3.0-or-later	https://github.com/nextcloud/	https://raw.githubusercontent.com/nextcloud/mail/v1.12.8/appinfo/info.xml
+nc_mail/lib/Db/MailAccountMapper.php	AGPL-3.0-or-later	https://github.com/nextcloud/	https://raw.githubusercontent.com/nextcloud/mail/1e777a1783254bd4b7f69f39a6c5123323f8b701/lib/Db/MailAccountMapper.php
-nc_mail/lib/Command/UpdateAccount.php	AGPL-3.0-or-later	https://github.com/nextcloud/	https://raw.githubusercontent.com/nextcloud/mail/v1.12.8/lib/Command/UpdateAccount.php
-nc_mail/lib/Db/MailAccountMapper.php	AGPL-3.0-or-later	https://github.com/nextcloud/	https://raw.githubusercontent.com/nextcloud/mail/v1.12.8/lib/Db/MailAccountMapper.php

dd-apps/docker/nextcloud/nc_mail/appinfo/info.xml

@@ -12,7 +12,7 @@
 - **🙈 We’re not reinventing the wheel!** Based on the great [Horde](https://horde.org) libraries.
 - **📬 Want to host your own mail server?** We do not have to reimplement this as you could set up [Mail-in-a-Box](https://mailinabox.email)!
 	]]></description>
-	<version>1.12.8</version>
+	<version>1.15.1</version>
 	<licence>agpl</licence>
 	<author>Greta Doçi</author>
 	<author homepage="https://github.com/nextcloud/groupware">Nextcloud Groupware Team</author>
@@ -29,8 +29,8 @@
 	<repository type="git">https://github.com/nextcloud/mail.git</repository>
 	<screenshot>https://user-images.githubusercontent.com/1374172/79554966-278e1600-809f-11ea-82ea-7a0d72a2704f.png</screenshot>
 	<dependencies>
-		<php min-version="7.3" max-version="8.0" />
+		<php min-version="7.4" max-version="8.1" />
-		<nextcloud min-version="21" max-version="24" />
+		<nextcloud min-version="23" max-version="25" />
 	</dependencies>
 	<background-jobs>
 		<job>OCA\Mail\BackgroundJob\CleanupJob</job>

dd-apps/docker/nextcloud/nc_mail/lib/Db/MailAccountMapper.php

@@ -37,7 +37,6 @@ use OCP\IUser;
  * @template-extends QBMapper<MailAccount>
  */
 class MailAccountMapper extends QBMapper {
-
 	/**
 	 * @param IDBConnection $db
 	 */
@@ -97,22 +96,6 @@ class MailAccountMapper extends QBMapper {
 
 		return $this->findEntities($query);
 	}
-	/**
-	 * Finds an mail account by user id and email address
-	 *
-	 * @return MailAccount
-	 * @throws DoesNotExistException
-	 */
-	public function findByUserIdAndEmail(string $userId, string $email): MailAccount {
-		$qb = $this->db->getQueryBuilder();
-		$query = $qb
-			->select('*')
-			->from($this->getTableName())
-			->where($qb->expr()->eq('user_id', $qb->createNamedParameter($userId, IQueryBuilder::PARAM_STR), IQueryBuilder::PARAM_STR))
-			->andWhere($qb->expr()->eq('email', $qb->createNamedParameter($email, IQueryBuilder::PARAM_STR), IQueryBuilder::PARAM_STR));
-
-		return $this->findEntity($query);
-	}
 
 	/**
 	 * @throws DoesNotExistException

dd-apps/docker/nextcloud/nextcloud.yml

@@ -1,5 +1,6 @@
 #
 #   Copyright © 2021,2022 IsardVDI S.L.
+#   Copyright © 2023 Evilham <contact@evilham.com>
 #
 #   This file is part of DD
 #
@@ -24,7 +25,8 @@ services:
       context: ${BUILD_APPS_ROOT_PATH}/docker/nextcloud
       dockerfile: Dockerfile
       args:
-        - IMG=${NEXTCLOUD_IMG}
+        # Update dd.conf.sample when bumping this version in main
+        - IMG=${NEXTCLOUD_IMG_OVERRIDE-nextcloud:24.0.8-fpm-alpine}
     container_name: dd-apps-nextcloud-app
     restart: unless-stopped
     depends_on:
@@ -36,7 +38,9 @@ services:
       - ${DATA_FOLDER}/saml/nextcloud:/saml:ro
       - ${DATA_FOLDER}/nc-mail-queue:/nc-mail-queue:rw
     environment:
+      # DD-specific settings
       - DOMAIN=${DOMAIN}
+      # General Nextcloud settings
       - NEXTCLOUD_ADMIN_USER=${NEXTCLOUD_ADMIN_USER}
       - NEXTCLOUD_ADMIN_PASSWORD=${NEXTCLOUD_ADMIN_PASSWORD}
       - POSTGRES_DB=nextcloud
@@ -44,20 +48,37 @@ services:
       - POSTGRES_PASSWORD=${NEXTCLOUD_POSTGRES_PASSWORD}
       - POSTGRES_HOST=dd-apps-postgresql
       - REDIS_HOST=dd-apps-redis
-      - NC_overwriteprotocol=https
       - NEXTCLOUD_TRUSTED_DOMAINS=nextcloud.${DOMAIN}
+      # SMTP settings
+      - SMTP_HOST=${SMTP_HOST}
+      - SMTP_PORT=${SMTP_PORT}
+      - SMTP_SECURE=${SMTP_PROTOCOL}
+      - SMTP_NAME=${SMTP_USER}
+      - SMTP_PASSWORD=${SMTP_PASSWORD}
+      - MAIL_FROM_ADDRESS=${SMTP_LOCAL_PART}
+      - MAIL_DOMAIN=${SMTP_DOMAIN}
+      # PHP-specific settings
+      - PHP_UPLOAD_LIMIT=${NEXTCLOUD_UPLOAD_LIMIT-512M}
+      - PHP_MEMORY_LIMIT=${NEXTCLOUD_MEMORY_LIMIT-512M}
+      # Proxy-specific settings
+      - OVERWRITEPROTOCOL=https
     networks:
       - dd_net
 
   dd-apps-nextcloud-nginx:
-    image: ${NGINX_IMG-nginx:1.21.6}
+    image: ${NGINX_ALPINE_IMG-nginx:1.22.1-alpine}
     container_name: dd-apps-nextcloud-nginx
     restart: unless-stopped
     links:
       - dd-apps-nextcloud-app
     volumes:
+      # Customised nginx config
       - ${BUILD_APPS_ROOT_PATH}/docker/nextcloud/nginx.conf:/etc/nginx/nginx.conf:ro
+      # By mounting these, nginx can serve static files when possible
       - ${SRC_FOLDER}/nextcloud:/var/www/html:ro
+      - ${BUILD_APPS_ROOT_PATH}/docker/nextcloud/src/themes/dd:/var/www/html/themes/dd:ro
+    # This image does not require access to any environment variables
+    environment: []
     healthcheck:
         test: |
           curl -L --max-redirs 1 \

dd-apps/docker/nextcloud/nginx.conf

@@ -20,45 +20,42 @@ http {
     access_log  /var/log/nginx/access.log  main;
 
     sendfile        on;
+    #tcp_nopush     on;
+
+    # Prevent nginx HTTP Server Detection
+    server_tokens   off;
+
     keepalive_timeout  65;
 
+    #gzip  on;
+
+    # Pass Real IP to Nginx from proxy
     set_real_ip_from  10.0.0.0/8;
     set_real_ip_from  172.16.0.0/12;
     set_real_ip_from  192.168.0.0/16;
     real_ip_header    X-Forwarded-For;
+
     upstream php-handler {
+        # Pass request to DD container
         server dd-apps-nextcloud-app:9000;
     }
 
     server {
         listen 80;
-        add_header Referrer-Policy "no-referrer" always;
-        add_header X-Content-Type-Options "nosniff" always;
-        add_header X-Download-Options "noopen" always;
-        add_header X-Frame-Options "SAMEORIGIN" always;
-        add_header X-Permitted-Cross-Domain-Policies "none" always;
-        add_header X-Robots-Tag "none" always;
-        add_header X-XSS-Protection "1; mode=block" always;
-        fastcgi_hide_header X-Powered-By;
-        root /var/www/html;
 
-        location = /robots.txt {
+        # HSTS settings
-            allow all;
+        # WARNING: Only add the preload option once you read about
-            log_not_found off;
+        # the consequences in https://hstspreload.org/. This option
-            access_log off;
+        # will add the domain to a hardcoded list that is shipped
-        }
+        # in all major browsers and getting removed from this list
-
+        # could take several months.
-        location = /.well-known/carddav {
+        #add_header Strict-Transport-Security "max-age=15768000; includeSubDomains; preload;" always;
-            return 301 $scheme://$host/remote.php/dav;
-        }
-
-        location = /.well-known/caldav {
-            return 301 $scheme://$host/remote.php/dav;
-        }
 
+        # set max upload size
         client_max_body_size 10G;
         fastcgi_buffers 64 4K;
 
+        # Enable gzip but do not remove ETag headers
         gzip on;
         gzip_vary on;
         gzip_comp_level 4;
@@ -66,53 +63,120 @@ http {
         gzip_proxied expired no-cache no-store private no_last_modified no_etag auth;
         gzip_types application/atom+xml application/javascript application/json application/ld+json application/manifest+json application/rss+xml application/vnd.geo+json application/vnd.ms-fontobject application/x-font-ttf application/x-web-app-manifest+json application/xhtml+xml application/xml font/opentype image/bmp image/svg+xml image/x-icon text/cache-manifest text/css text/plain text/vcard text/vnd.rim.location.xloc text/vtt text/x-component text/x-cross-domain-policy;
 
-        location / {
+        # Pagespeed is not supported by Nextcloud, so if your server is built
-            rewrite ^ /index.php;
+        # with the `ngx_pagespeed` module, uncomment this line to disable it.
+        #pagespeed off;
+
+        # HTTP response headers borrowed from Nextcloud `.htaccess`
+        add_header Referrer-Policy                      "no-referrer"   always;
+        add_header X-Content-Type-Options               "nosniff"       always;
+        add_header X-Download-Options                   "noopen"        always;
+        add_header X-Frame-Options                      "SAMEORIGIN"    always;
+        add_header X-Permitted-Cross-Domain-Policies    "none"          always;
+        add_header X-Robots-Tag                         "none"          always;
+        add_header X-XSS-Protection                     "1; mode=block" always;
+
+        # Remove X-Powered-By, which is an information leak
+        fastcgi_hide_header X-Powered-By;
+
+        # Path to the root of your installation
+        root /var/www/html;
+
+        # Specify how to handle directories -- specifying `/index.php$request_uri`
+        # here as the fallback means that Nginx always exhibits the desired behaviour
+        # when a client requests a path that corresponds to a directory that exists
+        # on the server. In particular, if that directory contains an index.php file,
+        # that file is correctly served; if it doesn't, then the request is passed to
+        # the front-end controller. This consistent behaviour means that we don't need
+        # to specify custom rules for certain paths (e.g. images and other assets,
+        # `/updater`, `/ocm-provider`, `/ocs-provider`), and thus
+        # `try_files $uri $uri/ /index.php$request_uri`
+        # always provides the desired behaviour.
+        index index.php index.html /index.php$request_uri;
+
+        # Rule borrowed from `.htaccess` to handle Microsoft DAV clients
+        location = / {
+            if ( $http_user_agent ~ ^DavClnt ) {
+                return 302 /remote.php/webdav/$is_args$args;
+            }
         }
 
-        location ~ ^\/(?:build|tests|config|lib|3rdparty|templates|data)\/ {
+        location = /robots.txt {
-            deny all;
+            allow all;
-        }
+            log_not_found off;
-        location ~ ^\/(?:\.|autotest|occ|issue|indie|db_|console) {
+            access_log off;
-            deny all;
         }
 
-        location ~ ^\/(?:index|remote|public|cron|core\/ajax\/update|status|ocs\/v[12]|updater\/.+|oc[ms]-provider\/.+)\.php(?:$|\/) {
+        # Make a regex exception for `/.well-known` so that clients can still
-            fastcgi_split_path_info ^(.+?\.php)(\/.*|)$;
+        # access it despite the existence of the regex rule
+        # `location ~ /(\.|autotest|...)` which would otherwise handle requests
+        # for `/.well-known`.
+        location ^~ /.well-known {
+            # The rules in this block are an adaptation of the rules
+            # in `.htaccess` that concern `/.well-known`.
+
+            # TODO: Test if this works fine with port / host / etc.
+            location = /.well-known/carddav { return 301 /remote.php/dav/; }
+            location = /.well-known/caldav  { return 301 /remote.php/dav/; }
+
+            location /.well-known/acme-challenge    { try_files $uri $uri/ =404; }
+            location /.well-known/pki-validation    { try_files $uri $uri/ =404; }
+
+            # Let Nextcloud's API for `/.well-known` URIs handle all other
+            # requests by passing them to the front-end controller.
+            return 301 /index.php$request_uri;
+        }
+
+        # Rules borrowed from `.htaccess` to hide certain paths from clients
+        location ~ ^/(?:build|tests|config|lib|3rdparty|templates|data)(?:$|/)  { return 404; }
+        location ~ ^/(?:\.|autotest|occ|issue|indie|db_|console)                { return 404; }
+
+        # Ensure this block, which passes PHP files to the PHP process, is above the blocks
+        # which handle static assets (as seen below). If this block is not declared first,
+        # then Nginx will encounter an infinite rewriting loop when it prepends `/index.php`
+        # to the URI, resulting in a HTTP 500 error response.
+        location ~ \.php(?:$|/) {
+            # Required for legacy support
+            rewrite ^/(?!index|remote|public|cron|core\/ajax\/update|status|ocs\/v[12]|updater\/.+|oc[ms]-provider\/.+|.+\/richdocumentscode\/proxy) /index.php$request_uri;
+
+            fastcgi_split_path_info ^(.+?\.php)(/.*)$;
             set $path_info $fastcgi_path_info;
+
             try_files $fastcgi_script_name =404;
+
             include fastcgi_params;
             fastcgi_param SCRIPT_FILENAME $document_root$fastcgi_script_name;
             fastcgi_param PATH_INFO $path_info;
-            fastcgi_param modHeadersAvailable true;
+            #fastcgi_param HTTPS on;
-            fastcgi_param front_controller_active true;
+
+            fastcgi_param modHeadersAvailable true;         # Avoid sending the security headers twice
+            fastcgi_param front_controller_active true;     # Enable pretty urls
             fastcgi_pass php-handler;
+
             fastcgi_intercept_errors on;
             fastcgi_request_buffering off;
             fastcgi_param SERVER_NAME $host;
         }
 
-        location ~ ^\/(?:updater|oc[ms]-provider)(?:$|\/) {
+        location ~ \.(?:css|js|svg|gif|map)$ {
-            try_files $uri/ =404;
+            try_files $uri /index.php$request_uri;
-            index index.php;
+            expires 6M;         # Cache-Control policy borrowed from `.htaccess`
+            access_log off;     # Optional: Don't log access to assets
         }
 
-        location ~ \.(?:css|js|woff2?|svg|gif|map)$ {
+        location ~ \.(?:woff2?|png|html|ttf|ico|jpg|jpeg|bcmap|mp4|webm)$ {
             try_files $uri /index.php$request_uri;
-            add_header Cache-Control "public, max-age=15778463";
+            expires 7d;         # Cache-Control policy borrowed from `.htaccess`
-            add_header Referrer-Policy "no-referrer" always;
+            access_log off;     # Optional: Don't log access to assets
-            add_header X-Content-Type-Options "nosniff" always;
-            add_header X-Download-Options "noopen" always;
-            add_header X-Frame-Options "SAMEORIGIN" always;
-            add_header X-Permitted-Cross-Domain-Policies "none" always;
-            add_header X-Robots-Tag "none" always;
-            add_header X-XSS-Protection "1; mode=block" always;
-            access_log off;
         }
 
-        location ~ \.(?:png|html|ttf|ico|jpg|jpeg|bcmap|mp4|webm)$ {
+        # Rule borrowed from `.htaccess`
-            try_files $uri /index.php$request_uri;
+        location /remote {
-            access_log off;
+            return 301 /remote.php$request_uri;
+        }
+
+        location / {
+            try_files $uri $uri/ /index.php$request_uri;
         }
     }
 }

dd-apps/docker/nextcloud/supervisord.conf

@@ -3,8 +3,8 @@ nodaemon=true
 logfile=/var/log/supervisord/supervisord.log
 pidfile=/var/run/supervisord/supervisord.pid
 childlogdir=/var/log/supervisord/
-logfile_maxbytes=50MB
+logfile_maxbytes=50MB                           ; maximum size of logfile before rotation
-logfile_backups=10
+logfile_backups=10                              ; number of backed up logfiles
 loglevel=error
 
 [program:php-fpm]
@@ -19,4 +19,4 @@ stdout_logfile=/dev/stdout
 stdout_logfile_maxbytes=0
 stderr_logfile=/dev/stderr
 stderr_logfile_maxbytes=0
-command=/cron.sh
+command=/cron.sh

dd-apps/docker/postgresql/postgresql.yml

@@ -23,10 +23,23 @@ services:
     image: ${POSTGRESQL_IMG-postgres:13.5-alpine3.15}
     container_name: dd-apps-postgresql
     restart: unless-stopped
-    env_file: .env
     environment:
+      # Postgres
       - POSTGRES_PASSWORD=${POSTGRES_PASSWORD}
       - POSTGRES_USER=${POSTGRES_USER}
+      # Etherpad
+      - ETHERPAD_POSTGRES_USER=${ETHERPAD_POSTGRES_USER:-etherpard}
+      - ETHERPAD_POSTGRES_PASSWORD=${ETHERPAD_POSTGRES_PASSWORD}
+      # Keycloak
+      - KEYCLOAK_DB_DATABASE=${KEYCLOAK_DB_DATABASE:-keycloak}
+      - KEYCLOAK_DB_USER=${KEYCLOAK_DB_USER:-keycloak}
+      - KEYCLOAK_DB_PASSWORD=${KEYCLOAK_DB_PASSWORD}
+      # Moodle
+      - MOODLE_POSTGRES_USER=${MOODLE_POSTGRES_USER:-moodle}
+      - MOODLE_POSTGRES_PASSWORD=${MOODLE_POSTGRES_PASSWORD}
+      # Nextcloud
+      - NEXTCLOUD_POSTGRES_USER=${NEXTCLOUD_POSTGRES_USER:-nextcloud}
+      - NEXTCLOUD_POSTGRES_PASSWORD=${NEXTCLOUD_POSTGRES_PASSWORD}
     volumes:
       - /etc/localtime:/etc/localtime:ro
       - ${DB_FOLDER}/postgres:/var/lib/postgresql/data

dd-apps/docker/wordpress/wordpress.yml

@@ -36,6 +36,7 @@ services:
     volumes:
       *wordpress-volumes
     environment:
+      DOMAIN: ${DOMAIN}
       WORDPRESS_DB_HOST: dd-apps-mariadb
       WORDPRESS_DB_NAME: "wordpress"
       WORDPRESS_DB_USER: ${WORDPRESS_MARIADB_USER}

dd-ctl

@@ -112,9 +112,11 @@ fi
 REPO_BRANCH="${1:-main}"
 
 
-cp dd.conf .env
 CUSTOM_PATH=$(pwd)
-. ./.env
+if [ -f dd.conf ]; then
+	cp dd.conf .env
+	. ./.env
+fi
 
 prerequisites_docker(){
 	# Remove uncompatible docker packages
@@ -175,14 +177,23 @@ ddupdate(){
 }
 
 build_compose(){
-	DD_DEFAULT_BUILD="$(git rev-parse --short HEAD)"
+	DD_DEFAULT_BUILD="$(git rev-parse --short=8 HEAD)"
 	export DD_BUILD="${DD_BUILD:-${DD_DEFAULT_BUILD}}"
 	setconf DD_BUILD "${DD_BUILD}" .env
 
 	setconf CUSTOM_PATH "$CUSTOM_PATH" .env
+
 	setconf BUILD_APPS_ROOT_PATH "$CUSTOM_PATH/dd-apps" .env
 	setconf BUILD_SSO_ROOT_PATH "$CUSTOM_PATH/dd-sso" .env
 
+	# SMTP (needed for e.g. Nextcloud's more granular settings)
+	SMTP_LOCAL_PART="$(echo "${SMTP_USER:-}" | cut -d '@' -f 1)"
+	export SMTP_LOCAL_PART
+	SMTP_DOMAIN="$(echo "${SMTP_USER:-}" | cut -d '@' -f 2)"
+	export SMTP_DOMAIN
+	setconf SMTP_LOCAL_PART "${SMTP_LOCAL_PART}" .env
+	setconf SMTP_DOMAIN "${SMTP_DOMAIN}" .env
+
 	# Choose HAProxy configuration flavour
 	if [ "${PROXY_PROTOCOL:-false}" = "true" ]; then
 		HAPROXY_YML="haproxy.proxy.yml"
@@ -207,7 +218,7 @@ build_compose(){
 	setconf HAPROXY_CFG "${HAPROXY_CFG}"
 	setconf HAPROXY_CFG "${HAPROXY_CFG}" .env
 
-	# Enable or disable ClamAV
+	# Enable or disable ClamAV's container
 	if [ "${DISABLE_CLAMAV:-true}" = "true" ]; then
 		# Current default (might change)
 		CLAMAV_YML="clamav.disabled.yml"
@@ -215,18 +226,16 @@ build_compose(){
 		CLAMAV_YML="clamav.yml"
 	fi
 
-	## Prepare apps environment
+	# Clean up redundant .env files
-	ln -sf "${CUSTOM_PATH}/.env" dd-apps/.env
+	for f in	dd-apps \
-	ln -sf "${CUSTOM_PATH}/.env" dd-apps/docker/postgresql && \
+			dd-apps/docker/postgresql dd-apps/docker/mariadb \
-	ln -sf "${CUSTOM_PATH}/.env" dd-apps/docker/mariadb && \
+			dd-apps/docker/moodle dd-apps/docker/nextcloud \
-	ln -sf "${CUSTOM_PATH}/.env" dd-apps/docker/moodle && \
+			dd-apps/docker/wordpress dd-apps/docker/etherpad \
-	ln -sf "${CUSTOM_PATH}/.env" dd-apps/docker/nextcloud && \
+			dd-sso dd-sso/docker-compose-parts; do
-	ln -sf "${CUSTOM_PATH}/.env" dd-apps/docker/wordpress && \
+		if [ -f "${f}/.env" ]; then
-	ln -sf "${CUSTOM_PATH}/.env" dd-apps/docker/etherpad
+			rm "${f}/.env"
-
+		fi
-	## Prepare sso environment
+	done
-	ln -sf "${CUSTOM_PATH}/.env" dd-sso/.env
-	ln -sf "${CUSTOM_PATH}/.env" dd-sso/docker-compose-parts/.env
 
 	# Clean up older custom data
 	rm -rf custom/system/keycloak-themes
@@ -234,12 +243,13 @@ build_compose(){
 
 	# Build compose ymls
 	docker-compose \
+			--env-file "${CUSTOM_PATH}/.env" \
+					\
 			-f "dd-sso/docker-compose-parts/$WAF_YML" \
 			-f "dd-sso/docker-compose-parts/$HAPROXY_YML"\
 			-f dd-sso/docker-compose-parts/api.yml \
 			-f dd-sso/docker-compose-parts/keycloak.yml \
 			-f dd-sso/docker-compose-parts/avatars.yml \
-			-f dd-apps/docker/postgresql/postgresql.yml \
 			-f dd-sso/docker-compose-parts/admin.yml \
 					\
 			-f dd-apps/docker/moodle/moodle.yml \
@@ -287,7 +297,8 @@ genpatches(){
 
 build(){
 	build_compose
-	docker-compose build --pull
+	docker-compose pull --ignore-pull-failures --include-deps
+	docker-compose build
 }
 
 build_compose_develop(){
@@ -356,20 +367,18 @@ setup_nextcloud(){
 		EOF
 	fi
 
+	#docker exec -u www-data dd-apps-nextcloud-app php occ app:install user_saml
+	docker exec -u www-data dd-apps-nextcloud-app php occ app:enable user_saml
+
+	# Installing apps may require an occ upgrade
+	nextcloud_upgrade
+
 	# Temporary patch while upstream lands our changes
 	# See: https://github.com/nextcloud/mail/pull/6908
 	for f in appinfo/info.xml lib/Command/UpdateAccount.php lib/Db/MailAccountMapper.php; do
 		install -m 0644 -o 82 -g 82 "dd-apps/docker/nextcloud/nc_mail/$f" "${SRC_FOLDER}/nextcloud/custom_apps/mail/$f"
 	done
 
-	# Custom forms
-	docker exec dd-apps-nextcloud-app apk add git npm composer
-	docker exec -u www-data dd-apps-nextcloud-app rm -rf /var/www/html/custom_apps/forms
-	docker exec -u www-data dd-apps-nextcloud-app git clone https://github.com/juanan3ip/form -b dev /var/www/html/custom_apps/forms
-	docker exec -u www-data dd-apps-nextcloud-app npm --prefix /var/www/html/custom_apps/forms install
-	docker exec -u www-data dd-apps-nextcloud-app composer -d/var/www/html/custom_apps/forms install --no-dev -o
-	docker exec -u www-data dd-apps-nextcloud-app php occ app:enable forms
-
 	# Disable Big Blue Button media check by default
 	docker exec -u www-data dd-apps-nextcloud-app php occ config:app:set bbb join.mediaCheck --value="false"
 	# Disable Big Blue Button listen only mode by default
@@ -384,11 +393,8 @@ setup_nextcloud(){
 	docker exec -u www-data dd-apps-nextcloud-app php occ --no-warnings config:system:set allow_local_remote_servers  --value=true
 	docker exec -u www-data dd-apps-nextcloud-app php occ --no-warnings maintenance:theme:update
 
-	#docker exec -u www-data dd-apps-nextcloud-app php occ app:install user_saml
-	docker exec -u www-data dd-apps-nextcloud-app php occ app:enable user_saml
 	docker exec dd-apps-nextcloud-app apk add jq
 	docker exec dd-apps-nextcloud-app sh -c 'jq ". + {\"pad\": [\"application/x-ownpad\"], \"calc\": [\"application/x-ownpad\"]}" /var/www/html/resources/config/mimetypemapping.dist.json > /var/www/html/config/mimetypemapping.json'
-	nextcloud_scan
 	# Open pads in a new tab/window
 	docker exec dd-apps-nextcloud-app sed -i.orig 's/^\(\s*\)\(var viewer = OC.generateUrl.*\)/\1\2\n\1window.open(viewer);\n\1return;/' /var/www/html/custom_apps/ownpad/js/ownpad.js
 
@@ -472,16 +478,25 @@ setup_nextcloud(){
 	# Add default file for moodle activities
 	if [ ! -f "$DATA_FOLDER/nextcloud/admin/files/template.docx" ]; then
 		cp dd-apps/docker/nextcloud/template.docx "$DATA_FOLDER/nextcloud/admin/files/"
-		nextcloud_scan
 	fi
 
-	configure_nextcloud_logo
+	# Custom forms
+	## This may be forcing the need for occ upgrade in the past
+	## Keep it towards the end
+	docker exec dd-apps-nextcloud-app apk add git npm composer
+	docker exec -u www-data dd-apps-nextcloud-app rm -rf /var/www/html/custom_apps/forms
+	docker exec -u www-data dd-apps-nextcloud-app git clone https://github.com/juanan3ip/form -b dev /var/www/html/custom_apps/forms
+	docker exec -u www-data dd-apps-nextcloud-app npm --prefix /var/www/html/custom_apps/forms install
+	docker exec -u www-data dd-apps-nextcloud-app composer -d/var/www/html/custom_apps/forms install --no-dev -o
+	docker exec -u www-data dd-apps-nextcloud-app php occ app:enable forms
 
 	nextcloud_upgrade
+	configure_nextcloud_logo
+	nextcloud_scan
 }
 
 nextcloud_upgrade(){
-	docker-compose exec -u www-data dd-apps-nextcloud-app ./occ upgrade
+	docker exec -i -u www-data dd-apps-nextcloud-app ./occ upgrade
 }
 
 nextcloud_scan(){
@@ -879,10 +894,10 @@ securize() {
 			ETHERPAD_POSTGRES_PASSWORD \
 			ETHERPAD_ADMIN_PASSWORD \
 			WORDPRESS_MARIADB_PASSWORD \
-			WORDPRESS_ADMIN_PASSWORD \
+			WORDPRESS_ADMIN_PASSWORD; do
-			IPA_ADMIN_PWD; do
 		setconf "${dd_var}" "$(genpwd)"
 	done
+	setconf "API_SECRET" "$(openssl rand -base64 32)"
 }
 
 setconf() {

dd-sso/admin/docker/requirements.pip3

@@ -23,6 +23,10 @@ Flask==2.1.3
 Flask-Login==0.6.2
 eventlet==0.33.1
 Flask-SocketIO==5.2.0
+# Flask-SocketIO depends on dnspython
+# but dnspython 2.3 removes dns.rdtypes.ANY, which is needed by Flask-SocketIO
+# so we keep it below version 2.3
+dnspython<2.3
 flasgger==0.9.5
 bcrypt==3.2.2
 # diceware can't be upgraded without issues

dd-sso/admin/src/admin/lib/nextcloud.py

@@ -275,10 +275,13 @@ class Nextcloud:
                     self._request("PUT", url, data=data, headers=headers)
                 )
                 if result["ocs"]["meta"]["statuscode"] == 102:
-                    raise ProviderItemExists
+                    if result["ocs"]["meta"].get("message") != "Invalid displayname":
-                if result["ocs"]["meta"]["statuscode"] == 104:
+                        # Workaround for https://github.com/nextcloud/server/issues/33751
+                        # Fixed on NC 25
+                        raise ProviderItemExists
+                elif result["ocs"]["meta"]["statuscode"] == 104:
                     raise ProviderGroupNotExists
-                if result["ocs"]["meta"]["statuscode"] != 100:
+                elif result["ocs"]["meta"]["statuscode"] != 100:
                     log.error("Get Nextcloud provider user add error: " + str(result))
                     raise ProviderOpError
             except:

dd-sso/docker-compose-parts/admin.yml

@@ -39,8 +39,6 @@ services:
     - ${DATA_FOLDER}/legal:/admin/admin/static/templates/pages/legal:rw
     - ${DATA_FOLDER}/dd-admin:/data:rw
     - ${DATA_FOLDER}/nc-mail-queue:/nc-mail-queue:rw
-    env_file:
-      - .env
     environment:
       - VERIFY="false"  # In development do not verify certificates
       - DOMAIN=${DOMAIN}
@@ -53,3 +51,5 @@ services:
       - AVATARS_SERVER_HOST=dd-sso-avatars:9000
       - AVATARS_ACCESS_KEY=${AVATARS_ACCESS_KEY:-AKIAIOSFODNN7EXAMPLE}
       - AVATARS_SECRET_KEY=${AVATARS_SECRET_KEY:-wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY}
+    # TODO: Check missing env vars and pass them on the environment
+    env_file: ${CUSTOM_PATH}/.env

dd-sso/docker-compose-parts/api.yml

@@ -28,6 +28,10 @@ services:
         DOMAIN: $DOMAIN
         NGINX_ALPINE_IMG: ${NGINX_ALPINE_IMG-nginx:1.21.6-alpine}
     container_name: dd-sso-api
+    environment:
+      DOMAIN: ${DOMAIN}
+      KEYCLOAK_USER: ${KEYCLOAK_USER}
+      KEYCLOAK_PASSWORD: ${KEYCLOAK_PASSWORD}
     volumes:
       - /etc/localtime:/etc/localtime:ro
       - ${CUSTOM_PATH}/custom/menu:/api/menu
@@ -37,8 +41,3 @@ services:
     restart: unless-stopped
     networks:
       - dd_net
-    # ports:
-    # - published: 7039
-    #   target: 7039
-    env_file:
-      - .env

dd-sso/docker-compose-parts/haproxy.no-ports.yml

@@ -28,6 +28,12 @@ services:
       dockerfile: Dockerfile
       target: production
     container_name: dd-sso-haproxy
+    environment:
+      DOMAIN: ${DOMAIN}
+      HAPROXY_CFG: ${HAPROXY_CFG:-haproxy.no-waf.no-haproxy.cfg}
+      LETSENCRYPT_DOMAIN: ${LETSENCRYPT_DOMAIN}
+      LETSENCRYPT_DOMAIN_ROOT: ${LETSENCRYPT_DOMAIN_ROOT}
+      LETSENCRYPT_EMAIL: ${LETSENCRYPT_EMAIL}
     restart: unless-stopped
     volumes:
       - /etc/localtime:/etc/localtime:ro
@@ -35,8 +41,6 @@ services:
       - ${SRC_FOLDER}/haproxy/certs:/certs:rw
     networks:
       - dd_net
-    env_file:
-      - .env
     logging:
       driver: "json-file"
       options:

dd-sso/docker-compose-parts/haproxy.proxy.yml

@@ -28,6 +28,12 @@ services:
       dockerfile: Dockerfile
       target: production
     container_name: dd-sso-haproxy
+    environment:
+      DOMAIN: ${DOMAIN}
+      HAPROXY_CFG: ${HAPROXY_CFG:-haproxy.no-waf.no-haproxy.cfg}
+      LETSENCRYPT_DOMAIN: ${LETSENCRYPT_DOMAIN}
+      LETSENCRYPT_DOMAIN_ROOT: ${LETSENCRYPT_DOMAIN_ROOT}
+      LETSENCRYPT_EMAIL: ${LETSENCRYPT_EMAIL}
     restart: unless-stopped
     volumes:
       - /etc/localtime:/etc/localtime:ro
@@ -45,8 +51,6 @@ services:
       target: 8888
     - published: 591
       target: 591
-    env_file:
-      - .env
     logging:
       driver: "json-file"
       options:

dd-sso/docker-compose-parts/haproxy.yml

@@ -28,6 +28,12 @@ services:
       dockerfile: Dockerfile
       target: production
     container_name: dd-sso-haproxy
+    environment:
+      DOMAIN: ${DOMAIN}
+      HAPROXY_CFG: ${HAPROXY_CFG:-haproxy.no-waf.no-haproxy.cfg}
+      LETSENCRYPT_DOMAIN: ${LETSENCRYPT_DOMAIN}
+      LETSENCRYPT_DOMAIN_ROOT: ${LETSENCRYPT_DOMAIN_ROOT}
+      LETSENCRYPT_EMAIL: ${LETSENCRYPT_EMAIL}
     restart: unless-stopped
     volumes:
       - /etc/localtime:/etc/localtime:ro
@@ -40,8 +46,6 @@ services:
       target: 80
     - published: 443
       target: 443
-    env_file:
-      - .env
     logging:
       driver: "json-file"
       options:

dd-sso/docker-compose-parts/postgresql.yml

@@ -1,38 +0,0 @@
-#
-#   Copyright © 2021,2022 IsardVDI S.L.
-#
-#   This file is part of DD
-#
-#   DD is free software: you can redistribute it and/or modify
-#   it under the terms of the GNU Affero General Public License as published by
-#   the Free Software Foundation, either version 3 of the License, or (at your
-#   option) any later version.
-#
-#   DD is distributed in the hope that it will be useful, but WITHOUT ANY
-#   WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
-#   FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for more
-#   details.
-#
-#   You should have received a copy of the GNU Affero General Public License
-#   along with DD. If not, see <https://www.gnu.org/licenses/>.
-#
-# SPDX-License-Identifier: AGPL-3.0-or-later
-version: '3.7'
-services:
-  dd-sso-postgresql:
-    image: ${POSTGRESQL_IMG-postgres:13.5-alpine3.15}
-    container_name: dd-sso-postgresql
-    restart: unless-stopped
-    env_file:
-      - .env
-    volumes:
-      - /etc/localtime:/etc/localtime:ro
-      - ${DB_FOLDER}/postgres:/var/lib/postgresql/data
-      - ${BUILD_SSO_ROOT_PATH}/init/databases:/docker-entrypoint-initdb.d
-    networks:
-      - dd_net
-    logging:
-      driver: "json-file"
-      options:
-        max-size: "5m"
-        max-file: "10"

dd-sso/docker-compose-parts/waf-modsecurity.yml

@@ -7,8 +7,8 @@ services:
       dockerfile: Dockerfile
       target: production
     container_name: dd-waf-apache
-    env_file:
+    environment:
-      - .env
+      DISABLE_WAF: ${DISABLE_WAF:-true}
     restart: unless-stopped
     volumes:
       - /etc/localtime:/etc/localtime:ro

dd-sso/docker/api/requirements.pip3

@@ -20,4 +20,8 @@
 Flask==2.0.1
 eventlet==0.33.0
 Flask-SocketIO==5.1.0
-python-keycloak==0.26.1
+# Flask-SocketIO depends on dnspython
+# but dnspython 2.3 removes dns.rdtypes.ANY, which is needed by Flask-SocketIO
+# so we keep it below version 2.3
+dnspython<2.3
+python-keycloak==0.26.1

dd.conf.sample

@@ -61,9 +61,9 @@ SMTP_PASSWORD=SuperSecret
 SMTP_PROTOCOL=tls
 
 ## DEFAULT CUSTOM ROLE NAMES
-CUSTOM_ROLE_MANAGER='manager'
+#CUSTOM_ROLE_MANAGER='manager'
-CUSTOM_ROLE_TEACHER='teacher'
+#CUSTOM_ROLE_TEACHER='teacher'
-CUSTOM_ROLE_STUDENT='student'
+#CUSTOM_ROLE_STUDENT='student'
 
 
 DDADMIN_USER=ddadmin
@@ -73,7 +73,7 @@ DDADMIN_EMAIL=theemail@mymailserver.com
 # ------ Api Secret -----------------------------------------------------------
 ## Generate your own SECRET! (or apply securize script)
 ## openssl rand -base64 32
-API_SECRET=LYY1kVYzbTSQx1yC4AauY7R6X34Jaz6+SY8CNC6RSno=
+#API_SECRET=LYY1kVYzbTSQx1yC4AauY7R6X34Jaz6+SY8CNC6RSno=
 
 ## ADMINAPP (https://admin.$DOMAIN)
 ##=============================================================================
@@ -130,22 +130,16 @@ ETHERPAD_POSTGRES_USER=etherpad
 ETHERPAD_POSTGRES_PASSWORD=3th3rpad
 ### ETHERPAD_API_KEY=NotImplemented
 
-## POSTGRES (https://sso.$DOMAIN/dd-sso-adminer)
+## POSTGRES
 ##=============================================================================
-### The adminer user/pwd is admin/$KEYCLOAK_PASSWORD
 POSTGRES_USER=admin
 POSTGRES_PASSWORD=postgrespostgres
 
-## MARIADB (https://sso.$DOMAIN/dd-sso-adminer)
+## MARIADB
 ##=============================================================================
-### The adminer user/pwd is admin/$KEYCLOAK_PASSWORD
 ### MARIADB_USER=root (it is the defaults in the container
 MARIADB_PASSWORD=SuperSecret
 
-## FREEIPA (disabled)
-##=============================================================================
-IPA_ADMIN_PWD=freeipafreeipa
-
 ## ACCEPT PROXY PROTOCOL ON 8888 (HTTP) AND 561 (HTTPS)
 #PROXY_PROTOCOL=false
 
@@ -162,7 +156,9 @@ MOODLE_MAX_EXECUTION_TIME=300
 MOODLE_MEMORY_LIMIT=128M
 
 ## NEXTCLOUD
-NEXTCLOUD_IMG=nextcloud:21.0.5-fpm-alpine
+# If enabled overrides nextcloud.yaml default image.
+# You usually do not want to change this, but instead rely on DD's version.
+#NEXTCLOUD_IMG_OVERRIDE=nextcloud:24.0.8-fpm-alpine
 
 ## ONLYOFFICE
 ONLYOFFICE_IMG=alehoho/oo-ce-docker-license:6.3.1.32
@@ -185,8 +181,7 @@ POSTGRESQL_IMG=postgres:14.1-alpine3.15
 #MARIADB_IMG=mariadb:10.6.5
 
 ## NGINX
-#NGINX_IMG=nginx:1.21.6
+#NGINX_ALPINE_IMG=nginx:1.22.1-alpine
-#NGINX_ALPINE_IMG=nginx:1.21.6-alpine
 
 ## REDIS
 #REDIS_IMG=redis:6.2.6-alpine3.15

docs/updating.md

@@ -0,0 +1 @@
+../UPDATING.md

mkdocs.yml

@@ -58,6 +58,7 @@ nav:
 - index.ca.md
 - index.es.md
 - install.ca.md
+- updating.md
 - customising.ca.md
 - integrations.ca.md
 - post-install.ca.md