From ac789f8d6c9e12d240b1025259fa9359a1ed078a Mon Sep 17 00:00:00 2001 From: Evilham Date: Sat, 24 Dec 2022 13:10:01 +0100 Subject: [PATCH 01/15] [dd-ctl] Move NC forms towards the end This may be causing issues when the plugin requires occ upgrade Alternatively, installing plugins may require it. While there, also remove one of the nextcloud_scan calls and delay it along with logo customisation until after a potential ./occ upgrade has taken place. --- dd-ctl | 32 ++++++++++++++++++-------------- 1 file changed, 18 insertions(+), 14 deletions(-) diff --git a/dd-ctl b/dd-ctl index 42cde0a..bf967d6 100755 --- a/dd-ctl +++ b/dd-ctl @@ -356,20 +356,18 @@ setup_nextcloud(){ EOF fi + #docker exec -u www-data dd-apps-nextcloud-app php occ app:install user_saml + docker exec -u www-data dd-apps-nextcloud-app php occ app:enable user_saml + + # Installing apps may require an occ upgrade + nextcloud_upgrade + # Temporary patch while upstream lands our changes # See: https://github.com/nextcloud/mail/pull/6908 for f in appinfo/info.xml lib/Command/UpdateAccount.php lib/Db/MailAccountMapper.php; do install -m 0644 -o 82 -g 82 "dd-apps/docker/nextcloud/nc_mail/$f" "${SRC_FOLDER}/nextcloud/custom_apps/mail/$f" done - # Custom forms - docker exec dd-apps-nextcloud-app apk add git npm composer - docker exec -u www-data dd-apps-nextcloud-app rm -rf /var/www/html/custom_apps/forms - docker exec -u www-data dd-apps-nextcloud-app git clone https://github.com/juanan3ip/form -b dev /var/www/html/custom_apps/forms - docker exec -u www-data dd-apps-nextcloud-app npm --prefix /var/www/html/custom_apps/forms install - docker exec -u www-data dd-apps-nextcloud-app composer -d/var/www/html/custom_apps/forms install --no-dev -o - docker exec -u www-data dd-apps-nextcloud-app php occ app:enable forms - # Disable Big Blue Button media check by default docker exec -u www-data dd-apps-nextcloud-app php occ config:app:set bbb join.mediaCheck --value="false" # Disable Big Blue Button listen only mode by default @@ -384,11 +382,8 @@ setup_nextcloud(){ docker exec -u www-data dd-apps-nextcloud-app php occ --no-warnings config:system:set allow_local_remote_servers --value=true docker exec -u www-data dd-apps-nextcloud-app php occ --no-warnings maintenance:theme:update - #docker exec -u www-data dd-apps-nextcloud-app php occ app:install user_saml - docker exec -u www-data dd-apps-nextcloud-app php occ app:enable user_saml docker exec dd-apps-nextcloud-app apk add jq docker exec dd-apps-nextcloud-app sh -c 'jq ". + {\"pad\": [\"application/x-ownpad\"], \"calc\": [\"application/x-ownpad\"]}" /var/www/html/resources/config/mimetypemapping.dist.json > /var/www/html/config/mimetypemapping.json' - nextcloud_scan # Open pads in a new tab/window docker exec dd-apps-nextcloud-app sed -i.orig 's/^\(\s*\)\(var viewer = OC.generateUrl.*\)/\1\2\n\1window.open(viewer);\n\1return;/' /var/www/html/custom_apps/ownpad/js/ownpad.js @@ -472,16 +467,25 @@ setup_nextcloud(){ # Add default file for moodle activities if [ ! -f "$DATA_FOLDER/nextcloud/admin/files/template.docx" ]; then cp dd-apps/docker/nextcloud/template.docx "$DATA_FOLDER/nextcloud/admin/files/" - nextcloud_scan fi - configure_nextcloud_logo + # Custom forms + ## This may be forcing the need for occ upgrade in the past + ## Keep it towards the end + docker exec dd-apps-nextcloud-app apk add git npm composer + docker exec -u www-data dd-apps-nextcloud-app rm -rf /var/www/html/custom_apps/forms + docker exec -u www-data dd-apps-nextcloud-app git clone https://github.com/juanan3ip/form -b dev /var/www/html/custom_apps/forms + docker exec -u www-data dd-apps-nextcloud-app npm --prefix /var/www/html/custom_apps/forms install + docker exec -u www-data dd-apps-nextcloud-app composer -d/var/www/html/custom_apps/forms install --no-dev -o + docker exec -u www-data dd-apps-nextcloud-app php occ app:enable forms nextcloud_upgrade + configure_nextcloud_logo + nextcloud_scan } nextcloud_upgrade(){ - docker-compose exec -u www-data dd-apps-nextcloud-app ./occ upgrade + docker exec -i -u www-data dd-apps-nextcloud-app ./occ upgrade } nextcloud_scan(){ From c7d172f916b80c415e078a10cdfeae653877955b Mon Sep 17 00:00:00 2001 From: Evilham Date: Fri, 23 Dec 2022 10:37:07 +0100 Subject: [PATCH 02/15] [dd-ctl] Improve image pulling before building Using this differently may have been triggering DockerHub rates --- dd-ctl | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/dd-ctl b/dd-ctl index bf967d6..c3471ce 100755 --- a/dd-ctl +++ b/dd-ctl @@ -287,7 +287,8 @@ genpatches(){ build(){ build_compose - docker-compose build --pull + docker-compose pull --ignore-pull-failures --include-deps + docker-compose build } build_compose_develop(){ From db4a6d14e9a05ee1b0334aaf87af31703a4ae4d0 Mon Sep 17 00:00:00 2001 From: Roger Garcia Date: Tue, 20 Dec 2022 17:21:55 +0100 Subject: [PATCH 03/15] Added new parameter IMG_NEXTCLOUD_OVERRIDE defaulting to nextcloud.yaml --- dd-apps/docker/nextcloud/nextcloud.yml | 2 +- dd.conf.sample | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/dd-apps/docker/nextcloud/nextcloud.yml b/dd-apps/docker/nextcloud/nextcloud.yml index d8c49f0..c3dda0e 100644 --- a/dd-apps/docker/nextcloud/nextcloud.yml +++ b/dd-apps/docker/nextcloud/nextcloud.yml @@ -24,7 +24,7 @@ services: context: ${BUILD_APPS_ROOT_PATH}/docker/nextcloud dockerfile: Dockerfile args: - - IMG=${NEXTCLOUD_IMG} + - IMG=${NEXTCLOUD_IMG_OVERRIDE-nextcloud:21.0.5-fpm-alpine} container_name: dd-apps-nextcloud-app restart: unless-stopped depends_on: diff --git a/dd.conf.sample b/dd.conf.sample index ef550ea..a45e39e 100644 --- a/dd.conf.sample +++ b/dd.conf.sample @@ -162,7 +162,8 @@ MOODLE_MAX_EXECUTION_TIME=300 MOODLE_MEMORY_LIMIT=128M ## NEXTCLOUD -NEXTCLOUD_IMG=nextcloud:21.0.5-fpm-alpine +# If enabled overrides nextcloud.yaml default image +#NEXTCLOUD_IMG_OVERRIDE=nextcloud:21.0.5-fpm-alpine ## ONLYOFFICE ONLYOFFICE_IMG=alehoho/oo-ce-docker-license:6.3.1.32 From cbb4e06e896da40430d66435589ab5ea8cf71420 Mon Sep 17 00:00:00 2001 From: Roger Garcia Date: Tue, 20 Dec 2022 17:21:55 +0100 Subject: [PATCH 04/15] Upgrade NC to version 22.2.10 --- dd-apps/docker/nextcloud/nextcloud.yml | 2 +- dd.conf.sample | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/dd-apps/docker/nextcloud/nextcloud.yml b/dd-apps/docker/nextcloud/nextcloud.yml index c3dda0e..dbbea39 100644 --- a/dd-apps/docker/nextcloud/nextcloud.yml +++ b/dd-apps/docker/nextcloud/nextcloud.yml @@ -24,7 +24,7 @@ services: context: ${BUILD_APPS_ROOT_PATH}/docker/nextcloud dockerfile: Dockerfile args: - - IMG=${NEXTCLOUD_IMG_OVERRIDE-nextcloud:21.0.5-fpm-alpine} + - IMG=${NEXTCLOUD_IMG_OVERRIDE-nextcloud:22.2.10-fpm-alpine} container_name: dd-apps-nextcloud-app restart: unless-stopped depends_on: diff --git a/dd.conf.sample b/dd.conf.sample index a45e39e..45b56e4 100644 --- a/dd.conf.sample +++ b/dd.conf.sample @@ -163,7 +163,7 @@ MOODLE_MEMORY_LIMIT=128M ## NEXTCLOUD # If enabled overrides nextcloud.yaml default image -#NEXTCLOUD_IMG_OVERRIDE=nextcloud:21.0.5-fpm-alpine +#NEXTCLOUD_IMG_OVERRIDE=nextcloud:22.2.10-fpm-alpine ## ONLYOFFICE ONLYOFFICE_IMG=alehoho/oo-ce-docker-license:6.3.1.32 From 57c87d7172579b8fbe0d8249f47ee0966621e252 Mon Sep 17 00:00:00 2001 From: Roger Garcia Date: Wed, 21 Dec 2022 09:48:14 +0100 Subject: [PATCH 05/15] Upgrade NC to version 23.0.11 --- dd-apps/docker/nextcloud/nextcloud.yml | 2 +- dd.conf.sample | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/dd-apps/docker/nextcloud/nextcloud.yml b/dd-apps/docker/nextcloud/nextcloud.yml index dbbea39..8880bfd 100644 --- a/dd-apps/docker/nextcloud/nextcloud.yml +++ b/dd-apps/docker/nextcloud/nextcloud.yml @@ -24,7 +24,7 @@ services: context: ${BUILD_APPS_ROOT_PATH}/docker/nextcloud dockerfile: Dockerfile args: - - IMG=${NEXTCLOUD_IMG_OVERRIDE-nextcloud:22.2.10-fpm-alpine} + - IMG=${NEXTCLOUD_IMG_OVERRIDE-nextcloud:23.0.11-fpm-alpine} container_name: dd-apps-nextcloud-app restart: unless-stopped depends_on: diff --git a/dd.conf.sample b/dd.conf.sample index 45b56e4..7310ea1 100644 --- a/dd.conf.sample +++ b/dd.conf.sample @@ -162,7 +162,7 @@ MOODLE_MAX_EXECUTION_TIME=300 MOODLE_MEMORY_LIMIT=128M ## NEXTCLOUD -# If enabled overrides nextcloud.yaml default image +# If enabled overrides nextcloud.yaml default image; you usually do not want to change this #NEXTCLOUD_IMG_OVERRIDE=nextcloud:22.2.10-fpm-alpine ## ONLYOFFICE From f55e830e2cd3071aceb604642fcfb5fdd8d5df17 Mon Sep 17 00:00:00 2001 From: Roger Garcia Date: Wed, 21 Dec 2022 09:48:49 +0100 Subject: [PATCH 06/15] Upgrade NC to version 24.0.8 --- dd-apps/docker/nextcloud/nextcloud.yml | 3 ++- dd.conf.sample | 5 +++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/dd-apps/docker/nextcloud/nextcloud.yml b/dd-apps/docker/nextcloud/nextcloud.yml index 8880bfd..0728122 100644 --- a/dd-apps/docker/nextcloud/nextcloud.yml +++ b/dd-apps/docker/nextcloud/nextcloud.yml @@ -24,7 +24,8 @@ services: context: ${BUILD_APPS_ROOT_PATH}/docker/nextcloud dockerfile: Dockerfile args: - - IMG=${NEXTCLOUD_IMG_OVERRIDE-nextcloud:23.0.11-fpm-alpine} + # Update dd.conf.sample when bumping this version in main + - IMG=${NEXTCLOUD_IMG_OVERRIDE-nextcloud:24.0.8-fpm-alpine} container_name: dd-apps-nextcloud-app restart: unless-stopped depends_on: diff --git a/dd.conf.sample b/dd.conf.sample index 7310ea1..407b43e 100644 --- a/dd.conf.sample +++ b/dd.conf.sample @@ -162,8 +162,9 @@ MOODLE_MAX_EXECUTION_TIME=300 MOODLE_MEMORY_LIMIT=128M ## NEXTCLOUD -# If enabled overrides nextcloud.yaml default image; you usually do not want to change this -#NEXTCLOUD_IMG_OVERRIDE=nextcloud:22.2.10-fpm-alpine +# If enabled overrides nextcloud.yaml default image. +# You usually do not want to change this, but instead rely on DD's version. +#NEXTCLOUD_IMG_OVERRIDE=nextcloud:24.0.8-fpm-alpine ## ONLYOFFICE ONLYOFFICE_IMG=alehoho/oo-ce-docker-license:6.3.1.32 From 3cfa0630d6b1c392ed72f8b098beb892ab1e103c Mon Sep 17 00:00:00 2001 From: Evilham Date: Fri, 23 Dec 2022 10:54:37 +0100 Subject: [PATCH 07/15] [docs] Add UPDATING documenting important and breaking changes The first example of these changes relates to the latest Nextcloud upgrades to major version 24. Operators are now expected to read the latest version this file before updating their instances. --- UPDATING.md | 47 +++++++++++++++++++++++++++++++++++++++++++++++ docs/updating.md | 1 + mkdocs.yml | 1 + 3 files changed, 49 insertions(+) create mode 100644 UPDATING.md create mode 120000 docs/updating.md diff --git a/UPDATING.md b/UPDATING.md new file mode 100644 index 0000000..80277ca --- /dev/null +++ b/UPDATING.md @@ -0,0 +1,47 @@ +# Updating + +It is quite important for operators to check this file before updating their +production instances, as it may contain information that is critical for good +continuity of the services. + + +# Kinds of breaking changes + +## Nextcloud upgrades + +Nextcloud upgrades must be carried out in a very specific order, if you follow +DD closely you should not have issues with this on 2023 and beyond. + +However, if you run an older instance, please find the +**2022-12 - Nextcloud 24** section on this document, and follow the commands +closely. + +Further major upgrades will be documented here too, but they should have +little to no impact. + + +# Important changes in reverse chronological order + +## 2022 + +### 2022-12 - Nextcloud 24 + +Starting with commits in december, DD's blessed Nextcloud version is 24.0.8. + +Work is already being carried out to upgrade this to major version 25. + +In order to safely upgrade to major version 24, the upgrade must be performed +step-wise: + + # Latest commit with NC major version 21 by default + ./dd-ctl update db4a6d14e9a05ee1b0334aaf87af31703a4ae4d0 + # Upgrade to NC version 22 + ./dd-ctl update cbb4e06e896da40430d66435589ab5ea8cf71420 + # Upgrade to NC version 23 + ./dd-ctl update 57c87d7172579b8fbe0d8249f47ee0966621e252 + # Upgrade to NC version 24.0.8 (latest minor on december 2022) + ./dd-ctl update f55e830e2cd3071aceb604642fcfb5fdd8d5df17 + # + # Upgrade to latest state of DD + # (read more recent notices of important changes before running this) + #./dd-ctl update diff --git a/docs/updating.md b/docs/updating.md new file mode 120000 index 0000000..535d93e --- /dev/null +++ b/docs/updating.md @@ -0,0 +1 @@ +../UPDATING.md \ No newline at end of file diff --git a/mkdocs.yml b/mkdocs.yml index c9e5cd0..0daee51 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -58,6 +58,7 @@ nav: - index.ca.md - index.es.md - install.ca.md +- updating.md - customising.ca.md - integrations.ca.md - post-install.ca.md From ea995c0336411f2fe2444fbcc1441b42ceca7134 Mon Sep 17 00:00:00 2001 From: Evilham Date: Tue, 3 Jan 2023 10:18:54 +0100 Subject: [PATCH 08/15] [dd-ctl] Force DD_DEFAULT_BUILD length to 8 hex digits This is used to tag Docker images in the registry / pull images form the registry, it has been observed in the wiled that: git rev-parse --short can have different default values for its length depending on the system. We currently specify the length to be 8 as specified here: https://git-scm.com/docs/git-rev-parse#Documentation/git-rev-parse.txt---shortlength --- dd-ctl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dd-ctl b/dd-ctl index c3471ce..d37f512 100755 --- a/dd-ctl +++ b/dd-ctl @@ -175,7 +175,7 @@ ddupdate(){ } build_compose(){ - DD_DEFAULT_BUILD="$(git rev-parse --short HEAD)" + DD_DEFAULT_BUILD="$(git rev-parse --short=8 HEAD)" export DD_BUILD="${DD_BUILD:-${DD_DEFAULT_BUILD}}" setconf DD_BUILD "${DD_BUILD}" .env From f4a3d38384a5976a5d101375d1615784fc43a4c4 Mon Sep 17 00:00:00 2001 From: Evilham Date: Thu, 12 Jan 2023 11:42:34 +0100 Subject: [PATCH 09/15] [NC] Update mail plugin patch for NC 24 This is still not on release line supporting NC 24 line (v1.15.1) and having an older appinfo may be producing issues. --- dd-apps/docker/nextcloud/dd-patch | 9 +++------ .../docker/nextcloud/nc_mail/appinfo/info.xml | 6 +++--- .../nc_mail/lib/Db/MailAccountMapper.php | 17 ----------------- 3 files changed, 6 insertions(+), 26 deletions(-) diff --git a/dd-apps/docker/nextcloud/dd-patch b/dd-apps/docker/nextcloud/dd-patch index 0084ea6..6b6c523 100644 --- a/dd-apps/docker/nextcloud/dd-patch +++ b/dd-apps/docker/nextcloud/dd-patch @@ -1,9 +1,6 @@ # Generate .orig and .patch files with ./dd-ctl genpatches # file license author source nginx.conf AGPL-3.0-or-later https://github.com/nextcloud/ https://raw.githubusercontent.com/nextcloud/docker/522559eefdd56d2e49259c3b0f4a0e92882cdb87/.examples/docker-compose/with-nginx-proxy/postgres/fpm/web/nginx.conf -#nc_mail/appinfo.xml AGPL-3.0-or-later https://github.com/nextcloud/ https://raw.githubusercontent.com/MaadixNet/mail/feature/occ-account-update-command/appinfo/info.xml -#nc_mail/lib/Command/UpdateAccount.php AGPL-3.0-or-later https://github.com/nextcloud/ https://raw.githubusercontent.com/MaadixNet/mail/feature/occ-account-update-command/lib/Command/UpdateAccount.php -#nc_mail/lib/Db/MailAccountMapper.php AGPL-3.0-or-later https://github.com/nextcloud/ https://raw.githubusercontent.com/MaadixNet/mail/feature/occ-account-update-command/lib/Db/MailAccountMapper.php -nc_mail/appinfo/info.xml AGPL-3.0-or-later https://github.com/nextcloud/ https://raw.githubusercontent.com/nextcloud/mail/v1.12.8/appinfo/info.xml -nc_mail/lib/Command/UpdateAccount.php AGPL-3.0-or-later https://github.com/nextcloud/ https://raw.githubusercontent.com/nextcloud/mail/v1.12.8/lib/Command/UpdateAccount.php -nc_mail/lib/Db/MailAccountMapper.php AGPL-3.0-or-later https://github.com/nextcloud/ https://raw.githubusercontent.com/nextcloud/mail/v1.12.8/lib/Db/MailAccountMapper.php +nc_mail/appinfo/info.xml AGPL-3.0-or-later https://github.com/nextcloud/ https://raw.githubusercontent.com/nextcloud/mail/v1.15.1/appinfo/info.xml +nc_mail/lib/Command/UpdateAccount.php AGPL-3.0-or-later https://github.com/nextcloud/ https://raw.githubusercontent.com/nextcloud/mail/1e777a1783254bd4b7f69f39a6c5123323f8b701/lib/Command/UpdateAccount.php +nc_mail/lib/Db/MailAccountMapper.php AGPL-3.0-or-later https://github.com/nextcloud/ https://raw.githubusercontent.com/nextcloud/mail/1e777a1783254bd4b7f69f39a6c5123323f8b701/lib/Db/MailAccountMapper.php diff --git a/dd-apps/docker/nextcloud/nc_mail/appinfo/info.xml b/dd-apps/docker/nextcloud/nc_mail/appinfo/info.xml index a7e8311..ba54672 100644 --- a/dd-apps/docker/nextcloud/nc_mail/appinfo/info.xml +++ b/dd-apps/docker/nextcloud/nc_mail/appinfo/info.xml @@ -12,7 +12,7 @@ - **🙈 We’re not reinventing the wheel!** Based on the great [Horde](https://horde.org) libraries. - **📬 Want to host your own mail server?** We do not have to reimplement this as you could set up [Mail-in-a-Box](https://mailinabox.email)! ]]> - 1.12.8 + 1.15.1 agpl Greta Doçi Nextcloud Groupware Team @@ -29,8 +29,8 @@ https://github.com/nextcloud/mail.git https://user-images.githubusercontent.com/1374172/79554966-278e1600-809f-11ea-82ea-7a0d72a2704f.png - - + + OCA\Mail\BackgroundJob\CleanupJob diff --git a/dd-apps/docker/nextcloud/nc_mail/lib/Db/MailAccountMapper.php b/dd-apps/docker/nextcloud/nc_mail/lib/Db/MailAccountMapper.php index 6c5113b..626708e 100644 --- a/dd-apps/docker/nextcloud/nc_mail/lib/Db/MailAccountMapper.php +++ b/dd-apps/docker/nextcloud/nc_mail/lib/Db/MailAccountMapper.php @@ -37,7 +37,6 @@ use OCP\IUser; * @template-extends QBMapper */ class MailAccountMapper extends QBMapper { - /** * @param IDBConnection $db */ @@ -97,22 +96,6 @@ class MailAccountMapper extends QBMapper { return $this->findEntities($query); } - /** - * Finds an mail account by user id and email address - * - * @return MailAccount - * @throws DoesNotExistException - */ - public function findByUserIdAndEmail(string $userId, string $email): MailAccount { - $qb = $this->db->getQueryBuilder(); - $query = $qb - ->select('*') - ->from($this->getTableName()) - ->where($qb->expr()->eq('user_id', $qb->createNamedParameter($userId, IQueryBuilder::PARAM_STR), IQueryBuilder::PARAM_STR)) - ->andWhere($qb->expr()->eq('email', $qb->createNamedParameter($email, IQueryBuilder::PARAM_STR), IQueryBuilder::PARAM_STR)); - - return $this->findEntity($query); - } /** * @throws DoesNotExistException From c9af7242c0c3f32883023be663e4c18d428966a9 Mon Sep 17 00:00:00 2001 From: Evilham Date: Fri, 13 Jan 2023 11:26:12 +0100 Subject: [PATCH 10/15] [NC] Work arround issue nextcloud/server#33751 That issue is fixed in NC 25, but it will likely not be backported to NC 24. It produces issues when modifying users and not modifying their display name. See also: https://github.com/nextcloud/server/issues/33751 --- dd-sso/admin/src/admin/lib/nextcloud.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/dd-sso/admin/src/admin/lib/nextcloud.py b/dd-sso/admin/src/admin/lib/nextcloud.py index 763dcfd..f0d26b4 100644 --- a/dd-sso/admin/src/admin/lib/nextcloud.py +++ b/dd-sso/admin/src/admin/lib/nextcloud.py @@ -275,10 +275,13 @@ class Nextcloud: self._request("PUT", url, data=data, headers=headers) ) if result["ocs"]["meta"]["statuscode"] == 102: - raise ProviderItemExists - if result["ocs"]["meta"]["statuscode"] == 104: + if result["ocs"]["meta"].get("message") != "Invalid displayname": + # Workaround for https://github.com/nextcloud/server/issues/33751 + # Fixed on NC 25 + raise ProviderItemExists + elif result["ocs"]["meta"]["statuscode"] == 104: raise ProviderGroupNotExists - if result["ocs"]["meta"]["statuscode"] != 100: + elif result["ocs"]["meta"]["statuscode"] != 100: log.error("Get Nextcloud provider user add error: " + str(result)) raise ProviderOpError except: From 1d077b71f9658421609c466471e029d1df8c5f89 Mon Sep 17 00:00:00 2001 From: Evilham Date: Fri, 20 Jan 2023 11:29:51 +0100 Subject: [PATCH 11/15] [sso-admin] Fix SocketIO transitive dependency (dnspython) Flask-SocketIO depends on dnspython but dnspython 2.3 removes dns.rdtypes.ANY, which is needed by Flask-SocketIO so we keep it below version 2.3 --- dd-sso/admin/docker/requirements.pip3 | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/dd-sso/admin/docker/requirements.pip3 b/dd-sso/admin/docker/requirements.pip3 index deaf8bd..e250bf0 100644 --- a/dd-sso/admin/docker/requirements.pip3 +++ b/dd-sso/admin/docker/requirements.pip3 @@ -23,6 +23,10 @@ Flask==2.1.3 Flask-Login==0.6.2 eventlet==0.33.1 Flask-SocketIO==5.2.0 +# Flask-SocketIO depends on dnspython +# but dnspython 2.3 removes dns.rdtypes.ANY, which is needed by Flask-SocketIO +# so we keep it below version 2.3 +dnspython<2.3 flasgger==0.9.5 bcrypt==3.2.2 # diceware can't be upgraded without issues From affbbde9d3d059260cda43d164c5222a44e92b1d Mon Sep 17 00:00:00 2001 From: Evilham Date: Tue, 10 Jan 2023 16:44:49 +0100 Subject: [PATCH 12/15] [NC] Reduce upstream diff and improve env vars By keeping the dfif against upstream we can follow them easier over time. And by having env vars explicit, we can get rid of the auxiliary env files. --- dd-apps/docker/nextcloud/dd-patch | 3 +- dd-apps/docker/nextcloud/nextcloud.yml | 24 +++- dd-apps/docker/nextcloud/nginx.conf | 162 +++++++++++++++------- dd-apps/docker/nextcloud/supervisord.conf | 6 +- 4 files changed, 140 insertions(+), 55 deletions(-) diff --git a/dd-apps/docker/nextcloud/dd-patch b/dd-apps/docker/nextcloud/dd-patch index 6b6c523..dfe1148 100644 --- a/dd-apps/docker/nextcloud/dd-patch +++ b/dd-apps/docker/nextcloud/dd-patch @@ -1,6 +1,7 @@ # Generate .orig and .patch files with ./dd-ctl genpatches # file license author source -nginx.conf AGPL-3.0-or-later https://github.com/nextcloud/ https://raw.githubusercontent.com/nextcloud/docker/522559eefdd56d2e49259c3b0f4a0e92882cdb87/.examples/docker-compose/with-nginx-proxy/postgres/fpm/web/nginx.conf +nginx.conf AGPL-3.0-or-later https://github.com/nextcloud/ https://raw.githubusercontent.com/nextcloud/docker/04ab3157e4f1197a9f8f4b91cf2430586356dd9a/.examples/docker-compose/with-nginx-proxy/postgres/fpm/web/nginx.conf +supervisord.conf AGPL-3.0-or-later https://github.com/nextcloud/ https://raw.githubusercontent.com/nextcloud/docker/04ab3157e4f1197a9f8f4b91cf2430586356dd9a/.examples/dockerfiles/cron/fpm-alpine/supervisord.conf nc_mail/appinfo/info.xml AGPL-3.0-or-later https://github.com/nextcloud/ https://raw.githubusercontent.com/nextcloud/mail/v1.15.1/appinfo/info.xml nc_mail/lib/Command/UpdateAccount.php AGPL-3.0-or-later https://github.com/nextcloud/ https://raw.githubusercontent.com/nextcloud/mail/1e777a1783254bd4b7f69f39a6c5123323f8b701/lib/Command/UpdateAccount.php nc_mail/lib/Db/MailAccountMapper.php AGPL-3.0-or-later https://github.com/nextcloud/ https://raw.githubusercontent.com/nextcloud/mail/1e777a1783254bd4b7f69f39a6c5123323f8b701/lib/Db/MailAccountMapper.php diff --git a/dd-apps/docker/nextcloud/nextcloud.yml b/dd-apps/docker/nextcloud/nextcloud.yml index 0728122..639b0fd 100644 --- a/dd-apps/docker/nextcloud/nextcloud.yml +++ b/dd-apps/docker/nextcloud/nextcloud.yml @@ -1,5 +1,6 @@ # # Copyright © 2021,2022 IsardVDI S.L. +# Copyright © 2023 Evilham # # This file is part of DD # @@ -37,7 +38,9 @@ services: - ${DATA_FOLDER}/saml/nextcloud:/saml:ro - ${DATA_FOLDER}/nc-mail-queue:/nc-mail-queue:rw environment: + # DD-specific settings - DOMAIN=${DOMAIN} + # General Nextcloud settings - NEXTCLOUD_ADMIN_USER=${NEXTCLOUD_ADMIN_USER} - NEXTCLOUD_ADMIN_PASSWORD=${NEXTCLOUD_ADMIN_PASSWORD} - POSTGRES_DB=nextcloud @@ -45,20 +48,37 @@ services: - POSTGRES_PASSWORD=${NEXTCLOUD_POSTGRES_PASSWORD} - POSTGRES_HOST=dd-apps-postgresql - REDIS_HOST=dd-apps-redis - - NC_overwriteprotocol=https - NEXTCLOUD_TRUSTED_DOMAINS=nextcloud.${DOMAIN} + # SMTP settings + - SMTP_HOST=${SMTP_HOST} + - SMTP_PORT=${SMTP_PORT} + - SMTP_SECURE=${SMTP_PROTOCOL} + - SMTP_NAME=${SMTP_USER} + - SMTP_PASSWORD=${SMTP_PASSWORD} + - MAIL_FROM_ADDRESS=${SMTP_LOCAL_PART} + - MAIL_DOMAIN=${SMTP_DOMAIN} + # PHP-specific settings + - PHP_UPLOAD_LIMIT=${NEXTCLOUD_UPLOAD_LIMIT-512M} + - PHP_MEMORY_LIMIT=${NEXTCLOUD_MEMORY_LIMIT-512M} + # Proxy-specific settings + - OVERWRITEPROTOCOL=https networks: - dd_net dd-apps-nextcloud-nginx: - image: ${NGINX_IMG-nginx:1.21.6} + image: ${NGINX_ALPINE_IMG-nginx:1.22.1-alpine} container_name: dd-apps-nextcloud-nginx restart: unless-stopped links: - dd-apps-nextcloud-app volumes: + # Customised nginx config - ${BUILD_APPS_ROOT_PATH}/docker/nextcloud/nginx.conf:/etc/nginx/nginx.conf:ro + # By mounting these, nginx can serve static files when possible - ${SRC_FOLDER}/nextcloud:/var/www/html:ro + - ${BUILD_APPS_ROOT_PATH}/docker/nextcloud/src/themes/dd:/var/www/html/themes/dd:ro + # This image does not require access to any environment variables + environment: [] healthcheck: test: | curl -L --max-redirs 1 \ diff --git a/dd-apps/docker/nextcloud/nginx.conf b/dd-apps/docker/nextcloud/nginx.conf index 9897fdd..e0a4cf6 100644 --- a/dd-apps/docker/nextcloud/nginx.conf +++ b/dd-apps/docker/nextcloud/nginx.conf @@ -20,45 +20,42 @@ http { access_log /var/log/nginx/access.log main; sendfile on; + #tcp_nopush on; + + # Prevent nginx HTTP Server Detection + server_tokens off; + keepalive_timeout 65; + #gzip on; + + # Pass Real IP to Nginx from proxy set_real_ip_from 10.0.0.0/8; set_real_ip_from 172.16.0.0/12; set_real_ip_from 192.168.0.0/16; real_ip_header X-Forwarded-For; + upstream php-handler { + # Pass request to DD container server dd-apps-nextcloud-app:9000; } server { listen 80; - add_header Referrer-Policy "no-referrer" always; - add_header X-Content-Type-Options "nosniff" always; - add_header X-Download-Options "noopen" always; - add_header X-Frame-Options "SAMEORIGIN" always; - add_header X-Permitted-Cross-Domain-Policies "none" always; - add_header X-Robots-Tag "none" always; - add_header X-XSS-Protection "1; mode=block" always; - fastcgi_hide_header X-Powered-By; - root /var/www/html; - location = /robots.txt { - allow all; - log_not_found off; - access_log off; - } - - location = /.well-known/carddav { - return 301 $scheme://$host/remote.php/dav; - } - - location = /.well-known/caldav { - return 301 $scheme://$host/remote.php/dav; - } + # HSTS settings + # WARNING: Only add the preload option once you read about + # the consequences in https://hstspreload.org/. This option + # will add the domain to a hardcoded list that is shipped + # in all major browsers and getting removed from this list + # could take several months. + #add_header Strict-Transport-Security "max-age=15768000; includeSubDomains; preload;" always; + # set max upload size client_max_body_size 10G; fastcgi_buffers 64 4K; + # Enable gzip but do not remove ETag headers gzip on; gzip_vary on; gzip_comp_level 4; @@ -66,53 +63,120 @@ http { gzip_proxied expired no-cache no-store private no_last_modified no_etag auth; gzip_types application/atom+xml application/javascript application/json application/ld+json application/manifest+json application/rss+xml application/vnd.geo+json application/vnd.ms-fontobject application/x-font-ttf application/x-web-app-manifest+json application/xhtml+xml application/xml font/opentype image/bmp image/svg+xml image/x-icon text/cache-manifest text/css text/plain text/vcard text/vnd.rim.location.xloc text/vtt text/x-component text/x-cross-domain-policy; - location / { - rewrite ^ /index.php; + # Pagespeed is not supported by Nextcloud, so if your server is built + # with the `ngx_pagespeed` module, uncomment this line to disable it. + #pagespeed off; + + # HTTP response headers borrowed from Nextcloud `.htaccess` + add_header Referrer-Policy "no-referrer" always; + add_header X-Content-Type-Options "nosniff" always; + add_header X-Download-Options "noopen" always; + add_header X-Frame-Options "SAMEORIGIN" always; + add_header X-Permitted-Cross-Domain-Policies "none" always; + add_header X-Robots-Tag "none" always; + add_header X-XSS-Protection "1; mode=block" always; + + # Remove X-Powered-By, which is an information leak + fastcgi_hide_header X-Powered-By; + + # Path to the root of your installation + root /var/www/html; + + # Specify how to handle directories -- specifying `/index.php$request_uri` + # here as the fallback means that Nginx always exhibits the desired behaviour + # when a client requests a path that corresponds to a directory that exists + # on the server. In particular, if that directory contains an index.php file, + # that file is correctly served; if it doesn't, then the request is passed to + # the front-end controller. This consistent behaviour means that we don't need + # to specify custom rules for certain paths (e.g. images and other assets, + # `/updater`, `/ocm-provider`, `/ocs-provider`), and thus + # `try_files $uri $uri/ /index.php$request_uri` + # always provides the desired behaviour. + index index.php index.html /index.php$request_uri; + + # Rule borrowed from `.htaccess` to handle Microsoft DAV clients + location = / { + if ( $http_user_agent ~ ^DavClnt ) { + return 302 /remote.php/webdav/$is_args$args; + } } - location ~ ^\/(?:build|tests|config|lib|3rdparty|templates|data)\/ { - deny all; - } - location ~ ^\/(?:\.|autotest|occ|issue|indie|db_|console) { - deny all; + location = /robots.txt { + allow all; + log_not_found off; + access_log off; } - location ~ ^\/(?:index|remote|public|cron|core\/ajax\/update|status|ocs\/v[12]|updater\/.+|oc[ms]-provider\/.+)\.php(?:$|\/) { - fastcgi_split_path_info ^(.+?\.php)(\/.*|)$; + # Make a regex exception for `/.well-known` so that clients can still + # access it despite the existence of the regex rule + # `location ~ /(\.|autotest|...)` which would otherwise handle requests + # for `/.well-known`. + location ^~ /.well-known { + # The rules in this block are an adaptation of the rules + # in `.htaccess` that concern `/.well-known`. + + # TODO: Test if this works fine with port / host / etc. + location = /.well-known/carddav { return 301 /remote.php/dav/; } + location = /.well-known/caldav { return 301 /remote.php/dav/; } + + location /.well-known/acme-challenge { try_files $uri $uri/ =404; } + location /.well-known/pki-validation { try_files $uri $uri/ =404; } + + # Let Nextcloud's API for `/.well-known` URIs handle all other + # requests by passing them to the front-end controller. + return 301 /index.php$request_uri; + } + + # Rules borrowed from `.htaccess` to hide certain paths from clients + location ~ ^/(?:build|tests|config|lib|3rdparty|templates|data)(?:$|/) { return 404; } + location ~ ^/(?:\.|autotest|occ|issue|indie|db_|console) { return 404; } + + # Ensure this block, which passes PHP files to the PHP process, is above the blocks + # which handle static assets (as seen below). If this block is not declared first, + # then Nginx will encounter an infinite rewriting loop when it prepends `/index.php` + # to the URI, resulting in a HTTP 500 error response. + location ~ \.php(?:$|/) { + # Required for legacy support + rewrite ^/(?!index|remote|public|cron|core\/ajax\/update|status|ocs\/v[12]|updater\/.+|oc[ms]-provider\/.+|.+\/richdocumentscode\/proxy) /index.php$request_uri; + + fastcgi_split_path_info ^(.+?\.php)(/.*)$; set $path_info $fastcgi_path_info; + try_files $fastcgi_script_name =404; + include fastcgi_params; fastcgi_param SCRIPT_FILENAME $document_root$fastcgi_script_name; fastcgi_param PATH_INFO $path_info; - fastcgi_param modHeadersAvailable true; - fastcgi_param front_controller_active true; + #fastcgi_param HTTPS on; + + fastcgi_param modHeadersAvailable true; # Avoid sending the security headers twice + fastcgi_param front_controller_active true; # Enable pretty urls fastcgi_pass php-handler; + fastcgi_intercept_errors on; fastcgi_request_buffering off; fastcgi_param SERVER_NAME $host; } - location ~ ^\/(?:updater|oc[ms]-provider)(?:$|\/) { - try_files $uri/ =404; - index index.php; + location ~ \.(?:css|js|svg|gif|map)$ { + try_files $uri /index.php$request_uri; + expires 6M; # Cache-Control policy borrowed from `.htaccess` + access_log off; # Optional: Don't log access to assets } - location ~ \.(?:css|js|woff2?|svg|gif|map)$ { + location ~ \.(?:woff2?|png|html|ttf|ico|jpg|jpeg|bcmap|mp4|webm)$ { try_files $uri /index.php$request_uri; - add_header Cache-Control "public, max-age=15778463"; - add_header Referrer-Policy "no-referrer" always; - add_header X-Content-Type-Options "nosniff" always; - add_header X-Download-Options "noopen" always; - add_header X-Frame-Options "SAMEORIGIN" always; - add_header X-Permitted-Cross-Domain-Policies "none" always; - add_header X-Robots-Tag "none" always; - add_header X-XSS-Protection "1; mode=block" always; - access_log off; + expires 7d; # Cache-Control policy borrowed from `.htaccess` + access_log off; # Optional: Don't log access to assets } - location ~ \.(?:png|html|ttf|ico|jpg|jpeg|bcmap|mp4|webm)$ { - try_files $uri /index.php$request_uri; - access_log off; + # Rule borrowed from `.htaccess` + location /remote { + return 301 /remote.php$request_uri; + } + + location / { + try_files $uri $uri/ /index.php$request_uri; } } } diff --git a/dd-apps/docker/nextcloud/supervisord.conf b/dd-apps/docker/nextcloud/supervisord.conf index 63366d0..4f76259 100644 --- a/dd-apps/docker/nextcloud/supervisord.conf +++ b/dd-apps/docker/nextcloud/supervisord.conf @@ -3,8 +3,8 @@ nodaemon=true logfile=/var/log/supervisord/supervisord.log pidfile=/var/run/supervisord/supervisord.pid childlogdir=/var/log/supervisord/ -logfile_maxbytes=50MB -logfile_backups=10 +logfile_maxbytes=50MB ; maximum size of logfile before rotation +logfile_backups=10 ; number of backed up logfiles loglevel=error [program:php-fpm] @@ -19,4 +19,4 @@ stdout_logfile=/dev/stdout stdout_logfile_maxbytes=0 stderr_logfile=/dev/stderr stderr_logfile_maxbytes=0 -command=/cron.sh \ No newline at end of file +command=/cron.sh From a72001dea5e2e1502219c97dcddd0eb18752dda0 Mon Sep 17 00:00:00 2001 From: Evilham Date: Tue, 10 Jan 2023 16:58:17 +0100 Subject: [PATCH 13/15] [docker] Cleanup environment files By having the environment explicit on each service, we both document the settings and have more control over what each service is allowed to see. This avoids weird things like nginx having access to postgresql's credentials on its environment. As a bonus: we are able to use one single environment file, which is basically dd.conf with some values that are dynamically-calculated and added from dd-ctl. --- dd-apps/docker/postgresql/postgresql.yml | 15 ++++++- dd-apps/docker/wordpress/wordpress.yml | 1 + dd-ctl | 42 ++++++++++++------- dd-sso/docker-compose-parts/admin.yml | 4 +- dd-sso/docker-compose-parts/api.yml | 9 ++-- .../docker-compose-parts/haproxy.no-ports.yml | 8 +++- dd-sso/docker-compose-parts/haproxy.proxy.yml | 8 +++- dd-sso/docker-compose-parts/haproxy.yml | 8 +++- dd-sso/docker-compose-parts/postgresql.yml | 38 ----------------- .../docker-compose-parts/waf-modsecurity.yml | 4 +- dd.conf.sample | 3 +- 11 files changed, 68 insertions(+), 72 deletions(-) delete mode 100644 dd-sso/docker-compose-parts/postgresql.yml diff --git a/dd-apps/docker/postgresql/postgresql.yml b/dd-apps/docker/postgresql/postgresql.yml index 1c510b7..5bca788 100644 --- a/dd-apps/docker/postgresql/postgresql.yml +++ b/dd-apps/docker/postgresql/postgresql.yml @@ -23,10 +23,23 @@ services: image: ${POSTGRESQL_IMG-postgres:13.5-alpine3.15} container_name: dd-apps-postgresql restart: unless-stopped - env_file: .env environment: + # Postgres - POSTGRES_PASSWORD=${POSTGRES_PASSWORD} - POSTGRES_USER=${POSTGRES_USER} + # Etherpad + - ETHERPAD_POSTGRES_USER=${ETHERPAD_POSTGRES_USER:-etherpard} + - ETHERPAD_POSTGRES_PASSWORD=${ETHERPAD_POSTGRES_PASSWORD} + # Keycloak + - KEYCLOAK_DB_DATABASE=${KEYCLOAK_DB_DATABASE:-keycloak} + - KEYCLOAK_DB_USER=${KEYCLOAK_DB_USER:-keycloak} + - KEYCLOAK_DB_PASSWORD=${KEYCLOAK_DB_PASSWORD} + # Moodle + - MOODLE_POSTGRES_USER=${MOODLE_POSTGRES_USER:-moodle} + - MOODLE_POSTGRES_PASSWORD=${MOODLE_POSTGRES_PASSWORD} + # Nextcloud + - NEXTCLOUD_POSTGRES_USER=${NEXTCLOUD_POSTGRES_USER:-nextcloud} + - NEXTCLOUD_POSTGRES_PASSWORD=${NEXTCLOUD_POSTGRES_PASSWORD} volumes: - /etc/localtime:/etc/localtime:ro - ${DB_FOLDER}/postgres:/var/lib/postgresql/data diff --git a/dd-apps/docker/wordpress/wordpress.yml b/dd-apps/docker/wordpress/wordpress.yml index b6a88a0..12acad3 100644 --- a/dd-apps/docker/wordpress/wordpress.yml +++ b/dd-apps/docker/wordpress/wordpress.yml @@ -36,6 +36,7 @@ services: volumes: *wordpress-volumes environment: + DOMAIN: ${DOMAIN} WORDPRESS_DB_HOST: dd-apps-mariadb WORDPRESS_DB_NAME: "wordpress" WORDPRESS_DB_USER: ${WORDPRESS_MARIADB_USER} diff --git a/dd-ctl b/dd-ctl index d37f512..8aa12c7 100755 --- a/dd-ctl +++ b/dd-ctl @@ -112,9 +112,11 @@ fi REPO_BRANCH="${1:-main}" -cp dd.conf .env CUSTOM_PATH=$(pwd) -. ./.env +if [ -f dd.conf ]; then + cp dd.conf .env + . ./.env +fi prerequisites_docker(){ # Remove uncompatible docker packages @@ -180,9 +182,18 @@ build_compose(){ setconf DD_BUILD "${DD_BUILD}" .env setconf CUSTOM_PATH "$CUSTOM_PATH" .env + setconf BUILD_APPS_ROOT_PATH "$CUSTOM_PATH/dd-apps" .env setconf BUILD_SSO_ROOT_PATH "$CUSTOM_PATH/dd-sso" .env + # SMTP (needed for e.g. Nextcloud's more granular settings) + SMTP_LOCAL_PART="$(echo "${SMTP_USER:-}" | cut -d '@' -f 1)" + export SMTP_LOCAL_PART + SMTP_DOMAIN="$(echo "${SMTP_USER:-}" | cut -d '@' -f 2)" + export SMTP_DOMAIN + setconf SMTP_LOCAL_PART "${SMTP_LOCAL_PART}" .env + setconf SMTP_DOMAIN "${SMTP_DOMAIN}" .env + # Choose HAProxy configuration flavour if [ "${PROXY_PROTOCOL:-false}" = "true" ]; then HAPROXY_YML="haproxy.proxy.yml" @@ -207,7 +218,7 @@ build_compose(){ setconf HAPROXY_CFG "${HAPROXY_CFG}" setconf HAPROXY_CFG "${HAPROXY_CFG}" .env - # Enable or disable ClamAV + # Enable or disable ClamAV's container if [ "${DISABLE_CLAMAV:-true}" = "true" ]; then # Current default (might change) CLAMAV_YML="clamav.disabled.yml" @@ -215,18 +226,16 @@ build_compose(){ CLAMAV_YML="clamav.yml" fi - ## Prepare apps environment - ln -sf "${CUSTOM_PATH}/.env" dd-apps/.env - ln -sf "${CUSTOM_PATH}/.env" dd-apps/docker/postgresql && \ - ln -sf "${CUSTOM_PATH}/.env" dd-apps/docker/mariadb && \ - ln -sf "${CUSTOM_PATH}/.env" dd-apps/docker/moodle && \ - ln -sf "${CUSTOM_PATH}/.env" dd-apps/docker/nextcloud && \ - ln -sf "${CUSTOM_PATH}/.env" dd-apps/docker/wordpress && \ - ln -sf "${CUSTOM_PATH}/.env" dd-apps/docker/etherpad - - ## Prepare sso environment - ln -sf "${CUSTOM_PATH}/.env" dd-sso/.env - ln -sf "${CUSTOM_PATH}/.env" dd-sso/docker-compose-parts/.env + # Clean up redundant .env files + for f in dd-apps \ + dd-apps/docker/postgresql dd-apps/docker/mariadb \ + dd-apps/docker/moodle dd-apps/docker/nextcloud \ + dd-apps/docker/wordpress dd-apps/docker/etherpad \ + dd-sso dd-sso/docker-compose-parts; do + if [ -f "${f}/.env" ]; then + rm "${f}/.env" + fi + done # Clean up older custom data rm -rf custom/system/keycloak-themes @@ -234,12 +243,13 @@ build_compose(){ # Build compose ymls docker-compose \ + --env-file "${CUSTOM_PATH}/.env" \ + \ -f "dd-sso/docker-compose-parts/$WAF_YML" \ -f "dd-sso/docker-compose-parts/$HAPROXY_YML"\ -f dd-sso/docker-compose-parts/api.yml \ -f dd-sso/docker-compose-parts/keycloak.yml \ -f dd-sso/docker-compose-parts/avatars.yml \ - -f dd-apps/docker/postgresql/postgresql.yml \ -f dd-sso/docker-compose-parts/admin.yml \ \ -f dd-apps/docker/moodle/moodle.yml \ diff --git a/dd-sso/docker-compose-parts/admin.yml b/dd-sso/docker-compose-parts/admin.yml index 054baf9..2dd9b24 100644 --- a/dd-sso/docker-compose-parts/admin.yml +++ b/dd-sso/docker-compose-parts/admin.yml @@ -39,8 +39,6 @@ services: - ${DATA_FOLDER}/legal:/admin/admin/static/templates/pages/legal:rw - ${DATA_FOLDER}/dd-admin:/data:rw - ${DATA_FOLDER}/nc-mail-queue:/nc-mail-queue:rw - env_file: - - .env environment: - VERIFY="false" # In development do not verify certificates - DOMAIN=${DOMAIN} @@ -53,3 +51,5 @@ services: - AVATARS_SERVER_HOST=dd-sso-avatars:9000 - AVATARS_ACCESS_KEY=${AVATARS_ACCESS_KEY:-AKIAIOSFODNN7EXAMPLE} - AVATARS_SECRET_KEY=${AVATARS_SECRET_KEY:-wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY} + # TODO: Check missing env vars and pass them on the environment + env_file: ${CUSTOM_PATH}/.env diff --git a/dd-sso/docker-compose-parts/api.yml b/dd-sso/docker-compose-parts/api.yml index 94e968e..42a6925 100644 --- a/dd-sso/docker-compose-parts/api.yml +++ b/dd-sso/docker-compose-parts/api.yml @@ -28,6 +28,10 @@ services: DOMAIN: $DOMAIN NGINX_ALPINE_IMG: ${NGINX_ALPINE_IMG-nginx:1.21.6-alpine} container_name: dd-sso-api + environment: + DOMAIN: ${DOMAIN} + KEYCLOAK_USER: ${KEYCLOAK_USER} + KEYCLOAK_PASSWORD: ${KEYCLOAK_PASSWORD} volumes: - /etc/localtime:/etc/localtime:ro - ${CUSTOM_PATH}/custom/menu:/api/menu @@ -37,8 +41,3 @@ services: restart: unless-stopped networks: - dd_net - # ports: - # - published: 7039 - # target: 7039 - env_file: - - .env diff --git a/dd-sso/docker-compose-parts/haproxy.no-ports.yml b/dd-sso/docker-compose-parts/haproxy.no-ports.yml index 2ef11c9..b6a7dc0 100644 --- a/dd-sso/docker-compose-parts/haproxy.no-ports.yml +++ b/dd-sso/docker-compose-parts/haproxy.no-ports.yml @@ -28,6 +28,12 @@ services: dockerfile: Dockerfile target: production container_name: dd-sso-haproxy + environment: + DOMAIN: ${DOMAIN} + HAPROXY_CFG: ${HAPROXY_CFG:-haproxy.no-waf.no-haproxy.cfg} + LETSENCRYPT_DOMAIN: ${LETSENCRYPT_DOMAIN} + LETSENCRYPT_DOMAIN_ROOT: ${LETSENCRYPT_DOMAIN_ROOT} + LETSENCRYPT_EMAIL: ${LETSENCRYPT_EMAIL} restart: unless-stopped volumes: - /etc/localtime:/etc/localtime:ro @@ -35,8 +41,6 @@ services: - ${SRC_FOLDER}/haproxy/certs:/certs:rw networks: - dd_net - env_file: - - .env logging: driver: "json-file" options: diff --git a/dd-sso/docker-compose-parts/haproxy.proxy.yml b/dd-sso/docker-compose-parts/haproxy.proxy.yml index d5830c5..3597131 100644 --- a/dd-sso/docker-compose-parts/haproxy.proxy.yml +++ b/dd-sso/docker-compose-parts/haproxy.proxy.yml @@ -28,6 +28,12 @@ services: dockerfile: Dockerfile target: production container_name: dd-sso-haproxy + environment: + DOMAIN: ${DOMAIN} + HAPROXY_CFG: ${HAPROXY_CFG:-haproxy.no-waf.no-haproxy.cfg} + LETSENCRYPT_DOMAIN: ${LETSENCRYPT_DOMAIN} + LETSENCRYPT_DOMAIN_ROOT: ${LETSENCRYPT_DOMAIN_ROOT} + LETSENCRYPT_EMAIL: ${LETSENCRYPT_EMAIL} restart: unless-stopped volumes: - /etc/localtime:/etc/localtime:ro @@ -45,8 +51,6 @@ services: target: 8888 - published: 591 target: 591 - env_file: - - .env logging: driver: "json-file" options: diff --git a/dd-sso/docker-compose-parts/haproxy.yml b/dd-sso/docker-compose-parts/haproxy.yml index c373259..2e1be97 100644 --- a/dd-sso/docker-compose-parts/haproxy.yml +++ b/dd-sso/docker-compose-parts/haproxy.yml @@ -28,6 +28,12 @@ services: dockerfile: Dockerfile target: production container_name: dd-sso-haproxy + environment: + DOMAIN: ${DOMAIN} + HAPROXY_CFG: ${HAPROXY_CFG:-haproxy.no-waf.no-haproxy.cfg} + LETSENCRYPT_DOMAIN: ${LETSENCRYPT_DOMAIN} + LETSENCRYPT_DOMAIN_ROOT: ${LETSENCRYPT_DOMAIN_ROOT} + LETSENCRYPT_EMAIL: ${LETSENCRYPT_EMAIL} restart: unless-stopped volumes: - /etc/localtime:/etc/localtime:ro @@ -40,8 +46,6 @@ services: target: 80 - published: 443 target: 443 - env_file: - - .env logging: driver: "json-file" options: diff --git a/dd-sso/docker-compose-parts/postgresql.yml b/dd-sso/docker-compose-parts/postgresql.yml deleted file mode 100644 index 60f9585..0000000 --- a/dd-sso/docker-compose-parts/postgresql.yml +++ /dev/null @@ -1,38 +0,0 @@ -# -# Copyright © 2021,2022 IsardVDI S.L. -# -# This file is part of DD -# -# DD is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as published by -# the Free Software Foundation, either version 3 of the License, or (at your -# option) any later version. -# -# DD is distributed in the hope that it will be useful, but WITHOUT ANY -# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS -# FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for more -# details. -# -# You should have received a copy of the GNU Affero General Public License -# along with DD. If not, see . -# -# SPDX-License-Identifier: AGPL-3.0-or-later -version: '3.7' -services: - dd-sso-postgresql: - image: ${POSTGRESQL_IMG-postgres:13.5-alpine3.15} - container_name: dd-sso-postgresql - restart: unless-stopped - env_file: - - .env - volumes: - - /etc/localtime:/etc/localtime:ro - - ${DB_FOLDER}/postgres:/var/lib/postgresql/data - - ${BUILD_SSO_ROOT_PATH}/init/databases:/docker-entrypoint-initdb.d - networks: - - dd_net - logging: - driver: "json-file" - options: - max-size: "5m" - max-file: "10" diff --git a/dd-sso/docker-compose-parts/waf-modsecurity.yml b/dd-sso/docker-compose-parts/waf-modsecurity.yml index 1817fb8..31295aa 100644 --- a/dd-sso/docker-compose-parts/waf-modsecurity.yml +++ b/dd-sso/docker-compose-parts/waf-modsecurity.yml @@ -7,8 +7,8 @@ services: dockerfile: Dockerfile target: production container_name: dd-waf-apache - env_file: - - .env + environment: + DISABLE_WAF: ${DISABLE_WAF:-true} restart: unless-stopped volumes: - /etc/localtime:/etc/localtime:ro diff --git a/dd.conf.sample b/dd.conf.sample index 407b43e..18b6159 100644 --- a/dd.conf.sample +++ b/dd.conf.sample @@ -187,8 +187,7 @@ POSTGRESQL_IMG=postgres:14.1-alpine3.15 #MARIADB_IMG=mariadb:10.6.5 ## NGINX -#NGINX_IMG=nginx:1.21.6 -#NGINX_ALPINE_IMG=nginx:1.21.6-alpine +#NGINX_ALPINE_IMG=nginx:1.22.1-alpine ## REDIS #REDIS_IMG=redis:6.2.6-alpine3.15 From e15a3b760c76b5cd0ecb0233f6b74d84a9a10181 Mon Sep 17 00:00:00 2001 From: Evilham Date: Wed, 11 Jan 2023 11:36:53 +0100 Subject: [PATCH 14/15] [dd.conf] Cleanup some variables While at it, also ensure that API_SECRET gets rotated when running ./dd-ctl securize --- dd-ctl | 4 ++-- dd.conf.sample | 18 ++++++------------ 2 files changed, 8 insertions(+), 14 deletions(-) diff --git a/dd-ctl b/dd-ctl index 8aa12c7..261e8dc 100755 --- a/dd-ctl +++ b/dd-ctl @@ -894,10 +894,10 @@ securize() { ETHERPAD_POSTGRES_PASSWORD \ ETHERPAD_ADMIN_PASSWORD \ WORDPRESS_MARIADB_PASSWORD \ - WORDPRESS_ADMIN_PASSWORD \ - IPA_ADMIN_PWD; do + WORDPRESS_ADMIN_PASSWORD; do setconf "${dd_var}" "$(genpwd)" done + setconf "API_SECRET" "$(openssl rand -base64 32)" } setconf() { diff --git a/dd.conf.sample b/dd.conf.sample index 18b6159..870a412 100644 --- a/dd.conf.sample +++ b/dd.conf.sample @@ -61,9 +61,9 @@ SMTP_PASSWORD=SuperSecret SMTP_PROTOCOL=tls ## DEFAULT CUSTOM ROLE NAMES -CUSTOM_ROLE_MANAGER='manager' -CUSTOM_ROLE_TEACHER='teacher' -CUSTOM_ROLE_STUDENT='student' +#CUSTOM_ROLE_MANAGER='manager' +#CUSTOM_ROLE_TEACHER='teacher' +#CUSTOM_ROLE_STUDENT='student' DDADMIN_USER=ddadmin @@ -73,7 +73,7 @@ DDADMIN_EMAIL=theemail@mymailserver.com # ------ Api Secret ----------------------------------------------------------- ## Generate your own SECRET! (or apply securize script) ## openssl rand -base64 32 -API_SECRET=LYY1kVYzbTSQx1yC4AauY7R6X34Jaz6+SY8CNC6RSno= +#API_SECRET=LYY1kVYzbTSQx1yC4AauY7R6X34Jaz6+SY8CNC6RSno= ## ADMINAPP (https://admin.$DOMAIN) ##============================================================================= @@ -130,22 +130,16 @@ ETHERPAD_POSTGRES_USER=etherpad ETHERPAD_POSTGRES_PASSWORD=3th3rpad ### ETHERPAD_API_KEY=NotImplemented -## POSTGRES (https://sso.$DOMAIN/dd-sso-adminer) +## POSTGRES ##============================================================================= -### The adminer user/pwd is admin/$KEYCLOAK_PASSWORD POSTGRES_USER=admin POSTGRES_PASSWORD=postgrespostgres -## MARIADB (https://sso.$DOMAIN/dd-sso-adminer) +## MARIADB ##============================================================================= -### The adminer user/pwd is admin/$KEYCLOAK_PASSWORD ### MARIADB_USER=root (it is the defaults in the container MARIADB_PASSWORD=SuperSecret -## FREEIPA (disabled) -##============================================================================= -IPA_ADMIN_PWD=freeipafreeipa - ## ACCEPT PROXY PROTOCOL ON 8888 (HTTP) AND 561 (HTTPS) #PROXY_PROTOCOL=false From ed44b8d3d292fdd29ac4eb83f2bb56a24dff9fac Mon Sep 17 00:00:00 2001 From: Evilham Date: Fri, 3 Feb 2023 11:40:13 +0100 Subject: [PATCH 15/15] [sso-api] Fix SocketIO transitive dependency (dnspython) Flask-SocketIO depends on dnspython but dnspython 2.3 removes dns.rdtypes.ANY, which is needed by Flask-SocketIO so we keep it below version 2.3 We had missed Flask-SocketIO being a dependency on API when fixing the issue for admin. --- dd-sso/docker/api/requirements.pip3 | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/dd-sso/docker/api/requirements.pip3 b/dd-sso/docker/api/requirements.pip3 index ece5a5a..b9177ad 100644 --- a/dd-sso/docker/api/requirements.pip3 +++ b/dd-sso/docker/api/requirements.pip3 @@ -20,4 +20,8 @@ Flask==2.0.1 eventlet==0.33.0 Flask-SocketIO==5.1.0 -python-keycloak==0.26.1 \ No newline at end of file +# Flask-SocketIO depends on dnspython +# but dnspython 2.3 removes dns.rdtypes.ANY, which is needed by Flask-SocketIO +# so we keep it below version 2.3 +dnspython<2.3 +python-keycloak==0.26.1