Merge branch 'main' into develop

merge-requests/18/head
elena 2022-08-04 09:48:39 +02:00
commit 8152d24b29
19 changed files with 153 additions and 105 deletions

View File

@ -57,8 +57,10 @@ environment in education:
- **Documents**: A document viewer and editor integrated with Nextcloud
- **Web pages**: A Wordpress instance with custom theme and custom plugins
- **Pad**: An Etherpad instance integrated with Nextcloud
- **Conferences**: BigBlueButton integrated with Moodle and Nextcloud (needs a standalone host)
- **Video calls**: BigBlueButton integrated with Moodle and Nextcloud (needs a standalone host)
- **Forms**: A forms Nextcloud plugin
- **Email, lists, chat, calendar, surveys...**
- **Import & export from other environments**
| | |
| ---------------------------- | ------------------------------- |
@ -98,6 +100,7 @@ public announcement on the
Using that version as a clean slate got us to the repo you see here, where
changes will be reviewed before going in and anyone is welcome.
We will reopen for the public the previous repository when evere we find the time to clean it up.
When in doubt about authorship, please check each file's license headers.
@ -117,4 +120,5 @@ The authorship of the previous commits is from:
- Raúl FS
- Unai Tolosa Pontesta
- Evilham
- Xnet
</details>

View File

@ -66,6 +66,10 @@ RUN mkdir -p \
COPY supervisord.conf /
# Temporary replacement for a real queue
RUN echo '*/1 * * * * /nc-queue.sh' >> /etc/crontabs/www-data
COPY nc-queue.sh /
ENV NEXTCLOUD_UPDATE=1
CMD ["/usr/bin/supervisord", "-c", "/supervisord.conf"]

View File

@ -0,0 +1,5 @@
#/bin/sh
find "${NC_MAIL_QUEUE_FOLDER:-/nc-mail-queue}" -name '*.sh' -exec sh -c \
'cd /var/www/html && {} && rm {}' \
';'

View File

@ -33,6 +33,7 @@ services:
- /etc/localtime:/etc/localtime:ro
- ${SRC_FOLDER}/nextcloud:/var/www/html
- ${DATA_FOLDER}/nextcloud:/var/www/html/data
- ${DATA_FOLDER}/nc-mail-queue:/nc-mail-queue:rw
environment:
- NEXTCLOUD_ADMIN_USER=${NEXTCLOUD_ADMIN_USER}
- NEXTCLOUD_ADMIN_PASSWORD=${NEXTCLOUD_ADMIN_PASSWORD}

View File

@ -1,22 +1,23 @@
#
# Copyright © 2021,2022 IsardVDI S.L.
#
# This file is part of DD
#
# DD is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or (at your
# option) any later version.
#
# DD is distributed in the hope that it will be useful, but WITHOUT ANY
# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
# FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for more
# details.
#
# You should have received a copy of the GNU Affero General Public License
# along with DD. If not, see <https://www.gnu.org/licenses/>.
#
# SPDX-License-Identifier: AGPL-3.0-or-later
/*
* Copyright © 2021,2022 IsardVDI S.L.
*
* This file is part of DD
*
* DD is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as published by
* the Free Software Foundation, either version 3 of the License, or (at your
* option) any later version.
*
* DD is distributed in the hope that it will be useful, but WITHOUT ANY
* WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
* FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for more
* details.
*
* You should have received a copy of the GNU Affero General Public License
* along with DD. If not, see <https://www.gnu.org/licenses/>.
*
* SPDX-License-Identifier: AGPL-3.0-or-later
*/
jQuery(document).ready(() => {
base_url = `${window.location.protocol}//${window.location.host.replace(/^nextcloud\./, 'api.')}`
$.getJSON(`${base_url}/json`, (result) => {

View File

@ -53,10 +53,10 @@
<link rel="apple-touch-icon-precomposed" href="<?php print_unescaped(image_path($_['appid'], 'favicon-touch.png')); ?>">
<link rel="mask-icon" sizes="any" href="<?php print_unescaped(image_path($_['appid'], 'favicon-mask.svg')); ?>" color="<?php p($theme->getColorPrimary()); ?>">
<link rel="manifest" href="<?php print_unescaped(image_path($_['appid'], 'manifest.json')); ?>">
<script nonce="<?php p(\OC::$server->getContentSecurityPolicyNonceManager()->getNonce()) ?>" src="/themes/digitaldemocratic/core/js/jquery_slim_3.2.1.js"></script>
<script nonce="<?php p(\OC::$server->getContentSecurityPolicyNonceManager()->getNonce()) ?>" src="/themes/dd/core/js/jquery_slim_3.2.1.js"></script>
<link rel="stylesheet" href="<?php p($api_url) ?>/css/font-awesome-4.7.0/css/font-awesome.min.css">
<link rel="stylesheet" href="<?php p($api_url) ?>/css/dd.css">
<script nonce="<?php p(\OC::$server->getContentSecurityPolicyNonceManager()->getNonce()) ?>" src="/themes/digitaldemocratic/core/js/navbar.js"></script>
<script nonce="<?php p(\OC::$server->getContentSecurityPolicyNonceManager()->getNonce()) ?>" src="/themes/dd/core/js/navbar.js"></script>
<?php emit_css_loading_tags($_); ?>
<?php emit_script_loading_tags($_); ?>
<?php print_unescaped($_['headers']); ?>
@ -137,7 +137,7 @@
</div>
<div id="product-logo">
<a href="https://xnet-x.net/ca/digital-democratic/" target="_blank">
<img src="/themes/digitaldemocratic/core/img/dd.svg" alt="" style="height: 16px; margin-right: 21px; margin-top: 16px"/>
<img src="/themes/dd/core/img/dd.svg" alt="" style="height: 16px; margin-right: 21px; margin-top: 16px"/>
</a>
</div>
</div>

8
dd-ctl
View File

@ -655,9 +655,11 @@ configure_nextcloud_logo(){
local instance_id=$(docker exec -u www-data dd-apps-nextcloud-app php occ config:system:get instanceid)
local cachebuster=$(docker exec -u www-data dd-apps-nextcloud-app php occ config:app:get theming cachebuster)
docker exec -u www-data dd-apps-nextcloud-app mkdir -p /var/www/html/data/appdata_$instance_id/theming/images
docker cp custom/img/logo.png dd-apps-nextcloud-app:/var/www/html/data/appdata_$instance_id/theming/images/logo
docker cp custom/img/background.png dd-apps-nextcloud-app:/var/www/html/data/appdata_$instance_id/theming/images/background
docker exec dd-apps-nextcloud-app chown www-data:www-data /var/www/html/data/appdata_$instance_id/theming/images/{logo,background}
nc_logo="${DATA_FOLDER}/nextcloud/appdata_$instance_id/theming/images/logo"
nc_background="${DATA_FOLDER}/nextcloud/appdata_$instance_id/theming/images/background"
cp custom/img/logo.png "${nc_logo}"
cp custom/img/background.png "${nc_background}"
chown 82:82 "${nc_logo}" "${nc_background}"
docker exec -u www-data dd-apps-nextcloud-app php occ config:app:set theming logoMime --value="image/png"
docker exec -u www-data dd-apps-nextcloud-app php occ config:app:set theming backgroundMime --value="image/png"
docker exec -u www-data dd-apps-nextcloud-app php occ config:app:set theming cachebuster --value="$(expr $cachebuster + 1 )"

3
dd-sso/.gitignore vendored
View File

@ -6,9 +6,6 @@ docker-compose.yml
**/custom.yaml
**/system.yaml
admin/src/node_modules
admin/src/admin/node_modules/
# Byte-compiled / optimized / DLL files
__pycache__/
*.py[cod]

View File

@ -20,6 +20,9 @@
FROM alpine:3.12.0 as production
MAINTAINER isard <info@isardvdi.com>
# Ensure python dependencies
COPY admin/docker/requirements.pip3 /requirements.pip3
RUN apk add python3 py3-pip py3-pyldap~=3.2.0
RUN pip3 install --upgrade pip
RUN apk add --no-cache --virtual .build_deps \
@ -27,37 +30,28 @@ RUN apk add --no-cache --virtual .build_deps \
python3-dev \
libffi-dev \
gcc python3-dev linux-headers musl-dev postgresql-dev
COPY admin/docker/requirements.pip3 /requirements.pip3
RUN pip3 install --no-cache-dir -r requirements.pip3
RUN apk del .build_deps
RUN apk add --no-cache curl py3-yaml yarn libpq openssl py3-pillow
# Add catalan words list (issue with newer diceweare)
RUN wget -O /usr/lib/python3.8/site-packages/diceware/wordlists/wordlist_cat_ascii.txt https://raw.githubusercontent.com/1ma/diceware-cat/master/cat-wordlist-ascii.txt
# SSH configuration
# ARG SSH_ROOT_PWD
# RUN apk add openssh
# RUN echo "root:$SSH_ROOT_PWD" |chpasswd
# RUN sed -i \
# -e 's|[#]*PermitRootLogin prohibit-password|PermitRootLogin yes|g' \
# -e 's|[#]*PasswordAuthentication yes|PasswordAuthentication yes|g' \
# -e 's|[#]*ChallengeResponseAuthentication yes|ChallengeResponseAuthentication yes|g' \
# -e 's|[#]*UsePAM yes|UsePAM yes|g' \
# -e 's|[#]#Port 22|Port 22|g' \
# /etc/ssh/sshd_config
# Let's test 0.26.1 python-keycloak version
# RUN apk add --no-cache git && \
# git clone -b delete_realm_roles https://github.com/isard-vdi/python-keycloak.git && \
# cd python-keycloak && \
# python3 setup.py install && \
# apk del git
# Add code and entrypoint
COPY admin/src /admin
RUN cd /admin/admin && yarn install
COPY admin/docker/run.sh /run.sh
#EXPOSE 7039
CMD [ "/run.sh" ]
# Ensure www-data group and user (82 is default in alpine)
RUN addgroup -g 82 -S www-data; adduser -u 82 -D -S -G www-data www-data
# Fix directory permissions
# Ensure node dependencies too
RUN cd /admin/admin && \
chown www-data:www-data "." && \
mkdir -p "${NODE_MODULES_FOLDER:-node_modules}" && \
chown www-data:www-data "${NODE_MODULES_FOLDER:-node_modules}" && \
HOME=/tmp su -s /bin/sh -m www-data -c \
"yarn install --modules-folder '${NODE_MODULES_FOLDER:-node_modules}'"
CMD [ "/run.sh" ]

View File

@ -30,7 +30,8 @@ mysql-connector-python==8.0.30
psycopg2==2.9.3
# python-keycloak can't be upgraded without issues
python-keycloak==0.26.1
minio==7.1.11
# minio can't be upgraded without issues
minio==7.0.3
urllib3==1.26.11
schema==0.7.5
Werkzeug==2.2.1

View File

@ -18,13 +18,15 @@
# along with DD. If not, see <https://www.gnu.org/licenses/>.
#
# SPDX-License-Identifier: AGPL-3.0-or-later
# ssh-keygen -A
## Only in development
cd /admin/admin
yarn install
## End Only in development
# We possibly need to fix bad old permissions
chown -R www-data:www-data \
/admin/custom \
/admin/moodledata/saml2 /admin/saml_certs \
"${DATA_FOLDER}" \
"${LEGAL_PATH}" \
"${NC_MAIL_QUEUE_FOLDER}"
cd /admin
export PYTHONWARNINGS="ignore:Unverified HTTPS request"
python3 start.py
#&
# /usr/sbin/sshd -D -e -f /etc/ssh/sshd_config
exec su -s /bin/sh -m www-data -c 'python3 start.py'

View File

@ -73,7 +73,9 @@ class AdminFlaskApp(Flask):
custom_dir: str
data_dir: str
domain : str
node_modules_dir : str
ready: bool = False
validators: Dict
def __init__(self, *args: Any, **kwargs: Any):
super().__init__(*args, **kwargs)
@ -82,7 +84,7 @@ class AdminFlaskApp(Flask):
self.url_map.strict_slashes = False
self._load_config()
# Minor setup tasks
self._load_validators()
self.validators = self._load_validators()
self._setup_routes()
self._setup_api_3p()
setup_api_views(self)
@ -141,6 +143,7 @@ class AdminFlaskApp(Flask):
try:
self.data_dir = os.environ.get("DATA_FOLDER", ".")
self.custom_dir = os.environ.get("CUSTOM_FOLDER", ".")
self.node_modules_dir = os.environ.get("NODE_MODULES_FOLDER", "node_modules")
# Handle secrets like Flask's session key
secret_key_file = os.path.join(self.secrets_dir, "secret_key")
if not os.path.exists(self.secrets_dir):
@ -202,19 +205,19 @@ class AdminFlaskApp(Flask):
@self.route("/build/<path:path>")
def send_build(path: str) -> Response:
return send_from_directory(
os.path.join(self.root_path, "node_modules/gentelella/build"), path
os.path.join(self.node_modules_dir, "gentelella/build"), path
)
@self.route("/vendors/<path:path>")
def send_vendors(path: str) -> Response:
return send_from_directory(
os.path.join(self.root_path, "node_modules/gentelella/vendors"), path
os.path.join(self.node_modules_dir, "gentelella/vendors"), path
)
@self.route("/node_modules/<path:path>")
def send_nodes(path: str) -> Response:
return send_from_directory(
os.path.join(self.root_path, "node_modules"), path
self.node_modules_dir, path
)
@self.route("/templates/<path:path>")

View File

@ -22,6 +22,8 @@ import json
import logging as log
import os
import traceback
from datetime import datetime
from pathlib import Path
from pprint import pprint
from time import sleep
@ -60,7 +62,7 @@ from .helpers import (
rand_password,
)
from typing import TYPE_CHECKING, cast, Any, Dict, Iterable, List, Optional
from typing import TYPE_CHECKING, cast, Any, Dict, Iterable, List, Optional, Tuple
if TYPE_CHECKING:
from admin.flaskapp import AdminFlaskApp
from admin.lib.callbacks import ThirdPartyCallbacks
@ -130,12 +132,41 @@ class Admin:
res = res and tp.delete_user(user_id)
return res
def nextcloud_mail_set(self, users : List[DDUser], extra_data : Dict) -> Dict:
# TODO: implement
return {}
def _nextcloud_mail_set_cmd(self, user : DDUser, kw : Dict) -> Tuple[str, str]:
account_name = 'DD' # Treating this as a constant
update_cmd = f"""mail:account:update \
--imap-host '{ kw['inbound_host'] }' --imap-port '{ kw['inbound_port'] }' --imap-ssl-mode '{ kw['inbound_ssl_mode'] }' \\
--imap-user '{ user['email'] }' --imap-password '{ user['password'] }' \\
--smtp-host '{ kw['outbound_host'] }' --smtp-port '{ kw['outbound_port'] }' --smtp-ssl-mode '{ kw['outbound_ssl_mode'] }' \\
--smtp-user '{ user['email'] }' --smtp-password '{ user['password'] }' \\
-- '{ user['user_id'] }' '{ user['email']}'"""
create_cmd = f"""mail:account:create '{ user['user_id'] }' '{ account_name }' '{ user['email'] }' \\
'{ kw['inbound_host'] }' '{ kw['inbound_port'] }' '{ kw['inbound_ssl_mode'] }' \\
'{ user['email'] }' '{ user['password'] }' \\
'{ kw['outbound_host'] }' '{ kw['outbound_port'] }' '{ kw['outbound_ssl_mode'] }' \\
'{ user['email'] }' '{ user['password'] }'"""
return (update_cmd, create_cmd)
def nextcloud_mail_delete(self, users : List[DDUser], extra_data : Dict) -> Dict:
# TODO: implement
def _nextcloud_mail_set_sh(self, users : List[DDUser], extra_data : Dict) -> str:
cmds = '\n'.join((f"./occ {u} || ./occ {c}" for u, c in (self._nextcloud_mail_set_cmd(u, extra_data) for u in users)))
return f"""#!/bin/sh -eu
{cmds}
"""
def nextcloud_mail_set(self, users : List[DDUser], extra_data : Dict) -> Dict:
# TODO: this could (and should) be nicer.
# Ideally we'd use the database as a queue instead of creating the
# shell scripts here.
d = Path(os.environ.get("NC_MAIL_QUEUE_FOLDER", "/nc-mail-queue"))
fn = datetime.utcnow().isoformat() + secrets.token_hex(4)
sh = d.joinpath(fn + '.sh')
tmp = d.joinpath(fn + '.tmp')
# Create executable file
tmp.touch(mode=0o750)
# Write script
tmp.write_text(self._nextcloud_mail_set_sh(users, extra_data))
# Put it in-place
tmp.rename(sh)
return {}
def check_connections(self, app : "AdminFlaskApp") -> None:
@ -1615,7 +1646,7 @@ class Admin:
internaluser : DDUser = [u for u in self.internal["users"] if u["id"] == user_id][0]
cohorts = self.moodle.get_cohorts()
for group in mdelete:
cohort = [c for c in cohorts if c["name"] == group[0]][0]
cohort = [c for c in cohorts if c["name"] == group][0]
try:
self.moodle.delete_user_in_cohort(
internaluser["moodle_id"], cohort["id"]
@ -1905,7 +1936,7 @@ class Admin:
" NEXTCLOUD USERS: Creating nextcloud user: "
+ u["username"]
+ " in groups "
+ str(list)
+ str(u.get("groups", []))
)
try:
# Quota is in MB

View File

@ -45,7 +45,7 @@ class Avatars:
# self.update_missing_avatars()
def add_user_default_avatar(self, userid : str, role : str="unknown") -> None:
path = os.path.join(self.avatars_path, role) + ".jpg",
path = os.path.join(self.avatars_path, role) + ".jpg"
self.mclient.fput_object(
self.bucket,
userid,

View File

@ -37,7 +37,7 @@ import stat
from copy import deepcopy
from datetime import datetime, timedelta
from pathlib import Path
from typing import Any, Dict, List, Optional, Union
from typing import Any, Callable, Dict, List, Optional, Union
import requests
from attr import field, frozen
@ -52,10 +52,15 @@ from jose.backends.rsa_backend import RSAKey
from jose.constants import ALGORITHMS
try:
# Python 3.8
from functools import cached_property as cache
except ImportError:
from functools import cache # type: ignore # Python 3.9+
except ImportError:
try:
from functools import cached_property as cache # type: ignore # Python 3.8
except ImportError:
from functools import lru_cache
def cache(call: Callable) -> property: # type: ignore # Python 3.7
return property(lru_cache()(call))
Data = Union[str, bytes]

View File

@ -46,7 +46,7 @@ def setup_mail_views(app: "AdminFlaskApp") -> None:
key = json.dumps(mail_3p.our_pubkey_jwk)
return key, 200, {"Content-Type": "application/json"}
@app.route("/ddapi/mailusers", methods=["GET", "POST", "PUT", "DELETE"])
@app.route("/ddapi/mailusers", methods=["GET", "POST", "PUT"])
@has_jws_token(app)
def ddapi_mail_users() -> JsonResponse:
users: List[Dict[str, Any]] = []
@ -66,8 +66,10 @@ def setup_mail_views(app: "AdminFlaskApp") -> None:
raise Error(
"internal_server", "Failure sending users", traceback.format_exc()
)
if request.method not in ["POST", "PUT", "DELETE"]:
if request.method not in ["POST", "PUT"]:
# Unsupported method
# Note we do not support DELETE as it is taken care of when the
# full Nextcloud user is deleted.
return json.dumps({}), 400, JsonHeaders
try:
@ -75,19 +77,19 @@ def setup_mail_views(app: "AdminFlaskApp") -> None:
Dict, mail_3p.verify_and_decrypt_incoming_json(request.get_data())
)
users = dec_data.pop("users")
for user in users:
if not app.validators["mail"].validate(user):
raise Error(
"bad_request",
"Data validation for mail failed: "
+ str(app.validators["mail"].errors),
traceback.format_exc(),
)
config = dec_data.pop("config", {})
# TODO: fix these validators
#for user in users:
# if not app.validators["mail"].validate(user):
# raise Error(
# "bad_request",
# "Data validation for mail failed: "
# + str(app.validators["mail"].errors),
# traceback.format_exc(),
# )
res: Dict
if request.method in ["POST", "PUT"]:
res = app.admin.nextcloud_mail_set(users, dec_data)
elif request.method == "DELETE":
res = app.admin.nextcloud_mail_delete(users, dec_data)
res = app.admin.nextcloud_mail_set(users, config)
return (
json.dumps(res),
200,

View File

@ -25,25 +25,19 @@ services:
context: ${BUILD_SSO_ROOT_PATH}
dockerfile: admin/docker/Dockerfile
target: production
# args: ## DEVELOPMENT
# SSH_ROOT_PWD: ${IPA_ADMIN_PWD}
# SSH_PORT: 2022
networks:
- dd_net
# ports:
# - "2022:22"
# - "9000:9000"
restart: unless-stopped
volumes:
- /etc/localtime:/etc/localtime:ro
- ${BUILD_SSO_ROOT_PATH}/admin/src:/admin # Revome in production
- ${BUILD_SSO_ROOT_PATH}/init/keycloak/jsons:/admin/keycloak-init:ro
- ${CUSTOM_PATH}/custom:/admin/custom
- ${CUSTOM_PATH}/custom:/admin/custom:rw
- ${DATA_FOLDER}/avatars:/admin/avatars:ro
- ${DATA_FOLDER}/moodle/saml2:/admin/moodledata/saml2:rw
- ${DATA_FOLDER}/saml_certs:/admin/saml_certs:rw
- ${DATA_FOLDER}/legal:/admin/admin/static/templates/pages/legal:rw
- ${DATA_FOLDER}/dd-admin:/data:rw
- ${DATA_FOLDER}/nc-mail-queue:/nc-mail-queue:rw
env_file:
- .env
environment:
@ -52,3 +46,5 @@ services:
- MANAGED_EMAIL_DOMAIN=${MANAGED_EMAIL_DOMAIN}
- DATA_FOLDER=/data
- CUSTOM_FOLDER=/admin/custom
- NC_MAIL_QUEUE_FOLDER=/nc-mail-queue
- LEGAL_PATH=/admin/admin/static/templates/pages/legal

View File

@ -19,9 +19,9 @@
# SPDX-License-Identifier: AGPL-3.0-or-later
version: '3.7'
services:
isard-sso-adminer:
dd-sso-adminer:
image: adminer
container_name: isard-sso-adminer
container_name: dd-sso-adminer
restart: unless-stopped
networks:
isard_net: {}

View File

@ -19,10 +19,10 @@
# SPDX-License-Identifier: AGPL-3.0-or-later
version: '3.7'
services:
isard-sso-pgtuner:
dd-sso-pgtuner:
image: jfcoz/postgresqltuner
container_name: isard-sso-pgtuner
container_name: dd-sso-pgtuner
restart: "no"
command: --host isard-apps-postgresql --user ${NEXTCLOUD_POSTGRES_USER} --password ${NEXTCLOUD_POSTGRES_PASSWORD} --database nextcloud
command: --host dd-apps-postgresql --user ${NEXTCLOUD_POSTGRES_USER} --password ${NEXTCLOUD_POSTGRES_PASSWORD} --database nextcloud
networks:
isard_net: {}