diff --git a/admin/src/admin/__init__.py b/admin/src/admin/__init__.py index bf623a5..fc48415 100644 --- a/admin/src/admin/__init__.py +++ b/admin/src/admin/__init__.py @@ -1,77 +1,96 @@ #!flask/bin/python # coding=utf-8 -import os import logging as log +import os -from flask import Flask, send_from_directory, render_template -app = Flask(__name__, static_url_path='') -app = Flask(__name__, template_folder='static/templates') +from flask import Flask, render_template, send_from_directory + +app = Flask(__name__, static_url_path="") +app = Flask(__name__, template_folder="static/templates") app.url_map.strict_slashes = False -''' +""" App secret key for encrypting cookies You can generate one with: import os os.urandom(24) And paste it here. -''' +""" app.secret_key = "Change this key!/\xf7\x83\xbe\x17\xfa\xa3zT\n\\]m\xa6\x8bF\xdd\r\xf7\x9e\x1d\x1f\x14'" -print('Starting isard-sso api...') +print("Starting isard-sso api...") from admin.lib.load_config import loadConfig + try: loadConfig(app) except: - print('Could not get environment variables...') + print("Could not get environment variables...") from admin.lib.postup import Postup + Postup() from admin.lib.admin import Admin -app.admin=Admin() -app.ready=False +app.admin = Admin() -''' +app.ready = False + +""" Debug should be removed on production! -''' +""" if app.debug: - log.warning('Debug mode: {}'.format(app.debug)) + log.warning("Debug mode: {}".format(app.debug)) else: - log.info('Debug mode: {}'.format(app.debug)) + log.info("Debug mode: {}".format(app.debug)) -''' +""" Serve static files -''' -@app.route('/build/') -def send_build(path): - return send_from_directory(os.path.join(app.root_path, 'node_modules/gentelella/build'), path) - -@app.route('/vendors/') -def send_vendors(path): - return send_from_directory(os.path.join(app.root_path, 'node_modules/gentelella/vendors'), path) +""" -@app.route('/templates/') + +@app.route("/build/") +def send_build(path): + return send_from_directory( + os.path.join(app.root_path, "node_modules/gentelella/build"), path + ) + + +@app.route("/vendors/") +def send_vendors(path): + return send_from_directory( + os.path.join(app.root_path, "node_modules/gentelella/vendors"), path + ) + + +@app.route("/templates/") def send_templates(path): - return send_from_directory(os.path.join(app.root_path, 'templates'), path) + return send_from_directory(os.path.join(app.root_path, "templates"), path) + # @app.route('/templates/') # def send_templates(path): # return send_from_directory(os.path.join(app.root_path, 'static/templates'), path) -@app.route('/static/') + +@app.route("/static/") def send_static_js(path): - return send_from_directory(os.path.join(app.root_path, 'static'), path) + return send_from_directory(os.path.join(app.root_path, "static"), path) -@app.route('/avatars/') + +@app.route("/avatars/") def send_avatars_img(path): - return send_from_directory(os.path.join(app.root_path, '../avatars/master-avatars'), path) + return send_from_directory( + os.path.join(app.root_path, "../avatars/master-avatars"), path + ) -@app.route('/custom/') + +@app.route("/custom/") def send_custom(path): - return send_from_directory(os.path.join(app.root_path, '../custom'), path) + return send_from_directory(os.path.join(app.root_path, "../custom"), path) + # @app.errorhandler(404) # def not_found_error(error): @@ -81,15 +100,7 @@ def send_custom(path): # def internal_error(error): # return render_template('page_500.html'), 500 -''' +""" Import all views -''' -from .views import LoginViews -from .views import WebViews -from .views import ApiViews -from .views import InternalViews - - - - - +""" +from .views import ApiViews, InternalViews, LoginViews, WebViews diff --git a/admin/src/admin/auth/authentication.py b/admin/src/admin/auth/authentication.py index 25114d4..cc2d632 100644 --- a/admin/src/admin/auth/authentication.py +++ b/admin/src/admin/auth/authentication.py @@ -1,8 +1,10 @@ -from admin import app -from flask_login import LoginManager, UserMixin import os -''' OIDC TESTS ''' +from flask_login import LoginManager, UserMixin + +from admin import app + +""" OIDC TESTS """ # from flask_oidc import OpenIDConnect # app.config.update({ # 'SECRET_KEY': 'u\x91\xcf\xfa\x0c\xb9\x95\xe3t\xba2K\x7f\xfd\xca\xa3\x9f\x90\x88\xb8\xee\xa4\xd6\xe4', @@ -18,7 +20,7 @@ import os # # 'OVERWRITE_REDIRECT_URI': 'https://sso.mydomain.duckdns.org//custom_callback', # # 'OIDC_CALLBACK_ROUTE': '//custom_callback' # oidc = OpenIDConnect(app) -''' OIDC TESTS ''' +""" OIDC TESTS """ login_manager = LoginManager() @@ -26,31 +28,33 @@ login_manager.init_app(app) login_manager.login_view = "login" -ram_users={ - os.environ["ADMINAPP_USER"]: { - 'id': os.environ["ADMINAPP_USER"], - 'password': os.environ["ADMINAPP_PASSWORD"], - 'role': 'manager' - }, - os.environ["KEYCLOAK_USER"]: { - 'id': os.environ["KEYCLOAK_USER"], - 'password': os.environ["KEYCLOAK_PASSWORD"], - 'role': 'admin', - }, - os.environ["WORDPRESS_MARIADB_USER"]: { - 'id': os.environ["WORDPRESS_MARIADB_USER"], - 'password': os.environ["WORDPRESS_MARIADB_PASSWORD"], - 'role': 'manager', - } +ram_users = { + os.environ["ADMINAPP_USER"]: { + "id": os.environ["ADMINAPP_USER"], + "password": os.environ["ADMINAPP_PASSWORD"], + "role": "manager", + }, + os.environ["KEYCLOAK_USER"]: { + "id": os.environ["KEYCLOAK_USER"], + "password": os.environ["KEYCLOAK_PASSWORD"], + "role": "admin", + }, + os.environ["WORDPRESS_MARIADB_USER"]: { + "id": os.environ["WORDPRESS_MARIADB_USER"], + "password": os.environ["WORDPRESS_MARIADB_PASSWORD"], + "role": "manager", + }, } + class User(UserMixin): - def __init__(self, dict): - self.id = dict['id'] - self.username = dict['id'] - self.password = dict['password'] - self.role = dict['role'] + def __init__(self, dict): + self.id = dict["id"] + self.username = dict["id"] + self.password = dict["password"] + self.role = dict["role"] + @login_manager.user_loader def user_loader(username): - return User(ram_users[username]) + return User(ram_users[username]) diff --git a/admin/src/admin/lib/admin.py b/admin/src/admin/lib/admin.py index 32abc6c..377e2de 100644 --- a/admin/src/admin/lib/admin.py +++ b/admin/src/admin/lib/admin.py @@ -1,34 +1,45 @@ +import json +import logging as log +import os +import traceback +from pprint import pprint +from time import sleep + +import diceware + from admin import app +from .avatars import Avatars +from .helpers import ( + filter_roles_list, + filter_roles_listofdicts, + get_gids_from_kgroup_ids, + get_group_from_group_id, + gid2kpath, + kpath2gid, + system_username, +) from .keycloak_client import KeycloakClient from .moodle import Moodle from .nextcloud import Nextcloud from .nextcloud_exc import ProviderItemExists -from .avatars import Avatars -from .helpers import filter_roles_list, filter_roles_listofdicts, system_username, get_gids_from_kgroup_ids, get_group_from_group_id, kpath2gid, gid2kpath - -import logging as log -from pprint import pprint -import traceback, os, json -from time import sleep - -import diceware options = diceware.handle_options(None) -options.wordlist = 'cat_ascii' +options.wordlist = "cat_ascii" options.num = 3 -from .helpers import rand_password, count_repeated -from .exceptions import UserExists, UserNotFound -from .events import Events - import secrets -MANAGER=os.environ['CUSTOM_ROLE_MANAGER'] -TEACHER=os.environ['CUSTOM_ROLE_TEACHER'] -STUDENT=os.environ['CUSTOM_ROLE_STUDENT'] +from .events import Events +from .exceptions import UserExists, UserNotFound +from .helpers import count_repeated, rand_password -class Admin(): +MANAGER = os.environ["CUSTOM_ROLE_MANAGER"] +TEACHER = os.environ["CUSTOM_ROLE_TEACHER"] +STUDENT = os.environ["CUSTOM_ROLE_STUDENT"] + + +class Admin: def __init__(self): self.check_connections() @@ -36,135 +47,151 @@ class Admin(): self.overwrite_admins() self.default_setup() - self.internal={} + self.internal = {} - ready=False + ready = False while not ready: try: self.resync_data() - ready=True + ready = True except: print(traceback.format_exc()) - log.error('Could not resync data, waiting for system to be online...') + log.error("Could not resync data, waiting for system to be online...") sleep(2) self.sync_groups_from_keycloak() - self.external={'users':[], - 'groups':[], - 'roles':[]} + self.external = {"users": [], "groups": [], "roles": []} - log.warning(' Updating missing user avatars with defaults') - self.av=Avatars() + log.warning(" Updating missing user avatars with defaults") + self.av = Avatars() # av.minio_delete_all_objects() # This will reset all avatars on usres - self.av.update_missing_avatars(self.internal['users']) + self.av.update_missing_avatars(self.internal["users"]) - log.warning(' SYSTEM READY TO HANDLE CONNECTIONS') + log.warning(" SYSTEM READY TO HANDLE CONNECTIONS") def check_connections(self): - ready=False + ready = False while not ready: try: - self.keycloak=KeycloakClient(verify=app.config['VERIFY']) - ready=True + self.keycloak = KeycloakClient(verify=app.config["VERIFY"]) + ready = True except: log.error(traceback.format_exc()) - log.error('Could not connect to keycloak, waiting to be online...') + log.error("Could not connect to keycloak, waiting to be online...") sleep(2) - log.warning('Keycloak connected.') + log.warning("Keycloak connected.") - ready=False + ready = False while not ready: try: - self.moodle=Moodle(verify=app.config['VERIFY']) - ready=True + self.moodle = Moodle(verify=app.config["VERIFY"]) + ready = True except: - log.error('Could not connect to moodle, waiting to be online...') + log.error("Could not connect to moodle, waiting to be online...") sleep(2) - log.warning('Moodle connected.') + log.warning("Moodle connected.") - ready=False + ready = False while not ready: try: - with open(os.path.join(app.root_path, "../moodledata/saml2/moodle."+app.config['DOMAIN']+".pem"),"r") as pem: - ready=True + with open( + os.path.join( + app.root_path, + "../moodledata/saml2/moodle." + app.config["DOMAIN"] + ".pem", + ), + "r", + ) as pem: + ready = True except IOError: - log.warning('Could not get moodle SAML2 pem certificate. Retrying...') + log.warning("Could not get moodle SAML2 pem certificate. Retrying...") sleep(2) - ready=False + ready = False while not ready: try: - self.nextcloud=Nextcloud(verify=app.config['VERIFY']) - ready=True + self.nextcloud = Nextcloud(verify=app.config["VERIFY"]) + ready = True except: - log.error('Could not connect to nextcloud, waiting to be online...') + log.error("Could not connect to nextcloud, waiting to be online...") sleep(2) - log.warning('Nextcloud connected.') + log.warning("Nextcloud connected.") def set_custom_roles(self): pass ## This function should be moved to postup.py def overwrite_admins(self): - log.warning('Setting defaults...') - dduser=os.environ['DDADMIN_USER'] - ddpassword=os.environ['DDADMIN_PASSWORD'] - ddmail=os.environ['DDADMIN_EMAIL'] + log.warning("Setting defaults...") + dduser = os.environ["DDADMIN_USER"] + ddpassword = os.environ["DDADMIN_PASSWORD"] + ddmail = os.environ["DDADMIN_EMAIL"] ### User admin in group admin try: - log.warning('KEYCLOAK: Adding group admin and user admin to this group') - admin_guid=self.keycloak.add_group('admin') + log.warning("KEYCLOAK: Adding group admin and user admin to this group") + admin_guid = self.keycloak.add_group("admin") except: pass - admin_guid=self.keycloak.get_group_by_path(path='/admin')['id'] + admin_guid = self.keycloak.get_group_by_path(path="/admin")["id"] try: - ## Add default admin user to group admin - admin_uid=self.keycloak.get_user_id('admin') - self.keycloak.group_user_add(admin_uid,admin_guid) - log.warning('KEYCLOAK: OK') + ## Add default admin user to group admin + admin_uid = self.keycloak.get_user_id("admin") + self.keycloak.group_user_add(admin_uid, admin_guid) + log.warning("KEYCLOAK: OK") except: print(traceback.format_exc()) - log.warning('KEYCLOAK: Seems to be there already') + log.warning("KEYCLOAK: Seems to be there already") ### ddadmin user try: - log.warning('KEYCLOAK: Adding user ddadmin and adding to group and role admin') - uid=self.keycloak.add_user(dduser,'DD','Admin',ddmail,ddpassword,group='admin',temporary=False) - self.keycloak.assign_realm_roles(uid,'admin') - log.warning('KEYCLOAK: OK') + log.warning( + "KEYCLOAK: Adding user ddadmin and adding to group and role admin" + ) + uid = self.keycloak.add_user( + dduser, + "DD", + "Admin", + ddmail, + ddpassword, + group="admin", + temporary=False, + ) + self.keycloak.assign_realm_roles(uid, "admin") + log.warning("KEYCLOAK: OK") except: - log.warning('KEYCLOAK: Seems to be there already') + log.warning("KEYCLOAK: Seems to be there already") try: - log.warning('MOODLE: Adding default group admin') - self.moodle.add_system_cohort('admin','system admins') - log.warning('MOODLE: OK') + log.warning("MOODLE: Adding default group admin") + self.moodle.add_system_cohort("admin", "system admins") + log.warning("MOODLE: OK") except: - log.warning('MOODLE: Seems to be there already') + log.warning("MOODLE: Seems to be there already") try: - log.warning('MOODLE: Adding user ddadmin and adding to siteadmins') - self.moodle.create_user(ddmail,dduser,ddpassword,'DD','Admin') - uid=self.moodle.get_user_by('username',dduser)['users'][0]['id'] + log.warning("MOODLE: Adding user ddadmin and adding to siteadmins") + self.moodle.create_user(ddmail, dduser, ddpassword, "DD", "Admin") + uid = self.moodle.get_user_by("username", dduser)["users"][0]["id"] self.moodle.add_user_to_siteadmin(uid) - log.warning('MOODLE: OK') + log.warning("MOODLE: OK") except: - log.warning('MOODLE: Seems to be there already') + log.warning("MOODLE: Seems to be there already") try: - log.warning('NEXTCLOUD: Adding default group admin') - self.nextcloud.add_group('admin') - log.warning('NEXTCLOUD: OK') + log.warning("NEXTCLOUD: Adding default group admin") + self.nextcloud.add_group("admin") + log.warning("NEXTCLOUD: OK") except ProviderItemExists: - log.warning('NEXTCLOUD: Seems to be there already') + log.warning("NEXTCLOUD: Seems to be there already") try: - log.warning('NEXTCLOUD: Adding user ddadmin and adding to group admin') - self.nextcloud.add_user(dduser,ddpassword,group='admin',email=ddmail,displayname='DD Admin') - log.warning('NEXTCLOUD: OK') + log.warning("NEXTCLOUD: Adding user ddadmin and adding to group admin") + self.nextcloud.add_user( + dduser, ddpassword, group="admin", email=ddmail, displayname="DD Admin" + ) + log.warning("NEXTCLOUD: OK") except ProviderItemExists: - log.warning('NEXTCLOUD: Seems to be there already') + log.warning("NEXTCLOUD: Seems to be there already") except: log.error(traceback.format_exc()) exit(1) @@ -172,91 +199,118 @@ class Admin(): def default_setup(self): ### Add default roles try: - log.warning('KEYCLOAK: Adding default roles') - self.keycloak.add_role(MANAGER,'Realm managers') - self.keycloak.add_role(TEACHER,'Realm teachers') - self.keycloak.add_role(STUDENT,'Realm students') - log.warning('KEYCLOAK: OK') + log.warning("KEYCLOAK: Adding default roles") + self.keycloak.add_role(MANAGER, "Realm managers") + self.keycloak.add_role(TEACHER, "Realm teachers") + self.keycloak.add_role(STUDENT, "Realm students") + log.warning("KEYCLOAK: OK") except: - log.warning('KEYCLOAK: Seems to be there already') + log.warning("KEYCLOAK: Seems to be there already") #### Add default groups try: - log.warning('KEYCLOAK: Adding default groups') + log.warning("KEYCLOAK: Adding default groups") self.keycloak.add_group(MANAGER) self.keycloak.add_group(TEACHER) self.keycloak.add_group(STUDENT) - log.warning('KEYCLOAK: OK') + log.warning("KEYCLOAK: OK") except: - log.warning('KEYCLOAK: Seems to be there already') + log.warning("KEYCLOAK: Seems to be there already") try: - log.warning('MOODLE: Adding default group manager') - self.moodle.add_system_cohort(MANAGER,'system managers') - log.warning('MOODLE: OK') + log.warning("MOODLE: Adding default group manager") + self.moodle.add_system_cohort(MANAGER, "system managers") + log.warning("MOODLE: OK") except: - log.warning('MOODLE: Seems to be there already') + log.warning("MOODLE: Seems to be there already") try: - log.warning('MOODLE: Adding default group teacher') - self.moodle.add_system_cohort(TEACHER,'system teachers') - log.warning('MOODLE: OK') + log.warning("MOODLE: Adding default group teacher") + self.moodle.add_system_cohort(TEACHER, "system teachers") + log.warning("MOODLE: OK") except: - log.warning('MOODLE: Seems to be there already') + log.warning("MOODLE: Seems to be there already") try: - log.warning('MOODLE: Adding default group student') - self.moodle.add_system_cohort(STUDENT,'system students') - log.warning('MOODLE: OK') + log.warning("MOODLE: Adding default group student") + self.moodle.add_system_cohort(STUDENT, "system students") + log.warning("MOODLE: OK") except: - log.warning('MOODLE: Seems to be there already') - - + log.warning("MOODLE: Seems to be there already") try: - log.warning('NEXTCLOUD: Adding default group manager') + log.warning("NEXTCLOUD: Adding default group manager") self.nextcloud.add_group(MANAGER) - log.warning('NEXTCLOUD: OK') + log.warning("NEXTCLOUD: OK") except ProviderItemExists: - log.warning('NEXTCLOUD: Seems to be there already') + log.warning("NEXTCLOUD: Seems to be there already") try: - log.warning('NEXTCLOUD: Adding default group teacher') + log.warning("NEXTCLOUD: Adding default group teacher") self.nextcloud.add_group(TEACHER) - log.warning('NEXTCLOUD: OK') + log.warning("NEXTCLOUD: OK") except ProviderItemExists: - log.warning('NEXTCLOUD: Seems to be there already') + log.warning("NEXTCLOUD: Seems to be there already") try: - log.warning('NEXTCLOUD: Adding default group student') + log.warning("NEXTCLOUD: Adding default group student") self.nextcloud.add_group(STUDENT) - log.warning('NEXTCLOUD: OK') + log.warning("NEXTCLOUD: OK") except ProviderItemExists: - log.warning('NEXTCLOUD: Seems to be there already') - + log.warning("NEXTCLOUD: Seems to be there already") try: - log.warning('KEYCLOAK: Adding default users system_teacher, system_manager and system_student users') - uid=self.keycloak.add_user('system_manager','Manager','System','fakemanager@fake.com','m@n@g3r',group=MANAGER,temporary=False) - self.keycloak.assign_realm_roles(uid,MANAGER) - uid=self.keycloak.add_user('system_teacher','Teacher','System','faketeacher@fake.com','t3@ch3r',group=TEACHER,temporary=False) - self.keycloak.assign_realm_roles(uid,TEACHER) - uid=self.keycloak.add_user('system_student','Student','System','fakestudent@fake.com','stud3nt',group=STUDENT,temporary=False) - self.keycloak.assign_realm_roles(uid,STUDENT) - log.warning('KEYCLOAK: OK') + log.warning( + "KEYCLOAK: Adding default users system_teacher, system_manager and system_student users" + ) + uid = self.keycloak.add_user( + "system_manager", + "Manager", + "System", + "fakemanager@fake.com", + "m@n@g3r", + group=MANAGER, + temporary=False, + ) + self.keycloak.assign_realm_roles(uid, MANAGER) + uid = self.keycloak.add_user( + "system_teacher", + "Teacher", + "System", + "faketeacher@fake.com", + "t3@ch3r", + group=TEACHER, + temporary=False, + ) + self.keycloak.assign_realm_roles(uid, TEACHER) + uid = self.keycloak.add_user( + "system_student", + "Student", + "System", + "fakestudent@fake.com", + "stud3nt", + group=STUDENT, + temporary=False, + ) + self.keycloak.assign_realm_roles(uid, STUDENT) + log.warning("KEYCLOAK: OK") except: - log.warning('KEYCLOAK: Seems to be there already') - - + log.warning("KEYCLOAK: Seems to be there already") def resync_data(self): - self.internal={'users':self._get_mix_users(), - 'groups':self._get_mix_groups(), - 'roles':self._get_roles()} + self.internal = { + "users": self._get_mix_users(), + "groups": self._get_mix_groups(), + "roles": self._get_roles(), + } return True def get_moodle_users(self): - return [u for u in self.moodle.get_users_with_groups_and_roles() if not system_username(u['username'])] + return [ + u + for u in self.moodle.get_users_with_groups_and_roles() + if not system_username(u["username"]) + ] ## TOO SLOW. Not used. # def get_moodle_users(self): @@ -265,40 +319,57 @@ class Admin(): # #self.moodle.get_user_by('email','%%')['users'] # return [{"id":u['id'], # "username":u['username'], - # "first": u['firstname'], - # "last": u['lastname'], + # "first": u['firstname'], + # "last": u['lastname'], # "email": u['email'], # "groups": u['groups'], - # "roles": u['roles']} + # "roles": u['roles']} # for u in users] def get_keycloak_users(self): # log.warning('Loading keycloak users... can take a long time...') users = self.keycloak.get_users_with_groups_and_roles() - return [{"id":u['id'], - "username":u['username'], - "first": u.get('first_name',None), - "last": u.get('last_name',None), - "enabled": u['enabled'], - "email": u.get('email',''), - "groups": u['group'], - "roles": filter_roles_list(u['role'])} - for u in users if not system_username(u['username'])] + return [ + { + "id": u["id"], + "username": u["username"], + "first": u.get("first_name", None), + "last": u.get("last_name", None), + "enabled": u["enabled"], + "email": u.get("email", ""), + "groups": u["group"], + "roles": filter_roles_list(u["role"]), + } + for u in users + if not system_username(u["username"]) + ] def get_nextcloud_users(self): - return [{"id":u['username'], - "username":u['username'], - "first": u['displayname'].split(' ')[0] if u['displayname'] is not None else '', - "last": u['displayname'].split(' ')[1] if u['displayname'] is not None and len(u['displayname'].split(' '))>1 else '', - "email": u.get('email',''), - "groups": u['groups'], + return [ + { + "id": u["username"], + "username": u["username"], + "first": u["displayname"].split(" ")[0] + if u["displayname"] is not None + else "", + "last": u["displayname"].split(" ")[1] + if u["displayname"] is not None and len(u["displayname"].split(" ")) > 1 + else "", + "email": u.get("email", ""), + "groups": u["groups"], "roles": False, - "quota": u['quota'], - "quota_used_bytes": str(int(int(u['total_bytes'])/1024/1024/2))+' MB' if u['total_bytes'] != None else "0 MB"} - # The theoretical bytes returned do not map to any known unit. The division is just an approach while we investigate. - for u in self.nextcloud.get_users_list() if u['username'] not in ['guest','ddadmin','admin'] and not u['username'].startswith('system')] - + "quota": u["quota"], + "quota_used_bytes": str(int(int(u["total_bytes"]) / 1024 / 1024 / 2)) + + " MB" + if u["total_bytes"] != None + else "0 MB", + } + # The theoretical bytes returned do not map to any known unit. The division is just an approach while we investigate. + for u in self.nextcloud.get_users_list() + if u["username"] not in ["guest", "ddadmin", "admin"] + and not u["username"].startswith("system") + ] ## TOO SLOW # def get_nextcloud_users(self): @@ -309,68 +380,78 @@ class Admin(): # u=self.nextcloud.get_user(user) # users_list.append({"id":u['id'], # "username":u['id'], - # "first": u['displayname'], - # "last": None, + # "first": u['displayname'], + # "last": None, # "email": u['email'], # "groups": u['groups'], # "roles": []}) # return users_list def get_mix_users(self): - return self.internal['users'] + return self.internal["users"] def _get_mix_users(self): - kgroups=self.keycloak.get_groups() + kgroups = self.keycloak.get_groups() - kusers=self.get_keycloak_users() - musers=self.get_moodle_users() - nusers=self.get_nextcloud_users() + kusers = self.get_keycloak_users() + musers = self.get_moodle_users() + nusers = self.get_nextcloud_users() - kusers_usernames=[u['username'] for u in kusers] - musers_usernames=[u['username'] for u in musers] - nusers_usernames=[u['username'] for u in nusers] + kusers_usernames = [u["username"] for u in kusers] + musers_usernames = [u["username"] for u in musers] + nusers_usernames = [u["username"] for u in nusers] - all_users_usernames=set(kusers_usernames+musers_usernames+nusers_usernames) + all_users_usernames = set( + kusers_usernames + musers_usernames + nusers_usernames + ) - users=[] + users = [] for username in all_users_usernames: - theuser={} - keycloak_exists=[u for u in kusers if u['username'] == username] + theuser = {} + keycloak_exists = [u for u in kusers if u["username"] == username] if len(keycloak_exists): - theuser=keycloak_exists[0] - theuser['keycloak']=True - theuser['keycloak_groups']=get_gids_from_kgroup_ids(theuser.pop('groups'),kgroups) - #self.keycloak.get_user_groups_paths(keycloak_exists[0]['id']) #keycloak_exists[0]['groups'] + theuser = keycloak_exists[0] + theuser["keycloak"] = True + theuser["keycloak_groups"] = get_gids_from_kgroup_ids( + theuser.pop("groups"), kgroups + ) + # self.keycloak.get_user_groups_paths(keycloak_exists[0]['id']) #keycloak_exists[0]['groups'] else: - theuser['id']=False - theuser['keycloak']=False - theuser['keycloak_groups']=[] + theuser["id"] = False + theuser["keycloak"] = False + theuser["keycloak_groups"] = [] - moodle_exists=[u for u in musers if u['username'] == username] + moodle_exists = [u for u in musers if u["username"] == username] if len(moodle_exists): - theuser={**moodle_exists[0], **theuser} - theuser['moodle']=True - theuser['moodle_groups']=moodle_exists[0]['groups'] - theuser['moodle_id']=moodle_exists[0]['id'] + theuser = {**moodle_exists[0], **theuser} + theuser["moodle"] = True + theuser["moodle_groups"] = moodle_exists[0]["groups"] + theuser["moodle_id"] = moodle_exists[0]["id"] else: - theuser['moodle']=False - theuser['moodle_groups']=[] + theuser["moodle"] = False + theuser["moodle_groups"] = [] - nextcloud_exists=[u for u in nusers if u['username'] == username] + nextcloud_exists = [u for u in nusers if u["username"] == username] if len(nextcloud_exists): - theuser={**nextcloud_exists[0], **theuser} - theuser['nextcloud']=True - theuser['nextcloud_groups']=nextcloud_exists[0]['groups'] - theuser['nextcloud_id']=nextcloud_exists[0]['id'] - theuser['quota']=theuser['quota'] if theuser.get('quota',False) and theuser['quota'] != None and theuser['quota'] != 'none' else False + theuser = {**nextcloud_exists[0], **theuser} + theuser["nextcloud"] = True + theuser["nextcloud_groups"] = nextcloud_exists[0]["groups"] + theuser["nextcloud_id"] = nextcloud_exists[0]["id"] + theuser["quota"] = ( + theuser["quota"] + if theuser.get("quota", False) + and theuser["quota"] != None + and theuser["quota"] != "none" + else False + ) else: - theuser['nextcloud']=False - theuser['nextcloud_groups']=[] - theuser['quota']=False - theuser['quota_used_bytes']=False + theuser["nextcloud"] = False + theuser["nextcloud_groups"] = [] + theuser["quota"] = False + theuser["quota_used_bytes"] = False - # if not len(theuser['roles']): + # if not len(theuser['roles']): # log.error(' SKIPPING USER WITHOUT ANY ROLE!!: '+theuser['username']+' . Should be fixed at keycloak level.') # continue @@ -378,57 +459,57 @@ class Admin(): return users def get_roles(self): - return self.internal['roles'] + return self.internal["roles"] def _get_roles(self): return filter_roles_listofdicts(self.keycloak.get_roles()) def get_keycloak_groups(self): - log.warning('Loading keycloak groups...') + log.warning("Loading keycloak groups...") return self.keycloak.get_groups() def get_moodle_groups(self): - log.warning('Loading moodle groups...') + log.warning("Loading moodle groups...") return self.moodle.get_cohorts() def get_nextcloud_groups(self): - log.warning('Loading nextcloud groups...') + log.warning("Loading nextcloud groups...") return self.nextcloud.get_groups_list() def get_mix_groups(self): - return self.internal['groups'] + return self.internal["groups"] def _get_mix_groups(self): - kgroups=self.get_keycloak_groups() - mgroups=self.get_moodle_groups() - ngroups=self.get_nextcloud_groups() + kgroups = self.get_keycloak_groups() + mgroups = self.get_moodle_groups() + ngroups = self.get_nextcloud_groups() - kgroups=[] if kgroups is None else kgroups - mgroups=[] if mgroups is None else mgroups - ngroups=[] if ngroups is None else ngroups + kgroups = [] if kgroups is None else kgroups + mgroups = [] if mgroups is None else mgroups + ngroups = [] if ngroups is None else ngroups - kgroups_names=[kpath2gid(g['path']) for g in kgroups] - mgroups_names=[g['name'] for g in mgroups] - ngroups_names=ngroups + kgroups_names = [kpath2gid(g["path"]) for g in kgroups] + mgroups_names = [g["name"] for g in mgroups] + ngroups_names = ngroups - all_groups_names=set(kgroups_names+mgroups_names+ngroups_names) - groups=[] + all_groups_names = set(kgroups_names + mgroups_names + ngroups_names) + groups = [] for name in all_groups_names: - thegroup={} - keycloak_exists=[g for g in kgroups if kpath2gid(g['path']) == name] + thegroup = {} + keycloak_exists = [g for g in kgroups if kpath2gid(g["path"]) == name] if len(keycloak_exists): # del keycloak_exists[0]['subGroups'] - thegroup=keycloak_exists[0] - thegroup['keycloak']=True - thegroup['name']=kpath2gid(thegroup['path']) + thegroup = keycloak_exists[0] + thegroup["keycloak"] = True + thegroup["name"] = kpath2gid(thegroup["path"]) # thegroup['path']=self.keycloak.get_group_path(keycloak_exists[0]['id']) - + else: - thegroup['id']=False - thegroup['keycloak']=False - thegroup['name']=False - thegroup['path']=False - moodle_exists=[g for g in mgroups if g['name'] == name] + thegroup["id"] = False + thegroup["keycloak"] = False + thegroup["name"] = False + thegroup["path"] = False + moodle_exists = [g for g in mgroups if g["name"] == name] if len(moodle_exists): # del moodle_exists[0]['idnumber'] # del moodle_exists[0]['descriptionformat'] @@ -436,614 +517,863 @@ class Admin(): # del moodle_exists[0]['visible'] # # thegroup['path']=moodle_exists[0]['name'] # thegroup={**moodle_exists[0], **thegroup} - thegroup['moodle']=True - thegroup['moodle_id']=moodle_exists[0]['id'] - thegroup['description']=moodle_exists[0]['description'] + thegroup["moodle"] = True + thegroup["moodle_id"] = moodle_exists[0]["id"] + thegroup["description"] = moodle_exists[0]["description"] else: - thegroup['moodle']=False + thegroup["moodle"] = False - nextcloud_exists=[g for g in ngroups if g == name] + nextcloud_exists = [g for g in ngroups if g == name] if len(nextcloud_exists): # nextcloud={"id":nextcloud_exists[0], # "name":nextcloud_exists[0], # "path":nextcloud_exists[0]} # thegroup={**nextcloud, **thegroup} - thegroup['nextcloud']=True - thegroup['nextcloud_id']=nextcloud_exists[0] ### is the path + thegroup["nextcloud"] = True + thegroup["nextcloud_id"] = nextcloud_exists[0] ### is the path else: - thegroup['nextcloud']=False + thegroup["nextcloud"] = False groups.append(thegroup) return groups def sync_groups_from_keycloak(self): self.resync_data() - for group in self.internal['groups']: - if not group['keycloak']: - if group['moodle']: + for group in self.internal["groups"]: + if not group["keycloak"]: + if group["moodle"]: try: - self.moodle.delete_cohorts([group['moodle_id']]) + self.moodle.delete_cohorts([group["moodle_id"]]) except: print(traceback.format_exc()) # log.error('MOODLE: User '+user['username']+' it is not in cohort '+group) - if group['nextcloud']: - self.nextcloud.delete_group(group['nextcloud_id']) + if group["nextcloud"]: + self.nextcloud.delete_group(group["nextcloud_id"]) self.resync_data() - for group in self.internal['groups']: - if group['keycloak']: - if not group['moodle']: - self.moodle.add_system_cohort(group['name'],description='') - if not group['nextcloud']: - self.nextcloud.add_group(group['name']) + for group in self.internal["groups"]: + if group["keycloak"]: + if not group["moodle"]: + self.moodle.add_system_cohort(group["name"], description="") + if not group["nextcloud"]: + self.nextcloud.add_group(group["name"]) self.resync_data() - def get_external_users(self): - return self.external['users'] + return self.external["users"] def get_external_groups(self): - return self.external['groups'] + return self.external["groups"] def get_external_roles(self): - return self.external['roles'] + return self.external["roles"] - def upload_csv_ug(self,data): - log.warning('Processing uploaded users...') - users=[] - total=len(data['data']) - item=1 - ev=Events('Processing uploaded users',total=len(data['data'])) - groups=[] - for u in data['data']: - log.warning('Processing ('+str(item)+'/'+str(total)+') uploaded user: '+u['username']) - user_groups=[g.strip() for g in u['groups'].split(',')] + def upload_csv_ug(self, data): + log.warning("Processing uploaded users...") + users = [] + total = len(data["data"]) + item = 1 + ev = Events("Processing uploaded users", total=len(data["data"])) + groups = [] + for u in data["data"]: + log.warning( + "Processing (" + + str(item) + + "/" + + str(total) + + ") uploaded user: " + + u["username"] + ) + user_groups = [g.strip() for g in u["groups"].split(",")] - pathslist=[] + pathslist = [] for group in user_groups: - pathpart='' - for part in kpath2gid(group).split('.'): - if pathpart=='': - pathpart=part + pathpart = "" + for part in kpath2gid(group).split("."): + if pathpart == "": + pathpart = part else: - pathpart=pathpart+'.'+part + pathpart = pathpart + "." + part pathslist.append(pathpart) - pathslist.append(u['role']) + pathslist.append(u["role"]) - groups=groups+user_groups + groups = groups + user_groups # pprint(u) - if u['quota'] == '': - u['quota'] = '500 MB' - if u['role'] == 'student': u['quota']='500MB' - if u['role'] == 'teacher': u['quota']='5 GB' - if u['role'] == 'manager': u['quota']='Unlimited' - if u['role'] == 'admin': u['quota']='Unlimited' + if u["quota"] == "": + u["quota"] = "500 MB" + if u["role"] == "student": + u["quota"] = "500MB" + if u["role"] == "teacher": + u["quota"] = "5 GB" + if u["role"] == "manager": + u["quota"] = "Unlimited" + if u["role"] == "admin": + u["quota"] = "Unlimited" - users.append({'provider':'external', - 'id':u['id'].strip(), - 'email': u['email'].strip(), - 'first': u['firstname'].strip(), - 'last': u['lastname'].strip(), - 'username': u['username'].strip(), - 'groups':user_groups, - 'gids': pathslist, - 'quota': u['quota'], - 'roles':[u['role'].strip()], - 'temporary': True if u['password_temporal'].lower() == 'yes' else False, - 'password': self.get_dice_pwd() if u['password']=='' else u['password']}) - item+=1 - ev.increment({'name':u['username']}) - self.external['users']=users + users.append( + { + "provider": "external", + "id": u["id"].strip(), + "email": u["email"].strip(), + "first": u["firstname"].strip(), + "last": u["lastname"].strip(), + "username": u["username"].strip(), + "groups": user_groups, + "gids": pathslist, + "quota": u["quota"], + "roles": [u["role"].strip()], + "temporary": True + if u["password_temporal"].lower() == "yes" + else False, + "password": self.get_dice_pwd() + if u["password"] == "" + else u["password"], + } + ) + item += 1 + ev.increment({"name": u["username"]}) + self.external["users"] = users - groups=list(dict.fromkeys(groups)) - sysgroups=[] + groups = list(dict.fromkeys(groups)) + sysgroups = [] for g in groups: - sysgroups.append({'provider':'external', - "id": g, - "name": kpath2gid(g), - "path": g, - "description": 'Imported with csv'}) - self.external['groups']=sysgroups + sysgroups.append( + { + "provider": "external", + "id": g, + "name": kpath2gid(g), + "path": g, + "description": "Imported with csv", + } + ) + self.external["groups"] = sysgroups return True def get_dice_pwd(self): return diceware.get_passphrase(options=options) def reset_external(self): - self.external={'users':[], - 'groups':[], - 'roles':[]} + self.external = {"users": [], "groups": [], "roles": []} return True - def upload_json_ga(self,data): - groups=[] - log.warning('Processing uploaded groups...') + def upload_json_ga(self, data): + groups = [] + log.warning("Processing uploaded groups...") try: - ev=Events('Processing uploaded groups','Group:',total=len(data['data']['groups']),table='groups') + ev = Events( + "Processing uploaded groups", + "Group:", + total=len(data["data"]["groups"]), + table="groups", + ) except: log.error(traceback.format_exc()) - for g in data['data']['groups']: + for g in data["data"]["groups"]: try: - group={'provider':'external', - "id": g['id'], - "mailid": g['email'].split('@')[0], - "name": g['name'], - "description": g['description']} - ev.increment({'name':g['name'],'data':group}) + group = { + "provider": "external", + "id": g["id"], + "mailid": g["email"].split("@")[0], + "name": g["name"], + "description": g["description"], + } + ev.increment({"name": g["name"], "data": group}) groups.append(group) except: pass - self.external['groups']=groups + self.external["groups"] = groups - log.warning('Processing uploaded users...') - users=[] - total=len(data['data']['users']) - item=1 - ev=Events('Processing uploaded users','User:',total=len(data['data']['users']),table='users') - for u in data['data']['users']: - log.warning('Processing ('+str(item)+'/'+str(total)+') uploaded user: '+u['primaryEmail'].split('@')[0]) - new_user={'provider':'external', - 'id':u['id'], - 'email': u['primaryEmail'], - 'first': u['name']['givenName'], - 'last': u['name']['familyName'], - 'username': u['primaryEmail'].split('@')[0], - 'groups':[u['orgUnitPath']], ## WARNING: Removing the first - 'roles':[], - 'password': self.get_dice_pwd()} + log.warning("Processing uploaded users...") + users = [] + total = len(data["data"]["users"]) + item = 1 + ev = Events( + "Processing uploaded users", + "User:", + total=len(data["data"]["users"]), + table="users", + ) + for u in data["data"]["users"]: + log.warning( + "Processing (" + + str(item) + + "/" + + str(total) + + ") uploaded user: " + + u["primaryEmail"].split("@")[0] + ) + new_user = { + "provider": "external", + "id": u["id"], + "email": u["primaryEmail"], + "first": u["name"]["givenName"], + "last": u["name"]["familyName"], + "username": u["primaryEmail"].split("@")[0], + "groups": [u["orgUnitPath"]], ## WARNING: Removing the first + "roles": [], + "password": self.get_dice_pwd(), + } users.append(new_user) - item+=1 - ev.increment({'name':u['primaryEmail'].split('@')[0],'data':new_user}) - self.external['users']=users + item += 1 + ev.increment({"name": u["primaryEmail"].split("@")[0], "data": new_user}) + self.external["users"] = users ## Add groups to users (now they only have their orgUnitPath) - for g in self.external['groups']: - for useringroup in data['data']['d_members'][g['mailid']]: - for u in self.external['users']: - if u['id'] == useringroup['id']: - u['groups']=u['groups']+[g['name']] + for g in self.external["groups"]: + for useringroup in data["data"]["d_members"][g["mailid"]]: + for u in self.external["users"]: + if u["id"] == useringroup["id"]: + u["groups"] = u["groups"] + [g["name"]] return True - def sync_external(self,ids): + def sync_external(self, ids): # self.resync_data() - log.warning('Starting sync to keycloak') + log.warning("Starting sync to keycloak") self.sync_to_keycloak_external() ### Now we only sycn external to keycloak and then they can be updated to others with UI buttons - log.warning('Starting sync to moodle') + log.warning("Starting sync to moodle") self.sync_to_moodle_external() - log.warning('Starting sync to nextcloud') + log.warning("Starting sync to nextcloud") self.sync_to_nextcloud_external() - log.warning('All syncs finished. Resyncing from apps...') + log.warning("All syncs finished. Resyncing from apps...") self.resync_data() - def sync_to_keycloak_external(self): ### This one works from the external, moodle and nextcloud from the internal - groups=[] - for u in self.external['users']: - groups=groups+u['groups'] - groups=list(dict.fromkeys(groups)) + def sync_to_keycloak_external( + self, + ): ### This one works from the external, moodle and nextcloud from the internal + groups = [] + for u in self.external["users"]: + groups = groups + u["groups"] + groups = list(dict.fromkeys(groups)) - total=len(groups) - i=0 - ev=Events('Syncing import groups to keycloak','Adding group:',total=len(groups)) + total = len(groups) + i = 0 + ev = Events( + "Syncing import groups to keycloak", "Adding group:", total=len(groups) + ) for g in groups: - i=i+1 - log.warning(' KEYCLOAK GROUPS: Adding group ('+str(i)+'/'+str(total)+'): '+g) - ev.increment({'name':g}) + i = i + 1 + log.warning( + " KEYCLOAK GROUPS: Adding group (" + + str(i) + + "/" + + str(total) + + "): " + + g + ) + ev.increment({"name": g}) self.keycloak.add_group_tree(g) - total=len(self.external['users']) - index=0 - ev=Events('Syncing import users to keycloak','Adding user:',total=len(self.external['users'])) - for u in self.external['users']: - index=index+1 + total = len(self.external["users"]) + index = 0 + ev = Events( + "Syncing import users to keycloak", + "Adding user:", + total=len(self.external["users"]), + ) + for u in self.external["users"]: + index = index + 1 # Add user - log.warning(' KEYCLOAK USERS: Adding user ('+str(index)+'/'+str(total)+'): '+u['username']) - ev.increment({'name':u['username'],'data':u}) - uid=self.keycloak.add_user(u['username'],u['first'],u['last'],u['email'],u['password'],temporary=u['temporary']) - self.av.add_user_default_avatar(uid,u['roles'][0]) + log.warning( + " KEYCLOAK USERS: Adding user (" + + str(index) + + "/" + + str(total) + + "): " + + u["username"] + ) + ev.increment({"name": u["username"], "data": u}) + uid = self.keycloak.add_user( + u["username"], + u["first"], + u["last"], + u["email"], + u["password"], + temporary=u["temporary"], + ) + self.av.add_user_default_avatar(uid, u["roles"][0]) # Add user to role and group rolename - if len(u['roles']) != 0: - log.warning(' KEYCLOAK USERS: Assign user '+u['username']+' with initial pwd '+ u['password']+' to role '+u['roles'][0]) - self.keycloak.assign_realm_roles(uid,u['roles'][0]) - gid=self.keycloak.get_group_by_path('/'+u['roles'][0])['id'] + if len(u["roles"]) != 0: + log.warning( + " KEYCLOAK USERS: Assign user " + + u["username"] + + " with initial pwd " + + u["password"] + + " to role " + + u["roles"][0] + ) + self.keycloak.assign_realm_roles(uid, u["roles"][0]) + gid = self.keycloak.get_group_by_path("/" + u["roles"][0])["id"] # self.keycloak.group_user_add(uid,gid) # Add user to groups - for g in u['groups']: - parts=g.split('/') - sub='' - if len(parts)==0: - log.warning(' KEYCLOAK USERS: Skip assign user '+u['username']+' to any group as does not have one') - continue # NO GROUP - for i in range(1,len(parts)): - sub=sub+'/'+parts[i] - if sub=='/': continue # User with no path - log.warning(' KEYCLOAK USERS: Assign user '+u['username']+' to group '+ str(sub)) - kuser=self.keycloak.get_group_by_path(path=sub) - gid=kuser['id'] - self.keycloak.group_user_add(uid,gid) + for g in u["groups"]: + parts = g.split("/") + sub = "" + if len(parts) == 0: + log.warning( + " KEYCLOAK USERS: Skip assign user " + + u["username"] + + " to any group as does not have one" + ) + continue # NO GROUP + for i in range(1, len(parts)): + sub = sub + "/" + parts[i] + if sub == "/": + continue # User with no path + log.warning( + " KEYCLOAK USERS: Assign user " + + u["username"] + + " to group " + + str(sub) + ) + kuser = self.keycloak.get_group_by_path(path=sub) + gid = kuser["id"] + self.keycloak.group_user_add(uid, gid) # We add it as it is needed for moodle and nextcloud - u['groups'].append(u['roles'][0]) + u["groups"].append(u["roles"][0]) self.resync_data() - def sync_to_moodle_external(self): # works from the internal (keycloak) + def sync_to_moodle_external(self): # works from the internal (keycloak) ### Process all groups from the users keycloak_groups key - groups=[] - for u in self.external['users']: - groups=groups+u['gids'] - groups=list(dict.fromkeys(groups)) + groups = [] + for u in self.external["users"]: + groups = groups + u["gids"] + groups = list(dict.fromkeys(groups)) ### Create all groups. Skip / in system groups - total=len(groups) - ev=Events('Syncing groups from external to moodle',total=len(groups)) + total = len(groups) + ev = Events("Syncing groups from external to moodle", total=len(groups)) for g in groups: - parts=g.split('.') - if not len(parts): - log.error(' MOODLE GROUPS: Group '+g+ ' empty') + parts = g.split(".") + if not len(parts): + log.error(" MOODLE GROUPS: Group " + g + " empty") continue - subpath=parts[0] - for i in range(1,len(parts)): + subpath = parts[0] + for i in range(1, len(parts)): try: - log.warning(' MOODLE GROUPS: Adding group as cohort ('+str(i)+'/'+str(total)+'): '+subpath) - ev.increment({'name':subpath}) + log.warning( + " MOODLE GROUPS: Adding group as cohort (" + + str(i) + + "/" + + str(total) + + "): " + + subpath + ) + ev.increment({"name": subpath}) # if parts[i] in ['admin','manager','teacher','student']: self.moodle.add_system_cohort(subpath) - if len(parts) != i+1: - subpath=subpath+'.'+parts[i+1] + if len(parts) != i + 1: + subpath = subpath + "." + parts[i + 1] except: - log.error(' MOODLE GROUPS: Group '+subpath+ ' probably already exists') - + log.error( + " MOODLE GROUPS: Group " + subpath + " probably already exists" + ) + ### Get all existing moodle cohorts - cohorts=self.moodle.get_cohorts() + cohorts = self.moodle.get_cohorts() ### Create users in moodle - ev=Events('Syncing users from external to moodle',total=len(self.internal['users'])) - for u in self.external['users']: - log.warning('MOODLE: Creating moodle user: '+u['username']) - ev.increment({'name':u['username']}) - if u['first'] == '': u['first']='-' - if u['last'] == '': u['last']='-' + ev = Events( + "Syncing users from external to moodle", total=len(self.internal["users"]) + ) + for u in self.external["users"]: + log.warning("MOODLE: Creating moodle user: " + u["username"]) + ev.increment({"name": u["username"]}) + if u["first"] == "": + u["first"] = "-" + if u["last"] == "": + u["last"] = "-" try: - self.moodle.create_user(u['email'],u['username'],'*12'+secrets.token_urlsafe(16),u['first'],u['last'])[0] + self.moodle.create_user( + u["email"], + u["username"], + "*12" + secrets.token_urlsafe(16), + u["first"], + u["last"], + )[0] except UserExists: - log.warning('MOODLE:The user: '+u['username']+' already exsits.') + log.warning("MOODLE:The user: " + u["username"] + " already exsits.") except: - log.error(' -->> Error creating on moodle the user: '+u['username']) + log.error(" -->> Error creating on moodle the user: " + u["username"]) log.error(traceback.format_exc()) # user_id=user['id'] # self.resync_data() ### Add user to their cohorts (groups) - ev=Events('Syncing users groups from external to moodle cohorts',total=len(self.internal['users'])) - cohorts=self.moodle.get_cohorts() - for u in self.external['users']: + ev = Events( + "Syncing users groups from external to moodle cohorts", + total=len(self.internal["users"]), + ) + cohorts = self.moodle.get_cohorts() + for u in self.external["users"]: try: - uid=self.moodle.get_user_by('username',u['username'])['users'][0]['id'] - for group in u['gids']: - cohort=[c for c in cohorts if c['name']==group][0] - self.moodle.add_user_to_cohort(uid,cohort['id']) + uid = self.moodle.get_user_by("username", u["username"])["users"][0][ + "id" + ] + for group in u["gids"]: + cohort = [c for c in cohorts if c["name"] == group][0] + self.moodle.add_user_to_cohort(uid, cohort["id"]) except: - log.error('Exception on getting user from moodle: '+u['username']) - log.error(self.moodle.get_user_by('username',u['username'])) + log.error("Exception on getting user from moodle: " + u["username"]) + log.error(self.moodle.get_user_by("username", u["username"])) # self.resync_data() def delete_all_moodle_cohorts(self): - cohorts=self.moodle.get_cohorts() - ids=[c['id'] for c in cohorts] + cohorts = self.moodle.get_cohorts() + ids = [c["id"] for c in cohorts] self.moodle.delete_cohorts(ids) def sync_to_nextcloud_external(self): - groups=[] - for u in self.external['users']: - groups=groups+u['gids'] - groups=list(dict.fromkeys(groups)) + groups = [] + for u in self.external["users"]: + groups = groups + u["gids"] + groups = list(dict.fromkeys(groups)) - total=len(groups) - i=0 - ev=Events('Syncing groups from external to nextcloud',total=len(groups)) + total = len(groups) + i = 0 + ev = Events("Syncing groups from external to nextcloud", total=len(groups)) for g in groups: - parts=g.split('.') - if not len(parts): - log.error(' NEXTCLOUD GROUPS: Group '+g+ ' empty') + parts = g.split(".") + if not len(parts): + log.error(" NEXTCLOUD GROUPS: Group " + g + " empty") continue - subpath=parts[0] - for i in range(1,len(parts)): + subpath = parts[0] + for i in range(1, len(parts)): try: - log.warning(' NEXTCLOUD GROUPS: Adding group as cohort ('+str(i)+'/'+str(total)+'): '+subpath) - ev.increment({'name':subpath}) + log.warning( + " NEXTCLOUD GROUPS: Adding group as cohort (" + + str(i) + + "/" + + str(total) + + "): " + + subpath + ) + ev.increment({"name": subpath}) # if parts[i] in ['admin','manager','teacher','student']: self.nextcloud.add_group(subpath) - if len(parts) != i+1: - subpath=subpath+'.'+parts[i+1] + if len(parts) != i + 1: + subpath = subpath + "." + parts[i + 1] except: - log.error(' NEXTCLOUD GROUPS: Group '+subpath+ ' probably already exists') + log.error( + " NEXTCLOUD GROUPS: Group " + + subpath + + " probably already exists" + ) - - ev=Events('Syncing users from external to nextcloud',total=len(self.internal['users'])) - for u in self.external['users']: - log.warning(' NEXTCLOUD USERS: Creating nextcloud user: '+u['username']+' in groups '+str(u['gids'])) + ev = Events( + "Syncing users from external to nextcloud", + total=len(self.internal["users"]), + ) + for u in self.external["users"]: + log.warning( + " NEXTCLOUD USERS: Creating nextcloud user: " + + u["username"] + + " in groups " + + str(u["gids"]) + ) try: - ev.increment({'name':u['username']}) - self.nextcloud.add_user_with_groups(u['username'],'*12'+secrets.token_urlsafe(16),u['quota'],u['gids'],u['email'],u['first']+' '+u['last']) + ev.increment({"name": u["username"]}) + self.nextcloud.add_user_with_groups( + u["username"], + "*12" + secrets.token_urlsafe(16), + u["quota"], + u["gids"], + u["email"], + u["first"] + " " + u["last"], + ) except ProviderItemExists: - log.warning('User '+u['username']+' already exists. Skipping...') + log.warning("User " + u["username"] + " already exists. Skipping...") continue except: log.error(traceback.format_exc()) - def sync_to_moodle(self): # works from the internal (keycloak) + def sync_to_moodle(self): # works from the internal (keycloak) ### Process all groups from the users keycloak_groups key - groups=[] - for u in self.internal['users']: - groups=groups+u['keycloak_groups'] - groups=list(dict.fromkeys(groups)) + groups = [] + for u in self.internal["users"]: + groups = groups + u["keycloak_groups"] + groups = list(dict.fromkeys(groups)) - ev=Events('Syncing groups from keycloak to moodle',total=len(groups)) - pathslist=[] + ev = Events("Syncing groups from keycloak to moodle", total=len(groups)) + pathslist = [] for group in groups: - pathpart='' - for part in group.split('.'): - if pathpart=='': - pathpart=part + pathpart = "" + for part in group.split("."): + if pathpart == "": + pathpart = part else: - pathpart=pathpart+'.'+part + pathpart = pathpart + "." + part pathslist.append(pathpart) - ev.increment({'name':pathpart}) + ev.increment({"name": pathpart}) try: - log.info('MOODLE: Adding group '+pathpart) + log.info("MOODLE: Adding group " + pathpart) self.moodle.add_system_cohort(pathpart) except: - #print(traceback.format_exc()) - log.error('MOODLE: Group '+pathpart+' probably already exists.') + # print(traceback.format_exc()) + log.error("MOODLE: Group " + pathpart + " probably already exists.") ### Get all existing moodle cohorts - cohorts=self.moodle.get_cohorts() + cohorts = self.moodle.get_cohorts() ### Create users in moodle - ev=Events('Syncing users from keycloak to moodle',total=len(self.internal['users'])) - for u in self.internal['users']: - if not u['moodle']: - log.warning('Creating moodle user: '+u['username']) - ev.increment({'name':u['username']}) - if u['first'] == '': u['first']=' ' - if u['last'] == '': u['last']=' ' + ev = Events( + "Syncing users from keycloak to moodle", total=len(self.internal["users"]) + ) + for u in self.internal["users"]: + if not u["moodle"]: + log.warning("Creating moodle user: " + u["username"]) + ev.increment({"name": u["username"]}) + if u["first"] == "": + u["first"] = " " + if u["last"] == "": + u["last"] = " " try: - self.moodle.create_user(u['email'],u['username'],'*12'+secrets.token_urlsafe(16),u['first'],u['last'])[0] + self.moodle.create_user( + u["email"], + u["username"], + "*12" + secrets.token_urlsafe(16), + u["first"], + u["last"], + )[0] except: - log.error(' -->> Error creating on moodle the user: '+u['username']) + log.error( + " -->> Error creating on moodle the user: " + u["username"] + ) # user_id=user['id'] self.resync_data() - ev=Events('Syncing users with moodle cohorts',total=len(self.internal['users'])) - cohorts=self.moodle.get_cohorts() - for u in self.internal['users']: - groups=u['keycloak_groups']+[u['roles'][0]] + ev = Events( + "Syncing users with moodle cohorts", total=len(self.internal["users"]) + ) + cohorts = self.moodle.get_cohorts() + for u in self.internal["users"]: + groups = u["keycloak_groups"] + [u["roles"][0]] for path in groups: try: - cohort=[c for c in cohorts if c['name']==path][0] + cohort = [c for c in cohorts if c["name"] == path][0] except: # print(traceback.format_exc()) - log.error(' MOODLE USER GROUPS: keycloak group '+path+' does not exist as moodle cohort. This should not happen. User '+u['username']+ ' not added.') + log.error( + " MOODLE USER GROUPS: keycloak group " + + path + + " does not exist as moodle cohort. This should not happen. User " + + u["username"] + + " not added." + ) try: - self.moodle.add_user_to_cohort(u['moodle_id'],cohort['id']) + self.moodle.add_user_to_cohort(u["moodle_id"], cohort["id"]) except: log.error(traceback.format_exc()) - log.error(' MOODLE USER GROUPS: User '+u['username']+' already exists in cohort '+cohort['name']) + log.error( + " MOODLE USER GROUPS: User " + + u["username"] + + " already exists in cohort " + + cohort["name"] + ) - ev.increment({'name':'User '+u['username']+' added to moodle cohorts','data':[]}) + ev.increment( + { + "name": "User " + u["username"] + " added to moodle cohorts", + "data": [], + } + ) self.resync_data() def sync_to_nextcloud(self): - groups=[] - for u in self.internal['users']: - groups=groups+u['keycloak_groups'] - groups=list(dict.fromkeys(groups)) + groups = [] + for u in self.internal["users"]: + groups = groups + u["keycloak_groups"] + groups = list(dict.fromkeys(groups)) - total=len(groups) - i=0 - ev=Events('Syncing groups from keycloak to nextcloud',total=len(groups)) + total = len(groups) + i = 0 + ev = Events("Syncing groups from keycloak to nextcloud", total=len(groups)) for g in groups: - parts=g.split('/') - subpath='' - for i in range(1,len(parts)): + parts = g.split("/") + subpath = "" + for i in range(1, len(parts)): try: - log.warning(' NEXTCLOUD GROUPS: Adding group ('+str(i)+'/'+str(total)+'): '+subpath) - ev.increment({'name':subpath}) - subpath=subpath+'/'+parts[i] + log.warning( + " NEXTCLOUD GROUPS: Adding group (" + + str(i) + + "/" + + str(total) + + "): " + + subpath + ) + ev.increment({"name": subpath}) + subpath = subpath + "/" + parts[i] self.nextcloud.add_group(subpath) except: - log.error('probably exists') - i=i+1 + log.error("probably exists") + i = i + 1 - ev=Events('Syncing users from keycloak to nextcloud',total=len(self.internal['users'])) - for u in self.internal['users']: - if not u['nextcloud']: - log.warning(' NEXTCLOUD USERS: Creating nextcloud user: '+u['username']+' in groups '+str(u['keycloak_groups'])) + ev = Events( + "Syncing users from keycloak to nextcloud", + total=len(self.internal["users"]), + ) + for u in self.internal["users"]: + if not u["nextcloud"]: + log.warning( + " NEXTCLOUD USERS: Creating nextcloud user: " + + u["username"] + + " in groups " + + str(u["keycloak_groups"]) + ) try: - ev.increment({'name':u['username']}) - self.nextcloud.add_user_with_groups(u['username'],'*12'+secrets.token_urlsafe(16),"500 GB",u['keycloak_groups'],u['email'],u['first']+' '+u['last']) + ev.increment({"name": u["username"]}) + self.nextcloud.add_user_with_groups( + u["username"], + "*12" + secrets.token_urlsafe(16), + "500 GB", + u["keycloak_groups"], + u["email"], + u["first"] + " " + u["last"], + ) except ProviderItemExists: - log.warning('User '+u['username']+' already exists. Skipping...') + log.warning( + "User " + u["username"] + " already exists. Skipping..." + ) continue except: log.error(traceback.format_exc()) - def delete_keycloak_user(self,userid): - user=[u for u in self.internal['users'] if u['id']==userid] - if len(user) and user[0]['keycloak']: - user=user[0] - keycloak_id=user['id'] + def delete_keycloak_user(self, userid): + user = [u for u in self.internal["users"] if u["id"] == userid] + if len(user) and user[0]["keycloak"]: + user = user[0] + keycloak_id = user["id"] else: return False - log.warning('Removing keycloak user: '+user['username']) + log.warning("Removing keycloak user: " + user["username"]) try: self.keycloak.delete_user(keycloak_id) except: log.error(traceback.format_exc()) - log.warning('Could not remove users: '+user['username']) - + log.warning("Could not remove users: " + user["username"]) + self.av.delete_user_avatar(userid) def delete_keycloak_users(self): - total=len(self.internal['users']) - i=0 - ev=Events('Deleting users from keycloak','Deleting user:',total=len(self.internal['users'])) - for u in self.internal['users']: - i=i+1 - if not u['keycloak']: continue + total = len(self.internal["users"]) + i = 0 + ev = Events( + "Deleting users from keycloak", + "Deleting user:", + total=len(self.internal["users"]), + ) + for u in self.internal["users"]: + i = i + 1 + if not u["keycloak"]: + continue # Do not remove admin users!!! What to do with managers??? - if ['admin'] in u['roles']: continue - log.info(' KEYCLOAK USERS: Removing user ('+str(i)+'/'+str(total)+'): '+u['username']) + if ["admin"] in u["roles"]: + continue + log.info( + " KEYCLOAK USERS: Removing user (" + + str(i) + + "/" + + str(total) + + "): " + + u["username"] + ) try: - ev.increment({'name':u['username'],'data':u}) - self.keycloak.delete_user(u['id']) + ev.increment({"name": u["username"], "data": u}) + self.keycloak.delete_user(u["id"]) except: - log.warning(' KEYCLOAK USERS: Could not remove user: '+u['username'] +'. Probably already not exists.') + log.warning( + " KEYCLOAK USERS: Could not remove user: " + + u["username"] + + ". Probably already not exists." + ) self.av.minio_delete_all_objects() - def delete_nextcloud_user(self,userid): - user=[u for u in self.internal['users'] if u['id']==userid] - if len(user) and user[0]['nextcloud']: - user=user[0] - nextcloud_id=user['nextcloud_id'] + def delete_nextcloud_user(self, userid): + user = [u for u in self.internal["users"] if u["id"] == userid] + if len(user) and user[0]["nextcloud"]: + user = user[0] + nextcloud_id = user["nextcloud_id"] else: return False - log.warning('Removing nextcloud user: '+user['username']) + log.warning("Removing nextcloud user: " + user["username"]) try: self.nextcloud.delete_user(nextcloud_id) except: log.error(traceback.format_exc()) - log.warning('Could not remove users: '+user['username']) + log.warning("Could not remove users: " + user["username"]) def delete_nextcloud_users(self): - ev=Events('Deleting users from nextcloud',total=len(self.internal['users'])) - for u in self.internal['users']: - - if u['nextcloud'] and not u['keycloak']: - if u['roles'] and 'admin' in u['roles']: continue - log.info('Removing nextcloud user: '+u['username']) + ev = Events("Deleting users from nextcloud", total=len(self.internal["users"])) + for u in self.internal["users"]: + + if u["nextcloud"] and not u["keycloak"]: + if u["roles"] and "admin" in u["roles"]: + continue + log.info("Removing nextcloud user: " + u["username"]) try: - ev.increment({'name':u['username']}) - self.nextcloud.delete_user(u['nextcloud_id']) + ev.increment({"name": u["username"]}) + self.nextcloud.delete_user(u["nextcloud_id"]) except: log.error(traceback.format_exc()) - log.warning('Could not remove user: '+u['username']) + log.warning("Could not remove user: " + u["username"]) - def delete_moodle_user(self,userid): - user=[u for u in self.internal['users'] if u['id']==userid] - if len(user) and user[0]['moodle']: - user=user[0] - moodle_id=user['moodle_id'] + def delete_moodle_user(self, userid): + user = [u for u in self.internal["users"] if u["id"] == userid] + if len(user) and user[0]["moodle"]: + user = user[0] + moodle_id = user["moodle_id"] else: return False - log.warning('Removing moodle user: '+user['username']) + log.warning("Removing moodle user: " + user["username"]) try: self.moodle.delete_users([moodle_id]) except: log.error(traceback.format_exc()) - log.warning('Could not remove users: '+user['username']) + log.warning("Could not remove users: " + user["username"]) def delete_moodle_users(self): - userids=[] - usernames=[] - for u in self.internal['users']: - if u['moodle'] and not u['keycloak']: - userids.append(u['moodle_id']) - usernames.append(u['username']) + userids = [] + usernames = [] + for u in self.internal["users"]: + if u["moodle"] and not u["keycloak"]: + userids.append(u["moodle_id"]) + usernames.append(u["username"]) if len(userids): - log.warning('Removing moodle users: '+','.join(usernames)) + log.warning("Removing moodle users: " + ",".join(usernames)) try: self.moodle.delete_users(userids) - app.socketio.emit('update', - json.dumps({'status':True,'item':'user','action':'delete','itemdata':u}), - namespace='//sio', - room='admin') + app.socketio.emit( + "update", + json.dumps( + { + "status": True, + "item": "user", + "action": "delete", + "itemdata": u, + } + ), + namespace="//sio", + room="admin", + ) except: log.error(traceback.format_exc()) - log.warning('Could not remove users: '+','.join(usernames)) - - + log.warning("Could not remove users: " + ",".join(usernames)) def delete_keycloak_groups(self): - for g in self.internal['groups']: - if not g['keycloak']: continue + for g in self.internal["groups"]: + if not g["keycloak"]: + continue # Do not remove admin group. It should not exist in keycloak, only in nextcloud - if g['name'] in ['admin','manager','teacher','student']: continue - log.info('Removing keycloak group: '+g['name']) + if g["name"] in ["admin", "manager", "teacher", "student"]: + continue + log.info("Removing keycloak group: " + g["name"]) try: - self.keycloak.delete_group(g['id']) + self.keycloak.delete_group(g["id"]) except: log.error(traceback.format_exc()) - log.warning('Could not remove group: '+g['name']) + log.warning("Could not remove group: " + g["name"]) - def external_roleassign(self,data): - for newuserid in data['ids']: - for externaluser in self.external['users']: - if externaluser['id'] == newuserid: - externaluser['roles']=[data['action']] - for role in ['admin','manager','teacher','student']: + def external_roleassign(self, data): + for newuserid in data["ids"]: + for externaluser in self.external["users"]: + if externaluser["id"] == newuserid: + externaluser["roles"] = [data["action"]] + for role in ["admin", "manager", "teacher", "student"]: try: - externaluser['gids'].remove(role) + externaluser["gids"].remove(role) # externaluser['groups'].remove('/'+role) except: pass - externaluser['gids'].append(data['action']) + externaluser["gids"].append(data["action"]) return True - def user_update_password(self,userid,password,temporary): - return self.keycloak.update_user_pwd(userid,password,temporary) + def user_update_password(self, userid, password, temporary): + return self.keycloak.update_user_pwd(userid, password, temporary) def update_users_from_keycloak(self): - kgroups=self.keycloak.get_groups() - users=[{'id':u['id'], - 'username':u['username'], - 'enabled':u['enabled'], - 'email':u['email'], - 'firstname':u['first'], - 'lastname':u['last'], - 'groups':get_gids_from_kgroup_ids(u['groups'],kgroups), - 'roles':u['roles'], - 'quota':'3 GB' if len(u['roles']) and u['roles'][0] != 'student' else '500 MB'} - for u in self.get_keycloak_users()] + kgroups = self.keycloak.get_groups() + users = [ + { + "id": u["id"], + "username": u["username"], + "enabled": u["enabled"], + "email": u["email"], + "firstname": u["first"], + "lastname": u["last"], + "groups": get_gids_from_kgroup_ids(u["groups"], kgroups), + "roles": u["roles"], + "quota": "3 GB" + if len(u["roles"]) and u["roles"][0] != "student" + else "500 MB", + } + for u in self.get_keycloak_users() + ] for user in users: - ev=Events('Updating users from keycloak','User:',total=len(users),table='users') + ev = Events( + "Updating users from keycloak", "User:", total=len(users), table="users" + ) self.user_update(user) - ev.increment({'name':user['username'],'data':user['groups']}) - - def user_update(self,user): - log.warning('Updating user moodle, nextcloud keycloak') - ev=Events('Updating user','Updating user in keycloak') + ev.increment({"name": user["username"], "data": user["groups"]}) + def user_update(self, user): + log.warning("Updating user moodle, nextcloud keycloak") + ev = Events("Updating user", "Updating user in keycloak") ## Get actual user role try: - internaluser = [u for u in self.internal['users'] if u['username'] == user['username']][0] + internaluser = [ + u for u in self.internal["users"] if u["username"] == user["username"] + ][0] except: - raise UserNotFound + raise UserNotFound + + if not len(user["roles"]): + user["roles"] = ["student"] + delete_roles = list( + set(["admin", "manager", "teacher", "student"]) - set(user["roles"]) + ) - if not len(user['roles']): user['roles']=['student'] - delete_roles=list(set(['admin','manager','teacher','student']) - set(user['roles'])) - ### keycloak groups - - kadd=list(set(user['groups']) - set(internaluser['keycloak_groups']+delete_roles)) - kdelete=list(set(internaluser['keycloak_groups']) - set(user['groups']) - set(user['roles'])) - if user['roles'][0] not in kadd: kadd.append(user['roles'][0]) + kadd = list( + set(user["groups"]) - set(internaluser["keycloak_groups"] + delete_roles) + ) + kdelete = list( + set(internaluser["keycloak_groups"]) + - set(user["groups"]) + - set(user["roles"]) + ) + if user["roles"][0] not in kadd: + kadd.append(user["roles"][0]) ### Moodle groups - madd=list(set(user['groups']) - set(internaluser['moodle_groups']+delete_roles)) - mdelete=list(set(internaluser['moodle_groups']) - set(user['groups']) - set(user['roles'])) - if user['roles'][0] not in madd: madd.append(user['roles'][0]) + madd = list( + set(user["groups"]) - set(internaluser["moodle_groups"] + delete_roles) + ) + mdelete = list( + set(internaluser["moodle_groups"]) + - set(user["groups"]) + - set(user["roles"]) + ) + if user["roles"][0] not in madd: + madd.append(user["roles"][0]) ### nextcloud groups - nadd=list(set(user['groups']) - set(internaluser['nextcloud_groups']+delete_roles)) - ndelete=list(set(internaluser['nextcloud_groups']) - set(user['groups']) - set(user['roles'])) - if user['roles'][0] not in nadd: nadd.append(user['roles'][0]) + nadd = list( + set(user["groups"]) - set(internaluser["nextcloud_groups"] + delete_roles) + ) + ndelete = list( + set(internaluser["nextcloud_groups"]) + - set(user["groups"]) + - set(user["roles"]) + ) + if user["roles"][0] not in nadd: + nadd.append(user["roles"][0]) #### Delete recursive to parent from this subgroup #### DISABLED DELETE BUT KEPT ADD, as we should then check if the user is still in another subgroups of this group. How? @@ -1055,22 +1385,22 @@ class Admin(): # pathpart=part # else: # pathpart=pathpart+'.'+part - # if repeated_subpaths[pathpart] > 1: + # if repeated_subpaths[pathpart] > 1: # ## We do not delete it as the user is in multiple subpaths of this path. # continue # pathslist.append(pathpart) # kdelete=pathslist - pathslist=[] + pathslist = [] for group in kadd: - pathpart='' - for part in group.split('.'): - if pathpart=='': - pathpart=part + pathpart = "" + for part in group.split("."): + if pathpart == "": + pathpart = part else: - pathpart=pathpart+'.'+part + pathpart = pathpart + "." + part pathslist.append(pathpart) - kadd=pathslist + kadd = pathslist # print('KADD WITH SUBPATHS') # print(kadd) @@ -1078,7 +1408,7 @@ class Admin(): # print('KDELETE') # print(kdelete) # print(count_repeated(kdelete)) - + # pathslist=[] # for group in mdelete: # pathpart='' @@ -1090,16 +1420,16 @@ class Admin(): # pathslist.append(pathpart) # mdelete=pathslist - pathslist=[] + pathslist = [] for group in madd: - pathpart='' - for part in group.split('.'): - if pathpart=='': - pathpart=part + pathpart = "" + for part in group.split("."): + if pathpart == "": + pathpart = part else: - pathpart=pathpart+'.'+part + pathpart = pathpart + "." + part pathslist.append(pathpart) - madd=pathslist + madd = pathslist # print('MADD WITH SUBPATHS') # print(madd) @@ -1119,16 +1449,16 @@ class Admin(): # pathslist.append(pathpart) # ndelete=pathslist - pathslist=[] + pathslist = [] for group in nadd: - pathpart='' - for part in group.split('.'): - if pathpart=='': - pathpart=part + pathpart = "" + for part in group.split("."): + if pathpart == "": + pathpart = part else: - pathpart=pathpart+'.'+part + pathpart = pathpart + "." + part pathslist.append(pathpart) - nadd=pathslist + nadd = pathslist # print('NADD WITH SUBPATHS') # print(nadd) @@ -1137,293 +1467,406 @@ class Admin(): # print(ndelete) # print(count_repeated(ndelete)) - self.update_keycloak_user(internaluser['id'],user,kdelete,kadd) - ev.update_text('Syncing data from applications...') + self.update_keycloak_user(internaluser["id"], user, kdelete, kadd) + ev.update_text("Syncing data from applications...") self.resync_data() - ev.update_text('Updating user in moodle') - self.update_moodle_user(internaluser['id'],user,mdelete,madd) + ev.update_text("Updating user in moodle") + self.update_moodle_user(internaluser["id"], user, mdelete, madd) - ev.update_text('Updating user in nextcloud') - self.update_nextcloud_user(internaluser['id'],user,ndelete,nadd) - - ev.update_text('User updated') + ev.update_text("Updating user in nextcloud") + self.update_nextcloud_user(internaluser["id"], user, ndelete, nadd) + + ev.update_text("User updated") return True - def update_keycloak_user(self,user_id,user,kdelete,kadd): + def update_keycloak_user(self, user_id, user, kdelete, kadd): # pprint(self.keycloak.get_user_realm_roles(user_id)) - self.keycloak.remove_user_realm_roles(user_id,'student') - self.keycloak.assign_realm_roles(user_id,user['roles'][0]) + self.keycloak.remove_user_realm_roles(user_id, "student") + self.keycloak.assign_realm_roles(user_id, user["roles"][0]) for group in kdelete: - group_id = self.keycloak.get_group_by_path(gid2kpath(group))['id'] - self.keycloak.group_user_remove(user_id,group_id) + group_id = self.keycloak.get_group_by_path(gid2kpath(group))["id"] + self.keycloak.group_user_remove(user_id, group_id) for group in kadd: - group_id = self.keycloak.get_group_by_path(gid2kpath(group))['id'] - self.keycloak.group_user_add(user_id,group_id) - self.keycloak.user_update(user_id,user['enabled'],user['email'],user['firstname'],user['lastname']) + group_id = self.keycloak.get_group_by_path(gid2kpath(group))["id"] + self.keycloak.group_user_add(user_id, group_id) + self.keycloak.user_update( + user_id, user["enabled"], user["email"], user["firstname"], user["lastname"] + ) self.resync_data() return True - def enable_users(self,data): - #data={'id':'','username':''} - ev=Events('Bulk actions','Enabling user:',total=len(data)) + def enable_users(self, data): + # data={'id':'','username':''} + ev = Events("Bulk actions", "Enabling user:", total=len(data)) for user in data: - ev.increment({'name':user['username'],'data':user['username']}) - self.keycloak.user_enable(user['id']) + ev.increment({"name": user["username"], "data": user["username"]}) + self.keycloak.user_enable(user["id"]) self.resync_data() - def disable_users(self,data): - #data={'id':'','username':''} - ev=Events('Bulk actions','Disabling user:',total=len(data)) + def disable_users(self, data): + # data={'id':'','username':''} + ev = Events("Bulk actions", "Disabling user:", total=len(data)) for user in data: - ev.increment({'name':user['username'],'data':user['username']}) - self.keycloak.user_disable(user['id']) + ev.increment({"name": user["username"], "data": user["username"]}) + self.keycloak.user_disable(user["id"]) self.resync_data() - def update_moodle_user(self,user_id,user,mdelete,madd): - internaluser=[u for u in self.internal['users'] if u['id']==user_id][0] - cohorts=self.moodle.get_cohorts() + def update_moodle_user(self, user_id, user, mdelete, madd): + internaluser = [u for u in self.internal["users"] if u["id"] == user_id][0] + cohorts = self.moodle.get_cohorts() for group in mdelete: - cohort=[c for c in cohorts if c['name']==group][0] + cohort = [c for c in cohorts if c["name"] == group][0] try: - self.moodle.delete_user_in_cohort(internaluser['moodle_id'],cohort['id']) + self.moodle.delete_user_in_cohort( + internaluser["moodle_id"], cohort["id"] + ) except: - log.error('MOODLE: User '+user['username']+' it is not in cohort '+group) + log.error( + "MOODLE: User " + user["username"] + " it is not in cohort " + group + ) - if not internaluser['moodle']: - self.add_moodle_user(username=user['username'], email=user['email'], first_name=user['firstname'], last_name=user['lastname']) + if not internaluser["moodle"]: + self.add_moodle_user( + username=user["username"], + email=user["email"], + first_name=user["firstname"], + last_name=user["lastname"], + ) self.resync_data() else: - self.moodle.update_user(username=user['username'], email=user['email'], first_name=user['firstname'], last_name=user['lastname'], enabled=user['enabled']) + self.moodle.update_user( + username=user["username"], + email=user["email"], + first_name=user["firstname"], + last_name=user["lastname"], + enabled=user["enabled"], + ) for group in madd: - cohort=[c for c in cohorts if c['name']==group][0] - self.moodle.add_user_to_cohort(internaluser['moodle_id'],cohort['id']) + cohort = [c for c in cohorts if c["name"] == group][0] + self.moodle.add_user_to_cohort(internaluser["moodle_id"], cohort["id"]) return True - def add_moodle_user(self, username,email,first_name,last_name,password='*12'+secrets.token_urlsafe(16)): - log.warning('Creating moodle user: '+username) - ev=Events('Add user',username) + def add_moodle_user( + self, + username, + email, + first_name, + last_name, + password="*12" + secrets.token_urlsafe(16), + ): + log.warning("Creating moodle user: " + username) + ev = Events("Add user", username) try: - self.moodle.create_user(email,username,password,first_name,last_name) - ev.update_text({'name':'Added to moodle','data':[]}) + self.moodle.create_user(email, username, password, first_name, last_name) + ev.update_text({"name": "Added to moodle", "data": []}) except UserExists: - log.error(' -->> User already exists') - error=Events('User already exists.',str(se),type='error') + log.error(" -->> User already exists") + error = Events("User already exists.", str(se), type="error") except SystemError as se: - log.error('Moodle create user error: '+str(se)) - error=Events('Moodle create user error',str(se),type='error') + log.error("Moodle create user error: " + str(se)) + error = Events("Moodle create user error", str(se), type="error") except: - log.error(' -->> Error creating on moodle the user: '+username) + log.error(" -->> Error creating on moodle the user: " + username) print(traceback.format_exc()) - error=Events('Internal error','Check logs',type='error') + error = Events("Internal error", "Check logs", type="error") + + def update_nextcloud_user(self, user_id, user, ndelete, nadd): - def update_nextcloud_user(self,user_id, user, ndelete,nadd): - ## TODO: Disable de user? Is really needed? it is disabled in keycloak, so can't login again ## ocs/v1.php/cloud/users/{userid}/disable - internaluser=[u for u in self.internal['users'] if u['id']==user_id][0] - if not internaluser['nextcloud']: + internaluser = [u for u in self.internal["users"] if u["id"] == user_id][0] + if not internaluser["nextcloud"]: try: - self.add_nextcloud_user(user['username'],user['email'],user['quota'],user['firstname'],user['lastname'],user['groups']) + self.add_nextcloud_user( + user["username"], + user["email"], + user["quota"], + user["firstname"], + user["lastname"], + user["groups"], + ) except: - log.warning('NEXTCLOUD: Ooops! User '+user['username']+' seems to be in NC but our db info says not....') + log.warning( + "NEXTCLOUD: Ooops! User " + + user["username"] + + " seems to be in NC but our db info says not...." + ) self.resync_data() else: - if not user['quota'] or user['quota'] == 'false': - self.nextcloud.update_user(user['username'],{"quota":"none","email":user['email'],"displayname":user['firstname']+' '+user['lastname']}) + if not user["quota"] or user["quota"] == "false": + self.nextcloud.update_user( + user["username"], + { + "quota": "none", + "email": user["email"], + "displayname": user["firstname"] + " " + user["lastname"], + }, + ) else: - self.nextcloud.update_user(user['username'],{"quota":user['quota'],"email":user['email'],"displayname":user['firstname']+' '+user['lastname']}) + self.nextcloud.update_user( + user["username"], + { + "quota": user["quota"], + "email": user["email"], + "displayname": user["firstname"] + " " + user["lastname"], + }, + ) for group in ndelete: - self.nextcloud.remove_user_from_group(user['username'],group) + self.nextcloud.remove_user_from_group(user["username"], group) for group in nadd: - self.nextcloud.add_user_to_group(user['username'],group) + self.nextcloud.add_user_to_group(user["username"], group) - def add_nextcloud_user(self,username,email,quota,first_name,last_name,groups,password='*12'+secrets.token_urlsafe(16)): - log.warning(' NEXTCLOUD USERS: Creating nextcloud user: '+username+' in groups '+str(groups)) - ev=Events('Add user',username) + def add_nextcloud_user( + self, + username, + email, + quota, + first_name, + last_name, + groups, + password="*12" + secrets.token_urlsafe(16), + ): + log.warning( + " NEXTCLOUD USERS: Creating nextcloud user: " + + username + + " in groups " + + str(groups) + ) + ev = Events("Add user", username) try: # Quota is "1 GB", "500 MB" - self.nextcloud.add_user_with_groups(username,password,quota,groups,email,first_name+' '+last_name) - ev.increment({'name':'Added to nextcloud','data':[]}) + self.nextcloud.add_user_with_groups( + username, password, quota, groups, email, first_name + " " + last_name + ) + ev.increment({"name": "Added to nextcloud", "data": []}) except ProviderItemExists: - log.warning('User '+username+' already exists. Skipping...') + log.warning("User " + username + " already exists. Skipping...") except: log.error(traceback.format_exc()) - def delete_users(self,data): - ev=Events('Bulk actions','Deleting users:',total=len(data)) + def delete_users(self, data): + ev = Events("Bulk actions", "Deleting users:", total=len(data)) for user in data: - ev.increment({'name':user['username'],'data':user['username']}) - self.delete_user(user['id']) + ev.increment({"name": user["username"], "data": user["username"]}) + self.delete_user(user["id"]) self.resync_data() - def delete_user(self,userid): - log.warning('Deleting user moodle, nextcloud keycloak') - ev=Events('Deleting user','Deleting from moodle') + def delete_user(self, userid): + log.warning("Deleting user moodle, nextcloud keycloak") + ev = Events("Deleting user", "Deleting from moodle") self.delete_moodle_user(userid) - ev.update_text('Deleting from nextcloud') + ev.update_text("Deleting from nextcloud") self.delete_nextcloud_user(userid) - ev.update_text('Deleting from keycloak') + ev.update_text("Deleting from keycloak") self.delete_keycloak_user(userid) - ev.update_text('Syncing data from applications...') + ev.update_text("Syncing data from applications...") self.resync_data() - ev.update_text('User deleted') + ev.update_text("User deleted") return True - def get_user(self,userid): - user = [u for u in self.internal['users'] if u['id']==userid] - if not len(user): return False + def get_user(self, userid): + user = [u for u in self.internal["users"] if u["id"] == userid] + if not len(user): + return False return user[0] - def get_user_username(self,username): - user = [u for u in self.internal['users'] if u['username']==username] - if not len(user): return False + def get_user_username(self, username): + user = [u for u in self.internal["users"] if u["username"] == username] + if not len(user): + return False return user[0] - def add_user(self,u): + def add_user(self, u): - pathslist=[] - for group in u['groups']: - pathpart='' - for part in group.split('.'): - if pathpart=='': - pathpart=part + pathslist = [] + for group in u["groups"]: + pathpart = "" + for part in group.split("."): + if pathpart == "": + pathpart = part else: - pathpart=pathpart+'.'+part + pathpart = pathpart + "." + part pathslist.append(pathpart) ### KEYCLOAK ####################### - ev=Events('Add user',u['username'],total=5) - log.warning(' KEYCLOAK USERS: Adding user: '+u['username']) - uid=self.keycloak.add_user(u['username'],u['first'],u['last'],u['email'],u['password'],enabled=u['enabled']) + ev = Events("Add user", u["username"], total=5) + log.warning(" KEYCLOAK USERS: Adding user: " + u["username"]) + uid = self.keycloak.add_user( + u["username"], + u["first"], + u["last"], + u["email"], + u["password"], + enabled=u["enabled"], + ) - self.av.add_user_default_avatar(uid,u['role']) + self.av.add_user_default_avatar(uid, u["role"]) # Add user to role and group rolename - log.warning(' KEYCLOAK USERS: Assign user '+u['username']+' with initial pwd '+ u['password']+' to role '+u['role']) - self.keycloak.assign_realm_roles(uid,u['role']) - gid=self.keycloak.get_group_by_path(path='/'+u['role'])['id'] - self.keycloak.group_user_add(uid,gid) - ev.increment({'name':'Added to system','data':[]}) + log.warning( + " KEYCLOAK USERS: Assign user " + + u["username"] + + " with initial pwd " + + u["password"] + + " to role " + + u["role"] + ) + self.keycloak.assign_realm_roles(uid, u["role"]) + gid = self.keycloak.get_group_by_path(path="/" + u["role"])["id"] + self.keycloak.group_user_add(uid, gid) + ev.increment({"name": "Added to system", "data": []}) # Add user to groups for path in pathslist: - path='/'+path.replace('.','/') - log.warning(' KEYCLOAK USERS: Assign user '+u['username']+' to group '+ path) - gid=self.keycloak.get_group_by_path(path=path)['id'] - self.keycloak.group_user_add(uid,gid) - ev.increment({'name':'Added to system groups','data':[]}) + path = "/" + path.replace(".", "/") + log.warning( + " KEYCLOAK USERS: Assign user " + u["username"] + " to group " + path + ) + gid = self.keycloak.get_group_by_path(path=path)["id"] + self.keycloak.group_user_add(uid, gid) + ev.increment({"name": "Added to system groups", "data": []}) - pathslist.append(u['role']) + pathslist.append(u["role"]) ### MOODLE ############################### # Add user - log.warning('Creating moodle user: '+u['username']) + log.warning("Creating moodle user: " + u["username"]) try: - moodle_id=self.moodle.create_user(u['email'],u['username'],'*12'+secrets.token_urlsafe(16),u['first'],u['last'])[0]['id'] - ev.increment({'name':'Added to moodle','data':[]}) + moodle_id = self.moodle.create_user( + u["email"], + u["username"], + "*12" + secrets.token_urlsafe(16), + u["first"], + u["last"], + )[0]["id"] + ev.increment({"name": "Added to moodle", "data": []}) except UserExists: - log.error(' -->> User already exists') - error=Events('User already exists.',str(se),type='error') + log.error(" -->> User already exists") + error = Events("User already exists.", str(se), type="error") except SystemError as se: - log.error('Moodle create user error: '+str(se)) - error=Events('Moodle create user error',str(se),type='error') + log.error("Moodle create user error: " + str(se)) + error = Events("Moodle create user error", str(se), type="error") except: - log.error(' -->> Error creating on moodle the user: '+u['username']) + log.error(" -->> Error creating on moodle the user: " + u["username"]) print(traceback.format_exc()) - error=Events('Internal error','Check logs',type='error') + error = Events("Internal error", "Check logs", type="error") # Add user to cohort ## Get all existing moodle cohorts - cohorts=self.moodle.get_cohorts() + cohorts = self.moodle.get_cohorts() for path in pathslist: - print('MOODLE ADD SUBPATH: '+path) + print("MOODLE ADD SUBPATH: " + path) try: - cohort=[c for c in cohorts if c['name']==path][0] + cohort = [c for c in cohorts if c["name"] == path][0] except: # print(traceback.format_exc()) - log.error(' MOODLE USER GROUPS: keycloak group '+path+' does not exist as moodle cohort. This should not happen. User '+u['username']+ ' not added.') + log.error( + " MOODLE USER GROUPS: keycloak group " + + path + + " does not exist as moodle cohort. This should not happen. User " + + u["username"] + + " not added." + ) try: - self.moodle.add_user_to_cohort(moodle_id,cohort['id']) + self.moodle.add_user_to_cohort(moodle_id, cohort["id"]) except: - log.error(' MOODLE USER GROUPS: User '+u['username']+' already exists in cohort '+cohort['name']) + log.error( + " MOODLE USER GROUPS: User " + + u["username"] + + " already exists in cohort " + + cohort["name"] + ) - ev.increment({'name':'Added to moodle cohorts','data':[]}) + ev.increment({"name": "Added to moodle cohorts", "data": []}) ### NEXTCLOUD ########################3 - log.warning(' NEXTCLOUD USERS: Creating nextcloud user: '+u['username']+' in groups '+str(list)) + log.warning( + " NEXTCLOUD USERS: Creating nextcloud user: " + + u["username"] + + " in groups " + + str(list) + ) try: # Quota is in MB - self.nextcloud.add_user_with_groups(u['username'],'*12'+secrets.token_urlsafe(16),u['quota'],pathslist,u['email'],u['first']+' '+u['last']) - ev.increment({'name':'Added to nextcloud','data':[]}) + self.nextcloud.add_user_with_groups( + u["username"], + "*12" + secrets.token_urlsafe(16), + u["quota"], + pathslist, + u["email"], + u["first"] + " " + u["last"], + ) + ev.increment({"name": "Added to nextcloud", "data": []}) except ProviderItemExists: - log.warning('User '+u['username']+' already exists. Skipping...') + log.warning("User " + u["username"] + " already exists. Skipping...") except: log.error(traceback.format_exc()) self.resync_data() - def add_group(self,g): + def add_group(self, g): # TODO: Check if exists # We add in keycloak with his name, will be shown in app with full path with dots - if g['parent'] != None: g['parent'] = gid2kpath(g['parent']) + if g["parent"] != None: + g["parent"] = gid2kpath(g["parent"]) - new_path=self.keycloak.add_group(g['name'],g['parent']) + new_path = self.keycloak.add_group(g["name"], g["parent"]) - if g['parent'] != None: - new_path=kpath2gid(new_path['path']) + if g["parent"] != None: + new_path = kpath2gid(new_path["path"]) else: - new_path=g['name'] + new_path = g["name"] - self.moodle.add_system_cohort(new_path,description=g['description']) + self.moodle.add_system_cohort(new_path, description=g["description"]) self.nextcloud.add_group(new_path) - def delete_group_by_id(self,group_id): + def delete_group_by_id(self, group_id): # TODO: Check if exists (None) group = self.keycloak.get_group_by_id(group_id) - to_be_deleted=[] + to_be_deleted = [] # Childs - for internalgroup in self.internal['groups']: - if internalgroup['name'].startswith(group['name']+'.'): to_be_deleted.append(internalgroup['name']) - to_be_deleted.append(kpath2gid(group['path'])) + for internalgroup in self.internal["groups"]: + if internalgroup["name"].startswith(group["name"] + "."): + to_be_deleted.append(internalgroup["name"]) + to_be_deleted.append(kpath2gid(group["path"])) try: - self.keycloak.delete_group(group['id']) + self.keycloak.delete_group(group["id"]) except: - log.error('KEYCLOAK: Could no delete group '+group['path']) + log.error("KEYCLOAK: Could no delete group " + group["path"]) - cohorts=self.moodle.get_cohorts() + cohorts = self.moodle.get_cohorts() for gid in to_be_deleted: - cohort = [c['id'] for c in cohorts if c['name'] == gid] + cohort = [c["id"] for c in cohorts if c["name"] == gid] self.moodle.delete_cohorts(cohort) self.nextcloud.delete_group(gid) + def delete_group_by_path(self, path): + group = self.keycloak.get_group_by_path(path) - def delete_group_by_path(self,path): - group =self.keycloak.get_group_by_path(path) - - to_be_deleted=[] + to_be_deleted = [] # Childs - for internalgroup in self.internal['groups']: - if internalgroup['name'].startswith(group['name']+'.'): to_be_deleted.append(internalgroup['name']) - to_be_deleted.append(kpath2gid(group['path'])) + for internalgroup in self.internal["groups"]: + if internalgroup["name"].startswith(group["name"] + "."): + to_be_deleted.append(internalgroup["name"]) + to_be_deleted.append(kpath2gid(group["path"])) try: - self.keycloak.delete_group(group['id']) + self.keycloak.delete_group(group["id"]) except: - log.error('KEYCLOAK: Could no delete group '+group['path']) + log.error("KEYCLOAK: Could no delete group " + group["path"]) - cohorts=self.moodle.get_cohorts() + cohorts = self.moodle.get_cohorts() for gid in to_be_deleted: - cohort = [c['id'] for c in cohorts if c['name'] == gid] + cohort = [c["id"] for c in cohorts if c["name"] == gid] self.moodle.delete_cohorts(cohort) self.nextcloud.delete_group(gid) - diff --git a/admin/src/admin/lib/avatars.py b/admin/src/admin/lib/avatars.py index f80f82c..65caab3 100644 --- a/admin/src/admin/lib/avatars.py +++ b/admin/src/admin/lib/avatars.py @@ -1,49 +1,61 @@ -from requests import get, post -from admin import app - import logging as log -from pprint import pprint import os +from pprint import pprint from minio import Minio from minio.commonconfig import REPLACE, CopySource from minio.deleteobjects import DeleteObject +from requests import get, post -class Avatars(): +from admin import app + + +class Avatars: def __init__(self): self.mclient = Minio( - "isard-sso-avatars:9000", - access_key="AKIAIOSFODNN7EXAMPLE", - secret_key="wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY", - secure=False - ) - self.bucket='master-avatars' + "isard-sso-avatars:9000", + access_key="AKIAIOSFODNN7EXAMPLE", + secret_key="wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY", + secure=False, + ) + self.bucket = "master-avatars" self._minio_set_realm() # self.update_missing_avatars() - def add_user_default_avatar(self,userid,role='unknown'): + def add_user_default_avatar(self, userid, role="unknown"): self.mclient.fput_object( - self.bucket, userid, os.path.join(app.root_path,"../custom/avatars/"+role+'.jpg'), - content_type="image/jpeg ", + self.bucket, + userid, + os.path.join(app.root_path, "../custom/avatars/" + role + ".jpg"), + content_type="image/jpeg ", + ) + log.warning( + " AVATARS: Updated avatar for user " + userid + " with role " + role ) - log.warning(' AVATARS: Updated avatar for user '+userid+' with role '+role) - def delete_user_avatar(self,userid): + def delete_user_avatar(self, userid): self.minio_delete_object(userid) - - def update_missing_avatars(self,users): - sys_roles=['admin','manager','teacher','student'] + + def update_missing_avatars(self, users): + sys_roles = ["admin", "manager", "teacher", "student"] for u in self.get_users_without_image(users): try: - img=[r+'.jpg' for r in sys_roles if r in u['roles']][0] + img = [r + ".jpg" for r in sys_roles if r in u["roles"]][0] except: - img='unknown.jpg' + img = "unknown.jpg" self.mclient.fput_object( - self.bucket, u['id'], os.path.join(app.root_path,"../custom/avatars/"+img), - content_type="image/jpeg ", + self.bucket, + u["id"], + os.path.join(app.root_path, "../custom/avatars/" + img), + content_type="image/jpeg ", + ) + log.warning( + " AVATARS: Updated avatar for user " + + u["username"] + + " with role " + + img.split(".")[0] ) - log.warning(' AVATARS: Updated avatar for user '+u['username']+' with role '+img.split('.')[0]) def _minio_set_realm(self): if not self.mclient.bucket_exists(self.bucket): @@ -57,14 +69,14 @@ class Avatars(): lambda x: DeleteObject(x.object_name), self.mclient.list_objects(self.bucket), ) - errors=self.mclient.remove_objects(self.bucket, delete_object_list) + errors = self.mclient.remove_objects(self.bucket, delete_object_list) for error in errors: - log.error(" AVATARS: Error occured when deleting avatar object: "+ error) + log.error(" AVATARS: Error occured when deleting avatar object: " + error) - def minio_delete_object(self,oid): - errors=self.mclient.remove_objects(self.bucket, [DeleteObject(oid)]) + def minio_delete_object(self, oid): + errors = self.mclient.remove_objects(self.bucket, [DeleteObject(oid)]) for error in errors: - log.error(" AVATARS: Error occured when deleting avatar object: "+ error) + log.error(" AVATARS: Error occured when deleting avatar object: " + error) - def get_users_without_image(self,users): - return [u for u in users if u['id'] and u['id'] not in self.minio_get_objects()] \ No newline at end of file + def get_users_without_image(self, users): + return [u for u in users if u["id"] and u["id"] not in self.minio_get_objects()] diff --git a/admin/src/admin/lib/events.py b/admin/src/admin/lib/events.py index c854855..ddcffe5 100644 --- a/admin/src/admin/lib/events.py +++ b/admin/src/admin/lib/events.py @@ -1,110 +1,166 @@ #!flask/bin/python # coding=utf-8 -from admin import app -import logging as log -import traceback - -from uuid import uuid4 -import json -from time import sleep -import sys,os -from flask import render_template, Response, request, redirect, url_for, jsonify -from flask_socketio import SocketIO, emit, join_room, leave_room, \ - close_room, rooms, disconnect, send import base64 +import json +import logging as log +import os +import sys +import traceback +from time import sleep +from uuid import uuid4 -class Events(): - def __init__(self,title,text='',total=0,table=False,type='info'): +from flask import Response, jsonify, redirect, render_template, request, url_for +from flask_socketio import ( + SocketIO, + close_room, + disconnect, + emit, + join_room, + leave_room, + rooms, + send, +) + +from admin import app + + +class Events: + def __init__(self, title, text="", total=0, table=False, type="info"): # notice, info, success, and error - self.eid=str(base64.b64encode(os.urandom(32))[:8]) - self.title=title - self.text=text - self.total=total - self.table=table - self.item=0 - self.type=type + self.eid = str(base64.b64encode(os.urandom(32))[:8]) + self.title = title + self.text = text + self.total = total + self.table = table + self.item = 0 + self.type = type self.create() def create(self): - log.info('START '+self.eid+': '+self.text) - app.socketio.emit('notify-create', - json.dumps({'id':self.eid, - 'title':self.title, - 'text':self.text, - 'type':self.type}), - namespace='/sio', - room='admin') + log.info("START " + self.eid + ": " + self.text) + app.socketio.emit( + "notify-create", + json.dumps( + { + "id": self.eid, + "title": self.title, + "text": self.text, + "type": self.type, + } + ), + namespace="/sio", + room="admin", + ) sleep(0.001) def __del__(self): - log.info('END '+self.eid+': '+self.text) - app.socketio.emit('notify-destroy', - json.dumps({'id':self.eid}), - namespace='/sio', - room='admin') + log.info("END " + self.eid + ": " + self.text) + app.socketio.emit( + "notify-destroy", + json.dumps({"id": self.eid}), + namespace="/sio", + room="admin", + ) sleep(0.001) - def update_text(self,text): - self.text=text - app.socketio.emit('notify-update', - json.dumps({'id':self.eid, - 'text':self.text,}), - namespace='/sio', - room='admin') + def update_text(self, text): + self.text = text + app.socketio.emit( + "notify-update", + json.dumps( + { + "id": self.eid, + "text": self.text, + } + ), + namespace="/sio", + room="admin", + ) sleep(0.001) - def append_text(self,text): - self.text=self.text+'
'+text - app.socketio.emit('notify-update', - json.dumps({'id':self.eid, - 'text':self.text,}), - namespace='/sio', - room='admin') + def append_text(self, text): + self.text = self.text + "
" + text + app.socketio.emit( + "notify-update", + json.dumps( + { + "id": self.eid, + "text": self.text, + } + ), + namespace="/sio", + room="admin", + ) sleep(0.001) - def increment(self,data={'name':'','data':[]}): - self.item+=1 - log.info('INCREMENT '+self.eid+': '+self.text) - app.socketio.emit('notify-increment', - json.dumps({'id':self.eid, - 'title':self.title, - 'text': '['+str(self.item)+'/'+str(self.total)+'] '+self.text+' '+data['name'], - 'item':self.item, - 'total':self.total, - 'table':self.table, - 'type':self.type, - 'data':data}), - namespace='/sio', - room='admin') + def increment(self, data={"name": "", "data": []}): + self.item += 1 + log.info("INCREMENT " + self.eid + ": " + self.text) + app.socketio.emit( + "notify-increment", + json.dumps( + { + "id": self.eid, + "title": self.title, + "text": "[" + + str(self.item) + + "/" + + str(self.total) + + "] " + + self.text + + " " + + data["name"], + "item": self.item, + "total": self.total, + "table": self.table, + "type": self.type, + "data": data, + } + ), + namespace="/sio", + room="admin", + ) sleep(0.0001) - def decrement(self,data={'name':'','data':[]}): - self.item-=1 - log.info('DECREMENT '+self.eid+': '+self.text) - app.socketio.emit('notify-decrement', - json.dumps({'id':self.eid, - 'title':self.title, - 'text': '['+str(self.item)+'/'+str(self.total)+'] '+self.text+' '+data['name'], - 'item':self.item, - 'total':self.total, - 'table':self.table, - 'type':self.type, - 'data':data}), - namespace='/sio', - room='admin') + def decrement(self, data={"name": "", "data": []}): + self.item -= 1 + log.info("DECREMENT " + self.eid + ": " + self.text) + app.socketio.emit( + "notify-decrement", + json.dumps( + { + "id": self.eid, + "title": self.title, + "text": "[" + + str(self.item) + + "/" + + str(self.total) + + "] " + + self.text + + " " + + data["name"], + "item": self.item, + "total": self.total, + "table": self.table, + "type": self.type, + "data": data, + } + ), + namespace="/sio", + room="admin", + ) sleep(0.001) def reload(self): - app.socketio.emit('reload', - json.dumps({}), - namespace='/sio', - room='admin') + app.socketio.emit("reload", json.dumps({}), namespace="/sio", room="admin") sleep(0.0001) - def table(self,event,table,data={}): + def table(self, event, table, data={}): # refresh, add, delete, update - app.socketio.emit('table_'+event, - json.dumps({'table':table,'data':data}), - namespace='/sio', - room='admin') - sleep(0.0001) \ No newline at end of file + app.socketio.emit( + "table_" + event, + json.dumps({"table": table, "data": data}), + namespace="/sio", + room="admin", + ) + sleep(0.0001) diff --git a/admin/src/admin/lib/exceptions.py b/admin/src/admin/lib/exceptions.py index 2eec0fc..4faa9ce 100644 --- a/admin/src/admin/lib/exceptions.py +++ b/admin/src/admin/lib/exceptions.py @@ -3,5 +3,6 @@ class UserExists(Exception): pass + class UserNotFound(Exception): - pass \ No newline at end of file + pass diff --git a/admin/src/admin/lib/helpers.py b/admin/src/admin/lib/helpers.py index 35abb33..c1d2f70 100644 --- a/admin/src/admin/lib/helpers.py +++ b/admin/src/admin/lib/helpers.py @@ -1,55 +1,74 @@ -import random, string -from pprint import pprint +import random +import string from collections import Counter +from pprint import pprint + def system_username(username): - return True if username in ['guest','ddadmin','admin'] or username.startswith('system_') else False + return ( + True + if username in ["guest", "ddadmin", "admin"] or username.startswith("system_") + else False + ) + def system_group(groupname): - return True if groupname in ['admin','manager','teacher','student'] else False + return True if groupname in ["admin", "manager", "teacher", "student"] else False -def get_group_from_group_id(group_id,groups): - return next((d for d in groups if d.get('id') == group_id), None) -def get_gid_from_kgroup_id(kgroup_id,groups): +def get_group_from_group_id(group_id, groups): + return next((d for d in groups if d.get("id") == group_id), None) + + +def get_gid_from_kgroup_id(kgroup_id, groups): # print(kgroup_id) # pprint(groups) # return get_group_from_group_id(kgroup_id,groups)['path'].replace('/','.')[1:] - return [g['path'].replace('/','.')[1:] for g in groups if g['id'] == kgroup_id][0] + return [g["path"].replace("/", ".")[1:] for g in groups if g["id"] == kgroup_id][0] + + +def get_gids_from_kgroup_ids(kgroup_ids, groups): + return [get_gid_from_kgroup_id(kgroup_id, groups) for kgroup_id in kgroup_ids] -def get_gids_from_kgroup_ids(kgroup_ids,groups): - return [get_gid_from_kgroup_id(kgroup_id,groups) for kgroup_id in kgroup_ids] def kpath2gid(path): # print(path.replace('/','.')[1:]) - return path.replace('/','.')[1:] + return path.replace("/", ".")[1:] + def gid2kpath(gid): - return '/'+gid.replace('.','/') + return "/" + gid.replace(".", "/") + def count_repeated(itemslist): print(Counter(itemslist)) + def groups_kname2gid(groups): - return [name.replace('.','/') for name in groups] + return [name.replace(".", "/") for name in groups] + def groups_path2id(groups): - return [g.replace('/','.')[1:] for g in groups] + return [g.replace("/", ".")[1:] for g in groups] + def groups_id2path(groups): - return ['/'+g.replace('.','/') for g in groups] + return ["/" + g.replace(".", "/") for g in groups] + def filter_roles_list(role_list): - client_roles=['admin','manager','teacher','student'] + client_roles = ["admin", "manager", "teacher", "student"] return [r for r in role_list if r in client_roles] + def filter_roles_listofdicts(role_listofdicts): - client_roles=['admin','manager','teacher','student'] - return [r for r in role_listofdicts if r['name'] in client_roles] + client_roles = ["admin", "manager", "teacher", "student"] + return [r for r in role_listofdicts if r["name"] in client_roles] + def rand_password(lenght): characters = string.ascii_letters + string.digits + string.punctuation - passwd = ''.join(random.choice(characters) for i in range(lenght)) + passwd = "".join(random.choice(characters) for i in range(lenght)) while not any(ele.isupper() for ele in passwd): - passwd = ''.join(random.choice(characters) for i in range(lenght)) - return passwd \ No newline at end of file + passwd = "".join(random.choice(characters) for i in range(lenght)) + return passwd diff --git a/admin/src/admin/lib/keycloak_client.py b/admin/src/admin/lib/keycloak_client.py index 9ea89dc..4aeda35 100644 --- a/admin/src/admin/lib/keycloak_client.py +++ b/admin/src/admin/lib/keycloak_client.py @@ -1,68 +1,82 @@ #!/usr/bin/env python # coding=utf-8 -import time ,os -from admin import app -from datetime import datetime, timedelta - +import json import logging as log +import os +import time import traceback -import yaml, json +from datetime import datetime, timedelta from pprint import pprint +import yaml from jinja2 import Environment, FileSystemLoader - from keycloak import KeycloakAdmin + +from admin import app + from .postgres import Postgres -class KeycloakClient(): - """https://www.keycloak.org/docs-api/13.0/rest-api/index.html - https://github.com/marcospereirampj/python-keycloak - https://gist.github.com/kaqfa/99829941121188d7cef8271f93f52f1f - """ - def __init__(self, - url="http://isard-sso-keycloak:8080/auth/", - username=os.environ['KEYCLOAK_USER'], - password=os.environ['KEYCLOAK_PASSWORD'], - realm='master', - verify=True): - self.url=url - self.username=username - self.password=password - self.realm=realm - self.verify=verify - self.keycloak_pg=Postgres('isard-apps-postgresql','keycloak',os.environ['KEYCLOAK_DB_USER'],os.environ['KEYCLOAK_DB_PASSWORD']) +class KeycloakClient: + """https://www.keycloak.org/docs-api/13.0/rest-api/index.html + https://github.com/marcospereirampj/python-keycloak + https://gist.github.com/kaqfa/99829941121188d7cef8271f93f52f1f + """ + + def __init__( + self, + url="http://isard-sso-keycloak:8080/auth/", + username=os.environ["KEYCLOAK_USER"], + password=os.environ["KEYCLOAK_PASSWORD"], + realm="master", + verify=True, + ): + self.url = url + self.username = username + self.password = password + self.realm = realm + self.verify = verify + + self.keycloak_pg = Postgres( + "isard-apps-postgresql", + "keycloak", + os.environ["KEYCLOAK_DB_USER"], + os.environ["KEYCLOAK_DB_PASSWORD"], + ) def connect(self): - self.keycloak_admin = KeycloakAdmin(server_url=self.url, - username=self.username, - password=self.password, - realm_name=self.realm, - verify=self.verify) -# from keycloak import KeycloakAdmin -# keycloak_admin = KeycloakAdmin(server_url="http://isard-sso-keycloak:8080/auth/",username="admin",password="keycloakkeycloak",realm_name="master",verify=False) + self.keycloak_admin = KeycloakAdmin( + server_url=self.url, + username=self.username, + password=self.password, + realm_name=self.realm, + verify=self.verify, + ) - ######## Example create group and subgroup - - # try: - # self.add_group('level1') - # except: - # self.delete_group(self.get_group('/level1')['id']) - # self.add_group('level1') - # self.add_group('level2',parent=self.get_group('/level1')['id']) - # pprint(self.get_groups()) + # from keycloak import KeycloakAdmin + # keycloak_admin = KeycloakAdmin(server_url="http://isard-sso-keycloak:8080/auth/",username="admin",password="keycloakkeycloak",realm_name="master",verify=False) - ######## Example roles - # try: - # self.add_role('superman') - # except: - # self.delete_role('superman') - # self.add_role('superman') - # pprint(self.get_roles()) + ######## Example create group and subgroup - ''' USERS ''' + # try: + # self.add_group('level1') + # except: + # self.delete_group(self.get_group('/level1')['id']) + # self.add_group('level1') + # self.add_group('level2',parent=self.get_group('/level1')['id']) + # pprint(self.get_groups()) - def get_user_id(self,username): + ######## Example roles + # try: + # self.add_role('superman') + # except: + # self.delete_role('superman') + # self.add_role('superman') + # pprint(self.get_roles()) + + """ USERS """ + + def get_user_id(self, username): self.connect() return self.keycloak_admin.get_user_id(username) @@ -85,24 +99,31 @@ class KeycloakClient(): group by u.id,u.username,u.email,u.first_name,u.last_name, u.realm_id, u.enabled, ua.value order by u.username""" - (headers,users)=self.keycloak_pg.select_with_headers(q) + (headers, users) = self.keycloak_pg.select_with_headers(q) - users_with_lists = [list(l[:-4])+([[]] if l[-4] == [None] else [list(set(l[-4]))]) +\ - ([[]] if l[-3] == [None] else [list(set(l[-3]))]) +\ - ([[]] if l[-3] == [None] else [list(set(l[-2]))]) +\ - ([[]] if l[-1] == [None] else [list(set(l[-1]))]) for l in users] - - users_with_lists = [list(l[:-4])+([[]] if l[-4] == [None] else [list(set(l[-4]))]) +\ - ([[]] if l[-3] == [None] else [list(set(l[-3]))]) +\ - ([[]] if l[-3] == [None] else [list(set(l[-2]))]) +\ - ([[]] if l[-1] == [None] else [list(set(l[-1]))]) for l in users_with_lists] + users_with_lists = [ + list(l[:-4]) + + ([[]] if l[-4] == [None] else [list(set(l[-4]))]) + + ([[]] if l[-3] == [None] else [list(set(l[-3]))]) + + ([[]] if l[-3] == [None] else [list(set(l[-2]))]) + + ([[]] if l[-1] == [None] else [list(set(l[-1]))]) + for l in users + ] + + users_with_lists = [ + list(l[:-4]) + + ([[]] if l[-4] == [None] else [list(set(l[-4]))]) + + ([[]] if l[-3] == [None] else [list(set(l[-3]))]) + + ([[]] if l[-3] == [None] else [list(set(l[-2]))]) + + ([[]] if l[-1] == [None] else [list(set(l[-1]))]) + for l in users_with_lists + ] list_dict_users = [dict(zip(headers, r)) for r in users_with_lists] # self.connect() # groups = self.keycloak_admin.get_groups() - # for user in list_dict_users: # new_user_groups = [] # for group_id in user['group']: @@ -113,31 +134,31 @@ class KeycloakClient(): # user['group']=new_user_groups return list_dict_users - - - def getparent(self,group_id, data): + def getparent(self, group_id, data): # Recursively get full path from any group_id in the tree path = "" for item in data: if group_id == item[0]: path = self.getparent(item[2], data) - path = f'{path}/{item[1]}' + path = f"{path}/{item[1]}" return path - def get_group_path(self,group_id): + def get_group_path(self, group_id): # Get full path using getparent recursive func # RETURNS: String with full path q = """SELECT * FROM keycloak_group""" - groups=self.keycloak_pg.select(q) - return self.getparent(group_id,groups) + groups = self.keycloak_pg.select(q) + return self.getparent(group_id, groups) - def get_user_groups_paths(self,user_id): + def get_user_groups_paths(self, user_id): # Get full paths for user grups # RETURNS list of paths - q = """SELECT group_id FROM user_group_membership WHERE user_id = '%s'""" % (user_id) - user_group_ids=self.keycloak_pg.select(q) + q = """SELECT group_id FROM user_group_membership WHERE user_id = '%s'""" % ( + user_id + ) + user_group_ids = self.keycloak_pg.select(q) - paths=[] + paths = [] for g in user_group_ids: paths.append(self.get_group_path(g[0])) return paths @@ -151,90 +172,116 @@ class KeycloakClient(): # user['roles']= [r['name'] for r in self.keycloak_admin.get_realm_roles_of_user(user_id=user['id'])] # return users - def add_user(self,username,first,last,email,password,group=False,temporary=True,enabled=True): + def add_user( + self, + username, + first, + last, + email, + password, + group=False, + temporary=True, + enabled=True, + ): # RETURNS string with keycloak user id (the main id in this app) self.connect() - username=username.lower() + username = username.lower() try: - uid=self.keycloak_admin.create_user({"email": email, - "username": username, - "enabled": enabled, - "firstName": first, - "lastName": last, - "credentials":[{"type":"password", - "value":password, - "temporary":temporary}]}) + uid = self.keycloak_admin.create_user( + { + "email": email, + "username": username, + "enabled": enabled, + "firstName": first, + "lastName": last, + "credentials": [ + {"type": "password", "value": password, "temporary": temporary} + ], + } + ) except: log.error(traceback.format_exc()) if group: - path = '/'+group if group[1:] != '/' else group + path = "/" + group if group[1:] != "/" else group try: - gid=self.keycloak_admin.get_group_by_path(path=path,search_in_subgroups=False)['id'] + gid = self.keycloak_admin.get_group_by_path( + path=path, search_in_subgroups=False + )["id"] except: - self.keycloak_admin.create_group({"name":group}) - gid=self.keycloak_admin.get_group_by_path(path)['id'] - self.keycloak_admin.group_user_add(uid,gid) + self.keycloak_admin.create_group({"name": group}) + gid = self.keycloak_admin.get_group_by_path(path)["id"] + self.keycloak_admin.group_user_add(uid, gid) return uid - def update_user_pwd(self,user_id,password,temporary=True): - # Updates - payload={"credentials":[{"type":"password", - "value":password, - "temporary":temporary}]} + def update_user_pwd(self, user_id, password, temporary=True): + # Updates + payload = { + "credentials": [ + {"type": "password", "value": password, "temporary": temporary} + ] + } self.connect() - return self.keycloak_admin.update_user( user_id, payload) + return self.keycloak_admin.update_user(user_id, payload) - def user_update(self,user_id,enabled,email,first,last,groups=[],roles=[]): + def user_update(self, user_id, enabled, email, first, last, groups=[], roles=[]): ## NOTE: Roles didn't seem to be updated/added. Also not confident with groups - # Updates - payload={"enabled":enabled, - "email":email, - "firstName":first, - "lastName":last, - "groups":groups, - "realmRoles":roles} + # Updates + payload = { + "enabled": enabled, + "email": email, + "firstName": first, + "lastName": last, + "groups": groups, + "realmRoles": roles, + } self.connect() - return self.keycloak_admin.update_user( user_id, payload) + return self.keycloak_admin.update_user(user_id, payload) - def user_enable(self,user_id): - payload={"enabled":True} + def user_enable(self, user_id): + payload = {"enabled": True} self.connect() - return self.keycloak_admin.update_user( user_id, payload) + return self.keycloak_admin.update_user(user_id, payload) - def user_disable(self,user_id): - payload={"enabled":False} + def user_disable(self, user_id): + payload = {"enabled": False} self.connect() - return self.keycloak_admin.update_user( user_id, payload) + return self.keycloak_admin.update_user(user_id, payload) - def group_user_remove(self,user_id,group_id): + def group_user_remove(self, user_id, group_id): self.connect() - return self.keycloak_admin.group_user_remove(user_id,group_id) + return self.keycloak_admin.group_user_remove(user_id, group_id) # def add_user_role(self,user_id,role_id): # self.connect() # return self.keycloak_admin.assign_role(client_id=client_id, user_id=user_id, role_id=role_id, role_name="test") - def remove_user_realm_roles(self,user_id,roles): + def remove_user_realm_roles(self, user_id, roles): self.connect() - roles = [r for r in self.get_user_realm_roles(user_id) if r['name'] in ['admin','manager','teacher','student']] - return self.keycloak_admin.delete_realm_roles_of_user(user_id,roles) + roles = [ + r + for r in self.get_user_realm_roles(user_id) + if r["name"] in ["admin", "manager", "teacher", "student"] + ] + return self.keycloak_admin.delete_realm_roles_of_user(user_id, roles) - def delete_user(self,userid): + def delete_user(self, userid): self.connect() return self.keycloak_admin.delete_user(user_id=userid) - def get_user_groups(self,userid): + def get_user_groups(self, userid): self.connect() return self.keycloak_admin.get_user_groups(user_id=userid) - def get_user_realm_roles(self,userid): + def get_user_realm_roles(self, userid): self.connect() return self.keycloak_admin.get_realm_roles_of_user(user_id=userid) - def add_user_client_role(self,client_id,user_id,role_id,role_name): + def add_user_client_role(self, client_id, user_id, role_id, role_name): self.connect() - return self.keycloak_admin.assign_client_role(client_id=client_id, user_id=user_id, role_id=role_id, role_name="test") + return self.keycloak_admin.assign_client_role( + client_id=client_id, user_id=user_id, role_id=role_id, role_name="test" + ) ## GROUPS def get_all_groups(self): @@ -242,75 +289,77 @@ class KeycloakClient(): self.connect() return self.keycloak_admin.get_groups() - def get_recursive_groups(self, l_groups,l=[]): + def get_recursive_groups(self, l_groups, l=[]): for d_group in l_groups: d = {} for key, value in d_group.items(): - if key == 'subGroups': - self.get_recursive_groups(value,l) + if key == "subGroups": + self.get_recursive_groups(value, l) else: - d[key]=value + d[key] = value l.append(d) return l - def get_groups(self,with_subgroups=True): + def get_groups(self, with_subgroups=True): ## RETURNS ALL GROUPS in root list self.connect() groups = self.keycloak_admin.get_groups() return self.get_recursive_groups(groups) - subgroups=[] - subgroups1=[] + subgroups = [] + subgroups1 = [] # This needs to be recursive function if with_subgroups: for group in groups: - if len(group['subGroups']): - for sg in group['subGroups']: + if len(group["subGroups"]): + for sg in group["subGroups"]: subgroups.append(sg) # for sgroup in subgroups: # if len(sgroup['subGroups']): # for sg1 in sgroup['subGroups']: # subgroups1.append(sg1) - - return groups+subgroups+subgroups1 - def get_group_by_id(self,group_id): + return groups + subgroups + subgroups1 + + def get_group_by_id(self, group_id): self.connect() return self.keycloak_admin.get_group(group_id=group_id) - def get_group_by_path(self,path,recursive=True): + def get_group_by_path(self, path, recursive=True): self.connect() - return self.keycloak_admin.get_group_by_path(path=path,search_in_subgroups=recursive) + return self.keycloak_admin.get_group_by_path( + path=path, search_in_subgroups=recursive + ) - def add_group(self,name,parent=None,skip_exists=False): + def add_group(self, name, parent=None, skip_exists=False): self.connect() - if parent != None: - parent=self.get_group_by_path(parent)['id'] - return self.keycloak_admin.create_group({"name":name}, parent=parent) + if parent != None: + parent = self.get_group_by_path(parent)["id"] + return self.keycloak_admin.create_group({"name": name}, parent=parent) - def delete_group(self,group_id): + def delete_group(self, group_id): self.connect() return self.keycloak_admin.delete_group(group_id=group_id) - def group_user_add(self,user_id,group_id): + def group_user_add(self, user_id, group_id): self.connect() return self.keycloak_admin.group_user_add(user_id, group_id) - def add_group_tree(self,path): - parts=path.split('/') - parent_path='/' - for i in range(1,len(parts)): + def add_group_tree(self, path): + parts = path.split("/") + parent_path = "/" + for i in range(1, len(parts)): if i == 1: try: - self.add_group(parts[i],None,skip_exists=True) + self.add_group(parts[i], None, skip_exists=True) except: - log.warning('KEYCLOAK: Group :'+parts[i]+ ' already exists.') - parent_path=parent_path+parts[i] + log.warning("KEYCLOAK: Group :" + parts[i] + " already exists.") + parent_path = parent_path + parts[i] else: try: - self.add_group(parts[i],parent_path,skip_exists=True) + self.add_group(parts[i], parent_path, skip_exists=True) except: - log.warning('KEYCLOAK: Group :'+parts[i]+ ' already exists.') - parent_path=parent_path+parts[i] + log.warning("KEYCLOAK: Group :" + parts[i] + " already exists.") + parent_path = parent_path + parts[i] # parts=path.split('/') # parent_path=None @@ -319,126 +368,149 @@ class KeycloakClient(): # try: # self.add_group(parts[i],parent_path,skip_exists=True) # except: - # if parent_path==None: + # if parent_path==None: # parent_path='/'+parts[i] # else: # parent_path=self.get_group_by_path(parent_path)['path'] # parent_path=parent_path+'/'+parts[i] # continue - - # if parent_path==None: + + # if parent_path==None: # parent_path='/'+parts[i] # else: # parent_path=parent_path+'/'+parts[i] - - # try: - # if i == 1: parent_id=self.add_group(parts[i]) - # except: - # # Main already exists?? What a fail! - # parent_id=self.get_group(parent_id)['id'] - # continue - # self.add_group(parts[i],parent_id) - def add_user_with_groups_and_role(self,username,first,last,email,password,role,groups): + # try: + # if i == 1: parent_id=self.add_group(parts[i]) + # except: + # # Main already exists?? What a fail! + # parent_id=self.get_group(parent_id)['id'] + # continue + # self.add_group(parts[i],parent_id) + + def add_user_with_groups_and_role( + self, username, first, last, email, password, role, groups + ): ## Add user - uid=self.add_user(username,first,last,email,password) + uid = self.add_user(username, first, last, email, password) ## Add user to role - log.info('User uid: '+str(uid)+ ' role: '+str(role)) + log.info("User uid: " + str(uid) + " role: " + str(role)) try: - therole=role[0] + therole = role[0] except: - therole='' - log.info(self.assign_realm_roles(uid,role)) + therole = "" + log.info(self.assign_realm_roles(uid, role)) ## Create groups in user for g in groups: - log.warning('Creating keycloak group: '+g) - parts=g.split('/') - parent_path=None - for i in range(1,len(parts)): + log.warning("Creating keycloak group: " + g) + parts = g.split("/") + parent_path = None + for i in range(1, len(parts)): # parent_id=None if parent_path==None else self.get_group(parent_path)['id'] try: - self.add_group(parts[i],parent_path,skip_exists=True) + self.add_group(parts[i], parent_path, skip_exists=True) except: - log.warning('Group '+str(parent_path)+ ' already exists. Skipping creation') + log.warning( + "Group " + + str(parent_path) + + " already exists. Skipping creation" + ) pass if parent_path is None: - thepath='/'+parts[i] + thepath = "/" + parts[i] else: - thepath=parent_path+'/'+parts[i] - if thepath=='/': - log.warning('Not adding the user '+username+' to any group as does not have any...') + thepath = parent_path + "/" + parts[i] + if thepath == "/": + log.warning( + "Not adding the user " + + username + + " to any group as does not have any..." + ) continue - gid=self.get_group_by_path(path=thepath)['id'] + gid = self.get_group_by_path(path=thepath)["id"] - log.warning('Adding '+username+' with uuid: '+uid+' to group '+g+' with uuid: '+gid) - self.keycloak_admin.group_user_add(uid,gid) + log.warning( + "Adding " + + username + + " with uuid: " + + uid + + " to group " + + g + + " with uuid: " + + gid + ) + self.keycloak_admin.group_user_add(uid, gid) - - if parent_path==None: parent_path='' - parent_path=parent_path+'/'+parts[i] + if parent_path == None: + parent_path = "" + parent_path = parent_path + "/" + parts[i] - # self.group_user_add(uid,gid) - ## ROLES def get_roles(self): self.connect() return self.keycloak_admin.get_realm_roles() - def get_role(self,name): + def get_role(self, name): self.connect() return self.keycloak_admin.get_realm_role(name) - def add_role(self,name,description=''): + def add_role(self, name, description=""): self.connect() - return self.keycloak_admin.create_realm_role({"name":name, "description":description}) + return self.keycloak_admin.create_realm_role( + {"name": name, "description": description} + ) - def delete_role(self,name): + def delete_role(self, name): self.connect() return self.keycloak_admin.delete_realm_role(name) - ## CLIENTS - def get_client_roles(self,client_id): + def get_client_roles(self, client_id): self.connect() return self.keycloak_admin.get_client_roles(client_id=client_id) - def add_client_role(self,client_id,name,description=''): + def add_client_role(self, client_id, name, description=""): self.connect() - return self.keycloak_admin.create_client_role(client_id, {'name': name, 'description':description, 'clientRole': True}) - + return self.keycloak_admin.create_client_role( + client_id, {"name": name, "description": description, "clientRole": True} + ) ## SYSTEM def get_server_info(self): self.connect() return self.keycloak_admin.get_server_info() - + def get_server_clients(self): self.connect() return self.keycloak_admin.get_clients() def get_server_rsa_key(self): self.connect() - rsa_key = [k for k in self.keycloak_admin.get_keys()['keys'] if k['type']=='RSA'][0] - return {'name':rsa_key['kid'],'certificate':rsa_key['certificate']} + rsa_key = [ + k for k in self.keycloak_admin.get_keys()["keys"] if k["type"] == "RSA" + ][0] + return {"name": rsa_key["kid"], "certificate": rsa_key["certificate"]} ## REALM def assign_realm_roles(self, user_id, role): self.connect() try: - role=[r for r in self.keycloak_admin.get_realm_roles() if r['name']==role] + role = [ + r for r in self.keycloak_admin.get_realm_roles() if r["name"] == role + ] except: return False return self.keycloak_admin.assign_realm_roles(user_id=user_id, roles=role) # return self.keycloak_admin.assign_realm_roles(user_id=user_id, client_id=None, roles=role) ## CLIENTS - def delete_client(self,clientid): + def delete_client(self, clientid): self.connect() return self.keycloak_admin.delete_client(clientid) - def add_client(self,client): + def add_client(self, client): self.connect() return self.keycloak_admin.create_client(client) diff --git a/admin/src/admin/lib/load_config.py b/admin/src/admin/lib/load_config.py index ffda344..58b8d9a 100644 --- a/admin/src/admin/lib/load_config.py +++ b/admin/src/admin/lib/load_config.py @@ -1,24 +1,39 @@ #!/usr/bin/env python # coding=utf-8 -from admin import app - -import os, sys import logging as log +import os +import sys import traceback -class loadConfig(): +from admin import app - def __init__(self, app=None): + +class loadConfig: + def __init__(self, app=None): try: - app.config.setdefault('DOMAIN', os.environ['DOMAIN']) - app.config.setdefault('KEYCLOAK_POSTGRES_USER', os.environ['KEYCLOAK_DB_USER']) - app.config.setdefault('KEYCLOAK_POSTGRES_PASSWORD', os.environ['KEYCLOAK_DB_PASSWORD']) - app.config.setdefault('MOODLE_POSTGRES_USER', os.environ['MOODLE_POSTGRES_USER']) - app.config.setdefault('MOODLE_POSTGRES_PASSWORD', os.environ['MOODLE_POSTGRES_PASSWORD']) - app.config.setdefault('NEXTCLOUD_POSTGRES_USER', os.environ['NEXTCLOUD_POSTGRES_USER']) - app.config.setdefault('NEXTCLOUD_POSTGRES_PASSWORD', os.environ['NEXTCLOUD_POSTGRES_PASSWORD']) - app.config.setdefault('VERIFY', True if os.environ['VERIFY']=="true" else False) + app.config.setdefault("DOMAIN", os.environ["DOMAIN"]) + app.config.setdefault( + "KEYCLOAK_POSTGRES_USER", os.environ["KEYCLOAK_DB_USER"] + ) + app.config.setdefault( + "KEYCLOAK_POSTGRES_PASSWORD", os.environ["KEYCLOAK_DB_PASSWORD"] + ) + app.config.setdefault( + "MOODLE_POSTGRES_USER", os.environ["MOODLE_POSTGRES_USER"] + ) + app.config.setdefault( + "MOODLE_POSTGRES_PASSWORD", os.environ["MOODLE_POSTGRES_PASSWORD"] + ) + app.config.setdefault( + "NEXTCLOUD_POSTGRES_USER", os.environ["NEXTCLOUD_POSTGRES_USER"] + ) + app.config.setdefault( + "NEXTCLOUD_POSTGRES_PASSWORD", os.environ["NEXTCLOUD_POSTGRES_PASSWORD"] + ) + app.config.setdefault( + "VERIFY", True if os.environ["VERIFY"] == "true" else False + ) except Exception as e: log.error(traceback.format_exc()) raise diff --git a/admin/src/admin/lib/moodle.py b/admin/src/admin/lib/moodle.py index 7febc06..25190f7 100644 --- a/admin/src/admin/lib/moodle.py +++ b/admin/src/admin/lib/moodle.py @@ -1,35 +1,43 @@ +import logging as log +import traceback +from pprint import pprint + from requests import get, post + from admin import app -import logging as log -from pprint import pprint -import traceback - -from .postgres import Postgres from .exceptions import UserExists, UserNotFound +from .postgres import Postgres # Module variables to connect to moodle api -class Moodle(): + +class Moodle: """https://github.com/mrcinv/moodle_api.py - https://docs.moodle.org/dev/Web_service_API_functions - https://docs.moodle.org/311/en/Using_web_services + https://docs.moodle.org/dev/Web_service_API_functions + https://docs.moodle.org/311/en/Using_web_services """ - def __init__(self, - key=app.config["MOODLE_WS_TOKEN"], - url="https://moodle."+app.config["DOMAIN"], - endpoint="/webservice/rest/server.php", - verify=app.config["VERIFY"]): + def __init__( + self, + key=app.config["MOODLE_WS_TOKEN"], + url="https://moodle." + app.config["DOMAIN"], + endpoint="/webservice/rest/server.php", + verify=app.config["VERIFY"], + ): self.key = key self.url = url self.endpoint = endpoint - self.verify=verify + self.verify = verify - self.moodle_pg=Postgres('isard-apps-postgresql','moodle',app.config['MOODLE_POSTGRES_USER'],app.config['MOODLE_POSTGRES_PASSWORD']) + self.moodle_pg = Postgres( + "isard-apps-postgresql", + "moodle", + app.config["MOODLE_POSTGRES_USER"], + app.config["MOODLE_POSTGRES_PASSWORD"], + ) - - def rest_api_parameters(self, in_args, prefix='', out_dict=None): + def rest_api_parameters(self, in_args, prefix="", out_dict=None): """Transform dictionary/array structure to a flat dictionary, with key names defining the structure. Example usage: @@ -37,23 +45,23 @@ class Moodle(): {'courses[0][id]':1, 'courses[0][name]':'course1'} """ - if out_dict==None: + if out_dict == None: out_dict = {} - if not type(in_args) in (list,dict): + if not type(in_args) in (list, dict): out_dict[prefix] = in_args return out_dict - if prefix == '': - prefix = prefix + '{0}' + if prefix == "": + prefix = prefix + "{0}" else: - prefix = prefix + '[{0}]' - if type(in_args)==list: + prefix = prefix + "[{0}]" + if type(in_args) == list: for idx, item in enumerate(in_args): self.rest_api_parameters(item, prefix.format(idx), out_dict) - elif type(in_args)==dict: + elif type(in_args) == dict: for key, item in in_args.items(): self.rest_api_parameters(item, prefix.format(key), out_dict) return out_dict - + def call(self, fname, **kwargs): """Calls moodle API function with function name fname and keyword arguments. Example: @@ -61,54 +69,69 @@ class Moodle(): courses = [{'id': 1, 'fullname': 'My favorite course'}]) """ parameters = self.rest_api_parameters(kwargs) - parameters.update({"wstoken": self.key, 'moodlewsrestformat': 'json', "wsfunction": fname}) - response = post(self.url+self.endpoint, parameters, verify=self.verify) + parameters.update( + {"wstoken": self.key, "moodlewsrestformat": "json", "wsfunction": fname} + ) + response = post(self.url + self.endpoint, parameters, verify=self.verify) response = response.json() - if type(response) == dict and response.get('exception'): + if type(response) == dict and response.get("exception"): raise SystemError(response) return response - def create_user(self, email, username, password, first_name='-', last_name='-'): - if len(self.get_user_by('username',username)['users']): + def create_user(self, email, username, password, first_name="-", last_name="-"): + if len(self.get_user_by("username", username)["users"]): raise UserExists try: - data = [{'username': username, 'email':email, - 'password': password, 'firstname':first_name, 'lastname':last_name}] - user = self.call('core_user_create_users', users=data) - return user #[{'id': 8, 'username': 'asdfw'}] + data = [ + { + "username": username, + "email": email, + "password": password, + "firstname": first_name, + "lastname": last_name, + } + ] + user = self.call("core_user_create_users", users=data) + return user # [{'id': 8, 'username': 'asdfw'}] except SystemError as se: - raise SystemError(se.args[0]['message']) + raise SystemError(se.args[0]["message"]) def update_user(self, username, email, first_name, last_name, enabled=True): - user = self.get_user_by('username',username)['users'][0] - if not len(user): + user = self.get_user_by("username", username)["users"][0] + if not len(user): raise UserNotFound try: - data = [{'id':user['id'],'username': username, 'email':email, - 'firstname':first_name, 'lastname':last_name, - 'suspended':0 if enabled else 1}] - user = self.call('core_user_update_users', users=data) + data = [ + { + "id": user["id"], + "username": username, + "email": email, + "firstname": first_name, + "lastname": last_name, + "suspended": 0 if enabled else 1, + } + ] + user = self.call("core_user_update_users", users=data) return user except SystemError as se: - raise SystemError(se.args[0]['message']) + raise SystemError(se.args[0]["message"]) def delete_user(self, user_id): - user = self.call('core_user_delete_users', userids=[user_id]) + user = self.call("core_user_delete_users", userids=[user_id]) return user def delete_users(self, userids): - user = self.call('core_user_delete_users', userids=userids) + user = self.call("core_user_delete_users", userids=userids) return user def get_user_by(self, key, value): - criteria = [{'key': key, 'value': value}] + criteria = [{"key": key, "value": value}] try: - user = self.call('core_user_get_users', criteria=criteria) + user = self.call("core_user_get_users", criteria=criteria) except: raise SystemError("Error calling Moodle API\n", traceback.format_exc()) return user - #{'users': [{'id': 8, 'username': 'asdfw', 'firstname': 'afowie', 'lastname': 'aokjdnfwe', 'fullname': 'afowie aokjdnfwe', 'email': 'awfewe@ads.com', 'department': '', 'firstaccess': 0, 'lastaccess': 0, 'auth': 'manual', 'suspended': False, 'confirmed': True, 'lang': 'ca', 'theme': '', 'timezone': '99', 'mailformat': 1, 'profileimageurlsmall': 'https://moodle.mydomain.duckdns.org/theme/image.php/cbe/core/1630941606/u/f2', 'profileimageurl': 'https://DOMAIN/theme/image.php/cbe/core/1630941606/u/f1'}], 'warnings': []} - + # {'users': [{'id': 8, 'username': 'asdfw', 'firstname': 'afowie', 'lastname': 'aokjdnfwe', 'fullname': 'afowie aokjdnfwe', 'email': 'awfewe@ads.com', 'department': '', 'firstaccess': 0, 'lastaccess': 0, 'auth': 'manual', 'suspended': False, 'confirmed': True, 'lang': 'ca', 'theme': '', 'timezone': '99', 'mailformat': 1, 'profileimageurlsmall': 'https://moodle.mydomain.duckdns.org/theme/image.php/cbe/core/1630941606/u/f2', 'profileimageurl': 'https://DOMAIN/theme/image.php/cbe/core/1630941606/u/f1'}], 'warnings': []} def get_users_with_groups_and_roles(self): q = """select u.id as id, username, firstname as first, lastname as last, email, json_agg(h.name) as groups, json_agg(r.shortname) as roles @@ -119,8 +142,13 @@ class Moodle(): left join mdl_role as r on r.id = ra.roleid where u.deleted = 0 group by u.id , username, first, last, email""" - (headers,users)=self.moodle_pg.select_with_headers(q) - users_with_lists = [list(l[:-2])+([[]] if l[-2] == [None] else [list(set(l[-2]))]) + ([[]] if l[-1] == [None] else [list(set(l[-1]))]) for l in users] + (headers, users) = self.moodle_pg.select_with_headers(q) + users_with_lists = [ + list(l[:-2]) + + ([[]] if l[-2] == [None] else [list(set(l[-2]))]) + + ([[]] if l[-1] == [None] else [list(set(l[-1]))]) + for l in users + ] list_dict_users = [dict(zip(headers, r)) for r in users_with_lists] return list_dict_users @@ -134,26 +162,32 @@ class Moodle(): def enroll_user_to_course(self, user_id, course_id, role_id=5): # 5 is student - data = [{'roleid': role_id, 'userid': user_id, 'courseid': course_id}] - enrolment = self.call('enrol_manual_enrol_users', enrolments=data) + data = [{"roleid": role_id, "userid": user_id, "courseid": course_id}] + enrolment = self.call("enrol_manual_enrol_users", enrolments=data) return enrolment - + def get_quiz_attempt(self, quiz_id, user_id): - attempts = self.call('mod_quiz_get_user_attempts', quizid=quiz_id, userid=user_id) + attempts = self.call( + "mod_quiz_get_user_attempts", quizid=quiz_id, userid=user_id + ) return attempts - + def get_cohorts(self): - cohorts = self.call('core_cohort_get_cohorts') + cohorts = self.call("core_cohort_get_cohorts") return cohorts - def add_system_cohort(self,name,description='',visible=True): - visible=1 if visible else 0 - data = [{'categorytype': {'type': 'system', 'value': ''}, - 'name': name, - 'idnumber': name, - 'description': description, - 'visible': visible}] - cohort = self.call('core_cohort_create_cohorts', cohorts=data) + def add_system_cohort(self, name, description="", visible=True): + visible = 1 if visible else 0 + data = [ + { + "categorytype": {"type": "system", "value": ""}, + "name": name, + "idnumber": name, + "description": description, + "visible": visible, + } + ] + cohort = self.call("core_cohort_create_cohorts", cohorts=data) return cohort # def add_users_to_cohort(self,users,cohort): @@ -161,52 +195,61 @@ class Moodle(): # user = self.call('core_cohort_add_cohort_members', criteria=criteria) # return user - def add_user_to_cohort(self,userid,cohortid): - members=[{'cohorttype':{'type':'id','value':cohortid}, - 'usertype':{'type':'id','value':userid}}] - user = self.call('core_cohort_add_cohort_members', members=members) + def add_user_to_cohort(self, userid, cohortid): + members = [ + { + "cohorttype": {"type": "id", "value": cohortid}, + "usertype": {"type": "id", "value": userid}, + } + ] + user = self.call("core_cohort_add_cohort_members", members=members) return user - def delete_user_in_cohort(self,userid,cohortid): - members=[{'cohortid':cohortid, - 'userid':userid}] - user = self.call('core_cohort_delete_cohort_members', members=members) + def delete_user_in_cohort(self, userid, cohortid): + members = [{"cohortid": cohortid, "userid": userid}] + user = self.call("core_cohort_delete_cohort_members", members=members) return user def get_cohort_members(self, cohort_ids): - members = self.call('core_cohort_get_cohort_members', cohortids=cohort_ids) - #[0]['userids'] + members = self.call("core_cohort_get_cohort_members", cohortids=cohort_ids) + # [0]['userids'] return members def delete_cohorts(self, cohortids): - deleted = self.call('core_cohort_delete_cohorts', cohortids=cohortids) + deleted = self.call("core_cohort_delete_cohorts", cohortids=cohortids) return deleted def get_user_cohorts(self, user_id): - user_cohorts=[] - cohorts=self.get_cohorts() + user_cohorts = [] + cohorts = self.get_cohorts() for cohort in cohorts: - if user_id in self.get_cohort_members(cohort['id']): user_cohorts.append(cohort) + if user_id in self.get_cohort_members(cohort["id"]): + user_cohorts.append(cohort) return user_cohorts - def add_user_to_siteadmin(self,user_id): + def add_user_to_siteadmin(self, user_id): q = """SELECT value FROM mdl_config WHERE name='siteadmins'""" - value=self.moodle_pg.select(q)[0][0] + value = self.moodle_pg.select(q)[0][0] if str(user_id) not in value: - value=value+','+str(user_id) - q = """UPDATE mdl_config SET value = '%s' WHERE name='siteadmins'""" % (value) + value = value + "," + str(user_id) + q = """UPDATE mdl_config SET value = '%s' WHERE name='siteadmins'""" % ( + value + ) self.moodle_pg.update(q) - log.warning('MOODLE:ADDING THE USER TO ADMINS: This needs a purge cache in moodle!') + log.warning( + "MOODLE:ADDING THE USER TO ADMINS: This needs a purge cache in moodle!" + ) # def add_role_to_user(self, user_id, role='admin', context='missing'): - # if role=='admin': + # if role=='admin': # role_id=1 # else: # return False # assignments = [{'roleid': role_id, 'userid': user_id, 'contextid': 0}] # self.call('core_role_assign_roles', assignments=assignments) - # userid=user_id, role_id=role_id) - # 'contextlevel': 1, + # userid=user_id, role_id=role_id) + # 'contextlevel': 1, + # define('CONTEXT_SYSTEM', 10); # define('CONTEXT_USER', 30); diff --git a/admin/src/admin/lib/mysql.py b/admin/src/admin/lib/mysql.py index ede7185..29e8dc3 100644 --- a/admin/src/admin/lib/mysql.py +++ b/admin/src/admin/lib/mysql.py @@ -1,32 +1,31 @@ #!/usr/bin/env python # coding=utf-8 +import json +import logging as log import time -from admin import app +import traceback from datetime import datetime, timedelta -import logging as log -import traceback -import yaml, json - import mysql.connector +import yaml -class Mysql(): +from admin import app - def __init__(self,host,database,user,password): + +class Mysql: + def __init__(self, host, database, user, password): self.conn = mysql.connector.connect( - host=host, - database=database, - user=user, - password=password) + host=host, database=database, user=user, password=password + ) - def select(self,sql): + def select(self, sql): self.cur = self.conn.cursor() self.cur.execute(sql) - data=self.cur.fetchall() + data = self.cur.fetchall() self.cur.close() return data - def update(self,sql): + def update(self, sql): self.cur = self.conn.cursor() self.cur.execute(sql) self.conn.commit() diff --git a/admin/src/admin/lib/nextcloud.py b/admin/src/admin/lib/nextcloud.py index 5e696a6..0ead939 100644 --- a/admin/src/admin/lib/nextcloud.py +++ b/admin/src/admin/lib/nextcloud.py @@ -1,38 +1,63 @@ #!/usr/bin/env python # coding=utf-8 -#from ..lib.log import * -from admin import app -import time,requests,json,pprint,os -import urllib -import traceback +import json import logging as log -from .nextcloud_exc import * +import os +import pprint +import time +import traceback +import urllib +import requests + +# from ..lib.log import * +from admin import app + +from .nextcloud_exc import * from .postgres import Postgres -class Nextcloud(): - def __init__(self, - url="https://nextcloud."+app.config['DOMAIN'], - username=os.environ['NEXTCLOUD_ADMIN_USER'], - password=os.environ['NEXTCLOUD_ADMIN_PASSWORD'], - verify=True): - self.verify_cert=verify - self.apiurl=url+'/ocs/v1.php/cloud/' - self.shareurl=url+'/ocs/v2.php/apps/files_sharing/api/v1/' - self.davurl=url+'/remote.php/dav/files/' - self.auth=(username,password) - self.user=username +class Nextcloud: + def __init__( + self, + url="https://nextcloud." + app.config["DOMAIN"], + username=os.environ["NEXTCLOUD_ADMIN_USER"], + password=os.environ["NEXTCLOUD_ADMIN_PASSWORD"], + verify=True, + ): - self.nextcloud_pg=Postgres('isard-apps-postgresql','nextcloud',app.config['NEXTCLOUD_POSTGRES_USER'],app.config['NEXTCLOUD_POSTGRES_PASSWORD']) + self.verify_cert = verify + self.apiurl = url + "/ocs/v1.php/cloud/" + self.shareurl = url + "/ocs/v2.php/apps/files_sharing/api/v1/" + self.davurl = url + "/remote.php/dav/files/" + self.auth = (username, password) + self.user = username - def _request(self,method,url,data={},headers={'OCS-APIRequest':'true'},auth=False): - if auth == False: auth=self.auth + self.nextcloud_pg = Postgres( + "isard-apps-postgresql", + "nextcloud", + app.config["NEXTCLOUD_POSTGRES_USER"], + app.config["NEXTCLOUD_POSTGRES_PASSWORD"], + ) + + def _request( + self, method, url, data={}, headers={"OCS-APIRequest": "true"}, auth=False + ): + if auth == False: + auth = self.auth try: - response = requests.request(method, url, data=data, auth=auth, verify=self.verify_cert, headers=headers) - if 'meta' in response.text: - if '997' in response.text: raise ProviderUnauthorized + response = requests.request( + method, + url, + data=data, + auth=auth, + verify=self.verify_cert, + headers=headers, + ) + if "meta" in response.text: + if "997" in response.text: + raise ProviderUnauthorized # if '998' in response.text: raise ProviderInvalidQuery return response.text @@ -48,15 +73,16 @@ class Nextcloud(): # except requests.exceptions.RequestException as err: # raise ProviderError except Exception as e: - if str(e) == 'an integer is required (got type bytes)': + if str(e) == "an integer is required (got type bytes)": raise ProviderConnError raise ProviderError def check_connection(self): - url = self.apiurl + "users/"+self.user+"?format=json" + url = self.apiurl + "users/" + self.user + "?format=json" try: - result = self._request('GET',url) - if json.loads(result)['ocs']['meta']['statuscode'] == 100: return True + result = self._request("GET", url) + if json.loads(result)["ocs"]["meta"]["statuscode"] == 100: + return True raise ProviderError except requests.exceptions.HTTPError as errh: raise ProviderConnError @@ -69,43 +95,43 @@ class Nextcloud(): except requests.exceptions.RequestException as err: raise ProviderError except Exception as e: - if str(e) == 'an integer is required (got type bytes)': + if str(e) == "an integer is required (got type bytes)": raise ProviderConnError raise ProviderError - def get_user(self,userid): - url = self.apiurl + "users/"+userid+"?format=json" + def get_user(self, userid): + url = self.apiurl + "users/" + userid + "?format=json" try: - result = json.loads(self._request('GET',url)) - if result['ocs']['meta']['statuscode'] == 100: return result['ocs']['data'] + result = json.loads(self._request("GET", url)) + if result["ocs"]["meta"]["statuscode"] == 100: + return result["ocs"]["data"] raise ProviderItemNotExists except: log.error(traceback.format_exc()) - raise + raise # 100 - successful -# q = """select u.uid as username, adn.value as displayname, ade.value as email, json_agg(gg.displayname) as admin_groups,json_agg(g.displayname) as groups -# from oc_users as u -# left join oc_group_user as gu on gu.uid = u.uid -# left join oc_groups as g on gu.gid = g.gid -# left join oc_group_admin as ga on ga.uid = u.uid -# left join oc_groups as gg on gg.gid = ga.gid -# left join oc_accounts_data as adn on adn.uid = u.uid and adn.name = 'displayname' -# left join oc_accounts_data as ade on ade.uid = u.uid and ade.name = 'email' -# group by u.uid, adn.value, ade.value""" -# cur.execute(q) -# users = cur.fetchall() -# fields = [a.name for a in cur.description] -# cur.close() -# conn.close() + # q = """select u.uid as username, adn.value as displayname, ade.value as email, json_agg(gg.displayname) as admin_groups,json_agg(g.displayname) as groups + # from oc_users as u + # left join oc_group_user as gu on gu.uid = u.uid + # left join oc_groups as g on gu.gid = g.gid + # left join oc_group_admin as ga on ga.uid = u.uid + # left join oc_groups as gg on gg.gid = ga.gid + # left join oc_accounts_data as adn on adn.uid = u.uid and adn.name = 'displayname' + # left join oc_accounts_data as ade on ade.uid = u.uid and ade.name = 'email' + # group by u.uid, adn.value, ade.value""" + # cur.execute(q) + # users = cur.fetchall() + # fields = [a.name for a in cur.description] + # cur.close() + # conn.close() - -# users_with_lists = [list(l[:-2])+([[]] if l[-2] == [None] else [list(set(l[-2]))]) + ([[]] if l[-1] == [None] else [list(set(l[-1]))]) for l in users] -# users_with_lists = [list(l[:-2])+([[]] if l[-2] == [None] else [list(set(l[-2]))]) + ([[]] if l[-1] == [None] else [list(set(l[-1]))]) for l in users_with_lists] -# list_dict_users = [dict(zip(fields, r)) for r in users_with_lists] + # users_with_lists = [list(l[:-2])+([[]] if l[-2] == [None] else [list(set(l[-2]))]) + ([[]] if l[-1] == [None] else [list(set(l[-1]))]) for l in users] + # users_with_lists = [list(l[:-2])+([[]] if l[-2] == [None] else [list(set(l[-2]))]) + ([[]] if l[-1] == [None] else [list(set(l[-1]))]) for l in users_with_lists] + # list_dict_users = [dict(zip(fields, r)) for r in users_with_lists] def get_users_list(self): # q = """select u.uid as username, adn.value as displayname, ade.value as email, json_agg(gg.displayname) as admin_groups,json_agg(g.displayname) as groups - # from oc_users as u + # from oc_users as u # left join oc_group_user as gu on gu.uid = u.uid # left join oc_groups as g on gu.gid = g.gid # left join oc_group_admin as ga on ga.uid = u.uid @@ -127,9 +153,19 @@ class Nextcloud(): left join oc_storages as s on s.id=CONCAT('home::',u.uid) left join oc_filecache as fc on fc.storage = numeric_id group by u.uid, adn.value, ade.value, pref.configvalue""" - (headers,users)=self.nextcloud_pg.select_with_headers(q) - users_with_lists = [list(l[:-2])+([[]] if l[-2] == [None] else [list(set(l[-2]))]) + ([[]] if l[-1] == [None] else [list(set(l[-1]))]) for l in users] - users_with_lists = [list(l[:-2])+([[]] if l[-2] == [None] else [list(set(l[-2]))]) + ([[]] if l[-1] == [None] else [list(set(l[-1]))]) for l in users_with_lists] + (headers, users) = self.nextcloud_pg.select_with_headers(q) + users_with_lists = [ + list(l[:-2]) + + ([[]] if l[-2] == [None] else [list(set(l[-2]))]) + + ([[]] if l[-1] == [None] else [list(set(l[-1]))]) + for l in users + ] + users_with_lists = [ + list(l[:-2]) + + ([[]] if l[-2] == [None] else [list(set(l[-2]))]) + + ([[]] if l[-1] == [None] else [list(set(l[-1]))]) + for l in users_with_lists + ] list_dict_users = [dict(zip(headers, r)) for r in users_with_lists] return list_dict_users @@ -143,33 +179,46 @@ class Nextcloud(): # raise ProviderOpError # except: # log.error(traceback.format_exc()) - # raise + # raise - def add_user(self,userid,userpassword,quota=False,group=False,email='',displayname=''): - data={'userid':userid,'password':userpassword,'quota':quota,'groups[]':group,'email':email,'displayname':displayname} - if not group: del data['groups[]'] - if not quota: del data['quota'] + def add_user( + self, userid, userpassword, quota=False, group=False, email="", displayname="" + ): + data = { + "userid": userid, + "password": userpassword, + "quota": quota, + "groups[]": group, + "email": email, + "displayname": displayname, + } + if not group: + del data["groups[]"] + if not quota: + del data["quota"] # if group: # data={'userid':userid,'password':userpassword,'quota':quota,'groups[]':group,'email':email,'displayname':displayname} # else: # data={'userid':userid,'password':userpassword,'quota':quota,'email':email,'displayname':displayname} url = self.apiurl + "users?format=json" headers = { - 'Content-Type': 'application/x-www-form-urlencoded', - 'OCS-APIRequest': 'true', + "Content-Type": "application/x-www-form-urlencoded", + "OCS-APIRequest": "true", } try: - result = json.loads(self._request('POST',url,data=data,headers=headers)) - if result['ocs']['meta']['statuscode'] == 100: return True - if result['ocs']['meta']['statuscode'] == 102: raise ProviderItemExists - if result['ocs']['meta']['statuscode'] == 104: + result = json.loads(self._request("POST", url, data=data, headers=headers)) + if result["ocs"]["meta"]["statuscode"] == 100: + return True + if result["ocs"]["meta"]["statuscode"] == 102: + raise ProviderItemExists + if result["ocs"]["meta"]["statuscode"] == 104: self.add_group(group) # raise ProviderGroupNotExists - log.error('Get Nextcloud provider user add error: '+str(result)) + log.error("Get Nextcloud provider user add error: " + str(result)) raise ProviderOpError except: log.error(traceback.format_exc()) - raise + raise # 100 - successful # 101 - invalid input data # 102 - username already exists @@ -184,89 +233,113 @@ class Nextcloud(): url = self.apiurl + "users/" + userid + "?format=json" headers = { - 'Content-Type': 'application/x-www-form-urlencoded', - 'OCS-APIRequest': 'true', + "Content-Type": "application/x-www-form-urlencoded", + "OCS-APIRequest": "true", } - for k,v in key_values.items(): - data={"key":k,"value":v} + for k, v in key_values.items(): + data = {"key": k, "value": v} try: - result = json.loads(self._request('PUT',url,data=data,headers=headers)) - if result['ocs']['meta']['statuscode'] == 100: return True - if result['ocs']['meta']['statuscode'] == 102: raise ProviderItemExists - if result['ocs']['meta']['statuscode'] == 104: raise ProviderGroupNotExists - log.error('Get Nextcloud provider user add error: '+str(result)) + result = json.loads( + self._request("PUT", url, data=data, headers=headers) + ) + if result["ocs"]["meta"]["statuscode"] == 100: + return True + if result["ocs"]["meta"]["statuscode"] == 102: + raise ProviderItemExists + if result["ocs"]["meta"]["statuscode"] == 104: + raise ProviderGroupNotExists + log.error("Get Nextcloud provider user add error: " + str(result)) raise ProviderOpError except: log.error(traceback.format_exc()) - raise + raise def add_user_to_group(self, userid, group_id): - data={'groupid':group_id} + data = {"groupid": group_id} url = self.apiurl + "users/" + userid + "/groups?format=json" headers = { - 'Content-Type': 'application/x-www-form-urlencoded', - 'OCS-APIRequest': 'true', + "Content-Type": "application/x-www-form-urlencoded", + "OCS-APIRequest": "true", } try: - result = json.loads(self._request('POST',url,data=data,headers=headers)) - if result['ocs']['meta']['statuscode'] == 100: return True - if result['ocs']['meta']['statuscode'] == 102: raise ProviderItemExists - if result['ocs']['meta']['statuscode'] == 104: raise ProviderGroupNotExists - log.error('Get Nextcloud provider user add error: '+str(result)) + result = json.loads(self._request("POST", url, data=data, headers=headers)) + if result["ocs"]["meta"]["statuscode"] == 100: + return True + if result["ocs"]["meta"]["statuscode"] == 102: + raise ProviderItemExists + if result["ocs"]["meta"]["statuscode"] == 104: + raise ProviderGroupNotExists + log.error("Get Nextcloud provider user add error: " + str(result)) raise ProviderOpError except: log.error(traceback.format_exc()) - raise + raise def remove_user_from_group(self, userid, group_id): - data={'groupid':group_id} + data = {"groupid": group_id} url = self.apiurl + "users/" + userid + "/groups?format=json" headers = { - 'Content-Type': 'application/x-www-form-urlencoded', - 'OCS-APIRequest': 'true', + "Content-Type": "application/x-www-form-urlencoded", + "OCS-APIRequest": "true", } try: - result = json.loads(self._request('DELETE',url,data=data,headers=headers)) - if result['ocs']['meta']['statuscode'] == 100: return True - if result['ocs']['meta']['statuscode'] == 102: raise ProviderItemExists - if result['ocs']['meta']['statuscode'] == 104: + result = json.loads( + self._request("DELETE", url, data=data, headers=headers) + ) + if result["ocs"]["meta"]["statuscode"] == 100: + return True + if result["ocs"]["meta"]["statuscode"] == 102: + raise ProviderItemExists + if result["ocs"]["meta"]["statuscode"] == 104: self.add_group(group) # raise ProviderGroupNotExists - log.error('Get Nextcloud provider user add error: '+str(result)) + log.error("Get Nextcloud provider user add error: " + str(result)) raise ProviderOpError except: log.error(traceback.format_exc()) - raise + raise - def add_user_with_groups(self,userid,userpassword,quota=False,groups=[],email='',displayname=''): - data={'userid':userid,'password':userpassword,'quota':quota,'groups[]':groups,'email':email,'displayname':displayname} + def add_user_with_groups( + self, userid, userpassword, quota=False, groups=[], email="", displayname="" + ): + data = { + "userid": userid, + "password": userpassword, + "quota": quota, + "groups[]": groups, + "email": email, + "displayname": displayname, + } # if not group: del data['groups[]'] - if not quota: del data['quota'] + if not quota: + del data["quota"] # if group: # data={'userid':userid,'password':userpassword,'quota':quota,'groups[]':group,'email':email,'displayname':displayname} # else: # data={'userid':userid,'password':userpassword,'quota':quota,'email':email,'displayname':displayname} url = self.apiurl + "users?format=json" headers = { - 'Content-Type': 'application/x-www-form-urlencoded', - 'OCS-APIRequest': 'true', + "Content-Type": "application/x-www-form-urlencoded", + "OCS-APIRequest": "true", } try: - result = json.loads(self._request('POST',url,data=data,headers=headers)) - if result['ocs']['meta']['statuscode'] == 100: return True - if result['ocs']['meta']['statuscode'] == 102: raise ProviderItemExists - if result['ocs']['meta']['statuscode'] == 104: + result = json.loads(self._request("POST", url, data=data, headers=headers)) + if result["ocs"]["meta"]["statuscode"] == 100: + return True + if result["ocs"]["meta"]["statuscode"] == 102: + raise ProviderItemExists + if result["ocs"]["meta"]["statuscode"] == 104: # self.add_group(group) None # raise ProviderGroupNotExists - log.error('Get Nextcloud provider user add error: '+str(result)) + log.error("Get Nextcloud provider user add error: " + str(result)) raise ProviderOpError except: log.error(traceback.format_exc()) - raise + raise # 100 - successful # 101 - invalid input data # 102 - username already exists @@ -276,149 +349,172 @@ class Nextcloud(): # 106 - no group specified (required for subadmins) # 107 - all errors that contain a hint - for example “Password is among the 1,000,000 most common ones. Please make it unique.” (this code was added in 12.0.6 & 13.0.1) - def delete_user(self,userid): - url = self.apiurl + "users/"+userid+"?format=json" + def delete_user(self, userid): + url = self.apiurl + "users/" + userid + "?format=json" try: - result = json.loads(self._request('DELETE',url)) - if result['ocs']['meta']['statuscode'] == 100: return True - if result['ocs']['meta']['statuscode'] == 101: raise ProviderUserNotExists + result = json.loads(self._request("DELETE", url)) + if result["ocs"]["meta"]["statuscode"] == 100: + return True + if result["ocs"]["meta"]["statuscode"] == 101: + raise ProviderUserNotExists log.error(traceback.format_exc()) raise ProviderOpError except: log.error(traceback.format_exc()) - raise + raise # 100 - successful # 101 - failure - def enable_user(self,userid): + def enable_user(self, userid): None - def disable_user(self,userid): + def disable_user(self, userid): None - def exists_user_folder(self,userid,userpassword,folder='IsardVDI'): - auth=(userid,userpassword) - url = self.davurl + userid +"/" + folder+"?format=json" + def exists_user_folder(self, userid, userpassword, folder="IsardVDI"): + auth = (userid, userpassword) + url = self.davurl + userid + "/" + folder + "?format=json" headers = { - 'Depth': '0', - 'Content-Type': 'application/x-www-form-urlencoded', - 'OCS-APIRequest': 'true', + "Depth": "0", + "Content-Type": "application/x-www-form-urlencoded", + "OCS-APIRequest": "true", } try: - result = self._request('PROPFIND',url,auth=auth,headers=headers) - if 'HTTP/1.1 200 OK' in result: return True + result = self._request("PROPFIND", url, auth=auth, headers=headers) + if "HTTP/1.1 200 OK" in result: + return True return False except: log.error(traceback.format_exc()) - raise + raise - def add_user_folder(self,userid,userpassword,folder='IsardVDI'): - auth=(userid,userpassword) - url = self.davurl + userid +"/" + folder+"?format=json" + def add_user_folder(self, userid, userpassword, folder="IsardVDI"): + auth = (userid, userpassword) + url = self.davurl + userid + "/" + folder + "?format=json" headers = { - 'Content-Type': 'application/x-www-form-urlencoded', - 'OCS-APIRequest': 'true', + "Content-Type": "application/x-www-form-urlencoded", + "OCS-APIRequest": "true", } try: - result = self._request('MKCOL',url,auth=auth,headers=headers) - if result=='': return True - if 'The resource you tried to create already exists' in result: raise ProviderItemExists - log.error(result.split('message>')[1].split('<')[0]) + result = self._request("MKCOL", url, auth=auth, headers=headers) + if result == "": + return True + if ( + "The resource you tried to create already exists" + in result + ): + raise ProviderItemExists + log.error(result.split("message>")[1].split("<")[0]) raise ProviderOpError except: log.error(traceback.format_exc()) - raise + raise - def exists_user_share_folder(self,userid,userpassword,folder='IsardVDI'): - auth=(userid,userpassword) + def exists_user_share_folder(self, userid, userpassword, folder="IsardVDI"): + auth = (userid, userpassword) url = self.shareurl + "shares?format=json" headers = { - 'Content-Type': 'application/x-www-form-urlencoded', - 'OCS-APIRequest': 'true', + "Content-Type": "application/x-www-form-urlencoded", + "OCS-APIRequest": "true", } try: - result = json.loads(self._request('GET', url, auth=auth, headers=headers)) - if result['ocs']['meta']['statuscode']==200: - share=[s for s in result['ocs']['data'] if s['path'] == '/'+folder] + result = json.loads(self._request("GET", url, auth=auth, headers=headers)) + if result["ocs"]["meta"]["statuscode"] == 200: + share = [s for s in result["ocs"]["data"] if s["path"] == "/" + folder] if len(share) >= 1: # Should we delete all but the first (0) one? - return {'token': share[0]['token'], - 'url': share[0]['url']} + return {"token": share[0]["token"], "url": share[0]["url"]} raise ProviderItemNotExists raise ProviderOpError except: log.error(traceback.format_exc()) - raise + raise - def add_user_share_folder(self,userid,userpassword,folder='IsardVDI'): - auth=(userid,userpassword) - data={'path':'/'+folder,'shareType':3} + def add_user_share_folder(self, userid, userpassword, folder="IsardVDI"): + auth = (userid, userpassword) + data = {"path": "/" + folder, "shareType": 3} url = self.shareurl + "shares?format=json" headers = { - 'Content-Type': 'application/x-www-form-urlencoded', - 'OCS-APIRequest': 'true', + "Content-Type": "application/x-www-form-urlencoded", + "OCS-APIRequest": "true", } try: - result = json.loads(self._request('POST',url, data=data, auth=auth, headers=headers)) - if result['ocs']['meta']['statuscode'] == 100 or result['ocs']['meta']['statuscode'] == 200: - return {'token': result['ocs']['data']['token'], - 'url': result['ocs']['data']['url']} - log.error('Add user share folder error: '+result['ocs']['meta']['message']) + result = json.loads( + self._request("POST", url, data=data, auth=auth, headers=headers) + ) + if ( + result["ocs"]["meta"]["statuscode"] == 100 + or result["ocs"]["meta"]["statuscode"] == 200 + ): + return { + "token": result["ocs"]["data"]["token"], + "url": result["ocs"]["data"]["url"], + } + log.error( + "Add user share folder error: " + result["ocs"]["meta"]["message"] + ) raise ProviderFolderNotExists except: log.error(traceback.format_exc()) - raise + raise - def get_group(self,userid): + def get_group(self, userid): None def get_groups_list(self): url = self.apiurl + "groups?format=json" try: - result = json.loads(self._request('GET',url)) - if result['ocs']['meta']['statuscode'] == 100: return [g for g in result['ocs']['data']['groups']] + result = json.loads(self._request("GET", url)) + if result["ocs"]["meta"]["statuscode"] == 100: + return [g for g in result["ocs"]["data"]["groups"]] raise ProviderOpError except: log.error(traceback.format_exc()) - raise + raise - def add_group(self,groupid): - data={'groupid':groupid} + def add_group(self, groupid): + data = {"groupid": groupid} url = self.apiurl + "groups?format=json" headers = { - 'Content-Type': 'application/x-www-form-urlencoded', - 'OCS-APIRequest': 'true', + "Content-Type": "application/x-www-form-urlencoded", + "OCS-APIRequest": "true", } try: - result = json.loads(self._request('POST',url, data=data, auth=self.auth, headers=headers)) - if result['ocs']['meta']['statuscode'] == 100: return True - if result['ocs']['meta']['statuscode'] == 102: raise ProviderItemExists + result = json.loads( + self._request("POST", url, data=data, auth=self.auth, headers=headers) + ) + if result["ocs"]["meta"]["statuscode"] == 100: + return True + if result["ocs"]["meta"]["statuscode"] == 102: + raise ProviderItemExists raise ProviderOpError except: log.error(traceback.format_exc()) - raise + raise # 100 - successful # 101 - invalid input data # 102 - group already exists # 103 - failed to add the group - def delete_group(self,groupid): - group = urllib.parse.quote(groupid, safe='') - url = self.apiurl + "groups/"+group+"?format=json" + def delete_group(self, groupid): + group = urllib.parse.quote(groupid, safe="") + url = self.apiurl + "groups/" + group + "?format=json" headers = { - 'Content-Type': 'application/x-www-form-urlencoded', - 'OCS-APIRequest': 'true', + "Content-Type": "application/x-www-form-urlencoded", + "OCS-APIRequest": "true", } try: - result = json.loads(self._request('DELETE',url, auth=self.auth, headers=headers)) - if result['ocs']['meta']['statuscode'] == 100: return True + result = json.loads( + self._request("DELETE", url, auth=self.auth, headers=headers) + ) + if result["ocs"]["meta"]["statuscode"] == 100: + return True log.error(traceback.format_exc()) raise ProviderOpError except: log.error(traceback.format_exc()) - raise + raise # 100 - successful # 101 - invalid input data # 102 - group already exists # 103 - failed to add the group - diff --git a/admin/src/admin/lib/nextcloud_exc.py b/admin/src/admin/lib/nextcloud_exc.py index 827e93f..25329e0 100644 --- a/admin/src/admin/lib/nextcloud_exc.py +++ b/admin/src/admin/lib/nextcloud_exc.py @@ -3,30 +3,38 @@ class ProviderUnauthorized(Exception): pass + class ProviderConnError(Exception): pass + class ProviderSslError(Exception): pass + class ProviderConnTimeout(Exception): pass + class ProviderError(Exception): pass + class ProviderItemExists(Exception): pass + class ProviderItemNotExists(Exception): pass + class ProviderGroupNotExists(Exception): pass + class ProviderFolderNotExists(Exception): pass - + class ProviderOpError(Exception): pass diff --git a/admin/src/admin/lib/postgres.py b/admin/src/admin/lib/postgres.py index 37b48cc..8399327 100644 --- a/admin/src/admin/lib/postgres.py +++ b/admin/src/admin/lib/postgres.py @@ -1,50 +1,48 @@ #!/usr/bin/env python # coding=utf-8 +import json +import logging as log import time -from admin import app +import traceback from datetime import datetime, timedelta -import logging as log -import traceback -import yaml, json - import psycopg2 +import yaml -class Postgres(): +from admin import app - def __init__(self,host,database,user,password): + +class Postgres: + def __init__(self, host, database, user, password): self.conn = psycopg2.connect( - host=host, - database=database, - user=user, - password=password) - + host=host, database=database, user=user, password=password + ) # def __del__(self): # self.cur.close() # self.conn.close() - def select(self,sql): + def select(self, sql): self.cur = self.conn.cursor() self.cur.execute(sql) - data=self.cur.fetchall() + data = self.cur.fetchall() self.cur.close() return data - def update(self,sql): + def update(self, sql): self.cur = self.conn.cursor() self.cur.execute(sql) self.conn.commit() self.cur.close() # return self.cur.fetchall() - def select_with_headers(self,sql): + def select_with_headers(self, sql): self.cur = self.conn.cursor() self.cur.execute(sql) - data=self.cur.fetchall() + data = self.cur.fetchall() fields = [a.name for a in self.cur.description] self.cur.close() - return (fields,data) + return (fields, data) # def update_moodle_saml_plugin(self): # plugin[('idpmetadata', 'NrtA5ynG0htowP3SXw7dBJRIAMxn-1PwuuXwOwNhlRwMIICmzCCAYMCBgF5jb0RCTANBgkqhkiG9w0BAQsFADARMQ8wDQYDVQQDDAZtYXN0ZXIwHhcNMjEwNTIxMDcwMjI4WhcNMzEwNTIxMDcwNDA4WjARMQ8wDQYDVQQDDAZtYXN0ZXIwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQCI8xh/C0+frz3kgWiUbziTDls71R2YiXLSVE+bw7gbEgZUGCLhoEI679azMtIxmnzM/snIX+yTb12+XoYkgbiLTMPQfnH+Kiab6g3HL3KPfhqS+yWkFxOoCp6Ibmp7yPlVWuHH+MBfO8OBr/r8Ao7heFbuzjiLd1KG67rcoaxfDgMuBoEomg1bgEjFgHaQIrSC6OZzH0h987/arqufZXeXlfyiqScMPUi+u5IpDWSwz06UKP0k8mxzNSlpZ93CKOUSsV0SMLxqg7FQ3SGiOk577bGW9o9BDTkkmSo3Up6smc0LzwvvUwuNd0B1irGkWZFQN9OXJnJYf1InEebIMtmPAgMBAAEwDQYJKoZIhvcNAQELBQADggEBADM34+qEGeBQ22luphVTuVJtGxcbxLx7DfsT0QfJD/OuxTTbNAa1VRyarb5juIAkqdj4y2quZna9ZXLecVo4RkwpzPoKoAkYA8b+kHnWqEwJi9iPrDvKb+GR0bBkLPN49YxIZ8IdKX/PRa3yuLHe+loiNsCaS/2ZK2KO46COsqU4QX1iVhF9kWphNLybjNAX45B6cJLsa1g0vXLdm3kv3SB4I2fErFVaOoDtFIjttoYlXdpUiThkPXBfr7N67P3dZHaS4tjJh+IZ8I6TINpcsH8dBkUhzYEIPHCePwSiC1w6WDBLNDuKt1mj1CZrLq+1x+Yhrs+QNRheEKGi89HZ8N0=urn:oasis:names:tc:SAML:2.0:nameid-format:persistenturn:oasis:names:tc:SAML:2.0:nameid-format:transienturn:oasis:names:tc:SAML:1.1:nameid-format:unspecifiedurn:oasis:names:tc:SAML:1.1:nameid-format:emailAddress')] @@ -52,4 +50,4 @@ class Postgres(): # cursor.execute(pg_update, (title, bookid)) # connection.commit() # count = cursor.rowcount - # print(count, "Successfully Updated!") \ No newline at end of file + # print(count, "Successfully Updated!") diff --git a/admin/src/admin/lib/postup.py b/admin/src/admin/lib/postup.py index 1d616d5..f8b048b 100644 --- a/admin/src/admin/lib/postup.py +++ b/admin/src/admin/lib/postup.py @@ -1,74 +1,111 @@ #!/usr/bin/env python # coding=utf-8 -import time, os -from admin import app -from datetime import datetime, timedelta - +import json import logging as log -import traceback -import yaml, json +import os +import random -import psycopg2 - -from .postgres import Postgres # from .keycloak import Keycloak # from .moodle import Moodle -import string, random +import string +import time +import traceback +from datetime import datetime, timedelta -class Postup(): +import psycopg2 +import yaml + +from admin import app + +from .postgres import Postgres + + +class Postup: def __init__(self): - ready=False + ready = False while not ready: try: - self.pg=Postgres('isard-apps-postgresql','moodle',app.config['MOODLE_POSTGRES_USER'],app.config['MOODLE_POSTGRES_PASSWORD']) - ready=True + self.pg = Postgres( + "isard-apps-postgresql", + "moodle", + app.config["MOODLE_POSTGRES_USER"], + app.config["MOODLE_POSTGRES_PASSWORD"], + ) + ready = True except: - log.warning('Could not connect to moodle database. Retrying...') + log.warning("Could not connect to moodle database. Retrying...") time.sleep(2) - log.info('Connected to moodle database.') + log.info("Connected to moodle database.") - ready=False + ready = False while not ready: try: - with open(os.path.join(app.root_path, "../moodledata/saml2/moodle."+app.config['DOMAIN']+".crt"),"r") as crt: - app.config.setdefault('SP_CRT', crt.read()) - ready=True + with open( + os.path.join( + app.root_path, + "../moodledata/saml2/moodle." + app.config["DOMAIN"] + ".crt", + ), + "r", + ) as crt: + app.config.setdefault("SP_CRT", crt.read()) + ready = True except IOError: - log.warning('Could not get moodle SAML2 crt certificate. Retrying...') + log.warning("Could not get moodle SAML2 crt certificate. Retrying...") time.sleep(2) except: log.error(traceback.format_exc()) - log.info('Got moodle srt certificate.') + log.info("Got moodle srt certificate.") - ready=False + ready = False while not ready: try: - with open(os.path.join(app.root_path, "../moodledata/saml2/moodle."+app.config['DOMAIN']+".pem"),"r") as pem: - app.config.setdefault('SP_PEM', pem.read()) - ready=True + with open( + os.path.join( + app.root_path, + "../moodledata/saml2/moodle." + app.config["DOMAIN"] + ".pem", + ), + "r", + ) as pem: + app.config.setdefault("SP_PEM", pem.read()) + ready = True except IOError: - log.warning('Could not get moodle SAML2 pem certificate. Retrying...') + log.warning("Could not get moodle SAML2 pem certificate. Retrying...") time.sleep(2) - log.info('Got moodle pem certificate.') + log.info("Got moodle pem certificate.") self.select_and_configure_theme() self.configure_tipnc() self.add_moodle_ws_token() - def select_and_configure_theme(self,theme='cbe'): + def select_and_configure_theme(self, theme="cbe"): try: - self.pg.update("""UPDATE "mdl_config" SET value = '%s' WHERE "name" = 'theme';""" % (theme)) + self.pg.update( + """UPDATE "mdl_config" SET value = '%s' WHERE "name" = 'theme';""" + % (theme) + ) except: log.error(traceback.format_exc()) exit(1) None try: - self.pg.update("""UPDATE "mdl_config_plugins" SET value = '%s' WHERE "plugin" = 'theme_cbe' AND "name" = 'host';""" % (os.environ['DOMAIN'])) - self.pg.update("""UPDATE "mdl_config_plugins" SET value = '%s' WHERE "plugin" = 'theme_cbe' AND "name" = 'logourl';""" % ("https://api."+os.environ['DOMAIN']+"/img/logo.png")) - self.pg.update("""UPDATE "mdl_config_plugins" SET value = '1' WHERE "plugin" = 'theme_cbe' AND "name" = 'header_api';""") - self.pg.update("""UPDATE "mdl_config_plugins" SET value = '1' WHERE "plugin" = 'theme_cbe' AND "name" = 'vclasses_direct';""") - self.pg.update("""UPDATE "mdl_config_plugins" SET value = '1' WHERE "plugin" = 'theme_cbe' AND "name" = 'uniquenamecourse';""") + self.pg.update( + """UPDATE "mdl_config_plugins" SET value = '%s' WHERE "plugin" = 'theme_cbe' AND "name" = 'host';""" + % (os.environ["DOMAIN"]) + ) + self.pg.update( + """UPDATE "mdl_config_plugins" SET value = '%s' WHERE "plugin" = 'theme_cbe' AND "name" = 'logourl';""" + % ("https://api." + os.environ["DOMAIN"] + "/img/logo.png") + ) + self.pg.update( + """UPDATE "mdl_config_plugins" SET value = '1' WHERE "plugin" = 'theme_cbe' AND "name" = 'header_api';""" + ) + self.pg.update( + """UPDATE "mdl_config_plugins" SET value = '1' WHERE "plugin" = 'theme_cbe' AND "name" = 'vclasses_direct';""" + ) + self.pg.update( + """UPDATE "mdl_config_plugins" SET value = '1' WHERE "plugin" = 'theme_cbe' AND "name" = 'uniquenamecourse';""" + ) except: log.error(traceback.format_exc()) exit(1) @@ -76,12 +113,27 @@ class Postup(): def configure_tipnc(self): try: - self.pg.update("""UPDATE "mdl_config_plugins" SET value = '%s' WHERE "plugin" = 'assignsubmission_tipnc' AND "name" = 'host';""" % ("https://nextcloud."+os.environ['DOMAIN']+"/")) - self.pg.update("""UPDATE "mdl_config_plugins" SET value = '%s' WHERE "plugin" = 'assignsubmission_tipnc' AND "name" = 'password';""" % (os.environ['NEXTCLOUD_ADMIN_PASSWORD'])) - self.pg.update("""UPDATE "mdl_config_plugins" SET value = 'template.docx' WHERE "plugin" = 'assignsubmission_tipnc' AND "name" = 'template';""") - self.pg.update("""UPDATE "mdl_config_plugins" SET value = '/apps/onlyoffice/' WHERE "plugin" = 'assignsubmission_tipnc' AND "name" = 'location';""") - self.pg.update("""UPDATE "mdl_config_plugins" SET value = '%s' WHERE "plugin" = 'assignsubmission_tipnc' AND "name" = 'user';""" % (os.environ['NEXTCLOUD_ADMIN_USER'])) - self.pg.update("""UPDATE "mdl_config_plugins" SET value = 'tasks' WHERE "plugin" = 'assignsubmission_tipnc' AND "name" = 'folder';""") + self.pg.update( + """UPDATE "mdl_config_plugins" SET value = '%s' WHERE "plugin" = 'assignsubmission_tipnc' AND "name" = 'host';""" + % ("https://nextcloud." + os.environ["DOMAIN"] + "/") + ) + self.pg.update( + """UPDATE "mdl_config_plugins" SET value = '%s' WHERE "plugin" = 'assignsubmission_tipnc' AND "name" = 'password';""" + % (os.environ["NEXTCLOUD_ADMIN_PASSWORD"]) + ) + self.pg.update( + """UPDATE "mdl_config_plugins" SET value = 'template.docx' WHERE "plugin" = 'assignsubmission_tipnc' AND "name" = 'template';""" + ) + self.pg.update( + """UPDATE "mdl_config_plugins" SET value = '/apps/onlyoffice/' WHERE "plugin" = 'assignsubmission_tipnc' AND "name" = 'location';""" + ) + self.pg.update( + """UPDATE "mdl_config_plugins" SET value = '%s' WHERE "plugin" = 'assignsubmission_tipnc' AND "name" = 'user';""" + % (os.environ["NEXTCLOUD_ADMIN_USER"]) + ) + self.pg.update( + """UPDATE "mdl_config_plugins" SET value = 'tasks' WHERE "plugin" = 'assignsubmission_tipnc' AND "name" = 'folder';""" + ) except: log.error(traceback.format_exc()) exit(1) @@ -89,18 +141,23 @@ class Postup(): def add_moodle_ws_token(self): try: - token=self.pg.select("""SELECT * FROM "mdl_external_tokens" WHERE "externalserviceid" = 3""")[0][1] - app.config.setdefault('MOODLE_WS_TOKEN',token) + token = self.pg.select( + """SELECT * FROM "mdl_external_tokens" WHERE "externalserviceid" = 3""" + )[0][1] + app.config.setdefault("MOODLE_WS_TOKEN", token) return except: # log.error(traceback.format_exc()) None try: - self.pg.update("""INSERT INTO "mdl_external_services" ("name", "enabled", "requiredcapability", "restrictedusers", "component", "timecreated", "timemodified", "shortname", "downloadfiles", "uploadfiles") VALUES - ('dd admin', 1, '', 1, NULL, 1621719763, 1621719850, 'dd_admin', 0, 0);""") + self.pg.update( + """INSERT INTO "mdl_external_services" ("name", "enabled", "requiredcapability", "restrictedusers", "component", "timecreated", "timemodified", "shortname", "downloadfiles", "uploadfiles") VALUES + ('dd admin', 1, '', 1, NULL, 1621719763, 1621719850, 'dd_admin', 0, 0);""" + ) - self.pg.update("""INSERT INTO "mdl_external_services_functions" ("externalserviceid", "functionname") VALUES + self.pg.update( + """INSERT INTO "mdl_external_services_functions" ("externalserviceid", "functionname") VALUES (3, 'core_course_update_courses'), (3, 'core_user_get_users'), (3, 'core_user_get_users_by_field'), @@ -116,17 +173,37 @@ class Postup(): (3, 'core_cohort_search_cohorts'), (3, 'core_cohort_update_cohorts'), (3, 'core_role_assign_roles'), - (3, 'core_cohort_get_cohorts');""") + (3, 'core_cohort_get_cohorts');""" + ) - self.pg.update("""INSERT INTO "mdl_external_services_users" ("externalserviceid", "userid", "iprestriction", "validuntil", "timecreated") VALUES - (3, 2, NULL, NULL, 1621719871);""") + self.pg.update( + """INSERT INTO "mdl_external_services_users" ("externalserviceid", "userid", "iprestriction", "validuntil", "timecreated") VALUES + (3, 2, NULL, NULL, 1621719871);""" + ) - b32=''.join(random.choices(string.ascii_uppercase + string.ascii_uppercase + string.ascii_lowercase, k = 32)) - b64=''.join(random.choices(string.ascii_uppercase + string.ascii_uppercase + string.ascii_lowercase, k = 64)) - self.pg.update("""INSERT INTO "mdl_external_tokens" ("token", "privatetoken", "tokentype", "userid", "externalserviceid", "sid", "contextid", "creatorid", "iprestriction", "validuntil", "timecreated", "lastaccess") VALUES - ('%s', '%s', 0, 2, 3, NULL, 1, 2, NULL, 0, 1621831206, NULL);""" % (b32,b64)) - - app.config.setdefault('MOODLE_WS_TOKEN',b32) + b32 = "".join( + random.choices( + string.ascii_uppercase + + string.ascii_uppercase + + string.ascii_lowercase, + k=32, + ) + ) + b64 = "".join( + random.choices( + string.ascii_uppercase + + string.ascii_uppercase + + string.ascii_lowercase, + k=64, + ) + ) + self.pg.update( + """INSERT INTO "mdl_external_tokens" ("token", "privatetoken", "tokentype", "userid", "externalserviceid", "sid", "contextid", "creatorid", "iprestriction", "validuntil", "timecreated", "lastaccess") VALUES + ('%s', '%s', 0, 2, 3, NULL, 1, 2, NULL, 0, 1621831206, NULL);""" + % (b32, b64) + ) + + app.config.setdefault("MOODLE_WS_TOKEN", b32) except: log.error(traceback.format_exc()) exit(1) diff --git a/admin/src/admin/views/ApiViews.py b/admin/src/admin/views/ApiViews.py index 4b3c7f6..a7fe37b 100644 --- a/admin/src/admin/views/ApiViews.py +++ b/admin/src/admin/views/ApiViews.py @@ -1,315 +1,488 @@ #!flask/bin/python # coding=utf-8 -from admin import app -import logging as log -import traceback - -from uuid import uuid4 -import time,json -import sys,os,re -from flask import render_template, Response, request, redirect, url_for, jsonify import concurrent.futures -from flask_login import current_user, login_required -from .decorators import is_admin - -from ..lib.helpers import system_group +import json +import logging as log +import os +import re +import sys # import Queue import threading -threads={'external':None} +import time +import traceback +from uuid import uuid4 + +from flask import Response, jsonify, redirect, render_template, request, url_for +from flask_login import current_user, login_required + +from admin import app + +from ..lib.helpers import system_group +from .decorators import is_admin + +threads = {"external": None} # q = Queue.Queue() from keycloak.exceptions import KeycloakGetError from ..lib.exceptions import UserExists, UserNotFound -@app.route('/api/resync') + +@app.route("/api/resync") @login_required def resync(): - return json.dumps(app.admin.resync_data()), 200, {'Content-Type': 'application/json'} + return ( + json.dumps(app.admin.resync_data()), + 200, + {"Content-Type": "application/json"}, + ) -@app.route('/api/users', methods=['GET','PUT']) -@app.route('/api/users/', methods=['POST', 'PUT', 'GET', 'DELETE']) + +@app.route("/api/users", methods=["GET", "PUT"]) +@app.route("/api/users/", methods=["POST", "PUT", "GET", "DELETE"]) @login_required def users(provider=False): - if request.method == 'DELETE': - if current_user.role != 'admin': return json.dumps({}), 301, {'Content-Type': 'application/json'} - if provider == 'keycloak': - return json.dumps(app.admin.delete_keycloak_users()), 200, {'Content-Type': 'application/json'} - if provider == 'nextcloud': - return json.dumps(app.admin.delete_nextcloud_users()), 200, {'Content-Type': 'application/json'} - if provider == 'moodle': - return json.dumps(app.admin.delete_moodle_users()), 200, {'Content-Type': 'application/json'} - if request.method == 'POST': - if current_user.role != 'admin': return json.dumps({}), 301, {'Content-Type': 'application/json'} - if provider == 'moodle': - return json.dumps(app.admin.sync_to_moodle()), 200, {'Content-Type': 'application/json'} - if provider == 'nextcloud': - return json.dumps(app.admin.sync_to_nextcloud()), 200, {'Content-Type': 'application/json'} - if request.method == 'PUT' and not provider: - if current_user.role != 'admin': return json.dumps({}), 301, {'Content-Type': 'application/json'} + if request.method == "DELETE": + if current_user.role != "admin": + return json.dumps({}), 301, {"Content-Type": "application/json"} + if provider == "keycloak": + return ( + json.dumps(app.admin.delete_keycloak_users()), + 200, + {"Content-Type": "application/json"}, + ) + if provider == "nextcloud": + return ( + json.dumps(app.admin.delete_nextcloud_users()), + 200, + {"Content-Type": "application/json"}, + ) + if provider == "moodle": + return ( + json.dumps(app.admin.delete_moodle_users()), + 200, + {"Content-Type": "application/json"}, + ) + if request.method == "POST": + if current_user.role != "admin": + return json.dumps({}), 301, {"Content-Type": "application/json"} + if provider == "moodle": + return ( + json.dumps(app.admin.sync_to_moodle()), + 200, + {"Content-Type": "application/json"}, + ) + if provider == "nextcloud": + return ( + json.dumps(app.admin.sync_to_nextcloud()), + 200, + {"Content-Type": "application/json"}, + ) + if request.method == "PUT" and not provider: + if current_user.role != "admin": + return json.dumps({}), 301, {"Content-Type": "application/json"} - if 'external' in threads.keys(): - if threads['external'] is not None and threads['external'].is_alive(): - return json.dumps({'msg':'Precondition failed: already working with users'}), 412, {'Content-Type': 'application/json'} + if "external" in threads.keys(): + if threads["external"] is not None and threads["external"].is_alive(): + return ( + json.dumps( + {"msg": "Precondition failed: already working with users"} + ), + 412, + {"Content-Type": "application/json"}, + ) else: - threads['external']=None + threads["external"] = None try: - threads['external'] = threading.Thread(target=app.admin.update_users_from_keycloak, args=()) - threads['external'].start() - return json.dumps({}), 200, {'Content-Type': 'application/json'} + threads["external"] = threading.Thread( + target=app.admin.update_users_from_keycloak, args=() + ) + threads["external"].start() + return json.dumps({}), 200, {"Content-Type": "application/json"} except: log.error(traceback.format_exc()) - return json.dumps({'msg':'Add user error.'}), 500, {'Content-Type': 'application/json'} + return ( + json.dumps({"msg": "Add user error."}), + 500, + {"Content-Type": "application/json"}, + ) # return json.dumps(app.admin.update_users_from_keycloak()), 200, {'Content-Type': 'application/json'} users = app.admin.get_mix_users() - if current_user.role != 'admin': + if current_user.role != "admin": for user in users: - user['keycloak_groups'] = [g for g in user['keycloak_groups'] if not system_group(g) ] - return json.dumps(users), 200, {'Content-Type': 'application/json'} - -@app.route('/api/users_bulk/', methods=['PUT']) + user["keycloak_groups"] = [ + g for g in user["keycloak_groups"] if not system_group(g) + ] + return json.dumps(users), 200, {"Content-Type": "application/json"} + + +@app.route("/api/users_bulk/", methods=["PUT"]) @login_required def users_bulk(action): - data=request.get_json(force=True) - if request.method == 'PUT': - if action == 'enable': - if 'external' in threads.keys(): - if threads['external'] is not None and threads['external'].is_alive(): - return json.dumps({'msg':'Precondition failed: already operating users'}), 412, {'Content-Type': 'application/json'} + data = request.get_json(force=True) + if request.method == "PUT": + if action == "enable": + if "external" in threads.keys(): + if threads["external"] is not None and threads["external"].is_alive(): + return ( + json.dumps( + {"msg": "Precondition failed: already operating users"} + ), + 412, + {"Content-Type": "application/json"}, + ) else: - threads['external']=None + threads["external"] = None try: - threads['external'] = threading.Thread(target=app.admin.enable_users, args=(data,)) - threads['external'].start() - return json.dumps({}), 200, {'Content-Type': 'application/json'} + threads["external"] = threading.Thread( + target=app.admin.enable_users, args=(data,) + ) + threads["external"].start() + return json.dumps({}), 200, {"Content-Type": "application/json"} except: log.error(traceback.format_exc()) - return json.dumps({'msg':'Enable users error.'}), 500, {'Content-Type': 'application/json'} - if action == 'disable': - if 'external' in threads.keys(): - if threads['external'] is not None and threads['external'].is_alive(): - return json.dumps({'msg':'Precondition failed: already operating users'}), 412, {'Content-Type': 'application/json'} + return ( + json.dumps({"msg": "Enable users error."}), + 500, + {"Content-Type": "application/json"}, + ) + if action == "disable": + if "external" in threads.keys(): + if threads["external"] is not None and threads["external"].is_alive(): + return ( + json.dumps( + {"msg": "Precondition failed: already operating users"} + ), + 412, + {"Content-Type": "application/json"}, + ) else: - threads['external']=None + threads["external"] = None try: - threads['external'] = threading.Thread(target=app.admin.disable_users, args=(data,)) - threads['external'].start() - return json.dumps({}), 200, {'Content-Type': 'application/json'} + threads["external"] = threading.Thread( + target=app.admin.disable_users, args=(data,) + ) + threads["external"].start() + return json.dumps({}), 200, {"Content-Type": "application/json"} except: log.error(traceback.format_exc()) - return json.dumps({'msg':'Disabling users error.'}), 500, {'Content-Type': 'application/json'} - if action == 'delete': - if 'external' in threads.keys(): - if threads['external'] is not None and threads['external'].is_alive(): - return json.dumps({'msg':'Precondition failed: already operating users'}), 412, {'Content-Type': 'application/json'} + return ( + json.dumps({"msg": "Disabling users error."}), + 500, + {"Content-Type": "application/json"}, + ) + if action == "delete": + if "external" in threads.keys(): + if threads["external"] is not None and threads["external"].is_alive(): + return ( + json.dumps( + {"msg": "Precondition failed: already operating users"} + ), + 412, + {"Content-Type": "application/json"}, + ) else: - threads['external']=None + threads["external"] = None try: - threads['external'] = threading.Thread(target=app.admin.delete_users, args=(data,)) - threads['external'].start() - return json.dumps({}), 200, {'Content-Type': 'application/json'} + threads["external"] = threading.Thread( + target=app.admin.delete_users, args=(data,) + ) + threads["external"].start() + return json.dumps({}), 200, {"Content-Type": "application/json"} except: log.error(traceback.format_exc()) - return json.dumps({'msg':'Deleting users error.'}), 500, {'Content-Type': 'application/json'} - return json.dumps({}), 405, {'Content-Type': 'application/json'} + return ( + json.dumps({"msg": "Deleting users error."}), + 500, + {"Content-Type": "application/json"}, + ) + return json.dumps({}), 405, {"Content-Type": "application/json"} + # Update pwd -@app.route('/api/user_password', methods=['GET']) -@app.route('/api/user_password/', methods=['PUT']) +@app.route("/api/user_password", methods=["GET"]) +@app.route("/api/user_password/", methods=["PUT"]) @login_required def user_password(userid=False): - if request.method == 'GET': - return json.dumps(app.admin.get_dice_pwd()), 200, {'Content-Type': 'application/json'} - if request.method == 'PUT': - data=request.get_json(force=True) - password=data['password'] - temporary=data.get('temporary',True) + if request.method == "GET": + return ( + json.dumps(app.admin.get_dice_pwd()), + 200, + {"Content-Type": "application/json"}, + ) + if request.method == "PUT": + data = request.get_json(force=True) + password = data["password"] + temporary = data.get("temporary", True) try: - res = app.admin.user_update_password(userid,password,temporary) - return json.dumps({}), 200, {'Content-Type': 'application/json'} + res = app.admin.user_update_password(userid, password, temporary) + return json.dumps({}), 200, {"Content-Type": "application/json"} except KeycloakGetError as e: log.error(e.error_message.decode("utf-8")) - return json.dumps({'msg':'Update password error.'}), 500, {'Content-Type': 'application/json'} + return ( + json.dumps({"msg": "Update password error."}), + 500, + {"Content-Type": "application/json"}, + ) + + return json.dumps({}), 405, {"Content-Type": "application/json"} - return json.dumps({}), 405, {'Content-Type': 'application/json'} # User -@app.route('/api/user', methods=['POST']) -@app.route('/api/user/', methods=['PUT', 'GET', 'DELETE']) +@app.route("/api/user", methods=["POST"]) +@app.route("/api/user/", methods=["PUT", "GET", "DELETE"]) @login_required def user(userid=None): - if request.method == 'DELETE': + if request.method == "DELETE": app.admin.delete_user(userid) - return json.dumps({}), 200, {'Content-Type': 'application/json'} - if request.method == 'POST': - data=request.get_json(force=True) - if app.admin.get_user_username(data['username']): - return json.dumps({'msg':'Add user error: already exists.'}), 409, {'Content-Type': 'application/json'} - data['enabled']=True if data.get('enabled',False) else False - data['quota']=data['quota'] if data['quota'] != 'false' else False - data['groups']=data['groups'] if data.get('groups',False) else [] - if 'external' in threads.keys(): - if threads['external'] is not None and threads['external'].is_alive(): - return json.dumps({'msg':'Precondition failed: already adding users'}), 412, {'Content-Type': 'application/json'} + return json.dumps({}), 200, {"Content-Type": "application/json"} + if request.method == "POST": + data = request.get_json(force=True) + if app.admin.get_user_username(data["username"]): + return ( + json.dumps({"msg": "Add user error: already exists."}), + 409, + {"Content-Type": "application/json"}, + ) + data["enabled"] = True if data.get("enabled", False) else False + data["quota"] = data["quota"] if data["quota"] != "false" else False + data["groups"] = data["groups"] if data.get("groups", False) else [] + if "external" in threads.keys(): + if threads["external"] is not None and threads["external"].is_alive(): + return ( + json.dumps({"msg": "Precondition failed: already adding users"}), + 412, + {"Content-Type": "application/json"}, + ) else: - threads['external']=None + threads["external"] = None try: - threads['external'] = threading.Thread(target=app.admin.add_user, args=(data,)) - threads['external'].start() - return json.dumps({}), 200, {'Content-Type': 'application/json'} + threads["external"] = threading.Thread( + target=app.admin.add_user, args=(data,) + ) + threads["external"].start() + return json.dumps({}), 200, {"Content-Type": "application/json"} except: log.error(traceback.format_exc()) - return json.dumps({'msg':'Add user error.'}), 500, {'Content-Type': 'application/json'} + return ( + json.dumps({"msg": "Add user error."}), + 500, + {"Content-Type": "application/json"}, + ) - if request.method == 'PUT': - data=request.get_json(force=True) - data['enabled']=True if data.get('enabled',False) else False - data['groups']=data['groups'] if data.get('groups',False) else [] - data['roles']=[data.pop('role-keycloak')] + if request.method == "PUT": + data = request.get_json(force=True) + data["enabled"] = True if data.get("enabled", False) else False + data["groups"] = data["groups"] if data.get("groups", False) else [] + data["roles"] = [data.pop("role-keycloak")] try: app.admin.user_update(data) - return json.dumps({}), 200, {'Content-Type': 'application/json'} + return json.dumps({}), 200, {"Content-Type": "application/json"} except UserNotFound: - return json.dumps({'msg':'User not found.'}), 404, {'Content-Type': 'application/json'} - if request.method == 'DELETE': + return ( + json.dumps({"msg": "User not found."}), + 404, + {"Content-Type": "application/json"}, + ) + if request.method == "DELETE": pass - if request.method == 'GET': + if request.method == "GET": user = app.admin.get_user(userid) - if not user: return json.dumps({'msg':'User not found.'}), 404, {'Content-Type': 'application/json'} - return json.dumps(user), 200, {'Content-Type': 'application/json'} + if not user: + return ( + json.dumps({"msg": "User not found."}), + 404, + {"Content-Type": "application/json"}, + ) + return json.dumps(user), 200, {"Content-Type": "application/json"} -@app.route('/api/roles') + +@app.route("/api/roles") @login_required def roles(): - sorted_roles = sorted(app.admin.get_roles(), key=lambda k: k['name']) + sorted_roles = sorted(app.admin.get_roles(), key=lambda k: k["name"]) if current_user.role != "admin": - sorted_roles = [sr for sr in sorted_roles if sr['name'] != 'admin'] - return json.dumps(sorted_roles), 200, {'Content-Type': 'application/json'} + sorted_roles = [sr for sr in sorted_roles if sr["name"] != "admin"] + return json.dumps(sorted_roles), 200, {"Content-Type": "application/json"} -@app.route('/api/group', methods=['POST','DELETE']) -@app.route('/api/group/', methods=['PUT', 'GET', 'DELETE']) + +@app.route("/api/group", methods=["POST", "DELETE"]) +@app.route("/api/group/", methods=["PUT", "GET", "DELETE"]) @login_required def group(group_id=False): - if request.method == 'POST': - data=request.get_json(force=True) - data['parent']=data['parent'] if data['parent'] != '' else None - return json.dumps(app.admin.add_group(data)), 200, {'Content-Type': 'application/json'} - if request.method == 'DELETE': + if request.method == "POST": + data = request.get_json(force=True) + data["parent"] = data["parent"] if data["parent"] != "" else None + return ( + json.dumps(app.admin.add_group(data)), + 200, + {"Content-Type": "application/json"}, + ) + if request.method == "DELETE": try: - data=request.get_json(force=True) + data = request.get_json(force=True) except: - data=False - + data = False + if data: - res = app.admin.delete_group_by_path(data['path']) + res = app.admin.delete_group_by_path(data["path"]) else: res = app.admin.delete_group_by_id(group_id) - return json.dumps(res), 200, {'Content-Type': 'application/json'} + return json.dumps(res), 200, {"Content-Type": "application/json"} -@app.route('/api/groups') -@app.route('/api/groups/', methods=['POST', 'PUT', 'GET', 'DELETE']) + +@app.route("/api/groups") +@app.route("/api/groups/", methods=["POST", "PUT", "GET", "DELETE"]) @login_required def groups(provider=False): - if request.method == 'GET': - sorted_groups = sorted(app.admin.get_mix_groups(), key=lambda k: k['name']) + if request.method == "GET": + sorted_groups = sorted(app.admin.get_mix_groups(), key=lambda k: k["name"]) if current_user.role != "admin": ## internal groups should be avoided as are assigned with the role - sorted_groups = [sg for sg in sorted_groups if not system_group(sg['name'])] + sorted_groups = [sg for sg in sorted_groups if not system_group(sg["name"])] else: sorted_groups = [sg for sg in sorted_groups] - return json.dumps(sorted_groups), 200, {'Content-Type': 'application/json'} - if request.method == 'DELETE': - if provider == 'keycloak': - return json.dumps(app.admin.delete_keycloak_groups()), 200, {'Content-Type': 'application/json'} + return json.dumps(sorted_groups), 200, {"Content-Type": "application/json"} + if request.method == "DELETE": + if provider == "keycloak": + return ( + json.dumps(app.admin.delete_keycloak_groups()), + 200, + {"Content-Type": "application/json"}, + ) + ### SYSADM USERS ONLY -@app.route('/api/external', methods=['POST', 'PUT', 'GET','DELETE']) + +@app.route("/api/external", methods=["POST", "PUT", "GET", "DELETE"]) @login_required def external(): - if 'external' in threads.keys(): - if threads['external'] is not None and threads['external'].is_alive(): - return json.dumps({}), 301, {'Content-Type': 'application/json'} + if "external" in threads.keys(): + if threads["external"] is not None and threads["external"].is_alive(): + return json.dumps({}), 301, {"Content-Type": "application/json"} else: - threads['external']=None + threads["external"] = None - if request.method == 'POST': - data=request.get_json(force=True) - if data['format']=='json-ga': - threads['external'] = threading.Thread(target=app.admin.upload_json_ga, args=(data,)) - threads['external'].start() - return json.dumps({}), 200, {'Content-Type': 'application/json'} - if data['format']=='csv-ug': + if request.method == "POST": + data = request.get_json(force=True) + if data["format"] == "json-ga": + threads["external"] = threading.Thread( + target=app.admin.upload_json_ga, args=(data,) + ) + threads["external"].start() + return json.dumps({}), 200, {"Content-Type": "application/json"} + if data["format"] == "csv-ug": valid = check_upload_errors(data) - if valid['pass']: - threads['external'] = threading.Thread(target=app.admin.upload_csv_ug, args=(data,)) - threads['external'].start() - return json.dumps({}), 200, {'Content-Type': 'application/json'} + if valid["pass"]: + threads["external"] = threading.Thread( + target=app.admin.upload_csv_ug, args=(data,) + ) + threads["external"].start() + return json.dumps({}), 200, {"Content-Type": "application/json"} else: - return json.dumps(valid), 422, {'Content-Type': 'application/json'} - if request.method == 'PUT': - data=request.get_json(force=True) - threads['external'] = threading.Thread(target=app.admin.sync_external, args=(data,)) - threads['external'].start() - return json.dumps({}), 200, {'Content-Type': 'application/json'} - if request.method == 'DELETE': - print('RESET') + return json.dumps(valid), 422, {"Content-Type": "application/json"} + if request.method == "PUT": + data = request.get_json(force=True) + threads["external"] = threading.Thread( + target=app.admin.sync_external, args=(data,) + ) + threads["external"].start() + return json.dumps({}), 200, {"Content-Type": "application/json"} + if request.method == "DELETE": + print("RESET") app.admin.reset_external() - return json.dumps({}), 200, {'Content-Type': 'application/json'} - return json.dumps({}), 500, {'Content-Type': 'application/json'} + return json.dumps({}), 200, {"Content-Type": "application/json"} + return json.dumps({}), 500, {"Content-Type": "application/json"} -@app.route('/api/external/users') + +@app.route("/api/external/users") @login_required def external_users_list(): - while threads['external'] is not None and threads['external'].is_alive(): - time.sleep(.5) - return json.dumps(app.admin.get_external_users()), 200, {'Content-Type': 'application/json'} + while threads["external"] is not None and threads["external"].is_alive(): + time.sleep(0.5) + return ( + json.dumps(app.admin.get_external_users()), + 200, + {"Content-Type": "application/json"}, + ) -@app.route('/api/external/groups') + +@app.route("/api/external/groups") @login_required def external_groups_list(): - while threads['external'] is not None and threads['external'].is_alive(): - time.sleep(.5) - return json.dumps(app.admin.get_external_groups()), 200, {'Content-Type': 'application/json'} + while threads["external"] is not None and threads["external"].is_alive(): + time.sleep(0.5) + return ( + json.dumps(app.admin.get_external_groups()), + 200, + {"Content-Type": "application/json"}, + ) -@app.route('/api/external/roles', methods=['PUT']) + +@app.route("/api/external/roles", methods=["PUT"]) @login_required def external_roles(): - if request.method == 'PUT': - return json.dumps(app.admin.external_roleassign(request.get_json(force=True))), 200, {'Content-Type': 'application/json'} + if request.method == "PUT": + return ( + json.dumps(app.admin.external_roleassign(request.get_json(force=True))), + 200, + {"Content-Type": "application/json"}, + ) -{'groups': '/alumnes/3er', -'firstname': 'Andreu', -'lastname': 'B', -'email': '12andreub@escolamontseny.cat', -'username': '12andreub', -'password': 'pepinillo', -'password_temporal': 'yes', -'role': 'student', -'quota': '500 MB', -'': '', -'id': '12andreub'} + +{ + "groups": "/alumnes/3er", + "firstname": "Andreu", + "lastname": "B", + "email": "12andreub@escolamontseny.cat", + "username": "12andreub", + "password": "pepinillo", + "password_temporal": "yes", + "role": "student", + "quota": "500 MB", + "": "", + "id": "12andreub", +} def check_upload_errors(data): - email_regex = r'\b[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Z|a-z]{2,}\b' - for u in data['data']: + email_regex = r"\b[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Z|a-z]{2,}\b" + for u in data["data"]: try: - user_groups=[g.strip() for g in u['groups'].split(',')] + user_groups = [g.strip() for g in u["groups"].split(",")] except: - return {'pass':False,'msg':'User '+u['username']+' has invalid groups: '+u['groups']} + return { + "pass": False, + "msg": "User " + u["username"] + " has invalid groups: " + u["groups"], + } - if not re.fullmatch(email_regex, u['email']): - return {'pass':False,'msg':'User '+u['username']+' has invalid email: '+u['email']} - - if u['role'] not in ['admin','manager','teacher','student']: - if u['role'] == '': - return {'pass':False,'msg':'User '+u['username']+' has no role assigned!'} - return {'pass':False,'msg':'User '+u['username']+' has invalid role: '+u['role']} - - if u['password_temporal'].lower() not in ['yes','no']: - return {'pass':False,'msg':'User '+u['username']+' has invalid password_temporal value (yes/no): '+u['password_temporal']} - return {'pass':True,'msg':''} \ No newline at end of file + if not re.fullmatch(email_regex, u["email"]): + return { + "pass": False, + "msg": "User " + u["username"] + " has invalid email: " + u["email"], + } + + if u["role"] not in ["admin", "manager", "teacher", "student"]: + if u["role"] == "": + return { + "pass": False, + "msg": "User " + u["username"] + " has no role assigned!", + } + return { + "pass": False, + "msg": "User " + u["username"] + " has invalid role: " + u["role"], + } + + if u["password_temporal"].lower() not in ["yes", "no"]: + return { + "pass": False, + "msg": "User " + + u["username"] + + " has invalid password_temporal value (yes/no): " + + u["password_temporal"], + } + return {"pass": True, "msg": ""} diff --git a/admin/src/admin/views/InternalViews.py b/admin/src/admin/views/InternalViews.py index 2ad8461..0e65705 100644 --- a/admin/src/admin/views/InternalViews.py +++ b/admin/src/admin/views/InternalViews.py @@ -1,107 +1,136 @@ #!flask/bin/python # coding=utf-8 -from admin import app +import json import logging as log +import os +import sys +import time import traceback -import time,json -import sys,os - from flask import request + +from admin import app + from .decorators import is_internal -@app.route('/api/internal/users', methods=['GET']) + +@app.route("/api/internal/users", methods=["GET"]) @is_internal def internal_users(): - if request.method == 'GET': - sorted_users = sorted(app.admin.get_mix_users(), key=lambda k: k['username']) + if request.method == "GET": + sorted_users = sorted(app.admin.get_mix_users(), key=lambda k: k["username"]) # group_users = [user for user in sorted_users if data['path'] in user['keycloak_groups']] - users=[] + users = [] for user in sorted_users: - if not user['enabled']: continue + if not user["enabled"]: + continue users.append(user_parser(user)) - return json.dumps(users), 200, {'Content-Type': 'application/json'} + return json.dumps(users), 200, {"Content-Type": "application/json"} -@app.route('/api/internal/users/filter', methods=['POST']) + +@app.route("/api/internal/users/filter", methods=["POST"]) @is_internal def internal_users_search(): - if request.method == 'POST': - data=request.get_json(force=True) + if request.method == "POST": + data = request.get_json(force=True) users = app.admin.get_mix_users() - result = [user_parser(user) for user in filter_users(users, data['text'])] - sorted_result = sorted(result, key=lambda k: k['id']) - return json.dumps(sorted_result), 200, {'Content-Type': 'application/json'} + result = [user_parser(user) for user in filter_users(users, data["text"])] + sorted_result = sorted(result, key=lambda k: k["id"]) + return json.dumps(sorted_result), 200, {"Content-Type": "application/json"} -@app.route('/api/internal/groups', methods=['GET']) + +@app.route("/api/internal/groups", methods=["GET"]) @is_internal def internal_groups(): - if request.method == 'GET': - sorted_groups = sorted(app.admin.get_mix_groups(), key=lambda k: k['name']) - groups=[] + if request.method == "GET": + sorted_groups = sorted(app.admin.get_mix_groups(), key=lambda k: k["name"]) + groups = [] for group in sorted_groups: - if not group['path'].startswith('/'): continue - groups.append({'id':group['path'], - 'name':group['name'], - 'description':group.get('description','')}) - return json.dumps(groups), 200, {'Content-Type': 'application/json'} + if not group["path"].startswith("/"): + continue + groups.append( + { + "id": group["path"], + "name": group["name"], + "description": group.get("description", ""), + } + ) + return json.dumps(groups), 200, {"Content-Type": "application/json"} -@app.route('/api/internal/group/users', methods=['POST']) + +@app.route("/api/internal/group/users", methods=["POST"]) @is_internal def internal_group_users(): - if request.method == 'POST': - data=request.get_json(force=True) - sorted_users = sorted(app.admin.get_mix_users(), key=lambda k: k['username']) + if request.method == "POST": + data = request.get_json(force=True) + sorted_users = sorted(app.admin.get_mix_users(), key=lambda k: k["username"]) # group_users = [user for user in sorted_users if data['path'] in user['keycloak_groups']] - users=[] + users = [] for user in sorted_users: - if data['path'] not in user['keycloak_groups'] or not user['enabled']: continue + if data["path"] not in user["keycloak_groups"] or not user["enabled"]: + continue users.append(user) - if data.get('text',False) and data['text'] != '': - result = [user_parser(user) for user in filter_users(users, data['text'])] + if data.get("text", False) and data["text"] != "": + result = [user_parser(user) for user in filter_users(users, data["text"])] else: result = [user_parser(user) for user in users] - return json.dumps(result), 200, {'Content-Type': 'application/json'} + return json.dumps(result), 200, {"Content-Type": "application/json"} -@app.route('/api/internal/roles', methods=['GET']) + +@app.route("/api/internal/roles", methods=["GET"]) @is_internal def internal_roles(): - if request.method == 'GET': - roles=[] - for role in sorted(app.admin.get_roles(), key=lambda k: k['name']): - if role['name'] == 'admin': continue - roles.append({'id':role['id'], - 'name':role['name'], - 'description':role.get('description','')}) - return json.dumps(roles), 200, {'Content-Type': 'application/json'} + if request.method == "GET": + roles = [] + for role in sorted(app.admin.get_roles(), key=lambda k: k["name"]): + if role["name"] == "admin": + continue + roles.append( + { + "id": role["id"], + "name": role["name"], + "description": role.get("description", ""), + } + ) + return json.dumps(roles), 200, {"Content-Type": "application/json"} -@app.route('/api/internal/role/users', methods=['POST']) + +@app.route("/api/internal/role/users", methods=["POST"]) @is_internal def internal_role_users(): - if request.method == 'POST': - data=request.get_json(force=True) - sorted_users = sorted(app.admin.get_mix_users(), key=lambda k: k['username']) + if request.method == "POST": + data = request.get_json(force=True) + sorted_users = sorted(app.admin.get_mix_users(), key=lambda k: k["username"]) # group_users = [user for user in sorted_users if data['path'] in user['keycloak_groups']] - users=[] + users = [] for user in sorted_users: - if data['role'] not in user['roles'] or not user['enabled']: continue + if data["role"] not in user["roles"] or not user["enabled"]: + continue users.append(user) - if data.get('text',False) and data['text'] != '': - result = [user_parser(user) for user in filter_users(users, data['text'])] + if data.get("text", False) and data["text"] != "": + result = [user_parser(user) for user in filter_users(users, data["text"])] else: result = [user_parser(user) for user in users] - return json.dumps(result), 200, {'Content-Type': 'application/json'} + return json.dumps(result), 200, {"Content-Type": "application/json"} + def user_parser(user): - return {'id':user['username'], - 'first':user['first'], - 'last':user['last'], - 'role':user['roles'][0] if len(user['roles']) else None, - 'email':user['email'], - 'groups':user['keycloak_groups']} + return { + "id": user["username"], + "first": user["first"], + "last": user["last"], + "role": user["roles"][0] if len(user["roles"]) else None, + "email": user["email"], + "groups": user["keycloak_groups"], + } + def filter_users(users, text): - return [user for user in users - if text in user['username'] or - text in user['first'] or - text in user['last'] or - text in user['email']] + return [ + user + for user in users + if text in user["username"] + or text in user["first"] + or text in user["last"] + or text in user["email"] + ] diff --git a/admin/src/admin/views/LoginViews.py b/admin/src/admin/views/LoginViews.py index 1dc2ced..61a9a9d 100644 --- a/admin/src/admin/views/LoginViews.py +++ b/admin/src/admin/views/LoginViews.py @@ -1,31 +1,42 @@ import os +from flask import flash, redirect, render_template, request, url_for +from flask_login import current_user, login_required, login_user, logout_user + from admin import app -from flask import render_template, flash, request, redirect, url_for -from ..auth.authentication import * -from flask_login import login_required, current_user, login_user, logout_user -@app.route('/', methods=['GET', 'POST']) -@app.route('/login', methods=['GET', 'POST']) +from ..auth.authentication import * + + +@app.route("/", methods=["GET", "POST"]) +@app.route("/login", methods=["GET", "POST"]) def login(): - if request.method == 'POST': - if request.form['user'] == '' or request.form['password'] == '': - flash("Can't leave it blank",'danger') - elif request.form['user'].startswith(' '): - flash('Username not found or incorrect password.','warning') - else: - ram_user=ram_users.get(request.form['user']) - if ram_user and request.form['password'] == ram_user['password']: - user=User({'id': ram_user['id'], 'password': ram_user['password'], 'role': ram_user['role'], 'active': True}) - login_user(user) - flash('Logged in successfully.','success') - return redirect(url_for('web_users')) - else: - flash('Username not found or incorrect password.','warning') - return render_template('login.html') + if request.method == "POST": + if request.form["user"] == "" or request.form["password"] == "": + flash("Can't leave it blank", "danger") + elif request.form["user"].startswith(" "): + flash("Username not found or incorrect password.", "warning") + else: + ram_user = ram_users.get(request.form["user"]) + if ram_user and request.form["password"] == ram_user["password"]: + user = User( + { + "id": ram_user["id"], + "password": ram_user["password"], + "role": ram_user["role"], + "active": True, + } + ) + login_user(user) + flash("Logged in successfully.", "success") + return redirect(url_for("web_users")) + else: + flash("Username not found or incorrect password.", "warning") + return render_template("login.html") -@app.route('/logout', methods=['GET']) + +@app.route("/logout", methods=["GET"]) @login_required def logout(): - logout_user() - return redirect(url_for('login')) + logout_user() + return redirect(url_for("login")) diff --git a/admin/src/admin/views/Socketio.py b/admin/src/admin/views/Socketio.py index e610291..9f1ba43 100644 --- a/admin/src/admin/views/Socketio.py +++ b/admin/src/admin/views/Socketio.py @@ -9,11 +9,11 @@ # @socketio.on('connect', namespace='//sio') # def socketio_connect(): # join_room('admin') -# socketio.emit('update', -# json.dumps('Joined'), -# namespace='//sio', +# socketio.emit('update', +# json.dumps('Joined'), +# namespace='//sio', # room='admin') - + # @socketio.on('disconnect', namespace='//sio') # def socketio_domains_disconnect(): -# None \ No newline at end of file +# None diff --git a/admin/src/admin/views/WebViews.py b/admin/src/admin/views/WebViews.py index b7e12ce..4e0f069 100644 --- a/admin/src/admin/views/WebViews.py +++ b/admin/src/admin/views/WebViews.py @@ -1,23 +1,34 @@ #!flask/bin/python # coding=utf-8 -from admin import app -import logging as log -import traceback - -from uuid import uuid4 -import time,json -import sys,os -from flask import render_template, Response, request, redirect, url_for, jsonify, send_file import concurrent.futures +import json +import logging as log +import os +import sys +import time +import traceback +from pprint import pprint +from uuid import uuid4 + +from flask import ( + Response, + jsonify, + redirect, + render_template, + request, + send_file, + url_for, +) from flask_login import login_required + +from admin import app + +from ..lib.avatars import Avatars from .decorators import is_admin -from pprint import pprint -from ..lib.avatars import Avatars +avatars = Avatars() -avatars=Avatars() - -''' OIDC TESTS ''' +""" OIDC TESTS """ # from ..auth.authentication import oidc # @app.route('/custom_callback') @@ -43,47 +54,60 @@ avatars=Avatars() # def logoutoidc(): # oidc.logout() # return 'Hi, you have been logged out! Return' -''' OIDC TESTS ''' +""" OIDC TESTS """ -@app.route('/users') + +@app.route("/users") @login_required def web_users(): - return render_template('pages/users.html', title="Users", nav="Users") + return render_template("pages/users.html", title="Users", nav="Users") -@app.route('/roles') + +@app.route("/roles") @login_required def web_roles(): - return render_template('pages/roles.html', title="Roles", nav="Roles") + return render_template("pages/roles.html", title="Roles", nav="Roles") -@app.route('/groups') + +@app.route("/groups") @login_required def web_groups(provider=False): - return render_template('pages/groups.html', title="Groups", nav="Groups") + return render_template("pages/groups.html", title="Groups", nav="Groups") -@app.route('/avatar/', methods=['GET']) + +@app.route("/avatar/", methods=["GET"]) @login_required def avatar(userid): - if userid != 'false': - return send_file('../avatars/master-avatars/'+userid, mimetype='image/jpeg') - return send_file('static/img/missing.jpg', mimetype='image/jpeg') + if userid != "false": + return send_file("../avatars/master-avatars/" + userid, mimetype="image/jpeg") + return send_file("static/img/missing.jpg", mimetype="image/jpeg") + ### SYS ADMIN -@app.route('/sysadmin/users') + +@app.route("/sysadmin/users") @login_required @is_admin def web_sysadmin_users(): - return render_template('pages/sysadmin/users.html', title="SysAdmin Users", nav="SysAdminUsers") + return render_template( + "pages/sysadmin/users.html", title="SysAdmin Users", nav="SysAdminUsers" + ) -@app.route('/sysadmin/groups') + +@app.route("/sysadmin/groups") @login_required @is_admin def web_sysadmin_groups(): - return render_template('pages/sysadmin/groups.html', title="SysAdmin Groups", nav="SysAdminGroups") - + return render_template( + "pages/sysadmin/groups.html", title="SysAdmin Groups", nav="SysAdminGroups" + ) -@app.route('/sysadmin/external') + +@app.route("/sysadmin/external") @login_required ## SysAdmin role def web_sysadmin_external(): - return render_template('pages/sysadmin/external.html', title="External", nav="External") + return render_template( + "pages/sysadmin/external.html", title="External", nav="External" + ) diff --git a/admin/src/admin/views/decorators.py b/admin/src/admin/views/decorators.py index 6264a84..55ef0fe 100644 --- a/admin/src/admin/views/decorators.py +++ b/admin/src/admin/views/decorators.py @@ -1,25 +1,36 @@ #!flask/bin/python # coding=utf-8 -from functools import wraps -from flask import request, redirect, url_for -from flask_login import current_user, logout_user import socket +from functools import wraps + +from flask import redirect, request, url_for +from flask_login import current_user, logout_user + def is_admin(fn): @wraps(fn) def decorated_view(*args, **kwargs): - if current_user.role == 'admin': return fn(*args, **kwargs) - return redirect(url_for('login')) + if current_user.role == "admin": + return fn(*args, **kwargs) + return redirect(url_for("login")) + return decorated_view + def is_internal(fn): @wraps(fn) def decorated_view(*args, **kwargs): - remote_addr=request.headers['X-Forwarded-For'].split(',')[0] if 'X-Forwarded-For' in request.headers else request.remote_addr.split(',')[0] + remote_addr = ( + request.headers["X-Forwarded-For"].split(",")[0] + if "X-Forwarded-For" in request.headers + else request.remote_addr.split(",")[0] + ) ## Now only checks if it is wordpress container, ## but we should check if it is internal net and not haproxy - if socket.gethostbyname('isard-apps-wordpress') == remote_addr: return fn(*args, **kwargs) + if socket.gethostbyname("isard-apps-wordpress") == remote_addr: + return fn(*args, **kwargs) logout_user() - return redirect(url_for('login')) - return decorated_view \ No newline at end of file + return redirect(url_for("login")) + + return decorated_view diff --git a/admin/src/moodle_saml.py b/admin/src/moodle_saml.py index 9e41910..c56a460 100644 --- a/admin/src/moodle_saml.py +++ b/admin/src/moodle_saml.py @@ -1,72 +1,89 @@ #!/usr/bin/env python # coding=utf-8 -import time, os -from datetime import datetime, timedelta -import pprint - +import json import logging as log +import os +import pprint +import random +import string +import time import traceback -import yaml, json +from datetime import datetime, timedelta import psycopg2 +import yaml -from admin.lib.postgres import Postgres from admin.lib.keycloak_client import KeycloakClient +from admin.lib.postgres import Postgres -import string, random +app = {} +app["config"] = {} -app={} -app['config']={} -class MoodleSaml(): +class MoodleSaml: def __init__(self): - ready=False + ready = False while not ready: try: - self.pg=Postgres('isard-apps-postgresql','moodle',os.environ['MOODLE_POSTGRES_USER'],os.environ['MOODLE_POSTGRES_PASSWORD']) - ready=True + self.pg = Postgres( + "isard-apps-postgresql", + "moodle", + os.environ["MOODLE_POSTGRES_USER"], + os.environ["MOODLE_POSTGRES_PASSWORD"], + ) + ready = True except: - log.warning('Could not connect to moodle database. Retrying...') + log.warning("Could not connect to moodle database. Retrying...") time.sleep(2) - log.info('Connected to moodle database.') + log.info("Connected to moodle database.") - ready=False + ready = False while not ready: try: - privatekey_pass=self.get_privatekey_pass() - log.warning("The key: "+str(privatekey_pass)) - if privatekey_pass.endswith(os.environ['DOMAIN']): - app['config']['MOODLE_SAML_PRIVATEKEYPASS']=privatekey_pass - ready=True + privatekey_pass = self.get_privatekey_pass() + log.warning("The key: " + str(privatekey_pass)) + if privatekey_pass.endswith(os.environ["DOMAIN"]): + app["config"]["MOODLE_SAML_PRIVATEKEYPASS"] = privatekey_pass + ready = True except: # print(traceback.format_exc()) - log.warning('Could not get moodle site identifier. Retrying...') + log.warning("Could not get moodle site identifier. Retrying...") time.sleep(2) - log.info('Got moodle site identifier.') + log.info("Got moodle site identifier.") - ready=False + ready = False while not ready: try: - with open(os.path.join("./moodledata/saml2/moodle."+os.environ['DOMAIN']+".crt"),"r") as crt: - app['config']['SP_CRT']=crt.read() - ready=True + with open( + os.path.join( + "./moodledata/saml2/moodle." + os.environ["DOMAIN"] + ".crt" + ), + "r", + ) as crt: + app["config"]["SP_CRT"] = crt.read() + ready = True except IOError: - log.warning('Could not get moodle SAML2 crt certificate. Retrying...') + log.warning("Could not get moodle SAML2 crt certificate. Retrying...") time.sleep(2) except: log.error(traceback.format_exc()) - log.info('Got moodle srt certificate.') + log.info("Got moodle srt certificate.") - ready=False + ready = False while not ready: try: - with open(os.path.join("./moodledata/saml2/moodle."+os.environ['DOMAIN']+".pem"),"r") as pem: - app['config']['SP_PEM']=pem.read() - ready=True + with open( + os.path.join( + "./moodledata/saml2/moodle." + os.environ["DOMAIN"] + ".pem" + ), + "r", + ) as pem: + app["config"]["SP_PEM"] = pem.read() + ready = True except IOError: - log.warning('Could not get moodle SAML2 pem certificate. Retrying...') + log.warning("Could not get moodle SAML2 pem certificate. Retrying...") time.sleep(2) - log.info('Got moodle pem certificate.') + log.info("Got moodle pem certificate.") ## This seems related to the fact that the certificate generated the first time does'nt work. ## And when regenerating the certificate de privatekeypass seems not to be used and instead it @@ -77,183 +94,259 @@ class MoodleSaml(): ## 3.- Cleanup all caches in moodle (Development tab) # with open(os.path.join("./moodledata/saml2/"+os.environ['MOODLE_SAML_PRIVATEKEYPASS'].replace("moodle."+os.environ['DOMAIN'],'')+'.idp.xml'),"w") as xml: # xml.write(self.parse_idp_metadata()) - with open(os.path.join("./moodledata/saml2/0f635d0e0f3874fff8b581c132e6c7a7.idp.xml"),"w") as xml: + with open( + os.path.join("./moodledata/saml2/0f635d0e0f3874fff8b581c132e6c7a7.idp.xml"), + "w", + ) as xml: xml.write(self.parse_idp_metadata()) - log.info('Written SP file on moodledata.') + log.info("Written SP file on moodledata.") try: self.activate_saml_plugin() except: - print('Error activating saml on moodle') - + print("Error activating saml on moodle") + try: self.set_moodle_saml_plugin() except: - print('Error setting saml on moodle') - + print("Error setting saml on moodle") + try: self.delete_keycloak_moodle_saml_plugin() except: - print('Error deleting saml on keycloak') - + print("Error deleting saml on keycloak") + try: self.add_keycloak_moodle_saml() except: - print('Error adding saml on keycloak') + print("Error adding saml on keycloak") - # SAML clients don't work well with composite roles so disabling and adding on realm + # SAML clients don't work well with composite roles so disabling and adding on realm # self.add_client_roles() def activate_saml_plugin(self): ## After you need to purge moodle caches: /var/www/html # php admin/cli/purge_caches.php - return self.pg.update("""UPDATE "mdl_config" SET value = 'email,saml2' WHERE "name" = 'auth'""") + return self.pg.update( + """UPDATE "mdl_config" SET value = 'email,saml2' WHERE "name" = 'auth'""" + ) def get_privatekey_pass(self): - return self.pg.select("""SELECT * FROM "mdl_config" WHERE "name" = 'siteidentifier'""")[0][2] + return self.pg.select( + """SELECT * FROM "mdl_config" WHERE "name" = 'siteidentifier'""" + )[0][2] def parse_idp_metadata(self): - keycloak=KeycloakClient() - rsa=keycloak.get_server_rsa_key() - keycloak=None - return ''+rsa['name']+''+rsa['certificate']+'urn:oasis:names:tc:SAML:2.0:nameid-format:persistenturn:oasis:names:tc:SAML:2.0:nameid-format:transienturn:oasis:names:tc:SAML:1.1:nameid-format:unspecifiedurn:oasis:names:tc:SAML:1.1:nameid-format:emailAddress' + keycloak = KeycloakClient() + rsa = keycloak.get_server_rsa_key() + keycloak = None + return ( + '' + + rsa["name"] + + "" + + rsa["certificate"] + + 'urn:oasis:names:tc:SAML:2.0:nameid-format:persistenturn:oasis:names:tc:SAML:2.0:nameid-format:transienturn:oasis:names:tc:SAML:1.1:nameid-format:unspecifiedurn:oasis:names:tc:SAML:1.1:nameid-format:emailAddress' + ) def set_keycloak_moodle_saml_plugin(self): - keycloak=KeycloakClient() + keycloak = KeycloakClient() keycloak.add_moodle_client() - keycloak=None + keycloak = None def delete_keycloak_moodle_saml_plugin(self): - keycloak=KeycloakClient() - keycloak.delete_client('a92d5417-92b6-4678-9cb9-51bc0edcee8c') - keycloak=None + keycloak = KeycloakClient() + keycloak.delete_client("a92d5417-92b6-4678-9cb9-51bc0edcee8c") + keycloak = None def set_moodle_saml_plugin(self): - config={'idpmetadata': self.parse_idp_metadata(), - 'certs_locked': '1', - 'duallogin': '1', - 'idpattr': 'username', - 'autocreate': '1', - 'anyauth': '1', - 'saml_role_siteadmin_map': 'admin', - 'saml_role_coursecreator_map': 'teacher', - 'saml_role_manager_map': 'manager', - 'field_map_email': 'email', - 'field_map_firstname': 'givenName', - 'field_map_lastname': 'sn'} + config = { + "idpmetadata": self.parse_idp_metadata(), + "certs_locked": "1", + "duallogin": "1", + "idpattr": "username", + "autocreate": "1", + "anyauth": "1", + "saml_role_siteadmin_map": "admin", + "saml_role_coursecreator_map": "teacher", + "saml_role_manager_map": "manager", + "field_map_email": "email", + "field_map_firstname": "givenName", + "field_map_lastname": "sn", + } for name in config.keys(): - self.pg.update("""UPDATE "mdl_config_plugins" SET value = '%s' WHERE "plugin" = 'auth_saml2' AND "name" = '%s'""" % (config[name],name)) - self.pg.update("""INSERT INTO "mdl_auth_saml2_idps" ("metadataurl", "entityid", "activeidp", "defaultidp", "adminidp", "defaultname", "displayname", "logo", "alias", "whitelist") VALUES - ('xml', 'https://sso.%s/auth/realms/master', 1, 0, 0, 'Login via SAML2', '', NULL, NULL, NULL);""" % (os.environ['DOMAIN'])) + self.pg.update( + """UPDATE "mdl_config_plugins" SET value = '%s' WHERE "plugin" = 'auth_saml2' AND "name" = '%s'""" + % (config[name], name) + ) + self.pg.update( + """INSERT INTO "mdl_auth_saml2_idps" ("metadataurl", "entityid", "activeidp", "defaultidp", "adminidp", "defaultname", "displayname", "logo", "alias", "whitelist") VALUES + ('xml', 'https://sso.%s/auth/realms/master', 1, 0, 0, 'Login via SAML2', '', NULL, NULL, NULL);""" + % (os.environ["DOMAIN"]) + ) def add_keycloak_moodle_saml(self): - client={ - "id" : "a92d5417-92b6-4678-9cb9-51bc0edcee8c", + client = { + "id": "a92d5417-92b6-4678-9cb9-51bc0edcee8c", "name": "moodle", "description": "moodle", - "clientId" : "https://moodle."+os.environ['DOMAIN']+"/auth/saml2/sp/metadata.php", - "surrogateAuthRequired" : False, - "enabled" : True, - "alwaysDisplayInConsole" : False, - "clientAuthenticatorType" : "client-secret", - "redirectUris" : [ "https://moodle."+os.environ['DOMAIN']+"/auth/saml2/sp/saml2-acs.php/moodle."+os.environ['DOMAIN']+"" ], - "webOrigins" : [ "https://moodle."+os.environ['DOMAIN']+"" ], - "notBefore" : 0, - "bearerOnly" : False, - "consentRequired" : False, - "standardFlowEnabled" : True, - "implicitFlowEnabled" : False, - "directAccessGrantsEnabled" : False, - "serviceAccountsEnabled" : False, - "publicClient" : False, - "frontchannelLogout" : True, - "protocol" : "saml", - "attributes" : { - "saml.force.post.binding" : True, - "saml.encrypt" : False, - "saml_assertion_consumer_url_post" : "https://moodle."+os.environ['DOMAIN']+"/auth/saml2/sp/saml2-acs.php/moodle."+os.environ['DOMAIN']+"", - "saml.server.signature" : True, - "saml.server.signature.keyinfo.ext" : False, - "saml.signing.certificate" : app['config']['SP_CRT'], - "saml_single_logout_service_url_redirect" : "https://moodle."+os.environ['DOMAIN']+"/auth/saml2/sp/saml2-logout.php/moodle."+os.environ['DOMAIN']+"", - "saml.signature.algorithm" : "RSA_SHA256", - "saml_force_name_id_format" : False, - "saml.client.signature" : True, - "saml.encryption.certificate" : app['config']['SP_PEM'], - "saml.authnstatement" : True, - "saml_name_id_format" : "username", - "saml_signature_canonicalization_method" : "http://www.w3.org/2001/10/xml-exc-c14n#" + "clientId": "https://moodle." + + os.environ["DOMAIN"] + + "/auth/saml2/sp/metadata.php", + "surrogateAuthRequired": False, + "enabled": True, + "alwaysDisplayInConsole": False, + "clientAuthenticatorType": "client-secret", + "redirectUris": [ + "https://moodle." + + os.environ["DOMAIN"] + + "/auth/saml2/sp/saml2-acs.php/moodle." + + os.environ["DOMAIN"] + + "" + ], + "webOrigins": ["https://moodle." + os.environ["DOMAIN"] + ""], + "notBefore": 0, + "bearerOnly": False, + "consentRequired": False, + "standardFlowEnabled": True, + "implicitFlowEnabled": False, + "directAccessGrantsEnabled": False, + "serviceAccountsEnabled": False, + "publicClient": False, + "frontchannelLogout": True, + "protocol": "saml", + "attributes": { + "saml.force.post.binding": True, + "saml.encrypt": False, + "saml_assertion_consumer_url_post": "https://moodle." + + os.environ["DOMAIN"] + + "/auth/saml2/sp/saml2-acs.php/moodle." + + os.environ["DOMAIN"] + + "", + "saml.server.signature": True, + "saml.server.signature.keyinfo.ext": False, + "saml.signing.certificate": app["config"]["SP_CRT"], + "saml_single_logout_service_url_redirect": "https://moodle." + + os.environ["DOMAIN"] + + "/auth/saml2/sp/saml2-logout.php/moodle." + + os.environ["DOMAIN"] + + "", + "saml.signature.algorithm": "RSA_SHA256", + "saml_force_name_id_format": False, + "saml.client.signature": True, + "saml.encryption.certificate": app["config"]["SP_PEM"], + "saml.authnstatement": True, + "saml_name_id_format": "username", + "saml_signature_canonicalization_method": "http://www.w3.org/2001/10/xml-exc-c14n#", }, - "authenticationFlowBindingOverrides" : { }, - "fullScopeAllowed" : True, - "nodeReRegistrationTimeout" : -1, - "protocolMappers" : [ { - "id" : "9296daa3-4fc4-4b80-b007-5070f546ae13", - "name" : "X500 sn", - "protocol" : "saml", - "protocolMapper" : "saml-user-property-mapper", - "consentRequired" : False, - "config" : { - "attribute.nameformat" : "urn:oasis:names:tc:SAML:2.0:attrname-format:uri", - "user.attribute" : "lastName", - "friendly.name" : "sn", - "attribute.name" : "urn:oid:2.5.4.4" - } - }, { - "id" : "ccecf6e4-d20a-4211-b67c-40200a6b2c5d", - "name" : "username", - "protocol" : "saml", - "protocolMapper" : "saml-user-property-mapper", - "consentRequired" : False, - "config" : { - "attribute.nameformat" : "Basic", - "user.attribute" : "username", - "friendly.name" : "username", - "attribute.name" : "username" - } - }, { - "id" : "53858403-eba2-4f6d-81d0-cced700b5719", - "name" : "X500 givenName", - "protocol" : "saml", - "protocolMapper" : "saml-user-property-mapper", - "consentRequired" : False, - "config" : { - "attribute.nameformat" : "urn:oasis:names:tc:SAML:2.0:attrname-format:uri", - "user.attribute" : "firstName", - "friendly.name" : "givenName", - "attribute.name" : "urn:oid:2.5.4.42" - } - }, { - "id" : "20034db5-1d0e-4e66-b815-fb0440c6d1e2", - "name" : "X500 email", - "protocol" : "saml", - "protocolMapper" : "saml-user-property-mapper", - "consentRequired" : False, - "config" : { - "attribute.nameformat" : "urn:oasis:names:tc:SAML:2.0:attrname-format:uri", - "user.attribute" : "email", - "friendly.name" : "email", - "attribute.name" : "urn:oid:1.2.840.113549.1.9.1" - } - } ], - "defaultClientScopes" : [ "web-origins", "role_list", "roles", "profile", "email" ], - "optionalClientScopes" : [ "address", "phone", "offline_access", "microprofile-jwt" ], - "access" : { - "view" : True, - "configure" : True, - "manage" : True - } + "authenticationFlowBindingOverrides": {}, + "fullScopeAllowed": True, + "nodeReRegistrationTimeout": -1, + "protocolMappers": [ + { + "id": "9296daa3-4fc4-4b80-b007-5070f546ae13", + "name": "X500 sn", + "protocol": "saml", + "protocolMapper": "saml-user-property-mapper", + "consentRequired": False, + "config": { + "attribute.nameformat": "urn:oasis:names:tc:SAML:2.0:attrname-format:uri", + "user.attribute": "lastName", + "friendly.name": "sn", + "attribute.name": "urn:oid:2.5.4.4", + }, + }, + { + "id": "ccecf6e4-d20a-4211-b67c-40200a6b2c5d", + "name": "username", + "protocol": "saml", + "protocolMapper": "saml-user-property-mapper", + "consentRequired": False, + "config": { + "attribute.nameformat": "Basic", + "user.attribute": "username", + "friendly.name": "username", + "attribute.name": "username", + }, + }, + { + "id": "53858403-eba2-4f6d-81d0-cced700b5719", + "name": "X500 givenName", + "protocol": "saml", + "protocolMapper": "saml-user-property-mapper", + "consentRequired": False, + "config": { + "attribute.nameformat": "urn:oasis:names:tc:SAML:2.0:attrname-format:uri", + "user.attribute": "firstName", + "friendly.name": "givenName", + "attribute.name": "urn:oid:2.5.4.42", + }, + }, + { + "id": "20034db5-1d0e-4e66-b815-fb0440c6d1e2", + "name": "X500 email", + "protocol": "saml", + "protocolMapper": "saml-user-property-mapper", + "consentRequired": False, + "config": { + "attribute.nameformat": "urn:oasis:names:tc:SAML:2.0:attrname-format:uri", + "user.attribute": "email", + "friendly.name": "email", + "attribute.name": "urn:oid:1.2.840.113549.1.9.1", + }, + }, + ], + "defaultClientScopes": [ + "web-origins", + "role_list", + "roles", + "profile", + "email", + ], + "optionalClientScopes": [ + "address", + "phone", + "offline_access", + "microprofile-jwt", + ], + "access": {"view": True, "configure": True, "manage": True}, } - keycloak=KeycloakClient() + keycloak = KeycloakClient() keycloak.add_client(client) - keycloak=None + keycloak = None def add_client_roles(self): - keycloak=KeycloakClient() - keycloak.add_client_role('a92d5417-92b6-4678-9cb9-51bc0edcee8c','admin','Moodle admins') - keycloak.add_client_role('a92d5417-92b6-4678-9cb9-51bc0edcee8c','manager','Moodle managers') - keycloak.add_client_role('a92d5417-92b6-4678-9cb9-51bc0edcee8c','teacher','Moodle teachers') - keycloak.add_client_role('a92d5417-92b6-4678-9cb9-51bc0edcee8c','student','Moodle students') - keycloak=None + keycloak = KeycloakClient() + keycloak.add_client_role( + "a92d5417-92b6-4678-9cb9-51bc0edcee8c", "admin", "Moodle admins" + ) + keycloak.add_client_role( + "a92d5417-92b6-4678-9cb9-51bc0edcee8c", "manager", "Moodle managers" + ) + keycloak.add_client_role( + "a92d5417-92b6-4678-9cb9-51bc0edcee8c", "teacher", "Moodle teachers" + ) + keycloak.add_client_role( + "a92d5417-92b6-4678-9cb9-51bc0edcee8c", "student", "Moodle students" + ) + keycloak = None -m=MoodleSaml() + +m = MoodleSaml() diff --git a/admin/src/nextcloud_saml.py b/admin/src/nextcloud_saml.py index ddef1c4..f574172 100644 --- a/admin/src/nextcloud_saml.py +++ b/admin/src/nextcloud_saml.py @@ -1,66 +1,81 @@ #!/usr/bin/env python # coding=utf-8 -import time, os -from datetime import datetime, timedelta -import pprint - +import json import logging as log +import os +import pprint +import random +import string +import time import traceback -import yaml, json +from datetime import datetime, timedelta import psycopg2 +import yaml -from admin.lib.postgres import Postgres from admin.lib.keycloak_client import KeycloakClient +from admin.lib.postgres import Postgres -import string, random +app = {} +app["config"] = {} -app={} -app['config']={} -class NextcloudSaml(): +class NextcloudSaml: def __init__(self): - self.url="http://isard-sso-keycloak:8080/auth/" - self.username=os.environ['KEYCLOAK_USER'] - self.password=os.environ['KEYCLOAK_PASSWORD'] - self.realm='master' - self.verify=True + self.url = "http://isard-sso-keycloak:8080/auth/" + self.username = os.environ["KEYCLOAK_USER"] + self.password = os.environ["KEYCLOAK_PASSWORD"] + self.realm = "master" + self.verify = True - ready=False + ready = False while not ready: try: - self.pg=Postgres('isard-apps-postgresql','nextcloud',os.environ['NEXTCLOUD_POSTGRES_USER'],os.environ['NEXTCLOUD_POSTGRES_PASSWORD']) - ready=True + self.pg = Postgres( + "isard-apps-postgresql", + "nextcloud", + os.environ["NEXTCLOUD_POSTGRES_USER"], + os.environ["NEXTCLOUD_POSTGRES_PASSWORD"], + ) + ready = True except: - log.warning('Could not connect to nextcloud database. Retrying...') + log.warning("Could not connect to nextcloud database. Retrying...") time.sleep(2) - log.info('Connected to nextcloud database.') + log.info("Connected to nextcloud database.") - ready=False + ready = False while not ready: try: - with open(os.path.join("./saml_certs/public.cert"),"r") as crt: - app['config']['PUBLIC_CERT']=crt.read() - ready=True + with open(os.path.join("./saml_certs/public.cert"), "r") as crt: + app["config"]["PUBLIC_CERT"] = crt.read() + ready = True except IOError: - log.warning('Could not get public certificate to be used in nextcloud. Retrying...') - log.warning(' You should generate them: /admin/saml_certs # openssl req -nodes -new -x509 -keyout private.key -out public.cert') + log.warning( + "Could not get public certificate to be used in nextcloud. Retrying..." + ) + log.warning( + " You should generate them: /admin/saml_certs # openssl req -nodes -new -x509 -keyout private.key -out public.cert" + ) time.sleep(2) except: log.error(traceback.format_exc()) - log.info('Got moodle srt certificate to be used in nextcloud.') + log.info("Got moodle srt certificate to be used in nextcloud.") - ready=False + ready = False while not ready: try: - with open(os.path.join("./saml_certs/private.key"),"r") as pem: - app['config']['PRIVATE_KEY']=pem.read() - ready=True + with open(os.path.join("./saml_certs/private.key"), "r") as pem: + app["config"]["PRIVATE_KEY"] = pem.read() + ready = True except IOError: - log.warning('Could not get private key to be used in nextcloud. Retrying...') - log.warning(' You should generate them: /admin/saml_certs # openssl req -nodes -new -x509 -keyout private.key -out public.cert') + log.warning( + "Could not get private key to be used in nextcloud. Retrying..." + ) + log.warning( + " You should generate them: /admin/saml_certs # openssl req -nodes -new -x509 -keyout private.key -out public.cert" + ) time.sleep(2) - log.info('Got moodle pem certificate to be used in nextcloud.') + log.info("Got moodle pem certificate to be used in nextcloud.") # ## This seems related to the fact that the certificate generated the first time does'nt work. # ## And when regenerating the certificate de privatekeypass seems not to be used and instead it @@ -77,30 +92,32 @@ class NextcloudSaml(): try: self.reset_saml() except: - print('Error resetting saml on nextcloud') - + print("Error resetting saml on nextcloud") + try: self.delete_keycloak_nextcloud_saml_plugin() except: - print('Error resetting saml on keycloak') - + print("Error resetting saml on keycloak") + try: self.set_nextcloud_saml_plugin() except: log.error(traceback.format_exc()) - print('Error adding saml on nextcloud') - + print("Error adding saml on nextcloud") + try: self.add_keycloak_nextcloud_saml() except: - print('Error adding saml on keycloak') + print("Error adding saml on keycloak") def connect(self): - self.keycloak= KeycloakClient(url=self.url, - username=self.username, - password=self.password, - realm=self.realm, - verify=self.verify) + self.keycloak = KeycloakClient( + url=self.url, + username=self.username, + password=self.password, + realm=self.realm, + verify=self.verify, + ) # def activate_saml_plugin(self): # ## After you need to purge moodle caches: /var/www/html # php admin/cli/purge_caches.php @@ -111,22 +128,23 @@ class NextcloudSaml(): def parse_idp_cert(self): self.connect() - rsa=self.keycloak.get_server_rsa_key() - self.keycloak=None - return rsa['certificate'] + rsa = self.keycloak.get_server_rsa_key() + self.keycloak = None + return rsa["certificate"] def set_keycloak_nextcloud_saml_plugin(self): self.connect() self.keycloak.add_nextcloud_client() - self.keycloak=None + self.keycloak = None def delete_keycloak_nextcloud_saml_plugin(self): self.connect() - self.keycloak.delete_client('bef873f0-2079-4876-8657-067de27d01b7') - self.keycloak=None + self.keycloak.delete_client("bef873f0-2079-4876-8657-067de27d01b7") + self.keycloak = None def set_nextcloud_saml_plugin(self): - self.pg.update("""INSERT INTO "oc_appconfig" ("appid", "configkey", "configvalue") VALUES + self.pg.update( + """INSERT INTO "oc_appconfig" ("appid", "configkey", "configvalue") VALUES ('user_saml', 'general-uid_mapping', 'username'), ('user_saml', 'type', 'saml'), ('user_saml', 'sp-privateKey', '%s'), @@ -144,155 +162,192 @@ class NextcloudSaml(): ('user_saml', 'security-wantAssertionsSigned', '1'), ('user_saml', 'general-idp0_display_name', 'SAML Login'), ('user_saml', 'sp-x509cert', '%s'), -('user_saml', 'idp-singleLogoutService.url', 'https://sso.%s/auth/realms/master/protocol/saml');""" % (app['config']['PRIVATE_KEY'],os.environ['DOMAIN'],os.environ['DOMAIN'],self.parse_idp_cert(),app['config']['PUBLIC_CERT'],os.environ['DOMAIN'])) - +('user_saml', 'idp-singleLogoutService.url', 'https://sso.%s/auth/realms/master/protocol/saml');""" + % ( + app["config"]["PRIVATE_KEY"], + os.environ["DOMAIN"], + os.environ["DOMAIN"], + self.parse_idp_cert(), + app["config"]["PUBLIC_CERT"], + os.environ["DOMAIN"], + ) + ) def reset_saml(self): - cfg_list=['general-uid_mapping', - 'sp-privateKey', - 'saml-attribute-mapping-email_mapping', - 'saml-attribute-mapping-quota_mapping', - 'saml-attribute-mapping-displayName_mapping', - 'saml-attribute-mapping-group_mapping', - 'idp-entityId', - 'idp-singleSignOnService.url', - 'idp-x509cert', - 'security-authnRequestsSigned', - 'security-logoutRequestSigned', - 'security-logoutResponseSigned', - 'security-wantMessagesSigned', - 'security-wantAssertionsSigned', - 'general-idp0_display_name', - 'type', - 'sp-x509cert', - 'idp-singleLogoutService.url'] + cfg_list = [ + "general-uid_mapping", + "sp-privateKey", + "saml-attribute-mapping-email_mapping", + "saml-attribute-mapping-quota_mapping", + "saml-attribute-mapping-displayName_mapping", + "saml-attribute-mapping-group_mapping", + "idp-entityId", + "idp-singleSignOnService.url", + "idp-x509cert", + "security-authnRequestsSigned", + "security-logoutRequestSigned", + "security-logoutResponseSigned", + "security-wantMessagesSigned", + "security-wantAssertionsSigned", + "general-idp0_display_name", + "type", + "sp-x509cert", + "idp-singleLogoutService.url", + ] for cfg in cfg_list: - self.pg.update("""DELETE FROM "oc_appconfig" WHERE appid = 'user_saml' AND configkey = '%s'""" % (cfg)) + self.pg.update( + """DELETE FROM "oc_appconfig" WHERE appid = 'user_saml' AND configkey = '%s'""" + % (cfg) + ) def add_keycloak_nextcloud_saml(self): - client={"id" : "bef873f0-2079-4876-8657-067de27d01b7", + client = { + "id": "bef873f0-2079-4876-8657-067de27d01b7", "name": "nextcloud", "description": "nextcloud", - "clientId" : "https://nextcloud."+os.environ['DOMAIN']+"/apps/user_saml/saml/metadata", - "surrogateAuthRequired" : False, - "enabled" : True, - "alwaysDisplayInConsole" : False, - "clientAuthenticatorType" : "client-secret", - "redirectUris" : [ "https://nextcloud."+os.environ['DOMAIN']+"/apps/user_saml/saml/acs" ], - "webOrigins" : [ "https://nextcloud."+os.environ['DOMAIN'] ], - "notBefore" : 0, - "bearerOnly" : False, - "consentRequired" : False, - "standardFlowEnabled" : True, - "implicitFlowEnabled" : False, - "directAccessGrantsEnabled" : False, - "serviceAccountsEnabled" : False, - "publicClient" : False, - "frontchannelLogout" : True, - "protocol" : "saml", - "attributes" : { - "saml.assertion.signature" : True, - "saml.force.post.binding" : True, - "saml_assertion_consumer_url_post" : "https://nextcloud."+os.environ['DOMAIN']+"/apps/user_saml/saml/acs", - "saml.server.signature" : True, - "saml.server.signature.keyinfo.ext" : False, - "saml.signing.certificate" : app['config']['PUBLIC_CERT'], - "saml_single_logout_service_url_redirect" : "https://nextcloud."+os.environ['DOMAIN']+"/apps/user_saml/saml/sls", - "saml.signature.algorithm" : "RSA_SHA256", - "saml_force_name_id_format" : False, - "saml.client.signature" : False, - "saml.authnstatement" : True, - "saml_name_id_format" : "username", - "saml_signature_canonicalization_method" : "http://www.w3.org/2001/10/xml-exc-c14n#" + "clientId": "https://nextcloud." + + os.environ["DOMAIN"] + + "/apps/user_saml/saml/metadata", + "surrogateAuthRequired": False, + "enabled": True, + "alwaysDisplayInConsole": False, + "clientAuthenticatorType": "client-secret", + "redirectUris": [ + "https://nextcloud." + os.environ["DOMAIN"] + "/apps/user_saml/saml/acs" + ], + "webOrigins": ["https://nextcloud." + os.environ["DOMAIN"]], + "notBefore": 0, + "bearerOnly": False, + "consentRequired": False, + "standardFlowEnabled": True, + "implicitFlowEnabled": False, + "directAccessGrantsEnabled": False, + "serviceAccountsEnabled": False, + "publicClient": False, + "frontchannelLogout": True, + "protocol": "saml", + "attributes": { + "saml.assertion.signature": True, + "saml.force.post.binding": True, + "saml_assertion_consumer_url_post": "https://nextcloud." + + os.environ["DOMAIN"] + + "/apps/user_saml/saml/acs", + "saml.server.signature": True, + "saml.server.signature.keyinfo.ext": False, + "saml.signing.certificate": app["config"]["PUBLIC_CERT"], + "saml_single_logout_service_url_redirect": "https://nextcloud." + + os.environ["DOMAIN"] + + "/apps/user_saml/saml/sls", + "saml.signature.algorithm": "RSA_SHA256", + "saml_force_name_id_format": False, + "saml.client.signature": False, + "saml.authnstatement": True, + "saml_name_id_format": "username", + "saml_signature_canonicalization_method": "http://www.w3.org/2001/10/xml-exc-c14n#", }, - "authenticationFlowBindingOverrides" : { }, - "fullScopeAllowed" : True, - "nodeReRegistrationTimeout" : -1, - "protocolMappers" : [ { - "id" : "e8e4acff-da2b-46aa-8bdb-ba42171671d6", - "name" : "username", - "protocol" : "saml", - "protocolMapper" : "saml-user-attribute-mapper", - "consentRequired" : False, - "config" : { - "attribute.nameformat" : "Basic", - "user.attribute" : "username", - "friendly.name" : "username", - "attribute.name" : "username" - } - }, { - "id" : "8ab13cd7-822a-40d5-a1e1-9f556aed2332", - "name" : "quota", - "protocol" : "saml", - "protocolMapper" : "saml-user-attribute-mapper", - "consentRequired" : False, - "config" : { - "attribute.nameformat" : "Basic", - "user.attribute" : "quota", - "friendly.name" : "quota", - "attribute.name" : "quota" - } - }, { - "id" : "28206b59-757b-4e3c-81cb-0b6053b1fd3d", - "name" : "email", - "protocol" : "saml", - "protocolMapper" : "saml-user-property-mapper", - "consentRequired" : False, - "config" : { - "attribute.nameformat" : "Basic", - "user.attribute" : "email", - "friendly.name" : "email", - "attribute.name" : "email" - } - }, { - "id" : "5176a593-180f-4924-b294-b83a0d8d5972", - "name" : "displayname", - "protocol" : "saml", - "protocolMapper" : "saml-javascript-mapper", - "consentRequired" : False, - "config" : { - "single" : False, - "Script" : "/**\n * Available variables: \n * user - the current user\n * realm - the current realm\n * clientSession - the current clientSession\n * userSession - the current userSession\n * keycloakSession - the current keycloakSession\n */\n\n\n//insert your code here...\nvar Output = user.getFirstName()+\" \"+user.getLastName();\nOutput;\n", - "attribute.nameformat" : "Basic", - "friendly.name" : "displayname", - "attribute.name" : "displayname" - } - }, { - "id" : "e51e04b9-f71a-42de-819e-dd9285246ada", - "name" : "Roles", - "protocol" : "saml", - "protocolMapper" : "saml-role-list-mapper", - "consentRequired" : False, - "config" : { - "single" : True, - "attribute.nameformat" : "Basic", - "friendly.name" : "Roles", - "attribute.name" : "Roles" - } - }, { - "id" : "9c101249-bb09-4cc8-8f75-5a18fcb307e6", - "name" : "group_list", - "protocol" : "saml", - "protocolMapper" : "saml-group-membership-mapper", - "consentRequired" : False, - "config" : { - "single" : True, - "attribute.nameformat" : "Basic", - "full.path" : False, - "friendly.name" : "member", - "attribute.name" : "member" - } - } ], - "defaultClientScopes" : [ "web-origins", "role_list", "roles", "profile", "email" ], - "optionalClientScopes" : [ "address", "phone", "offline_access", "microprofile-jwt" ], - "access" : { - "view" : True, - "configure" : True, - "manage" : True - } + "authenticationFlowBindingOverrides": {}, + "fullScopeAllowed": True, + "nodeReRegistrationTimeout": -1, + "protocolMappers": [ + { + "id": "e8e4acff-da2b-46aa-8bdb-ba42171671d6", + "name": "username", + "protocol": "saml", + "protocolMapper": "saml-user-attribute-mapper", + "consentRequired": False, + "config": { + "attribute.nameformat": "Basic", + "user.attribute": "username", + "friendly.name": "username", + "attribute.name": "username", + }, + }, + { + "id": "8ab13cd7-822a-40d5-a1e1-9f556aed2332", + "name": "quota", + "protocol": "saml", + "protocolMapper": "saml-user-attribute-mapper", + "consentRequired": False, + "config": { + "attribute.nameformat": "Basic", + "user.attribute": "quota", + "friendly.name": "quota", + "attribute.name": "quota", + }, + }, + { + "id": "28206b59-757b-4e3c-81cb-0b6053b1fd3d", + "name": "email", + "protocol": "saml", + "protocolMapper": "saml-user-property-mapper", + "consentRequired": False, + "config": { + "attribute.nameformat": "Basic", + "user.attribute": "email", + "friendly.name": "email", + "attribute.name": "email", + }, + }, + { + "id": "5176a593-180f-4924-b294-b83a0d8d5972", + "name": "displayname", + "protocol": "saml", + "protocolMapper": "saml-javascript-mapper", + "consentRequired": False, + "config": { + "single": False, + "Script": '/**\n * Available variables: \n * user - the current user\n * realm - the current realm\n * clientSession - the current clientSession\n * userSession - the current userSession\n * keycloakSession - the current keycloakSession\n */\n\n\n//insert your code here...\nvar Output = user.getFirstName()+" "+user.getLastName();\nOutput;\n', + "attribute.nameformat": "Basic", + "friendly.name": "displayname", + "attribute.name": "displayname", + }, + }, + { + "id": "e51e04b9-f71a-42de-819e-dd9285246ada", + "name": "Roles", + "protocol": "saml", + "protocolMapper": "saml-role-list-mapper", + "consentRequired": False, + "config": { + "single": True, + "attribute.nameformat": "Basic", + "friendly.name": "Roles", + "attribute.name": "Roles", + }, + }, + { + "id": "9c101249-bb09-4cc8-8f75-5a18fcb307e6", + "name": "group_list", + "protocol": "saml", + "protocolMapper": "saml-group-membership-mapper", + "consentRequired": False, + "config": { + "single": True, + "attribute.nameformat": "Basic", + "full.path": False, + "friendly.name": "member", + "attribute.name": "member", + }, + }, + ], + "defaultClientScopes": [ + "web-origins", + "role_list", + "roles", + "profile", + "email", + ], + "optionalClientScopes": [ + "address", + "phone", + "offline_access", + "microprofile-jwt", + ], + "access": {"view": True, "configure": True, "manage": True}, } self.connect() self.keycloak.add_client(client) - self.keycloak=None + self.keycloak = None -n=NextcloudSaml() \ No newline at end of file + +n = NextcloudSaml() diff --git a/admin/src/nextcloud_saml_onlycerts.py b/admin/src/nextcloud_saml_onlycerts.py index 5690039..e5c80ed 100644 --- a/admin/src/nextcloud_saml_onlycerts.py +++ b/admin/src/nextcloud_saml_onlycerts.py @@ -1,66 +1,81 @@ #!/usr/bin/env python # coding=utf-8 -import time, os -from datetime import datetime, timedelta -import pprint - +import json import logging as log +import os +import pprint +import random +import string +import time import traceback -import yaml, json +from datetime import datetime, timedelta import psycopg2 +import yaml -from admin.lib.postgres import Postgres from admin.lib.keycloak_client import KeycloakClient +from admin.lib.postgres import Postgres -import string, random +app = {} +app["config"] = {} -app={} -app['config']={} -class NextcloudSaml(): +class NextcloudSaml: def __init__(self): - self.url="http://isard-sso-keycloak:8080/auth/" - self.username=os.environ['KEYCLOAK_USER'] - self.password=os.environ['KEYCLOAK_PASSWORD'] - self.realm='master' - self.verify=True + self.url = "http://isard-sso-keycloak:8080/auth/" + self.username = os.environ["KEYCLOAK_USER"] + self.password = os.environ["KEYCLOAK_PASSWORD"] + self.realm = "master" + self.verify = True - ready=False + ready = False while not ready: try: - self.pg=Postgres('isard-apps-postgresql','nextcloud',os.environ['NEXTCLOUD_POSTGRES_USER'],os.environ['NEXTCLOUD_POSTGRES_PASSWORD']) - ready=True + self.pg = Postgres( + "isard-apps-postgresql", + "nextcloud", + os.environ["NEXTCLOUD_POSTGRES_USER"], + os.environ["NEXTCLOUD_POSTGRES_PASSWORD"], + ) + ready = True except: - log.warning('Could not connect to nextcloud database. Retrying...') + log.warning("Could not connect to nextcloud database. Retrying...") time.sleep(2) - log.info('Connected to nextcloud database.') + log.info("Connected to nextcloud database.") - ready=False + ready = False while not ready: try: - with open(os.path.join("./saml_certs/public.cert"),"r") as crt: - app['config']['PUBLIC_CERT']=crt.read() - ready=True + with open(os.path.join("./saml_certs/public.cert"), "r") as crt: + app["config"]["PUBLIC_CERT"] = crt.read() + ready = True except IOError: - log.warning('Could not get public certificate to be used in nextcloud. Retrying...') - log.warning(' You should generate them: /admin/saml_certs # openssl req -nodes -new -x509 -keyout private.key -out public.cert') + log.warning( + "Could not get public certificate to be used in nextcloud. Retrying..." + ) + log.warning( + " You should generate them: /admin/saml_certs # openssl req -nodes -new -x509 -keyout private.key -out public.cert" + ) time.sleep(2) except: log.error(traceback.format_exc()) - log.info('Got moodle srt certificate to be used in nextcloud.') + log.info("Got moodle srt certificate to be used in nextcloud.") - ready=False + ready = False while not ready: try: - with open(os.path.join("./saml_certs/private.key"),"r") as pem: - app['config']['PRIVATE_KEY']=pem.read() - ready=True + with open(os.path.join("./saml_certs/private.key"), "r") as pem: + app["config"]["PRIVATE_KEY"] = pem.read() + ready = True except IOError: - log.warning('Could not get private key to be used in nextcloud. Retrying...') - log.warning(' You should generate them: /admin/saml_certs # openssl req -nodes -new -x509 -keyout private.key -out public.cert') + log.warning( + "Could not get private key to be used in nextcloud. Retrying..." + ) + log.warning( + " You should generate them: /admin/saml_certs # openssl req -nodes -new -x509 -keyout private.key -out public.cert" + ) time.sleep(2) - log.info('Got moodle pem certificate to be used in nextcloud.') + log.info("Got moodle pem certificate to be used in nextcloud.") # ## This seems related to the fact that the certificate generated the first time does'nt work. # ## And when regenerating the certificate de privatekeypass seems not to be used and instead it @@ -77,20 +92,22 @@ class NextcloudSaml(): try: self.reset_saml_certs() except: - print('Error resetting saml on nextcloud') + print("Error resetting saml on nextcloud") try: self.set_nextcloud_saml_plugin_certs() except: log.error(traceback.format_exc()) - print('Error adding saml on nextcloud') + print("Error adding saml on nextcloud") def connect(self): - self.keycloak= KeycloakClient(url=self.url, - username=self.username, - password=self.password, - realm=self.realm, - verify=self.verify) + self.keycloak = KeycloakClient( + url=self.url, + username=self.username, + password=self.password, + realm=self.realm, + verify=self.verify, + ) # def activate_saml_plugin(self): # ## After you need to purge moodle caches: /var/www/html # php admin/cli/purge_caches.php @@ -101,22 +118,30 @@ class NextcloudSaml(): def parse_idp_cert(self): self.connect() - rsa=self.keycloak.get_server_rsa_key() - self.keycloak=None - return rsa['certificate'] + rsa = self.keycloak.get_server_rsa_key() + self.keycloak = None + return rsa["certificate"] def set_nextcloud_saml_plugin_certs(self): - self.pg.update("""INSERT INTO "oc_appconfig" ("appid", "configkey", "configvalue") VALUES + self.pg.update( + """INSERT INTO "oc_appconfig" ("appid", "configkey", "configvalue") VALUES ('user_saml', 'sp-privateKey', '%s'), ('user_saml', 'idp-x509cert', '%s'), -('user_saml', 'sp-x509cert', '%s');""" % (app['config']['PRIVATE_KEY'],self.parse_idp_cert(),app['config']['PUBLIC_CERT'])) - +('user_saml', 'sp-x509cert', '%s');""" + % ( + app["config"]["PRIVATE_KEY"], + self.parse_idp_cert(), + app["config"]["PUBLIC_CERT"], + ) + ) def reset_saml_certs(self): - cfg_list=['sp-privateKey', - 'idp-x509cert', - 'sp-x509cert'] + cfg_list = ["sp-privateKey", "idp-x509cert", "sp-x509cert"] for cfg in cfg_list: - self.pg.update("""DELETE FROM "oc_appconfig" WHERE appid = 'user_saml' AND configkey = '%s'""" % (cfg)) + self.pg.update( + """DELETE FROM "oc_appconfig" WHERE appid = 'user_saml' AND configkey = '%s'""" + % (cfg) + ) -n=NextcloudSaml() \ No newline at end of file + +n = NextcloudSaml() diff --git a/admin/src/postgres.py b/admin/src/postgres.py index ee6adfd..2749fe1 100644 --- a/admin/src/postgres.py +++ b/admin/src/postgres.py @@ -1,50 +1,47 @@ #!/usr/bin/env python # coding=utf-8 -import time -from datetime import datetime, timedelta -import pprint - +import json import logging as log +import pprint +import time import traceback -import yaml, json +from datetime import datetime, timedelta import psycopg2 +import yaml -class Postgres(): - def __init__(self,host,database,user,password): +class Postgres: + def __init__(self, host, database, user, password): self.conn = psycopg2.connect( - host=host, - database=database, - user=user, - password=password) - + host=host, database=database, user=user, password=password + ) # def __del__(self): # self.cur.close() # self.conn.close() - def select(self,sql): + def select(self, sql): self.cur = self.conn.cursor() self.cur.execute(sql) - data=self.cur.fetchall() + data = self.cur.fetchall() self.cur.close() return data - def update(self,sql): + def update(self, sql): self.cur = self.conn.cursor() self.cur.execute(sql) self.conn.commit() self.cur.close() # return self.cur.fetchall() - def select_with_headers(self,sql): + def select_with_headers(self, sql): self.cur = self.conn.cursor() self.cur.execute(sql) - data=self.cur.fetchall() + data = self.cur.fetchall() fields = [a.name for a in self.cur.description] self.cur.close() - return (fields,data) + return (fields, data) # def update_moodle_saml_plugin(self): # plugin[('idpmetadata', 'NrtA5ynG0htowP3SXw7dBJRIAMxn-1PwuuXwOwNhlRwMIICmzCCAYMCBgF5jb0RCTANBgkqhkiG9w0BAQsFADARMQ8wDQYDVQQDDAZtYXN0ZXIwHhcNMjEwNTIxMDcwMjI4WhcNMzEwNTIxMDcwNDA4WjARMQ8wDQYDVQQDDAZtYXN0ZXIwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQCI8xh/C0+frz3kgWiUbziTDls71R2YiXLSVE+bw7gbEgZUGCLhoEI679azMtIxmnzM/snIX+yTb12+XoYkgbiLTMPQfnH+Kiab6g3HL3KPfhqS+yWkFxOoCp6Ibmp7yPlVWuHH+MBfO8OBr/r8Ao7heFbuzjiLd1KG67rcoaxfDgMuBoEomg1bgEjFgHaQIrSC6OZzH0h987/arqufZXeXlfyiqScMPUi+u5IpDWSwz06UKP0k8mxzNSlpZ93CKOUSsV0SMLxqg7FQ3SGiOk577bGW9o9BDTkkmSo3Up6smc0LzwvvUwuNd0B1irGkWZFQN9OXJnJYf1InEebIMtmPAgMBAAEwDQYJKoZIhvcNAQELBQADggEBADM34+qEGeBQ22luphVTuVJtGxcbxLx7DfsT0QfJD/OuxTTbNAa1VRyarb5juIAkqdj4y2quZna9ZXLecVo4RkwpzPoKoAkYA8b+kHnWqEwJi9iPrDvKb+GR0bBkLPN49YxIZ8IdKX/PRa3yuLHe+loiNsCaS/2ZK2KO46COsqU4QX1iVhF9kWphNLybjNAX45B6cJLsa1g0vXLdm3kv3SB4I2fErFVaOoDtFIjttoYlXdpUiThkPXBfr7N67P3dZHaS4tjJh+IZ8I6TINpcsH8dBkUhzYEIPHCePwSiC1w6WDBLNDuKt1mj1CZrLq+1x+Yhrs+QNRheEKGi89HZ8N0=urn:oasis:names:tc:SAML:2.0:nameid-format:persistenturn:oasis:names:tc:SAML:2.0:nameid-format:transienturn:oasis:names:tc:SAML:1.1:nameid-format:unspecifiedurn:oasis:names:tc:SAML:1.1:nameid-format:emailAddress')] @@ -52,4 +49,4 @@ class Postgres(): # cursor.execute(pg_update, (title, bookid)) # connection.commit() # count = cursor.rowcount - # print(count, "Successfully Updated!") \ No newline at end of file + # print(count, "Successfully Updated!") diff --git a/admin/src/scripts/avatars.py b/admin/src/scripts/avatars.py index 6b7f300..0d210f9 100644 --- a/admin/src/scripts/avatars.py +++ b/admin/src/scripts/avatars.py @@ -1,64 +1,82 @@ #!/usr/bin/env python -import time ,os -from datetime import datetime, timedelta - +import json import logging as log +import os +import time import traceback -import yaml, json +from datetime import datetime, timedelta from pprint import pprint +import yaml from keycloak import KeycloakAdmin -from postgres import Postgres - from minio import Minio from minio.commonconfig import REPLACE, CopySource from minio.deleteobjects import DeleteObject -class DefaultAvatars(): - def __init__(self, - url="http://isard-sso-keycloak:8080/auth/", - username=os.environ['KEYCLOAK_USER'], - password=os.environ['KEYCLOAK_PASSWORD'], - realm='master', - verify=True): - self.url=url - self.username=username - self.password=password - self.realm=realm - self.verify=verify +from postgres import Postgres - self.keycloak_pg=Postgres('isard-apps-postgresql','keycloak',os.environ['KEYCLOAK_DB_USER'],os.environ['KEYCLOAK_DB_PASSWORD']) + +class DefaultAvatars: + def __init__( + self, + url="http://isard-sso-keycloak:8080/auth/", + username=os.environ["KEYCLOAK_USER"], + password=os.environ["KEYCLOAK_PASSWORD"], + realm="master", + verify=True, + ): + self.url = url + self.username = username + self.password = password + self.realm = realm + self.verify = verify + + self.keycloak_pg = Postgres( + "isard-apps-postgresql", + "keycloak", + os.environ["KEYCLOAK_DB_USER"], + os.environ["KEYCLOAK_DB_PASSWORD"], + ) self.mclient = Minio( - "isard-sso-avatars:9000", - access_key="AKIAIOSFODNN7EXAMPLE", - secret_key="wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY", - secure=False - ) - self.bucket='master-avatars' + "isard-sso-avatars:9000", + access_key="AKIAIOSFODNN7EXAMPLE", + secret_key="wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY", + secure=False, + ) + self.bucket = "master-avatars" self._minio_set_realm() self.update_missing_avatars() def connect(self): - self.keycloak_admin = KeycloakAdmin(server_url=self.url, - username=self.username, - password=self.password, - realm_name=self.realm, - verify=self.verify) + self.keycloak_admin = KeycloakAdmin( + server_url=self.url, + username=self.username, + password=self.password, + realm_name=self.realm, + verify=self.verify, + ) def update_missing_avatars(self): - sys_roles=['admin','manager','teacher','student'] + sys_roles = ["admin", "manager", "teacher", "student"] for u in self.get_users_without_image(): try: - img=[r+'.jpg' for r in sys_roles if r in u['role']][0] + img = [r + ".jpg" for r in sys_roles if r in u["role"]][0] except: - img='unknown.jpg' + img = "unknown.jpg" self.mclient.fput_object( - self.bucket, u['id'], "custom/avatars/"+img, - content_type="image/jpeg ", + self.bucket, + u["id"], + "custom/avatars/" + img, + content_type="image/jpeg ", + ) + log.warning( + " AVATARS: Updated avatar for user " + + u["username"] + + " with role " + + img.split(".")[0] ) - log.warning(' AVATARS: Updated avatar for user '+u['username']+' with role '+img.split('.')[0]) def _minio_set_realm(self): if not self.mclient.bucket_exists(self.bucket): @@ -72,17 +90,17 @@ class DefaultAvatars(): lambda x: DeleteObject(x.object_name), self.mclient.list_objects(self.bucket), ) - errors=self.mclient.remove_objects(self.bucket, delete_object_list) + errors = self.mclient.remove_objects(self.bucket, delete_object_list) for error in errors: log.error(" AVATARS: Error occured when deleting avatar object", error) def get_users(self): self.connect() - users=self.get_users_with_groups_and_roles() + users = self.get_users_with_groups_and_roles() return users def get_users_without_image(self): - return [u for u in self.get_users() if u['id'] not in self.minio_get_objects()] + return [u for u in self.get_users() if u["id"] not in self.minio_get_objects()] def get_users_with_groups_and_roles(self): q = """select u.id, u.username, u.email, u.first_name, u.last_name, u.realm_id, ua.value as quota @@ -99,20 +117,28 @@ class DefaultAvatars(): group by u.id,u.username,u.email,u.first_name,u.last_name, u.realm_id, ua.value order by u.username""" - (headers,users)=self.keycloak_pg.select_with_headers(q) + (headers, users) = self.keycloak_pg.select_with_headers(q) - users_with_lists = [list(l[:-4])+([[]] if l[-4] == [None] else [list(set(l[-4]))]) +\ - ([[]] if l[-3] == [None] else [list(set(l[-3]))]) +\ - ([[]] if l[-3] == [None] else [list(set(l[-2]))]) +\ - ([[]] if l[-1] == [None] else [list(set(l[-1]))]) for l in users] - - users_with_lists = [list(l[:-4])+([[]] if l[-4] == [None] else [list(set(l[-4]))]) +\ - ([[]] if l[-3] == [None] else [list(set(l[-3]))]) +\ - ([[]] if l[-3] == [None] else [list(set(l[-2]))]) +\ - ([[]] if l[-1] == [None] else [list(set(l[-1]))]) for l in users_with_lists] + users_with_lists = [ + list(l[:-4]) + + ([[]] if l[-4] == [None] else [list(set(l[-4]))]) + + ([[]] if l[-3] == [None] else [list(set(l[-3]))]) + + ([[]] if l[-3] == [None] else [list(set(l[-2]))]) + + ([[]] if l[-1] == [None] else [list(set(l[-1]))]) + for l in users + ] + + users_with_lists = [ + list(l[:-4]) + + ([[]] if l[-4] == [None] else [list(set(l[-4]))]) + + ([[]] if l[-3] == [None] else [list(set(l[-3]))]) + + ([[]] if l[-3] == [None] else [list(set(l[-2]))]) + + ([[]] if l[-1] == [None] else [list(set(l[-1]))]) + for l in users_with_lists + ] list_dict_users = [dict(zip(headers, r)) for r in users_with_lists] return list_dict_users -da=DefaultAvatars() +da = DefaultAvatars() diff --git a/admin/src/scripts/reset_pwds.py b/admin/src/scripts/reset_pwds.py index cbc502a..04664ba 100644 --- a/admin/src/scripts/reset_pwds.py +++ b/admin/src/scripts/reset_pwds.py @@ -1,77 +1,97 @@ #!/usr/bin/env python -import time ,os -from datetime import datetime, timedelta - +import json import logging as log +import os +import time import traceback -import yaml, json +from datetime import datetime, timedelta from pprint import pprint +import diceware +import yaml from jinja2 import Environment, FileSystemLoader - from keycloak import KeycloakAdmin + from postgres import Postgres -import diceware options = diceware.handle_options(None) -options.wordlist = 'cat_ascii' +options.wordlist = "cat_ascii" options.num = 3 -class KeycloakClient(): - """https://www.keycloak.org/docs-api/13.0/rest-api/index.html - https://github.com/marcospereirampj/python-keycloak - https://gist.github.com/kaqfa/99829941121188d7cef8271f93f52f1f - """ - def __init__(self, - url="http://isard-sso-keycloak:8080/auth/", - username=os.environ['KEYCLOAK_USER'], - password=os.environ['KEYCLOAK_PASSWORD'], - realm='master', - verify=True): - self.url=url - self.username=username - self.password=password - self.realm=realm - self.verify=verify - self.keycloak_pg=Postgres('isard-apps-postgresql','keycloak',os.environ['KEYCLOAK_DB_USER'],os.environ['KEYCLOAK_DB_PASSWORD']) +class KeycloakClient: + """https://www.keycloak.org/docs-api/13.0/rest-api/index.html + https://github.com/marcospereirampj/python-keycloak + https://gist.github.com/kaqfa/99829941121188d7cef8271f93f52f1f + """ + + def __init__( + self, + url="http://isard-sso-keycloak:8080/auth/", + username=os.environ["KEYCLOAK_USER"], + password=os.environ["KEYCLOAK_PASSWORD"], + realm="master", + verify=True, + ): + self.url = url + self.username = username + self.password = password + self.realm = realm + self.verify = verify + + self.keycloak_pg = Postgres( + "isard-apps-postgresql", + "keycloak", + os.environ["KEYCLOAK_DB_USER"], + os.environ["KEYCLOAK_DB_PASSWORD"], + ) def connect(self): - self.keycloak_admin = KeycloakAdmin(server_url=self.url, - username=self.username, - password=self.password, - realm_name=self.realm, - verify=self.verify) - + self.keycloak_admin = KeycloakAdmin( + server_url=self.url, + username=self.username, + password=self.password, + realm_name=self.realm, + verify=self.verify, + ) def update_pwds(self): self.get_users() def get_users(self): self.connect() - users=self.get_users_with_groups_and_roles() - userupdate=[] + users = self.get_users_with_groups_and_roles() + userupdate = [] for u in users: - if u['username'] not in ['admin','ddadmin'] and not u['username'].startswith('system_'): - print('Generating password for user '+u['username']) - userupdate.append({'id':u['id'], - 'username':u['username'], - 'password': diceware.get_passphrase(options=options)}) - with open("user_temp_passwd.csv","w") as csv: + if u["username"] not in ["admin", "ddadmin"] and not u[ + "username" + ].startswith("system_"): + print("Generating password for user " + u["username"]) + userupdate.append( + { + "id": u["id"], + "username": u["username"], + "password": diceware.get_passphrase(options=options), + } + ) + with open("user_temp_passwd.csv", "w") as csv: for user in userupdate: - csv.write("%s,%s,%s\n"%(user['id'],user['username'],user['password'])) + csv.write( + "%s,%s,%s\n" % (user["id"], user["username"], user["password"]) + ) for u in userupdate: - print('Updating keycloak password for user '+u['username']) - self.update_user_pwd(u['id'],u['password']) + print("Updating keycloak password for user " + u["username"]) + self.update_user_pwd(u["id"], u["password"]) - def update_user_pwd(self,user_id,password,temporary=True): - payload={"credentials":[{"type":"password", - "value":password, - "temporary":temporary}]} + def update_user_pwd(self, user_id, password, temporary=True): + payload = { + "credentials": [ + {"type": "password", "value": password, "temporary": temporary} + ] + } self.connect() - self.keycloak_admin.update_user( user_id, payload) - + self.keycloak_admin.update_user(user_id, payload) def get_users_with_groups_and_roles(self): q = """select u.id, u.username, u.email, u.first_name, u.last_name, u.realm_id, ua.value as quota @@ -88,7 +108,7 @@ class KeycloakClient(): group by u.id,u.username,u.email,u.first_name,u.last_name, u.realm_id, ua.value order by u.username""" - # q = """select u.id, u.username, u.email, u.first_name, u.last_name, u.realm_id, ua.value as quota, g.id, g.path, g.name, + # q = """select u.id, u.username, u.email, u.first_name, u.last_name, u.realm_id, ua.value as quota, g.id, g.path, g.name, # --,json_agg(g."name") as group, json_agg(g_parent."name") as group_parent1, json_agg(g_parent2."name") as group_parent2 # --,json_agg(r.name) as role # from user_entity as u @@ -96,7 +116,7 @@ class KeycloakClient(): # left join user_group_membership as ugm on ugm.user_id = u.id # left join keycloak_group as g on g.id = ugm.group_id # --left join keycloak_group as g_parent on g.parent_group = g_parent.id - # --left join keycloak_group as g_parent2 on g_parent.parent_group = g_parent2.id + # --left join keycloak_group as g_parent2 on g_parent.parent_group = g_parent2.id # left join user_role_mapping as rm on rm.user_id = u.id # left join keycloak_role as r on r.id = rm.role_id # --group by u.id,u.username,u.email,u.first_name,u.last_name, u.realm_id, ua.value @@ -111,25 +131,34 @@ class KeycloakClient(): # left join user_group_membership as ugm on ugm.user_id = u.id # left join keycloak_group as g on g.id = ugm.group_id # left join keycloak_group as g_parent on g.parent_group = g_parent.id - # left join keycloak_group as g_parent2 on g_parent.parent_group = g_parent2.id + # left join keycloak_group as g_parent2 on g_parent.parent_group = g_parent2.id # left join user_role_mapping as rm on rm.user_id = u.id # left join keycloak_role as r on r.id = rm.role_id # group by u.id,u.username,u.email,u.first_name,u.last_name, u.realm_id, ua.value # order by u.username""" - (headers,users)=self.keycloak_pg.select_with_headers(q) + (headers, users) = self.keycloak_pg.select_with_headers(q) - users_with_lists = [list(l[:-4])+([[]] if l[-4] == [None] else [list(set(l[-4]))]) +\ - ([[]] if l[-3] == [None] else [list(set(l[-3]))]) +\ - ([[]] if l[-3] == [None] else [list(set(l[-2]))]) +\ - ([[]] if l[-1] == [None] else [list(set(l[-1]))]) for l in users] - - users_with_lists = [list(l[:-4])+([[]] if l[-4] == [None] else [list(set(l[-4]))]) +\ - ([[]] if l[-3] == [None] else [list(set(l[-3]))]) +\ - ([[]] if l[-3] == [None] else [list(set(l[-2]))]) +\ - ([[]] if l[-1] == [None] else [list(set(l[-1]))]) for l in users_with_lists] + users_with_lists = [ + list(l[:-4]) + + ([[]] if l[-4] == [None] else [list(set(l[-4]))]) + + ([[]] if l[-3] == [None] else [list(set(l[-3]))]) + + ([[]] if l[-3] == [None] else [list(set(l[-2]))]) + + ([[]] if l[-1] == [None] else [list(set(l[-1]))]) + for l in users + ] + + users_with_lists = [ + list(l[:-4]) + + ([[]] if l[-4] == [None] else [list(set(l[-4]))]) + + ([[]] if l[-3] == [None] else [list(set(l[-3]))]) + + ([[]] if l[-3] == [None] else [list(set(l[-2]))]) + + ([[]] if l[-1] == [None] else [list(set(l[-1]))]) + for l in users_with_lists + ] list_dict_users = [dict(zip(headers, r)) for r in users_with_lists] return list_dict_users -k=KeycloakClient() + +k = KeycloakClient() k.update_pwds() diff --git a/admin/src/scripts/temporary_no.py b/admin/src/scripts/temporary_no.py index 8c38f8c..b851037 100644 --- a/admin/src/scripts/temporary_no.py +++ b/admin/src/scripts/temporary_no.py @@ -1,70 +1,88 @@ #!/usr/bin/env python -import time ,os -from datetime import datetime, timedelta - +import json import logging as log +import os +import time import traceback -import yaml, json +from datetime import datetime, timedelta from pprint import pprint +import yaml from jinja2 import Environment, FileSystemLoader - from keycloak import KeycloakAdmin + from postgres import Postgres -class KeycloakClient(): +class KeycloakClient: """https://www.keycloak.org/docs-api/13.0/rest-api/index.html - https://github.com/marcospereirampj/python-keycloak - https://gist.github.com/kaqfa/99829941121188d7cef8271f93f52f1f + https://github.com/marcospereirampj/python-keycloak + https://gist.github.com/kaqfa/99829941121188d7cef8271f93f52f1f """ - def __init__(self, - url="http://isard-sso-keycloak:8080/auth/", - username=os.environ['KEYCLOAK_USER'], - password=os.environ['KEYCLOAK_PASSWORD'], - realm='master', - verify=True): - self.url=url - self.username=username - self.password=password - self.realm=realm - self.verify=verify - self.keycloak_pg=Postgres('isard-apps-postgresql','keycloak',os.environ['KEYCLOAK_DB_USER'],os.environ['KEYCLOAK_DB_PASSWORD']) + def __init__( + self, + url="http://isard-sso-keycloak:8080/auth/", + username=os.environ["KEYCLOAK_USER"], + password=os.environ["KEYCLOAK_PASSWORD"], + realm="master", + verify=True, + ): + self.url = url + self.username = username + self.password = password + self.realm = realm + self.verify = verify + + self.keycloak_pg = Postgres( + "isard-apps-postgresql", + "keycloak", + os.environ["KEYCLOAK_DB_USER"], + os.environ["KEYCLOAK_DB_PASSWORD"], + ) def connect(self): - self.keycloak_admin = KeycloakAdmin(server_url=self.url, - username=self.username, - password=self.password, - realm_name=self.realm, - verify=self.verify) - + self.keycloak_admin = KeycloakAdmin( + server_url=self.url, + username=self.username, + password=self.password, + realm_name=self.realm, + verify=self.verify, + ) def update_pwds(self): self.get_users() def get_users(self): self.connect() - users=self.get_users_with_groups_and_roles() - userupdate=[] + users = self.get_users_with_groups_and_roles() + userupdate = [] for u in users: - if u['username'] not in ['admin','ddadmin'] and not u['username'].startswith('system_'): - print('Generating password for user '+u['username']) - userupdate.append({'id':u['id'], - 'username':u['username'], - 'password': diceware.get_passphrase(options=options)}) - with open("user_temp_passwd.csv","w") as csv: + if u["username"] not in ["admin", "ddadmin"] and not u[ + "username" + ].startswith("system_"): + print("Generating password for user " + u["username"]) + userupdate.append( + { + "id": u["id"], + "username": u["username"], + "password": diceware.get_passphrase(options=options), + } + ) + with open("user_temp_passwd.csv", "w") as csv: for user in userupdate: - csv.write("%s,%s,%s\n"%(user['id'],user['username'],user['password'])) + csv.write( + "%s,%s,%s\n" % (user["id"], user["username"], user["password"]) + ) for u in userupdate: - print('Updating keycloak password for user '+u['username']) - self.update_user_pwd(u['id'],u['password']) + print("Updating keycloak password for user " + u["username"]) + self.update_user_pwd(u["id"], u["password"]) - def update_user_pwd_temporary(self,temporary=False): - payload={"credentials":[{"temporary":temporary}]} + def update_user_pwd_temporary(self, temporary=False): + payload = {"credentials": [{"temporary": temporary}]} self.connect() - self.keycloak_admin.update_user( user_id, payload) + self.keycloak_admin.update_user(user_id, payload) def get_users_with_groups_and_roles(self): q = """select u.id, u.username, u.email, u.first_name, u.last_name, u.realm_id, ua.value as quota @@ -81,7 +99,7 @@ class KeycloakClient(): group by u.id,u.username,u.email,u.first_name,u.last_name, u.realm_id, ua.value order by u.username""" - # q = """select u.id, u.username, u.email, u.first_name, u.last_name, u.realm_id, ua.value as quota, g.id, g.path, g.name, + # q = """select u.id, u.username, u.email, u.first_name, u.last_name, u.realm_id, ua.value as quota, g.id, g.path, g.name, # --,json_agg(g."name") as group, json_agg(g_parent."name") as group_parent1, json_agg(g_parent2."name") as group_parent2 # --,json_agg(r.name) as role # from user_entity as u @@ -89,7 +107,7 @@ class KeycloakClient(): # left join user_group_membership as ugm on ugm.user_id = u.id # left join keycloak_group as g on g.id = ugm.group_id # --left join keycloak_group as g_parent on g.parent_group = g_parent.id - # --left join keycloak_group as g_parent2 on g_parent.parent_group = g_parent2.id + # --left join keycloak_group as g_parent2 on g_parent.parent_group = g_parent2.id # left join user_role_mapping as rm on rm.user_id = u.id # left join keycloak_role as r on r.id = rm.role_id # --group by u.id,u.username,u.email,u.first_name,u.last_name, u.realm_id, ua.value @@ -104,25 +122,34 @@ class KeycloakClient(): # left join user_group_membership as ugm on ugm.user_id = u.id # left join keycloak_group as g on g.id = ugm.group_id # left join keycloak_group as g_parent on g.parent_group = g_parent.id - # left join keycloak_group as g_parent2 on g_parent.parent_group = g_parent2.id + # left join keycloak_group as g_parent2 on g_parent.parent_group = g_parent2.id # left join user_role_mapping as rm on rm.user_id = u.id # left join keycloak_role as r on r.id = rm.role_id # group by u.id,u.username,u.email,u.first_name,u.last_name, u.realm_id, ua.value # order by u.username""" - (headers,users)=self.keycloak_pg.select_with_headers(q) + (headers, users) = self.keycloak_pg.select_with_headers(q) - users_with_lists = [list(l[:-4])+([[]] if l[-4] == [None] else [list(set(l[-4]))]) +\ - ([[]] if l[-3] == [None] else [list(set(l[-3]))]) +\ - ([[]] if l[-3] == [None] else [list(set(l[-2]))]) +\ - ([[]] if l[-1] == [None] else [list(set(l[-1]))]) for l in users] - - users_with_lists = [list(l[:-4])+([[]] if l[-4] == [None] else [list(set(l[-4]))]) +\ - ([[]] if l[-3] == [None] else [list(set(l[-3]))]) +\ - ([[]] if l[-3] == [None] else [list(set(l[-2]))]) +\ - ([[]] if l[-1] == [None] else [list(set(l[-1]))]) for l in users_with_lists] + users_with_lists = [ + list(l[:-4]) + + ([[]] if l[-4] == [None] else [list(set(l[-4]))]) + + ([[]] if l[-3] == [None] else [list(set(l[-3]))]) + + ([[]] if l[-3] == [None] else [list(set(l[-2]))]) + + ([[]] if l[-1] == [None] else [list(set(l[-1]))]) + for l in users + ] + + users_with_lists = [ + list(l[:-4]) + + ([[]] if l[-4] == [None] else [list(set(l[-4]))]) + + ([[]] if l[-3] == [None] else [list(set(l[-3]))]) + + ([[]] if l[-3] == [None] else [list(set(l[-2]))]) + + ([[]] if l[-1] == [None] else [list(set(l[-1]))]) + for l in users_with_lists + ] list_dict_users = [dict(zip(headers, r)) for r in users_with_lists] return list_dict_users -k=KeycloakClient() + +k = KeycloakClient() k.update_user_pwd_temporary() diff --git a/admin/src/scripts/temporary_no.py.mod b/admin/src/scripts/temporary_no.py.mod index 7aecb49..e2fe662 100644 --- a/admin/src/scripts/temporary_no.py.mod +++ b/admin/src/scripts/temporary_no.py.mod @@ -1,15 +1,16 @@ #!/usr/bin/env python -import time ,os -from datetime import datetime, timedelta - +import json import logging as log +import os +import time import traceback -import yaml, json +from datetime import datetime, timedelta from pprint import pprint +import yaml from jinja2 import Environment, FileSystemLoader - from keycloak import KeycloakAdmin + from postgres import Postgres diff --git a/admin/src/start.py b/admin/src/start.py index cc79a8b..c7ea01c 100644 --- a/admin/src/start.py +++ b/admin/src/start.py @@ -1,28 +1,46 @@ #!flask/bin/python # coding=utf-8 from gevent import monkey + monkey.patch_all() -from flask_socketio import SocketIO, emit, join_room, leave_room, \ - close_room, rooms, disconnect, send import json + +from flask_socketio import ( + SocketIO, + close_room, + disconnect, + emit, + join_room, + leave_room, + rooms, + send, +) + from admin import app app.socketio = SocketIO(app) -@app.socketio.on('connect', namespace='/sio') + +@app.socketio.on("connect", namespace="/sio") def socketio_connect(): - join_room('admin') - app.socketio.emit('update', - json.dumps('Joined'), - namespace='/sio', - room='admin') - -@app.socketio.on('disconnect', namespace='/sio') + join_room("admin") + app.socketio.emit("update", json.dumps("Joined"), namespace="/sio", room="admin") + + +@app.socketio.on("disconnect", namespace="/sio") def socketio_disconnect(): None -if __name__ == '__main__': - app.socketio.run(app,host='0.0.0.0', port=9000, debug=False, ssl_context='adhoc', async_mode="threading") #, logger=logger, engineio_logger=engineio_logger) + +if __name__ == "__main__": + app.socketio.run( + app, + host="0.0.0.0", + port=9000, + debug=False, + ssl_context="adhoc", + async_mode="threading", + ) # , logger=logger, engineio_logger=engineio_logger) # , cors_allowed_origins="*" -# /usr/lib/python3.8/site-packages/certifi \ No newline at end of file +# /usr/lib/python3.8/site-packages/certifi diff --git a/admin/src/wordpress_saml.py b/admin/src/wordpress_saml.py index b00eb38..f150068 100644 --- a/admin/src/wordpress_saml.py +++ b/admin/src/wordpress_saml.py @@ -1,67 +1,84 @@ #!/usr/bin/env python # coding=utf-8 -import time, os -from datetime import datetime, timedelta -import pprint - +import json import logging as log +import os +import pprint +import random +import string +import time import traceback -import yaml, json +from datetime import datetime, timedelta import psycopg2 +import yaml -from admin.lib.mysql import Mysql from admin.lib.keycloak_client import KeycloakClient +from admin.lib.mysql import Mysql -import string, random +app = {} +app["config"] = {} -app={} -app['config']={} -class WordpressSaml(): +class WordpressSaml: def __init__(self): - self.url="http://isard-sso-keycloak:8080/auth/" - self.username=os.environ['KEYCLOAK_USER'] - self.password=os.environ['KEYCLOAK_PASSWORD'] - self.realm='master' - self.verify=True + self.url = "http://isard-sso-keycloak:8080/auth/" + self.username = os.environ["KEYCLOAK_USER"] + self.password = os.environ["KEYCLOAK_PASSWORD"] + self.realm = "master" + self.verify = True - ready=False + ready = False while not ready: try: - self.db=Mysql('isard-apps-mariadb','wordpress','root',os.environ['MARIADB_PASSWORD']) - ready=True + self.db = Mysql( + "isard-apps-mariadb", + "wordpress", + "root", + os.environ["MARIADB_PASSWORD"], + ) + ready = True except: - log.warning('Could not connect to wordpress database. Retrying...') + log.warning("Could not connect to wordpress database. Retrying...") time.sleep(2) - log.info('Connected to wordpress database.') + log.info("Connected to wordpress database.") - ready=False + ready = False while not ready: try: - with open(os.path.join("./saml_certs/public.cert"),"r") as crt: - app['config']['PUBLIC_CERT_RAW']=crt.read() - app['config']['PUBLIC_CERT']=self.cert_prepare(app['config']['PUBLIC_CERT_RAW']) - ready=True + with open(os.path.join("./saml_certs/public.cert"), "r") as crt: + app["config"]["PUBLIC_CERT_RAW"] = crt.read() + app["config"]["PUBLIC_CERT"] = self.cert_prepare( + app["config"]["PUBLIC_CERT_RAW"] + ) + ready = True except IOError: - log.warning('Could not get public certificate to be used in wordpress. Retrying...') - log.warning(' You should generate them: /admin/saml_certs # openssl req -nodes -new -x509 -keyout private.key -out public.cert') + log.warning( + "Could not get public certificate to be used in wordpress. Retrying..." + ) + log.warning( + " You should generate them: /admin/saml_certs # openssl req -nodes -new -x509 -keyout private.key -out public.cert" + ) time.sleep(2) except: log.error(traceback.format_exc()) - log.info('Got moodle srt certificate to be used in wordpress.') + log.info("Got moodle srt certificate to be used in wordpress.") - ready=False + ready = False while not ready: try: - with open(os.path.join("./saml_certs/private.key"),"r") as pem: - app['config']['PRIVATE_KEY']=self.cert_prepare(pem.read()) - ready=True + with open(os.path.join("./saml_certs/private.key"), "r") as pem: + app["config"]["PRIVATE_KEY"] = self.cert_prepare(pem.read()) + ready = True except IOError: - log.warning('Could not get private key to be used in wordpress. Retrying...') - log.warning(' You should generate them: /admin/saml_certs # openssl req -nodes -new -x509 -keyout private.key -out public.cert') + log.warning( + "Could not get private key to be used in wordpress. Retrying..." + ) + log.warning( + " You should generate them: /admin/saml_certs # openssl req -nodes -new -x509 -keyout private.key -out public.cert" + ) time.sleep(2) - log.info('Got moodle pem certificate to be used in wordpress.') + log.info("Got moodle pem certificate to be used in wordpress.") # ## This seems related to the fact that the certificate generated the first time does'nt work. # ## And when regenerating the certificate de privatekeypass seems not to be used and instead it @@ -79,32 +96,34 @@ class WordpressSaml(): self.reset_saml() except: print(traceback.format_exc()) - print('Error resetting saml on wordpress') + print("Error resetting saml on wordpress") try: self.delete_keycloak_wordpress_saml_plugin() except: - print('Error resetting saml on keycloak') - + print("Error resetting saml on keycloak") + try: self.set_wordpress_saml_plugin() except: print(traceback.format_exc()) - print('Error adding saml on wordpress') - + print("Error adding saml on wordpress") + try: self.add_keycloak_wordpress_saml() except: - print('Error adding saml on keycloak') + print("Error adding saml on keycloak") # SAML clients don't work well with composite roles so disabling and adding on realm # self.add_client_roles() def connect(self): - self.keycloak= KeycloakClient(url=self.url, - username=self.username, - password=self.password, - realm=self.realm, - verify=self.verify) + self.keycloak = KeycloakClient( + url=self.url, + username=self.username, + password=self.password, + realm=self.realm, + verify=self.verify, + ) # def activate_saml_plugin(self): # ## After you need to purge moodle caches: /var/www/html # php admin/cli/purge_caches.php @@ -113,28 +132,29 @@ class WordpressSaml(): # def get_privatekey_pass(self): # return self.db.select("""SELECT * FROM "mdl_config" WHERE "name" = 'siteidentifier'""")[0][2] - def cert_prepare(self,cert): - return ''.join(cert.split('-----')[2].splitlines()) + def cert_prepare(self, cert): + return "".join(cert.split("-----")[2].splitlines()) def parse_idp_cert(self): self.connect() - rsa=self.keycloak.get_server_rsa_key() - self.keycloak=None - return rsa['certificate'] + rsa = self.keycloak.get_server_rsa_key() + self.keycloak = None + return rsa["certificate"] def set_keycloak_wordpress_saml_plugin(self): self.connect() self.keycloak.add_wordpress_client() - self.keycloak=None + self.keycloak = None def delete_keycloak_wordpress_saml_plugin(self): self.connect() - self.keycloak.delete_client('630601f8-25d1-4822-8741-c93affd2cd84') - self.keycloak=None + self.keycloak.delete_client("630601f8-25d1-4822-8741-c93affd2cd84") + self.keycloak = None def set_wordpress_saml_plugin(self): # ('active_plugins', 'a:2:{i:0;s:33:\"edwiser-bridge/edwiser-bridge.php\";i:1;s:17:\"onelogin_saml.php\";}', 'yes'), - self.db.update("""INSERT INTO wp_options (option_name, option_value, autoload) VALUES + self.db.update( + """INSERT INTO wp_options (option_name, option_value, autoload) VALUES ('onelogin_saml_enabled', 'on', 'yes'), ('onelogin_saml_idp_entityid', 'Saml Login', 'yes'), ('onelogin_saml_idp_sso', 'https://sso.%s/auth/realms/master/protocol/saml', 'yes'), @@ -194,103 +214,139 @@ class WordpressSaml(): ('onelogin_saml_advanced_settings_sp_x509cert', '%s', 'yes'), ('onelogin_saml_advanced_settings_sp_privatekey', '%s', 'yes'), ('onelogin_saml_advanced_signaturealgorithm', 'http://www.w3.org/2000/09/xmldsig#rsa-sha1', 'yes'), -('onelogin_saml_advanced_digestalgorithm', 'http://www.w3.org/2000/09/xmldsig#sha1', 'yes');""" % (os.environ['DOMAIN'],os.environ['DOMAIN'],self.parse_idp_cert(),app['config']['PUBLIC_CERT'],app['config']['PRIVATE_KEY'])) - +('onelogin_saml_advanced_digestalgorithm', 'http://www.w3.org/2000/09/xmldsig#sha1', 'yes');""" + % ( + os.environ["DOMAIN"], + os.environ["DOMAIN"], + self.parse_idp_cert(), + app["config"]["PUBLIC_CERT"], + app["config"]["PRIVATE_KEY"], + ) + ) def reset_saml(self): - self.db.update("""DELETE FROM wp_options WHERE option_name LIKE 'onelogin_saml_%'""") + self.db.update( + """DELETE FROM wp_options WHERE option_name LIKE 'onelogin_saml_%'""" + ) def add_keycloak_wordpress_saml(self): - client={"id" : "630601f8-25d1-4822-8741-c93affd2cd84", - "clientId" : "php-saml", - "surrogateAuthRequired" : False, - "enabled" : True, - "alwaysDisplayInConsole" : False, - "clientAuthenticatorType" : "client-secret", - "redirectUris" : [ "https://wp."+os.environ['DOMAIN']+"/wp-login.php?saml_acs" ], - "webOrigins" : [ "https://wp."+os.environ['DOMAIN'] ], - "notBefore" : 0, - "bearerOnly" : False, - "consentRequired" : False, - "standardFlowEnabled" : True, - "implicitFlowEnabled" : False, - "directAccessGrantsEnabled" : False, - "serviceAccountsEnabled" : False, - "publicClient" : False, - "frontchannelLogout" : True, - "protocol" : "saml", - "attributes" : { - "saml.force.post.binding" : True, - "saml_assertion_consumer_url_post" : "https://wp."+os.environ['DOMAIN']+"/wp-login.php?saml_acs", - "saml.server.signature" : True, - "saml.server.signature.keyinfo.ext" : False, - "saml.signing.certificate" : app['config']['PUBLIC_CERT_RAW'], - "saml_single_logout_service_url_redirect" : "https://wp."+os.environ['DOMAIN']+"/wp-login.php?saml_sls", - "saml.signature.algorithm" : "RSA_SHA256", - "saml_force_name_id_format" : False, - "saml.client.signature" : True, - "saml.authnstatement" : True, - "saml_name_id_format" : "username", - "saml_signature_canonicalization_method" : "http://www.w3.org/2001/10/xml-exc-c14n#" + client = { + "id": "630601f8-25d1-4822-8741-c93affd2cd84", + "clientId": "php-saml", + "surrogateAuthRequired": False, + "enabled": True, + "alwaysDisplayInConsole": False, + "clientAuthenticatorType": "client-secret", + "redirectUris": [ + "https://wp." + os.environ["DOMAIN"] + "/wp-login.php?saml_acs" + ], + "webOrigins": ["https://wp." + os.environ["DOMAIN"]], + "notBefore": 0, + "bearerOnly": False, + "consentRequired": False, + "standardFlowEnabled": True, + "implicitFlowEnabled": False, + "directAccessGrantsEnabled": False, + "serviceAccountsEnabled": False, + "publicClient": False, + "frontchannelLogout": True, + "protocol": "saml", + "attributes": { + "saml.force.post.binding": True, + "saml_assertion_consumer_url_post": "https://wp." + + os.environ["DOMAIN"] + + "/wp-login.php?saml_acs", + "saml.server.signature": True, + "saml.server.signature.keyinfo.ext": False, + "saml.signing.certificate": app["config"]["PUBLIC_CERT_RAW"], + "saml_single_logout_service_url_redirect": "https://wp." + + os.environ["DOMAIN"] + + "/wp-login.php?saml_sls", + "saml.signature.algorithm": "RSA_SHA256", + "saml_force_name_id_format": False, + "saml.client.signature": True, + "saml.authnstatement": True, + "saml_name_id_format": "username", + "saml_signature_canonicalization_method": "http://www.w3.org/2001/10/xml-exc-c14n#", + }, + "authenticationFlowBindingOverrides": {}, + "fullScopeAllowed": True, + "nodeReRegistrationTimeout": -1, + "protocolMappers": [ + { + "id": "72c6175e-bd07-4c27-abd6-4e4ae38d834b", + "name": "username", + "protocol": "saml", + "protocolMapper": "saml-user-attribute-mapper", + "consentRequired": False, + "config": { + "attribute.nameformat": "Basic", + "user.attribute": "username", + "friendly.name": "username", + "attribute.name": "username", + }, }, - "authenticationFlowBindingOverrides" : { }, - "fullScopeAllowed" : True, - "nodeReRegistrationTimeout" : -1, - "protocolMappers" : [ { - "id" : "72c6175e-bd07-4c27-abd6-4e4ae38d834b", - "name" : "username", - "protocol" : "saml", - "protocolMapper" : "saml-user-attribute-mapper", - "consentRequired" : False, - "config" : { - "attribute.nameformat" : "Basic", - "user.attribute" : "username", - "friendly.name" : "username", - "attribute.name" : "username" - } - }, { - "id" : "abd6562f-4732-4da9-987f-b1a6ad6605fa", - "name" : "roles", - "protocol" : "saml", - "protocolMapper" : "saml-role-list-mapper", - "consentRequired" : False, - "config" : { - "single" : True, - "attribute.nameformat" : "Basic", - "friendly.name" : "Roles", - "attribute.name" : "Role" - } - }, { - "id" : "50aafb71-d91c-4bc7-bb60-e1ae0222aab3", - "name" : "email", - "protocol" : "saml", - "protocolMapper" : "saml-user-property-mapper", - "consentRequired" : False, - "config" : { - "attribute.nameformat" : "Basic", - "user.attribute" : "email", - "friendly.name" : "email", - "attribute.name" : "email" - } - } ], - "defaultClientScopes" : [ "web-origins", "role_list", "roles", "profile", "email" ], - "optionalClientScopes" : [ "address", "phone", "offline_access", "microprofile-jwt" ], - "access" : { - "view" : True, - "configure" : True, - "manage" : True - } - } + { + "id": "abd6562f-4732-4da9-987f-b1a6ad6605fa", + "name": "roles", + "protocol": "saml", + "protocolMapper": "saml-role-list-mapper", + "consentRequired": False, + "config": { + "single": True, + "attribute.nameformat": "Basic", + "friendly.name": "Roles", + "attribute.name": "Role", + }, + }, + { + "id": "50aafb71-d91c-4bc7-bb60-e1ae0222aab3", + "name": "email", + "protocol": "saml", + "protocolMapper": "saml-user-property-mapper", + "consentRequired": False, + "config": { + "attribute.nameformat": "Basic", + "user.attribute": "email", + "friendly.name": "email", + "attribute.name": "email", + }, + }, + ], + "defaultClientScopes": [ + "web-origins", + "role_list", + "roles", + "profile", + "email", + ], + "optionalClientScopes": [ + "address", + "phone", + "offline_access", + "microprofile-jwt", + ], + "access": {"view": True, "configure": True, "manage": True}, + } self.connect() self.keycloak.add_client(client) - self.keycloak=None + self.keycloak = None def add_client_roles(self): self.connect() - self.keycloak.add_client_role('630601f8-25d1-4822-8741-c93affd2cd84','admin','Wordpress admins') - self.keycloak.add_client_role('630601f8-25d1-4822-8741-c93affd2cd84','manager','Wordpress managers') - self.keycloak.add_client_role('630601f8-25d1-4822-8741-c93affd2cd84','teacher','Wordpress teachers') - self.keycloak.add_client_role('630601f8-25d1-4822-8741-c93affd2cd84','student','Wordpress students') - self.keycloak=None + self.keycloak.add_client_role( + "630601f8-25d1-4822-8741-c93affd2cd84", "admin", "Wordpress admins" + ) + self.keycloak.add_client_role( + "630601f8-25d1-4822-8741-c93affd2cd84", "manager", "Wordpress managers" + ) + self.keycloak.add_client_role( + "630601f8-25d1-4822-8741-c93affd2cd84", "teacher", "Wordpress teachers" + ) + self.keycloak.add_client_role( + "630601f8-25d1-4822-8741-c93affd2cd84", "student", "Wordpress students" + ) + self.keycloak = None -nw=WordpressSaml() \ No newline at end of file + +nw = WordpressSaml() diff --git a/admin/src/wordpress_saml_onlycerts.py b/admin/src/wordpress_saml_onlycerts.py index 2cbc39c..e7a8e66 100644 --- a/admin/src/wordpress_saml_onlycerts.py +++ b/admin/src/wordpress_saml_onlycerts.py @@ -1,67 +1,84 @@ #!/usr/bin/env python # coding=utf-8 -import time, os -from datetime import datetime, timedelta -import pprint - +import json import logging as log +import os +import pprint +import random +import string +import time import traceback -import yaml, json +from datetime import datetime, timedelta import psycopg2 +import yaml -from admin.lib.mysql import Mysql from admin.lib.keycloak_client import KeycloakClient +from admin.lib.mysql import Mysql -import string, random +app = {} +app["config"] = {} -app={} -app['config']={} -class WordpressSaml(): +class WordpressSaml: def __init__(self): - self.url="http://isard-sso-keycloak:8080/auth/" - self.username=os.environ['KEYCLOAK_USER'] - self.password=os.environ['KEYCLOAK_PASSWORD'] - self.realm='master' - self.verify=True + self.url = "http://isard-sso-keycloak:8080/auth/" + self.username = os.environ["KEYCLOAK_USER"] + self.password = os.environ["KEYCLOAK_PASSWORD"] + self.realm = "master" + self.verify = True - ready=False + ready = False while not ready: try: - self.db=Mysql('isard-apps-mariadb','wordpress','root',os.environ['MARIADB_PASSWORD']) - ready=True + self.db = Mysql( + "isard-apps-mariadb", + "wordpress", + "root", + os.environ["MARIADB_PASSWORD"], + ) + ready = True except: - log.warning('Could not connect to wordpress database. Retrying...') + log.warning("Could not connect to wordpress database. Retrying...") time.sleep(2) - log.info('Connected to wordpress database.') + log.info("Connected to wordpress database.") - ready=False + ready = False while not ready: try: - with open(os.path.join("./saml_certs/public.cert"),"r") as crt: - app['config']['PUBLIC_CERT_RAW']=crt.read() - app['config']['PUBLIC_CERT']=self.cert_prepare(app['config']['PUBLIC_CERT_RAW']) - ready=True + with open(os.path.join("./saml_certs/public.cert"), "r") as crt: + app["config"]["PUBLIC_CERT_RAW"] = crt.read() + app["config"]["PUBLIC_CERT"] = self.cert_prepare( + app["config"]["PUBLIC_CERT_RAW"] + ) + ready = True except IOError: - log.warning('Could not get public certificate to be used in wordpress. Retrying...') - log.warning(' You should generate them: /admin/saml_certs # openssl req -nodes -new -x509 -keyout private.key -out public.cert') + log.warning( + "Could not get public certificate to be used in wordpress. Retrying..." + ) + log.warning( + " You should generate them: /admin/saml_certs # openssl req -nodes -new -x509 -keyout private.key -out public.cert" + ) time.sleep(2) except: log.error(traceback.format_exc()) - log.info('Got moodle srt certificate to be used in wordpress.') + log.info("Got moodle srt certificate to be used in wordpress.") - ready=False + ready = False while not ready: try: - with open(os.path.join("./saml_certs/private.key"),"r") as pem: - app['config']['PRIVATE_KEY']=self.cert_prepare(pem.read()) - ready=True + with open(os.path.join("./saml_certs/private.key"), "r") as pem: + app["config"]["PRIVATE_KEY"] = self.cert_prepare(pem.read()) + ready = True except IOError: - log.warning('Could not get private key to be used in wordpress. Retrying...') - log.warning(' You should generate them: /admin/saml_certs # openssl req -nodes -new -x509 -keyout private.key -out public.cert') + log.warning( + "Could not get private key to be used in wordpress. Retrying..." + ) + log.warning( + " You should generate them: /admin/saml_certs # openssl req -nodes -new -x509 -keyout private.key -out public.cert" + ) time.sleep(2) - log.info('Got moodle pem certificate to be used in wordpress.') + log.info("Got moodle pem certificate to be used in wordpress.") # ## This seems related to the fact that the certificate generated the first time does'nt work. # ## And when regenerating the certificate de privatekeypass seems not to be used and instead it @@ -79,14 +96,14 @@ class WordpressSaml(): self.reset_saml_certs() except: print(traceback.format_exc()) - print('Error resetting saml certs on wordpress') + print("Error resetting saml certs on wordpress") try: self.set_wordpress_saml_plugin_certs() except: print(traceback.format_exc()) - print('Error adding saml on wordpress') - + print("Error adding saml on wordpress") + # SAML clients don't work well with composite roles so disabling and adding on realm # self.add_client_roles() @@ -98,32 +115,47 @@ class WordpressSaml(): # return self.db.select("""SELECT * FROM "mdl_config" WHERE "name" = 'siteidentifier'""")[0][2] def connect(self): - self.keycloak= KeycloakClient(url=self.url, - username=self.username, - password=self.password, - realm=self.realm, - verify=self.verify) + self.keycloak = KeycloakClient( + url=self.url, + username=self.username, + password=self.password, + realm=self.realm, + verify=self.verify, + ) - def cert_prepare(self,cert): - return ''.join(cert.split('-----')[2].splitlines()) + def cert_prepare(self, cert): + return "".join(cert.split("-----")[2].splitlines()) def parse_idp_cert(self): self.connect() - rsa=self.keycloak.get_server_rsa_key() - self.keycloak=None - return rsa['certificate'] + rsa = self.keycloak.get_server_rsa_key() + self.keycloak = None + return rsa["certificate"] def set_wordpress_saml_plugin_certs(self): # ('active_plugins', 'a:2:{i:0;s:33:\"edwiser-bridge/edwiser-bridge.php\";i:1;s:17:\"onelogin_saml.php\";}', 'yes'), - self.db.update("""INSERT INTO wp_options (option_name, option_value, autoload) VALUES + self.db.update( + """INSERT INTO wp_options (option_name, option_value, autoload) VALUES ('onelogin_saml_idp_x509cert', '%s', 'yes'), ('onelogin_saml_advanced_settings_sp_x509cert', '%s', 'yes'), -('onelogin_saml_advanced_settings_sp_privatekey', '%s', 'yes');""" % (self.parse_idp_cert(),app['config']['PUBLIC_CERT'],app['config']['PRIVATE_KEY'])) - +('onelogin_saml_advanced_settings_sp_privatekey', '%s', 'yes');""" + % ( + self.parse_idp_cert(), + app["config"]["PUBLIC_CERT"], + app["config"]["PRIVATE_KEY"], + ) + ) def reset_saml_certs(self): - self.db.update("""DELETE FROM wp_options WHERE option_name = 'onelogin_saml_idp_x509cert'""") - self.db.update("""DELETE FROM wp_options WHERE option_name = 'onelogin_saml_advanced_settings_sp_x509cert'""") - self.db.update("""DELETE FROM wp_options WHERE option_name = 'onelogin_saml_advanced_settings_sp_privatekey'""") + self.db.update( + """DELETE FROM wp_options WHERE option_name = 'onelogin_saml_idp_x509cert'""" + ) + self.db.update( + """DELETE FROM wp_options WHERE option_name = 'onelogin_saml_advanced_settings_sp_x509cert'""" + ) + self.db.update( + """DELETE FROM wp_options WHERE option_name = 'onelogin_saml_advanced_settings_sp_privatekey'""" + ) -nw=WordpressSaml() + +nw = WordpressSaml()