From d54f47c06f751f638b7c065c347436ca7d3dc825 Mon Sep 17 00:00:00 2001 From: root Date: Thu, 27 May 2021 11:25:53 +0200 Subject: [PATCH] progress --- admin/src/admin/lib/admin.py | 83 +++++++++++++------ admin/src/admin/lib/helpers.py | 8 ++ admin/src/admin/lib/keycloak.py | 72 ++++++++-------- admin/src/admin/lib/load_config.py | 4 + admin/src/admin/lib/moodle.py | 4 +- admin/src/admin/lib/nextcloud.py | 39 ++++++--- admin/src/admin/lib/postgres.py | 13 ++- admin/src/admin/static/js/users.js | 23 ++++- .../admin/static/templates/pages/users.html | 2 + admin/src/admin/views/MenuViews.py | 11 ++- 10 files changed, 180 insertions(+), 79 deletions(-) create mode 100644 admin/src/admin/lib/helpers.py diff --git a/admin/src/admin/lib/admin.py b/admin/src/admin/lib/admin.py index ed80c45..66c6ff5 100644 --- a/admin/src/admin/lib/admin.py +++ b/admin/src/admin/lib/admin.py @@ -5,9 +5,10 @@ from .nextcloud import Nextcloud import logging as log from pprint import pprint +import traceback from .nextcloud_exc import * - +from .helpers import filter_roles_list, filter_roles_listofdicts class Admin(): def __init__(self): self.keycloak=Keycloak(verify=app.config['VERIFY']) @@ -43,31 +44,42 @@ class Admin(): # for u in users] def get_keycloak_users(self): - log.warning('Loading keycloak users... can take a long time...') + # log.warning('Loading keycloak users... can take a long time...') users = self.keycloak.get_users_with_groups_and_roles() return [{"id":u['id'], "username":u['username'], - "first": u.get('firstName',None), - "last": u.get('lastName',None), + "first": u.get('first_name',None), + "last": u.get('last_name',None), "email": u.get('email',''), - "groups": u['groups'], - "roles": u['roles']} + "groups": u['group'], + "roles": filter_roles_list(u['role'])} for u in users] def get_nextcloud_users(self): - log.warning('Loading nextcloud users... can take a long time...') - users = self.nextcloud.get_users_list() - users_list=[] - for user in users: - u=self.nextcloud.get_user(user) - users_list.append({"id":u['id'], - "username":u['id'], - "first": u['displayname'], - "last": None, - "email": u['email'], - "groups": u['groups'], - "roles": []}) - return users_list + return [{"id":u['username'], + "username":u['username'], + "first": u['displayname'].split(' ')[0] if u['displayname'] is not None else '', + "last": u['displayname'].split(' ')[1] if u['displayname'] is not None and len(u['displayname'].split(' '))>1 else '', + "email": u.get('email',''), + "groups": u['groups'], + "roles": False} + for u in self.nextcloud.get_users_list()] + + ## TOO SLOW + # def get_nextcloud_users(self): + # log.warning('Loading nextcloud users... can take a long time...') + # users = self.nextcloud.get_users_list() + # users_list=[] + # for user in users: + # u=self.nextcloud.get_user(user) + # users_list.append({"id":u['id'], + # "username":u['id'], + # "first": u['displayname'], + # "last": None, + # "email": u['email'], + # "groups": u['groups'], + # "roles": []}) + # return users_list # def get_ram_users(self): # return self.internal['users'] @@ -75,8 +87,7 @@ class Admin(): def get_mix_users(self): kusers=self.get_keycloak_users() musers=self.get_moodle_users() - # nusers=self.get_nextcloud_users() - nusers=[] + nusers=self.get_nextcloud_users() kusers_usernames=[u['username'] for u in kusers] musers_usernames=[u['username'] for u in musers] @@ -120,7 +131,7 @@ class Admin(): return users def get_roles(self): - return self.keycloak.get_roles() + return filter_roles_listofdicts(self.keycloak.get_roles()) def get_keycloak_groups(self): log.warning('Loading keycloak groups... can take a long time...') @@ -205,7 +216,6 @@ class Admin(): "description": g['description']}) self.external['groups']=groups - users=[] for u in data['data']['users']: users.append({'provider':data['provider'], @@ -214,7 +224,7 @@ class Admin(): 'first': u['name']['givenName'], 'last': u['name']['familyName'], 'username': u['primaryEmail'].split('@')[0], - 'groups':[u['orgUnitPath']], + 'groups':[u['orgUnitPath'][1:]], ## WARNING: Removing the first 'roles':[]}) self.external['users']=users @@ -246,3 +256,28 @@ class Admin(): except: log.error(traceback.format_exc()) #
Las contraseñas deben tener al menos 1 dígito(s).
Las contraseñas deben tener al menos 1 mayúscula(s).
Las contraseñas deben tener al menos 1 caracter(es) no alfanumérico(s) como *,-, + + def delete_keycloak_users(self): + users=self.get_keycloak_users() + for u in users: + # Do not remove admin users!!! What to do with managers??? + if 'admin' in u['roles']: continue + if 'manager' in u['roles']: continue + log.info('Removing keycloak user: '+u['username']) + try: + self.keycloak.delete_user(u['id']) + except: + log.error(traceback.format_exc()) + log.warning('Could not remove user: '+u['username']) + + def delete_keycloak_groups(self): + groups=self.get_keycloak_groups() + for g in groups: + # Do not remove admin group. It should not exist in keycloak, only in nextcloud + if g['name'] == ['admin']: continue + log.info('Removing keycloak group: '+g['name']) + try: + self.keycloak.delete_group(g['id']) + except: + log.error(traceback.format_exc()) + log.warning('Could not remove group: '+g['name']) \ No newline at end of file diff --git a/admin/src/admin/lib/helpers.py b/admin/src/admin/lib/helpers.py new file mode 100644 index 0000000..24922e1 --- /dev/null +++ b/admin/src/admin/lib/helpers.py @@ -0,0 +1,8 @@ + +def filter_roles_list(role_list): + client_roles=['admin','manager','teacher','student'] + return [r for r in role_list if r in client_roles] + +def filter_roles_listofdicts(role_listofdicts): + client_roles=['admin','manager','teacher','student'] + return [r for r in role_listofdicts if r['name'] in client_roles] \ No newline at end of file diff --git a/admin/src/admin/lib/keycloak.py b/admin/src/admin/lib/keycloak.py index 7776d03..7195023 100644 --- a/admin/src/admin/lib/keycloak.py +++ b/admin/src/admin/lib/keycloak.py @@ -12,6 +12,7 @@ from pprint import pprint from jinja2 import Environment, FileSystemLoader from keycloak import KeycloakAdmin +from .postgres import Postgres class Keycloak(): """https://www.keycloak.org/docs-api/13.0/rest-api/index.html @@ -30,6 +31,8 @@ class Keycloak(): self.realm=realm self.verify=verify + self.keycloak_pg=Postgres('isard-apps-postgresql','keycloak',app.config['KEYCLOAK_POSTGRES_USER'],app.config['KEYCLOAK_POSTGRES_PASSWORD']) + def connect(self): self.keycloak_admin = KeycloakAdmin(server_url=self.url, username=self.username, @@ -67,46 +70,43 @@ class Keycloak(): self.connect() return self.keycloak_admin.get_users({}) -# q = """select u.username,u.email,u.first_name,u.last_name, u.realm_id, ua.value as quota -# ,json_agg(g."name") as group, json_agg(g_parent."name") as group_parent1, json_agg(g_parent2."name") as group_parent2 -# ,json_agg(r.name) as role -# from user_entity as u -# left join user_attribute as ua on ua.user_id=u.id and ua.name = 'quota' -# left join user_group_membership as ugm on ugm.user_id = u.id -# left join keycloak_group as g on g.id = ugm.group_id -# left join keycloak_group as g_parent on g.parent_group = g_parent.id -# left join keycloak_group as g_parent2 on g_parent.parent_group = g_parent2.id -# left join user_role_mapping as rm on rm.user_id = u.id -# left join keycloak_role as r on r.id = rm.role_id -# group by u.username,u.email,u.first_name,u.last_name, u.realm_id, ua.value -# order by u.username""" -# cur.execute(q) -# users = cur.fetchall() -# fields = [a.name for a in cur.description] -# cur.close() -# conn.close() + def get_users_with_groups_and_roles(self): + q = """select u.id, u.username, u.email, u.first_name, u.last_name, u.realm_id, ua.value as quota + ,json_agg(g."name") as group, json_agg(g_parent."name") as group_parent1, json_agg(g_parent2."name") as group_parent2 + ,json_agg(r.name) as role + from user_entity as u + left join user_attribute as ua on ua.user_id=u.id and ua.name = 'quota' + left join user_group_membership as ugm on ugm.user_id = u.id + left join keycloak_group as g on g.id = ugm.group_id + left join keycloak_group as g_parent on g.parent_group = g_parent.id + left join keycloak_group as g_parent2 on g_parent.parent_group = g_parent2.id + left join user_role_mapping as rm on rm.user_id = u.id + left join keycloak_role as r on r.id = rm.role_id + group by u.id,u.username,u.email,u.first_name,u.last_name, u.realm_id, ua.value + order by u.username""" + (headers,users)=self.keycloak_pg.select_with_headers(q) + users_with_lists = [list(l[:-4])+([[]] if l[-4] == [None] else [list(set(l[-4]))]) +\ + ([[]] if l[-3] == [None] else [list(set(l[-3]))]) +\ + ([[]] if l[-3] == [None] else [list(set(l[-2]))]) +\ + ([[]] if l[-1] == [None] else [list(set(l[-1]))]) for l in users] + + users_with_lists = [list(l[:-4])+([[]] if l[-4] == [None] else [list(set(l[-4]))]) +\ + ([[]] if l[-3] == [None] else [list(set(l[-3]))]) +\ + ([[]] if l[-3] == [None] else [list(set(l[-2]))]) +\ + ([[]] if l[-1] == [None] else [list(set(l[-1]))]) for l in users_with_lists] -# users_with_lists = [list(l[:-4])+([[]] if l[-4] == [None] else [list(set(l[-4]))]) +\ -# ([[]] if l[-3] == [None] else [list(set(l[-3]))]) +\ -# ([[]] if l[-3] == [None] else [list(set(l[-2]))]) +\ -# ([[]] if l[-1] == [None] else [list(set(l[-1]))]) for l in users] - -# users_with_lists = [list(l[:-4])+([[]] if l[-4] == [None] else [list(set(l[-4]))]) +\ -# ([[]] if l[-3] == [None] else [list(set(l[-3]))]) +\ -# ([[]] if l[-3] == [None] else [list(set(l[-2]))]) +\ -# ([[]] if l[-1] == [None] else [list(set(l[-1]))]) for l in users_with_lists] - -# list_dict_users = [dict(zip(fields, r)) for r in users_with_lists] + list_dict_users = [dict(zip(headers, r)) for r in users_with_lists] + return list_dict_users ## Too slow - def get_users_with_groups_and_roles(self): - self.connect() - users=self.keycloak_admin.get_users({}) - for user in users: - user['groups']=[g['path'] for g in self.keycloak_admin.get_user_groups(user_id=user['id'])] - user['roles']= [r['name'] for r in self.keycloak_admin.get_realm_roles_of_user(user_id=user['id'])] - return users + # def get_users_with_groups_and_roles(self): + # self.connect() + # users=self.keycloak_admin.get_users({}) + # for user in users: + # user['groups']=[g['path'] for g in self.keycloak_admin.get_user_groups(user_id=user['id'])] + # user['roles']= [r['name'] for r in self.keycloak_admin.get_realm_roles_of_user(user_id=user['id'])] + # return users def add_user(self,username,first,last,email,password,group=False): # Returns user id diff --git a/admin/src/admin/lib/load_config.py b/admin/src/admin/lib/load_config.py index d8b5347..ffda344 100644 --- a/admin/src/admin/lib/load_config.py +++ b/admin/src/admin/lib/load_config.py @@ -12,8 +12,12 @@ class loadConfig(): def __init__(self, app=None): try: app.config.setdefault('DOMAIN', os.environ['DOMAIN']) + app.config.setdefault('KEYCLOAK_POSTGRES_USER', os.environ['KEYCLOAK_DB_USER']) + app.config.setdefault('KEYCLOAK_POSTGRES_PASSWORD', os.environ['KEYCLOAK_DB_PASSWORD']) app.config.setdefault('MOODLE_POSTGRES_USER', os.environ['MOODLE_POSTGRES_USER']) app.config.setdefault('MOODLE_POSTGRES_PASSWORD', os.environ['MOODLE_POSTGRES_PASSWORD']) + app.config.setdefault('NEXTCLOUD_POSTGRES_USER', os.environ['NEXTCLOUD_POSTGRES_USER']) + app.config.setdefault('NEXTCLOUD_POSTGRES_PASSWORD', os.environ['NEXTCLOUD_POSTGRES_PASSWORD']) app.config.setdefault('VERIFY', True if os.environ['VERIFY']=="true" else False) except Exception as e: log.error(traceback.format_exc()) diff --git a/admin/src/admin/lib/moodle.py b/admin/src/admin/lib/moodle.py index 413f56f..33514ad 100644 --- a/admin/src/admin/lib/moodle.py +++ b/admin/src/admin/lib/moodle.py @@ -24,7 +24,7 @@ class Moodle(): self.endpoint = endpoint self.verify=verify - self.pg=Postgres('isard-apps-postgresql','moodle',app.config['MOODLE_POSTGRES_USER'],app.config['MOODLE_POSTGRES_PASSWORD']) + self.moodle_pg=Postgres('isard-apps-postgresql','moodle',app.config['MOODLE_POSTGRES_USER'],app.config['MOODLE_POSTGRES_PASSWORD']) def rest_api_parameters(self, in_args, prefix='', out_dict=None): @@ -85,7 +85,7 @@ class Moodle(): left join mdl_role_assignments AS ra ON ra.id = u.id left join mdl_role as r on r.id = ra.roleid group by u.id , username, first, last, email""" - (headers,users)=self.pg.select_with_headers(q) + (headers,users)=self.moodle_pg.select_with_headers(q) users_with_lists = [list(l[:-2])+([[]] if l[-2] == [None] else [list(set(l[-2]))]) + ([[]] if l[-1] == [None] else [list(set(l[-1]))]) for l in users] list_dict_users = [dict(zip(headers, r)) for r in users_with_lists] return list_dict_users diff --git a/admin/src/admin/lib/nextcloud.py b/admin/src/admin/lib/nextcloud.py index ce45e98..ee72b96 100644 --- a/admin/src/admin/lib/nextcloud.py +++ b/admin/src/admin/lib/nextcloud.py @@ -8,6 +8,8 @@ import traceback import logging as log from .nextcloud_exc import * +from .postgres import Postgres + class Nextcloud(): def __init__(self, url="https://nextcloud."+app.config['DOMAIN'], @@ -23,6 +25,8 @@ class Nextcloud(): self.auth=(username,password) self.user=username + self.nextcloud_pg=Postgres('isard-apps-postgresql','nextcloud',app.config['NEXTCLOUD_POSTGRES_USER'],app.config['NEXTCLOUD_POSTGRES_PASSWORD']) + def _request(self,method,url,data={},headers={'OCS-APIRequest':'true'},auth=False): if auth == False: auth=self.auth try: @@ -95,18 +99,33 @@ class Nextcloud(): # users_with_lists = [list(l[:-2])+([[]] if l[-2] == [None] else [list(set(l[-2]))]) + ([[]] if l[-1] == [None] else [list(set(l[-1]))]) for l in users] # users_with_lists = [list(l[:-2])+([[]] if l[-2] == [None] else [list(set(l[-2]))]) + ([[]] if l[-1] == [None] else [list(set(l[-1]))]) for l in users_with_lists] # list_dict_users = [dict(zip(fields, r)) for r in users_with_lists] + def get_users_list(self): + q = """select u.uid as username, adn.value as displayname, ade.value as email, json_agg(gg.displayname) as admin_groups,json_agg(g.displayname) as groups + from oc_users as u + left join oc_group_user as gu on gu.uid = u.uid + left join oc_groups as g on gu.gid = g.gid + left join oc_group_admin as ga on ga.uid = u.uid + left join oc_groups as gg on gg.gid = ga.gid + left join oc_accounts_data as adn on adn.uid = u.uid and adn.name = 'displayname' + left join oc_accounts_data as ade on ade.uid = u.uid and ade.name = 'email' + group by u.uid, adn.value, ade.value""" + (headers,users)=self.nextcloud_pg.select_with_headers(q) + users_with_lists = [list(l[:-2])+([[]] if l[-2] == [None] else [list(set(l[-2]))]) + ([[]] if l[-1] == [None] else [list(set(l[-1]))]) for l in users] + users_with_lists = [list(l[:-2])+([[]] if l[-2] == [None] else [list(set(l[-2]))]) + ([[]] if l[-1] == [None] else [list(set(l[-1]))]) for l in users_with_lists] + list_dict_users = [dict(zip(headers, r)) for r in users_with_lists] + return list_dict_users ### Too slow... - def get_users_list(self): - url = self.apiurl + "users?format=json" - try: - result = json.loads(self._request('GET',url)) - if result['ocs']['meta']['statuscode'] == 100: return result['ocs']['data']['users'] - log.error('Get Nextcloud provider users list error: '+str(result)) - raise ProviderOpError - except: - log.error(traceback.format_exc()) - raise + # def get_users_list(self): + # url = self.apiurl + "users?format=json" + # try: + # result = json.loads(self._request('GET',url)) + # if result['ocs']['meta']['statuscode'] == 100: return result['ocs']['data']['users'] + # log.error('Get Nextcloud provider users list error: '+str(result)) + # raise ProviderOpError + # except: + # log.error(traceback.format_exc()) + # raise def add_user(self,userid,userpassword,quota,group=False,email='',displayname=''): if group: diff --git a/admin/src/admin/lib/postgres.py b/admin/src/admin/lib/postgres.py index ab2f441..6c7652b 100644 --- a/admin/src/admin/lib/postgres.py +++ b/admin/src/admin/lib/postgres.py @@ -5,7 +5,7 @@ from admin import app from datetime import datetime, timedelta import pprint -import logging +import logging as log import traceback import yaml, json @@ -19,25 +19,32 @@ class Postgres(): database=database, user=user, password=password) - self.cur = self.conn.cursor() + # def __del__(self): # self.cur.close() # self.conn.close() def select(self,sql): + self.cur = self.conn.cursor() self.cur.execute(sql) - return self.cur.fetchall() + data=self.cur.fetchall() + self.cur.close() + return data def update(self,sql): + self.cur = self.conn.cursor() self.cur.execute(sql) self.conn.commit() + self.cur.close() # return self.cur.fetchall() def select_with_headers(self,sql): + self.cur = self.conn.cursor() self.cur.execute(sql) data=self.cur.fetchall() fields = [a.name for a in self.cur.description] + self.cur.close() return (fields,data) # def update_moodle_saml_plugin(self): diff --git a/admin/src/admin/static/js/users.js b/admin/src/admin/static/js/users.js index 0d2ff32..3298d8c 100644 --- a/admin/src/admin/static/js/users.js +++ b/admin/src/admin/static/js/users.js @@ -14,6 +14,24 @@ $(document).ready(function() { $('#modalAdd').parsley(); }); + $('.btn-delete_keycloak').on('click', function () { + $.ajax({ + type: "DELETE", + url:"/isard-sso-admin/users/keycloak", + success: function(data) + { + console.log('SUCCESS') + // $("#modalImport").modal('hide'); + // users_table.ajax.reload(); + // groups_table.ajax.reload(); + }, + error: function(data) + { + alert('Something went wrong on our side...') + } + }); + }); + $('.btn-sync_to_moodle').on('click', function () { $.ajax({ type: "POST", @@ -73,6 +91,7 @@ $(document).ready(function() { { "data": "id", "width": "10px" }, { "data": "keycloak", "width": "10px" }, { "data": "keycloak_groups", "width": "10px" }, + { "data": "roles", "width": "10px" }, { "data": "moodle", "width": "10px" }, { "data": "moodle_groups", "width": "10px" }, { "data": "nextcloud", "width": "10px" }, @@ -93,7 +112,7 @@ $(document).ready(function() { }; }}, { - "targets": 4, + "targets": 5, "render": function ( data, type, full, meta ) { if(full.moodle){ return '' @@ -102,7 +121,7 @@ $(document).ready(function() { }; }}, { - "targets": 6, + "targets": 7, "render": function ( data, type, full, meta ) { if(full.nextcloud){ return '' diff --git a/admin/src/admin/static/templates/pages/users.html b/admin/src/admin/static/templates/pages/users.html index 37b17d8..84b7cb3 100644 --- a/admin/src/admin/static/templates/pages/users.html +++ b/admin/src/admin/static/templates/pages/users.html @@ -16,6 +16,7 @@