From 0bad243ea86adcbe5cbbf66fba8c741a647fe4a5 Mon Sep 17 00:00:00 2001 From: root Date: Tue, 14 Sep 2021 10:29:09 +0200 Subject: [PATCH] testing --- admin/src/admin/lib/admin.py | 187 +++++++++++++++--- admin/src/admin/lib/keycloak_client.py | 46 +++-- .../src/admin/static/js/sysadmin/external.js | 17 +- .../templates/pages/sysadmin/external.html | 4 +- admin/src/admin/static/templates/sidebar.html | 1 + admin/src/admin/views/ApiViews.py | 45 ++++- 6 files changed, 249 insertions(+), 51 deletions(-) diff --git a/admin/src/admin/lib/admin.py b/admin/src/admin/lib/admin.py index 7f68119..c447b0e 100644 --- a/admin/src/admin/lib/admin.py +++ b/admin/src/admin/lib/admin.py @@ -497,9 +497,28 @@ class Admin(): groups=[] for u in data['data']: log.warning('Processing ('+str(item)+'/'+str(total)+') uploaded user: '+u['username']) - user_groups=["/" + g.strip() for g in u['groups'].split(',')] - + user_groups=[g.strip() for g in u['groups'].split(',')] + + pathslist=[] + for group in user_groups: + pathpart='' + for part in kpath2gid(group).split('.'): + if pathpart=='': + pathpart=part + else: + pathpart=pathpart+'.'+part + pathslist.append(pathpart) + pathslist.append(u['role']) + groups=groups+user_groups + # pprint(u) + if u['quota'] == '': + u['quota'] = '500 MB' + if u['role'] == 'student': u['quota']='500MB' + if u['role'] == 'teacher': u['quota']='5 GB' + if u['role'] == 'manager': u['quota']='Unlimited' + if u['role'] == 'admin': u['quota']='Unlimited' + users.append({'provider':'external', 'id':u['id'].strip(), 'email': u['email'].strip(), @@ -507,20 +526,22 @@ class Admin(): 'last': u['lastname'].strip(), 'username': u['username'].strip(), 'groups':user_groups, + 'gids': pathslist, + 'quota': u['quota'], 'roles':[u['role'].strip()], - 'password': self.get_dice_pwd()}) + 'temporary': True if u['password_temporal'].lower() == 'yes' else False, + 'password': self.get_dice_pwd() if u['password']=='' else u['password']}) item+=1 - ev.increment({'name':u['username'].split('@')[0]}) + ev.increment({'name':u['username']}) self.external['users']=users groups=list(dict.fromkeys(groups)) - sysgroups=[] for g in groups: sysgroups.append({'provider':'external', "id": g, - "mailid": g, - "name": g, + "name": kpath2gid(g), + "path": g, "description": 'Imported with csv'}) self.external['groups']=sysgroups return True @@ -584,16 +605,18 @@ class Admin(): return True def sync_external(self,ids): + # self.resync_data() log.warning('Starting sync to keycloak') - self.sync_to_keycloak() + self.sync_to_keycloak_external() ### Now we only sycn external to keycloak and then they can be updated to others with UI buttons log.warning('Starting sync to moodle') - self.sync_to_moodle() + self.sync_to_moodle_external() log.warning('Starting sync to nextcloud') - self.sync_to_nextcloud() - log.warning('All syncs finished') + self.sync_to_nextcloud_external() + log.warning('All syncs finished. Resyncing from apps...') + self.resync_data() - def sync_to_keycloak(self): ### This one works from the external, moodle and nextcloud from the internal + def sync_to_keycloak_external(self): ### This one works from the external, moodle and nextcloud from the internal groups=[] for u in self.external['users']: groups=groups+u['groups'] @@ -606,6 +629,7 @@ class Admin(): i=i+1 log.warning(' KEYCLOAK GROUPS: Adding group ('+str(i)+'/'+str(total)+'): '+g) ev.increment({'name':g}) + print(g) self.keycloak.add_group_tree(g) total=len(self.external['users']) @@ -616,14 +640,14 @@ class Admin(): # Add user log.warning(' KEYCLOAK USERS: Adding user ('+str(index)+'/'+str(total)+'): '+u['username']) ev.increment({'name':u['username'],'data':u}) - uid=self.keycloak.add_user(u['username'],u['first'],u['last'],u['email'],u['password']) + uid=self.keycloak.add_user(u['username'],u['first'],u['last'],u['email'],u['password'],temporary=u['temporary']) self.av.add_user_default_avatar(uid,u['roles'][0]) # Add user to role and group rolename if len(u['roles']) != 0: log.warning(' KEYCLOAK USERS: Assign user '+u['username']+' with initial pwd '+ u['password']+' to role '+u['roles'][0]) self.keycloak.assign_realm_roles(uid,u['roles'][0]) - gid=self.keycloak.get_group_by_path(path='/'+u['roles'][0])['id'] - self.keycloak.group_user_add(uid,gid) + gid=self.keycloak.get_group_by_path('/'+u['roles'][0])['id'] + # self.keycloak.group_user_add(uid,gid) # Add user to groups for g in u['groups']: parts=g.split('/') @@ -635,10 +659,119 @@ class Admin(): sub=sub+'/'+parts[i] if sub=='/': continue # User with no path log.warning(' KEYCLOAK USERS: Assign user '+u['username']+' to group '+ str(sub)) - gid=self.keycloak.get_group_by_path(path=sub)['id'] + kuser=self.keycloak.get_group_by_path(path=sub) + gid=kuser['id'] self.keycloak.group_user_add(uid,gid) + # We add it as it is needed for moodle and nextcloud + u['groups'].append(u['roles'][0]) self.resync_data() + def sync_to_moodle_external(self): # works from the internal (keycloak) + ### Process all groups from the users keycloak_groups key + groups=[] + for u in self.external['users']: + groups=groups+u['gids'] + groups=list(dict.fromkeys(groups)) + + ### Create all groups. Skip / in system groups + total=len(groups) + ev=Events('Syncing groups from external to moodle',total=len(groups)) + for g in groups: + parts=g.split('.') + if not len(parts): + log.error(' MOODLE GROUPS: Group '+g+ ' empty') + continue + subpath=parts[0] + for i in range(1,len(parts)): + try: + log.warning(' MOODLE GROUPS: Adding group as cohort ('+str(i)+'/'+str(total)+'): '+subpath) + ev.increment({'name':subpath}) + # if parts[i] in ['admin','manager','teacher','student']: + self.moodle.add_system_cohort(subpath) + if len(parts) != i+1: + subpath=subpath+'.'+parts[i+1] + except: + log.error(' MOODLE GROUPS: Group '+subpath+ ' probably already exists') + + ### Get all existing moodle cohorts + cohorts=self.moodle.get_cohorts() + + ### Create users in moodle + ev=Events('Syncing users from external to moodle',total=len(self.internal['users'])) + for u in self.external['users']: + log.warning('MOODLE: Creating moodle user: '+u['username']) + ev.increment({'name':u['username']}) + if u['first'] == '': u['first']='-' + if u['last'] == '': u['last']='-' + try: + self.moodle.create_user(u['email'],u['username'],'*12'+secrets.token_urlsafe(16),u['first'],u['last'])[0] + except UserExists: + log.warning('MOODLE:The user: '+u['username']+' already exsits.') + except: + log.error(' -->> Error creating on moodle the user: '+u['username']) + log.error(traceback.format_exc()) + # user_id=user['id'] + + # self.resync_data() + ### Add user to their cohorts (groups) + ev=Events('Syncing users groups from external to moodle cohorts',total=len(self.internal['users'])) + cohorts=self.moodle.get_cohorts() + for u in self.external['users']: + try: + uid=self.moodle.get_user_by('username',u['username'])['users'][0]['id'] + for group in u['gids']: + cohort=[c for c in cohorts if c['name']==group][0] + self.moodle.add_user_to_cohort(uid,cohort['id']) + except: + log.error('Exception on getting user from moodle: '+u['username']) + log.error(self.moodle.get_user_by('username',u['username'])) + # self.resync_data() + + def delete_all_moodle_cohorts(self): + cohorts=self.moodle.get_cohorts() + ids=[c['id'] for c in cohorts] + self.moodle.delete_cohorts(ids) + + def sync_to_nextcloud_external(self): + groups=[] + for u in self.external['users']: + groups=groups+u['gids'] + groups=list(dict.fromkeys(groups)) + + total=len(groups) + i=0 + ev=Events('Syncing groups from external to nextcloud',total=len(groups)) + + for g in groups: + parts=g.split('.') + if not len(parts): + log.error(' NEXTCLOUD GROUPS: Group '+g+ ' empty') + continue + subpath=parts[0] + for i in range(1,len(parts)): + try: + log.warning(' NEXTCLOUD GROUPS: Adding group as cohort ('+str(i)+'/'+str(total)+'): '+subpath) + ev.increment({'name':subpath}) + # if parts[i] in ['admin','manager','teacher','student']: + self.nextcloud.add_group(subpath) + if len(parts) != i+1: + subpath=subpath+'.'+parts[i+1] + except: + log.error(' NEXTCLOUD GROUPS: Group '+subpath+ ' probably already exists') + + + ev=Events('Syncing users from external to nextcloud',total=len(self.internal['users'])) + for u in self.external['users']: + log.warning(' NEXTCLOUD USERS: Creating nextcloud user: '+u['username']+' in groups '+str(u['gids'])) + try: + ev.increment({'name':u['username']}) + self.nextcloud.add_user_with_groups(u['username'],'*12'+secrets.token_urlsafe(16),u['quota'],u['gids'],u['email'],u['first']+' '+u['last']) + except ProviderItemExists: + log.warning('User '+u['username']+' already exists. Skipping...') + continue + except: + log.error(traceback.format_exc()) + def sync_to_moodle(self): # works from the internal (keycloak) ### Process all groups from the users keycloak_groups key groups=[] @@ -678,7 +811,7 @@ class Admin(): if u['first'] == '': u['first']=' ' if u['last'] == '': u['last']=' ' try: - self.moodle.create_user(u['email'],u['username'],secrets.token_urlsafe(16),u['first'],u['last'])[0] + self.moodle.create_user(u['email'],u['username'],'*12'+secrets.token_urlsafe(16),u['first'],u['last'])[0] except: log.error(' -->> Error creating on moodle the user: '+u['username']) # user_id=user['id'] @@ -711,11 +844,6 @@ class Admin(): index=index+1 self.resync_data() - def delete_all_moodle_cohorts(self): - cohorts=self.moodle.get_cohorts() - ids=[c['id'] for c in cohorts] - self.moodle.delete_cohorts(ids) - def sync_to_nextcloud(self): groups=[] for u in self.internal['users']: @@ -744,7 +872,7 @@ class Admin(): log.warning(' NEXTCLOUD USERS: Creating nextcloud user: '+u['username']+' in groups '+str(u['keycloak_groups'])) try: ev.increment({'name':u['username']}) - self.nextcloud.add_user_with_groups(u['username'],secrets.token_urlsafe(16),"500 GB",u['keycloak_groups'],u['email'],u['first']+' '+u['last']) + self.nextcloud.add_user_with_groups(u['username'],'*12'+secrets.token_urlsafe(16),"500 GB",u['keycloak_groups'],u['email'],u['first']+' '+u['last']) except ProviderItemExists: log.warning('User '+u['username']+' already exists. Skipping...') continue @@ -864,6 +992,13 @@ class Admin(): for externaluser in self.external['users']: if externaluser['id'] == newuserid: externaluser['roles']=[data['action']] + for role in ['admin','manager','teacher','student']: + try: + externaluser['gids'].remove(role) + # externaluser['groups'].remove('/'+role) + except: + pass + externaluser['gids'].append(data['action']) return True def user_update_password(self,userid,password,temporary): @@ -1057,7 +1192,7 @@ class Admin(): return True - def add_moodle_user(self, username,email,first_name,last_name,password='*'+secrets.token_urlsafe(16)): + def add_moodle_user(self, username,email,first_name,last_name,password='*12'+secrets.token_urlsafe(16)): log.warning('Creating moodle user: '+username) ev=Events('Add user',username) try: @@ -1097,7 +1232,7 @@ class Admin(): for group in nadd: self.nextcloud.add_user_to_group(user['username'],group) - def add_nextcloud_user(self,username,email,quota,first_name,last_name,groups,password=secrets.token_urlsafe(16)): + def add_nextcloud_user(self,username,email,quota,first_name,last_name,groups,password='*12'+secrets.token_urlsafe(16)): log.warning(' NEXTCLOUD USERS: Creating nextcloud user: '+username+' in groups '+str(groups)) ev=Events('Add user',username) try: @@ -1209,7 +1344,7 @@ class Admin(): log.warning(' NEXTCLOUD USERS: Creating nextcloud user: '+u['username']+' in groups '+str(list)) try: # Quota is in MB - self.nextcloud.add_user_with_groups(u['username'],secrets.token_urlsafe(16),u['quota'],pathslist,u['email'],u['first']+' '+u['last']) + self.nextcloud.add_user_with_groups(u['username'],'*12'+secrets.token_urlsafe(16),u['quota'],pathslist,u['email'],u['first']+' '+u['last']) ev.increment({'name':'Added to nextcloud','data':[]}) except ProviderItemExists: log.warning('User '+u['username']+' already exists. Skipping...') diff --git a/admin/src/admin/lib/keycloak_client.py b/admin/src/admin/lib/keycloak_client.py index 8842d1d..3820e83 100644 --- a/admin/src/admin/lib/keycloak_client.py +++ b/admin/src/admin/lib/keycloak_client.py @@ -286,23 +286,39 @@ class KeycloakClient(): def add_group_tree(self,path): parts=path.split('/') - parent_path=None + parent_path='/' for i in range(1,len(parts)): - # print('Adding group name '+parts[i]+' with parent path '+str(parent_path)) - try: - self.add_group(parts[i],parent_path,skip_exists=True) - except: - if parent_path==None: - parent_path='/'+parts[i] - else: - parent_path=self.get_group_by_path(parent_path)['path'] - parent_path=parent_path+'/'+parts[i] - continue - - if parent_path==None: - parent_path='/'+parts[i] + if i == 1: + try: + self.add_group(parts[i],None,skip_exists=True) + except: + log.warning('KEYCLOAK: Group :'+parts[i]+ ' already exists.') + parent_path=parent_path+parts[i] else: - parent_path=parent_path+'/'+parts[i] + try: + self.add_group(parts[i],parent_path,skip_exists=True) + except: + log.warning('KEYCLOAK: Group :'+parts[i]+ ' already exists.') + parent_path=parent_path+parts[i] + + # parts=path.split('/') + # parent_path=None + # for i in range(1,len(parts)): + # # print('Adding group name '+parts[i]+' with parent path '+str(parent_path)) + # try: + # self.add_group(parts[i],parent_path,skip_exists=True) + # except: + # if parent_path==None: + # parent_path='/'+parts[i] + # else: + # parent_path=self.get_group_by_path(parent_path)['path'] + # parent_path=parent_path+'/'+parts[i] + # continue + + # if parent_path==None: + # parent_path='/'+parts[i] + # else: + # parent_path=parent_path+'/'+parts[i] # try: # if i == 1: parent_id=self.add_group(parts[i]) diff --git a/admin/src/admin/static/js/sysadmin/external.js b/admin/src/admin/static/js/sysadmin/external.js index c63aa30..f9345fd 100644 --- a/admin/src/admin/static/js/sysadmin/external.js +++ b/admin/src/admin/static/js/sysadmin/external.js @@ -77,7 +77,7 @@ $(document).ready(function() { $('.btn-download').on('click', function () { data=users_table.rows().data() - csv_data='TYPE,EXTID,EMAIL,FIRST,LSAT,USERNAME,GROUPS,ROLE,PASSWORD'+ '\r\n' + csv_data='TYPE,EXT_ID,EMAIL,FIRST,LAST,USERNAME,PATHS,GROUPS,QUOTA,ROLE,PASS_TEMP,PASSWORD'+ '\r\n' csv_data=csv_data+convertToCSV(data) console.log(csv_data) exportCSVFile(csv_data, 'users_data') @@ -105,10 +105,10 @@ $(document).ready(function() { // users_table.ajax.reload(); // groups_table.ajax.reload(); }, - error: function(data) - { - console.log('Ajax timeout') - } + error: function(xhr, status, error) { + var err = eval("(" + xhr.responseText + ")"); + alert(JSON.parse(xhr.responseText).msg) + } }); } }); @@ -193,8 +193,10 @@ $(document).ready(function() { { "data": "first", "width": "10px"}, { "data": "last", "width": "10px"}, { "data": "email", "width": "10px"}, + { "data": "gids", "width": "10px"}, { "data": "groups", "width": "10px"}, { "data": "roles", "width": "10px"}, + { "data": "quota", "width": "10px"}, { "data": "password", "width": "10px"}, ], "order": [[3, 'asc']], @@ -207,6 +209,11 @@ $(document).ready(function() { { "targets": 6, "render": function ( data, type, full, meta ) { + return "
  • " + full.gids.join("
  • ") + "
    • " + }}, + { + "targets": 7, + "render": function ( data, type, full, meta ) { return "
  • " + full.groups.join("
  • ") + "
    • " }} ] diff --git a/admin/src/admin/static/templates/pages/sysadmin/external.html b/admin/src/admin/static/templates/pages/sysadmin/external.html index d809327..597973a 100644 --- a/admin/src/admin/static/templates/pages/sysadmin/external.html +++ b/admin/src/admin/static/templates/pages/sysadmin/external.html @@ -47,8 +47,10 @@ First Last email - groups + gids + paths roles + quota password diff --git a/admin/src/admin/static/templates/sidebar.html b/admin/src/admin/static/templates/sidebar.html index c3efcf7..6447c29 100644 --- a/admin/src/admin/static/templates/sidebar.html +++ b/admin/src/admin/static/templates/sidebar.html @@ -16,6 +16,7 @@
  • Users
  • Groups
  • Roles
    • +
  • Import
    • {% if current_user.role == 'admin' %}

    System Admin

    diff --git a/admin/src/admin/views/ApiViews.py b/admin/src/admin/views/ApiViews.py index a39756a..8ddb7b0 100644 --- a/admin/src/admin/views/ApiViews.py +++ b/admin/src/admin/views/ApiViews.py @@ -6,7 +6,7 @@ import traceback from uuid import uuid4 import time,json -import sys,os +import sys,os,re from flask import render_template, Response, request, redirect, url_for, jsonify import concurrent.futures from flask_login import current_user, login_required @@ -198,9 +198,13 @@ def external(): threads['external'].start() return json.dumps({}), 200, {'Content-Type': 'application/json'} if data['format']=='csv-ug': - threads['external'] = threading.Thread(target=app.admin.upload_csv_ug, args=(data,)) - threads['external'].start() - return json.dumps({}), 200, {'Content-Type': 'application/json'} + valid = check_upload_errors(data) + if valid['pass']: + threads['external'] = threading.Thread(target=app.admin.upload_csv_ug, args=(data,)) + threads['external'].start() + return json.dumps({}), 200, {'Content-Type': 'application/json'} + else: + return json.dumps(valid), 422, {'Content-Type': 'application/json'} if request.method == 'PUT': data=request.get_json(force=True) threads['external'] = threading.Thread(target=app.admin.sync_external, args=(data,)) @@ -227,3 +231,36 @@ def external_groups_list(): def external_roles(): if request.method == 'PUT': return json.dumps(app.admin.external_roleassign(request.get_json(force=True))), 200, {'Content-Type': 'application/json'} + +{'groups': '/alumnes/3er', +'firstname': 'Andreu', +'lastname': 'B', +'email': '12andreub@escolamontseny.cat', +'username': '12andreub', +'password': 'pepinillo', +'password_temporal': 'yes', +'role': 'student', +'quota': '500 MB', +'': '', +'id': '12andreub'} + + +def check_upload_errors(data): + email_regex = r'\b[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Z|a-z]{2,}\b' + for u in data['data']: + try: + user_groups=[g.strip() for g in u['groups'].split(',')] + except: + return {'pass':False,'msg':'User '+u['username']+' has invalid groups: '+u['groups']} + + if not re.fullmatch(email_regex, u['email']): + return {'pass':False,'msg':'User '+u['username']+' has invalid email: '+u['email']} + + if u['role'] not in ['admin','manager','teacher','student']: + if u['role'] == '': + return {'pass':False,'msg':'User '+u['username']+' has no role assigned!'} + return {'pass':False,'msg':'User '+u['username']+' has invalid role: '+u['role']} + + if u['password_temporal'].lower() not in ['yes','no']: + return {'pass':False,'msg':'User '+u['username']+' has invalid password_temporal value (yes/no): '+u['password_temporal']} + return {'pass':True,'msg':''} \ No newline at end of file