Merge branch 'new-csv-import' into 'develop'

Fixed import from CSV

See merge request isard/isard-sso!51
Josep Maria Viñolas Auquer 2021-09-14 18:14:12 +00:00
commit ce50a22638
6 changed files with 249 additions and 51 deletions

View File

@ -497,9 +497,28 @@ class Admin():
groups=[]
for u in data['data']:
log.warning('Processing ('+str(item)+'/'+str(total)+') uploaded user: '+u['username'])
user_groups=["/" + g.strip() for g in u['groups'].split(',')]
user_groups=[g.strip() for g in u['groups'].split(',')]
pathslist=[]
for group in user_groups:
pathpart=''
for part in kpath2gid(group).split('.'):
if pathpart=='':
pathpart=part
else:
pathpart=pathpart+'.'+part
pathslist.append(pathpart)
pathslist.append(u['role'])
groups=groups+user_groups
# pprint(u)
if u['quota'] == '':
u['quota'] = '500 MB'
if u['role'] == 'student': u['quota']='500MB'
if u['role'] == 'teacher': u['quota']='5 GB'
if u['role'] == 'manager': u['quota']='Unlimited'
if u['role'] == 'admin': u['quota']='Unlimited'
users.append({'provider':'external',
'id':u['id'].strip(),
'email': u['email'].strip(),
@ -507,20 +526,22 @@ class Admin():
'last': u['lastname'].strip(),
'username': u['username'].strip(),
'groups':user_groups,
'gids': pathslist,
'quota': u['quota'],
'roles':[u['role'].strip()],
'password': self.get_dice_pwd()})
'temporary': True if u['password_temporal'].lower() == 'yes' else False,
'password': self.get_dice_pwd() if u['password']=='' else u['password']})
item+=1
ev.increment({'name':u['username'].split('@')[0]})
ev.increment({'name':u['username']})
self.external['users']=users
groups=list(dict.fromkeys(groups))
sysgroups=[]
for g in groups:
sysgroups.append({'provider':'external',
"id": g,
"mailid": g,
"name": g,
"name": kpath2gid(g),
"path": g,
"description": 'Imported with csv'})
self.external['groups']=sysgroups
return True
@ -584,16 +605,18 @@ class Admin():
return True
def sync_external(self,ids):
# self.resync_data()
log.warning('Starting sync to keycloak')
self.sync_to_keycloak()
self.sync_to_keycloak_external()
### Now we only sycn external to keycloak and then they can be updated to others with UI buttons
log.warning('Starting sync to moodle')
self.sync_to_moodle()
self.sync_to_moodle_external()
log.warning('Starting sync to nextcloud')
self.sync_to_nextcloud()
log.warning('All syncs finished')
self.sync_to_nextcloud_external()
log.warning('All syncs finished. Resyncing from apps...')
self.resync_data()
def sync_to_keycloak(self): ### This one works from the external, moodle and nextcloud from the internal
def sync_to_keycloak_external(self): ### This one works from the external, moodle and nextcloud from the internal
groups=[]
for u in self.external['users']:
groups=groups+u['groups']
@ -606,6 +629,7 @@ class Admin():
i=i+1
log.warning(' KEYCLOAK GROUPS: Adding group ('+str(i)+'/'+str(total)+'): '+g)
ev.increment({'name':g})
print(g)
self.keycloak.add_group_tree(g)
total=len(self.external['users'])
@ -616,14 +640,14 @@ class Admin():
# Add user
log.warning(' KEYCLOAK USERS: Adding user ('+str(index)+'/'+str(total)+'): '+u['username'])
ev.increment({'name':u['username'],'data':u})
uid=self.keycloak.add_user(u['username'],u['first'],u['last'],u['email'],u['password'])
uid=self.keycloak.add_user(u['username'],u['first'],u['last'],u['email'],u['password'],temporary=u['temporary'])
self.av.add_user_default_avatar(uid,u['roles'][0])
# Add user to role and group rolename
if len(u['roles']) != 0:
log.warning(' KEYCLOAK USERS: Assign user '+u['username']+' with initial pwd '+ u['password']+' to role '+u['roles'][0])
self.keycloak.assign_realm_roles(uid,u['roles'][0])
gid=self.keycloak.get_group_by_path(path='/'+u['roles'][0])['id']
self.keycloak.group_user_add(uid,gid)
gid=self.keycloak.get_group_by_path('/'+u['roles'][0])['id']
# self.keycloak.group_user_add(uid,gid)
# Add user to groups
for g in u['groups']:
parts=g.split('/')
@ -635,10 +659,119 @@ class Admin():
sub=sub+'/'+parts[i]
if sub=='/': continue # User with no path
log.warning(' KEYCLOAK USERS: Assign user '+u['username']+' to group '+ str(sub))
gid=self.keycloak.get_group_by_path(path=sub)['id']
kuser=self.keycloak.get_group_by_path(path=sub)
gid=kuser['id']
self.keycloak.group_user_add(uid,gid)
# We add it as it is needed for moodle and nextcloud
u['groups'].append(u['roles'][0])
self.resync_data()
def sync_to_moodle_external(self): # works from the internal (keycloak)
### Process all groups from the users keycloak_groups key
groups=[]
for u in self.external['users']:
groups=groups+u['gids']
groups=list(dict.fromkeys(groups))
### Create all groups. Skip / in system groups
total=len(groups)
ev=Events('Syncing groups from external to moodle',total=len(groups))
for g in groups:
parts=g.split('.')
if not len(parts):
log.error(' MOODLE GROUPS: Group '+g+ ' empty')
continue
subpath=parts[0]
for i in range(1,len(parts)):
try:
log.warning(' MOODLE GROUPS: Adding group as cohort ('+str(i)+'/'+str(total)+'): '+subpath)
ev.increment({'name':subpath})
# if parts[i] in ['admin','manager','teacher','student']:
self.moodle.add_system_cohort(subpath)
if len(parts) != i+1:
subpath=subpath+'.'+parts[i+1]
except:
log.error(' MOODLE GROUPS: Group '+subpath+ ' probably already exists')
### Get all existing moodle cohorts
cohorts=self.moodle.get_cohorts()
### Create users in moodle
ev=Events('Syncing users from external to moodle',total=len(self.internal['users']))
for u in self.external['users']:
log.warning('MOODLE: Creating moodle user: '+u['username'])
ev.increment({'name':u['username']})
if u['first'] == '': u['first']='-'
if u['last'] == '': u['last']='-'
try:
self.moodle.create_user(u['email'],u['username'],'*12'+secrets.token_urlsafe(16),u['first'],u['last'])[0]
except UserExists:
log.warning('MOODLE:The user: '+u['username']+' already exsits.')
except:
log.error(' -->> Error creating on moodle the user: '+u['username'])
log.error(traceback.format_exc())
# user_id=user['id']
# self.resync_data()
### Add user to their cohorts (groups)
ev=Events('Syncing users groups from external to moodle cohorts',total=len(self.internal['users']))
cohorts=self.moodle.get_cohorts()
for u in self.external['users']:
try:
uid=self.moodle.get_user_by('username',u['username'])['users'][0]['id']
for group in u['gids']:
cohort=[c for c in cohorts if c['name']==group][0]
self.moodle.add_user_to_cohort(uid,cohort['id'])
except:
log.error('Exception on getting user from moodle: '+u['username'])
log.error(self.moodle.get_user_by('username',u['username']))
# self.resync_data()
def delete_all_moodle_cohorts(self):
cohorts=self.moodle.get_cohorts()
ids=[c['id'] for c in cohorts]
self.moodle.delete_cohorts(ids)
def sync_to_nextcloud_external(self):
groups=[]
for u in self.external['users']:
groups=groups+u['gids']
groups=list(dict.fromkeys(groups))
total=len(groups)
i=0
ev=Events('Syncing groups from external to nextcloud',total=len(groups))
for g in groups:
parts=g.split('.')
if not len(parts):
log.error(' NEXTCLOUD GROUPS: Group '+g+ ' empty')
continue
subpath=parts[0]
for i in range(1,len(parts)):
try:
log.warning(' NEXTCLOUD GROUPS: Adding group as cohort ('+str(i)+'/'+str(total)+'): '+subpath)
ev.increment({'name':subpath})
# if parts[i] in ['admin','manager','teacher','student']:
self.nextcloud.add_group(subpath)
if len(parts) != i+1:
subpath=subpath+'.'+parts[i+1]
except:
log.error(' NEXTCLOUD GROUPS: Group '+subpath+ ' probably already exists')
ev=Events('Syncing users from external to nextcloud',total=len(self.internal['users']))
for u in self.external['users']:
log.warning(' NEXTCLOUD USERS: Creating nextcloud user: '+u['username']+' in groups '+str(u['gids']))
try:
ev.increment({'name':u['username']})
self.nextcloud.add_user_with_groups(u['username'],'*12'+secrets.token_urlsafe(16),u['quota'],u['gids'],u['email'],u['first']+' '+u['last'])
except ProviderItemExists:
log.warning('User '+u['username']+' already exists. Skipping...')
continue
except:
log.error(traceback.format_exc())
def sync_to_moodle(self): # works from the internal (keycloak)
### Process all groups from the users keycloak_groups key
groups=[]
@ -678,7 +811,7 @@ class Admin():
if u['first'] == '': u['first']=' '
if u['last'] == '': u['last']=' '
try:
self.moodle.create_user(u['email'],u['username'],secrets.token_urlsafe(16),u['first'],u['last'])[0]
self.moodle.create_user(u['email'],u['username'],'*12'+secrets.token_urlsafe(16),u['first'],u['last'])[0]
except:
log.error(' -->> Error creating on moodle the user: '+u['username'])
# user_id=user['id']
@ -711,11 +844,6 @@ class Admin():
index=index+1
self.resync_data()
def delete_all_moodle_cohorts(self):
cohorts=self.moodle.get_cohorts()
ids=[c['id'] for c in cohorts]
self.moodle.delete_cohorts(ids)
def sync_to_nextcloud(self):
groups=[]
for u in self.internal['users']:
@ -744,7 +872,7 @@ class Admin():
log.warning(' NEXTCLOUD USERS: Creating nextcloud user: '+u['username']+' in groups '+str(u['keycloak_groups']))
try:
ev.increment({'name':u['username']})
self.nextcloud.add_user_with_groups(u['username'],secrets.token_urlsafe(16),"500 GB",u['keycloak_groups'],u['email'],u['first']+' '+u['last'])
self.nextcloud.add_user_with_groups(u['username'],'*12'+secrets.token_urlsafe(16),"500 GB",u['keycloak_groups'],u['email'],u['first']+' '+u['last'])
except ProviderItemExists:
log.warning('User '+u['username']+' already exists. Skipping...')
continue
@ -864,6 +992,13 @@ class Admin():
for externaluser in self.external['users']:
if externaluser['id'] == newuserid:
externaluser['roles']=[data['action']]
for role in ['admin','manager','teacher','student']:
try:
externaluser['gids'].remove(role)
# externaluser['groups'].remove('/'+role)
except:
pass
externaluser['gids'].append(data['action'])
return True
def user_update_password(self,userid,password,temporary):
@ -1057,7 +1192,7 @@ class Admin():
return True
def add_moodle_user(self, username,email,first_name,last_name,password='*'+secrets.token_urlsafe(16)):
def add_moodle_user(self, username,email,first_name,last_name,password='*12'+secrets.token_urlsafe(16)):
log.warning('Creating moodle user: '+username)
ev=Events('Add user',username)
try:
@ -1097,7 +1232,7 @@ class Admin():
for group in nadd:
self.nextcloud.add_user_to_group(user['username'],group)
def add_nextcloud_user(self,username,email,quota,first_name,last_name,groups,password=secrets.token_urlsafe(16)):
def add_nextcloud_user(self,username,email,quota,first_name,last_name,groups,password='*12'+secrets.token_urlsafe(16)):
log.warning(' NEXTCLOUD USERS: Creating nextcloud user: '+username+' in groups '+str(groups))
ev=Events('Add user',username)
try:
@ -1209,7 +1344,7 @@ class Admin():
log.warning(' NEXTCLOUD USERS: Creating nextcloud user: '+u['username']+' in groups '+str(list))
try:
# Quota is in MB
self.nextcloud.add_user_with_groups(u['username'],secrets.token_urlsafe(16),u['quota'],pathslist,u['email'],u['first']+' '+u['last'])
self.nextcloud.add_user_with_groups(u['username'],'*12'+secrets.token_urlsafe(16),u['quota'],pathslist,u['email'],u['first']+' '+u['last'])
ev.increment({'name':'Added to nextcloud','data':[]})
except ProviderItemExists:
log.warning('User '+u['username']+' already exists. Skipping...')

View File

@ -286,23 +286,39 @@ class KeycloakClient():
def add_group_tree(self,path):
parts=path.split('/')
parent_path=None
parent_path='/'
for i in range(1,len(parts)):
# print('Adding group name '+parts[i]+' with parent path '+str(parent_path))
try:
self.add_group(parts[i],parent_path,skip_exists=True)
except:
if parent_path==None:
parent_path='/'+parts[i]
else:
parent_path=self.get_group_by_path(parent_path)['path']
parent_path=parent_path+'/'+parts[i]
continue
if parent_path==None:
parent_path='/'+parts[i]
if i == 1:
try:
self.add_group(parts[i],None,skip_exists=True)
except:
log.warning('KEYCLOAK: Group :'+parts[i]+ ' already exists.')
parent_path=parent_path+parts[i]
else:
parent_path=parent_path+'/'+parts[i]
try:
self.add_group(parts[i],parent_path,skip_exists=True)
except:
log.warning('KEYCLOAK: Group :'+parts[i]+ ' already exists.')
parent_path=parent_path+parts[i]
# parts=path.split('/')
# parent_path=None
# for i in range(1,len(parts)):
# # print('Adding group name '+parts[i]+' with parent path '+str(parent_path))
# try:
# self.add_group(parts[i],parent_path,skip_exists=True)
# except:
# if parent_path==None:
# parent_path='/'+parts[i]
# else:
# parent_path=self.get_group_by_path(parent_path)['path']
# parent_path=parent_path+'/'+parts[i]
# continue
# if parent_path==None:
# parent_path='/'+parts[i]
# else:
# parent_path=parent_path+'/'+parts[i]
# try:
# if i == 1: parent_id=self.add_group(parts[i])

View File

@ -77,7 +77,7 @@ $(document).ready(function() {
$('.btn-download').on('click', function () {
data=users_table.rows().data()
csv_data='TYPE,EXTID,EMAIL,FIRST,LSAT,USERNAME,GROUPS,ROLE,PASSWORD'+ '\r\n'
csv_data='TYPE,EXT_ID,EMAIL,FIRST,LAST,USERNAME,PATHS,GROUPS,QUOTA,ROLE,PASS_TEMP,PASSWORD'+ '\r\n'
csv_data=csv_data+convertToCSV(data)
console.log(csv_data)
exportCSVFile(csv_data, 'users_data')
@ -105,10 +105,10 @@ $(document).ready(function() {
// users_table.ajax.reload();
// groups_table.ajax.reload();
},
error: function(data)
{
console.log('Ajax timeout')
}
error: function(xhr, status, error) {
var err = eval("(" + xhr.responseText + ")");
alert(JSON.parse(xhr.responseText).msg)
}
});
}
});
@ -193,8 +193,10 @@ $(document).ready(function() {
{ "data": "first", "width": "10px"},
{ "data": "last", "width": "10px"},
{ "data": "email", "width": "10px"},
{ "data": "gids", "width": "10px"},
{ "data": "groups", "width": "10px"},
{ "data": "roles", "width": "10px"},
{ "data": "quota", "width": "10px"},
{ "data": "password", "width": "10px"},
],
"order": [[3, 'asc']],
@ -207,6 +209,11 @@ $(document).ready(function() {
{
"targets": 6,
"render": function ( data, type, full, meta ) {
return "<li>" + full.gids.join("</li><li>") + "</li>"
}},
{
"targets": 7,
"render": function ( data, type, full, meta ) {
return "<li>" + full.groups.join("</li><li>") + "</li>"
}}
]

View File

@ -47,8 +47,10 @@
<th>First</th>
<th>Last</th>
<th>email</th>
<th>groups</th>
<th>gids</th>
<th>paths</th>
<th>roles</th>
<th>quota</th>
<th>password</th>
</tr>
</thead>

View File

@ -16,6 +16,7 @@
<li><a href="/users"><i class="fa fa-user"></i> Users</a></li>
<li><a href="/groups"><i class="fa fa-users"></i> Groups</a></li>
<li><a href="/roles"><i class="fa fa-user-secret"></i> Roles</a></li>
<li><a href="/sysadmin/external"><i class="fa fa-external-link"></i> Import</a></li>
{% if current_user.role == 'admin' %}
<h3>System Admin</h3>

View File

@ -6,7 +6,7 @@ import traceback
from uuid import uuid4
import time,json
import sys,os
import sys,os,re
from flask import render_template, Response, request, redirect, url_for, jsonify
import concurrent.futures
from flask_login import current_user, login_required
@ -198,9 +198,13 @@ def external():
threads['external'].start()
return json.dumps({}), 200, {'Content-Type': 'application/json'}
if data['format']=='csv-ug':
threads['external'] = threading.Thread(target=app.admin.upload_csv_ug, args=(data,))
threads['external'].start()
return json.dumps({}), 200, {'Content-Type': 'application/json'}
valid = check_upload_errors(data)
if valid['pass']:
threads['external'] = threading.Thread(target=app.admin.upload_csv_ug, args=(data,))
threads['external'].start()
return json.dumps({}), 200, {'Content-Type': 'application/json'}
else:
return json.dumps(valid), 422, {'Content-Type': 'application/json'}
if request.method == 'PUT':
data=request.get_json(force=True)
threads['external'] = threading.Thread(target=app.admin.sync_external, args=(data,))
@ -227,3 +231,36 @@ def external_groups_list():
def external_roles():
if request.method == 'PUT':
return json.dumps(app.admin.external_roleassign(request.get_json(force=True))), 200, {'Content-Type': 'application/json'}
{'groups': '/alumnes/3er',
'firstname': 'Andreu',
'lastname': 'B',
'email': '12andreub@escolamontseny.cat',
'username': '12andreub',
'password': 'pepinillo',
'password_temporal': 'yes',
'role': 'student',
'quota': '500 MB',
'': '',
'id': '12andreub'}
def check_upload_errors(data):
email_regex = r'\b[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Z|a-z]{2,}\b'
for u in data['data']:
try:
user_groups=[g.strip() for g in u['groups'].split(',')]
except:
return {'pass':False,'msg':'User '+u['username']+' has invalid groups: '+u['groups']}
if not re.fullmatch(email_regex, u['email']):
return {'pass':False,'msg':'User '+u['username']+' has invalid email: '+u['email']}
if u['role'] not in ['admin','manager','teacher','student']:
if u['role'] == '':
return {'pass':False,'msg':'User '+u['username']+' has no role assigned!'}
return {'pass':False,'msg':'User '+u['username']+' has invalid role: '+u['role']}
if u['password_temporal'].lower() not in ['yes','no']:
return {'pass':False,'msg':'User '+u['username']+' has invalid password_temporal value (yes/no): '+u['password_temporal']}
return {'pass':True,'msg':''}