diff --git a/admin/docker/requirements.pip3 b/admin/docker/requirements.pip3 index 07e82d0..c2479e8 100644 --- a/admin/docker/requirements.pip3 +++ b/admin/docker/requirements.pip3 @@ -23,4 +23,6 @@ diceware==0.9.6 #gevent==1.4.0 #greenlet==0.4.15 python-engineio==3.8.1 -python-socketio==4.1.0 \ No newline at end of file +python-socketio==4.1.0 + +minio==7.0.3 \ No newline at end of file diff --git a/admin/src/admin/lib/admin.py b/admin/src/admin/lib/admin.py index 26ffa09..ab3327d 100644 --- a/admin/src/admin/lib/admin.py +++ b/admin/src/admin/lib/admin.py @@ -18,13 +18,7 @@ options = diceware.handle_options(None) options.wordlist = 'cat_ascii' options.num = 3 - - -from flask_socketio import SocketIO, emit, join_room, leave_room, \ - close_room, rooms, disconnect, send -socketio = SocketIO(app) -from ..views.Socketio import socketio -# socketio = SocketIO(app) +from .events import Events class Admin(): def __init__(self): @@ -39,9 +33,6 @@ class Admin(): sleep(2) log.warning('Keycloak connected.') - # print(self.keycloak.get_user_groups_paths('320b0713-c480-4928-9105-041309d72191')) - # exit(1) - ready=False while not ready: try: @@ -73,7 +64,6 @@ class Admin(): self.default_setup() self.internal={} - # self.resync_data() ready=False while not ready: @@ -87,24 +77,11 @@ class Admin(): self.external={'users':[], 'groups':[], 'roles':[]} + log.warning(' Updating missing user avatars with defaults') + av=Avatars() + # av.minio_delete_all_objects() # This will reset all avatars on usres + av.update_missing_avatars(self.internal['users']) log.warning(' SYSTEM READY TO HANDLE CONNECTIONS') - # self.test_cohorts() - # self.delete_all_moodle_cohorts() - - ########## Testing: get user group - # cids=[c['id'] for c in self.moodle.get_cohorts()] - # relations=self.moodle.get_cohort_members(cids) - # for r in relations: - # print(self.get_internalgroup_from_moodlecohortid(r['cohortid'])['path']) - - # def get_internalgroup_from_moodlecohortid(self,cohort_id): - # for g in self.internal['groups']: - # if not g['moodle']: continue - # if g['moodle_id'] == cohort_id: return g - # return '' - - - ## This function should be moved to postup.py def default_setup(self): @@ -122,9 +99,9 @@ class Admin(): self.keycloak_admin.group_user_add(admin_uid,gid) log.warning('KEYCLOAK: OK') except: + # print(traceback.format_exc()) log.warning('KEYCLOAK: Seems to be there already') - #### Add default groups try: log.warning('KEYCLOAK: Adding default groups') @@ -240,37 +217,6 @@ class Admin(): except: log.warning('MOODLE: Seems to be there already') - - ### testing - # def test_cohorts(self): - # cohorts=self.moodle.get_cohorts() - - # testc=[c for c in cohorts if c['name'] in ['teacher','/teacher','student','/student']] - # pprint(testc) - - # groups=[] - # for u in self.internal['users']: - # groups=groups+u['keycloak_groups'] - # groups=list(dict.fromkeys(groups)) - # pprint(groups) - # pprint([g for g in groups if 'teacher' in g['keycloak_groups']]) - # exit(1) - # total=len(groups) - # i=0 - # for g in groups: - # parts=g.split('/') - # subpath='' - # for i in range(1,len(parts)): - # try: - # log.warning(' MOODLE GROUPS: Adding group as cohort ('+str(i)+'/'+str(total)+'): '+subpath) - # subpath=subpath+'/'+parts[i] - # self.moodle.add_system_cohort(subpath) - # except: - # log.error('probably exists') - # i=i+1 - - ### end testing - def resync_data(self): self.internal={'users':self._get_mix_users(), 'groups':self._get_mix_groups(), @@ -380,8 +326,7 @@ class Admin(): theuser['nextcloud_groups']=[] del theuser['groups'] users.append(theuser) - - # pprint([u['moodle_groups'] for u in users]) + return users def get_roles(self): @@ -465,27 +410,66 @@ class Admin(): def get_external_roles(self): return self.external['roles'] - def upload_json(self,data): + def upload_csv_ug(self,data): + log.warning('Processing uploaded users...') + users=[] + total=len(data['data']) + item=1 + ev=Events('Processing uploaded users',total=len(data['data'])) + groups=[] + for u in data['data']: + log.warning('Processing ('+str(item)+'/'+str(total)+') uploaded user: '+u['username']) + user_groups=["/" + g.strip() for g in u['groups'].split(',')] + + groups=groups+user_groups + users.append({'provider':'external', + 'id':u['id'].strip(), + 'email': u['email'].strip(), + 'first': u['firstname'].strip(), + 'last': u['lastname'].strip(), + 'username': u['username'].strip(), + 'groups':user_groups, + 'roles':[u['role'].strip()], + 'password':diceware.get_passphrase(options=options)}) + item+=1 + ev.increment({'name':u['username'].split('@')[0]}) + self.external['users']=users + + groups=list(dict.fromkeys(groups)) + + sysgroups=[] + for g in groups: + sysgroups.append({'provider':'external', + "id": g, + "mailid": g, + "name": g, + "description": 'Imported with csv'}) + self.external['groups']=sysgroups + return True + + def upload_json_ga(self,data): groups=[] log.warning('Processing uploaded groups...') + ev=Events('Processing uploaded groups',total=len(data['data']['groups'])) for g in data['data']['groups']: - # for m in data['data']['d_members']: - - # data['provider'] - groups.append({'provider':'external', - "id": g['id'], - "mailid": g['email'].split('@')[0], - "name": g['name'], - "description": g['description']}) + try: + ev.increment({'name':g['name']}) + groups.append({'provider':'external', + "id": g['id'], + "mailid": g['email'].split('@')[0], + "name": g['name'], + "description": g['description']}) + except: + pass self.external['groups']=groups log.warning('Processing uploaded users...') users=[] total=len(data['data']['users']) - i=1 + item=1 + ev=Events('Processing uploaded users',total=len(data['data']['users'])) for u in data['data']['users']: - log.warning('Processing ('+str(i)+'/'+str(total)+') uploaded user: '+u['primaryEmail'].split('@')[0]) - # data['provider'] + log.warning('Processing ('+str(item)+'/'+str(total)+') uploaded user: '+u['primaryEmail'].split('@')[0]) users.append({'provider':'external', 'id':u['id'], 'email': u['primaryEmail'], @@ -494,12 +478,9 @@ class Admin(): 'username': u['primaryEmail'].split('@')[0], 'groups':[u['orgUnitPath']], ## WARNING: Removing the first 'roles':[], - 'password':diceware.get_passphrase(options=options)}) - socketio.emit('update', - json.dumps({'status':True,'item':'user','action':'delete','itemdata':''}), - namespace='/isard-sso-admin/sio', - room='admin') - i=i+1 + 'password': diceware.get_passphrase(options=options)}) + item+=1 + ev.increment({'name':u['primaryEmail'].split('@')[0]}) self.external['users']=users ## Add groups to users (now they only have their orgUnitPath) @@ -513,6 +494,7 @@ class Admin(): def sync_external(self,ids): log.warning('Starting sync to keycloak') self.sync_to_keycloak() + ### Now we only sycn external to keycloak and then they can be updated to others with UI buttons # log.warning('Starting sync to moodle') # self.sync_to_moodle() # log.warning('Starting sync to nextcloud') @@ -527,18 +509,21 @@ class Admin(): total=len(groups) i=0 + ev=Events('Syncing import groups to keycloak',total=len(groups)) for g in groups: i=i+1 - # print('ADDING FULL PATH: '+str(g)) log.warning(' KEYCLOAK GROUPS: Adding group ('+str(i)+'/'+str(total)+'): '+g) + ev.increment({'name':g}) self.keycloak.add_group_tree(g) total=len(self.external['users']) index=0 + ev=Events('Syncing import users to keycloak',total=len(self.external['users'])) for u in self.external['users']: index=index+1 # Add user log.warning(' KEYCLOAK USERS: Adding user ('+str(index)+'/'+str(total)+'): '+u['username']) + ev.increment({'name':u['username']}) uid=self.keycloak.add_user(u['username'],u['first'],u['last'],u['email'],u['password']) # Add user to role and group rolename @@ -573,12 +558,14 @@ class Admin(): ### Create all groups. Skip / in system groups total=len(groups) i=0 + ev=Events('Syncing groups from keycloak to moodle',total=len(groups)) for g in groups: parts=g.split('/') subpath='' for i in range(1,len(parts)): try: log.warning(' MOODLE GROUPS: Adding group as cohort ('+str(i)+'/'+str(total)+'): '+subpath) + ev.increment({'name':subpath}) if parts[i] in ['admin','manager','teacher','student']: subpath=parts[i] else: @@ -592,19 +579,28 @@ class Admin(): cohorts=self.moodle.get_cohorts() ### Create users in moodle + ev=Events('Syncing users from keycloak to moodle',total=len(self.internal['users'])) for u in self.internal['users']: if not u['moodle']: - log.error('Creating moodle user: '+u['username']) - user=self.moodle.create_user(u['email'],u['username'],'1Random 1String',u['first'],u['last'])[0] + log.warning('Creating moodle user: '+u['username']) + ev.increment({'name':u['username']}) + if u['first'] == '': u['first']=' ' + if u['last'] == '': u['last']=' ' + try: + # pprint(u) + pprint(self.moodle.create_user(u['email'],u['username'],'1Random 1String',u['first'],u['last'])[0]) + except: + log.error(' -->> Error creating on moodle the user: '+u['username']) # user_id=user['id'] self.resync_data() ### Add user to their cohorts (groups) + ev=Events('Syncing users groups from keycloak to moodle cohorts',total=len(self.internal['users'])) for u in self.internal['users']: total=len(u['keycloak_groups']) index=0 + ev.increment({'name':u['username']}) for g in u['keycloak_groups']: - log.error('for user groups') parts=g.split('/') subpath='' # pprint(parts) @@ -617,7 +613,6 @@ class Admin(): try: cohort=[c for c in cohorts if c['name']==subpath][0] except: - # pprint(subpath) log.error(' MOODLE USER GROUPS: keycloak group '+subpath+' does not exist as moodle cohort. This should not happen. User '+u['username']+ ' not added.') try: @@ -640,25 +635,26 @@ class Admin(): total=len(groups) i=0 + ev=Events('Syncing groups from keycloak to nextcloud',total=len(groups)) for g in groups: parts=g.split('/') subpath='' for i in range(1,len(parts)): try: log.warning(' NEXTCLOUD GROUPS: Adding group ('+str(i)+'/'+str(total)+'): '+subpath) + ev.increment({'name':subpath}) subpath=subpath+'/'+parts[i] self.nextcloud.add_group(subpath) except: log.error('probably exists') i=i+1 + ev=Events('Syncing users from keycloak to nextcloud',total=len(self.internal['users'])) for u in self.internal['users']: - # print('User '+u['username']) if not u['nextcloud']: - # print(' Is not in nextcloud') log.warning(' NEXTCLOUD USERS: Creating nextcloud user: '+u['username']+' in groups '+str(u['keycloak_groups'])) - # group=u['keycloak_groups'][0] if len(u['keycloak_groups']) else False try: + ev.increment({'name':u['username']}) self.nextcloud.add_user_with_groups(u['username'],'1Random 1String',500000000000,u['keycloak_groups'],u['email'],u['first']+' '+u['last']) except ProviderItemExists: log.warning('User '+u['username']+' already exists. Skipping...') @@ -673,12 +669,11 @@ class Admin(): i=i+1 if not u['keycloak']: continue # Do not remove admin users!!! What to do with managers??? - if 'admin' in u['roles']: continue - # if 'manager' in u['roles']: continue + if ['admin'] in u['roles']: continue log.info(' KEYCLOAK USERS: Removing user ('+str(i)+'/'+str(total)+'): '+u['username']) try: self.keycloak.delete_user(u['id']) - socketio.emit('update', + app.socketio.emit('update', json.dumps({'status':True,'item':'user','action':'delete','itemdata':u}), namespace='/isard-sso-admin/sio', room='admin') @@ -692,7 +687,7 @@ class Admin(): log.info('Removing nextcloud user: '+u['username']) try: self.nextcloud.delete_user(u['nextcloud_id']) - socketio.emit('update', + app.socketio.emit('update', json.dumps({'status':True,'item':'user','action':'delete','itemdata':u}), namespace='/isard-sso-admin/sio', room='admin') @@ -711,7 +706,7 @@ class Admin(): log.warning('Removing moodle users: '+','.join(usernames)) try: self.moodle.delete_users(userids) - socketio.emit('update', + app.socketio.emit('update', json.dumps({'status':True,'item':'user','action':'delete','itemdata':u}), namespace='/isard-sso-admin/sio', room='admin') diff --git a/admin/src/admin/lib/avatars.py b/admin/src/admin/lib/avatars.py index 1db2318..36a0cf3 100644 --- a/admin/src/admin/lib/avatars.py +++ b/admin/src/admin/lib/avatars.py @@ -4,30 +4,52 @@ from admin import app import logging as log from pprint import pprint import os -from os import listdir -from os.path import isfile, join -from .postgres import Postgres - -# Module variables to connect to moodle api +from minio import Minio +from minio.commonconfig import REPLACE, CopySource +from minio.deleteobjects import DeleteObject class Avatars(): - def __init__(self): - self.keycloak_pg=Postgres('isard-apps-postgresql','keycloak',os.environ['KEYCLOAK_DB_USER'],os.environ['KEYCLOAK_DB_PASSWORD']) + self.mclient = Minio( + "isard-sso-avatars:9000", + access_key="AKIAIOSFODNN7EXAMPLE", + secret_key="wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY", + secure=False + ) + self.bucket='master-avatars' + self._minio_set_realm() + # self.update_missing_avatars() - def username2id(self,username): - q = """select id, username from user_entity where username = '%s'""" % (username) - try: - return self.keycloak_pg.select(q)[0][0] - except: - pass - return False + def update_missing_avatars(self,users): + sys_roles=['admin','manager','teacher','student'] + for u in self.get_users_without_image(users): + try: + img=[r+'.jpg' for r in sys_roles if r in u['roles']][0] + except: + img='unknown.jpg' - def get_files(self): - path='avatars/master-avatars/' - onlyfiles = [f for f in listdir(path) if isfile(join(path, f))] - pprint(onlyfiles) + self.mclient.fput_object( + self.bucket, u['id'], os.path.join(app.root_path,"../custom/avatars/"+img), + content_type="image/jpeg ", + ) + log.warning(' AVATARS: Updated avatar for user '+u['username']+' with role '+img.split('.')[0]) - # def generate_missing(self, users): - # for u in users: + def _minio_set_realm(self): + if not self.mclient.bucket_exists(self.bucket): + self.mclient.make_bucket(self.bucket) + + def minio_get_objects(self): + return [o.object_name for o in self.mclient.list_objects(self.bucket)] + + def minio_delete_all_objects(self): + delete_object_list = map( + lambda x: DeleteObject(x.object_name), + self.mclient.list_objects(self.bucket), + ) + errors=self.mclient.remove_objects(self.bucket, delete_object_list) + for error in errors: + log.error(" AVATARS: Error occured when deleting avatar object", error) + + def get_users_without_image(self,users): + return [u for u in users if u['id'] not in self.minio_get_objects()] \ No newline at end of file diff --git a/admin/src/admin/lib/events.py b/admin/src/admin/lib/events.py new file mode 100644 index 0000000..6f622e4 --- /dev/null +++ b/admin/src/admin/lib/events.py @@ -0,0 +1,73 @@ +#!flask/bin/python +# coding=utf-8 +from admin import app +import logging as log +import traceback + +from uuid import uuid4 +import json +from time import sleep +import sys,os +from flask import render_template, Response, request, redirect, url_for, jsonify +from flask_socketio import SocketIO, emit, join_room, leave_room, \ + close_room, rooms, disconnect, send +import base64 + +class Events(): + def __init__(self,title,text='',total=0): + self.eid=str(base64.b64encode(os.urandom(32))[:8]) + self.title=title + self.text=text + self.total=total + self.item=0 + self.create() + + def create(self): + app.socketio.emit('notify-create', + json.dumps({'id':self.eid, + 'title':self.title, + 'text':self.text}), + namespace='/isard-sso-admin/sio', + room='admin') + sleep(0.1) + + def __del__(self): + app.socketio.emit('notify-destroy', + json.dumps({'id':self.eid}), + namespace='/isard-sso-admin/sio', + room='admin') + sleep(0.1) + + def update_text(self,text): + self.text=text + app.socketio.emit('notify-update', + json.dumps({'id':self.eid, + 'text':self.text,}), + namespace='/isard-sso-admin/sio', + room='admin') + sleep(0.1) + + def increment(self,data={}): + self.item+=1 + app.socketio.emit('notify-update', + json.dumps({'id':self.eid, + 'text': '['+str(self.item)+'/'+str(self.total)+'] ', + 'item':self.item, + 'total':self.total, + 'data':data}), + namespace='/isard-sso-admin/sio', + room='admin') + sleep(0.1) + + def decrement(self,data={}): + self.item-=1 + app.socketio.emit('notify-update', + json.dumps({'id':self.eid, + 'text':'['+str(self.item)+'/'+str(self.total)+'] ', + 'item':self.item, + 'total':self.total, + 'data':data}), + namespace='/isard-sso-admin/sio', + room='admin') + sleep(0.1) + diff --git a/admin/src/admin/static/js/external.js b/admin/src/admin/static/js/external.js index 932d44b..a01a8b1 100644 --- a/admin/src/admin/static/js/external.js +++ b/admin/src/admin/static/js/external.js @@ -17,7 +17,7 @@ $(document).ready(function() { $.each(users_table.rows().data(),function(key, value){ ids[value['id']]=value['roles'] }); - console.log(ids) + // console.log(ids) $.ajax({ type: "PUT", url:"/isard-sso-admin/external", @@ -46,25 +46,30 @@ $(document).ready(function() { }) $("#modalImport #send").on('click', function(e){ - console.log(users_table.rows().data()) + // console.log(users_table.rows().data()) var form = $('#modalImportForm'); form.parsley().validate(); if (form.parsley().isValid()){ formdata = form.serializeObject() - formdata['data']=JSON.parse(filecontents) + if($('#format').val() == 'csv-ug'){ + formdata['data']=parseCSV(filecontents) + }else{ + formdata['data']=JSON.parse(filecontents) + } $.ajax({ type: "POST", url:"/isard-sso-admin/external", data: JSON.stringify(formdata), success: function(data) { - $("#modalImport").modal('hide'); - users_table.ajax.reload(); - groups_table.ajax.reload(); + console.log('SUCCESS') + // $("#modalImport").modal('hide'); + // users_table.ajax.reload(); + // groups_table.ajax.reload(); }, error: function(data) { - alert('Something went wrong on our side...') + console.log('Ajax timeout') } }); } @@ -194,20 +199,46 @@ $(document).ready(function() { }); function readFile (evt) { - path = ""; - items = []; - var files = evt.target.files; - var file = files[0]; - var reader = new FileReader(); - reader.onload = function(event) { - filecontents=event.target.result; - $.each(JSON.parse(filecontents), walker); - populate_path(items) + if($('#format').val() == 'json-ga'){ + path = ""; + items = []; + var files = evt.target.files; + var file = files[0]; + var reader = new FileReader(); + reader.onload = function(event) { + filecontents=event.target.result; + $.each(JSON.parse(filecontents), walker); + populate_path(items) + } + reader.readAsText(file, 'UTF-8') + } + if($('#format').val() == 'csv-ug'){ + var files = evt.target.files; + var file = files[0]; + var reader = new FileReader(); + reader.onload = function(event) { + filecontents=event.target.result; + // $.each(JSON.parse(filecontents), walker); + // populate_path(items) + } + reader.readAsText(file, 'UTF-8') } - reader.readAsText(file, 'UTF-8') } +function parseCSV(){ + lines=filecontents.split('\n') + header=lines[0].split(';') + users=[] + $.each(lines, function(n, l){ + if(n!=0 && l.length > 10){ + usr=toObject(header,l.split(';')) + usr['id']=usr['username'] + users.push(usr) + } + }) + return users; +} function toObject(names, values) { var result = {}; for (var i = 0; i < names.length; i++) diff --git a/admin/src/admin/static/js/status_socket.js b/admin/src/admin/static/js/status_socket.js index 7053b12..c710b7b 100644 --- a/admin/src/admin/static/js/status_socket.js +++ b/admin/src/admin/static/js/status_socket.js @@ -1,19 +1,86 @@ - // SocketIO - socket = io.connect(location.protocol+'//' + document.domain +'/isard-sso-admin/sio'); - console.log(location.protocol+'//' + document.domain +'/isard-sso-admin/sio') - socket.on('connect', function() { - // connection_done(); - console.log('Listening status socket'); - }); +notice={} +$lost=0; - socket.on('connect_error', function(data) { - // connection_lost(); - }); - - socket.on('update', function(data) { - var data = JSON.parse(data); - console.log('Status update') - console.log(data) - // var data = JSON.parse(data); - // drawUserQuota(data); +socket = io.connect(location.protocol+'//' + document.domain +'/isard-sso-admin/sio'); +console.log(location.protocol+'//' + document.domain +'/isard-sso-admin/sio') +socket.on('connect', function() { + if($lost){location.reload();} + console.log('Listening status socket'); +}); + +socket.on('connect_error', function(data) { + $lost=$lost+1; + $('#modal-lostconnection').modal({ + backdrop: 'static', + keyboard: false + }).modal('show'); +}); + +socket.on('notify-create', function(data) { + var data = JSON.parse(data); + notice[data.id] = new PNotify({ + title: data.title, + text: data.text, + hide: false }); +}); + +socket.on('notify-destroy', function(data) { + var data = JSON.parse(data); + notice[data.id].remove() +}); + +socket.on('notify-update', function(data) { + var data = JSON.parse(data); + // console.log(data.text) + notice[data.id].update({ + text: data.text + }) +}); + + // new PNotify({ + // title: "Quota for creating desktops full.", + // text: "Can't create another desktop, user quota full.", + // hide: true, + // delay: 3000, + // icon: 'fa fa-alert-sign', + // opacity: 1, + // type: 'error' + // }); + +// socket.on('update', function(data) { +// var data = JSON.parse(data); +// console.log('Status update') +// console.log(data) +// // var data = JSON.parse(data); +// // drawUserQuota(data); +// }); + +socket.on('update', function(data) { + var data = JSON.parse(data); + console.log('Status update') + console.log(data) + // var data = JSON.parse(data); + // drawUserQuota(data); +}); + + // {'event':'traceback', + // 'id':u['id'], + // 'item':'group', + // 'action':'add' + // 'name':g['name'], + // 'progress':str(item)+'/'+str(total), + // 'status':False, + // 'msg':, + // 'payload':{'traceback':traceback.format_exc(), + // 'data':g}) + +socket.on('progress', function(data) { + var data = JSON.parse(data); + console.log(data) + // $('.modal-progress #item').html(data.item) +}); + + + +//// diff --git a/admin/src/admin/static/js/users.js b/admin/src/admin/static/js/users.js index efcea8b..63a5705 100644 --- a/admin/src/admin/static/js/users.js +++ b/admin/src/admin/static/js/users.js @@ -220,7 +220,7 @@ $(document).ready(function() { "columnDefs": [ { "targets": 1, "render": function ( data, type, full, meta ) { - return '' + return '' }}, { "targets": 6, diff --git a/admin/src/admin/static/templates/base.html b/admin/src/admin/static/templates/base.html index 2a2558c..92455a4 100644 --- a/admin/src/admin/static/templates/base.html +++ b/admin/src/admin/static/templates/base.html @@ -59,26 +59,9 @@ - + + {% include 'pages/modals/common_modals.html' %} + diff --git a/admin/src/admin/static/templates/pages/modals/common_modals.html b/admin/src/admin/static/templates/pages/modals/common_modals.html new file mode 100644 index 0000000..b54a6be --- /dev/null +++ b/admin/src/admin/static/templates/pages/modals/common_modals.html @@ -0,0 +1,44 @@ + + + \ No newline at end of file diff --git a/admin/src/admin/static/templates/pages/modals/external_modals.html b/admin/src/admin/static/templates/pages/modals/external_modals.html index 118abe1..bd8c3d5 100644 --- a/admin/src/admin/static/templates/pages/modals/external_modals.html +++ b/admin/src/admin/static/templates/pages/modals/external_modals.html @@ -32,7 +32,8 @@
diff --git a/admin/src/admin/static/templates/pages/users.html b/admin/src/admin/static/templates/pages/users.html index 425c9b1..0fd62fb 100644 --- a/admin/src/admin/static/templates/pages/users.html +++ b/admin/src/admin/static/templates/pages/users.html @@ -27,7 +27,7 @@ - + diff --git a/admin/src/admin/views/MenuViews.py b/admin/src/admin/views/MenuViews.py index 93cf2e1..088e591 100644 --- a/admin/src/admin/views/MenuViews.py +++ b/admin/src/admin/views/MenuViews.py @@ -8,9 +8,14 @@ from uuid import uuid4 import time,json import sys,os from flask import render_template, Response, request, redirect, url_for, jsonify +import concurrent.futures from pprint import pprint +# from flask_socketio import SocketIO, emit, join_room, leave_room, \ +# close_room, rooms, disconnect, send +# socketio = SocketIO(app) + @app.route('/isard-sso-admin/resync') # @login_required def resync(): @@ -70,7 +75,12 @@ def groups_list(): # @login_required def external(): if request.method == 'POST': - return json.dumps(app.admin.upload_json(request.get_json(force=True))), 200, {'Content-Type': 'application/json'} + data=request.get_json(force=True) + if data['format']=='json-ga': + app.admin.upload_json_ga(data) + if data['format']=='csv-ug': + app.admin.upload_csv_ug(data) + return json.dumps({}), 200, {'Content-Type': 'application/json'} if request.method == 'PUT': return json.dumps(app.admin.sync_external(request.get_json(force=True))), 200, {'Content-Type': 'application/json'} return render_template('pages/external.html', title="External", nav="External") @@ -89,4 +99,15 @@ def external_groups_list(): # @login_required def external_roles(): if request.method == 'PUT': - return json.dumps(app.admin.external_roleassign(request.get_json(force=True))), 200, {'Content-Type': 'application/json'} \ No newline at end of file + return json.dumps(app.admin.external_roleassign(request.get_json(force=True))), 200, {'Content-Type': 'application/json'} + + def WaitStatus(self, desktop_id, original_status, transition_status, final_status): + with concurrent.futures.ThreadPoolExecutor(max_workers=1) as executor: + future = executor.submit(lambda p: self._wait_for_domain_status(*p), [desktop_id, original_status, transition_status, final_status]) + try: + result = future.result() + except ReqlTimeoutError: + raise DesktopActionTimeout + except DesktopWaitFailed: + raise DesktopActionFailed + return True \ No newline at end of file diff --git a/admin/src/admin/views/Socketio.py b/admin/src/admin/views/Socketio.py index b2b968d..4f8cd0b 100644 --- a/admin/src/admin/views/Socketio.py +++ b/admin/src/admin/views/Socketio.py @@ -1,19 +1,19 @@ -from flask_socketio import SocketIO, emit, join_room, leave_room, \ - close_room, rooms, disconnect, send -from admin import app -import json +# from flask_socketio import SocketIO, emit, join_room, leave_room, \ +# close_room, rooms, disconnect, send +# from admin import app +# import json -socketio = SocketIO(app) -# from ...start import socketio +# # socketio = SocketIO(app) +# # from ...start import socketio -@socketio.on('connect', namespace='/isard-sso-admin/sio') -def socketio_connect(): - join_room('admin') - socketio.emit('update', - json.dumps('Joined'), - namespace='/isard-sso-admin/sio', - room='admin') +# @socketio.on('connect', namespace='/isard-sso-admin/sio') +# def socketio_connect(): +# join_room('admin') +# socketio.emit('update', +# json.dumps('Joined'), +# namespace='/isard-sso-admin/sio', +# room='admin') -@socketio.on('disconnect', namespace='/isard-sso-admin/sio') -def socketio_domains_disconnect(): - None \ No newline at end of file +# @socketio.on('disconnect', namespace='/isard-sso-admin/sio') +# def socketio_domains_disconnect(): +# None \ No newline at end of file diff --git a/admin/src/claus.py b/admin/src/claus.py deleted file mode 100644 index cb330ba..0000000 --- a/admin/src/claus.py +++ /dev/null @@ -1,5 +0,0 @@ -import diceware -options = diceware.handle_options(None) -options.wordlist = 'cat_ascii' -options.num = 3 -print(diceware.get_passphrase(options=options)) diff --git a/admin/src/scripts/README.md b/admin/src/scripts/README.md new file mode 100644 index 0000000..6163ed2 --- /dev/null +++ b/admin/src/scripts/README.md @@ -0,0 +1,7 @@ +# Scripts for cli + +Take care at using this at your own risk! +The reset_pwd.py for example will reset ALL USERS password and dump a file with pwd that should be removed! +The avatars.py will reset all users avatars to defaults. + +If you still want to play with them copy it one folder up to be used. diff --git a/admin/src/scripts/avatars.py b/admin/src/scripts/avatars.py new file mode 100644 index 0000000..6b7f300 --- /dev/null +++ b/admin/src/scripts/avatars.py @@ -0,0 +1,118 @@ +#!/usr/bin/env python +import time ,os +from datetime import datetime, timedelta + +import logging as log +import traceback +import yaml, json +from pprint import pprint + +from keycloak import KeycloakAdmin +from postgres import Postgres + +from minio import Minio +from minio.commonconfig import REPLACE, CopySource +from minio.deleteobjects import DeleteObject + +class DefaultAvatars(): + def __init__(self, + url="http://isard-sso-keycloak:8080/auth/", + username=os.environ['KEYCLOAK_USER'], + password=os.environ['KEYCLOAK_PASSWORD'], + realm='master', + verify=True): + self.url=url + self.username=username + self.password=password + self.realm=realm + self.verify=verify + + self.keycloak_pg=Postgres('isard-apps-postgresql','keycloak',os.environ['KEYCLOAK_DB_USER'],os.environ['KEYCLOAK_DB_PASSWORD']) + + self.mclient = Minio( + "isard-sso-avatars:9000", + access_key="AKIAIOSFODNN7EXAMPLE", + secret_key="wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY", + secure=False + ) + self.bucket='master-avatars' + self._minio_set_realm() + self.update_missing_avatars() + + def connect(self): + self.keycloak_admin = KeycloakAdmin(server_url=self.url, + username=self.username, + password=self.password, + realm_name=self.realm, + verify=self.verify) + + def update_missing_avatars(self): + sys_roles=['admin','manager','teacher','student'] + for u in self.get_users_without_image(): + try: + img=[r+'.jpg' for r in sys_roles if r in u['role']][0] + except: + img='unknown.jpg' + + self.mclient.fput_object( + self.bucket, u['id'], "custom/avatars/"+img, + content_type="image/jpeg ", + ) + log.warning(' AVATARS: Updated avatar for user '+u['username']+' with role '+img.split('.')[0]) + + def _minio_set_realm(self): + if not self.mclient.bucket_exists(self.bucket): + self.mclient.make_bucket(self.bucket) + + def minio_get_objects(self): + return [o.object_name for o in self.mclient.list_objects(self.bucket)] + + def minio_delete_all_objects(self): + delete_object_list = map( + lambda x: DeleteObject(x.object_name), + self.mclient.list_objects(self.bucket), + ) + errors=self.mclient.remove_objects(self.bucket, delete_object_list) + for error in errors: + log.error(" AVATARS: Error occured when deleting avatar object", error) + + def get_users(self): + self.connect() + users=self.get_users_with_groups_and_roles() + return users + + def get_users_without_image(self): + return [u for u in self.get_users() if u['id'] not in self.minio_get_objects()] + + def get_users_with_groups_and_roles(self): + q = """select u.id, u.username, u.email, u.first_name, u.last_name, u.realm_id, ua.value as quota + ,json_agg(g."name") as group, json_agg(g_parent."name") as group_parent1, json_agg(g_parent2."name") as group_parent2 + ,json_agg(r.name) as role + from user_entity as u + left join user_attribute as ua on ua.user_id=u.id and ua.name = 'quota' + left join user_group_membership as ugm on ugm.user_id = u.id + left join keycloak_group as g on g.id = ugm.group_id + left join keycloak_group as g_parent on g.parent_group = g_parent.id + left join keycloak_group as g_parent2 on g_parent.parent_group = g_parent2.id + left join user_role_mapping as rm on rm.user_id = u.id + left join keycloak_role as r on r.id = rm.role_id + group by u.id,u.username,u.email,u.first_name,u.last_name, u.realm_id, ua.value + order by u.username""" + + (headers,users)=self.keycloak_pg.select_with_headers(q) + + users_with_lists = [list(l[:-4])+([[]] if l[-4] == [None] else [list(set(l[-4]))]) +\ + ([[]] if l[-3] == [None] else [list(set(l[-3]))]) +\ + ([[]] if l[-3] == [None] else [list(set(l[-2]))]) +\ + ([[]] if l[-1] == [None] else [list(set(l[-1]))]) for l in users] + + users_with_lists = [list(l[:-4])+([[]] if l[-4] == [None] else [list(set(l[-4]))]) +\ + ([[]] if l[-3] == [None] else [list(set(l[-3]))]) +\ + ([[]] if l[-3] == [None] else [list(set(l[-2]))]) +\ + ([[]] if l[-1] == [None] else [list(set(l[-1]))]) for l in users_with_lists] + + list_dict_users = [dict(zip(headers, r)) for r in users_with_lists] + return list_dict_users + +da=DefaultAvatars() + diff --git a/admin/src/reset_pwds.py b/admin/src/scripts/reset_pwds.py similarity index 100% rename from admin/src/reset_pwds.py rename to admin/src/scripts/reset_pwds.py diff --git a/admin/src/start.py b/admin/src/start.py index 6f18bcb..571ac3d 100644 --- a/admin/src/start.py +++ b/admin/src/start.py @@ -11,19 +11,26 @@ from admin import app # socketio.init_app(app, cors_allowed_origins="*") -from admin.views.Socketio import * -# socketio = SocketIO(app) -# @socketio.on('connect', namespace='/isard-sso-admin/sio') -# def socketio_connect(): -# join_room('admin') -# socketio.emit('update', -# json.dumps('Joined'), -# namespace='/isard-sso-admin/sio', -# room='admin') +# from admin.views.Socketio import * + +app.socketio = SocketIO(app) +# app.socketio.init_app(app, cors_allowed_origins="*") +@app.socketio.on('connect', namespace='/isard-sso-admin/sio') +def socketio_connect(): + join_room('admin') + app.socketio.emit('update', + json.dumps('Joined'), + namespace='/isard-sso-admin/sio', + room='admin') + + # socketio.emit('update', + # json.dumps('DATA SENT'), + # namespace='/isard-sso-admin/sio', + # room='admin') -# @socketio.on('disconnect', namespace='/isard-sso-admin/sio') -# def socketio_domains_disconnect(): -# None +@app.socketio.on('disconnect', namespace='/isard-sso-admin/sio') +def socketio_domains_disconnect(): + None if __name__ == '__main__': - socketio.run(app,host='0.0.0.0', port=9000, debug=False, cors_allowed_origins="*") #, logger=logger, engineio_logger=engineio_logger) + app.socketio.run(app,host='0.0.0.0', port=9000, debug=False, cors_allowed_origins="*", async_mode="threading") #, logger=logger, engineio_logger=engineio_logger) diff --git a/docker/haproxy/haproxy.conf b/docker/haproxy/haproxy.conf index 1b9540b..7ffdcb0 100644 --- a/docker/haproxy/haproxy.conf +++ b/docker/haproxy/haproxy.conf @@ -93,6 +93,10 @@ backend be_sso backend be_admin mode http + option forwardfor + timeout queue 600s + timeout server 600s + timeout connect 600s acl authorized http_auth(AuthUsers) http-request auth realm AuthUsers unless authorized acl existing-x-forwarded-host req.hdr(X-Forwarded-Host) -m found diff --git a/minio-client-test.py b/minio-client-test.py deleted file mode 100644 index a6b309c..0000000 --- a/minio-client-test.py +++ /dev/null @@ -1,63 +0,0 @@ -from minio import Minio -from minio.commonconfig import REPLACE, CopySource -# Create client with anonymous access. -# client = Minio("isard-apps-avatars") - -# # Create client with access and secret key. -# client = Minio("s3.amazonaws.com", "ACCESS-KEY", "SECRET-KEY") - -# Create client with access key and secret key with specific region. -client = Minio( - "isard-sso-avatars:9000", - access_key="AKIAIOSFODNN7EXAMPLE", - secret_key="wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY", - secure=False -) - -buckets = client.list_buckets() -for bucket in buckets: - print(bucket.name, bucket.creation_date) -response = client.get_object("master-avatars", "89423d20-3915-4e67-b227-86f099f1816a") -print(response) - -result = client.copy_object( - "master-avatars", - "prova", - CopySource("master-avatars", "89423d20-3915-4e67-b227-86f099f1816a"), -) -client.remove_object("master-avatars", "prova") -result = client.fput_object( - "master-avatars", "test", "admin/static/img/background.png", - content_type="image/jpeg ", -) -objects = client.list_objects("master-avatars") -for obj in objects: - print(obj.key) -exit(1) -try: - response = client.get_object("master-avatars", "my-object") - print(response) - # Read data from response. -finally: - response.close() - response.release_conn() - - # region="my-region", -# # Create client with custom HTTP client using proxy server. -# import urllib3 -# client = Minio( -# "SERVER:PORT", -# access_key="ACCESS_KEY", -# secret_key="SECRET_KEY", -# secure=True, -# http_client=urllib3.ProxyManager( -# "https://PROXYSERVER:PROXYPORT/", -# timeout=urllib3.Timeout.DEFAULT_TIMEOUT, -# cert_reqs="CERT_REQUIRED", -# retries=urllib3.Retry( -# total=5, -# backoff_factor=0.2, -# status_forcelist=[500, 502, 503, 504], -# ), -# ), -# )