EDUCATIC
/
digitaldemocratic
mirror of https://gitlab.com/digitaldemocratic/digitaldemocratic
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

added csv and gevents

root 2021-06-03 00:20:20 +02:00
parent a41d563f5f
commit 8b65eaecf1
20 changed files with 578 additions and 271 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
admin/docker/requirements.pip3
View File

@ -24,3 +24,5 @@ diceware==0.9.6
#greenlet==0.4.15 #greenlet==0.4.15
python-engineio==3.8.1 python-engineio==3.8.1
python-socketio==4.1.0 python-socketio==4.1.0
minio==7.0.3

179
admin/src/admin/lib/admin.py
View File

@ -18,13 +18,7 @@ options = diceware.handle_options(None)
options.wordlist = 'cat_ascii' options.wordlist = 'cat_ascii'
options.num = 3 options.num = 3
from .events import Events
from flask_socketio import SocketIO, emit, join_room, leave_room, \
close_room, rooms, disconnect, send
socketio = SocketIO(app)
from ..views.Socketio import socketio
# socketio = SocketIO(app)
class Admin(): class Admin():
def __init__(self): def __init__(self):
@ -39,9 +33,6 @@ class Admin():
sleep(2) sleep(2)
log.warning('Keycloak connected.') log.warning('Keycloak connected.')
# print(self.keycloak.get_user_groups_paths('320b0713-c480-4928-9105-041309d72191'))
# exit(1)
ready=False ready=False
while not ready: while not ready:
try: try:
@ -73,7 +64,6 @@ class Admin():
self.default_setup() self.default_setup()
self.internal={} self.internal={}
# self.resync_data()
ready=False ready=False
while not ready: while not ready:
@ -87,24 +77,11 @@ class Admin():
self.external={'users':[], self.external={'users':[],
'groups':[], 'groups':[],
'roles':[]} 'roles':[]}
log.warning(' Updating missing user avatars with defaults')
av=Avatars()
# av.minio_delete_all_objects() # This will reset all avatars on usres
av.update_missing_avatars(self.internal['users'])
log.warning(' SYSTEM READY TO HANDLE CONNECTIONS') log.warning(' SYSTEM READY TO HANDLE CONNECTIONS')
# self.test_cohorts()
# self.delete_all_moodle_cohorts()
########## Testing: get user group
# cids=[c['id'] for c in self.moodle.get_cohorts()]
# relations=self.moodle.get_cohort_members(cids)
# for r in relations:
# print(self.get_internalgroup_from_moodlecohortid(r['cohortid'])['path'])
# def get_internalgroup_from_moodlecohortid(self,cohort_id):
# for g in self.internal['groups']:
# if not g['moodle']: continue
# if g['moodle_id'] == cohort_id: return g
# return ''
## This function should be moved to postup.py ## This function should be moved to postup.py
def default_setup(self): def default_setup(self):
@ -122,9 +99,9 @@ class Admin():
self.keycloak_admin.group_user_add(admin_uid,gid) self.keycloak_admin.group_user_add(admin_uid,gid)
log.warning('KEYCLOAK: OK') log.warning('KEYCLOAK: OK')
except: except:
# print(traceback.format_exc())
log.warning('KEYCLOAK: Seems to be there already') log.warning('KEYCLOAK: Seems to be there already')
#### Add default groups #### Add default groups
try: try:
log.warning('KEYCLOAK: Adding default groups') log.warning('KEYCLOAK: Adding default groups')
@ -240,37 +217,6 @@ class Admin():
except: except:
log.warning('MOODLE: Seems to be there already') log.warning('MOODLE: Seems to be there already')
### testing
# def test_cohorts(self):
# cohorts=self.moodle.get_cohorts()
# testc=[c for c in cohorts if c['name'] in ['teacher','/teacher','student','/student']]
# pprint(testc)
# groups=[]
# for u in self.internal['users']:
# groups=groups+u['keycloak_groups']
# groups=list(dict.fromkeys(groups))
# pprint(groups)
# pprint([g for g in groups if 'teacher' in g['keycloak_groups']])
# exit(1)
# total=len(groups)
# i=0
# for g in groups:
# parts=g.split('/')
# subpath=''
# for i in range(1,len(parts)):
# try:
# log.warning(' MOODLE GROUPS: Adding group as cohort ('+str(i)+'/'+str(total)+'): '+subpath)
# subpath=subpath+'/'+parts[i]
# self.moodle.add_system_cohort(subpath)
# except:
# log.error('probably exists')
# i=i+1
### end testing
def resync_data(self): def resync_data(self):
self.internal={'users':self._get_mix_users(), self.internal={'users':self._get_mix_users(),
'groups':self._get_mix_groups(), 'groups':self._get_mix_groups(),
@ -381,7 +327,6 @@ class Admin():
del theuser['groups'] del theuser['groups']
users.append(theuser) users.append(theuser)
# pprint([u['moodle_groups'] for u in users])
return users return users
def get_roles(self): def get_roles(self):
@ -465,27 +410,66 @@ class Admin():
def get_external_roles(self): def get_external_roles(self):
return self.external['roles'] return self.external['roles']
def upload_json(self,data): def upload_csv_ug(self,data):
log.warning('Processing uploaded users...')
users=[]
total=len(data['data'])
item=1
ev=Events('Processing uploaded users',total=len(data['data']))
groups=[]
for u in data['data']:
log.warning('Processing ('+str(item)+'/'+str(total)+') uploaded user: '+u['username'])
user_groups=["/" + g.strip() for g in u['groups'].split(',')]
groups=groups+user_groups
users.append({'provider':'external',
'id':u['id'].strip(),
'email': u['email'].strip(),
'first': u['firstname'].strip(),
'last': u['lastname'].strip(),
'username': u['username'].strip(),
'groups':user_groups,
'roles':[u['role'].strip()],
'password':diceware.get_passphrase(options=options)})
item+=1
ev.increment({'name':u['username'].split('@')[0]})
self.external['users']=users
groups=list(dict.fromkeys(groups))
sysgroups=[]
for g in groups:
sysgroups.append({'provider':'external',
"id": g,
"mailid": g,
"name": g,
"description": 'Imported with csv'})
self.external['groups']=sysgroups
return True
def upload_json_ga(self,data):
groups=[] groups=[]
log.warning('Processing uploaded groups...') log.warning('Processing uploaded groups...')
ev=Events('Processing uploaded groups',total=len(data['data']['groups']))
for g in data['data']['groups']: for g in data['data']['groups']:
# for m in data['data']['d_members']: try:
ev.increment({'name':g['name']})
# data['provider'] groups.append({'provider':'external',
groups.append({'provider':'external', "id": g['id'],
"id": g['id'], "mailid": g['email'].split('@')[0],
"mailid": g['email'].split('@')[0], "name": g['name'],
"name": g['name'], "description": g['description']})
"description": g['description']}) except:
pass
self.external['groups']=groups self.external['groups']=groups
log.warning('Processing uploaded users...') log.warning('Processing uploaded users...')
users=[] users=[]
total=len(data['data']['users']) total=len(data['data']['users'])
i=1 item=1
ev=Events('Processing uploaded users',total=len(data['data']['users']))
for u in data['data']['users']: for u in data['data']['users']:
log.warning('Processing ('+str(i)+'/'+str(total)+') uploaded user: '+u['primaryEmail'].split('@')[0]) log.warning('Processing ('+str(item)+'/'+str(total)+') uploaded user: '+u['primaryEmail'].split('@')[0])
# data['provider']
users.append({'provider':'external', users.append({'provider':'external',
'id':u['id'], 'id':u['id'],
'email': u['primaryEmail'], 'email': u['primaryEmail'],
@ -494,12 +478,9 @@ class Admin():
'username': u['primaryEmail'].split('@')[0], 'username': u['primaryEmail'].split('@')[0],
'groups':[u['orgUnitPath']], ## WARNING: Removing the first 'groups':[u['orgUnitPath']], ## WARNING: Removing the first
'roles':[], 'roles':[],
'password':diceware.get_passphrase(options=options)}) 'password': diceware.get_passphrase(options=options)})
socketio.emit('update', item+=1
json.dumps({'status':True,'item':'user','action':'delete','itemdata':''}), ev.increment({'name':u['primaryEmail'].split('@')[0]})
namespace='/isard-sso-admin/sio',
room='admin')
i=i+1
self.external['users']=users self.external['users']=users
## Add groups to users (now they only have their orgUnitPath) ## Add groups to users (now they only have their orgUnitPath)
@ -513,6 +494,7 @@ class Admin():
def sync_external(self,ids): def sync_external(self,ids):
log.warning('Starting sync to keycloak') log.warning('Starting sync to keycloak')
self.sync_to_keycloak() self.sync_to_keycloak()
### Now we only sycn external to keycloak and then they can be updated to others with UI buttons
# log.warning('Starting sync to moodle') # log.warning('Starting sync to moodle')
# self.sync_to_moodle() # self.sync_to_moodle()
# log.warning('Starting sync to nextcloud') # log.warning('Starting sync to nextcloud')
@ -527,18 +509,21 @@ class Admin():
total=len(groups) total=len(groups)
i=0 i=0
ev=Events('Syncing import groups to keycloak',total=len(groups))
for g in groups: for g in groups:
i=i+1 i=i+1
# print('ADDING FULL PATH: '+str(g))
log.warning(' KEYCLOAK GROUPS: Adding group ('+str(i)+'/'+str(total)+'): '+g) log.warning(' KEYCLOAK GROUPS: Adding group ('+str(i)+'/'+str(total)+'): '+g)
ev.increment({'name':g})
self.keycloak.add_group_tree(g) self.keycloak.add_group_tree(g)
total=len(self.external['users']) total=len(self.external['users'])
index=0 index=0
ev=Events('Syncing import users to keycloak',total=len(self.external['users']))
for u in self.external['users']: for u in self.external['users']:
index=index+1 index=index+1
# Add user # Add user
log.warning(' KEYCLOAK USERS: Adding user ('+str(index)+'/'+str(total)+'): '+u['username']) log.warning(' KEYCLOAK USERS: Adding user ('+str(index)+'/'+str(total)+'): '+u['username'])
ev.increment({'name':u['username']})
uid=self.keycloak.add_user(u['username'],u['first'],u['last'],u['email'],u['password']) uid=self.keycloak.add_user(u['username'],u['first'],u['last'],u['email'],u['password'])
# Add user to role and group rolename # Add user to role and group rolename
@ -573,12 +558,14 @@ class Admin():
### Create all groups. Skip / in system groups ### Create all groups. Skip / in system groups
total=len(groups) total=len(groups)
i=0 i=0
ev=Events('Syncing groups from keycloak to moodle',total=len(groups))
for g in groups: for g in groups:
parts=g.split('/') parts=g.split('/')
subpath='' subpath=''
for i in range(1,len(parts)): for i in range(1,len(parts)):
try: try:
log.warning(' MOODLE GROUPS: Adding group as cohort ('+str(i)+'/'+str(total)+'): '+subpath) log.warning(' MOODLE GROUPS: Adding group as cohort ('+str(i)+'/'+str(total)+'): '+subpath)
ev.increment({'name':subpath})
if parts[i] in ['admin','manager','teacher','student']: if parts[i] in ['admin','manager','teacher','student']:
subpath=parts[i] subpath=parts[i]
else: else:
@ -592,19 +579,28 @@ class Admin():
cohorts=self.moodle.get_cohorts() cohorts=self.moodle.get_cohorts()
### Create users in moodle ### Create users in moodle
ev=Events('Syncing users from keycloak to moodle',total=len(self.internal['users']))
for u in self.internal['users']: for u in self.internal['users']:
if not u['moodle']: if not u['moodle']:
log.error('Creating moodle user: '+u['username']) log.warning('Creating moodle user: '+u['username'])
user=self.moodle.create_user(u['email'],u['username'],'1Random 1String',u['first'],u['last'])[0] ev.increment({'name':u['username']})
if u['first'] == '': u['first']=' '
if u['last'] == '': u['last']=' '
try:
# pprint(u)
pprint(self.moodle.create_user(u['email'],u['username'],'1Random 1String',u['first'],u['last'])[0])
except:
log.error(' -->> Error creating on moodle the user: '+u['username'])
# user_id=user['id'] # user_id=user['id']
self.resync_data() self.resync_data()
### Add user to their cohorts (groups) ### Add user to their cohorts (groups)
ev=Events('Syncing users groups from keycloak to moodle cohorts',total=len(self.internal['users']))
for u in self.internal['users']: for u in self.internal['users']:
total=len(u['keycloak_groups']) total=len(u['keycloak_groups'])
index=0 index=0
ev.increment({'name':u['username']})
for g in u['keycloak_groups']: for g in u['keycloak_groups']:
log.error('for user groups')
parts=g.split('/') parts=g.split('/')
subpath='' subpath=''
# pprint(parts) # pprint(parts)
@ -617,7 +613,6 @@ class Admin():
try: try:
cohort=[c for c in cohorts if c['name']==subpath][0] cohort=[c for c in cohorts if c['name']==subpath][0]
except: except:
# pprint(subpath)
log.error(' MOODLE USER GROUPS: keycloak group '+subpath+' does not exist as moodle cohort. This should not happen. User '+u['username']+ ' not added.') log.error(' MOODLE USER GROUPS: keycloak group '+subpath+' does not exist as moodle cohort. This should not happen. User '+u['username']+ ' not added.')
try: try:
@ -640,25 +635,26 @@ class Admin():
total=len(groups) total=len(groups)
i=0 i=0
ev=Events('Syncing groups from keycloak to nextcloud',total=len(groups))
for g in groups: for g in groups:
parts=g.split('/') parts=g.split('/')
subpath='' subpath=''
for i in range(1,len(parts)): for i in range(1,len(parts)):
try: try:
log.warning(' NEXTCLOUD GROUPS: Adding group ('+str(i)+'/'+str(total)+'): '+subpath) log.warning(' NEXTCLOUD GROUPS: Adding group ('+str(i)+'/'+str(total)+'): '+subpath)
ev.increment({'name':subpath})
subpath=subpath+'/'+parts[i] subpath=subpath+'/'+parts[i]
self.nextcloud.add_group(subpath) self.nextcloud.add_group(subpath)
except: except:
log.error('probably exists') log.error('probably exists')
i=i+1 i=i+1
ev=Events('Syncing users from keycloak to nextcloud',total=len(self.internal['users']))
for u in self.internal['users']: for u in self.internal['users']:
# print('User '+u['username'])
if not u['nextcloud']: if not u['nextcloud']:
# print(' Is not in nextcloud')
log.warning(' NEXTCLOUD USERS: Creating nextcloud user: '+u['username']+' in groups '+str(u['keycloak_groups'])) log.warning(' NEXTCLOUD USERS: Creating nextcloud user: '+u['username']+' in groups '+str(u['keycloak_groups']))
# group=u['keycloak_groups'][0] if len(u['keycloak_groups']) else False
try: try:
ev.increment({'name':u['username']})
self.nextcloud.add_user_with_groups(u['username'],'1Random 1String',500000000000,u['keycloak_groups'],u['email'],u['first']+' '+u['last']) self.nextcloud.add_user_with_groups(u['username'],'1Random 1String',500000000000,u['keycloak_groups'],u['email'],u['first']+' '+u['last'])
except ProviderItemExists: except ProviderItemExists:
log.warning('User '+u['username']+' already exists. Skipping...') log.warning('User '+u['username']+' already exists. Skipping...')
@ -673,12 +669,11 @@ class Admin():
i=i+1 i=i+1
if not u['keycloak']: continue if not u['keycloak']: continue
# Do not remove admin users!!! What to do with managers??? # Do not remove admin users!!! What to do with managers???
if 'admin' in u['roles']: continue if ['admin'] in u['roles']: continue
# if 'manager' in u['roles']: continue
log.info(' KEYCLOAK USERS: Removing user ('+str(i)+'/'+str(total)+'): '+u['username']) log.info(' KEYCLOAK USERS: Removing user ('+str(i)+'/'+str(total)+'): '+u['username'])
try: try:
self.keycloak.delete_user(u['id']) self.keycloak.delete_user(u['id'])
socketio.emit('update', app.socketio.emit('update',
json.dumps({'status':True,'item':'user','action':'delete','itemdata':u}), json.dumps({'status':True,'item':'user','action':'delete','itemdata':u}),
namespace='/isard-sso-admin/sio', namespace='/isard-sso-admin/sio',
room='admin') room='admin')
@ -692,7 +687,7 @@ class Admin():
log.info('Removing nextcloud user: '+u['username']) log.info('Removing nextcloud user: '+u['username'])
try: try:
self.nextcloud.delete_user(u['nextcloud_id']) self.nextcloud.delete_user(u['nextcloud_id'])
socketio.emit('update', app.socketio.emit('update',
json.dumps({'status':True,'item':'user','action':'delete','itemdata':u}), json.dumps({'status':True,'item':'user','action':'delete','itemdata':u}),
namespace='/isard-sso-admin/sio', namespace='/isard-sso-admin/sio',
room='admin') room='admin')
@ -711,7 +706,7 @@ class Admin():
log.warning('Removing moodle users: '+','.join(usernames)) log.warning('Removing moodle users: '+','.join(usernames))
try: try:
self.moodle.delete_users(userids) self.moodle.delete_users(userids)
socketio.emit('update', app.socketio.emit('update',
json.dumps({'status':True,'item':'user','action':'delete','itemdata':u}), json.dumps({'status':True,'item':'user','action':'delete','itemdata':u}),
namespace='/isard-sso-admin/sio', namespace='/isard-sso-admin/sio',
room='admin') room='admin')

62
admin/src/admin/lib/avatars.py
View File

@ -4,30 +4,52 @@ from admin import app
import logging as log import logging as log
from pprint import pprint from pprint import pprint
import os import os
from os import listdir
from os.path import isfile, join
from .postgres import Postgres from minio import Minio
from minio.commonconfig import REPLACE, CopySource
# Module variables to connect to moodle api from minio.deleteobjects import DeleteObject
class Avatars(): class Avatars():
def __init__(self): def __init__(self):
self.keycloak_pg=Postgres('isard-apps-postgresql','keycloak',os.environ['KEYCLOAK_DB_USER'],os.environ['KEYCLOAK_DB_PASSWORD']) self.mclient = Minio(
"isard-sso-avatars:9000",
access_key="AKIAIOSFODNN7EXAMPLE",
secret_key="wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY",
secure=False
)
self.bucket='master-avatars'
self._minio_set_realm()
# self.update_missing_avatars()
def username2id(self,username): def update_missing_avatars(self,users):
q = """select id, username from user_entity where username = '%s'""" % (username) sys_roles=['admin','manager','teacher','student']
try: for u in self.get_users_without_image(users):
return self.keycloak_pg.select(q)[0][0] try:
except: img=[r+'.jpg' for r in sys_roles if r in u['roles']][0]
pass except:
return False img='unknown.jpg'
def get_files(self): self.mclient.fput_object(
path='avatars/master-avatars/' self.bucket, u['id'], os.path.join(app.root_path,"../custom/avatars/"+img),
onlyfiles = [f for f in listdir(path) if isfile(join(path, f))] content_type="image/jpeg ",
pprint(onlyfiles) )
log.warning(' AVATARS: Updated avatar for user '+u['username']+' with role '+img.split('.')[0])
# def generate_missing(self, users): def _minio_set_realm(self):
# for u in users: if not self.mclient.bucket_exists(self.bucket):
self.mclient.make_bucket(self.bucket)
def minio_get_objects(self):
return [o.object_name for o in self.mclient.list_objects(self.bucket)]
def minio_delete_all_objects(self):
delete_object_list = map(
lambda x: DeleteObject(x.object_name),
self.mclient.list_objects(self.bucket),
)
errors=self.mclient.remove_objects(self.bucket, delete_object_list)
for error in errors:
log.error(" AVATARS: Error occured when deleting avatar object", error)
def get_users_without_image(self,users):
return [u for u in users if u['id'] not in self.minio_get_objects()]

73
admin/src/admin/lib/events.py Normal file
View File

@ -0,0 +1,73 @@
#!flask/bin/python
# coding=utf-8
from admin import app
import logging as log
import traceback
from uuid import uuid4
import json
from time import sleep
import sys,os
from flask import render_template, Response, request, redirect, url_for, jsonify
from flask_socketio import SocketIO, emit, join_room, leave_room, \
close_room, rooms, disconnect, send
import base64
class Events():
def __init__(self,title,text='',total=0):
self.eid=str(base64.b64encode(os.urandom(32))[:8])
self.title=title
self.text=text
self.total=total
self.item=0
self.create()
def create(self):
app.socketio.emit('notify-create',
json.dumps({'id':self.eid,
'title':self.title,
'text':self.text}),
namespace='/isard-sso-admin/sio',
room='admin')
sleep(0.1)
def __del__(self):
app.socketio.emit('notify-destroy',
json.dumps({'id':self.eid}),
namespace='/isard-sso-admin/sio',
room='admin')
sleep(0.1)
def update_text(self,text):
self.text=text
app.socketio.emit('notify-update',
json.dumps({'id':self.eid,
'text':self.text,}),
namespace='/isard-sso-admin/sio',
room='admin')
sleep(0.1)
def increment(self,data={}):
self.item+=1
app.socketio.emit('notify-update',
json.dumps({'id':self.eid,
'text': '['+str(self.item)+'/'+str(self.total)+'] ',
'item':self.item,
'total':self.total,
'data':data}),
namespace='/isard-sso-admin/sio',
room='admin')
sleep(0.1)
def decrement(self,data={}):
self.item-=1
app.socketio.emit('notify-update',
json.dumps({'id':self.eid,
'text':'['+str(self.item)+'/'+str(self.total)+'] ',
'item':self.item,
'total':self.total,
'data':data}),
namespace='/isard-sso-admin/sio',
room='admin')
sleep(0.1)

65
admin/src/admin/static/js/external.js
View File

@ -17,7 +17,7 @@ $(document).ready(function() {
$.each(users_table.rows().data(),function(key, value){ $.each(users_table.rows().data(),function(key, value){
ids[value['id']]=value['roles'] ids[value['id']]=value['roles']
}); });
console.log(ids) // console.log(ids)
$.ajax({ $.ajax({
type: "PUT", type: "PUT",
url:"/isard-sso-admin/external", url:"/isard-sso-admin/external",
@ -46,25 +46,30 @@ $(document).ready(function() {
}) })
$("#modalImport #send").on('click', function(e){ $("#modalImport #send").on('click', function(e){
console.log(users_table.rows().data()) // console.log(users_table.rows().data())
var form = $('#modalImportForm'); var form = $('#modalImportForm');
form.parsley().validate(); form.parsley().validate();
if (form.parsley().isValid()){ if (form.parsley().isValid()){
formdata = form.serializeObject() formdata = form.serializeObject()
formdata['data']=JSON.parse(filecontents) if($('#format').val() == 'csv-ug'){
formdata['data']=parseCSV(filecontents)
}else{
formdata['data']=JSON.parse(filecontents)
}
$.ajax({ $.ajax({
type: "POST", type: "POST",
url:"/isard-sso-admin/external", url:"/isard-sso-admin/external",
data: JSON.stringify(formdata), data: JSON.stringify(formdata),
success: function(data) success: function(data)
{ {
$("#modalImport").modal('hide'); console.log('SUCCESS')
users_table.ajax.reload(); // $("#modalImport").modal('hide');
groups_table.ajax.reload(); // users_table.ajax.reload();
// groups_table.ajax.reload();
}, },
error: function(data) error: function(data)
{ {
alert('Something went wrong on our side...') console.log('Ajax timeout')
} }
}); });
} }
@ -194,20 +199,46 @@ $(document).ready(function() {
}); });
function readFile (evt) { function readFile (evt) {
path = ""; if($('#format').val() == 'json-ga'){
items = []; path = "";
var files = evt.target.files; items = [];
var file = files[0]; var files = evt.target.files;
var reader = new FileReader(); var file = files[0];
reader.onload = function(event) { var reader = new FileReader();
filecontents=event.target.result; reader.onload = function(event) {
$.each(JSON.parse(filecontents), walker); filecontents=event.target.result;
populate_path(items) $.each(JSON.parse(filecontents), walker);
populate_path(items)
}
reader.readAsText(file, 'UTF-8')
}
if($('#format').val() == 'csv-ug'){
var files = evt.target.files;
var file = files[0];
var reader = new FileReader();
reader.onload = function(event) {
filecontents=event.target.result;
// $.each(JSON.parse(filecontents), walker);
// populate_path(items)
}
reader.readAsText(file, 'UTF-8')
} }
reader.readAsText(file, 'UTF-8')
} }
function parseCSV(){
lines=filecontents.split('\n')
header=lines[0].split(';')
users=[]
$.each(lines, function(n, l){
if(n!=0 && l.length > 10){
usr=toObject(header,l.split(';'))
usr['id']=usr['username']
users.push(usr)
}
})
return users;
}
function toObject(names, values) { function toObject(names, values) {
var result = {}; var result = {};
for (var i = 0; i < names.length; i++) for (var i = 0; i < names.length; i++)

99
admin/src/admin/static/js/status_socket.js
View File

@ -1,19 +1,86 @@
// SocketIO notice={}
socket = io.connect(location.protocol+'//' + document.domain +'/isard-sso-admin/sio'); $lost=0;
console.log(location.protocol+'//' + document.domain +'/isard-sso-admin/sio')
socket.on('connect', function() {
// connection_done();
console.log('Listening status socket');
});
socket.on('connect_error', function(data) { socket = io.connect(location.protocol+'//' + document.domain +'/isard-sso-admin/sio');
// connection_lost(); console.log(location.protocol+'//' + document.domain +'/isard-sso-admin/sio')
}); socket.on('connect', function() {
if($lost){location.reload();}
console.log('Listening status socket');
});
socket.on('update', function(data) { socket.on('connect_error', function(data) {
var data = JSON.parse(data); $lost=$lost+1;
console.log('Status update') $('#modal-lostconnection').modal({
console.log(data) backdrop: 'static',
// var data = JSON.parse(data); keyboard: false
// drawUserQuota(data); }).modal('show');
});
socket.on('notify-create', function(data) {
var data = JSON.parse(data);
notice[data.id] = new PNotify({
title: data.title,
text: data.text,
hide: false
}); });
});
socket.on('notify-destroy', function(data) {
var data = JSON.parse(data);
notice[data.id].remove()
});
socket.on('notify-update', function(data) {
var data = JSON.parse(data);
// console.log(data.text)
notice[data.id].update({
text: data.text
})
});
// new PNotify({
// title: "Quota for creating desktops full.",
// text: "Can't create another desktop, user quota full.",
// hide: true,
// delay: 3000,
// icon: 'fa fa-alert-sign',
// opacity: 1,
// type: 'error'
// });
// socket.on('update', function(data) {
// var data = JSON.parse(data);
// console.log('Status update')
// console.log(data)
// // var data = JSON.parse(data);
// // drawUserQuota(data);
// });
socket.on('update', function(data) {
var data = JSON.parse(data);
console.log('Status update')
console.log(data)
// var data = JSON.parse(data);
// drawUserQuota(data);
});
// {'event':'traceback',
// 'id':u['id'],
// 'item':'group',
// 'action':'add'
// 'name':g['name'],
// 'progress':str(item)+'/'+str(total),
// 'status':False,
// 'msg':,
// 'payload':{'traceback':traceback.format_exc(),
// 'data':g})
socket.on('progress', function(data) {
var data = JSON.parse(data);
console.log(data)
// $('.modal-progress #item').html(data.item)
});
////

2
admin/src/admin/static/js/users.js
View File

@ -220,7 +220,7 @@ $(document).ready(function() {
"columnDefs": [ { "columnDefs": [ {
"targets": 1, "targets": 1,
"render": function ( data, type, full, meta ) { "render": function ( data, type, full, meta ) {
return '<object data="static/img/usera.jpg" type="image/png" width="25" height="25"><img src="/isard-sso-admin/avatars/'+full.id+'" title="'+full.id+'" width="25" height="25"></object>' return '<img src="/isard-sso-admin/avatars/'+full.id+'" title="'+full.id+'" width="25" height="25"></object>'
}}, }},
{ {
"targets": 6, "targets": 6,

23
admin/src/admin/static/templates/base.html
View File

@ -59,26 +59,9 @@
<!-- /footer content --> <!-- /footer content -->
</div> </div>
</div> </div>
<div id="modal-lostconnection" class="modal fade" role="dialog" style="width:50%;margin-left:30%;margin-top:10%;z-index: 100000;">
<div class="modal-admin"> {% include 'pages/modals/common_modals.html' %}
<div class="modal-content">
<div class="row text-center"><h2 style="margin-bottom:5px">Connection lost</h2></div>
<hr>
<div class="row">
<div class="col-md-1 col-sm-1 col-xs-12"></div>
<div class="col-md-10 col-sm-10 col-xs-12">
<div class="row text-center">Unable to contact server. There should be a problem with network or a heavy load.</div>
<br>
<div class="row text-center" style="margin-bottom:25px">
<i class="fa fa-circle-o-notch fa-spin fa-fw"></i> Trying to reconnect...
</div>
</div>
<div class="col-md-1 col-sm-1 col-xs-12">
</div>
</div>
</div>
</div>
</div>
<!-- jQuery --> <!-- jQuery -->
<script src="/isard-sso-admin/vendors/jquery/dist/jquery.js"></script> <script src="/isard-sso-admin/vendors/jquery/dist/jquery.js"></script>

44
admin/src/admin/static/templates/pages/modals/common_modals.html Normal file
View File

@ -0,0 +1,44 @@
<div id="modal-lostconnection" class="modal fade" role="dialog" style="width:50%;margin-left:30%;margin-top:10%;z-index: 100000;">
<div class="modal-admin">
<div class="modal-content">
<div class="row text-center"><h2 style="margin-bottom:5px">Connection lost</h2></div>
<hr>
<div class="row">
<div class="col-md-1 col-sm-1 col-xs-12"></div>
<div class="col-md-10 col-sm-10 col-xs-12">
<div class="row text-center">Unable to contact server. There should be a problem with network or a heavy load.</div>
<br>
<div class="row text-center">
<i class="fa fa-circle-o-notch fa-spin fa-fw"></i> Trying to reconnect...
</div>
<br>
<div class="col-md-1 col-sm-1 col-xs-12">
</div>
</div>
</div>
</div>
</div>
</div>
<div id="modal-info" class="modal fade" role="dialog" style="width:100%;margin-left:10%;margin-top:10%;z-index: 100000;">
<div class="modal-admin">
<div class="modal-content">
<div class="row text-center"><h2 style="margin-bottom:5px">Connection lost</h2></div>
<hr>
<div class="row">
<div class="col-md-10 col-sm-10 col-xs-12">
<div class="row text-center">Unable to contact server. There should be a problem with network or a heavy load.</div>
<br>
<div class="row text-center">
<i class="fa fa-circle-o-notch fa-spin fa-fw"></i> Trying to reconnect...
</div>
<br>
<div class="col-md-1 col-sm-1 col-xs-12">
</div>
</div>
</div>
</div>
</div>
</div>

3
admin/src/admin/static/templates/pages/modals/external_modals.html
View File

@ -32,7 +32,8 @@
<label class="control-label col-md-3 col-sm-3 col-xs-12" for="format">Format: <span class="required">*</span></label> <label class="control-label col-md-3 col-sm-3 col-xs-12" for="format">Format: <span class="required">*</span></label>
<div class="col-md-6 col-sm-6 col-xs-12"> <div class="col-md-6 col-sm-6 col-xs-12">
<select id="format" name="format" class="form-control format" required> <select id="format" name="format" class="form-control format" required>
<option value="json">JSON dump</option> <option value="json-ga">GAdminconsole JSON</option>
<option value="csv-ug">CSV user with groups</option>
</select> </select>
</div> </div>
</div> </div>

4
admin/src/admin/static/templates/pages/users.html
View File

@ -27,7 +27,7 @@
<button class="btn btn-primary btn-xs btn-sync_to_moodle"> <button class="btn btn-primary btn-xs btn-sync_to_moodle">
<i class="fa fa-refresh" aria-hidden="true"></i> Sync to Moodle <i class="fa fa-refresh" aria-hidden="true"></i> Sync to Moodle
</button> </button>
<!-- <button class="btn btn-danger btn-xs btn-delete_keycloak"> <button class="btn btn-danger btn-xs btn-delete_keycloak">
<i class="fa fa-trash"></i> Delete all keycloak <i class="fa fa-trash"></i> Delete all keycloak
</button> </button>
<button class="btn btn-danger btn-xs btn-delete_nextcloud"> <button class="btn btn-danger btn-xs btn-delete_nextcloud">
@ -35,7 +35,7 @@
</button> </button>
<button class="btn btn-danger btn-xs btn-delete_moodle"> <button class="btn btn-danger btn-xs btn-delete_moodle">
<i class="fa fa-trash"></i> Delete missing keycloak in moodle <i class="fa fa-trash"></i> Delete missing keycloak in moodle
</button> --> </button>
<table id="users" class="table" width="100%"> <table id="users" class="table" width="100%">
<thead> <thead>
<tr> <tr>

23
admin/src/admin/views/MenuViews.py
View File

@ -8,9 +8,14 @@ from uuid import uuid4
import time,json import time,json
import sys,os import sys,os
from flask import render_template, Response, request, redirect, url_for, jsonify from flask import render_template, Response, request, redirect, url_for, jsonify
import concurrent.futures
from pprint import pprint from pprint import pprint
# from flask_socketio import SocketIO, emit, join_room, leave_room, \
# close_room, rooms, disconnect, send
# socketio = SocketIO(app)
@app.route('/isard-sso-admin/resync') @app.route('/isard-sso-admin/resync')
# @login_required # @login_required
def resync(): def resync():
@ -70,7 +75,12 @@ def groups_list():
# @login_required # @login_required
def external(): def external():
if request.method == 'POST': if request.method == 'POST':
return json.dumps(app.admin.upload_json(request.get_json(force=True))), 200, {'Content-Type': 'application/json'} data=request.get_json(force=True)
if data['format']=='json-ga':
app.admin.upload_json_ga(data)
if data['format']=='csv-ug':
app.admin.upload_csv_ug(data)
return json.dumps({}), 200, {'Content-Type': 'application/json'}
if request.method == 'PUT': if request.method == 'PUT':
return json.dumps(app.admin.sync_external(request.get_json(force=True))), 200, {'Content-Type': 'application/json'} return json.dumps(app.admin.sync_external(request.get_json(force=True))), 200, {'Content-Type': 'application/json'}
return render_template('pages/external.html', title="External", nav="External") return render_template('pages/external.html', title="External", nav="External")
@ -90,3 +100,14 @@ def external_groups_list():
def external_roles(): def external_roles():
if request.method == 'PUT': if request.method == 'PUT':
return json.dumps(app.admin.external_roleassign(request.get_json(force=True))), 200, {'Content-Type': 'application/json'} return json.dumps(app.admin.external_roleassign(request.get_json(force=True))), 200, {'Content-Type': 'application/json'}
def WaitStatus(self, desktop_id, original_status, transition_status, final_status):
with concurrent.futures.ThreadPoolExecutor(max_workers=1) as executor:
future = executor.submit(lambda p: self._wait_for_domain_status(*p), [desktop_id, original_status, transition_status, final_status])
try:
result = future.result()
except ReqlTimeoutError:
raise DesktopActionTimeout
except DesktopWaitFailed:
raise DesktopActionFailed
return True

32
admin/src/admin/views/Socketio.py
View File

@ -1,19 +1,19 @@
from flask_socketio import SocketIO, emit, join_room, leave_room, \ # from flask_socketio import SocketIO, emit, join_room, leave_room, \
close_room, rooms, disconnect, send # close_room, rooms, disconnect, send
from admin import app # from admin import app
import json # import json
socketio = SocketIO(app) # # socketio = SocketIO(app)
# from ...start import socketio # # from ...start import socketio
@socketio.on('connect', namespace='/isard-sso-admin/sio') # @socketio.on('connect', namespace='/isard-sso-admin/sio')
def socketio_connect(): # def socketio_connect():
join_room('admin') # join_room('admin')
socketio.emit('update', # socketio.emit('update',
json.dumps('Joined'), # json.dumps('Joined'),
namespace='/isard-sso-admin/sio', # namespace='/isard-sso-admin/sio',
room='admin') # room='admin')
@socketio.on('disconnect', namespace='/isard-sso-admin/sio') # @socketio.on('disconnect', namespace='/isard-sso-admin/sio')
def socketio_domains_disconnect(): # def socketio_domains_disconnect():
None # None

5
admin/src/claus.py
View File

@ -1,5 +0,0 @@
import diceware
options = diceware.handle_options(None)
options.wordlist = 'cat_ascii'
options.num = 3
print(diceware.get_passphrase(options=options))

7
admin/src/scripts/README.md Normal file
View File

@ -0,0 +1,7 @@
# Scripts for cli
Take care at using this at your own risk!
The reset_pwd.py for example will reset ALL USERS password and dump a file with pwd that should be removed!
The avatars.py will reset all users avatars to defaults.
If you still want to play with them copy it one folder up to be used.

118
admin/src/scripts/avatars.py Normal file
View File

@ -0,0 +1,118 @@
#!/usr/bin/env python
import time ,os
from datetime import datetime, timedelta
import logging as log
import traceback
import yaml, json
from pprint import pprint
from keycloak import KeycloakAdmin
from postgres import Postgres
from minio import Minio
from minio.commonconfig import REPLACE, CopySource
from minio.deleteobjects import DeleteObject
class DefaultAvatars():
def __init__(self,
url="http://isard-sso-keycloak:8080/auth/",
username=os.environ['KEYCLOAK_USER'],
password=os.environ['KEYCLOAK_PASSWORD'],
realm='master',
verify=True):
self.url=url
self.username=username
self.password=password
self.realm=realm
self.verify=verify
self.keycloak_pg=Postgres('isard-apps-postgresql','keycloak',os.environ['KEYCLOAK_DB_USER'],os.environ['KEYCLOAK_DB_PASSWORD'])
self.mclient = Minio(
"isard-sso-avatars:9000",
access_key="AKIAIOSFODNN7EXAMPLE",
secret_key="wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY",
secure=False
)
self.bucket='master-avatars'
self._minio_set_realm()
self.update_missing_avatars()
def connect(self):
self.keycloak_admin = KeycloakAdmin(server_url=self.url,
username=self.username,
password=self.password,
realm_name=self.realm,
verify=self.verify)
def update_missing_avatars(self):
sys_roles=['admin','manager','teacher','student']
for u in self.get_users_without_image():
try:
img=[r+'.jpg' for r in sys_roles if r in u['role']][0]
except:
img='unknown.jpg'
self.mclient.fput_object(
self.bucket, u['id'], "custom/avatars/"+img,
content_type="image/jpeg ",
)
log.warning(' AVATARS: Updated avatar for user '+u['username']+' with role '+img.split('.')[0])
def _minio_set_realm(self):
if not self.mclient.bucket_exists(self.bucket):
self.mclient.make_bucket(self.bucket)
def minio_get_objects(self):
return [o.object_name for o in self.mclient.list_objects(self.bucket)]
def minio_delete_all_objects(self):
delete_object_list = map(
lambda x: DeleteObject(x.object_name),
self.mclient.list_objects(self.bucket),
)
errors=self.mclient.remove_objects(self.bucket, delete_object_list)
for error in errors:
log.error(" AVATARS: Error occured when deleting avatar object", error)
def get_users(self):
self.connect()
users=self.get_users_with_groups_and_roles()
return users
def get_users_without_image(self):
return [u for u in self.get_users() if u['id'] not in self.minio_get_objects()]
def get_users_with_groups_and_roles(self):
q = """select u.id, u.username, u.email, u.first_name, u.last_name, u.realm_id, ua.value as quota
,json_agg(g."name") as group, json_agg(g_parent."name") as group_parent1, json_agg(g_parent2."name") as group_parent2
,json_agg(r.name) as role
from user_entity as u
left join user_attribute as ua on ua.user_id=u.id and ua.name = 'quota'
left join user_group_membership as ugm on ugm.user_id = u.id
left join keycloak_group as g on g.id = ugm.group_id
left join keycloak_group as g_parent on g.parent_group = g_parent.id
left join keycloak_group as g_parent2 on g_parent.parent_group = g_parent2.id
left join user_role_mapping as rm on rm.user_id = u.id
left join keycloak_role as r on r.id = rm.role_id
group by u.id,u.username,u.email,u.first_name,u.last_name, u.realm_id, ua.value
order by u.username"""
(headers,users)=self.keycloak_pg.select_with_headers(q)
users_with_lists = [list(l[:-4])+([[]] if l[-4] == [None] else [list(set(l[-4]))]) +\
([[]] if l[-3] == [None] else [list(set(l[-3]))]) +\
([[]] if l[-3] == [None] else [list(set(l[-2]))]) +\
([[]] if l[-1] == [None] else [list(set(l[-1]))]) for l in users]
users_with_lists = [list(l[:-4])+([[]] if l[-4] == [None] else [list(set(l[-4]))]) +\
([[]] if l[-3] == [None] else [list(set(l[-3]))]) +\
([[]] if l[-3] == [None] else [list(set(l[-2]))]) +\
([[]] if l[-1] == [None] else [list(set(l[-1]))]) for l in users_with_lists]
list_dict_users = [dict(zip(headers, r)) for r in users_with_lists]
return list_dict_users
da=DefaultAvatars()

0
admin/src/reset_pwds.py → admin/src/scripts/reset_pwds.py
View File

33
admin/src/start.py
View File

@ -11,19 +11,26 @@ from admin import app
# socketio.init_app(app, cors_allowed_origins="*") # socketio.init_app(app, cors_allowed_origins="*")
from admin.views.Socketio import * # from admin.views.Socketio import *
# socketio = SocketIO(app)
# @socketio.on('connect', namespace='/isard-sso-admin/sio')
# def socketio_connect():
# join_room('admin')
# socketio.emit('update',
# json.dumps('Joined'),
# namespace='/isard-sso-admin/sio',
# room='admin')
# @socketio.on('disconnect', namespace='/isard-sso-admin/sio') app.socketio = SocketIO(app)
# def socketio_domains_disconnect(): # app.socketio.init_app(app, cors_allowed_origins="*")
# None @app.socketio.on('connect', namespace='/isard-sso-admin/sio')
def socketio_connect():
join_room('admin')
app.socketio.emit('update',
json.dumps('Joined'),
namespace='/isard-sso-admin/sio',
room='admin')
# socketio.emit('update',
# json.dumps('DATA SENT'),
# namespace='/isard-sso-admin/sio',
# room='admin')
@app.socketio.on('disconnect', namespace='/isard-sso-admin/sio')
def socketio_domains_disconnect():
None
if __name__ == '__main__': if __name__ == '__main__':
socketio.run(app,host='0.0.0.0', port=9000, debug=False, cors_allowed_origins="*") #, logger=logger, engineio_logger=engineio_logger) app.socketio.run(app,host='0.0.0.0', port=9000, debug=False, cors_allowed_origins="*", async_mode="threading") #, logger=logger, engineio_logger=engineio_logger)

4
docker/haproxy/haproxy.conf
View File

@ -93,6 +93,10 @@ backend be_sso
backend be_admin backend be_admin
mode http mode http
option forwardfor
timeout queue 600s
timeout server 600s
timeout connect 600s
acl authorized http_auth(AuthUsers) acl authorized http_auth(AuthUsers)
http-request auth realm AuthUsers unless authorized http-request auth realm AuthUsers unless authorized
acl existing-x-forwarded-host req.hdr(X-Forwarded-Host) -m found acl existing-x-forwarded-host req.hdr(X-Forwarded-Host) -m found

63
minio-client-test.py
View File

@ -1,63 +0,0 @@
from minio import Minio
from minio.commonconfig import REPLACE, CopySource
# Create client with anonymous access.
# client = Minio("isard-apps-avatars")
# # Create client with access and secret key.
# client = Minio("s3.amazonaws.com", "ACCESS-KEY", "SECRET-KEY")
# Create client with access key and secret key with specific region.
client = Minio(
"isard-sso-avatars:9000",
access_key="AKIAIOSFODNN7EXAMPLE",
secret_key="wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY",
secure=False
)
buckets = client.list_buckets()
for bucket in buckets:
print(bucket.name, bucket.creation_date)
response = client.get_object("master-avatars", "89423d20-3915-4e67-b227-86f099f1816a")
print(response)
result = client.copy_object(
"master-avatars",
"prova",
CopySource("master-avatars", "89423d20-3915-4e67-b227-86f099f1816a"),
)
client.remove_object("master-avatars", "prova")
result = client.fput_object(
"master-avatars", "test", "admin/static/img/background.png",
content_type="image/jpeg ",
)
objects = client.list_objects("master-avatars")
for obj in objects:
print(obj.key)
exit(1)
try:
response = client.get_object("master-avatars", "my-object")
print(response)
# Read data from response.
finally:
response.close()
response.release_conn()
# region="my-region",
# # Create client with custom HTTP client using proxy server.
# import urllib3
# client = Minio(
# "SERVER:PORT",
# access_key="ACCESS_KEY",
# secret_key="SECRET_KEY",
# secure=True,
# http_client=urllib3.ProxyManager(
# "https://PROXYSERVER:PROXYPORT/",
# timeout=urllib3.Timeout.DEFAULT_TIMEOUT,
# cert_reqs="CERT_REQUIRED",
# retries=urllib3.Retry(
# total=5,
# backoff_factor=0.2,
# status_forcelist=[500, 502, 503, 504],
# ),
# ),
# )