EDUCATIC
/
digitaldemocratic
mirror of https://gitlab.com/digitaldemocratic/digitaldemocratic
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

added csv and gevents

root 2021-06-03 00:20:20 +02:00
parent a41d563f5f
commit 8b65eaecf1
20 changed files with 578 additions and 271 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
admin/docker/requirements.pip3
View File

@ -23,4 +23,6 @@ diceware==0.9.6
#gevent==1.4.0
#greenlet==0.4.15
python-engineio==3.8.1
python-socketio==4.1.0
python-socketio==4.1.0
minio==7.0.3

181
admin/src/admin/lib/admin.py
View File

@ -18,13 +18,7 @@ options = diceware.handle_options(None)
options.wordlist = 'cat_ascii'
options.num = 3
from flask_socketio import SocketIO, emit, join_room, leave_room, \
close_room, rooms, disconnect, send
socketio = SocketIO(app)
from ..views.Socketio import socketio
# socketio = SocketIO(app)
from .events import Events
class Admin():
def __init__(self):
@ -39,9 +33,6 @@ class Admin():
sleep(2)
log.warning('Keycloak connected.')
# print(self.keycloak.get_user_groups_paths('320b0713-c480-4928-9105-041309d72191'))
# exit(1)
ready=False
while not ready:
try:
@ -73,7 +64,6 @@ class Admin():
self.default_setup()
self.internal={}
# self.resync_data()
ready=False
while not ready:
@ -87,24 +77,11 @@ class Admin():
self.external={'users':[],
'groups':[],
'roles':[]}
log.warning(' Updating missing user avatars with defaults')
av=Avatars()
# av.minio_delete_all_objects() # This will reset all avatars on usres
av.update_missing_avatars(self.internal['users'])
log.warning(' SYSTEM READY TO HANDLE CONNECTIONS')
# self.test_cohorts()
# self.delete_all_moodle_cohorts()
########## Testing: get user group
# cids=[c['id'] for c in self.moodle.get_cohorts()]
# relations=self.moodle.get_cohort_members(cids)
# for r in relations:
# print(self.get_internalgroup_from_moodlecohortid(r['cohortid'])['path'])
# def get_internalgroup_from_moodlecohortid(self,cohort_id):
# for g in self.internal['groups']:
# if not g['moodle']: continue
# if g['moodle_id'] == cohort_id: return g
# return ''
## This function should be moved to postup.py
def default_setup(self):
@ -122,9 +99,9 @@ class Admin():
self.keycloak_admin.group_user_add(admin_uid,gid)
log.warning('KEYCLOAK: OK')
except:
# print(traceback.format_exc())
log.warning('KEYCLOAK: Seems to be there already')
#### Add default groups
try:
log.warning('KEYCLOAK: Adding default groups')
@ -240,37 +217,6 @@ class Admin():
except:
log.warning('MOODLE: Seems to be there already')
### testing
# def test_cohorts(self):
# cohorts=self.moodle.get_cohorts()
# testc=[c for c in cohorts if c['name'] in ['teacher','/teacher','student','/student']]
# pprint(testc)
# groups=[]
# for u in self.internal['users']:
# groups=groups+u['keycloak_groups']
# groups=list(dict.fromkeys(groups))
# pprint(groups)
# pprint([g for g in groups if 'teacher' in g['keycloak_groups']])
# exit(1)
# total=len(groups)
# i=0
# for g in groups:
# parts=g.split('/')
# subpath=''
# for i in range(1,len(parts)):
# try:
# log.warning(' MOODLE GROUPS: Adding group as cohort ('+str(i)+'/'+str(total)+'): '+subpath)
# subpath=subpath+'/'+parts[i]
# self.moodle.add_system_cohort(subpath)
# except:
# log.error('probably exists')
# i=i+1
### end testing
def resync_data(self):
self.internal={'users':self._get_mix_users(),
'groups':self._get_mix_groups(),
@ -380,8 +326,7 @@ class Admin():
theuser['nextcloud_groups']=[]
del theuser['groups']
users.append(theuser)
# pprint([u['moodle_groups'] for u in users])
return users
def get_roles(self):
@ -465,27 +410,66 @@ class Admin():
def get_external_roles(self):
return self.external['roles']
def upload_json(self,data):
def upload_csv_ug(self,data):
log.warning('Processing uploaded users...')
users=[]
total=len(data['data'])
item=1
ev=Events('Processing uploaded users',total=len(data['data']))
groups=[]
for u in data['data']:
log.warning('Processing ('+str(item)+'/'+str(total)+') uploaded user: '+u['username'])
user_groups=["/" + g.strip() for g in u['groups'].split(',')]
groups=groups+user_groups
users.append({'provider':'external',
'id':u['id'].strip(),
'email': u['email'].strip(),
'first': u['firstname'].strip(),
'last': u['lastname'].strip(),
'username': u['username'].strip(),
'groups':user_groups,
'roles':[u['role'].strip()],
'password':diceware.get_passphrase(options=options)})
item+=1
ev.increment({'name':u['username'].split('@')[0]})
self.external['users']=users
groups=list(dict.fromkeys(groups))
sysgroups=[]
for g in groups:
sysgroups.append({'provider':'external',
"id": g,
"mailid": g,
"name": g,
"description": 'Imported with csv'})
self.external['groups']=sysgroups
return True
def upload_json_ga(self,data):
groups=[]
log.warning('Processing uploaded groups...')
ev=Events('Processing uploaded groups',total=len(data['data']['groups']))
for g in data['data']['groups']:
# for m in data['data']['d_members']:
# data['provider']
groups.append({'provider':'external',
"id": g['id'],
"mailid": g['email'].split('@')[0],
"name": g['name'],
"description": g['description']})
try:
ev.increment({'name':g['name']})
groups.append({'provider':'external',
"id": g['id'],
"mailid": g['email'].split('@')[0],
"name": g['name'],
"description": g['description']})
except:
pass
self.external['groups']=groups
log.warning('Processing uploaded users...')
users=[]
total=len(data['data']['users'])
i=1
item=1
ev=Events('Processing uploaded users',total=len(data['data']['users']))
for u in data['data']['users']:
log.warning('Processing ('+str(i)+'/'+str(total)+') uploaded user: '+u['primaryEmail'].split('@')[0])
# data['provider']
log.warning('Processing ('+str(item)+'/'+str(total)+') uploaded user: '+u['primaryEmail'].split('@')[0])
users.append({'provider':'external',
'id':u['id'],
'email': u['primaryEmail'],
@ -494,12 +478,9 @@ class Admin():
'username': u['primaryEmail'].split('@')[0],
'groups':[u['orgUnitPath']], ## WARNING: Removing the first
'roles':[],
'password':diceware.get_passphrase(options=options)})
socketio.emit('update',
json.dumps({'status':True,'item':'user','action':'delete','itemdata':''}),
namespace='/isard-sso-admin/sio',
room='admin')
i=i+1
'password': diceware.get_passphrase(options=options)})
item+=1
ev.increment({'name':u['primaryEmail'].split('@')[0]})
self.external['users']=users
## Add groups to users (now they only have their orgUnitPath)
@ -513,6 +494,7 @@ class Admin():
def sync_external(self,ids):
log.warning('Starting sync to keycloak')
self.sync_to_keycloak()
### Now we only sycn external to keycloak and then they can be updated to others with UI buttons
# log.warning('Starting sync to moodle')
# self.sync_to_moodle()
# log.warning('Starting sync to nextcloud')
@ -527,18 +509,21 @@ class Admin():
total=len(groups)
i=0
ev=Events('Syncing import groups to keycloak',total=len(groups))
for g in groups:
i=i+1
# print('ADDING FULL PATH: '+str(g))
log.warning(' KEYCLOAK GROUPS: Adding group ('+str(i)+'/'+str(total)+'): '+g)
ev.increment({'name':g})
self.keycloak.add_group_tree(g)
total=len(self.external['users'])
index=0
ev=Events('Syncing import users to keycloak',total=len(self.external['users']))
for u in self.external['users']:
index=index+1
# Add user
log.warning(' KEYCLOAK USERS: Adding user ('+str(index)+'/'+str(total)+'): '+u['username'])
ev.increment({'name':u['username']})
uid=self.keycloak.add_user(u['username'],u['first'],u['last'],u['email'],u['password'])
# Add user to role and group rolename
@ -573,12 +558,14 @@ class Admin():
### Create all groups. Skip / in system groups
total=len(groups)
i=0
ev=Events('Syncing groups from keycloak to moodle',total=len(groups))
for g in groups:
parts=g.split('/')
subpath=''
for i in range(1,len(parts)):
try:
log.warning(' MOODLE GROUPS: Adding group as cohort ('+str(i)+'/'+str(total)+'): '+subpath)
ev.increment({'name':subpath})
if parts[i] in ['admin','manager','teacher','student']:
subpath=parts[i]
else:
@ -592,19 +579,28 @@ class Admin():
cohorts=self.moodle.get_cohorts()
### Create users in moodle
ev=Events('Syncing users from keycloak to moodle',total=len(self.internal['users']))
for u in self.internal['users']:
if not u['moodle']:
log.error('Creating moodle user: '+u['username'])
user=self.moodle.create_user(u['email'],u['username'],'1Random 1String',u['first'],u['last'])[0]
log.warning('Creating moodle user: '+u['username'])
ev.increment({'name':u['username']})
if u['first'] == '': u['first']=' '
if u['last'] == '': u['last']=' '
try:
# pprint(u)
pprint(self.moodle.create_user(u['email'],u['username'],'1Random 1String',u['first'],u['last'])[0])
except:
log.error(' -->> Error creating on moodle the user: '+u['username'])
# user_id=user['id']
self.resync_data()
### Add user to their cohorts (groups)
ev=Events('Syncing users groups from keycloak to moodle cohorts',total=len(self.internal['users']))
for u in self.internal['users']:
total=len(u['keycloak_groups'])
index=0
ev.increment({'name':u['username']})
for g in u['keycloak_groups']:
log.error('for user groups')
parts=g.split('/')
subpath=''
# pprint(parts)
@ -617,7 +613,6 @@ class Admin():
try:
cohort=[c for c in cohorts if c['name']==subpath][0]
except:
# pprint(subpath)
log.error(' MOODLE USER GROUPS: keycloak group '+subpath+' does not exist as moodle cohort. This should not happen. User '+u['username']+ ' not added.')
try:
@ -640,25 +635,26 @@ class Admin():
total=len(groups)
i=0
ev=Events('Syncing groups from keycloak to nextcloud',total=len(groups))
for g in groups:
parts=g.split('/')
subpath=''
for i in range(1,len(parts)):
try:
log.warning(' NEXTCLOUD GROUPS: Adding group ('+str(i)+'/'+str(total)+'): '+subpath)
ev.increment({'name':subpath})
subpath=subpath+'/'+parts[i]
self.nextcloud.add_group(subpath)
except:
log.error('probably exists')
i=i+1
ev=Events('Syncing users from keycloak to nextcloud',total=len(self.internal['users']))
for u in self.internal['users']:
# print('User '+u['username'])
if not u['nextcloud']:
# print(' Is not in nextcloud')
log.warning(' NEXTCLOUD USERS: Creating nextcloud user: '+u['username']+' in groups '+str(u['keycloak_groups']))
# group=u['keycloak_groups'][0] if len(u['keycloak_groups']) else False
try:
ev.increment({'name':u['username']})
self.nextcloud.add_user_with_groups(u['username'],'1Random 1String',500000000000,u['keycloak_groups'],u['email'],u['first']+' '+u['last'])
except ProviderItemExists:
log.warning('User '+u['username']+' already exists. Skipping...')
@ -673,12 +669,11 @@ class Admin():
i=i+1
if not u['keycloak']: continue
# Do not remove admin users!!! What to do with managers???
if 'admin' in u['roles']: continue
# if 'manager' in u['roles']: continue
if ['admin'] in u['roles']: continue
log.info(' KEYCLOAK USERS: Removing user ('+str(i)+'/'+str(total)+'): '+u['username'])
try:
self.keycloak.delete_user(u['id'])
socketio.emit('update',
app.socketio.emit('update',
json.dumps({'status':True,'item':'user','action':'delete','itemdata':u}),
namespace='/isard-sso-admin/sio',
room='admin')
@ -692,7 +687,7 @@ class Admin():
log.info('Removing nextcloud user: '+u['username'])
try:
self.nextcloud.delete_user(u['nextcloud_id'])
socketio.emit('update',
app.socketio.emit('update',
json.dumps({'status':True,'item':'user','action':'delete','itemdata':u}),
namespace='/isard-sso-admin/sio',
room='admin')
@ -711,7 +706,7 @@ class Admin():
log.warning('Removing moodle users: '+','.join(usernames))
try:
self.moodle.delete_users(userids)
socketio.emit('update',
app.socketio.emit('update',
json.dumps({'status':True,'item':'user','action':'delete','itemdata':u}),
namespace='/isard-sso-admin/sio',
room='admin')

62
admin/src/admin/lib/avatars.py
View File

@ -4,30 +4,52 @@ from admin import app
import logging as log
from pprint import pprint
import os
from os import listdir
from os.path import isfile, join
from .postgres import Postgres
# Module variables to connect to moodle api
from minio import Minio
from minio.commonconfig import REPLACE, CopySource
from minio.deleteobjects import DeleteObject
class Avatars():
def __init__(self):
self.keycloak_pg=Postgres('isard-apps-postgresql','keycloak',os.environ['KEYCLOAK_DB_USER'],os.environ['KEYCLOAK_DB_PASSWORD'])
self.mclient = Minio(
"isard-sso-avatars:9000",
access_key="AKIAIOSFODNN7EXAMPLE",
secret_key="wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY",
secure=False
)
self.bucket='master-avatars'
self._minio_set_realm()
# self.update_missing_avatars()
def username2id(self,username):
q = """select id, username from user_entity where username = '%s'""" % (username)
try:
return self.keycloak_pg.select(q)[0][0]
except:
pass
return False
def update_missing_avatars(self,users):
sys_roles=['admin','manager','teacher','student']
for u in self.get_users_without_image(users):
try:
img=[r+'.jpg' for r in sys_roles if r in u['roles']][0]
except:
img='unknown.jpg'
def get_files(self):
path='avatars/master-avatars/'
onlyfiles = [f for f in listdir(path) if isfile(join(path, f))]
pprint(onlyfiles)
self.mclient.fput_object(
self.bucket, u['id'], os.path.join(app.root_path,"../custom/avatars/"+img),
content_type="image/jpeg ",
)
log.warning(' AVATARS: Updated avatar for user '+u['username']+' with role '+img.split('.')[0])
# def generate_missing(self, users):
# for u in users:
def _minio_set_realm(self):
if not self.mclient.bucket_exists(self.bucket):
self.mclient.make_bucket(self.bucket)
def minio_get_objects(self):
return [o.object_name for o in self.mclient.list_objects(self.bucket)]
def minio_delete_all_objects(self):
delete_object_list = map(
lambda x: DeleteObject(x.object_name),
self.mclient.list_objects(self.bucket),
)
errors=self.mclient.remove_objects(self.bucket, delete_object_list)
for error in errors:
log.error(" AVATARS: Error occured when deleting avatar object", error)
def get_users_without_image(self,users):
return [u for u in users if u['id'] not in self.minio_get_objects()]

73
admin/src/admin/lib/events.py Normal file
View File

@ -0,0 +1,73 @@
#!flask/bin/python
# coding=utf-8
from admin import app
import logging as log
import traceback
from uuid import uuid4
import json
from time import sleep
import sys,os
from flask import render_template, Response, request, redirect, url_for, jsonify
from flask_socketio import SocketIO, emit, join_room, leave_room, \
close_room, rooms, disconnect, send
import base64
class Events():
def __init__(self,title,text='',total=0):
self.eid=str(base64.b64encode(os.urandom(32))[:8])
self.title=title
self.text=text
self.total=total
self.item=0
self.create()
def create(self):
app.socketio.emit('notify-create',
json.dumps({'id':self.eid,
'title':self.title,
'text':self.text}),
namespace='/isard-sso-admin/sio',
room='admin')
sleep(0.1)
def __del__(self):
app.socketio.emit('notify-destroy',
json.dumps({'id':self.eid}),
namespace='/isard-sso-admin/sio',
room='admin')
sleep(0.1)
def update_text(self,text):
self.text=text
app.socketio.emit('notify-update',
json.dumps({'id':self.eid,
'text':self.text,}),
namespace='/isard-sso-admin/sio',
room='admin')
sleep(0.1)
def increment(self,data={}):
self.item+=1
app.socketio.emit('notify-update',
json.dumps({'id':self.eid,
'text': '['+str(self.item)+'/'+str(self.total)+'] ',
'item':self.item,
'total':self.total,
'data':data}),
namespace='/isard-sso-admin/sio',
room='admin')
sleep(0.1)
def decrement(self,data={}):
self.item-=1
app.socketio.emit('notify-update',
json.dumps({'id':self.eid,
'text':'['+str(self.item)+'/'+str(self.total)+'] ',
'item':self.item,
'total':self.total,
'data':data}),
namespace='/isard-sso-admin/sio',
room='admin')
sleep(0.1)

65
admin/src/admin/static/js/external.js
View File

@ -17,7 +17,7 @@ $(document).ready(function() {
$.each(users_table.rows().data(),function(key, value){
ids[value['id']]=value['roles']
});
console.log(ids)
// console.log(ids)
$.ajax({
type: "PUT",
url:"/isard-sso-admin/external",
@ -46,25 +46,30 @@ $(document).ready(function() {
})
$("#modalImport #send").on('click', function(e){
console.log(users_table.rows().data())
// console.log(users_table.rows().data())
var form = $('#modalImportForm');
form.parsley().validate();
if (form.parsley().isValid()){
formdata = form.serializeObject()
formdata['data']=JSON.parse(filecontents)
if($('#format').val() == 'csv-ug'){
formdata['data']=parseCSV(filecontents)
}else{
formdata['data']=JSON.parse(filecontents)
}
$.ajax({
type: "POST",
url:"/isard-sso-admin/external",
data: JSON.stringify(formdata),
success: function(data)
{
$("#modalImport").modal('hide');
users_table.ajax.reload();
groups_table.ajax.reload();
console.log('SUCCESS')
// $("#modalImport").modal('hide');
// users_table.ajax.reload();
// groups_table.ajax.reload();
},
error: function(data)
{
alert('Something went wrong on our side...')
console.log('Ajax timeout')
}
});
}
@ -194,20 +199,46 @@ $(document).ready(function() {
});
function readFile (evt) {
path = "";
items = [];
var files = evt.target.files;
var file = files[0];
var reader = new FileReader();
reader.onload = function(event) {
filecontents=event.target.result;
$.each(JSON.parse(filecontents), walker);
populate_path(items)
if($('#format').val() == 'json-ga'){
path = "";
items = [];
var files = evt.target.files;
var file = files[0];
var reader = new FileReader();
reader.onload = function(event) {
filecontents=event.target.result;
$.each(JSON.parse(filecontents), walker);
populate_path(items)
}
reader.readAsText(file, 'UTF-8')
}
if($('#format').val() == 'csv-ug'){
var files = evt.target.files;
var file = files[0];
var reader = new FileReader();
reader.onload = function(event) {
filecontents=event.target.result;
// $.each(JSON.parse(filecontents), walker);
// populate_path(items)
}
reader.readAsText(file, 'UTF-8')
}
reader.readAsText(file, 'UTF-8')
}
function parseCSV(){
lines=filecontents.split('\n')
header=lines[0].split(';')
users=[]
$.each(lines, function(n, l){
if(n!=0 && l.length > 10){
usr=toObject(header,l.split(';'))
usr['id']=usr['username']
users.push(usr)
}
})
return users;
}
function toObject(names, values) {
var result = {};
for (var i = 0; i < names.length; i++)

101
admin/src/admin/static/js/status_socket.js
View File

@ -1,19 +1,86 @@
// SocketIO
socket = io.connect(location.protocol+'//' + document.domain +'/isard-sso-admin/sio');
console.log(location.protocol+'//' + document.domain +'/isard-sso-admin/sio')
socket.on('connect', function() {
// connection_done();
console.log('Listening status socket');
});
notice={}
$lost=0;
socket.on('connect_error', function(data) {
// connection_lost();
});
socket.on('update', function(data) {
var data = JSON.parse(data);
console.log('Status update')
console.log(data)
// var data = JSON.parse(data);
// drawUserQuota(data);
socket = io.connect(location.protocol+'//' + document.domain +'/isard-sso-admin/sio');
console.log(location.protocol+'//' + document.domain +'/isard-sso-admin/sio')
socket.on('connect', function() {
if($lost){location.reload();}
console.log('Listening status socket');
});
socket.on('connect_error', function(data) {
$lost=$lost+1;
$('#modal-lostconnection').modal({
backdrop: 'static',
keyboard: false
}).modal('show');
});
socket.on('notify-create', function(data) {
var data = JSON.parse(data);
notice[data.id] = new PNotify({
title: data.title,
text: data.text,
hide: false
});
});
socket.on('notify-destroy', function(data) {
var data = JSON.parse(data);
notice[data.id].remove()
});
socket.on('notify-update', function(data) {
var data = JSON.parse(data);
// console.log(data.text)
notice[data.id].update({
text: data.text
})
});
// new PNotify({
// title: "Quota for creating desktops full.",
// text: "Can't create another desktop, user quota full.",
// hide: true,
// delay: 3000,
// icon: 'fa fa-alert-sign',
// opacity: 1,
// type: 'error'
// });
// socket.on('update', function(data) {
// var data = JSON.parse(data);
// console.log('Status update')
// console.log(data)
// // var data = JSON.parse(data);
// // drawUserQuota(data);
// });
socket.on('update', function(data) {
var data = JSON.parse(data);
console.log('Status update')
console.log(data)
// var data = JSON.parse(data);
// drawUserQuota(data);
});
// {'event':'traceback',
// 'id':u['id'],
// 'item':'group',
// 'action':'add'
// 'name':g['name'],
// 'progress':str(item)+'/'+str(total),
// 'status':False,
// 'msg':,
// 'payload':{'traceback':traceback.format_exc(),
// 'data':g})
socket.on('progress', function(data) {
var data = JSON.parse(data);
console.log(data)
// $('.modal-progress #item').html(data.item)
});
////

2
admin/src/admin/static/js/users.js
View File

@ -220,7 +220,7 @@ $(document).ready(function() {
"columnDefs": [ {
"targets": 1,
"render": function ( data, type, full, meta ) {
return '<object data="static/img/usera.jpg" type="image/png" width="25" height="25"><img src="/isard-sso-admin/avatars/'+full.id+'" title="'+full.id+'" width="25" height="25"></object>'
return '<img src="/isard-sso-admin/avatars/'+full.id+'" title="'+full.id+'" width="25" height="25"></object>'
}},
{
"targets": 6,

23
admin/src/admin/static/templates/base.html
View File

@ -59,26 +59,9 @@
<!-- /footer content -->
</div>
</div>
<div id="modal-lostconnection" class="modal fade" role="dialog" style="width:50%;margin-left:30%;margin-top:10%;z-index: 100000;">
<div class="modal-admin">
<div class="modal-content">
<div class="row text-center"><h2 style="margin-bottom:5px">Connection lost</h2></div>
<hr>
<div class="row">
<div class="col-md-1 col-sm-1 col-xs-12"></div>
<div class="col-md-10 col-sm-10 col-xs-12">
<div class="row text-center">Unable to contact server. There should be a problem with network or a heavy load.</div>
<br>
<div class="row text-center" style="margin-bottom:25px">
<i class="fa fa-circle-o-notch fa-spin fa-fw"></i> Trying to reconnect...
</div>
</div>
<div class="col-md-1 col-sm-1 col-xs-12">
</div>
</div>
</div>
</div>
</div>
{% include 'pages/modals/common_modals.html' %}
<!-- jQuery -->
<script src="/isard-sso-admin/vendors/jquery/dist/jquery.js"></script>

44
admin/src/admin/static/templates/pages/modals/common_modals.html Normal file
View File

@ -0,0 +1,44 @@
<div id="modal-lostconnection" class="modal fade" role="dialog" style="width:50%;margin-left:30%;margin-top:10%;z-index: 100000;">
<div class="modal-admin">
<div class="modal-content">
<div class="row text-center"><h2 style="margin-bottom:5px">Connection lost</h2></div>
<hr>
<div class="row">
<div class="col-md-1 col-sm-1 col-xs-12"></div>
<div class="col-md-10 col-sm-10 col-xs-12">
<div class="row text-center">Unable to contact server. There should be a problem with network or a heavy load.</div>
<br>
<div class="row text-center">
<i class="fa fa-circle-o-notch fa-spin fa-fw"></i> Trying to reconnect...
</div>
<br>
<div class="col-md-1 col-sm-1 col-xs-12">
</div>
</div>
</div>
</div>
</div>
</div>
<div id="modal-info" class="modal fade" role="dialog" style="width:100%;margin-left:10%;margin-top:10%;z-index: 100000;">
<div class="modal-admin">
<div class="modal-content">
<div class="row text-center"><h2 style="margin-bottom:5px">Connection lost</h2></div>
<hr>
<div class="row">
<div class="col-md-10 col-sm-10 col-xs-12">
<div class="row text-center">Unable to contact server. There should be a problem with network or a heavy load.</div>
<br>
<div class="row text-center">
<i class="fa fa-circle-o-notch fa-spin fa-fw"></i> Trying to reconnect...
</div>
<br>
<div class="col-md-1 col-sm-1 col-xs-12">
</div>
</div>
</div>
</div>
</div>
</div>

3
admin/src/admin/static/templates/pages/modals/external_modals.html
View File

@ -32,7 +32,8 @@
<label class="control-label col-md-3 col-sm-3 col-xs-12" for="format">Format: <span class="required">*</span></label>
<div class="col-md-6 col-sm-6 col-xs-12">
<select id="format" name="format" class="form-control format" required>
<option value="json">JSON dump</option>
<option value="json-ga">GAdminconsole JSON</option>
<option value="csv-ug">CSV user with groups</option>
</select>
</div>
</div>

4
admin/src/admin/static/templates/pages/users.html
View File

@ -27,7 +27,7 @@
<button class="btn btn-primary btn-xs btn-sync_to_moodle">
<i class="fa fa-refresh" aria-hidden="true"></i> Sync to Moodle
</button>
<!-- <button class="btn btn-danger btn-xs btn-delete_keycloak">
<button class="btn btn-danger btn-xs btn-delete_keycloak">
<i class="fa fa-trash"></i> Delete all keycloak
</button>
<button class="btn btn-danger btn-xs btn-delete_nextcloud">
@ -35,7 +35,7 @@
</button>
<button class="btn btn-danger btn-xs btn-delete_moodle">
<i class="fa fa-trash"></i> Delete missing keycloak in moodle
</button> -->
</button>
<table id="users" class="table" width="100%">
<thead>
<tr>

25
admin/src/admin/views/MenuViews.py
View File

@ -8,9 +8,14 @@ from uuid import uuid4
import time,json
import sys,os
from flask import render_template, Response, request, redirect, url_for, jsonify
import concurrent.futures
from pprint import pprint
# from flask_socketio import SocketIO, emit, join_room, leave_room, \
# close_room, rooms, disconnect, send
# socketio = SocketIO(app)
@app.route('/isard-sso-admin/resync')
# @login_required
def resync():
@ -70,7 +75,12 @@ def groups_list():
# @login_required
def external():
if request.method == 'POST':
return json.dumps(app.admin.upload_json(request.get_json(force=True))), 200, {'Content-Type': 'application/json'}
data=request.get_json(force=True)
if data['format']=='json-ga':
app.admin.upload_json_ga(data)
if data['format']=='csv-ug':
app.admin.upload_csv_ug(data)
return json.dumps({}), 200, {'Content-Type': 'application/json'}
if request.method == 'PUT':
return json.dumps(app.admin.sync_external(request.get_json(force=True))), 200, {'Content-Type': 'application/json'}
return render_template('pages/external.html', title="External", nav="External")
@ -89,4 +99,15 @@ def external_groups_list():
# @login_required
def external_roles():
if request.method == 'PUT':
return json.dumps(app.admin.external_roleassign(request.get_json(force=True))), 200, {'Content-Type': 'application/json'}
return json.dumps(app.admin.external_roleassign(request.get_json(force=True))), 200, {'Content-Type': 'application/json'}
def WaitStatus(self, desktop_id, original_status, transition_status, final_status):
with concurrent.futures.ThreadPoolExecutor(max_workers=1) as executor:
future = executor.submit(lambda p: self._wait_for_domain_status(*p), [desktop_id, original_status, transition_status, final_status])
try:
result = future.result()
except ReqlTimeoutError:
raise DesktopActionTimeout
except DesktopWaitFailed:
raise DesktopActionFailed
return True

32
admin/src/admin/views/Socketio.py
View File

@ -1,19 +1,19 @@
from flask_socketio import SocketIO, emit, join_room, leave_room, \
close_room, rooms, disconnect, send
from admin import app
import json
# from flask_socketio import SocketIO, emit, join_room, leave_room, \
# close_room, rooms, disconnect, send
# from admin import app
# import json
socketio = SocketIO(app)
# from ...start import socketio
# # socketio = SocketIO(app)
# # from ...start import socketio
@socketio.on('connect', namespace='/isard-sso-admin/sio')
def socketio_connect():
join_room('admin')
socketio.emit('update',
json.dumps('Joined'),
namespace='/isard-sso-admin/sio',
room='admin')
# @socketio.on('connect', namespace='/isard-sso-admin/sio')
# def socketio_connect():
# join_room('admin')
# socketio.emit('update',
# json.dumps('Joined'),
# namespace='/isard-sso-admin/sio',
# room='admin')
@socketio.on('disconnect', namespace='/isard-sso-admin/sio')
def socketio_domains_disconnect():
None
# @socketio.on('disconnect', namespace='/isard-sso-admin/sio')
# def socketio_domains_disconnect():
# None

5
admin/src/claus.py
View File

@ -1,5 +0,0 @@
import diceware
options = diceware.handle_options(None)
options.wordlist = 'cat_ascii'
options.num = 3
print(diceware.get_passphrase(options=options))

7
admin/src/scripts/README.md Normal file
View File

@ -0,0 +1,7 @@
# Scripts for cli
Take care at using this at your own risk!
The reset_pwd.py for example will reset ALL USERS password and dump a file with pwd that should be removed!
The avatars.py will reset all users avatars to defaults.
If you still want to play with them copy it one folder up to be used.

118
admin/src/scripts/avatars.py Normal file
View File

@ -0,0 +1,118 @@
#!/usr/bin/env python
import time ,os
from datetime import datetime, timedelta
import logging as log
import traceback
import yaml, json
from pprint import pprint
from keycloak import KeycloakAdmin
from postgres import Postgres
from minio import Minio
from minio.commonconfig import REPLACE, CopySource
from minio.deleteobjects import DeleteObject
class DefaultAvatars():
def __init__(self,
url="http://isard-sso-keycloak:8080/auth/",
username=os.environ['KEYCLOAK_USER'],
password=os.environ['KEYCLOAK_PASSWORD'],
realm='master',
verify=True):
self.url=url
self.username=username
self.password=password
self.realm=realm
self.verify=verify
self.keycloak_pg=Postgres('isard-apps-postgresql','keycloak',os.environ['KEYCLOAK_DB_USER'],os.environ['KEYCLOAK_DB_PASSWORD'])
self.mclient = Minio(
"isard-sso-avatars:9000",
access_key="AKIAIOSFODNN7EXAMPLE",
secret_key="wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY",
secure=False
)
self.bucket='master-avatars'
self._minio_set_realm()
self.update_missing_avatars()
def connect(self):
self.keycloak_admin = KeycloakAdmin(server_url=self.url,
username=self.username,
password=self.password,
realm_name=self.realm,
verify=self.verify)
def update_missing_avatars(self):
sys_roles=['admin','manager','teacher','student']
for u in self.get_users_without_image():
try:
img=[r+'.jpg' for r in sys_roles if r in u['role']][0]
except:
img='unknown.jpg'
self.mclient.fput_object(
self.bucket, u['id'], "custom/avatars/"+img,
content_type="image/jpeg ",
)
log.warning(' AVATARS: Updated avatar for user '+u['username']+' with role '+img.split('.')[0])
def _minio_set_realm(self):
if not self.mclient.bucket_exists(self.bucket):
self.mclient.make_bucket(self.bucket)
def minio_get_objects(self):
return [o.object_name for o in self.mclient.list_objects(self.bucket)]
def minio_delete_all_objects(self):
delete_object_list = map(
lambda x: DeleteObject(x.object_name),
self.mclient.list_objects(self.bucket),
)
errors=self.mclient.remove_objects(self.bucket, delete_object_list)
for error in errors:
log.error(" AVATARS: Error occured when deleting avatar object", error)
def get_users(self):
self.connect()
users=self.get_users_with_groups_and_roles()
return users
def get_users_without_image(self):
return [u for u in self.get_users() if u['id'] not in self.minio_get_objects()]
def get_users_with_groups_and_roles(self):
q = """select u.id, u.username, u.email, u.first_name, u.last_name, u.realm_id, ua.value as quota
,json_agg(g."name") as group, json_agg(g_parent."name") as group_parent1, json_agg(g_parent2."name") as group_parent2
,json_agg(r.name) as role
from user_entity as u
left join user_attribute as ua on ua.user_id=u.id and ua.name = 'quota'
left join user_group_membership as ugm on ugm.user_id = u.id
left join keycloak_group as g on g.id = ugm.group_id
left join keycloak_group as g_parent on g.parent_group = g_parent.id
left join keycloak_group as g_parent2 on g_parent.parent_group = g_parent2.id
left join user_role_mapping as rm on rm.user_id = u.id
left join keycloak_role as r on r.id = rm.role_id
group by u.id,u.username,u.email,u.first_name,u.last_name, u.realm_id, ua.value
order by u.username"""
(headers,users)=self.keycloak_pg.select_with_headers(q)
users_with_lists = [list(l[:-4])+([[]] if l[-4] == [None] else [list(set(l[-4]))]) +\
([[]] if l[-3] == [None] else [list(set(l[-3]))]) +\
([[]] if l[-3] == [None] else [list(set(l[-2]))]) +\
([[]] if l[-1] == [None] else [list(set(l[-1]))]) for l in users]
users_with_lists = [list(l[:-4])+([[]] if l[-4] == [None] else [list(set(l[-4]))]) +\
([[]] if l[-3] == [None] else [list(set(l[-3]))]) +\
([[]] if l[-3] == [None] else [list(set(l[-2]))]) +\
([[]] if l[-1] == [None] else [list(set(l[-1]))]) for l in users_with_lists]
list_dict_users = [dict(zip(headers, r)) for r in users_with_lists]
return list_dict_users
da=DefaultAvatars()

0
admin/src/reset_pwds.py → admin/src/scripts/reset_pwds.py
View File

33
admin/src/start.py
View File

@ -11,19 +11,26 @@ from admin import app
# socketio.init_app(app, cors_allowed_origins="*")
from admin.views.Socketio import *
# socketio = SocketIO(app)
# @socketio.on('connect', namespace='/isard-sso-admin/sio')
# def socketio_connect():
# join_room('admin')
# socketio.emit('update',
# json.dumps('Joined'),
# namespace='/isard-sso-admin/sio',
# room='admin')
# from admin.views.Socketio import *
app.socketio = SocketIO(app)
# app.socketio.init_app(app, cors_allowed_origins="*")
@app.socketio.on('connect', namespace='/isard-sso-admin/sio')
def socketio_connect():
join_room('admin')
app.socketio.emit('update',
json.dumps('Joined'),
namespace='/isard-sso-admin/sio',
room='admin')
# socketio.emit('update',
# json.dumps('DATA SENT'),
# namespace='/isard-sso-admin/sio',
# room='admin')
# @socketio.on('disconnect', namespace='/isard-sso-admin/sio')
# def socketio_domains_disconnect():
# None
@app.socketio.on('disconnect', namespace='/isard-sso-admin/sio')
def socketio_domains_disconnect():
None
if __name__ == '__main__':
socketio.run(app,host='0.0.0.0', port=9000, debug=False, cors_allowed_origins="*") #, logger=logger, engineio_logger=engineio_logger)
app.socketio.run(app,host='0.0.0.0', port=9000, debug=False, cors_allowed_origins="*", async_mode="threading") #, logger=logger, engineio_logger=engineio_logger)

4
docker/haproxy/haproxy.conf
View File

@ -93,6 +93,10 @@ backend be_sso
backend be_admin
mode http
option forwardfor
timeout queue 600s
timeout server 600s
timeout connect 600s
acl authorized http_auth(AuthUsers)
http-request auth realm AuthUsers unless authorized
acl existing-x-forwarded-host req.hdr(X-Forwarded-Host) -m found

63
minio-client-test.py
View File

@ -1,63 +0,0 @@
from minio import Minio
from minio.commonconfig import REPLACE, CopySource
# Create client with anonymous access.
# client = Minio("isard-apps-avatars")
# # Create client with access and secret key.
# client = Minio("s3.amazonaws.com", "ACCESS-KEY", "SECRET-KEY")
# Create client with access key and secret key with specific region.
client = Minio(
"isard-sso-avatars:9000",
access_key="AKIAIOSFODNN7EXAMPLE",
secret_key="wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY",
secure=False
)
buckets = client.list_buckets()
for bucket in buckets:
print(bucket.name, bucket.creation_date)
response = client.get_object("master-avatars", "89423d20-3915-4e67-b227-86f099f1816a")
print(response)
result = client.copy_object(
"master-avatars",
"prova",
CopySource("master-avatars", "89423d20-3915-4e67-b227-86f099f1816a"),
)
client.remove_object("master-avatars", "prova")
result = client.fput_object(
"master-avatars", "test", "admin/static/img/background.png",
content_type="image/jpeg ",
)
objects = client.list_objects("master-avatars")
for obj in objects:
print(obj.key)
exit(1)
try:
response = client.get_object("master-avatars", "my-object")
print(response)
# Read data from response.
finally:
response.close()
response.release_conn()
# region="my-region",
# # Create client with custom HTTP client using proxy server.
# import urllib3
# client = Minio(
# "SERVER:PORT",
# access_key="ACCESS_KEY",
# secret_key="SECRET_KEY",
# secure=True,
# http_client=urllib3.ProxyManager(
# "https://PROXYSERVER:PROXYPORT/",
# timeout=urllib3.Timeout.DEFAULT_TIMEOUT,
# cert_reqs="CERT_REQUIRED",
# retries=urllib3.Retry(
# total=5,
# backoff_factor=0.2,
# status_forcelist=[500, 502, 503, 504],
# ),
# ),
# )