diff --git a/admin/docker/requirements.pip3 b/admin/docker/requirements.pip3 index 07e82d0..c2479e8 100644 --- a/admin/docker/requirements.pip3 +++ b/admin/docker/requirements.pip3 @@ -23,4 +23,6 @@ diceware==0.9.6 #gevent==1.4.0 #greenlet==0.4.15 python-engineio==3.8.1 -python-socketio==4.1.0 \ No newline at end of file +python-socketio==4.1.0 + +minio==7.0.3 \ No newline at end of file diff --git a/admin/src/admin/lib/admin.py b/admin/src/admin/lib/admin.py index 17cacef..3187f40 100644 --- a/admin/src/admin/lib/admin.py +++ b/admin/src/admin/lib/admin.py @@ -20,12 +20,6 @@ options.num = 3 from .events import Events -# from flask_socketio import SocketIO, emit, join_room, leave_room, \ -# close_room, rooms, disconnect, send -# socketio = SocketIO(app) -# from ..views.Socketio import socketio -# socketio = SocketIO(app) - class Admin(): def __init__(self): ready=False @@ -39,9 +33,6 @@ class Admin(): sleep(2) log.warning('Keycloak connected.') - # print(self.keycloak.get_user_groups_paths('320b0713-c480-4928-9105-041309d72191')) - # exit(1) - ready=False while not ready: try: @@ -73,7 +64,6 @@ class Admin(): self.default_setup() self.internal={} - # self.resync_data() ready=False while not ready: @@ -87,24 +77,11 @@ class Admin(): self.external={'users':[], 'groups':[], 'roles':[]} + log.warning(' Updating missing user avatars with defaults') + av=Avatars() + # av.minio_delete_all_objects() # This will reset all avatars on usres + av.update_missing_avatars(self.internal['users']) log.warning(' SYSTEM READY TO HANDLE CONNECTIONS') - # self.test_cohorts() - # self.delete_all_moodle_cohorts() - - ########## Testing: get user group - # cids=[c['id'] for c in self.moodle.get_cohorts()] - # relations=self.moodle.get_cohort_members(cids) - # for r in relations: - # print(self.get_internalgroup_from_moodlecohortid(r['cohortid'])['path']) - - # def get_internalgroup_from_moodlecohortid(self,cohort_id): - # for g in self.internal['groups']: - # if not g['moodle']: continue - # if g['moodle_id'] == cohort_id: return g - # return '' - - - ## This function should be moved to postup.py def default_setup(self): @@ -122,9 +99,9 @@ class Admin(): self.keycloak_admin.group_user_add(admin_uid,gid) log.warning('KEYCLOAK: OK') except: + # print(traceback.format_exc()) log.warning('KEYCLOAK: Seems to be there already') - #### Add default groups try: log.warning('KEYCLOAK: Adding default groups') @@ -240,37 +217,6 @@ class Admin(): except: log.warning('MOODLE: Seems to be there already') - - ### testing - # def test_cohorts(self): - # cohorts=self.moodle.get_cohorts() - - # testc=[c for c in cohorts if c['name'] in ['teacher','/teacher','student','/student']] - # pprint(testc) - - # groups=[] - # for u in self.internal['users']: - # groups=groups+u['keycloak_groups'] - # groups=list(dict.fromkeys(groups)) - # pprint(groups) - # pprint([g for g in groups if 'teacher' in g['keycloak_groups']]) - # exit(1) - # total=len(groups) - # i=0 - # for g in groups: - # parts=g.split('/') - # subpath='' - # for i in range(1,len(parts)): - # try: - # log.warning(' MOODLE GROUPS: Adding group as cohort ('+str(i)+'/'+str(total)+'): '+subpath) - # subpath=subpath+'/'+parts[i] - # self.moodle.add_system_cohort(subpath) - # except: - # log.error('probably exists') - # i=i+1 - - ### end testing - def resync_data(self): self.internal={'users':self._get_mix_users(), 'groups':self._get_mix_groups(), @@ -380,8 +326,7 @@ class Admin(): theuser['nextcloud_groups']=[] del theuser['groups'] users.append(theuser) - - # pprint([u['moodle_groups'] for u in users]) + return users def get_roles(self): @@ -525,17 +470,6 @@ class Admin(): ev=Events('Processing uploaded users',total=len(data['data']['users'])) for u in data['data']['users']: log.warning('Processing ('+str(item)+'/'+str(total)+') uploaded user: '+u['primaryEmail'].split('@')[0]) - - # self.e.send({'event':'progress', - # 'id':u['id'], - # 'item':'user', - # 'action':'passwdgen', - # 'name':u['primaryEmail'].split('@')[0], - # 'progress':str(item)+'/'+str(total), - # 'status':True, - # 'msg':'Generating password', - # 'payload':None}) - # data['provider'] users.append({'provider':'external', 'id':u['id'], 'email': u['primaryEmail'], @@ -545,11 +479,6 @@ class Admin(): 'groups':[u['orgUnitPath']], ## WARNING: Removing the first 'roles':[], 'password': diceware.get_passphrase(options=options)}) - # app.socketio.emit('update', - # json.dumps({'status':True,'item':'user','action':'delete','itemdata':''}), - # namespace='/isard-sso-admin/sio', - # room='admin') - # sleep(0) item+=1 ev.increment({'name':u['primaryEmail'].split('@')[0]}) self.external['users']=users @@ -565,6 +494,7 @@ class Admin(): def sync_external(self,ids): log.warning('Starting sync to keycloak') self.sync_to_keycloak() + ### Now we only sycn external to keycloak and then they can be updated to others with UI buttons # log.warning('Starting sync to moodle') # self.sync_to_moodle() # log.warning('Starting sync to nextcloud') @@ -583,7 +513,6 @@ class Admin(): ev=Events('Syncing import groups to keycloak',total=len(groups)) for g in groups: i=i+1 - # print('ADDING FULL PATH: '+str(g)) log.warning(' KEYCLOAK GROUPS: Adding group ('+str(i)+'/'+str(total)+'): '+g) ev.increment({'name':g}) self.keycloak.add_group_tree(g) @@ -685,7 +614,6 @@ class Admin(): try: cohort=[c for c in cohorts if c['name']==subpath][0] except: - # pprint(subpath) log.error(' MOODLE USER GROUPS: keycloak group '+subpath+' does not exist as moodle cohort. This should not happen. User '+u['username']+ ' not added.') try: @@ -724,11 +652,8 @@ class Admin(): ev=Events('Syncing users from keycloak to nextcloud',total=len(self.internal['users'])) for u in self.internal['users']: - # print('User '+u['username']) if not u['nextcloud']: - # print(' Is not in nextcloud') log.warning(' NEXTCLOUD USERS: Creating nextcloud user: '+u['username']+' in groups '+str(u['keycloak_groups'])) - # group=u['keycloak_groups'][0] if len(u['keycloak_groups']) else False try: ev.increment({'name':u['username']}) self.nextcloud.add_user_with_groups(u['username'],'1Random 1String',500000000000,u['keycloak_groups'],u['email'],u['first']+' '+u['last']) @@ -745,8 +670,7 @@ class Admin(): i=i+1 if not u['keycloak']: continue # Do not remove admin users!!! What to do with managers??? - if 'admin' in u['roles']: continue - # if 'manager' in u['roles']: continue + if ['admin'] in u['roles']: continue log.info(' KEYCLOAK USERS: Removing user ('+str(i)+'/'+str(total)+'): '+u['username']) try: self.keycloak.delete_user(u['id']) diff --git a/admin/src/admin/lib/avatars.py b/admin/src/admin/lib/avatars.py index 1db2318..36a0cf3 100644 --- a/admin/src/admin/lib/avatars.py +++ b/admin/src/admin/lib/avatars.py @@ -4,30 +4,52 @@ from admin import app import logging as log from pprint import pprint import os -from os import listdir -from os.path import isfile, join -from .postgres import Postgres - -# Module variables to connect to moodle api +from minio import Minio +from minio.commonconfig import REPLACE, CopySource +from minio.deleteobjects import DeleteObject class Avatars(): - def __init__(self): - self.keycloak_pg=Postgres('isard-apps-postgresql','keycloak',os.environ['KEYCLOAK_DB_USER'],os.environ['KEYCLOAK_DB_PASSWORD']) + self.mclient = Minio( + "isard-sso-avatars:9000", + access_key="AKIAIOSFODNN7EXAMPLE", + secret_key="wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY", + secure=False + ) + self.bucket='master-avatars' + self._minio_set_realm() + # self.update_missing_avatars() - def username2id(self,username): - q = """select id, username from user_entity where username = '%s'""" % (username) - try: - return self.keycloak_pg.select(q)[0][0] - except: - pass - return False + def update_missing_avatars(self,users): + sys_roles=['admin','manager','teacher','student'] + for u in self.get_users_without_image(users): + try: + img=[r+'.jpg' for r in sys_roles if r in u['roles']][0] + except: + img='unknown.jpg' - def get_files(self): - path='avatars/master-avatars/' - onlyfiles = [f for f in listdir(path) if isfile(join(path, f))] - pprint(onlyfiles) + self.mclient.fput_object( + self.bucket, u['id'], os.path.join(app.root_path,"../custom/avatars/"+img), + content_type="image/jpeg ", + ) + log.warning(' AVATARS: Updated avatar for user '+u['username']+' with role '+img.split('.')[0]) - # def generate_missing(self, users): - # for u in users: + def _minio_set_realm(self): + if not self.mclient.bucket_exists(self.bucket): + self.mclient.make_bucket(self.bucket) + + def minio_get_objects(self): + return [o.object_name for o in self.mclient.list_objects(self.bucket)] + + def minio_delete_all_objects(self): + delete_object_list = map( + lambda x: DeleteObject(x.object_name), + self.mclient.list_objects(self.bucket), + ) + errors=self.mclient.remove_objects(self.bucket, delete_object_list) + for error in errors: + log.error(" AVATARS: Error occured when deleting avatar object", error) + + def get_users_without_image(self,users): + return [u for u in users if u['id'] not in self.minio_get_objects()] \ No newline at end of file diff --git a/admin/src/admin/static/js/users.js b/admin/src/admin/static/js/users.js index efcea8b..63a5705 100644 --- a/admin/src/admin/static/js/users.js +++ b/admin/src/admin/static/js/users.js @@ -220,7 +220,7 @@ $(document).ready(function() { "columnDefs": [ { "targets": 1, "render": function ( data, type, full, meta ) { - return '' + return '' }}, { "targets": 6, diff --git a/admin/src/admin/static/templates/pages/users.html b/admin/src/admin/static/templates/pages/users.html index 425c9b1..0fd62fb 100644 --- a/admin/src/admin/static/templates/pages/users.html +++ b/admin/src/admin/static/templates/pages/users.html @@ -27,7 +27,7 @@ - + diff --git a/admin/src/claus.py b/admin/src/claus.py deleted file mode 100644 index cb330ba..0000000 --- a/admin/src/claus.py +++ /dev/null @@ -1,5 +0,0 @@ -import diceware -options = diceware.handle_options(None) -options.wordlist = 'cat_ascii' -options.num = 3 -print(diceware.get_passphrase(options=options)) diff --git a/admin/src/scripts/README.md b/admin/src/scripts/README.md new file mode 100644 index 0000000..6163ed2 --- /dev/null +++ b/admin/src/scripts/README.md @@ -0,0 +1,7 @@ +# Scripts for cli + +Take care at using this at your own risk! +The reset_pwd.py for example will reset ALL USERS password and dump a file with pwd that should be removed! +The avatars.py will reset all users avatars to defaults. + +If you still want to play with them copy it one folder up to be used. diff --git a/admin/src/scripts/avatars.py b/admin/src/scripts/avatars.py new file mode 100644 index 0000000..6b7f300 --- /dev/null +++ b/admin/src/scripts/avatars.py @@ -0,0 +1,118 @@ +#!/usr/bin/env python +import time ,os +from datetime import datetime, timedelta + +import logging as log +import traceback +import yaml, json +from pprint import pprint + +from keycloak import KeycloakAdmin +from postgres import Postgres + +from minio import Minio +from minio.commonconfig import REPLACE, CopySource +from minio.deleteobjects import DeleteObject + +class DefaultAvatars(): + def __init__(self, + url="http://isard-sso-keycloak:8080/auth/", + username=os.environ['KEYCLOAK_USER'], + password=os.environ['KEYCLOAK_PASSWORD'], + realm='master', + verify=True): + self.url=url + self.username=username + self.password=password + self.realm=realm + self.verify=verify + + self.keycloak_pg=Postgres('isard-apps-postgresql','keycloak',os.environ['KEYCLOAK_DB_USER'],os.environ['KEYCLOAK_DB_PASSWORD']) + + self.mclient = Minio( + "isard-sso-avatars:9000", + access_key="AKIAIOSFODNN7EXAMPLE", + secret_key="wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY", + secure=False + ) + self.bucket='master-avatars' + self._minio_set_realm() + self.update_missing_avatars() + + def connect(self): + self.keycloak_admin = KeycloakAdmin(server_url=self.url, + username=self.username, + password=self.password, + realm_name=self.realm, + verify=self.verify) + + def update_missing_avatars(self): + sys_roles=['admin','manager','teacher','student'] + for u in self.get_users_without_image(): + try: + img=[r+'.jpg' for r in sys_roles if r in u['role']][0] + except: + img='unknown.jpg' + + self.mclient.fput_object( + self.bucket, u['id'], "custom/avatars/"+img, + content_type="image/jpeg ", + ) + log.warning(' AVATARS: Updated avatar for user '+u['username']+' with role '+img.split('.')[0]) + + def _minio_set_realm(self): + if not self.mclient.bucket_exists(self.bucket): + self.mclient.make_bucket(self.bucket) + + def minio_get_objects(self): + return [o.object_name for o in self.mclient.list_objects(self.bucket)] + + def minio_delete_all_objects(self): + delete_object_list = map( + lambda x: DeleteObject(x.object_name), + self.mclient.list_objects(self.bucket), + ) + errors=self.mclient.remove_objects(self.bucket, delete_object_list) + for error in errors: + log.error(" AVATARS: Error occured when deleting avatar object", error) + + def get_users(self): + self.connect() + users=self.get_users_with_groups_and_roles() + return users + + def get_users_without_image(self): + return [u for u in self.get_users() if u['id'] not in self.minio_get_objects()] + + def get_users_with_groups_and_roles(self): + q = """select u.id, u.username, u.email, u.first_name, u.last_name, u.realm_id, ua.value as quota + ,json_agg(g."name") as group, json_agg(g_parent."name") as group_parent1, json_agg(g_parent2."name") as group_parent2 + ,json_agg(r.name) as role + from user_entity as u + left join user_attribute as ua on ua.user_id=u.id and ua.name = 'quota' + left join user_group_membership as ugm on ugm.user_id = u.id + left join keycloak_group as g on g.id = ugm.group_id + left join keycloak_group as g_parent on g.parent_group = g_parent.id + left join keycloak_group as g_parent2 on g_parent.parent_group = g_parent2.id + left join user_role_mapping as rm on rm.user_id = u.id + left join keycloak_role as r on r.id = rm.role_id + group by u.id,u.username,u.email,u.first_name,u.last_name, u.realm_id, ua.value + order by u.username""" + + (headers,users)=self.keycloak_pg.select_with_headers(q) + + users_with_lists = [list(l[:-4])+([[]] if l[-4] == [None] else [list(set(l[-4]))]) +\ + ([[]] if l[-3] == [None] else [list(set(l[-3]))]) +\ + ([[]] if l[-3] == [None] else [list(set(l[-2]))]) +\ + ([[]] if l[-1] == [None] else [list(set(l[-1]))]) for l in users] + + users_with_lists = [list(l[:-4])+([[]] if l[-4] == [None] else [list(set(l[-4]))]) +\ + ([[]] if l[-3] == [None] else [list(set(l[-3]))]) +\ + ([[]] if l[-3] == [None] else [list(set(l[-2]))]) +\ + ([[]] if l[-1] == [None] else [list(set(l[-1]))]) for l in users_with_lists] + + list_dict_users = [dict(zip(headers, r)) for r in users_with_lists] + return list_dict_users + +da=DefaultAvatars() + diff --git a/admin/src/reset_pwds.py b/admin/src/scripts/reset_pwds.py similarity index 100% rename from admin/src/reset_pwds.py rename to admin/src/scripts/reset_pwds.py diff --git a/minio-client-test.py b/minio-client-test.py deleted file mode 100644 index a6b309c..0000000 --- a/minio-client-test.py +++ /dev/null @@ -1,63 +0,0 @@ -from minio import Minio -from minio.commonconfig import REPLACE, CopySource -# Create client with anonymous access. -# client = Minio("isard-apps-avatars") - -# # Create client with access and secret key. -# client = Minio("s3.amazonaws.com", "ACCESS-KEY", "SECRET-KEY") - -# Create client with access key and secret key with specific region. -client = Minio( - "isard-sso-avatars:9000", - access_key="AKIAIOSFODNN7EXAMPLE", - secret_key="wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY", - secure=False -) - -buckets = client.list_buckets() -for bucket in buckets: - print(bucket.name, bucket.creation_date) -response = client.get_object("master-avatars", "89423d20-3915-4e67-b227-86f099f1816a") -print(response) - -result = client.copy_object( - "master-avatars", - "prova", - CopySource("master-avatars", "89423d20-3915-4e67-b227-86f099f1816a"), -) -client.remove_object("master-avatars", "prova") -result = client.fput_object( - "master-avatars", "test", "admin/static/img/background.png", - content_type="image/jpeg ", -) -objects = client.list_objects("master-avatars") -for obj in objects: - print(obj.key) -exit(1) -try: - response = client.get_object("master-avatars", "my-object") - print(response) - # Read data from response. -finally: - response.close() - response.release_conn() - - # region="my-region", -# # Create client with custom HTTP client using proxy server. -# import urllib3 -# client = Minio( -# "SERVER:PORT", -# access_key="ACCESS_KEY", -# secret_key="SECRET_KEY", -# secure=True, -# http_client=urllib3.ProxyManager( -# "https://PROXYSERVER:PROXYPORT/", -# timeout=urllib3.Timeout.DEFAULT_TIMEOUT, -# cert_reqs="CERT_REQUIRED", -# retries=urllib3.Retry( -# total=5, -# backoff_factor=0.2, -# status_forcelist=[500, 502, 503, 504], -# ), -# ), -# )