Will use name as group and based on paths with replaced / with dots .

admin/src/admin/lib/admin.py

@@ -6,7 +6,7 @@ from .nextcloud import Nextcloud
 from .nextcloud_exc import ProviderItemExists
 from .avatars import Avatars
 
-from .helpers import filter_roles_list, filter_roles_listofdicts
+from .helpers import filter_roles_list, filter_roles_listofdicts, system_username, get_gids_from_kgroup_ids, get_group_from_group_id, kpath2gid, gid2kpath
 
 import logging as log
 from pprint import pprint
@@ -18,14 +18,49 @@ options = diceware.handle_options(None)
 options.wordlist = 'cat_ascii'
 options.num = 3
 
-from .helpers import rand_password
+from .helpers import rand_password, count_repeated
 from .exceptions import UserExists, UserNotFound
 from .events import Events
 
 import secrets
 
+MANAGER=os.environ['CUSTOM_ROLE_MANAGER']
+TEACHER=os.environ['CUSTOM_ROLE_TEACHER']
+STUDENT=os.environ['CUSTOM_ROLE_STUDENT']
+
 class Admin():
     def __init__(self):
+        self.check_connections()
+
+        self.set_custom_roles()
+        self.overwrite_admins()
+
+        self.default_setup()
+        self.internal={}
+
+        ready=False
+        while not ready:
+            try:
+                self.resync_data()
+                ready=True
+            except:
+                print(traceback.format_exc())
+                log.error('Could not resync data, waiting for system to be online...')
+                sleep(2)
+
+        self.sync_groups_from_keycloak()
+        self.external={'users':[],
+                        'groups':[],
+                        'roles':[]}
+
+        log.warning(' Updating missing user avatars with defaults')
+        self.av=Avatars()
+        # av.minio_delete_all_objects() # This will reset all avatars on usres
+        self.av.update_missing_avatars(self.internal['users'])
+
+        log.warning(' SYSTEM READY TO HANDLE CONNECTIONS')
+
+    def check_connections(self):
         ready=False
         while not ready:
             try:
@@ -66,30 +101,11 @@ class Admin():
                 sleep(2)
         log.warning('Nextcloud connected.')
 
-        self.default_setup()
+    def set_custom_roles(self):
-        self.internal={}
+        pass
-
-        ready=False
-        while not ready:
-            try:
-                self.resync_data()
-                ready=True
-            except Exception as e:
-                print(e)
-                log.error('Could not connect to moodle, waiting to be online...')
-                sleep(2)
-
-        self.external={'users':[],
-                        'groups':[],
-                        'roles':[]}
-        log.warning(' Updating missing user avatars with defaults')
-        self.av=Avatars()
-        # av.minio_delete_all_objects() # This will reset all avatars on usres
-        self.av.update_missing_avatars(self.internal['users'])
-        log.warning(' SYSTEM READY TO HANDLE CONNECTIONS')
 
     ## This function should be moved to postup.py
-    def default_setup(self):
+    def overwrite_admins(self):
         log.warning('Setting defaults...')
         dduser=os.environ['DDADMIN_USER']
         ddpassword=os.environ['DDADMIN_PASSWORD']
@@ -111,118 +127,22 @@ class Admin():
             print(traceback.format_exc())
             log.warning('KEYCLOAK: Seems to be there already')
 
-        #### Add default groups
+        ### ddadmin user
         try:
-            log.warning('KEYCLOAK: Adding default groups')
+            log.warning('KEYCLOAK: Adding user ddadmin and adding to group and role admin')
-            self.keycloak.add_group('manager')
+            uid=self.keycloak.add_user(dduser,'DD','Admin',ddmail,ddpassword,group='admin',temporary=False)
-            self.keycloak.add_group('teacher')
+            self.keycloak.assign_realm_roles(uid,'admin')
-            self.keycloak.add_group('student')
             log.warning('KEYCLOAK: OK')
         except:
             log.warning('KEYCLOAK: Seems to be there already')
 
         try:
             log.warning('MOODLE: Adding default group admin')
-            self.moodle.add_system_cohort('/admin','system admins')
+            self.moodle.add_system_cohort('admin','system admins')
             log.warning('MOODLE: OK')
         except:
             log.warning('MOODLE: Seems to be there already')
 
-        try:
-            log.warning('MOODLE: Adding default group manager')
-            self.moodle.add_system_cohort('/manager','system managers')
-            log.warning('MOODLE: OK')
-        except:
-            log.warning('MOODLE: Seems to be there already')
-
-        try:
-            log.warning('MOODLE: Adding default group teacher')
-            self.moodle.add_system_cohort('/teacher','system teacher')
-            log.warning('MOODLE: OK')
-        except:
-            log.warning('MOODLE: Seems to be there already')
-
-        try:
-            log.warning('MOODLE: Adding default group student')
-            self.moodle.add_system_cohort('/student','system student')
-            log.warning('MOODLE: OK')
-        except:
-            log.warning('MOODLE: Seems to be there already')
-
-        try:
-            log.warning('NEXTCLOUD: Adding default group admin')
-            self.nextcloud.add_group('/admin')
-            log.warning('NEXTCLOUD: OK')
-        except ProviderItemExists:
-            log.warning('NEXTCLOUD: Seems to be there already')
-
-        try:
-            log.warning('NEXTCLOUD: Adding default group manager')
-            self.nextcloud.add_group('/manager')
-            log.warning('NEXTCLOUD: OK')
-        except ProviderItemExists:
-            log.warning('NEXTCLOUD: Seems to be there already')
-
-        try:
-            log.warning('NEXTCLOUD: Adding default group teacher')
-            self.nextcloud.add_group('/teacher')
-            log.warning('NEXTCLOUD: OK')
-        except ProviderItemExists:
-            log.warning('NEXTCLOUD: Seems to be there already')
-
-        try:
-            log.warning('NEXTCLOUD: Adding default group student')
-            self.nextcloud.add_group('/student')
-            log.warning('NEXTCLOUD: OK')
-        except ProviderItemExists:
-            log.warning('NEXTCLOUD: Seems to be there already')
-
-
-        ### Add default roles
-        try:
-            log.warning('KEYCLOAK: Adding default roles')
-            self.keycloak.add_role('manager','Realm managers')
-            self.keycloak.add_role('teacher','Realm teachers')
-            self.keycloak.add_role('student','Realm students')
-            log.warning('KEYCLOAK: OK')
-        except:
-            log.warning('KEYCLOAK: Seems to be there already')
-
-
-        ### ddadmin user
-        try:
-            log.warning('KEYCLOAK: Adding user ddadmin and adding to group and role admin')
-            ## Assign group admin to this dduser for nextcloud
-            uid=self.keycloak.add_user(dduser,'DD','Admin',ddmail,ddpassword,group='admin',temporary=False)
-            ## Assign role admin to this user for keycloak, moodle and wordpress
-            self.keycloak.assign_realm_roles(uid,'admin')
-            log.warning('KEYCLOAK: OK')
-        except:
-            log.warning('KEYCLOAK: Seems to be there already')
-
-
-        try:
-            log.warning('KEYCLOAK: Adding default users system_teacher, system_manager and system_student users')
-            uid=self.keycloak.add_user('system_manager','Manager','System','fakemanager@fake.com','m@n@g3r',group='manager',temporary=False)
-            self.keycloak.assign_realm_roles(uid,'manager')
-            uid=self.keycloak.add_user('system_teacher','Teacher','System','faketeacher@fake.com','t3@ch3r',group='teacher',temporary=False)
-            self.keycloak.assign_realm_roles(uid,'teacher')
-            uid=self.keycloak.add_user('system_student','Student','System','fakestudent@fake.com','stud3nt',group='student',temporary=False)
-            self.keycloak.assign_realm_roles(uid,'student')
-            log.warning('KEYCLOAK: OK')
-        except:
-            log.warning('KEYCLOAK: Seems to be there already')
-
-        try:
-            log.warning('NEXTCLOUD: Adding user ddadmin and adding to group admin')
-            self.nextcloud.add_user(dduser,ddpassword,group='/admin',email=ddmail,displayname='DD Admin')
-            log.warning('NEXTCLOUD: OK')
-        except ProviderItemExists:
-            log.warning('NEXTCLOUD: Seems to be there already')
-        except:
-            log.error(traceback.format_exc())
-            exit(1)
-
         try:
             log.warning('MOODLE: Adding user ddadmin and adding to siteadmins')
             self.moodle.create_user(ddmail,dduser,ddpassword,'DD','Admin')
@@ -232,6 +152,103 @@ class Admin():
         except:
             log.warning('MOODLE: Seems to be there already')
 
+        try:
+            log.warning('NEXTCLOUD: Adding default group admin')
+            self.nextcloud.add_group('admin')
+            log.warning('NEXTCLOUD: OK')
+        except ProviderItemExists:
+            log.warning('NEXTCLOUD: Seems to be there already')
+
+        try:
+            log.warning('NEXTCLOUD: Adding user ddadmin and adding to group admin')
+            self.nextcloud.add_user(dduser,ddpassword,group='admin',email=ddmail,displayname='DD Admin')
+            log.warning('NEXTCLOUD: OK')
+        except ProviderItemExists:
+            log.warning('NEXTCLOUD: Seems to be there already')
+        except:
+            log.error(traceback.format_exc())
+            exit(1)
+
+    def default_setup(self):
+        ### Add default roles
+        try:
+            log.warning('KEYCLOAK: Adding default roles')
+            self.keycloak.add_role(MANAGER,'Realm managers')
+            self.keycloak.add_role(TEACHER,'Realm teachers')
+            self.keycloak.add_role(STUDENT,'Realm students')
+            log.warning('KEYCLOAK: OK')
+        except:
+            log.warning('KEYCLOAK: Seems to be there already')
+
+        #### Add default groups
+        try:
+            log.warning('KEYCLOAK: Adding default groups')
+            self.keycloak.add_group(MANAGER)
+            self.keycloak.add_group(TEACHER)
+            self.keycloak.add_group(STUDENT)
+            log.warning('KEYCLOAK: OK')
+        except:
+            log.warning('KEYCLOAK: Seems to be there already')
+
+        try:
+            log.warning('MOODLE: Adding default group manager')
+            self.moodle.add_system_cohort(MANAGER,'system managers')
+            log.warning('MOODLE: OK')
+        except:
+            log.warning('MOODLE: Seems to be there already')
+
+        try:
+            log.warning('MOODLE: Adding default group teacher')
+            self.moodle.add_system_cohort(TEACHER,'system teachers')
+            log.warning('MOODLE: OK')
+        except:
+            log.warning('MOODLE: Seems to be there already')
+
+        try:
+            log.warning('MOODLE: Adding default group student')
+            self.moodle.add_system_cohort(STUDENT,'system students')
+            log.warning('MOODLE: OK')
+        except:
+            log.warning('MOODLE: Seems to be there already')
+
+
+
+        try:
+            log.warning('NEXTCLOUD: Adding default group manager')
+            self.nextcloud.add_group(MANAGER)
+            log.warning('NEXTCLOUD: OK')
+        except ProviderItemExists:
+            log.warning('NEXTCLOUD: Seems to be there already')
+
+        try:
+            log.warning('NEXTCLOUD: Adding default group teacher')
+            self.nextcloud.add_group(TEACHER)
+            log.warning('NEXTCLOUD: OK')
+        except ProviderItemExists:
+            log.warning('NEXTCLOUD: Seems to be there already')
+
+        try:
+            log.warning('NEXTCLOUD: Adding default group student')
+            self.nextcloud.add_group(STUDENT)
+            log.warning('NEXTCLOUD: OK')
+        except ProviderItemExists:
+            log.warning('NEXTCLOUD: Seems to be there already')
+
+
+        try:
+            log.warning('KEYCLOAK: Adding default users system_teacher, system_manager and system_student users')
+            uid=self.keycloak.add_user('system_manager','Manager','System','fakemanager@fake.com','m@n@g3r',group=MANAGER,temporary=False)
+            self.keycloak.assign_realm_roles(uid,MANAGER)
+            uid=self.keycloak.add_user('system_teacher','Teacher','System','faketeacher@fake.com','t3@ch3r',group=TEACHER,temporary=False)
+            self.keycloak.assign_realm_roles(uid,TEACHER)
+            uid=self.keycloak.add_user('system_student','Student','System','fakestudent@fake.com','stud3nt',group=STUDENT,temporary=False)
+            self.keycloak.assign_realm_roles(uid,STUDENT)
+            log.warning('KEYCLOAK: OK')
+        except:
+            log.warning('KEYCLOAK: Seems to be there already')
+
+
+
     def resync_data(self):
         self.internal={'users':self._get_mix_users(),
                         'groups':self._get_mix_groups(),
@@ -239,7 +256,7 @@ class Admin():
         return True
 
     def get_moodle_users(self):
-        return [u for u in self.moodle.get_users_with_groups_and_roles() if u['username'] not in ['guest','ddadmin','admin'] and not u['username'].startswith('system_')]
+        return [u for u in self.moodle.get_users_with_groups_and_roles() if not system_username(u['username'])]
 
     ## TOO SLOW. Not used.
     # def get_moodle_users(self):
@@ -257,6 +274,7 @@ class Admin():
 
     def get_keycloak_users(self):
         # log.warning('Loading keycloak users... can take a long time...')
+
         users = self.keycloak.get_users_with_groups_and_roles()
         return [{"id":u['id'],
                 "username":u['username'],
@@ -266,7 +284,7 @@ class Admin():
                 "email": u.get('email',''),
                 "groups": u['group'],
                 "roles": filter_roles_list(u['role'])} 
-                for u in users if u['username'] not in ['guest','ddadmin','admin'] and not u['username'].startswith('system')]
+                for u in users if not system_username(u['username'])]
 
     def get_nextcloud_users(self):
         return [{"id":u['username'],
@@ -281,6 +299,7 @@ class Admin():
                 # The theoretical bytes returned do not map to any known unit. The division is just an approach while we investigate.
                 for u in self.nextcloud.get_users_list() if u['username'] not in ['guest','ddadmin','admin'] and not u['username'].startswith('system')]
 
+
     ## TOO SLOW
     # def get_nextcloud_users(self):
     #     log.warning('Loading nextcloud users... can take a long time...')
@@ -301,6 +320,8 @@ class Admin():
         return self.internal['users']
 
     def _get_mix_users(self):
+        kgroups=self.keycloak.get_groups()
+
         kusers=self.get_keycloak_users()
         musers=self.get_moodle_users()
         nusers=self.get_nextcloud_users()
@@ -315,10 +336,12 @@ class Admin():
         for username in all_users_usernames:
             theuser={}
             keycloak_exists=[u for u in kusers if u['username'] == username]
+
             if len(keycloak_exists):
                 theuser=keycloak_exists[0]
                 theuser['keycloak']=True
-                theuser['keycloak_groups']=self.keycloak.get_user_groups_paths(keycloak_exists[0]['id']) #keycloak_exists[0]['groups']
+                theuser['keycloak_groups']=get_gids_from_kgroup_ids(theuser.pop('groups'),kgroups)
+                #self.keycloak.get_user_groups_paths(keycloak_exists[0]['id']) #keycloak_exists[0]['groups']
             else:
                 theuser['id']=False
                 theuser['keycloak']=False
@@ -340,17 +363,16 @@ class Admin():
                 theuser['nextcloud']=True
                 theuser['nextcloud_groups']=nextcloud_exists[0]['groups']
                 theuser['nextcloud_id']=nextcloud_exists[0]['id']
-                theuser['quota']=theuser['quota'] if theuser['quota'] != None and theuser['quota'] != 'none' else False
+                theuser['quota']=theuser['quota'] if theuser.get('quota',False) and theuser['quota'] != None and theuser['quota'] != 'none' else False
             else:
                 theuser['nextcloud']=False
                 theuser['nextcloud_groups']=[]
                 theuser['quota']=False
                 theuser['quota_used_bytes']=False
-            del theuser['groups']
 
-            if not len(theuser['roles']): 
+            # if not len(theuser['roles']): 
-                log.error('                     SKIPPING USER WITHOUT ANY ROLE!!: '+theuser['username']+' . Should be fixed at keycloak level.')
+            #     log.error('                     SKIPPING USER WITHOUT ANY ROLE!!: '+theuser['username']+' . Should be fixed at keycloak level.')
-                continue
+            #     continue
 
             users.append(theuser)
         return users
@@ -377,7 +399,7 @@ class Admin():
         return self.internal['groups']
 
     def _get_mix_groups(self):
-        kgroups=self.get_keycloak_groups() 
+        kgroups=self.get_keycloak_groups()
         mgroups=self.get_moodle_groups()
         ngroups=self.get_nextcloud_groups()
 
@@ -385,40 +407,47 @@ class Admin():
         mgroups=[] if mgroups is None else mgroups
         ngroups=[] if ngroups is None else ngroups
 
-        kgroups_names=[g['path'] for g in kgroups]
+        kgroups_names=[kpath2gid(g['path']) for g in kgroups]
         mgroups_names=[g['name'] for g in mgroups]
         ngroups_names=ngroups
 
         all_groups_names=set(kgroups_names+mgroups_names+ngroups_names)
-
         groups=[]
         for name in all_groups_names:
             thegroup={}
-            keycloak_exists=[g for g in kgroups if g['path'] == name]
+            keycloak_exists=[g for g in kgroups if kpath2gid(g['path']) == name]
             if len(keycloak_exists):
+                # del keycloak_exists[0]['subGroups']
                 thegroup=keycloak_exists[0]
                 thegroup['keycloak']=True
-                thegroup['path']=self.keycloak.get_group_path(keycloak_exists[0]['id'])
+                thegroup['name']=kpath2gid(thegroup['path'])
-                del thegroup['subGroups']
+                # thegroup['path']=self.keycloak.get_group_path(keycloak_exists[0]['id'])
+                
             else:
                 thegroup['id']=False
                 thegroup['keycloak']=False
-
+                thegroup['name']=False
+                thegroup['path']=False
             moodle_exists=[g for g in mgroups if g['name'] == name]
             if len(moodle_exists):
-                thegroup['path']=moodle_exists[0]['name']
+                # del moodle_exists[0]['idnumber']
-                thegroup={**moodle_exists[0], **thegroup}
+                # del moodle_exists[0]['descriptionformat']
+                # del moodle_exists[0]['theme']
+                # del moodle_exists[0]['visible']
+                # # thegroup['path']=moodle_exists[0]['name']
+                # thegroup={**moodle_exists[0], **thegroup}
                 thegroup['moodle']=True
                 thegroup['moodle_id']=moodle_exists[0]['id']
+                thegroup['description']=moodle_exists[0]['description']
             else:
                 thegroup['moodle']=False
 
             nextcloud_exists=[g for g in ngroups if g == name]
             if len(nextcloud_exists):
-                nextcloud={"id":nextcloud_exists[0],
+                # nextcloud={"id":nextcloud_exists[0],
-                            "name":nextcloud_exists[0],
+                #             "name":nextcloud_exists[0],
-                            "path":nextcloud_exists[0]}
+                #             "path":nextcloud_exists[0]}
-                thegroup={**nextcloud, **thegroup}
+                # thegroup={**nextcloud, **thegroup}
                 thegroup['nextcloud']=True
                 thegroup['nextcloud_id']=nextcloud_exists[0]  ### is the path
             else:
@@ -427,6 +456,29 @@ class Admin():
             groups.append(thegroup)
         return groups
 
+    def sync_groups_from_keycloak(self):
+        self.resync_data()
+        for group in self.internal['groups']:
+            if not group['keycloak']:
+                if group['moodle']:
+                    try:
+                        self.moodle.delete_cohorts([group['moodle_id']])
+                    except:
+                        print(traceback.format_exc())
+                        # log.error('MOODLE: User '+user['username']+' it is not in cohort '+group)
+                if group['nextcloud']:
+                    self.nextcloud.delete_group(group['nextcloud_id'])
+
+        self.resync_data()
+        for group in self.internal['groups']:
+            if group['keycloak']:
+                if not group['moodle']:
+                    self.moodle.add_system_cohort(group['name'],description='')
+                if not group['nextcloud']:
+                    self.nextcloud.add_group(group['name'])
+        self.resync_data()
+
+  
     def get_external_users(self):
         return self.external['users']
 
@@ -626,7 +678,7 @@ class Admin():
                 if u['first'] == '': u['first']=' '
                 if u['last'] == '': u['last']=' '
                 try:
-                    pprint(self.moodle.create_user(u['email'],u['username'],secrets.token_urlsafe(16),u['first'],u['last'])[0])
+                    self.moodle.create_user(u['email'],u['username'],secrets.token_urlsafe(16),u['first'],u['last'])[0]
                 except:
                     log.error('  -->> Error creating on moodle the user: '+u['username'])
                 # user_id=user['id']
@@ -817,6 +869,24 @@ class Admin():
     def user_update_password(self,userid,password,temporary):
         return self.keycloak.update_user_pwd(userid,password,temporary)
 
+    def update_users_from_keycloak(self):
+        kgroups=self.keycloak.get_groups()
+        users=[{'id':u['id'],
+                'username':u['username'],
+                'enabled':u['enabled'],
+                'email':u['email'],
+                'firstname':u['first'],
+                'lastname':u['last'],
+                'groups':get_gids_from_kgroup_ids(u['groups'],kgroups),
+                'roles':u['roles'],
+                'quota':'3 GB' if len(u['roles']) and u['roles'][0] != 'student' else '500 MB'} 
+                for u in self.get_keycloak_users()]
+
+        for user in users:
+            ev=Events('Updating users from keycloak','User:',total=len(users),table='users')
+            self.user_update(user)
+            ev.increment({'name':user['username'],'data':user['groups']})
+
     def user_update(self,user):
         log.warning('Updating user moodle, nextcloud keycloak')
         ev=Events('Updating user','Updating user in keycloak')
@@ -828,116 +898,216 @@ class Admin():
         except:
             raise UserNotFound 
 
-        if set(user['groups']) != set(internaluser['keycloak_groups']):
+        if not len(user['roles']): user['roles']=['student']
-            add_user_groups = list(set(user['groups']) - set(internaluser['keycloak_groups']))
+        delete_roles=list(set(['admin','manager','teacher','student']) - set(user['roles']))
-            user_groups_with_system=user['groups']+['/admin','/manager','/teacher','/student']
+        
-            remove_user_groups = list(set(internaluser['keycloak_groups']) - set(user_groups_with_system))
+        ### keycloak groups
-        else:
-            add_user_groups=[]
-            remove_user_groups=[]
 
-        print('PREVIOUS ADD USER GROUPS')
-        print(add_user_groups)
-        print('PREVIOUS REMOVE USER GROUPS')
-        print(remove_user_groups)
 
-        if user['roles'][0] not in internaluser['roles']:
+        kadd=list(set(user['groups']) - set(internaluser['keycloak_groups']+delete_roles))
-            # Remove internaluser['roles']
+        kdelete=list(set(internaluser['keycloak_groups']) - set(user['groups']) - set(user['roles']))
-            add_user_roles=user['roles']
+        if user['roles'][0] not in kadd: kadd.append(user['roles'][0])
-            if '/'+user['roles'][0] not in add_user_groups:
-                add_user_groups.append('/'+user['roles'][0])
-            remove_user_roles=internaluser['roles'] #Remove the previous role
-            if '/'+user['roles'][0] not in remove_user_groups:
-                remove_user_groups.append('/'+internaluser['roles'][0])
-        else:
-            add_user_roles=[]
-            remove_user_roles=[]
-            # Add user['role']
 
-        print('ADD USER GROUPS')
+        ### Moodle groups
-        print(add_user_groups)
+        madd=list(set(user['groups']) - set(internaluser['moodle_groups']+delete_roles))
-        print('REMOVE USER GROUPS')
+        mdelete=list(set(internaluser['moodle_groups']) - set(user['groups']) - set(user['roles']))
-        print(remove_user_groups)
+        if user['roles'][0] not in madd: madd.append(user['roles'][0])
-        print('ADD USER ROLES')
-        print(add_user_roles)
-        print('REMOVE USER ROLES')
-        print(remove_user_roles)
 
-        self.update_keycloak_user(internaluser['id'],user,remove_user_roles,remove_user_groups)
+        ### nextcloud groups
+        nadd=list(set(user['groups']) - set(internaluser['nextcloud_groups']+delete_roles))
+        ndelete=list(set(internaluser['nextcloud_groups']) - set(user['groups']) - set(user['roles']))
+        if user['roles'][0] not in nadd: nadd.append(user['roles'][0])
+
+        #### Delete recursive to parent from this subgroup
+        #### DISABLED DELETE BUT KEPT ADD, as we should then check if the user is still in another subgroups of this group. How?
+        # pathslist=[]
+        # for group in kdelete:
+        #     pathpart=''
+        #     for part in group.split('.'):
+        #         if pathpart=='':
+        #             pathpart=part
+        #         else:
+        #             pathpart=pathpart+'.'+part
+        #         if repeated_subpaths[pathpart] > 1: 
+        #             ## We do not delete it as the user is in multiple subpaths of this path.
+        #             continue
+        #         pathslist.append(pathpart)
+        # kdelete=pathslist
+
+        pathslist=[]
+        for group in kadd:
+            pathpart=''
+            for part in group.split('.'):
+                if pathpart=='':
+                    pathpart=part
+                else:
+                    pathpart=pathpart+'.'+part
+                pathslist.append(pathpart)
+        kadd=pathslist
+
+        # print('KADD WITH SUBPATHS')
+        # print(kadd)
+        # print(count_repeated(kadd))
+        # print('KDELETE')
+        # print(kdelete)
+        # print(count_repeated(kdelete))
+        
+        # pathslist=[]
+        # for group in mdelete:
+        #     pathpart=''
+        #     for part in group.split('.'):
+        #         if pathpart=='':
+        #             pathpart=part
+        #         else:
+        #             pathpart=pathpart+'.'+part
+        #         pathslist.append(pathpart)
+        # mdelete=pathslist
+
+        pathslist=[]
+        for group in madd:
+            pathpart=''
+            for part in group.split('.'):
+                if pathpart=='':
+                    pathpart=part
+                else:
+                    pathpart=pathpart+'.'+part
+                pathslist.append(pathpart)
+        madd=pathslist
+
+        # print('MADD WITH SUBPATHS')
+        # print(madd)
+        # print(count_repeated(madd))
+        # print('MDELETE WITH SUBPATHS')
+        # print(mdelete)
+        # print(count_repeated(mdelete))
+
+        # pathslist=[]
+        # for group in ndelete:
+        #     pathpart=''
+        #     for part in group.split('.'):
+        #         if pathpart=='':
+        #             pathpart=part
+        #         else:
+        #             pathpart=pathpart+'.'+part
+        #         pathslist.append(pathpart)
+        # ndelete=pathslist
+
+        pathslist=[]
+        for group in nadd:
+            pathpart=''
+            for part in group.split('.'):
+                if pathpart=='':
+                    pathpart=part
+                else:
+                    pathpart=pathpart+'.'+part
+                pathslist.append(pathpart)
+        nadd=pathslist
+
+        # print('NADD WITH SUBPATHS')
+        # print(nadd)
+        # print(count_repeated(nadd))
+        # print('NDELETE WITH SUBPATHS')
+        # print(ndelete)
+        # print(count_repeated(ndelete))
+
+        self.update_keycloak_user(internaluser['id'],user,kdelete,kadd)
         ev.update_text('Syncing data from applications...')
         self.resync_data()
 
         ev.update_text('Updating user in moodle')
-        self.update_moodle_user(internaluser['id'],user,remove_user_groups)
+        self.update_moodle_user(internaluser['id'],user,mdelete,madd)
 
         ev.update_text('Updating user in nextcloud')
-        self.update_nextcloud_user(internaluser['id'],user,remove_user_groups)
+        self.update_nextcloud_user(internaluser['id'],user,ndelete,nadd)
  
         ev.update_text('User updated')
         return True
 
-    def update_keycloak_user(self,user_id,user,remove_user_roles,remove_user_groups):
+    def update_keycloak_user(self,user_id,user,kdelete,kadd):
-        if len(remove_user_roles):
-            toremove=[]
-            # pprint(self.keycloak.get_user_realm_roles(user_id))
-            for role in remove_user_roles:
-                toremove.append(self.keycloak.get_role(role))
-
-                # Also remove from group identical to role:
-                group_id = self.keycloak.get_group_by_path('/'+role)['id']
-                self.keycloak.group_user_remove(user_id,group_id)
-
-            self.keycloak.remove_user_roles(user_id,toremove)
-
         self.keycloak.assign_realm_roles(user_id,user['roles'][0])
-        # Also add it to the group identical to role
+        for group in kdelete:
-        group_id = self.keycloak.get_group_by_path('/'+user['roles'][0])['id']
+            group_id = self.keycloak.get_group_by_path(gid2kpath(group))['id']
-        self.keycloak.group_user_add(user_id,group_id)
+            self.keycloak.group_user_remove(user_id,group_id)
-
+        for group in kadd:
-        if len(remove_user_groups):
+            group_id = self.keycloak.get_group_by_path(gid2kpath(group))['id']
-            for group in remove_user_groups:
-                group_id = self.keycloak.get_group_by_path(group)['id']
-                self.keycloak.group_user_remove(user_id,group_id)
-
-        for group in user['groups']:
-            group_id = self.keycloak.get_group_by_path(group)['id']
             self.keycloak.group_user_add(user_id,group_id)
-
-
-
         self.keycloak.user_update(user_id,user['enabled'],user['email'],user['firstname'],user['lastname'])
-        # So we should add it to the correct role
+
-        
         return True
 
-    def update_moodle_user(self,user_id,user,remove_user_groups):
+    def update_moodle_user(self,user_id,user,mdelete,madd):
         internaluser=[u for u in self.internal['users'] if u['id']==user_id][0]
         cohorts=self.moodle.get_cohorts()
-        if len(remove_user_groups):
+        for group in mdelete:
-            for group in remove_user_groups:
+            cohort=[c for c in cohorts if c['name']==group][0]
-                cohort=[c for c in cohorts if c['name']==group][0]
+            try:
-                try:
+                self.moodle.delete_user_in_cohort(internaluser['moodle_id'],cohort['id'])
-                    self.moodle.delete_user_in_cohort(user_id,cohort['id'])
+            except:
-                except:
+                log.error('MOODLE: User '+user['username']+' it is not in cohort '+group)
-                    log.error('MOODLE: User '+user['username']+' it is not in cohort '+group)
+
-        
+        if not internaluser['moodle']:
-        for group in user['groups']:
+            self.add_moodle_user(username=user['username'], email=user['email'], first_name=user['firstname'], last_name=user['lastname'])
+            self.resync_data()
+        else:
+            self.moodle.update_user(username=user['username'], email=user['email'], first_name=user['firstname'], last_name=user['lastname'], enabled=user['enabled'])
+
+        for group in madd:
             cohort=[c for c in cohorts if c['name']==group][0]
             self.moodle.add_user_to_cohort(internaluser['moodle_id'],cohort['id'])
-        self.moodle.update_user(username=user['username'], email=user['email'], first_name=user['firstname'], last_name=user['lastname'], enabled=user['enabled'])
+
         return True
 
-    def update_nextcloud_user(self,user_id, user, remove_user_groups):
+    def add_moodle_user(self, username,email,first_name,last_name,password='*'+secrets.token_urlsafe(16)):
-        self.nextcloud.update_user(user['username'],{"quota":user['quota'],"email":user['email'],"displayname":user['firstname']+' '+user['lastname']})
+        log.warning('Creating moodle user: '+username)
+        ev=Events('Add user',username)
+        try:
+            self.moodle.create_user(email,username,password,first_name,last_name)
+            ev.update_text({'name':'Added to moodle','data':[]})
+        except UserExists:
+            log.error('  -->> User already exists')
+            error=Events('User already exists.',str(se),type='error')
+        except SystemError as se:
+            log.error('Moodle create user error: '+str(se))
+            error=Events('Moodle create user error',str(se),type='error')
+        except:
+            log.error('  -->> Error creating on moodle the user: '+username)
+            print(traceback.format_exc())
+            error=Events('Internal error','Check logs',type='error')
+
+    def update_nextcloud_user(self,user_id, user, ndelete,nadd):
+        
         ## TODO: Disable de user? Is really needed? it is disabled in keycloak, so can't login again
         ## ocs/v1.php/cloud/users/{userid}/disable
-        if len(remove_user_groups):
+        internaluser=[u for u in self.internal['users'] if u['id']==user_id][0]
-            for group in remove_user_groups:
+        if not internaluser['nextcloud']:
-                self.nextcloud.remove_user_from_group(user['username'],group)
+            try:
+                self.add_nextcloud_user(user['username'],user['email'],user['quota'],user['firstname'],user['lastname'],user['groups'])
+            except:
+                log.warning('NEXTCLOUD: Ooops! User '+user['username']+' seems to be in NC but our db info says not....')
+            self.resync_data()
+        else:
+            if not user['quota'] or user['quota'] == 'false':
+                self.nextcloud.update_user(user['username'],{"quota":"none","email":user['email'],"displayname":user['firstname']+' '+user['lastname']})
+            else:
+                self.nextcloud.update_user(user['username'],{"quota":user['quota'],"email":user['email'],"displayname":user['firstname']+' '+user['lastname']})
 
-        for group in user['groups']:
+        for group in ndelete:
+            self.nextcloud.remove_user_from_group(user['username'],group)
+
+        for group in nadd:
             self.nextcloud.add_user_to_group(user['username'],group)
 
+    def add_nextcloud_user(self,username,email,quota,first_name,last_name,groups,password=secrets.token_urlsafe(16)):
+        log.warning('  NEXTCLOUD USERS: Creating nextcloud user: '+username+' in groups '+str(groups))
+        ev=Events('Add user',username)
+        try:
+            # Quota is "1 GB", "500 MB"
+            self.nextcloud.add_user_with_groups(username,password,quota,groups,email,first_name+' '+last_name)
+            ev.increment({'name':'Added to nextcloud','data':[]})
+        except ProviderItemExists:
+            log.warning('User '+username+' already exists. Skipping...')
+        except:
+            log.error(traceback.format_exc())
 
     def delete_user(self,userid):
         log.warning('Deleting user moodle, nextcloud keycloak')
@@ -963,11 +1133,26 @@ class Admin():
         return user[0]
 
     def add_user(self,u):
+
+        pathslist=[]
+        for group in u['groups']:
+            pathpart=''
+            for part in group.split('.'):
+                if pathpart=='':
+                    pathpart=part
+                else:
+                    pathpart=pathpart+'.'+part
+                pathslist.append(pathpart)
+        print(pathslist)
+
         ### KEYCLOAK
+        #######################
         ev=Events('Add user',u['username'],total=5)
         log.warning(' KEYCLOAK USERS: Adding user: '+u['username'])
         uid=self.keycloak.add_user(u['username'],u['first'],u['last'],u['email'],u['password'],enabled=u['enabled'])
+
         self.av.add_user_default_avatar(uid,u['role'])
+
         # Add user to role and group rolename
         log.warning(' KEYCLOAK USERS: Assign user '+u['username']+' with initial pwd '+ u['password']+' to role '+u['role'])
         self.keycloak.assign_realm_roles(uid,u['role'])
@@ -975,24 +1160,20 @@ class Admin():
         self.keycloak.group_user_add(uid,gid)
         ev.increment({'name':'Added to system','data':[]})
         # Add user to groups
-        for g in u['groups']:
+        for path in pathslist:
-            parts=g.split('/')
+            path='/'+path.replace('.','/')
-            sub=''
+            log.warning(' KEYCLOAK USERS: Assign user '+u['username']+' to group '+ path)
-            if len(parts)==0: 
+            gid=self.keycloak.get_group_by_path(path=path)['id']
-                log.warning(' KEYCLOAK USERS: Skip assign user '+u['username']+' to any group as does not have one')
+            self.keycloak.group_user_add(uid,gid)
-                continue # NO GROUP
-            for i in range(1,len(parts)):
-                sub=sub+'/'+parts[i]
-                if sub=='/': continue # User with no path
-                log.warning(' KEYCLOAK USERS: Assign user '+u['username']+' to group '+ str(sub))
-                gid=self.keycloak.get_group_by_path(path=sub)['id']
-                self.keycloak.group_user_add(uid,gid)
         ev.increment({'name':'Added to system groups','data':[]})
+
+        pathslist.append(u['role'])
         ### MOODLE
+        ###############################
         # Add user
         log.warning('Creating moodle user: '+u['username'])
         try:
-            self.moodle.create_user(u['email'],u['username'],'*'+secrets.token_urlsafe(16),u['first'],u['last'])
+            moodle_id=self.moodle.create_user(u['email'],u['username'],'*12'+secrets.token_urlsafe(16),u['first'],u['last'])[0]['id']
             ev.increment({'name':'Added to moodle','data':[]})
         except UserExists:
             log.error('  -->> User already exists')
@@ -1008,24 +1189,27 @@ class Admin():
         # Add user to cohort
         ## Get all existing moodle cohorts
         cohorts=self.moodle.get_cohorts()
-        for g in u['groups']:
+        for path in pathslist:
+            print('MOODLE ADD SUBPATH: '+path)
             try:
-                cohort=[c for c in cohorts if c['name']==g][0]
+                cohort=[c for c in cohorts if c['name']==path][0]
             except:
                 # print(traceback.format_exc())
-                log.error(' MOODLE USER GROUPS: keycloak group '+g+' does not exist as moodle cohort. This should not happen. User '+u['username']+ ' not added.')
+                log.error(' MOODLE USER GROUPS: keycloak group '+path+' does not exist as moodle cohort. This should not happen. User '+u['username']+ ' not added.')
 
             try:
-                self.moodle.add_user_to_cohort(u['username'],cohort['id'])
+                self.moodle.add_user_to_cohort(moodle_id,cohort['id'])
             except:
                 log.error(' MOODLE USER GROUPS: User '+u['username']+' already exists in cohort '+cohort['name'])
 
         ev.increment({'name':'Added to moodle cohorts','data':[]})
+
         ### NEXTCLOUD
-        log.warning('  NEXTCLOUD USERS: Creating nextcloud user: '+u['username']+' in groups '+str(u['groups']))
+        ########################3
+        log.warning('  NEXTCLOUD USERS: Creating nextcloud user: '+u['username']+' in groups '+str(list))
         try:
             # Quota is in MB
-            self.nextcloud.add_user_with_groups(u['username'],secrets.token_urlsafe(16),u['quota'],u['groups'],u['email'],u['first']+' '+u['last'])
+            self.nextcloud.add_user_with_groups(u['username'],secrets.token_urlsafe(16),u['quota'],pathslist,u['email'],u['first']+' '+u['last'])
             ev.increment({'name':'Added to nextcloud','data':[]})
         except ProviderItemExists:
             log.warning('User '+u['username']+' already exists. Skipping...')
@@ -1037,41 +1221,58 @@ class Admin():
     def add_group(self,g):
         # TODO: Check if exists
 
+        # We add in keycloak with his name, will be shown in app with full path with dots
+        if g['parent'] != None: g['parent'] = gid2kpath(g['parent'])
+
         new_path=self.keycloak.add_group(g['name'],g['parent'])
+
         if g['parent'] != None:
-            new_path=new_path['path']
+            new_path=kpath2gid(new_path['path'])
         else:
-            new_path='/'+g['name']
+            new_path=g['name']
-        self.moodle.add_system_cohort(new_path,g['description'])
+
+        self.moodle.add_system_cohort(new_path,description=g['description'])
         self.nextcloud.add_group(new_path)
 
     def delete_group_by_id(self,group_id):
-        # TODO: Check if exists
+        # TODO: Check if exists (None)
         group = self.keycloak.get_group_by_id(group_id)
 
+        to_be_deleted=[]
+        # Childs
+        for internalgroup in self.internal['groups']:
+            if internalgroup['name'].startswith(group['name']+'.'): to_be_deleted.append(internalgroup['name'])
+        to_be_deleted.append(kpath2gid(group['path']))
+
         try:
             self.keycloak.delete_group(group['id'])
         except:
             log.error('KEYCLOAK: Could no delete group '+group['path'])
+
         cohorts=self.moodle.get_cohorts()
+        for gid in to_be_deleted:
+            cohort = [c['id'] for c in cohorts if c['name'] == gid]
+            self.moodle.delete_cohorts(cohort)
+            self.nextcloud.delete_group(gid)
 
-        cohort = [c['id'] for c in cohorts if c['name'] == group['path']]
-        self.moodle.delete_cohorts(cohort)
-
-        self.nextcloud.delete_group(group['path'])
-        # group = [g for g in self.internal['groups'] if g['id'] == group_id][0]
 
     def delete_group_by_path(self,path):
         group =self.keycloak.get_group_by_path(path)
-        if group != None:
+
-            try:
+        to_be_deleted=[]
-                self.keycloak.delete_group(group['id'])
+        # Childs
-            except:
+        for internalgroup in self.internal['groups']:
-                log.error('KEYCLOAK: Could no delete group '+group['path'])
+            if internalgroup['name'].startswith(group['name']+'.'): to_be_deleted.append(internalgroup['name'])
+        to_be_deleted.append(kpath2gid(group['path']))
+
+        try:
+            self.keycloak.delete_group(group['id'])
+        except:
+            log.error('KEYCLOAK: Could no delete group '+group['path'])
+
         cohorts=self.moodle.get_cohorts()
-        cohort = [c['id'] for c in cohorts if c['name'] == path]
+        for gid in to_be_deleted:
-        if len(cohort):
+            cohort = [c['id'] for c in cohorts if c['name'] == gid]
             self.moodle.delete_cohorts(cohort)
-
+            self.nextcloud.delete_group(gid)
-        self.nextcloud.delete_group(path)
 

admin/src/admin/lib/helpers.py

@@ -1,5 +1,43 @@
-import random
+import random, string
-import string
+from pprint import pprint
+from collections import Counter
+
+def system_username(username):
+     return True if username in ['guest','ddadmin','admin'] or username.startswith('system_') else False
+
+def system_group(groupname):
+    return True if groupname in ['admin','manager','teacher','student'] else False
+
+def get_group_from_group_id(group_id,groups):
+    return next((d for d in groups if d.get('id') == group_id), None)
+
+def get_gid_from_kgroup_id(kgroup_id,groups):
+    # print(kgroup_id)
+    # pprint(groups)
+    # return get_group_from_group_id(kgroup_id,groups)['path'].replace('/','.')[1:]
+    return [g['path'].replace('/','.')[1:] for g in groups if g['id'] == kgroup_id][0]
+
+def get_gids_from_kgroup_ids(kgroup_ids,groups):
+    return [get_gid_from_kgroup_id(kgroup_id,groups) for kgroup_id in kgroup_ids]
+
+def kpath2gid(path):
+    # print(path.replace('/','.')[1:])
+    return path.replace('/','.')[1:]
+
+def gid2kpath(gid):
+    return '/'+gid.replace('.','/')
+
+def count_repeated(itemslist):
+    print(Counter(itemslist))
+
+def groups_kname2gid(groups):
+    return [name.replace('.','/') for name in groups]
+
+def groups_path2id(groups):
+    return [g.replace('/','.')[1:] for g in groups]
+
+def groups_id2path(groups):
+    return ['/'+g.replace('.','/') for g in groups]
 
 def filter_roles_list(role_list):
     client_roles=['admin','manager','teacher','student']

admin/src/admin/lib/keycloak_client.py

@@ -72,7 +72,7 @@ class KeycloakClient():
 
     def get_users_with_groups_and_roles(self):
         q = """select u.id, u.username, u.email, u.first_name, u.last_name, u.realm_id, u.enabled, ua.value as quota
-        ,json_agg(g."name") as group, json_agg(g_parent."name") as group_parent1, json_agg(g_parent2."name") as group_parent2
+        ,json_agg(g."id") as group, json_agg(g_parent."name") as group_parent1, json_agg(g_parent2."name") as group_parent2
         ,json_agg(r.name) as role
         from user_entity as u
         left join user_attribute as ua on ua.user_id=u.id and ua.name = 'quota'
@@ -98,6 +98,19 @@ class KeycloakClient():
                                         ([[]] if l[-1] == [None] else [list(set(l[-1]))]) for l in users_with_lists]
 
         list_dict_users = [dict(zip(headers, r)) for r in users_with_lists]
+
+        # self.connect()
+        # groups = self.keycloak_admin.get_groups()
+
+        
+        # for user in list_dict_users:
+        #     new_user_groups = []
+        #     for group_id in user['group']:
+        #         found = [g for g in groups if g['id'] == group_id][0]
+        #         new_user_groups.append({'id':found['id'],
+        #                                 'name':found['name'],
+        #                                 'path':found['path']})
+        #     user['group']=new_user_groups
         return list_dict_users
 
 
@@ -194,7 +207,6 @@ class KeycloakClient():
 
     def remove_user_roles(self,user_id,roles):
         self.connect()
-        print(roles)
         return self.keycloak_admin.delete_realm_roles_of_user(user_id,roles)
 
     def delete_user(self,userid):
@@ -214,9 +226,27 @@ class KeycloakClient():
         return self.keycloak_admin.assign_client_role(client_id=client_id, user_id=user_id, role_id=role_id, role_name="test")
 
     ## GROUPS
+    def get_all_groups(self):
+        ## RETURNS ONLY MAIN GROUPS WITH NESTED subGroups list
+        self.connect()
+        return self.keycloak_admin.get_groups()
+
+    def get_recursive_groups(self, l_groups,l=[]):
+        for d_group in l_groups:
+            d = {}
+            for key, value in d_group.items():
+                if key == 'subGroups':
+                    self.get_recursive_groups(value,l)
+                else:
+                    d[key]=value
+            l.append(d)
+        return l
+
     def get_groups(self,with_subgroups=True):
+        ## RETURNS ALL GROUPS in root list
         self.connect()
         groups = self.keycloak_admin.get_groups()
+        return self.get_recursive_groups(groups)
         subgroups=[]
         subgroups1=[]
         # This needs to be recursive function

admin/src/admin/lib/moodle.py

@@ -75,7 +75,7 @@ class Moodle():
             data = [{'username': username, 'email':email, 
                     'password': password, 'firstname':first_name, 'lastname':last_name}]
             user = self.call('core_user_create_users', users=data)
-            return user
+            return user #[{'id': 8, 'username': 'asdfw'}]
         except SystemError as se:
             raise SystemError(se.args[0]['message'])
 
@@ -107,6 +107,8 @@ class Moodle():
         except:
             raise SystemError("Error calling Moodle API\n", traceback.format_exc())
         return user
+        #{'users': [{'id': 8, 'username': 'asdfw', 'firstname': 'afowie', 'lastname': 'aokjdnfwe', 'fullname': 'afowie aokjdnfwe', 'email': 'awfewe@ads.com', 'department': '', 'firstaccess': 0, 'lastaccess': 0, 'auth': 'manual', 'suspended': False, 'confirmed': True, 'lang': 'ca', 'theme': '', 'timezone': '99', 'mailformat': 1, 'profileimageurlsmall': 'https://moodle.santantoni.duckdns.org/theme/image.php/cbe/core/1630941606/u/f2', 'profileimageurl': 'https://DOMAIN/theme/image.php/cbe/core/1630941606/u/f1'}], 'warnings': []}
+
 
     def get_users_with_groups_and_roles(self):
         q = """select u.id as id, username, firstname as first, lastname as last, email, json_agg(h.name) as groups,  json_agg(r.shortname) as roles
@@ -166,7 +168,9 @@ class Moodle():
         return user
 
     def delete_user_in_cohort(self,userid,cohortid):
-        user = self.call('core_cohort_delete_cohort_members', cohortid=cohortid, userid=userid)
+        members=[{'cohortid':cohortid,
+                    'userid':userid}]
+        user = self.call('core_cohort_delete_cohort_members', members=members)
         return user
 
     def get_cohort_members(self, cohort_ids):

admin/src/admin/lib/nextcloud.py

@@ -116,7 +116,7 @@ class Nextcloud():
 
         # With quotas
         q = """select u.uid as username, configvalue as quota, sum(size) as total_bytes, adn.value as displayname, ade.value as email, json_agg(gg.displayname) as admin_groups,json_agg(g.displayname) as groups
-                from oc_users as u 
+                from oc_accounts as u 
                 left join oc_group_user as gu on gu.uid = u.uid
                 left join oc_groups as g on gu.gid = g.gid
                 left join oc_group_admin as ga on ga.uid = u.uid

admin/src/admin/static/js/groups.js

@@ -21,7 +21,7 @@ $(document).ready(function() {
                     }
                 })
                 $(".groups-select").append(
-                    '<option value="' + element.path + '">' + element.path + '</option>'
+                    '<option value="' + element.name + '">' + element.name + '</option>'
                 )
             });
             $('.groups-select').select2();

admin/src/admin/static/js/sysadmin/users.js

@@ -230,6 +230,25 @@ $(document).ready(function() {
         });
     });
 
+
+    $('.btn-sync_from_keycloak').on('click', function () {
+        $.ajax({
+            type: "PUT",
+            url:"/api/users",
+            success: function(data)
+            {
+                console.log('SUCCESS')
+                // $("#modalImport").modal('hide');
+                // users_table.ajax.reload();
+                // groups_table.ajax.reload();
+            },
+            error: function(data)
+            {
+                alert('Something went wrong on our side...')
+            }
+        });
+    });
+
     $('.btn-sync_to_nextcloud').on('click', function () {
         $.ajax({
             type: "POST",

admin/src/admin/static/js/users.js

@@ -18,7 +18,7 @@ $(document).ready(function() {
                     }
                 })
                 $(".groups-select").append(
-                    '<option value="' + element.path + '">' + element.path + '</option>'
+                    '<option value="' + element.name + '">' + element.name + '</option>'
                 )
             });
             $('.groups-select').select2();
@@ -307,7 +307,11 @@ $(document).ready(function() {
                             {
                                 "targets": 2,
                                 "render": function ( data, type, full, meta ) {
-                                    return full.roles[0][0].toUpperCase() + full.roles[0].slice(1);
+                                    if(full.roles.length){
+                                        return full.roles[0][0].toUpperCase() + full.roles[0].slice(1);
+                                    }else{
+                                        return '-'
+                                    }
                                 }},
                             {
                                 "targets": 4,

admin/src/admin/static/templates/pages/modals/users_modals.html

@@ -42,7 +42,7 @@
                 <div class="col-md-6 col-xs-12">
                   <label class="control-label" for="name">Username <span class="required">*</span>
                   </label>
-                  <input id="username" class="roundbox" maxlength="40" pattern="^[-_àèìòùáéíóúñçÀÈÌÒÙÁÉÍÓÚÑÇ .a-zA-Z0-9]+$" data-parsley-length="[4, 40]" name="username" placeholder="Username" data-parsley-trigger="change" required type="text" style="width:100%">
+                  <input id="username" class="roundbox" maxlength="40" pattern="^[-_.a-z0-9]+$" data-parsley-length="[4, 40]" name="username" placeholder="Username" data-parsley-trigger="change" required type="text" style="width:100%">
                 </div>
                 <div class="col-md-6 col-xs-12">
                   <label class="control-label" for="email">Email <span class="required">*</span>

admin/src/admin/static/templates/pages/sysadmin/users.html

@@ -21,6 +21,9 @@
         <div class="clearfix"></div>
 			</div>
 			<div class="x_content">
+				<button class="btn btn-success btn-xs btn-sync_from_keycloak">
+					<i class="fa fa-refresh" aria-hidden="true"></i> Sync all from keycloak
+				</button>
 				<button class="btn btn-primary btn-xs btn-sync_to_nextcloud">
 					<i class="fa fa-refresh" aria-hidden="true"></i> Sync to Nextcloud
 				</button>

admin/src/admin/views/ApiViews.py

@@ -12,6 +12,8 @@ import concurrent.futures
 from flask_login import current_user, login_required
 from .decorators import is_admin
 
+from ..lib.helpers import system_group
+
 # import Queue
 import threading
 threads={}
@@ -26,7 +28,7 @@ from ..lib.exceptions import UserExists, UserNotFound
 def resync():
     return json.dumps(app.admin.resync_data()), 200, {'Content-Type': 'application/json'}
 
-@app.route('/api/users', methods=['GET'])
+@app.route('/api/users', methods=['GET','PUT'])
 @app.route('/api/users/<provider>', methods=['POST', 'PUT', 'GET', 'DELETE'])
 @login_required
 def users(provider=False):
@@ -44,12 +46,28 @@ def users(provider=False):
             return json.dumps(app.admin.sync_to_moodle()), 200, {'Content-Type': 'application/json'}
         if provider == 'nextcloud':
             return json.dumps(app.admin.sync_to_nextcloud()), 200, {'Content-Type': 'application/json'}
+    if request.method == 'PUT' and not provider:
+        if current_user.role != 'admin': return json.dumps({}), 301, {'Content-Type': 'application/json'}
+
+        if 'external' in threads.keys():
+            if threads['external'] is not None and threads['external'].is_alive():
+                return json.dumps({'msg':'Precondition failed: already working with users'}), 412, {'Content-Type': 'application/json'}
+            else:
+                threads['external']=None
+        try:
+            threads['external'] = threading.Thread(target=app.admin.update_users_from_keycloak, args=())
+            threads['external'].start()
+            return json.dumps({}), 200, {'Content-Type': 'application/json'}
+        except:
+            log.error(traceback.format_exc())
+            return json.dumps({'msg':'Add user error.'}), 500, {'Content-Type': 'application/json'}
+
+        # return json.dumps(app.admin.update_users_from_keycloak()), 200, {'Content-Type': 'application/json'}
 
     users = app.admin.get_mix_users()
-
     if current_user.role != 'admin':
         for user in users:
-            user['keycloak_groups'] = [g for g in user['keycloak_groups'] if g not in ['/admin','/manager','/teacher','/student']]
+            user['keycloak_groups'] = [g for g in user['keycloak_groups'] if not system_group(g) ]
     return json.dumps(users), 200, {'Content-Type': 'application/json'}
     
 
@@ -137,7 +155,6 @@ def group(group_id=False):
     if request.method == 'DELETE':
         try:
             data=request.get_json(force=True)
-            print(data)
         except:
             data=False
         
@@ -155,9 +172,9 @@ def groups(provider=False):
         sorted_groups = sorted(app.admin.get_mix_groups(), key=lambda k: k['name'])
         if current_user.role != "admin":
             ## internal groups should be avoided as are assigned with the role
-            sorted_groups = [sg for sg in sorted_groups if sg['path'] not in ['/admin','/manager','/teacher','/student'] and sg['path'].startswith('/')]
+            sorted_groups = [sg for sg in sorted_groups if not system_group(sg['name'])]
         else:
-            sorted_groups = [sg for sg in sorted_groups if sg['path'].startswith('/')]
+            sorted_groups = [sg for sg in sorted_groups]
         return json.dumps(sorted_groups), 200, {'Content-Type': 'application/json'}
     if request.method == 'DELETE':
         if provider == 'keycloak':

docker-compose-parts/admin.devel.yml

@@ -2,6 +2,4 @@
 version: '3.7'
 services:
   isard-sso-admin:
-    volumes:
+    entrypoint: /bin/sleep infinity
-    - ${BUILD_ROOT_PATH}/admin/src:/admin:rw
-    command: /bin/sleep infinity