From 48b4e00286af07f94bcfafcd8c92ac52c31896c8 Mon Sep 17 00:00:00 2001 From: Tim Schubert Date: Tue, 14 Aug 2018 18:24:17 +0200 Subject: [PATCH] Rewrite creating repo and course --- abgabesystem.py | 362 ++++++++++++++++++++++-------------------------- config.yml | 3 +- gitlab.rb | 4 +- 3 files changed, 171 insertions(+), 198 deletions(-) diff --git a/abgabesystem.py b/abgabesystem.py index ea273e3..dbcd30d 100644 --- a/abgabesystem.py +++ b/abgabesystem.py @@ -1,7 +1,6 @@ #!/usr/bin/env python3 import argparse -import yaml import gitlab import logging as log import csv @@ -10,77 +9,6 @@ import subprocess import os -class Course(yaml.YAMLObject): - """Group for a course - - - name: name of the course - - base: the project containig the official solutions - - students: path to the CSV file that can be exported from Stud.IP - - deploy_key: a deploy key for deploying student repos to CI jobs - """ - - yaml_tag = 'Course' - - def __init__(self, name, studentsfile, deploy_key): - self.name = name - self.base = 'solutions' - self.students = studentsfile - self.deploy_key = deploy_key - - def sync_group(self, gl): - """Creates the group for the course - """ - found = gl.groups.list(search=self.name) - print(found) - - if len(found) > 0: - for g in found: - if g.name == self.name: - log.info('Found existing group %s' % found[0].name) - return g - - path = self.name.replace(' ', '_').lower() - log.info('%s: Creating group' % self.name) - group = gl.groups.create({ - 'name': self.name, - 'path': path, - 'visibility': 'internal' - }) - return group - - def sync_base(self, gl): - """Creates the project containig the official solutions - - All student projects will fork from this projects and can be updated using - - ``` - git remote add upstream - git pull upstream master - ``` - """ - - found = self.group.projects.list(search=self.base) - if len(found) == 0: - self.base = gl.projects.create({ - 'name': self.base, - 'namespace_id': self.group.id, - 'visibility': 'internal' - }) - log.info('%s: Created project base repo' % self.name) - data = { - 'branch': 'master', - 'commit_message': 'Initial commit', - 'actions': [ - { - 'action': 'create', - 'file_path': 'README.md', - 'content': 'README' - } - ] - } - self.base.commits.create(data) - - class Student(): """A Gitlab user @@ -103,73 +31,6 @@ class Student(): yield Student(line['Nutzernamen'], line['E-Mail'], line['Vorname'] + ' ' + line['Nachname'], line['Gruppe']) - def sync_user(self, gl, ldap): - """Creates a dummy user for users that do not exist in gitlab - but in LDAP and have not logged in yet""" - - found = gl.users.list(search=self.user) - user = None - if len(found) > 0: - user = found[0] - else: - log.info('Creating student %s' % self.user) - user = gl.users.create({ - 'email': self.email, - 'username': self.user, - 'name': self.name, - 'provider': ldap['provider'], - 'skip_confirmation': True, - 'extern_uid': 'uid=%s,%s' % (self.user, ldap['main']['base']), - 'password': secrets.token_urlsafe(nbytes=32) - }) - user.customattributes.set('group', self.group) - - return user - - -def sync_project(gl, course, student): - """Create user projects as forks from course/solutions in namespace of - course and add user as developer (NOT master) user should not be able - to modify protected TAG or force-push on protected branch users can - later invite other users into their projects""" - - projects = course.group.projects.list(search=student.user.username) - project = None - if len(projects) == 0: - base = course.group.projects.list(search=course.base)[0] - base = gl.projects.get(base.id) - - log.info('Creating project %s' % student.user.username) - fork = base.forks.create({ - 'namespace': student.user.username, - 'name': student.user.username - }) - project = gl.projects.get(fork.id) - project.path = student.user.username - project.name = student.user.username - project.visibility = 'private' - project.save() - course.group.transfer_project(to_project_id=fork.id) - else: - project = gl.projects.get(id=projects[0].id) - - try: - student_member = project.members.get(student.user.id) - student_member.access_level = gitlab.DEVELOPER_ACCESS - student_member.save() - except gitlab.exceptions.GitlabGetError as e: - student_member = project.members.create({'user_id': student.user.id, 'access_level': - gitlab.DEVELOPER_ACCESS}) - deploy_key = project.keys.create({ - 'title': course.name, - 'key': course.deploy_key - }) - - project.keys.enable(deploy_key.id) - project.container_registry_enabled = False - project.lfs_enabled = False - project.save() - def create_tag(project, tag, ref): """Creates protected tag on ref @@ -186,56 +47,159 @@ def create_tag(project, tag, ref): }) -def sync(gl, conf, args): - """Syncs groups and students from Stud.IP to Gitlab and create student - projects +def get_students(gl, students_csv): + """Returns already existing GitLab users for students from provided CSV file that have an account. """ - course = conf['course'] - print(course.name) - course.group = course.sync_group(gl) - course.sync_base(gl) + for student in Student.from_csv(students_csv): + users = gl.users.list(search=student.user) + if len(users) > 0: + yield users[0] - with open(course.students, encoding='latin1') as csvfile: - for student in Student.from_csv(csvfile): + +def create_user(gl, student, ldap_base, ldap_provider): + """Creates a GitLab user account student. + Requires admin privileges. + """ + + user = gl.users.create({ + 'email': student.email, + 'username': student.user, + 'name': student.name, + 'provider': ldap_provider, + 'skip_confirmation': True, + 'extern_uid': 'uid=%s,%s' % (student.user, ldap_base), + 'password': secrets.token_urlsafe(nbytes=32) + }) + user.customattributes.set('group', student.group) + + return user + + +def create_users(gl, args): + with open(args.students, encoding='iso8859') as students_csv: + for student in Student.from_csv(students_csv): try: - student.user = student.sync_user(gl, conf['ldap']) - print("%s %s" % (student.user.username, student.user.name)) - sync_project(gl, course, student) - except gitlab.exceptions.GitlabCreateError as e: - log.warn(e) + create_user(gl, student, args.ldap_base, args.ldap_provider) + except gitlab.exceptions.GitlabCreateError: + log.warn('Failed to create user: %s' % student.user) -def list_projects(gl, conf, args): - """Prints all git URLs for the student's projects +def fork_reference(gl, reference, namespace, deploy_key): + """Create fork of solutions for student. """ - groups = gl.groups.list(search=conf['course'].name) - print(groups) - if len(groups) == 0: - pass - for g in groups: - if (g.name == args.course): - for project in g.projects.list(all=True): - project = gl.projects.get(project.id) - print(project.ssh_url_to_repo) + + fork = reference.forks.create({ + 'namespace': namespace.id + }) + project = gl.projects.get(fork.id) + project.visibility = 'private' + project.container_registry_enabled = False + project.lfs_enabled = False + deploy_key = project.keys.create({ + 'title': "Deploy Key", + 'key': deploy_key + }) + project.keys.enable(deploy_key.id) + project.save() + + return project -def get_base_project(gl, conf, args): - """The project of a course containing the official solutions""" +def create_project(gl, group, user, reference, deploy_key): + """Creates a namespace (subgroup) and forks the project with + the reference solutions into that namespace + """ - return conf['course']['base'] + subgroup = None + + try: + subgroup = gl.groups.create({ + 'name': user.username, + 'path': user.username, + 'parent_id': group.id + }) + except gitlab.exceptions.GitlabError as e: + subgroups = group.subgroups.list(search=user.username) + if len(subgroups) > 0 and subgroup[0].name == user.username: + subgroup = subgroups[0] + subgroup = gl.groups.get(subgroup.id, lazy=True) + else: + raise(e) + try: + subgroup.members.create({ + 'user_id': user.id, + 'access_level': gitlab.DEVELOPER_ACCESS, + }) + except gitlab.exceptions.GitlabError: + log.warning('Failed to add student %s to its own group' % user.username) + + try: + fork_reference(gl, reference, subgroup, deploy_key) + except gitlab.exceptions.GitlabCreateError as e: + log.warning(e.error_message) -def deadline(gl, conf, args): +def setup_course(gl, group, students_csv, deploy_key): + + solution = None + reference_project = None + + try: + solution = gl.groups.create({ + 'name': 'solutions', + 'path': 'solutions', + 'parent_id': group.id, + }) + except gitlab.exceptions.GitlabCreateError as e: + log.info('Failed to create solutions group. %s' % e.error_message) + solutions = group.subgroups.list(search='solutions') + if len(solutions) > 0 and solutions[0].name == 'solutions': + solution = gl.groups.get(solutions[0].id, lazy=True) + else: + raise(gitlab.exceptions.GitlabCreateError(error_message='Failed to setup solutions subgroup')) + + try: + reference_project = gl.projects.create({ + 'name': 'solutions', + 'namespace_id': solution.id + }) + except gitlab.exceptions.GitlabCreateError as e: + log.info('Failed to setup group structure. %s' % e.error_message) + projects = solution.projects.list(search='solutions') + if len(projects) > 0 and projects[0].name == 'solutions': + reference_project = gl.projects.get(projects[0].id) + else: + raise(gitlab.exceptions.GitlabCreateError(error_message='Failed to setup reference solutions')) + + if solution is None or reference_project is None: + raise(gitlab.exceptions.GitlabCreateError(error_message='Failed to setup course')) + + for user in get_students(gl, students_csv): + create_project(gl, solution, user, reference_project, deploy_key) + + +def projects(gl, args): + groups = gl.groups.list(search=args.course) + if len(groups) == 0 and groups[0].name == args.course: + log.warn('This group does not exist') + else: + group = groups[0] + with open(args.deploy_key, 'r') as key, open(args.students, encoding='iso8859') as students_csv: + key = key.read() + setup_course(gl, group, students_csv, key) + + +def deadline(gl, args): """Checks deadlines for course and triggers deadline if it is reached""" deadline_name = args.tag_name - course = conf['course'] + course = args.course group = None for g in gl.groups.list(search=course.name): - if g.name == conf['course'].name: + if g.name == args.course: group = g - course.group = gl.groups.get(group.id) + group = gl.groups.get(group.id) for project in course.group.projects.list(all=True): project = gl.projects.get(project.id) print(project.name) @@ -245,41 +209,44 @@ def deadline(gl, conf, args): print(e) -def plagiates(gl, conf, args): +def plagiates(gl, args): """Runs the plagiarism checker (JPlag) for the solutions and a given tag name """ - groups = gl.groups.list(search=conf['course'].name) + groups = gl.groups.list(search=args.course) tag = args.tag_name print(groups) if len(groups) == 0: pass for g in groups: - if g.name == conf['course'].name: + if g.name == args.course: try: os.mkdir('repos') except os.FileExistsError as e: print(e) os.chdir('repos') for project in g.projects.list(all=True): - project = gl.projects.get(project.id) + project = gl.projects.get(project.id, lazy=True) try: subprocess.run( ['git', 'clone', '--branch', tag, project.ssh_url_to_repo]) - except subprocess.CalledProcessError as e: print(e) os.chdir('..') subprocess.run( - ['java', '-jar', '/app/jplag.jar', '-s', 'repos', '-p', 'java', '-r', 'results', '-bc', conf['course'].base, '-l', 'java17']) + ['java', '-jar', '/app/jplag.jar', '-s', 'repos', '-p', 'java', '-r', 'results', '-bc', args.reference, '-l', 'java17']) -def parseconf(conf): - """Reads course from config file""" - - with open(args.config[0], 'r') as conf: - return yaml.load(conf) +def course(gl, args): + try: + group = gl.groups.create({ + 'name': args.course, + 'path': args.course + }) + log.info('Created group %s' % args.course) + except gitlab.exceptions.GitlabCreateError as e: + log.warning('Failed to create group %s. %s' % (args.course, e.error_message)) if __name__ == '__main__': @@ -289,21 +256,29 @@ if __name__ == '__main__': log.info('authenticated') parser = argparse.ArgumentParser() - parser.add_argument( - '--config', type=str, nargs=1, help='path to config file', - default=['config.yml']) subparsers = parser.add_subparsers(title='subcommands') - sync_parser = subparsers.add_parser( - 'sync', - help='students and courses from Stud.IP and LDAP') - sync_parser.set_defaults(func=sync) + user_parser = subparsers.add_parser( + 'users', + help='Creates users from LDAP') + user_parser.set_defaults(func=create_users) + user_parser.add_argument('-s', '--students', dest='students') + user_parser.add_argument('-b', '--ldap-base', dest='ldap_base') + user_parser.add_argument('-p', '--ldap-provider', dest='ldap_provider') + + course_parser = subparsers.add_parser( + 'courses', + help='Create course') + course_parser.set_defaults(func=course) + course_parser.add_argument('-c', '--course', dest='course') projects_parser = subparsers.add_parser( 'projects', - description='list projects for course') - projects_parser.set_defaults(func=list_projects) - projects_parser.add_argument('course') + help='Setup projects') + projects_parser.set_defaults(func=projects) + projects_parser.add_argument('-c', '--course', dest='course') + projects_parser.add_argument('-d', '--deploy-key', dest='deploy_key') + projects_parser.add_argument('-s', '--students', dest='students') deadline_parser = subparsers.add_parser( 'deadline', @@ -318,11 +293,10 @@ if __name__ == '__main__': plagiates_parser.add_argument('tag_name') args = parser.parse_args() - conf = parseconf(args.config) log.basicConfig(filename='example.log', filemode='w', level=log.DEBUG) if 'func' in args: - args.func(gl, conf, args) + args.func(gl, args) else: parser.print_help() diff --git a/config.yml b/config.yml index 341ef5f..c6980d1 100644 --- a/config.yml +++ b/config.yml @@ -1,6 +1,5 @@ ldap: - main: - base: 'ou=people,dc=tu-bs,dc=de' + base: 'ou=people,dc=tu-bs,dc=de' provider: main course: !!python/object:abgabesystem.Course diff --git a/gitlab.rb b/gitlab.rb index ac74b31..261e6c1 100644 --- a/gitlab.rb +++ b/gitlab.rb @@ -68,7 +68,7 @@ gitlab_rails['gitlab_email_reply_to'] = 'noreply@ips1.ibr.cs.tu-bs.de' # gitlab_rails['gitlab_default_projects_features_wiki'] = true # gitlab_rails['gitlab_default_projects_features_snippets'] = true # gitlab_rails['gitlab_default_projects_features_builds'] = true -gitlab_rails['gitlab_default_projects_features_container_registry'] = false +# gitlab_rails['gitlab_default_projects_features_container_registry'] = true ### Automatic issue closing ###! See https://docs.gitlab.com/ce/customization/issue_closing.html for more @@ -214,7 +214,7 @@ main: # 'main' is the GitLab 'provider ID' of this LDAP server # lowercase_usernames: false # block_auto_created_users: false base: 'ou=people,dc=tu-bs,dc=de' - user_filter: '(ou=Student TU Braunschweig)' + #user_filter: '(ou=Student TU Braunschweig)' # ## EE only # group_base: '' # admin_group: ''