integrates invenio-cern-sync and jobs

- closes CERNDocumentServer#196

Pipfile

@@ -18,6 +18,7 @@ uwsgitop = ">=0.11"
 uwsgi-tools = ">=1.1.1"
 flask-mail = ">=0.9.0,<0.10.0"
 invenio-preservation-sync = "==0.1.0"
+# invenio-cern-sync = {git = "https://github.com/cerndocumentserver/invenio-cern-sync.git", ref = "v1.0.0"}
 
 [requires]
 python_version = "3.9"

invenio.cfg

@@ -192,6 +192,7 @@ DATACITE_DATACENTER_SYMBOL = ""
 # See https://github.com/inveniosoftware/invenio-accounts/blob/master/invenio_accounts/config.py
 ACCOUNTS_DEFAULT_USERS_VERIFIED = True  # ensure that users are verified by default
 ACCOUNTS_DEFAULT_USER_VISIBILITY = "public"  # enables users to be searchable for invites
+ACCOUNTS_DEFAULT_EMAIL_VISIBILITY = "public"
 ACCOUNTS_LOCAL_LOGIN_ENABLED = True  # enable local login
 PERMANENT_SESSION_LIFETIME = timedelta(days=10)
 SECURITY_REGISTERABLE = True  # local login: allow users to register
@@ -264,59 +265,55 @@ OAUTHCLIENT_SIGNUP_FORM = confirm_registration_form
 
 OAUTH_REMOTE_APP_NAME = "cern"
 
+# Invenio-CERN-Sync/CERN SSO
+# ==========================
+from invenio_cern_sync.sso import cern_remote_app_name, cern_keycloak
 OAUTHCLIENT_REMOTE_APPS = {
-    OAUTH_REMOTE_APP_NAME: _keycloak_helper.remote_app,
+    cern_remote_app_name: cern_keycloak.remote_app,
 }
 
 CERN_APP_CREDENTIALS = {
     "consumer_key": "CHANGE ME",
     "consumer_secret": "CHANGE ME",
 }
+CERN_SYNC_KEYCLOAK_BASE_URL = "https://auth.cern.ch/"
+CERN_SYNC_AUTHZ_BASE_URL = "https://authorization-service-api.web.cern.ch/"
+INVENIO_CERN_SYNC_KEYCLOAK_BASE_URL = "https://auth.cern.ch/"  # set env var when testing
+
+
+OAUTHCLIENT_CERN_REALM_URL = cern_keycloak.realm_url
+OAUTHCLIENT_CERN_USER_INFO_URL = cern_keycloak.user_info_url
+OAUTHCLIENT_CERN_VERIFY_EXP = True
+OAUTHCLIENT_CERN_VERIFY_AUD = False
+OAUTHCLIENT_CERN_USER_INFO_FROM_ENDPOINT = True
 
 from invenio_oauthclient.views.client import auto_redirect_login
 
 ACCOUNTS_LOGIN_VIEW_FUNCTION = auto_redirect_login  # autoredirect to external login if enabled
 OAUTHCLIENT_AUTO_REDIRECT_TO_EXTERNAL_LOGIN = True  # autoredirect to external login
 
+from invenio_cern_sync.users.profile import CERNUserProfileSchema
+ACCOUNTS_USER_PROFILE_SCHEMA = CERNUserProfileSchema()
+
 # Invenio-UserProfiles
 # ====================
-USERPROFILES_READ_ONLY = False  # allow users to change profile info (name, email, etc...)
-USERPROFILES_EXTEND_SECURITY_FORMS = True
+USERPROFILES_READ_ONLY = True  # disable change of user profile
+USERPROFILES_EXTEND_SECURITY_FORMS = True  # automatically use user's email address as account email
 
 # OAI-PMH
 # =======
-# See https://github.com/inveniosoftware/invenio-oaiserver/blob/master/invenio_oaiserver/config.py
 OAISERVER_ID_PREFIX = "cds-rdm.com"
 """The prefix that will be applied to the generated OAI-PMH ids."""
 
 # Invenio-Search
 # ==============
 SEARCH_INDEX_PREFIX = "cds-rdm-"
 
-# Celery
-# ======
-CELERY_BEAT_SCHEDULE = {
-    **APP_RDM_CELERY_BEAT_SCHEDULE,
-    "user-sync": {
-        "task": "cds_rdm.tasks.sync_users",
-        "schedule": crontab(minute=0, hour=3),  # Every day at 03:00 UTC
-    },
-    "groups-sync": {
-        "task": "cds_rdm.tasks.sync_groups",
-        "schedule": crontab(minute=0, hour=2),  # Every day at 02:00 UTC
-    },
-}
-
 ###############################################################################
 # CDS-RDM configuration
 ###############################################################################
 CDS_SERVICE_ELEMENT_URL = "https://cern.service-now.com/service-portal?id=service_element&name=CDS-Service"
 
-# AUTH/LDAP
-CERN_LDAP_URL = "ldap://xldap.cern.ch"
-CERN_AUTHORIZATION_SERVICE_API = "https://authorization-service-api-qa.web.cern.ch/api/v1.0/"
-CERN_AUTHORIZATION_SERVICE_API_GROUP = "Group"
-
 # Permissions: define who can create new communities
 CDS_EMAILS_ALLOW_CREATE_COMMUNITIES = []
 CDS_GROUPS_ALLOW_CREATE_COMMUNITIES = []
@@ -514,6 +511,8 @@ RDM_CUSTOM_FIELDS_UI = [
         ]
     }
 ]
+RDM_FILES_DEFAULT_QUOTA_SIZE = 50 * 10**9     # 50GB
+RDM_FILES_DEFAULT_MAX_FILE_SIZE = 50 * 10**9  # 50GB
 
 JOBS_ADMINISTRATION_ENABLED = True
 

site/cds_rdm/cli.py

@@ -15,14 +15,12 @@
 from invenio_pidstore.models import PersistentIdentifier
 from invenio_rdm_records.proxies import current_rdm_records_service
 from invenio_rdm_records.records.api import RDMDraft, RDMRecord
-from invenio_rdm_records.records.models import (
-    RDMDraftMetadata,
-    RDMFileDraftMetadata,
-    RDMFileRecordMetadata,
-    RDMParentCommunity,
-    RDMRecordMetadata,
-    RDMVersionsState,
-)
+from invenio_rdm_records.records.models import (RDMDraftMetadata,
+                                                RDMFileDraftMetadata,
+                                                RDMFileRecordMetadata,
+                                                RDMParentCommunity,
+                                                RDMRecordMetadata,
+                                                RDMVersionsState)
 from invenio_requests.proxies import current_requests_service
 from invenio_requests.records.api import Request
 from invenio_requests.records.models import RequestMetadata

site/cds_rdm/files.py

@@ -26,7 +26,8 @@
     HTTPKerberosAuth = type("obj", (object,), {})
     DISABLED = 3
     # use base PyFSFileStorage instead
-    from invenio_files_rest.storage.pyfs import PyFSFileStorage as BaseFileStorage
+    from invenio_files_rest.storage.pyfs import \
+        PyFSFileStorage as BaseFileStorage
 
 
 class OffloadFileStorage(BaseFileStorage):

site/cds_rdm/jobs.py

@@ -0,0 +1,43 @@
+# -*- coding: utf-8 -*-
+#
+# This file is part of Invenio.
+# Copyright (C) 2023 CERN.
+#
+# Invenio is free software; you can redistribute it and/or modify it
+# under the terms of the MIT License; see LICENSE file for more details.
+
+"""Jobs."""
+
+from invenio_jobs.jobs import JobType
+
+from .tasks import sync_groups, sync_users
+
+
+class SyncUsers(JobType):
+    """Sync users with CERN database."""
+
+    id = "sync_cern_users"
+    title = "Sync CERN users"
+    description = "Sync CERN users with the AuthZ service"
+
+    task = sync_users
+
+    @classmethod
+    def build_task_arguments(cls, _, since=None, **kwargs):
+        """Build task arguments."""
+        return {"since": since}
+
+
+class SyncGroups(JobType):
+    """Sync groups with CERN database."""
+
+    id = "sync_cern_groups"
+    title = "Sync CERN groups"
+    description = "Sync CERN groups with the AuthZ service"
+
+    task = sync_groups
+
+    @classmethod
+    def build_task_arguments(cls, _, since=None, **kwargs):
+        """Build task arguments."""
+        return {"since": since}