Add recurring handling of user, resource and vgrid map refresh to janitor

mig/lib/janitor.py

@@ -1,3 +1,3 @@
 #!/usr/bin/python
 # -*- coding: utf-8 -*-
 #
@@ -43,6 +43,8 @@
 from mig.shared.pwcrypto import verify_reset_token
 from mig.shared.serial import load
 from mig.shared.userdb import default_db_path, load_user_dict
+from mig.shared.vgridaccess import get_resource_map, get_user_map, \
+    get_vgrid_map, last_refresh, RESOURCES, USERS, VGRIDS
 
 REMIND_REQ_DAYS = 5
 EXPIRE_REQ_DAYS = 30
@@ -56,7 +58,7 @@
 SECS_PER_HOUR = 60 * SECS_PER_MINUTE
 SECS_PER_DAY = 24 * SECS_PER_HOUR
 
 task_triggers = {}
 
 
 def _lookup_last_run(configuration, target):
@@ -493,7 +495,17 @@
         now = time.time()
     handled = 0
     _logger.debug("handle pending cache updates")
-    # TODO: actually handle vgrid/user/resource/... cache updates
+    # NOTE: we use get_X_map here to throttle down and only refresh when needed
+    user_map = get_user_map(configuration, caching=False)
+    if last_refresh[USERS] > now:
+        handled += 1
+    if configuration.site_enable_resources:
+        resource_map = get_resource_map(configuration, caching=False)
+        if last_refresh[RESOURCES] > now:
+            handled += 1
+    vgrid_map = get_vgrid_map(configuration, caching=False)
+    if last_refresh[VGRIDS] > now:
+        handled += 1
     if handled > 0:
         _logger.info("handled %d pending cache updates" % handled)
     else:
@@ -501,7 +513,7 @@
     return handled
 
 
 def handle_janitor_tasks(configuration, now=None):
     """A wrapper to take care of all regular janitor tasks like clean up and
     cache updates.
     Returns the number of actual tasks completed to let the main thread know if

mig/shared/functionality/resman.py

@@ -1,10 +1,10 @@
 #!/usr/bin/python
 # -*- coding: utf-8 -*-
 #
 # --- BEGIN_HEADER ---
 #
 # resman - manage resources
-# Copyright (C) 2003-2021  The MiG Project lead by Brian Vinter
+# Copyright (C) 2003-2025  The MiG Project lead by the Science HPC Center at UCPH
 #
 # This file is part of MiG.
 #
@@ -20,7 +20,8 @@
 #
 # You should have received a copy of the GNU General Public License
 # along with this program; if not, write to the Free Software
-# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301,
+# USA.
 #
 # -- END_HEADER ---
 #
@@ -41,7 +42,7 @@
 from mig.shared.modified import pending_resources_update, pending_vgrids_update
 from mig.shared.resource import anon_to_real_res_map
 from mig.shared.vgridaccess import user_visible_res_confs, get_resource_map, \
-    OWNERS, CONF
+    load_resource_map, CONF, OWNERS
 
 list_operations = ['showlist', 'list']
 show_operations = ['show', 'showlist']
@@ -56,7 +57,7 @@
     return ['resources', defaults]
 
 
 def main(client_id, user_arguments_dict):
     """Main function used by front end"""
 
     (configuration, logger, output_objects, op_name) = \
@@ -85,7 +86,7 @@
                                '''Resources are not enabled on this system'''})
         return (output_objects, returnvalues.SYSTEM_ERROR)
 
     if not operation in allowed_operations:
         output_objects.append({'object_type': 'text', 'text':
                                '''Operation must be one of %s.''' %
                                ', '.join(allowed_operations)})
@@ -155,7 +156,11 @@
         logger.info("get vgrid and resource map with caching %s" % caching)
         visible_res_confs = user_visible_res_confs(configuration, client_id,
                                                    caching)
-        res_map = get_resource_map(configuration, caching)
+        # NOTE: prevent refresh if janitor is enabled as it handles all updates
+        if configuration.site_enable_janitor:
+            (res_map, _) = load_resource_map(configuration, caching=caching)
+        else:
+            res_map = get_resource_map(configuration, caching)
         anon_map = anon_to_real_res_map(configuration.resource_home)
 
         # NOTE: use simple pending check if caching to avoid lock during update

mig/shared/functionality/vgridman.py

@@ -1,10 +1,10 @@
 #!/usr/bin/python
 # -*- coding: utf-8 -*-
 #
 # --- BEGIN_HEADER ---
 #
 # vgridman - backend to manage vgrids
 # Copyright (C) 2003-2025  The MiG Project lead by the Science HPC Center at UCPH
 #
 # This file is part of MiG.
 #
@@ -20,7 +20,8 @@
 #
 # You should have received a copy of the GNU General Public License
 # along with this program; if not, write to the Free Software
-# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301,
+# USA.
 #
 # -- END_HEADER ---
 #
@@ -31,7 +32,7 @@
 
 from mig.shared import returnvalues
 from mig.shared.base import get_site_base_url
 from mig.shared.defaults import default_vgrid, all_vgrids, default_pager_entries, \
     csrf_field
 from mig.shared.functional import validate_input_and_cert
 from mig.shared.handlers import get_csrf_limit, make_csrf_token
@@ -40,7 +41,8 @@
 from mig.shared.modified import pending_vgrids_update
 from mig.shared.useradm import get_full_user_map
 from mig.shared.vgrid import vgrid_create_allowed
-from mig.shared.vgridaccess import get_vgrid_map, VGRIDS, OWNERS, MEMBERS, SETTINGS
+from mig.shared.vgridaccess import get_vgrid_map, load_vgrid_map, MEMBERS, \
+    OWNERS, SETTINGS, VGRIDS
 
 list_operations = ['showlist', 'list']
 show_operations = ['show', 'showlist']
@@ -55,7 +57,7 @@
     return ['vgrids', defaults]
 
 
 def main(client_id, user_arguments_dict):
     """Main function used by front end"""
 
     (configuration, logger, output_objects, op_name) = \
@@ -80,7 +82,7 @@
     operation = accepted['operation'][-1]
     caching = (accepted['caching'][-1].lower() in ('true', 'yes'))
 
     if not operation in allowed_operations:
         output_objects.append({'object_type': 'error_text', 'text':
                                '''Operation must be one of %s.''' %
                                ', '.join(allowed_operations)})
@@ -96,7 +98,7 @@
     user_settings = title_entry.get('user_settings', {})
     collaboration_links = user_settings.get(
         'SITE_COLLABORATION_LINKS', 'default')
     if not collaboration_links in configuration.site_collaboration_links or \
             collaboration_links == 'default':
         active_vgrid_links += configuration.site_default_vgrid_links
     elif collaboration_links == 'advanced':
@@ -115,7 +117,7 @@
     if operation in show_operations:
 
         # jquery support for tablesorter and confirmation on request and leave
         # table initially sorted by col. 2 (admin), then 3 (member), then 0 (name)
 
         # NOTE: We distinguish between caching on page load and forced refresh
         refresh_helper = 'ajax_vgridman("%s", %s, %%s)'
@@ -185,7 +187,11 @@
 
     if operation in list_operations:
         logger.info("get vgrid map with caching %s" % caching)
-        vgrid_map = get_vgrid_map(configuration, caching=caching)
+        # NOTE: prevent refresh if janitor is enabled as it handles all updates
+        if configuration.site_enable_janitor:
+            (vgrid_map, _) = load_vgrid_map(configuration, caching=caching)
+        else:
+            vgrid_map = get_vgrid_map(configuration, caching=caching)
         # NOTE: use simple pending check if caching to avoid lock during update
         if caching:
             pending_updates = pending_vgrids_update(configuration)
@@ -232,16 +238,16 @@
                 # can own it or leave it. Do not add any page links.
 
                 vgrid_obj['privatemonitorlink'] = {'object_type': 'link',
                                                    'destination': 'showvgridmonitor.py?vgrid_name=%s'
                                                    % vgrid_name,
                                                    'class': 'monitorlink iconspace',
                                                    'title': 'View %s monitor' %
                                                    vgrid_name,
                                                    'text': 'View'}
                 vgrid_obj['memberlink'] = {'object_type': 'link',
                                            'destination': '',
                                            'class': 'infolink iconspace',
                                            'title': 'Every user is member of the %s %s'
                                            % (default_vgrid, label),
                                            'text': ''}
                 vgrid_obj['administratelink'] = {'object_type': 'link',