Add support for adding request contextual data to configuration objects.

mig/shared/conf.py

@@ -20,7 +20,7 @@
 #
 # You should have received a copy of the GNU General Public License
 # along with this program; if not, write to the Free Software
 # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
 #
 # -- END_HEADER ---
 #
@@ -32,6 +32,7 @@
 import os
 import sys
 
+from mig.shared.configuration import RuntimeConfiguration
 from mig.shared.defaults import MIG_ENV
 from mig.shared.fileio import unpickle
 
@@ -42,7 +43,6 @@
     and disable_auth_log arguments are passed on to allow skipping the default
     log initialization and disabling auth log for unit tests.
     """
-    from mig.shared.configuration import Configuration
     if config_file:
         _config_file = config_file
     elif os.environ.get('MIG_CONF', None):
@@ -63,8 +63,8 @@
         skip_log = True
         disable_auth_log = True
 
-    configuration = Configuration(_config_file, False, skip_log,
+    configuration = RuntimeConfiguration(_config_file, False, skip_log,
                                   disable_auth_log)
     return configuration
 
 
@@ -85,7 +85,7 @@
         return (True, resource_config)
 
 
 def get_resource_fields(resource_home, unique_resource_name, fields, logger):
     """Return a dictionary mapping fields to resource_config values.
     Missing fields are left out of the result dictionary.
     """
@@ -102,7 +102,7 @@
     return results
 
 
 def get_resource_exe(resource_config, exe_name, logger):
     for exe in resource_config['EXECONFIG']:
 
         # find the right exe entry
@@ -119,7 +119,7 @@
     return (False, msg)
 
 
 def get_resource_all_exes(resource_config, logger):
     msg = ''
     if 'EXECONFIG' not in resource_config:
         msg = 'No exe hosts configured!'
@@ -128,7 +128,7 @@
     return (True, resource_config['EXECONFIG'])
 
 
 def get_all_exe_names(unique_resource_name):
     exe_names = []
     conf = get_configuration_object()
     (status, resource_config) = \
@@ -142,7 +142,7 @@
     return [exe['name'] for exe in exe_units]
 
 
 def get_all_exe_vgrids(unique_resource_name):
     """Return a dictionary mapping exe names to assigned vgrids"""
     exe_vgrids = {}
     conf = get_configuration_object()
@@ -158,7 +158,7 @@
     return exe_vgrids
 
 
 def get_resource_store(resource_config, store_name, logger):
     for store in resource_config['STORECONFIG']:
 
         # find the right store entry
@@ -175,7 +175,7 @@
     return (False, msg)
 
 
 def get_resource_all_stores(resource_config, logger):
     msg = ''
     if 'STORECONFIG' not in resource_config:
         msg = 'No store hosts configured!'
@@ -184,7 +184,7 @@
     return (True, resource_config['STORECONFIG'])
 
 
 def get_all_store_names(unique_resource_name):
     store_names = []
     conf = get_configuration_object()
     (status, resource_config) = \

mig/shared/configuration.py

@@ -1,3 +1,3 @@
 #!/usr/bin/python
 # -*- coding: utf-8 -*-
 #
@@ -20,7 +20,7 @@
 #
 # You should have received a copy of the GNU General Public License
 # along with this program; if not, write to the Free Software
 # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
 #
 # -- END_HEADER ---
 #
@@ -158,7 +158,7 @@
                     #    "reading conf content from file in %r" % cache)
                     content = read_file(path, logger)
                     if cache and content:
                         # logger.debug("caching conf content salt in %r" % cache)
                         write_file(content, cache, logger)
                 if not content:
                     logger.warning("salt file not found or empty: %s" % path)
@@ -232,7 +232,7 @@
         'server_home': '~/state/server_home/',
         'webserver_home': '~/state/webserver_home/',
         'sessid_to_mrsl_link_home': '~/state/sessid_to_mrsl_link_home/',
         'sessid_to_jupyter_mount_link_home': '~/state/sessid_to_jupyter_mount_link_home/',
         'mig_system_files': '~/state/mig_system_files/',
         'mig_system_storage': '~/state/mig_system_storage',
         'mig_system_run': '~/state/mig_system_run/',
@@ -370,7 +370,7 @@
         'architectures': 'X86 AMD64 IA64 SPARC SPARC64 ITANIUM SUN4U SPARC-T1',
         'scriptlanguages': 'sh python java',
         'jobtypes': 'batch interactive bulk all',
         'lrmstypes': 'Native Native-execution-leader Batch Batch-execution-leader',
     }
     scheduler_section = {'algorithm': 'FairFit',
                          'expire_after': '99999999999',
@@ -408,7 +408,7 @@
         'QUOTA': quota_section,
     }
     for section in defaults:
         if not section in config.sections():
             config.add_section(section)
 
     modified = False
@@ -422,7 +422,7 @@
                 modified = True
     if modified:
         backup_path = '%s.%d' % (config_file, time.time())
         print('Backing up existing configuration to %s as update removes all comments'
               % backup_path)
         fd = open(config_file, 'r')
         backup_fd = open(backup_path, 'w')
@@ -710,7 +710,7 @@
                        'VERIFYFILES', 'VGRID', 'SANDBOX'],
     'job_cond_yellow': ['DISK', 'MEMORY', 'CPUTIME'],
     'job_cond_orange': ['CPUCOUNT', 'NODECOUNT'],
     'job_cond_red': ['EXECUTABLES', 'INPUTFILES', 'REGISTERED', 'SEEN_WITHIN_X'],
     'enable_suggest': False,
     'suggest_threshold': 'GREEN',
 
@@ -790,7 +790,7 @@
         try:
             if self.logger:
                 self.logger.info('reloading configuration and reopening log')
         except:
             pass
 
         try:
@@ -2855,6 +2855,47 @@
         return peers_dict
 
 
+class RuntimeConfiguration(Configuration):
+    """A more specific version of the Configuration which additionally supports
+    the notion of a context.
+
+    Contextual information that is relevant to the duration of a request is
+    required in certain cases e.g. to support templating. Given Configuration
+    objects are threaded into and throough almost all the necessary codepaths
+    to make this information available, they are an attractive place to put
+    this - but a Configuration is currently loaded from static per-site data.
+
+    Resolv this ambiguity with this subclass - a raw Confioguration will
+    continute to represent the static data while a specialised but entirely
+    compatible object is handed to request processing codepaths.
+    """
+
+    def __init__(self, config_file, verbose=False, skip_log=False,
+                 disable_auth_log=False):
+        super().__init__(config_file, verbose, skip_log, disable_auth_log)
+        self._context = None
+
+    def context(self, namespace=None):
+        """Retrieve the context or a previously registered namespace.
+        """
+
+        if self._context is None:
+            self._context = {}
+        if namespace is None:
+            return self._context
+        # allow the KeyError to escape if the registered namespace is missing
+        return self._context[namespace]
+
+    def context_set(self, value, namespace=None):
+        """Attach a value as named namespace within the active congifuration.
+        """
+        assert namespace is not None
+
+        context = self.context()
+        context[namespace] = value
+        return value
+
+
 if '__main__' == __name__:
     conf = Configuration(os.path.expanduser('~/mig/server/MiGserver.conf'),
                          True)

tests/test_mig_shared_configuration.py

@@ -1,3 +1,3 @@
 # -*- coding: utf-8 -*-
 #
 # --- BEGIN_HEADER ---
@@ -42,7 +42,7 @@
 
 def _to_dict(obj):
     return {k: v for k, v in inspect.getmembers(obj)
-            if not (k.startswith('__') or _is_method(v))}
+            if not (k.startswith('_') or _is_method(v))}
 
 
 class MigSharedConfiguration(MigTestCase):