diff --git a/.appveyor/install.bat b/.appveyor/install.bat index d009e54d0f..a13ee5affd 100644 --- a/.appveyor/install.bat +++ b/.appveyor/install.bat @@ -3,7 +3,7 @@ for /F "tokens=*" %%g in ('C:\\%WINPYTHON%\\python.exe -c "import sys; print(sys REM use mingw 32 bit until #3291 is resolved set PATH=C:\\%WINPYTHON%;C:\\%WINPYTHON%\\Scripts;C:\\ProgramData\\chocolatey\\bin;C:\\MinGW\\bin;C:\\MinGW\\msys\\1.0\\bin;C:\\cygwin\\bin;C:\\msys64\\usr\\bin;C:\\msys64\\mingw64\\bin;%PATH% C:\\%WINPYTHON%\\python.exe -m pip install -U --progress-bar off pip setuptools wheel -C:\\%WINPYTHON%\\python.exe -m pip install -U --progress-bar off coverage codecov +C:\\%WINPYTHON%\\python.exe -m pip install -U --progress-bar off urllib3 coverage codecov set STATIC_DEPS=true & C:\\%WINPYTHON%\\python.exe -m pip install -U --progress-bar off lxml C:\\%WINPYTHON%\\python.exe -m pip install -U --progress-bar off -r requirements.txt REM install 3rd party tools to test with diff --git a/CHANGES.txt b/CHANGES.txt index d7db09d77b..80c1a5b853 100755 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -56,6 +56,20 @@ RELEASE 4.2.0 - Sat, 31 Jul 2021 18:12:46 -0700 - As part of experimental ninja tool, allow SetOption() to set both disable_execute_ninja and disable_ninja. + From Adam Gross: + - Added support for remote caching. This feature allows for fetch and push of build outputs + to a Bazel remote cache server or any other similar server that supports /ac/ and /cas/ + GET and PUT requests using SHA-256 file names. See https://github.com/buchgr/bazel-remote + for more details on the server. New parameters introduced: + --remote-cache-fetch-enabled: Enables fetch of build output from the server + --remote-cache-push-enabled: Enables push of build output to the server + --remote-cache-url: Required if fetch or push is enabled + --remote-cache-connections: Connection count (defaults to 100) + - Added support for a new parameter --use-scheduler-v2 that opts into a newer, more aggressive + parallel scanner. This scanner avoids waiting on jobs if the job queue is full and instead + scans for tasks. This scanner is expected to improve the performance of your build as long + as you don't have very large actions that cause poor scanning performance. + From David H: - Fix Issue #3906 - `IMPLICIT_COMMAND_DEPENDENCIES` was not properly disabled when set to any string value (For example ['none','false','no','off']) diff --git a/SCons/Job.py b/SCons/Job.py index f87a3bbfe6..bc94b6d20c 100644 --- a/SCons/Job.py +++ b/SCons/Job.py @@ -21,13 +21,14 @@ # OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION # WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -"""Serial and Parallel classes to execute build tasks. +"""Serial, Parallel, and ParallelV2 classes to execute build tasks. The Jobs class provides a higher level interface to start, stop, and wait on jobs. """ import SCons.compat +import SCons.Node import os import signal @@ -64,7 +65,8 @@ class Jobs: methods for starting, stopping, and waiting on all N jobs. """ - def __init__(self, num, taskmaster): + def __init__(self, num, taskmaster, remote_cache=None, + use_scheduler_v2=False): """ Create 'num' jobs using the given taskmaster. @@ -76,19 +78,29 @@ def __init__(self, num, taskmaster): allocated. If more than one job is requested but the Parallel class can't do it, it gets reset to 1. Wrapping interfaces that care should check the value of 'num_jobs' after initialization. + + 'remote_cache' can be set to a RemoteCache.RemoteCache object. + + 'use_scheduler_v2' can be set to True to opt into the newer and more + aggressive scheduler. """ self.job = None - if num > 1: - stack_size = explicit_stack_size - if stack_size is None: - stack_size = default_stack_size - try: + stack_size = explicit_stack_size + if stack_size is None: + stack_size = default_stack_size + + try: + if ((remote_cache and remote_cache.fetch_enabled) or + use_scheduler_v2): + self.job = ParallelV2(taskmaster, num, stack_size, remote_cache) + elif num > 1: self.job = Parallel(taskmaster, num, stack_size) - self.num_jobs = num - except NameError: - pass + self.num_jobs = num + except NameError: + pass + if self.job is None: self.job = Serial(taskmaster) self.num_jobs = 1 @@ -359,7 +371,6 @@ def __init__(self, taskmaster, num, stack_size): self.taskmaster = taskmaster self.interrupted = InterruptState() self.tp = ThreadPool(num, stack_size, self.interrupted) - self.maxjobs = num def start(self): @@ -399,28 +410,199 @@ def start(self): # Let any/all completed tasks finish up before we go # back and put the next batch of tasks on the queue. while True: - task, ok = self.tp.get() + self.process_result() jobs = jobs - 1 - if ok: - task.executed() - else: - if self.interrupted(): - try: - raise SCons.Errors.BuildError( - task.targets[0], errstr=interrupt_msg) - except: - task.exception_set() - - # Let the failed() callback function arrange - # for the build to stop if that's appropriate. - task.failed() + if self.tp.resultsQueue.empty(): + break + + self.tp.cleanup() + self.taskmaster.cleanup() + + def process_result(self): + task, ok = self.tp.get() + + if ok: + task.executed() + else: + if self.interrupted(): + try: + raise SCons.Errors.BuildError( + task.targets[0], errstr=interrupt_msg) + except: + task.exception_set() + + # Let the failed() callback function arrange + # for the build to stop if that's appropriate. + task.failed() + + task.postprocess() + + class ParallelV2(Parallel): + """ + This class is an extension of the Parallel class that provides two main + improvements: + + 1. Minimizes time waiting for jobs by fetching tasks. + 2. Supports remote caching. + """ + __slots__ = ['remote_cache'] + + def __init__(self, taskmaster, num, stack_size, remote_cache): + super(ParallelV2, self).__init__(taskmaster, num, stack_size) + + self.remote_cache = remote_cache + def get_next_task_to_execute(self, limit): + """ + Finds the next task that is ready for execution. If limit is 0, + this function fetches until a task is found ready to execute. + Otherwise, this function will fetch up to "limit" number of tasks. + + Returns tuple with: + 1. Task to execute. + 2. False if a call to next_task returned None, True otherwise. + """ + count = 0 + while limit == 0 or count < limit: + task = self.taskmaster.next_task() + if task is None: + return None, False + + try: + # prepare task for execution + task.prepare() + except: + task.exception_set() + task.failed() task.postprocess() + else: + if task.needs_execute(): + return task, True + else: + task.executed() + task.postprocess() - if self.tp.resultsQueue.empty(): + count = count + 1 + + # We hit the limit of tasks to retrieve. + return None, True + + def start(self): + fetch_response_queue = queue.Queue(0) + if self.remote_cache: + self.remote_cache.set_fetch_response_queue( + fetch_response_queue) + + jobs = 0 + tasks_left = True + pending_fetches = 0 + cache_hits = 0 + cache_misses = 0 + cache_skips = 0 + cache_suspended = 0 + + while True: + fetch_limit = 0 if jobs == 0 and pending_fetches == 0 else 1 + if tasks_left: + task, tasks_left = \ + self.get_next_task_to_execute(fetch_limit) + else: + task = None + + if not task and not tasks_left and jobs == 0 and \ + pending_fetches == 0: + # No tasks left, no jobs, no cache fetches. + break + + while jobs > 0: + # Break if there are no results available and one of the + # following is true: + # 1. There are tasks left. + # 2. There is at least one job slot open and at least one + # remote cache fetch pending. + # Otherwise we want to wait for jobs because the most + # important factor for build speed is keeping the job + # queue full. + if ((tasks_left or + (jobs < self.maxjobs and pending_fetches > 0)) + and self.tp.resultsQueue.empty()): + break + + self.process_result() + jobs = jobs - 1 + + # Tasks could have been unblocked, so we should check + # again. + tasks_left = True + + while pending_fetches > 0: + # Trimming the remote cache fetch queue is the least + # important job, so we only block if there are no responses + # available, no tasks left to fetch, and no active jobs. + if ((tasks_left or jobs > 0) and + fetch_response_queue.empty()): break + cache_task, cache_hit, target_infos = \ + fetch_response_queue.get() + pending_fetches = pending_fetches - 1 + + if cache_hit: + cache_hits = cache_hits + 1 + cache_task.executed(target_infos=target_infos) + cache_task.postprocess() + + # Tasks could have been unblocked, so we should check + # again. + tasks_left = True + else: + cache_misses = cache_misses + 1 + self.tp.put(cache_task) + jobs = jobs + 1 + + if task: + # Tasks should first go to the remote cache if enabled. + if self.remote_cache: + fetch_pending, task_cacheable = \ + self.remote_cache.fetch_task(task) + else: + fetch_pending = task_cacheable = False + + if fetch_pending: + pending_fetches = pending_fetches + 1 + else: + # Fetch is not pending because remote cache is not + # being used or the task was not cacheable. + # + # Count the number of non-cacheable tasks but don't + # count tasks with 1 target that is an alias, because + # they are not actually run. + if (len(task.targets) > 1 or + not isinstance(task.targets[0], + SCons.Node.Alias.Alias)): + if task_cacheable: + cache_suspended = cache_suspended + 1 + else: + cache_skips = cache_skips + 1 + self.tp.put(task) + jobs = jobs + 1 + + # Instruct the remote caching layer to log information about + # the cache hit rate. + cache_count = cache_hits + cache_misses + cache_suspended + task_count = cache_count + cache_skips + if self.remote_cache and task_count > 0: + reset_count = self.remote_cache.reset_count + total_failures = self.remote_cache.total_failure_count + hit_pct = (cache_hits * 100.0 / cache_count if cache_count + else 0.0) + cacheable_pct = cache_count * 100.0 / task_count + self.remote_cache.log_stats( + hit_pct, cache_count, cache_hits, cache_misses, + cache_suspended, cacheable_pct, cache_skips, task_count, + total_failures, reset_count) + self.tp.cleanup() self.taskmaster.cleanup() diff --git a/SCons/Node/FS.py b/SCons/Node/FS.py index 5f05a861bb..74f216e297 100644 --- a/SCons/Node/FS.py +++ b/SCons/Node/FS.py @@ -1118,6 +1118,9 @@ class LocalFS: really need this one? """ + def access(self, path, mode): + return os.access(path, mode) + def chmod(self, path, mode): return os.chmod(path, mode) @@ -3006,20 +3009,10 @@ def push_to_cache(self): if self.exists(): self.get_build_env().get_CacheDir().push(self) - def retrieve_from_cache(self): - """Try to retrieve the node's content from a cache - - This method is called from multiple threads in a parallel build, - so only do thread safe stuff here. Do thread unsafe stuff in - built(). - - Returns true if the node was successfully retrieved. + def should_retrieve_from_cache(self): + """Returns whether this node should be retrieved from the cache """ - if self.nocache: - return None - if not self.is_derived(): - return None - return self.get_build_env().get_CacheDir().retrieve(self) + return not self.nocache and self.is_derived() def visited(self): if self.exists() and self.executor is not None: @@ -3274,7 +3267,7 @@ def builder_set(self, builder): SCons.Node.Node.builder_set(self, builder) self.changed_since_last_build = 5 - def built(self): + def built(self, csig=None, size=0): """Called just after this File node is successfully built. Just like for 'release_target_info' we try to release @@ -3284,7 +3277,7 @@ def built(self): @see: release_target_info """ - SCons.Node.Node.built(self) + SCons.Node.Node.built(self, csig, size) if (not SCons.Node.interactive and not hasattr(self.attributes, 'keep_targetinfo')): diff --git a/SCons/Node/__init__.py b/SCons/Node/__init__.py index ec742a686b..a27d99582f 100644 --- a/SCons/Node/__init__.py +++ b/SCons/Node/__init__.py @@ -681,6 +681,14 @@ def push_to_cache(self): """ pass + def should_retrieve_from_cache(self): + """Returns whether this node should be retrieved from the cache + + By default nodes are not cacheable. Child classes should override this + method if the CacheDir class supports them. + """ + return False + def retrieve_from_cache(self): """Try to retrieve the node's content from a cache @@ -690,7 +698,8 @@ def retrieve_from_cache(self): Returns true if the node was successfully retrieved. """ - return 0 + return (self.should_retrieve_from_cache() and + self.get_build_env().get_CacheDir().retrieve(self)) # # Taskmaster interface subsystem @@ -757,7 +766,7 @@ def build(self, **kw): e.node = self raise - def built(self): + def built(self, csig=None, size=0): """Called just after this node is successfully built.""" # Clear the implicit dependency caches of any Nodes @@ -793,7 +802,15 @@ def built(self): if not self.exists() and do_store_info: SCons.Warnings.warn(SCons.Warnings.TargetNotBuiltWarning, "Cannot find target " + str(self) + " after building") - self.ninfo.update(self) + + # If we already retrieved the NodeInfo from the cache, provide it now. + if csig: + self.ninfo = self.NodeInfo() + self.ninfo.update(self) + self.ninfo.csig = str(csig) + self.ninfo.size = size + else: + self.ninfo.update(self) def visited(self): """Called just after this node has been visited (with or diff --git a/SCons/RemoteCache.py b/SCons/RemoteCache.py new file mode 100644 index 0000000000..ec9bac8b72 --- /dev/null +++ b/SCons/RemoteCache.py @@ -0,0 +1,946 @@ +"""SCons.RemoteCache + +This class owns the logic related to pushing to and fetching from a Bazel +remote cache server. That server stores Task metadata in the /ac/ section +(action cache) and Node binary data in the /cas/ section (content-addressible +storage). + +The Bazel remote cache uses LRU cache eviction for binaries in the +content-addressible storage. This means that Task metadata could exist in the +action cache /ac/ section but one or more of the Node binary data could be +missing due to eviction. Thus, in order for a Task to be fulfilled from cache, +we must verify that the Task metadata exists in the action cache and all Node +binary data exists in the content-addressible storage. + +This class can be configured to do cache fetch, cache push, or both. + +The first step of fetching a Task is to issue a GET request to the /ac/ +portion of the Bazel remote cache server. The server does validation of the +/ac/ request so it should only succeed if (1) the record exists on the server +and (2) the /cas/ storage has not evicted the nodes. + +We use two ThreadPoolExecutor instances. The first instance is responsible for +fetching a task from cache; a request to that executor is done when an entire +task fetch is completed, either resulting in a cache hit or cache miss. The +second instance is response for singular network accesses; a request to that +executor is done when the specified network request is completed. + +The second ThreadPoolExecutor is needed because the urllib3 API is synchronous +and we want multi-target tasks to be fetched in parallel. Requests to the first +ThreadPoolExecutor will wait on requests to the second ThreadPoolExecutor. If +we only used the one ThreadPoolExecutor and that executor was full of task +fetch requests, the cache would hang because there would be no room for the +individual network requests. We avoid that starvation by using the second +ThreadPoolExecutor. + +For security purposes and due to requirements by the Bazel remote cache server, +/ac/ and /cas/ requests always use SHA-256. This requires that SCons uses +SHA-256 content signatures. The security reasons are primarily to avoid +collisions with other actions. + +""" + +# +# Copyright 2020 VMware, Inc. +# +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to +# the following conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY +# KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE +# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +# + +__revision__ = "__FILE__ __REVISION__ __DATE__ __DEVELOPER__" + +import SCons.Node.FS +import SCons.Script +import SCons.Util + +from concurrent.futures import ThreadPoolExecutor + +import datetime +import json +import os +import queue +import random +import stat +import sys +import time +import threading + +try: + import urllib3 + urllib3_exception = None +except ImportError as e: + urllib3_exception = e + + +def raise_if_not_supported(): + # Verify that all required packages are present. + if urllib3_exception: + raise SCons.Errors.UserError( + 'Remote caching was requested but it is not supported in this ' + 'deployment. urllib3 is required but missing: %s' % + urllib3_exception) + + +class RemoteCache(object): + __slots__ = [ + 'backoff_remission_sec', + 'cache_ok_since', + 'connection_pool', + 'current_reset_backoff_multiplier', + 'debug_file', + 'failure_count', + 'fetch_enabled', + 'fetch_enabled_currently', + 'fetch_response_queue', + 'fs', + 'executor', + 'log', + 'metadata_request_timeout_sec', + 'metadata_version', + 'os_platform', + 'push_enabled', + 'push_enabled_currently', + 'request_executor', + 'request_failure_threshold', + 'reset_backoff_multiplier', + 'reset_count', + 'reset_delay_sec', + 'reset_skew_sec', + 'server_address', + 'server_path', + 'sessions', + 'stats_download_requests', + 'stats_download_total_bytes', + 'stats_download_total_ms', + 'stats_metadata_requests', + 'stats_metadata_total_ms', + 'stats_upload_requests', + 'stats_upload_total_bytes', + 'stats_upload_total_ms', + 'stats_lock', + 'total_failure_count', + 'transfer_request_timeout_sec' + ] + + def __init__(self, worker_count, server_address, fetch_enabled, + push_enabled, cache_debug): + """ + Initializes the cache. Supported parameters: + worker_count: Number of threads to create. + server_address: URL of remote server. Only the server name is + required, but the following can also be provided: + scheme, port, and url. + fetch_enabled: True if fetch is enabled. + push_enabled: True if push is enabled. + cache_debug: File to debug log to, '-' for stdout, None otherwise. + """ + self.debug_file = None + self.sessions = {} + self.fetch_enabled = fetch_enabled + self.push_enabled = push_enabled + self.log = SCons.Util.display + self.fs = SCons.Node.FS.default_fs # TODO: Use something better? + self.fetch_response_queue = None + self.failure_count = 0 + self.total_failure_count = 0 + self.cache_ok_since = 0.0 + self.reset_count = 0 + self.fetch_enabled_currently = self.fetch_enabled + self.push_enabled_currently = self.push_enabled + self.current_reset_backoff_multiplier = 1.0 + self.stats_lock = threading.Lock() + self.stats_download_requests = 0 + self.stats_download_total_bytes = 0 + self.stats_download_total_ms = 0 + self.stats_metadata_requests = 0 + self.stats_metadata_total_ms = 0 + self.stats_upload_requests = 0 + self.stats_upload_total_bytes = 0 + self.stats_upload_total_ms = 0 + + if cache_debug == '-': + self.debug_file = sys.stdout + elif cache_debug: + try: + self.debug_file = open(cache_debug, 'w') + except Exception as e: + self.log('WARNING: Unable to open cache debug file "%s" for ' + 'write with exception: %s.' % (cache_debug, e)) + + # This is hardcoded and expected to change if we ever change the + # contents of _get_task_metadata_signature. That allows us to + # continue to add more contents to the metadata over time and + # put it in a different place in the /ac/ section of the remote + # cache. + self.metadata_version = 1 + + # Include the platform in the signature. This is for the case where + # we have python actions that behave differently based on OS (so + # simply hashing the code isn't enough). + # + # Note: before python 3.3, 'linux2' was returned for linux kernels + # >2.6. Now it's just 'linux'. Other platforms (FreeBSD, etc) + # have also added version numbers. For our uses, we just strip + # them all. As a side effect, 'win32' becomes 'win'. + self.os_platform = sys.platform.rstrip('0123456789') + + # Maximum number of failures that we allow before disabling remote + # caching. + self.request_failure_threshold = 10 + + # Per-request timeouts. The first is for /ac/. The second is for /cas/. + self.metadata_request_timeout_sec = 10 + self.transfer_request_timeout_sec = 120 # 2 min + + # Time until we attempt a cache restart (+/- skew) + self.reset_delay_sec = 240.0 # 4 min + self.reset_skew_sec = 60.0 # 1 min + + # Exponential backoff multiplier + self.reset_backoff_multiplier = 1.5 + + # Time without errors before exponential backoff resets + self.backoff_remission_sec = 180.0 # 3 min + + # Generate the server address using the passed in data. + url_info = urllib3.util.parse_url(server_address) + if not url_info or not url_info.host: + raise SCons.Errors.UserError('An invalid remote cache server URL ' + '"%s" was provided' % server_address) + elif (url_info.scheme and + url_info.scheme.lower() not in ['http', 'https']): + raise SCons.Errors.UserError('Remote cache server URL must ' + 'start with http:// or https://') + self.server_address = '%s://%s' % ( + url_info.scheme if url_info.scheme else 'http', url_info.host) + if url_info.port: + self.server_address = '%s:%d' % (self.server_address, + url_info.port) + + # It is also allowable for the URL to contain a path, to which /ac/ and + # /cas/ requests should be appended. + self.server_path = url_info.path.rstrip('/') if url_info.path else '' + + # Create the thread pool and the connection pool that it uses. + # Have urllib3 perform 1 retry per request (its default is 3). 1 retry + # is helpful if in case of an unstable WiFi connection, while not + # taking too long to fail if the server is unreachable. + self.executor = ThreadPoolExecutor(max_workers=worker_count) + self.connection_pool = urllib3.connectionpool.connection_from_url( + self.server_address, maxsize=worker_count, block=True, + retries=1) + self.request_executor = ThreadPoolExecutor(max_workers=worker_count) + + self._debug('Remote cache server is configured as %s and max ' + 'connection count is %d.' % + (self.server_address + self.server_path, worker_count)) + + def _get_node_data_url(self, csig): + """Retrieves the URL for the specified node.""" + return '%s/cas/%s' % (self.server_path, csig) + + def _get_task_cache_url(self, task): + """Retrieves the URL for the specified task's metadata.""" + return '%s/ac/%s' % (self.server_path, + self._get_task_metadata_signature(task)) + + def _get_task_metadata_signature(self, task): + """ + Retrieves the SHA-256 signature that represents the task in the remote + cache. This is used to look up task metadata in the remote cache. + + Important note 1: if you ever change the output of _get_task_metadata, + you must also change metadata_version. + + Important note 2: When SCons supports SHA1 for content signatures, the + "alternative-hash-md5" string will need to be dynamic. + """ + sig_info = [ + 'scons-metadata-version-%d' % self.metadata_version, + 'os-platform=%s' % self.os_platform, + ] + [t.get_cachedir_bsig() for t in task.targets] + return SCons.Util.hash_signature(';'.join(sig_info)) + + def _get_task_metadata(self, task): + """ + Retrieves the JSON metadata that we want to push to the action cache, + dumped to a string and encoded using UTF-8. + + As we are using the Bazel remote cache server, which validates the + entries placed into /ac/, this must match the JSON encoding of Bazel's + ActionResult protobuf message from remote_execution.proto. + """ + output_files = [] + is_posix = os.name == 'posix' + for t in task.targets: + output_file = { + 'path': t.path, + 'digest': { + 'hash': t.get_csig(), + 'sizeBytes': t.get_size(), + }, + } + if is_posix: + output_file['isExecutable'] = t.fs.access(t.abspath, os.X_OK) + output_files += [output_file] + return json.dumps({'outputFiles': output_files}).encode('utf-8') + + def set_fetch_response_queue(self, queue): + """ + Sets the queue used to report cache fetch results if fetching + is enabled. + """ + self.fetch_response_queue = queue + + def fetch_task(self, task): + """ + Dispatches a request to a helper thread to fetch a task from the + remote cache. + + Returns tuple with two booleans: + [0]: True if we submitted the task to the thread pool and False + in all other cases. + [1]: True if the task is cacheable *and* caching was enabled at + some point in the build. False otherwise. + """ + if not (self.fetch_enabled and task.is_cacheable()): + # Caching is not enabled in general or for this task. + return False, False + elif (not self.fetch_enabled_currently and + not self._try_reset_failure()): + # The cache is currently disabled. + return False, True + else: + self.executor.submit(self._fetch, task) + return True, True + + def push_task(self, task): + """ + Dispatches a request to a helper thread to push a task to the + remote cache. + """ + if not self.push_enabled or not task.is_cacheable(): + return + + if self.push_enabled_currently or self._try_reset_failure(): + self.executor.submit(self._push, task) + + def _request_nodes(self, node_tuples): + """ + Makes a GET request to the server for each of the specific Node content + signatures. + + Returns True if all requests succeeded, False otherwise. + """ + if not node_tuples: + return True + + if len(node_tuples) == 1: + # Optimize this common case by not going to the ThreadPoolExecutor. + # That would just cause unnecessary delays due to thread scheduling + # and waiting for locks. + return self._request(node_tuples[0], None) + else: + # We need to make more than one request and doing it serially would + # cause unnecessary delays for high-latency connections. urllib3 is + # a synchronous API so we work around that by doing the requests + # in a helper thread pool. + responses_left = len(node_tuples) + all_files_present = True + result_queue = queue.Queue(0) + + for node_tuple in node_tuples: + self.request_executor.submit(self._request, node_tuple, + result_queue) + + while responses_left > 0: + success = result_queue.get() + all_files_present = \ + all_files_present and success + responses_left -= 1 + + return all_files_present + + def _track_request(self, verb, url, target_path, *args, **kwargs): + """ + Wraps a GET request, tracking response time and optionally logging + details of the response. + + Supported parameters: + verb (String): Verb to request (PUT or GET). + url (String): URL to make the request to. + target_path (String): None if it's a metadata request. Otherwise, + the relative or absolute path for the target + we are putting/getting. + args/kwargs: Params to be passed into connection_pool.request. + """ + start = datetime.datetime.now() + is_metadata_request = target_path is None + + response = None + try: + if verb == 'GET': + response = self.connection_pool.request(verb, url, *args, + **kwargs) + else: + response = self.connection_pool.urlopen(verb, url, *args, + **kwargs) + exception = None + except Exception as e: + exception = e + + if self.debug_file: + with self.stats_lock: + ms = self._get_delta_ms(start) + if is_metadata_request: + self.stats_metadata_requests += 1 + self.stats_metadata_total_ms += ms + elif response is not None and self._success(response): + # /cas/ requests only provide good tracking data if they + # succeeded. Otherwise, the actual uploaded/downloaded size + # is unknown and would skew our averages. + if verb == 'GET': + self.stats_download_requests += 1 + self.stats_download_total_ms += ms + self.stats_download_total_bytes += len(response.data) + else: + self.stats_upload_requests += 1 + self.stats_upload_total_ms += ms + self.stats_upload_total_bytes += len(kwargs['body']) + + ms = self._get_delta_ms(start) + # target_path could be relative or absolute. Convert to relative. + target_log = (('for target %s ' % self.fs.File(target_path).path) + if target_path else '') + + if exception: + self._debug( + '%s FAILED exception %s: URL %s %s(%d ms elapsed).' % + (verb, exception, url, target_log, ms)) + else: + if self._success(response): + self._debug('%s SUCCESS: URL %s %s(%d ms elapsed).' % + (verb, url, target_log, ms)) + else: + self._debug( + '%s FAILED %d (%s): URL %s %s(%d ms elapsed).' % + (verb, response.status, response.reason, url, + target_log, ms)) + + if exception: + raise exception + + return response + + def _validate_file(self, request_data, csig, size, path): + """Validates that the downloaded file data is correct + + This function verifies that request_data matches the expected size in + bytes and SHA-256 content signature. + + Returns True if it was successfully validated, False otherwise. + """ + actual_size = len(request_data) + if actual_size != size: + # Validate the size of the downloaded data. + self.log('Size mismatch downloading file "%s". Expected size %d, ' + 'got %d.' % (path, size, actual_size)) + return False + + # Validate the hash of the downloaded data. We only use SHA-256 in + # so we only need to check for that. + csig_length = len(csig) + if csig_length == 64: + actual_csig = SCons.Util.hash_signature(request_data) + else: + self.log('WARNING: Not validating csig %s (path "%s") because ' + 'hash length %d is of an unknown format.' % + (csig, path, csig_length)) + actual_csig = None + + if actual_csig is not None and actual_csig != csig: + self.log('Hash mismatch downloading file "%s". Expected hash ' + '%s, but got %s.' % (path, csig, actual_csig)) + return False + + return True + + def _request(self, node_tuple, result_queue): + """ + Implementation of sending a GET request to the remote cache for a + specific node. + + This function can be called multiple times from different threads + in the ThreadPoolExecutor, so it must be thread-safe. + + Params: + node_tuple (tuple): Tuple of node absolute path, csig, size, and + is_executable. is_executable is only used for + GET requests on Posix systems. + result_queue (Queue): Optional queue to post Boolean result to. + + Returns True if the request succeeded, False otherwise. + """ + (path, csig, size, is_executable) = node_tuple + + try: + # XXX TODO: Stream responses to GET requests by passing + # preload_content=False. + response = self._track_request( + 'GET', self._get_node_data_url(csig), path, + timeout=self.transfer_request_timeout_sec) + + if self._success(response): + # Validate that the data has the correct signature and size. + success = self._validate_file(response.data, csig, size, path) + if success: + with open(path, 'wb') as f: + f.write(response.data) + + if os.name == 'posix' and is_executable: + self.fs.chmod( + path, self.fs.stat(path).st_mode | + stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH) + + if result_queue: + result_queue.put(success) + return success + except urllib3.exceptions.HTTPError as e: + # Host is not available. Immediately disable caching. + self.log('GET request failed for file %s with exception: %s' % + (path, e)) + self._handle_failure() + except Exception as e: + import traceback + self.log('Unexpected exception making GET request for file %s: ' + '%s, %s' % (path, e, traceback.format_exc())) + self._handle_failure() + + if result_queue: + result_queue.put(False) + return False + + def _success(self, response): + """Wraps some logic to determine whether the request succeeded.""" + return response.status < 400 or response.status >= 600 + + def _get_nodes_from_task_metadata(self, task_metadata): + """Retrieves node tuples from the task metadata. + + Returns a list of tuples with the following entries: + 1. Absolute path to the file. + 2. File content signature. + 3. Size in bytes. + 4. True if the file should be marked as executable on Posix, False + otherwise. + """ + nodes = [] + for output_file in task_metadata['outputFiles']: + path = output_file.get('path', None) + if path: + path = self.fs.File(path).abspath + digest = output_file.get('digest', None) + csig = digest.get('hash', None) + size = int(digest.get('sizeBytes', '0')) + is_executable = output_file.get('isExecutable', False) + + if path and digest and csig: + nodes.append((path, csig, size, is_executable)) + return nodes + + def _get_delta_ms(self, start): + """ + Helper function to return the difference in milliseconds between now + and the specified start time. + """ + delta = datetime.datetime.now() - start + return delta.total_seconds() * 1000 + + def _fetch(self, task): + """ + Implementation of fetching a task from a remote cache. + + This function can be called multiple times from different threads + in the ThreadPoolExecutor, so it must be thread-safe. + """ + try: + url = self._get_task_cache_url(task) + + # Fetch the task metadata from the server, if it exists. + # Catch JSON decoding errors in case we received a partial + # response from the server or the stored cache data is incomplete. + response = self._track_request( + 'GET', url, None, + headers={'Accept': 'application/json'}, + timeout=self.metadata_request_timeout_sec) + action_result = None + if self._success(response): + try: + action_result = json.loads(response.data.decode('utf-8')) + except json.JSONDecodeError as e: + if self.debug_file: + self._debug('Cache miss due to bad JSON data in the ' + 'action cache for task with url %s, ' + 'targets %s. Exception: %s' % + (url, [str(t) for t in task.targets], e)) + except Exception as e: + self.log('Received unexpected exception %s when trying to ' + 'decode JSON data for task with url %s, targets ' + '%s' % (e, url, [str(t) for t in task.targets])) + + if action_result: + nodes = self._get_nodes_from_task_metadata(action_result) + + if not nodes: + # The task metadata couldn't be processed. + if self.debug_file: + self._debug('Cache miss for task with url %s, targets ' + '%s due to bad /ac/ record.' % + (url, [str(t) for t in task.targets])) + elif self._request_nodes(nodes): + # All downloads were successful. The main python thread + # marks nodes as built, so we must deliver csig and size + # info in a format expected by the + # Task.executed_with_callbacks function. + target_infos = [(csig, size) for _, csig, size, _ in nodes] + + # Mark each target as cached so it isn't unnecessarily + # pushed back to cache. + for t in task.targets: + t.cached = 1 + + # Optionally print information about the cache hit. + if self.debug_file: + self._debug_cache_result(True, task, url) + + # Optionally push the result to the response queue. + if self.fetch_response_queue: + self.fetch_response_queue.put((task, True, + target_infos)) + return True + elif self.debug_file: + self._debug('Cache miss for task with url %s, targets ' + '%s due to missing /cas/ record.' % + (url, [str(t) for t in task.targets])) + elif self.debug_file: + self._debug_cache_result(False, task, url) + except urllib3.exceptions.HTTPError as e: + # Host could not be reached. + self.log('GET request failed for task (targets=%s) with ' + 'exception: %s' % ([str(t) for t in task.targets], e)) + self._handle_failure() + except Exception as e: + import traceback + self.log('Unexpected exception fetching task (targets=%s): ' + '%s, %s' % + ([str(t) for t in task.targets], e, + traceback.format_exc())) + self._handle_failure() + + # This is the failure path. Unlink any retrieved files and indicate + # failure. + for t in task.targets: + if t.fs.exists(t.abspath): + t.fs.unlink(t.abspath) + + # Push the result to the response queue. + if self.fetch_response_queue: + self.fetch_response_queue.put((task, False, None)) + else: + self.log('Unexpectedly unable to find a response queue to ' + 'push the remote cache fetch result to.') + + def _push(self, task): + """ + Implementation of pushing a task to a remote cache. + + This function can be called multiple times from different threads + in the ThreadPoolExecutor, so it must be thread-safe. + + :param task: Task instance to push. + """ + try: + # Push each node first then push the task info. + for t in task.targets: + response = self._track_request( + 'PUT', self._get_node_data_url(t.get_csig()), t.path, + body=t.get_contents(), + timeout=self.transfer_request_timeout_sec) + + if not self._success(response): + # If the write failed, don't continue, because we don't + # want to write the metadata. + return + + # All target writes succeeded, so write the metadata. + response = self._track_request( + 'PUT', self._get_task_cache_url(task), None, + body=self._get_task_metadata(task), + headers={'Content-Type': 'application/json'}, + timeout=self.metadata_request_timeout_sec) + + except urllib3.exceptions.HTTPError as e: + # Host is not available. + self.log('PUT request failed for task (targets=%s) with ' + 'exception: %s' % ([str(t) for t in task.targets], e)) + self._handle_failure() + except Exception as e: + import traceback + self.log('Unexpected exception pushing task (targets=%s): ' + '%s, %s' % + ([str(t) for t in task.targets], e, + traceback.format_exc())) + self._handle_failure() + + def close(self): + """Releases any resources that this class acquired.""" + if self.executor is not None: + # Async fetches shouldn't still be pending at this point, but + # async pushes could. + start = datetime.datetime.now() + self.executor.shutdown(wait=True) + if self.debug_file: + # Log a debug message if shutting down the network request + # executor took a while. This happens if there are large + # cache pushes at the end of the build. + ms = self._get_delta_ms(start) + if ms > 100: + self._debug('Shutting down took %d ms.' % ms) + + self.executor = None + + for _, session in self.sessions.items(): + session.close() + self.sessions.clear() + + if self.debug_file not in [None, sys.stdout]: + self.debug_file.close() + self.debug_file = None + + def _get_monotonic_now(self): + """ """ + if sys.version_info[:2] >= (3, 3): + return time.monotonic() + else: + return time.clock() + + def _handle_failure(self): + """Disables remote caching if there were too many failures.""" + self.failure_count += 1 + self.total_failure_count += 1 + + now = self._get_monotonic_now() + cache_ok_since = self.cache_ok_since + + if (self.failure_count > self.request_failure_threshold and + (self.push_enabled_currently or + self.fetch_enabled_currently)): + + cache_ok_duration = now - cache_ok_since + + # Note: backoff_remission_sec above is not scaled by the + # multiplier today. We may revisit that design choice later. + if (self.reset_count and + (not self.backoff_remission_sec or + cache_ok_duration < self.backoff_remission_sec)): + # This is an exponential backoff scenario. + backoff_multiplier = ( + self.current_reset_backoff_multiplier * + self.reset_backoff_multiplier) + else: + # Either this is the first failure or the enough time + # passed to reset the exponential backoff. + backoff_multiplier = 1.0 + + reset_text = 'RemoteCache: Request failure threshold was reached.' + + if self.reset_delay_sec: + # If the server is down, we don't want everybody trying to + # hit it all at once, so add some randomness. + # random.randint() requires that reset_skew_sec be an int. + reset_skew_sec = int(self.reset_skew_sec * backoff_multiplier) + reset_delay_sec = self.reset_delay_sec * backoff_multiplier + + skew = random.randint(-reset_skew_sec, reset_skew_sec) + delay = reset_delay_sec + skew + + # Set cache_ok_since into the future. + self.cache_ok_since = now + delay + + reset_text += (' Suspending remote caching, attempting ' + 'restart in %2.1f seconds.' % delay) + else: + reset_text += ' Disabling remote caching.' + + self.log(reset_text) + self.current_reset_backoff_multiplier = backoff_multiplier + self.push_enabled_currently = False + self.fetch_enabled_currently = False + elif cache_ok_since < now: + # The cache has behaved unhealthy up until this point. + # Note: cache_ok_since points into the future when the cache is + # disabled and we're waiting for a reset. + self.cache_ok_since = now + + def _try_reset_failure(self): + """ + Attempts to restart the cache fetch and push logic if the appropriate + amount of time has passed. Exponenial backoff is implemented as well. + :return: False is the reset deadline hasn't passed yet. + """ + if not self.reset_delay_sec: + return False + + now = self._get_monotonic_now() + + # When the cache is suspended, cache_ok_since is set into the future. + # once we reach that point, we re-enable it. + if now < self.cache_ok_since: + return False + + self.log('RemoteCache: Resuming remote caching attempts.') + self.reset_count += 1 + self.failure_count = 0 + self.fetch_enabled_currently = self.fetch_enabled + self.push_enabled_currently = self.push_enabled + return True + + def _debug(self, msg): + """ + Prints a debug message if --cache-debug was set. + Caller is responsible for checking that self.debug_file is not None. + """ + if self.debug_file: + self.debug_file.write(msg + '\n') + + def _debug_cache_result(self, hit, task, url): + """ + Prints the result of a task lookup to debug_file. + Caller is responsible for checking that self.debug_file is not None. + """ + t = task.targets[0] + executor = t.get_executor() + try: + # Print the raw executor contents without any subst calls. This + # makes it easier to compare hit and miss records across builds. + # Avoid using executor.get_contents() because it uses ''.join() + # and doesn't print well. + actions = [action.get_contents(executor.get_all_targets(), + executor.get_all_sources(), + executor.get_build_env()).decode() + for action in executor.get_action_list()] + except Exception: + # Actions that run Python functions fail because they can't + # be printed raw. Instead, convert directly to string. + actions = [str(executor)] + + def format_container(container): + """ + Returns the contents of the container, preserving space by putting + consecutive entries from the same directory on the same line. It is + extremely important that order is maintained in the outputted list, + so this function cannot do any sorting. This also means that the + same directory may show up multiple times in the returned string + if all files from that directory are not adjacent in the + container. + """ + lastdir = None + result = '' + for c in container: + dir = getattr(c, 'dir', None) + name = getattr(c, 'name', '') + # Note: this code uses concatenation instead of % formatting + # for better performance. + if dir is None: + result += '\n\t' + str(c) + elif dir != lastdir: + result += '\n\t' + str(dir) + ': ' + str(name) + else: + result += ', ' + str(name) + lastdir = dir + return result if result else ' None' + + # To be cacheable, there needs to be at least one file target, which + # means that we expect sources, depends, and implicit to be set. + # However, ignore may not be set. + self._debug('Cache %s for task with url %s.\nTargets:%s\n' + 'Sources:%s\nDepends:%s\nImplicit:%s\nIgnore:%s\n' + 'Actions:\n\t%s' % + ('hit' if hit else 'miss', url, + format_container(task.targets), + format_container(t.sources), + format_container(t.depends), + format_container(t.implicit), + format_container(getattr(t, 'ignore', [])), + '\n\t'.join(actions))) + + def _print_operation_stats(self, operation, requests, bytes, ms): + """Prints statistics about the specified operation.""" + if requests > 0: + mb = float(bytes) / 1048576 + seconds = float(ms) / 1000 + self._debug('%s stats: %d requests on %d MB took %d ms. Average ' + '%2.2f ms per MB and %2.2f MB per second.' % + (operation, requests, mb, ms, + 0 if mb == 0 else float(ms) / mb, + 0 if seconds == 0 else mb / seconds)) + + def log_stats(self, hit_pct, cache_count, cache_hits, cache_misses, + cache_suspended, cacheable_pct, cache_skips, task_count, + total_failures, reset_count): + """ + Prints the statistics that were collected during a build. + """ + message = ( + '%2.1f percent cache hit rate on %d cacheable tasks with %d hits, ' + '%d misses, %d w/cache suspended. %2.1f percent of total tasks ' + 'cacheable, due to %d/%d tasks marked not cacheable. Saw %d total ' + 'failures, %d cache restarts.' % + (hit_pct, cache_count, cache_hits, cache_misses, + cache_suspended, + cacheable_pct, cache_skips, task_count, + total_failures, reset_count) + ) + self.log('RemoteCache: %s' % message) + + if self.debug_file: + self._debug(message) + + if self.stats_metadata_requests > 0: + self._debug('Task metadata stats: %d ms average RTT on %d ' + 'requests.' % + (self.stats_metadata_total_ms / + self.stats_metadata_requests, + self.stats_metadata_requests)) + + # The upload and download stats use the same log format. + self._print_operation_stats('Download', + self.stats_download_requests, + self.stats_download_total_bytes, + self.stats_download_total_ms) + self._print_operation_stats('Upload', + self.stats_upload_requests, + self.stats_upload_total_bytes, + self.stats_upload_total_ms) + +# Local Variables: +# tab-width:4 +# indent-tabs-mode:nil +# End: +# vim: set expandtab tabstop=4 shiftwidth=4: diff --git a/SCons/RemoteCacheTests.py b/SCons/RemoteCacheTests.py new file mode 100644 index 0000000000..964ea18b0f --- /dev/null +++ b/SCons/RemoteCacheTests.py @@ -0,0 +1,354 @@ +# MIT License +# +# Copyright 2020 VMware, Inc. +# +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to +# the following conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY +# KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE +# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +# + +from hashlib import sha256 +import json +import os +import queue +import stat +import unittest +from unittest import mock + +import TestSCons + +import SCons.RemoteCache + +# If the Python version running the tests doesn't have urllib3, then +# RemoteCache.py will not have the urllib3 attribute. However, the various +# mock.patch decorators below depend upon the attribute existing. +if not hasattr(SCons.RemoteCache, 'urllib3'): + SCons.RemoteCache.urllib3 = None + + +class Url(): + """Test version of the urllib3's Url class.""" + __slots__ = ['host', 'scheme', 'port', 'path'] + + def __init__(self): + self.host = None + self.scheme = None + self.port = 0 + self.path = None + + +def MockParseUrl(address): + """Naive implementation of URL parsing. Just enough for tests.""" + url = Url() + components = address.split('://', 1) + if len(components) > 1: + if components[0]: + url.scheme = components[0] + else: + # Invalid URL starting with :// + return None + + components = components[-1].split('/', 1) + port_components = components[0].split(':', 1) + url.host = port_components[0] + if len(port_components) > 1: + url.port = int(port_components[1]) + else: + url.port = 443 if url.scheme == 'https' else 80 + if len(components) > 1: + url.path = components[1] + return url + + +class MockResponse(mock.MagicMock): + """Mock version of a response class that comes from urllib3.response.""" + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + + for key, val in kwargs.items(): + setattr(self, key, val) + +class MockConnectionPool(mock.MagicMock): + """Mock version of the urllib3.ConnectionPool class.""" + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + + self.pending_responses = [] + + def add_pending_response(self, **kwargs): + self.pending_responses.append(MockResponse(**kwargs)) + + def request(self, verb, url, *args, **kwargs): + return self.pending_responses.pop(0) + + def urlopen(self, verb, url, *args, **kwargs): + return self.pending_responses.pop(0) + + +def MockConnectionFromUrl(*args, **kwargs): + """ + Mock side effect for function urllib3.ConnectionPool.connection_from_url. + Returns a mock version of urllib3.ConnectionPool. + """ + return MockConnectionPool(*args, **kwargs) + + +class Task(): + """Test version of the Task class.""" + slots = ['targets', 'cacheable'] + + def __init__(self, targets, cacheable): + self.targets = targets + self.cacheable = cacheable + + def is_cacheable(self): + return self.cacheable + + +def CreateRemoteCache(mock_urllib3, worker_count, server_address, + fetch_enabled, push_enabled): + """ + Creates an instance of the RemoteCache class using various mock classes. + """ + # Initialize all mocks. + mock_urllib3.util.parse_url = mock.MagicMock(side_effect=MockParseUrl) + mock_urllib3.connectionpool.connection_from_url = mock.MagicMock( + side_effect=MockConnectionFromUrl) + + # Create the remote cache and queue. + cache = SCons.RemoteCache.RemoteCache( + worker_count, server_address, fetch_enabled, push_enabled, None) + mock_urllib3.util.parse_url.assert_called_with(server_address) + assert cache is not None, cache + q = queue.Queue(0) + cache.set_fetch_response_queue(q) + + return cache, q + + +def GetJSONMetadata(files): + """ + Retrieves the JSON metadata for the specified files. Files can either be a + single tuple or a list of tuples. Each tuple should have the following: + + 1. Filename. + 2. Contents. + 3. Hash. + 4. (Optional) True if the file is executable on posix. Default is False. + """ + if isinstance(files, tuple): + files = [files] + + output_files = [] + for file in files: + if len(file) == 4: + (filename, contents, hash, isExecutable) = file + else: + (filename, contents, hash) = file + isExecutable = False + + output_file = { + 'path': filename, + 'digest': { + 'hash': hash, + 'sizeBytes': len(contents), + }, + } + if os.name == 'posix': + output_file['isExecutable'] = isExecutable + output_files.append(output_file) + + return json.dumps({'outputFiles': output_files}).encode() + + +class TestCaseBase(unittest.TestCase): + """Base test case that does common test setup.""" + def setUp(self): + SCons.Util.set_hash_format('sha256') + self.test = TestSCons.TestSCons() + self.env = self.test.Environment() + self.name = self.__class__.__name__ + + +class FetchOneTargetTestCase(TestCaseBase): + """Fetches one task from cache.""" + @mock.patch('SCons.RemoteCache.urllib3') + def runTest(self, mock_urllib3): + cache, q = CreateRemoteCache(mock_urllib3, 4, 'test', True, False) + + # Test making a request. + contents = b'a' + hash = sha256(contents).hexdigest() + data = GetJSONMetadata((self.name, contents, hash)) + cache.connection_pool.add_pending_response(status=200, data=data) + cache.connection_pool.add_pending_response(status=200, data=contents) + + t = self.env.File(self.name) + task = Task([t], True) + fetch_result = cache.fetch_task(task) + assert fetch_result == (True, True), fetch_result + + task2, hit, target_infos = q.get() + assert task == task2 + assert t.cached + assert t.get_contents() == contents, t.get_contents() + assert t.get_csig() == hash, t.get_csig() + assert hit + assert target_infos == [(hash, len(contents))] + assert not cache.connection_pool.pending_responses, \ + cache.connection_pool.pending_responses + + +class FetchMultipleTargetsTestCase(TestCaseBase): + """ + Tests fetching a task with multiple targets. This will result in /ac/ fetch + and multiple /cas/ fetches. + """ + @mock.patch('SCons.RemoteCache.urllib3') + def runTest(self, mock_urllib3): + cache, q = CreateRemoteCache(mock_urllib3, 4, 'test', True, False) + + # Test making a request. + filename1 = self.name + contents1 = b'a' + filename2 = filename1 + '2' + contents2 = b'b' + hash1 = sha256(contents1).hexdigest() + hash2 = sha256(contents2).hexdigest() + data = GetJSONMetadata([(filename1, contents1, hash1), + (filename2, contents2, hash2)]) + cache.connection_pool.add_pending_response(status=200, data=data) + cache.connection_pool.add_pending_response(status=200, data=contents1) + cache.connection_pool.add_pending_response(status=200, data=contents2) + + t1 = self.env.File(filename1) + t2 = self.env.File(filename2) + task = Task([t1, t2], True) + fetch_result = cache.fetch_task(task) + assert fetch_result == (True, True), fetch_result + + task2, hit, target_infos = q.get() + assert task == task2 + assert t1.cached + assert t1.get_contents() == contents1, t1.get_contents() + assert t1.get_csig() == hash1, t1.get_csig() + assert t2.cached + assert t2.get_contents() == contents2, t2.get_contents() + assert t2.get_csig() == hash2, t2.get_csig() + assert hit + assert target_infos == [(hash1, len(contents1)), + (hash2, len(contents2))] + assert not cache.connection_pool.pending_responses, \ + cache.connection_pool.pending_responses + + +class PushMultipleTargetsTestCase(TestCaseBase): + """ + Tests pushing a task with multiple targets. This will result in /cas/ + pushes and one /ac/ push. + """ + @mock.patch('SCons.RemoteCache.urllib3') + def runTest(self, mock_urllib3): + cache, q = CreateRemoteCache(mock_urllib3, 4, 'test', False, True) + + # Push two targets. If it's a posix machine, mark one of them as + # executable. + targets = [] + file_infos = [(self.name, b'a', sha256(b'a').hexdigest(), True), + (self.name + '2', b'b', sha256(b'b').hexdigest(), False)] + for name, contents, _, executable in file_infos: + t = self.env.File(name) + with open(t.abspath, 'w') as f: + f.write(contents.decode()) + + if os.name == 'posix' and executable: + self.env.fs.chmod( + t.abspath, + stat.S_IXUSR | self.env.fs.stat(t.abspath)[stat.ST_MODE]) + + targets.append(t) + + data = GetJSONMetadata(file_infos) + cache.connection_pool.add_pending_response(status=200, data=file_infos[0][1]) + cache.connection_pool.add_pending_response(status=200, data=file_infos[1][1]) + cache.connection_pool.add_pending_response(status=200, data=data) + + task = Task(targets, True) + cache.push_task(task) + connection_pool = cache.connection_pool + cache.close() # This will wait for ThreadPool requests to complete. + assert not connection_pool.pending_responses, \ + connection_pool.pending_responses + + +class FetchMetadataFailureTestCase(TestCaseBase): + """Tests a failed /ac/ request.""" + @mock.patch('SCons.RemoteCache.urllib3') + def runTest(self, mock_urllib3): + cache, q = CreateRemoteCache(mock_urllib3, 4, 'test', True, False) + + cache.connection_pool.add_pending_response(status=404) + + t = self.env.File(self.name) + task = Task([t], True) + fetch_result = cache.fetch_task(task) + assert fetch_result == (True, True), fetch_result + + task2, hit, target_infos = q.get() + assert task == task2 + assert not t.cached + assert not hit + assert target_infos is None + assert not cache.connection_pool.pending_responses, \ + cache.connection_pool.pending_responses + + +class FetchNodeContentsFailureTestCase(TestCaseBase): + """Tests a successful /ac/ request but failed /cas/ request.""" + @mock.patch('SCons.RemoteCache.urllib3') + def runTest(self, mock_urllib3): + cache, q = CreateRemoteCache(mock_urllib3, 4, 'test', True, False) + + contents = b'a' + hash = sha256(contents).hexdigest() + data = GetJSONMetadata((self.name, contents, hash)) + cache.connection_pool.add_pending_response(status=200, data=data) + cache.connection_pool.add_pending_response(status=404) + + t = self.env.File(self.name) + task = Task([t], True) + fetch_result = cache.fetch_task(task) + assert fetch_result == (True, True), fetch_result + + task2, hit, target_infos = q.get() + assert task == task2 + assert not t.cached + assert not hit + assert target_infos is None + assert not cache.connection_pool.pending_responses, \ + cache.connection_pool.pending_responses + + +if __name__ == "__main__": + unittest.main() + +# Local Variables: +# tab-width:4 +# indent-tabs-mode:nil +# End: +# vim: set expandtab tabstop=4 shiftwidth=4: diff --git a/SCons/Script/Main.py b/SCons/Script/Main.py index 9aaa289db0..51baad60d4 100644 --- a/SCons/Script/Main.py +++ b/SCons/Script/Main.py @@ -57,6 +57,7 @@ import SCons.Node.FS import SCons.Platform import SCons.Platform.virtualenv +import SCons.RemoteCache import SCons.SConf import SCons.Script import SCons.Taskmaster @@ -226,7 +227,7 @@ def do_failed(self, status=2): exit_status = status this_build_status = status - def executed(self): + def executed(self, target_infos=None): t = self.targets[0] if self.top and not t.has_builder() and not t.side_effect: if not t.exists(): @@ -247,9 +248,11 @@ def executed(self): self.do_failed() else: print("scons: Nothing to be done for `%s'." % t) - SCons.Taskmaster.OutOfDateTask.executed(self) + SCons.Taskmaster.OutOfDateTask.executed( + self, target_infos=target_infos) else: - SCons.Taskmaster.OutOfDateTask.executed(self) + SCons.Taskmaster.OutOfDateTask.executed( + self, target_infos=target_infos) def failed(self): # Handle the failure of a build task. The primary purpose here @@ -423,7 +426,7 @@ def execute(self): this_build_status = 1 self.tm.stop() - def executed(self): + def executed(self, target_infos=None): pass @@ -1118,7 +1121,12 @@ def _main(parser): # Hash format and chunksize are set late to support SetOption being called # in a SConscript or SConstruct file. - SCons.Util.set_hash_format(options.hash_format) + # Remote caching requires SHA-256. + if (options.remote_cache_fetch_enabled or + options.remote_cache_push_enabled): + SCons.Util.set_hash_format('sha256') + else: + SCons.Util.set_hash_format(options.hash_format) if options.md5_chunksize: SCons.Node.FS.File.hash_chunksize = options.md5_chunksize * 1024 @@ -1152,7 +1160,8 @@ def _build_targets(fs, options, targets, target_top): if options.diskcheck: SCons.Node.FS.set_diskcheck(options.diskcheck) - SCons.CacheDir.cache_enabled = not options.cache_disable + SCons.CacheDir.cache_enabled = (not options.cache_disable and + not options.remote_cache_url) SCons.CacheDir.cache_readonly = options.cache_readonly SCons.CacheDir.cache_debug = options.cache_debug SCons.CacheDir.cache_force = options.cache_force @@ -1271,6 +1280,22 @@ def order(dependencies): """Leave the order of dependencies alone.""" return dependencies + if options.remote_cache_fetch_enabled or options.remote_cache_push_enabled: + if not options.remote_cache_url: + raise Exception('--remote-cache-url is required when remote ' + 'caching is enabled.') + + SCons.RemoteCache.raise_if_not_supported() + + remote_cache = SCons.RemoteCache.RemoteCache( + options.remote_cache_connections, + options.remote_cache_url, + options.remote_cache_fetch_enabled, + options.remote_cache_push_enabled, + options.cache_debug) + else: + remote_cache = None + def tmtrace_cleanup(tfile): tfile.close() @@ -1281,7 +1306,8 @@ def tmtrace_cleanup(tfile): atexit.register(tmtrace_cleanup, tmtrace) else: tmtrace = None - taskmaster = SCons.Taskmaster.Taskmaster(nodes, task_class, order, tmtrace) + taskmaster = SCons.Taskmaster.Taskmaster(nodes, task_class, order, tmtrace, + remote_cache) # Let the BuildTask objects get at the options to respond to the # various print_* settings, tree_printer list, etc. @@ -1301,7 +1327,8 @@ def tmtrace_cleanup(tfile): # to check if python configured with threads. global num_jobs num_jobs = options.num_jobs - jobs = SCons.Job.Jobs(num_jobs, taskmaster) + jobs = SCons.Job.Jobs(num_jobs, taskmaster, remote_cache, + options.use_scheduler_v2) if num_jobs > 1: msg = None if jobs.num_jobs == 1 or not python_has_threads: diff --git a/SCons/Script/SConsOptions.py b/SCons/Script/SConsOptions.py index d38380b882..fb0a1b9a51 100644 --- a/SCons/Script/SConsOptions.py +++ b/SCons/Script/SConsOptions.py @@ -141,8 +141,13 @@ def __getattr__(self, attr): 'no_progress', 'num_jobs', 'random', + 'remote_cache_connections', + 'remote_cache_fetch_enabled', + 'remote_cache_push_enabled', + 'remote_cache_url', 'silent', 'stack_size', + 'use_scheduler_v2', 'warn', # TODO: Remove these once we update the AddOption() API to allow setting @@ -898,6 +903,27 @@ def opt_implicit_deps(option, opt, value, parser): action="store_true", help="Build dependencies in random order.") + op.add_option('--remote-cache-connections', + dest='remote_cache_connections', default=100, + action='store', nargs=1, type='int', + help='Allow N connections to the server.', + metavar='N') + + op.add_option('--remote-cache-fetch-enabled', + dest='remote_cache_fetch_enabled', default=False, + action='store_true', + help='Whether to fetch nodes from the remote cache') + + op.add_option('--remote-cache-push-enabled', + dest='remote_cache_push_enabled', default=False, + action='store_true', + help='Whether to push nodes to the remote cache') + + op.add_option('--remote-cache-url', + dest='remote_cache_url', default='', + action='store', nargs=1, + help='Server URL for remote caching.') + op.add_option('-s', '--silent', '--quiet', dest="silent", default=False, action="store_true", @@ -965,6 +991,12 @@ def opt_tree(option, opt, value, parser, tree_options=tree_options): help="Search up directory tree for SConstruct, " "build Default() targets from local SConscript.") + op.add_option('--use-scheduler-v2', + dest='use_scheduler_v2', default=False, + action='store_true', + help='Whether to use the more aggressive Parallel scheduler ' + 'on a multi-CPU build.') + def opt_version(option, opt, value, parser): sys.stdout.write(parser.version + '\n') sys.exit(0) diff --git a/SCons/Taskmaster.py b/SCons/Taskmaster.py index d57179545b..bceb78960f 100644 --- a/SCons/Taskmaster.py +++ b/SCons/Taskmaster.py @@ -221,21 +221,8 @@ def execute(self): if not t.retrieve_from_cache(): break cached_targets.append(t) - if len(cached_targets) < len(self.targets): - # Remove targets before building. It's possible that we - # partially retrieved targets from the cache, leaving - # them in read-only mode. That might cause the command - # to fail. - # - for t in cached_targets: - try: - t.fs.unlink(t.get_internal_path()) - except (IOError, OSError): - pass + if not self.process_cached_targets(cached_targets): self.targets[0].build() - else: - for t in cached_targets: - t.cached = 1 except SystemExit: exc_value = sys.exc_info()[1] raise SCons.Errors.ExplicitExit(self.targets[0], exc_value.code) @@ -249,11 +236,37 @@ def execute(self): buildError.exc_info = sys.exc_info() raise buildError - def executed_without_callbacks(self): + def process_cached_targets(self, cached_targets): + """ + Processes the list of cached targets, updating the task based on + whether all targets were retrieved from cache. + Returns True if all targets were retrieved from cache, False otherwise. + """ + if len(cached_targets) < len(self.targets): + # Remove targets before building. It's possible that we + # partially retrieved targets from the cache, leaving + # them in read-only mode. That might cause the command + # to fail. + # + for t in cached_targets: + try: + t.fs.unlink(t.get_internal_path()) + except (IOError, OSError): + pass + return False + else: + for t in cached_targets: + t.cached = 1 + return True + + def executed_without_callbacks(self, target_infos=None): """ Called when the task has been successfully executed and the Taskmaster instance doesn't want to call the Node's callback methods. + + target_infos is unused. See executed_with_callbacks + documentation for its contents and purpose. """ T = self.tm.trace if T: T.write(self.trace_message('Task.executed_without_callbacks()', @@ -265,7 +278,7 @@ def executed_without_callbacks(self): side_effect.set_state(NODE_NO_STATE) t.set_state(NODE_EXECUTED) - def executed_with_callbacks(self): + def executed_with_callbacks(self, target_infos=None): """ Called when the task has been successfully executed and the Taskmaster instance wants to call the Node's callback @@ -277,12 +290,25 @@ def executed_with_callbacks(self): In any event, we always call "visited()", which will handle any post-visit actions that must take place regardless of whether or not the target was an actual built target or a source Node. + + target_infos is optional. When provided, it is expected to be a list of + the same length as self.targets. Each list entry should be a tuple with + three entries: (1) the target file csig, (2) the target file sha256 + csig, and (3) the target file size. It is expected to be in the same + order as self.targets. """ global print_prepare T = self.tm.trace if T: T.write(self.trace_message('Task.executed_with_callbacks()', self.node)) + if target_infos and len(target_infos) != len(self.targets): + raise Exception('executed_with_callbacks: Unexpected contents of ' + 'target_infos. Expected %d infos, got %d.' % + (len(self.targets), len(target_infos))) + + changed = False + target_infos_index = 0 for t in self.targets: if t.get_state() == NODE_EXECUTING: for side_effect in t.side_effects: @@ -290,7 +316,15 @@ def executed_with_callbacks(self): t.set_state(NODE_EXECUTED) if not t.cached: t.push_to_cache() - t.built() + changed = True + if target_infos: + # This was a remote cache hit, so we already have the node + # size and csig. Avoid unnecessary I/O by providing it now. + (csig, size) = target_infos[target_infos_index] + target_infos_index = target_infos_index + 1 + t.built(csig, size) + else: + t.built() t.visited() if (not print_prepare and (not hasattr(self, 'options') or not self.options.debug_includes)): @@ -298,6 +332,13 @@ def executed_with_callbacks(self): else: t.visited() + # Push the entire task to remote cache now that all targets have been + # marked as built. That clears memoizations, which allows us to + # properly retrieve csigs. + if (changed and self.tm.remote_cache and + self.tm.remote_cache.push_enabled): + self.tm.remote_cache.push_task(self) + executed = executed_with_callbacks def failed(self): @@ -546,6 +587,12 @@ def _exception_raise(self): # raise e.__class__, e.__class__(e), sys.exc_info()[2] # exec("raise exc_type(exc_value).with_traceback(exc_traceback)") + def is_cacheable(self): + """Checks whether all targets for the task are cacheable.""" + for t in self.targets: + if not t.should_retrieve_from_cache(): + return False + return True class AlwaysTask(Task): @@ -592,7 +639,8 @@ class Taskmaster: The Taskmaster for walking the dependency DAG. """ - def __init__(self, targets=[], tasker=None, order=None, trace=None): + def __init__(self, targets=[], tasker=None, order=None, trace=None, + remote_cache=None): self.original_top = targets self.top_targets_left = targets[:] self.top_targets_left.reverse() @@ -607,6 +655,7 @@ def __init__(self, targets=[], tasker=None, order=None, trace=None): self.trace = trace self.next_candidate = self.find_next_candidate self.pending_children = set() + self.remote_cache = remote_cache def find_next_candidate(self): """ @@ -1027,6 +1076,10 @@ def cleanup(self): """ Check for dependency cycles. """ + if self.remote_cache: + self.remote_cache.close() + self.remote_cache = None + if not self.pending_children: return diff --git a/SCons/Tool/ninja/Overrides.py b/SCons/Tool/ninja/Overrides.py index eb8dbb5cb2..23c740a968 100644 --- a/SCons/Tool/ninja/Overrides.py +++ b/SCons/Tool/ninja/Overrides.py @@ -73,7 +73,8 @@ def _print_cmd_str(*_args, **_kwargs): pass -def ninja_always_serial(self, num, taskmaster): +def ninja_always_serial(self, num, taskmaster, remote_cache=None, + use_scheduler_v2=False): """Replacement for SCons.Job.Jobs constructor which always uses the Serial Job class.""" # We still set self.num_jobs to num even though it's a lie. The # only consumer of this attribute is the Parallel Job class AND diff --git a/bin/files b/bin/files index 08b1caa3bd..bece30fff7 100644 --- a/bin/files +++ b/bin/files @@ -26,6 +26,7 @@ ./SCons/Platform/posix.py ./SCons/Platform/sunos.py ./SCons/Platform/win32.py +./SCons/RemoteCache.py ./SCons/Scanner/C.py ./SCons/Scanner/D.py ./SCons/Scanner/Fortran.py diff --git a/setup.cfg b/setup.cfg index 51aca60e95..273c40c885 100644 --- a/setup.cfg +++ b/setup.cfg @@ -43,6 +43,7 @@ zip_safe = False python_requires = >=3.5 install_requires = setuptools + urllib3 >= 1.8 setup_requires = setuptools include_package_data = True diff --git a/test/RemoteCache/.exclude_tests b/test/RemoteCache/.exclude_tests new file mode 100644 index 0000000000..1fc8fab382 --- /dev/null +++ b/test/RemoteCache/.exclude_tests @@ -0,0 +1,2 @@ +RemoteCacheTestServer.py +RemoteCacheUtils.py diff --git a/test/RemoteCache/CachePushAndFetch.py b/test/RemoteCache/CachePushAndFetch.py new file mode 100644 index 0000000000..102b9a3cba --- /dev/null +++ b/test/RemoteCache/CachePushAndFetch.py @@ -0,0 +1,101 @@ +#!/usr/bin/env python +# +# MIT License +# +# Copyright The SCons Foundation +# +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to +# the following conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY +# KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE +# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +# + +""" +Tests remote cache push and then fetch. In this test, cache misses are +expected for the first compilation with cache hits for the second compilation. +This is because the first compilation is the cache producer for the second +compilation. +""" + +import os +import stat +import sys + +import RemoteCacheUtils +import TestSCons + +test = TestSCons.TestSCons() +RemoteCacheUtils.skip_test_if_no_urllib3(test) +test.file_fixture('test_main.c') +test.dir_fixture('CachePushAndFetch') +server_url = RemoteCacheUtils.start_test_server(test.workpath()) + +arguments = [ + '--remote-cache-fetch-enabled', + '--remote-cache-push-enabled', + '--remote-cache-url=' + server_url, + '--cache-debug=%s' % test.workpath('cache.txt'), +] + +# Populate the cache. The expected compiler output depends on the platform. +# TODO: Do we need to call SCons.Tool.MSCommon.msvc_exists() on Windows and +# handle any other compilers? +if sys.platform == 'win32': + expected_compiler_output = """\ +cl /Fotest_main.obj /c test_main.c /nologo +test_main.c +link /nologo /OUT:main.exe test_main.obj""" +else: + expected_compiler_output = """\ +gcc -o test_main.o -c test_main.c +gcc -o main test_main.o""" + +test.run(arguments=arguments, + stdout=test.wrap_stdout("""\ +{expected_compiler_output} +RemoteCache: 0.0 percent cache hit rate on 2 cacheable tasks with 0 hits, 2 \ +misses, 0 w/cache suspended. 66.7 percent of total tasks cacheable, due to \ +1/3 tasks marked not cacheable. Saw 0 total failures, 0 cache restarts. +""".format(expected_compiler_output=expected_compiler_output))) + +# Clean the build directory. +test.run(arguments='-c .') + +# Run and confirm that we had cache hits. +test.run(arguments=arguments, + stdout=test.wrap_stdout("""\ +RemoteCache: 100.0 percent cache hit rate on 2 cacheable tasks with 2 hits, \ +0 misses, 0 w/cache suspended. 66.7 percent of total tasks cacheable, due to \ +1/3 tasks marked not cacheable. Saw 0 total failures, 0 cache restarts. +""")) + +# Confirm that cache hits set the execute bits only for the executable. +if os.name == 'posix': + object_mode = os.stat(test.workpath('test_main.o')).st_mode + executable_mode = os.stat(test.workpath('main')).st_mode + executable_bits = stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH + assert object_mode & executable_bits == 0, object_mode + assert executable_mode & executable_bits == executable_bits, \ + executable_mode + +test.pass_test() + +# Local Variables: +# tab-width:4 +# indent-tabs-mode:nil +# End: +# vim: set expandtab tabstop=4 shiftwidth=4: diff --git a/test/RemoteCache/CachePushAndFetch/SConstruct b/test/RemoteCache/CachePushAndFetch/SConstruct new file mode 100644 index 0000000000..d5b063de33 --- /dev/null +++ b/test/RemoteCache/CachePushAndFetch/SConstruct @@ -0,0 +1,2 @@ +env = Environment() +env.Program('main', 'test_main.c') \ No newline at end of file diff --git a/test/RemoteCache/RemoteCacheTestServer.py b/test/RemoteCache/RemoteCacheTestServer.py new file mode 100644 index 0000000000..5522a6e55c --- /dev/null +++ b/test/RemoteCache/RemoteCacheTestServer.py @@ -0,0 +1,81 @@ +#!/usr/bin/env python +# +# MIT License +# +# Copyright The SCons Foundation +# +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to +# the following conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY +# KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE +# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +# + +""" +RemoteCacheTestServer.py + +Test Python script that acts like a Bazel remote cache server. +""" + +import argparse +import hashlib +import http.server +import os + +parser = argparse.ArgumentParser( + description='Test loopback remote cache server') +parser.add_argument('address', help='Address to listen on') +parser.add_argument('port', type=int, help='Port to listen on') +args = vars(parser.parse_args()) + + +class HandlerClass(http.server.SimpleHTTPRequestHandler): + """ + Subclass of SimpleHTTPRequestHandler to handle PUT and GET requests, which + are the only requests that the SCons remote caching code makes. + SimpleHTTPRequestHandler automatically supports GET requests, so we only + need to implement PUT requests. http.client has code to handle a request + XYZ by calling the do_XYZ function, so we only need to implement do_PUT. + """ + def do_PUT(self): + path = self.translate_path(self.path) + dir, file = os.path.split(path) + if not os.path.exists(dir): + os.makedirs(dir) + + length = int(self.headers['Content-Length']) + data = self.rfile.read(length) + + if os.path.basename(dir) == 'cas': + # For Content Addressable Storage entries, validate that the last + # path segment matches the sha256 of the content; return + # Unprocessable Entity otherwise. + actual_sha256 = hashlib.sha256(data).hexdigest() + if file != actual_sha256: + self.send_response(422) + self.end_headers() + return + + with open(path, 'wb') as f: + f.write(data) + + # No Content is a standard answer for a successful resource PUT. + self.send_response(204) + self.end_headers() + +with http.server.HTTPServer((args['address'], args['port']), + HandlerClass) as httpd: + httpd.serve_forever() diff --git a/test/RemoteCache/RemoteCacheUtils.py b/test/RemoteCache/RemoteCacheUtils.py new file mode 100644 index 0000000000..d941da1979 --- /dev/null +++ b/test/RemoteCache/RemoteCacheUtils.py @@ -0,0 +1,64 @@ +#!/usr/bin/env python +# +# MIT License +# +# Copyright The SCons Foundation +# +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to +# the following conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY +# KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE +# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +# + +""" +Test utilities shared between remote cache tests. +""" + +import atexit +import os +import random +import subprocess +import sys + + +def start_test_server(directory): + """ + Starts a test script which pretends to be a Bazel remote cache server + Returns the full url to the test server, which should be passed into SCons + using --remote-cache-url + """ + host = 'localhost' + port = str(random.randint(49152, 65535)) + process = subprocess.Popen( + [sys.executable, + os.path.join(os.path.dirname(__file__), 'RemoteCacheTestServer.py'), + host, port], + cwd=directory, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) + + def ShutdownServer(): + process.kill() + atexit.register(ShutdownServer) + + return 'http://%s:%s' % (host, port) + + +def skip_test_if_no_urllib3(test): + """Skips the test if urllib3 is not available""" + try: + import urllib3 # noqa: F401 + except ImportError: + test.skip_test('urllib3 not found; skipping test')