Merge pull request #313 from WeBankPartners/dev_search_upgrade

Dev search upgrade

Author: pobu168

Dockerfile

@@ -7,6 +7,7 @@ ADD artifacts-corepy/requirements.txt /tmp/requirements.txt
 ADD artifacts-corepy/dist/* /tmp/
 RUN mkdir -p /etc/artifacts_corepy/
 RUN mkdir -p /var/log/artifacts_corepy/
+RUN mkdir -p /tmp/artifacts/
 ADD artifacts-corepy/etc/* /etc/artifacts_corepy/
 ADD nexus-data.tar.gz /nexus-data-init
 

Dockerfile_nonexus

@@ -6,6 +6,7 @@ ADD artifacts-corepy/requirements.txt /tmp/requirements.txt
 ADD artifacts-corepy/dist/* /tmp/
 RUN mkdir -p /etc/artifacts_corepy/
 RUN mkdir -p /var/log/artifacts_corepy/
+RUN mkdir -p /tmp/artifacts/
 ADD artifacts-corepy/etc/* /etc/artifacts_corepy/
 # Install && Clean up
 RUN apt update && apt-get -y install gcc swig libssl-dev python3-dev && \

artifacts-corepy/artifacts_corepy/server/base.py

@@ -39,7 +39,7 @@ def decrypt_rsa(secret_key, encrypt_text):
                   'cleanup_keep_unit_field', 'delete_op', 'log_level','ci_typeid_app_root_ci', 'ci_typeid_db_root_ci',
                   'ci_typeid_app_template_ci', 'ci_typeid_db_template_ci', 'push_nexus_server',
                   'push_nexus_repository', 'push_nexus_username', 'push_nexus_password', 's3_server_url',
-                  'db_script_extension', 'global_variable_prefix')
+                  'db_script_extension', 'global_variable_prefix', 'cache_cleanup_interval_min')
 def get_env_value(value, origin_value):
     prefix = 'ENV@'
     encrypt_prefix = 'RSA@'

artifacts-corepy/artifacts_corepy/server/scheduler.py

@@ -36,7 +36,13 @@
 
 def cleanup_cached_dir():
     try:
-        max_delta = 24 * 60 * 60
+        interval_min = 10
+        try:
+            interval_min = int(CONF.pakcage_cache_cleanup_interval_min)
+        except Exception as e:
+            LOG.error("Invalid package_cache_cleanup_interval_min: %s",
+                      CONF.pakcage_cache_cleanup_interval_min)
+        max_delta = interval_min * 60
         base_dir = CONF.pakcage_cache_dir
         for name in list(os.listdir(base_dir)):
             fullpath = os.path.join(base_dir, name)
@@ -192,7 +198,7 @@ def main():
     except Exception as e:
         LOG.exception(e)
     scheduler = BlockingScheduler(jobstores=jobstores, executors=executors, job_defaults=job_defaults, timezone=tz_info)
-    scheduler.add_job(cleanup_cached_dir, 'cron', hour='*')
+    scheduler.add_job(cleanup_cached_dir, 'cron', minute="*/5")
     scheduler.add_job(rotate_log, 'cron', hour=3, minute=5)
 
     cron_values = CONF.cleanup.cron.split()

artifacts-corepy/etc/artifacts_corepy.conf

@@ -57,7 +57,8 @@
         "push_nexus_server": "ENV@ARTIFACTS_PUSH_NEXUS_SERVER_URL",
         "push_nexus_repository": "ENV@ARTIFACTS_PUSH_NEXUS_REPOSITORY",
         "push_nexus_username": "ENV@ARTIFACTS_PUSH_NEXUS_USERNAME",
-        "push_nexus_password": "ENV@ARTIFACTS_PUSH_NEXUS_PASSWORD"
+        "push_nexus_password": "ENV@ARTIFACTS_PUSH_NEXUS_PASSWORD",
+        "cache_cleanup_interval_min": "ENV@ARTIFACTS_CACHE_CLEANUP_INTERVAL_MIN"
     },
     "log": {
     	"gunicorn_access": "/var/log/artifacts_corepy/access.log",
@@ -113,6 +114,7 @@
     "nexus_sort_as_string": "${nexus_sort_as_string}",
     "jwt_signing_key": "${jwt_signing_key}",
     "pakcage_cache_dir": "/tmp/artifacts/",
+    "pakcage_cache_cleanup_interval_min": "${cache_cleanup_interval_min}",
     "cleanup": {
         "cron": "${cleanup_corn}",
         "keep_topn": "${cleanup_keep_topn}",

register.xml

@@ -63,6 +63,7 @@
         <systemParameter name="PUSH_NEXUS_REPOSITORY" scopeType="plugins" defaultValue="" />
         <systemParameter name="ARTIFACTS_DEPLOY_PACKAGE_FIELD_MAP" scopeType="plugins" defaultValue="{}" />
         <systemParameter name="ARTIFACTS_DIFF_CONF_TEMPLATE_MAP" scopeType="plugins" defaultValue="{}" />
+        <systemParameter name="ARTIFACTS_CACHE_CLEANUP_INTERVAL_MIN" scopeType="plugins" defaultValue="10" />
 
     </systemParameters>
 
@@ -75,7 +76,7 @@
 
     <!-- 6.运行资源 - 描述部署运行本插件包需要的基础资源(如主机、虚拟机、容器、数据库等) -->
     <resourceDependencies>
-        <docker imageName="{{REPOSITORY}}:{{VERSION}}" containerName="{{REPOSITORY}}-{{VERSION}}" portBindings="{{ALLOCATE_PORT}}:9000,{{MONITOR_PORT}}:8081,5000:8082" volumeBindings="{{BASE_MOUNT_PATH}}/artifacts/log:/var/log/artifacts_corepy,{{BASE_MOUNT_PATH}}/certs:/certs,{{BASE_MOUNT_PATH}}/nexus-data:/nexus-data,/etc/localtime:/etc/localtime" envVariables="ARTIFACTS_UPLOAD_ENABLED={{UPLOAD_ENABLED}},ARTIFACTS_UPLOAD_NEXUS_ENABLED={{UPLOAD_NEXUS_ENABLED}},ARTIFACTS_CITYPE_SYSTEM_DESIGN={{ARTIFACTS_CITYPE_SYSTEM_DESIGN}},ARTIFACTS_CITYPE_UNIT_DESIGN={{ARTIFACTS_CITYPE_UNIT_DESIGN}},ARTIFACTS_CITYPE_DIFF_CONFIG={{ARTIFACTS_CITYPE_DIFF_CONFIG}},ARTIFACTS_CITYPE_DEPLOY_PACKAGE={{ARTIFACTS_CITYPE_DEPLOY_PACKAGE}},ARTIFACTS_ENCRYPT_VARIABLE_PREFIX={{ARTIFACTS_ENCRYPT_VARIABLE_PREFIX}},ARTIFACTS_FILE_VARIABLE_PREFIX={{ARTIFACTS_FILE_VARIABLE_PREFIX}},ARTIFACTS_DEFAULT_SPECIAL_REPLACE={{ARTIFACTS_DEFAULT_SPECIAL_REPLACE}},ARTIFACTS_LOCAL_NEXUS_SERVER_URL={{LOCAL_NEXUS_SERVER_URL}},ARTIFACTS_LOCAL_NEXUS_USERNAME={{LOCAL_NEXUS_USERNAME}},ARTIFACTS_LOCAL_NEXUS_PASSWORD={{LOCAL_NEXUS_PASSWORD}},ARTIFACTS_LOCAL_NEXUS_REPOSITORY={{LOCAL_NEXUS_REPOSITORY}},ARTIFACTS_USE_REMOTE_NEXUS_ONLY={{USE_REMOTE_NEXUS_ONLY}},ARTIFACTS_CMDB_ARTIFACT_PATH={{CMDB_ARTIFACT_PATH}},ARTIFACTS_NEXUS_SERVER_URL={{NEXUS_SERVER_URL}},ARTIFACTS_NEXUS_USERNAME={{NEXUS_USERNAME}},ARTIFACTS_NEXUS_PASSWORD={{NEXUS_PASSWORD}},ARTIFACTS_NEXUS_REPOSITORY={{NEXUS_REPOSITORY}},WECUBE_S3_ACCESS_KEY={{S3_ACCESS_KEY}},WECUBE_S3_SECRET_KEY={{S3_SECRET_KEY}},WECUBE_S3_SERVER_URL={{S3_SERVER_URL}},ARTIFACTS_DIFF_CONF_EXTENSION={{ARTIFACTS_DIFF_CONF_EXTENSION}},ARTIFACTS_DB_SCRIPT_EXTENSION={{ARTIFACTS_DB_SCRIPT_EXTENSION}},ARTIFACTS_VARIABLE_EXPRESSION={{ARTIFACTS_VARIABLE_EXPRESSION}},WECUBE_GATEWAY_URL={{GATEWAY_URL}},WECUBE_JWT_SIGNING_KEY={{JWT_SIGNING_KEY}},ARTIFACTS_NEXUS_SORT_AS_STRING={{NEXUS_SORT_AS_STRING}},ARTIFACTS_LOCAL_NEXUS_CONNECTOR_PORT=5000,ARTIFACTS_NEXUS_CONNECTOR_PORT={{NEXUS_CONNECTOR_PORT}},ARTIFACTS_SYSTEM_DESIGN_VIEW={{ARTIFACTS_SYSTEM_DESIGN_VIEW}},ARTIFACTS_CLEANUP_CRON={{ARTIFACTS_CLEANUP_CRON}},ARTIFACTS_CLEANUP_KEEP_TOPN={{ARTIFACTS_CLEANUP_KEEP_TOPN}},ARTIFACTS_CLEANUP_KEEP_UNIT_FIELD={{ARTIFACTS_CLEANUP_KEEP_UNIT_FIELD}},SUB_SYSTEM_CODE={{SUB_SYSTEM_CODE}},SUB_SYSTEM_KEY={{SUB_SYSTEM_KEY}},ARTIFACTS_LOG_LEVEL={{ARTIFACTS_LOG_LEVEL}},ARTIFACTS_CITYPE_APP_ROOT_CI={{ARTIFACTS_CITYPE_APP_ROOT_CI}},ARTIFACTS_CITYPE_DB_ROOT_CI={{ARTIFACTS_CITYPE_DB_ROOT_CI}},ARTIFACTS_CITYPE_APP_TEMPLATE_CI={{ARTIFACTS_CITYPE_APP_TEMPLATE_CI}},ARTIFACTS_CITYPE_DB_TEMPLATE_CI={{ARTIFACTS_CITYPE_DB_TEMPLATE_CI}},ARTIFACTS_PUSH_NEXUS_SERVER_URL={{PUSH_NEXUS_SERVER_URL}},ARTIFACTS_PUSH_NEXUS_USERNAME={{PUSH_NEXUS_USERNAME}},ARTIFACTS_PUSH_NEXUS_PASSWORD={{PUSH_NEXUS_PASSWORD}},ARTIFACTS_PUSH_NEXUS_REPOSITORY={{PUSH_NEXUS_REPOSITORY}},ARTIFACTS_DEPLOY_PACKAGE_FIELD_MAP={{ARTIFACTS_DEPLOY_PACKAGE_FIELD_MAP}},ARTIFACTS_DIFF_CONF_TEMPLATE_MAP={{ARTIFACTS_DIFF_CONF_TEMPLATE_MAP}},ARTIFACTS_GLOBAL_VARIABLE_PREFIX={{ARTIFACTS_GLOBAL_VARIABLE_PREFIX}}" />
+        <docker imageName="{{REPOSITORY}}:{{VERSION}}" containerName="{{REPOSITORY}}-{{VERSION}}" portBindings="{{ALLOCATE_PORT}}:9000,{{MONITOR_PORT}}:8081,5000:8082" volumeBindings="{{BASE_MOUNT_PATH}}/artifacts/cachedir:/tmp/artifacts,{{BASE_MOUNT_PATH}}/artifacts/log:/var/log/artifacts_corepy,{{BASE_MOUNT_PATH}}/certs:/certs,{{BASE_MOUNT_PATH}}/nexus-data:/nexus-data,/etc/localtime:/etc/localtime" envVariables="ARTIFACTS_UPLOAD_ENABLED={{UPLOAD_ENABLED}},ARTIFACTS_UPLOAD_NEXUS_ENABLED={{UPLOAD_NEXUS_ENABLED}},ARTIFACTS_CITYPE_SYSTEM_DESIGN={{ARTIFACTS_CITYPE_SYSTEM_DESIGN}},ARTIFACTS_CITYPE_UNIT_DESIGN={{ARTIFACTS_CITYPE_UNIT_DESIGN}},ARTIFACTS_CITYPE_DIFF_CONFIG={{ARTIFACTS_CITYPE_DIFF_CONFIG}},ARTIFACTS_CITYPE_DEPLOY_PACKAGE={{ARTIFACTS_CITYPE_DEPLOY_PACKAGE}},ARTIFACTS_ENCRYPT_VARIABLE_PREFIX={{ARTIFACTS_ENCRYPT_VARIABLE_PREFIX}},ARTIFACTS_FILE_VARIABLE_PREFIX={{ARTIFACTS_FILE_VARIABLE_PREFIX}},ARTIFACTS_DEFAULT_SPECIAL_REPLACE={{ARTIFACTS_DEFAULT_SPECIAL_REPLACE}},ARTIFACTS_LOCAL_NEXUS_SERVER_URL={{LOCAL_NEXUS_SERVER_URL}},ARTIFACTS_LOCAL_NEXUS_USERNAME={{LOCAL_NEXUS_USERNAME}},ARTIFACTS_LOCAL_NEXUS_PASSWORD={{LOCAL_NEXUS_PASSWORD}},ARTIFACTS_LOCAL_NEXUS_REPOSITORY={{LOCAL_NEXUS_REPOSITORY}},ARTIFACTS_USE_REMOTE_NEXUS_ONLY={{USE_REMOTE_NEXUS_ONLY}},ARTIFACTS_CMDB_ARTIFACT_PATH={{CMDB_ARTIFACT_PATH}},ARTIFACTS_NEXUS_SERVER_URL={{NEXUS_SERVER_URL}},ARTIFACTS_NEXUS_USERNAME={{NEXUS_USERNAME}},ARTIFACTS_NEXUS_PASSWORD={{NEXUS_PASSWORD}},ARTIFACTS_NEXUS_REPOSITORY={{NEXUS_REPOSITORY}},WECUBE_S3_ACCESS_KEY={{S3_ACCESS_KEY}},WECUBE_S3_SECRET_KEY={{S3_SECRET_KEY}},WECUBE_S3_SERVER_URL={{S3_SERVER_URL}},ARTIFACTS_DIFF_CONF_EXTENSION={{ARTIFACTS_DIFF_CONF_EXTENSION}},ARTIFACTS_DB_SCRIPT_EXTENSION={{ARTIFACTS_DB_SCRIPT_EXTENSION}},ARTIFACTS_VARIABLE_EXPRESSION={{ARTIFACTS_VARIABLE_EXPRESSION}},WECUBE_GATEWAY_URL={{GATEWAY_URL}},WECUBE_JWT_SIGNING_KEY={{JWT_SIGNING_KEY}},ARTIFACTS_NEXUS_SORT_AS_STRING={{NEXUS_SORT_AS_STRING}},ARTIFACTS_LOCAL_NEXUS_CONNECTOR_PORT=5000,ARTIFACTS_NEXUS_CONNECTOR_PORT={{NEXUS_CONNECTOR_PORT}},ARTIFACTS_SYSTEM_DESIGN_VIEW={{ARTIFACTS_SYSTEM_DESIGN_VIEW}},ARTIFACTS_CLEANUP_CRON={{ARTIFACTS_CLEANUP_CRON}},ARTIFACTS_CLEANUP_KEEP_TOPN={{ARTIFACTS_CLEANUP_KEEP_TOPN}},ARTIFACTS_CLEANUP_KEEP_UNIT_FIELD={{ARTIFACTS_CLEANUP_KEEP_UNIT_FIELD}},SUB_SYSTEM_CODE={{SUB_SYSTEM_CODE}},SUB_SYSTEM_KEY={{SUB_SYSTEM_KEY}},ARTIFACTS_LOG_LEVEL={{ARTIFACTS_LOG_LEVEL}},ARTIFACTS_CITYPE_APP_ROOT_CI={{ARTIFACTS_CITYPE_APP_ROOT_CI}},ARTIFACTS_CITYPE_DB_ROOT_CI={{ARTIFACTS_CITYPE_DB_ROOT_CI}},ARTIFACTS_CITYPE_APP_TEMPLATE_CI={{ARTIFACTS_CITYPE_APP_TEMPLATE_CI}},ARTIFACTS_CITYPE_DB_TEMPLATE_CI={{ARTIFACTS_CITYPE_DB_TEMPLATE_CI}},ARTIFACTS_PUSH_NEXUS_SERVER_URL={{PUSH_NEXUS_SERVER_URL}},ARTIFACTS_PUSH_NEXUS_USERNAME={{PUSH_NEXUS_USERNAME}},ARTIFACTS_PUSH_NEXUS_PASSWORD={{PUSH_NEXUS_PASSWORD}},ARTIFACTS_PUSH_NEXUS_REPOSITORY={{PUSH_NEXUS_REPOSITORY}},ARTIFACTS_DEPLOY_PACKAGE_FIELD_MAP={{ARTIFACTS_DEPLOY_PACKAGE_FIELD_MAP}},ARTIFACTS_DIFF_CONF_TEMPLATE_MAP={{ARTIFACTS_DIFF_CONF_TEMPLATE_MAP}},ARTIFACTS_GLOBAL_VARIABLE_PREFIX={{ARTIFACTS_GLOBAL_VARIABLE_PREFIX}},ARTIFACTS_CACHE_CLEANUP_INTERVAL_MIN={{ARTIFACTS_CACHE_CLEANUP_INTERVAL_MIN}}" />
         <s3 bucketName="wecube-artifacts" />
     </resourceDependencies>