Merge pull request #314 from WeBankPartners/dev

Dev

Author: zgyzgyhero

Dockerfile

@@ -7,6 +7,7 @@ ADD artifacts-corepy/requirements.txt /tmp/requirements.txt
 ADD artifacts-corepy/dist/* /tmp/
 RUN mkdir -p /etc/artifacts_corepy/
 RUN mkdir -p /var/log/artifacts_corepy/
+RUN mkdir -p /tmp/artifacts/
 ADD artifacts-corepy/etc/* /etc/artifacts_corepy/
 ADD nexus-data.tar.gz /nexus-data-init
 

Dockerfile_nonexus

@@ -1,11 +1,12 @@
-FROM python:3.7-slim
+FROM python:3.8-slim-buster
 LABEL maintainer = "Webank CTB Team"
 RUN sed -i 's/deb.debian.org/mirrors.tencentyun.com/g' /etc/apt/sources.list
 RUN sed -i 's/security.debian.org/mirrors.tencentyun.com/g' /etc/apt/sources.list
 ADD artifacts-corepy/requirements.txt /tmp/requirements.txt
 ADD artifacts-corepy/dist/* /tmp/
 RUN mkdir -p /etc/artifacts_corepy/
 RUN mkdir -p /var/log/artifacts_corepy/
+RUN mkdir -p /tmp/artifacts/
 ADD artifacts-corepy/etc/* /etc/artifacts_corepy/
 # Install && Clean up
 RUN apt update && apt-get -y install gcc swig libssl-dev python3-dev && \

Makefile

@@ -11,8 +11,6 @@ clean_py:
 build_py: clean_py
 	pip3 install wheel
 	cd artifacts-corepy && python3 setup.py bdist_wheel
-	cd artifacts-ui && npm rebuild node-sass
-	cd artifacts-ui && npm install --save core-js@2
 	cd artifacts-ui && npm run plugin
 
 image_py: build_py

README.md

@@ -1 +1,16 @@
-# wecube-plugins-artifacts
+# wecube-plugins-artifacts
+
+
+接口说明：
+组合物料包：包含原始物料包，包的配置，差异化变量配置信息，便于跨环境的物料使用
+
+1. 导出组合物料包：
+   即下载组合物料包到本地文件系统
+   GET /artifacts/packages/{deploy_package_id}/download
+   响应：文件流下载
+2. 推送组合物料包：
+   推送组合物料包到指定nexus上，nexus通过系统参数PUSH_NEXUS_\*配置
+   POST /artifacts/unit-designs/{unit_design_id}/packages/{deploy_package_id}/push
+   请求：json {}
+   响应：新包配置数据
+

artifacts-corepy/artifacts_corepy/apps/package/api.py

@@ -1239,9 +1239,15 @@ def update_file_variable(self, package_cached_dir, files):
         '''
         解析文件差异化变量
         '''
-        spliters = [s.strip() for s in CONF.encrypt_variable_prefix.split(',')]
-        spliters.extend([s.strip() for s in CONF.file_variable_prefix.split(',')])
-        spliters.extend([s.strip() for s in CONF.default_special_replace.split(',')])
+        spliters = []
+        if CONF.encrypt_variable_prefix.strip():
+            spliters = [s.strip() for s in CONF.encrypt_variable_prefix.split(',')]
+        if CONF.file_variable_prefix.strip():
+            spliters.extend([s.strip() for s in CONF.file_variable_prefix.split(',')])
+        if CONF.default_special_replace.strip():
+            spliters.extend([s.strip() for s in CONF.default_special_replace.split(',')])
+        if CONF.global_variable_prefix.strip():
+            spliters.extend([s.strip() for s in CONF.global_variable_prefix.split(',')])
         spliters = [s for s in spliters if s]
         for i in files:
             filepath = os.path.join(package_cached_dir, i['filename'])

artifacts-corepy/artifacts_corepy/apps/package/apiv2.py

@@ -5,14 +5,17 @@
 import cgi
 import falcon
 import os
+import json
 import urllib.parse
+from talos.core import utils
 from talos.core import config
 from talos.core.i18n import _
 from talos.common import controller as base_controller
 
 from artifacts_corepy.common.controller import Collection, Item, POSTCollection
 from artifacts_corepy.common import exceptions
 from artifacts_corepy.apps.package import apiv2 as package_api
+from artifacts_corepy.common import constant
 
 CONF = config.CONF
 
@@ -45,6 +48,22 @@ def on_get(self, req, resp, **kwargs):
             'message': 'success'
         }
 
+
+class CollectionProcessDef(Collection):
+    allow_methods = ('GET', )
+    name = 'artifacts.process.defs'
+    resource = package_api.ProcessDef
+
+class CollectionUser(Collection):
+    allow_methods = ('GET', )
+    name = 'artifacts.users.list'
+    resource = package_api.User
+    
+class CollectionCiData(POSTCollection):
+    allow_methods = ('POST',)
+    name = 'artifacts.diffconfigs'
+    resource = package_api.CiData
+
 class CollectionSystemDesign(Collection):
     allow_methods = ('GET', )
     name = 'artifacts.system-designs'
@@ -87,10 +106,23 @@ class CollectionUnitDesignPackages(POSTCollection):
     resource = package_api.UnitDesignPackages
 
 
+class CollectionPackageStatistics(POSTCollection):
+    allow_methods = ('POST', )
+    name = 'artifacts.packages.statistics'
+    resource = package_api.UnitDesignPackages
+    
+    def list(self, req, criteria, **kwargs):
+        return self.make_resource(req).get_package_statistics(req.json, **kwargs)
+
 class CollectionUnitDesignNexusPackages(POSTCollection):
     allow_methods = ('POST', )
     name = 'artifacts.unit-design.nexus.packages'
     resource = package_api.UnitDesignNexusPackages
+    
+class ItemUnitDesignNexusPackages(Item):
+    allow_methods = ('GET', )
+    name = 'artifacts.unit-design.nexus.path'
+    resource = package_api.UnitDesignNexusPackages
 
 
 class CollectionUnitDesignNexusPackageUpload(object):
@@ -99,18 +131,24 @@ class CollectionUnitDesignNexusPackageUpload(object):
 
     def on_post(self, req, resp, **kwargs):
         download_url = req.params.get('downloadUrl', None)
+        baseline_package = req.params.get('baseline_package', None)
+        package_type = req.params.get('package_type', None)
+        if not package_type:
+            raise exceptions.ValidationError(message=_('missing query param: package_type'))
+        elif package_type not in [constant.PackageType.app, constant.PackageType.db, constant.PackageType.mixed, constant.PackageType.image, constant.PackageType.rule]:
+            raise exceptions.ValidationError(message=_('invalid package_type param value: %s') % package_type)
         if not download_url:
-            raise exceptions.ValidationError(message=_('missing query: downloadUrl'))
-        form = cgi.FieldStorage(fp=req.stream, environ=req.env)
+            raise exceptions.ValidationError(message=_('missing query param: downloadUrl'))
+        # form = cgi.FieldStorage(fp=req.stream, environ=req.env)
         resp.json = {
             'code': 200,
             'status': 'OK',
-            'data': self.upload(req, download_url, **kwargs),
+            'data': self.upload(req, download_url, baseline_package, package_type, **kwargs),
             'message': 'success'
         }
 
-    def upload(self, req, download_url, **kwargs):
-        return self.resource().upload_from_nexus(download_url, **kwargs)
+    def upload(self, req, download_url, baseline_package, package_type, **kwargs):
+        return self.resource().upload_from_nexus(download_url, baseline_package, package_type, **kwargs)
 
 
 class CollectionUnitDesignPackageUpload(object):
@@ -119,15 +157,25 @@ class CollectionUnitDesignPackageUpload(object):
 
     def on_post(self, req, resp, **kwargs):
         form = cgi.FieldStorage(fp=req.stream, environ=req.env)
+        baseline_package = None
+        package_type = None
+        if 'baseline_package' in form:
+            baseline_package = form.getvalue('baseline_package', None)
+        if 'package_type' in form:
+            package_type = form.getvalue('package_type', None)
+        if not package_type:
+            raise exceptions.ValidationError(message=_('missing form param: package_type'))
+        elif package_type not in [constant.PackageType.app, constant.PackageType.db, constant.PackageType.mixed, constant.PackageType.image, constant.PackageType.rule]:
+            raise exceptions.ValidationError(message=_('invalid package_type param value: %s') % package_type)
         resp.json = {
             'code': 200,
             'status': 'OK',
-            'data': self.upload(req, form['file'].filename, form['file'].type, form['file'].file, **kwargs),
+            'data': self.upload(req, form['file'].filename, form['file'].type, form['file'].file, baseline_package, package_type, **kwargs),
             'message': 'success'
         }
 
-    def upload(self, req, filename, filetype, fileobj, **kwargs):
-        return self.resource().upload(filename, filetype, fileobj, **kwargs)
+    def upload(self, req, filename, filetype, fileobj, baseline_package, package_type, **kwargs):
+        return self.resource().upload(filename, filetype, fileobj, baseline_package, package_type, **kwargs)
 
 
 class ItemPackage(Item):
@@ -305,10 +353,43 @@ class PushComposePackage(base_controller.Controller):
     resource = package_api.UnitDesignPackages
 
     def on_post(self, req, resp, **kwargs):
+        body_param = {}
+        if hasattr(req, 'json'):
+            body_param = req.json
         resp.json = {
             'code': 200,
             'status': 'OK',
-            'data': self.resource().push_compose_package(**kwargs),
+            'data': self.resource().push_compose_package(body_param, **kwargs),
+            'message': 'success'
+        }
+        resp.status = falcon.HTTP_200
+        
+class SystemConfig(base_controller.Controller):
+    allow_methods = ('GET',)
+    name = 'artifacts.systemconfig'
+    resource = package_api.UnitDesignPackages
+
+    def on_get(self, req, resp, **kwargs):
+        local_nexus_server = CONF.nexus.server
+        remote_nexus_server = CONF.wecube.nexus.server
+        push_nexus_server = CONF.pushnexus.server
+        local_nexus_server = local_nexus_server.strip()
+        remote_nexus_server = remote_nexus_server.strip()
+        push_nexus_server = push_nexus_server.strip()
+        if utils.bool_from_string(CONF.use_remote_nexus_only):
+            local_nexus_server = remote_nexus_server
+        resp.json = {
+            'code': 200,
+            'status': 'OK',
+            'data': {
+                'upload_enabled': bool(utils.bool_from_string(CONF.wecube.upload_enabled) and local_nexus_server),
+                'upload_from_nexus_enabled': bool(utils.bool_from_string(CONF.wecube.upload_nexus_enabled) and remote_nexus_server),
+                'push_to_nexus_enabled': True if push_nexus_server else False,
+                'variable_prefix_encrypt': [] if not CONF.encrypt_variable_prefix.strip() else [s.strip() for s in CONF.encrypt_variable_prefix.split(',')],
+                'variable_prefix_file': [] if not CONF.file_variable_prefix.strip() else [s.strip() for s in CONF.file_variable_prefix.split(',')],
+                'variable_prefix_default': [] if not CONF.default_special_replace.strip() else [s.strip() for s in CONF.default_special_replace.split(',')],
+                'variable_prefix_global': [] if not CONF.global_variable_prefix.strip() else [s.strip() for s in CONF.global_variable_prefix.split(',')],
+            },
             'message': 'success'
         }
         resp.status = falcon.HTTP_200

artifacts-corepy/artifacts_corepy/apps/package/controller.py

@@ -64,6 +64,10 @@ def __call__(self, req, resp, package_name, entity_name, action_name):
 
 
 def add_routes(api):
+    # process
+    api.add_route('/artifacts/process/definitions', controller.CollectionProcessDef())
+    api.add_route('/artifacts/users', controller.CollectionUser())
+    api.add_route('/artifacts/cidata/{citype}/query', controller.CollectionCiData())
     # cmdb api forward
     api.add_route('/artifacts/system-design-versions', controller.CollectionSystemDesign())
     api.add_route('/artifacts/system-design-versions/{rid}', controller.ItemSystemDesign())
@@ -74,6 +78,7 @@ def add_routes(api):
     api.add_route('/artifacts/enum/system/codes/{cat_id}', controller.ItemEnumCodes())
     api.add_route('/artifacts/ci-types/{ci_type_id}/operations', controller.ItemCITypeOperations())
     api.add_route('/artifacts/unit-designs/{unit_design_id}/packages/query', controller.CollectionUnitDesignPackages())
+    api.add_route('/artifacts/unit-designs/{unit_design_id}/packages/statistics', controller.CollectionPackageStatistics())
     api.add_route('/artifacts/ci-types/{ci_type_id}/references/by', controller.ItemCiReferences())
     api.add_route('/artifacts/ci-types/{ci_type_id}/attributes', controller.ItemCiAttributes())
     api.add_route('/artifacts/ci-types/{ci_type_id}/ci-data/batch-delete', controller.CiDelete())
@@ -86,6 +91,8 @@ def add_routes(api):
     # nexus query
     api.add_route('/artifacts/unit-designs/{unit_design_id}/packages/queryNexusDirectiry',
                   controller.CollectionUnitDesignNexusPackages())
+    api.add_route('/artifacts/unit-designs/{unit_design_id}/packages/queryNexusPath',
+                  controller.ItemUnitDesignNexusPackages())
     # nexus upload
     api.add_route('/artifacts/unit-designs/{unit_design_id}/packages/uploadNexusPackage',
                   controller.CollectionUnitDesignNexusPackageUpload())
@@ -122,3 +129,7 @@ def add_routes(api):
                   controller.DownloadComposePackage())
     api.add_route('/artifacts/unit-designs/{unit_design_id}/packages/{deploy_package_id}/push',
                   controller.PushComposePackage())
+    
+    # system config
+    api.add_route('/artifacts/sysconfig',
+                  controller.SystemConfig())

artifacts-corepy/artifacts_corepy/apps/package/route.py

@@ -8,6 +8,7 @@ class PackageType(object):
     db = 'DB'
     mixed = 'APP&DB'
     image = 'IMAGE'
+    rule = 'RULE'
     default = 'APP&DB'
 
 

artifacts-corepy/artifacts_corepy/common/constant.py

@@ -24,13 +24,15 @@ def __init__(self, server, username, password):
         self.username = username
         self.password = password
 
-    def list(self, repository, path, extensions=None, continue_token=None, search_url='/service/rest/v1/search/assets'):
+    def list(self, repository, path, extensions=None, continue_token=None, search_url='/service/rest/v1/search/assets', filename=None):
         results = []
         url = self.server + search_url
         # group必须以/开头且结尾不包含/
         group = path.lstrip('/')
         group = '/' + group.rstrip('/')
         query = {'repository': repository, 'group': group}
+        if filename:
+            query['q'] = filename
         if continue_token:
             query['continuationToken'] = continue_token
         LOG.info('GET %s', url)
@@ -50,10 +52,11 @@ def list(self, repository, path, extensions=None, continue_token=None, search_ur
         results.extend([{
             'name': i['path'].split('/')[-1],
             'downloadUrl': i['downloadUrl'],
-            'md5': i.get('checksum', {}).get('md5', None)
+            'md5': i.get('checksum', {}).get('md5', None) or i.get('checksum', {}).get('sha1', None) or 'N/A',
+            'lastModified': i['lastModified'][:19]+'Z' if i.get('lastModified', None) else None
         } for i in filtered_items])
         if resp_json['continuationToken']:
-            results.extend(self.list(repository, path, extensions, resp_json['continuationToken']))
+            results.extend(self.list(repository, path, extensions, continue_token=resp_json['continuationToken'], filename=filename))
         return results
 
     def upload(self, repository, path, filename, filetype, fileobj, upload_url='/service/rest/v1/components'):