diff --git a/.github/scripts/setup_build_ubuntu-20.04.sh b/.github/scripts/setup_build_ubuntu-20.04.sh index b81d0cf2d..bc29ea9bb 100755 --- a/.github/scripts/setup_build_ubuntu-20.04.sh +++ b/.github/scripts/setup_build_ubuntu-20.04.sh @@ -3,4 +3,4 @@ # Install required packages for CodeCompass build sudo apt-get install -y git cmake make g++ libboost-all-dev llvm-10-dev clang-10 \ libclang-10-dev odb libodb-dev thrift-compiler libthrift-dev default-jdk libssl-dev \ - libgraphviz-dev libmagic-dev libgit2-dev ctags doxygen libgtest-dev npm libldap2-dev + libgraphviz-dev libmagic-dev libgit2-dev ctags doxygen libgtest-dev npm libldap2-dev libyaml-cpp-dev diff --git a/.github/scripts/setup_runtime_ubuntu-20.04.sh b/.github/scripts/setup_runtime_ubuntu-20.04.sh index 81efc0413..ecc8be72c 100755 --- a/.github/scripts/setup_runtime_ubuntu-20.04.sh +++ b/.github/scripts/setup_runtime_ubuntu-20.04.sh @@ -4,4 +4,4 @@ sudo apt-get install -y git cmake make g++ graphviz \ libboost-filesystem1.71.0 libboost-log1.71.0 libboost-program-options1.71.0 \ libllvm10 clang-10 libclang1-10 libthrift-0.13.0 default-jre libssl1.1 libmagic1 \ - libgit2-28 ctags googletest libldap-2.4-2 + libgit2-28 ctags googletest libldap-2.4-2 libyaml-cpp0.6 diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3eb3e8099..7e637303a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -256,4 +256,4 @@ jobs: if: ${{ github.repository == 'Ericsson/CodeCompass' && (github.ref_name == 'master' || startsWith(github.ref_name, 'release/') == true) }} uses: ./.github/workflows/tarball.yml secrets: - GITLAB_TRIGGER_TOKEN: ${{ secrets.GITLAB_TRIGGER_TOKEN }} + GITLAB_TRIGGER_TOKEN: ${{ secrets.GITLAB_TRIGGER_TOKEN }} \ No newline at end of file diff --git a/doc/deps.md b/doc/deps.md index 2154f39fe..5a6a8a953 100644 --- a/doc/deps.md +++ b/doc/deps.md @@ -40,6 +40,7 @@ be installed from the official repository of the given Linux distribution. - **`libgtest-dev`**: For testing CodeCompass. ***See [Known issues](#known-issues)!*** - **`libldap2-dev`**: For LDAP authentication. +- **`libyaml-cpp-dev`**: For Helm chart analysis. ## Quick guide @@ -52,7 +53,7 @@ known issues. sudo apt install git cmake make g++ gcc-7-plugin-dev libboost-all-dev \ llvm-10-dev clang-10 libclang-10-dev \ default-jdk libssl1.0-dev libgraphviz-dev libmagic-dev libgit2-dev ctags doxygen \ - libldap2-dev libgtest-dev npm + libldap2-dev libyaml-cpp-dev libgtest-dev npm ``` #### Ubuntu 20.04 ("Focal Fossa") LTS @@ -62,7 +63,7 @@ sudo apt install git cmake make g++ libboost-all-dev \ llvm-10-dev clang-10 libclang-10-dev \ odb libodb-dev thrift-compiler libthrift-dev \ default-jdk libssl-dev libgraphviz-dev libmagic-dev libgit2-dev ctags doxygen \ - libldap2-dev libgtest-dev npm + libldap2-dev libyaml-cpp-dev libgtest-dev npm ``` #### Database engine support diff --git a/doc/helm_documentation.md b/doc/helm_documentation.md new file mode 100644 index 000000000..ebc4dfa0b --- /dev/null +++ b/doc/helm_documentation.md @@ -0,0 +1,161 @@ +# Helm chart static analysis plugin +## Developers' documentation + +The Helm plugin serves the purpose of analyzing Helm charts and storing data about the microservices and their relations in the database. + +The plugin uses the _yaml-cpp_ library to parse YAML nodes. +The analysis is executed file-by-file. +**This plugin heavily relies on the rules and conventions of Helm and Kubernetes.** + +## Helm file analysis steps + +The Helm parser receives one or more root directories of Helm charts. +The directory is traversed in a recursive manner. +The plugin is capable of multi-threaded analysis, where each thread receives a file for analysis. + +### File purpose analysis + +First the parser checks the purpose of the YAML file in analysis. +The following "purposes" (i.e. file types) are checked: + +- _Helm charts_: the actual chart files which contains the name and other metadata of the microservice. + Within this category, _integration charts_ and _subcharts_ are distinguished. + Processing is somewhat different based on the subtype (see details below). + Files named _Chart.yaml_ or _Chart.yml_ are classified into this category. + +- _Helm values_: files which contain vital information of microservice relations and default values (e.g. for environment variables). + Files named _values.yaml_ or _values.yml_ are classified into this category. + These files are collected in a container for further analysis. + +- _Helm templates_: files that define microservice relations and metadata, such as ConfigMaps, Secrets, Certificates etc. + Files that are inside a directory named _templates_ are classified into this category. + These files are collected in a container for further analysis. + +- _Docker compose files_: files named _compose.yml, compose.yaml, docker_compose.yml, or docker_compose.yaml_ are classified into this category. + +- _CI files_: any file that does not belong to any category above and contains the substring _"ci"_, is classified as a continuous integration file. + +_Note:_ At this point, the plugin is more specifically aimed at the analysis of Helm charts. +However, it can be used for generic YAML parsing as well, so the non-Helm types are left in. + +### YAML node analysis + +After the file type check, the keys and values in the file are analyzed in a recursive manner. +_yaml-cpp_ distinguishes the following node types: scalar, sequence, map, and null and undefined. +These types appear within the `SymbolType` field. +The nodes are processed according to their type. + +Generally, every node is broken down to the smallest, scalar node which is persisted in the database. +However, a scalar holds the type of node in which it was included: e.g. if a scalar is originally a key in a map, +its `AstType` will be `MAP`. Maps and sequences need to be parsed in slightly different ways, +because their iterators are not interchangeable. In the end, all nodes are broken down to scalars, +which are then persisted in the database with their containing type. + +In order to provide better syntax highlight, a `SymbolType` field holds the place of a node: +`Key`, `Value`, `NestedKey`, `NestedValue`, or `Other`. + +The location of nodes within files is calculated by the `Mark()` method of _yaml_cpp_, +and the content of a node is extracted with the `Dump()` method. + +### Exception handling + +If any error happens during file analysis, the error will be saved as a `BuildLog` object. +The erroneous file will remain in _Not parsed_ status. + +Errors usually happen in Helm charts if the user did not execute the `helm template` command on the charts before parsing, +or when there are syntax errors in the file. + +### Chart analysis + +As mentioned above, Helm charts should be handled differently if they are integration charts or subcharts. + +#### Integration charts + +If the file is an integration chart - a chart which describes the entire software, not just a component -, +tha parser check if there is a _dependencies_ key in the file. +If not, the service that the file describes is registered in the database. +If there is, the listed microservices are registered in the database. +If a listed service contains an _alias_ key, its value should be persisted as the name of the service. +Otherwise, the value of the obligatory _name_ key is the name of the service. + +_Note: this approach might be refined in the future to adapt to more possible architecture description formats._ + +#### Subcharts + +If a _Chart.yaml_ file is contained by a _charts_ directory, it should be considered a subchart +(i.e. a component in a microservice-based software). +The described microservice should be persisted in the database. + +_Note: in case of parsing multiple projects, keep in mind that a full-value microservice (i.e. +a microservice which is not defined in a subchart, but not necessarily an integration service) +can define a component in within another project. Thus it can be a subchart in on project, and +an integration chart on its own at the same time._ + +### Template file analysis + +This analysis is implemented in the `TemplateAnalyzer` class. + +#### Dependencies + +After basic YAML parsing, the previously collected template files are analyzed to collect +microservice dependencies. +The content of each template file is investigated. +Any template object should be persisted in the database as a `HelmTemplate` object with +the file that it is defined in. +Based on the content and its format, the analysis branches off in the following directions: + +- _Service dependency:_ if the value of the `kind` key in the template file is _Service_. + This type defines additional services that derive from the original service. + In this case, the new service is persisted in the database as an _EXTERNAL_ service, and an edge + is created between the two services. The defined service depends on the defining service. +- _Mount dependencies:_ these types are usually collected within big deployment files. + The dependencies listed as the value of a _volumes_ key should be analyzed. + Please note that there can be multiple _volumes_ keys in a deployment file. + We consider two subtypes: + - _ConfigMaps_: these contain lots of important information that is needed for K8S pod definition. + - _Secrets_: these contain confidential information within services. + Both types can be identified by a `kind` key in the list. +- _Certificate dependencies:_ while the previous types could be identified by a `kind` key + within a file, a certificate can be defined by several template types. + An approximate heuristics is to check the value of the `kind` key for containing the + "Certificate" or the "InternalUserCA" substring. (_WARNING:_ naming conventions may vary + in every project.) Certificate dependencies define further secrets which should also + be persisted in the database. + +#### Resources + +All template files are checked to see if they contain resource usage information. +The following resources are collected: + +- _CPU:_ listed with `resources` and `requests` keys. This resource is calculated in + the number of CPU cores which can be a fraction. +- _Memory:_ listed with `resources` and `requests` keys. Calculated in gigabytes. +- _Storage:_ listed within a `volumeClaimTemplates` list, with a `storage` key. + Calculated in gigabytes. + +### Analysis of _values_ files + +This analysis is implemented in the `ValueAnalyzer` class. + +After basic YAML parsing, the previously collected _values.yaml_ files are analyzed to find +further connection points based on references. +Values files otherwise contain default values which might be overridden during deployment. + +Every values file is checked for references for microservices. +If the analyzer finds a reference, a new `HelmTemplate` object is persisted in the database, +and a new edge is defined between the microservices. + +## Frontend functionality + +The plugin is capable of providing the following functionality: + +- _Syntax highlight:_ the YAML nodes are colored according to the symbol type. +- _InfoTree:_ some very basic metadata is available of the nodes (name, symbol type, value type). +- _Microservice navigator:_ there is an accordion menu on the left side in which the detected + microservices are listed. The diagrams are available by right clicking the services. +- _Diagrams:_ + - Dependent services + - Config maps + - Secrets + - Resource usage + diff --git a/doc/usage.md b/doc/usage.md index 7bf61704c..82ae9865f 100644 --- a/doc/usage.md +++ b/doc/usage.md @@ -74,7 +74,7 @@ For full documentation see: (`-E SQL_ASCII` flag is recommended!) - [Start PostgreSQL database](https://www.postgresql.org/docs/12/app-postgres.html) -## 1. Generate compilation database +## 1/a. Generate compilation database If you want to parse a C++ project, you have to create a [compilation database file](http://clang.llvm.org/docs/JSONCompilationDatabase.html). @@ -103,6 +103,19 @@ installation. The first command line argument is the output file name and the second argument is the build command which compiles your project. This can be a simple compiler invocation or starting a build system. +## 1/b. Prepare Helm charts +If you want to parse a Helm chart, and the chart files contain template files, +you need to run the `helm template` command on the +charts first, in order to bring the chart to a pure YAML format. + +1. Run `helm template` on the chart. Save the output of the command in a separate directory, + e.g. if you run the command from the root of the chart, + `helm template . --output-dir=./output` +2. Copy all files and directories from the chart to the output directory (except the original + template files), in the original directory order: Chart.yaml, values.yaml files, files + directories, etc. This is needed because of the recursive directory traversal. +3. Parse the project as described in below, with the output directory as input. + ## 2. Parse the project For parsing a project with CodeCompass, the following command has to be emitted: diff --git a/docker/dev/Dockerfile b/docker/dev/Dockerfile index fb18edf59..6c6504b0c 100644 --- a/docker/dev/Dockerfile +++ b/docker/dev/Dockerfile @@ -25,6 +25,7 @@ RUN set -x && apt-get update -qq \ libmagic-dev \ libsqlite3-dev \ libssl-dev \ + libyaml-cpp-dev \ llvm-10 clang-10 llvm-10-dev libclang-10-dev \ npm \ thrift-compiler libthrift-dev \ diff --git a/docker/runtime/Dockerfile b/docker/runtime/Dockerfile index 272d1985e..a26503dd2 100644 --- a/docker/runtime/Dockerfile +++ b/docker/runtime/Dockerfile @@ -71,6 +71,7 @@ RUN set -x && apt-get update -qq && \ libldap-2.4-2 \ libmagic-dev \ libthrift-dev \ + libyaml-cpp-dev \ ctags \ tini && \ apt-get clean && \ diff --git a/plugins/helm/CMakeLists.txt b/plugins/helm/CMakeLists.txt new file mode 100644 index 000000000..fec4231fb --- /dev/null +++ b/plugins/helm/CMakeLists.txt @@ -0,0 +1,5 @@ +add_subdirectory(model) +add_subdirectory(parser) +add_subdirectory(service) + +install_webplugin(webgui) \ No newline at end of file diff --git a/plugins/helm/model/CMakeLists.txt b/plugins/helm/model/CMakeLists.txt new file mode 100644 index 000000000..dbef84fec --- /dev/null +++ b/plugins/helm/model/CMakeLists.txt @@ -0,0 +1,15 @@ +set(ODB_SOURCES + include/model/helmtemplate.h + include/model/microservice.h + include/model/microserviceedge.h + include/model/msresource.h + include/model/yamlastnode.h + include/model/yamlfile.h + include/model/yamlcontent.h) + +generate_odb_files("${ODB_SOURCES}") + +add_odb_library(helmmodel ${ODB_CXX_SOURCES}) +add_dependencies(helmmodel model) + +install_sql() diff --git a/plugins/helm/model/include/model/helmtemplate.h b/plugins/helm/model/include/model/helmtemplate.h new file mode 100644 index 000000000..afe7bab84 --- /dev/null +++ b/plugins/helm/model/include/model/helmtemplate.h @@ -0,0 +1,63 @@ +#ifndef CC_MODEL_HELM_H +#define CC_MODEL_HELM_H + +#include + +#include +#include +#include + +#include +#include + +namespace cc +{ +namespace model +{ + +typedef uint64_t HelmTemplateId; + +#pragma db object +struct HelmTemplate +{ + enum class DependencyType + { + SERVICE, + MOUNT, + CERTIFICATE, + RESOURCE, + OTHER + }; + + #pragma db id + HelmTemplateId id; + + #pragma db not_null + FileId file; + + std::string name; + + #pragma db not_null + DependencyType dependencyType; + + #pragma db not_null + std::string kind; + + #pragma db not_null + MicroserviceId depends; + + bool operator==(HelmTemplate& rhs); +}; + +inline std::uint64_t createIdentifier(const HelmTemplate& helm_) +{ + return util::fnvHash( + helm_.name + + helm_.kind + + std::to_string(helm_.depends) + + std::to_string(helm_.file)); +} +} +} + +#endif // CC_MODEL_HELM_H diff --git a/plugins/helm/model/include/model/microservice.h b/plugins/helm/model/include/model/microservice.h new file mode 100644 index 000000000..4ab1391ad --- /dev/null +++ b/plugins/helm/model/include/model/microservice.h @@ -0,0 +1,47 @@ +#ifndef CC_MODEL_MICROSERVICE_H +#define CC_MODEL_MICROSERVICE_H + +#include +#include +#include + +#include "model/file.h" + +#include "util/hash.h" + +namespace cc +{ +namespace model +{ + +typedef std::uint64_t MicroserviceId; + +#pragma db object +struct Microservice +{ + enum class ServiceType + { + INTERNAL, + EXTERNAL + }; + + #pragma db id + MicroserviceId serviceId; + + #pragma db not_null + std::string name; + + FileId file; + + ServiceType type; +}; + +inline std::uint64_t createIdentifier(const Microservice& service_) +{ + return util::fnvHash( + service_.name); +} +} +} + +#endif // CC_MODEL_MICROSERVICE_H diff --git a/plugins/helm/model/include/model/microserviceedge.h b/plugins/helm/model/include/model/microserviceedge.h new file mode 100644 index 000000000..e16b50d16 --- /dev/null +++ b/plugins/helm/model/include/model/microserviceedge.h @@ -0,0 +1,68 @@ +#ifndef CC_MODEL_YAMLEDGE_H +#define CC_MODEL_YAMLEDGE_H + +#include + +#include +#include "model/helmtemplate.h" + +#include + +namespace cc +{ +namespace model +{ + +struct MicroserviceEdge; +typedef std::shared_ptr MicroserviceEdgePtr; + +typedef std::uint64_t MicroserviceEdgeId; + +#pragma db object +struct MicroserviceEdge +{ + #pragma db id + MicroserviceEdgeId id; + + #pragma db not_null + uint64_t helperId; + + #pragma db not_null + #pragma db on_delete(cascade) + std::shared_ptr from; + + #pragma db not_null + #pragma db on_delete(cascade) + std::shared_ptr to; + + #pragma db not_null + #pragma db on_delete(cascade) + std::shared_ptr connection; + + #pragma db not_null + std::string type; + + std::string toString() const; +}; + +inline std::string MicroserviceEdge::toString() const +{ + return std::string("MicroserviceEdge") + .append("\nfrom = ").append(std::to_string(from->serviceId)) + .append("\nto = ").append(std::to_string(to->serviceId)) + .append("\ntype = "); +} + +inline std::uint64_t createIdentifier(const MicroserviceEdge& edge_) +{ + return util::fnvHash( + std::to_string(edge_.from->serviceId) + + std::to_string(edge_.to->serviceId) + + std::to_string(edge_.helperId) + + edge_.type); +} + +} +} + +#endif // CC_MODEL_YAMLEDGE_H diff --git a/plugins/helm/model/include/model/msresource.h b/plugins/helm/model/include/model/msresource.h new file mode 100644 index 000000000..e09c61088 --- /dev/null +++ b/plugins/helm/model/include/model/msresource.h @@ -0,0 +1,58 @@ +#ifndef CC_MODEL_MSRESOURCE_H +#define CC_MODEL_MSRESOURCE_H + +#include +#include +#include + +#include "model/file.h" +#include "microservice.h" + +#include "util/hash.h" + +namespace cc +{ +namespace model +{ + +#pragma db object +struct MSResource +{ + enum class ResourceType + { + CPU, + MEMORY, + STORAGE + }; + + #pragma db id auto + uint64_t id; + + #pragma db not_null + ResourceType type; + + #pragma db not_null + MicroserviceId service; + + #pragma db not_null + float amount; + + #pragma db not_null + std::string unit; +}; + +inline std::string resourceTypeToString(MSResource::ResourceType type_) +{ + switch (type_) + { + case MSResource::ResourceType::CPU: return "CPU"; + case MSResource::ResourceType::MEMORY: return "Memory"; + case MSResource::ResourceType::STORAGE: return "Storage"; + } + + return std::string(); +} +} +} + +#endif // CC_MODEL_MSRESOURCE_H diff --git a/plugins/helm/model/include/model/yamlastnode.h b/plugins/helm/model/include/model/yamlastnode.h new file mode 100644 index 000000000..1cac562c7 --- /dev/null +++ b/plugins/helm/model/include/model/yamlastnode.h @@ -0,0 +1,150 @@ +#ifndef CC_MODEL_YAMLASTNODE_H +#define CC_MODEL_YAMLASTNODE_H + +#include +#include + +#include +#include +#include + +#include +#include + +#include + +namespace cc +{ +namespace model +{ + +typedef std::uint64_t YamlAstNodeId; + +#pragma db object +struct YamlAstNode +{ + enum class SymbolType + { + Key, + Value, + NestedKey, + NestedValue, + Other + }; + + enum class AstType + { + NULLTYPE, + SCALAR, + MAP, + SEQUENCE, + UNDEFINED = 50 + }; + + #pragma db id + YamlAstNodeId id = 0; + + std::string astValue; + + #pragma db null + FileLoc location; + + std::uint64_t entityHash; + + SymbolType symbolType = SymbolType::Other; + + AstType astType = AstType::NULLTYPE; + + std::string toString() const; + + bool operator< (const YamlAstNode& other) const { return id < other.id; } + bool operator==(const YamlAstNode& other) const { return id == other.id; } +}; + +typedef std::shared_ptr YamlAstNodePtr; + +inline std::string symbolTypeToString(YamlAstNode::SymbolType type_) +{ + switch (type_) + { + case YamlAstNode::SymbolType::Key: return "Key"; + case YamlAstNode::SymbolType::Value: return "Value"; + case YamlAstNode::SymbolType::NestedKey: return "NestedKey"; + case YamlAstNode::SymbolType::NestedValue: return "NestedValue"; + case YamlAstNode::SymbolType::Other: return "Other"; + } + + return std::string(); +} + +inline std::string astTypeToString(YamlAstNode::AstType type_) +{ + switch (type_) + { + case YamlAstNode::AstType::NULLTYPE: return "Null"; + case YamlAstNode::AstType::SCALAR: return "Scalar"; + case YamlAstNode::AstType::MAP: return "Map"; + case YamlAstNode::AstType::SEQUENCE: return "Sequence"; + case YamlAstNode::AstType::UNDEFINED: return "Undefined"; + } + + return std::string(); +} + +inline std::string YamlAstNode::toString() const +{ + return std::string("YamlAstNode") + .append("\nid = ").append(std::to_string(id)) + .append("\nastValue = ").append(astValue) + .append("\nlocation = ").append(location.file->path).append(" (") + .append(std::to_string( + static_cast(location.range.start.line))).append(":") + .append(std::to_string( + static_cast(location.range.start.column))).append(" - ") + .append(std::to_string( + static_cast(location.range.end.line))).append(":") + .append(std::to_string( + static_cast(location.range.end.column))).append(")") + .append("\nsymbolType = ").append(symbolTypeToString(symbolType)) + .append("\nastType = ").append(astTypeToString(astType)); +} + +inline std::uint64_t createIdentifier(const YamlAstNode& astNode_) +{ + using SymbolTypeInt + = std::underlying_type::type; + using AstTypeInt + = std::underlying_type::type; + + std::string res; + + res + .append(astNode_.astValue).append(":") + .append(std::to_string(astNode_.entityHash)).append(":") + .append(std::to_string( + static_cast(astNode_.symbolType))).append(":") + .append(std::to_string( + static_cast(astNode_.astType))).append(":"); + + if (astNode_.location.file) + res + .append(std::to_string( + astNode_.location.file->id)).append(":") + .append(std::to_string( + astNode_.location.range.start.line)).append(":") + .append(std::to_string( + astNode_.location.range.start.column)).append(":") + .append(std::to_string( + astNode_.location.range.end.line)).append(":") + .append(std::to_string( + astNode_.location.range.end.column)).append(":"); + else + res.append("null"); + + return util::fnvHash(res); +} + +} +} + +#endif // CC_MODEL_YAMLASTNODE_H diff --git a/plugins/helm/model/include/model/yamlcontent.h b/plugins/helm/model/include/model/yamlcontent.h new file mode 100644 index 000000000..9df54aaee --- /dev/null +++ b/plugins/helm/model/include/model/yamlcontent.h @@ -0,0 +1,50 @@ +#ifndef CC_MODEL_YAMLCONTENT_H +#define CC_MODEL_YAMLCONTENT_H + +#include + +#include +#include +#include + +#include +#include + +namespace cc +{ +namespace model +{ + +struct YamlContent; + +typedef std::shared_ptr YamlContentPtr; + +#pragma db object +struct YamlContent +{ + #pragma db id auto + std::uint64_t id; + + std::string key; + + std::string value; + + #pragma db not_null + FileId file; + + std::string toString() const; +}; + +inline std::string YamlContent::toString() const +{ + return std::string("YamlContent") + .append("\nid = ").append(std::to_string(id)) + .append("\nkey = ").append(key) + .append("\nvalue = ").append(value) + .append("\nfile id = ").append(std::to_string(file)); +} + +} //model +} //cc + +#endif // CC_MODEL_YAMLCONTENT_H diff --git a/plugins/helm/model/include/model/yamlfile.h b/plugins/helm/model/include/model/yamlfile.h new file mode 100644 index 000000000..05a84c1aa --- /dev/null +++ b/plugins/helm/model/include/model/yamlfile.h @@ -0,0 +1,72 @@ +#ifndef CC_MODEL_YAML_H +#define CC_MODEL_YAML_H + +#include + +#include +#include +#include + +#include + +namespace cc +{ +namespace model +{ +struct YamlFile; + +typedef std::shared_ptr YamlFilePtr; + +#pragma db object +struct YamlFile +{ + enum Type + { + DOCKER_COMPOSE, + HELM_CHART, + HELM_SUBCHART, + HELM_TEMPLATE, + HELM_VALUES, + CI, + OTHER + }; + + #pragma db id auto + std::uint64_t id; + + #pragma db not_null + FileId file; + + #pragma db not_null + Type type; + + std::string toString() const; +}; + +inline std::string typeToString(YamlFile::Type type_) +{ + switch (type_) + { + case YamlFile::Type::DOCKER_COMPOSE: return "docker-compose"; + case YamlFile::Type::HELM_CHART: return "Helm chart"; + case YamlFile::Type::HELM_TEMPLATE: return "Helm template"; + case YamlFile::Type::HELM_VALUES: return "Helm values"; + case YamlFile::Type::CI: return "CI"; + case YamlFile::Type::OTHER: return "Other"; + } + + return std::string(); +} + +inline std::string YamlFile::toString() const +{ + return std::string("YamlFile") + .append("\nid = ").append(std::to_string(id)) + .append("\nfile id = ").append(std::to_string(file)) + .append("\ntype = ").append(typeToString(type)); +} + +} //model +} //cc + +#endif // CC_MODEL_YAML_H diff --git a/plugins/helm/parser/CMakeLists.txt b/plugins/helm/parser/CMakeLists.txt new file mode 100644 index 000000000..29feb3f03 --- /dev/null +++ b/plugins/helm/parser/CMakeLists.txt @@ -0,0 +1,26 @@ +find_package(yaml-cpp REQUIRED) + +include_directories( + include + ${PROJECT_SOURCE_DIR}/model/include + ${PROJECT_SOURCE_DIR}/util/include + ${PROJECT_SOURCE_DIR}/parser/include + ${CMAKE_SOURCE_DIR}/parser/include + ${PLUGIN_DIR}/model/include + ${YAML_CPP_INCLUDE_DIRS}) + +add_library(helmparser SHARED + src/templateanalyzer.cpp + src/valueanalyzer.cpp + src/helmparser.cpp) + +target_link_libraries(helmparser + helmmodel + model + util + ${Boost_LIBRARIES} + ${YAML_CPP_LIBRARIES}) + +install(TARGETS helmparser + LIBRARY DESTINATION ${INSTALL_LIB_DIR} + DESTINATION ${INSTALL_PARSER_DIR}) diff --git a/plugins/helm/parser/include/parser/helmparser.h b/plugins/helm/parser/include/parser/helmparser.h new file mode 100644 index 000000000..a528ab554 --- /dev/null +++ b/plugins/helm/parser/include/parser/helmparser.h @@ -0,0 +1,105 @@ +#ifndef CC_PARSER_YAML_PARSER_H +#define CC_PARSER_YAML_PARSER_H + +#include + +#include +#include + +#include + +#include "yaml-cpp/yaml.h" + +namespace cc +{ +namespace parser +{ +class YamlParser : public AbstractParser +{ +public: + YamlParser(ParserContext& ctx_); + ~YamlParser(); + virtual bool cleanupDatabase() override; + virtual bool parse() override; + +private: + /** + * This method classifies a YAML file according to the purpose + * it serves, e.g. Helm chart or other type of configuration file. + * The classification is done based on naming conventions. + */ + void processFileType( + model::FilePtr& file_, + YAML::Node& loadedFile); + + void processIntegrationChart( + model::FilePtr& file_, + YAML::Node& loadedFile); + + /** + * The first-level keys in a YAML files usually have great significance, + * so they should be collected in a separate collection. + * @param file_ + * @param loadedFile + */ + void processRootKeys( + model::FilePtr& file_, + YAML::Node& loadedFile); + + bool collectAstNodes(model::FilePtr file_); + + /** + * This method is used to decide the type of root keys and values. + */ + void chooseCoreNodeType( + YAML::Node& node_, + model::FilePtr file_, + model::YamlAstNode::SymbolType symbolType_); + + /** + * These methods handle the different key and value types + * in YAML files. + */ + void processAtomicNode( + YAML::Node& node_, + model::FilePtr file_, + model::YamlAstNode::SymbolType symbolType_, + model::YamlAstNode::AstType astType_); + void processMap( + YAML::Node& node_, + model::FilePtr file_, + model::YamlAstNode::SymbolType symbolType_); + void processSequence( + YAML::Node& node_, + model::FilePtr file_, + model::YamlAstNode::SymbolType symbolType_); + + model::Range getNodeLocation(YAML::Node& node_); + + /** + * A method to recursively traverse the input directory and + * find YAML files. + */ + util::DirIterCallback getParserCallback(); + + bool accept(const std::string& path_) const; + + std::unordered_set _fileIdCache; + std::map _fileAstCache; + std::map _valuesAstCache; + std::unique_ptr> _pool; + std::atomic _visitedFileCount; + std::vector _astNodes; + std::vector _yamlFiles; + std::vector _rootPairs; + std::vector _buildLogs; + std::vector _processedMS; + + std::mutex _mutex; + bool _areDependenciesListed; +}; + +} // namespace parser +} // namespace cc + +#endif // CC_PARSER_YAML_PARSER_H diff --git a/plugins/helm/parser/src/helmparser.cpp b/plugins/helm/parser/src/helmparser.cpp new file mode 100644 index 000000000..b4e85aa1d --- /dev/null +++ b/plugins/helm/parser/src/helmparser.cpp @@ -0,0 +1,544 @@ +#include +#include +#include +#include +#include + +#include "yaml-cpp/yaml.h" + +#include + +#include +#include +#include +#include +#include + +#include + +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "parser/helmparser.h" +#include "valueanalyzer.h" +#include "templateanalyzer.h" + +namespace cc +{ +namespace parser +{ + +namespace fs = boost::filesystem; + +YamlParser::YamlParser(ParserContext& ctx_): AbstractParser(ctx_) +{ + _areDependenciesListed = true; + + util::OdbTransaction {_ctx.db} ([&, this] { + for (const model::YamlFile& yf + : _ctx.db->query()) + { + _fileIdCache.insert(yf.file); + } + }); + + int threadNum = _ctx.options["jobs"].as(); + _pool = util::make_thread_pool( + threadNum, [&, this](const std::string& path_) + { + LOG(info) << "[yamlparser] Processing " << path_; + model::FilePtr file = _ctx.srcMgr.getFile(path_); + if (file) + { + if (_fileIdCache.find(file->id) == _fileIdCache.end()) + { + if (accept(file->path)) + { + bool success = collectAstNodes(file); + ++this->_visitedFileCount; + file->parseStatus = success + ? model::File::PSFullyParsed + : model::File::PSPartiallyParsed; + file->type = "YAML"; + _ctx.srcMgr.updateFile(*file); + } + } + else + LOG(debug) << "YamlParser already parsed this file: " << file->path; + } + }); + +} + +bool YamlParser::cleanupDatabase() +{ + if (!_fileIdCache.empty()) + { + try + { + util::OdbTransaction {_ctx.db} ([this] { + for (const model::File& file + : _ctx.db->query( + odb::query::id.in_range( + _fileIdCache.begin(), _fileIdCache.end()))) + { + auto it = _ctx.fileStatus.find(file.path); + if (it != _ctx.fileStatus.end() && + (it->second == cc::parser::IncrementalStatus::DELETED || + it->second == cc::parser::IncrementalStatus::MODIFIED || + it->second == cc::parser::IncrementalStatus::ACTION_CHANGED)) + { + LOG(info) << "[yamlparser] Database cleanup: " << file.path; + + _ctx.db->erase_query( + odb::query::file == file.id); + _fileIdCache.erase(file.id); + } + } + }); + } + catch (odb::database_exception&) + { + LOG(fatal) << "[yamlparser] Transaction failed!"; + return false; + } + } + return true; +} + +bool YamlParser::parse() +{ + this->_visitedFileCount = 0; + + for (const std::string& input : + _ctx.options["input"].as>()) + { + if (fs::is_directory(input)) + { + LOG(info) << "[yamlparser] Yaml parse path: " << input; + + //--- Parse YAML files ---// + + util::OdbTransaction trans(_ctx.db); + trans([&, this]() { + auto cb = getParserCallback(); + /*--- Call non-empty iter-callback for all files + in the current root directory. ---*/ + try + { + util::iterateDirectoryRecursive(input, cb); + } + catch (std::exception& ex_) + { + LOG(warning) + << "[yamlparser] Yaml parser threw an exception: " << ex_.what(); + } + catch (...) + { + LOG(warning) + << "[yamlparser] Yaml parser failed with unknown exception!"; + } + }); + } + } + _pool->wait(); + LOG(info) << "[yamlparser] Processed files: " << this->_visitedFileCount; + + _ctx.srcMgr.persistFiles(); + + //--- Collect relations ---/ + TemplateAnalyzer templateAnalyzer(_ctx, _fileAstCache); + templateAnalyzer.init(); + + ValueAnalyzer relationCollector(_ctx, _valuesAstCache, templateAnalyzer.getTemplateCounter()); + relationCollector.init(); + + return true; +} + +util::DirIterCallback YamlParser::getParserCallback() +{ + return [this](const std::string& currPath_) + { + boost::filesystem::path path(currPath_); + + if (boost::filesystem::is_regular_file(path)) + { + _pool->enqueue(currPath_); + } + + return true; + }; +} + +bool YamlParser::accept(const std::string& path_) const +{ + std::string ext = boost::filesystem::extension(path_); + return ext == ".yaml" || ext == ".yml"; +} + +void YamlParser::processFileType(model::FilePtr& file_, YAML::Node& loadedFile) +{ + util::OdbTransaction {_ctx.db} ([&] { + model::YamlFilePtr file = std::make_shared(); + file->file = file_->id; + + if (file_->filename == "Chart.yaml" || file_->filename == "Chart.yml") + { + if (file_->path.find("charts/") != std::string::npos) + { + file->type = model::YamlFile::Type::HELM_SUBCHART; + + if (std::find(_processedMS.begin(), _processedMS.end(), YAML::Dump(loadedFile["name"])) == _processedMS.end()) + { + model::Microservice service; + service.file = file_->id; + service.name = YAML::Dump(loadedFile["name"]); + service.type = model::Microservice::ServiceType::INTERNAL; + service.serviceId = cc::model::createIdentifier(service); + _ctx.db->persist(service); + _processedMS.push_back(service.name); + } + } + else + { + file->type = model::YamlFile::Type::HELM_CHART; + processIntegrationChart(file_, loadedFile); + } + + _mutex.lock(); + _fileAstCache.insert({file_->path, loadedFile}); + _mutex.unlock(); + } + else if (file_->filename == "values.yaml" || file_->filename == "values.yml") + { + file->type = model::YamlFile::Type::HELM_VALUES; + + // If a values.yaml file belongs to a subchart in the chart set, + // it should not be parsed since it contains default values + // that may provide false results in the microservice architecture + // dependency mapping. + _mutex.lock(); + _valuesAstCache.insert({file_->path, loadedFile}); + _mutex.unlock(); + /*if (file_->path.find("charts/") == std::string::npos) + { + model::Microservice service; + service.file = file_->id; + service.name = fs::path(file_->path).parent_path().filename().string(); + service.serviceId = cc::model::createIdentifier(service); + _ctx.db->persist(service); + }*/ + } + else if (file_->path.find("templates/") != std::string::npos) + { + file->type = model::YamlFile::Type::HELM_TEMPLATE; + + _mutex.lock(); + _fileAstCache.insert({file_->path, loadedFile}); + _mutex.unlock(); + } + else if (file_->filename == "compose.yaml" || file_->filename == "compose.yml" + || file_->filename == "docker-compose.yaml" || file_->filename == "docker-compose.yml") + file->type = model::YamlFile::Type::DOCKER_COMPOSE; + else if (file_->filename.find("ci") != std::string::npos) + file->type = model::YamlFile::Type::CI; + + _ctx.db->persist(file); + }); +} + +/* + * Integration charts (root charts) may contain several + * aliased microservices that are otherwise not contained + * in any subcharts. + */ +void YamlParser::processIntegrationChart(model::FilePtr& file_, YAML::Node& loadedFile_) +{ + if (!loadedFile_["dependencies"] || !loadedFile_["dependencies"].IsSequence()) + { + //_areDependenciesListed = false; + + model::Microservice service; + service.file = file_->id; + service.type = model::Microservice::ServiceType::INTERNAL; + service.name = YAML::Dump(loadedFile_["name"]); + + if (std::find(_processedMS.begin(), _processedMS.end(), service.name) == _processedMS.end()) + { + service.serviceId = cc::model::createIdentifier(service); + _ctx.db->persist(service); + _processedMS.push_back(service.name); + } + + return; + } + + util::OdbTransaction {_ctx.db} ([&] + { + for (auto iter = loadedFile_["dependencies"].begin(); + iter != loadedFile_["dependencies"].end(); + ++iter) + { + model::Microservice service; + service.file = file_->id; + service.type = model::Microservice::ServiceType::INTERNAL; + + if ((*iter)["alias"]) + service.name = YAML::Dump((*iter)["alias"]); + else + service.name = YAML::Dump((*iter)["name"]); + + if (std::find(_processedMS.begin(), _processedMS.end(), service.name) == _processedMS.end()) + { + service.serviceId = cc::model::createIdentifier(service); + _ctx.db->persist(service); + _processedMS.push_back(service.name); + } + } + }); +} + +void YamlParser::processRootKeys(model::FilePtr& file_, YAML::Node& loadedFile) +{ + util::OdbTransaction {_ctx.db} ([&]{ + for (const auto &pair: loadedFile) + { + model::YamlContentPtr content = std::make_shared(); + content->file = file_->id; + content->key = Dump(pair.first); + content->value = Dump(pair.second); + + _ctx.db->persist(content); + } + }); +} + +bool YamlParser::collectAstNodes(model::FilePtr file_) +{ + try + { + YAML::Node currentFile = YAML::LoadFile(file_->path); + + processFileType(file_, currentFile); + processRootKeys(file_, currentFile); + + for (auto it = currentFile.begin(); it != currentFile.end(); ++it) + { + chooseCoreNodeType(it->first, file_, model::YamlAstNode::SymbolType::Key); + chooseCoreNodeType(it->second, file_, model::YamlAstNode::SymbolType::Value); + } + } + catch (YAML::ParserException& e) + { + LOG(warning) << "[yamlparser] Exception thrown in : " << file_->path << ": " << e.what(); + + util::OdbTransaction {_ctx.db} ([&] + { + model::BuildLog buildLog; + buildLog.location.file = file_; + buildLog.location.range.start.line = e.mark.line + 1; + buildLog.location.range.start.column = e.mark.column; + buildLog.location.range.end.line = e.mark.line + 1; + buildLog.location.range.end.column = e.mark.column; + buildLog.log.message = e.what(); + buildLog.log.type = model::BuildLogMessage::Error; + + _mutex.lock(); + _buildLogs.push_back(buildLog); + _mutex.unlock(); + }); + + return false; + } + + return true; +} + +void YamlParser::chooseCoreNodeType( + YAML::Node& node_, + model::FilePtr file_, + model::YamlAstNode::SymbolType symbolType_) +{ + switch (node_.Type()) + { + case YAML::NodeType::Scalar: + processAtomicNode(node_, file_, symbolType_, + model::YamlAstNode::AstType::SCALAR); + break; + case YAML::NodeType::Sequence: + processSequence(node_, file_, symbolType_); + break; + case YAML::NodeType::Map: + processMap(node_, file_, symbolType_); + break; + case YAML::NodeType::Null: + case YAML::NodeType::Undefined: + break; + } +} + +void YamlParser::processAtomicNode( + YAML::Node& node_, + model::FilePtr file_, + model::YamlAstNode::SymbolType symbolType_, + model::YamlAstNode::AstType astType_) +{ + (util::OdbTransaction(_ctx.db))([&, this] + { + model::YamlAstNodePtr currentNode = std::make_shared(); + currentNode->astValue = YAML::Dump(node_); + currentNode->location.file = _ctx.srcMgr.getFile(file_->path); + currentNode->location.range = getNodeLocation(node_); + currentNode->astType = astType_; + currentNode->symbolType = symbolType_; + currentNode->entityHash = util::fnvHash(YAML::Dump(node_)); + currentNode->id = model::createIdentifier(*currentNode); + + _mutex.lock(); + if (!std::count_if(_astNodes.begin(), _astNodes.end(), + [&](const auto& other){ + return currentNode->id == other->id; + })) + { + _astNodes.push_back(currentNode); + } + _mutex.unlock(); + }); +} + +void YamlParser::processMap( + YAML::Node& node_, + model::FilePtr file_, + model::YamlAstNode::SymbolType symbolType_) +{ + for (auto it = node_.begin(); it != node_.end(); ++it) + { + switch (it->first.Type()) + { + case YAML::NodeType::Scalar: + processAtomicNode(it->first, file_, + model::YamlAstNode::SymbolType::NestedKey, + model::YamlAstNode::AstType::MAP); + break; + case YAML::NodeType::Sequence: + processSequence(it->first, file_, symbolType_); + break; + case YAML::NodeType::Map: + processMap(it->first, file_, symbolType_); + break; + case YAML::NodeType::Null: + case YAML::NodeType::Undefined: + break; + } + + switch (it->second.Type()) + { + case YAML::NodeType::Scalar: + processAtomicNode(it->second, file_, + model::YamlAstNode::SymbolType::NestedValue, + model::YamlAstNode::AstType::MAP); + break; + case YAML::NodeType::Sequence: + processSequence(it->second, file_, symbolType_); + break; + case YAML::NodeType::Map: + processMap(it->second, file_, symbolType_); + break; + case YAML::NodeType::Null: + case YAML::NodeType::Undefined: + break; + } + } +} + +void YamlParser::processSequence( + YAML::Node& node_, + model::FilePtr file_, + model::YamlAstNode::SymbolType symbolType_) +{ + for (size_t i = 0; i < node_.size(); ++i) + { + YAML::Node temp = node_[i]; + switch (temp.Type()) + { + case YAML::NodeType::Scalar: + processAtomicNode(temp, file_, + model::YamlAstNode::SymbolType::Value, + model::YamlAstNode::AstType::SEQUENCE); + break; + case YAML::NodeType::Sequence: + processSequence(temp, file_, symbolType_); + break; + case YAML::NodeType::Map: + processMap(temp, file_, symbolType_); + break; + case YAML::NodeType::Null: + case YAML::NodeType::Undefined: + break; + } + } +} + +model::Range YamlParser::getNodeLocation(YAML::Node& node_) +{ + model::Range location; + location.start.line = node_.Mark().line + 1; + location.start.column = node_.Mark().column; + std::string nodeValue = YAML::Dump(node_); + auto count = std::count(nodeValue.begin(), nodeValue.end(), '\n'); + location.end.line = node_.Mark().line + count + 1; + + if (count > 0) + { + auto pos = nodeValue.find_last_of('\n'); + location.end.column = nodeValue.size() - pos - 1; + } + else + { + location.end.column = node_.Mark().column + nodeValue.size() + 1; + } + + return location; +} + +YamlParser::~YamlParser() +{ + (util::OdbTransaction(_ctx.db))([this]{ + util::persistAll(_astNodes, _ctx.db); + + for (model::BuildLog& log : _buildLogs) + _ctx.db->persist(log); + }); +} + +#pragma clang diagnostic push +#pragma clang diagnostic ignored "-Wreturn-type-c-linkage" +extern "C" +{ +boost::program_options::options_description getOptions() +{ + boost::program_options::options_description description("Yaml Plugin"); + return description; +} + +std::shared_ptr make(ParserContext& ctx_) +{ + return std::make_shared(ctx_); +} +} +#pragma clang diagnostic pop + +} +} diff --git a/plugins/helm/parser/src/templateanalyzer.cpp b/plugins/helm/parser/src/templateanalyzer.cpp new file mode 100644 index 000000000..b4e8ff80e --- /dev/null +++ b/plugins/helm/parser/src/templateanalyzer.cpp @@ -0,0 +1,603 @@ +#include +#include + +#include "templateanalyzer.h" + +namespace cc +{ +namespace parser +{ + +std::unordered_set TemplateAnalyzer::_edgeCache; +std::vector TemplateAnalyzer::_microserviceCache; +std::mutex TemplateAnalyzer::_edgeCacheMutex; + +TemplateAnalyzer::TemplateAnalyzer( + cc::parser::ParserContext& ctx_, + std::map& fileAstCache_) + : _ctx(ctx_), _fileAstCache(fileAstCache_) +{ + fillDependencyPairsMap(); + fillResourceTypePairsMap(); + + std::lock_guard cacheLock(_edgeCacheMutex); + if (_edgeCache.empty()) + { + util::OdbTransaction{_ctx.db}([this] + { + for (const model::MicroserviceEdge& edge : _ctx.db->query()) + { + _edgeCache.insert(edge.id); + } + }); + } + + if (_microserviceCache.empty()) + { + util::OdbTransaction{_ctx.db}([this] + { + for (const model::Microservice& service : _ctx.db->query()) + { + _microserviceCache.push_back(service); + } + }); + } + + templateCounter = 0; +} + +TemplateAnalyzer::~TemplateAnalyzer() +{ + (util::OdbTransaction(_ctx.db))([this]{ + for (model::HelmTemplate& helmTemplate : _newTemplates) + _ctx.db->persist(helmTemplate); + + for (model::MSResource& msResource : _msResources) + _ctx.db->persist(msResource); + + util::persistAll(_newEdges, _ctx.db); + }); +} + +void TemplateAnalyzer::init() +{ + (util::OdbTransaction(_ctx.db))([this]{ + std::for_each(_fileAstCache.begin(), _fileAstCache.end(), + [&, this](std::pair pair) + { + auto currentService = std::find_if(_microserviceCache.begin(), + _microserviceCache.end(), + [&](model::Microservice& service) + { + return pair.first.find(service.name) != std::string::npos; + }); + + if (currentService != _microserviceCache.end()) + visitKeyValuePairs(pair.first, pair.second, *currentService); + }); + }); +} + +bool TemplateAnalyzer::visitKeyValuePairs( + std::string path_, + YAML::Node& currentNode_, + model::Microservice& service_) +{ + typedef model::HelmTemplate::DependencyType DependencyType; + + if (!currentNode_["metadata"] || + !currentNode_["metadata"].IsMap() || + !currentNode_["kind"] || + !currentNode_["metadata"]["name"]) + return false; + + auto typeIter = _dependencyPairs.find(YAML::Dump(currentNode_["kind"])); + auto type = typeIter->second; + + if (typeIter == _dependencyPairs.end()) + { + if (YAML::Dump(currentNode_["kind"]).find("Certificate") != std::string::npos || + YAML::Dump(currentNode_["kind"]).find("InternalUserCA") != std::string::npos) + { + type = DependencyType::CERTIFICATE; + } + else + { + auto volumesNode = findKey("volumes", currentNode_); + if (volumesNode.IsDefined()) + type = DependencyType::MOUNT; + else + return false; + } + } + + switch (type) + { + case DependencyType::SERVICE: + processServiceDeps(path_, currentNode_, service_); + break; + case DependencyType::MOUNT: + processMountDeps(path_, currentNode_, service_); + break; + case DependencyType::CERTIFICATE: + processCertificateDeps(path_, currentNode_, service_); + break; + case DependencyType::RESOURCE: + case DependencyType::OTHER: + break; + } + + processResources(path_, currentNode_, service_); + processStorageResources(path_, currentNode_, service_); + + return true; +} + +void TemplateAnalyzer::processServiceDeps( + const std::string& path_, + YAML::Node& currentFile_, + model::Microservice& service_) +{ + /* --- Process Service templates --- */ + + // Find MS in database. + auto serviceIter = std::find_if(_microserviceCache.begin(), _microserviceCache.end(), + [&](const model::Microservice& service) + { + return service.name == YAML::Dump(currentFile_["metadata"]["name"]); + }); + + // Persist template data to db. + model::HelmTemplate helmTemplate;// = std::make_shared(); + helmTemplate.dependencyType = model::HelmTemplate::DependencyType::SERVICE; + + auto filePtr = _ctx.db->query_one(odb::query::path == path_); + helmTemplate.file = filePtr->id; + helmTemplate.kind = YAML::Dump(currentFile_["kind"]); + helmTemplate.name = ""; + + // If the MS is not present in the db, + // it is an external / central MS, + // and should be added to the db. + if (serviceIter == _microserviceCache.end()) + { + model::Microservice externalService; + externalService.name = YAML::Dump(currentFile_["metadata"]["name"]); + externalService.type = model::Microservice::ServiceType::EXTERNAL; + externalService.file = filePtr->id; + externalService.serviceId = createIdentifier(externalService); + _ctx.db->persist(externalService); + + helmTemplate.depends = externalService.serviceId; + } + else + { + helmTemplate.depends = serviceIter->serviceId; + } + + helmTemplate.id = createIdentifier(helmTemplate); + addHelmTemplate(helmTemplate); + addEdge(service_.serviceId, helmTemplate.depends, helmTemplate.id, helmTemplate.kind); +} + +void TemplateAnalyzer::processMountDeps( + const std::string& path_, + YAML::Node& currentFile_, + model::Microservice& service_) +{ + /* --- Processing ConfigMap templates --- */ + + //auto volumesNode = findKey("volumes", currentFile_); + std::vector nodes; + findKeys("volumes", nodes, currentFile_); + + for (auto volumesNode = nodes.begin(); volumesNode != nodes.end(); ++volumesNode) + { + if (!volumesNode->IsDefined()) + return; + + for (auto volume = volumesNode->begin(); volume != volumesNode->end(); ++volume) + { + if ((*volume)["configMap"] && (*volume)["configMap"]["name"]) + { + model::HelmTemplate helmTemplate; + helmTemplate.dependencyType = model::HelmTemplate::DependencyType::MOUNT; + helmTemplate.kind = "ConfigMap"; + auto filePtr = _ctx.db->query_one(odb::query::path == path_); + helmTemplate.file = filePtr->id; + helmTemplate.name = YAML::Dump((*volume)["configMap"]["name"]); + + auto serviceIter = std::find_if(_microserviceCache.begin(), _microserviceCache.end(), + [&](const model::Microservice &service) + { + return (YAML::Dump((*volume)["configMap"]["name"])).find(service.name) != + std::string::npos; + }); + + if (serviceIter == _microserviceCache.end()) + { + helmTemplate.depends = -1; + helmTemplate.id = createIdentifier(helmTemplate); + } + else + { + helmTemplate.depends = serviceIter->serviceId; + helmTemplate.id = createIdentifier(helmTemplate); + addEdge(service_.serviceId, helmTemplate.depends, helmTemplate.id, helmTemplate.kind); + } + + addHelmTemplate(helmTemplate); + } + else if ((*volume)["secret"] && (*volume)["secret"]["secretName"]) + { + model::HelmTemplate helmTemplate; + helmTemplate.dependencyType = model::HelmTemplate::DependencyType::MOUNT; + helmTemplate.kind = "Secret"; + auto filePtr = _ctx.db->query_one(odb::query::path == path_); + helmTemplate.file = filePtr->id; + helmTemplate.name = YAML::Dump((*volume)["secret"]["secretName"]); + + auto serviceIter = std::find_if(_microserviceCache.begin(), _microserviceCache.end(), + [&](const model::Microservice &service) + { + return (YAML::Dump((*volume)["secret"]["secretName"])).find(service.name) != + std::string::npos; + }); + + if (serviceIter == _microserviceCache.end()) + { + /*auto dependentServiceIt = std::find_if(_newTemplates.begin(), _newTemplates.end(), + [&](const model::HelmTemplate &dependentService) + { + return (YAML::Dump((*volume)["secret"]["secretName"])).find(service.name) != + std::string::npos; + });*/ + helmTemplate.depends = -1; + helmTemplate.id = createIdentifier(helmTemplate); + } + else + { + helmTemplate.depends = serviceIter->serviceId; + helmTemplate.id = createIdentifier(helmTemplate); + addEdge(service_.serviceId, helmTemplate.depends, helmTemplate.id, helmTemplate.kind); + } + + addHelmTemplate(helmTemplate); + } + } + } +} + +void TemplateAnalyzer::processCertificateDeps( + const std::string& path_, + YAML::Node& currentFile_, + model::Microservice& service_) +{ + model::HelmTemplate helmTemplate; + helmTemplate.dependencyType = model::HelmTemplate::DependencyType::CERTIFICATE; + helmTemplate.kind = YAML::Dump(currentFile_["kind"]); + auto filePtr = _ctx.db->query_one(odb::query::path == path_); + helmTemplate.file = filePtr->id; + helmTemplate.name = YAML::Dump(currentFile_["metadata"]["name"]); + + auto keyName = findKey("generatedSecretName", currentFile_); + + auto serviceIter = std::find_if(_microserviceCache.begin(), _microserviceCache.end(), + [&](const model::Microservice& service) + { + return (YAML::Dump(keyName)).find(service.name) != std::string::npos; + }); + + if (serviceIter == _microserviceCache.end()) + { + helmTemplate.depends = -1; + helmTemplate.id = createIdentifier(helmTemplate); + } + else + { + helmTemplate.depends = serviceIter->serviceId; + helmTemplate.id = createIdentifier(helmTemplate); + addEdge(service_.serviceId, helmTemplate.depends, helmTemplate.id, helmTemplate.kind); + } + + addHelmTemplate(helmTemplate); + + model::HelmTemplate secretTemplate; + secretTemplate.dependencyType = model::HelmTemplate::DependencyType::CERTIFICATE; + secretTemplate.kind = "Secret"; + secretTemplate.file = filePtr->id; + secretTemplate.name = YAML::Dump(keyName); + + if (serviceIter == _microserviceCache.end()) + { + secretTemplate.depends = -1; + secretTemplate.id = createIdentifier(secretTemplate); + } + else + { + secretTemplate.depends = serviceIter->serviceId; + secretTemplate.id = createIdentifier(secretTemplate); + addEdge(service_.serviceId, secretTemplate.depends, secretTemplate.id, secretTemplate.kind); + } + + addHelmTemplate(secretTemplate); +} + +void TemplateAnalyzer::processResources( + const std::string& path_, + YAML::Node& currentFile_, + model::Microservice& service_) +{ + /* If the resource keys are not found, return. */ + YAML::Node resourcesKey = findKey("resources", currentFile_); + if (!resourcesKey.IsDefined()) + return; + + YAML::Node requestsKey = findKey("requests", resourcesKey); + if (!requestsKey.IsDefined()) + return; + + /* Collect the resources. */ + + for (const auto& pair : requestsKey) + { + auto it = _msResourcePairs.find(YAML::Dump(pair.first)); + if (it == _msResourcePairs.end()) + return; + + model::MSResource resource; + resource.type = it->second; + resource.service = service_.serviceId; + + auto convertedAmount = convertUnit(YAML::Dump(pair.second), it->second); + resource.amount = convertedAmount.first; + resource.unit = convertedAmount.second; + + _msResources.push_back(resource); + } +} + +void TemplateAnalyzer::processStorageResources( + const std::string& path_, + YAML::Node& currentFile_, + model::Microservice& service_) +{ + /* --- Collect storage resources --- */ + YAML::Node volumeClaimsKey = findKey("volumeClaimTemplates", currentFile_); + if (!volumeClaimsKey.IsDefined()) + return; + + YAML::Node storageKey = findKey("storage", volumeClaimsKey); + if (!storageKey.IsDefined()) + { + LOG(info) << " storageKey not found"; + return; + } + + auto it = _msResourcePairs.find("storage"); + if (it == _msResourcePairs.end()) + return; + + model::MSResource resource; + resource.type = it->second; + resource.service = service_.serviceId; + + LOG(info) << YAML::Dump(storageKey); + auto convertedAmount = convertUnit(YAML::Dump(storageKey), it->second); + resource.amount = convertedAmount.first; + resource.unit = convertedAmount.second; + + _msResources.push_back(resource); +} + +std::pair TemplateAnalyzer::convertUnit( + std::string amount_, + model::MSResource::ResourceType type_) +{ + switch (type_) + { + case model::MSResource::ResourceType::CPU: + { + int m = amount_.find('m'); + if (m != std::string::npos) + { + amount_.erase(m, 1); + return std::make_pair(std::stof(amount_) / 1000, ""); + } + else + { + return std::make_pair(std::stof(amount_), ""); + } + } + + case model::MSResource::ResourceType::MEMORY: + case model::MSResource::ResourceType::STORAGE: + { + int m = amount_.find('M'); + if (m != std::string::npos) + { + amount_.erase(m, 2); + LOG(info) << std::stof(amount_); + return std::make_pair(std::stof(amount_) / 1024, "Gi"); + } + + int g = amount_.find('G'); + if (g != std::string::npos) + { + amount_.erase(g, 2); + LOG(info) << std::stof(amount_); + return std::make_pair(std::stof(amount_), "Gi"); + } + break; + } + } + + return {}; +} + +YAML::Node TemplateAnalyzer::findKey( + const std::string& key_, + const YAML::Node& node_) +{ + switch (node_.Type()) + { + case YAML::NodeType::Scalar: + case YAML::NodeType::Null: + case YAML::NodeType::Undefined: + break; + case YAML::NodeType::Sequence: + for (auto elem : node_) + if (elem.IsMap()) + return findKey(key_, elem); + break; + case YAML::NodeType::Map: + if (node_[key_]) + return node_[key_]; + else + for (auto iter = node_.begin(); iter != node_.end(); ++iter) + { + YAML::Node temp = findKey(key_, iter->second); + if (temp.IsDefined()) + return temp; + } + break; + } + + return YAML::Node(YAML::NodeType::Undefined); +} + +std::vector TemplateAnalyzer::findKeys( + const std::string& key_, + std::vector& nodes_, + YAML::Node& node_) +{ + switch (node_.Type()) + { + case YAML::NodeType::Scalar: + case YAML::NodeType::Null: + case YAML::NodeType::Undefined: + break; + case YAML::NodeType::Sequence: + for (auto elem : node_) + if (elem.IsMap()) + findKeys(key_, nodes_, elem); + break; + case YAML::NodeType::Map: + if (node_[key_]) + { + nodes_.push_back(node_[key_]); + //return findKeys(key_, nodes_, node_); + //return nodes_; + } + else + for (auto iter = node_.begin(); iter != node_.end(); ++iter) + { + findKeys(key_, nodes_, iter->second); + //if (temp.IsDefined()) + //{ + //nodes_.push_back(temp); + //return nodes_; + //} + } + break; + } + + return nodes_; +} + +void TemplateAnalyzer::addHelmTemplate(model::HelmTemplate& helmTemplate_) +{ + auto it = std::find_if(_newTemplates.begin(), _newTemplates.end(), + [&](auto& helm) + { + return helm.id == helmTemplate_.id; + }); + + if (it == _newTemplates.end()) + _newTemplates.push_back(helmTemplate_); +} + +void TemplateAnalyzer::addEdge( + const model::MicroserviceId& from_, + const model::MicroserviceId& to_, + const model::HelmTemplateId& connect_, + std::string type_) +{ + static std::mutex m; + std::lock_guard guard(m); + + model::MicroserviceEdgePtr edge = std::make_shared(); + + edge->from = std::make_shared(); + edge->from->serviceId = from_; + edge->to = std::make_shared(); + edge->to->serviceId = to_; + + edge->connection = std::make_shared(); + edge->connection->id = connect_; + + edge->type = std::move(type_); + edge->helperId = ++templateCounter; + edge->id = model::createIdentifier(*edge); + + if (_edgeCache.insert(edge->id).second) + { + _newEdges.push_back(edge); + } +} + +int TemplateAnalyzer::LCSubStr(std::string& s1, std::string& s2, int m, int n) +{ + // Create a table to store + // lengths of longest + // common suffixes of substrings. + // Note that LCSuff[i][j] contains + // length of longest common suffix + // of X[0..i-1] and Y[0..j-1]. + + int LCSuff[m + 1][n + 1]; + int result = 0; // To store length of the + // longest common substring + + /* Following steps build LCSuff[m+1][n+1] in + bottom up fashion. */ + for (int i = 0; i <= m; i++) + { + for (int j = 0; j <= n; j++) + { + // The first row and first column + // entries have no logical meaning, + // they are used only for simplicity + // of program + if (i == 0 || j == 0) + LCSuff[i][j] = 0; + + else if (s1[i - 1] == s2[j - 1]) { + LCSuff[i][j] = LCSuff[i - 1][j - 1] + 1; + result = std::max(result, LCSuff[i][j]); + } + else + LCSuff[i][j] = 0; + } + } + return result; +} + +void TemplateAnalyzer::fillDependencyPairsMap() +{ + _dependencyPairs.insert({"Service", model::HelmTemplate::DependencyType::SERVICE}); + _dependencyPairs.insert({"ConfigMap", model::HelmTemplate::DependencyType::MOUNT}); + _dependencyPairs.insert({"Secret", model::HelmTemplate::DependencyType::MOUNT}); + _dependencyPairs.insert({"Certificate", model::HelmTemplate::DependencyType::CERTIFICATE}); + _dependencyPairs.insert({"VolumeClaim", model::HelmTemplate::DependencyType::RESOURCE}); +} + +void TemplateAnalyzer::fillResourceTypePairsMap() +{ + _msResourcePairs.insert({"cpu", model::MSResource::ResourceType::CPU}); + _msResourcePairs.insert({"memory", model::MSResource::ResourceType::MEMORY}); + _msResourcePairs.insert({"storage", model::MSResource::ResourceType::STORAGE}); +} + +} +} \ No newline at end of file diff --git a/plugins/helm/parser/src/templateanalyzer.h b/plugins/helm/parser/src/templateanalyzer.h new file mode 100644 index 000000000..c32ac06cf --- /dev/null +++ b/plugins/helm/parser/src/templateanalyzer.h @@ -0,0 +1,140 @@ +#ifndef CC_PARSER_TEMPLATEANALYZER_H +#define CC_PARSER_TEMPLATEANALYZER_H + +#include "yaml-cpp/yaml.h" + +#include "model/file.h" + +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +namespace cc +{ +namespace parser +{ + +class TemplateAnalyzer +{ +public: + TemplateAnalyzer( + ParserContext& ctx_, + std::map& fileAstCache_); + + ~TemplateAnalyzer(); + + void init(); + uint64_t getTemplateCounter() { return templateCounter; } + +private: + bool visitKeyValuePairs( + std::string path_, + YAML::Node& currentFile_, + model::Microservice& service_); + + /** + * + * @param path_ The currently processed file path. + * @param currentFile_ The currently processed file as a YAML node. + * @param service_ The microservice in which the file is defined. + */ + void processServiceDeps( + const std::string& path_, + YAML::Node& currentFile_, + model::Microservice& service_); + + /** + * + * @param path_ The currently processed file path. + * @param currentFile_ The currently processed file as a YAML node. + * @param service_ The microservice in which the file is defined. + */ + void processMountDeps( + const std::string& path_, + YAML::Node& currentFile_, + model::Microservice& service_); + + /** + * + * @param path_ The currently processed file path. + * @param currentFile_ The currently processed file as a YAML node. + * @param service_ The microservice in which the file is defined. + */ + void processCertificateDeps( + const std::string& path_, + YAML::Node& currentFile_, + model::Microservice& service_); + + /** + * Collect and store the various resources that a + * cluster uses: CPU, memory, storage. + * @param path_ The currently processed file path. + * @param currentFile_ The currently processed file as a YAML node. + * @param service_ The microservice in which the file is defined. + */ + void processResources( + const std::string& path_, + YAML::Node& currentFile_, + model::Microservice& service_); + + void processStorageResources( + const std::string& path_, + YAML::Node& currentFile_, + model::Microservice& service_); + + void addHelmTemplate(model::HelmTemplate& helmTemplate_); + + void addEdge( + const model::MicroserviceId& from_, + const model::MicroserviceId& to_, + const model::HelmTemplateId& connect_, + std::string type_); + + void fillDependencyPairsMap(); + void fillResourceTypePairsMap(); + + std::pair convertUnit( + std::string amount_, + model::MSResource::ResourceType type_); + + YAML::Node findKey( + const std::string& key_, + const YAML::Node& currentFile_); + + std::vector findKeys( + const std::string& key_, + std::vector& nodes_, + YAML::Node& node_); + + int LCSubStr(std::string& s1, std::string& s2, int m, int n); + + std::map _dependencyPairs; + std::map _msResourcePairs; + + static std::unordered_set _edgeCache; + std::vector _newEdges; + std::vector _newTemplates; + uint64_t templateCounter; + + static std::vector _microserviceCache; + model::Microservice _currentService; + + std::vector _msResources; + + static std::mutex _edgeCacheMutex; + + ParserContext& _ctx; + std::map& _fileAstCache; +}; +} +} + + +#endif // CC_PARSER_TEMPLATEANALYZER_H diff --git a/plugins/helm/parser/src/valueanalyzer.cpp b/plugins/helm/parser/src/valueanalyzer.cpp new file mode 100644 index 000000000..77e341b15 --- /dev/null +++ b/plugins/helm/parser/src/valueanalyzer.cpp @@ -0,0 +1,160 @@ +#include +#include + +#include + +#include "valueanalyzer.h" + +namespace cc +{ +namespace parser +{ + +namespace fs = boost::filesystem; + +std::unordered_set ValueAnalyzer::_edgeCache; +std::vector ValueAnalyzer::_microserviceCache; +std::mutex ValueAnalyzer::_edgeCacheMutex; + +ValueAnalyzer::ValueAnalyzer( + ParserContext& ctx_, + std::map& fileAstCache_, + std::uint64_t templateIdCounter) + : _templateCounter(templateIdCounter), _ctx(ctx_), _fileAstCache(fileAstCache_) +{ + std::lock_guard cacheLock(_edgeCacheMutex); + + if (_edgeCache.empty()) + { + util::OdbTransaction{_ctx.db}([this] + { + for (const model::MicroserviceEdge& edge : _ctx.db->query()) + { + _edgeCache.insert(edge.id); + } + }); + } + + if (_microserviceCache.empty()) + { + util::OdbTransaction{_ctx.db}([this] + { + for (const model::Microservice& service : _ctx.db->query()) + { + _microserviceCache.push_back(service); + } + }); + } +} + +ValueAnalyzer::~ValueAnalyzer() +{ + _ctx.srcMgr.persistFiles(); + + (util::OdbTransaction(_ctx.db))([this]{ + for (model::HelmTemplate& helmTemplate : _newTemplates) + _ctx.db->persist(helmTemplate); + + util::persistAll(_newEdges, _ctx.db); + }); +} + +void ValueAnalyzer::init() +{ + (util::OdbTransaction(_ctx.db))([this]{ + std::for_each(_fileAstCache.begin(), _fileAstCache.end(), + [&, this](std::pair pair) + { + auto filePtr = _ctx.db->query_one(odb::query::path == pair.first); + auto currentService = std::find_if(_microserviceCache.begin(), + _microserviceCache.end(), + [&](model::Microservice& service) + { + return pair.first.find(service.name) != std::string::npos; + }); + + if (currentService != _microserviceCache.end()) + visitKeyValuePairs(pair.second, *currentService, filePtr); + }); + }); +} + +bool ValueAnalyzer::visitKeyValuePairs( + YAML::Node& currentNode_, + model::Microservice& service_, + const model::FilePtr& file_) +{ + for (auto it = currentNode_.begin(); it != currentNode_.end(); ++it) + { + if (it->second.IsDefined() && !it->second.IsScalar()) + visitKeyValuePairs(it->second, service_, file_); + else + { + std::string current(YAML::Dump(it->second)); + auto iter = std::find_if(_microserviceCache.begin(), + _microserviceCache.end(), + [&, this](const model::Microservice& other) { + return current == other.name; + }); + + if (iter != _microserviceCache.end()) + { + model::HelmTemplate helmTemplate; + helmTemplate.name = ""; + helmTemplate.dependencyType = model::HelmTemplate::DependencyType::SERVICE; + helmTemplate.depends = service_.serviceId; + helmTemplate.kind = "Service"; + helmTemplate.file = file_->id; + helmTemplate.id = createIdentifier(helmTemplate); + addHelmTemplate(helmTemplate); + + addEdge(service_.serviceId, iter->serviceId, helmTemplate.id, "Service"); + } + } + } +} + +void ValueAnalyzer::addHelmTemplate( + model::HelmTemplate& helmTemplate_) +{ + auto it = std::find_if(_newTemplates.begin(), _newTemplates.end(), + [&](auto& helm) + { + return helm.id == helmTemplate_.id; + }); + + if (it == _newTemplates.end()) + _newTemplates.push_back(helmTemplate_); +} + +void ValueAnalyzer::addEdge( + const model::MicroserviceId& from_, + const model::MicroserviceId& to_, + const model::HelmTemplateId& connect_, + std::string type_) +{ + static std::mutex m; + std::lock_guard guard(m); + + model::MicroserviceEdgePtr edge = std::make_shared(); + + edge->from = std::make_shared(); + edge->from->serviceId = from_; + edge->to = std::make_shared(); + edge->to->serviceId = to_; + + edge->connection = std::make_shared(); + edge->connection->id = connect_; + + edge->type = std::move(type_); + edge->helperId = ++_templateCounter; + edge->id = model::createIdentifier(*edge); + + if (_edgeCache.insert(edge->id).second) + { + _newEdges.push_back(edge); + } +} + +} +} \ No newline at end of file diff --git a/plugins/helm/parser/src/valueanalyzer.h b/plugins/helm/parser/src/valueanalyzer.h new file mode 100644 index 000000000..d0062adf9 --- /dev/null +++ b/plugins/helm/parser/src/valueanalyzer.h @@ -0,0 +1,71 @@ +#ifndef CC_PARSER_YAMLRELATIONCOLLECTOR_H +#define CC_PARSER_YAMLRELATIONCOLLECTOR_H + +#include "yaml-cpp/yaml.h" + +#include "model/file.h" + +#include +#include +#include +#include +#include +#include + +#include + +namespace cc +{ +namespace parser +{ + +class ValueAnalyzer +{ +public: + ValueAnalyzer( + ParserContext& ctx_, + std::map& fileAstCache_, + uint64_t templateIdCounter); + + void init(); + + ~ValueAnalyzer(); + +private: + YAML::Node findValue( + std::string value_, + YAML::Node& currentFile_); + + void addEdge( + const model::MicroserviceId& from_, + const model::MicroserviceId& to_, + const model::HelmTemplateId& connect_, + std::string type_); + + bool visitKeyValuePairs( + YAML::Node& currentNode_, + model::Microservice& service_, + const model::FilePtr& file_); + + void addHelmTemplate( + model::HelmTemplate& helmTemplate_); + + static std::unordered_set _edgeCache; + std::vector _newEdges; + std::vector _newTemplates; + uint64_t _templateCounter; + + static std::vector _microserviceCache; + model::Microservice _currentService; + + static std::mutex _edgeCacheMutex; + + //YAML::Node& _loadedFile; + ParserContext& _ctx; + std::map& _fileAstCache; +}; + +} +} + +#endif // CC_PARSER_YAMLRELATIONCOLLECTOR_H diff --git a/plugins/helm/service/CMakeLists.txt b/plugins/helm/service/CMakeLists.txt new file mode 100644 index 000000000..4b63b4d4f --- /dev/null +++ b/plugins/helm/service/CMakeLists.txt @@ -0,0 +1,62 @@ +include_directories( + include + ${CMAKE_CURRENT_BINARY_DIR}/gen-cpp + ${PROJECT_SOURCE_DIR}/util/include + ${PROJECT_SOURCE_DIR}/webserver/include + ${PROJECT_BINARY_DIR}/service/language/gen-cpp + ${PROJECT_BINARY_DIR}/service/project/gen-cpp + ${PROJECT_SOURCE_DIR}/service/project/include + ${PLUGIN_DIR}/model/include) + +include_directories(SYSTEM + ${THRIFT_LIBTHRIFT_INCLUDE_DIRS}) + +add_custom_command( + OUTPUT + ${CMAKE_CURRENT_BINARY_DIR}/gen-cpp/helm_constants.cpp + ${CMAKE_CURRENT_BINARY_DIR}/gen-cpp/helm_constants.h + ${CMAKE_CURRENT_BINARY_DIR}/gen-cpp/helm_types.cpp + ${CMAKE_CURRENT_BINARY_DIR}/gen-cpp/helm_types.h + ${CMAKE_CURRENT_BINARY_DIR}/gen-cpp/HelmService.cpp + ${CMAKE_CURRENT_BINARY_DIR}/gen-cpp/HelmService.h + ${CMAKE_CURRENT_BINARY_DIR}/gen-cpp + ${CMAKE_CURRENT_BINARY_DIR}/gen-js + COMMAND + ${THRIFT_EXECUTABLE} --gen cpp --gen js + -o ${CMAKE_CURRENT_BINARY_DIR} + -I ${PROJECT_SOURCE_DIR}/service + ${CMAKE_CURRENT_SOURCE_DIR}/helm.thrift + DEPENDS + ${CMAKE_CURRENT_SOURCE_DIR}/helm.thrift + COMMENT + "Generating Thrift for helm.thrift") + +add_library(helmthrift STATIC + ${CMAKE_CURRENT_BINARY_DIR}/gen-cpp/helm_constants.cpp + ${CMAKE_CURRENT_BINARY_DIR}/gen-cpp/helm_types.cpp + ${CMAKE_CURRENT_BINARY_DIR}/gen-cpp/HelmService.cpp) + +target_compile_options(helmthrift PUBLIC -fPIC) + +add_dependencies(helmthrift projectthrift) +add_dependencies(helmthrift languagethrift) + +add_library(helmservice SHARED + src/helmservice.cpp + src/helmfilediagram.cpp + src/plugin.cpp) + +target_compile_options(helmservice PUBLIC -Wno-unknown-pragmas) + +target_link_libraries(helmservice + util + model + helmmodel + gvc + projectservice + languagethrift + helmthrift + ${THRIFT_LIBTHRIFT_LIBRARIES}) + +install(TARGETS helmservice DESTINATION ${INSTALL_SERVICE_DIR}) +install_js_thrift() diff --git a/plugins/helm/service/helm.thrift b/plugins/helm/service/helm.thrift new file mode 100644 index 000000000..534e4f17a --- /dev/null +++ b/plugins/helm/service/helm.thrift @@ -0,0 +1,50 @@ +include "project/common.thrift" +include "project/project.thrift" +include "language/language.thrift" + +namespace cpp cc.service.language +namespace java cc.service.language + +typedef string MicroserviceId + +enum ServiceType +{ + Internal = 0, + External = 1 +} + +struct MicroserviceInfo +{ + 1: MicroserviceId serviceId, + 2: string name, + 3: common.FileId fileId + 4: ServiceType type +} + +service HelmService extends language.LanguageService +{ + map getMicroserviceDiagramTypes(1:MicroserviceId serviceId) + throws (1:common.InvalidId ex) + + string getMicroserviceDiagram( + 1:MicroserviceId serviceId, + 2:i32 diagramId) + + list getMicroserviceList( + 1:ServiceType type) + + list getMicroserviceTypes() + /** + * This function returns a JSON string which represents the file hierarcy with + * the given file in the root. Every JSON object belongs to a directory or a + * file. Such an object contains the required metrics given in metricsTypes + * parameter. The result collects only those files of which the file type is + * contained by fileTypeFilter. + */ + /*string getYamlFileDiagram( + 1:common.FileId fileId) + + string getYamlFileInfo( + 1:common.FileId fileId)*/ + +} diff --git a/plugins/helm/service/include/service/helmservice.h b/plugins/helm/service/include/service/helmservice.h new file mode 100644 index 000000000..a19c37a34 --- /dev/null +++ b/plugins/helm/service/include/service/helmservice.h @@ -0,0 +1,188 @@ +#ifndef CC_SERVICE_YAML_H +#define CC_SERVICE_YAML_H + +#include +#include + +#include + +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include + +#include + +namespace cc +{ +namespace service +{ +namespace language +{ + +class HelmServiceHandler : virtual public HelmServiceIf +{ +public: + HelmServiceHandler( + std::shared_ptr db_, + std::shared_ptr datadir_, + const cc::webserver::ServerContext& context_); + + void getFileTypes(std::vector& return_) override; + + void getAstNodeInfo( + AstNodeInfo& return_, + const core::AstNodeId& astNodeId_) override; + + void getAstNodeInfoByPosition( + AstNodeInfo& return_, + const core::FilePosition& fpos_) override; + + void getSourceText( + std::string& return_, + const core::AstNodeId& astNodeId_) override; + + void getDocumentation( + std::string& return_, + const core::AstNodeId& astNodeId_) override; + + void getProperties( + std::map& return_, + const core::AstNodeId& astNodeId_) override; + + void getDiagramTypes( + std::map& return_, + const core::AstNodeId& astNodeId_) override; + + void getDiagram( + std::string& return_, + const core::AstNodeId& astNodeId_, + const std::int32_t diagramId_) override; + + void getDiagramLegend( + std::string& return_, + const std::int32_t diagramId_) override; + + void getFileDiagramTypes( + std::map& return_, + const core::FileId& fileId_) override; + + void getFileDiagram( + std::string& return_, + const core::FileId& fileId_, + const int32_t diagramId_) override; + + void getFileDiagramLegend( + std::string& return_, + const std::int32_t diagramId_) override; + + void getReferenceTypes( + std::map& return_, + const core::AstNodeId& astNodeId) override; + + void getReferences( + std::vector& return_, + const core::AstNodeId& astNodeId_, + const std::int32_t referenceId_, + const std::vector& tags_) override; + + std::int32_t getReferenceCount( + const core::AstNodeId& astNodeId_, + const std::int32_t referenceId_) override; + + void getReferencesInFile( + std::vector& return_, + const core::AstNodeId& astNodeId_, + const std::int32_t referenceId_, + const core::FileId& fileId_, + const std::vector& tags_) override; + + void getReferencesPage( + std::vector& return_, + const core::AstNodeId& astNodeId_, + const std::int32_t referenceId_, + const std::int32_t pageSize_, + const std::int32_t pageNo_) override; + + void getFileReferenceTypes( + std::map& return_, + const core::FileId& fileId_) override; + + void getFileReferences( + std::vector& return_, + const core::FileId& fileId_, + const std::int32_t referenceId_) override; + + std::int32_t getFileReferenceCount( + const core::FileId& fileId_, + const std::int32_t referenceId_) override; + + void getSyntaxHighlight( + std::vector& return_, + const core::FileRange& range_) override; + + model::YamlAstNode queryYamlAstNode( + const core::AstNodeId& astNodeId_); + + /* --- Extending language service --- */ + void getMicroserviceDiagramTypes( + std::map& return_, + const language::MicroserviceId& serviceId_); + + void getMicroserviceDiagram( + std::string& return_, + const language::MicroserviceId & fileId_, + const int32_t diagramId_); + + void getMicroserviceList( + std::vector& return_, + ServiceType::type type_); + + void getMicroserviceTypes( + std::vector& return_); + +private: + enum DiagramType + { + YAML_FILE_INFO, + ROOT_KEYS, + MICROSERVICES, + SERVICES, + CONFIGMAPS, + SECRETS, + CERTIFICATES, + RESOURCES + }; + + inline cc::service::language::ServiceType::type convertToThriftType( + model::Microservice::ServiceType type_); + inline model::Microservice::ServiceType convertToModelType( + cc::service::language::ServiceType::type type_); + + std::shared_ptr _db; + util::OdbTransaction _transaction; + + std::shared_ptr _datadir; + const cc::webserver::ServerContext& _context; +}; + +} // yaml +} // service +} // cc + +#endif diff --git a/plugins/helm/service/src/helmfilediagram.cpp b/plugins/helm/service/src/helmfilediagram.cpp new file mode 100644 index 000000000..e2a8e7bd5 --- /dev/null +++ b/plugins/helm/service/src/helmfilediagram.cpp @@ -0,0 +1,605 @@ +#include +#include +#include + +#include + +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include + +#include "helmfilediagram.h" + +namespace cc +{ +namespace service +{ +namespace language +{ + +namespace fs = boost::filesystem; + +typedef odb::query EdgeQuery; +typedef odb::result EdgeResult; +typedef odb::query MicroserviceQuery; +typedef odb::result MicroserviceResult; +typedef odb::query MSResourceQuery; +typedef odb::result MSResourceResult; +typedef odb::query HelmTemplateQuery; +typedef odb::result HelmTemplateResult; + +HelmFileDiagram::HelmFileDiagram( + std::shared_ptr db_, + std::shared_ptr datadir_, + const cc::webserver::ServerContext& context_) + : _db(db_), + _transaction(db_), + _helmHandler(db_, datadir_, context_), + _projectHandler(db_, datadir_, context_) +{ +} + +void HelmFileDiagram::getYamlFileInfo( + util::Graph& graph_, + const core::FileId& fileId_) +{ + std::string htmlContent = ""; + _transaction([&, this](){ + + FilePathResult yamlPath = _db->query( + FilePathQuery::id == std::stoull(fileId_)); + + YamlResult yamlInfo = _db->query( + YamlQuery::file == std::stoull(fileId_)); + + YamlContentResult yamlContent = _db->query( + YamlContentQuery::file == std::stoull(fileId_)); + + core::FileInfo fileInfo; + _projectHandler.getFileInfo(fileInfo, fileId_); + util::Graph::Node node = addNode(graph_, fileInfo); + + int numOfDataPairs = yamlContent.size(); + std::string type = typeToString(yamlInfo.begin()->type); + std::string path = yamlPath.begin()->path; + + htmlContent += "

FileType: " + type + "

"; + htmlContent += "

FilePath: " + path + "

"; + htmlContent += "

Key-data pairs: " + + std::to_string(numOfDataPairs) + "

"; + + htmlContent += graphHtmlTag("p", + graphHtmlTag("strong", "Main Keys:")); + + htmlContent += "
    "; + + for (const model::YamlContent& yc : yamlContent) + { + htmlContent += graphHtmlTag("li", yc.key); + } + + htmlContent += "
"; + graph_.setNodeAttribute(node, "FileInfo", htmlContent, true); + }); +} + + +void HelmFileDiagram::getYamlFileDiagram( + util::Graph& graph_, + const core::FileId& fileId_) +{ + std::string thAttr + = "style=\"background-color:lightGray; font-weight:bold; height:50px\""; + + std::string tdAttr = "style=\"height:30px\""; + std::string table = ""; + + _transaction([&, this](){ + typedef odb::result YamlResult; + typedef odb::query YamlQuery; + + YamlResult yamlContent = _db->query( + YamlQuery::file == std::stoull(fileId_)); + + core::FileInfo fileInfo; + _projectHandler.getFileInfo(fileInfo, fileId_); + util::Graph::Node node = addNode(graph_, fileInfo); + + table += graphHtmlTag("tr", + graphHtmlTag("th", "Key", thAttr) + + graphHtmlTag("th", "Value", thAttr)); + + for (const model::YamlContent& yc : yamlContent) + { + table += graphHtmlTag("tr", + graphHtmlTag("td", yc.key, tdAttr) + + graphHtmlTag("td", yc.value, tdAttr)); + } + table.append("
"); + + graph_.setNodeAttribute(node, "content", table, true); + }); +} + +void HelmFileDiagram::getMicroserviceDiagram( + util::Graph& graph_, + const core::FileId& fileId_) +{ + core::FileInfo fileInfo; + _projectHandler.getFileInfo(fileInfo, fileId_); + util::Graph::Node currentNode = addNode(graph_, fileInfo); + + util::bfsBuild(graph_, currentNode, std::bind(&HelmFileDiagram::getMicroservices, + this, std::placeholders::_1, std::placeholders::_2), + microserviceNodeDecoration, {}, 1); +} + +std::vector HelmFileDiagram::getMicroservices( + util::Graph& graph_, + const util::Graph::Node& node_) +{ + std::vector microservices; + + _transaction([&, this] { + for (const model::Microservice& service : _db->query()) + { + microservices.push_back(addNode(graph_, service)); + } + }); + + return microservices; +} + +/* ---- Dependent microservices ---- */ + +void HelmFileDiagram::getDependentServicesDiagram( + util::Graph& graph_, + const language::MicroserviceId& serviceId_) +{ + util::Graph::Node currentNode; + + _transaction([&, this]{ + MicroserviceResult res = _db->query( + MicroserviceQuery::serviceId == std::stoull(serviceId_)); + + currentNode = addNode(graph_, *res.begin()); + }); + + std::vector serviceNodes = getDependencies(graph_, currentNode); + std::vector revServiceNodes = getRevDependencies(graph_, currentNode); +} + +std::vector HelmFileDiagram::getDependencies( + util::Graph& graph_, + const util::Graph::Node& node_) +{ + return getDependentServices(graph_, node_); +} + +std::vector HelmFileDiagram::getRevDependencies( + util::Graph& graph_, + const util::Graph::Node& node_) +{ + return getDependentServices(graph_, node_, true); +} + +std::vector HelmFileDiagram::getDependentServices( + util::Graph& graph_, + const util::Graph::Node& node_, + bool reverse_) +{ + std::vector dependencies; + std::multimap serviceIds = getDependentServiceIds(graph_, node_, reverse_); + + for (const auto& serviceId : serviceIds) + { + _transaction([&, this]{ + MicroserviceResult res = _db->query( + MicroserviceQuery::serviceId == serviceId.first); + + util::Graph::Node newNode = addNode(graph_, *res.begin()); + dependencies.push_back(newNode); + util::Graph::Edge edge; + edge = reverse_ ? graph_.createEdge(newNode, node_) : graph_.createEdge(node_, newNode); + decorateEdge(graph_, edge, {{"label", serviceId.second}}); + }); + } + + return dependencies; +} + +std::multimap HelmFileDiagram::getDependentServiceIds( + util::Graph&, + const util::Graph::Node& node_, + bool reverse_) { + std::multimap dependencies; + + _transaction([&, this] { + EdgeResult res = _db->query( + (reverse_ + ? EdgeQuery::to->serviceId + : EdgeQuery::from->serviceId) == std::stoull(node_) + && (EdgeQuery::type == "Service")); + + for (const model::MicroserviceEdge &edge: res) + { + model::MicroserviceId serviceId = reverse_ ? edge.from->serviceId : edge.to->serviceId; + if (std::to_string(serviceId) != node_) + dependencies.insert({serviceId, edge.type}); + } + }); + + return dependencies; +} + +/* ---- Generated config maps ---- */ + +void HelmFileDiagram::getConfigMapsDiagram( + util::Graph& graph_, + const language::MicroserviceId& serviceId_) +{ + util::Graph::Node currentNode; + + _transaction([&, this]{ + MicroserviceResult res = _db->query( + MicroserviceQuery::serviceId == std::stoull(serviceId_)); + + currentNode = addNode(graph_, *res.begin()); + }); + + std::vector configMapNodes = getConfigMaps(graph_, currentNode); +} + +std::vector HelmFileDiagram::getConfigMaps( + util::Graph& graph_, + const util::Graph::Node& node_) +{ + return getDependentConfigMaps(graph_, node_); +} + +std::vector HelmFileDiagram::getRevConfigMaps( + util::Graph& graph_, + const util::Graph::Node& node_) +{ + return getDependentConfigMaps(graph_, node_, true); +} + +std::vector HelmFileDiagram::getDependentConfigMaps( + util::Graph& graph_, + const util::Graph::Node& node_, + bool reverse_) +{ + std::vector dependencies; + std::multimap configMapIds = getDependentConfigMapIds(graph_, node_, reverse_); + for (const auto& configMapId : configMapIds) + { + _transaction([&, this]{ + MicroserviceResult res = _db->query( + MicroserviceQuery::serviceId == configMapId.first); + + util::Graph::Node newNode = addNode(graph_, *res.begin()); + dependencies.push_back(newNode); + util::Graph::Edge edge = graph_.createEdge(node_, newNode); + decorateEdge(graph_, edge, {{"label", configMapId.second}}); + }); + } + + return dependencies; +} + +std::multimap HelmFileDiagram::getDependentConfigMapIds( + util::Graph& graph_, + const util::Graph::Node& node_, + bool reverse_) +{ + std::multimap dependencies; + + _transaction([&, this]{ + EdgeResult res = _db->query( + (reverse_ + ? EdgeQuery::to->serviceId + : EdgeQuery::from->serviceId) == std::stoull(node_) + && (EdgeQuery::type == "ConfigMap")); + + for (const model::MicroserviceEdge& edge : res) + { + auto helm = _db->query_one( + HelmTemplateQuery::id == edge.connection->id); + model::MicroserviceId serviceId = reverse_ ? edge.from->serviceId : edge.to->serviceId; + dependencies.insert({serviceId, helm->name}); + } + }); + + return dependencies; +} + +/* ---- Secrets ---- */ + +void HelmFileDiagram::getSecretsDiagram( + util::Graph& graph_, + const language::MicroserviceId& serviceId_) +{ + util::Graph::Node currentNode; + + _transaction([&, this]{ + MicroserviceResult res = _db->query( + MicroserviceQuery::serviceId == std::stoull(serviceId_)); + + currentNode = addNode(graph_, *res.begin()); + }); + + std::vector secretNodes = getSecrets(graph_, currentNode); +} + +std::vector HelmFileDiagram::getSecrets( + util::Graph& graph_, + const util::Graph::Node& node_) +{ + return getDependentSecrets(graph_, node_); +} + +std::vector HelmFileDiagram::getRevSecrets( + util::Graph& graph_, + const util::Graph::Node& node_) +{ + return getDependentSecrets(graph_, node_, true); +} + +std::vector HelmFileDiagram::getDependentSecrets( + util::Graph& graph_, + const util::Graph::Node& node_, + bool reverse_) +{ + std::vector dependencies; + std::multimap secretIds = getDependentSecretIds(graph_, node_, reverse_); + + for (const auto& secretId : secretIds) + { + _transaction([&, this]{ + MicroserviceResult res = _db->query( + MicroserviceQuery::serviceId == secretId.first); + + util::Graph::Node newNode = addNode(graph_, *res.begin()); + dependencies.push_back(newNode); + util::Graph::Edge edge = graph_.createEdge(node_, newNode); + decorateEdge(graph_, edge, {{"label", secretId.second}}); + }); + } + + return dependencies; +} + +std::multimap HelmFileDiagram::getDependentSecretIds( + util::Graph&, + const util::Graph::Node& node_, + bool reverse_) +{ + std::multimap dependencies; + + _transaction([&, this]{ + EdgeResult res = _db->query( + (reverse_ + ? EdgeQuery::to->serviceId + : EdgeQuery::from->serviceId) == std::stoull(node_) + && (EdgeQuery::type == "Secret")); + + for (const model::MicroserviceEdge& edge : res) + { + auto helm = _db->query_one( + HelmTemplateQuery::id == edge.connection->id); + model::MicroserviceId serviceId = reverse_ ? edge.from->serviceId : edge.to->serviceId; + dependencies.insert({serviceId, helm->name}); + } + }); + + return dependencies; +} + +void HelmFileDiagram::getResourcesDiagram( + util::Graph& graph_, + const language::MicroserviceId& serviceId_) +{ + util::Graph::Node currentNode; + + _transaction([&, this]{ + MicroserviceResult res = _db->query( + MicroserviceQuery::serviceId == std::stoull(serviceId_)); + + currentNode = addNode(graph_, *res.begin()); + }); + + std::vector secretNodes = getResources(graph_, currentNode); +} + +std::vector HelmFileDiagram::getResources( + util::Graph& graph_, + const util::Graph::Node& node_) +{ + std::vector resources; + + _transaction([&, this]{ + MSResourceResult cpu = _db->query( + MSResourceQuery::service == std::stoull(node_) && + MSResourceQuery::type == model::MSResource::ResourceType::CPU); + + float cpuSum = 0.0f; + for (auto it = cpu.begin(); it != cpu.end(); ++it) + cpuSum += it->amount; + + util::Graph::Node cpuNode = addNode(graph_, model::MSResource::ResourceType::CPU, cpuSum); + resources.push_back(cpuNode); + util::Graph::Edge cpuEdge = graph_.createEdge(node_, cpuNode); + decorateEdge(graph_, cpuEdge, {{"label", resourceTypeToString(model::MSResource::ResourceType::CPU)}}); + + MSResourceResult memory = _db->query( + MSResourceQuery::service == std::stoull(node_) && + MSResourceQuery::type == model::MSResource::ResourceType::MEMORY); + + float memorySum = 0.0f; + for (auto it = memory.begin(); it != memory.end(); ++it) + memorySum += it->amount; + + util::Graph::Node memoryNode = addNode(graph_, model::MSResource::ResourceType::MEMORY, memorySum); + resources.push_back(memoryNode); + util::Graph::Edge memoryEdge = graph_.createEdge(node_, memoryNode); + decorateEdge(graph_, memoryEdge, {{"label", resourceTypeToString(model::MSResource::ResourceType::MEMORY)}}); + + MSResourceResult storage = _db->query( + MSResourceQuery::service == std::stoull(node_) && + MSResourceQuery::type == model::MSResource::ResourceType::STORAGE); + + float storageSum = 0.0f; + for (auto it = storage.begin(); it != storage.end(); ++it) + storageSum += it->amount; + + util::Graph::Node storageNode = addNode(graph_, model::MSResource::ResourceType::STORAGE, storageSum); + resources.push_back(storageNode); + util::Graph::Edge storageEdge = graph_.createEdge(node_, storageNode); + decorateEdge(graph_, storageEdge, {{"label", resourceTypeToString(model::MSResource::ResourceType::STORAGE)}}); + }); + + return resources; +} + +std::string HelmFileDiagram::graphHtmlTag( + const std::string& tag_, + const std::string& content_, + const std::string& attr_) +{ + return std::string("<") + .append(tag_) + .append(" ") + .append(attr_) + .append(">") + .append(content_) + .append(""); +} + +util::Graph::Node HelmFileDiagram::addNode( + util::Graph& graph_, + const core::FileInfo& fileInfo_) +{ + util::Graph::Node node_ = graph_.getOrCreateNode(fileInfo_.id); + graph_.setNodeAttribute(node_, "label", getLastNParts(fileInfo_.path, 3)); + + if (fileInfo_.type == model::File::DIRECTORY_TYPE) + { + decorateNode(graph_, node_, directoryNodeDecoration); + } + else if (fileInfo_.type == model::File::BINARY_TYPE) + { + decorateNode(graph_, node_, binaryFileNodeDecoration); + } + else + { + std::string ext = boost::filesystem::extension(fileInfo_.path); + std::transform(ext.begin(), ext.end(), ext.begin(), ::tolower); + + if (ext == ".yaml" || ext == ".yml") + decorateNode(graph_, node_, sourceFileNodeDecoration); + } + + return node_; +} + +util::Graph::Node HelmFileDiagram::addNode( + util::Graph& graph_, + const model::Microservice& service_) +{ + util::Graph::Node node_ = graph_.getOrCreateNode(std::to_string(service_.serviceId)); + graph_.setNodeAttribute(node_, "label", service_.name); + + decorateNode(graph_, node_, microserviceNodeDecoration); + + return node_; +} + +util::Graph::Node HelmFileDiagram::addNode( + util::Graph& graph_, + const model::MSResource::ResourceType& type_, + float amount_) +{ + util::Graph::Node node_ = graph_.getOrCreateNode(resourceTypeToString(type_)); + graph_.setNodeAttribute(node_, "label", std::to_string(std::ceil(amount_ * 100.0) / 100.0)); + + decorateNode(graph_, node_, sourceFileNodeDecoration); + + return node_; +} + +std::string HelmFileDiagram::getLastNParts( + const std::string& path_, + std::size_t n_) +{ + if (path_.empty() || n_ == 0) + return ""; + + std::size_t p; + for (p = path_.rfind('/'); + --n_ > 0 && p - 1 < path_.size(); + p = path_.rfind('/', p - 1)); + + return p > 0 && p < path_.size() ? "..." + path_.substr(p) : path_; +} + +void HelmFileDiagram::decorateNode( + util::Graph& graph_, + const util::Graph::Node& node_, + const Decoration& decoration_) const +{ + for (const auto& attr : decoration_) + graph_.setNodeAttribute(node_, attr.first, attr.second); +} + +void HelmFileDiagram::decorateEdge( + util::Graph& graph_, + const util::Graph::Edge& edge_, + const Decoration& decoration_) const +{ + for (const auto& attr : decoration_) + graph_.setEdgeAttribute(edge_, attr.first, attr.second); +} + +const HelmFileDiagram::Decoration + HelmFileDiagram::sourceFileNodeDecoration = { + {"shape", "box"}, + {"style", "filled"}, + {"fillcolor", "#116db6"}, + {"fontcolor", "white"} +}; + +const HelmFileDiagram::Decoration + HelmFileDiagram::directoryNodeDecoration = { + {"shape", "folder"} +}; + +const HelmFileDiagram::Decoration + HelmFileDiagram::binaryFileNodeDecoration = { + {"shape", "box3d"}, + {"style", "filled"}, + {"fillcolor", "#f18a21"}, + {"fontcolor", "white"} +}; + +const HelmFileDiagram::Decoration HelmFileDiagram::microserviceNodeDecoration = { + {"shape", "folder"}, + {"color", "blue"} +}; + +const HelmFileDiagram::Decoration HelmFileDiagram::dependsEdgeDecoration = { + {"label", "depends on"} +}; + +} +} +} \ No newline at end of file diff --git a/plugins/helm/service/src/helmfilediagram.h b/plugins/helm/service/src/helmfilediagram.h new file mode 100644 index 000000000..438a59fbd --- /dev/null +++ b/plugins/helm/service/src/helmfilediagram.h @@ -0,0 +1,187 @@ +#ifndef CC_SERVICE_LANGUAGE_YAMLFILEDIAGRAM_H +#define CC_SERVICE_LANGUAGE_YAMLFILEDIAGRAM_H + +#include +#include + +#include +#include +#include + +namespace cc +{ +namespace service +{ +namespace language +{ + +typedef odb::result FilePathResult; +typedef odb::query FilePathQuery; +typedef odb::query YamlQuery; +typedef odb::result YamlResult; +typedef odb::result YamlContentResult; +typedef odb::query YamlContentQuery; + +class HelmFileDiagram +{ +public: + HelmFileDiagram( + std::shared_ptr db_, + std::shared_ptr datadir_, + const cc::webserver::ServerContext& context_); + + void getYamlFileInfo( + util::Graph& graph_, + const core::FileId& fileId_); + + void getYamlFileDiagram( + util::Graph& graph_, + const core::FileId& fileId_); + + void getMicroserviceDiagram( + util::Graph& graph_, + const core::FileId& fileId_); + + void getDependentServicesDiagram( + util::Graph& graph_, + const language::MicroserviceId& serviceId_); + + void getConfigMapsDiagram( + util::Graph& graph_, + const language::MicroserviceId& serviceId_); + + void getSecretsDiagram( + util::Graph& graph_, + const language::MicroserviceId& serviceId_); + + void getResourcesDiagram( + util::Graph& graph_, + const language::MicroserviceId& serviceId_); + +private: + typedef std::vector> Decoration; + + std::vector getMicroservices( + util::Graph& graph_, + const util::Graph::Node& node_); + + std::vector getDependencies( + util::Graph& graph_, + const util::Graph::Node& node_); + + std::vector getRevDependencies( + util::Graph& graph_, + const util::Graph::Node& node_); + + /** + * This method should return the relations between + * previously detected microservices. It works if + * the user chooses a file within a microservice. + */ + std::vector getDependentServices( + util::Graph& graph_, + const util::Graph::Node& node_, + bool reverse_ = false); + + std::multimap getDependentServiceIds( + util::Graph&, + const util::Graph::Node& node_, + bool reverse_); + + /* Config map diagram functions */ + + std::vector getConfigMaps( + util::Graph& graph_, + const util::Graph::Node& node_); + + std::vector getRevConfigMaps( + util::Graph& graph_, + const util::Graph::Node& node_); + + std::vector getDependentConfigMaps( + util::Graph& graph_, + const util::Graph::Node& node_, + bool reverse_ = false); + + std::multimap getDependentConfigMapIds( + util::Graph&, + const util::Graph::Node& node_, + bool reverse_); + + /* ---- Secret diagram functions ---- */ + + std::vector getSecrets( + util::Graph& graph_, + const util::Graph::Node& node_); + + std::vector getRevSecrets( + util::Graph& graph_, + const util::Graph::Node& node_); + + std::vector getDependentSecrets( + util::Graph& graph_, + const util::Graph::Node& node_, + bool reverse_ = false); + + std::multimap getDependentSecretIds( + util::Graph&, + const util::Graph::Node& node_, + bool reverse_); + + /* ---- Resource diagram functions ---- */ + + std::vector getResources( + util::Graph& graph_, + const util::Graph::Node& node_); + + std::string graphHtmlTag( + const std::string& tag_, + const std::string& content_, + const std::string& attr_ = ""); + + util::Graph::Node addNode( + util::Graph& graph_, + const core::FileInfo& fileInfo_); + + util::Graph::Node addNode( + util::Graph& graph_, + const model::Microservice& service_); + + util::Graph::Node addNode( + util::Graph& graph_, + const model::MSResource::ResourceType& type_, + float amount_); + + std::string getLastNParts( + const std::string& path_, + std::size_t n_); + + void decorateNode( + util::Graph& graph_, + const util::Graph::Node& node_, + const Decoration& decoration_) const; + + void decorateEdge( + util::Graph& graph_, + const util::Graph::Node& node_, + const Decoration& decoration_) const; + + static const Decoration sourceFileNodeDecoration; + static const Decoration binaryFileNodeDecoration; + static const Decoration directoryNodeDecoration; + static const Decoration microserviceNodeDecoration; + + static const Decoration dependsEdgeDecoration; + + std::shared_ptr _db; + util::OdbTransaction _transaction; + HelmServiceHandler _helmHandler; + core::ProjectServiceHandler _projectHandler; +}; + +} // language +} // service +} // cc + + +#endif //CC_SERVICE_LANGUAGE_YAMLFILEDIAGRAM_H diff --git a/plugins/helm/service/src/helmservice.cpp b/plugins/helm/service/src/helmservice.cpp new file mode 100644 index 000000000..281cb297f --- /dev/null +++ b/plugins/helm/service/src/helmservice.cpp @@ -0,0 +1,452 @@ +#include +#include +#include + +#include + +#include +#include +#include + +#include "helmfilediagram.h" + +namespace +{ + typedef odb::query AstQuery; + typedef odb::result AstResult; + typedef odb::query FileQuery; + typedef odb::result FileResult; + typedef odb::query MicroserviceQuery; + typedef odb::result MicroserviceResult; + + /** + * This struct transforms a model::YamlAstNode to an AstNodeInfo Thrift + * object. + */ + struct CreateAstNodeInfo + { + typedef std::map> TagMap; + + CreateAstNodeInfo(const TagMap& tags_ = {}) : _tags(tags_) + { + } + + /** + * Returns the Thrift object for this Yaml AST node. + */ + cc::service::language::AstNodeInfo operator()( + const cc::model::YamlAstNode& astNode_) + { + cc::service::language::AstNodeInfo ret; + + ret.__set_id(std::to_string(astNode_.id)); + ret.__set_entityHash(astNode_.entityHash); + ret.__set_astNodeType(cc::model::astTypeToString(astNode_.astType)); + ret.__set_symbolType(cc::model::symbolTypeToString(astNode_.symbolType)); + ret.__set_astNodeValue(astNode_.astValue); + + ret.range.range.startpos.line = astNode_.location.range.start.line; + ret.range.range.startpos.column = astNode_.location.range.start.column; + ret.range.range.endpos.line = astNode_.location.range.end.line; + ret.range.range.endpos.column = astNode_.location.range.end.column; + + if (astNode_.location.file) + ret.range.file = std::to_string(astNode_.location.file.object_id()); + + TagMap::const_iterator it = _tags.find(astNode_.id); + if (it != _tags.end()) + ret.__set_tags(it->second); + + return ret; + } + + const std::map>& _tags; + std::shared_ptr _db; + }; +} + +namespace cc +{ +namespace service +{ +namespace language +{ + +HelmServiceHandler::HelmServiceHandler( + std::shared_ptr db_, + std::shared_ptr datadir_, + const cc::webserver::ServerContext& context_) + : _db(db_), + _transaction(db_), + _datadir(datadir_), + _context(context_) +{ +} + +void HelmServiceHandler::getFileTypes(std::vector& return_) +{ + return_.emplace_back("YAML"); + return_.emplace_back("Dir"); +} + +void HelmServiceHandler::getAstNodeInfo( + AstNodeInfo& return_, + const core::AstNodeId& astNodeId_) +{ + return_ = _transaction([this, &astNodeId_]() { + return CreateAstNodeInfo()(queryYamlAstNode(astNodeId_)); + }); +} + +void HelmServiceHandler::getAstNodeInfoByPosition( + AstNodeInfo& return_, + const core::FilePosition& fpos_) +{ + _transaction([&, this](){ + //--- Query nodes at the given position ---// + + AstResult nodes = _db->query( + AstQuery::location.file == std::stoull(fpos_.file) && + // StartPos <= Pos + ((AstQuery::location.range.start.line == fpos_.pos.line && + AstQuery::location.range.start.column <= fpos_.pos.column) || + AstQuery::location.range.start.line < fpos_.pos.line) && + // Pos < EndPos + ((AstQuery::location.range.end.line == fpos_.pos.line && + AstQuery::location.range.end.column > fpos_.pos.column) || + AstQuery::location.range.end.line > fpos_.pos.line)); + + if (!nodes.empty()) + { + model::YamlAstNode temp = *(nodes.begin()); + + return_ = _transaction([this,&temp](){ + return CreateAstNodeInfo()(temp); + }); + } + }); +} + +void HelmServiceHandler::getSourceText( + std::string& return_, + const core::AstNodeId& astNodeId_) +{ + return_ = _transaction([this, &astNodeId_](){ + model::YamlAstNode astNode = queryYamlAstNode(astNodeId_); + + if (astNode.location.file) + return cc::util::textRange( + astNode.location.file.load()->content.load()->content, + astNode.location.range.start.line, + astNode.location.range.start.column, + astNode.location.range.end.line, + astNode.location.range.end.column); + + return std::string(); + }); +} + +void HelmServiceHandler::getDocumentation( + std::string& return_, + const core::AstNodeId& astNodeId_) {} + +void HelmServiceHandler::getProperties( + std::map& return_, + const core::AstNodeId& astNodeId_) +{ + _transaction([&, this]() { + model::YamlAstNode node = queryYamlAstNode(astNodeId_); + + return_["Name"] = node.astValue; + return_["Symbol type"] = symbolTypeToString(node.symbolType); + return_["Value type"] = astTypeToString(node.astType); + }); +} + +void HelmServiceHandler::getDiagramTypes( + std::map& return_, + const core::AstNodeId& astNodeId_) {} + +void HelmServiceHandler::getDiagram( + std::string& return_, + const core::AstNodeId& astNodeId_, + const std::int32_t diagramId_) {} + +void HelmServiceHandler::getDiagramLegend( + std::string& return_, + const std::int32_t diagramId_) {} + +void HelmServiceHandler::getFileDiagramTypes( + std::map& return_, + const core::FileId& fileId_) +{ + model::FilePtr file = _transaction([&, this](){ + return _db->query_one( + FileQuery::id == std::stoull(fileId_)); + }); + + if (file) + { + if (file->type == "YAML") + { + return_["File Info"] = YAML_FILE_INFO; + return_["Root keys"] = ROOT_KEYS; + //return_["Dependencies"] = DEPENDENCIES; + } + else if (file->type == "Dir") + { + return_["Microservices"] = MICROSERVICES; + } + } +} + +void HelmServiceHandler::getFileDiagram( + std::string& return_, + const core::FileId& fileId_, + const int32_t diagramId_) +{ + HelmFileDiagram diagram(_db, _datadir, _context); + util::Graph graph; + graph.setAttribute("rankdir", "LR"); + + switch (diagramId_) + { + case ROOT_KEYS: + diagram.getYamlFileDiagram(graph, fileId_); + return_ = graph.output(util::Graph::DOT); + break; + case YAML_FILE_INFO: + diagram.getYamlFileInfo(graph, fileId_); + return_ = graph.output(util::Graph::DOT); + break; + case MICROSERVICES: + diagram.getMicroserviceDiagram(graph, fileId_); + return_ = graph.output(util::Graph::SVG); + break; + case SERVICES: + diagram.getDependentServicesDiagram(graph, fileId_); + return_ = graph.output(util::Graph::SVG); + break; + case CONFIGMAPS: + diagram.getConfigMapsDiagram(graph, fileId_); + return_ = graph.output(util::Graph::SVG); + break; + case SECRETS: + diagram.getSecretsDiagram(graph, fileId_); + return_ = graph.output(util::Graph::SVG); + break; + case CERTIFICATES: + break; + case RESOURCES: + diagram.getResourcesDiagram(graph, fileId_); + return_ = graph.output(util::Graph::SVG); + break; + } +} + +void HelmServiceHandler::getFileDiagramLegend( + std::string& return_, + const std::int32_t diagramId_) {} + +void HelmServiceHandler::getReferenceTypes( + std::map& return_, + const core::AstNodeId& astNodeId) {} + +void HelmServiceHandler::getReferences( + std::vector& return_, + const core::AstNodeId& astNodeId_, + const std::int32_t referenceId_, + const std::vector& tags_) {} + +std::int32_t HelmServiceHandler::getReferenceCount( + const core::AstNodeId& astNodeId_, + const std::int32_t referenceId_) {} + +void HelmServiceHandler::getReferencesInFile( + std::vector& return_, + const core::AstNodeId& astNodeId_, + const std::int32_t referenceId_, + const core::FileId& fileId_, + const std::vector& tags_) {} + +void HelmServiceHandler::getReferencesPage( + std::vector& return_, + const core::AstNodeId& astNodeId_, + const std::int32_t referenceId_, + const std::int32_t pageSize_, + const std::int32_t pageNo_) {} + +void HelmServiceHandler::getFileReferenceTypes( + std::map& return_, + const core::FileId& fileId_) {} + +void HelmServiceHandler::getFileReferences( + std::vector& return_, + const core::FileId& fileId_, + const std::int32_t referenceId_) {} + +std::int32_t HelmServiceHandler::getFileReferenceCount( + const core::FileId& fileId_, + const std::int32_t referenceId_) {} + +void HelmServiceHandler::getSyntaxHighlight( + std::vector& return_, + const core::FileRange& range_) +{ + std::vector content; + + _transaction([&, this]() + { + //--- Load the file content and break it into lines ---// + + model::FilePtr file = _db->query_one( + FileQuery::id == std::stoull(range_.file)); + + if (!file || !file->content.load()) + return; + + std::istringstream s(file->content->content); + std::string line; + while (std::getline(s, line)) + content.push_back(line); + + //--- Iterate over AST node elements ---// + + for (const model::YamlAstNode& node : _db->query( + AstQuery::location.file == std::stoull(range_.file) && + AstQuery::location.range.start.line >= range_.range.startpos.line && + AstQuery::location.range.end.line < range_.range.endpos.line && + AstQuery::location.range.end.line != model::Position::npos)) + { + if (node.astValue.empty()) + continue; + + for (std::size_t i = node.location.range.start.line - 1; + i < node.location.range.end.line && i < content.size(); + ++i) + { + SyntaxHighlight syntax; + syntax.range.startpos.line = i + 1; + syntax.range.startpos.column = node.location.range.start.column;//ri->position() + 1; + syntax.range.endpos.line = i + 1; + syntax.range.endpos.column = + syntax.range.startpos.column + node.astValue.length() + 1; + + std::string symbolClass = + "cm-" + model::symbolTypeToString(node.symbolType); + syntax.className = symbolClass; + + return_.push_back(std::move(syntax)); + } + } + }); +} + +/* --- Extending language service --- */ + +void HelmServiceHandler::getMicroserviceDiagramTypes( + std::map& return_, + const language::MicroserviceId& serviceId_) +{ + return_["Dependent services"] = SERVICES; + return_["Config maps"] = CONFIGMAPS; + return_["Secrets"] = SECRETS; + return_["Resource usage"] = RESOURCES; +} + +void HelmServiceHandler::getMicroserviceDiagram( + std::string& return_, + const language::MicroserviceId& serviceId_, + const int32_t diagramId_) +{ + HelmFileDiagram diagram(_db, _datadir, _context); + util::Graph graph; + graph.setAttribute("rankdir", "LR"); + + switch (diagramId_) + { + case SERVICES: + diagram.getDependentServicesDiagram(graph, serviceId_); + return_ = graph.output(util::Graph::SVG); + break; + } +} + +void HelmServiceHandler::getMicroserviceList( + std::vector& return_, + ServiceType::type type_) +{ + _transaction([&, this]() { + MicroserviceResult services = _db->query( + MicroserviceQuery::type == convertToModelType(type_)); + + for (const auto& service : services) + { + MicroserviceInfo msInfo; + msInfo.serviceId = std::to_string(service.serviceId); + msInfo.fileId = std::to_string(service.file); + msInfo.name = service.name; + msInfo.type = convertToThriftType(service.type); + + return_.push_back(msInfo); + } + + std::sort(return_.begin(), return_.end(), + [](MicroserviceInfo& lhs, MicroserviceInfo& rhs) + { + return lhs.name < rhs.name; + }); + }); +} + +void HelmServiceHandler::getMicroserviceTypes( + std::vector& return_) +{ + typedef model::Microservice::ServiceType MSServiceType; + return_.push_back(convertToThriftType(MSServiceType::INTERNAL)); + return_.push_back(convertToThriftType(MSServiceType::EXTERNAL)); +} + +inline cc::service::language::ServiceType::type HelmServiceHandler::convertToThriftType( + model::Microservice::ServiceType type_) +{ + typedef model::Microservice::ServiceType MSServiceType; + switch(type_) + { + case MSServiceType::INTERNAL: return ServiceType::Internal; + case MSServiceType::EXTERNAL: return ServiceType::External; + } +} + +inline model::Microservice::ServiceType HelmServiceHandler::convertToModelType( + cc::service::language::ServiceType::type type_) +{ + typedef model::Microservice::ServiceType MSServiceType; + switch(type_) + { + case ServiceType::Internal: return MSServiceType::INTERNAL; + case ServiceType::External: return MSServiceType::EXTERNAL; + } +} + +model::YamlAstNode HelmServiceHandler::queryYamlAstNode( + const core::AstNodeId &astNodeId_) +{ + return _transaction([&, this](){ + model::YamlAstNode node; + + if (!_db->find(std::stoull(astNodeId_), node)) + { + core::InvalidId ex; + ex.__set_msg("Invalid YamlAstNode ID"); + ex.__set_nodeid(astNodeId_); + throw ex; + } + + return node; + }); +} + +} +} +} diff --git a/plugins/helm/service/src/plugin.cpp b/plugins/helm/service/src/plugin.cpp new file mode 100644 index 000000000..7efdc6f67 --- /dev/null +++ b/plugins/helm/service/src/plugin.cpp @@ -0,0 +1,42 @@ +#include + +#include + +/* These two methods are used by the plugin manager to allow dynamic loading + of CodeCompass Service plugins. Clang (>= version 6.0) gives a warning that + these C-linkage specified methods return types that are not proper from a + C code. + + These codes are NOT to be called from any C code. The C linkage is used to + turn off the name mangling so that the dynamic loader can easily find the + symbol table needed to set the plugin up. +*/ +#pragma clang diagnostic push +#pragma clang diagnostic ignored "-Wreturn-type-c-linkage" +extern "C" +{ + boost::program_options::options_description getOptions() + { + namespace po = boost::program_options; + + po::options_description description("Helm Plugin"); + + description.add_options() + ("dummy-result", po::value()->default_value("Dummy result"), + "This value will be returned by the dummy service."); + + return description; + } + + void registerPlugin( + const cc::webserver::ServerContext& context_, + cc::webserver::PluginHandler* pluginHandler_) + { + cc::webserver::registerPluginSimple( + context_, + pluginHandler_, + CODECOMPASS_SERVICE_FACTORY_WITH_CFG(Helm, language), + "HelmService"); + } +} +#pragma clang diagnostic pop diff --git a/plugins/helm/webgui/js/helmDiagram.js b/plugins/helm/webgui/js/helmDiagram.js new file mode 100644 index 000000000..a9dcfed24 --- /dev/null +++ b/plugins/helm/webgui/js/helmDiagram.js @@ -0,0 +1,78 @@ +require([ + 'dojo/_base/declare', + 'dojo/dom-construct', + 'dojo/topic', + 'dojo/dom-style', + 'dijit/Menu', + 'dijit/MenuItem', + 'dijit/PopupMenuItem', + 'dijit/form/Button', + 'dijit/form/CheckBox', + 'dijit/form/Select', + 'dijit/layout/ContentPane', + 'codecompass/viewHandler', + 'codecompass/urlHandler', + 'codecompass/model'], +function (declare, dom, topic, style, Menu, MenuItem, PopupMenuItem, Button, CheckBox, Select, + ContentPane, viewHandler, urlHandler, model) { + + model.addService('yamlservice', 'YamlService', YamlServiceClient); + + var fileDiagramHandler = { + id : 'yaml-file-diagram-handler', + + getDiagram : function (diagramType, fileId, callback) { + model.yamlservice.getFileDiagram(fileId, diagramType, callback); + }, + + getDiagramLegend : function (diagramType) { + return model.yamlservice.getFileDiagramLegend(diagramType); + }, + + mouseOverInfo : function (diagramType, fileId) { + return { + fileId : fileId, + selection : [1,1,1,1] + }; + } + }; + + viewHandler.registerModule(fileDiagramHandler, { + type : viewHandler.moduleType.Diagram + }); + + var fileDiagrams = { + id : 'yaml-file-diagrams', + render : function (serviceInfo) { + var submenu = new Menu(); + + var diagramTypes = model.yamlservice.getMicroserviceDiagramTypes(serviceInfo); + for (let diagramType in diagramTypes) + submenu.addChild(new MenuItem({ + label : diagramType, + type : diagramType, + onClick : function () { + var that = this; + + //topic.publish('codecompass/openFile', { fileId : serviceInfo.id }); + + topic.publish('codecompass/openDiagram', { + handler : 'yaml-file-diagram-handler', + diagramType : diagramTypes[that.type], + node : serviceInfo + }); + } + })); + + if (Object.keys(diagramTypes).length !== 0) + return new PopupMenuItem({ + label : 'YAML Diagrams', + popup : submenu + }); + } + }; + + viewHandler.registerModule(fileDiagrams, { + type : viewHandler.moduleType.MicroserviceContextMenu + }); +}); \ No newline at end of file diff --git a/plugins/helm/webgui/js/helmInfoTree.js b/plugins/helm/webgui/js/helmInfoTree.js new file mode 100644 index 000000000..2b87c5ae3 --- /dev/null +++ b/plugins/helm/webgui/js/helmInfoTree.js @@ -0,0 +1,75 @@ +require([ + 'codecompass/model', + 'codecompass/viewHandler', + 'codecompass/util'], +function (model, viewHandler, util) { + + model.addService('yamlservice', 'YamlService', YamlServiceClient); + + function createRootNode(elementInfo) { + var rootLabel + = '' + + (elementInfo instanceof AstNodeInfo + ? elementInfo.symbolType + : 'File') + + ''; + + var rootValue + = '' + + (elementInfo instanceof AstNodeInfo + ? elementInfo.astNodeValue + : elementInfo.name) + + ''; + + var label + = '' + + rootLabel + ': ' + rootValue + + ''; + + return { + id : 'root', + name : label, + cssClass : 'icon-info', + hasChildren : true, + getChildren : function () { + return that._store.query({ parent : 'root' }); + } + }; + } + + var yamlInfoTree = { + id : 'yaml-infotree', + render: function (elementInfo) { + var ret = []; + console.log(elementInfo); + + ret.push(createRootNode(elementInfo)); + + if (elementInfo instanceof AstNodeInfo) { + var props = model.yamlservice.getProperties(elementInfo.id); + + for (var propName in props) { + var propId = propName.replace(/ /g, '-'); + var label + = '' + propName + ': ' + + '' + props[propName] + ''; + + ret.push({ + name : label, + parent : 'root', + nodeInfo : elementInfo, + cssClass : 'icon-' + propId, + hasChildren : false + }); + } + } + + return ret; + } + }; + + viewHandler.registerModule(yamlInfoTree, { + type : viewHandler.moduleType.InfoTree, + service : model.yamlservice + }); +}); diff --git a/plugins/helm/webgui/js/helmMenu.js b/plugins/helm/webgui/js/helmMenu.js new file mode 100644 index 000000000..a85c5e041 --- /dev/null +++ b/plugins/helm/webgui/js/helmMenu.js @@ -0,0 +1,91 @@ +require([ + 'dojo/topic', + 'dijit/Menu', + 'dijit/MenuItem', + 'dijit/PopupMenuItem', + 'codecompass/astHelper', + 'codecompass/model', + 'codecompass/urlHandler', + 'codecompass/viewHandler'], +function (topic, Menu, MenuItem, PopupMenuItem, astHelper, model, urlHandler, viewHandler) { + + model.addService('yamlservice', 'YamlService', YamlServiceClient); + + var infoTree = { + id : 'yaml-text-infotree', + render : function (nodeInfo, fileInfo) { + return new MenuItem({ + label : 'Info Tree', + onClick : function () { + if (!nodeInfo || !fileInfo) + return; + + topic.publish('codecompass/infotree', { + fileType : fileInfo.type, + elementInfo : nodeInfo + }); + + if (window.gtag) { + window.gtag ('event', 'info_tree', { + 'event_category' : urlHandler.getState('wsid'), + 'event_label' : urlHandler.getFileInfo().name + + ': ' + + nodeInfo.astNodeValue + }); + } + } + }); + } + }; + + viewHandler.registerModule(infoTree, { + type : viewHandler.moduleType.TextContextMenu, + service : model.yamlservice + }); + + var diagrams = { + id : 'yaml-text-diagrams', + render : function (nodeInfo, fileInfo) { + if (!nodeInfo || !fileInfo) + return; + + var submenu = new Menu(); + + var diagramTypes = model.yamlservice.getDiagramTypes(nodeInfo.id); + for (diagramType in diagramTypes) + submenu.addChild(new MenuItem({ + label : diagramType, + type : diagramType, + onClick : function () { + var that = this; + + topic.publish('codecompass/openDiagram', { + handler : 'yaml-ast-diagram', + diagramType : diagramTypes[that.type], + node : nodeInfo.id + }); + } + })); + + submenu.addChild(new MenuItem({ + label : "CodeBites", + onClick : function () { + topic.publish('codecompass/codebites', { + node : nodeInfo + }); + } + })); + + if (Object.keys(diagramTypes).length !== 0) + return new PopupMenuItem({ + label : 'Diagrams', + popup : submenu + }); + } + }; + + viewHandler.registerModule(diagrams, { + type : viewHandler.moduleType.TextContextMenu, + service : model.yamlservice + }); +}); diff --git a/plugins/helm/webgui/js/helmNavigator.js b/plugins/helm/webgui/js/helmNavigator.js new file mode 100644 index 000000000..6d3b0777e --- /dev/null +++ b/plugins/helm/webgui/js/helmNavigator.js @@ -0,0 +1,140 @@ +require([ + 'dojo/on', + 'dojo/query', + 'dijit/Tooltip', + 'dijit/tree/ObjectStoreModel', + 'dojo/_base/declare', + 'dojo/store/Memory', + 'dojo/store/Observable', + 'dijit/Tree', + 'dojo/topic', + 'codecompass/view/component/HtmlTree', + 'codecompass/model', + 'codecompass/viewHandler', + 'codecompass/util', + 'codecompass/view/component/ContextMenu'], + function (on, query, Tooltip, ObjectStoreModel, declare, Memory, Observable, Tree, topic, + HtmlTree, model, viewHandler, util, ContextMenu) { + + model.addService('yamlservice', 'YamlService', YamlServiceClient); + + var YamlNavigator = declare(Tree, { + _numOfMicroserviesToLoad : 20, + + constructor : function () { + var that = this; + + this._data = []; + + this._store = new Observable(new Memory({ + data: this._data, + getChildren: function (node) { + return node.getChildren ? node.getChildren(node) : []; + } + })); + + this._dataModel = new ObjectStoreModel({ + store: that._store, + query: { id: 'root' }, + mayHaveChildren: function (node) { + return node.hasChildren; + } + }); + + this._contextMenu = new ContextMenu(); + + this._data.push({ + id: 'root', + name: 'List of microservices', + cssClass: 'icon-list', + hasChildren: true, + getChildren: function () { + return that._store.query({parent: 'root'}); + } + }); + + model.yamlservice.getMicroserviceTypes().forEach(function (type) { + + that._store.put({ + id : type, + name : that.serviceTypeToString(type), + cssClass : 'icon-repository', + hasChildren : true, + loaded : true, + parent : 'root', + getChildren : function (type) { + return that.getMicroservices(type); + } + }); + }); + + this.set('model', this._dataModel); + this.set('openOnClick', false); + }, + + getMicroservices : function (serviceType) { + var that = this; + + var ret = []; + + model.yamlservice.getMicroserviceList(serviceType.id).forEach(function (service) { + ret.push({ + id: service.serviceId, + name: service.name, + cssClass: 'icon-head', + hasChildren: false + }); + }); + + return ret; + }, + + startup : function () { + this.inherited(arguments); + var that = this; + + var contextMenu = new ContextMenu({ + targetNodeIds : [this.id], + selector : '.dijitTreeNode' + }); + + on(this, '.dijitTreeNode:contextmenu', function (event) { + var serviceInfo = dijit.byNode( + query(event.target).closest('.dijitTreeNode')[0]).item.id; + console.log(serviceInfo); + + that.buildContextMenu(contextMenu, serviceInfo); + }); + }, + + buildContextMenu : function (contextMenu, serviceInfo) { + contextMenu.clear(); + + viewHandler.getModules({ + type : viewHandler.moduleType.MicroserviceContextMenu, + serviceId : serviceInfo.id + }).forEach(function (menuItem) { + var item = menuItem.render(serviceInfo); + if (item) + contextMenu.addChild(item); + }); + }, + + serviceTypeToString : function (serviceType) { + switch (serviceType) { + case ServiceType.Internal: return 'Internal'; + case ServiceType.External: return 'External'; + } + } + + }); + + var navigator = new YamlNavigator({ + id : 'yamlnavigator', + title : 'Microservice Navigator' + }); + + viewHandler.registerModule(navigator, { + type : viewHandler.moduleType.Accordion + }); +}); \ No newline at end of file diff --git a/webgui-new/package.json b/webgui-new/package.json index 706601caa..33dc7959a 100644 --- a/webgui-new/package.json +++ b/webgui-new/package.json @@ -7,7 +7,7 @@ "build": "next build && next export", "start": "NODE_ENV=production ts-node src/server.ts", "lint": "next lint", - "thrift-ts": "thrift-typescript --fallbackNamespace none" + "thrift-ts": "thrift-typescript --fallbackNamespace none --outDir generated" }, "devDependencies": { "@creditkarma/thrift-typescript": "^3.7.6", diff --git a/webgui-new/thrift-codegen.sh b/webgui-new/thrift-codegen.sh index 17787bf43..2508cafab 100755 --- a/webgui-new/thrift-codegen.sh +++ b/webgui-new/thrift-codegen.sh @@ -40,7 +40,6 @@ for file in $(find ./thrift -type f -name '*.thrift'); do done npm run thrift-ts -mv codegen generated rm -rf ./thrift for file in $(find ./generated -type f -name '*.ts'); do @@ -51,3 +50,6 @@ done sed -i 's/import \* as SearchResult from "\.\/SearchResult";/import \* as SearchRes from "\.\/SearchResult";/g' ./generated/SearchService.ts sed -i 's/SearchResult\.SearchResult/SearchRes.SearchResult/g' ./generated/SearchService.ts +# Add missing inclusion dependency in HelmService for LanguageService +sed -i '/import \* as __ROOT_NAMESPACE__ from "\.\/";/a import \* as LanguageService from "\.\/LanguageService";' ./generated/HelmService.ts +sed -i 's/__ROOT_NAMESPACE__\.LanguageService/LanguageService/g' ./generated/HelmService.ts diff --git a/webgui/scripts/codecompass/viewHandler.js b/webgui/scripts/codecompass/viewHandler.js index 08bcc4dd6..13db250da 100644 --- a/webgui/scripts/codecompass/viewHandler.js +++ b/webgui/scripts/codecompass/viewHandler.js @@ -31,6 +31,7 @@ function (lang, Deferred, model) { InfoTree : 6, /** Info tree item (eg. CppInforTree)**/ FileManagerContextMenu : 7, /** File manager context menu (eg. Diagram directory)**/ InfoPage : 8, /** Info page (eg. Credit, User Guide) **/ + MicroserviceContextMenu: 9 /** Microservice navigator context menu **/ }, /** diff --git a/webgui/style/codecompass.css b/webgui/style/codecompass.css index f0dd6898e..c03fbebe0 100644 --- a/webgui/style/codecompass.css +++ b/webgui/style/codecompass.css @@ -709,3 +709,8 @@ div.CodeMirror-linenumber { .cm-s-default .cm-Function { font-weight: bold; } .cm-s-default .cm-Type { color:rgb(43, 145, 175); } .cm-s-default .cm-Enum { color:rgb(47, 79, 79); } + +.cm-s-default .cm-Key { color: #c91010; } +.cm-s-default .cm-Value {color: #9a59b5;} +.cm-s-default .cm-NestedKey {color: #ff5500;} +.cm-s-default .cm-NestedValue {color: #9a59b5;}