MINIFICPP-2650 - Processor c api

Extensions.md

@@ -45,9 +45,11 @@ extern "C" MinifiExtension* InitExtension(MinifiConfig* /*config*/) {
     .name = minifi::utils::toStringView(MAKESTRING(MODULE_NAME)),
     .version = minifi::utils::toStringView(minifi::AgentBuild::VERSION),
     .deinit = nullptr,
-    .user_data = nullptr
+    .user_data = nullptr,
+    .processors_count = 0,
+    .processors_ptr = nullptr
   };
-  return MinifiCreateExtension(&ext_create_info);
+  return MinifiCreateExtension(minifi::utils::toStringView(MINIFI_API_VERSION), &ext_create_info);
 }
 ```
 

cmake/Extensions.cmake

@@ -100,6 +100,64 @@ macro(register_extension extension-name extension-display-name extension-guard d
     endif()
 endmacro()
 
+macro(register_c_api_extension extension-name extension-display-name extension-guard description)
+    set(${extension-guard} ${extension-name} PARENT_SCOPE)
+    get_property(extensions GLOBAL PROPERTY EXTENSION-OPTIONS)
+    set_property(GLOBAL APPEND PROPERTY EXTENSION-OPTIONS ${extension-name})
+    target_compile_definitions(${extension-name}
+            PRIVATE "EXTENSION_NAME=${extension-name}" "EXTENSION_VERSION=${PROJECT_VERSION_MAJOR}.${PROJECT_VERSION_MINOR}.${PROJECT_VERSION_PATCH}")
+    set_target_properties(${extension-name} PROPERTIES
+            ENABLE_EXPORTS True
+            POSITION_INDEPENDENT_CODE ON)
+    if(WIN32)
+        set_target_properties(${extension-name} PROPERTIES
+                RUNTIME_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/bin"
+                ARCHIVE_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/bin"
+                WINDOWS_EXPORT_ALL_SYMBOLS TRUE)
+    else()
+        set_target_properties(${extension-name} PROPERTIES LIBRARY_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/bin")
+    endif()
+
+    if (${ARGC} GREATER 5)
+        set(ARG_FLAGS ${ARGV5})
+    endif()
+
+    if (NOT "CI_ONLY_INSTALL" IN_LIST ARG_FLAGS OR CI_BUILD)
+        get_component_name(${extension-name} component-name)
+
+        if(WIN32)
+            install(TARGETS ${extension-name} RUNTIME DESTINATION extensions COMPONENT ${component-name})
+        else()
+            if("${CMAKE_CXX_COMPILER_ID}" STREQUAL "GNU" AND NOT APPLE)
+                target_link_options(${extension-name} PRIVATE "-Wl,--disable-new-dtags")
+            endif()
+            if (APPLE)
+                set_target_properties(${extension-name} PROPERTIES INSTALL_RPATH "@loader_path")
+            else()
+                set_target_properties(${extension-name} PROPERTIES INSTALL_RPATH "$ORIGIN")
+            endif()
+            if (MINIFI_PACKAGING_TYPE STREQUAL "RPM")
+                install(TARGETS ${extension-name}
+                        LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR}/${PROJECT_NAME}/extensions/
+                        COMPONENT ${component-name})
+                set(RPM_EXPECTED_EXTENSION_LIST ${RPM_EXPECTED_EXTENSION_LIST} /usr/${CMAKE_INSTALL_LIBDIR}/${PROJECT_NAME}/extensions/lib${extension-name}.so)
+                set(RPM_EXPECTED_EXTENSION_LIST ${RPM_EXPECTED_EXTENSION_LIST} PARENT_SCOPE)
+            elseif (MINIFI_PACKAGING_TYPE STREQUAL "TGZ")
+                install(TARGETS ${extension-name} LIBRARY DESTINATION extensions COMPONENT ${component-name})
+            else()
+                message(FATAL_ERROR "Invalid MINIFI_PACKAGING_TYPE")
+            endif()
+        endif()
+    endif()
+
+    ADD_FEATURE_INFO("${extension-display-name}" ${extension-guard} "${description}")
+    mark_as_advanced(${extension-guard})
+    # check for test directory
+    if(${ARGC} GREATER 4 AND NOT "${ARGV4}" STREQUAL "")
+        register_extension_test(${ARGV4})
+    endif()
+endmacro()
+
 ### TESTING MACROS
 
 define_property(GLOBAL PROPERTY EXTENSION-TESTS

core-framework/CMakeLists.txt

@@ -1,3 +1,5 @@
+add_subdirectory(common)
+
 file(GLOB SOURCES
         src/*.cpp
         src/core/*.cpp
@@ -13,7 +15,7 @@ file(GLOB SOURCES
 
 add_minifi_library(minifi-core-framework STATIC ${SOURCES})
 target_include_directories(minifi-core-framework PUBLIC include)
-target_link_libraries(minifi-core-framework PUBLIC minifi-api ZLIB::ZLIB concurrentqueue RapidJSON spdlog Threads::Threads gsl-lite libsodium range-v3 expected-lite date::date date::tz asio magic_enum OpenSSL::Crypto OpenSSL::SSL CURL::libcurl RapidJSON)
+target_link_libraries(minifi-core-framework PUBLIC minifi-api minifi-core-framework-common ZLIB::ZLIB concurrentqueue RapidJSON spdlog Threads::Threads gsl-lite libsodium range-v3 expected-lite date::date date::tz asio magic_enum OpenSSL::Crypto OpenSSL::SSL CURL::libcurl RapidJSON)
 if(NOT WIN32)
     target_link_libraries(minifi-core-framework PUBLIC OSSP::libuuid++)
 endif()

core-framework/common/CMakeLists.txt

@@ -0,0 +1,22 @@
+file(GLOB SOURCES
+        src/*.cpp
+        src/core/*.cpp
+        src/core/extension/*.cpp
+        src/io/*.cpp
+        src/http/*.cpp
+        src/utils/*.cpp
+        src/utils/crypto/*.cpp
+        src/utils/crypto/ciphers/*.cpp
+        src/utils/crypto/property_encryption/*.cpp
+        src/utils/net/*.cpp
+        src/utils/file/*.cpp)
+
+add_minifi_library(minifi-core-framework-common STATIC ${SOURCES})
+target_include_directories(minifi-core-framework-common PUBLIC include)
+target_link_libraries(minifi-core-framework-common PUBLIC minifi-api-common ZLIB::ZLIB concurrentqueue RapidJSON spdlog Threads::Threads gsl-lite libsodium range-v3 expected-lite date::date date::tz asio magic_enum OpenSSL::Crypto OpenSSL::SSL CURL::libcurl RapidJSON)
+if(NOT WIN32)
+    target_link_libraries(minifi-core-framework-common PUBLIC OSSP::libuuid++)
+endif()
+if ("${CMAKE_CXX_COMPILER_ID}" STREQUAL "GNU" AND CMAKE_CXX_COMPILER_VERSION VERSION_LESS 9)
+    target_link_libraries(minifi-core-framework-common PUBLIC stdc++fs)
+endif()

core-framework/include/utils/Enum.h → core-framework/common/include/utils/Enum.h

@@ -18,9 +18,9 @@
 #pragma once
 
 #include <optional>
+#include <stdexcept>
 #include <string>
 #include <string_view>
-#include <stdexcept>
 
 #include "magic_enum.hpp"
 
@@ -34,10 +34,9 @@ T enumCast(std::string_view str, bool case_insensitive = false) {
   } else {
     enum_optional_value = magic_enum::enum_cast<T>(str);
   }
-  if (enum_optional_value) {
-    return enum_optional_value.value();
-  }
-  throw std::runtime_error("Cannot convert \"" + std::string(str) + "\" to enum class value of enum type \"" + std::string(magic_enum::enum_type_name<T>()) + "\"");
+  if (enum_optional_value) { return enum_optional_value.value(); }
+  throw std::runtime_error("Cannot convert \"" + std::string(str) + "\" to enum class value of enum type \"" +
+      std::string(magic_enum::enum_type_name<T>()) + "\"");
 }
 
 }  // namespace org::apache::nifi::minifi::utils

core-framework/common/include/utils/IdHash.h

@@ -0,0 +1,40 @@
+/**
+* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#pragma once
+
+#include "minifi-cpp/utils/Id.h"
+#include "utils/Hash.h"
+
+namespace std {
+template<>
+struct hash<org::apache::nifi::minifi::utils::Identifier> {
+  size_t operator()(const org::apache::nifi::minifi::utils::Identifier& id) const noexcept {
+    static_assert(sizeof(org::apache::nifi::minifi::utils::Identifier) % sizeof(size_t) == 0);
+    constexpr int slices = sizeof(org::apache::nifi::minifi::utils::Identifier) / sizeof(size_t);
+    const auto get_slice = [](const org::apache::nifi::minifi::utils::Identifier& id, size_t idx) -> size_t {
+      size_t result{};
+      memcpy(&result, reinterpret_cast<const unsigned char*>(&id.data_) + idx * sizeof(size_t), sizeof(size_t));
+      return result;
+    };
+    size_t hash = get_slice(id, 0);
+    for (size_t i = 1; i < slices; ++i) {
+      hash = org::apache::nifi::minifi::utils::hash_combine(hash, get_slice(id, i));
+    }
+    return hash;
+  }
+};
+}  // namespace std

core-framework/include/utils/StringUtils.h → core-framework/common/include/utils/StringUtils.h

@@ -452,6 +452,7 @@ nonstd::expected<T, std::error_code> parseNumber(std::string_view input) {
 
 std::string partAfterLastOccurrenceOf(std::string_view input, char delimiter);
 
+std::string snakeCaseToPascalCase(std::string_view input);
 
 template<typename ... Bases>
 struct overload : Bases ... {

core-framework/include/utils/TimeUtil.h → core-framework/common/include/utils/TimeUtil.h

@@ -33,7 +33,6 @@
 #include <memory>
 
 #include "StringUtils.h"
-#include "minifi-cpp/utils/TimeUtil.h"
 
 // libc++ doesn't define operator<=> on durations, and apparently the operator rewrite rules don't automagically make one
 #if defined(_LIBCPP_VERSION)
@@ -249,10 +248,4 @@ inline date::local_seconds roundToNextSecond(date::local_seconds tp) {
   return std::chrono::floor<std::chrono::seconds>(tp) + std::chrono::seconds(1);
 }
 
-#ifdef WIN32
-// The tzdata location is set as a global variable in date-tz library
-// We need to set it from from libminifi to effect calls made from libminifi (on Windows)
-void dateSetInstall(const std::string& install);
-#endif
-
 }  // namespace org::apache::nifi::minifi::utils::timeutils

core-framework/common/include/utils/UnicodeConversion.h

@@ -0,0 +1,37 @@
+/**
+*
+* Licensed to the Apache Software Foundation (ASF) under one or more
+* contributor license agreements.  See the NOTICE file distributed with
+* this work for additional information regarding copyright ownership.
+* The ASF licenses this file to You under the Apache License, Version 2.0
+* (the "License"); you may not use this file except in compliance with
+* the License.  You may obtain a copy of the License at
+*
+*     http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+#pragma once
+
+#include <atlbase.h>
+#include <atlconv.h>
+#include <string>
+
+namespace org::apache::nifi::minifi::utils {
+
+inline std::string to_string(const std::wstring& utf16_string) {
+  ATL::CW2A utf8_string(utf16_string.c_str(), CP_UTF8);
+  return {LPSTR{utf8_string}};
+}
+
+inline std::wstring to_wstring(const std::string& utf8_string) {
+  ATL::CA2W utf16_string(utf8_string.c_str(), CP_UTF8);
+  return {LPWSTR{utf16_string}};
+}
+
+}  // namespace org::apache::nifi::minifi::utils

core-framework/common/src/utils/Id.cpp

@@ -0,0 +1,133 @@
+/**
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "minifi-cpp/utils/Id.h"
+
+#include "utils/StringUtils.h"
+#include "minifi-cpp/utils/gsl.h"
+
+namespace org::apache::nifi::minifi::utils {
+
+Identifier::Identifier(const Data& data) : data_(data) {}
+
+Identifier& Identifier::operator=(const Data& data) {
+  data_ = data;
+  return *this;
+}
+
+Identifier& Identifier::operator=(const std::string& idStr) {
+  const auto id = Identifier::parse(idStr);
+  if (!id) {
+    throw std::runtime_error("Couldn't parse UUID");
+  }
+  *this = id.value();
+  return *this;
+}
+
+bool Identifier::isNil() const {
+  return *this == Identifier{};
+}
+
+bool Identifier::operator!=(const Identifier& other) const {
+  return !(*this == other);
+}
+
+bool Identifier::operator==(const Identifier& other) const {
+  return data_ == other.data_;
+}
+
+bool Identifier::operator<(const Identifier &other) const {
+  return data_ < other.data_;
+}
+
+SmallString<36> Identifier::to_string() const {
+  SmallString<36> uuidStr;
+  // xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx is 36 long: 16 bytes * 2 hex digits / byte + 4 hyphens
+  int byteIdx = 0;
+  int charIdx = 0;
+
+  // [xxxxxxxx]-xxxx-xxxx-xxxx-xxxxxxxxxxxx
+  while (byteIdx < 4) {
+    uuidStr[charIdx++] = hex_lut[data_[byteIdx] >> 4];
+    uuidStr[charIdx++] = hex_lut[data_[byteIdx++] & 0xf];
+  }
+  // xxxxxxxx[-]xxxx-xxxx-xxxx-xxxxxxxxxxxx
+  uuidStr[charIdx++] = '-';
+
+  // xxxxxxxx-[xxxx-xxxx-xxxx-]xxxxxxxxxxxx - 3x 2 bytes and a hyphen
+  for (int idx = 0; idx < 3; ++idx) {
+    uuidStr[charIdx++] = hex_lut[data_[byteIdx] >> 4];
+    uuidStr[charIdx++] = hex_lut[data_[byteIdx++] & 0xf];
+    uuidStr[charIdx++] = hex_lut[data_[byteIdx] >> 4];
+    uuidStr[charIdx++] = hex_lut[data_[byteIdx++] & 0xf];
+    uuidStr[charIdx++] = '-';
+  }
+
+  // xxxxxxxx-xxxx-xxxx-xxxx-[xxxxxxxxxxxx] - the rest, i.e. until byte 16
+  while (byteIdx < 16) {
+    uuidStr[charIdx++] = hex_lut[data_[byteIdx] >> 4];
+    uuidStr[charIdx++] = hex_lut[data_[byteIdx++] & 0xf];
+  }
+
+  // null terminator
+  uuidStr[charIdx] = 0;
+  return uuidStr;
+}
+
+std::optional<Identifier> Identifier::parse(const std::string &str) {
+  Identifier id;
+  // xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx is 36 long: 16 bytes * 2 hex digits / byte + 4 hyphens
+  if (str.length() != 36) return {};
+  int charIdx = 0;
+  int byteIdx = 0;
+  auto input = reinterpret_cast<const uint8_t*>(str.c_str());
+
+  // [xxxxxxxx]-xxxx-xxxx-xxxx-xxxxxxxxxxxx
+  while (byteIdx < 4) {
+    if (!parseByte(id.data_, input, charIdx, byteIdx)) return {};
+  }
+  // xxxxxxxx[-]xxxx-xxxx-xxxx-xxxxxxxxxxxx
+  if (input[charIdx++] != '-') return {};
+
+  // xxxxxxxx-[xxxx-xxxx-xxxx-]xxxxxxxxxxxx - 3x 2 bytes and a hyphen
+  for (size_t idx = 0; idx < 3; ++idx) {
+    if (!parseByte(id.data_, input, charIdx, byteIdx)) return {};
+    if (!parseByte(id.data_, input, charIdx, byteIdx)) return {};
+    if (input[charIdx++] != '-') return {};
+  }
+
+  // xxxxxxxx-xxxx-xxxx-xxxx-[xxxxxxxxxxxx] - the rest, i.e. until byte 16
+  while (byteIdx < 16) {
+    if (!parseByte(id.data_, input, charIdx, byteIdx)) return {};
+  }
+  return id;
+}
+
+bool Identifier::parseByte(Data &data, const uint8_t *input, int &charIdx, int &byteIdx) {
+  uint8_t upper = 0;
+  uint8_t lower = 0;
+  if (!string::from_hex(input[charIdx], upper)
+      || !string::from_hex(input[charIdx + 1], lower)) {
+    return false;
+  }
+  charIdx += 2;
+  data[byteIdx++] = (upper << 4) | lower;
+  return true;
+}
+
+}  // namespace org::apache::nifi::minifi::utils