Full Rewrite

.gitattributes

@@ -0,0 +1,3 @@
+# Set shell line endings to LF, even on Windows.
+# See https://help.github.com/articles/dealing-with-line-endings/
+*.sh text eol=lf

.gitignore

@@ -1,2 +1,6 @@
+# Maven
 target/
-*.swp
+
+# JetBrains
+.idea
+*.iml

assembly.xml

@@ -0,0 +1,31 @@
+<?xml version="1.0" encoding="UTF-8"?>
+
+<!--
+  Licensed under the Apache License, Version 2.0 (the "License");
+  you may not use this file except in compliance with the License.
+  You may obtain a copy of the License at
+
+  http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License.
+
+  Copyright Clairvoyant 2019
+-->
+
+<assembly>
+  <id>assemble</id>
+  <formats>
+    <format>jar</format>
+  </formats>
+  <includeBaseDirectory>false</includeBaseDirectory>
+  <fileSets>
+    <fileSet>
+      <directory>${project.basedir}/src</directory>
+      <outputDirectory>./</outputDirectory>
+    </fileSet>
+  </fileSets>
+</assembly>

pom.xml

@@ -0,0 +1,111 @@
+<?xml version="1.0" encoding="UTF-8"?>
+
+<!--
+  Licensed under the Apache License, Version 2.0 (the "License");
+  you may not use this file except in compliance with the License.
+  You may obtain a copy of the License at
+
+  http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License.
+
+  Copyright Clairvoyant 2019
+-->
+
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
+  <modelVersion>4.0.0</modelVersion>
+
+  <groupId>com.clairvoyant.csd</groupId>
+  <artifactId>AIRFLOW</artifactId>
+  <version>1.10.3</version>
+  <name>Airflow CSD</name>
+  <packaging>pom</packaging>
+
+  <build>
+    <plugins>
+      <plugin>
+        <groupId>org.apache.maven.plugins</groupId>
+        <artifactId>maven-assembly-plugin</artifactId>
+        <version>3.1.1</version>
+        <configuration>
+          <appendAssemblyId>false</appendAssemblyId>
+          <descriptors>
+            <descriptor>assembly.xml</descriptor>
+          </descriptors>
+        </configuration>
+        <executions>
+          <execution>
+            <id>make-assembly</id>
+            <phase>package</phase>
+            <goals>
+              <goal>single</goal>
+            </goals>
+          </execution>
+        </executions>
+      </plugin>
+      <plugin>
+        <groupId>com.cloudera.enterprise</groupId>
+        <artifactId>schema-validator-maven-plugin</artifactId>
+        <version>5.16.2</version>
+        <executions>
+          <execution>
+            <id>validate-schema</id>
+          </execution>
+        </executions>
+      </plugin>
+    </plugins>
+
+    <pluginManagement>
+      <plugins>
+        <plugin>
+          <groupId>com.cloudera.enterprise</groupId>
+          <artifactId>schema-validator-maven-plugin</artifactId>
+          <executions>
+            <execution>
+              <id>validate-schema</id>
+              <phase>test</phase>
+              <goals>
+                <goal>validate</goal>
+              </goals>
+              <configuration>
+                <sourceDirectory>src</sourceDirectory>
+                <strictMode>true</strictMode>
+              </configuration>
+            </execution>
+          </executions>
+        </plugin>
+      </plugins>
+    </pluginManagement>
+  </build>
+
+  <repositories>
+    <repository>
+      <id>Central</id>
+      <name>Maven Repository</name>
+      <url>https://repo1.maven.org/maven2/</url>
+    </repository>
+    <repository>
+      <id>Cloudera</id>
+      <name>Cloudera Rel Repository</name>
+      <url>https://repository.cloudera.com/content/repositories/releases/</url>
+    </repository>
+  </repositories>
+
+  <pluginRepositories>
+    <pluginRepository>
+      <id>Central</id>
+      <name>Maven Repository</name>
+      <url>https://repo1.maven.org/maven2/</url>
+    </pluginRepository>
+    <pluginRepository>
+      <id>Cloudera</id>
+      <name>Cloudera Rel Repository</name>
+      <url>https://repository.cloudera.com/content/repositories/releases/</url>
+    </pluginRepository>
+  </pluginRepositories>
+
+</project>

src/_aux/airflow.cfg

@@ -0,0 +1,179 @@
+# -*- coding: utf-8 -*-
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# Copyright Clairvoyant 2019
+#
+
+#####################################################################
+# Generated by Cloudera Manager and should not be modified directly #
+#####################################################################
+
+[core]
+dags_folder={{AIRFLOW__CORE__DAGS_FOLDER}}
+base_log_folder={{AIRFLOW__CORE__BASE_LOG_FOLDER}}
+plugins_folder={{AIRFLOW__CORE__PLUGINS_FOLDER}}
+executor=CeleryExecutor
+sql_engine_encoding=utf-8
+sql_alchemy_pool_enabled=True
+#sql_alchemy_conn={{AIRFLOW__CORE__SQL_ALCHEMY_CONN}}
+sql_alchemy_pool_size={{AIRFLOW__CORE__SQL_ALCHEMY_POOL_SIZE}}
+sql_alchemy_pool_recycle={{AIRFLOW__CORE__SQL_ALCHEMY_POOL_RECYCLE}}
+sql_alchemy_reconnect_timeout={{AIRFLOW__CORE__SQL_ALCHEMY_RECONNECT_TIMEOUT}}
+parallelism={{AIRFLOW__CORE__PARALLELISM}}
+dag_concurrency={{AIRFLOW__CORE__DAG_CONCURRENCY}}
+dags_are_paused_at_creation={{AIRFLOW__CORE__DAGS_ARE_PAUSED_AT_CREATION}}
+non_pooled_task_slot_count={{AIRFLOW__CORE__NON_POOLED_TASK_SLOT_COUNT}}
+max_active_runs_per_dag={{AIRFLOW__CORE__MAX_ACTIVE_RUNS_PER_DAG}}
+load_examples={{AIRFLOW__CORE__LOAD_EXAMPLES}}
+#fernet_key={{AIRFLOW__CORE__FERNET_KEY}}
+donot_pickle={{AIRFLOW__CORE__DONOT_PICKLE}}
+dagbag_import_timeout={{AIRFLOW__CORE__DAGBAG_IMPORT_TIMEOUT}}
+default_impersonation={{AIRFLOW__CORE__DEFAULT_IMPERSONATION}}
+security={{AIRFLOW__CORE__SECURITY}}
+secure_mode={{AIRFLOW__CORE__SECURE_MODE}}
+unit_test_mode=False
+enable_xcom_pickling=False
+killed_task_cleanup_time={{AIRFLOW__CORE__KILLED_TASK_CLEANUP_TIME}}
+dag_run_conf_overrides_params={{AIRFLOW__CORE__DAG_RUN_CONF_OVERRIDES_PARAMS}}
+worker_precheck={{AIRFLOW__CORE__WORKER_PRECHECK}}
+dag_discovery_safe_mode={{AIRFLOW__CORE__DAG_DISCOVERY_SAFE_MODE}}
+
+[cli]
+api_client={{AIRFLOW__CLI__API_CLIENT}}
+endpoint_url={{AIRFLOW__CLI__ENDPOINT_URL}}
+
+[api]
+auth_backend={{AIRFLOW__API__AUTH_BACKEND}}
+
+[lineage]
+backend={{AIRFLOW__LINEAGE__BACKEND}}
+
+[atlas]
+host={{AIRFLOW__ATLAS__HOST}}
+port={{AIRFLOW__ATLAS__PORT}}
+username={{AIRFLOW__ATLAS__USERNAME}}
+#password={{AIRFLOW__ATLAS__PASSWORD}}
+
+[operators]
+default_owner={{AIRFLOW__OPERATORS__DEFAULT_OWNER}}
+default_cpus={{AIRFLOW__OPERATORS__DEFAULT_CPUS}}
+default_ram={{AIRFLOW__OPERATORS__DEFAULT_RAM}}
+default_disk={{AIRFLOW__OPERATORS__DEFAULT_DISK}}
+default_gpus={{AIRFLOW__OPERATORS__DEFAULT_GPUS}}
+
+[hive]
+default_hive_mapred_queue={{AIRFLOW__HIVE__DEFAULT_HIVE_MAPRED_QUEUE}}
+
+[webserver]
+base_url={{AIRFLOW__WEBSERVER__BASE_URL}}
+web_server_host={{AIRFLOW__WEBSERVER__WEB_SERVER_HOST}}
+web_server_port={{AIRFLOW__WEBSERVER__WEB_SERVER_PORT}}
+web_server_ssl_cert={{AIRFLOW__WEBSERVER__WEB_SERVER_SSL_CERT}}
+web_server_ssl_key={{AIRFLOW__WEBSERVER__WEB_SERVER_SSL_KEY}}
+web_server_master_timeout={{AIRFLOW__WEBSERVER__WEB_SERVER_MASTER_TIMEOUT}}
+web_server_worker_timeout={{AIRFLOW__WEBSERVER__WEB_SERVER_WORKER_TIMEOUT}}
+worker_refresh_batch_size={{AIRFLOW__WEBSERVER__WORKER_REFRESH_BATCH_SIZE}}
+worker_refresh_interval={{AIRFLOW__WEBSERVER__WORKER_REFRESH_INTERVAL}}
+#secret_key={{AIRFLOW__WEBSERVER__SECRET_KEY}}
+workers={{AIRFLOW__WEBSERVER__WORKERS}}
+worker_class={{AIRFLOW__WEBSERVER__WORKER_CLASS}}
+expose_config={{AIRFLOW__WEBSERVER__EXPOSE_CONFIG}}
+authenticate={{AIRFLOW__WEBSERVER__AUTHENTICATE}}
+auth_backend={{AIRFLOW__WEBSERVER__AUTH_BACKEND}}
+filter_by_owner={{AIRFLOW__WEBSERVER__FILTER_BY_OWNER}}
+owner_mode={{AIRFLOW__WEBSERVER__OWNER_MODE}}
+dag_default_view={{AIRFLOW__WEBSERVER__DAG_DEFAULT_VIEW}}
+dag_orientation={{AIRFLOW__WEBSERVER__DAG_ORIENTATION}}
+log_fetch_timeout_sec={{AIRFLOW__WEBSERVER__LOG_FETCH_TIMEOUT_SEC}}
+hide_paused_dags_by_default={{AIRFLOW__WEBSERVER__HIDE_PAUSED_DAGS_BY_DEFAULT}}
+page_size={{AIRFLOW__WEBSERVER__PAGE_SIZE}}
+rbac={{AIRFLOW__WEBSERVER__RBAC}}
+navbar_color={{AIRFLOW__WEBSERVER__NAVBAR_COLOR}}
+default_dag_run_display_number={{AIRFLOW__WEBSERVER__DEFAULT_DAG_RUN_DISPLAY_NUMBER}}
+enable_proxy_fix={{AIRFLOW__WEBSERVER__ENABLE_PROXY_FIX}}
+cookie_secure={{AIRFLOW__WEBSERVER__COOKIE_SECURE}}
+cookie_samesite={{AIRFLOW__WEBSERVER__COOKIE_SAMESITE}}
+
+[email]
+email_backend={{AIRFLOW__EMAIL__EMAIL_BACKEND}}
+
+[smtp]
+smtp_host={{AIRFLOW__SMTP__SMTP_HOST}}
+smtp_port={{AIRFLOW__SMTP__SMTP_PORT}}
+smtp_starttls={{AIRFLOW__SMTP__STARTTLS}}
+smtp_ssl={{AIRFLOW__SMTP__SMTP_SSL}}
+smtp_user={{AIRFLOW__SMTP__SMTP_USER}}
+#smtp_password={{AIRFLOW__SMTP__SMTP_PASSWORD}}
+smtp_mail_from={{AIRFLOW__SMTP__SMTP_MAIL_FROM}}
+
+[celery]
+celery_app_name={{AIRFLOW__CELERY__CELERY_APP_NAME}}
+worker_autoscale={{AIRFLOW__CELERY__WORKER_AUTOSCALE}}
+worker_log_server_port={{AIRFLOW__CELERY__WORKER_LOG_SERVER_PORT}}
+#broker_url={{AIRFLOW__CELERY__BROKER_URL}}
+#result_backend={{AIRFLOW__CELERY__RESULT_BACKEND}}
+flower_host={{AIRFLOW__CELERY__FLOWER_HOST}}
+flower_url_prefix={{AIRFLOW__CELERY__FLOWER_URL_PREFIX}}
+flower_port={{AIRFLOW__CELERY__FLOWER_PORT}}
+flower_basic_auth={{AIRFLOW__CELERY__FLOWER_BASIC_AUTH}}
+default_queue={{AIRFLOW__CELERY__DEFAULT_QUEUE}}
+sync_parallelism={{AIRFLOW__CELERY__SYNC_PARALLELISM}}
+celery_config_options={{AIRFLOW__CELERY__CELERY_CONFIG_OPTIONS}}
+ssl_active={{AIRFLOW__CELERY__SSL_ACTIVE}}
+ssl_key={{AIRFLOW__CELERY__SSL_KEY}}
+ssl_cert={{AIRFLOW__CELERY__SSL_CERT}}
+ssl_cacert={{AIRFLOW__CELERY__SSL_CACERT}}
+
+[scheduler]
+job_heartbeat_sec={{AIRFLOW__SCHEDULER__JOB_HEARTBEAT_SEC}}
+scheduler_heartbeat_sec={{AIRFLOW__SCHEDULER__SCHEDULER_HEARTBEAT_SEC}}
+run_duration={{AIRFLOW__SCHEDULER__RUN_DURATION}}
+min_file_process_interval={{AIRFLOW__SCHEDULER__MIN_FILE_PROCESS_INTERVAL}}
+dag_dir_list_interval={{AIRFLOW__SCHEDULER__DAG_DIR_LIST_INTERVAL}}
+print_stats_interval={{AIRFLOW__SCHEDULER__PRINT_STATS_INTERVAL}}
+scheduler_health_check_threshold={{AIRFLOW__SCHEDULER__SCHEDULER_HEATH_CHECK_THRESHOLD}}
+child_process_log_directory={{AIRFLOW__SCHEDULER__CHILD_PROCESS_LOG_DIRECTORY}}
+scheduler_zombie_task_threshold={{AIRFLOW__SCHEDULER__SCHEDULER_ZOMBIE_TASK_THRESHOLD}}
+catchup_by_default={{AIRFLOW__SCHEDULER__CATCHUP_BY_DEFAULT}}
+max_tis_per_query={{AIRFLOW__SCHEDULER__MAX_TIS_PER_QUERY}}
+statsd_on={{AIRFLOW__SCHEDULER__STATSD_ON}}
+statsd_host={{AIRFLOW__SCHEDULER__STATSD_HOST}}
+statsd_port={{AIRFLOW__SCHEDULER__STATSD_PORT}}
+statsd_prefix={{AIRFLOW__SCHEDULER__STATSD_PREFIX}}
+max_threads={{AIRFLOW__SCHEDULER__MAX_THREADS}}
+use_job_schedule={{AIRFLOW__SCHEDULER__USE_JOB_SCHEDULE}}
+
+[ldap]
+uri={{AIRFLOW__LDAP__URI}}
+user_filter={{AIRFLOW__LDAP__USER_FILTER}}
+user_name_attr={{AIRFLOW__LDAP__USER_NAME_ATTR}}
+group_member_attr={{AIRFLOW__LDAP__GROUP_MEMBER_ATTR}}
+superuser_filter={{AIRFLOW__LDAP__SUPERUSER_FILTER}}
+data_profiler_filter={{AIRFLOW__LDAP__DATA_PROFILER_FILTER}}
+bind_user={{AIRFLOW__LDAP__BIND_USER}}
+#bind_password={{AIRFLOW__LDAP__BIND_PASSWORD}}
+basedn={{AIRFLOW__LDAP__BASEDN}}
+cacert={{AIRFLOW__LDAP__CACERT}}
+search_scope={{AIRFLOW__LDAP__SEARCH_SCOPE}}
+ignore_malformed_schema={{AIRFLOW__LDAP__IGNORE_MALFORMED_SCHEMA}}
+
+[kerberos]
+ccache={{AIRFLOW__KERBEROS__CCACHE}}
+principal={{AIRFLOW__KERBEROS__PRINCIPAL}}
+reinit_frequency={{AIRFLOW__KERBEROS__REINIT_FREQUENCY}}
+kinit_path={{AIRFLOW__KERBEROS__KINIT_PATH}}
+keytab={{AIRFLOW__KERBEROS__KEYTAB}}
+
+[admin]
+hide_sensitive_variable_fields={{AIRFLOW__ADMIN__HIDE_SENSITIVE_VARIABLE_FIELDS}}