ED-2069: Add OCI oss support superseed PR 1617,1615,1608,1602

ansible/artifacts-download.yml

@@ -40,3 +40,17 @@
         aws_access_key_id: "{{ cloud_artifact_storage_accountname }}"
         aws_secret_access_key: "{{ cloud_artifact_storage_secret }}"
       when: cloud_service_provider == "aws"
+
+
+    - name: download artifact from oci oss
+      include_role:
+        name: oci-cloud-storage
+        apply:
+          environment:
+           OCI_CLI_AUTH: "instance_principal"           
+        tasks_from: download.yml
+      vars:
+        local_file_or_folder_path: "{{ artifact_path }}"
+        oss_bucket_name: "{{ cloud_storage_artifacts_bucketname }}"
+        oss_object_name: "{{ artifact }}"
+      when: cloud_service_provider == "oci"

ansible/artifacts-upload.yml

@@ -41,3 +41,16 @@
         aws_access_key_id: "{{ cloud_artifact_storage_accountname }}"
         aws_secret_access_key: "{{ cloud_artifact_storage_secret }}"
       when: cloud_service_provider == "aws"
+
+    - name: upload artifact to oci oss
+      include_role:
+        name: oci-cloud-storage
+        apply:
+          environment:
+           OCI_CLI_AUTH: "instance_principal"        
+        tasks_from: upload.yml
+      vars:
+        local_file_or_folder_path: "{{ artifact_path }}"
+        oss_bucket_name: "{{ cloud_storage_artifacts_bucketname }}"
+        oss_path: "{{ artifact }}"
+      when: cloud_service_provider == "oci"

ansible/inventory/env/group_vars/all.yml

@@ -76,6 +76,10 @@ postgres:
   db_port: 5432
   db_admin_user: analytics
   db_admin_password: "{{dp_vault_pgdb_admin_password}}"
+  dp_db_name: analytics
+  dp_ssl_mode: require
+  dp_ssl: false
+  dp_sslfactory: org.postgresql.ssl.NonValidatingFactory
 
 postgres_address_space: 0.0.0.0/0 # Postgres trust address space
 

ansible/lpa_data-products_deploy.yml

@@ -7,5 +7,6 @@
   environment:
     AZURE_STORAGE_ACCOUNT: "{{sunbird_private_storage_account_name}}"
     AZURE_STORAGE_KEY: "{{sunbird_private_storage_account_key}}"
+    OCI_CLI_AUTH: "instance_principal"
   roles:
    - data-products-deploy

ansible/oci-bds-spark.provision.yml

@@ -0,0 +1,18 @@
+- hosts: local
+  become: yes
+  vars_files:
+    - "{{inventory_dir}}/secrets.yml"
+  roles:
+    - oci-bds-spark-cluster
+  tags:
+    - copy-script
+
+- hosts: bds-livy-node
+  become: yes
+  gather_facts: no
+  vars_files:
+    - "{{inventory_dir}}/secrets.yml"
+  roles:
+    - provision-oci-spark-cluster
+  tags:
+    - spark-provision

ansible/roles/analytics-bootstrap-always/meta/main.yml

@@ -1,6 +1,7 @@
 ---
 dependencies:
   - { role: jdk11 , become: yes }
-  - { role: azure-cli  , become: yes }
+  - { role: azure-cli  , become: yes, when: cloud_service_provider == "azure" }
+  - { role: oci-cli  , become: yes, when: cloud_service_provider == "oci" }
 
 

ansible/roles/analytics-bootstrap-spark/tasks/main.yml

@@ -10,6 +10,36 @@
   with_items:
     - {var: 'azure_storage_key', value: '{{ sunbird_private_storage_account_name }}'}
     - {var: 'azure_storage_secret', value: '{{ sunbird_private_storage_account_key }}'}
+
+
+- name: Adding PATH for oci cli Vars to bashrc file of spark.
+  become: yes
+  become_user: "{{ analytics_user }}"
+  lineinfile:
+    path: '{{ analytics_user_home }}/.bashrc'
+    line: 'export PATH={{ analytics_user_home }}/bin:$PATH'
+    regexp: "export PATH={{ analytics_user_home }}/bin.*"
+  when: cloud_service_provider == "oci"
+
+- name: Configure OCI cli 
+  become: yes
+  become_user: "{{ analytics_user }}"
+  file:
+    path: "{{ analytics_user_home }}/.oci"
+    state: directory  
+  when: cloud_service_provider == "oci"
+
+- name: Create OCI cli config location
+  become: yes
+  become_user: "{{ analytics_user }}"
+  template: src=oci-key.j2 dest={{ analytics_user_home }}/.oci/oci-key.pem mode=600 owner={{ analytics_user }} group={{ analytics_group }}
+  when: cloud_service_provider == "oci"
+
+- name: Create OCI cli config file
+  become: yes
+  become_user: "{{ analytics_user }}"
+  template: src=oci-cli-config.j2 dest={{ analytics_user_home }}/.oci/config mode=600 owner={{ analytics_user }} group={{ analytics_group }}
+  when: cloud_service_provider == "oci"
 
 - name: Adding ENV Vars to spark servers environment.
   become: yes

ansible/roles/analytics-bootstrap-spark/templates/oci-cli-config.j2

@@ -0,0 +1,6 @@
+[DEFAULT]
+user={{oci_cli_user_ocid }}
+fingerprint={{oci_cli_fingerprint}}
+key_file=/home/analytics/.oci/oci-key.pem
+tenancy={{oci_cli_tenancy}}
+region={{oci_cli_region}}

ansible/roles/analytics-bootstrap-spark/templates/oci-key.j2

@@ -0,0 +1 @@
+{{ oci_cli_key_content }}

ansible/roles/analytics-spark-provision/defaults/main.yml

@@ -3,7 +3,7 @@ analytics:
    soft_path: /mount/data/analytics
    base_path: /home/analytics 
 scala_version: 2.12.10
-spark_version: 3.1.3
+spark_version: 3.2.1
 model_version: "2.0" 
 spark:
   home: "{{ analytics.home }}/spark-{{ spark_version }}-bin-hadoop2.7"

ansible/roles/analytics-spark-provision/tasks/main.yml

@@ -117,28 +117,31 @@
     recurse: yes
   become: yes
 
+# kenneth changed to install Ruby 2.6 as per R.4.6.0
 - name: Install latest ruby
   become: yes
   become_user: "{{ analytics_user }}"
-  shell: "export PATH=$PATH:/home/analytics/.rvm/bin && rvm install ruby-2.5"
+  shell: "export PATH=$PATH:/home/analytics/.rvm/bin && rvm install ruby-2.6"
 
 - name: Add ruby repository
   become: yes
   apt_repository:
     repo: ppa:brightbox/ruby-ng
 
+# kenneth changed to install ruby-dev 2.6 as per R.4.6.0 - there is no ruby2.2-dev in bionic
 - name: Install latest ruby-dev
   become: yes
   apt:
-    name: "ruby2.5-dev"
+    name: "ruby2.6-dev"
     state: installed
     update_cache: true
     cache_valid_time: 3600
 
+# changed to ruby 2.6 as per R.4.6.0
 - name: Install ruby-kafka
   become: yes
   become_user: "{{ analytics_user }}"
-  shell: "bash -ilc 'export PATH=$PATH:/home/analytics/.rvm/bin && rvm --default use ruby-2.5 && gem install ruby-kafka'"
+  shell: "bash -ilc 'export PATH=$PATH:/home/analytics/.rvm/bin && rvm --default use ruby-2.6 && gem install --user-install --no-document ruby-kafka'"
 
 - name: Download Kafka-2.11
   become: yes

ansible/roles/content-snapshot-indexer/defaults/main.yml

@@ -48,6 +48,7 @@ cloud_storage:
   provider: "azure"
   account_name: "{{sunbird_public_storage_account_name}}"
   account_key: "{{sunbird_public_storage_account_key}}"
+  account_endpoint: "{{sunbird_public_storage_account_endpoint}}"
 
 cassandra:
   host: "{{lp_cassandra_host}}" ## LMS-Cassandra IP Address.

ansible/roles/content-snapshot-indexer/templates/conf/ESCloudUploader.j2

@@ -8,4 +8,5 @@ cloudStorage.container="{{ cloud_storage.container }}"
 cloudStorage.objectKey="{{ cloud_storage.object_key }}"
 cloudStorage.provider="{{ cloud_storage.provider }}"
 cloudStorage.accountName="{{ cloud_storage.account_name }}"
-cloudStorage.accountKey="{{ cloud_storage.account_key }}"
+cloudStorage.accountKey="{{ cloud_storage.account_key }}"
+cloudStorage.accountEndpoint="{{ cloud_storage.account_endpoint }}"

ansible/roles/data-products-deploy/defaults/main.yml

@@ -1,6 +1,7 @@
 analytics_user: analytics
 analytics_group: analytics
 spark_output_temp_dir: /mount/data/analytics/tmp/
+oci_install_loc: /home/{{analytics_user}}/bin/
 
 bucket: "telemetry-data-store"
 secor_bucket: "telemetry-data-store"
@@ -279,4 +280,15 @@ assessment_metric_primary_category: "{{ exhaust_job_assessment_primary_category
 
 # Default s3 variables
 sunbird_private_s3_storage_key: ""
-sunbird_private_s3_storage_secret: ""
+sunbird_private_s3_storage_secret: ""
+
+
+# jets3t s3 config, allows us to configure for s3-like object stores
+jets3t_s3_request_signature_version: "{{ s3_request_signature_version }}"
+jets3t_s3_endpoint_host: "{% if s3_storage_endpoint %}{{ s3_storage_endpoint | regex_replace('^[a-z]+://(.*)$', '\\1') }}{% endif %}"
+jets3t_s3_disable_dns_buckets: "{{ s3_path_style_access }}"
+jets3t_s3_https_only: "{{ s3_https_only }}"
+jets3t_s3_default_bucket_location: "{{ s3_default_bucket_location }}"
+
+
+spark_pg_db_name: "analytics"

ansible/roles/data-products-deploy/tasks/main.yml

@@ -5,6 +5,19 @@
   tags:
     - always
 
+- name: Ensure oci oss bucket exists
+  command: "{{oci_install_loc}}/oci os bucket get --name {{ bucket }}"
+  register: check_bucket
+  when: dp_object_store_type == "oci"
+  tags:
+    - always
+
+- name: Create oci oss bucket 
+  command: "{{oci_install_loc}}/oci os bucket create -c {{oci_bucket_compartment}} --name {{bucket}}"
+  when: dp_object_store_type == "oci" and check_bucket.rc !=0
+  tags:
+    - always
+
 - name: Copy Core Data Products
   copy: src={{ analytics_batch_module_artifact }} dest={{ analytics.home }}/models-{{ model_version }}
   tags:
@@ -14,6 +27,15 @@
   command: az storage blob upload --overwrite -c {{ bucket }} --name models-{{ model_version }}/{{ analytics_batch_module_artifact }} -f {{ analytics.home }}/models-{{ model_version }}/{{ analytics_batch_module_artifact }}
   async: 3600
   poll: 10
+  when: dp_object_store_type == "azure"
+  tags:
+    - dataproducts-spark-cluster 
+
+- name: Copy Core Data Products to oci oss
+  command: "{{oci_install_loc}}/oci os object put -bn {{ bucket }} --name models-{{ model_version }}/{{ analytics_batch_module_artifact }} --file {{ analytics.home }}/models-{{ model_version }}/{{ analytics_batch_module_artifact }} --force"
+  async: 3600
+  poll: 10
+  when: dp_object_store_type == "oci"
   tags:
     - dataproducts-spark-cluster 
 
@@ -27,8 +49,17 @@
   command: az storage blob upload --overwrite -c {{ bucket }} --name models-{{ model_version }}/data-products-1.0.jar -f {{ analytics.home }}/models-{{ model_version }}/data-products-1.0/data-products-1.0.jar
   async: 3600
   poll: 10
+  when: dp_object_store_type == "azure"
   tags:
-    - ed-dataproducts-spark-cluster      
+    - ed-dataproducts-spark-cluster   
+
+- name: Copy Ed Data Products to oci oss
+  command: "{{oci_install_loc}}/oci os object put -bn {{ bucket }} --name models-{{ model_version }}/data-products-1.0.jar --file {{ analytics.home }}/models-{{ model_version }}/data-products-1.0/data-products-1.0.jar --force"
+  async: 3600
+  poll: 10
+  when: dp_object_store_type == "oci"
+  tags:
+    - ed-dataproducts-spark-cluster     
 
 - name: Copy Framework Library
   copy: src={{ analytics_core_artifact }} dest={{ analytics.home }}/models-{{ model_version }}
@@ -39,6 +70,15 @@
   command: az storage blob upload --overwrite --debug -c {{ bucket }} --name models-{{ model_version }}/{{ analytics_core_artifact }} -f {{ analytics.home }}/models-{{ model_version }}/{{ analytics_core_artifact }}
   async: 3600
   poll: 10
+  when: dp_object_store_type == "azure"
+  tags:
+    - framework-spark-cluster
+
+- name: Copy Framework Library to oci oss
+  command: "{{oci_install_loc}}/oci os object put -bn {{ bucket }} --name models-{{ model_version }}/{{ analytics_core_artifact }} --file {{ analytics.home }}/models-{{ model_version }}/{{ analytics_core_artifact }} --force"
+  async: 3600
+  poll: 10
+  when: dp_object_store_type == "oci"
   tags:
     - framework-spark-cluster
 
@@ -51,6 +91,15 @@
   command: az storage blob upload --overwrite -c {{ bucket }} --name models-{{ model_version }}/{{ scruid_artifact }} -f {{ analytics.home }}/models-{{ model_version }}/{{ scruid_artifact }}
   async: 3600
   poll: 10
+  when: dp_object_store_type == "azure"
+  tags:
+    - framework-spark-cluster  
+
+- name: Copy Scruid Library to oci oss
+  command: "{{oci_install_loc}}/oci os object put -bn {{ bucket }} --name models-{{ model_version }}/{{ scruid_artifact }} --file {{ analytics.home }}/models-{{ model_version }}/{{ scruid_artifact }} --force"
+  async: 3600
+  poll: 10
+  when: dp_object_store_type == "oci"
   tags:
     - framework-spark-cluster  
 
@@ -87,6 +136,17 @@
     line: 'spark_output_temp_dir="/var/log/sparkapp/tmp/"'
   tags:
     - framework-spark-cluster
+  when: dp_object_store_type != "oci"
+
+- name: Update spark temp dir value for cluster
+  lineinfile:
+    path: '{{ analytics.home }}/models-{{ model_version }}/application.conf'
+    regexp: '^spark_output_temp_dir="/mount/data/analytics/tmp/"'
+    line: 'spark_output_temp_dir="/var/log/spark/"'
+  tags:
+    - framework-spark-cluster
+  when: dp_object_store_type == "oci"
+
 
 - name: Update logger kafka config for cluster
   lineinfile:
@@ -100,9 +160,18 @@
   command: az storage blob upload --overwrite -c {{ bucket }} -f {{ analytics.home }}/models-{{ model_version }}/application.conf --name models-{{ model_version }}/application.conf
   async: 3600
   poll: 10
+  when: dp_object_store_type == "azure"
   tags:
     - framework-spark-cluster    
 
+- name: Copy configuration file to oci oss
+  command: "{{oci_install_loc}}/oci os object put -bn {{ bucket }} --file {{ analytics.home }}/models-{{ model_version }}/application.conf --name models-{{ model_version }}/application.conf --force"
+  async: 3600
+  poll: 10
+  when: dp_object_store_type == "oci"
+  tags:
+    - framework-spark-cluster  
+
 - name: Copy log4j2 xml file
   template: src=log4j2.xml.j2 dest={{ analytics.home }}/models-{{ model_version }}/log4j2.xml mode=755 owner={{ analytics_user }} group={{ analytics_group }}
   tags: [ dataproducts, framework, ed-dataproducts ]
@@ -271,7 +340,7 @@
     - spark-jobs
 
 - name: Copy collection-summary ingestion spec
-  copy: src="collection-summary-ingestion-spec.json" dest={{ analytics.home }}/scripts/ mode=755 owner={{ analytics_user }} group={{ analytics_group }}
+  template: src="collection-summary-ingestion-spec.j2" dest={{ analytics.home }}/scripts/collection-summary-ingestion-spec.json mode=755 owner={{ analytics_user }} group={{ analytics_group }}
   tags:
     - ed-dataproducts
 
@@ -300,7 +369,7 @@
     - spark-jobs 
 
 - name: Copy sourcing-summary ingestion spec
-  copy: src="sourcing-ingestion-spec.json" dest={{ analytics.home }}/scripts/ mode=755 owner={{ analytics_user }} group={{ analytics_group }}
+  template: src="sourcing-ingestion-spec.j2" dest={{ analytics.home }}/scripts/sourcing-ingestion-spec.json mode=755 owner={{ analytics_user }} group={{ analytics_group }}
   tags:
     - ed-dataproducts
 
@@ -335,6 +404,24 @@
     - run-job
     - config-update
 
+- name: Copy jets3t.properties file
+  template: src=jets3t.j2 dest={{ analytics_cluster.home }}/jets3t.properties
+  delegate_to: localhost  
+  tags:
+    - replay-job
+    - run-job
+    - config-update
+
+- name: Copy JetS3t.properties to oci oss
+  command: "{{oci_install_loc}}/oci os object put -bn {{ bucket }} --name models-{{ model_version }}/jets3t.properties --file {{ analytics_cluster.home }}/jets3t.properties --content-type auto --force"
+  async: 3600
+  poll: 10
+  when: dp_object_store_type == "oci"
+  tags:
+    - replay-job
+    - run-job
+    - config-update 
+
 - name: Replay Job
   shell: "nohup {{ analytics_cluster.home }}/submit-script.sh --job {{ job_id }} --mode {{ mode }} --partitions {{ partitions }} --parallelisation {{ parallelisation }} --startDate {{ start_date }} --endDate {{ end_date }} --sparkMaster {{ sparkMaster }} --selectedPartitions {{ selected_partitions }} &"
   async: "{{ (pause_min * 60) }}"
@@ -376,10 +463,10 @@
   shell: |
     if echo "{{jobs}}" | grep 'druid'
     then
-     python {{ analytics_cluster.home }}/update-job-requests.py {{ jobs }} {{ batch_size }} druid {{env}}_report_config
+     python {{ analytics_cluster.home }}/update-job-requests.py {{ jobs }} {{ batch_size }} druid {{spark_env}}_report_config
     elif echo "{{jobs}}" | grep 'exhaust'
      then
-     python {{ analytics_cluster.home }}/update-job-requests.py {{ jobs }} {{ batch_size }} exhaust {{env}}_job_request
+     python {{ analytics_cluster.home }}/update-job-requests.py {{ jobs }} {{ batch_size }} exhaust {{spark_env}}_job_request
     fi
   tags:
     - parallel-jobs-submit