cypienta · Nidheesh-Panchal · Jul 19, 2024 · Jul 19, 2024 · Jul 22, 2024 · Jul 22, 2024
diff --git a/docs/requirements.txt b/docs/requirements.txt
@@ -1,2 +1,5 @@
 sphinx==7.1.2
 sphinx-rtd-theme==1.3.0rc1
+sphinx-material
+sphinx-book-theme
+sphinx-toolbox
diff --git a/docs/source/conf.py b/docs/source/conf.py
@@ -3,11 +3,13 @@
 # -- Project information
 
 project = 'Cypienta'
-copyright = '2024, Cypienta'
+copyright = '2025, Cypienta'
 author = 'Cypienta'
 
-release = '0.1'
-version = '0.1.0'
+# release = '1.0'
+release = '0.9'
+# version = '1.0.0'
+version = '0.9.0'
 
 # -- General configuration
 
@@ -17,6 +19,7 @@
     'sphinx.ext.autodoc',
     'sphinx.ext.autosummary',
     'sphinx.ext.intersphinx',
+    'sphinx_toolbox.sidebar_links',
 ]
 
 intersphinx_mapping = {
@@ -29,7 +32,9 @@
 
 # -- Options for HTML output
 
-html_theme = 'sphinx_rtd_theme'
+# html_theme = 'sphinx_rtd_theme'
+# html_theme = 'sphinx_material'
+html_theme = "sphinx_book_theme"
 
 # -- Options for EPUB output
-epub_show_urls = 'footnote'
+# epub_show_urls = 'footnote'
diff --git a/docs/source/dag/dags.rst b/docs/source/dag/dags.rst
@@ -0,0 +1,69 @@
+Overview of Airflow DAGs
+=================================
+
+Functionality of DAGs
+---------------------------------
+
+The fleet of airflow DAGs will be responsible for end-to-end flow for the Cypienta Correlation Pipeline.
+
+#. **s3_trigger:**
+
+    - The DAG is triggered from the periodic schedule in the Cypienta UI.
+    - Gets the list of file in the upload folders for multiple data sources ``mapping/input/``. If the list of files is not empty then the skip_input DAG is triggered. Else, it exits.
+
+#. **skip_input:**
+
+    - The DAG is triggered by s3_trigger DAG so that the list of input files can be processed.
+    - Maintains a queue of input files to be processed by the pipeline.
+    - The DAG is triggered from pipeline_part_1, pipeline_part_2, to clear the input queue.
+
+#. **pipeline_part_1:**
+
+    - The DAG is triggered by skip_input DAG to process the input files.
+    - It triggers the following tasks in sequence:
+
+        - enrich_with_technique
+        - update_lookup_table - trigger update_lookup_table DAG
+
+    - The DAG then triggers pipeline_part_2 concurrently, one for each clustering agent to be processed.
+
+#. **pipeline_part_2:**
+
+    - The DAG is triggered by pipeline_part_1 DAG to process the batches per clustering agent.
+    - It triggers the following tasks in sequence:
+
+        - clustering part 1 - batches for clustering part 1 are ran concurrently
+
+    - The DAG then triggers pipeline_part_3 concurrently for per clustering agent to be processed. Each clustering agent DAG run will run only single batch of data in sequential manner.
+
+#. **pipeline_part_3:**
+
+    - The DAG is triggered by pipeline_part_2 DAG to process the batches in sequential order per clustering agent.
+    - It triggers the following tasks in sequence:
+
+        - clustering part 2
+        - retrigger pipeline_part_3 if there are more batches to process
+
+    - Once the current batch is processed successfully, it triggers the pipeline_part_4 DAG for the pertinent clustering agent.
+
+#. **pipeline_part_4:**
+
+    - The DAG is triggered by pipeline_part_3 DAG.
+    - It triggers the following tasks in sequence:
+
+        - flow - batches for flows are ran concurrently
+        - create campaign
+
+#. **snapshot:**
+
+    - The DAG is triggered by the pipeline_part_4 DAG to create a snapshot of the current state of the pipeline.
+    - It snapshots the database and restarts it.
+
+#. **restore:**
+
+    - This DAG is triggered by the failure callback of the pipeline_part_1, pipeline_part_2, pipeline_part_3, and pipeline_part_4 DAG.
+    - It restores the database to the last saved snapshot and restarts it.
+
+#. **update_lookup_table:**
+
+    - The DAG is triggered by the pipeline_part_1 DAG to update the lookup table for techniques.
diff --git a/docs/source/deploy_ui/airflow.rst b/docs/source/deploy_ui/airflow.rst
@@ -0,0 +1,51 @@
+Airflow Configuration and Errors
+================================
+
+Airflow Errors
+--------------
+
+1.  Go to the Cypienta UI and login with your credentials.
+
+    .. image:: resources/ui_login.png
+        :alt: Login to UI
+        :align: center
+
+
+    .. note::
+        The default credentials are present in :doc:`start_using` page.
+
+2.  On the left hand side panel, click on ``Airflow``.
+
+    .. image:: resources/bastet_airflow.png
+        :alt: Airflow
+        :align: center
+
+3. The tab ``Error`` shows the Airflow Error List that will show the errors that have occurred in the Airflow pipeline.
+
+4. Click on the ``Go to Airflow`` link at the top right to go to the Airflow UI.
+
+
+Airflow Scheduler
+-----------------
+
+1.  On the Cypienta UI, on the left hand side panel, click on ``Airflow``.
+
+    .. image:: resources/bastet_airflow.png
+        :alt: Airflow
+        :align: center
+
+3. Click on the tab ``Schedule``. The default schedule is setup using a cron tab which will trigger the Pipeline every 5 hours.
+
+    .. image:: resources/airflow_schedule.png
+        :alt: Airflow
+        :align: center
+
+4. To edit the pipeline schedule, click on the ``Edit`` button. The schedule can be edited using the cron tab format.
+
+    .. image:: resources/airflow_schedule_edit.png
+        :alt: Airflow
+        :align: center
+
+    .. note::
+
+        To learn more about cron expressions and create a schedule in cron tab, `Click here <https://crontab.cronhub.io/>`__.
diff --git a/docs/source/deploy_ui/mapping.rst b/docs/source/deploy_ui/mapping.rst
@@ -0,0 +1,94 @@
+Map Alert fields to Cypienta Internal Format
+============================================
+
+1.  Go to the Cypienta UI and login with your credentials.
+
+    .. image:: resources/ui_login.png
+        :alt: Login to UI
+        :align: center
+
+
+    .. note::
+        The default ``Username`` is ``cypienta`` and the default ``Password`` is ``cypienta``
+
+2.  On the left hand side panel, click on ``Add Alerts``.
+
+    .. image:: resources/add_alerts.png
+        :alt: Add Alerts
+        :align: center
+
+3.  Drag and drop a file or click on the drag and drop area to upload a file. Once the file is selected, click on ``Upload File``.
+
+    .. note::
+        The maximum file size that can be uploaded is 10MB. The file should be in CSV/XML/JSON format.
+
+        CSV: The CSV format file must have filename with extension as ``.csv``. The first row will be considered as the header row and there must be atleast 1 alert in the file.
+
+        JSON: The JSON format file must have filename with extension as ``.json``. The file must have a json list of alerts format and must contain atleast 1 alert.
+
+        XML: The XML format file must have filename with extension as ``.xml``. The file must have a list element for ``alerts`` and must contain atleast 1 alert.
+
+4. Once the file is uploaded, the system will automatically map and suggest the fields to the internal format.
+
+    .. image:: resources/alerts_mapping.png
+        :alt: Alerts Mapping
+        :align: center
+
+5. There are 5 required fields that must be mapped to the internal format. The fields are:
+
+    -  ``Id``: This field is used to uniquely identify the alert.
+    -  ``Time``: This field is used to represent the time of the alert and must be in datetime format.
+    -  ``Name``: Human readable alert text.
+    -  ``Src``: Source of the alerts for network alerts.
+    -  ``Dst``: Destination of the alerts for network alerts.
+
+    The remaining fields are optional and can be mapped as per the requirement.
+
+    -  ``Event_feature``
+    -  ``Node_feature``
+
+    The mapping of fields to internal is put in a interactive UI element partitioned in 2 different sections. The left section shows the fields from the uploaded file which are currently not selected for any internal mapping and the right section shows the fields that are chosen for the internal format.
+    The user may click on the plus button to map a field to the internal format. The user may also click on the minus button to remove a field from the internal format.
+    To select all the fields currently in the ``Unused fields`` section, click on ``Choose all`` button. To remove all the fields currently in the ``Chosen fields`` section, click on ``Remove all`` button.
+
+    The top 3 values for the fields in the ``Chosen fields`` section are displayed as sample values, if available.
+
+    All chosen fields for required field mappings have a drop down for selecting the priority. If user wants to select multiple options for fetching the field value for the required fields, the user may select the priority for the field. The priority is used to select the value of the field from the multiple options available in the alert. Field value with priority 1 is given the highest priority and if fount to be not empty, other values are ignored.
+
+    For node feature field mapping, each selected field will have a drop down with options ``src``, ``dst``, and ``both``. The user must select any one option for each field to associate the node feature field to the source, destination or both.
+
+
+    .. image:: resources/node_feature.png
+        :alt: Alerts Mapping
+        :align: center
+
+6. Once all the required fields have atleast 1 chosen field. Click on ``Activate mapping for ingestion`` to save the mapping and start the ingestion process.
+
+    .. image:: resources/activate_mapping.png
+        :alt: Activate Mapping
+        :align: center
+
+7. Click on ``OK`` to confirm saving the mapping.
+
+    .. image:: resources/save_mapping.png
+        :alt: Confirm Mapping
+        :align: center
+
+8. Give a unique name for the mapping and then click ``OK``. A recommended mapping name is a source type of the alerts.
+
+    .. image:: resources/save_mapping_name.png
+        :alt: Mapping Name
+        :align: center
+
+9. Once the mapping is saved, an alert with a bucket prefix will appear. Note down this bucket prefix for future use. The alerts can then directly be uploaded to the bucket prefix path to automatically map the alerts to the internal format and start the ingestion process.
+   Click on ``OK`` to close the alert.
+
+    .. image:: resources/mapping_saved.png
+        :alt: Bucket Prefix
+        :align: center
+
+10. Another alert shows that the pipeline have been triggered with the uploaded input file. Click on ``OK`` to close the alert.
+
+    .. image:: resources/pipeline_triggered.png
+        :alt: Pipeline Triggered
+        :align: center
diff --git a/docs/source/deploy_ui/resources/activate_mapping.png b/docs/source/deploy_ui/resources/activate_mapping.png
diff --git a/docs/source/deploy_ui/resources/add_alerts.png b/docs/source/deploy_ui/resources/add_alerts.png
diff --git a/docs/source/deploy_ui/resources/add_label.png b/docs/source/deploy_ui/resources/add_label.png
diff --git a/docs/source/deploy_ui/resources/add_rule.png b/docs/source/deploy_ui/resources/add_rule.png
diff --git a/docs/source/deploy_ui/resources/airflow_schedule.png b/docs/source/deploy_ui/resources/airflow_schedule.png
diff --git a/docs/source/deploy_ui/resources/airflow_schedule_edit.png b/docs/source/deploy_ui/resources/airflow_schedule_edit.png
diff --git a/docs/source/deploy_ui/resources/alerts_mapping.png b/docs/source/deploy_ui/resources/alerts_mapping.png
diff --git a/docs/source/deploy_ui/resources/bastet_airflow.png b/docs/source/deploy_ui/resources/bastet_airflow.png
diff --git a/docs/source/deploy_ui/resources/gen_ai_add_key.png b/docs/source/deploy_ui/resources/gen_ai_add_key.png
diff --git a/docs/source/deploy_ui/resources/gen_ai_create_summary.png b/docs/source/deploy_ui/resources/gen_ai_create_summary.png
diff --git a/docs/source/deploy_ui/resources/gen_ai_summary.png b/docs/source/deploy_ui/resources/gen_ai_summary.png
diff --git a/docs/source/deploy_ui/resources/mapping_saved.png b/docs/source/deploy_ui/resources/mapping_saved.png
diff --git a/docs/source/deploy_ui/resources/new_label.png b/docs/source/deploy_ui/resources/new_label.png
diff --git a/docs/source/deploy_ui/resources/new_rule.png b/docs/source/deploy_ui/resources/new_rule.png
diff --git a/docs/source/deploy_ui/resources/node_feature.png b/docs/source/deploy_ui/resources/node_feature.png
diff --git a/docs/source/deploy_ui/resources/pipeline_triggered.png b/docs/source/deploy_ui/resources/pipeline_triggered.png
diff --git a/docs/source/deploy_ui/resources/save_mapping.png b/docs/source/deploy_ui/resources/save_mapping.png
diff --git a/docs/source/deploy_ui/resources/save_mapping_name.png b/docs/source/deploy_ui/resources/save_mapping_name.png
diff --git a/docs/source/deploy_ui/resources/select_labels.png b/docs/source/deploy_ui/resources/select_labels.png
diff --git a/docs/source/deploy_ui/resources/select_rules.png b/docs/source/deploy_ui/resources/select_rules.png
diff --git a/docs/source/deploy_ui/resources/ui_tune_cluster_config.png b/docs/source/deploy_ui/resources/ui_tune_cluster_config.png
diff --git a/docs/source/deploy_ui/resources/view_labels.png b/docs/source/deploy_ui/resources/view_labels.png
diff --git a/docs/source/deploy_ui/start_using.rst b/docs/source/deploy_ui/start_using.rst
@@ -3,6 +3,10 @@ Start using Cypienta UI
 
 Once all your resources are deployed and the ECS app is up and in ``Running`` status. You can start using the Cypienta UI.
 
+.. note::
+    The Cypienta UI will include those events that have MITRE ATT&CK techniques associated with them or recognized with the ``enrich with technique`` step of the pipeline. If no techniques are associated with the events, they will not be used as part of clustering step, and will not be visible in the UI.
+
+
 Start using Cypienta UI
 -----------------------
 
@@ -26,6 +30,9 @@ Start using Cypienta UI
     .. image:: resources/home_page.png
         :alt: Home page
         :align: center
+
+    .. note::
+        The default ``Username`` is ``cypienta`` and the default ``Password`` is ``cypienta``
 
 
 How to use the Hide feature for events in UI
@@ -37,6 +44,11 @@ How to use the Hide feature for events in UI
         :alt: Campaign list
         :align: center
 
+
+    .. note::
+        The ``Campaigns`` page will show all the clusters that were created from the events. Each event will only be present in one cluster. Clusters with single event will not be visible on UI.
+        The ``Flows`` page will show all the flows that were created from the events. Each event can be present in none or more than one flow.
+
 2. Click on any campaign that you want to modify:
 
     .. image:: resources/hide_open_campaign.png
@@ -66,6 +78,7 @@ How to use the Hide feature for events in UI
 
    You can see the list of hidden events now has an event that was selected earlier and hide action was taken.
 
+
 Edit recognized techniques for events
 -------------------------------------
 
@@ -141,4 +154,84 @@ How to use "Cut Events" feature
 
     .. image:: resources/cut_completed.png
         :alt: cut completed
+        :align: center
+
+
+How to add Rules and Labels for campaigns
+-----------------------------------------
+
+1. On the left hand side panel, click on ``Cluster`` drop down and select ``Rules``.
+
+    .. image:: resources/select_rules.png
+        :alt: select rules
+        :align: center
+
+2. Click on ``Add Rule`` button to add a new rule.
+
+    .. image:: resources/add_rule.png
+        :alt: add rule
+        :align: center
+
+3. Fill in the details for the rule. Give a distinguishable name to the rule. Select the metric on which you want to set a rule.
+   Select the condition and value for the rule. Do not select any of the campaigns in the ``Campaigns`` field and click on ``Save``.
+
+    .. image:: resources/new_rule.png
+        :alt: add rule details
+        :align: center
+
+4. Now to utilize the new rule we need to add a label to the campaign. On the left hand side panel, click on ``Cluster`` drop down and select ``Labels``.
+
+    .. image:: resources/select_labels.png
+        :alt: select labels
+        :align: center
+
+5. Click on ``Add Label`` button to add a new label.
+
+    .. image:: resources/add_label.png
+        :alt: add label
+        :align: center
+
+6. Fill in the details for the label. Give a distinguishable name to the label, which will be applied to all campaigns. Select the rules that you want to apply to the label and click on ``Save``.
+
+    .. image:: resources/new_label.png
+        :alt: add label details
+        :align: center
+
+7. Now go back to the ``Clusters`` page to see the list of Campaigns and you will see the label applied to all the campaigns.
+
+    .. image:: resources/view_labels.png
+        :alt: label applied
+        :align: center
+
+    .. note::
+        Applying new or edited rules or labels to all campaigns may take some time. Refresh the campaigns page to check if the changes have been applied.
+
+
+Generate summary using Open AI
+------------------------------
+
+1. On the left hand side panel, click on ``GenAI``
+
+    .. image:: resources/gen_ai_add_key.png
+        :alt: gen ai config
+        :align: center
+
+2. Add your API key in the input field and click on ``Add API key``.
+
+3. On the left hand side panel, click on ``Campaigns``
+
+    .. image:: resources/campaign_list.png
+        :alt: Campaign list
+        :align: center
+
+4. Select any campaign for which you want to generate a summary. Click on ``Generate Summary`` button.
+
+    .. image:: resources/gen_ai_create_summary.png
+        :alt: gen ai summary
+        :align: center
+
+4. Click on the ``Diamond`` tab and view the summary created for your selected campaign.
+
+    .. image:: resources/gen_ai_summary.png
+        :alt: gen ai summary
         :align: center