diff --git a/_search-plugins/ubi/data-structures.md b/_search-plugins/ubi/data-structures.md deleted file mode 100644 index 0c64c3254ba..00000000000 --- a/_search-plugins/ubi/data-structures.md +++ /dev/null @@ -1,204 +0,0 @@ ---- -layout: default -title: UBI client data structures -parent: User Behavior Insights -has_children: false -nav_order: 10 ---- - -# UBI client data structures - -Data structures are used to create events that follow the [User Behavior Insights (UBI) event schema specification](https://github.com/o19s/ubi). -For more information about the schema, see [UBI index schemas]({{site.url}}{{site.baseurl}}/search-plugins/ubi/schemas/). - - -You must provide an implementation for the following functions: -- `getClientId()` -- `getQueryId()` - -You can also optionally provide an implementation for the following functions: -- `getSessionId()` -- `getPageId()` - - -The following JavaScript structures can be used as a starter implementation to serialize UBI events into schema-compatible JSON: -```js -/********************************************************************************************* - * Ubi Event data structures - * The following structures help ensure adherence to the UBI event schema - *********************************************************************************************/ - - - -export class UbiEventData { - constructor(object_type, id=null, description=null, details=null) { - this.object_id_field = object_type; - this.object_id = id; - this.description = description; - this.object_detail = details; - } -} -export class UbiPosition{ - constructor({ordinal=null, x=null, y=null, trail=null}={}) { - this.ordinal = ordinal; - this.x = x; - this.y = y; - if(trail) - this.trail = trail; - else { - const trail = getTrail(); - if(trail && trail.length > 0) - this.trail = trail; - } - } -} - - -export class UbiEventAttributes { - /** - * Tries to prepopulate common event attributes - * The developer can add an `object` that the user interacted with and - * the site `position` information relevant to the event - * - * Attributes, other than `object` or `position` can be added in the form: - * attributes['item1'] = 1 - * attributes['item2'] = '2' - * - * @param {*} attributes: object with general event attributes - * @param {*} object: the data object the user interacted with - * @param {*} position: the site position information - */ - constructor({attributes={}, object=null, position=null}={}) { - if(attributes != null){ - Object.assign(this, attributes); - } - if(object != null && Object.keys(object).length > 0){ - this.object = object; - } - if(position != null && Object.keys(position).length > 0){ - this.position = position; - } - this.setDefaultValues(); - } - - setDefaultValues(){ - try{ - if(!this.hasOwnProperty('dwell_time') && typeof TimeMe !== 'undefined'){ - this.dwell_time = TimeMe.getTimeOnPageInSeconds(window.location.pathname); - } - - if(!this.hasOwnProperty('browser')){ - this.browser = window.navigator.userAgent; - } - - if(!this.hasOwnProperty('page_id')){ - this.page_id = window.location.pathname; - } - if(!this.hasOwnProperty('session_id')){ - this.session_id = getSessionId(); - } - - if(!this.hasOwnProperty('page_id')){ - this.page_id = getPageId(); - } - - if(!this.hasOwnProperty('position') || this.position == null){ - const trail = getTrail(); - if(trail.length > 0){ - this.position = new UbiPosition({trail:trail}); - } - } - // ToDo: set IP - } - catch(error){ - console.log(error); - } - } -} - - - -export class UbiEvent { - constructor(action_name, {message_type='INFO', message=null, event_attributes={}, data_object={}}={}) { - this.action_name = action_name; - this.client_id = getClientId(); - this.query_id = getQueryId(); - this.timestamp = Date.now(); - - this.message_type = message_type; - if( message ) - this.message = message; - - this.event_attributes = new UbiEventAttributes({attributes:event_attributes, object:data_object}); - } - - /** - * Use to suppress null objects in the json output - * @param key - * @param value - * @returns - */ - static replacer(key, value){ - if(value == null || - (value.constructor == Object && Object.keys(value).length === 0)) { - return undefined; - } - return value; - } - - /** - * - * @returns json string - */ - toJson() { - return JSON.stringify(this, UbiEvent.replacer); - } -} -``` -{% include copy.html %} - -# Sample usage - -```js -export function logUbiMessage(event_type, message_type, message){ - let e = new UbiEvent(event_type, { - message_type:message_type, - message:message - }); - logEvent(e); -} - -export function logDwellTime(action_name, page, seconds){ - console.log(`${page} => ${seconds}`); - let e = new UbiEvent(action_name, { - message:`On page ${page} for ${seconds} seconds`, - event_attributes:{ - session_id: getSessionId()}, - dwell_seconds:seconds - }, - data_object:TimeMe - }); - logEvent(e); -} - -/** - * ordinal is the number within a list of results - * for the item that was clicked - */ -export function logItemClick(item, ordinal){ - let e = new UbiEvent('item_click', { - message:`Item ${item['object_id']} was clicked`, - event_attributes:{session_id: getSessionId()}, - data_object:item, - }); - e.event_attributes.position.ordinal = ordinal; - logEvent(e); -} - -export function logEvent( event ){ - // some configured http client - return client.index( index = 'ubi_events', body = event.toJson()); -} - -``` -{% include copy.html %} diff --git a/_search-plugins/ubi/dsl-queries.md b/_search-plugins/ubi/dsl-queries.md index 0802680dc90..b2b997e83a3 100644 --- a/_search-plugins/ubi/dsl-queries.md +++ b/_search-plugins/ubi/dsl-queries.md @@ -95,7 +95,4 @@ GET ubi_events/_search ``` {% include copy.html %} -You can run the preceding queries in the OpenSearch Dashboards [Query Workbench]({{site.url}}{{site.baseurl}}/search-plugins/sql/workbench/). - -A demo workbench with sample data can be found here: -[http://chorus-opensearch-edition.dev.o19s.com:5601/app/OpenSearch-query-workbench](http://chorus-OpenSearch-edition.dev.o19s.com:5601/app/OpenSearch-query-workbench). +You can run the preceding queries in the OpenSearch Dashboards [Dev Tools]({{site.url}}{{site.baseurl}}/dashboards/dev-tools/index-dev/) console. diff --git a/_search-plugins/ubi/index.md b/_search-plugins/ubi/index.md index 0aeb46c1338..0cb532a0f61 100644 --- a/_search-plugins/ubi/index.md +++ b/_search-plugins/ubi/index.md @@ -11,7 +11,7 @@ redirect_from: **Introduced 2.15** {: .label .label-purple } -**References UBI Specification 1.0.0** +**References UBI Specification 1.3.0** {: .label .label-purple } User Behavior Insights (UBI) is a schema for capturing user search behavior. Search behavior consists of the queries that the user submits, the results that are presented to them, and the actions they take on those results. The UBI schema links all user interactions (events) to the search result they were performed on. That is, it not only captures the chronological sequence of events but also captures the causal links between events. Analysis of this behavior is used for improving the quality of search results. @@ -20,33 +20,66 @@ Client applications such as web pages or apps capture user behavior and send UBI In principle, queries sent to the server and results returned by the server can be sent to the UBI endpoint from the client. But as an optimization, they can instead be sent directly to the UBI endpoint from the server, without incurring a round-trip to the client. That is the function of the UBI plugin and is not a requirement to adopt UBI. -UBI includes the following elements: -* A machine-readable [schema](https://github.com/o19s/ubi) that faciliates interoperablity of the UBI specification. -* A client-side JavaScript [example reference implementation]({{site.url}}{{site.baseurl}}/search-plugins/ubi/data-structures/) that shows how to capture events and send them to the OpenSearch UBI plugin. -* An OpenSearch [plugin](https://github.com/opensearch-project/user-behavior-insights) that captures server-side behavior. - - -The UBI documentation is organized into two categories: *Explanation and reference* and *Tutorials and how-to guides*: +> "how our users are using our product, whether search results were useful for them and whether they clicked on top-n results we gave and all related stuff" -- Data scientist working on search. -*Explanation and reference* - -| Link | Description | -| :--------- | :------- | -| [UBI Request/Response Specification](https://github.com/o19s/ubi/) | The industry-standard schema for UBI requests and responses. The current version references UBI Specification 1.0.0. | -| [UBI index schema]({{site.url}}{{site.baseurl}}/search-plugins/ubi/schemas/) | Documentation on the individual OpenSearch query and event stores. | +UBI includes the following elements: +* A machine-readable [schema](https://github.com/o19s/ubi) that facilitates interoperability of the UBI specification. +* [ubi.js](https://github.com/opensearch-project/user-behavior-insights/tree/main/ubi-javascript-collector/ubi.js): An (optional) client-side JavaScript library for capturing searches and events. +* An (optional) OpenSearch [plugin](https://github.com/opensearch-project/user-behavior-insights) that streamlines the recording of query data. +Advanced features in OpenSearch, such as the [Search Relevance Workbench]({{site.url}}{{site.baseurl}}/search-plugins/search-relevance/using-search-relevance-workbench/) and the [Hybrid Search Optimizer]({{site.url}}{{site.baseurl}}/search-plugins/search-relevance/optimize-hybrid-search/), build on the data collected according to the UBI specification. -*Tutorials and how-to guides* + -| Link | Description | -| :--------- | :------- | -| [UBI plugin](https://github.com/opensearch-project/user-behavior-insights) | How to install and use the UBI plugin. | -| [UBI client data structures]({{site.url}}{{site.baseurl}}/search-plugins/ubi/data-structures/) | Sample JavaScript structures for populating the event store. | -| [Example UBI query DSL queries]({{site.url}}{{site.baseurl}}/search-plugins/ubi/dsl-queries/) | How to write queries for UBI data in OpenSearch query DSL. | -| [Example UBI SQL queries]({{site.url}}{{site.baseurl}}/search-plugins/ubi/sql-queries/) | How to write analytic queries for UBI data in SQL. | -| [UBI dashboard tutorial]({{site.url}}{{site.baseurl}}/search-plugins/ubi/ubi-dashboard-tutorial/) | How to build a dashboard containing UBI data. | -| [Chorus Opensearch Edition](https://github.com/o19s/chorus-opensearch-edition/?tab=readme-ov-file#structured-learning-using-chorus-opensearch-edition) katas | A series of structured tutorials that teach you how to use UBI with OpenSearch through a demo e-commerce store. | +
+ Tutorials+
|
+
+ How To Guides+
|
+
+ Explanation+
|
+
+ Reference+
|
+
- `query_id` (size 100): The unique identifier of a query, which is typically a UUID but can be any string. - The `query_id` is either provided by the client or generated at index time by the UBI plugin. The `query_id` values in both the **UBI queries** and **UBI events** indexes must be consistent. + The `query_id` is either provided by the client or generated at query time by the UBI plugin. The `query_id` values in both the **UBI queries** and **UBI events** indexes must be consistent.
- `client_id`: The client that issues the query. This is typically a web browser used by a unique user. The `client_id` in both the **UBI queries** and **UBI events** indexes must be consistent. -- `timestamp`: When the event occurred, either in UNIX format or formatted as `2018-11-13T20:20:39+00:00`. +- `timestamp`: The time at which the event occurred in ISO 8601 format, such as `2018-11-13T20:20:39+00:00Z`. - `message_type` (size 100): A logical bin for grouping actions (each with an `action_name`). For example, `QUERY` or `CONVERSION`. @@ -193,18 +193,12 @@ The following are the predefined, minimal fields in the `ubi_events` index: - `event_attributes.position.ordinal`: Tracks the list position that a user can select (for example, selecting the third element can be described as `event{onClick, results[4]}`). - - `event_attributes.position.{x,y}`: Tracks x and y values defined by the client. - - - `event_attributes.position.page_depth`: Tracks the page depth of the results. - - - `event_attributes.position.scroll_depth`: Tracks the scroll depth of the page results. - - - `event_attributes.position.trail`: A text field that tracks the path/trail that a user took to get to this location. - + - `event_attributes.position.xy.{x,y}`: Tracks x and y values defined by the client. + - `event_attributes.object`: Contains identifying information about the object returned by the query (for example, a book, product, or post). The `object` structure can refer to the object by internal ID or object ID. The `object_id` is the ID that links prior queries to this object. This field comprises the following subfields: - - `event_attributes.object.internal_id`: A unique ID that OpenSearch can use to internally index the object, for example, the `_id` field in the indexes. + - `event_attributes.object.internal_id`: The unique ID that OpenSearch uses to internally index the object, for example, the `_id` field in the indexes.
@@ -214,7 +208,7 @@ The following are the predefined, minimal fields in the `ubi_events` index:
- - `event_attributes.object.object_id_field`: Indicates the type/class of the object and the name of the search index field that contains the `object_id`.
+ - `event_attributes.object.object_id_field`: Indicates the type/class of the object and the name of the search index field that contains the `object_id`, such as `ssn`, `isbn`, or `ean`.
- `event_attributes.object.description`: An optional description of the object.
diff --git a/_search-plugins/ubi/sql-queries.md b/_search-plugins/ubi/sql-queries.md
index 517c81e1d60..0c7423809a4 100644
--- a/_search-plugins/ubi/sql-queries.md
+++ b/_search-plugins/ubi/sql-queries.md
@@ -8,18 +8,15 @@ nav_order: 20
# Sample UBI SQL queries
-You can run sample User Behavior Insights (UBI) SQL queries in the OpenSearch Dashboards [Query Workbench]({{site.url}}{{site.baseurl}}/dashboards/query-workbench/).
-
-To query a demo workbench with synthetic data, see
-[http://chorus-opensearch-edition.dev.o19s.com:5601/app/opensearch-query-workbench](http://chorus-opensearch-edition.dev.o19s.com:5601/app/opensearch-query-workbench).
+You can run sample User Behavior Insights (UBI) SQL queries through the OpenSearch Dashboards [Query Workbench]({{site.url}}{{site.baseurl}}/dashboards/query-workbench/).
## Queries with zero results
-Queries can be executed on events on either the server (`ubi_queries`) or client (`ubi_events`) side.
+Queries can be executed on events on either the queries index (`ubi_queries`) or events index (`ubi_events`).
### Server-side queries
-The UBI-activated search server logs the queries and their results, so in order to find all queries with *no* results, search for empty `query_response_hit_ids`:
+A UBI-enabled search server logs the queries and their results as they are made, so in order to find all queries with *no* results, search for empty `query_response_hit_ids`:
```sql
select
diff --git a/_search-plugins/ubi/ubi-dashboard-tutorial.md b/_search-plugins/ubi/ubi-dashboard-tutorial.md
index eeee480d449..aa5b304cb96 100644
--- a/_search-plugins/ubi/ubi-dashboard-tutorial.md
+++ b/_search-plugins/ubi/ubi-dashboard-tutorial.md
@@ -11,6 +11,8 @@ nav_order: 25
Whether you've been collecting user events and queries for a while or [you've uploaded some sample events](https://github.com/o19s/chorus-OpenSearch-edition/blob/main/katas/003_import_preexisting_event_data.md), you're now ready to visualize the data collected through User Behavior Insights (UBI) in a dashboard in OpenSearch.
+> This tutorial is a nice way to learn how to make custom dashboards.
+
To quickly view a dashboard without completing the full tutorial, do the following:
1. Download and save the [sample UBI dashboard]({{site.url}}{{site.baseurl}}/assets/examples/ubi-dashboard.ndjson).
1. On the top menu, go to **Management > Dashboard Management**.
@@ -30,12 +32,12 @@ In OpenSearch Management, navigate to **Dashboards Management > Index patterns**
OpenSearch Dashboards accesses your indexes using index patterns. To visualize your users' online search behavior, you must create an index pattern in order to access the indexes that UBI creates. For more information, see [Index patterns]({{site.url}}{{site.baseurl}}/dashboards/management/index-patterns/).
-After you select **Create index pattern**, a list of indexes in your OpenSearch instance is displayed. The UBI stores may be hidden by default, so make sure to select **Include system and hidden indexes**, as shown in the following image.
+After you select **Create index pattern**, a list of indexes in your OpenSearch instance is displayed. The UBI stores may be hidden by default, so make sure to select **Include system and hidden indexes**, as shown in the following image.

You can group indexes into the same data source for your dashboard using wildcards. For this tutorial you'll combine the query and event stores into the `ubi_*` pattern.
-OpenSearch Dashboards prompts you to filter on any `date` field in your schema so that you can look at things like trending queries over the last 15 minutes. However, for your first dashboard, select **I don't want to use the time filter**, as shown in the following image.
+OpenSearch Dashboards prompts you to filter on any `date` field in your schema so that you can look at things like trending queries over the last 15 minutes. However, for your first dashboard, select **I don't want to use the time filter**, as shown in the following image.
@@ -58,17 +60,17 @@ Save the visualization so that it's added to your new dashboard. Now that you ha
Now you'll add a word cloud for trending searches by creating a new visualization, similarly to the previous step.
-In the **New Visualization** window, select **Tag Cloud**, and then select the index pattern you created in step 2. Choose the tag cloud visualization of the terms in the `message` field where the JavaScript client logs the raw search text. Note: The true query, as processed by OpenSearch with filters, boosting, and so on, resides in the `ubi_queries` index. However, you'll view the `message` field of the `ubi_events` index, where the JavaScript client captures the text that the user actually typed.
+In the **New Visualization** window, select **Tag Cloud**, and then select the index pattern you created in Step 2. Choose the tag cloud visualization of the terms in the `message` field where the JavaScript client logs the raw search text. Note: The true query, as processed by OpenSearch with filters, boosting, and so on, resides in the `ubi_queries` index. However, you'll view the `message` field of the `ubi_events` index, where the JavaScript client captures the text that the user actually typed.
The following image shows the tag cloud visualization on the `message` field.

The underlying queries can be found at [SQL trending queries]({{site.url}}{{site.baseurl}}/search-plugins/ubi/sql-queries/#trending-queries).
-{: .note}
+{: .note}
The resulting visualization may contain different information than you're looking for. The `message` field is updated with every event, and as a result, it can contain error messages, debug messages, click information, and other unwanted data.
-To view only search terms for query events, you need to add a filter to your visualization. Because during setup you provided a `message_type` of `QUERY` for each search event, you can filter by that message type to isolate the specific users' searches. To do this, select **Add filter** and then select **QUERY** in the **Edit filter** panel, as shown in the following image.
+To view only search terms for query events, you need to add a filter to your visualization. Because during setup you provided a `message_type` of `QUERY` for each search event, you can filter by that message type to isolate the specific users' searches. To do this, select **Add filter** and then select **QUERY** in the **Edit filter** panel, as shown in the following image.

There should now be two visualizations (the pie chart and the tag cloud) displayed on your dashboard, as shown in the following image.
@@ -76,9 +78,9 @@ There should now be two visualizations (the pie chart and the tag cloud) display
## 5. Add a histogram of item clicks
-Now you'll add a histogram visualization to your dashboard, similarly to the previous step. In the **New Visualization** window, select **Vertical Bar**. Then select the index pattern you created in step 2.
+Now you'll add a histogram visualization to your dashboard, similarly to the previous step. In the **New Visualization** window, select **Vertical Bar**. Then select the index pattern you created in Step 2.
-Examine the `event_attributes.position.ordinal` data field. This field contains the position of the item in a list selected by the user. For the histogram visualization, the x-axis represents the ordinal number of the selected item (n). The y-axis represents the number of times that the nth item was clicked, as shown in the following image.
+Examine the `event_attributes.position.ordinal` data field. This field contains the position of the item in a list selected by the user. For the histogram visualization, the x-axis represents the ordinal number of the selected item (n). The y-axis represents the number of times that the nth item was clicked, as shown in the following image.

@@ -89,6 +91,4 @@ Now you can further filter the displayed data. For example, you can see how the
You can filter event messages containing the word `*laptop*` by adding wildcards, as shown in the following image.
-.
-
-
+.
diff --git a/_search-plugins/ubi/ubi-javascript-collector.md b/_search-plugins/ubi/ubi-javascript-collector.md
new file mode 100644
index 00000000000..042d2d99a1c
--- /dev/null
+++ b/_search-plugins/ubi/ubi-javascript-collector.md
@@ -0,0 +1,121 @@
+---
+layout: default
+title: UBI JavaScript Collector
+parent: User Behavior Insights
+has_children: false
+nav_order: 10
+---
+
+# How to use UBI JavaScript collector
+
+UBI comes with a very basic JavaScript client that manages the lifecycle of the `query_id` for a specific search and can create UBI event data structures and store them for specific actions.
+
+For more information about the schema, see [UBI index schemas]({{site.url}}{{site.baseurl}}/search-plugins/ubi/schemas/).
+
+We recommend using the client as a starting point to address your specific needs.
+
+## Installation
+
+The client is a single file ([ubi.js](https://github.com/opensearch-project/user-behavior-insights/tree/main/ubi-javascript-collector/ubi.js)) and only has a dependency on the `axios` library.
+Download it from [https://github.com/opensearch-project/user-behavior-insights/tree/main/ubi-javascript-collector/ubi.js](https://github.com/opensearch-project/user-behavior-insights/tree/main/ubi-javascript-collector/ubi.js).
+
+To reference the events and create the client, use the following code:
+
+```js
+import { UbiEvent } from './ubi';
+import { UbiEventAttributes } from './ubi'
+import { UbiClient } from './ubi'
+
+const ubiClient = new UbiClient('http://localhost:9200');
+```
+{% include copy.html %}
+
+
+## Creating an event
+
+The following code tracks adding an item to a shopping cart in an e-commerce application. It uses the `UbiEvent` and `UbiEventAttributes` classes to encapsulate event details, which can then be sent to the tracking system:
+```js
+var event = new UbiEvent(
+ 'add_to_cart',
+ client_id,
+ session_id,
+ getQueryId(),
+ new UbiEventAttributes('product', item.primary_ean, item.title, item),
+ item.title + ' (' + item.id + ')'
+);
+```
+{% include copy.html %}
+
+### Parameters
+
+1. **Event Name**:
+ - `'add_to_cart'` -- This string indicates the type of event being tracked.
+
+2. **Client ID**:
+ - `client_id` -- A variable that holds the unique identifier for the client. This helps in distinguishing between different users or sessions.
+
+3. **Session ID**:
+ - `session_id` -- A variable that contains the unique identifier for the user session. This is used to track user interactions within a specific session.
+
+4. **Query ID**:
+ - `getQueryId()` -- A function call that retrieves the current query ID, which may represent a specific search or interaction context.
+
+5. **UbiEventAttributes**:
+ - This is an instance of the `UbiEventAttributes` class, which encapsulates additional details about the event:
+ - **Type**:
+ - `'product'` -- Specifies that the attribute type is related to a product.
+ - **Primary EAN**:
+ - `item.primary_ean` -- This is the product's unique identifier in EAN format.
+ - **Title**:
+ - `item.title` -- The name or description of the product.
+ - **Item**:
+ - `item` -- The complete product object containing all relevant details.
+
+6. **Event Label**:
+ - `item.title + ' (' + item.id + ')'` -- This creates a descriptive label for the event that includes the product title and its unique identifier (ID).
+
+The method `getQueryId()` refers to a helper method that generates a unique query ID (and stores it in the session).
+The following is a sample method:
+
+```js
+function generateQueryId(){
+ const query_id = generateGuid();
+ sessionStorage.setItem('query_id', query_id);
+ return query_id;
+}
+
+function generateGuid() {
+ let id = '';
+ try{
+ id = crypto.randomUUID();
+ }
+ catch(error){
+ // crypto.randomUUID only works in https, not http context, so fallback.
+ id ='10000000-1000-4000-8000-100000000000'.replace(/[018]/g, c =>
+ (c ^ crypto.getRandomValues(new Uint8Array(1))[0] & 15 >> c / 4).toString(16)
+ );
+ }
+ return id;
+};
+```
+{% include copy.html %}
+
+## Tracking the event
+
+You can send the event to the backend by calling the `trackEvent` method:
+
+```js
+ubiClient.trackEvent(event);
+```
+
+
+## Tracking queries
+
+You can optionally track queries using the client (instead of using the UBI plugin for OpenSearch).
+
+The code is similar to that used for tracking events:
+
+```js
+const query = new UbiQuery(APPLICATION, client_id, query_id, value, "_id", {});
+ubiClient.trackQuery(query)
+```