diff --git a/src/_data/catalog/destination_categories.yml b/src/_data/catalog/destination_categories.yml index d752536048..9d1dc61343 100644 --- a/src/_data/catalog/destination_categories.yml +++ b/src/_data/catalog/destination_categories.yml @@ -1,5 +1,5 @@ # AUTOGENERATED FROM PUBLIC API. DO NOT EDIT -# destination categories last updated 2025-07-07 +# destination categories last updated 2025-07-10 items: - display_name: A/B Testing slug: a-b-testing diff --git a/src/_data/catalog/destinations.yml b/src/_data/catalog/destinations.yml index 9bc940dbeb..c8a1914d2f 100644 --- a/src/_data/catalog/destinations.yml +++ b/src/_data/catalog/destinations.yml @@ -1,5 +1,5 @@ # AUTOGENERATED FROM PUBLIC API. DO NOT EDIT -# destination data last updated 2025-07-07 +# destination data last updated 2025-07-10 items: - id: 637e8d185e2dec264895ea89 display_name: 1Flow @@ -37744,7 +37744,7 @@ items: previous_names: - Extensible Webhook website: http://www.segment.com - status: PUBLIC_BETA + status: PUBLIC categories: - Raw Data logo: @@ -45881,7 +45881,7 @@ items: hidden: false defaultTrigger: null fields: - - id: ejWurq2rBDsCgnuu7gUkfP + - id: d8ZX5a19Xv36HNTk5SxWeP sortOrder: 0 fieldKey: eventType label: Event Type @@ -45896,7 +45896,7 @@ items: dynamic: false allowNull: false hidden: false - - id: 49UBKvAQRCHcvU8p3bBgMU + - id: bqoumn1Za2cwqcsb7u4tv9 sortOrder: 1 fieldKey: eventProperties label: Event Properties @@ -45914,7 +45914,7 @@ items: dynamic: false allowNull: false hidden: false - - id: mGVhUaP3RuwTsTZdpEoQPX + - id: d4ukxc9VWsrFEewLkk54PL sortOrder: 2 fieldKey: deduplicationId label: Event ID @@ -45931,7 +45931,7 @@ items: dynamic: false allowNull: false hidden: false - - id: Gjy2F7ujet3ZtFNa36vvB + - id: VBRCSArFbFmTaNmNexuD9 sortOrder: 3 fieldKey: customerId label: Customer ID @@ -45952,7 +45952,7 @@ items: dynamic: false allowNull: false hidden: false - - id: uQUG6igtnQaaHMtkBDLBPy + - id: 7dbozzEhtjFwbmQj3AEre9 sortOrder: 4 fieldKey: anonymousId label: Anonymous ID @@ -45967,7 +45967,7 @@ items: dynamic: false allowNull: false hidden: false - - id: u7Ren1bQoSKSZEX7q31aAP + - id: jmHz964gJgZF7BbQnGsL5s sortOrder: 5 fieldKey: email label: Email @@ -45990,7 +45990,7 @@ items: hidden: false defaultTrigger: event = "Signed Up" fields: - - id: vCYC98ivGTnZFPD2BdigAk + - id: kziVHoixgvK2GzbQ1NawYZ sortOrder: 0 fieldKey: customerId label: Customer ID @@ -46011,7 +46011,7 @@ items: dynamic: false allowNull: false hidden: false - - id: 8ovNDhpRH8b7ftYUSbiYuq + - id: enVaDD5pXR2utLkhcHmfEE sortOrder: 1 fieldKey: anonymousId label: Anonymous ID @@ -46026,7 +46026,7 @@ items: dynamic: false allowNull: false hidden: false - - id: w2BSBoSzA6sPjTPhxCimyP + - id: 5VYcjBeByFwwK5bAX8un46 sortOrder: 2 fieldKey: email label: Email @@ -46041,7 +46041,7 @@ items: dynamic: false allowNull: false hidden: false - - id: 9RPeb3FwcZ2PkX4kTfjwf9 + - id: 7mWFogh1knG4oBnPXG9tu8 sortOrder: 3 fieldKey: isNewCustomer label: New Customer Flag @@ -46056,7 +46056,7 @@ items: dynamic: false allowNull: false hidden: false - - id: tRnUs96hC8qtBpDJULC5Wv + - id: ms3Vb5UwXcb3XmEoYB86YR sortOrder: 4 fieldKey: loyaltyStatus label: Loyalty Program Status @@ -46073,7 +46073,7 @@ items: dynamic: false allowNull: false hidden: false - - id: 8mSVU4ZFrZLdaQ5zhWMCQg + - id: 4CeDLqyBbfKR8MbPPk3Mgf sortOrder: 5 fieldKey: firstName label: First Name @@ -46088,7 +46088,7 @@ items: dynamic: false allowNull: false hidden: false - - id: wmrSZxSGDkg49HzeX9x38s + - id: 2CQGvV5FaGA9TNrsZPhvYp sortOrder: 6 fieldKey: lastName label: Last Name @@ -46103,7 +46103,7 @@ items: dynamic: false allowNull: false hidden: false - - id: pnSYzM79hHRUAnfqmXhPR1 + - id: dRZRctiaZEPpSZdoiFZvZo sortOrder: 7 fieldKey: name label: Name @@ -46118,7 +46118,7 @@ items: dynamic: false allowNull: false hidden: false - - id: pZ8DujrjiSURj9Tgf5G72Z + - id: mZKhWaiZnd8AsnZvdMHyHW sortOrder: 8 fieldKey: age label: Age @@ -46133,7 +46133,7 @@ items: dynamic: false allowNull: false hidden: false - - id: wtxL8ncWxq4eCaL69QZEPN + - id: rNwRi3hVNgUzpNKRHqAsiw sortOrder: 9 fieldKey: birthday label: Birthday @@ -46150,7 +46150,7 @@ items: dynamic: false allowNull: false hidden: false - - id: 2nW3vfptTX13YhyjkgTFuA + - id: 5yPzkQBn4VbvZr3R5sH3Ko sortOrder: 10 fieldKey: coupon label: Coupon Code @@ -46165,7 +46165,7 @@ items: dynamic: false allowNull: false hidden: false - - id: 9kvGtxJGeTxMBQDW5ki3XL + - id: vS2KjQwtYgtNGBJ7Zmx474 sortOrder: 11 fieldKey: attributionId label: Friendbuy Attribution ID @@ -46182,7 +46182,7 @@ items: dynamic: false allowNull: false hidden: false - - id: m7KAr3ogqs7qoA6tdMZ2Jw + - id: 9NTxhvRdYhfmaesQPr9DXB sortOrder: 12 fieldKey: referralCode label: Friendbuy Referral ID @@ -46199,7 +46199,7 @@ items: dynamic: false allowNull: false hidden: false - - id: ijWBVDgTHCNy8m25YMVW8o + - id: xahUp79NtFA9zY9Emra6L1 sortOrder: 13 fieldKey: friendbuyAttributes label: Custom Attributes @@ -46225,7 +46225,7 @@ items: hidden: false defaultTrigger: event = "Order Completed" fields: - - id: 5WjHrBCkCj3edpz7bngu1L + - id: 9AEP912d257Xi9c5DujmN1 sortOrder: 0 fieldKey: orderId label: Order ID @@ -46240,7 +46240,7 @@ items: dynamic: false allowNull: false hidden: false - - id: neQbZ7y9G8D8GBmi6L7TwG + - id: kNo8MNntCJevg9EkC5gJM7 sortOrder: 1 fieldKey: amount label: Purchase Amount @@ -46255,7 +46255,7 @@ items: dynamic: false allowNull: false hidden: false - - id: mcbEjuL8xaFp2Y5feaGzQj + - id: j15ZWtXFyANdJvk71hRi2w sortOrder: 2 fieldKey: currency label: Currency @@ -46270,7 +46270,7 @@ items: dynamic: false allowNull: false hidden: false - - id: hRbLRY8WxGxMyXtnR4Vy2z + - id: dJ2nuUARwbXM14vvepSTda sortOrder: 3 fieldKey: coupon label: Coupon @@ -46285,7 +46285,7 @@ items: dynamic: false allowNull: false hidden: false - - id: tTuq8xv1LZjdK429XHQQJE + - id: umiNBqS58WvUXTHehdiXGF sortOrder: 4 fieldKey: attributionId label: Friendbuy Attribution ID @@ -46302,7 +46302,7 @@ items: dynamic: false allowNull: false hidden: false - - id: d7zKntMD3XhSsyUXycTVM8 + - id: hkozD7eyipEVgargQUVYwS sortOrder: 5 fieldKey: referralCode label: Friendbuy Referral ID @@ -46319,7 +46319,7 @@ items: dynamic: false allowNull: false hidden: false - - id: f5ftrEEpvZodKvCevvxyYs + - id: 56BkojcpCkT52pA25TQSoH sortOrder: 6 fieldKey: giftCardCodes label: Gift Card Codes @@ -46334,7 +46334,7 @@ items: dynamic: false allowNull: false hidden: false - - id: r1qtFhFk8h4czXxQinHwXD + - id: bnXCmz11HimhNbtQVNCZoS sortOrder: 7 fieldKey: products label: Products @@ -46349,7 +46349,7 @@ items: dynamic: false allowNull: false hidden: false - - id: xouR34VFZfRSqVhkeydA6w + - id: 6nvbgosu9x2FaXvyXiSuJk sortOrder: 8 fieldKey: customerId label: Customer ID @@ -46370,7 +46370,7 @@ items: dynamic: false allowNull: false hidden: false - - id: 9pJKLMZLpmwU4SHnhcwmfz + - id: rC5X9PghsymRb75A2nGgmm sortOrder: 9 fieldKey: anonymousId label: Anonymous ID @@ -46385,7 +46385,7 @@ items: dynamic: false allowNull: false hidden: false - - id: kQ8ngcFNgb3f7zsHLNHzgB + - id: 66GbxZRJbF6ttB1F2bTDyN sortOrder: 10 fieldKey: email label: Email @@ -46400,7 +46400,7 @@ items: dynamic: false allowNull: false hidden: false - - id: sD9VkFwa7tNc4msLjRQs1R + - id: qArtRrSKhYN11gJPr77rKq sortOrder: 11 fieldKey: isNewCustomer label: New Customer Flag @@ -46415,7 +46415,7 @@ items: dynamic: false allowNull: false hidden: false - - id: fua5c3GJUihvNuS5YS5YSm + - id: 2K3X7unfcYgtZDAz5X7sDr sortOrder: 12 fieldKey: loyaltyStatus label: Loyalty Program Status @@ -46432,7 +46432,7 @@ items: dynamic: false allowNull: false hidden: false - - id: 6Y5kFPumK9qLifaG8DxiPt + - id: g2USvWypGhiZFSFbuPJyZa sortOrder: 13 fieldKey: firstName label: First Name @@ -46447,7 +46447,7 @@ items: dynamic: false allowNull: false hidden: false - - id: 8PAax2B2zXMw2HanvYJgB4 + - id: j8o9jrMEF1ZTBW9k3mcVkU sortOrder: 14 fieldKey: lastName label: Last Name @@ -46462,7 +46462,7 @@ items: dynamic: false allowNull: false hidden: false - - id: fvujcPZpLpYweszLWLacGG + - id: ha4KHTBeHtxzvci9Fv866j sortOrder: 15 fieldKey: name label: Name @@ -46477,7 +46477,7 @@ items: dynamic: false allowNull: false hidden: false - - id: 2raZiFQAUFEXDVJsXmSrXs + - id: i5NDZnXEdeVikZvcezs7rP sortOrder: 16 fieldKey: age label: Age @@ -46492,7 +46492,7 @@ items: dynamic: false allowNull: false hidden: false - - id: efyUW87iUbKEbrFCovPfJe + - id: iD6EntrdLxhf8p1GkeaNGH sortOrder: 17 fieldKey: birthday label: Birthday @@ -46509,7 +46509,7 @@ items: dynamic: false allowNull: false hidden: false - - id: 57iKyKbadLvmSPS7yG4bez + - id: hQtVzytUyGTzu1eDb1ntvW sortOrder: 18 fieldKey: friendbuyAttributes label: Custom Attributes @@ -46535,7 +46535,7 @@ items: hidden: false defaultTrigger: type = "identify" fields: - - id: cRpZUanEFouGKUmPaAynae + - id: mt1tsyzXdsgx2q2MuzezJ sortOrder: 0 fieldKey: customerId label: Customer ID @@ -46550,7 +46550,7 @@ items: dynamic: false allowNull: false hidden: false - - id: tXzWmhtx5zMDkzLPqbuWPX + - id: ihjXAgoPZ8wvRyVqRzjTzS sortOrder: 1 fieldKey: anonymousId label: Anonymous ID @@ -46565,7 +46565,7 @@ items: dynamic: false allowNull: false hidden: false - - id: 4LBJGNXckZ45Q7DMaNsxE1 + - id: oHNrPAeF3PWfWk7C7A5D7Y sortOrder: 2 fieldKey: email label: Email @@ -46580,7 +46580,7 @@ items: dynamic: false allowNull: false hidden: false - - id: mCHjjrNqWxsrfPTJV8NUkj + - id: tQ23DP7HXw8CuaeDL7ceCt sortOrder: 3 fieldKey: firstName label: First Name @@ -46595,7 +46595,7 @@ items: dynamic: false allowNull: false hidden: false - - id: jhU9DPhtN9NPcnRhfWSDAn + - id: sk3uGGQ6c9F2u7paaNyUY1 sortOrder: 4 fieldKey: lastName label: Last Name @@ -46610,7 +46610,7 @@ items: dynamic: false allowNull: false hidden: false - - id: cW7Z2tJ8BQoMBpnkk9LPLS + - id: 6dDCzVwiMqG89kcYQBbNgA sortOrder: 5 fieldKey: name label: Name @@ -46628,7 +46628,7 @@ items: dynamic: false allowNull: false hidden: false - - id: ggEfxnbXSzb7PH86VvNjAr + - id: vzUvVM9Ymu75883KmjpzfA sortOrder: 6 fieldKey: age label: Age @@ -46643,7 +46643,7 @@ items: dynamic: false allowNull: false hidden: false - - id: BGVd2mbDda3Dapz45iF5P + - id: jpUb81CTqjs1wsBuh79QPv sortOrder: 7 fieldKey: birthday label: Birthday @@ -46660,7 +46660,7 @@ items: dynamic: false allowNull: false hidden: false - - id: 5z8txbByuGPQpKNi79fQjk + - id: f9E9V3HSCKMU6QKX6spNCv sortOrder: 8 fieldKey: language label: Language @@ -46675,7 +46675,7 @@ items: dynamic: false allowNull: false hidden: false - - id: 7h2KBTf6Pje2S7qSgRD5y1 + - id: iD3yVFcXbQdH2BxwXAyXTY sortOrder: 9 fieldKey: addressCountry label: Country @@ -46690,7 +46690,7 @@ items: dynamic: false allowNull: false hidden: false - - id: hb4URs3QmKh3BCU4eCQtHf + - id: ceWgsjeVL7GUZbQ2GEPanU sortOrder: 10 fieldKey: addressState label: State @@ -46705,7 +46705,7 @@ items: dynamic: false allowNull: false hidden: false - - id: iQitSpMkfs7E1EAhY9Z39B + - id: ppZPTSRHVbp4aB5Tjsp8Sq sortOrder: 11 fieldKey: addressCity label: City @@ -46720,7 +46720,7 @@ items: dynamic: false allowNull: false hidden: false - - id: j64R2HBx8LC4eBotQ6T4sX + - id: vG9t2zsqR1eLgkWVzjiDmQ sortOrder: 12 fieldKey: addressPostalCode label: State @@ -46735,7 +46735,7 @@ items: dynamic: false allowNull: false hidden: false - - id: jywnQKwXgWqM1Z36LyRxkh + - id: 5VWdFYYiFLzAsCYsmR1cAV sortOrder: 13 fieldKey: customerSince label: Customer Since @@ -46750,7 +46750,7 @@ items: dynamic: false allowNull: false hidden: false - - id: hhsGTynpidBroAJAvb8Xfj + - id: 6GFUvQ4oN9jT6WgyocDEdK sortOrder: 14 fieldKey: loyaltyStatus label: Loyalty Status @@ -46767,7 +46767,7 @@ items: dynamic: false allowNull: false hidden: false - - id: gVYGWF7cBNUC1mbKfb8hF8 + - id: 4HFSh4tmpZhnXr361sgsrB sortOrder: 15 fieldKey: isNewCustomer label: New Customer Flag @@ -46782,7 +46782,7 @@ items: dynamic: false allowNull: false hidden: false - - id: 8A3b5omAUfFNiqdXYdFnRa + - id: 6di6K8ZRAzpK7oU7V6tjRq sortOrder: 16 fieldKey: friendbuyAttributes label: Custom Attributes @@ -46810,7 +46810,7 @@ items: hidden: false defaultTrigger: type = "page" fields: - - id: 9m4ebCrh63BiJ5JZfHfLjx + - id: 7d99TsT6iyvqgegs4jDRNb sortOrder: 0 fieldKey: name label: Page Name @@ -46825,7 +46825,7 @@ items: dynamic: false allowNull: false hidden: false - - id: gpbuBNz73y2QzrqcKAssZK + - id: vp5V75NmVcHSRDYBcPPLVP sortOrder: 1 fieldKey: category label: Page Category @@ -46840,7 +46840,7 @@ items: dynamic: false allowNull: false hidden: false - - id: ja1bDYBCX5nbifXwAARRwu + - id: nNSJNxaVnyki7GF3avoBXN sortOrder: 2 fieldKey: title label: Page Title @@ -46856,64 +46856,6 @@ items: allowNull: false hidden: false presets: - - actionId: fk2xKEXiXd9qEkQb24nzXh - name: Track Page - fields: - name: - '@path': $.name - category: - '@path': $.category - title: - '@path': $.properties.title - trigger: type = "page" - - actionId: vj1sYaj9saWmcRUVbFaPf5 - name: Track Purchase - fields: - orderId: - '@path': $.properties.order_id - amount: - '@path': $.properties.total - currency: - '@path': $.properties.currency - coupon: - '@path': $.properties.coupon - attributionId: - '@path': $.properties.attributionId - referralCode: - '@path': $.properties.referralCode - giftCardCodes: - '@path': $.properties.giftCardCodes - products: - '@path': $.properties.products - customerId: - '@if': - exists: - '@path': $.properties.customerId - then: - '@path': $.properties.customerId - else: - '@path': $.userId - anonymousId: - '@path': $.anonymousId - email: - '@path': $.properties.email - isNewCustomer: - '@path': $.properties.isNewCustomer - loyaltyStatus: - '@path': $.properties.loyaltyStatus - firstName: - '@path': $.properties.firstName - lastName: - '@path': $.properties.lastName - name: - '@path': $.properties.name - age: - '@path': $.properties.age - birthday: - '@path': $.properties.birthday - friendbuyAttributes: - '@path': $.properties.friendbuyAttributes - trigger: event = "Order Completed" - actionId: otSYqZFafJi5aae1yTwynq name: Track Sign Up fields: @@ -46990,6 +46932,64 @@ items: friendbuyAttributes: '@path': $.traits.friendbuyAttributes trigger: type = "identify" + - actionId: fk2xKEXiXd9qEkQb24nzXh + name: Track Page + fields: + name: + '@path': $.name + category: + '@path': $.category + title: + '@path': $.properties.title + trigger: type = "page" + - actionId: vj1sYaj9saWmcRUVbFaPf5 + name: Track Purchase + fields: + orderId: + '@path': $.properties.order_id + amount: + '@path': $.properties.total + currency: + '@path': $.properties.currency + coupon: + '@path': $.properties.coupon + attributionId: + '@path': $.properties.attributionId + referralCode: + '@path': $.properties.referralCode + giftCardCodes: + '@path': $.properties.giftCardCodes + products: + '@path': $.properties.products + customerId: + '@if': + exists: + '@path': $.properties.customerId + then: + '@path': $.properties.customerId + else: + '@path': $.userId + anonymousId: + '@path': $.anonymousId + email: + '@path': $.properties.email + isNewCustomer: + '@path': $.properties.isNewCustomer + loyaltyStatus: + '@path': $.properties.loyaltyStatus + firstName: + '@path': $.properties.firstName + lastName: + '@path': $.properties.lastName + name: + '@path': $.properties.name + age: + '@path': $.properties.age + birthday: + '@path': $.properties.birthday + friendbuyAttributes: + '@path': $.properties.friendbuyAttributes + trigger: event = "Order Completed" partnerOwned: true - id: 54521fd625e721e32a72eeb8 display_name: FullStory @@ -70387,7 +70387,7 @@ items: hidden: false defaultTrigger: type = "track" fields: - - id: mq1JBc7UE5jezBA9BJuNkH + - id: q3tMM7GJnQDrR9434wLyrx sortOrder: 0 fieldKey: event_name label: Event Name @@ -70401,7 +70401,7 @@ items: choices: null dynamic: false allowNull: false - - id: xynLS9fBioXaiHxTk5q2Qc + - id: iBmfEFXhgW2y4YDCxB3k3m sortOrder: 1 fieldKey: revenue label: Revenue @@ -70417,7 +70417,7 @@ items: choices: null dynamic: false allowNull: false - - id: d1fFHeUCUmYupZCkzBFhW4 + - id: qmCjcgmFhLVmGUvmWUdkN7 sortOrder: 2 fieldKey: currency label: Currency @@ -70433,7 +70433,7 @@ items: choices: null dynamic: false allowNull: false - - id: pY9vJcUt4Ps96fZu51uAn3 + - id: ptGRqqjrbV2MeMi1rJV6xA sortOrder: 3 fieldKey: event_metadata label: Event Metadata @@ -70455,7 +70455,7 @@ items: hidden: false defaultTrigger: type = "group" fields: - - id: 6aEhSk9Xxi6zdwt78AunMp + - id: bDPnWGcGHWXWj5nANtZYZP sortOrder: 0 fieldKey: company label: Company @@ -70490,7 +70490,7 @@ items: choices: null dynamic: false allowNull: false - - id: 5zBgiK62Rp8w3YGB7VjGg9 + - id: juHWuh64cJFTmu4fQby8Vk sortOrder: 1 fieldKey: hide_default_launcher label: Hide Default Launcher @@ -70524,7 +70524,7 @@ items: hidden: false defaultTrigger: type = "identify" or type = "page" fields: - - id: szzhV3uoFuH1cDpqyMNoYx + - id: b38rUKWz2doCTdnf7uaJ4B sortOrder: 0 fieldKey: user_id label: User ID @@ -70538,7 +70538,7 @@ items: choices: null dynamic: false allowNull: false - - id: kczvBPohUPGu38k8otGk5g + - id: anqafKKQ66Se8ttGaXkRSr sortOrder: 1 fieldKey: custom_traits label: Custom Attributes @@ -70550,7 +70550,7 @@ items: choices: null dynamic: false allowNull: false - - id: cMvFExAeUenaZTCdtPaNVu + - id: uwLDCEbBSriBL2tQ53zKAf sortOrder: 2 fieldKey: name label: Name @@ -70564,7 +70564,7 @@ items: choices: null dynamic: false allowNull: false - - id: d5wyJQybuiaoFJ87JfMcx4 + - id: ocfjoKATo9M2xFmRZDX1YL sortOrder: 3 fieldKey: phone label: Phone Number @@ -70578,7 +70578,7 @@ items: choices: null dynamic: false allowNull: false - - id: ran9B5T8nV1Jqskruk8skp + - id: aHcMwcmDQLK6vtkPdDFYVo sortOrder: 4 fieldKey: unsubscribed_from_emails label: Unsubscribed From Emails @@ -70590,7 +70590,7 @@ items: choices: null dynamic: false allowNull: false - - id: tW4W819YNUhtF5itADRUkc + - id: utTEacb4Z3YodS9LYxFwyZ sortOrder: 5 fieldKey: language_override label: Language Override @@ -70604,7 +70604,7 @@ items: choices: null dynamic: false allowNull: false - - id: uBbuEwKUaikLUw6VH5MQTh + - id: qKZkWiHyT7MD8n4F23XJKo sortOrder: 6 fieldKey: email label: Email Address @@ -70618,7 +70618,7 @@ items: choices: null dynamic: false allowNull: false - - id: d1oAVy86TTzDk3KYxXzvFB + - id: faSozTVDyr41hu1AJJ7K9a sortOrder: 7 fieldKey: created_at label: User Creation Time @@ -70638,7 +70638,7 @@ items: choices: null dynamic: false allowNull: false - - id: hdGhrbjyGsJ7RiFQtbGpG6 + - id: 2VuNZhEmxiTWjU8ka9DBrM sortOrder: 8 fieldKey: avatar_image_url label: Avatar @@ -70652,7 +70652,7 @@ items: choices: null dynamic: false allowNull: false - - id: 8GWMZnfBhwBaoKhhWMCVPY + - id: oSYzfbVpP53GcMGRz6hSKo sortOrder: 9 fieldKey: user_hash label: User Hash @@ -70675,7 +70675,7 @@ items: choices: null dynamic: false allowNull: false - - id: eAKgAWEvqBx9eYghqpZBNQ + - id: 2UMW9XztJGHE7RUwFy344C sortOrder: 10 fieldKey: intercom_user_jwt label: Intercom User JWT @@ -70699,7 +70699,7 @@ items: choices: null dynamic: false allowNull: false - - id: wbgKoz36MtxFcic8Pz3rHD + - id: sc4asvfuszwnTY6nGGJqQp sortOrder: 11 fieldKey: company label: Company @@ -70734,7 +70734,7 @@ items: choices: null dynamic: false allowNull: false - - id: ihytDbuPQgRhpJptC5HmZX + - id: uqrhkyxQA2vRV5s74KeBgK sortOrder: 12 fieldKey: companies label: Companies @@ -70771,7 +70771,7 @@ items: choices: null dynamic: false allowNull: false - - id: 9WCnDuuQPojuykHHxzpwqo + - id: 4yGDpFYKBjzGT8k8DETUW9 sortOrder: 13 fieldKey: hide_default_launcher label: Hide Default Launcher @@ -70798,53 +70798,6 @@ items: dynamic: false allowNull: false presets: - - actionId: 9e8n4VX1tR81fawEiAtGPg - name: Identify Company - fields: - company: - company_id: - '@path': $.groupId - name: - '@path': $.traits.name - created_at: - '@if': - exists: - '@path': $.traits.createdAt - then: - '@path': $.traits.createdAt - else: - '@path': $.traits.created_at - plan: - '@path': $.traits.plan - size: - '@path': $.traits.size - website: - '@path': $.traits.website - industry: - '@path': $.traits.industry - monthly_spend: - '@path': $.traits.monthly_spend - hide_default_launcher: - '@if': - exists: - '@path': $.context.Intercom.hideDefaultLauncher - then: - '@path': $.context.Intercom.hideDefaultLauncher - else: - '@path': $.context.Intercom.hide_default_launcher - trigger: type = "group" - - actionId: obbhMxCCfbwRwxViADd4Dq - name: Track Event - fields: - event_name: - '@path': $.event - revenue: - '@path': $.properties.revenue - currency: - '@path': $.properties.currency - event_metadata: - '@path': $.properties - trigger: type = "track" - actionId: soJTzcZj8roZAwGZow5YNp name: Identify User fields: @@ -70939,6 +70892,53 @@ items: else: '@path': $.context.Intercom.hide_default_launcher trigger: type = "identify" or type = "page" + - actionId: 9e8n4VX1tR81fawEiAtGPg + name: Identify Company + fields: + company: + company_id: + '@path': $.groupId + name: + '@path': $.traits.name + created_at: + '@if': + exists: + '@path': $.traits.createdAt + then: + '@path': $.traits.createdAt + else: + '@path': $.traits.created_at + plan: + '@path': $.traits.plan + size: + '@path': $.traits.size + website: + '@path': $.traits.website + industry: + '@path': $.traits.industry + monthly_spend: + '@path': $.traits.monthly_spend + hide_default_launcher: + '@if': + exists: + '@path': $.context.Intercom.hideDefaultLauncher + then: + '@path': $.context.Intercom.hideDefaultLauncher + else: + '@path': $.context.Intercom.hide_default_launcher + trigger: type = "group" + - actionId: obbhMxCCfbwRwxViADd4Dq + name: Track Event + fields: + event_name: + '@path': $.event + revenue: + '@path': $.properties.revenue + currency: + '@path': $.properties.currency + event_metadata: + '@path': $.properties + trigger: type = "track" partnerOwned: false - id: 54521fd725e721e32a72eec5 display_name: Iron.io @@ -92734,14 +92734,14 @@ items: - id: 682db7017819e7e055f55cb4 display_name: Ortto (Actions) name: Ortto (Actions) - slug: ortto-actions + slug: actions-ortto hidden: false endpoints: - US regions: - us-west-2 - eu-west-1 - url: connections/destinations/catalog/ortto-actions + url: connections/destinations/catalog/actions-ortto previous_names: - Ortto (Actions) website: https://ortto.com/ @@ -111033,144 +111033,6 @@ items: hidden: false presets: [] partnerOwned: false -- id: 6189bc6dc616c2d82f61349c - display_name: Segment Data Lakes (Azure) - name: Segment Data Lakes (Azure) - slug: segment-data-lakes-azure - hidden: true - endpoints: - - US - regions: - - us-west-2 - - eu-west-1 - url: connections/destinations/catalog/segment-data-lakes-azure - previous_names: - - Azure Data Lakes - - Segment Data Lakes (Azure) - website: https://segment.com/docs/connections/storage/data-lakes/ - status: PUBLIC - categories: - - Raw Data - - Email - - CRM - logo: - url: https://cdn.filepicker.io/api/file/blchmYvQ36JOzWPWZ7Qa - mark: - url: https://cdn.filepicker.io/api/file/SIWnw2VqRPqF7abxuHZK - methods: - track: true - identify: true - group: true - alias: true - screen: false - page: true - platforms: - browser: true - mobile: true - server: true - warehouse: false - cloudAppObject: false - linkedAudiences: false - components: - - code: https://github.com/segmentio/integrations/tree/master/integrations/hubspot - owner: SEGMENT - type: SERVER - browserUnbundlingSupported: false - browserUnbundlingPublic: true - replay: false - connection_modes: - device: - web: false - mobile: false - server: false - cloud: - web: true - mobile: true - server: true - settings: - - name: azureStorageAccount - type: string - defaultValue: '' - description: The name of the Azure Storage Account - required: true - label: Azure Storage Account Name - - name: azureStorageContainer - type: string - defaultValue: '' - description: The name of the storage container - required: true - label: Azure Storage Container Name - - name: azureSubscriptionId - type: string - defaultValue: '' - description: Azure Subscription ID - required: true - label: Azure Subscription ID - - name: azureTenantId - type: string - defaultValue: '' - description: Azure Tenant ID - required: true - label: Azure Tenant ID - - name: databricksClusterId - type: string - defaultValue: '' - description: Databricks Cluster ID - required: true - label: Databricks Cluster ID - - name: databricksInstanceUrl - type: string - defaultValue: '' - description: Databricks Instance URL - required: true - label: Databricks Instance URL - - name: databricksWorkspaceName - type: string - defaultValue: '' - description: Databricks Workspace Name - required: true - label: Databricks Workspace Name - - name: databricksWorkspaceResourceGroup - type: string - defaultValue: '' - description: Databricks Workspace Resource Group - required: true - label: Databricks Workspace Resource Group Id - - name: datePartition - type: select - defaultValue: '' - description: >- - This drop down can be used to select the type of date partition required - to store the tables. - required: false - label: Date Partition - - name: partitionBySlugName - type: boolean - defaultValue: false - description: Enable to partition the data by source_slug_name instead of source_id - required: false - label: Partition by Slug Name - - name: region - type: string - defaultValue: '' - description: Azure Account Region - required: true - label: Region - - name: servicePrincipalClientId - type: string - defaultValue: '' - description: Service Principal Client ID - required: true - label: Service Principal Client ID - - name: servicePrincipalClientSecret - type: string - defaultValue: '' - description: Service Principal Client Secret - required: true - label: Service Principal Client Secret - actions: [] - presets: [] - partnerOwned: false - id: 639c2dbb1309fdcad13951b6 display_name: Segment Profiles name: Segment Profiles diff --git a/src/_data/catalog/destinations_private.yml b/src/_data/catalog/destinations_private.yml index d78ab8da5e..43df3f3dfa 100644 --- a/src/_data/catalog/destinations_private.yml +++ b/src/_data/catalog/destinations_private.yml @@ -1,5 +1,5 @@ # AUTOGENERATED FROM PUBLIC API. DO NOT EDIT -# destination data last updated 2025-07-07 +# destination data last updated 2025-07-10 items: - id: 54521fd925e721e32a72eee1 display_name: Pardot diff --git a/src/_data/catalog/source_categories.yml b/src/_data/catalog/source_categories.yml index 8e23322b82..595bb73691 100644 --- a/src/_data/catalog/source_categories.yml +++ b/src/_data/catalog/source_categories.yml @@ -1,5 +1,5 @@ # AUTOGENERATED FROM PUBLIC API. DO NOT EDIT -# source categories last updated 2025-07-07 +# source categories last updated 2025-07-10 items: - display_name: A/B testing slug: a-b-testing diff --git a/src/_data/catalog/sources.yml b/src/_data/catalog/sources.yml index 63f73ab407..a08d455fe8 100644 --- a/src/_data/catalog/sources.yml +++ b/src/_data/catalog/sources.yml @@ -1,5 +1,5 @@ # AUTOGENERATED FROM PUBLIC API. DO NOT EDIT -# sources last updated 2025-07-07 +# sources last updated 2025-07-10 items: - id: 8HWbgPTt3k display_name: .NET diff --git a/src/api/public-api/query-language.md b/src/api/public-api/query-language.md index 73e93ae2c7..9b60bf4054 100644 --- a/src/api/public-api/query-language.md +++ b/src/api/public-api/query-language.md @@ -59,17 +59,25 @@ The following tables list the query languages's available functions. | `trait` | | | ----------- | --------------------------------------------------------------------------------------------------- | -| Syntax | `trait({s: String})`
`s` - the name of the the trait to reference | +| Syntax | `trait({s: String})`
`s` - the name of the trait to reference | | Return Type | `ScalarExtractor` | | Description | Similar to the event operator, the trait operator is used to specify profile trait filter criteria. | -| Notes | You can reference other audiences by using the audience key as the trait name. | +| Notes | You can reference other audiences by using the audience key as the trait name. The `.` character indicates traversal through nested structures. If the trait name contains a literal period (`.`), it must be escaped using `\\\\`. | | Example | `trait('total_spend')` | +| `entity` | | +| ----------- | --------------------------------------------------------------------------------------------------- | +| Syntax | `entity({s: String})`
`s` - the relationship slug of the entity to build an extractor for | +| Return Type | `VectorExtractor` | +| Description | Similar to the event operator, the entity operator is used to specify entity filter criteria. | +| Notes | Entity is only used with Linked Audiences. | +| Example | `entity('accounts')` | + | `property` | | | ----------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | | Syntax | `property({s: String})`
`s` - the name of the property to build an extractor for
In the context of funnel audiences, you can add a parent prefix to reference the parent event.
`property(parent: {s: String})` | | Return Type | `ScalarExtractor` | -| Notes | Only valid within a `where` function or a Reducer. | +| Notes | Only valid within a `where` function or a Reducer. The `.` character indicates traversal through nested structures. If the trait name contains a literal period (`.`), it must be escaped using `\\\\`. | | Example | `property('total')` | | `context` | | @@ -79,11 +87,12 @@ The following tables list the query languages's available functions. | Notes | Only valid within a `where` function or a Reducer. | | Example | `context('page.url')` | -| `literal` | | -| -------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | -| Syntax | `literal({a: Any})`
`a` - the value to treat as a literal expression | +| `literal` | | +| -------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| Syntax | `literal({a: Any})`
`a` - the value to treat as a literal expression | | Operations allowed in call-chain | None allowed; typically used within another function, like a comparison (with syntactic sugar, this would appear on the right side of the comparison). The outer function or comparison dictates the operations allowed in the call-chain. | -| Example | `literal(100)`
| +| Notes | Literals can be `int`, `float`, `string`, `date` or `timestamp`, where `date` uses the format `yyyy-mm-dd` and `timestamp` follows the [ISO 8601](https://en.wikipedia.org/wiki/ISO_8601){:target="_blank"} format. `string` is generally interchangeable with all other formats, except when used in a `property` chained to an `entity`. In this case, Segment recommends matching the data type of the entity property being targeted. | +| Example | `literal(100)`, `literal('value')`, or `literal(2022-10-17T00:00:00)` | @@ -269,19 +278,28 @@ The following tables list the query languages's available functions. | Syntax | `one_of({a: Array})`
`a` - array of possible values | | Return Type | `Comparator` | | Description | Matches when the value exactly matches one of the values from the parameter array. | -| Example | `one_of('shoes','shirts')` | - -| `before_date` | | -| ------------- | --------------------------------------------------------- | -| Syntax | `before_date({t: Timestamp})`
`t` - ISO 8601 timestamp | -| Return Type | `Comparator` | -| Example | `before_date('2023-12-07T18:50:00Z')` | - -| `after_date` | | -| ------------ | -------------------------------------------------------- | -| Syntax | `after_date({t: Timestamp})`
`t` - ISO 8601 timestamp | -| Return Type | `Comparator` | -| Example | `after_date('2023-12-07T18:50:00Z')` | +| Example | `one_of('shoes','shirts')` | + +| `none_of` | | +| ----------- | ----------------------------------------------------------------------------------------- | +| Syntax | `none_of({a: Array})`
`a` - array of possible values | +| Return Type | `Comparator` | +| Description | Matches when the value does not exactly match one of the values from the parameter array. | +| Example | `none_of('shoes','shirts')` | + +| `before_date` | | +| ------------- | ---------------------------------------------------------------------------- | +| Syntax | `before_date({t: Timestamp})`
`t` - ISO 8601 timestamp | +| Return Type | `Comparator` | +| Notes | `string` format can also be used | +| Example | `before_date(2023-12-07T18:50:00Z)` or `before_date('2023-12-07T18:50:00Z')` | + +| `after_date` | | +| ------------ | -------------------------------------------------------------------------- | +| Syntax | `after_date({t: Timestamp})`
`t` - ISO 8601 timestamp | +| Return Type | `Comparator` | +| Notes | `string` format can also be used | +| Example | `after_date(2023-12-07T18:50:00Z)` or `after_date('2023-12-07T18:50:00Z')` | | `within_last` | | | ------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | @@ -366,7 +384,7 @@ The following tables list the query languages's available functions. | `ScalarExtractor` (extends `Extractor`, `Scalar`) | | | ------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | | Base Type | `Extractor`, `Scalar` | -| Operations allowed in call-chain | `equals`, `differs`, `absent`, `exists`, `greater_than`, `at_least`, `less_than`, `at_most`, `contains`, `omits`, `starts_with`, `ends_with`, `one_of`, `before_date`, `after_date`, `within_last`, `before_last`, `after_next` (inherited from `Scalar`) | +| Operations allowed in call-chain | `equals`, `differs`, `absent`, `exists`, `greater_than`, `at_least`, `less_than`, `at_most`, `contains`, `omits`, `starts_with`, `ends_with`, `one_of`, `none_of`, `before_date`, `after_date`, `within_last`, `before_last`, `after_next` (inherited from `Scalar`) | | Notes | A `ScalarExtractor` represents extractions of a single data element, like a field value or a trait value. | | `EventPropertyExtractor` (extends `Extractor`) | | @@ -391,7 +409,7 @@ The following tables list the query languages's available functions. | `Scalar` | | | -------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| Operations allowed in call-chain | `equals`, `differs`, `absent`, `exists`, `greater_than`, `at_least`, `less_than`, `at_most`, `contains`, `omits`, `starts_with`, `ends_with`, `one_of`, `before_date`, `after_date`, `within_last`, `before_last`, `after_next`, `within_next` | +| Operations allowed in call-chain | `equals`, `differs`, `absent`, `exists`, `greater_than`, `at_least`, `less_than`, `at_most`, `contains`, `omits`, `starts_with`, `ends_with`, `one_of`, `none_of`, `before_date`, `after_date`, `within_last`, `before_last`, `after_next`, `within_next` | | `ListScalar` | | | -------------------------------- | ------- | @@ -474,6 +492,30 @@ This example collects all accounts where any associated users performed the `Sho ANY event('Shoes Bought').count() >= 1 ``` +#### Associated with Orders that have an association to Order Products + +This example collects all users who have at least 1 association to an `orders` entity where the `orders` entity has at least 1 association to an `order-products` entity: + +```sql +entity('orders').where(entity('order-products').count() >= 1).count() >= 1 +``` + +#### Associated to Orders or is a VIP user + +This example collects all users who have at least 1 association to an `order` entity or have a `VIP` trait equal to true: + +```sql +entity('orders').count() >= 1 OR trait('VIP') = 'true' +``` + +#### Associated with orders that have a total greater than 500 + +This example collects all users with at least 1 association to an `orders` entity where the `orders` entity has a `total` property greater than 500: + +```sql +entity('orders').where(property('total') > 500).count() >= 1 +``` + ### Computed Traits Suppose you wanted to calculate the average spend based on all `Shoes Bought` events performed within the last 30 days for each user. diff --git a/src/connections/destinations/catalog/actions-amazon-conversions-api/index.md b/src/connections/destinations/catalog/actions-amazon-conversions-api/index.md index 89d415803d..9c2c0071e8 100644 --- a/src/connections/destinations/catalog/actions-amazon-conversions-api/index.md +++ b/src/connections/destinations/catalog/actions-amazon-conversions-api/index.md @@ -89,7 +89,7 @@ Amazon requires at least one match key to identify the user associated with each | postalCode | User's postal code (is hashed) | | maid | Mobile advertising ID (ADID, IDFA, or FIREADID) | | rampId | RAMP ID for attribution to traffic events | -| matchId | Custom match ID for the user | +| matchId | Match ID serves as an anonymous, opaque unique identifier that corresponds to individual users within an advertiser's system, such as loyalty membership identifications and order references. This functionality enables advertisers to precisely monitor campaign effectiveness while maintaining customer data privacy, eliminating the need to share sensitive information like hashed email addresses or phone numbers with Amazon, particularly when analyzing complex customer journeys across multiple channels and devices. The advertisers who implement the Amazon Advertising Tag (AAT) on their websites can transmit match_id as a parameter in conjunction with online event tracking. Amazon's system subsequently correlates these identifiers with users through cookies or hashed Personally Identifiable Information (PII). In instances where users complete offline conversions, advertisers can report these activities through the Conversions API (CAPI) utilizing the corresponding match_id, ensuring seamless cross-channel attribution. | Segment automatically maps these fields from standard identity traits when available. @@ -139,7 +139,7 @@ You can include custom attributes with your events to provide additional context #### How does deduplication work? -Amazon Conversions API uses the `clientDedupeId` field to prevent duplicate events. By default, Segment maps the messageId to this field. For events with the same clientDedupeId, only the latest event will be processed. +Amazon Conversions API uses the `clientDedupeId` field to prevent duplicate events. By default, Segment maps the messageId to this field. For events with the same clientDedupeId, only the latest event will be processed. Deduplication occurs across all event types, rather than being limited to individual event types. #### What regions are supported? @@ -163,3 +163,14 @@ Personally identifiable information (PII) fields like email, phone, name, and ad #### How can I verify events in Amazon? After you start sending events, you should see them in your Amazon Advertising dashboard under conversion tracking reports. Allow some time for the events to be processed and attributed. + +#### Which advertising accounts can access Amazon Conversions API through Segment? + +- Self-Service (SS) Accounts: Yes +- Managed Service (MS) Accounts: No +- Manager Accounts (MA): At present, this functionality is not available. However, we are actively monitoring market demand and user requirements. Should sufficient interest emerge from our user base, we will evaluate the possibility of implementing this feature in a future release. + +#### I am an advertiser on Amazon Demand Side Platform. My agency can't provide access to their entity as they manage other clients. How do I get access to a specific advertiser account? + +1. Request your agency to grant access to the specific advertiser account intended for campaign execution. Specifically, **View and Edit** permissions must be assigned for the Events Manager. Please see the [help article](https://advertising.amazon.com/help/G3ZZ6ZXQWG34DPPM){:target="_blank"}. +2. Utilize the Amazon Conversions API (CAPI) to transmit data to the advertiser account for which access was granted in the previous step. diff --git a/src/connections/destinations/catalog/actions-ortto-audiences/index.md b/src/connections/destinations/catalog/actions-ortto-audiences/index.md new file mode 100644 index 0000000000..a6ef792f4f --- /dev/null +++ b/src/connections/destinations/catalog/actions-ortto-audiences/index.md @@ -0,0 +1,33 @@ +--- +title: Ortto (Audiences) Destination +id: 686799c65fcebef2a98c8141 +beta: true +private: true +--- + +{% include content/plan-grid.md name="actions" %} + +The **Ortto (Audiences)** destination allows you to send Engage Audiences from Segment to Ortto, where they can be used as [Audiences](https://help.ortto.com/a-115-working-with-audiences){:target="_blank"} for marketing automation, personalization, and reporting. + +This destination is maintained by **Ortto**. For support or questions, please [contact the Ortto Support team](mailto:help@ortto.com){:target="_blank"}. + +## Getting started + +Follow these steps to set up the Ortto (Audiences) destination in your Segment workspace: + +1. In your Segment workspace, click **Engage** in the left navigation bar, then select your space. +2. Select **Engage Settings** and open the Destinations tab. +3. Search for `Ortto (Audiences)` and click **Add Destination**. +4. Confirm the source. This is typically the source connected to the Engage space you're working in. +5. [Log in to your Ortto account](https://ortto.app/login){:target="_blank"} and either: + - Create a new **Segment (Actions)** data source, or + - Use an existing one. +6. Copy the API key from your Ortto data source. +7. In Segment, open the destination's **Settings** tab and enter your Ortto API key. +8. Toggle **Enable Destination** to **ON** and click **Save Changes**. +9. Go to the **Mappings** tab in your Ortto (Audiences) destination and add a **Sync Audience** mapping. +10. Enable the mapping. +11. Navigate to the **Audiences** tab in your Engage space and select the audience you want to sync. +12. Click **Add Destination**. + +{% include components/actions-fields.html %} diff --git a/src/connections/destinations/catalog/actions-webhook-extensible/index.md b/src/connections/destinations/catalog/actions-webhook-extensible/index.md index da2f29b77e..a9846e2771 100644 --- a/src/connections/destinations/catalog/actions-webhook-extensible/index.md +++ b/src/connections/destinations/catalog/actions-webhook-extensible/index.md @@ -8,8 +8,6 @@ redirect_from: '/connections/destinations/catalog/extensible-webhook/' Segment's Extensible Webhook destination lets you send custom data payloads to any webhook endpoint. With support for flexible payload configuration, multiple authentication methods, and real-time data flow, Extensible Webhooks can help you integrate with internal systems or tools not covered by Segment’s standard destinations. -This destination is not currently available in EU regions. - Segment maintains this destination. For any issues, [contact Segment Support](friends@segment.com). ## Overview diff --git a/src/connections/destinations/catalog/adobe-analytics/mobile.md b/src/connections/destinations/catalog/adobe-analytics/mobile.md index 911d9e5e52..3412a531f8 100644 --- a/src/connections/destinations/catalog/adobe-analytics/mobile.md +++ b/src/connections/destinations/catalog/adobe-analytics/mobile.md @@ -14,7 +14,7 @@ Before you start sending data from your mobile application to Adobe Analytics, y - First, enable the Segment-Adobe Analytics destination from in your Segment workspace. - From your Adobe Mobile Services dashboard, check and customize the settings on the "Manage App Settings" tab. -- Download these settings as the `ADBMobileConfig.json` file by clicking the **Config JSON** link at the bottom of the same tab. Follow the instructions in Adobe's documentation [here for iOS](https://marketing.adobe.com/resources/help/en_US/mobile/ios/dev_qs.html){:target="_blank”} and [here for Android](https://marketing.adobe.com/resources/help/en_US/mobile/android/dev_qs.html){:target="_blank”}. +- Download these settings as the `ADBMobileConfig.json` file by clicking the **Config JSON** link at the bottom of the same tab. Follow the instructions in Adobe's [Configuration documentation](https://developer.adobe.com/client-sdks/home/base/mobile-core/configuration/){:target="_blank”}. - Finally, follow the instructions below for each mobile environment to bundle Segment's Adobe Analytics SDK in your project. > success "" diff --git a/src/connections/destinations/catalog/eagleeye-actions/index.md b/src/connections/destinations/catalog/eagleeye-actions/index.md index f6a1347d34..3169d1b1cb 100644 --- a/src/connections/destinations/catalog/eagleeye-actions/index.md +++ b/src/connections/destinations/catalog/eagleeye-actions/index.md @@ -5,23 +5,23 @@ id: 682db61f6c600fdb90251392 {% include content/plan-grid.md name="actions" %} -[Eagle Eye](https://eagleeye.com/?utm_source=segmentio&utm_medium=docs&utm_campaign=partners){:target="_blank”} delivers real-time loyalty and promotions solutions for enterprise brands, enabling seamless integration with leading customer data platforms and marketing activation tools. The platform connects digital and in-store experiences, accelerating personalization and campaign delivery through extensible APIs and pre-built connectors. +[Eagle Eye](https://eagleeye.com/?utm_source=segmentio&utm_medium=docs&utm_campaign=partners){:target="_blank"} delivers real-time loyalty and promotions solutions for enterprise brands, enabling seamless integration with leading customer data platforms and marketing activation tools. The platform connects digital and in-store experiences, accelerating personalization and campaign delivery through extensible APIs and pre-built connectors. -This integration requires both an Eagle Eye Source and Destination. The source sends events into Segment, and the destination processes those events to trigger specific behaviorial actions in Eagle Eye. +This integration requires both an Eagle Eye Source and Destination. The source sends events into Segment, and the destination processes those events to trigger specific behavioral actions in Eagle Eye. -> warning "" -> This destination must be used in tandem with a corresponding Eagle Eye Source to trigger behaviorial actions in Eagle Eye. +> warning "Required Source Integration" +> This destination must be used in tandem with a corresponding Eagle Eye Source to trigger behavioral actions in Eagle Eye. -This destination is maintained by Eagle Eye. For any issues with the destination, [contact their Support team](mailto:support@eagleeye.com). +This destination is maintained by Eagle Eye. For any issues with the destination, [contact their Support team](mailto:support@eagleeye.com){:target="_blank"}. ## Getting started To set up the Eagle Eye (Actions) destination in Segment: -1. From your workspace's [Destination catalog page](https://app.segment.com/goto-my-workspace/destinations/catalog){:target="_blank”} search for "Eagle Eye". +1. From your workspace's [Destination catalog page](https://app.segment.com/goto-my-workspace/destinations/catalog){:target="_blank"} search for "Eagle Eye". 2. Select "Eagle Eye (Actions)" and click **Add Destination**. 3. Select an existing Source to connect to Eagle Eye (Actions). -4. Set up your EE Connector for Segment by following the [CDP and MAP Connectors guide](https://developer.eagleeye.com/eagleeye-developer/docs/eagle-eye-connect-cdp-map){:target="_blank”}. You'll also need to refer to the [Segment-specific page](https://developer.eagleeye.com/docs/segment){:target="_blank”} for additional information required during setup. +4. Set up your EE Connector for Segment by following the [CDP and MAP Connectors guide](https://developer.eagleeye.com/eagleeye-developer/docs/eagle-eye-connect-cdp-map){:target="_blank"}. You'll also need to refer to the [Segment-specific page](https://developer.eagleeye.com/docs/segment){:target="_blank"} for additional information required during setup. 5. Go to the Eagle Eye (Actions) destination settings in Segment and enter the **Connector URL** and **Connector External Key** obtained in the previous step. {% include components/actions-fields.html %} @@ -30,15 +30,15 @@ To set up the Eagle Eye (Actions) destination in Segment: This guide assumes you're using the Eagle Eye Connector for Segment to send events. Any event is considered valid as long as it includes the properties "User identity value" and (optionally) "Wallet transaction reference". -To set this up, you must first configure [Social Behavioral Action Triggers](https://developer.eagleeye.com/eagleeye-developer/docs/segment#set-up-social-behavioural-action-triggers){:target="_blank”} within the Eagle Eye AIR dashboard. You can find this under **Triggers > Search** / **Create Behavioral Action**. +To set this up, you must first configure [Social Behavioral Action Triggers](https://developer.eagleeye.com/eagleeye-developer/docs/segment#set-up-social-behavioural-action-triggers){:target="_blank"} within the Eagle Eye AIR dashboard. You can find this under **Triggers > Search** / **Create Behavioral Action**. To map incoming event data and trigger behavioral actions in the Eagle Eye AIR dashboard, follow these steps: 1. In the Eagle Eye (Actions) destination, go to the Mappings tab and click **New Mapping**. 2. From the action list, select **Trigger Behavioral Action**. 3. Define your event trigger conditions. For example, if you want to trigger an action only for customers who just joined the program, select "Event Type is Track" and "Event Name is EE Loyalty Program Joined" (or any other event that contains the information you need). -4. Under **Map fields**, ensure that at least "User identity value" and "Behavioral Action trigger reference" are mapped. The trigger reference will be the same as the one set within the Eagle Eye AIR dashboard when creating the behavioral action. +4. Under **Map fields**, ensure that at least "User identity value" and "Behavioral Action trigger reference" are mapped. The trigger reference will be the same as the one set within the Eagle Eye AIR dashboard when creating the behavioral action. 5. (Optional) If your behavioral action requires a "Wallet transaction reference", you can pull it from one of the event fields. For example, an event property that may contain the transaction reference, such as `properties.order_id`. 6. Click **Next**, give the mapping a name and hit **Save** or **Save and enable**. -For more information on events sent through the Eagle Eye Connector for Segment and their properties, refer to the [Data Model](https://developer.eagleeye.com/eagleeye-developer/docs/segment#data-model){:target="_blank”} documentation. +For more information on events sent through the Eagle Eye Connector for Segment and their properties, refer to the [Data Model](https://developer.eagleeye.com/eagleeye-developer/docs/segment#data-model){:target="_blank"} documentation. diff --git a/src/connections/storage/catalog/data-lakes/index.md b/src/connections/storage/catalog/data-lakes/index.md index 9d96da8d11..d05220b839 100644 --- a/src/connections/storage/catalog/data-lakes/index.md +++ b/src/connections/storage/catalog/data-lakes/index.md @@ -7,9 +7,7 @@ redirect_from: {% include content/plan-grid.md name="data-lakes" %} Segment Data Lakes provide a way to collect large quantities of data in a format that's optimized for targeted data science and data analytics workflows. You can read [more information about Data Lakes](/docs/connections/storage/data-lakes/) and learn [how they differ from Warehouses](/docs/connections/storage/data-lakes/comparison/) in Segment's Data Lakes documentation. -Segment supports two type of data-lakes: -- [AWS Data Lakes](/docs/connections/storage/catalog/data-lakes/#set-up-segment-data-lakes) -- [Segment Data Lakes (Azure)](/docs/connections/storage/catalog/data-lakes/#set-up-segment-data-lakes-azure) +Segment only supports [AWS Data Lakes](/docs/connections/storage/catalog/data-lakes/#set-up-segment-data-lakes). > success "" > You can also set up your Segment Data Lakes using [Lake Formation](/docs/connections/storage/data-lakes/lake-formation/), a fully managed service built on top of the AWS Glue Data Catalog. @@ -83,302 +81,6 @@ Replay processing time can vary depending on the volume of data and number of ev Segment creates a separate EMR cluster to run replays, then destroys it when the replay finishes. This ensures that regular Data Lakes syncs are not interrupted, and helps the replay finish faster. -## Set up Segment Data Lakes (Azure) - -To set up Segment Data Lakes (Azure), create your Azure resources and then enable the Data Lakes destination in the Segment app. - -### Prerequisites - -Before you can configure your Azure resources, you must complete the following prerequisites: -- [Create an Azure subscription](https://azure.microsoft.com/en-us/free/){:target="_blank”} -- [Create an Azure resource group](https://docs.microsoft.com/en-us/azure/azure-resource-manager/management/manage-resource-groups-portal#create-resource-groups){:target="_blank”} -- Create an account with `Microsoft.Authorization/roleAssignments/write` permissions -- Configure the [Azure Command Line Interface (Azure CLI)](https://docs.microsoft.com/en-us/cli/azure/install-azure-cli){:target="_blank”} - -### Step 1 - Create an ALDS-enabled storage account - -1. Sign in to your [Azure environment](https://portal.azure.com){:target="_blank”}. -2. From the [Azure home page](https://portal.azure.com/#home){:target="_blank”}, select **Create a resource**. -3. Search for and select **Storage account**. -4. On the Storage account resource page, select the **Storage account** plan and click **Create**. -5. On the **Basic** tab, select an existing subscription and resource group, give your storage account a name, and update any necessary instance details. -6. Click **Next: Advanced**. -7. On the **Advanced Settings** tab in the Security section, select the following options: - - Require secure transfer for REST API operations - - Enable storage account key access - - Minimum TLS version: Version 1.2 -8. In the Data Lake Storage Gen2 section, select **Enable hierarchical namespace**. In the Blob storage selection, select the **Hot** option. -9. Click **Next: Networking**. -10. On the **Networking** page, select **Disable public access and use private access**. -11. Click **Review + create**. Take note of your location and storage account name, and review your chosen settings. When you are satisfied with your selections, click **Create**. -12. After your resource is deployed, click **Go to resource**. -13. On the storage account overview page, select the **Containers** button in the Data storage tab. -14. Select **Container**. Give your container a name, and select the **Private** level of public access. Click **Create**. - -> warning " " -> Before continuing, note the Location, Storage account name, and the Azure storage container name: you'll need this information when configuring the Segment Data Lakes (Azure) destination in the Segment app. - -### Step 2 - Set up Key Vault - -1. From the [home page of your Azure portal](https://portal.azure.com/#home){:target="_blank”}, select **Create a resource**. -2. Search for and select **Key Vault**. -3. On the Key Vault resource page, select the **Key Vault** plan and click **Create**. -4. On the **Basic** tab, select an existing subscription and resource group, give your Key Vault a name, and update the **Days to retain deleted vaults** setting, if desired. -5. Click **Review + create**. -6. Review your chosen settings. When you are satisfied with your selections, click **Review + create**. -7. After your resource is deployed, click **Go to resource**. -8. On the Key Vault page, select the **Access control (IAM)** tab. -9. Click **Add** and select **Add role assignment**. -10. On the **Roles** tab, select the `Key Vault Secrets User` role. Click **Next**. -11. On the **Members** tab, select a **User, group, or service principal**. -12. Click **Select members**. -13. Search for and select the `Databricks Resource Provider` service principal. -14. Click **Select**. -15. Under the **Members** header, verify that you selected the Databricks Resource Provider. Click **Review + assign**. - -### Step 3 - Set up Azure MySQL database - -1. From the [home page of your Azure portal](https://portal.azure.com/#home){:target="_blank”}, select **Create a resource**. -2. Search for and select **Azure Database for MySQL**. -3. On the Azure Database for MySQL resource page, select the **Azure Database for MySQL** plan and click **Create**. -4. Select **Single server** and click **Create**. -5. On the **Basic** tab, select an existing subscription and resource group, enter server details and create an administrator account. Due to the configurations required for the setup, Data Lakes supports MySQL version **5.7** only. Before you proceed, please ensure you have the correct MySQL server version selected. -6. Click **Review + create**. -7. Review your chosen settings. When you are satisfied with your selections, click **Create**. -8. After your resource is deployed, click **Go to resource**. -9. From the resource page, select the **Connection security** tab. -10. Under the Firewall rules section, select **Yes** to allow access to Azure services, and click the **Allow current client IP address (xx.xxx.xxx.xx)** button to allow access from your current IP address. -11. Click **Save** to save the changes you made on the **Connection security** page, and select the **Server parameters** tab. -12. Update the `lower_case_table_names` value to 2, and click **Save**. -13. Select the **Overview** tab and click the **Restart** button to restart your database. Restarting your database updates the `lower_case_table_name` setting. -14. Once the server restarts successfully, open your Azure CLI. -15. Sign into the MySQL server from your command line by entering the following command: - ```curl - mysql --host=/[HOSTNAME] --port=3306 --user=[USERNAME] --password=[PASSWORD] - ``` -16. Run the `CREATE DATABASE` command to create your Hive Metastore: - ```sql - CREATE DATABASE ; - ``` - -> warning " " -> Before continuing, note the MySQL server URL, username and password for the admin account, and your database name: you'll need this information when configuring the Segment Data Lakes (Azure) destination in the Segment app. - - -### Step 4 - Set up Databricks - -> info "Databricks pricing tier" -> If you create a Databricks instance only for Segment Data Lakes (Azure) usage, only the standard pricing tier is required. However, if you use your Databricks instance for other applications, you may require premium pricing. - -1. From the [home page of your Azure portal](https://portal.azure.com/#home){:target="_blank”}, select **Create a resource**. -2. Search for and select **Azure Databricks**. -3. On the Azure Database for MySQL resource page, select the **Azure Databricks** plan and click **Create**. -4. On the **Basic** tab, select an existing subscription and resource group, enter a name for your workspace, select the region you'd like to house your Databricks instance in, and select a pricing tier. For those using the Databricks instance only for Segment Data Lakes (Azure), a Standard pricing tier is appropriate. If you plan to use your Databricks instance for more than just Segment Data Lakes (Azure), you may require the premium pricing tier. -5. Click **Review + create**. -6. Review your chosen settings. When you are satisfied with your selections, click **Create**. -7. After your resource is deployed, click **Go to resource**. -8. On the Azure Databricks Service overview page, click **Launch Workspace**. -9. On the Databricks page, select **Create a cluster**. -10. On the Compute page, select **Create Cluster**. -11. Enter a name for your cluster and select the `Standard_DS4_v2` worker type. Set the minimum number of workers to 2, and the maximum number of workers to 8. __Segment recommends deselecting the "Terminate after X minutes" setting, as the time it takes to restart a cluster may delay your Data Lake syncs.__ -12. Click **Create Cluster**. -13. Open [your Azure portal](https://portal.azure.com/#home){:target="_blank”} and select the Key Vault you created in a previous step. -14. On the Key Vault page, select the JSON View link to view the Resource ID and vaultURI. Take note of these values, as you'll need them in the next step to configure your Databricks instance. -15. Open `https://#secrets/createScope` and enter the following information to connect your Databricks instance to the Key Vault you created in an earlier step: - - **Scope Name**: Set this value to `segment`. - - **Manage Principal**: Select **All Users**. - - **DNS Name**: Set this value to the Vault URI of your Key Vault instance. - - **Resource ID**: The Resource ID of your Azure Key Vault instance. -16. When you've entered all of your information, click **Create**. - -> warning " " -> Before continuing, note the Cluster ID, Workspace name, Workspace URL, and the Azure Resource Group for your Databricks Workspace: you'll need this information when configuring the Segment Data Lakes (Azure) destination in the Segment app. - -### Step 5 - Set up a Service Principal - -1. Open the Databricks instance you created in [Step 4 - Set up Databricks](#step-4---set-up-databricks). -2. Click **Settings** and select **User settings**. -3. On the Access tokens page, click **Generate new token**. -4. Enter a comment for your token, select the lifetime of your ticket, and click **Generate**. -5. Copy your token, as you'll use this to add your service principal to your workspace. -6. Open your Azure CLI and create a new service principal using the following commands:
-``` powershell -az login -az ad sp create-for-rbac --name -``` -7. In your Azure portal, select the Databricks instance you created in [Step 4 - Set up Databricks](#step-4---set-up-databricks). -8. On the overview page for your Databricks instance, select **Access control (IAM)**. -9. Click **Add** and select **Add role assignment**. -10. On the **Roles** tab, select the `Managed Application Operator` role. Click **Next**. -11. On the **Members** tab, select a **User, group, or service principal**. -12. Click **Select members**. -13. Search for and select the Service Principal you created above. -14. Click **Select**. -15. Under the **Members** header, verify that you selected your Service Principal. Click **Review + assign**. -16. Return to the Azure home page. Select your storage account. -17. On the overview page for your storage account, select **Access control (IAM)**. -18. Click **Add** and select **Add role assignment**. -19. On the **Roles** tab, select the `Storage Blob Data Contributor` role. Click **Next**. -20. On the **Members** tab, select a **User, group, or service principal**. -21. Click **Select members**. -22. Search for and select the Service Principal you created above. -23. Click **Select**. -24. Under the **Members** header, verify that you selected your Service Principal. Click **Review + assign**. -25. Open your Key Vault. In the sidebar, select **Secrets**. -26. Click **Generate/Import**. -27. On the Create a secret page, select **Manual**. Enter the name `spsecret` for your secret, and enter the name of the secret you created in Databricks in the **Value** field. -28. From your Azure CLI, call the Databricks SCIM API to add your service principal to your workspace, replacing ` `with the URL of your Databricks workspace, ` `with the access token you created in an earlier step, and `` with the client ID of your service principal:
-```curl -curl -X POST 'https:///api/2.0/preview/scim/v2/ServicePrincipals' \ - --header 'Content-Type: application/scim+json' \ - --header 'Authorization: Bearer ' \ - --data-raw '{ - "schemas":[ - "urn:ietf:params:scim:schemas:core:2.0:ServicePrincipal" - ], - "applicationId":"", - "displayName": "test-sp", - "entitlements":[ - { - "value":"allow-cluster-create" - } - ] - }' -``` -29. Open Databricks and navigate to your cluster. Select **Permissions**. -30. In the permissions menu, grant your service principal **Can Manage** permissions. - -> warning " " -> Before continuing, note the Client ID and Client Secret for your Service Principal: you'll need this information when configuring the Segment Data Lakes (Azure) destination in the Segment app. - -### Step 6 - Configure Databricks Cluster - -> warning "Optional configuration settings for log4j vulnerability" -> While Databricks released a statement that clusters are likely unaffected by the log4j vulnerability, out of an abundance of caution, Databricks recommends updating to log4j 2.15+ or adding the following options to the Spark configuration:
`spark.driver.extraJavaOptions "-Dlog4j2.formatMsgNoLookups=true"`
`spark.executor.extraJavaOptions "-Dlog4j2.formatMsgNoLookups=true"` - -1. Connect to a [Hive metastore](https://docs.databricks.com/data/metastores/external-hive-metastore.html){:target="_blank”} on your Databricks cluster using the following Spark configuration, replacing the variables (``) with information from your workspace:
-```py -## Configs so we can read from the storage account -spark.hadoop.fs.azure.account.oauth.provider.type..dfs.core.windows.net org.apache.hadoop.fs.azurebfs.oauth2.ClientCredsTokenProvider -spark.hadoop.fs.azure.account.oauth2.client.endpoint..dfs.core.windows.net https://login.microsoftonline.com//oauth2/token -spark.hadoop.fs.azure.account.oauth2.client.secret..dfs.core.windows.net -spark.hadoop.fs.azure.account.auth.type..dfs.core.windows.net OAuth -spark.hadoop.fs.azure.account.oauth2.client.id..dfs.core.windows.net -## -## -spark.hadoop.javax.jdo.option.ConnectionDriverName org.mariadb.jdbc.Driver -spark.hadoop.javax.jdo.option.ConnectionURL jdbc:mysql://:/?useSSL=true&requireSSL=true&enabledSslProtocolSuites=TLSv1.2 -spark.hadoop.javax.jdo.option.ConnectionUserName -spark.hadoop.javax.jdo.option.ConnectionPassword -## -## -## -spark.hive.mapred.supports.subdirectories true -spark.sql.storeAssignmentPolicy Legacy -mapreduce.input.fileinputformat.input.dir.recursive true -spark.sql.hive.convertMetastoreParquet false -## -datanucleus.autoCreateSchema true -datanucleus.autoCreateTables true -spark.sql.hive.metastore.schema.verification false -datanucleus.fixedDatastore false -## -spark.sql.hive.metastore.version 2.3.7 -spark.sql.hive.metastore.jars builtin -``` - -3. Log in to your Databricks instance and open your cluster. -4. On the overview page for your cluster, select **Edit**. -5. Open the **Advanced options** toggle and paste the Spark config you copied above, replacing the variables (``) with information from your workspace. -6. Select **Confirm and restart**. On the popup window, select **Confirm**. -7. Log in to your Azure MySQL database using the following command:
-```curl -mysql --host=[HOSTNAME] --port=3306 --user=[USERNAME] --password=[PASSWORD] -``` -8. Once you've logged in to your MySQL database, run the following commands:
-```sql -USE -INSERT INTO VERSION (VER_ID, SCHEMA_VERSION) VALUES (0, '2.3.7'); -``` -9. Log in to your Databricks cluster. -10. Click **Create** and select **Notebook**. -11. Give your cluster a name, select **SQL** as the default language, and make sure it's located in the cluster you created in [Step 4 - Set up Databricks](#step-4---set-up-databricks). -12. Click **Create**. -13. On the overview page for your new notebook, run the following command:
-```sql -CREATE TABLE test (id string); -``` -14. Open your cluster. -15. On the overview page for your cluster, select **Edit**. -16. Open the **Advanced options** toggle and paste the following code snippet:
-```py -datanucleus.autoCreateSchema false -datanucleus.autoCreateTables false -spark.sql.hive.metastore.schema.verification true -datanucleus.fixedDatastore true -``` -17. Select **Confirm and restart**. On the popup window, select **Confirm**. - -### Step 7 - Enable the Data Lakes destination in the Segment app - -After you set up the necessary resources in Azure, the next step is to set up the Data Lakes destination in Segment: - -1. In the [Segment App](https://app.segment.com/goto-my-workspace/overview){:target="_blank”}, click **Add Destination**. -2. Search for and select **Segment Data Lakes (Azure)**. -2. Click the **Configure Data Lakes** button, and select the source you'd like to receive data from. Click **Next**. -3. In the **Connection Settings** section, enter the following values: - - **Azure Storage Account**: The name of the Azure Storage account that you set up in [Step 1 - Create an ALDS-enabled storage account](#step-1---create-an-alds-enabled-storage-account). - ![img.png](images/storageaccount.png) - - **Azure Storage Container**: The name of the Azure Storage Container you created in [Step 1 - Create an ALDS-enabled storage account](#step-1---create-an-alds-enabled-storage-account). - ![img_1.png](images/storagecontainer.png) - - **Azure Subscription ID**: The ID of your [Azure subscription](https://docs.microsoft.com/en-us/azure/azure-portal/get-subscription-tenant-id){:target="_blank”}.
Please add it as it is in the Azure portal, in the format `********-****-****-****-************` - - **Azure Tenant ID**: The Tenant ID of your [Azure Active directory](https://docs.microsoft.com/en-us/azure/active-directory/fundamentals/active-directory-how-to-find-tenant){:target="_blank”}.
Please add it as it is in the Azure portal, in the format `********-****-****-****-************` - - **Databricks Cluster ID**: The ID of your [Databricks cluster](https://docs.databricks.com/workspace/workspace-details.html#cluster-url-and-id){:target="_blank”}. - - **Databricks Instance URL**: The ID of your [Databricks workspace](https://docs.databricks.com/workspace/workspace-details.html#workspace-instance-names-urls-and-ids){:target="_blank”}.
The correct format for adding the URL is 'adb-0000000000000000.00.azureatabricks.net' - - **Databricks Workspace Name**: The name of your [Databricks workspace](https://docs.databricks.com/workspace/workspace-details.html#workspace-instance-names-urls-and-ids){:target="_blank”}. - - **Databricks Workspace Resource Group**: The resource group that hosts your Azure Databricks instance. This is visible in Azure on the overview page for your Databricks instance. - - **Region**: The location of the Azure Storage account you set up in [Step 1 - Create an ALDS-enabled storage account](#step-1---create-an-alds-enabled-storage-account). - - **Service Principal Client ID**: The Client ID of the Service Principal that you set up in [Step 5 - Set up a Service Principal](#step-5---set-up-a-service-principal). - - **Service Principal Client Secret**: The Secret for the Service Principal that you set up in [Step 5 - Set up a Service Principal](#step-5---set-up-a-service-principal). - - -### (Optional) Set up your Segment Data Lake (Azure) using Terraform - -Instead of manually configuring your Data Lake, you can create it using the script in the [`terraform-segment-data-lakes`](https://github.com/segmentio/terraform-segment-data-lakes){:target="_blank”} GitHub repository. - -> warning "" -> This script requires Terraform versions 0.12+. - -Before you can run the Terraform script, create a Databricks workspace in the Azure UI using the instructions in [Step 4 - Set up Databricks](#step-4---set-up-databricks). Note the **Workspace URL**, as you will need it to run the script. - -In the setup file, set the following local variables: - -```js - -locals { -region = "" -resource_group = "" -storage_account = " -```py -spark.sql.hive.metastore.schema.verification.record.version false -``` -
After you've added to your config, restart your cluster so that your changes can take effect. If you continue to encounter errors, [contact Segment Support](https://segment.com/help/contact/){:target="_blank"}. - -#### What do I do if I get a "Version table does not exist" error when setting up the Azure MySQL database? -Check your Spark configs to ensure that the information you entered about the database is correct, then restart the cluster. The Databricks cluster automatically initializes the Hive Metastore, so an issue with your config file will stop the table from being created. If you continue to encounter errors, [contact Segment Support](https://segment.com/help/contact/){:target="_blank"}. diff --git a/src/connections/storage/data-lakes/index.md b/src/connections/storage/data-lakes/index.md index d2a823fb09..620d6d71e8 100644 --- a/src/connections/storage/data-lakes/index.md +++ b/src/connections/storage/data-lakes/index.md @@ -19,16 +19,16 @@ Data lakes typically have four layers: ![A graphic showing the information flowing from the metadata into the query, compute, and metadata layers, and then into the storage layer](images/data_lakes_overview_graphic.png) -Segment Data Lakes sends Segment data to a cloud data store, either AWS S3 or Azure Data Lake Storage Gen2 (ADLS), in a format optimized to reduce processing for data analytics and data science workloads. Segment data is great for building machine learning models for personalization and recommendations, and for other large scale advanced analytics. Data Lakes reduces the amount of processing required to get real value out of your data. +Segment Data Lakes sends Segment data to a cloud data store, AWS S3, in a format optimized to reduce processing for data analytics and data science workloads. Segment data is great for building machine learning models for personalization and recommendations, and for other large scale advanced analytics. Data Lakes reduces the amount of processing required to get real value out of your data. > warning "Segment Data Lakes deletion policies" -> Segment Data Lakes (AWS) and Segment Data Lakes (Azure) do not support Segment's [user deletion and suppression](/docs/privacy/user-deletion-and-suppression/) capabilities, as you retain your data in systems that you manage. +> Segment Data Lakes (AWS) does not support Segment's [user deletion and suppression](/docs/privacy/user-deletion-and-suppression/) capabilities, as you retain your data in systems that you manage. To learn more about Segment Data Lakes, check out the Segment blog post [Introducing Segment Data Lakes](https://segment.com/blog/introducing-segment-data-lakes/){:target="_blank"}. ## How Data Lakes work -Segment supports Data Lakes hosted on two cloud providers: Amazon Web Services (AWS) and Microsoft Azure. Each cloud provider has a similar system for managing data, but offer different query engines, post-processing systems, and analytics options. +Segment supports Data Lakes hosted on Amazon Web Services (AWS). Each cloud provider has a similar system for managing data, but offer different query engines, post-processing systems, and analytics options. ### How Segment Data Lakes (AWS) works @@ -40,16 +40,6 @@ Segment sends data to S3 by orchestrating the processing in an EMR (Elastic MapR ![A diagram visualizing data flowing from a Segment user into your account and into a Glue catalog/S3 bucket](images/dl_vpc.png) -### How Segment Data Lakes (Azure) works - -Data Lakes store Segment data in ADLS in a read-optimized encoding format (Parquet) which makes the data more accessible and actionable. To help you zero-in on the right data, Data Lakes also creates logical data partitions and event tables, and integrates metadata with existing schema management tools, like the Hive Metastore. The resulting data set is optimized for use with systems like Power BI and Azure HDInsight or machine learning vendors like Azure Databricks or Azure Synapse Analytics. - -![A diagram showing data flowing from Segment, through DataBricks, Parquet and Azure Data Lake Storage Gen2 into the Hive Metastore, and then into your post-processing systems](images/Azure_DL_setup.png) - -## Set up Segment Data Lakes (Azure) - -For detailed Segment Data Lakes (Azure) setup instructions, see the [Data Lakes setup page](/docs/connections/storage/catalog/data-lakes/). - ### Set up Segment Data Lakes (AWS) When setting up your data lake using the [Data Lakes catalog page](/docs/connections/storage/catalog/data-lakes/), be sure to consider the EMR and AWS IAM components listed below. @@ -64,18 +54,6 @@ Data Lakes uses an IAM role to grant Segment secure access to your AWS account. - **external_ids**: External IDs are the part of the IAM role which Segment uses to assume the role providing access to your AWS account. You will define the external ID in the IAM role as the Segment Workspace ID in which you want to connect to Data Lakes. The Segment Workspace ID can be retrieved from the [Segment app](https://app.segment.com/goto-my-workspace/overview){:target="_blank"} by navigating to **Settings > General Settings > ID**. - **s3_bucket**: Name of the S3 bucket used by the Data Lake. -### Set up Segment Data Lakes (Azure) - -To connect Segment Data Lakes (Azure), you must set up the following components in your Azure environment: - -- [Azure Storage Account](/docs/connections/storage/catalog/data-lakes/#step-1---create-an-alds-enabled-storage-account): An Azure storage account contains all of your Azure Storage data objects, including blobs, file shares, queues, tables, and disks. -- [Azure KeyVault Instance](/docs/connections/storage/catalog/data-lakes/#step-2---set-up-key-vault): Azure KeyVault provides a secure store for your keys, secrets, and certificates. -- [Azure MySQL Database](/docs/connections/storage/catalog/data-lakes/#step-3---set-up-azure-mysql-database): The MySQL database is a relational database service based on the MySQL Community Edition, versions 5.6, 5.7, and 8.0. -- [Databricks Instance](/docs/connections/storage/catalog/data-lakes/#step-4---set-up-databricks): Azure Databricks is a data analytics cluster that offers multiple environments (Databricks SQL, Databricks Data Science and Engineering, and Databricks Machine Learning) for you to develop data-intensive applications. -- [Databricks Cluster](/docs/connections/storage/catalog/data-lakes/#step-6---configure-databricks-cluster): The Databricks cluster is a cluster of computation resources that you can use to run data science and analytics workloads. -- [Service Principal](/docs/connections/storage/catalog/data-lakes/#step-5---set-up-a-service-principal): Service principals are identities used to access specific resources. - -For more information about configuring Segment Data Lakes (Azure), see the [Data Lakes setup page](/docs/connections/storage/catalog/data-lakes/#set-up-segment-data-lakes-azure). ## Data Lakes schema @@ -128,15 +106,6 @@ The schema inferred by Segment is stored in a Glue database within Glue Data Cat > info "" > The recommended IAM role permissions grant Segment access to create the Glue databases on your behalf. If you do not grant Segment these permissions, you must manually create the Glue databases for Segment to write to. -### Segment Data Lakes (Azure) schema - -Segment Data Lakes (Azure) applies a consistent schema to make raw data accessible for queries. A transformer automatically calculates the desired schema and uploads a schema JSON file for each event type to your Azure Data Lake Storage (ADLS) in the `/staging/` directory. - -Segment partitions the data in ALDS by the Segment source, event type, then the day and hour an event was received by Segment, to ensure that the data is actionable and accessible. - -The file path looks like this: -`//staging//` - ### Data types Data Lakes infers the data type for an event it receives. Groups of events are polled every hour to infer the data type for that each event. @@ -181,7 +150,7 @@ Segment doesn't support User deletions in Data Lakes, but supports [user suppres ### How does Data Lakes handle schema evolution? -As the data schema evolves, both Segment Data Lakes (AWS) and Segment Data Lakes (Azure) can detect new columns and add them to Glue Data Catalog or Azure Data Lake Storage (ADLS). However, Segment can't update existing data types. To update Segment-created data types, please reach out to [AWS Support](https://aws.amazon.com/contact-us/){:target="_blank"} or [Azure Support](https://support.microsoft.com/en-us/topic/contact-microsoft-azure-support-2315e669-8b1f-493b-5fb1-d88a8736ffe4){:target="_blank"}. +As the data schema evolves, Segment Data Lakes (AWS) can detect new columns and add them to Glue Data Catalog. However, Segment can't update existing data types. To update Segment-created data types, please reach out to [AWS Support](https://aws.amazon.com/contact-us/){:target="_blank"}. ### How does Data Lakes work with Protocols? @@ -206,9 +175,6 @@ Data Lakes offers 12 syncs in a 24 hour period and doesn't offer a custom sync s ### What is the cost to use AWS Glue? You can find details on Amazon's [pricing for Glue](https://aws.amazon.com/glue/pricing/){:target="_blank"} page. For reference, Data Lakes creates 1 table per event type in your source, and adds 1 partition per hour to the event table. -### What is the cost to use Microsoft Azure? -You can find details on Microsoft's [pricing for Azure](https://azure.microsoft.com/en-us/pricing/){:target="_blank"} page. For reference, Data Lakes creates 1 table per event type in your source, and adds 1 partition per hour to the event table. - ### What limits does AWS Glue have? AWS Glue has limits across various factors, such as number of databases per account, tables per account, and so on. See the [full list of Glue limits](https://docs.aws.amazon.com/general/latest/gr/glue.html#limits_glue){:target="_blank"} for more information. @@ -222,9 +188,3 @@ Segment stops creating new tables for the events after you exceed this limit. Ho You should also read the [additional considerations in Amazon's documentation](https://docs.aws.amazon.com/emr/latest/ReleaseGuide/emr-hive-metastore-glue.html){:target="_blank"} when using AWS Glue Data Catalog. -### What analytics tools are available to use with Segment Data Lakes (Azure)? -Segment Data Lakes (Azure) supports the following analytics tools: - - PowerBI - - Azure HDInsight - - Azure Synapse Analytics - - Databricks diff --git a/src/guides/what-is-replay.md b/src/guides/what-is-replay.md index 2d29e99b29..ef3faba484 100644 --- a/src/guides/what-is-replay.md +++ b/src/guides/what-is-replay.md @@ -19,11 +19,11 @@ This means you can send a limited sample of your data to a new tool to test it o With Replays, you're protected from outages and errors. If a destination which you rely on experiences an outage, or is temporarily unable to accept incoming data, you can use Replays to re-send data to that tool once the service recovers. You can also use Replays to recover from errors caused by misconfigurations in your Segment systems. For example, if you send data in the wrong format, or want to apply [destination filters](/docs/connections/destinations/destination-filters/). In this case, you can change your mapping using a destination filter, clear out the bad data, and replay it to that destination. You can also use this to update the schema in your data warehouse when it changes. -For more information, [Contact us](https://segment.com/help/contact/) and our Success Engineers will walk you through the process. +For more information, [contact Segment support](https://segment.com/help/contact/){:target="_blank"}. ## Replays considerations -Replays are currently only available for Business Tier customers, and due to their complex nature are not self-serve. [Contact us](https://segment.com/help/contact/) to learn more, or to request a replay for your workspace. When requesting a replay, include the workspace, the source to replay from, the destination tool or tools, and the time period. +Replays are currently only available for Business Tier customers, and due to their complex nature are not self-serve. [Contact Segment support](https://segment.com/help/contact/){:target="_blank"} to learn more, or to request a replay for your workspace. When requesting a replay, include the workspace, the source to replay from, the destination tool or tools, and the time period. Replays can process unlimited data, but they're rate limited to respect limitations in downstream partner tools. If you're also sending data to the destination being replayed to in real time, then, when determining your replay's limit, you'll want to take into account the rate limit being used by real-time events. You should also account for a small margin of your rate limit to allow events to be retried. @@ -38,13 +38,19 @@ You can initiate replays for some or all events, but you can't apply conditional The destination is not required to be enabled in order for a replay to be successful, including Destination Functions. - The destination must be connected to the source, but can remain disabled while the replay is running. - Destination filters are still considered when you run replays on disabled destinations. -- There are a few exceptions for destinations that must be enabled for the replay to be successful : Amazon S3 and Google Cloud Source (GCS). +- There are a few exceptions for destinations that must be enabled for the replay to be successful: Amazon S3 and Google Cloud Source (GCS). ### Replay-eligible destinations Replays are available for any destinations which support cloud-mode data (meaning data routed through Segment) and which also process timestamps. Destinations that are only available in device-mode (meaning where data is sent directly from the users' devices to the destination tool) cannot receive Replays. -Not all destinations support data deduplication, so you may need to delete, archive, or remove any older versions of the data before initiating a replay. [contact Segment support](https://segment.com/help/contact/){:target="_blank"} if you have questions or want help. +Not all destinations support data deduplication, so you may need to delete, archive, or remove any older versions of the data before initiating a replay. [Contact Segment support](https://segment.com/help/contact/){:target="_blank"} if you have questions or want help. + +#### What happens to integrations and `consent` objects during a replay? + +When Segment replays events, it does not retain the original integrations object flag, but it does preserve the `consent` object. + +During replays, Segment replaces the 'integrations' object with a structure that includes `All:false` and `[integration name]:true`. As a result, Segment removes the original integrations flag during replay. The replay tool doesn't overwrite the original `consent` object. If an event was previously dropped for a specific destination due to the `consent` object, the event drops within Segment again during replay. ### Replays & Destination Filters