feat: add FDv2 file data source initiator

this commit will introduce the implementation of FDv2 File initiator

Author: joker23

packages/shared/common/src/internal/fdv2/FDv2ChangeSetBuilder.ts

@@ -0,0 +1,96 @@
+import { PutObject, DeleteObject, Event } from './proto';
+
+// eventually this will be the same as the IntentCode type, but for now we'll use a simpler type
+type supportedIntentCodes = 'xfer-full';
+
+/**
+ * FDv2ChangeSetBuilder is a helper for constructing a change set for FDv2.
+ * The main use case for this builder is to help construct a change set from
+ * a FDv1 payload.
+ *
+ * @experimental
+ * This type is not stable, and not subject to any backwards
+ * compatibility guarantees or semantic versioning. It is not suitable for production usage.
+ */
+export default class FDv2ChangeSetBuilder {
+  private intent?: supportedIntentCodes;
+  private events: Event[] = [];
+
+  /**
+   * Begins a new change set with a given intent.
+   */
+  start(intent: supportedIntentCodes): this {
+    this.intent = intent;
+    this.events = [];
+
+    return this;
+  }
+
+  /**
+   * Returns the completed changeset.
+   * NOTE: currently, this builder is not designed to continuously build changesets, rather
+   * it is designed to construct a single changeset at a time. We can easily expand this by
+   * resetting some values in the future.
+   */
+  finish(): Array<Event> {
+    if (this.intent === undefined) {
+      throw new Error('changeset: cannot complete without a server-intent');
+    }
+
+    // NOTE: currently the only use case for this builder is to
+    // construct a change set for a file data intializer which only supports
+    // FDv1 format. As such, we need to use dummy values to satisfy the FDv2
+    // protocol.
+    const events: Array<Event> = [
+      {
+        event: 'server-intent',
+        data: {
+          payloads: [{
+              id: 'dummy-id',
+              target: 1,
+              intentCode: this.intent,
+              reason: 'payload-missing',
+            },
+          ]
+        }
+      },
+      ...this.events,
+      {
+        event: 'payload-transferred',
+        data: {
+          // IMPORTANT: the selector MUST be empty or "live" data synchronizers
+          // will not work as it would try to resume from a bogus state.
+          state: '',
+          version: 1,
+          id: 'dummy-id',
+        }
+      },
+    ];
+
+    return events;
+  }
+
+  /**
+   * Adds a new object to the changeset.
+   */
+  putObject(obj: PutObject): this {
+    this.events.push({
+      event: 'put-object',
+      data: obj,
+    });
+
+    return this
+  }
+
+  /**
+   * Adds a deletion to the changeset.
+   */
+  deleteObject(obj: DeleteObject): this {
+    this.events.push({
+      event: 'delete-object',
+      data: obj
+    });
+
+    return this
+  }
+}

packages/shared/common/src/internal/fdv2/index.ts

@@ -6,6 +6,7 @@ import {
   Update,
 } from './payloadProcessor';
 import { PayloadStreamReader } from './payloadStreamReader';
+import FDv2ChangeSetBuilder from './FDv2ChangeSetBuilder';
 
 export {
   FDv2EventsCollection,
@@ -14,4 +15,5 @@ export {
   PayloadProcessor,
   PayloadStreamReader,
   Update,
+  FDv2ChangeSetBuilder,
 };

packages/shared/common/src/internal/fdv2/payloadProcessor.ts

@@ -221,8 +221,10 @@ export class PayloadProcessor {
   private _processPayloadTransferred = (data: PayloadTransferred) => {
     // if the following properties haven't been provided by now, we should reset
     if (
-      !this._tempId || // server intent hasn't been received yet.
-      !data.state ||
+      // server intent hasn't been received yet.
+      !this._tempId ||
+      // selector can be an empty string if we are using a file data initilizer
+      (data.state === null || data.state === undefined) ||
       !data.version
     ) {
       this._resetAll(); // a reset is best defensive action since payload transferred terminates a payload

packages/shared/common/src/internal/fdv2/proto.ts

@@ -1,14 +1,16 @@
+export type EventType = 'server-intent' | 'put-object' | 'delete-object' | 'payload-transferred' | 'goodbye' | 'error'| 'heart-beat';
+export type IntentCode = 'xfer-full' | 'xfer-changes' | 'none';
+export type ObjectKind = 'flag' | 'segment';
+
 export interface Event {
-  event: string;
-  data: any;
+  event: EventType;
+  data: ServerIntentData | PutObject | DeleteObject | PayloadTransferred | GoodbyeObject | ErrorObject;
 }
 
 export interface ServerIntentData {
   payloads: PayloadIntent[];
 }
 
-export type IntentCode = 'xfer-full' | 'xfer-changes' | 'none';
-
 export interface PayloadIntent {
   id: string;
   target: number;
@@ -17,19 +19,31 @@ export interface PayloadIntent {
 }
 
 export interface PutObject {
-  kind: string;
+  kind: ObjectKind;
   key: string;
   version: number;
   object: any;
 }
 
 export interface DeleteObject {
-  kind: string;
+  kind: ObjectKind;
   key: string;
   version: number;
 }
 
+export interface GoodbyeObject {
+  reason: string;
+  silent: boolean;
+  catastrophe: boolean;
+}
+
+export interface ErrorObject {
+  payload_id: string;
+  reason: string;
+}
+
 export interface PayloadTransferred {
   state: string;
   version: number;
+  id?: string;
 }

packages/shared/sdk-server/src/LDClientImpl.ts

@@ -43,6 +43,7 @@ import BigSegmentStoreStatusProvider from './BigSegmentStatusProviderImpl';
 import { createPluginEnvironmentMetadata } from './createPluginEnvironmentMetadata';
 import { createPayloadListener } from './data_sources/createPayloadListenerFDv2';
 import { createStreamListeners } from './data_sources/createStreamListeners';
+import FileDataInitializerFDv2 from './data_sources/fileDataInitilizerFDv2';
 import DataSourceUpdates from './data_sources/DataSourceUpdates';
 import OneShotInitializerFDv2 from './data_sources/OneShotInitializerFDv2';
 import PollingProcessor from './data_sources/PollingProcessor';
@@ -325,14 +326,22 @@ function constructFDv2(
     const initializers: subsystem.LDDataSourceFactory[] = [];
 
     // use one shot initializer for performance and cost if we can do a combination of polling and streaming
-    if (isStandardOptions(dataSystem.dataSource)) {
+    if (dataSystem.dataSource?.initializerOptions?.type === 'polling') {
       initializers.push(
         () =>
           new OneShotInitializerFDv2(
             new Requestor(config, platform.requests, baseHeaders, '/sdk/poll', config.logger),
             config.logger,
           ),
       );
+    } else if (dataSystem.dataSource?.initializerOptions?.type === 'file') {
+      initializers.push(
+        () =>
+          new FileDataInitializerFDv2(
+            config,
+            platform,
+          ),
+      );
     }
 
     const synchronizers: subsystem.LDDataSourceFactory[] = [];

packages/shared/sdk-server/src/api/options/LDDataSystemOptions.ts

@@ -75,14 +75,33 @@ export type DataSourceOptions =
   | StreamingDataSourceOptions
   | PollingDataSourceOptions;
 
+/**
+ * Initializer option to read data from a file.
+ *
+ * NOTE: right now we only support data sources that are in FDv1 format.
+ */
+export interface FileDataInitializerOptions {
+  type: 'file';
+  paths: Array<string>;
+  yamlParser?: (data: string) => any;
+};
+
+/**
+ * Initializer option to initilize the SDK from doing a one time full payload transfer.
+ * This will be the default initializer used by the standard data source type.
+ */
+export interface PollingDataInitializerOptions {
+  type: 'polling';
+};
+
 /**
  * This standard data source is the recommended datasource for most customers. It will use
  * a combination of streaming and polling to initialize the SDK, provide real time updates,
  * and can switch between streaming and polling automatically to provide redundancy.
  */
 export interface StandardDataSourceOptions {
   dataSourceOptionsType: 'standard';
-
+  initializerOptions?: FileDataInitializerOptions | PollingDataInitializerOptions;
   /**
    * Sets the initial reconnect delay for the streaming connection, in seconds. Default if omitted.
    *
@@ -106,7 +125,7 @@ export interface StandardDataSourceOptions {
  */
 export interface StreamingDataSourceOptions {
   dataSourceOptionsType: 'streamingOnly';
-
+  initializerOptions?: FileDataInitializerOptions | PollingDataInitializerOptions;
   /**
    * Sets the initial reconnect delay for the streaming connection, in seconds. Default if omitted.
    *
@@ -124,7 +143,7 @@ export interface StreamingDataSourceOptions {
  */
 export interface PollingDataSourceOptions {
   dataSourceOptionsType: 'pollingOnly';
-
+  initializerOptions?: FileDataInitializerOptions | PollingDataInitializerOptions;
   /**
    * The time between polling requests, in seconds. Default if omitted.
    */