[DataAvailability] introduce api package in access engine

Makefile

@@ -138,8 +138,8 @@ endif
 
 .PHONY: generate-openapi
 generate-openapi:
-	swagger-codegen generate -l go -i https://raw.githubusercontent.com/onflow/flow/master/openapi/access.yaml -D packageName=models,modelDocs=false,models -o engine/access/rest/http/models;
-	go fmt ./engine/access/rest/http/models
+	swagger-codegen generate -l go -i https://raw.githubusercontent.com/onflow/flow/master/openapi/access.yaml -D packageName=models,modelDocs=false,models -o engine/access/api/rest/http/models;
+	go fmt ./engine/access/api/rest/http/models
 
 .PHONY: generate
 generate: generate-proto generate-mocks generate-fvm-env-wrappers
@@ -199,21 +199,21 @@ generate-mocks: install-mock-generators
 	mockery --name 'API' --dir="./access" --case=underscore --output="./access/mock" --outpkg="mock"
 	mockery --name 'Blocks' --dir="./access/validator" --case=underscore --output="./access/validator/mock" --outpkg="mock"
 	mockery --name 'API' --dir="./engine/protocol" --case=underscore --output="./engine/protocol/mock" --outpkg="mock"
-	mockery --name '.*' --dir="./engine/access/state_stream" --case=underscore --output="./engine/access/state_stream/mock" --outpkg="mock"
+	mockery --name '.*' --dir="./engine/access/api/state_stream" --case=underscore --output="./engine/access/api/state_stream/mock" --outpkg="mock"
 	mockery --name 'BlockTracker' --dir="./engine/access/subscription/tracker" --case=underscore --output="./engine/access/subscription/tracker/mock"  --outpkg="mock"
 	mockery --name 'ExecutionDataTracker' --dir="./engine/access/subscription/tracker" --case=underscore --output="./engine/access/subscription/tracker/mock"  --outpkg="mock"
-	mockery --name 'DataProvider' --dir="./engine/access/rest/websockets/data_providers" --case=underscore --output="./engine/access/rest/websockets/data_providers/mock"  --outpkg="mock"
-	mockery --name 'DataProviderFactory' --dir="./engine/access/rest/websockets/data_providers" --case=underscore --output="./engine/access/rest/websockets/data_providers/mock"  --outpkg="mock"
-	mockery --name 'LinkGenerator' --dir="./engine/access/rest/common/models" --case=underscore --output="./engine/access/rest/common/models/mock"  --outpkg="mock"
-	mockery --name 'WebsocketConnection' --dir="./engine/access/rest/websockets" --case=underscore --output="./engine/access/rest/websockets/mock"  --outpkg="mock"
-	mockery --name 'ConnectionFactory' --dir="./engine/access/rpc/connection" --case=underscore --output="./engine/access/rpc/connection/mock" --outpkg="mock"
-	mockery --name 'Communicator' --dir="./engine/access/rpc/backend/node_communicator" --case=underscore --output="./engine/access/rpc/backend/node_communicator/mock" --outpkg="mock"
-	mockery --name 'AccountProvider' --dir="./engine/access/rpc/backend/accounts/provider" --case=underscore --output="./engine/access/rpc/backend/accounts/provider/mock" --outpkg="mock"
-	mockery --name 'EventProvider' --dir="./engine/access/rpc/backend/events/provider" --case=underscore --output="./engine/access/rpc/backend/events/provider/mock" --outpkg="mock"
-	mockery --name 'TransactionProvider' --dir="./engine/access/rpc/backend/transactions/provider" --case=underscore --output="./engine/access/rpc/backend/transactions/provider/mock" --outpkg="mock"
-	mockery --name 'Provider' --dir="./engine/access/rpc/backend/transactions/error_messages" --case=underscore --output="./engine/access/rpc/backend/transactions/error_messages/mock" --outpkg="mock"
-	mockery --name 'TransactionSender' --dir="./engine/access/rpc/backend/transactions/retrier" --case=underscore --output="./engine/access/rpc/backend/transactions/retrier/mock" --outpkg="mock"
-	mockery --name 'Retrier' --dir="./engine/access/rpc/backend/transactions/retrier" --case=underscore --output="./engine/access/rpc/backend/transactions/retrier/mock" --outpkg="mock"
+	mockery --name 'DataProvider' --dir="./engine/access/api/rest/websockets/data_providers" --case=underscore --output="./engine/access/api/rest/websockets/data_providers/mock"  --outpkg="mock"
+	mockery --name 'DataProviderFactory' --dir="./engine/access/api/rest/websockets/data_providers" --case=underscore --output="./engine/access/api/rest/websockets/data_providers/mock"  --outpkg="mock"
+	mockery --name 'LinkGenerator' --dir="./engine/access/api/rest/common/models" --case=underscore --output="./engine/access/api/rest/common/models/mock"  --outpkg="mock"
+	mockery --name 'WebsocketConnection' --dir="./engine/access/api/rest/websockets" --case=underscore --output="./engine/access/api/rest/websockets/mock"  --outpkg="mock"
+	mockery --name 'ConnectionFactory' --dir="./engine/access/api/rpc/connection" --case=underscore --output="./engine/access/api/rpc/connection/mock" --outpkg="mock"
+	mockery --name 'Communicator' --dir="./engine/access/api/rpc/backend/node_communicator" --case=underscore --output="./engine/access/api/rpc/backend/node_communicator/mock" --outpkg="mock"
+	mockery --name 'AccountProvider' --dir="./engine/access/api/rpc/backend/accounts/provider" --case=underscore --output="./engine/access/api/rpc/backend/accounts/provider/mock" --outpkg="mock"
+	mockery --name 'EventProvider' --dir="./engine/access/api/rpc/backend/events/provider" --case=underscore --output="./engine/access/api/rpc/backend/events/provider/mock" --outpkg="mock"
+	mockery --name 'TransactionProvider' --dir="./engine/access/api/rpc/backend/transactions/provider" --case=underscore --output="./engine/access/api/rpc/backend/transactions/provider/mock" --outpkg="mock"
+	mockery --name 'Provider' --dir="./engine/access/api/rpc/backend/transactions/error_messages" --case=underscore --output="./engine/access/api/rpc/backend/transactions/error_messages/mock" --outpkg="mock"
+	mockery --name 'TransactionSender' --dir="./engine/access/api/rpc/backend/transactions/retrier" --case=underscore --output="./engine/access/api/rpc/backend/transactions/retrier/mock" --outpkg="mock"
+	mockery --name 'Retrier' --dir="./engine/access/api/rpc/backend/transactions/retrier" --case=underscore --output="./engine/access/api/rpc/backend/transactions/retrier/mock" --outpkg="mock"
 	mockery --name '.*' --dir=model/fingerprint --case=underscore --output="./model/fingerprint/mock" --outpkg="mock"
 	mockery --name 'ExecForkActor' --structname 'ExecForkActorMock' --dir=module/mempool/consensus/mock/ --case=underscore --output="./module/mempool/consensus/mock/" --outpkg="mock"
 	mockery --name '.*' --dir=engine/verification/fetcher/ --case=underscore --output="./engine/verification/fetcher/mock" --outpkg="mockfetcher"
@@ -222,6 +222,9 @@ generate-mocks: install-mock-generators
 	mockery --name 'ScriptExecutor' --dir=module/execution --case=underscore --output="module/execution/mock" --outpkg="mock"
 	mockery --name 'StorageSnapshot' --dir=fvm/storage/snapshot --case=underscore --output="fvm/storage/snapshot/mock" --outpkg="mock"
 	mockery --name 'Core' --dir=module/executiondatasync/optimistic_sync --case=underscore --output="module/executiondatasync/optimistic_sync/mock" --outpkg="mock"
+	mockery --name 'ExecutionResultInfoProvider' --dir=module/executiondatasync/optimistic_sync --case=underscore --output="module/executiondatasync/optimistic_sync/mock" --outpkg="mock"
+	mockery --name 'ExecutionStateCache' --dir=module/executiondatasync/optimistic_sync --case=underscore --output="module/executiondatasync/optimistic_sync/mock" --outpkg="mock"
+	mockery --name 'Snapshot' --dir=module/executiondatasync/optimistic_sync --case=underscore --output="module/executiondatasync/optimistic_sync/mock" --outpkg="mock"
 	mockery --name 'Requester' --dir=engine/access/ingestion/tx_error_messages --case=underscore --output="engine/access/ingestion/tx_error_messages/mock" --outpkg="mock"
 	mockery --name 'ExecutionDataRequester' --dir=module/state_synchronization/requester --case=underscore --output="module/state_synchronization/requester/mock" --outpkg="mock"
 

access/api.go

@@ -8,6 +8,7 @@ import (
 	"github.com/onflow/flow-go/engine/access/subscription"
 	accessmodel "github.com/onflow/flow-go/model/access"
 	"github.com/onflow/flow-go/model/flow"
+	"github.com/onflow/flow-go/module/executiondatasync/optimistic_sync"
 )
 
 type AccountsAPI interface {
@@ -28,17 +29,19 @@ type EventsAPI interface {
 	GetEventsForHeightRange(
 		ctx context.Context,
 		eventType string,
-		startHeight,
+		startHeight uint64,
 		endHeight uint64,
-		requiredEventEncodingVersion entities.EventEncodingVersion,
-	) ([]flow.BlockEvents, error)
+		requiredEventEncodingVersion entities.EventEncodingVersion, // TODO: we should depend on access models instead of rpc models
+		criteria optimistic_sync.Criteria,
+	) ([]flow.BlockEvents, accessmodel.ExecutorMetadata, error)
 
 	GetEventsForBlockIDs(
 		ctx context.Context,
 		eventType string,
 		blockIDs []flow.Identifier,
-		requiredEventEncodingVersion entities.EventEncodingVersion,
-	) ([]flow.BlockEvents, error)
+		requiredEventEncodingVersion entities.EventEncodingVersion, // TODO: we should depend on access models instead of rpc models
+		criteria optimistic_sync.Criteria,
+	) ([]flow.BlockEvents, accessmodel.ExecutorMetadata, error)
 }
 
 type ScriptsAPI interface {
@@ -53,6 +56,7 @@ type TransactionsAPI interface {
 	GetTransaction(ctx context.Context, id flow.Identifier) (*flow.TransactionBody, error)
 	GetTransactionsByBlockID(ctx context.Context, blockID flow.Identifier) ([]*flow.TransactionBody, error)
 
+	// TODO: we should depend on access models instead of rpc models
 	GetTransactionResult(ctx context.Context, txID flow.Identifier, blockID flow.Identifier, collectionID flow.Identifier, encodingVersion entities.EventEncodingVersion) (*accessmodel.TransactionResult, error)
 	GetTransactionResultByIndex(ctx context.Context, blockID flow.Identifier, index uint32, encodingVersion entities.EventEncodingVersion) (*accessmodel.TransactionResult, error)
 	GetTransactionResultsByBlockID(ctx context.Context, blockID flow.Identifier, encodingVersion entities.EventEncodingVersion) ([]*accessmodel.TransactionResult, error)
@@ -72,7 +76,7 @@ type TransactionStreamAPI interface {
 	SubscribeTransactionStatuses(
 		ctx context.Context,
 		txID flow.Identifier,
-		requiredEventEncodingVersion entities.EventEncodingVersion,
+		requiredEventEncodingVersion entities.EventEncodingVersion, // TODO: we should depend on access models instead of rpc models
 	) subscription.Subscription
 
 	// SendAndSubscribeTransactionStatuses sends a transaction to the execution node and subscribes to its status updates.

access/legacy/handler.go

@@ -12,6 +12,7 @@ import (
 	"github.com/onflow/flow-go/access"
 	"github.com/onflow/flow-go/access/legacy/convert"
 	"github.com/onflow/flow-go/model/flow"
+	"github.com/onflow/flow-go/module/executiondatasync/optimistic_sync"
 )
 
 type Handler struct {
@@ -320,7 +321,7 @@ func (h *Handler) GetEventsForHeightRange(
 	startHeight := req.GetStartHeight()
 	endHeight := req.GetEndHeight()
 
-	results, err := h.api.GetEventsForHeightRange(ctx, eventType, startHeight, endHeight, entities.EventEncodingVersion_JSON_CDC_V0)
+	results, _, err := h.api.GetEventsForHeightRange(ctx, eventType, startHeight, endHeight, entities.EventEncodingVersion_JSON_CDC_V0, optimistic_sync.Criteria{})
 	if err != nil {
 		return nil, err
 	}
@@ -338,7 +339,7 @@ func (h *Handler) GetEventsForBlockIDs(
 	eventType := req.GetType()
 	blockIDs := convert.MessagesToIdentifiers(req.GetBlockIds())
 
-	results, err := h.api.GetEventsForBlockIDs(ctx, eventType, blockIDs, entities.EventEncodingVersion_JSON_CDC_V0)
+	results, _, err := h.api.GetEventsForBlockIDs(ctx, eventType, blockIDs, entities.EventEncodingVersion_JSON_CDC_V0, optimistic_sync.Criteria{})
 	if err != nil {
 		return nil, err
 	}

admin/commands/storage/backfill_tx_error_messages_test.go

@@ -14,13 +14,13 @@ import (
 
 	"github.com/onflow/flow-go/admin"
 	"github.com/onflow/flow-go/admin/commands"
+	"github.com/onflow/flow-go/engine/access/api/rpc/backend"
+	"github.com/onflow/flow-go/engine/access/api/rpc/backend/node_communicator"
+	"github.com/onflow/flow-go/engine/access/api/rpc/backend/transactions/error_messages"
+	connectionmock "github.com/onflow/flow-go/engine/access/api/rpc/connection/mock"
 	"github.com/onflow/flow-go/engine/access/index"
 	"github.com/onflow/flow-go/engine/access/ingestion/tx_error_messages"
 	accessmock "github.com/onflow/flow-go/engine/access/mock"
-	"github.com/onflow/flow-go/engine/access/rpc/backend"
-	"github.com/onflow/flow-go/engine/access/rpc/backend/node_communicator"
-	"github.com/onflow/flow-go/engine/access/rpc/backend/transactions/error_messages"
-	connectionmock "github.com/onflow/flow-go/engine/access/rpc/connection/mock"
 	commonrpc "github.com/onflow/flow-go/engine/common/rpc"
 	"github.com/onflow/flow-go/model/flow"
 	syncmock "github.com/onflow/flow-go/module/state_synchronization/mock"

cmd/access/README.md

@@ -71,7 +71,7 @@ As the collections arrive, it persists the collections and the transactions with
 
 The `requester` engine requests collections from the collection nodes on behalf of the `ingestion` engine.
 
-### [RPC](../../engine/access/rpc)
+### [RPC](../../engine/access/api/rpc)
 
 The `rpc` engine is the GRPC server which responds to the [Access API](https://docs.onflow.org/access-api/) requests from clients.
 It also supports GRPCWebproxy requests.