Skip to content

IIC-559 Add support for 16 VFs #461

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 2 commits into from
Sep 24, 2025
Merged

IIC-559 Add support for 16 VFs #461

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
67bbe27
IIC-559 Add support for 16 VFs
saynb Aug 13, 2025
88370e9
Adding VSI devmem cmds back
saynb Sep 23, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
175 changes: 100 additions & 75 deletions ipu-plugin/pkg/ipuplugin/bridgeport.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@ package ipuplugin
import (
"context"
"fmt"
"strconv"
"sync"

"github.com/intel/ipu-opi-plugins/ipu-plugin/pkg/p4rtclient"
"github.com/intel/ipu-opi-plugins/ipu-plugin/pkg/types"
Expand All @@ -28,65 +28,98 @@ import (
"google.golang.org/protobuf/types/known/emptypb"
)

const (
outerVlanId = 0 // hardcoded s-tag
)

/*
interfaces slice will be populated with PortIds on ACC(port representators->PRs on ACC, for Host VFs),
for example, if ACC interface name is enp0s1f0d4, PortId(vportId) will be 4.
Will also include port representators->PRs needed for Network Functions.
interfaces = {HOST_VF_START_ID(22) to HOST_VF_END_ID(37), NF_PR_START_ID(6) to NF_PR_END_ID(13)}
intfMap is a map, that has key:value, between interfaceId and whether it is available(true or false) for use.
*/
var interfaces []uint
var intfMap map[uint]bool
var intfMapInit bool = false

// Note: 3 reserved(last digit of interface name, for example, enp0s1f0d8, is 8) in exlude list in deviceplugin.
var interfaces [3]string = [3]string{"6", "7", "8"}
var intfMap map[string]bool
// through bridgeport and network function service APIs(grpc),
// resource allocation apis can get invoked concurrently.
// serialize access to resources-> intfMap and interfaces
var ResourceMutex sync.Mutex

func initMap() error {
if intfMapInit == false {
intfMap = make(map[string]bool)
var index uint
if !intfMapInit {
for index = HOST_VF_START_ID; index <= HOST_VF_END_ID; index = index + 1 {
interfaces = append(interfaces, index)
}
for index = NF_PR_START_ID; index <= NF_PR_END_ID; index = index + 1 {
interfaces = append(interfaces, index)
}
intfMap = make(map[uint]bool)
for _, intf := range interfaces {
intfMap[intf] = false
}
if len(interfaces) != len(intfMap) {
log.Errorf("initMap setup error\n")
return fmt.Errorf("initMap setup error\n")
log.Errorf("initMap setup error")
return fmt.Errorf("initMap setup error")
}
intfMapInit = true
}
return nil
}

// in-order(sorted by interface name->interfaces) allocation, based on available ACC interfaces(for Host VF)
func allocateAccInterface() (error, string) {
var intfName string = ""
log.Debugf("allocateAccInterface\n")
if intfMapInit == false {
// in-order(sorted by interface IDs) allocation. Based on available ACC interfaces(for Host VF
// and NF PRs). Currently there are 2 ranges, first range(sorted) is for available Host-VF interface IDs
// (HOST_VF_START_ID to HOST_VF_END_ID) and second range(sorted) for NF PRs(NF_PR_START_ID to NF_PR_END_ID)
func AllocateAccInterface(allocPr string) (uint, error) {
var intfId uint = 0
start, end := 0, 0

ResourceMutex.Lock()
defer ResourceMutex.Unlock()

found := false
log.Debugf("AllocateAccInterface\n")
if !intfMapInit {
initMap()
}
for _, key := range interfaces {
log.Debugf("intfName->%v\n", key)
if allocPr == types.HostVfPr {
start = 0
end = HOST_VF_END_ID - HOST_VF_START_ID
} else {
start = HOST_VF_END_ID - HOST_VF_START_ID + 1
end = start + NF_PR_END_ID - NF_PR_START_ID
}
for i := start; i <= end; i++ {
key := interfaces[i]
value, present := intfMap[key]
if present == true && value == false {
if present && !value {
log.Debugf("Found avail Intf->%v: \n", key)
intfMap[key] = true
intfName = key
intfId = key
found = true
break
}
}
if intfName != "" {
return nil, intfName
if found {
return intfId, nil
}
log.Errorf("Interface not available\n")
return fmt.Errorf("Interface not available\n"), intfName
log.Errorf("AllocateAccInterface: Interface not available")
return intfId, fmt.Errorf("AllocateAccInterface: interface not available")
}

func freeAccInterface(intfName string) error {
log.Debugf("freeAccInterface\n")
value, present := intfMap[intfName]
if present == true && value == true {
log.Debugf("Found allocated Intf->%v: \n", intfName)
intfMap[intfName] = false
func FreeAccInterface(intfId uint) error {
log.Debugf("FreeAccInterface\n")

ResourceMutex.Lock()
defer ResourceMutex.Unlock()

value, present := intfMap[intfId]
if present && value {
log.Debugf("Found allocated Intf->%v: \n", intfId)
intfMap[intfId] = false
return nil
}
log.Errorf("Interface->%s not found in freeAccInterface\n", intfName)
return fmt.Errorf("Interface->%s not found in freeAccInterface\n", intfName)
log.Errorf("Interface->%v not found in FreeAccInterface", intfId)
return fmt.Errorf("interface->%v not found in FreeAccInterface", intfId)
}

// CreateBridgePort executes the creation of the port
Expand All @@ -109,12 +142,6 @@ func (s *server) CreateBridgePort(_ context.Context, in *pb.CreateBridgePortRequ
if in.BridgePort.Spec.LogicalBridges == nil || len(in.BridgePort.Spec.LogicalBridges) < 1 {
return nil, fmt.Errorf("vlan id is not provided")
}
vlan := s.getFirstVlanID(in.BridgePort.Spec.LogicalBridges)

if vlan < 2 || vlan > 4094 {
s.log.WithField("vlan", vlan).Debug("invalid vlan")
return nil, fmt.Errorf("invalid vlan %d, vlan must be within 2-4094 range", vlan)
}

if vfVsi < 1 {
s.log.WithField("vfVsi", vfVsi).Debug("invalid VSI")
Expand All @@ -127,32 +154,33 @@ func (s *server) CreateBridgePort(_ context.Context, in *pb.CreateBridgePortRequ

CheckAndAddPeerToPeerP4Rules(s.p4rtClient)

err, intfName := allocateAccInterface()
intfId, err := AllocateAccInterface(types.HostVfPr)
if err != nil {
return nil, fmt.Errorf("error from allocateAccInterface->%v", err)
return nil, fmt.Errorf("error from AllocateAccInterface->%v", err)
}

intIndex, err := strconv.Atoi(string(intfName))
if err != nil {
log.Errorf("error->%v converting, intfName->%v", err, intfName)
return nil, fmt.Errorf("error->%v converting, intfName->%v", err, intfName)
} else {
log.Infof("intIndex->%v, fullIntfName->%v", intIndex, AccIntfNames[intIndex])
}
log.Infof("intfId->%v, fullIntfName->%v", intfId, AccApfInfo[intfId].Name)

if err := s.bridgeCtlr.AddPort(AccIntfNames[intIndex]); err != nil {
log.Errorf("failed to add port to bridge: %v, for interface->%v", err, AccIntfNames[intIndex])
freeAccInterface(intfName)
return nil, fmt.Errorf("failed to add port to bridge: %v, for interface->%v", err, AccIntfNames[intIndex])
if err := s.bridgeCtlr.AddPort(AccApfInfo[intfId].Name); err != nil {
log.Errorf("failed to add port to bridge: %v, for interface->%v", err, AccApfInfo[intfId].Name)
FreeAccInterface(intfId)
return nil, fmt.Errorf("failed to add port to bridge: %v, for interface->%v", err, AccApfInfo[intfId].Name)
}

// Add FXP rules
log.Infof("AddHostVfP4Rules, path->%s, 1->%v, 2->%v", s.p4rtClient.GetBin(), in.BridgePort.Spec.MacAddress, AccApfMacList[intIndex])
p4rtclient.AddHostVfP4Rules(s.p4rtClient, in.BridgePort.Spec.MacAddress, AccApfMacList[intIndex])
log.Infof("AddHostVfP4Rules, path->%s, 1->%v, 2->%v", s.p4rtClient.GetBin(), in.BridgePort.Spec.MacAddress, AccApfInfo[intfId].Mac)
err = p4rtclient.AddHostVfP4Rules(s.p4rtClient, in.BridgePort.Spec.MacAddress, AccApfInfo[intfId].Mac)

if err != nil {
log.Errorf("CBP: err-> %v, from AddHostVfP4Rules", err)
DeletePortWrapper(s.bridgeCtlr, intfId)
FreeAccInterface(intfId)
return nil, fmt.Errorf("CBP: err-> %v, from AddHostVfP4Rules", err)
}

resp := proto.Clone(in.BridgePort).(*pb.BridgePort)
resp.Status = &pb.BridgePortStatus{OperStatus: pb.BPOperStatus_BP_OPER_STATUS_UP}
pbBridgePortInfo := &types.BridgePortInfo{PbBrPort: resp, PortInterface: intfName}
pbBridgePortInfo := &types.BridgePortInfo{PbBrPort: resp, PortId: intfId}
s.Ports[in.BridgePort.Name] = pbBridgePortInfo
return resp, nil
}
Expand All @@ -178,26 +206,24 @@ func (s *server) DeleteBridgePort(_ context.Context, in *pb.DeleteBridgePortRequ
// in such case avoid delete call loop from CNI Agent which otherwise will repeatedly call DeleteBridgePort as retry
return &emptypb.Empty{}, nil
}

FreeAccInterface(brPortInfo.PortId)
delete(s.Ports, in.Name)

portInfo = brPortInfo.PbBrPort

intIndex, err := strconv.Atoi(string(brPortInfo.PortInterface))
if err != nil {
log.Errorf("error->%v converting, intfName->%v", err, brPortInfo.PortInterface)
return nil, fmt.Errorf("error->%v converting, intfName->%v", err, brPortInfo.PortInterface)
} else {
log.Infof("intIndex->%v, fullIntfName->%v", intIndex, AccIntfNames[intIndex])
}
intfId := brPortInfo.PortId
log.Infof("intfIndex->%v, fullIntfName->%v", intfId, AccApfInfo[intfId].Name)

if err := s.bridgeCtlr.DeletePort(AccIntfNames[intIndex]); err != nil {
log.Errorf("unable to delete port from bridge: %v, for interface->%v", err, AccIntfNames[intIndex])
return nil, fmt.Errorf("unable to delete port from bridge: %v, for interface->%v", err, AccIntfNames[intIndex])
if err := s.bridgeCtlr.DeletePort(AccApfInfo[intfId].Name); err != nil {
log.Errorf("unable to delete port from bridge: %v, for interface->%v", err, AccApfInfo[intfId].Name)
return nil, fmt.Errorf("unable to delete port from bridge: %v, for interface->%v", err, AccApfInfo[intfId].Name)
}
freeAccInterface(brPortInfo.PortInterface)

// Delete FXP rules
log.Infof("DeleteHostVfP4Rules, path->%s, 1->%v, 2->%v", s.p4rtClient.GetBin(), portInfo.Spec.MacAddress, AccApfMacList[intIndex])
p4rtclient.DeleteHostVfP4Rules(s.p4rtClient, portInfo.Spec.MacAddress, AccApfMacList[intIndex])
log.Infof("DeleteHostVfP4Rules, path->%s, 1->%v, 2->%v", s.p4rtClient.GetBin(), portInfo.Spec.MacAddress, AccApfInfo[intfId].Mac)
p4rtclient.DeleteHostVfP4Rules(s.p4rtClient, portInfo.Spec.MacAddress, AccApfInfo[intfId].Mac)

delete(s.Ports, in.Name)
return &emptypb.Empty{}, nil
}

Expand All @@ -219,11 +245,10 @@ func (s *server) ListBridgePorts(_ context.Context, in *pb.ListBridgePortsReques
return &pb.ListBridgePortsResponse{}, nil
}

func (s *server) getFirstVlanID(bridges []string) int {
vlanId, err := strconv.Atoi(bridges[0])
if err != nil {
s.log.Errorf("unable to parse vlan ID %s. conversion error %s", bridges[0], err)
return 0
func DeletePortWrapper(bridgeCtlr types.BridgeController, intfId uint) error {
if err := bridgeCtlr.DeletePort(AccApfInfo[intfId].Name); err != nil {
log.Errorf("deletePortWrapper:failed to delete port to bridge: %v, for interface->%v", err, AccApfInfo[intfId].Name)
return err
}
return vlanId
return nil
}
19 changes: 2 additions & 17 deletions ipu-plugin/pkg/ipuplugin/bridgeport_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -65,21 +65,6 @@ var _ = Describe("bridgeport", Serial, func() {
Expect(err).To(HaveOccurred())
})
})
Context("when no vlan id is not in valid vlan range", func() {
It("should return error", func() {
fakeMacAddr := []byte{0xaa, 0xbb, 0xcc, 0xdd, 0xee, 0xff}
fakeReq := &pb.CreateBridgePortRequest{
BridgePort: &pb.BridgePort{
Spec: &pb.BridgePortSpec{
MacAddress: fakeMacAddr,
LogicalBridges: []string{"0"},
},
},
}
_, err := ipuServer.CreateBridgePort(context.TODO(), fakeReq)
Expect(err).To(HaveOccurred())
})
})
Context("when no VF VSI is 0", func() {
It("should return error", func() {
fakeMacAddr := []byte{0xaa, 0x00, 0xcc, 0xdd, 0xee, 0xff} // the second octet here is the VSI number
Expand Down Expand Up @@ -107,7 +92,7 @@ var _ = Describe("bridgeport", Serial, func() {
},
}
fakePortBridgeInfo := &types.BridgePortInfo{
fakePort, "fakePort1",
fakePort, 10,
}

fakeReq := &pb.CreateBridgePortRequest{BridgePort: fakePort}
Expand All @@ -131,7 +116,7 @@ var _ = Describe("bridgeport", Serial, func() {
},
}
fakePortBridgeInfo := &types.BridgePortInfo{
fakePort, "fakePort1",
fakePort, 10,
}
ipuServer.Ports["fakePort"] = fakePortBridgeInfo // fakePort already exists in internal Map

Expand Down
14 changes: 12 additions & 2 deletions ipu-plugin/pkg/ipuplugin/deviceplugin.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,8 @@ type DevicePluginService struct {
var (
//TODO: Use (GetFilteredPfs), to find interface names to be excluded.
//excluding d3(host-acc), reserving D4,D1(QSPF ports) D6-D8(for max 3 host VFs), D9-D10(for single NF)
exclude = []string{"enp0s1f0", "enp0s1f0d1", "enp0s1f0d2", "enp0s1f0d3", "enp0s1f0d4", "enp0s1f0d5", "enp0s1f0d6",
exclude = []string{"enp0s1f0", "enp0s1f0d1", "enp0s1f0d2", "enp0s1f0d3",
"enp0s1f0d4", "enp0s1f0d5", "enp0s1f0d6",
"enp0s1f0d7", "enp0s1f0d8", "enp0s1f0d9", "enp0s1f0d10"}
sysClassNet = "/sys/class/net"
sysBusPciDevices = "/sys/bus/pci/devices"
Expand Down Expand Up @@ -266,6 +267,15 @@ func discoverHostDevices(mode string) (map[string]*pb.Device, error) {
if mode != types.IpuMode && mode != types.HostMode {
return make(map[string]*pb.Device), fmt.Errorf("Invalid mode->%v", mode)
}
//Note: It is expected that VSP's-Init(on ACC) gets invoked prior to GetDevices,
//this check is meant to catch any anomalies.
if mode == types.IpuMode {
if len(AccApfsAvailForCNI) == 0 {
log.Errorf("discoverHostDevices: Error, AccApfsAvailForCNI not setup")
return make(map[string]*pb.Device), fmt.Errorf("discoverHostDevices: Error, AccApfsAvailForCNI not setup")
}
}

if mode == types.IpuMode {
if accDevs == nil {
accDevs = make(map[string]*pb.Device)
Expand Down Expand Up @@ -296,7 +306,7 @@ func discoverHostDevices(mode string) (map[string]*pb.Device, error) {
device_code := strings.TrimSpace(string(deviceCodeByte))
if mode == types.IpuMode {
if device_code == deviceCode {
if !slices.Contains(exclude, file.Name()) {
if slices.Contains(AccApfsAvailForCNI, file.Name()) {
accDevs[file.Name()] = &pb.Device{ID: file.Name(), Health: pluginapi.Healthy}
}
}
Expand Down
8 changes: 4 additions & 4 deletions ipu-plugin/pkg/ipuplugin/ipuplugin.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -234,8 +234,8 @@ func (s *server) Run() error {
}

func cleanUpRulesOnExit(p4rtClient types.P4RTClient) error {
log.Infof("DeletePhyPortRules, path->%s, 1->%v, 2->%v", p4rtClient.GetBin(), AccApfMacList[PHY_PORT0_SECONDARY_INTF_INDEX], AccApfMacList[PHY_PORT0_PRIMARY_INTF_INDEX])
p4rtclient.DeletePhyPortRules(p4rtClient, AccApfMacList[PHY_PORT0_SECONDARY_INTF_INDEX], AccApfMacList[PHY_PORT0_PRIMARY_INTF_INDEX])
log.Infof("DeletePhyPortRules, path->%s, 1->%v, 2->%v", p4rtClient.GetBin(), AccApfInfo[PHY_PORT0_SECONDARY_INTF_INDEX].Mac, AccApfInfo[PHY_PORT0_PRIMARY_INTF_INDEX].Mac)
p4rtclient.DeletePhyPortRules(p4rtClient, AccApfInfo[PHY_PORT0_SECONDARY_INTF_INDEX].Mac, AccApfInfo[PHY_PORT0_PRIMARY_INTF_INDEX].Mac)

vfMacList, err := utils.GetVfMacList()
if err != nil {
Expand All @@ -252,8 +252,8 @@ func cleanUpRulesOnExit(p4rtClient types.P4RTClient) error {
log.Infof("DeleteLAGP4Rules, path->%s", p4rtClient.GetBin())
p4rtclient.DeleteLAGP4Rules(p4rtClient)

log.Infof("DeleteRHPrimaryNetworkVportP4Rules, path->%s, 1->%v", p4rtClient, AccApfMacList[PHY_PORT0_PRIMARY_INTF_INDEX])
p4rtclient.DeleteRHPrimaryNetworkVportP4Rules(p4rtClient, AccApfMacList[PHY_PORT0_PRIMARY_INTF_INDEX])
log.Infof("DeleteRHPrimaryNetworkVportP4Rules, path->%s, 1->%v", p4rtClient, AccApfInfo[PHY_PORT0_PRIMARY_INTF_INDEX].Mac)
p4rtclient.DeleteRHPrimaryNetworkVportP4Rules(p4rtClient, AccApfInfo[PHY_PORT0_PRIMARY_INTF_INDEX].Mac)
return nil
}

Expand Down
Loading
Loading