Skip to content

Commit 0d4faeb

Browse files
saynbsaynb
committed
IIC-559 Add support for 16 VFs
Adding space to the grep command of cli_client
1 parent c02e7e5 commit 0d4faeb

File tree

9 files changed

+390
-166
lines changed

9 files changed

+390
-166
lines changed

‎ipu-plugin/pkg/ipuplugin/bridgeport.go‎

Lines changed: 100 additions & 75 deletions
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,7 @@ package ipuplugin
1717
import (
1818
"context"
1919
"fmt"
20-
"strconv"
20+
"sync"
2121

2222
"github.com/intel/ipu-opi-plugins/ipu-plugin/pkg/p4rtclient"
2323
"github.com/intel/ipu-opi-plugins/ipu-plugin/pkg/types"
@@ -28,65 +28,98 @@ import (
2828
"google.golang.org/protobuf/types/known/emptypb"
2929
)
3030

31-
const (
32-
outerVlanId = 0 // hardcoded s-tag
33-
)
34-
31+
/*
32+
interfaces slice will be populated with PortIds on ACC(port representators->PRs on ACC, for Host VFs),
33+
for example, if ACC interface name is enp0s1f0d4, PortId(vportId) will be 4.
34+
Will also include port representators->PRs needed for Network Functions.
35+
interfaces = {HOST_VF_START_ID(22) to HOST_VF_END_ID(37), NF_PR_START_ID(6) to NF_PR_END_ID(13)}
36+
intfMap is a map, that has key:value, between interfaceId and whether it is available(true or false) for use.
37+
*/
38+
var interfaces []uint
39+
var intfMap map[uint]bool
3540
var intfMapInit bool = false
3641

37-
// Note: 3 reserved(last digit of interface name, for example, enp0s1f0d8, is 8) in exlude list in deviceplugin.
38-
var interfaces [3]string = [3]string{"6", "7", "8"}
39-
var intfMap map[string]bool
42+
// through bridgeport and network function service APIs(grpc),
43+
// resource allocation apis can get invoked concurrently.
44+
// serialize access to resources-> intfMap and interfaces
45+
var ResourceMutex sync.Mutex
4046

4147
func initMap() error {
42-
if intfMapInit == false {
43-
intfMap = make(map[string]bool)
48+
var index uint
49+
if !intfMapInit {
50+
for index = HOST_VF_START_ID; index <= HOST_VF_END_ID; index = index + 1 {
51+
interfaces = append(interfaces, index)
52+
}
53+
for index = NF_PR_START_ID; index <= NF_PR_END_ID; index = index + 1 {
54+
interfaces = append(interfaces, index)
55+
}
56+
intfMap = make(map[uint]bool)
4457
for _, intf := range interfaces {
4558
intfMap[intf] = false
4659
}
4760
if len(interfaces) != len(intfMap) {
48-
log.Errorf("initMap setup error\n")
49-
return fmt.Errorf("initMap setup error\n")
61+
log.Errorf("initMap setup error")
62+
return fmt.Errorf("initMap setup error")
5063
}
5164
intfMapInit = true
5265
}
5366
return nil
5467
}
5568

56-
// in-order(sorted by interface name->interfaces) allocation, based on available ACC interfaces(for Host VF)
57-
func allocateAccInterface() (error, string) {
58-
var intfName string = ""
59-
log.Debugf("allocateAccInterface\n")
60-
if intfMapInit == false {
69+
// in-order(sorted by interface IDs) allocation. Based on available ACC interfaces(for Host VF
70+
// and NF PRs). Currently there are 2 ranges, first range(sorted) is for available Host-VF interface IDs
71+
// (HOST_VF_START_ID to HOST_VF_END_ID) and second range(sorted) for NF PRs(NF_PR_START_ID to NF_PR_END_ID)
72+
func AllocateAccInterface(allocPr string) (uint, error) {
73+
var intfId uint = 0
74+
start, end := 0, 0
75+
76+
ResourceMutex.Lock()
77+
defer ResourceMutex.Unlock()
78+
79+
found := false
80+
log.Debugf("AllocateAccInterface\n")
81+
if !intfMapInit {
6182
initMap()
6283
}
63-
for _, key := range interfaces {
64-
log.Debugf("intfName->%v\n", key)
84+
if allocPr == types.HostVfPr {
85+
start = 0
86+
end = HOST_VF_END_ID - HOST_VF_START_ID
87+
} else {
88+
start = HOST_VF_END_ID - HOST_VF_START_ID + 1
89+
end = start + NF_PR_END_ID - NF_PR_START_ID
90+
}
91+
for i := start; i <= end; i++ {
92+
key := interfaces[i]
6593
value, present := intfMap[key]
66-
if present == true && value == false {
94+
if present && !value {
6795
log.Debugf("Found avail Intf->%v: \n", key)
6896
intfMap[key] = true
69-
intfName = key
97+
intfId = key
98+
found = true
7099
break
71100
}
72101
}
73-
if intfName != "" {
74-
return nil, intfName
102+
if found {
103+
return intfId, nil
75104
}
76-
log.Errorf("Interface not available\n")
77-
return fmt.Errorf("Interface not available\n"), intfName
105+
log.Errorf("AllocateAccInterface: Interface not available")
106+
return intfId, fmt.Errorf("AllocateAccInterface: interface not available")
78107
}
79108

80-
func freeAccInterface(intfName string) error {
81-
log.Debugf("freeAccInterface\n")
82-
value, present := intfMap[intfName]
83-
if present == true && value == true {
84-
log.Debugf("Found allocated Intf->%v: \n", intfName)
85-
intfMap[intfName] = false
109+
func FreeAccInterface(intfId uint) error {
110+
log.Debugf("FreeAccInterface\n")
111+
112+
ResourceMutex.Lock()
113+
defer ResourceMutex.Unlock()
114+
115+
value, present := intfMap[intfId]
116+
if present && value {
117+
log.Debugf("Found allocated Intf->%v: \n", intfId)
118+
intfMap[intfId] = false
86119
return nil
87120
}
88-
log.Errorf("Interface->%s not found in freeAccInterface\n", intfName)
89-
return fmt.Errorf("Interface->%s not found in freeAccInterface\n", intfName)
121+
log.Errorf("Interface->%v not found in FreeAccInterface", intfId)
122+
return fmt.Errorf("interface->%v not found in FreeAccInterface", intfId)
90123
}
91124

92125
// CreateBridgePort executes the creation of the port
@@ -109,12 +142,6 @@ func (s *server) CreateBridgePort(_ context.Context, in *pb.CreateBridgePortRequ
109142
if in.BridgePort.Spec.LogicalBridges == nil || len(in.BridgePort.Spec.LogicalBridges) < 1 {
110143
return nil, fmt.Errorf("vlan id is not provided")
111144
}
112-
vlan := s.getFirstVlanID(in.BridgePort.Spec.LogicalBridges)
113-
114-
if vlan < 2 || vlan > 4094 {
115-
s.log.WithField("vlan", vlan).Debug("invalid vlan")
116-
return nil, fmt.Errorf("invalid vlan %d, vlan must be within 2-4094 range", vlan)
117-
}
118145

119146
if vfVsi < 1 {
120147
s.log.WithField("vfVsi", vfVsi).Debug("invalid VSI")
@@ -127,32 +154,33 @@ func (s *server) CreateBridgePort(_ context.Context, in *pb.CreateBridgePortRequ
127154

128155
CheckAndAddPeerToPeerP4Rules(s.p4rtClient)
129156

130-
err, intfName := allocateAccInterface()
157+
intfId, err := AllocateAccInterface(types.HostVfPr)
131158
if err != nil {
132-
return nil, fmt.Errorf("error from allocateAccInterface->%v", err)
159+
return nil, fmt.Errorf("error from AllocateAccInterface->%v", err)
133160
}
134161

135-
intIndex, err := strconv.Atoi(string(intfName))
136-
if err != nil {
137-
log.Errorf("error->%v converting, intfName->%v", err, intfName)
138-
return nil, fmt.Errorf("error->%v converting, intfName->%v", err, intfName)
139-
} else {
140-
log.Infof("intIndex->%v, fullIntfName->%v", intIndex, AccIntfNames[intIndex])
141-
}
162+
log.Infof("intfId->%v, fullIntfName->%v", intfId, AccApfInfo[intfId].Name)
142163

143-
if err := s.bridgeCtlr.AddPort(AccIntfNames[intIndex]); err != nil {
144-
log.Errorf("failed to add port to bridge: %v, for interface->%v", err, AccIntfNames[intIndex])
145-
freeAccInterface(intfName)
146-
return nil, fmt.Errorf("failed to add port to bridge: %v, for interface->%v", err, AccIntfNames[intIndex])
164+
if err := s.bridgeCtlr.AddPort(AccApfInfo[intfId].Name); err != nil {
165+
log.Errorf("failed to add port to bridge: %v, for interface->%v", err, AccApfInfo[intfId].Name)
166+
FreeAccInterface(intfId)
167+
return nil, fmt.Errorf("failed to add port to bridge: %v, for interface->%v", err, AccApfInfo[intfId].Name)
147168
}
148169

149170
// Add FXP rules
150-
log.Infof("AddHostVfP4Rules, path->%s, 1->%v, 2->%v", s.p4rtClient.GetBin(), in.BridgePort.Spec.MacAddress, AccApfMacList[intIndex])
151-
p4rtclient.AddHostVfP4Rules(s.p4rtClient, in.BridgePort.Spec.MacAddress, AccApfMacList[intIndex])
171+
log.Infof("AddHostVfP4Rules, path->%s, 1->%v, 2->%v", s.p4rtClient.GetBin(), in.BridgePort.Spec.MacAddress, AccApfInfo[intfId].Mac)
172+
err = p4rtclient.AddHostVfP4Rules(s.p4rtClient, in.BridgePort.Spec.MacAddress, AccApfInfo[intfId].Mac)
173+
174+
if err != nil {
175+
log.Errorf("CBP: err-> %v, from AddHostVfP4Rules", err)
176+
DeletePortWrapper(s.bridgeCtlr, intfId)
177+
FreeAccInterface(intfId)
178+
return nil, fmt.Errorf("CBP: err-> %v, from AddHostVfP4Rules", err)
179+
}
152180

153181
resp := proto.Clone(in.BridgePort).(*pb.BridgePort)
154182
resp.Status = &pb.BridgePortStatus{OperStatus: pb.BPOperStatus_BP_OPER_STATUS_UP}
155-
pbBridgePortInfo := &types.BridgePortInfo{PbBrPort: resp, PortInterface: intfName}
183+
pbBridgePortInfo := &types.BridgePortInfo{PbBrPort: resp, PortId: intfId}
156184
s.Ports[in.BridgePort.Name] = pbBridgePortInfo
157185
return resp, nil
158186
}
@@ -178,26 +206,24 @@ func (s *server) DeleteBridgePort(_ context.Context, in *pb.DeleteBridgePortRequ
178206
// in such case avoid delete call loop from CNI Agent which otherwise will repeatedly call DeleteBridgePort as retry
179207
return &emptypb.Empty{}, nil
180208
}
209+
210+
FreeAccInterface(brPortInfo.PortId)
211+
delete(s.Ports, in.Name)
212+
181213
portInfo = brPortInfo.PbBrPort
182214

183-
intIndex, err := strconv.Atoi(string(brPortInfo.PortInterface))
184-
if err != nil {
185-
log.Errorf("error->%v converting, intfName->%v", err, brPortInfo.PortInterface)
186-
return nil, fmt.Errorf("error->%v converting, intfName->%v", err, brPortInfo.PortInterface)
187-
} else {
188-
log.Infof("intIndex->%v, fullIntfName->%v", intIndex, AccIntfNames[intIndex])
189-
}
215+
intfId := brPortInfo.PortId
216+
log.Infof("intfIndex->%v, fullIntfName->%v", intfId, AccApfInfo[intfId].Name)
190217

191-
if err := s.bridgeCtlr.DeletePort(AccIntfNames[intIndex]); err != nil {
192-
log.Errorf("unable to delete port from bridge: %v, for interface->%v", err, AccIntfNames[intIndex])
193-
return nil, fmt.Errorf("unable to delete port from bridge: %v, for interface->%v", err, AccIntfNames[intIndex])
218+
if err := s.bridgeCtlr.DeletePort(AccApfInfo[intfId].Name); err != nil {
219+
log.Errorf("unable to delete port from bridge: %v, for interface->%v", err, AccApfInfo[intfId].Name)
220+
return nil, fmt.Errorf("unable to delete port from bridge: %v, for interface->%v", err, AccApfInfo[intfId].Name)
194221
}
195-
freeAccInterface(brPortInfo.PortInterface)
222+
196223
// Delete FXP rules
197-
log.Infof("DeleteHostVfP4Rules, path->%s, 1->%v, 2->%v", s.p4rtClient.GetBin(), portInfo.Spec.MacAddress, AccApfMacList[intIndex])
198-
p4rtclient.DeleteHostVfP4Rules(s.p4rtClient, portInfo.Spec.MacAddress, AccApfMacList[intIndex])
224+
log.Infof("DeleteHostVfP4Rules, path->%s, 1->%v, 2->%v", s.p4rtClient.GetBin(), portInfo.Spec.MacAddress, AccApfInfo[intfId].Mac)
225+
p4rtclient.DeleteHostVfP4Rules(s.p4rtClient, portInfo.Spec.MacAddress, AccApfInfo[intfId].Mac)
199226

200-
delete(s.Ports, in.Name)
201227
return &emptypb.Empty{}, nil
202228
}
203229

@@ -219,11 +245,10 @@ func (s *server) ListBridgePorts(_ context.Context, in *pb.ListBridgePortsReques
219245
return &pb.ListBridgePortsResponse{}, nil
220246
}
221247

222-
func (s *server) getFirstVlanID(bridges []string) int {
223-
vlanId, err := strconv.Atoi(bridges[0])
224-
if err != nil {
225-
s.log.Errorf("unable to parse vlan ID %s. conversion error %s", bridges[0], err)
226-
return 0
248+
func DeletePortWrapper(bridgeCtlr types.BridgeController, intfId uint) error {
249+
if err := bridgeCtlr.DeletePort(AccApfInfo[intfId].Name); err != nil {
250+
log.Errorf("deletePortWrapper:failed to delete port to bridge: %v, for interface->%v", err, AccApfInfo[intfId].Name)
251+
return err
227252
}
228-
return vlanId
253+
return nil
229254
}

‎ipu-plugin/pkg/ipuplugin/bridgeport_test.go‎

Lines changed: 2 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -65,21 +65,6 @@ var _ = Describe("bridgeport", Serial, func() {
6565
Expect(err).To(HaveOccurred())
6666
})
6767
})
68-
Context("when no vlan id is not in valid vlan range", func() {
69-
It("should return error", func() {
70-
fakeMacAddr := []byte{0xaa, 0xbb, 0xcc, 0xdd, 0xee, 0xff}
71-
fakeReq := &pb.CreateBridgePortRequest{
72-
BridgePort: &pb.BridgePort{
73-
Spec: &pb.BridgePortSpec{
74-
MacAddress: fakeMacAddr,
75-
LogicalBridges: []string{"0"},
76-
},
77-
},
78-
}
79-
_, err := ipuServer.CreateBridgePort(context.TODO(), fakeReq)
80-
Expect(err).To(HaveOccurred())
81-
})
82-
})
8368
Context("when no VF VSI is 0", func() {
8469
It("should return error", func() {
8570
fakeMacAddr := []byte{0xaa, 0x00, 0xcc, 0xdd, 0xee, 0xff} // the second octet here is the VSI number
@@ -107,7 +92,7 @@ var _ = Describe("bridgeport", Serial, func() {
10792
},
10893
}
10994
fakePortBridgeInfo := &types.BridgePortInfo{
110-
fakePort, "fakePort1",
95+
fakePort, 10,
11196
}
11297

11398
fakeReq := &pb.CreateBridgePortRequest{BridgePort: fakePort}
@@ -131,7 +116,7 @@ var _ = Describe("bridgeport", Serial, func() {
131116
},
132117
}
133118
fakePortBridgeInfo := &types.BridgePortInfo{
134-
fakePort, "fakePort1",
119+
fakePort, 10,
135120
}
136121
ipuServer.Ports["fakePort"] = fakePortBridgeInfo // fakePort already exists in internal Map
137122

‎ipu-plugin/pkg/ipuplugin/deviceplugin.go‎

Lines changed: 12 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -25,7 +25,8 @@ type DevicePluginService struct {
2525
var (
2626
//TODO: Use (GetFilteredPfs), to find interface names to be excluded.
2727
//excluding d3(host-acc), reserving D4,D1(QSPF ports) D6-D8(for max 3 host VFs), D9-D10(for single NF)
28-
exclude = []string{"enp0s1f0", "enp0s1f0d1", "enp0s1f0d2", "enp0s1f0d3", "enp0s1f0d4", "enp0s1f0d5", "enp0s1f0d6",
28+
exclude = []string{"enp0s1f0", "enp0s1f0d1", "enp0s1f0d2", "enp0s1f0d3",
29+
"enp0s1f0d4", "enp0s1f0d5", "enp0s1f0d6",
2930
"enp0s1f0d7", "enp0s1f0d8", "enp0s1f0d9", "enp0s1f0d10"}
3031
sysClassNet = "/sys/class/net"
3132
sysBusPciDevices = "/sys/bus/pci/devices"
@@ -266,6 +267,15 @@ func discoverHostDevices(mode string) (map[string]*pb.Device, error) {
266267
if mode != types.IpuMode && mode != types.HostMode {
267268
return make(map[string]*pb.Device), fmt.Errorf("Invalid mode->%v", mode)
268269
}
270+
//Note: It is expected that VSP's-Init(on ACC) gets invoked prior to GetDevices,
271+
//this check is meant to catch any anomalies.
272+
if mode == types.IpuMode {
273+
if len(AccApfsAvailForCNI) == 0 {
274+
log.Errorf("discoverHostDevices: Error, AccApfsAvailForCNI not setup")
275+
return make(map[string]*pb.Device), fmt.Errorf("discoverHostDevices: Error, AccApfsAvailForCNI not setup")
276+
}
277+
}
278+
269279
if mode == types.IpuMode {
270280
if accDevs == nil {
271281
accDevs = make(map[string]*pb.Device)
@@ -296,7 +306,7 @@ func discoverHostDevices(mode string) (map[string]*pb.Device, error) {
296306
device_code := strings.TrimSpace(string(deviceCodeByte))
297307
if mode == types.IpuMode {
298308
if device_code == deviceCode {
299-
if !slices.Contains(exclude, file.Name()) {
309+
if slices.Contains(AccApfsAvailForCNI, file.Name()) {
300310
accDevs[file.Name()] = &pb.Device{ID: file.Name(), Health: pluginapi.Healthy}
301311
}
302312
}

‎ipu-plugin/pkg/ipuplugin/ipuplugin.go‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -234,8 +234,8 @@ func (s *server) Run() error {
234234
}
235235

236236
func cleanUpRulesOnExit(p4rtClient types.P4RTClient) error {
237-
log.Infof("DeletePhyPortRules, path->%s, 1->%v, 2->%v", p4rtClient.GetBin(), AccApfMacList[PHY_PORT0_SECONDARY_INTF_INDEX], AccApfMacList[PHY_PORT0_PRIMARY_INTF_INDEX])
238-
p4rtclient.DeletePhyPortRules(p4rtClient, AccApfMacList[PHY_PORT0_SECONDARY_INTF_INDEX], AccApfMacList[PHY_PORT0_PRIMARY_INTF_INDEX])
237+
log.Infof("DeletePhyPortRules, path->%s, 1->%v, 2->%v", p4rtClient.GetBin(), AccApfInfo[PHY_PORT0_SECONDARY_INTF_INDEX].Mac, AccApfInfo[PHY_PORT0_PRIMARY_INTF_INDEX].Mac)
238+
p4rtclient.DeletePhyPortRules(p4rtClient, AccApfInfo[PHY_PORT0_SECONDARY_INTF_INDEX].Mac, AccApfInfo[PHY_PORT0_PRIMARY_INTF_INDEX].Mac)
239239

240240
vfMacList, err := utils.GetVfMacList()
241241
if err != nil {
@@ -252,8 +252,8 @@ func cleanUpRulesOnExit(p4rtClient types.P4RTClient) error {
252252
log.Infof("DeleteLAGP4Rules, path->%s", p4rtClient.GetBin())
253253
p4rtclient.DeleteLAGP4Rules(p4rtClient)
254254

255-
log.Infof("DeleteRHPrimaryNetworkVportP4Rules, path->%s, 1->%v", p4rtClient, AccApfMacList[PHY_PORT0_PRIMARY_INTF_INDEX])
256-
p4rtclient.DeleteRHPrimaryNetworkVportP4Rules(p4rtClient, AccApfMacList[PHY_PORT0_PRIMARY_INTF_INDEX])
255+
log.Infof("DeleteRHPrimaryNetworkVportP4Rules, path->%s, 1->%v", p4rtClient, AccApfInfo[PHY_PORT0_PRIMARY_INTF_INDEX].Mac)
256+
p4rtclient.DeleteRHPrimaryNetworkVportP4Rules(p4rtClient, AccApfInfo[PHY_PORT0_PRIMARY_INTF_INDEX].Mac)
257257
return nil
258258
}
259259

0 commit comments

Comments
 (0)