Skip to content

Commit ab13de3

Browse files
zyb521zyb521
committed
[feat] curvebs: The formatting and deployment phases support separate partitioning of WAL and data
Signed-off-by: zyb521 <[email protected]>
1 parent 9e46bbe commit ab13de3

File tree

6 files changed

+63
-16
lines changed

6 files changed

+63
-16
lines changed

‎internal/configure/format.go‎

Lines changed: 17 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,8 @@ import (
3232

3333
const (
3434
DEFAULT_CONTAINER_IMAGE = "opencurvedocker/curvebs:v1.2"
35+
FORMAT_TYPE_DATA = "data"
36+
FORMAT_TYPE_WAL = "wal"
3537
)
3638

3739
/*
@@ -40,14 +42,18 @@ const (
4042
* - machine2
4143
* - machine3
4244
* disk:
43-
* - /dev/sda:/data/chunkserver0:10 # device:mount_path:format_percent
44-
* - /dev/sdb:/data/chunkserver1:10
45-
* - /dev/sdc:/data/chunkserver2:10
45+
* - data:/dev/sda:/data/chunkserver0:10 # fortmat_type:device:mount_path:format_percent
46+
* - data:/dev/sdb:/data/chunkserver1:10
47+
* - data:/dev/sdc:/data/chunkserver2:10
48+
* - wal:/dev/nvme0n1p1:/data/wal/chunkserver0:10
49+
* - wal:/dev/nvme0n1p2:/data/wal/chunkserver1:10
50+
* - wal:/dev/nvme0n1p3:/data/wal/chunkserver2:10
4651
*/
4752
type (
4853
FormatConfig struct {
4954
ContainerIamge string
5055
Host string
56+
Type string
5157
Device string
5258
MountPoint string
5359
FormtPercent int
@@ -64,12 +70,15 @@ type (
6470

6571
func NewFormatConfig(containerImage, host, disk string) (*FormatConfig, error) {
6672
items := strings.Split(disk, ":")
67-
if len(items) != 3 {
73+
if len(items) != 4 {
6874
return nil, errno.ERR_INVALID_DISK_FORMAT.S(disk)
6975
}
7076

71-
device, mountPoint, percent := items[0], items[1], items[2]
72-
if !strings.HasPrefix(device, "/") {
77+
formatType, device, mountPoint, percent := items[0], items[1], items[2], items[3]
78+
if formatType != FORMAT_TYPE_DATA && formatType != FORMAT_TYPE_WAL {
79+
return nil, errno.ERR_INVALID_FORMAT_TYPE.
80+
F("formatType: %s", formatType)
81+
} else if !strings.HasPrefix(device, "/") {
7382
return nil, errno.ERR_INVALID_DEVICE.
7483
F("device: %s", device)
7584
} else if !strings.HasPrefix(mountPoint, "/") {
@@ -89,6 +98,7 @@ func NewFormatConfig(containerImage, host, disk string) (*FormatConfig, error) {
8998
return &FormatConfig{
9099
ContainerIamge: containerImage,
91100
Host: host,
101+
Type: formatType,
92102
Device: device,
93103
MountPoint: mountPoint,
94104
FormtPercent: formatPercent,
@@ -136,6 +146,7 @@ func ParseFormat(filename string) ([]*FormatConfig, error) {
136146

137147
func (fc *FormatConfig) GetContainerImage() string { return fc.ContainerIamge }
138148
func (fc *FormatConfig) GetHost() string { return fc.Host }
149+
func (fc *FormatConfig) GetFormatType() string { return fc.Type }
139150
func (fc *FormatConfig) GetDevice() string { return fc.Device }
140151
func (fc *FormatConfig) GetMountPoint() string { return fc.MountPoint }
141152
func (fc *FormatConfig) GetFormatPercent() int { return fc.FormtPercent }

‎internal/configure/topology/dc_get.go‎

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -42,6 +42,7 @@ const (
4242
LAYOUT_SERVICE_CONF_DIR = "/conf"
4343
LAYOUT_SERVICE_LOG_DIR = "/logs"
4444
LAYOUT_SERVICE_DATA_DIR = "/data"
45+
LAYOUT_SERVICE_WAL_DIR = "/wal"
4546
LAYOUT_TOOLS_DIR = "/tools"
4647
LAYOUT_TOOLS_V2_DIR = "/tools-v2"
4748
LAYOUT_CURVEBS_CHUNKFILE_POOL_DIR = "chunkfilepool"
@@ -131,6 +132,7 @@ func (dc *DeployConfig) GetContainerImage() string {
131132
}
132133
func (dc *DeployConfig) GetLogDir() string { return dc.getString(CONFIG_LOG_DIR) }
133134
func (dc *DeployConfig) GetDataDir() string { return dc.getString(CONFIG_DATA_DIR) }
135+
func (dc *DeployConfig) GetWalDir() string { return dc.getString(CONFIG_WAL_DIR) }
134136
func (dc *DeployConfig) GetCoreDir() string { return dc.getString(CONFIG_CORE_DIR) }
135137
func (dc *DeployConfig) GetListenIp() string { return dc.getString(CONFIG_LISTEN_IP) }
136138
func (dc *DeployConfig) GetListenPort() int { return dc.getInt(CONFIG_LISTEN_PORT) }
@@ -181,7 +183,8 @@ func (dc *DeployConfig) GetListenExternalPort() int {
181183
* │ ├── conf
182184
* │ ├── data
183185
* │ ├── log
184-
* │ └── sbin
186+
* │ ├── sbin
187+
* │ └── wal
185188
* ├── snapshotclone
186189
* │ ├── conf
187190
* │ ├── data
@@ -212,6 +215,7 @@ type (
212215
ServiceConfDir string // /curvebs/mds/conf
213216
ServiceLogDir string // /curvebs/mds/logs
214217
ServiceDataDir string // /curvebs/mds/data
218+
ServiceWalDir string // /curvebs/chunkserver/wal
215219
ServiceConfPath string // /curvebs/mds/conf/mds.conf
216220
ServiceConfSrcPath string // /curvebs/conf/mds.conf
217221
ServiceConfFiles []ConfFile
@@ -236,6 +240,9 @@ type (
236240
ChunkfilePoolRootDir string // /curvebs/chunkserver/data
237241
ChunkfilePoolDir string // /curvebs/chunkserver/data/chunkfilepool
238242
ChunkfilePoolMetaPath string // /curvebs/chunkserver/data/chunkfilepool.meta
243+
WalfilePoolRootDir string // /curvebs/chunkserver/wal
244+
WalfilePoolDir string // /curvebs/chunkserver/wal/walfilepool
245+
WalfilePoolMetaPath string // /curvebs/chunkserver/wal/walfilepool.meta
239246

240247
// core
241248
CoreSystemDir string
@@ -292,6 +299,7 @@ func (dc *DeployConfig) GetProjectLayout() Layout {
292299
ServiceConfDir: serviceRootDir + LAYOUT_SERVICE_CONF_DIR,
293300
ServiceLogDir: serviceRootDir + LAYOUT_SERVICE_LOG_DIR,
294301
ServiceDataDir: serviceRootDir + LAYOUT_SERVICE_DATA_DIR,
302+
ServiceWalDir: serviceRootDir + LAYOUT_SERVICE_WAL_DIR,
295303
ServiceConfPath: fmt.Sprintf("%s/%s.conf", serviceConfDir, role),
296304
ServiceConfSrcPath: fmt.Sprintf("%s/%s.conf", confSrcDir, role),
297305
ServiceConfFiles: serviceConfFiles,

‎internal/configure/topology/dc_item.go‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -124,6 +124,13 @@ var (
124124
nil,
125125
)
126126

127+
CONFIG_WAL_DIR = itemset.insert(
128+
"wal_dir",
129+
REQUIRE_STRING,
130+
true,
131+
nil,
132+
)
133+
127134
CONFIG_CORE_DIR = itemset.insert(
128135
"core_dir",
129136
REQUIRE_STRING,

‎internal/errno/errno.go‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -355,6 +355,7 @@ var (
355355
ERR_MOUNT_POINT_REQUIRE_ABSOLUTE_PATH = EC(341002, "mount point must be an absolute path")
356356
ERR_FORMAT_PERCENT_REQUIRES_INTERGET = EC(341003, "format percentage requires an integer")
357357
ERR_FORMAT_PERCENT_MUST_BE_BETWEEN_1_AND_100 = EC(341004, "format percentage must be between 1 and 100")
358+
ERR_INVALID_FORMAT_TYPE = EC(341005, "format type must be (data) or (wal)")
358359

359360
// 350: configure (client.yaml: parse failed)
360361
ERR_PARSE_CLIENT_CONFIGURE_FAILED = EC(350000, "parse client configure failed")

‎internal/task/task/bs/format.go‎

Lines changed: 17 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -231,22 +231,33 @@ func NewFormatChunkfilePoolTask(curveadm *cli.CurveAdm, fc *configure.FormatConf
231231
}
232232

233233
// new task
234+
formatTpye := fc.GetFormatType()
234235
device := fc.GetDevice()
235236
mountPoint := fc.GetMountPoint()
236237
usagePercent := fc.GetFormatPercent()
237-
subname := fmt.Sprintf("host=%s device=%s mountPoint=%s usage=%d%%",
238-
fc.GetHost(), device, mountPoint, usagePercent)
239-
t := task.NewTask("Start Format Chunkfile Pool", subname, hc.GetSSHConfig())
238+
subname := fmt.Sprintf("host=%s formatTpye=%s device=%s mountPoint=%s usage=%d%%",
239+
fc.GetHost(), formatTpye, device, mountPoint, usagePercent)
240+
t := task.NewTask("Start Format filePool", subname, hc.GetSSHConfig())
240241

241242
// add step to task
242243
var oldContainerId, containerId, oldUUID string
244+
var filePoolRootDir, filePoolDir, filePoolMetaPath string
243245
containerName := device2ContainerName(device)
244246
layout := topology.GetCurveBSProjectLayout()
245-
chunkfilePoolRootDir := layout.ChunkfilePoolRootDir
247+
248+
if formatTpye == configure.FORMAT_TYPE_WAL {
249+
filePoolRootDir = layout.WalfilePoolRootDir
250+
filePoolDir = layout.WalfilePoolDir
251+
filePoolMetaPath = layout.WalfilePoolMetaPath
252+
} else {
253+
filePoolRootDir = layout.ChunkfilePoolRootDir
254+
filePoolDir = layout.ChunkfilePoolDir
255+
filePoolMetaPath = layout.ChunkfilePoolMetaPath
256+
}
246257
formatScript := scripts.SCRIPT_FORMAT
247258
formatScriptPath := fmt.Sprintf("%s/format.sh", layout.ToolsBinDir)
248259
formatCommand := fmt.Sprintf("%s %s %d %d %s %s", formatScriptPath, layout.FormatBinaryPath,
249-
usagePercent, DEFAULT_CHUNKFILE_SIZE, layout.ChunkfilePoolDir, layout.ChunkfilePoolMetaPath)
260+
usagePercent, DEFAULT_CHUNKFILE_SIZE, filePoolDir, filePoolMetaPath)
250261

251262
// 1: skip if formating container exist
252263
t.AddStep(&step.ListContainers{
@@ -320,7 +331,7 @@ func NewFormatChunkfilePoolTask(curveadm *cli.CurveAdm, fc *configure.FormatConf
320331
Entrypoint: "/bin/bash",
321332
Name: containerName,
322333
Remove: true,
323-
Volumes: []step.Volume{{HostPath: mountPoint, ContainerPath: chunkfilePoolRootDir}},
334+
Volumes: []step.Volume{{HostPath: mountPoint, ContainerPath: filePoolRootDir}},
324335
Out: &containerId,
325336
ExecOptions: curveadm.ExecOptions(),
326337
})

‎internal/task/task/common/create_container.go‎

Lines changed: 12 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -106,6 +106,7 @@ func getArguments(dc *topology.DeployConfig) string {
106106
// only chunkserver need so many arguments, but who cares
107107
layout := dc.GetProjectLayout()
108108
dataDir := layout.ServiceDataDir
109+
walDir := layout.ServiceWalDir
109110
chunkserverArguments := map[string]interface{}{
110111
// chunkserver
111112
"conf": layout.ServiceConfPath,
@@ -115,11 +116,11 @@ func getArguments(dc *topology.DeployConfig) string {
115116
"chunkServerPort": dc.GetListenPort(),
116117
"chunkFilePoolDir": dataDir,
117118
"chunkFilePoolMetaPath": fmt.Sprintf("%s/chunkfilepool.meta", dataDir),
118-
"walFilePoolDir": dataDir,
119-
"walFilePoolMetaPath": fmt.Sprintf("%s/walfilepool.meta", dataDir),
119+
"walFilePoolDir": walDir,
120+
"walFilePoolMetaPath": fmt.Sprintf("%s/walfilepool.meta", walDir),
120121
"copySetUri": fmt.Sprintf("local://%s/copysets", dataDir),
121122
"recycleUri": fmt.Sprintf("local://%s/recycler", dataDir),
122-
"raftLogUri": fmt.Sprintf("curve://%s/copysets", dataDir),
123+
"raftLogUri": fmt.Sprintf("curve://%s/copysets", walDir),
123124
"raftSnapshotUri": fmt.Sprintf("curve://%s/copysets", dataDir),
124125
"chunkServerStoreUri": fmt.Sprintf("local://%s", dataDir),
125126
"chunkServerMetaUri": fmt.Sprintf("local://%s/chunkserver.dat", dataDir),
@@ -158,6 +159,7 @@ func getMountVolumes(dc *topology.DeployConfig, serviceMountDevice bool) []step.
158159
layout := dc.GetProjectLayout()
159160
logDir := dc.GetLogDir()
160161
dataDir := dc.GetDataDir()
162+
walDir := dc.GetWalDir()
161163
coreDir := dc.GetCoreDir()
162164

163165
if len(logDir) > 0 {
@@ -175,6 +177,13 @@ func getMountVolumes(dc *topology.DeployConfig, serviceMountDevice bool) []step.
175177
})
176178
}
177179

180+
if len(walDir) > 0 {
181+
volumes = append(volumes, step.Volume{
182+
HostPath: walDir,
183+
ContainerPath: layout.ServiceWalDir,
184+
})
185+
}
186+
178187
if len(coreDir) > 0 {
179188
volumes = append(volumes, step.Volume{
180189
HostPath: coreDir,

0 commit comments

Comments
 (0)