自建k8s部署aws-Ebs-Csi-Driver

1

https://github.com/kubernetes-sigs/aws-ebs-csi-driver/blob/v1.13.0/docs/example-iam-policy.json

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30

kubectl create secret generic aws-secret \
    --namespace kube-system \
    --from-literal "key_id=${AWS_ACCESS_KEY_ID}" \
    --from-literal "access_key=${AWS_SECRET_ACCESS_KEY}"
``

```yaml
# helm repo add aws-ebs-csi-driver https://kubernetes-sigs.github.io/aws-ebs-csi-driver

# helm upgrade --install aws-ebs-csi-driver --version 2.13.0\
    --namespace kube-system \
    aws-ebs-csi-driver/aws-ebs-csi-driver \
    --set sidecars.provisioner.image.repository=registry.cn-hangzhou.aliyuncs.com/seam/csi-provisioner \
    --set sidecars.attacher.image.repository=registry.cn-hangzhou.aliyuncs.com/seam/csi-attacher \
    --set sidecars.snapshotter.image.repository=registry.cn-hangzhou.aliyuncs.com/seam/csi-snapshotter \
    --set sidecars.livenessProbe.image.repository=registry.cn-hangzhou.aliyuncs.com/seam/livenessprobe \
    --set sidecars.resizer.image.repository=registry.cn-hangzhou.aliyuncs.com/seam/csi-resizer \
    --set sidecars.nodeDriverRegistrar.image.repository=registry.cn-hangzhou.aliyuncs.com/seam/csi-node-driver-registrar \
    --set node.kubeletPath=/data/k8s/kubelet
# kubectl get pod -n kube-system -l "app.kubernetes.io/name=aws-ebs-csi-driver,app.kubernetes.io/instance=aws-ebs-csi-driver"
NAME                                 READY   STATUS    RESTARTS   AGE
ebs-csi-controller-5cbfd45dc-2fq9q   6/6     Running   0          102s
ebs-csi-controller-5cbfd45dc-jgpl9   6/6     Running   0          102s
ebs-csi-node-2s8lj                   3/3     Running   0          101s
ebs-csi-node-4jstr                   3/3     Running   0          101s
ebs-csi-node-72w69                   3/3     Running   0          101s
ebs-csi-node-759rd                   0/3     Pending   0          101s
ebs-csi-node-cq86s                   3/3     Running   0          101s
ebs-csi-node-jnfxk                   0/3     Pending   0          101s
ebs-csi-node-m48nn                   3/3     Running   0          101s

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48

cat << EOF | kubectl apply -f -
apiVersion: storage.k8s.io/v1
kind: StorageClass
metadata:
  name: ebs-sc
provisioner: ebs.csi.aws.com
volumeBindingMode: WaitForFirstConsumer
parameters:
  csi.storage.k8s.io/fstype: xfs
  type: gp3
  iopsPerGB: "500"
  tagSpecification_1: "businessid=000000040629041578"

---
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
  name: ebs-claim
spec:
  accessModes:
    - ReadWriteOnce
  storageClassName: ebs-sc
  resources:
    requests:
      storage: 10Gi
---

apiVersion: v1
kind: Pod
metadata:
  name: app
spec:
#   nodeSelector:
#     hostname: 10.86.183.104
  containers:
  - name: app
    image: centos
    command: ["/bin/sh"]
    args: ["-c", "while true; do echo $(date -u) >> /data/out.txt; sleep 5; done"]
    volumeMounts:
    - name: persistent-storage
      mountPath: /data
  volumes:
  - name: persistent-storage
    persistentVolumeClaim:
      claimName: ebs-claim
EOF

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19

cat << EOF | kubectl apply -f -
apiVersion: storage.k8s.io/v1
kind: StorageClass
metadata:
  name: ebs-sc
parameters:
  type: gp3       # io1, io2, gp2, gp3, sc1, st1, standard, sbp1, sbg1
  csi.storage.k8s.io/fstype: ext4  # xfs, ext2, ext3, ext4
  iops: "3000"
  # iopsPerGB: 500  每GB存储的iops,跟iops二选一
  throughput: "125"
  tagSpecification_1: "businessid=000000040629041578"
  tagSpecification_2: "pvcnamespace={{ .PVCNamespace }}"
  tagSpecification_3: "pvcname={{ .PVCName }}"
  tagSpecification_4: "pvname={{ .PVName }}"
provisioner: ebs.csi.aws.com
reclaimPolicy: Delete
volumeBindingMode: WaitForFirstConsumer
EOF

  1
  2
  3
  4
  5
  6
  7
  8
  9
 10
 11
 12
 13
 14
 15
 16
 17
 18
 19
 20
 21
 22
 23
 24
 25
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110

cat << EOF > fio-job.yaml
# NOTE: For details of params to construct an fio job, refer to this link:
# https://fio.readthedocs.io/en/latest/fio_doc.html


---
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
  name: ebs-fio
spec:
  accessModes:
    - ReadWriteOnce
  storageClassName: ebs-sc
  resources:
    requests:
      storage: 6Gi

---
apiVersion: batch/v1
kind: Job
metadata:
  generateName: dbench-
spec:
  template:
    spec:
      containers:
      - name: dbench
        image: openebs/perf-test:latest
        imagePullPolicy: IfNotPresent
        env:

          ## storage mount point on which testfiles are created

          - name: DBENCH_MOUNTPOINT
            value: /data

          ##########################################################
          # I/O PROFILE COVERAGE FOR SPECIFIC PERF CHARACTERISTICS #
          ##########################################################

          ## quick: {read, write} iops, {read, write} bw (all random)
          ## detailed: {quick}, {read, write} latency & mixed 75r:25w (all random), {read, write} bw (all sequential)
          ## custom: a single user-defined job run with params specified in env 'CUSTOM'

          - name: DBENCH_TYPE
            value: detailed

          ####################################################
          # STANDARD TUNABLES FOR DBENCH_TYPE=QUICK/DETAILED #
          ####################################################

          ## active data size for the bench test

          - name: FIO_SIZE
            value: 2G

          ## use un-buffered i/o (usually O_DIRECT)

          - name: FIO_DIRECT
            value: '1'

          ## no of independent threads doing the same i/o

          - name: FIO_NUMJOBS
            value: '1'

          ## space b/w starting offsets on a file in case of parallel file i/o

          - name: FIO_OFFSET_INCREMENT
            value: 250M

          ## nature of i/o to file. commonly supported: libaio, sync, 

          - name: FIO_IOENGINE
            value: libaio

          ## additional runtime options which will be appended to the above params
          ## ensure options used are not mutually exclusive w/ above params
          ## ex: '--group_reporting=1, stonewall, --ramptime=<val> etc..,
          
          - name: OPTIONS
            value: ''

          ####################################################
          # CUSTOM JOB SPEC FOR DBENCH_TYPE=CUSTOM           #
          ####################################################

          ## this will execute a single job run with the params specified 
          ## ex: '--bs=16k --iodepth=64 --ioengine=sync --size=500M --name=custom --readwrite=randrw --rwmixread=80 --random_distribution=pareto' 

          - name: CUSTOM
            value: ''
          
        volumeMounts:
        - name: dbench-pv
          mountPath: /data
      restartPolicy: Never
      nodeSelector:
        hostname: shopline-xinjiapo-aws-dev-vm-8c32g-10.90.208.109
      volumes:
      - name: dbench-pv
        persistentVolumeClaim:
          claimName: ebs-fio
  backoffLimit: 4
---
EOF

kubectl create -f fio-job.yaml

  1
  2
  3
  4
  5
  6
  7
  8
  9
 10
 11
 12
 13
 14
 15
 16
 17
 18
 19
 20
 21
 22
 23
 24
 25
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290

root@k8s-master-7b4d73fd8d:~# kubectl get pvc
NAME      STATUS   VOLUME                                     CAPACITY   ACCESS MODES   STORAGECLASS   AGE
ebs-fio   Bound    pvc-d603d626-1013-4fef-b1c7-b406d2d93546   6Gi        RWO            ebs-sc         4s
root@k8s-master-7b4d73fd8d:~# kubectl get pod
NAME                 READY   STATUS    RESTARTS   AGE
dbench-dgkpv-457kk   1/1     Running   0          110s
root@k8s-master-7b4d73fd8d:~#
root@k8s-master-7b4d73fd8d:~#
root@k8s-master-7b4d73fd8d:~#
root@k8s-master-7b4d73fd8d:~# kubectl get pod
NAME                 READY   STATUS    RESTARTS   AGE
dbench-dgkpv-457kk   1/1     Running   0          2m59s
root@k8s-master-7b4d73fd8d:~# kubectl logs -f dbench-dgkpv-457kk
Working dir: /data

Testing Read IOPS...
read_iops: (g=0): rw=randread, bs=(R) 4096B-4096B, (W) 4096B-4096B, (T) 4096B-4096B, ioengine=libaio, iodepth=64
fio-3.13
Starting 1 process
read_iops: Laying out IO file (1 file / 2048MiB)

read_iops: (groupid=0, jobs=1): err= 0: pid=18: Tue Dec 27 07:59:57 2022
  read: IOPS=2995, BW=11.7MiB/s (12.3MB/s)(176MiB/15029msec)
   bw (  KiB/s): min=11912, max=12056, per=100.00%, avg=11999.47, stdev=27.31, samples=30
   iops        : min= 2978, max= 3014, avg=2999.87, stdev= 6.83, samples=30
  cpu          : usr=0.31%, sys=0.89%, ctx=9728, majf=0, minf=15
  IO depths    : 1=0.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=100.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.1%, >=64=0.0%
     issued rwts: total=45018,0,0,0 short=0,0,0,0 dropped=0,0,0,0
     latency   : target=0, window=0, percentile=100.00%, depth=64

Run status group 0 (all jobs):
   READ: bw=11.7MiB/s (12.3MB/s), 11.7MiB/s-11.7MiB/s (12.3MB/s-12.3MB/s), io=176MiB (185MB), run=15029-15029msec

Disk stats (read/write):
  nvme1n1: ios=53995/8, merge=0/3, ticks=1024884/151, in_queue=1025035, util=99.45%


Testing Write IOPS...
write_iops: (g=0): rw=randwrite, bs=(R) 4096B-4096B, (W) 4096B-4096B, (T) 4096B-4096B, ioengine=libaio, iodepth=64
fio-3.13
Starting 1 process

write_iops: (groupid=0, jobs=1): err= 0: pid=34: Tue Dec 27 08:00:14 2022
  write: IOPS=2996, BW=11.7MiB/s (12.3MB/s)(176MiB/15020msec); 0 zone resets
   bw (  KiB/s): min=11952, max=12040, per=99.97%, avg=11996.77, stdev=24.26, samples=30
   iops        : min= 2988, max= 3010, avg=2999.17, stdev= 6.05, samples=30
  cpu          : usr=0.40%, sys=1.58%, ctx=15976, majf=0, minf=15
  IO depths    : 1=0.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=100.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.1%, >=64=0.0%
     issued rwts: total=0,45000,0,0 short=0,0,0,0 dropped=0,0,0,0
     latency   : target=0, window=0, percentile=100.00%, depth=64

Run status group 0 (all jobs):
  WRITE: bw=11.7MiB/s (12.3MB/s), 11.7MiB/s-11.7MiB/s (12.3MB/s-12.3MB/s), io=176MiB (185MB), run=15020-15020msec

Disk stats (read/write):
  nvme1n1: ios=0/52548, merge=0/0, ticks=0/1033990, in_queue=1033990, util=99.48%


Testing Read Bandwidth...
read_bw: (g=0): rw=randread, bs=(R) 128KiB-128KiB, (W) 128KiB-128KiB, (T) 128KiB-128KiB, ioengine=libaio, iodepth=64
fio-3.13
Starting 1 process

read_bw: (groupid=0, jobs=1): err= 0: pid=50: Tue Dec 27 08:00:32 2022
  read: IOPS=996, BW=125MiB/s (131MB/s)(1889MiB/15099msec)
   bw (  KiB/s): min=126208, max=129792, per=99.92%, avg=128016.97, stdev=950.10, samples=30
   iops        : min=  986, max= 1014, avg=1000.03, stdev= 7.32, samples=30
  cpu          : usr=0.07%, sys=0.78%, ctx=3493, majf=0, minf=15
  IO depths    : 1=0.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=100.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.1%, >=64=0.0%
     issued rwts: total=15050,0,0,0 short=0,0,0,0 dropped=0,0,0,0
     latency   : target=0, window=0, percentile=100.00%, depth=64

Run status group 0 (all jobs):
   READ: bw=125MiB/s (131MB/s), 125MiB/s-125MiB/s (131MB/s-131MB/s), io=1889MiB (1981MB), run=15099-15099msec

Disk stats (read/write):
  nvme1n1: ios=18012/1, merge=0/0, ticks=1023152/29, in_queue=1023181, util=99.46%


Testing Write Bandwidth...
write_bw: (g=0): rw=randwrite, bs=(R) 128KiB-128KiB, (W) 128KiB-128KiB, (T) 128KiB-128KiB, ioengine=libaio, iodepth=64
fio-3.13
Starting 1 process

write_bw: (groupid=0, jobs=1): err= 0: pid=66: Tue Dec 27 08:00:50 2022
  write: IOPS=996, BW=125MiB/s (131MB/s)(1885MiB/15069msec); 0 zone resets
   bw (  KiB/s): min=126976, max=129532, per=99.97%, avg=128067.90, stdev=808.75, samples=30
   iops        : min=  992, max= 1011, avg=1000.40, stdev= 6.15, samples=30
  cpu          : usr=0.46%, sys=0.83%, ctx=4183, majf=0, minf=15
  IO depths    : 1=0.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=100.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.1%, >=64=0.0%
     issued rwts: total=0,15019,0,0 short=0,0,0,0 dropped=0,0,0,0
     latency   : target=0, window=0, percentile=100.00%, depth=64

Run status group 0 (all jobs):
  WRITE: bw=125MiB/s (131MB/s), 125MiB/s-125MiB/s (131MB/s-131MB/s), io=1885MiB (1977MB), run=15069-15069msec

Disk stats (read/write):
  nvme1n1: ios=0/17530, merge=0/0, ticks=0/1028369, in_queue=1028369, util=99.44%


Testing Read Latency...
read_latency: (g=0): rw=randread, bs=(R) 4096B-4096B, (W) 4096B-4096B, (T) 4096B-4096B, ioengine=libaio, iodepth=4
fio-3.13
Starting 1 process

read_latency: (groupid=0, jobs=1): err= 0: pid=82: Tue Dec 27 08:01:07 2022
  read: IOPS=2999, BW=11.7MiB/s (12.3MB/s)(176MiB/15001msec)
    slat (nsec): min=1155, max=74266, avg=2327.18, stdev=1536.53
    clat (usec): min=325, max=4499, avg=1330.59, stdev=195.50
     lat (usec): min=341, max=4501, avg=1333.00, stdev=195.41
    clat percentiles (usec):
     |  1.00th=[  889],  5.00th=[ 1020], 10.00th=[ 1090], 20.00th=[ 1188],
     | 30.00th=[ 1254], 40.00th=[ 1287], 50.00th=[ 1319], 60.00th=[ 1352],
     | 70.00th=[ 1401], 80.00th=[ 1467], 90.00th=[ 1582], 95.00th=[ 1663],
     | 99.00th=[ 1811], 99.50th=[ 1893], 99.90th=[ 2147], 99.95th=[ 2409],
     | 99.99th=[ 3982]
   bw (  KiB/s): min=11968, max=12024, per=99.95%, avg=11993.33, stdev=15.02, samples=30
   iops        : min= 2992, max= 3006, avg=2998.33, stdev= 3.75, samples=30
  lat (usec)   : 500=0.05%, 750=0.29%, 1000=3.57%
  lat (msec)   : 2=95.90%, 4=0.18%, 10=0.01%
  cpu          : usr=0.93%, sys=1.76%, ctx=44371, majf=0, minf=15
  IO depths    : 1=0.0%, 2=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued rwts: total=44997,0,0,0 short=0,0,0,0 dropped=0,0,0,0
     latency   : target=0, window=0, percentile=100.00%, depth=4

Run status group 0 (all jobs):
   READ: bw=11.7MiB/s (12.3MB/s), 11.7MiB/s-11.7MiB/s (12.3MB/s-12.3MB/s), io=176MiB (184MB), run=15001-15001msec

Disk stats (read/write):
  nvme1n1: ios=53995/1, merge=0/0, ticks=67606/2, in_queue=67609, util=99.44%


Testing Write Latency...
write_latency: (g=0): rw=randwrite, bs=(R) 4096B-4096B, (W) 4096B-4096B, (T) 4096B-4096B, ioengine=libaio, iodepth=4
fio-3.13
Starting 1 process

write_latency: (groupid=0, jobs=1): err= 0: pid=98: Tue Dec 27 08:01:25 2022
  write: IOPS=2999, BW=11.7MiB/s (12.3MB/s)(176MiB/15001msec); 0 zone resets
    slat (nsec): min=1207, max=46414, avg=2452.55, stdev=1529.09
    clat (usec): min=554, max=13927, avg=1330.43, stdev=385.44
     lat (usec): min=556, max=13929, avg=1332.96, stdev=385.41
    clat percentiles (usec):
     |  1.00th=[  717],  5.00th=[  922], 10.00th=[ 1037], 20.00th=[ 1156],
     | 30.00th=[ 1221], 40.00th=[ 1270], 50.00th=[ 1319], 60.00th=[ 1369],
     | 70.00th=[ 1418], 80.00th=[ 1483], 90.00th=[ 1582], 95.00th=[ 1680],
     | 99.00th=[ 1975], 99.50th=[ 2180], 99.90th=[ 6521], 99.95th=[ 8979],
     | 99.99th=[13042]
   bw (  KiB/s): min=11856, max=12144, per=99.97%, avg=11995.73, stdev=40.18, samples=30
   iops        : min= 2964, max= 3036, avg=2998.93, stdev=10.04, samples=30
  lat (usec)   : 750=1.44%, 1000=6.72%
  lat (msec)   : 2=90.97%, 4=0.63%, 10=0.20%, 20=0.04%
  cpu          : usr=0.61%, sys=1.91%, ctx=44616, majf=0, minf=15
  IO depths    : 1=0.0%, 2=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued rwts: total=0,44999,0,0 short=0,0,0,0 dropped=0,0,0,0
     latency   : target=0, window=0, percentile=100.00%, depth=4

Run status group 0 (all jobs):
  WRITE: bw=11.7MiB/s (12.3MB/s), 11.7MiB/s-11.7MiB/s (12.3MB/s-12.3MB/s), io=176MiB (184MB), run=15001-15001msec

Disk stats (read/write):
  nvme1n1: ios=0/52559, merge=0/0, ticks=0/67299, in_queue=67299, util=99.42%


Testing Read Sequential Speed...
read_seq: (g=0): rw=read, bs=(R) 1024KiB-1024KiB, (W) 1024KiB-1024KiB, (T) 1024KiB-1024KiB, ioengine=libaio, iodepth=16
fio-3.13
Starting 1 thread

read_seq: (groupid=0, jobs=1): err= 0: pid=114: Tue Dec 27 08:01:43 2022
  read: IOPS=124, BW=125MiB/s (131MB/s)(1927MiB/15402msec)
   bw (  KiB/s): min=126722, max=131072, per=99.94%, avg=128034.00, stdev=1278.01, samples=30
   iops        : min=  123, max=  128, avg=124.93, stdev= 1.26, samples=30
  cpu          : usr=0.04%, sys=0.45%, ctx=1926, majf=0, minf=0
  IO depths    : 1=0.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=100.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=99.9%, 8=0.0%, 16=0.1%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued rwts: total=1912,0,0,0 short=0,0,0,0 dropped=0,0,0,0
     latency   : target=0, window=0, percentile=100.00%, depth=16

Run status group 0 (all jobs):
   READ: bw=125MiB/s (131MB/s), 125MiB/s-125MiB/s (131MB/s-131MB/s), io=1927MiB (2021MB), run=15402-15402msec

Disk stats (read/write):
  nvme1n1: ios=9133/0, merge=0/0, ticks=1036255/0, in_queue=1036255, util=99.48%


Testing Write Sequential Speed...
write_seq: (g=0): rw=write, bs=(R) 1024KiB-1024KiB, (W) 1024KiB-1024KiB, (T) 1024KiB-1024KiB, ioengine=libaio, iodepth=16
...
fio-3.13
Starting 4 threads
write_seq: Laying out IO file (1 file / 2798MiB)

write_seq: (groupid=0, jobs=1): err= 0: pid=130: Tue Dec 27 08:02:01 2022
  write: IOPS=35, BW=36.3MiB/s (38.0MB/s)(571MiB/15740msec); 0 zone resets
   bw (  KiB/s): min=18432, max=63488, per=27.94%, avg=36322.81, stdev=14085.31, samples=31
   iops        : min=   18, max=   62, avg=35.35, stdev=13.75, samples=31
  cpu          : usr=0.08%, sys=0.15%, ctx=394, majf=0, minf=0
  IO depths    : 1=0.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=100.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=99.8%, 8=0.0%, 16=0.2%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued rwts: total=0,556,0,0 short=0,0,0,0 dropped=0,0,0,0
     latency   : target=0, window=0, percentile=100.00%, depth=16
write_seq: (groupid=0, jobs=1): err= 0: pid=131: Tue Dec 27 08:02:01 2022
  write: IOPS=29, BW=30.9MiB/s (32.4MB/s)(480MiB/15538msec); 0 zone resets
   bw (  KiB/s): min=20480, max=53248, per=23.60%, avg=30680.58, stdev=8976.14, samples=31
   iops        : min=   20, max=   52, avg=29.87, stdev= 8.71, samples=31
  cpu          : usr=0.09%, sys=0.12%, ctx=286, majf=0, minf=0
  IO depths    : 1=0.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=100.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=99.8%, 8=0.0%, 16=0.2%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued rwts: total=0,465,0,0 short=0,0,0,0 dropped=0,0,0,0
     latency   : target=0, window=0, percentile=100.00%, depth=16
write_seq: (groupid=0, jobs=1): err= 0: pid=132: Tue Dec 27 08:02:01 2022
  write: IOPS=26, BW=27.7MiB/s (29.0MB/s)(438MiB/15834msec); 0 zone resets
   bw (  KiB/s): min=20480, max=38912, per=21.14%, avg=27480.71, stdev=4979.95, samples=31
   iops        : min=   20, max=   38, avg=26.81, stdev= 4.83, samples=31
  cpu          : usr=0.08%, sys=0.11%, ctx=250, majf=0, minf=0
  IO depths    : 1=0.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=100.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=99.8%, 8=0.0%, 16=0.2%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued rwts: total=0,423,0,0 short=0,0,0,0 dropped=0,0,0,0
     latency   : target=0, window=0, percentile=100.00%, depth=16
write_seq: (groupid=0, jobs=1): err= 0: pid=133: Tue Dec 27 08:02:01 2022
  write: IOPS=32, BW=33.9MiB/s (35.5MB/s)(521MiB/15382msec); 0 zone resets
   bw (  KiB/s): min=20439, max=59392, per=26.20%, avg=34061.50, stdev=12825.18, samples=30
   iops        : min=   19, max=   58, avg=33.20, stdev=12.57, samples=30
  cpu          : usr=0.06%, sys=0.17%, ctx=325, majf=0, minf=0
  IO depths    : 1=0.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=100.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=99.8%, 8=0.0%, 16=0.2%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued rwts: total=0,506,0,0 short=0,0,0,0 dropped=0,0,0,0
     latency   : target=0, window=0, percentile=100.00%, depth=16

Run status group 0 (all jobs):
  WRITE: bw=127MiB/s (133MB/s), 27.7MiB/s-36.3MiB/s (29.0MB/s-38.0MB/s), io=2010MiB (2108MB), run=15382-15834msec

Disk stats (read/write):
  nvme1n1: ios=0/9182, merge=0/0, ticks=0/1929233, in_queue=1929234, util=99.47%


Testing Read/Write Mixed...
rw_mix: (g=0): rw=randrw, bs=(R) 4096B-4096B, (W) 4096B-4096B, (T) 4096B-4096B, ioengine=libaio, iodepth=64
fio-3.13
Starting 1 process

rw_mix: (groupid=0, jobs=1): err= 0: pid=149: Tue Dec 27 08:02:18 2022
  read: IOPS=3058, BW=11.0MiB/s (12.5MB/s)(180MiB/15031msec)
   bw (  KiB/s): min=11568, max=12816, per=100.00%, avg=12247.93, stdev=284.96, samples=30
   iops        : min= 2892, max= 3204, avg=3061.93, stdev=71.22, samples=30
  write: IOPS=1015, BW=4068KiB/s (4166kB/s)(59.7MiB/15031msec); 0 zone resets
   bw (  KiB/s): min= 3680, max= 4424, per=99.95%, avg=4065.83, stdev=178.92, samples=30
   iops        : min=  920, max= 1106, avg=1016.43, stdev=44.69, samples=30
  cpu          : usr=0.39%, sys=1.32%, ctx=9977, majf=0, minf=15
  IO depths    : 1=0.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=100.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.1%, >=64=0.0%
     issued rwts: total=45976,15259,0,0 short=0,0,0,0 dropped=0,0,0,0
     latency   : target=0, window=0, percentile=100.00%, depth=64

Run status group 0 (all jobs):
   READ: bw=11.0MiB/s (12.5MB/s), 11.0MiB/s-11.0MiB/s (12.5MB/s-12.5MB/s), io=180MiB (188MB), run=15031-15031msec
  WRITE: bw=4068KiB/s (4166kB/s), 4068KiB/s-4068KiB/s (4166kB/s-4166kB/s), io=59.7MiB (62.6MB), run=15031-15031msec

Disk stats (read/write):
  nvme1n1: ios=35746/18257, merge=0/0, ticks=675623/349719, in_queue=1025341, util=99.47%


All tests complete.

==================
= Dbench Summary =
==================
Random Read/Write IOPS: 2995/2996. BW: 125MiB/s / 125MiB/s
Average Latency (usec) Read/Write: 1333.00/1332.96
Sequential Read/Write: 125MiB/s / 127MiB/s
Mixed Random Read/Write IOPS: 3058/1015

  1
  2
  3
  4
  5
  6
  7
  8
  9
 10
 11
 12
 13
 14
 15
 16
 17
 18
 19
 20
 21
 22
 23
 24
 25
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278

# kubectl logs -f dbench-fzmlc-74frw
Working dir: /data

Testing Read IOPS...
read_iops: (g=0): rw=randread, bs=(R) 4096B-4096B, (W) 4096B-4096B, (T) 4096B-4096B, ioengine=libaio, iodepth=64
fio-3.13
Starting 1 process
read_iops: Laying out IO file (1 file / 2048MiB)

read_iops: (groupid=0, jobs=1): err= 0: pid=17: Tue Dec 27 07:50:41 2022
  read: IOPS=4995, BW=19.5MiB/s (20.5MB/s)(293MiB/15023msec)
   bw (  KiB/s): min=19920, max=20056, per=100.00%, avg=20000.53, stdev=31.23, samples=30
   iops        : min= 4980, max= 5014, avg=5000.13, stdev= 7.81, samples=30
  cpu          : usr=0.35%, sys=1.48%, ctx=15343, majf=0, minf=15
  IO depths    : 1=0.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=100.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.1%, >=64=0.0%
     issued rwts: total=75052,0,0,0 short=0,0,0,0 dropped=0,0,0,0
     latency   : target=0, window=0, percentile=100.00%, depth=64

Run status group 0 (all jobs):
   READ: bw=19.5MiB/s (20.5MB/s), 19.5MiB/s-19.5MiB/s (20.5MB/s-20.5MB/s), io=293MiB (308MB), run=15023-15023msec

Disk stats (read/write):
  nvme1n1: ios=90002/0, merge=0/0, ticks=1023770/0, in_queue=1023770, util=99.48%


Testing Write IOPS...
write_iops: (g=0): rw=randwrite, bs=(R) 4096B-4096B, (W) 4096B-4096B, (T) 4096B-4096B, ioengine=libaio, iodepth=64
fio-3.13
Starting 1 process

write_iops: (groupid=0, jobs=1): err= 0: pid=33: Tue Dec 27 07:50:59 2022
  write: IOPS=4995, BW=19.5MiB/s (20.5MB/s)(293MiB/15015msec); 0 zone resets
   bw (  KiB/s): min=15360, max=24616, per=100.00%, avg=20001.60, stdev=1219.20, samples=30
   iops        : min= 3840, max= 6154, avg=5000.40, stdev=304.80, samples=30
  cpu          : usr=0.51%, sys=1.77%, ctx=18684, majf=0, minf=15
  IO depths    : 1=0.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=100.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.1%, >=64=0.0%
     issued rwts: total=0,75008,0,0 short=0,0,0,0 dropped=0,0,0,0
     latency   : target=0, window=0, percentile=100.00%, depth=64

Run status group 0 (all jobs):
  WRITE: bw=19.5MiB/s (20.5MB/s), 19.5MiB/s-19.5MiB/s (20.5MB/s-20.5MB/s), io=293MiB (307MB), run=15015-15015msec

Disk stats (read/write):
  nvme1n1: ios=0/87614, merge=0/0, ticks=0/1028234, in_queue=1028235, util=99.46%


Testing Read Bandwidth...
read_bw: (g=0): rw=randread, bs=(R) 128KiB-128KiB, (W) 128KiB-128KiB, (T) 128KiB-128KiB, ioengine=libaio, iodepth=64
fio-3.13
Starting 1 process

read_bw: (groupid=0, jobs=1): err= 0: pid=49: Tue Dec 27 07:51:17 2022
  read: IOPS=996, BW=125MiB/s (131MB/s)(1889MiB/15096msec)
   bw (  KiB/s): min=126720, max=129536, per=99.99%, avg=128102.40, stdev=925.71, samples=30
   iops        : min=  990, max= 1012, avg=1000.80, stdev= 7.23, samples=30
  cpu          : usr=0.07%, sys=0.78%, ctx=3325, majf=0, minf=15
  IO depths    : 1=0.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=100.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.1%, >=64=0.0%
     issued rwts: total=15047,0,0,0 short=0,0,0,0 dropped=0,0,0,0
     latency   : target=0, window=0, percentile=100.00%, depth=64

Run status group 0 (all jobs):
   READ: bw=125MiB/s (131MB/s), 125MiB/s-125MiB/s (131MB/s-131MB/s), io=1889MiB (1980MB), run=15096-15096msec

Disk stats (read/write):
  nvme1n1: ios=18015/1, merge=0/0, ticks=1024772/24, in_queue=1024797, util=99.48%


Testing Write Bandwidth...
write_bw: (g=0): rw=randwrite, bs=(R) 128KiB-128KiB, (W) 128KiB-128KiB, (T) 128KiB-128KiB, ioengine=libaio, iodepth=64
fio-3.13
Starting 1 process

write_bw: (groupid=0, jobs=1): err= 0: pid=65: Tue Dec 27 07:51:34 2022
  write: IOPS=996, BW=125MiB/s (131MB/s)(1892MiB/15122msec); 0 zone resets
   bw (  KiB/s): min=126464, max=129792, per=99.98%, avg=128093.87, stdev=935.14, samples=30
   iops        : min=  988, max= 1014, avg=1000.73, stdev= 7.31, samples=30
  cpu          : usr=0.44%, sys=0.81%, ctx=4142, majf=0, minf=15
  IO depths    : 1=0.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=100.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.1%, >=64=0.0%
     issued rwts: total=0,15073,0,0 short=0,0,0,0 dropped=0,0,0,0
     latency   : target=0, window=0, percentile=100.00%, depth=64

Run status group 0 (all jobs):
  WRITE: bw=125MiB/s (131MB/s), 125MiB/s-125MiB/s (131MB/s-131MB/s), io=1892MiB (1984MB), run=15122-15122msec

Disk stats (read/write):
  nvme1n1: ios=0/17540, merge=0/0, ticks=0/1029415, in_queue=1029415, util=99.44%


Testing Read Latency...
read_latency: (g=0): rw=randread, bs=(R) 4096B-4096B, (W) 4096B-4096B, (T) 4096B-4096B, ioengine=libaio, iodepth=4
fio-3.13
Starting 1 process

read_latency: (groupid=0, jobs=1): err= 0: pid=81: Tue Dec 27 07:51:52 2022
  read: IOPS=4999, BW=19.5MiB/s (20.5MB/s)(293MiB/15001msec)
    slat (nsec): min=1143, max=55215, avg=2088.09, stdev=1211.27
    clat (usec): min=282, max=4215, avg=797.49, stdev=196.70
     lat (usec): min=283, max=4218, avg=799.65, stdev=196.61
    clat percentiles (usec):
     |  1.00th=[  392],  5.00th=[  490], 10.00th=[  553], 20.00th=[  627],
     | 30.00th=[  693], 40.00th=[  742], 50.00th=[  791], 60.00th=[  832],
     | 70.00th=[  889], 80.00th=[  963], 90.00th=[ 1057], 95.00th=[ 1123],
     | 99.00th=[ 1270], 99.50th=[ 1336], 99.90th=[ 1483], 99.95th=[ 1614],
     | 99.99th=[ 3064]
   bw (  KiB/s): min=19960, max=20016, per=100.00%, avg=19998.62, stdev=16.29, samples=29
   iops        : min= 4990, max= 5004, avg=4999.66, stdev= 4.07, samples=29
  lat (usec)   : 500=5.63%, 750=36.72%, 1000=42.04%
  lat (msec)   : 2=15.60%, 4=0.01%, 10=0.01%
  cpu          : usr=1.05%, sys=3.25%, ctx=72026, majf=0, minf=15
  IO depths    : 1=0.0%, 2=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued rwts: total=74997,0,0,0 short=0,0,0,0 dropped=0,0,0,0
     latency   : target=0, window=0, percentile=100.00%, depth=4

Run status group 0 (all jobs):
   READ: bw=19.5MiB/s (20.5MB/s), 19.5MiB/s-19.5MiB/s (20.5MB/s-20.5MB/s), io=293MiB (307MB), run=15001-15001msec

Disk stats (read/write):
  nvme1n1: ios=88747/1, merge=0/0, ticks=66494/1, in_queue=66494, util=99.47%


Testing Write Latency...
write_latency: (g=0): rw=randwrite, bs=(R) 4096B-4096B, (W) 4096B-4096B, (T) 4096B-4096B, ioengine=libaio, iodepth=4
fio-3.13
Starting 1 process

write_latency: (groupid=0, jobs=1): err= 0: pid=97: Tue Dec 27 07:52:09 2022
  write: IOPS=5071, BW=19.8MiB/s (20.8MB/s)(297MiB/15001msec); 0 zone resets
    slat (nsec): min=1182, max=79256, avg=2385.32, stdev=1587.33
    clat (usec): min=480, max=16903, avg=785.79, stdev=286.58
     lat (usec): min=481, max=16909, avg=788.24, stdev=286.64
    clat percentiles (usec):
     |  1.00th=[  537],  5.00th=[  570], 10.00th=[  603], 20.00th=[  644],
     | 30.00th=[  676], 40.00th=[  717], 50.00th=[  734], 60.00th=[  775],
     | 70.00th=[  824], 80.00th=[  889], 90.00th=[  996], 95.00th=[ 1106],
     | 99.00th=[ 1483], 99.50th=[ 1811], 99.90th=[ 4047], 99.95th=[ 4883],
     | 99.99th=[11469]
   bw (  KiB/s): min=17392, max=22680, per=100.00%, avg=20294.45, stdev=1067.71, samples=29
   iops        : min= 4348, max= 5670, avg=5073.59, stdev=266.98, samples=29
  lat (usec)   : 500=0.03%, 750=54.10%, 1000=36.29%
  lat (msec)   : 2=9.19%, 4=0.28%, 10=0.10%, 20=0.01%
  cpu          : usr=0.54%, sys=3.17%, ctx=74197, majf=0, minf=15
  IO depths    : 1=0.0%, 2=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued rwts: total=0,76078,0,0 short=0,0,0,0 dropped=0,0,0,0
     latency   : target=0, window=0, percentile=100.00%, depth=4

Run status group 0 (all jobs):
  WRITE: bw=19.8MiB/s (20.8MB/s), 19.8MiB/s-19.8MiB/s (20.8MB/s-20.8MB/s), io=297MiB (312MB), run=15001-15001msec

Disk stats (read/write):
  nvme1n1: ios=0/86374, merge=0/0, ticks=0/65667, in_queue=65667, util=99.39%


Testing Read Sequential Speed...
read_seq: (g=0): rw=read, bs=(R) 1024KiB-1024KiB, (W) 1024KiB-1024KiB, (T) 1024KiB-1024KiB, ioengine=libaio, iodepth=16
fio-3.13
Starting 1 thread

read_seq: (groupid=0, jobs=1): err= 0: pid=113: Tue Dec 27 07:52:27 2022
  read: IOPS=124, BW=125MiB/s (131MB/s)(1927MiB/15402msec)
   bw (  KiB/s): min=126976, max=131072, per=99.96%, avg=128068.27, stdev=1495.65, samples=30
   iops        : min=  124, max=  128, avg=125.07, stdev= 1.46, samples=30
  cpu          : usr=0.00%, sys=0.51%, ctx=1928, majf=0, minf=0
  IO depths    : 1=0.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=100.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=99.9%, 8=0.0%, 16=0.1%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued rwts: total=1912,0,0,0 short=0,0,0,0 dropped=0,0,0,0
     latency   : target=0, window=0, percentile=100.00%, depth=16

Run status group 0 (all jobs):
   READ: bw=125MiB/s (131MB/s), 125MiB/s-125MiB/s (131MB/s-131MB/s), io=1927MiB (2021MB), run=15402-15402msec

Disk stats (read/write):
  nvme1n1: ios=9132/1, merge=0/0, ticks=1035562/26, in_queue=1035589, util=99.47%


Testing Write Sequential Speed...
write_seq: (g=0): rw=write, bs=(R) 1024KiB-1024KiB, (W) 1024KiB-1024KiB, (T) 1024KiB-1024KiB, ioengine=libaio, iodepth=16
...
fio-3.13
Starting 4 threads
write_seq: Laying out IO file (1 file / 2798MiB)

write_seq: (groupid=0, jobs=1): err= 0: pid=129: Tue Dec 27 07:52:46 2022
  write: IOPS=35, BW=36.5MiB/s (38.3MB/s)(563MiB/15420msec); 0 zone resets
   bw (  KiB/s): min=16384, max=69632, per=28.02%, avg=36715.67, stdev=16672.60, samples=30
   iops        : min=   16, max=   68, avg=35.73, stdev=16.23, samples=30
  cpu          : usr=0.10%, sys=0.15%, ctx=391, majf=0, minf=0
  IO depths    : 1=0.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=100.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=99.8%, 8=0.0%, 16=0.2%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued rwts: total=0,548,0,0 short=0,0,0,0 dropped=0,0,0,0
     latency   : target=0, window=0, percentile=100.00%, depth=16
write_seq: (groupid=0, jobs=1): err= 0: pid=130: Tue Dec 27 07:52:46 2022
  write: IOPS=42, BW=43.8MiB/s (45.9MB/s)(692MiB/15808msec); 0 zone resets
   bw (  KiB/s): min=18432, max=75776, per=32.92%, avg=43137.03, stdev=18668.78, samples=31
   iops        : min=   18, max=   74, avg=42.10, stdev=18.19, samples=31
  cpu          : usr=0.21%, sys=0.07%, ctx=531, majf=0, minf=0
  IO depths    : 1=0.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=100.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=99.9%, 8=0.0%, 16=0.1%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued rwts: total=0,677,0,0 short=0,0,0,0 dropped=0,0,0,0
     latency   : target=0, window=0, percentile=100.00%, depth=16
write_seq: (groupid=0, jobs=1): err= 0: pid=131: Tue Dec 27 07:52:46 2022
  write: IOPS=23, BW=24.9MiB/s (26.2MB/s)(382MiB/15316msec); 0 zone resets
   bw (  KiB/s): min=14336, max=32768, per=19.06%, avg=24980.40, stdev=4968.21, samples=30
   iops        : min=   14, max=   32, avg=24.30, stdev= 4.84, samples=30
  cpu          : usr=0.04%, sys=0.12%, ctx=213, majf=0, minf=0
  IO depths    : 1=0.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=100.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=99.7%, 8=0.0%, 16=0.3%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued rwts: total=0,367,0,0 short=0,0,0,0 dropped=0,0,0,0
     latency   : target=0, window=0, percentile=100.00%, depth=16
write_seq: (groupid=0, jobs=1): err= 0: pid=132: Tue Dec 27 07:52:46 2022
  write: IOPS=24, BW=25.2MiB/s (26.4MB/s)(386MiB/15332msec); 0 zone resets
   bw (  KiB/s): min=18432, max=34816, per=19.22%, avg=25190.40, stdev=4874.72, samples=30
   iops        : min=   18, max=   34, avg=24.60, stdev= 4.76, samples=30
  cpu          : usr=0.00%, sys=0.18%, ctx=213, majf=0, minf=0
  IO depths    : 1=0.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=100.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=99.7%, 8=0.0%, 16=0.3%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued rwts: total=0,371,0,0 short=0,0,0,0 dropped=0,0,0,0
     latency   : target=0, window=0, percentile=100.00%, depth=16

Run status group 0 (all jobs):
  WRITE: bw=128MiB/s (134MB/s), 24.9MiB/s-43.8MiB/s (26.2MB/s-45.9MB/s), io=2023MiB (2121MB), run=15316-15808msec

Disk stats (read/write):
  nvme1n1: ios=0/9167, merge=0/0, ticks=0/1723282, in_queue=1723282, util=99.46%


Testing Read/Write Mixed...
rw_mix: (g=0): rw=randrw, bs=(R) 4096B-4096B, (W) 4096B-4096B, (T) 4096B-4096B, ioengine=libaio, iodepth=64
fio-3.13
Starting 1 process

rw_mix: (groupid=0, jobs=1): err= 0: pid=148: Tue Dec 27 07:53:03 2022
  read: IOPS=4633, BW=18.1MiB/s (18.0MB/s)(272MiB/15013msec)
   bw (  KiB/s): min=17896, max=19352, per=100.00%, avg=18547.20, stdev=344.80, samples=30
   iops        : min= 4474, max= 4838, avg=4636.80, stdev=86.20, samples=30
  write: IOPS=1564, BW=6263KiB/s (6413kB/s)(91.8MiB/15013msec); 0 zone resets
   bw (  KiB/s): min= 5768, max= 6632, per=100.00%, avg=6264.00, stdev=203.09, samples=30
   iops        : min= 1442, max= 1658, avg=1566.00, stdev=50.77, samples=30
  cpu          : usr=0.66%, sys=1.75%, ctx=15214, majf=0, minf=15
  IO depths    : 1=0.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=100.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.1%, >=64=0.0%
     issued rwts: total=69556,23486,0,0 short=0,0,0,0 dropped=0,0,0,0
     latency   : target=0, window=0, percentile=100.00%, depth=64

Run status group 0 (all jobs):
   READ: bw=18.1MiB/s (18.0MB/s), 18.1MiB/s-18.1MiB/s (18.0MB/s-18.0MB/s), io=272MiB (285MB), run=15013-15013msec
  WRITE: bw=6263KiB/s (6413kB/s), 6263KiB/s-6263KiB/s (6413kB/s-6413kB/s), io=91.8MiB (96.3MB), run=15013-15013msec

Disk stats (read/write):
  nvme1n1: ios=61823/28174, merge=0/0, ticks=698506/324761, in_queue=1023267, util=99.46%


All tests complete.

==================
= Dbench Summary =
==================
Random Read/Write IOPS: 4995/4995. BW: 125MiB/s / 125MiB/s
Average Latency (usec) Read/Write: 799.65/788.24
Sequential Read/Write: 125MiB/s / 128MiB/s
Mixed Random Read/Write IOPS: 4633/1564

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31

kubectl apply -f - <<EOF
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
  name: system:nodes:volumeattachments
rules:
- apiGroups:
  - storage.k8s.io
  resources:
  - volumeattachments
  verbs:
  - create
  - watch
  - delete

---

apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
  name: system:nodes:volumeattachments
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: ClusterRole
  name: system:nodes:volumeattachments
subjects:
- apiGroup: rbac.authorization.k8s.io
  kind: Group
  name: system:nodes

EOF