Skip to content

Commit 0b1884a

Browse files
Millefeuille42Millefeuille42
committed
refactor: use VHD header to fetch block size
Signed-off-by: Mathieu Labourier <mathieu.labourier@vates.tech>
1 parent 3ecfefa commit 0b1884a

File tree

9 files changed

+149
-45
lines changed

9 files changed

+149
-45
lines changed

‎libs/sm/VDI.py‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -101,6 +101,7 @@ def __init__(self, sr, uuid):
101101
self.description = ''
102102
self.vbds = []
103103
self.size = 0
104+
self._block_size = -1
104105
self.utilisation = 0
105106
self.vdi_type = ''
106107
self.has_child = 0
@@ -120,6 +121,12 @@ def __init__(self, sr, uuid):
120121

121122
self.load(uuid)
122123

124+
@property
125+
def block_size(self):
126+
if self._block_size < 0:
127+
self._block_size = vhdutil.getBlockSize(self.path)
128+
return self._block_size
129+
123130
@staticmethod
124131
def from_uuid(session, vdi_uuid):
125132

‎libs/sm/cleanup.py‎

Lines changed: 18 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -535,12 +535,19 @@ def __init__(self, sr, uuid, raw):
535535
self.sizeVirt = -1
536536
self._sizeVHD = -1
537537
self._sizeAllocated = -1
538+
self._block_size = -1
538539
self._hidden = False
539540
self.parent = None
540541
self.children = []
541542
self._vdiRef = None
542543
self._clearRef()
543544

545+
@property
546+
def block_size(self):
547+
if self._block_size < 0:
548+
self._block_size = vhdutil.getBlockSize(self.path)
549+
return self._block_size
550+
544551
@staticmethod
545552
def extractUuid(path):
546553
raise NotImplementedError("Implement in sub class")
@@ -1075,14 +1082,16 @@ def _getCoalescedSizeData(self):
10751082
blocksParent = self.parent.getVHDBlocks()
10761083
numBlocks = Util.countBits(blocksChild, blocksParent)
10771084
Util.log("Num combined blocks = %d" % numBlocks)
1078-
sizeData = numBlocks * vhdutil.VHD_BLOCK_SIZE
1085+
sizeData = numBlocks * self.block_size
10791086
assert(sizeData <= self.sizeVirt)
10801087
return sizeData
10811088

10821089
def _calcExtraSpaceForCoalescing(self):
10831090
sizeData = self._getCoalescedSizeData()
1084-
sizeCoalesced = sizeData + vhdutil.calcOverheadBitmap(sizeData) + \
1085-
vhdutil.calcOverheadEmpty(self.sizeVirt)
1091+
sizeCoalesced = sizeData + vhdutil.calcOverheadBitmap(
1092+
sizeData,
1093+
self.block_size
1094+
) + vhdutil.calcOverheadEmpty(self.sizeVirt)
10861095
Util.log("Coalesced size = %s" % Util.num2str(sizeCoalesced))
10871096
return sizeCoalesced - self.parent.getSizeVHD()
10881097

@@ -1238,7 +1247,7 @@ def deflate(self):
12381247
self._sizeAllocated = -1
12391248

12401249
def inflateFully(self):
1241-
self.inflate(lvhdutil.calcSizeVHDLV(self.sizeVirt))
1250+
self.inflate(lvhdutil.calcSizeVHDLV(self.sizeVirt, self.block_size))
12421251

12431252
def inflateParentForCoalesce(self):
12441253
"""Inflate the parent only as much as needed for the purposes of
@@ -1461,7 +1470,10 @@ def _queryVHDBlocks(self):
14611470
def _calcExtraSpaceForCoalescing(self):
14621471
if self.parent.raw:
14631472
return 0 # raw parents are never deflated in the first place
1464-
sizeCoalesced = lvhdutil.calcSizeVHDLV(self._getCoalescedSizeData())
1473+
sizeCoalesced = lvhdutil.calcSizeVHDLV(
1474+
self._getCoalescedSizeData(),
1475+
self.block_size
1476+
)
14651477
Util.log("Coalesced size = %s" % Util.num2str(sizeCoalesced))
14661478
return sizeCoalesced - self.parent.sizeLV
14671479

@@ -2775,7 +2787,7 @@ def _finishCoalesceLeaf(self, parent):
27752787
parent.deflate()
27762788

27772789
def _calcExtraSpaceNeeded(self, child, parent):
2778-
return lvhdutil.calcSizeVHDLV(parent.sizeVirt) - parent.sizeLV
2790+
return lvhdutil.calcSizeVHDLV(parent.sizeVirt, parent.block_size) - parent.sizeLV
27792791

27802792
def _handleInterruptedCoalesceLeaf(self):
27812793
entries = self.journaler.getAll(VDI.JRN_LEAF)

‎libs/sm/drivers/FileSR.py‎

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -561,7 +561,10 @@ def create(self, sr_uuid, vdi_uuid, size):
561561

562562
if self.vdi_type == vhdutil.VDI_TYPE_VHD:
563563
try:
564-
size = vhdutil.validate_and_round_vhd_size(int(size))
564+
size = vhdutil.validate_and_round_vhd_size(
565+
int(size),
566+
vhdutil.DEFAULT_VHD_BLOCK_SIZE
567+
)
565568
mb = 1024 * 1024
566569
size_mb = size // mb
567570
util.ioretry(lambda: self._create(str(size_mb), self.path))
@@ -656,7 +659,7 @@ def resize(self, sr_uuid, vdi_uuid, size):
656659
return VDI.VDI.get_params(self)
657660

658661
# We already checked it is a VDI_TYPE_VHD
659-
size = vhdutil.validate_and_round_vhd_size(int(size))
662+
size = vhdutil.validate_and_round_vhd_size(int(size), self.block_size)
660663

661664
jFile = JOURNAL_FILE_PREFIX + self.uuid
662665
try:

‎libs/sm/drivers/LVHDSR.py‎

Lines changed: 26 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -716,7 +716,10 @@ def scan(self, uuid):
716716
util.roundup(lvutil.LVM_SIZE_INCREMENT,
717717
vhdutil.calcOverheadEmpty(lvhdutil.MSIZE))
718718
else:
719-
utilisation = lvhdutil.calcSizeVHDLV(int(size))
719+
utilisation = lvhdutil.calcSizeVHDLV(
720+
int(size),
721+
vhdutil.getBlockSize(lvPath)
722+
)
720723

721724
vdi_ref = self.session.xenapi.VDI.db_introduce(
722725
vdi_uuid,
@@ -984,7 +987,10 @@ def _undoCloneOp(self, lvs, origUuid, baseUuid, clonUuid):
984987

985988
# inflate the parent to fully-allocated size
986989
if base.vdiType == vhdutil.VDI_TYPE_VHD:
987-
fullSize = lvhdutil.calcSizeVHDLV(vhdInfo.sizeVirt)
990+
fullSize = lvhdutil.calcSizeVHDLV(
991+
vhdInfo.sizeVirt,
992+
vhdutil.getBlockSize(basePath)
993+
)
988994
lvhdutil.inflate(self.journaler, self.uuid, baseUuid, fullSize)
989995

990996
# rename back
@@ -1173,7 +1179,7 @@ def _undoAllVHDJournals(self):
11731179
util.SMlog("Found VHD journal %s, reverting %s" % (uuid, vdi.path))
11741180
self.lvActivator.activate(uuid, vdi.lvname, False)
11751181
self.lvmCache.activateNoRefcount(jlvName)
1176-
fullSize = lvhdutil.calcSizeVHDLV(vdi.size)
1182+
fullSize = lvhdutil.calcSizeVHDLV(vdi.size, vdi.block_size)
11771183
lvhdutil.inflate(self.journaler, self.uuid, vdi.uuid, fullSize)
11781184
try:
11791185
jFile = os.path.join(self.path, jlvName)
@@ -1184,7 +1190,7 @@ def _undoAllVHDJournals(self):
11841190
util.SMlog("VHD revert failed but VHD ok: removing journal")
11851191
# Attempt to reclaim unused space
11861192
vhdInfo = vhdutil.getVHDInfo(vdi.path, lvhdutil.extractUuid, False)
1187-
NewSize = lvhdutil.calcSizeVHDLV(vhdInfo.sizeVirt)
1193+
NewSize = lvhdutil.calcSizeVHDLV(vhdInfo.sizeVirt, vdi.block_size)
11881194
if NewSize < fullSize:
11891195
lvhdutil.deflate(self.lvmCache, vdi.lvname, int(NewSize))
11901196
lvhdutil.lvRefreshOnAllSlaves(self.session, self.uuid,
@@ -1343,7 +1349,10 @@ def create(self, sr_uuid, vdi_uuid, size):
13431349
if self.exists:
13441350
raise xs_errors.XenError('VDIExists')
13451351

1346-
size = vhdutil.validate_and_round_vhd_size(int(size))
1352+
size = vhdutil.validate_and_round_vhd_size(
1353+
int(size),
1354+
vhdutil.DEFAULT_VHD_BLOCK_SIZE
1355+
)
13471356

13481357
util.SMlog("LVHDVDI.create: type = %s, %s (size=%s)" % \
13491358
(self.vdi_type, self.path, size))
@@ -1356,7 +1365,10 @@ def create(self, sr_uuid, vdi_uuid, size):
13561365
lvSize = util.roundup(lvutil.LVM_SIZE_INCREMENT,
13571366
vhdutil.calcOverheadEmpty(lvhdutil.MSIZE))
13581367
elif self.sr.provision == "thick":
1359-
lvSize = lvhdutil.calcSizeVHDLV(int(size))
1368+
lvSize = lvhdutil.calcSizeVHDLV(
1369+
int(size),
1370+
vhdutil.DEFAULT_VHD_BLOCK_SIZE
1371+
)
13601372

13611373
self.sr._ensureSpaceAvailable(lvSize)
13621374

@@ -1459,7 +1471,10 @@ def attach(self, sr_uuid, vdi_uuid):
14591471
needInflate = False
14601472
else:
14611473
self._loadThis()
1462-
if self.utilisation >= lvhdutil.calcSizeVHDLV(self.size):
1474+
if (
1475+
self.utilisation >=
1476+
lvhdutil.calcSizeVHDLV(self.size, self.block_size)
1477+
):
14631478
needInflate = False
14641479

14651480
if needInflate:
@@ -1479,7 +1494,7 @@ def detach(self, sr_uuid, vdi_uuid):
14791494
util.SMlog("LVHDVDI.detach for %s" % self.uuid)
14801495
self._loadThis()
14811496
already_deflated = (self.utilisation < \
1482-
lvhdutil.calcSizeVHDLV(self.size))
1497+
lvhdutil.calcSizeVHDLV(self.size, self.block_size))
14831498
needDeflate = True
14841499
if self.vdi_type == vhdutil.VDI_TYPE_RAW or already_deflated:
14851500
needDeflate = False
@@ -1520,7 +1535,7 @@ def resize(self, sr_uuid, vdi_uuid, size):
15201535
'(current size: %d, new size: %d)' % (self.size, size))
15211536
raise xs_errors.XenError('VDISize', opterr='shrinking not allowed')
15221537

1523-
size = vhdutil.validate_and_round_vhd_size(int(size))
1538+
size = vhdutil.validate_and_round_vhd_size(int(size), self.block_size)
15241539

15251540
if size == self.size:
15261541
return VDI.VDI.get_params(self)
@@ -1530,7 +1545,7 @@ def resize(self, sr_uuid, vdi_uuid, size):
15301545
lvSizeNew = util.roundup(lvutil.LVM_SIZE_INCREMENT, size)
15311546
else:
15321547
lvSizeOld = self.utilisation
1533-
lvSizeNew = lvhdutil.calcSizeVHDLV(size)
1548+
lvSizeNew = lvhdutil.calcSizeVHDLV(size, self.block_size)
15341549
if self.sr.provision == "thin":
15351550
# VDI is currently deflated, so keep it deflated
15361551
lvSizeNew = lvSizeOld
@@ -1696,7 +1711,7 @@ def _snapshot(self, snapType, cloneOp=False, cbtlog=None, cbt_consistency=None):
16961711
self.issnap = self.session.xenapi.VDI.get_is_a_snapshot( \
16971712
self.sr.srcmd.params['vdi_ref'])
16981713

1699-
fullpr = lvhdutil.calcSizeVHDLV(self.size)
1714+
fullpr = lvhdutil.calcSizeVHDLV(self.size, self.block_size)
17001715
thinpr = util.roundup(lvutil.LVM_SIZE_INCREMENT, \
17011716
vhdutil.calcOverheadEmpty(lvhdutil.MSIZE))
17021717
lvSizeOrig = thinpr

‎libs/sm/lvhdutil.py‎

Lines changed: 10 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -95,11 +95,11 @@ def calcSizeLV(sizeVHD):
9595
return util.roundup(LVM_SIZE_INCREMENT, sizeVHD)
9696

9797

98-
def calcSizeVHDLV(sizeVirt):
98+
def calcSizeVHDLV(sizeVirt, block_size):
9999
# all LVHD VDIs have the metadata area preallocated for the maximum
100100
# possible virtual size (for fast online VDI.resize)
101101
metaOverhead = vhdutil.calcOverheadEmpty(MSIZE)
102-
bitmapOverhead = vhdutil.calcOverheadBitmap(sizeVirt)
102+
bitmapOverhead = vhdutil.calcOverheadBitmap(sizeVirt, block_size)
103103
return calcSizeLV(sizeVirt + metaOverhead + bitmapOverhead)
104104

105105

@@ -208,7 +208,12 @@ def setSizeVirt(journaler, srUuid, vdiUuid, size, jFile):
208208
lvName = LV_PREFIX[vhdutil.VDI_TYPE_VHD] + vdiUuid
209209
vgName = VG_PREFIX + srUuid
210210
path = os.path.join(VG_LOCATION, vgName, lvName)
211-
inflate(journaler, srUuid, vdiUuid, calcSizeVHDLV(size))
211+
inflate(
212+
journaler,
213+
srUuid,
214+
vdiUuid,
215+
calcSizeVHDLV(size, vhdutil.getBlockSize(path))
216+
)
212217
vhdutil.setSizeVirt(path, size, jFile)
213218

214219

@@ -233,7 +238,8 @@ def attachThin(journaler, srUuid, vdiUuid):
233238
_tryAcquire(lock)
234239
lvmCache.refresh()
235240
vhdInfo = vhdutil.getVHDInfoLVM(lvName, extractUuid, vgName)
236-
newSize = calcSizeVHDLV(vhdInfo.sizeVirt)
241+
path = os.path.join(VG_LOCATION, vgName, lvName)
242+
newSize = calcSizeVHDLV(vhdInfo.sizeVirt, vhdutil.getBlockSize(path))
237243
currSizeLV = lvmCache.getSize(lvName)
238244
if newSize <= currSizeLV:
239245
return

‎libs/sm/vhdutil.py‎

Lines changed: 22 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -30,7 +30,7 @@
3030
MAX_CHAIN_SIZE = 30 # max VHD parent chain size
3131
VHD_UTIL = "/usr/bin/vhd-util"
3232
OPT_LOG_ERR = "--debug"
33-
VHD_BLOCK_SIZE = 2 * 1024 * 1024
33+
DEFAULT_VHD_BLOCK_SIZE = 2 * 1024 * 1024
3434
VHD_FOOTER_SIZE = 512
3535

3636
# lock to lock the entire SR for short ops
@@ -82,9 +82,9 @@ def calcOverheadEmpty(virtual_size):
8282
return overhead
8383

8484

85-
def calcOverheadBitmap(virtual_size):
86-
num_blocks = virtual_size // VHD_BLOCK_SIZE
87-
if virtual_size % VHD_BLOCK_SIZE:
85+
def calcOverheadBitmap(virtual_size, block_size):
86+
num_blocks = virtual_size // block_size
87+
if virtual_size % block_size:
8888
num_blocks += 1
8989
return num_blocks * 4096
9090

@@ -93,10 +93,18 @@ def ioretry(cmd, text=True):
9393
return util.ioretry(lambda: util.pread2(cmd, text=text),
9494
errlist=[errno.EIO, errno.EAGAIN])
9595

96+
def getBlockSize(path):
97+
cmd = [VHD_UTIL, "read", "-pn", path]
98+
ret = ioretry(cmd)
99+
for field in ret.split('\n'):
100+
field = field.lstrip()
101+
if not field.startswith("Block size"): continue
102+
return int(field.split(':')[1].lstrip().split()[0])
103+
raise util.SMException("Unable to find block size in VHD with path: {}".format(path))
104+
96105

97-
def convertAllocatedSizeToBytes(size):
98-
# Assume we have standard 2MB allocation blocks
99-
return size * 2 * 1024 * 1024
106+
def convertAllocatedSizeToBytes(size, block_size):
107+
return size * block_size
100108

101109

102110
def getVHDInfo(path, extractUuidFunction, includeParent=True):
@@ -120,7 +128,10 @@ def getVHDInfo(path, extractUuidFunction, includeParent=True):
120128
vhdInfo.parentUuid = extractUuidFunction(fields[nextIndex])
121129
nextIndex += 1
122130
vhdInfo.hidden = int(fields[nextIndex].replace("hidden: ", ""))
123-
vhdInfo.sizeAllocated = convertAllocatedSizeToBytes(int(fields[nextIndex+1]))
131+
vhdInfo.sizeAllocated = convertAllocatedSizeToBytes(
132+
int(fields[nextIndex+1]),
133+
getBlockSize(path)
134+
)
124135
vhdInfo.path = path
125136
return vhdInfo
126137

@@ -279,7 +290,7 @@ def setSizePhys(path, size, debug=True):
279290
def getAllocatedSize(path):
280291
cmd = [VHD_UTIL, "query", OPT_LOG_ERR, '-a', '-n', path]
281292
ret = ioretry(cmd)
282-
return convertAllocatedSizeToBytes(int(ret))
293+
return convertAllocatedSizeToBytes(int(ret), getBlockSize(path))
283294

284295
def killData(path):
285296
"zero out the disk (kill all data inside the VHD file)"
@@ -406,7 +417,7 @@ def repair(path):
406417
ioretry([VHD_UTIL, 'repair', '-n', path])
407418

408419

409-
def validate_and_round_vhd_size(size):
420+
def validate_and_round_vhd_size(size, block_size):
410421
""" Take the supplied vhd size, in bytes, and check it is positive and less
411422
that the maximum supported size, rounding up to the next block boundary
412423
"""
@@ -419,7 +430,7 @@ def validate_and_round_vhd_size(size):
419430
if size < MIN_VHD_SIZE:
420431
size = MIN_VHD_SIZE
421432

422-
size = util.roundup(VHD_BLOCK_SIZE, size)
433+
size = util.roundup(block_size, size)
423434

424435
return size
425436

‎tests/test_FileSR.py‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -327,6 +327,7 @@ def test_create_vdi_vhd(self, mock_vhdutil):
327327
vdi = FakeFileVDI(sr, vdi_uuid)
328328
vdi.vdi_type = vhdutil.VDI_TYPE_VHD
329329
mock_vhdutil.validate_and_round_vhd_size.side_effect = vhdutil.validate_and_round_vhd_size
330+
mock_vhdutil.DEFAULT_VHD_BLOCK_SIZE = vhdutil.DEFAULT_VHD_BLOCK_SIZE
330331

331332
# Act
332333
vdi.create(sr_uuid, vdi_uuid, 20 * 1024 * 1024)

‎tests/test_LVHDSR.py‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -307,9 +307,13 @@ def setUp(self):
307307
self.mock_lvhdutil.LV_PREFIX = lvhdutil.LV_PREFIX
308308
vhdutil_patcher = mock.patch('sm.drivers.LVHDSR.vhdutil', autospec=True)
309309
self.mock_vhdutil = vhdutil_patcher.start()
310+
self.mock_vhdutil.getBlockSize.return_value = vhdutil.DEFAULT_VHD_BLOCK_SIZE
310311
self.mock_vhdutil.VDI_TYPE_VHD = vhdutil.VDI_TYPE_VHD
311312
self.mock_vhdutil.VDI_TYPE_RAW = vhdutil.VDI_TYPE_RAW
312313
self.mock_vhdutil.MAX_CHAIN_SIZE = vhdutil.MAX_CHAIN_SIZE
314+
vdi_vhdutil_patcher = mock.patch('sm.VDI.vhdutil', autospec=True)
315+
self.mock_vdi_vhdutil = vdi_vhdutil_patcher.start()
316+
self.mock_vdi_vhdutil.getBlockSize.return_value = vhdutil.DEFAULT_VHD_BLOCK_SIZE
313317
lvutil_patcher = mock.patch('sm.drivers.LVHDSR.lvutil', autospec=True)
314318
self.mock_lvutil = lvutil_patcher.start()
315319
vdi_util_patcher = mock.patch('sm.VDI.util', autospec=True)

0 commit comments

Comments
 (0)