VMBuilder.disk

1 # 2 # Uncomplicated VM Builder 3 # Copyright (C) 2007-2010 Canonical Ltd. 4 # 5 # See AUTHORS for list of contributors 6 # 7 # This program is free software: you can redistribute it and/or modify 8 # it under the terms of the GNU General Public License version 3, as 9 # published by the Free Software Foundation. 10 # 11 # This program is distributed in the hope that it will be useful, 12 # but WITHOUT ANY WARRANTY; without even the implied warranty of 13 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 14 # GNU General Public License for more details. 15 # 16 # You should have received a copy of the GNU General Public License 17 # along with this program. If not, see <http://www.gnu.org/licenses/>. 18 # 19 # Virtual disk management 20 21 import fcntl 22 import logging 23 import os 24 import os.path 25 import re 26 import stat 27 import string 28 import time 29 from VMBuilder.util import run_cmd 30 from VMBuilder.exception import VMBuilderUserError, VMBuilderException 31 from struct import unpack 32 33 TYPE_EXT2 = 0 34 TYPE_EXT3 = 1 35 TYPE_XFS = 2 36 TYPE_SWAP = 3 37 TYPE_EXT4 = 4 38

39 -class Disk(object):

40 """ 41 Virtual disk. 42 43 @type vm: Hypervisor 44 @param vm: The Hypervisor to which the disk belongs 45 @type filename: string 46 @param filename: filename of the disk image 47 @type size: string or number 48 @param size: The size of the disk image to create (passed to 49 L{parse_size}). If specified and filename already exists, 50 L{VMBuilderUserError} will be raised. Otherwise, a disk image of 51 this size will be created once L{create}() is called. 52 """ 53

54 - def __init__(self, vm, filename, size=None):

55 self.vm = vm 56 "The hypervisor to which the disk belongs." 57 58 self.filename = filename 59 "The filename of the disk image." 60 61 self.partitions = [] 62 "The list of partitions on the disk. Is kept in order by L{add_part}." 63 64 self.preallocated = False 65 "Whether the file existed already (True if it did, False if we had to create it)." 66 67 self.size = 0 68 "The size of the disk. For preallocated disks, this is detected." 69 70 if not os.path.exists(self.filename): 71 if not size: 72 raise VMBuilderUserError('%s does not exist, but no size was given.' % (self.filename)) 73 self.size = parse_size(size) 74 else: 75 if size: 76 raise VMBuilderUserError('%s exists, but size was given.' % (self.filename)) 77 self.preallocated = True 78 self.size = detect_size(self.filename) 79 80 self.format_type = None 81 "The format type of the disks. Only used for converted disks."

82

83 - def devletters(self):

84 """ 85 @rtype: string 86 @return: the series of letters that ought to correspond to the device inside 87 the VM. E.g. the first disk of a VM would return 'a', while the 702nd would return 'zz' 88 """ 89 90 return index_to_devname(self.vm.disks.index(self))

91

92 - def create(self):

93 """ 94 Creates the disk image (if it doesn't already exist). 95 96 Once this method returns succesfully, L{filename} can be 97 expected to points to point to whatever holds the virtual disk 98 (be it a file, partition, logical volume, etc.). 99 """ 100 if not os.path.exists(self.filename): 101 logging.info('Creating disk image: "%s" of size: %dMB' % (self.filename, self.size)) 102 run_cmd(qemu_img_path(), 'create', '-f', 'raw', self.filename, '%dM' % self.size)

103

104 - def partition(self):

105 """ 106 Partitions the disk image. First adds a partition table and then 107 adds the individual partitions. 108 109 Should only be called once and only after you've added all partitions. 110 """ 111 112 logging.info('Adding partition table to disk image: %s' % self.filename) 113 run_cmd('parted', '--script', self.filename, 'mklabel', 'msdos') 114 115 # Partition the disk 116 for part in self.partitions: 117 part.create(self)

118

119 - def map_partitions(self):

120 """ 121 Create loop devices corresponding to the partitions. 122 123 Once this has returned succesfully, each partition's map device 124 is set as its L{filename<Disk.Partition.filename>} attribute. 125 126 Call this after L{partition}. 127 """ 128 logging.info('Creating loop devices corresponding to the created partitions') 129 self.vm.add_clean_cb(lambda : self.unmap(ignore_fail=True)) 130 kpartx_output = run_cmd('kpartx', '-av', self.filename) 131 parts = [] 132 for line in kpartx_output.split('\n'): 133 if line == "" or line.startswith("gpt:") or line.startswith("dos:"): 134 continue 135 if line.startswith("add"): 136 parts.append(line) 137 continue 138 logging.error('Skipping unknown line in kpartx output (%s)' % line) 139 mapdevs = [] 140 for line in parts: 141 mapdevs.append(line.split(' ')[2]) 142 for (part, mapdev) in zip(self.partitions, mapdevs): 143 part.set_filename('/dev/mapper/%s' % mapdev)

144

145 - def mkfs(self):

146 """ 147 Creates the partitions' filesystems 148 """ 149 logging.info("Creating file systems") 150 for part in self.partitions: 151 part.mkfs()

152

153 - def get_grub_id(self):

154 """ 155 @rtype: string 156 @return: name of the disk as known by grub 157 """ 158 return '(hd%d)' % self.get_index()

159

160 - def get_index(self):

161 """ 162 @rtype: number 163 @return: index of the disk (starting from 0 for the hypervisor's first disk) 164 """ 165 return self.vm.disks.index(self)

166

167 - def unmap(self, ignore_fail=False):

168 """ 169 Destroy all mapping devices 170 171 Unsets L{Partition}s' and L{Filesystem}s' filename attribute 172 """ 173 # first sleep to give the loopback devices a chance to settle down 174 time.sleep(3) 175 176 tries = 0 177 max_tries = 3 178 while tries < max_tries: 179 try: 180 run_cmd('kpartx', '-d', self.filename, ignore_fail=False) 181 break 182 except: 183 pass 184 tries += 1 185 time.sleep(3) 186 187 if tries >= max_tries: 188 # try it one last time 189 logging.info("Could not unmount '%s' after '%d' attempts. Final attempt" % (self.filename, tries)) 190 run_cmd('kpartx', '-d', self.filename, ignore_fail=ignore_fail) 191 192 for part in self.partitions: 193 logging.debug("Removing partition %s" % part.filename) 194 parted_oldmap=part.filename[len("/dev/mapper/"):-1]+"p"+part.filename[-1] 195 dmsetup_output = run_cmd('dmsetup', 'info', parted_oldmap, ignore_fail=True) 196 for line in dmsetup_output.split('\n'): 197 if line.startswith("State:") and line.endswith("ACTIVE"): 198 logging.debug("Removing parted old map with 'dmsetup remove %s'" % parted_oldmap) 199 dmsetup_output=run_cmd('dmsetup', 'remove', parted_oldmap, ignore_fail=ignore_fail) 200 part.set_filename(None)

201

202 - def add_part(self, begin, length, type, mntpnt):

203 """ 204 Add a partition to the disk 205 206 @type begin: number 207 @param begin: Start offset of the new partition (in megabytes) 208 @type length: 209 @param length: Size of the new partition (in megabytes) 210 @type type: string 211 @param type: Type of the new partition. Valid options are: ext2 ext3 xfs swap linux-swap 212 @type mntpnt: string 213 @param mntpnt: Intended mountpoint inside the guest of the new partition 214 """ 215 length = parse_size(length) 216 end = begin+length-1 217 logging.debug("add_part - begin %d, length %d, end %d, type %s, mntpnt %s" % (begin, length, end, type, mntpnt)) 218 for part in self.partitions: 219 if (begin >= part.begin and begin <= part.end) or \ 220 (end >= part.begin and end <= part.end): 221 raise VMBuilderUserError('Partitions are overlapping') 222 if begin < 0 or end > self.size: 223 raise VMBuilderUserError('Partition is out of bounds. start=%d, end=%d, disksize=%d' % (begin,end,self.size)) 224 part = self.Partition(disk=self, begin=begin, end=end, type=str_to_type(type), mntpnt=mntpnt) 225 self.partitions.append(part) 226 227 # We always keep the partitions in order, so that the output from kpartx matches our understanding 228 self.partitions.sort(cmp=lambda x,y: x.begin - y.begin)

229

230 - def convert(self, destdir, format):

231 """ 232 Convert the disk image 233 234 @type destdir: string 235 @param destdir: Target location of converted disk image 236 @type format: string 237 @param format: The target format (as understood by qemu-img or vdi) 238 @rtype: string 239 @return: the name of the converted image 240 """ 241 if self.preallocated: 242 # We don't convert preallocated disk images. That would be silly. 243 return self.filename 244 245 filename = os.path.basename(self.filename) 246 if '.' in filename: 247 filename = filename[:filename.rindex('.')] 248 destfile = '%s/%s.%s' % (destdir, filename, format) 249 250 logging.info('Converting %s to %s, format %s' % (self.filename, format, destfile)) 251 if format == 'vdi': 252 run_cmd(vbox_manager_path(), 'convertfromraw', '-format', 'VDI', self.filename, destfile) 253 else: 254 run_cmd(qemu_img_path(), 'convert', '-O', format, self.filename, destfile) 255 os.unlink(self.filename) 256 self.filename = os.path.abspath(destfile) 257 self.format_type = format 258 return destfile

259

260 - class Partition(object):

261 - def __init__(self, disk, begin, end, type, mntpnt):

262 self.disk = disk 263 "The disk on which this Partition resides." 264 265 self.begin = begin 266 "The start of the partition" 267 268 self.end = end 269 "The end of the partition" 270 271 self.type = type 272 "The partition type" 273 274 self.mntpnt = mntpnt 275 "The destined mount point" 276 277 self.filename = None 278 "The filename of this partition (the map device)" 279 280 self.fs = Filesystem(vm=self.disk.vm, type=self.type, mntpnt=self.mntpnt) 281 "The enclosed filesystem"

282

283 - def set_filename(self, filename):

284 self.filename = filename 285 self.fs.filename = filename

286

287 - def parted_fstype(self):

288 """ 289 @rtype: string 290 @return: the filesystem type of the partition suitable for passing to parted 291 """ 292 return { TYPE_EXT2: 'ext2', TYPE_EXT3: 'ext2', TYPE_EXT4: 'ext2', TYPE_XFS: 'ext2', TYPE_SWAP: 'linux-swap(new)' }[self.type]

293

294 - def create(self, disk):

295 """Adds partition to the disk image (does not mkfs or anything like that)""" 296 logging.info('Adding type %d partition to disk image: %s' % (self.type, disk.filename)) 297 run_cmd('parted', '--script', '--', disk.filename, 'mkpart', 'primary', self.parted_fstype(), self.begin, self.end)

298

299 - def mkfs(self):

300 """Adds Filesystem object""" 301 self.fs.mkfs()

302

303 - def get_grub_id(self):

304 """The name of the partition as known by grub""" 305 return '(hd%d,%d)' % (self.disk.get_index(), self.get_index())

306

307 - def get_suffix(self):

308 """Returns 'a4' for a device that would be called /dev/sda4 in the guest. 309 This allows other parts of VMBuilder to set the prefix to something suitable.""" 310 return '%s%d' % (self.disk.devletters(), self.get_index() + 1)

311

312 - def get_index(self):

313 """Index of the disk (starting from 0)""" 314 return self.disk.partitions.index(self)

315

316 - def set_type(self, type):

317 try: 318 if int(type) == type: 319 self.type = type 320 else: 321 self.type = str_to_type(type) 322 except ValueError: 323 self.type = str_to_type(type)

324

325 -class Filesystem(object):

326 - def __init__(self, vm=None, size=0, type=None, mntpnt=None, filename=None, devletter='a', device='', dummy=False):

327 self.vm = vm 328 self.filename = filename 329 self.size = parse_size(size) 330 self.devletter = devletter 331 self.device = device 332 self.dummy = dummy 333 334 self.set_type(type) 335 336 self.mntpnt = mntpnt 337 338 self.preallocated = False 339 "Whether the file existed already (True if it did, False if we had to create it)."

340

341 - def create(self):

342 logging.info('Creating filesystem: %s, size: %d, dummy: %s' % (self.mntpnt, self.size, repr(self.dummy))) 343 if not os.path.exists(self.filename): 344 logging.info('Not preallocated, so we create it.') 345 if not self.filename: 346 if self.mntpnt: 347 self.filename = re.sub('[^\w\s/]', '', self.mntpnt).strip().lower() 348 self.filename = re.sub('[\w/]', '_', self.filename) 349 if self.filename == '_': 350 self.filename = 'root' 351 elif self.type == TYPE_SWAP: 352 self.filename = 'swap' 353 else: 354 raise VMBuilderException('mntpnt not set') 355 356 self.filename = '%s/%s' % (self.vm.workdir, self.filename) 357 while os.path.exists('%s.img' % self.filename): 358 self.filename += '_' 359 self.filename += '.img' 360 logging.info('A name wasn\'t specified either, so we make one up: %s' % self.filename) 361 run_cmd(qemu_img_path(), 'create', '-f', 'raw', self.filename, '%dM' % self.size) 362 self.mkfs()

363

364 - def mkfs(self):

365 if not self.filename: 366 raise VMBuilderException('We can\'t mkfs if filename is not set. Did you forget to call .create()?') 367 if not self.dummy: 368 cmd = self.mkfs_fstype() + [self.filename] 369 run_cmd(*cmd) 370 # Let udev have a chance to extract the UUID for us 371 run_cmd('udevadm', 'settle') 372 if os.path.exists("/sbin/vol_id"): 373 self.uuid = run_cmd('vol_id', '--uuid', self.filename).rstrip() 374 elif os.path.exists("/sbin/blkid"): 375 self.uuid = run_cmd('blkid', '-c', '/dev/null', '-sUUID', '-ovalue', self.filename).rstrip()

376

377 - def mkfs_fstype(self):

378 map = { TYPE_EXT2: ['mkfs.ext2', '-F'], TYPE_EXT3: ['mkfs.ext3', '-F'], TYPE_EXT4: ['mkfs.ext4', '-F'], TYPE_XFS: ['mkfs.xfs'], TYPE_SWAP: ['mkswap'] } 379 380 if not self.vm.distro.has_256_bit_inode_ext3_support(): 381 map[TYPE_EXT3] = ['mkfs.ext3', '-I 128', '-F'] 382 383 return map[self.type]

384

385 - def fstab_fstype(self):

386 return { TYPE_EXT2: 'ext2', TYPE_EXT3: 'ext3', TYPE_EXT4: 'ext4', TYPE_XFS: 'xfs', TYPE_SWAP: 'swap' }[self.type]

387

388 - def fstab_options(self):

389 return 'defaults'

390

391 - def mount(self, rootmnt):

392 if (self.type != TYPE_SWAP) and not self.dummy: 393 logging.debug('Mounting %s', self.mntpnt) 394 self.mntpath = '%s%s' % (rootmnt, self.mntpnt) 395 if not os.path.exists(self.mntpath): 396 os.makedirs(self.mntpath) 397 run_cmd('mount', '-o', 'loop', self.filename, self.mntpath) 398 self.vm.add_clean_cb(self.umount)

399

400 - def umount(self):

401 self.vm.cancel_cleanup(self.umount) 402 if (self.type != TYPE_SWAP) and not self.dummy: 403 logging.debug('Unmounting %s', self.mntpath) 404 run_cmd('umount', self.mntpath)

405

406 - def get_suffix(self):

407 """Returns 'a4' for a device that would be called /dev/sda4 in the guest.. 408 This allows other parts of VMBuilder to set the prefix to something suitable.""" 409 if self.device: 410 return self.device 411 else: 412 return '%s%d' % (self.devletters(), self.get_index() + 1)

413

414 - def devletters(self):

415 """ 416 @rtype: string 417 @return: the series of letters that ought to correspond to the device inside 418 the VM. E.g. the first filesystem of a VM would return 'a', while the 702nd would return 'zz' 419 """ 420 return self.devletter

421

422 - def get_index(self):

423 """Index of the disk (starting from 0)""" 424 return self.vm.filesystems.index(self)

425

426 - def set_type(self, type):

427 try: 428 if int(type) == type: 429 self.type = type 430 else: 431 self.type = str_to_type(type) 432 except ValueError: 433 self.type = str_to_type(type)

434

435 -def parse_size(size_str):

436 """Takes a size like qemu-img would accept it and returns the size in MB""" 437 try: 438 return int(size_str) 439 except ValueError: 440 pass 441 442 try: 443 num = int(size_str[:-1]) 444 except ValueError: 445 raise VMBuilderUserError("Invalid size: %s" % size_str) 446 447 if size_str[-1:] == 'g' or size_str[-1:] == 'G': 448 return num * 1024 449 if size_str[-1:] == 'm' or size_str[-1:] == 'M': 450 return num 451 if size_str[-1:] == 'k' or size_str[-1:] == 'K': 452 return num / 1024

453 454 str_to_type_map = { 'ext2': TYPE_EXT2, 455 'ext3': TYPE_EXT3, 456 'ext4': TYPE_EXT4, 457 'xfs': TYPE_XFS, 458 'swap': TYPE_SWAP, 459 'linux-swap': TYPE_SWAP } 460

461 -def str_to_type(type):

462 try: 463 return str_to_type_map[type] 464 except KeyError: 465 raise Exception('Unknown partition type: %s' % type)

466

467 -def rootpart(disks):

468 """Returns the partition which contains the root dir""" 469 return path_to_partition(disks, '/')

470

471 -def bootpart(disks):

472 """Returns the partition which contains /boot""" 473 return path_to_partition(disks, '/boot/foo')

474

475 -def path_to_partition(disks, path):

476 parts = get_ordered_partitions(disks) 477 parts.reverse() 478 for part in parts: 479 if path.startswith(part.mntpnt): 480 return part 481 raise VMBuilderException("Couldn't find partition path %s belongs to" % path)

482

483 -def create_filesystems(vm):

484 for filesystem in vm.filesystems: 485 filesystem.create()

486

487 -def create_partitions(vm):

488 for disk in vm.disks: 489 disk.create(vm.workdir)

490

491 -def get_ordered_filesystems(vm):

492 """Returns filesystems (self hosted as well as contained in partitions 493 in an order suitable for mounting them""" 494 fss = list(vm.filesystems) 495 for disk in vm.disks: 496 fss += [part.fs for part in disk.partitions] 497 fss.sort(lambda x,y: len(x.mntpnt or '')-len(y.mntpnt or '')) 498 return fss

499

500 -def get_ordered_partitions(disks):

501 """Returns partitions from disks in an order suitable for mounting them""" 502 parts = [] 503 for disk in disks: 504 parts += disk.partitions 505 parts.sort(lambda x,y: len(x.mntpnt or '')-len(y.mntpnt or '')) 506 return parts

507

508 -def devname_to_index(devname):

509 return devname_to_index_rec(devname) - 1

510

511 -def devname_to_index_rec(devname):

512 if not devname: 513 return 0 514 return 26 * devname_to_index_rec(devname[:-1]) + (string.ascii_lowercase.index(devname[-1]) + 1)

515

516 -def index_to_devname(index, suffix=''):

517 if index < 0: 518 return suffix 519 return index_to_devname(index / 26 -1, string.ascii_lowercase[index % 26]) + suffix

520

521 -def detect_size(filename):

522 st = os.stat(filename) 523 if stat.S_ISREG(st.st_mode): 524 return st.st_size / 1024*1024 525 elif stat.S_ISBLK(st.st_mode): 526 # I really wish someone would make these available in Python 527 BLKGETSIZE64 = 2148012658 528 fp = open(filename, 'r') 529 fd = fp.fileno() 530 s = fcntl.ioctl(fd, BLKGETSIZE64, ' '*8) 531 return unpack('L', s)[0] / 1024*1024 532 533 raise VMBuilderException('No idea how to find the size of %s' % filename)

534

535 -def qemu_img_path():

536 exes = ['kvm-img', 'qemu-img'] 537 for dir in os.environ['PATH'].split(os.path.pathsep): 538 for exe in exes: 539 path = '%s%s%s' % (dir, os.path.sep, exe) 540 if os.access(path, os.X_OK): 541 return path

542

543 -def vbox_manager_path():

544 exe = 'VBoxManage' 545 for dir in os.environ['PATH'].split(os.path.pathsep): 546 path = '%s%s%s' % (dir, os.path.sep, exe) 547 if os.access(path, os.X_OK): 548 return path

549

Source Code for Module VMBuilder.disk