1# -*- coding: utf-8 -*- 2# 3# Copyright 2013 Oliver Tappe 4# Distributed under the terms of the MIT License. 5 6# -- Modules ------------------------------------------------------------------ 7 8import codecs 9import glob 10import json 11import os 12import re 13import shutil 14from functools import cmp_to_key 15from subprocess import check_call, check_output 16from textwrap import dedent 17 18from .Configuration import Configuration 19from .DependencyResolver import DependencyResolver 20from .Options import getOption 21from .Port import Port 22from .Utils import prefixLines, sysExit, touchFile, versionCompare, warn 23 24# -- Repository class --------------------------------------------------------- 25 26class Repository(object): 27 28 currentFormatVersion = 2 29 30 def __init__(self, treePath, outputDirectory, repositoryPath, 31 packagesPath, shellVariables, 32 policy, preserveFlags, quiet=False, verbose=False): 33 self.treePath = treePath 34 self.outputDirectory = outputDirectory 35 self.path = repositoryPath 36 self.inputSourcePackagesPath \ 37 = self.outputDirectory + '/input-source-packages' 38 self.packagesPath = packagesPath 39 self.shellVariables = shellVariables 40 self.policy = policy 41 self.quiet = quiet 42 self.verbose = verbose 43 44 self._formatVersionFilePath = self.path + '/.formatVersion' 45 self._portIdForPackageIdFilePath \ 46 = self.path + '/.portIdForPackageIdMap' 47 self._portNameForPackageNameFilePath \ 48 = self.path + '/.portNameForPackageNameMap' 49 50 # check repository format 51 formatVersion = self._readFormatVersion() 52 if formatVersion > Repository.currentFormatVersion: 53 sysExit('The version of the repository format used in\n\t%s' 54 '\nis newer than the one supported by haikuporter.\n' 55 'Please upgrade haikuporter.' % self.path) 56 57 Port.setRepositoryDir(self.path) 58 59 # update repository if it exists and isn't empty, populate it otherwise 60 self._initAllPorts() 61 self._initPortForPackageMaps() 62 if (os.path.isdir(self.path) and os.listdir(self.path) 63 and os.path.exists(self._portIdForPackageIdFilePath) 64 and os.path.exists(self._portNameForPackageNameFilePath) 65 and formatVersion == Repository.currentFormatVersion): 66 if not getOption('noRepositoryUpdate'): 67 self._updateRepository() 68 else: 69 if getOption('noRepositoryUpdate'): 70 sysExit('no or outdated repository found but no update ' 71 'allowed') 72 if formatVersion < Repository.currentFormatVersion: 73 warn('Found old repository format - repopulating the ' 74 'repository ...') 75 self._populateRepository(preserveFlags) 76 self._writeFormatVersion() 77 self._writePortForPackageMaps() 78 self._activePorts = None 79 80 def getPortIdForPackageId(self, packageId): 81 """return the port-ID for the given package-ID""" 82 83 return self._portIdForPackageId.get(packageId, None) 84 85 def getPortNameForPackageName(self, packageName): 86 """return the port name for the given package name""" 87 88 return self._portNameForPackageName.get(packageName, None) 89 90 @property 91 def allPorts(self): 92 return self._allPorts 93 94 @property 95 def activePorts(self): 96 if self._activePorts is not None: 97 return self._activePorts 98 99 self._activePorts = [] 100 for portName in self._portVersionsByName.keys(): 101 activePortVersion = self.getActiveVersionOf(portName) 102 if not activePortVersion: 103 continue 104 self._activePorts.append( 105 self._allPorts[portName + '-' + activePortVersion]) 106 107 return self._activePorts 108 109 @property 110 def portVersionsByName(self): 111 return self._portVersionsByName 112 113 def getActiveVersionOf(self, portName, warnAboutSkippedVersions=False): 114 """return the highest buildable version of the port with the given 115 name""" 116 if portName not in self._portVersionsByName: 117 return None 118 119 versions = self._portVersionsByName[portName] 120 for version in reversed(versions): 121 portID = portName + '-' + version 122 port = self._allPorts[portID] 123 if port.hasBrokenRecipe: 124 if warnAboutSkippedVersions: 125 warn('skipping %s, as the recipe is broken' % portID) 126 try: 127 port.parseRecipeFileRaisingExceptions(True) 128 except SystemExit as e: 129 print(e.code) 130 continue 131 if not port.isBuildableOnTargetArchitecture(): 132 if warnAboutSkippedVersions: 133 status = port.statusOnTargetArchitecture 134 warn(('skipping %s, as it is %s on the target ' 135 + 'architecture.') % (portID, status)) 136 continue 137 return version 138 139 return None 140 141 def getActivePort(self, portName): 142 """return the highest buildable version of the port with the given 143 name""" 144 version = self._portNameVersionForPortName(portName) 145 if version is None: 146 return None 147 148 return self._allPorts[version] 149 150 def _portNameVersionForPortName(self, portName): 151 portVersion = self.getActiveVersionOf(portName) 152 if not portVersion: 153 return None 154 return portName + '-' + portVersion 155 156 def searchPorts(self, regExp, returnPortNameVersions=False): 157 """Search for one or more ports in the HaikuPorts tree, returning 158 a list of found matches""" 159 if regExp: 160 if getOption('literalSearchStrings'): 161 regExp = re.escape(regExp) 162 reSearch = re.compile(regExp) 163 164 ports = [] 165 portNames = self.portVersionsByName.keys() 166 for portName in portNames: 167 if not regExp or reSearch.search(portName): 168 if returnPortNameVersions: 169 portNameVersion = self._portNameVersionForPortName(portName) 170 if portNameVersion is not None: 171 ports.append(portNameVersion) 172 else: 173 ports.append(portName) 174 175 return sorted(ports) 176 177 def _fileNameForPackageName(self, packageName): 178 portName = self._portNameForPackageName[packageName] 179 portVersion = self.getActiveVersionOf(portName) 180 if not portVersion: 181 return None 182 183 port = self._allPorts[portName + '-' + portVersion] 184 for package in port.packages: 185 if package.name == packageName: 186 return package.hpkgName 187 188 def searchPackages(self, regExp, returnFileNames=True): 189 """Search for one or more packages in the HaikuPorts tree, returning 190 a list of found matches""" 191 if regExp: 192 if getOption('literalSearchStrings'): 193 regExp = re.escape(regExp) 194 reSearch = re.compile(regExp) 195 196 packages = [] 197 packageNames = self._portNameForPackageName.keys() 198 for packageName in packageNames: 199 if not regExp or reSearch.search(packageName): 200 if returnFileNames: 201 packageName = self._fileNameForPackageName(packageName) 202 if not packageName: 203 continue 204 205 packages.append(packageName) 206 207 return sorted(packages) 208 209 def _initAllPorts(self): 210 # Collect all ports into a dictionary that can be keyed by 211 # name + '-' + version. Additionally, we keep a sorted list of 212 # available versions for each port name. 213 self._allPorts = {} 214 self._portVersionsByName = {} 215 216 ## REFACTOR into separate methods 217 218 # every existing input source package defines a port (which overrules 219 # any corresponding port in the recipe tree) 220 if os.path.exists(self.inputSourcePackagesPath): 221 for fileName in sorted(os.listdir(self.inputSourcePackagesPath)): 222 if not ('_source-' in fileName 223 or '_source_rigged-' in fileName): 224 continue 225 226 recipeFilePath \ 227 = self._partiallyExtractSourcePackageIfNeeded(fileName) 228 229 recipeName = os.path.basename(recipeFilePath) 230 name, version = recipeName[:-7].split('-') 231 if name not in self._portVersionsByName: 232 self._portVersionsByName[name] = [version] 233 else: 234 self._portVersionsByName[name].append(version) 235 236 portPath = os.path.dirname(recipeFilePath) 237 if self.outputDirectory == self.treePath: 238 portOutputPath = portPath 239 else: 240 portOutputPath = self.outputDirectory \ 241 + '/input-source-packages/' + name 242 self._allPorts[name + '-' + version] \ 243 = Port(name, version, '<source-package>', portPath, 244 portOutputPath, self.shellVariables, 245 self.policy) 246 247 # collect ports from the recipe tree 248 for category in sorted(os.listdir(self.treePath)): 249 categoryPath = self.treePath + '/' + category 250 if (not os.path.isdir(categoryPath) or category[0] == '.' 251 or '-' not in category): 252 continue 253 for port in sorted(os.listdir(categoryPath)): 254 portPath = categoryPath + '/' + port 255 portOutputPath = (self.outputDirectory + '/' + category + '/' 256 + port) 257 if not os.path.isdir(portPath) or port[0] == '.': 258 continue 259 for recipe in os.listdir(portPath): 260 recipePath = portPath + '/' + recipe 261 if (not os.path.isfile(recipePath) 262 or not recipe.endswith('.recipe')): 263 continue 264 portElements = recipe[:-7].split('-') 265 if len(portElements) == 2: 266 name, version = portElements 267 versionedName = name + '-' + version 268 if versionedName in self._allPorts: 269 # this version of the current port already was 270 # defined - skip 271 if not self.quiet and not getOption('doBootstrap'): 272 otherPort = self._allPorts[versionedName] 273 if otherPort.category == '<source-package>': 274 warn('%s/%s is overruled by input ' 275 'source package' % (category, 276 versionedName)) 277 else: 278 warn('%s/%s is overruled by duplicate ' 279 'in %s - please remove one of them' 280 % (category, versionedName, 281 otherPort.category)) 282 continue 283 if name not in self._portVersionsByName: 284 self._portVersionsByName[name] = [version] 285 else: 286 self._portVersionsByName[name].append(version) 287 self._allPorts[name + '-' + version] = Port(name, 288 version, category, portPath, portOutputPath, 289 self.shellVariables, self.policy) 290 else: 291 # invalid argument 292 if not self.quiet: 293 print("Warning: Couldn't parse port/version info: " 294 + recipe) 295 296 # Create ports for the secondary architectures. Not all make sense or 297 # are supported, but we won't know until we have parsed the recipe file. 298 secondaryArchitectures = Configuration.getSecondaryTargetArchitectures() 299 if secondaryArchitectures: 300 for port in tuple(self._allPorts.values()): 301 for architecture in secondaryArchitectures: 302 newPort = Port(port.baseName, port.version, port.category, 303 port.baseDir, port.outputDir, self.shellVariables, 304 port.policy, architecture) 305 self._allPorts[newPort.versionedName] = newPort 306 307 name = newPort.name 308 version = newPort.version 309 if name not in self._portVersionsByName: 310 self._portVersionsByName[name] = [version] 311 else: 312 self._portVersionsByName[name].append(version) 313 314 # Sort version list of each port 315 for portName in list(self._portVersionsByName.keys()): 316 self._portVersionsByName[portName].sort( 317 key=cmp_to_key(versionCompare)) 318 319 def _initPortForPackageMaps(self): 320 """Initialize dictionaries that map package names/IDs to port 321 names/IDs""" 322 323 self._portIdForPackageId = {} 324 if os.path.exists(self._portIdForPackageIdFilePath): 325 try: 326 with open(self._portIdForPackageIdFilePath, 'r') as fh: 327 self._portIdForPackageId = json.load(fh) 328 except BaseException as e: 329 print(e) 330 331 self._portNameForPackageName = {} 332 if os.path.exists(self._portNameForPackageNameFilePath): 333 try: 334 with open(self._portNameForPackageNameFilePath, 'r') as fh: 335 self._portNameForPackageName = json.load(fh) 336 except BaseException as e: 337 print(e) 338 339 def _writePortForPackageMaps(self): 340 """Writes dictionaries that map package names/IDs to port 341 names/IDs to a file""" 342 343 try: 344 with open(self._portIdForPackageIdFilePath, 'w') as fh: 345 json.dump(self._portIdForPackageId, fh, sort_keys=True, 346 indent=4, separators=(',', ' : ')) 347 except BaseException as e: 348 print(e) 349 350 try: 351 with open(self._portNameForPackageNameFilePath, 'w') as fh: 352 json.dump(self._portNameForPackageName, fh, sort_keys=True, 353 indent=4, separators=(',', ' : ')) 354 except BaseException as e: 355 print(e) 356 357 def _readFormatVersion(self): 358 """Read format version of repository from file""" 359 360 formatVersion = 0 361 if os.path.exists(self._formatVersionFilePath): 362 try: 363 with open(self._formatVersionFilePath, 'r') as fh: 364 data = json.load(fh) 365 formatVersion = data.get('formatVersion', 0) 366 except BaseException as e: 367 print(e) 368 return formatVersion 369 370 def _writeFormatVersion(self): 371 """Writes the version of the repository format into a file""" 372 373 try: 374 data = { 375 'formatVersion': Repository.currentFormatVersion 376 } 377 with open(self._formatVersionFilePath, 'w') as fh: 378 json.dump(data, fh, indent=4, separators=(',', ' : ')) 379 except BaseException as e: 380 print(e) 381 382 def _populateRepository(self, preserveFlags): 383 """Remove and refill the repository with all DependencyInfo-files from 384 parseable recipes""" 385 386 if os.path.exists(self.path): 387 shutil.rmtree(self.path) 388 389 self._portNameForPackageName = {} 390 self._portIdForPackageId = {} 391 self._updateRepository(None, preserveFlags) 392 return 393 394 def supportBackwardsCompatibility(self, name, version): 395 self._updateRepository({'name': name, 'version': version}) 396 397 def _updateRepository(self, explicitPortVersion=None, preserveFlags=True): 398 """Update all DependencyInfo-files in the repository as needed""" 399 400 allPorts = self.allPorts 401 402 activePorts = [] 403 updatedPorts = {} 404 405 # check for all known ports if their recipe has been changed 406 if os.path.exists(self.path): 407 if not self.quiet: 408 print('Checking if any dependency-infos need to be updated ...') 409 else: 410 os.makedirs(self.path) 411 if not self.quiet: 412 print('Populating repository ...') 413 414 skippedDir = os.path.join(self.path, '.skipped') 415 if not os.path.exists(skippedDir): 416 os.mkdir(skippedDir) 417 418 for portName in sorted(self._portVersionsByName.keys(), 419 key=str.lower): 420 421 if explicitPortVersion and explicitPortVersion['name'] == portName: 422 versions = [explicitPortVersion['version']] 423 forceAllowUnstable = True 424 else: 425 versions = reversed(self._portVersionsByName[portName]) 426 forceAllowUnstable = False 427 428 for version in versions: 429 portID = portName + '-' + version 430 port = allPorts[portID] 431 skippedFlag = os.path.join(skippedDir, portID) 432 433 # ignore recipes that were skipped last time unless they've 434 # been changed since then 435 if (os.path.exists(skippedFlag) 436 and (os.path.getmtime(port.recipeFilePath) 437 <= os.path.getmtime(skippedFlag))): 438 continue 439 440 # update all dependency-infos of port if the recipe is newer 441 # than the main dependency-info of that port 442 mainDependencyInfoFile = os.path.join(self.path, 443 port.dependencyInfoName) 444 if (os.path.exists(mainDependencyInfoFile) 445 and (os.path.getmtime(port.recipeFilePath) 446 <= os.path.getmtime(mainDependencyInfoFile))): 447 activePorts.append(portID) 448 break 449 450 # try to parse updated recipe 451 try: 452 port.parseRecipeFile(False, forceAllowUnstable, 453 forceAllowUnstable) 454 455 if not port.isBuildableOnTargetArchitecture( 456 forceAllowUnstable): 457 touchFile(skippedFlag) 458 if not self.quiet: 459 status = port.statusOnTargetArchitecture 460 print(('\t%s is still marked as %s on target ' 461 + 'architecture') % (portID, status)) 462 continue 463 464 if os.path.exists(skippedFlag): 465 os.remove(skippedFlag) 466 467 if not preserveFlags and port.checkFlag('build'): 468 if not self.quiet: 469 print('\t[build-flag reset]') 470 port.unsetFlag('build') 471 472 if not self.quiet: 473 print('\tupdating dependency infos of ' + portID) 474 475 port.writeDependencyInfosIntoRepository() 476 updatedPorts[portID] = port 477 break 478 479 except SystemExit as e: 480 # take notice of broken recipe file 481 touchFile(skippedFlag) 482 if not os.path.exists(mainDependencyInfoFile): 483 if not self.quiet: 484 print('\trecipe for %s is still broken:' % portID) 485 print(prefixLines('\t', e.code)) 486 487 # This also drops mappings for updated ports to remove any possibly 488 # removed sub-packages. 489 self._removeStalePortForPackageMappings(activePorts) 490 491 # Add port for package mappings for updated ports. 492 for portID, port in updatedPorts.items(): 493 for package in port.packages: 494 self._portIdForPackageId[package.versionedName] \ 495 = port.versionedName 496 self._portNameForPackageName[package.name] \ 497 = port.name 498 activePorts.append(portID) 499 500 # Note that removing stale dependency infos uses the port for package 501 # mappings to determine what to keep. This step must therefore come 502 # after the stale port for package mapping removal. 503 self._removeStaleDependencyInfos(activePorts) 504 505 def _removeStaleDependencyInfos(self, activePorts): 506 """check for any dependency-infos that no longer have a corresponding 507 recipe file""" 508 509 allPorts = self.allPorts 510 511 if not self.quiet: 512 print("Looking for stale dependency-infos ...") 513 dependencyInfos = glob.glob(self.path + '/*.DependencyInfo') 514 for dependencyInfo in dependencyInfos: 515 dependencyInfoFileName = os.path.basename(dependencyInfo) 516 packageID \ 517 = dependencyInfoFileName[:dependencyInfoFileName.rindex('.')] 518 portID = self.getPortIdForPackageId(packageID) 519 520 if not portID or portID not in activePorts: 521 if not self.quiet: 522 print('\tremoving ' + dependencyInfoFileName) 523 os.remove(dependencyInfo) 524 525 if not getOption('noPackageObsoletion'): 526 # obsolete corresponding package, if any 527 self._removePackagesForDependencyInfo(dependencyInfo) 528 529 def _removePackagesForDependencyInfo(self, dependencyInfo): 530 """remove all packages for the given dependency-info""" 531 532 (packageSpec, _) = os.path.basename(dependencyInfo).rsplit('.', 1) 533 packages = glob.glob(self.packagesPath + '/' + packageSpec + '-*.hpkg') 534 obsoleteDir = self.packagesPath + '/.obsolete' 535 for package in packages: 536 packageFileName = os.path.basename(package) 537 if not self.quiet: 538 print('\tobsoleting package ' + packageFileName) 539 obsoletePackage = obsoleteDir + '/' + packageFileName 540 if not os.path.exists(obsoleteDir): 541 os.mkdir(obsoleteDir) 542 os.rename(package, obsoletePackage) 543 544 def _removeStalePortForPackageMappings(self, activePorts): 545 """drops any port-for-package mappings that refer to non-existing or 546 broken ports""" 547 548 for packageId, portId in tuple(self._portIdForPackageId.items()): 549 if portId not in activePorts: 550 del self._portIdForPackageId[packageId] 551 552 for packageName, portName in tuple(self._portNameForPackageName.items()): 553 if portName not in self._portVersionsByName: 554 del self._portNameForPackageName[packageName] 555 continue 556 557 for version in self._portVersionsByName[portName]: 558 portId = portName + '-' + version 559 if portId in activePorts: 560 break 561 else: 562 # no version exists of this port that is not broken 563 del self._portNameForPackageName[packageName] 564 565 def _partiallyExtractSourcePackageIfNeeded(self, sourcePackageName): 566 """extract the recipe and potentially contained patches/licenses from 567 a source package, unless that has already been done""" 568 569 sourcePackagePath \ 570 = self.inputSourcePackagesPath + '/' + sourcePackageName 571 (name, version, revision, _) = sourcePackageName.split('-') 572 # determine port name by dropping '_source' or '_source_rigged' 573 if name.endswith('_source_rigged'): 574 name = name[:-14] 575 elif name.endswith('_source'): 576 name = name[:-7] 577 relativeBasePath \ 578 = 'develop/sources/%s-%s-%s' % (name, version, revision) 579 recipeName = name + '-' + version + '.recipe' 580 recipeFilePath = (self.inputSourcePackagesPath + '/' + relativeBasePath 581 + '/' + recipeName) 582 583 if (not os.path.exists(recipeFilePath) 584 or (os.path.getmtime(recipeFilePath) 585 <= os.path.getmtime(sourcePackagePath))): 586 # extract recipe, patches and licenses (but skip everything else) 587 allowedEntries = [ 588 relativeBasePath + '/' + recipeName, 589 relativeBasePath + '/additional-files', 590 relativeBasePath + '/licenses', 591 relativeBasePath + '/patches', 592 ] 593 entries = check_output([Configuration.getPackageCommand(), 'list', 594 '-p', sourcePackagePath]).decode('utf-8').splitlines() 595 entries = [ 596 entry for entry in entries if entry in allowedEntries 597 ] 598 check_call([Configuration.getPackageCommand(), 'extract', 599 '-C', self.inputSourcePackagesPath, sourcePackagePath] 600 + entries) 601 602 # override all SOURCE_URIs in recipe to point to the source package 603 textToAdd = dedent(r''' 604 # Added by haikuporter: 605 SOURCE_URI='pkg:%s' 606 for i in {2..1000}; do 607 eval currentSrcUri=\$SOURCE_URI_$i 608 if [ -z "$currentSrcUri" ]; then 609 break 610 fi 611 eval SOURCE_URI_$i="$SOURCE_URI" 612 done 613 for i in {001..999}; do 614 eval currentSrcUri=\$SOURCE_URI_$i 615 if [ -z "$currentSrcUri" ]; then 616 break 617 fi 618 eval SOURCE_URI_$i="$SOURCE_URI" 619 done 620 '''[1:]) % sourcePackagePath 621 with codecs.open(recipeFilePath, 'a', 'utf-8') as recipeFile: 622 recipeFile.write('\n' + textToAdd) 623 624 return recipeFilePath 625 626 def checkRepositoryConsistency(self, verbose): 627 """Check consistency of the repository by dependency solving all 628 dependency infos.""" 629 630 repositories = [self.path] 631 systemPackagesDirectory = getOption('systemPackagesDirectory') 632 if systemPackagesDirectory: 633 repositories.append(systemPackagesDirectory) 634 635 resolver = DependencyResolver(None, Port.requiresTypes, repositories, 636 quiet=True) 637 638 for port in sorted(self.activePorts, key=lambda port: port.name): 639 for package in port.packages: 640 if verbose: 641 print('checking package {} of {}'.format( 642 package.revisionedName, port.versionedName)) 643 644 try: 645 resolver.determineRequiredPackagesFor( 646 [package.dependencyInfoFile(self.path)]) 647 except LookupError as error: 648 print('{}:\n{}\n'.format(package.revisionedName, 649 prefixLines('\t', str(error)))) 650