1# -*- coding: utf-8 -*- 2# 3# Copyright 2013 Ingo Weinhold 4# Distributed under the terms of the MIT License. 5 6# -- Modules ------------------------------------------------------------------ 7 8import os 9import re 10import subprocess 11 12from .Options import getOption 13from .PackageInfo import PackageInfo, ResolvableExpression 14from .ProvidesManager import ProvidesManager 15from .ShellScriptlets import getScriptletPrerequirements 16from .Utils import printError, sysExit, warn 17 18 19class RestartDependencyResolutionException(Exception): 20 def __init__(self, packageNode, message): 21 Exception.__init__(self) 22 self.packageNode = packageNode 23 self.message = message 24 25# -- PackageNode class -------------------------------------------------------- 26 27class PackageNode(object): 28 def __init__(self, packageInfo, isBuildhostPackage): 29 self.packageInfo = packageInfo 30 self.realPath = os.path.realpath(packageInfo.path) 31 self.isBuildhostPackage = isBuildhostPackage 32 self.dependencyCount = 0 33 34 def __eq__(self, other): 35 return (self.packageInfo.name == other.packageInfo.name 36 and self.packageInfo.version == other.packageInfo.version 37 and self.realPath == other.realPath 38 and self.isBuildhostPackage == other.isBuildhostPackage) 39 40 def __str__(self): 41 return '%s-%s :: %s ::%s' % (self.packageInfo.name, 42 self.packageInfo.version, 43 self.realPath, 44 self.isBuildhostPackage) 45 46 @property 47 def versionedName(self): 48 return self.packageInfo.versionedName 49 50 @property 51 def path(self): 52 return self.packageInfo.path 53 54 def bumpDependencyCount(self): 55 self.dependencyCount += 1 56 57# -- DependencyResolver class ---------------------------------------------------- 58 59class DependencyResolver(object): 60 61 packageInfoCache = {} 62 63 def __init__(self, buildPlatform, requiresTypes, repositories, **kwargs): 64 self._providesManager = ProvidesManager() 65 self._platform = buildPlatform 66 self._requiresTypes = requiresTypes 67 self._repositories = repositories 68 self._stopAtHpkgs = kwargs.get('stopAtHpkgs', False) 69 self._ignoreBase = kwargs.get('ignoreBase', False) 70 self._presentDependencyPackages = kwargs.get( 71 'presentDependencyPackages', None) 72 self._quiet = kwargs.get('quiet', False) 73 self._updateDependencies = getOption('updateDependencies') \ 74 and len(repositories) > 2 75 self._satisfiedPackagesCache = [] 76 77 self._populateProvidesManager() 78 79 def determineRequiredPackagesFor(self, dependencyInfoFiles): 80 packageInfos = [ 81 self._parsePackageInfo(dif, True) for dif in dependencyInfoFiles 82 ] 83 84 errorMessages = [] 85 while True: 86 try: 87 self._packageNodes = [] 88 if self._presentDependencyPackages: 89 del self._presentDependencyPackages[:] 90 91 self._pending = [ 92 PackageNode(pi, False) for pi in packageInfos 93 ] 94 95 self._traversed = set([ 96 str(packageNode) for packageNode in self._pending 97 ]) 98 99 self._buildDependencyGraph() 100 break 101 102 except RestartDependencyResolutionException as exception: 103 errorMessages.append(exception.message) 104 if exception.packageNode.packageInfo in packageInfos: 105 # The resolution failure has bubbled to the top, we failed. 106 raise LookupError('\n'.join(errorMessages)) 107 108 self._providesManager.removeProvidesOfPackageInfo( 109 exception.packageNode.packageInfo) 110 continue 111 112 self._sortPackageNodesTopologically() 113 114 result = [ 115 node.path for node in self._packageNodes 116 ] 117 118 self._satisfiedPackagesCache += result 119 return result 120 121 def _populateProvidesManager(self): 122 for repository in self._repositories: 123 for entry in os.listdir(repository): 124 if not (entry.endswith('.DependencyInfo') 125 or entry.endswith('.hpkg') 126 or entry.endswith('.PackageInfo')): 127 continue 128 packageInfo = self._parsePackageInfo(repository + '/' + entry, 129 not entry.endswith('.hpkg')) 130 if packageInfo is None: 131 continue 132 self._providesManager.addProvidesFromPackageInfo(packageInfo) 133 134 def _buildDependencyGraph(self): 135 numberOfInitialPackages = len(self._pending) 136 numberOfHandledPackages = 0 137 while self._pending: 138 packageNode = self._pending.pop(0) 139 140 if 'REQUIRES' in self._requiresTypes: 141 self._addAllImmediateRequiresOf(packageNode) 142 if 'BUILD_REQUIRES' in self._requiresTypes: 143 self._addAllImmediateBuildRequiresOf(packageNode) 144 if 'BUILD_PREREQUIRES' in self._requiresTypes: 145 self._addAllImmediateBuildPrerequiresOf(packageNode) 146 if 'TEST_REQUIRES' in self._requiresTypes: 147 self._addAllImmediateTestRequiresOf(packageNode) 148 if 'SCRIPTLET_PREREQUIRES' in self._requiresTypes: 149 self._addScriptletPrerequiresOf(packageNode) 150 151 # when the batch of passed in packages has been handled, we need 152 # to activate the REQUIRES, too, since these are needed to run 153 # all the following packages 154 numberOfHandledPackages += 1 155 if (numberOfHandledPackages == numberOfInitialPackages 156 and 'REQUIRES' not in self._requiresTypes): 157 self._requiresTypes.append('REQUIRES') 158 159 def _sortPackageNodesTopologically(self): 160 sortedPackageNodes = [] 161 while self._packageNodes: 162 lowestDependencyCount = 1000000 163 nodesWithLowestDependencyCount = [] 164 for node in self._packageNodes: 165 if lowestDependencyCount > node.dependencyCount: 166 lowestDependencyCount = node.dependencyCount 167 nodesWithLowestDependencyCount = [node] 168 elif lowestDependencyCount == node.dependencyCount: 169 nodesWithLowestDependencyCount.append(node) 170 171 sortedPackageNodes += nodesWithLowestDependencyCount 172 self._packageNodes = [ 173 node for node in self._packageNodes 174 if node not in nodesWithLowestDependencyCount 175 ] 176 177 self._packageNodes = sortedPackageNodes 178 179 def _addAllImmediateRequiresOf(self, requiredPackageInfo): 180 packageInfo = requiredPackageInfo.packageInfo 181 forBuildhost = requiredPackageInfo.isBuildhostPackage 182 183 for requires in packageInfo.requires: 184 self._addImmediate(requiredPackageInfo, requires, 'requires', 185 forBuildhost) 186 187 def _addAllImmediateBuildRequiresOf(self, requiredPackageInfo): 188 packageInfo = requiredPackageInfo.packageInfo 189 forBuildhost = requiredPackageInfo.isBuildhostPackage 190 191 for requires in packageInfo.buildRequires: 192 self._addImmediate(requiredPackageInfo, requires, 'build-requires', 193 forBuildhost) 194 195 def _addAllImmediateBuildPrerequiresOf(self, requiredPackageInfo): 196 packageInfo = requiredPackageInfo.packageInfo 197 198 for requires in packageInfo.buildPrerequires: 199 self._addImmediate(requiredPackageInfo, requires, 200 'build-prerequires', True) 201 202 def _addAllImmediateTestRequiresOf(self, requiredPackageInfo): 203 packageInfo = requiredPackageInfo.packageInfo 204 205 for requires in packageInfo.testRequires: 206 self._addImmediate(requiredPackageInfo, requires, 207 'test-requires', False) 208 209 def _addScriptletPrerequiresOf(self, requiredPackageInfo): 210 scriptletPrerequirements = getScriptletPrerequirements() 211 for requires in scriptletPrerequirements: 212 self._addImmediate(requiredPackageInfo, 213 ResolvableExpression(requires), 'scriptlet-prerequires', True) 214 215 def _addImmediate(self, parent, requires, typeString, forBuildhost): 216 implicitProvides = [] 217 if self._platform: 218 implicitProvides = self._platform.getImplicitProvides(forBuildhost) 219 220 isImplicit = requires.name in implicitProvides 221 # Skip, if this is one of the implicit provides of the build platform, 222 # unless we are collecting the source packages for the bootstrap, in 223 # case of which we try to add all requires (as in that case the actual 224 # buildhost is Haiku and we need to put the corresponding source 225 # packages onto the bootstrap image). 226 if isImplicit and not getOption('createSourcePackagesForBootstrap'): 227 return 228 229 # if a prerequires type is requested, priorize any hpkg fitting the 230 # version requirements, and not the latest recipe. 231 isPrerequiresType = typeString.endswith('-prerequires') 232 provides = self._providesManager.getMatchingProvides(requires, 233 isPrerequiresType, self._ignoreBase) 234 235 if not provides: 236 if isImplicit: 237 return 238 if getOption('getDependencies'): 239 try: 240 print('Fetching package for ' + str(requires) + ' ...') 241 output = subprocess.check_output(['pkgman', 'install', '-y', 242 str(requires).replace(' ', '')], stderr=subprocess.PIPE).decode('utf-8') 243 for pkg in re.findall(r'://.*/([^/\n]+\.hpkg)', output): 244 pkginfo = PackageInfo('/boot/system/packages/' + pkg) 245 self._providesManager.addProvidesFromPackageInfo(pkginfo) 246 provides = self._providesManager.getMatchingProvides(requires, 247 isPrerequiresType, self._ignoreBase) 248 except subprocess.CalledProcessError as e: 249 # `pkgman install -y` failed, propagate the why. 250 error = e.stderr.decode('utf-8') 251 output = e.output.decode('utf-8').splitlines() 252 for index, line in enumerate(output): 253 if line.startswith('Encountered problems:'): 254 break 255 else: 256 index = -1 257 lines = ['\t' + line for line in output[index + 1:]] 258 lines = '\n'.join(lines) 259 raise RestartDependencyResolutionException(parent, 260 'failed to install package for {}.\n{}'.format(requires, error or lines)) 261 else: 262 message = '%s "%s" of package "%s" could not be resolved' \ 263 % (typeString, str(requires), parent.versionedName) 264 if not self._quiet: 265 printError(message) 266 raise RestartDependencyResolutionException(parent, message) 267 268 if provides.packageInfo.path in self._satisfiedPackagesCache: 269 return 270 271 requiredPackageInfo = PackageNode(provides.packageInfo, forBuildhost) 272 if requiredPackageInfo.path.endswith('.hpkg'): 273 if (self._presentDependencyPackages is not None 274 and requiredPackageInfo.path 275 not in self._presentDependencyPackages): 276 self._presentDependencyPackages.append(requiredPackageInfo.path) 277 278 self._addPackageNode(requiredPackageInfo, not self._stopAtHpkgs) 279 else: 280 parent.bumpDependencyCount() 281 self._addPackageNode(requiredPackageInfo, True) 282 283 def _addPackageNode(self, requiredPackageInfo, addToPending): 284 if str(requiredPackageInfo) not in self._traversed: 285 self._traversed.add(str(requiredPackageInfo)) 286 self._packageNodes.append(requiredPackageInfo) 287 if addToPending: 288 self._pending.append(requiredPackageInfo) 289 290 def _parsePackageInfo(self, packageInfoFile, fatal): 291 if packageInfoFile in DependencyResolver.packageInfoCache: 292 return DependencyResolver.packageInfoCache[packageInfoFile] 293 294 try: 295 packageInfo = PackageInfo(packageInfoFile) 296 DependencyResolver.packageInfoCache[packageInfoFile] = packageInfo 297 except subprocess.CalledProcessError: 298 message = 'failed to parse "%s"' % packageInfoFile 299 sysExit(message) if fatal else warn(message) 300 return None 301 302 return packageInfo 303