1# -*- coding: utf-8 -*- 2# 3# Copyright 2013 Ingo Weinhold 4# Distributed under the terms of the MIT License. 5 6# -- Modules ------------------------------------------------------------------ 7 8import glob 9import os 10import re 11from subprocess import check_output 12 13from .ConfigParser import ConfigParser 14from .Configuration import Configuration 15from .Utils import isCommandAvailable, sysExit, warn 16 17allowedWritableTopLevelDirectories = [ 18 'cache', 19 'non-packaged', 20 'settings', 21 'var' 22] 23 24allowedTopLevelEntries = [ 25 '.PackageInfo', 26 'add-ons', 27 'apps', 28 'bin', 29 'boot', 30 'data', 31 'develop', 32 'documentation', 33 'lib', 34 'preferences', 35 'servers' 36] + allowedWritableTopLevelDirectories 37 38# -- Policy checker class ----------------------------------------------------- 39 40class Policy(object): 41 42 # log for policy warnings and errors 43 violationsByPort = {} 44 45 def __init__(self, strict): 46 self.strict = strict 47 48 def setPort(self, port, requiredPackages): 49 self.port = port 50 self.secondaryArchitecture = port.secondaryArchitecture 51 self.secondaryArchSuffix = '_' + self.secondaryArchitecture \ 52 if self.secondaryArchitecture else '' 53 self.secondaryArchSubDir = '/' + self.secondaryArchitecture \ 54 if self.secondaryArchitecture else '' 55 56 # Get the provides of all of the port's packages. We need them to find 57 # dependencies between packages of the port. 58 self.portPackagesProvides = {} 59 for package in port.packages: 60 self.portPackagesProvides[package.name] = ( 61 self._parseResolvableExpressionList( 62 package.recipeKeys['PROVIDES'])) 63 64 # Create a map with the packages' provides. We need that later when 65 # checking the created package. 66 self.requiredPackagesProvides = {} 67 for package in requiredPackages: 68 provides = self._getPackageProvides(package) 69 self.requiredPackagesProvides[os.path.basename(package)] = provides 70 71 def checkPackage(self, package, packageFile): 72 self.package = package 73 self.packageFile = packageFile 74 self.violationEncountered = False 75 76 self.provides = self._parseResolvableExpressionListForKey('PROVIDES') 77 self.requires = self._parseResolvableExpressionListForKey('REQUIRES') 78 79 self._checkTopLevelEntries() 80 self._checkProvides() 81 self._checkLibraryDependencies() 82 self._checkMisplacedDevelopLibraries() 83 self._checkGlobalWritableFiles() 84 self._checkUserSettingsFiles() 85 self._checkPostInstallScripts() 86 87 if self.strict and self.violationEncountered: 88 sysExit("packaging policy violation(s) in strict mode") 89 90 def _checkTopLevelEntries(self): 91 for entry in os.listdir(self.package.packagingDir): 92 if entry not in allowedTopLevelEntries: 93 self._violation('Invalid top-level package entry "%s"' % entry) 94 95 def _parseResolvableExpressionListForKey(self, keyName): 96 return self._parseResolvableExpressionList( 97 self.package.recipeKeys[keyName]) 98 99 def _parseResolvableExpressionList(self, theList): 100 names = set() 101 for item in theList: 102 match = re.match(r'[^-/=!<>\s]+', item) 103 if match: 104 names.add(match.group(0)) 105 return names 106 107 def _checkProvides(self): 108 # check if the package provides itself 109 if self.package.name not in self.provides: 110 self._violation('no matching self provides for "%s"' 111 % self.package.name) 112 113 # everything in bin/ must be declared as cmd:* 114 binDir = os.path.join(self.package.packagingDir, 'bin') 115 if os.path.exists(binDir): 116 for entry in os.listdir(binDir): 117 # ignore secondary architecture subdir 118 if entry == self.package.secondaryArchitecture: 119 continue 120 name = self._normalizeResolvableName('cmd:' + entry) 121 if name.lower() not in self.provides: 122 self._violation('no matching provides "%s" for "%s"' 123 % (name, 'bin/' + entry)) 124 125 # library entries in lib[/<arch>] must be declared as lib:*[_<arch>] 126 libDir = os.path.join(self.package.packagingDir, 127 'lib' + self.secondaryArchSubDir) 128 if os.path.exists(libDir): 129 for entry in os.listdir(libDir): 130 suffixIndex = entry.find('.so') 131 if suffixIndex < 0: 132 continue 133 134 name = self._normalizeResolvableName( 135 'lib:' + entry[:suffixIndex] + self.secondaryArchSuffix) 136 if name.lower() not in self.provides: 137 self._violation('no matching provides "%s" for "%s"' 138 % (name, 'lib/' + entry)) 139 140 # library entries in develop/lib[<arch>] must be declared as 141 # devel:*[_<arch>] 142 developLibDir = os.path.join(self.package.packagingDir, 143 'develop/lib' + self.secondaryArchSubDir) 144 if os.path.exists(developLibDir): 145 for entry in os.listdir(developLibDir): 146 suffixIndex = entry.find('.so') 147 if suffixIndex < 0: 148 suffixIndex = entry.find('.a') 149 if suffixIndex < 0: 150 continue 151 152 name = self._normalizeResolvableName( 153 'devel:' + entry[:suffixIndex] + self.secondaryArchSuffix) 154 if name.lower() not in self.provides: 155 self._violation('no matching provides "%s" for "%s"' 156 % (name, 'develop/lib/' + entry)) 157 158 def _normalizeResolvableName(self, name): 159 # make name a valid resolvable name by replacing '-' with '_' 160 return name.replace('-', '_').lower() 161 162 def _checkLibraryDependencies(self): 163 # If there's no readelf (i.e. no binutils), there probably aren't any 164 # executables/libraries. 165 if not isCommandAvailable('readelf'): 166 return 167 168 # check all files in bin/, apps/ and lib[/<arch>] 169 for directory in ['bin', 'apps', 'lib' + self.secondaryArchSubDir]: 170 dir = os.path.join(self.package.packagingDir, directory) 171 if not os.path.exists(dir): 172 continue 173 174 for entry in os.listdir(dir): 175 path = os.path.join(dir, entry) 176 if os.path.isfile(path): 177 self._checkLibraryDependenciesOfFile(dir, path) 178 elif directory != "bin" and os.path.isdir(path): 179 for entry2 in os.listdir(path): 180 path2 = os.path.join(path, entry2) 181 if os.path.isfile(path2) and os.access(path2, os.X_OK): 182 self._checkLibraryDependenciesOfFile(path, path2) 183 184 def _checkLibraryDependenciesOfFile(self, dirPath, path): 185 # skip static libraries outright 186 if path.endswith('.a'): 187 return 188 189 # try to read the dynamic section of the file 190 try: 191 with open(os.devnull, "w") as devnull: 192 output = check_output(['readelf', '--dynamic', path], 193 stderr=devnull).decode('utf-8') 194 except: 195 return 196 197 libraries = set() 198 rpath = None 199 # extract the library names from the "(NEEDED)" lines of the output 200 for line in output.split('\n'): 201 if line.find('(NEEDED)') >= 0: 202 match = re.match(r'[^[]*\[(.*)].*', line) 203 if match: 204 libraries.add(os.path.basename(match.group(1))) 205 if line.find('(RPATH)') >= 0: 206 match = re.match(r'[^[]*\[(.*)].*', line) 207 if match: 208 rpath = match.group(1) 209 210 for library in libraries: 211 if self._isMissingLibraryDependency(library, dirPath, rpath): 212 if (library.startswith('libgcc') or 213 library.startswith('libstdc++') or 214 library.startswith('libsupc++')): 215 continue 216 self._violation('"%s" needs library "%s", but the ' 217 'package doesn\'t seem to declare that as a ' 218 'requirement' % (path, library)) 219 220 def _isMissingLibraryDependency(self, library, dirPath, rpath): 221 if library.startswith('_APP_'): 222 return False 223 224 # the library might be provided by the package ($libDir) 225 libDir = os.path.join(self.package.packagingDir, 226 'lib' + self.secondaryArchSubDir + '/' + library) 227 if os.path.exists(libDir): 228 return False 229 if len(glob.glob(libDir + '*')) == 1: 230 return False 231 232 # the library might be provided by the package (%A/lib) 233 libDir = os.path.join(dirPath, 'lib/' + library) 234 if os.path.exists(libDir): 235 return False 236 if len(glob.glob(libDir + '*')) == 1: 237 return False 238 239 # the library might be provided by the package, same dir (%A) 240 libDir = os.path.join(dirPath, library) 241 if os.path.exists(libDir): 242 return False 243 if len(glob.glob(libDir + '*')) == 1: 244 return False 245 246 # the library might be provided by the package in rpath 247 if rpath is not None: 248 for rpath1 in rpath.split(':'): 249 if rpath1.find('/.self/') != -1: 250 rpathDir = os.path.join(self.package.packagingDir, 251 rpath1[rpath1.find('/.self/') + len('/.self/'):] + '/' + library) 252 if os.path.exists(rpathDir): 253 return False 254 elif rpath1.find('$ORIGIN') != -1: 255 rpathDir = os.path.join(dirPath, 256 rpath1[rpath1.find('$ORIGIN/') + len('$ORIGIN/'):] + '/' + library) 257 if os.path.exists(rpathDir): 258 return False 259 260 # not provided by the package -- check whether it is required explicitly 261 suffixIndex = library.find('.so') 262 resolvableName = None 263 if suffixIndex >= 0: 264 resolvableName = self._normalizeResolvableName( 265 'lib:' + library[:suffixIndex] + self.secondaryArchSuffix) 266 if resolvableName in self.requires: 267 return False 268 269 # The library might be provided by a sibling package. 270 providingPackage = None 271 for packageName in self.portPackagesProvides.keys(): 272 packageProvides = self.portPackagesProvides[packageName] 273 if resolvableName in packageProvides: 274 providingPackage = packageName 275 break 276 277 if not providingPackage: 278 # Could be required implicitly by requiring (anything from) the 279 # package that provides the library. Find the library in the file 280 # system. 281 libraryPath = None 282 for directory in ['/boot/system/lib']: 283 path = directory + self.secondaryArchSubDir + '/' + library 284 if os.path.exists(path): 285 libraryPath = path 286 break 287 288 if not libraryPath: 289 # Don't complain if we're running on non-haiku host. 290 if os.path.exists('/boot/system/lib'): 291 self._violation('can\'t find used library "%s"' % library) 292 return False 293 294 # Find out which package the library belongs to. 295 providingPackage = self._getPackageProvidingPath(libraryPath) 296 if not providingPackage: 297 print('Warning: failed to determine the package providing "%s"' 298 % libraryPath) 299 return False 300 301 # Chop off ".hpkg" and the version part from the file name to get 302 # the package name. 303 packageName = providingPackage[:-5] 304 index = packageName.find('-') 305 if index >= 0: 306 packageName = packageName[:index] 307 308 packageProvides = self.requiredPackagesProvides.get( 309 providingPackage, []) 310 311 # Check whether the package is required. 312 if packageName in self.requires: 313 return False 314 315 # check whether any of the package's provides are required 316 for name in packageProvides: 317 if name in self.requires: 318 return False 319 320 return True 321 322 def _getPackageProvidingPath(self, path): 323 try: 324 with open(os.devnull, "w") as devnull: 325 output = check_output( 326 ['catattr', '-d', 'SYS:PACKAGE_FILE', path], stderr=devnull).decode('utf-8') 327 if output.endswith('\n'): 328 output = output[:-1] 329 return output 330 except: 331 return None 332 333 def _getPackageProvides(self, package): 334 # get the package listing 335 try: 336 with open(os.devnull, "w") as devnull: 337 output = check_output( 338 [Configuration.getPackageCommand(), 'list', package], 339 stderr=devnull).decode('utf-8') 340 except: 341 return None 342 343 # extract the provides 344 provides = [] 345 for line in output.split('\n'): 346 index = line.find('provides:') 347 if index >= 0: 348 index += 9 349 provides.append(line[index:].strip()) 350 351 return self._parseResolvableExpressionList(provides) 352 353 def _checkMisplacedDevelopLibraries(self): 354 libDir = os.path.join(self.package.packagingDir, 355 'lib' + self.secondaryArchSubDir) 356 if not os.path.exists(libDir): 357 return 358 359 for entry in os.listdir(libDir): 360 if not entry.endswith('.a') and not entry.endswith('.la'): 361 continue 362 363 path = libDir + '/' + entry 364 self._violation('development library entry "%s" should be placed ' 365 'in "develop/lib%s"' % (path, self.secondaryArchSubDir)) 366 367 def _checkGlobalWritableFiles(self): 368 # Create a map for the declared global writable files and check them 369 # while at it. 370 types = {False: 'file', True: 'directory'} 371 globalWritableFiles = {} 372 fileTypes = {} 373 for item in self.package.recipeKeys['GLOBAL_WRITABLE_FILES']: 374 if item.strip().startswith('#'): 375 continue 376 377 components = ConfigParser.splitItemAndUnquote(item) 378 if components: 379 path = components[0] 380 directory = path 381 index = path.find('/') 382 if index >= 0: 383 directory = path[0:index] 384 385 isDirectory = False 386 updateType = None 387 if len(components) > 1: 388 if components[1] == 'directory': 389 isDirectory = True 390 if len(components) > 2: 391 updateType = components[2] 392 else: 393 updateType = components[1] 394 395 fileType = types[isDirectory] 396 fileTypes[components[0]] = fileType 397 398 if directory not in allowedWritableTopLevelDirectories: 399 self._violation('Package declares invalid global writable ' 400 '%s "%s"' % (fileType, components[0])) 401 402 globalWritableFiles[components[0]] = updateType 403 404 if updateType: 405 absPath = os.path.join(self.package.packagingDir, path) 406 if not os.path.exists(absPath): 407 self._violation('Package declares non-existent global ' 408 'writable %s "%s" as included' % (fileType, path)) 409 elif os.path.isdir(absPath) != isDirectory: 410 self._violation('Package declares non-existent global ' 411 'writable %s "%s", but it\'s a %s' 412 % (fileType, path, types[not isDirectory])) 413 414 # iterate through the writable directories in the package 415 for directory in allowedWritableTopLevelDirectories: 416 dir = os.path.join(self.package.packagingDir, directory) 417 if os.path.exists(dir): 418 self._checkGlobalWritableFilesRecursively(globalWritableFiles, 419 fileTypes, directory) 420 421 def _checkGlobalWritableFilesRecursively(self, globalWritableFiles, 422 fileTypes, path): 423 if path in globalWritableFiles: 424 if not globalWritableFiles[path]: 425 self._violation('Included "%s" declared as not included global ' 426 'writable %s' % (path, fileTypes[path])) 427 return 428 429 absPath = os.path.join(self.package.packagingDir, path) 430 if not os.path.isdir(absPath): 431 self._violation('Included file "%s" not declared as global ' 432 'writable file' % path) 433 return 434 435 # entry is a directory -- recurse 436 for entry in os.listdir(absPath): 437 self._checkGlobalWritableFilesRecursively(globalWritableFiles, 438 fileTypes, path + '/' + entry) 439 440 def _checkUserSettingsFiles(self): 441 for item in self.package.recipeKeys['USER_SETTINGS_FILES']: 442 if item.strip().startswith('#'): 443 continue 444 445 components = ConfigParser.splitItemAndUnquote(item) 446 if not components: 447 continue 448 449 if not components[0].startswith('settings/'): 450 self._violation('Package declares invalid user settings ' 451 'file "%s"' % components[0]) 452 if len(components) > 1 and components[1] == 'directory': 453 continue 454 455 if len(components) > 2: 456 template = os.path.join(self.package.packagingDir, 457 components[2]) 458 if not os.path.exists(template): 459 self._violation('Package declares non-existent template ' 460 '"%s" for user settings file "%s" as included' 461 % (components[2], components[0])) 462 463 def _checkPostInstallScripts(self): 464 # check whether declared scripts exist 465 declaredFiles = set() 466 for script in self.package.recipeKeys['POST_INSTALL_SCRIPTS']: 467 if script.lstrip().startswith('#'): 468 continue 469 470 components = ConfigParser.splitItemAndUnquote(script) 471 if not components: 472 continue 473 script = components[0] 474 declaredFiles.add(script) 475 476 absScript = os.path.join(self.package.packagingDir, script) 477 if not os.path.exists(absScript): 478 self._violation('Package declares non-existent post-install ' 479 'script "%s"' % script) 480 481 # check whether existing scripts are declared 482 postInstallDir = 'boot/post-install' 483 dir = os.path.join(self.package.packagingDir, postInstallDir) 484 if os.path.exists(dir): 485 for script in os.listdir(dir): 486 path = postInstallDir + '/' + script 487 if path not in declaredFiles: 488 self._violation('script "%s" not declared as post-install ' 489 'script' % path) 490 491 def _violation(self, message): 492 self.violationEncountered = True 493 if self.strict: 494 violation = 'POLICY ERROR: ' + message 495 else: 496 violation = 'POLICY WARNING: ' + message 497 warn(violation) 498 if self.port.versionedName not in Policy.violationsByPort: 499 Policy.violationsByPort[self.port.versionedName] = [violation] 500 else: 501 Policy.violationsByPort[self.port.versionedName].append(violation) 502