1# -*- coding: utf-8 -*- 2# 3# Copyright 2007-2011 Brecht Machiels 4# Copyright 2009-2010 Chris Roberts 5# Copyright 2009-2011 Scott McCreary 6# Copyright 2009 Alexander Deynichenko 7# Copyright 2009 HaikuBot (aka RISC) 8# Copyright 2010-2011 Jack Laxson (Jrabbit) 9# Copyright 2011 Ingo Weinhold 10# Copyright 2013-2014 Oliver Tappe 11# Distributed under the terms of the MIT License. 12 13# -- Modules ------------------------------------------------------------------ 14 15import hashlib 16import os 17import shutil 18from subprocess import CalledProcessError, check_call, check_output 19 20from .Configuration import Configuration 21from .Options import getOption 22from .SourceFetcher import (createSourceFetcher, foldSubdirIntoSourceDir, 23 parseCheckoutUri) 24from .Utils import (ensureCommandIsAvailable, info, readStringFromFile, 25 storeStringInFile, sysExit, warn) 26 27# -- A source archive (or checkout) ------------------------------------------- 28 29class Source(object): 30 def __init__(self, port, index, uris, fetchTargetName, checksum, 31 sourceDir, patches, additionalFiles): 32 self.index = index 33 self.uris = uris 34 self.fetchTargetName = fetchTargetName 35 self.checksum = checksum 36 self.patches = patches 37 self.additionalFiles = additionalFiles 38 39 ## REFACTOR use property setters to handle branching based on instance 40 ## variables 41 42 if index == '1': 43 self.sourceBaseDir = port.sourceBaseDir 44 else: 45 self.sourceBaseDir = port.sourceBaseDir + '-' + index 46 47 if sourceDir: 48 index = sourceDir.find('/') 49 if index > 0: 50 self.sourceExportSubdir = sourceDir[index + 1:] 51 sourceDir = sourceDir[:index] 52 else: 53 self.sourceExportSubdir = None 54 self.sourceSubDir = sourceDir 55 self.sourceDir = self.sourceBaseDir + '/' + sourceDir 56 else: 57 self.sourceDir = self.sourceBaseDir 58 self.sourceSubDir = None 59 self.sourceExportSubdir = None 60 61 # PATCHES refers to patch files relative to the patches directory, 62 # make those absolute paths. 63 if self.patches and port.patchesDir: 64 self.patches = [ 65 port.patchesDir + '/' + patch for patch in self.patches 66 ] 67 68 # ADDITIONAL_FILES refers to the files relative to the additional-files 69 # directory, make those absolute paths. 70 if self.additionalFiles: 71 self.additionalFiles = [ 72 port.additionalFilesDir + '/' + additionalFile 73 for additionalFile in self.additionalFiles 74 ] 75 76 # determine filename of first URI 77 uriFileName = self.uris[0] 78 uriExtension = '' 79 hashPos = uriFileName.find('#') 80 if hashPos >= 0: 81 uriExtension = uriFileName[hashPos:] 82 uriFileName = uriFileName[:hashPos] 83 uriFileName = uriFileName[uriFileName.rindex('/') + 1:] 84 85 # set local filename from URI, unless specified explicitly 86 if not self.fetchTargetName: 87 self.fetchTargetName = uriFileName 88 89 downloadMirror = Configuration.getDownloadMirror() 90 if downloadMirror: 91 # add fallback URI using a general source tarball mirror (some 92 # original source sites aren't very reliable) 93 recipeDirName = os.path.basename(port.baseDir) 94 self.uris.append(downloadMirror + '/' + recipeDirName + '/' 95 + uriFileName + uriExtension) 96 97 self.sourceFetcher = None 98 self.fetchTarget = port.downloadDir + '/' + self.fetchTargetName 99 self.fetchTargetIsArchive = True 100 101 self.gitEnv = { 102 'GIT_COMMITTER_EMAIL': Configuration.getPackagerEmail(), 103 'GIT_COMMITTER_NAME': Configuration.getPackagerName().encode("utf-8"), 104 'GIT_AUTHOR_EMAIL': Configuration.getPackagerEmail(), 105 'GIT_AUTHOR_NAME': Configuration.getPackagerName().encode("utf-8"), 106 } 107 108 def fetch(self, port): 109 """Fetch the source from one of the URIs given in the recipe. 110 If the sources have already been fetched, setup an appropriate 111 source fetcher object 112 """ 113 114 # create download dir 115 downloadDir = os.path.dirname(self.fetchTarget) 116 if not os.path.exists(downloadDir): 117 os.mkdir(downloadDir) 118 119 # check if we've already downloaded the sources 120 uriFile = self.fetchTarget + '.uri' 121 if os.path.exists(self.fetchTarget): 122 if os.path.exists(uriFile): 123 # create a source fetcher corresponding to the base URI found 124 # in the uri file and update to a different revision, if needed 125 storedUri = readStringFromFile(uriFile) 126 (unusedType, storedBaseUri, storedRev) \ 127 = parseCheckoutUri(storedUri) 128 129 for uri in self.uris: 130 (unusedType, baseUri, rev) = parseCheckoutUri(uri) 131 if baseUri == storedBaseUri: 132 self.sourceFetcher \ 133 = createSourceFetcher(uri, self.fetchTarget) 134 if rev != storedRev: 135 self.sourceFetcher.updateToRev(rev) 136 storeStringInFile(uri, self.fetchTarget + '.uri') 137 port.unsetFlag('unpack', self.index) 138 port.unsetFlag('patchset', self.index) 139 else: 140 info('Skipping download of source for ' 141 + self.fetchTargetName) 142 break 143 else: 144 warn("Stored SOURCE_URI is no longer in recipe, automatic " 145 u"repository update won't work") 146 self.sourceFetcher \ 147 = createSourceFetcher(storedUri, self.fetchTarget) 148 149 return 150 else: 151 # Remove the fetch target, as it isn't complete 152 if os.path.isdir(self.fetchTarget): 153 shutil.rmtree(self.fetchTarget) 154 else: 155 os.remove(self.fetchTarget) 156 157 # download the sources 158 for uri in self.uris: 159 try: 160 info('\nDownloading: ' + uri + ' ...') 161 sourceFetcher = createSourceFetcher(uri, self.fetchTarget) 162 sourceFetcher.fetch() 163 164 # ok, fetching the source was successful, we keep the source 165 # fetcher and store the URI that the source came from for 166 # later runs 167 self.sourceFetcher = sourceFetcher 168 storeStringInFile(uri, self.fetchTarget + '.uri') 169 return 170 except Exception as e: 171 if isinstance(e, CalledProcessError): 172 info(e.output) 173 if uri != self.uris[-1]: 174 warn(('Unable to fetch source from %s (error: %s), ' 175 + 'trying next location.') % (uri, e)) 176 else: 177 warn(('Unable to fetch source from %s (error: %s)') 178 % (uri, e)) 179 180 # failed to fetch source 181 sysExit('Failed to fetch source from all known locations.') 182 183 def clean(self): 184 if os.path.exists(self.fetchTarget): 185 print('Removing source %s ...' % self.fetchTarget) 186 if os.path.isdir(self.fetchTarget): 187 shutil.rmtree(self.fetchTarget) 188 else: 189 os.remove(self.fetchTarget) 190 191 uriFile = self.fetchTarget + '.uri' 192 if os.path.exists(uriFile): 193 os.remove(uriFile) 194 195 def unpack(self, port): 196 """Unpack the source into the source directory""" 197 198 # Check to see if the source was already unpacked. 199 if port.checkFlag('unpack', self.index) and not getOption('force'): 200 if not os.path.exists(self.sourceBaseDir): 201 warn('Source dir has changed or been removed, unpacking in new dir') 202 port.unsetFlag('unpack', self.index) 203 else: 204 info('Skipping unpack of ' + self.fetchTargetName) 205 return 206 207 # re-create source directory 208 if os.path.exists(self.sourceBaseDir): 209 info('Cleaning source dir for ' + self.fetchTargetName) 210 shutil.rmtree(self.sourceBaseDir) 211 os.makedirs(self.sourceBaseDir) 212 213 info('Unpacking source of ' + self.fetchTargetName) 214 self.sourceFetcher.unpack(self.sourceBaseDir, self.sourceSubDir, 215 self.sourceExportSubdir) 216 217 if not os.path.exists(self.sourceDir): 218 sysExit(self.sourceSubDir + ' doesn\'t exist in sources! Define SOURCE_DIR in recipe?') 219 220 port.setFlag('unpack', self.index) 221 222 def populateAdditionalFiles(self, baseDir): 223 if not self.additionalFiles: 224 return 225 226 additionalFilesDir = os.path.join(baseDir, 'additional-files') 227 if self.index != '1': 228 additionalFilesDir += '-' + self.index 229 230 if not os.path.exists(additionalFilesDir): 231 os.mkdir(additionalFilesDir) 232 233 for additionalFile in self.additionalFiles: 234 if os.path.isdir(additionalFile): 235 shutil.copytree(additionalFile, 236 os.path.join(additionalFilesDir, 237 os.path.basename(additionalFile))) 238 else: 239 shutil.copy(additionalFile, additionalFilesDir) 240 241 def validateChecksum(self, port): 242 """Make sure that the SHA256-checksum matches the expectations""" 243 244 if not self.sourceFetcher.sourceShouldBeValidated: 245 return 246 247 # Check to see if the source was already unpacked. 248 if port.checkFlag('validate', self.index) and not getOption('force'): 249 info('Skipping checksum validation of ' + self.fetchTargetName) 250 return 251 252 info('Validating checksum of ' + self.fetchTargetName) 253 sha256 = hashlib.sha256() 254 255 with open(self.fetchTarget, 'rb') as f: 256 while True: 257 data = f.read(16384) 258 if not data: 259 break 260 sha256.update(data) 261 262 if self.checksum is not None: 263 if sha256.hexdigest() != self.checksum: 264 sysExit('Expected SHA-256: ' + self.checksum + '\n' 265 + 'Found SHA-256: ' + sha256.hexdigest()) 266 else: 267 warn('----- CHECKSUM TEMPLATE -----') 268 warn('CHECKSUM_SHA256%(index)s="%(digest)s"' % { 269 "digest": sha256.hexdigest(), 270 "index": ("_" + self.index) if self.index != "1" else ""}) 271 warn('-----------------------------') 272 273 if self.checksum is None: 274 if not Configuration.shallAllowUnsafeSources(): 275 sysExit('No checksum found in recipe!') 276 else: 277 warn('No checksum found in recipe!') 278 279 port.setFlag('validate', self.index) 280 281 @property 282 def isFromSourcePackage(self): 283 """Determines whether or not this source comes from a source package""" 284 285 return self.uris[0].lower().startswith('pkg:') 286 287 @property 288 def isFromRiggedSourcePackage(self): 289 """Determines whether or not this source comes from a source package 290 that has been rigged (i.e. does have the patches already applied)""" 291 292 return (self.uris[0].lower().startswith('pkg:') 293 and '_source_rigged-' in self.uris[0].lower()) 294 295 def referencesFiles(self, files): 296 if self.patches: 297 for patch in self.patches: 298 if patch in files: 299 return True 300 301 if self.additionalFiles: 302 for additionalFile in self.additionalFiles: 303 if os.path.isdir(additionalFile): 304 # ensure there is a path separator at the end 305 additionalFile = os.path.join(additionalFile, '') 306 for fileName in files: 307 if os.path.commonprefix([additionalFile, fileName]) \ 308 == additionalFile: 309 return True 310 elif additionalFile in files: 311 return True 312 313 return False 314 315 def patch(self, port): 316 """Apply any patches to this source""" 317 318 # Check to see if the source has already been patched. 319 if port.checkFlag('patchset', self.index) and not getOption('force'): 320 info('Skipping patchset for ' + self.fetchTargetName) 321 return True 322 323 if not getOption('noGitRepo'): 324 # use an implicit git repository for improved patch handling. 325 ensureCommandIsAvailable('git') 326 if not self._isInGitWorkingDirectory(self.sourceDir): 327 # import sources into pristine git repository 328 self._initImplicitGitRepo() 329 elif self.patches: 330 # reset existing git repsitory before appling patchset(s) again 331 self.reset() 332 else: 333 # make sure the patches can still be applied if no git repo 334 ensureCommandIsAvailable('patch') 335 336 patched = False 337 try: 338 # Apply patches 339 for patch in self.patches: 340 if not os.path.exists(patch): 341 sysExit('patch file "' + patch + '" not found.') 342 343 if getOption('noGitRepo'): 344 info('Applying patch(set) "%s" ...' % patch) 345 output = check_output(['patch', '--ignore-whitespace', '-p1', '-i', 346 patch], cwd=self.sourceDir).decode('utf-8') 347 info(output) 348 else: 349 if patch.endswith('.patchset'): 350 info('Applying patchset "%s" ...' % patch) 351 output = check_output(['git', 'am', '--ignore-whitespace', '-3', 352 '--keep-cr', patch], cwd=self.sourceDir, 353 env=self.gitEnv).decode('utf-8') 354 info(output) 355 else: 356 info('Applying patch "%s" ...' % patch) 357 output = check_output(['git', 'apply', '--ignore-whitespace', 358 '-p1', '--index', patch], 359 cwd=self.sourceDir).decode('utf-8') 360 info(output) 361 output = check_output(['git', 'commit', '-q', '-m', 362 'applying patch %s' 363 % os.path.basename(patch)], 364 cwd=self.sourceDir, env=self.gitEnv).decode('utf-8') 365 info(output) 366 patched = True 367 except: 368 # Don't leave behind half-patched sources. 369 if patched and not getOption('noGitRepo'): 370 self.reset() 371 raise 372 373 if patched: 374 port.setFlag('patchset', self.index) 375 376 return patched 377 378 def reset(self): 379 """Reset source to original state""" 380 381 output = check_output(['git', 'reset', '--hard', 'ORIGIN'], cwd=self.sourceDir).decode('utf-8') 382 info(output) 383 output = check_output(['git', 'clean', '-f', '-d'], cwd=self.sourceDir).decode('utf-8') 384 info(output) 385 386 def commitPatchPhase(self): 387 """Commit changes done in patch phase.""" 388 389 # see if there are any changes at all 390 changes = check_output(['git', 'status', '--porcelain'], 391 cwd=self.sourceDir).decode('utf-8') 392 if not changes: 393 info("Patch function hasn't changed anything for " 394 + self.fetchTargetName) 395 return 396 397 info('Committing changes done in patch function for ' 398 + self.fetchTargetName) 399 output = check_output(['git', 'commit', '-a', '-q', '-m', 'patch function'], 400 cwd=self.sourceDir, env=self.gitEnv).decode('utf-8') 401 info(output) 402 output = check_output(['git', 'tag', '--no-sign', '-f', 'PATCH_FUNCTION', 'HEAD'], 403 cwd=self.sourceDir).decode('utf-8') 404 info(output) 405 406 def extractPatchset(self, patchSetFilePath, archPatchSetFilePath): 407 """Extract the current set of patches applied to git repository, 408 taking care to not include the programatic changes introduced 409 during the patch phase""" 410 411 if not os.path.exists(self.sourceDir): 412 sysExit("Can't extract patchset for " + self.sourceDir 413 + u" as the source directory doesn't exist yet") 414 415 print('Extracting patchset for ' + self.fetchTargetName + " to " + patchSetFilePath) 416 needToRebase = True 417 try: 418 # check if the tag 'PATCH_FUNCTION' exists 419 with open(os.devnull, "w") as devnull: 420 check_call(['git', 'rev-parse', '--verify', 'PATCH_FUNCTION'], 421 stdout=devnull, stderr=devnull, cwd=self.sourceDir) 422 except: 423 # no PATCH_FUNCTION tag, so there's nothing to rebase 424 needToRebase = False 425 426 if needToRebase: 427 # the tag exists, so we drop the respective commit 428 check_call(['git', 'rebase', '-q', '--onto', 'PATCH_FUNCTION^', 429 'PATCH_FUNCTION', 'haikuport'], cwd=self.sourceDir, 430 env=self.gitEnv) 431 432 patchSetDirectory = os.path.dirname(patchSetFilePath) 433 if not os.path.exists(patchSetDirectory): 434 os.mkdir(patchSetDirectory) 435 with open(patchSetFilePath, 'w') as patchSetFile: 436 check_call(['git', '-c', 'core.abbrev=auto', 'format-patch', '-kp', 437 '--stdout', 'ORIGIN'], stdout=patchSetFile, 438 cwd=self.sourceDir, env=self.gitEnv) 439 440 if needToRebase: 441 # put PATCH_FUNCTION back in 442 check_call(['git', 'rebase', '-q', 'PATCH_FUNCTION', 'haikuport'], 443 cwd=self.sourceDir, env=self.gitEnv) 444 445 # warn if there's a correpsonding arch-specific patchset file 446 if os.path.exists(archPatchSetFilePath): 447 warn('arch-specific patchset file %s requires manual update' 448 % os.path.basename(archPatchSetFilePath)) 449 450 # if there's a corresponding patch file, remove it, as we now have 451 # the patchset 452 patchFilePath = patchSetFilePath[:-3] 453 if os.path.exists(patchFilePath): 454 warn('removing obsolete patch file ' 455 + os.path.basename(patchFilePath)) 456 os.remove(patchFilePath) 457 # if there's a corresponding diff file, remove it, as we now have 458 # the patchset 459 diffFilePath = patchFilePath[:-6] + '.diff' 460 if os.path.exists(diffFilePath): 461 warn('removing obsolete diff file ' 462 + os.path.basename(diffFilePath)) 463 os.remove(diffFilePath) 464 465 def exportSources(self, targetDir, rigged): 466 """Export sources into a folder""" 467 468 if not os.path.exists(targetDir): 469 os.makedirs(targetDir) 470 if rigged: 471 # export the sources in 'rigged' state, i.e. in directly usable 472 # form, with patches already applied 473 check_call('tar c --exclude=.git . | tar x -C %s' % targetDir, 474 cwd=self.sourceDir, shell=True) 475 else: 476 # unpack the archive into the targetDir 477 if self.sourceSubDir: 478 os.mkdir(targetDir + '/' + self.sourceSubDir) 479 self.sourceFetcher.unpack(targetDir, self.sourceSubDir, 480 self.sourceExportSubdir) 481 if self.sourceSubDir: 482 foldSubdirIntoSourceDir(self.sourceSubDir, targetDir) 483 484 def adjustToChroot(self, port): 485 """Adjust directories to chroot()-ed environment""" 486 487 self.fetchTarget = None 488 489 # adjust all relevant directories 490 pathLengthToCut = len(port.workDir) 491 self.sourceBaseDir = self.sourceBaseDir[pathLengthToCut:] 492 self.sourceDir = self.sourceDir[pathLengthToCut:] 493 self.additionalFiles = [ 494 additionalFile[pathLengthToCut:] 495 for additionalFile in self.additionalFiles 496 ] 497 498 def _initImplicitGitRepo(self): 499 """Import sources into git repository""" 500 501 ensureCommandIsAvailable('git') 502 info(check_output(['git', 'init', '-b', 'main'], cwd=self.sourceDir).decode('utf-8')) 503 info(check_output(['git', 'config', 'gc.auto', '0'], cwd=self.sourceDir).decode('utf-8')) 504 # Disable automatic garbage collection. This works around an issue 505 # with git failing to do that with the haikuwebkit repository. 506 info(check_output(['git', 'symbolic-ref', 'HEAD', 'refs/heads/haikuport'], 507 cwd=self.sourceDir).decode('utf-8')) 508 info(check_output(['git', 'add', '-f', '.'], cwd=self.sourceDir).decode('utf-8')) 509 info(check_output(['git', 'commit', '-m', 'import', '-q'], 510 cwd=self.sourceDir, env=self.gitEnv).decode('utf-8')) 511 info(check_output(['git', 'tag', '--no-sign', 'ORIGIN'], 512 cwd=self.sourceDir).decode('utf-8')) 513 514 def _isInGitWorkingDirectory(self, path): 515 """Returns whether the given source directory path is in a git working 516 directory. path must be under self.sourceBaseDir.""" 517 518 while (path == self.sourceBaseDir 519 or path.startswith(self.sourceBaseDir + '/')): 520 if os.path.exists(path + '/.git'): 521 return True 522 if path == self.sourceBaseDir: 523 return False 524 path = path[0:path.rfind('/')] 525 526 return False 527