1# -*- coding: utf-8 -*-
2#
3# Copyright 2013 Oliver Tappe
4# Distributed under the terms of the MIT License.
5
6# -- Modules ------------------------------------------------------------------
7
8import codecs
9import glob
10import json
11import os
12import re
13import shutil
14from functools import cmp_to_key
15from subprocess import check_call, check_output
16from textwrap import dedent
17
18from .Configuration import Configuration
19from .DependencyResolver import DependencyResolver
20from .Options import getOption
21from .Port import Port
22from .Utils import prefixLines, sysExit, touchFile, versionCompare, warn
23
24# -- Repository class ---------------------------------------------------------
25
26class Repository(object):
27
28	currentFormatVersion = 2
29
30	def __init__(self, treePath, outputDirectory, repositoryPath,
31			packagesPath, shellVariables,
32			policy, preserveFlags, quiet=False, verbose=False):
33		self.treePath = treePath
34		self.outputDirectory = outputDirectory
35		self.path = repositoryPath
36		self.inputSourcePackagesPath \
37			= self.outputDirectory + '/input-source-packages'
38		self.packagesPath = packagesPath
39		self.shellVariables = shellVariables
40		self.policy = policy
41		self.quiet = quiet
42		self.verbose = verbose
43
44		self._formatVersionFilePath = self.path + '/.formatVersion'
45		self._portIdForPackageIdFilePath \
46			= self.path + '/.portIdForPackageIdMap'
47		self._portNameForPackageNameFilePath \
48			= self.path + '/.portNameForPackageNameMap'
49
50		# check repository format
51		formatVersion = self._readFormatVersion()
52		if formatVersion > Repository.currentFormatVersion:
53			sysExit('The version of the repository format used in\n\t%s'
54					'\nis newer than the one supported by haikuporter.\n'
55					'Please upgrade haikuporter.' % self.path)
56
57		Port.setRepositoryDir(self.path)
58
59		# update repository if it exists and isn't empty, populate it otherwise
60		self._initAllPorts()
61		self._initPortForPackageMaps()
62		if (os.path.isdir(self.path) and os.listdir(self.path)
63			and os.path.exists(self._portIdForPackageIdFilePath)
64			and os.path.exists(self._portNameForPackageNameFilePath)
65			and formatVersion == Repository.currentFormatVersion):
66			if not getOption('noRepositoryUpdate'):
67				self._updateRepository()
68		else:
69			if getOption('noRepositoryUpdate'):
70				sysExit('no or outdated repository found but no update '
71					'allowed')
72			if formatVersion < Repository.currentFormatVersion:
73				warn('Found old repository format - repopulating the '
74					'repository ...')
75			self._populateRepository(preserveFlags)
76			self._writeFormatVersion()
77		self._writePortForPackageMaps()
78		self._activePorts = None
79
80	def getPortIdForPackageId(self, packageId):
81		"""return the port-ID for the given package-ID"""
82
83		return self._portIdForPackageId.get(packageId, None)
84
85	def getPortNameForPackageName(self, packageName):
86		"""return the port name for the given package name"""
87
88		return self._portNameForPackageName.get(packageName, None)
89
90	@property
91	def allPorts(self):
92		return self._allPorts
93
94	@property
95	def activePorts(self):
96		if self._activePorts is not None:
97			return self._activePorts
98
99		self._activePorts = []
100		for portName in self._portVersionsByName.keys():
101			activePortVersion = self.getActiveVersionOf(portName)
102			if not activePortVersion:
103				continue
104			self._activePorts.append(
105				self._allPorts[portName + '-' + activePortVersion])
106
107		return self._activePorts
108
109	@property
110	def portVersionsByName(self):
111		return self._portVersionsByName
112
113	def getActiveVersionOf(self, portName, warnAboutSkippedVersions=False):
114		"""return the highest buildable version of the port with the given
115		   name"""
116		if portName not in self._portVersionsByName:
117			return None
118
119		versions = self._portVersionsByName[portName]
120		for version in reversed(versions):
121			portID = portName + '-' + version
122			port = self._allPorts[portID]
123			if port.hasBrokenRecipe:
124				if warnAboutSkippedVersions:
125					warn('skipping %s, as the recipe is broken' % portID)
126					try:
127						port.parseRecipeFileRaisingExceptions(True)
128					except SystemExit as e:
129						print(e.code)
130				continue
131			if not port.isBuildableOnTargetArchitecture():
132				if warnAboutSkippedVersions:
133					status = port.statusOnTargetArchitecture
134					warn(('skipping %s, as it is %s on the target '
135						+ 'architecture.') % (portID, status))
136				continue
137			return version
138
139		return None
140
141	def getActivePort(self, portName):
142		"""return the highest buildable version of the port with the given
143		   name"""
144		version = self._portNameVersionForPortName(portName)
145		if version is None:
146			return None
147
148		return self._allPorts[version]
149
150	def _portNameVersionForPortName(self, portName):
151		portVersion = self.getActiveVersionOf(portName)
152		if not portVersion:
153			return None
154		return portName + '-' + portVersion
155
156	def searchPorts(self, regExp, returnPortNameVersions=False):
157		"""Search for one or more ports in the HaikuPorts tree, returning
158		   a list of found matches"""
159		if regExp:
160			if getOption('literalSearchStrings'):
161				regExp = re.escape(regExp)
162			reSearch = re.compile(regExp)
163
164		ports = []
165		portNames = self.portVersionsByName.keys()
166		for portName in portNames:
167			if not regExp or reSearch.search(portName):
168				if returnPortNameVersions:
169					portNameVersion = self._portNameVersionForPortName(portName)
170					if portNameVersion is not None:
171						ports.append(portNameVersion)
172				else:
173					ports.append(portName)
174
175		return sorted(ports)
176
177	def _fileNameForPackageName(self, packageName):
178		portName = self._portNameForPackageName[packageName]
179		portVersion = self.getActiveVersionOf(portName)
180		if not portVersion:
181			return None
182
183		port = self._allPorts[portName + '-' + portVersion]
184		for package in port.packages:
185			if package.name == packageName:
186				return package.hpkgName
187
188	def searchPackages(self, regExp, returnFileNames=True):
189		"""Search for one or more packages in the HaikuPorts tree, returning
190		   a list of found matches"""
191		if regExp:
192			if getOption('literalSearchStrings'):
193				regExp = re.escape(regExp)
194			reSearch = re.compile(regExp)
195
196		packages = []
197		packageNames = self._portNameForPackageName.keys()
198		for packageName in packageNames:
199			if not regExp or reSearch.search(packageName):
200				if returnFileNames:
201					packageName = self._fileNameForPackageName(packageName)
202					if not packageName:
203						continue
204
205				packages.append(packageName)
206
207		return sorted(packages)
208
209	def _initAllPorts(self):
210		# Collect all ports into a dictionary that can be keyed by
211		# name + '-' + version. Additionally, we keep a sorted list of
212		# available versions for each port name.
213		self._allPorts = {}
214		self._portVersionsByName = {}
215
216		## REFACTOR into separate methods
217
218		# every existing input source package defines a port (which overrules
219		# any corresponding port in the recipe tree)
220		if os.path.exists(self.inputSourcePackagesPath):
221			for fileName in sorted(os.listdir(self.inputSourcePackagesPath)):
222				if not ('_source-' in fileName
223						or '_source_rigged-' in fileName):
224					continue
225
226				recipeFilePath \
227					= self._partiallyExtractSourcePackageIfNeeded(fileName)
228
229				recipeName = os.path.basename(recipeFilePath)
230				name, version = recipeName[:-7].split('-')
231				if name not in self._portVersionsByName:
232					self._portVersionsByName[name] = [version]
233				else:
234					self._portVersionsByName[name].append(version)
235
236				portPath = os.path.dirname(recipeFilePath)
237				if self.outputDirectory == self.treePath:
238					portOutputPath = portPath
239				else:
240					portOutputPath = self.outputDirectory \
241						+ '/input-source-packages/' + name
242				self._allPorts[name + '-' + version] \
243					= Port(name, version, '<source-package>', portPath,
244						portOutputPath, self.shellVariables,
245						self.policy)
246
247		# collect ports from the recipe tree
248		for category in sorted(os.listdir(self.treePath)):
249			categoryPath = self.treePath + '/' + category
250			if (not os.path.isdir(categoryPath) or category[0] == '.'
251				or '-' not in category):
252				continue
253			for port in sorted(os.listdir(categoryPath)):
254				portPath = categoryPath + '/' + port
255				portOutputPath = (self.outputDirectory + '/' + category + '/'
256					+ port)
257				if not os.path.isdir(portPath) or port[0] == '.':
258					continue
259				for recipe in os.listdir(portPath):
260					recipePath = portPath + '/' + recipe
261					if (not os.path.isfile(recipePath)
262						or not recipe.endswith('.recipe')):
263						continue
264					portElements = recipe[:-7].split('-')
265					if len(portElements) == 2:
266						name, version = portElements
267						versionedName = name + '-' + version
268						if versionedName in self._allPorts:
269							# this version of the current port already was
270							# defined - skip
271							if not self.quiet and not getOption('doBootstrap'):
272								otherPort = self._allPorts[versionedName]
273								if otherPort.category == '<source-package>':
274									warn('%s/%s	 is overruled by input '
275										'source package' % (category,
276											versionedName))
277								else:
278									warn('%s/%s	 is overruled by duplicate '
279										'in %s - please remove one of them'
280										% (category, versionedName,
281											otherPort.category))
282							continue
283						if name not in self._portVersionsByName:
284							self._portVersionsByName[name] = [version]
285						else:
286							self._portVersionsByName[name].append(version)
287						self._allPorts[name + '-' + version] = Port(name,
288							version, category, portPath, portOutputPath,
289							self.shellVariables, self.policy)
290					else:
291						# invalid argument
292						if not self.quiet:
293							print("Warning: Couldn't parse port/version info: "
294								+ recipe)
295
296		# Create ports for the secondary architectures. Not all make sense or
297		# are supported, but we won't know until we have parsed the recipe file.
298		secondaryArchitectures = Configuration.getSecondaryTargetArchitectures()
299		if secondaryArchitectures:
300			for port in tuple(self._allPorts.values()):
301				for architecture in secondaryArchitectures:
302					newPort = Port(port.baseName, port.version, port.category,
303						port.baseDir, port.outputDir, self.shellVariables,
304						port.policy, architecture)
305					self._allPorts[newPort.versionedName] = newPort
306
307					name = newPort.name
308					version = newPort.version
309					if name not in self._portVersionsByName:
310						self._portVersionsByName[name] = [version]
311					else:
312						self._portVersionsByName[name].append(version)
313
314		# Sort version list of each port
315		for portName in list(self._portVersionsByName.keys()):
316			self._portVersionsByName[portName].sort(
317				key=cmp_to_key(versionCompare))
318
319	def _initPortForPackageMaps(self):
320		"""Initialize dictionaries that map package names/IDs to port
321		   names/IDs"""
322
323		self._portIdForPackageId = {}
324		if os.path.exists(self._portIdForPackageIdFilePath):
325			try:
326				with open(self._portIdForPackageIdFilePath, 'r') as fh:
327					self._portIdForPackageId = json.load(fh)
328			except BaseException as e:
329				print(e)
330
331		self._portNameForPackageName = {}
332		if os.path.exists(self._portNameForPackageNameFilePath):
333			try:
334				with open(self._portNameForPackageNameFilePath, 'r') as fh:
335					self._portNameForPackageName = json.load(fh)
336			except BaseException as e:
337				print(e)
338
339	def _writePortForPackageMaps(self):
340		"""Writes dictionaries that map package names/IDs to port
341		   names/IDs to a file"""
342
343		try:
344			with open(self._portIdForPackageIdFilePath, 'w') as fh:
345				json.dump(self._portIdForPackageId, fh, sort_keys=True,
346					indent=4, separators=(',', ' : '))
347		except BaseException as e:
348			print(e)
349
350		try:
351			with open(self._portNameForPackageNameFilePath, 'w') as fh:
352				json.dump(self._portNameForPackageName, fh, sort_keys=True,
353					indent=4, separators=(',', ' : '))
354		except BaseException as e:
355			print(e)
356
357	def _readFormatVersion(self):
358		"""Read format version of repository from file"""
359
360		formatVersion = 0
361		if os.path.exists(self._formatVersionFilePath):
362			try:
363				with open(self._formatVersionFilePath, 'r') as fh:
364					data = json.load(fh)
365				formatVersion = data.get('formatVersion', 0)
366			except BaseException as e:
367				print(e)
368		return formatVersion
369
370	def _writeFormatVersion(self):
371		"""Writes the version of the repository format into a file"""
372
373		try:
374			data = {
375				'formatVersion': Repository.currentFormatVersion
376			}
377			with open(self._formatVersionFilePath, 'w') as fh:
378				json.dump(data, fh, indent=4, separators=(',', ' : '))
379		except BaseException as e:
380			print(e)
381
382	def _populateRepository(self, preserveFlags):
383		"""Remove and refill the repository with all DependencyInfo-files from
384		   parseable recipes"""
385
386		if os.path.exists(self.path):
387			shutil.rmtree(self.path)
388
389		self._portNameForPackageName = {}
390		self._portIdForPackageId = {}
391		self._updateRepository(None, preserveFlags)
392		return
393
394	def supportBackwardsCompatibility(self, name, version):
395		self._updateRepository({'name': name, 'version': version})
396
397	def _updateRepository(self, explicitPortVersion=None, preserveFlags=True):
398		"""Update all DependencyInfo-files in the repository as needed"""
399
400		allPorts = self.allPorts
401
402		activePorts = []
403		updatedPorts = {}
404
405		# check for all known ports if their recipe has been changed
406		if os.path.exists(self.path):
407			if not self.quiet:
408				print('Checking if any dependency-infos need to be updated ...')
409		else:
410			os.makedirs(self.path)
411			if not self.quiet:
412				print('Populating repository ...')
413
414		skippedDir = os.path.join(self.path, '.skipped')
415		if not os.path.exists(skippedDir):
416			os.mkdir(skippedDir)
417
418		for portName in sorted(self._portVersionsByName.keys(),
419				key=str.lower):
420
421			if explicitPortVersion and explicitPortVersion['name'] == portName:
422				versions = [explicitPortVersion['version']]
423				forceAllowUnstable = True
424			else:
425				versions = reversed(self._portVersionsByName[portName])
426				forceAllowUnstable = False
427
428			for version in versions:
429				portID = portName + '-' + version
430				port = allPorts[portID]
431				skippedFlag = os.path.join(skippedDir, portID)
432
433				# ignore recipes that were skipped last time unless they've
434				# been changed since then
435				if (os.path.exists(skippedFlag)
436					and (os.path.getmtime(port.recipeFilePath)
437						<= os.path.getmtime(skippedFlag))):
438					continue
439
440				# update all dependency-infos of port if the recipe is newer
441				# than the main dependency-info of that port
442				mainDependencyInfoFile = os.path.join(self.path,
443					port.dependencyInfoName)
444				if (os.path.exists(mainDependencyInfoFile)
445					and (os.path.getmtime(port.recipeFilePath)
446						<= os.path.getmtime(mainDependencyInfoFile))):
447					activePorts.append(portID)
448					break
449
450				# try to parse updated recipe
451				try:
452					port.parseRecipeFile(False, forceAllowUnstable,
453						forceAllowUnstable)
454
455					if not port.isBuildableOnTargetArchitecture(
456							forceAllowUnstable):
457						touchFile(skippedFlag)
458						if not self.quiet:
459							status = port.statusOnTargetArchitecture
460							print(('\t%s is still marked as %s on target '
461								+ 'architecture') % (portID, status))
462						continue
463
464					if os.path.exists(skippedFlag):
465						os.remove(skippedFlag)
466
467					if not preserveFlags and port.checkFlag('build'):
468						if not self.quiet:
469							print('\t[build-flag reset]')
470						port.unsetFlag('build')
471
472					if not self.quiet:
473						print('\tupdating dependency infos of ' + portID)
474
475					port.writeDependencyInfosIntoRepository()
476					updatedPorts[portID] = port
477					break
478
479				except SystemExit as e:
480					# take notice of broken recipe file
481					touchFile(skippedFlag)
482					if not os.path.exists(mainDependencyInfoFile):
483						if not self.quiet:
484							print('\trecipe for %s is still broken:' % portID)
485							print(prefixLines('\t', e.code))
486
487		# This also drops mappings for updated ports to remove any possibly
488		# removed sub-packages.
489		self._removeStalePortForPackageMappings(activePorts)
490
491		# Add port for package mappings for updated ports.
492		for portID, port in updatedPorts.items():
493			for package in port.packages:
494				self._portIdForPackageId[package.versionedName] \
495					= port.versionedName
496				self._portNameForPackageName[package.name] \
497					= port.name
498			activePorts.append(portID)
499
500		# Note that removing stale dependency infos uses the port for package
501		# mappings to determine what to keep. This step must therefore come
502		# after the stale port for package mapping removal.
503		self._removeStaleDependencyInfos(activePorts)
504
505	def _removeStaleDependencyInfos(self, activePorts):
506		"""check for any dependency-infos that no longer have a corresponding
507		   recipe file"""
508
509		allPorts = self.allPorts
510
511		if not self.quiet:
512			print("Looking for stale dependency-infos ...")
513		dependencyInfos = glob.glob(self.path + '/*.DependencyInfo')
514		for dependencyInfo in dependencyInfos:
515			dependencyInfoFileName = os.path.basename(dependencyInfo)
516			packageID \
517				= dependencyInfoFileName[:dependencyInfoFileName.rindex('.')]
518			portID = self.getPortIdForPackageId(packageID)
519
520			if not portID or portID not in activePorts:
521				if not self.quiet:
522					print('\tremoving ' + dependencyInfoFileName)
523				os.remove(dependencyInfo)
524
525				if not getOption('noPackageObsoletion'):
526					# obsolete corresponding package, if any
527					self._removePackagesForDependencyInfo(dependencyInfo)
528
529	def _removePackagesForDependencyInfo(self, dependencyInfo):
530		"""remove all packages for the given dependency-info"""
531
532		(packageSpec, _) = os.path.basename(dependencyInfo).rsplit('.', 1)
533		packages = glob.glob(self.packagesPath + '/' + packageSpec + '-*.hpkg')
534		obsoleteDir = self.packagesPath + '/.obsolete'
535		for package in packages:
536			packageFileName = os.path.basename(package)
537			if not self.quiet:
538				print('\tobsoleting package ' + packageFileName)
539			obsoletePackage = obsoleteDir + '/' + packageFileName
540			if not os.path.exists(obsoleteDir):
541				os.mkdir(obsoleteDir)
542			os.rename(package, obsoletePackage)
543
544	def _removeStalePortForPackageMappings(self, activePorts):
545		"""drops any port-for-package mappings that refer to non-existing or
546		   broken ports"""
547
548		for packageId, portId in tuple(self._portIdForPackageId.items()):
549			if portId not in activePorts:
550				del self._portIdForPackageId[packageId]
551
552		for packageName, portName in tuple(self._portNameForPackageName.items()):
553			if portName not in self._portVersionsByName:
554				del self._portNameForPackageName[packageName]
555				continue
556
557			for version in self._portVersionsByName[portName]:
558				portId = portName + '-' + version
559				if portId in activePorts:
560					break
561			else:
562				# no version exists of this port that is not broken
563				del self._portNameForPackageName[packageName]
564
565	def _partiallyExtractSourcePackageIfNeeded(self, sourcePackageName):
566		"""extract the recipe and potentially contained patches/licenses from
567		   a source package, unless that has already been done"""
568
569		sourcePackagePath \
570			= self.inputSourcePackagesPath + '/' + sourcePackageName
571		(name, version, revision, _) = sourcePackageName.split('-')
572		# determine port name by dropping '_source' or '_source_rigged'
573		if name.endswith('_source_rigged'):
574			name = name[:-14]
575		elif name.endswith('_source'):
576			name = name[:-7]
577		relativeBasePath \
578			= 'develop/sources/%s-%s-%s' % (name, version, revision)
579		recipeName = name + '-' + version + '.recipe'
580		recipeFilePath = (self.inputSourcePackagesPath + '/' + relativeBasePath
581			+ '/' + recipeName)
582
583		if (not os.path.exists(recipeFilePath)
584			or (os.path.getmtime(recipeFilePath)
585				<= os.path.getmtime(sourcePackagePath))):
586			# extract recipe, patches and licenses (but skip everything else)
587			allowedEntries = [
588				relativeBasePath + '/' + recipeName,
589				relativeBasePath + '/additional-files',
590				relativeBasePath + '/licenses',
591				relativeBasePath + '/patches',
592			]
593			entries = check_output([Configuration.getPackageCommand(), 'list',
594				'-p', sourcePackagePath]).decode('utf-8').splitlines()
595			entries = [
596				entry for entry in entries if entry in allowedEntries
597			]
598			check_call([Configuration.getPackageCommand(), 'extract',
599					'-C', self.inputSourcePackagesPath, sourcePackagePath]
600				+ entries)
601
602			# override all SOURCE_URIs in recipe to point to the source package
603			textToAdd = dedent(r'''
604				# Added by haikuporter:
605				SOURCE_URI='pkg:%s'
606				for i in {2..1000}; do
607					eval currentSrcUri=\$SOURCE_URI_$i
608					if [ -z "$currentSrcUri" ]; then
609						break
610					fi
611					eval SOURCE_URI_$i="$SOURCE_URI"
612				done
613				for i in {001..999}; do
614					eval currentSrcUri=\$SOURCE_URI_$i
615					if [ -z "$currentSrcUri" ]; then
616						break
617					fi
618					eval SOURCE_URI_$i="$SOURCE_URI"
619				done
620				'''[1:]) % sourcePackagePath
621			with codecs.open(recipeFilePath, 'a', 'utf-8') as recipeFile:
622				recipeFile.write('\n' + textToAdd)
623
624		return recipeFilePath
625
626	def checkRepositoryConsistency(self, verbose):
627		"""Check consistency of the repository by dependency solving all
628			dependency infos."""
629
630		repositories = [self.path]
631		systemPackagesDirectory = getOption('systemPackagesDirectory')
632		if systemPackagesDirectory:
633			repositories.append(systemPackagesDirectory)
634
635		resolver = DependencyResolver(None, Port.requiresTypes, repositories,
636			quiet=True)
637
638		for port in sorted(self.activePorts, key=lambda port: port.name):
639			for package in port.packages:
640				if verbose:
641					print('checking package {} of {}'.format(
642							package.revisionedName, port.versionedName))
643
644				try:
645					resolver.determineRequiredPackagesFor(
646						[package.dependencyInfoFile(self.path)])
647				except LookupError as error:
648					print('{}:\n{}\n'.format(package.revisionedName,
649							prefixLines('\t', str(error))))
650