1# -*- coding: utf-8 -*-
2#
3# Copyright 2013 Ingo Weinhold
4# Distributed under the terms of the MIT License.
5
6# -- Modules ------------------------------------------------------------------
7
8import glob
9import os
10import re
11from subprocess import check_output
12
13from .ConfigParser import ConfigParser
14from .Configuration import Configuration
15from .Utils import isCommandAvailable, sysExit, warn
16
17allowedWritableTopLevelDirectories = [
18	'cache',
19	'non-packaged',
20	'settings',
21	'var'
22]
23
24allowedTopLevelEntries = [
25	'.PackageInfo',
26	'add-ons',
27	'apps',
28	'bin',
29	'boot',
30	'data',
31	'develop',
32	'documentation',
33	'lib',
34	'preferences',
35	'servers'
36] + allowedWritableTopLevelDirectories
37
38# -- Policy checker class -----------------------------------------------------
39
40class Policy(object):
41
42	# log for policy warnings and errors
43	violationsByPort = {}
44
45	def __init__(self, strict):
46		self.strict = strict
47
48	def setPort(self, port, requiredPackages):
49		self.port = port
50		self.secondaryArchitecture = port.secondaryArchitecture
51		self.secondaryArchSuffix = '_' + self.secondaryArchitecture \
52			if self.secondaryArchitecture else ''
53		self.secondaryArchSubDir = '/' + self.secondaryArchitecture \
54			if self.secondaryArchitecture else ''
55
56		# Get the provides of all of the port's packages. We need them to find
57		# dependencies between packages of the port.
58		self.portPackagesProvides = {}
59		for package in port.packages:
60			self.portPackagesProvides[package.name] = (
61				self._parseResolvableExpressionList(
62					package.recipeKeys['PROVIDES']))
63
64		# Create a map with the packages' provides. We need that later when
65		# checking the created package.
66		self.requiredPackagesProvides = {}
67		for package in requiredPackages:
68			provides = self._getPackageProvides(package)
69			self.requiredPackagesProvides[os.path.basename(package)] = provides
70
71	def checkPackage(self, package, packageFile):
72		self.package = package
73		self.packageFile = packageFile
74		self.violationEncountered = False
75
76		self.provides = self._parseResolvableExpressionListForKey('PROVIDES')
77		self.requires = self._parseResolvableExpressionListForKey('REQUIRES')
78
79		self._checkTopLevelEntries()
80		self._checkProvides()
81		self._checkLibraryDependencies()
82		self._checkMisplacedDevelopLibraries()
83		self._checkGlobalWritableFiles()
84		self._checkUserSettingsFiles()
85		self._checkPostInstallScripts()
86
87		if self.strict and self.violationEncountered:
88			sysExit("packaging policy violation(s) in strict mode")
89
90	def _checkTopLevelEntries(self):
91		for entry in os.listdir(self.package.packagingDir):
92			if entry not in allowedTopLevelEntries:
93				self._violation('Invalid top-level package entry "%s"' % entry)
94
95	def _parseResolvableExpressionListForKey(self, keyName):
96		return self._parseResolvableExpressionList(
97			self.package.recipeKeys[keyName])
98
99	def _parseResolvableExpressionList(self, theList):
100		names = set()
101		for item in theList:
102			match = re.match(r'[^-/=!<>\s]+', item)
103			if match:
104				names.add(match.group(0))
105		return names
106
107	def _checkProvides(self):
108		# check if the package provides itself
109		if self.package.name not in self.provides:
110			self._violation('no matching self provides for "%s"'
111				% self.package.name)
112
113		# everything in bin/ must be declared as cmd:*
114		binDir = os.path.join(self.package.packagingDir, 'bin')
115		if os.path.exists(binDir):
116			for entry in os.listdir(binDir):
117				# ignore secondary architecture subdir
118				if entry == self.package.secondaryArchitecture:
119					continue
120				name = self._normalizeResolvableName('cmd:' + entry)
121				if name.lower() not in self.provides:
122					self._violation('no matching provides "%s" for "%s"'
123						% (name, 'bin/' + entry))
124
125		# library entries in lib[/<arch>] must be declared as lib:*[_<arch>]
126		libDir = os.path.join(self.package.packagingDir,
127			'lib' + self.secondaryArchSubDir)
128		if os.path.exists(libDir):
129			for entry in os.listdir(libDir):
130				suffixIndex = entry.find('.so')
131				if suffixIndex < 0:
132					continue
133
134				name = self._normalizeResolvableName(
135					'lib:' + entry[:suffixIndex] + self.secondaryArchSuffix)
136				if name.lower() not in self.provides:
137					self._violation('no matching provides "%s" for "%s"'
138						% (name, 'lib/' + entry))
139
140		# library entries in develop/lib[<arch>] must be declared as
141		# devel:*[_<arch>]
142		developLibDir = os.path.join(self.package.packagingDir,
143			'develop/lib' + self.secondaryArchSubDir)
144		if os.path.exists(developLibDir):
145			for entry in os.listdir(developLibDir):
146				suffixIndex = entry.find('.so')
147				if suffixIndex < 0:
148					suffixIndex = entry.find('.a')
149					if suffixIndex < 0:
150						continue
151
152				name = self._normalizeResolvableName(
153					'devel:' + entry[:suffixIndex] + self.secondaryArchSuffix)
154				if name.lower() not in self.provides:
155					self._violation('no matching provides "%s" for "%s"'
156						% (name, 'develop/lib/' + entry))
157
158	def _normalizeResolvableName(self, name):
159		# make name a valid resolvable name by replacing '-' with '_'
160		return name.replace('-', '_').lower()
161
162	def _checkLibraryDependencies(self):
163		# If there's no readelf (i.e. no binutils), there probably aren't any
164		# executables/libraries.
165		if not isCommandAvailable('readelf'):
166			return
167
168		# check all files in bin/, apps/ and lib[/<arch>]
169		for directory in ['bin', 'apps', 'lib' + self.secondaryArchSubDir]:
170			dir = os.path.join(self.package.packagingDir, directory)
171			if not os.path.exists(dir):
172				continue
173
174			for entry in os.listdir(dir):
175				path = os.path.join(dir, entry)
176				if os.path.isfile(path):
177					self._checkLibraryDependenciesOfFile(dir, path)
178				elif directory != "bin" and os.path.isdir(path):
179					for entry2 in os.listdir(path):
180						path2 = os.path.join(path, entry2)
181						if os.path.isfile(path2) and os.access(path2, os.X_OK):
182							self._checkLibraryDependenciesOfFile(path, path2)
183
184	def _checkLibraryDependenciesOfFile(self, dirPath, path):
185		# skip static libraries outright
186		if path.endswith('.a'):
187			return
188
189		# try to read the dynamic section of the file
190		try:
191			with open(os.devnull, "w") as devnull:
192				output = check_output(['readelf', '--dynamic', path],
193					stderr=devnull).decode('utf-8')
194		except:
195			return
196
197		libraries = set()
198		rpath = None
199		# extract the library names from the "(NEEDED)" lines of the output
200		for line in output.split('\n'):
201			if line.find('(NEEDED)') >= 0:
202				match = re.match(r'[^[]*\[(.*)].*', line)
203				if match:
204					libraries.add(os.path.basename(match.group(1)))
205			if line.find('(RPATH)') >= 0:
206				match = re.match(r'[^[]*\[(.*)].*', line)
207				if match:
208					rpath = match.group(1)
209
210		for library in libraries:
211			if self._isMissingLibraryDependency(library, dirPath, rpath):
212				if (library.startswith('libgcc') or
213					library.startswith('libstdc++') or
214					library.startswith('libsupc++')):
215					continue
216				self._violation('"%s" needs library "%s", but the '
217					'package doesn\'t seem to declare that as a '
218					'requirement' % (path, library))
219
220	def _isMissingLibraryDependency(self, library, dirPath, rpath):
221		if library.startswith('_APP_'):
222			return False
223
224		# the library might be provided by the package ($libDir)
225		libDir = os.path.join(self.package.packagingDir,
226			'lib' + self.secondaryArchSubDir + '/' + library)
227		if os.path.exists(libDir):
228			return False
229		if len(glob.glob(libDir + '*')) == 1:
230			return False
231
232		# the library might be provided by the package (%A/lib)
233		libDir = os.path.join(dirPath, 'lib/' + library)
234		if os.path.exists(libDir):
235			return False
236		if len(glob.glob(libDir + '*')) == 1:
237			return False
238
239		# the library might be provided by the package, same dir (%A)
240		libDir = os.path.join(dirPath, library)
241		if os.path.exists(libDir):
242			return False
243		if len(glob.glob(libDir + '*')) == 1:
244			return False
245
246		# the library might be provided by the package in rpath
247		if rpath is not None:
248			for rpath1 in rpath.split(':'):
249				if rpath1.find('/.self/') != -1:
250					rpathDir = os.path.join(self.package.packagingDir,
251						rpath1[rpath1.find('/.self/') + len('/.self/'):] + '/' + library)
252					if os.path.exists(rpathDir):
253						return False
254				elif rpath1.find('$ORIGIN') != -1:
255					rpathDir = os.path.join(dirPath,
256						rpath1[rpath1.find('$ORIGIN/') + len('$ORIGIN/'):] + '/' + library)
257					if os.path.exists(rpathDir):
258						return False
259
260		# not provided by the package -- check whether it is required explicitly
261		suffixIndex = library.find('.so')
262		resolvableName = None
263		if suffixIndex >= 0:
264			resolvableName = self._normalizeResolvableName(
265				'lib:' + library[:suffixIndex] + self.secondaryArchSuffix)
266			if resolvableName in self.requires:
267				return False
268
269		# The library might be provided by a sibling package.
270		providingPackage = None
271		for packageName in self.portPackagesProvides.keys():
272			packageProvides = self.portPackagesProvides[packageName]
273			if resolvableName in packageProvides:
274				providingPackage = packageName
275				break
276
277		if not providingPackage:
278			# Could be required implicitly by requiring (anything from) the
279			# package that provides the library. Find the library in the file
280			# system.
281			libraryPath = None
282			for directory in ['/boot/system/lib']:
283				path = directory + self.secondaryArchSubDir + '/' + library
284				if os.path.exists(path):
285					libraryPath = path
286					break
287
288			if not libraryPath:
289				# Don't complain if we're running on non-haiku host.
290				if os.path.exists('/boot/system/lib'):
291					self._violation('can\'t find used library "%s"' % library)
292				return False
293
294			# Find out which package the library belongs to.
295			providingPackage = self._getPackageProvidingPath(libraryPath)
296			if not providingPackage:
297				print('Warning: failed to determine the package providing "%s"'
298					% libraryPath)
299				return False
300
301			# Chop off ".hpkg" and the version part from the file name to get
302			# the package name.
303			packageName = providingPackage[:-5]
304			index = packageName.find('-')
305			if index >= 0:
306				packageName = packageName[:index]
307
308			packageProvides = self.requiredPackagesProvides.get(
309				providingPackage, [])
310
311		# Check whether the package is required.
312		if packageName in self.requires:
313			return False
314
315		# check whether any of the package's provides are required
316		for name in packageProvides:
317			if name in self.requires:
318				return False
319
320		return True
321
322	def _getPackageProvidingPath(self, path):
323		try:
324			with open(os.devnull, "w") as devnull:
325				output = check_output(
326					['catattr', '-d', 'SYS:PACKAGE_FILE', path], stderr=devnull).decode('utf-8')
327				if output.endswith('\n'):
328					output = output[:-1]
329				return output
330		except:
331			return None
332
333	def _getPackageProvides(self, package):
334		# get the package listing
335		try:
336			with open(os.devnull, "w") as devnull:
337				output = check_output(
338					[Configuration.getPackageCommand(), 'list', package],
339					stderr=devnull).decode('utf-8')
340		except:
341			return None
342
343		# extract the provides
344		provides = []
345		for line in output.split('\n'):
346			index = line.find('provides:')
347			if index >= 0:
348				index += 9
349				provides.append(line[index:].strip())
350
351		return self._parseResolvableExpressionList(provides)
352
353	def _checkMisplacedDevelopLibraries(self):
354		libDir = os.path.join(self.package.packagingDir,
355			'lib' + self.secondaryArchSubDir)
356		if not os.path.exists(libDir):
357			return
358
359		for entry in os.listdir(libDir):
360			if not entry.endswith('.a') and not entry.endswith('.la'):
361				continue
362
363			path = libDir + '/' + entry
364			self._violation('development library entry "%s" should be placed '
365				'in "develop/lib%s"' % (path, self.secondaryArchSubDir))
366
367	def _checkGlobalWritableFiles(self):
368		# Create a map for the declared global writable files and check them
369		# while at it.
370		types = {False: 'file', True: 'directory'}
371		globalWritableFiles = {}
372		fileTypes = {}
373		for item in self.package.recipeKeys['GLOBAL_WRITABLE_FILES']:
374			if item.strip().startswith('#'):
375				continue
376
377			components = ConfigParser.splitItemAndUnquote(item)
378			if components:
379				path = components[0]
380				directory = path
381				index = path.find('/')
382				if index >= 0:
383					directory = path[0:index]
384
385				isDirectory = False
386				updateType = None
387				if len(components) > 1:
388					if components[1] == 'directory':
389						isDirectory = True
390						if len(components) > 2:
391							updateType = components[2]
392					else:
393						updateType = components[1]
394
395				fileType = types[isDirectory]
396				fileTypes[components[0]] = fileType
397
398				if directory not in allowedWritableTopLevelDirectories:
399					self._violation('Package declares invalid global writable '
400						'%s "%s"' % (fileType, components[0]))
401
402				globalWritableFiles[components[0]] = updateType
403
404				if updateType:
405					absPath = os.path.join(self.package.packagingDir, path)
406					if not os.path.exists(absPath):
407						self._violation('Package declares non-existent global '
408							'writable %s "%s" as included' % (fileType, path))
409					elif os.path.isdir(absPath) != isDirectory:
410						self._violation('Package declares non-existent global '
411							'writable %s "%s", but it\'s a %s'
412							% (fileType, path, types[not isDirectory]))
413
414		# iterate through the writable directories in the package
415		for directory in allowedWritableTopLevelDirectories:
416			dir = os.path.join(self.package.packagingDir, directory)
417			if os.path.exists(dir):
418				self._checkGlobalWritableFilesRecursively(globalWritableFiles,
419					fileTypes, directory)
420
421	def _checkGlobalWritableFilesRecursively(self, globalWritableFiles,
422			fileTypes, path):
423		if path in globalWritableFiles:
424			if not globalWritableFiles[path]:
425				self._violation('Included "%s" declared as not included global '
426					'writable %s' % (path, fileTypes[path]))
427			return
428
429		absPath = os.path.join(self.package.packagingDir, path)
430		if not os.path.isdir(absPath):
431			self._violation('Included file "%s" not declared as global '
432				'writable file' % path)
433			return
434
435		# entry is a directory -- recurse
436		for entry in os.listdir(absPath):
437			self._checkGlobalWritableFilesRecursively(globalWritableFiles,
438				fileTypes, path + '/' + entry)
439
440	def _checkUserSettingsFiles(self):
441		for item in self.package.recipeKeys['USER_SETTINGS_FILES']:
442			if item.strip().startswith('#'):
443				continue
444
445			components = ConfigParser.splitItemAndUnquote(item)
446			if not components:
447				continue
448
449			if not components[0].startswith('settings/'):
450				self._violation('Package declares invalid user settings '
451					'file "%s"' % components[0])
452			if len(components) > 1 and components[1] == 'directory':
453				continue
454
455			if len(components) > 2:
456				template = os.path.join(self.package.packagingDir,
457					components[2])
458				if not os.path.exists(template):
459					self._violation('Package declares non-existent template '
460						'"%s" for user settings file "%s" as included'
461						% (components[2], components[0]))
462
463	def _checkPostInstallScripts(self):
464		# check whether declared scripts exist
465		declaredFiles = set()
466		for script in self.package.recipeKeys['POST_INSTALL_SCRIPTS']:
467			if script.lstrip().startswith('#'):
468				continue
469
470			components = ConfigParser.splitItemAndUnquote(script)
471			if not components:
472				continue
473			script = components[0]
474			declaredFiles.add(script)
475
476			absScript = os.path.join(self.package.packagingDir, script)
477			if not os.path.exists(absScript):
478				self._violation('Package declares non-existent post-install '
479					'script "%s"' % script)
480
481		# check whether existing scripts are declared
482		postInstallDir = 'boot/post-install'
483		dir = os.path.join(self.package.packagingDir, postInstallDir)
484		if os.path.exists(dir):
485			for script in os.listdir(dir):
486				path = postInstallDir + '/' + script
487				if path not in declaredFiles:
488					self._violation('script "%s" not declared as post-install '
489						'script' % path)
490
491	def _violation(self, message):
492		self.violationEncountered = True
493		if self.strict:
494			violation = 'POLICY ERROR: ' + message
495		else:
496			violation = 'POLICY WARNING: ' + message
497		warn(violation)
498		if self.port.versionedName not in Policy.violationsByPort:
499			Policy.violationsByPort[self.port.versionedName] = [violation]
500		else:
501			Policy.violationsByPort[self.port.versionedName].append(violation)
502