DependencyScanningFilesystem.cpp revision 360784 
1//===- DependencyScanningFilesystem.cpp - clang-scan-deps fs --------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
9#include "clang/Tooling/DependencyScanning/DependencyScanningFilesystem.h"
10#include "clang/Lex/DependencyDirectivesSourceMinimizer.h"
11#include "llvm/Support/MemoryBuffer.h"
12#include "llvm/Support/Threading.h"
13
14using namespace clang;
15using namespace tooling;
16using namespace dependencies;
17
18CachedFileSystemEntry CachedFileSystemEntry::createFileEntry(
19    StringRef Filename, llvm::vfs::FileSystem &FS, bool Minimize) {
20  // Load the file and its content from the file system.
21  llvm::ErrorOr<std::unique_ptr<llvm::vfs::File>> MaybeFile =
22      FS.openFileForRead(Filename);
23  if (!MaybeFile)
24    return MaybeFile.getError();
25  llvm::ErrorOr<llvm::vfs::Status> Stat = (*MaybeFile)->status();
26  if (!Stat)
27    return Stat.getError();
28
29  llvm::vfs::File &F = **MaybeFile;
30  llvm::ErrorOr<std::unique_ptr<llvm::MemoryBuffer>> MaybeBuffer =
31      F.getBuffer(Stat->getName());
32  if (!MaybeBuffer)
33    return MaybeBuffer.getError();
34
35  llvm::SmallString<1024> MinimizedFileContents;
36  // Minimize the file down to directives that might affect the dependencies.
37  const auto &Buffer = *MaybeBuffer;
38  SmallVector<minimize_source_to_dependency_directives::Token, 64> Tokens;
39  if (!Minimize || minimizeSourceToDependencyDirectives(
40                       Buffer->getBuffer(), MinimizedFileContents, Tokens)) {
41    // Use the original file unless requested otherwise, or
42    // if the minimization failed.
43    // FIXME: Propage the diagnostic if desired by the client.
44    CachedFileSystemEntry Result;
45    Result.MaybeStat = std::move(*Stat);
46    Result.Contents.reserve(Buffer->getBufferSize() + 1);
47    Result.Contents.append(Buffer->getBufferStart(), Buffer->getBufferEnd());
48    // Implicitly null terminate the contents for Clang's lexer.
49    Result.Contents.push_back('\0');
50    Result.Contents.pop_back();
51    return Result;
52  }
53
54  CachedFileSystemEntry Result;
55  size_t Size = MinimizedFileContents.size();
56  Result.MaybeStat = llvm::vfs::Status(Stat->getName(), Stat->getUniqueID(),
57                                       Stat->getLastModificationTime(),
58                                       Stat->getUser(), Stat->getGroup(), Size,
59                                       Stat->getType(), Stat->getPermissions());
60  // The contents produced by the minimizer must be null terminated.
61  assert(MinimizedFileContents.data()[MinimizedFileContents.size()] == '\0' &&
62         "not null terminated contents");
63  // Even though there's an implicit null terminator in the minimized contents,
64  // we want to temporarily make it explicit. This will ensure that the
65  // std::move will preserve it even if it needs to do a copy if the
66  // SmallString still has the small capacity.
67  MinimizedFileContents.push_back('\0');
68  Result.Contents = std::move(MinimizedFileContents);
69  // Now make the null terminator implicit again, so that Clang's lexer can find
70  // it right where the buffer ends.
71  Result.Contents.pop_back();
72
73  // Compute the skipped PP ranges that speedup skipping over inactive
74  // preprocessor blocks.
75  llvm::SmallVector<minimize_source_to_dependency_directives::SkippedRange, 32>
76      SkippedRanges;
77  minimize_source_to_dependency_directives::computeSkippedRanges(Tokens,
78                                                                 SkippedRanges);
79  PreprocessorSkippedRangeMapping Mapping;
80  for (const auto &Range : SkippedRanges) {
81    if (Range.Length < 16) {
82      // Ignore small ranges as non-profitable.
83      // FIXME: This is a heuristic, its worth investigating the tradeoffs
84      // when it should be applied.
85      continue;
86    }
87    Mapping[Range.Offset] = Range.Length;
88  }
89  Result.PPSkippedRangeMapping = std::move(Mapping);
90
91  return Result;
92}
93
94CachedFileSystemEntry
95CachedFileSystemEntry::createDirectoryEntry(llvm::vfs::Status &&Stat) {
96  assert(Stat.isDirectory() && "not a directory!");
97  auto Result = CachedFileSystemEntry();
98  Result.MaybeStat = std::move(Stat);
99  return Result;
100}
101
102DependencyScanningFilesystemSharedCache::
103    DependencyScanningFilesystemSharedCache() {
104  // This heuristic was chosen using a empirical testing on a
105  // reasonably high core machine (iMacPro 18 cores / 36 threads). The cache
106  // sharding gives a performance edge by reducing the lock contention.
107  // FIXME: A better heuristic might also consider the OS to account for
108  // the different cost of lock contention on different OSes.
109  NumShards = std::max(2u, llvm::hardware_concurrency() / 4);
110  CacheShards = std::make_unique<CacheShard[]>(NumShards);
111}
112
113/// Returns a cache entry for the corresponding key.
114///
115/// A new cache entry is created if the key is not in the cache. This is a
116/// thread safe call.
117DependencyScanningFilesystemSharedCache::SharedFileSystemEntry &
118DependencyScanningFilesystemSharedCache::get(StringRef Key) {
119  CacheShard &Shard = CacheShards[llvm::hash_value(Key) % NumShards];
120  std::unique_lock<std::mutex> LockGuard(Shard.CacheLock);
121  auto It = Shard.Cache.try_emplace(Key);
122  return It.first->getValue();
123}
124
125/// Whitelist file extensions that should be minimized, treating no extension as
126/// a source file that should be minimized.
127///
128/// This is kinda hacky, it would be better if we knew what kind of file Clang
129/// was expecting instead.
130static bool shouldMinimize(StringRef Filename) {
131  StringRef Ext = llvm::sys::path::extension(Filename);
132  if (Ext.empty())
133    return true; // C++ standard library
134  return llvm::StringSwitch<bool>(Ext)
135    .CasesLower(".c", ".cc", ".cpp", ".c++", ".cxx", true)
136    .CasesLower(".h", ".hh", ".hpp", ".h++", ".hxx", true)
137    .CasesLower(".m", ".mm", true)
138    .CasesLower(".i", ".ii", ".mi", ".mmi", true)
139    .CasesLower(".def", ".inc", true)
140    .Default(false);
141}
142
143
144static bool shouldCacheStatFailures(StringRef Filename) {
145  StringRef Ext = llvm::sys::path::extension(Filename);
146  if (Ext.empty())
147    return false; // This may be the module cache directory.
148  return shouldMinimize(Filename); // Only cache stat failures on source files.
149}
150
151llvm::ErrorOr<const CachedFileSystemEntry *>
152DependencyScanningWorkerFilesystem::getOrCreateFileSystemEntry(
153    const StringRef Filename) {
154  if (const CachedFileSystemEntry *Entry = getCachedEntry(Filename)) {
155    return Entry;
156  }
157
158  // FIXME: Handle PCM/PCH files.
159  // FIXME: Handle module map files.
160
161  bool KeepOriginalSource = IgnoredFiles.count(Filename) ||
162                            !shouldMinimize(Filename);
163  DependencyScanningFilesystemSharedCache::SharedFileSystemEntry
164      &SharedCacheEntry = SharedCache.get(Filename);
165  const CachedFileSystemEntry *Result;
166  {
167    std::unique_lock<std::mutex> LockGuard(SharedCacheEntry.ValueLock);
168    CachedFileSystemEntry &CacheEntry = SharedCacheEntry.Value;
169
170    if (!CacheEntry.isValid()) {
171      llvm::vfs::FileSystem &FS = getUnderlyingFS();
172      auto MaybeStatus = FS.status(Filename);
173      if (!MaybeStatus) {
174        if (!shouldCacheStatFailures(Filename))
175          // HACK: We need to always restat non source files if the stat fails.
176          //   This is because Clang first looks up the module cache and module
177          //   files before building them, and then looks for them again. If we
178          //   cache the stat failure, it won't see them the second time.
179          return MaybeStatus.getError();
180        else
181          CacheEntry = CachedFileSystemEntry(MaybeStatus.getError());
182      } else if (MaybeStatus->isDirectory())
183        CacheEntry = CachedFileSystemEntry::createDirectoryEntry(
184            std::move(*MaybeStatus));
185      else
186        CacheEntry = CachedFileSystemEntry::createFileEntry(
187            Filename, FS, !KeepOriginalSource);
188    }
189
190    Result = &CacheEntry;
191  }
192
193  // Store the result in the local cache.
194  setCachedEntry(Filename, Result);
195  return Result;
196}
197
198llvm::ErrorOr<llvm::vfs::Status>
199DependencyScanningWorkerFilesystem::status(const Twine &Path) {
200  SmallString<256> OwnedFilename;
201  StringRef Filename = Path.toStringRef(OwnedFilename);
202  const llvm::ErrorOr<const CachedFileSystemEntry *> Result =
203      getOrCreateFileSystemEntry(Filename);
204  if (!Result)
205    return Result.getError();
206  return (*Result)->getStatus();
207}
208
209namespace {
210
211/// The VFS that is used by clang consumes the \c CachedFileSystemEntry using
212/// this subclass.
213class MinimizedVFSFile final : public llvm::vfs::File {
214public:
215  MinimizedVFSFile(std::unique_ptr<llvm::MemoryBuffer> Buffer,
216                   llvm::vfs::Status Stat)
217      : Buffer(std::move(Buffer)), Stat(std::move(Stat)) {}
218
219  llvm::ErrorOr<llvm::vfs::Status> status() override { return Stat; }
220
221  const llvm::MemoryBuffer *getBufferPtr() const { return Buffer.get(); }
222
223  llvm::ErrorOr<std::unique_ptr<llvm::MemoryBuffer>>
224  getBuffer(const Twine &Name, int64_t FileSize, bool RequiresNullTerminator,
225            bool IsVolatile) override {
226    return std::move(Buffer);
227  }
228
229  std::error_code close() override { return {}; }
230
231private:
232  std::unique_ptr<llvm::MemoryBuffer> Buffer;
233  llvm::vfs::Status Stat;
234};
235
236llvm::ErrorOr<std::unique_ptr<llvm::vfs::File>>
237createFile(const CachedFileSystemEntry *Entry,
238           ExcludedPreprocessorDirectiveSkipMapping *PPSkipMappings) {
239  if (Entry->isDirectory())
240    return llvm::ErrorOr<std::unique_ptr<llvm::vfs::File>>(
241        std::make_error_code(std::errc::is_a_directory));
242  llvm::ErrorOr<StringRef> Contents = Entry->getContents();
243  if (!Contents)
244    return Contents.getError();
245  auto Result = std::make_unique<MinimizedVFSFile>(
246      llvm::MemoryBuffer::getMemBuffer(*Contents, Entry->getName(),
247                                       /*RequiresNullTerminator=*/false),
248      *Entry->getStatus());
249  if (!Entry->getPPSkippedRangeMapping().empty() && PPSkipMappings)
250    (*PPSkipMappings)[Result->getBufferPtr()] =
251        &Entry->getPPSkippedRangeMapping();
252  return llvm::ErrorOr<std::unique_ptr<llvm::vfs::File>>(
253      std::unique_ptr<llvm::vfs::File>(std::move(Result)));
254}
255
256} // end anonymous namespace
257
258llvm::ErrorOr<std::unique_ptr<llvm::vfs::File>>
259DependencyScanningWorkerFilesystem::openFileForRead(const Twine &Path) {
260  SmallString<256> OwnedFilename;
261  StringRef Filename = Path.toStringRef(OwnedFilename);
262
263  const llvm::ErrorOr<const CachedFileSystemEntry *> Result =
264      getOrCreateFileSystemEntry(Filename);
265  if (!Result)
266    return Result.getError();
267  return createFile(Result.get(), PPSkipMappings);
268}
269