tool.cpp File Reference

Go to the source code of this file.

Functions
std::mutex &	outputRegistryMutex ()
std::unordered_set< std::string > &	outputRegistry ()
bool	registerOutputPath (const std::string &path)
void	unregisterOutputPath (const std::string &path)
int	main (int argc, const char **argv)

Function Documentation

main()

int main	(	int	argc,
		const char **	argv
	)

Definition at line 786 of file defer/tool.cpp.

                                      {
  InitLLVM InitLLVM(argc, argv);
 
  // Capture CWD before anything else can change it
  g_originalCwd = fs::current_path();
 
  // Hide all the LLVM internal options that aren't relevant to our tool
  cl::HideUnrelatedOptions(ToolCategory);
 
  cl::ParseCommandLineOptions(argc, argv, "ascii-defer transformation tool\n");
 
  fs::path outputDir = fs::path(OutputDirectoryOption.getValue());
  // Make output directory absolute relative to original CWD
  if (!outputDir.is_absolute()) {
    outputDir = g_originalCwd / outputDir;
  }
  fs::path inputRoot;
  if (!InputRootOption.getValue().empty()) {
    inputRoot = fs::path(InputRootOption.getValue());
  } else {
    inputRoot = fs::current_path();
  }
 
  // Make input root absolute for reliable path computation
  if (!inputRoot.is_absolute()) {
    std::error_code ec;
    inputRoot = fs::absolute(inputRoot, ec);
    if (ec) {
      llvm::errs() << "Failed to resolve input root path: " << ec.message() << "\n";
      return 1;
    }
  }
 
  std::vector<std::string> sourcePaths;
  for (const auto &path : SourcePaths) {
    if (!path.empty()) {
      sourcePaths.push_back(path);
    }
  }
 
  if (sourcePaths.empty()) {
    llvm::errs() << "No translation units specified for transformation. Provide positional source paths.\n";
    return 1;
  }
 
  if (fs::exists(outputDir)) {
    if (!fs::is_directory(outputDir)) {
      llvm::errs() << "Output path exists and is not a directory: " << outputDir.c_str() << "\n";
      return 1;
    }
  } else {
    std::error_code errorCode;
    fs::create_directories(outputDir, errorCode);
    if (errorCode) {
      llvm::errs() << "Failed to create output directory: " << outputDir.c_str() << " - " << errorCode.message()
                   << "\n";
      return 1;
    }
  }
 
  // Load compilation database
  std::string buildPath = BuildPath.getValue();
  if (buildPath.empty()) {
    buildPath = ".";
  }
  std::string errorMessage;
  std::unique_ptr<tooling::CompilationDatabase> compilations =
      tooling::CompilationDatabase::loadFromDirectory(buildPath, errorMessage);
  if (!compilations) {
    llvm::errs() << "Error loading compilation database from '" << buildPath << "': " << errorMessage << "\n";
    return 1;
  }
 
  tooling::ClangTool tool(*compilations, sourcePaths);
 
  // Build the list of arguments to prepend for system header resolution
  // LibTooling uses CC1 mode internally which has different include path handling than
  // the clang driver. We use -Xclang to pass CC1-level flags that properly configure
  // system include paths for LibTooling's CompilerInvocation.
  std::vector<std::string> prependArgs;
 
  // Try to find clang resource directory at runtime
  // Priority: 1) CLANG_RESOURCE_DIR compile-time path, 2) Runtime detection via common paths
  std::string resourceDir;
 
#ifdef CLANG_RESOURCE_DIR
  if (llvm::sys::fs::exists(CLANG_RESOURCE_DIR)) {
    resourceDir = CLANG_RESOURCE_DIR;
    llvm::errs() << "Using embedded clang resource directory: " << resourceDir << "\n";
  } else {
    llvm::errs() << "Embedded clang resource directory not found: " << CLANG_RESOURCE_DIR << "\n";
  }
#endif
 
  // Runtime detection if embedded path doesn't work
  if (resourceDir.empty()) {
    // Common locations for clang resource directories
    std::vector<std::string> searchPaths;
 
#ifdef __APPLE__
    // Homebrew LLVM on Apple Silicon
    searchPaths.push_back("/opt/homebrew/opt/llvm/lib/clang");
    // Homebrew LLVM on Intel Mac
    searchPaths.push_back("/usr/local/opt/llvm/lib/clang");
    // Xcode's clang
    searchPaths.push_back("/Applications/Xcode.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/lib/clang");
    // CommandLineTools clang
    searchPaths.push_back("/Library/Developer/CommandLineTools/usr/lib/clang");
#endif
#ifdef __linux__
    // System LLVM installations
    searchPaths.push_back("/usr/lib/llvm-22/lib/clang");
    searchPaths.push_back("/usr/lib/llvm-21/lib/clang");
    searchPaths.push_back("/usr/lib/llvm-20/lib/clang");
    searchPaths.push_back("/usr/lib/clang");
    searchPaths.push_back("/usr/local/lib/clang");
#endif
    // Universal fallback
    searchPaths.push_back("/usr/local/lib/clang");
 
    for (const auto& basePath : searchPaths) {
      if (!llvm::sys::fs::exists(basePath)) {
        continue;
      }
 
      // Find the highest version subdirectory
      std::error_code ec;
      std::string bestVersion;
      int bestMajor = 0;
 
      for (llvm::sys::fs::directory_iterator dir(basePath, ec), dirEnd;
           !ec && dir != dirEnd; dir.increment(ec)) {
        std::string name = llvm::sys::path::filename(dir->path()).str();
        // Parse version number (e.g., "22", "21.1.0", "21")
        int major = 0;
        if (sscanf(name.c_str(), "%d", &major) == 1 && major > bestMajor) {
          bestMajor = major;
          bestVersion = dir->path();
        }
      }
 
      if (!bestVersion.empty()) {
        resourceDir = bestVersion;
        llvm::errs() << "Found clang resource directory at runtime: " << resourceDir << "\n";
        break;
      }
    }
 
    if (resourceDir.empty()) {
      llvm::errs() << "Warning: Could not find clang resource directory\n";
    }
  }
 
  if (!resourceDir.empty()) {
    prependArgs.push_back(std::string("-resource-dir=") + resourceDir);
  }
 
  // Add target triple - LibTooling needs this to validate architecture-specific flags
  // Without a target, flags like -mavx2 cause "unsupported option for target ''" errors
#ifdef __APPLE__
  #ifdef __arm64__
    prependArgs.push_back("-target");
    prependArgs.push_back("arm64-apple-darwin");
    llvm::errs() << "Using target: arm64-apple-darwin\n";
  #else
    prependArgs.push_back("-target");
    prependArgs.push_back("x86_64-apple-darwin");
    llvm::errs() << "Using target: x86_64-apple-darwin\n";
  #endif
#elif defined(__linux__)
  #ifdef __aarch64__
    prependArgs.push_back("-target");
    prependArgs.push_back("aarch64-linux-gnu");
    llvm::errs() << "Using target: aarch64-linux-gnu\n";
  #else
    prependArgs.push_back("-target");
    prependArgs.push_back("x86_64-linux-gnu");
    llvm::errs() << "Using target: x86_64-linux-gnu\n";
  #endif
#endif
 
  // Override the sysroot for macOS. Homebrew's LLVM config file sets -isysroot
  // to CommandLineTools SDK, but we strip that from compile_commands.json and
  // set our own explicitly to ensure consistent behavior.
  std::string selectedSDK;
#ifdef __APPLE__
  {
    const char* sdkPaths[] = {
        // Xcode SDK (preferred - most complete headers and frameworks)
        "/Applications/Xcode.app/Contents/Developer/Platforms/"
        "MacOSX.platform/Developer/SDKs/MacOSX.sdk",
        // CommandLineTools SDK (fallback for users without Xcode)
        "/Library/Developer/CommandLineTools/SDKs/MacOSX.sdk",
    };
 
    for (const char* sdk : sdkPaths) {
      if (llvm::sys::fs::exists(sdk)) {
        selectedSDK = sdk;
        break;
      }
    }
 
    if (!selectedSDK.empty()) {
      prependArgs.push_back("-isysroot");
      prependArgs.push_back(selectedSDK);
      llvm::errs() << "Using macOS SDK: " << selectedSDK << "\n";
    } else {
      llvm::errs() << "Warning: No macOS SDK found, system headers may not be available\n";
    }
  }
#endif
 
  // Build list of system include paths to add as -isystem paths.
  // LibTooling (cc1 mode) doesn't automatically add system include paths like
  // the clang driver does, so we add them explicitly:
  // 1. Clang's builtin headers (stdbool.h, stddef.h, etc.) - FIRST so they shadow SDK builtins
  // 2. SDK's usr/include (stdio.h, stdlib.h, etc.) - for system headers
  std::vector<std::string> appendArgs;
  if (!resourceDir.empty()) {
    std::string builtinInclude = resourceDir + "/include";
    if (llvm::sys::fs::exists(builtinInclude)) {
      appendArgs.push_back("-isystem");
      appendArgs.push_back(builtinInclude);
      llvm::errs() << "Added clang builtin -isystem: " << builtinInclude << "\n";
    }
  }
  // NOTE: We intentionally do NOT add SDK's usr/include to the -isystem path.
  // LibTooling on LLVM 21 has a bug where __has_include_next() evaluates even
  // in non-taken preprocessor branches and creates VFS entries that can't be opened.
  // The clang builtin stdbool.h has:
  //   #if defined(__MVS__) && __has_include_next(<stdbool.h>)
  // Even though __MVS__ is not defined on macOS, LLVM 21's LibTooling still tries
  // to resolve __has_include_next(<stdbool.h>), looks in SDK/usr/include, and fails
  // because stdbool.h doesn't exist there (it's a builtin header).
  //
  // By not adding SDK/usr/include, the __has_include_next won't find any entry
  // and will return false without trying to open a non-existent file.
  // The -isysroot flag still provides access to SDK headers through framework paths.
 
  // Single consolidated argument adjuster that:
  // 1. Preserves compiler path (first arg)
  // 2. Inserts prepend args immediately after compiler (-nostdlibinc, -resource-dir, -isysroot)
  // 3. Strips unnecessary flags from remaining args
  // 4. Adds system include paths as -isystem at the end (after project -I paths)
  // 5. Adds defer tool parsing define BEFORE the "--" separator
  std::string inputRootStr = inputRoot.string();
  auto consolidatedAdjuster = [prependArgs, appendArgs, inputRootStr](const tooling::CommandLineArguments &args, StringRef) {
    // Helper to check if a path is under the project root
    auto isProjectPath = [&inputRootStr](const std::string &path) -> bool {
      if (inputRootStr.empty()) return false;
      // Normalize both paths for comparison
      std::error_code ec;
      auto normalizedPath = fs::canonical(path, ec);
      if (ec) return false; // Path doesn't exist or can't be resolved
      auto normalizedRoot = fs::canonical(inputRootStr, ec);
      if (ec) return false;
      // Check if the path starts with the root
      std::string pathStr = normalizedPath.string();
      std::string rootStr = normalizedRoot.string();
      if (pathStr.find(rootStr) != 0) return false;
      // Exclude .deps-cache directory - these are cached dependencies that use
      // angled includes (<header.h>) and need -isystem, not -iquote
      if (pathStr.find("/.deps-cache/") != std::string::npos) return false;
      return true;
    };
    tooling::CommandLineArguments result;
 
    if (args.empty()) {
      return result;
    }
 
    // First: preserve the compiler path (first argument)
    result.push_back(args[0]);
 
    // Second: add the prepend args right after the compiler
    for (const auto &arg : prependArgs) {
      result.push_back(arg);
    }
 
    // Third: process remaining arguments, stripping unnecessary flags
    // IMPORTANT: Convert -I to -iquote for project include paths.
    // Clang include search order for <header.h>:  -I paths -> -isystem paths -> system paths
    // Clang include search order for "header.h":  -iquote paths -> -I paths -> -isystem paths -> system paths
    //
    // The problem: Project -I paths are searched BEFORE -isystem paths for <stdio.h>.
    // This causes LibTooling to look for <stdio.h> in the project's lib/ directory first.
    //
    // The fix: Convert project -I to -iquote. Then <stdio.h> skips project paths entirely
    // and finds system headers in -isystem paths instead.
    //
    // ALSO: Collect all -isystem paths and reorder them so clang builtins come FIRST.
    // LibTooling on LLVM 21 has a VFS bug where it creates phantom entries for headers
    // in -isystem directories even when the header doesn't exist there.
    std::vector<std::string> collectedIsystemPaths;
    bool foundSeparator = false;
    size_t separatorIndex = 0;
    for (size_t i = 1; i < args.size(); ++i) {
      const std::string &arg = args[i];
 
      // When we hit "--", note its position and break
      if (arg == "--") {
        foundSeparator = true;
        separatorIndex = i;
        break;
      }
 
      // Skip sanitizer flags
      if (arg.find("-fsanitize") != std::string::npos)
        continue;
      if (arg.find("-fno-sanitize") != std::string::npos)
        continue;
      // Skip debug info flags (not needed for AST parsing)
      if (arg == "-g" || arg == "-g2" || arg == "-g3")
        continue;
      if (arg == "-fno-eliminate-unused-debug-types")
        continue;
      if (arg == "-fno-inline")
        continue;
      // Strip -resource-dir flags and their arguments - we added our embedded path
      if (arg == "-resource-dir") {
        ++i;
        continue;
      }
      if (arg.find("-resource-dir=") == 0)
        continue;
      // Strip -isysroot flags and their arguments - we added our embedded SDK path
      if (arg == "-isysroot") {
        ++i;
        continue;
      }
      if (arg.find("-isysroot=") == 0 || (arg.find("-isysroot") == 0 && arg.length() > 9))
        continue;
 
      // Collect -isystem paths instead of passing them through
      // We'll add them at the end in the correct order (clang builtins first)
      if (arg == "-isystem" && i + 1 < args.size()) {
        collectedIsystemPaths.push_back(args[++i]);
        continue;
      }
      if (arg.find("-isystem") == 0 && arg.length() > 8) {
        collectedIsystemPaths.push_back(arg.substr(8));
        continue;
      }
 
      // Convert -I to -iquote for project include paths
      // Convert -I to -isystem for dependency paths (so they come after our clang builtins)
      // This prevents <stdbool.h> from being searched in dependency directories
      // before our clang builtin path
      if (arg == "-I" && i + 1 < args.size()) {
        // -I /path/to/dir (separate argument)
        const std::string &includePath = args[++i];
        if (isProjectPath(includePath)) {
          result.push_back("-iquote");
          result.push_back(includePath);
        } else {
          // Collect dependency -I paths to add as -isystem after our builtins
          collectedIsystemPaths.push_back(includePath);
        }
        continue;
      }
      if (arg.find("-I") == 0 && arg.length() > 2) {
        // -I/path/to/dir (combined form)
        std::string includePath = arg.substr(2);
        if (isProjectPath(includePath)) {
          result.push_back("-iquote");
          result.push_back(includePath);
        } else {
          // Collect dependency -I paths to add as -isystem after our builtins
          collectedIsystemPaths.push_back(includePath);
        }
        continue;
      }
 
      result.push_back(arg);
    }
 
    // Fourth: add system include paths in the correct order
    // Order matters for LibTooling on LLVM 21 - clang builtins MUST come first
    // to shadow any phantom VFS entries that might be created for other paths
    for (const auto &arg : appendArgs) {
      result.push_back(arg);
    }
    // Then add the collected -isystem paths from the compilation database
    for (const auto &path : collectedIsystemPaths) {
      result.push_back("-isystem");
      result.push_back(path);
    }
 
    // Fifth: add the defer tool define and separator
    result.push_back("-DASCIICHAT_DEFER_TOOL_PARSING");
    if (foundSeparator) {
      result.push_back("--");
      // Copy any remaining args after "--"
      for (size_t i = separatorIndex + 1; i < args.size(); ++i) {
        result.push_back(args[i]);
      }
    }
 
    return result;
  };
  tool.appendArgumentsAdjuster(consolidatedAdjuster);
 
  // Debug: Print the final command line arguments
  tool.appendArgumentsAdjuster([](const tooling::CommandLineArguments &args, StringRef filename) {
    llvm::errs() << "Final command for " << filename << ":\n";
    for (const auto &arg : args) {
      llvm::errs() << "  " << arg << "\n";
    }
    return args;
  });
 
  DeferActionFactory actionFactory(outputDir, inputRoot);
  const int executionResult = tool.run(&actionFactory);
  if (executionResult != 0) {
    llvm::errs() << "Defer transformation failed with code " << executionResult << "\n";
  }
  return executionResult;
}

outputRegistry()

std::unordered_set< std::string > & outputRegistry

(

)

Definition at line 39 of file defer/tool.cpp.

                                              {
  static std::unordered_set<std::string> registry;
  return registry;
}

Referenced by registerOutputPath(), and unregisterOutputPath().

outputRegistryMutex()

std::mutex & outputRegistryMutex

(

)

Definition at line 34 of file defer/tool.cpp.

                                {
  static std::mutex mutex;
  return mutex;
}

Referenced by registerOutputPath(), and unregisterOutputPath().

registerOutputPath()

bool registerOutputPath

(

const std::string &

path

)

Definition at line 44 of file defer/tool.cpp.

                                               {
  std::lock_guard<std::mutex> guard(outputRegistryMutex());
  auto &registry = outputRegistry();
  bool inserted = registry.insert(path).second;
  return inserted;
}

References outputRegistry(), and outputRegistryMutex().

unregisterOutputPath()

void unregisterOutputPath

(

const std::string &

path

)

Definition at line 51 of file defer/tool.cpp.

                                                 {
  std::lock_guard<std::mutex> guard(outputRegistryMutex());
  outputRegistry().erase(path);
}

References outputRegistry(), and outputRegistryMutex().