ninja-build: vendor as full-fork recipe (path=source, patches baked)
This commit is contained in:
@@ -27,7 +27,6 @@ recipes/**/source.tar
|
||||
recipes/**/source.tar.tmp
|
||||
recipes/**/source.pre-preservation-test/
|
||||
local/recipes/archives/uutils-tar/source
|
||||
local/recipes/dev/ninja-build/source
|
||||
|
||||
# Bump-recipe cache used by local/scripts/bump-graphics-recipes.sh.
|
||||
# Cache is fetched from upstream release indexes; reproducible from script run.
|
||||
|
||||
@@ -1,9 +1,12 @@
|
||||
[source]
|
||||
git = "https://github.com/ninja-build/ninja"
|
||||
rev = "v1.13.1"
|
||||
patches = [
|
||||
"redox.patch",
|
||||
]
|
||||
# Vendored local fork (full-fork model): builds from the committed source/
|
||||
# tree (offline, reproducible). source/ = pristine upstream + the patches below
|
||||
# BAKED IN. .patch files kept tracked so a version bump can re-apply them via
|
||||
# sync-recipe-source.sh.
|
||||
# upstream: https://github.com/ninja-build/ninja
|
||||
# rev: v1.13.1
|
||||
# patch (baked): redox.patch
|
||||
path = "source"
|
||||
|
||||
[build]
|
||||
template = "cmake"
|
||||
|
||||
@@ -0,0 +1,26 @@
|
||||
# Copyright 2014 Google Inc. All Rights Reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
# This isn't meant to be authoritative, but it's good enough to be useful.
|
||||
# Still use your best judgement for formatting decisions: clang-format
|
||||
# sometimes makes strange choices.
|
||||
|
||||
BasedOnStyle: Google
|
||||
AllowShortFunctionsOnASingleLine: Inline
|
||||
AllowShortIfStatementsOnASingleLine: false
|
||||
AllowShortLoopsOnASingleLine: false
|
||||
ConstructorInitializerAllOnOneLineOrOnePerLine: false
|
||||
Cpp11BracedListStyle: false
|
||||
IndentCaseLabels: false
|
||||
DerivePointerBinding: false
|
||||
@@ -0,0 +1,19 @@
|
||||
---
|
||||
Checks: '
|
||||
,readability-avoid-const-params-in-decls,
|
||||
,readability-inconsistent-declaration-parameter-name,
|
||||
,readability-non-const-parameter,
|
||||
,readability-redundant-string-cstr,
|
||||
,readability-redundant-string-init,
|
||||
,readability-simplify-boolean-expr,
|
||||
,cppcoreguidelines-pro-type-cstyle-cast,
|
||||
'
|
||||
WarningsAsErrors: '
|
||||
,readability-avoid-const-params-in-decls,
|
||||
,readability-inconsistent-declaration-parameter-name,
|
||||
,readability-non-const-parameter,
|
||||
,readability-redundant-string-cstr,
|
||||
,readability-redundant-string-init,
|
||||
,readability-simplify-boolean-expr,
|
||||
,cppcoreguidelines-pro-type-cstyle-cast,
|
||||
'
|
||||
@@ -0,0 +1,11 @@
|
||||
root = true
|
||||
|
||||
[*]
|
||||
charset = utf-8
|
||||
indent_style = space
|
||||
indent_size = 2
|
||||
insert_final_newline = true
|
||||
end_of_line = lf
|
||||
|
||||
[CMakeLists.txt]
|
||||
indent_style = tab
|
||||
@@ -0,0 +1,7 @@
|
||||
version: 2
|
||||
updates:
|
||||
# Maintain dependencies for GitHub Actions
|
||||
- package-ecosystem: "github-actions"
|
||||
directory: "/"
|
||||
schedule:
|
||||
interval: "weekly"
|
||||
@@ -0,0 +1,70 @@
|
||||
name: ci-linux-musl
|
||||
|
||||
on:
|
||||
workflow_dispatch:
|
||||
pull_request:
|
||||
push:
|
||||
branches: ['**']
|
||||
tags-ignore: ['**'] # Don't trigger on tag pushes
|
||||
release:
|
||||
types: [published]
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.ref }}
|
||||
cancel-in-progress: true
|
||||
|
||||
permissions: {}
|
||||
|
||||
jobs:
|
||||
build:
|
||||
runs-on: ubuntu-24.04
|
||||
container: alpine:edge
|
||||
permissions:
|
||||
contents: read
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
build_method: ["python", "cmake"]
|
||||
|
||||
steps:
|
||||
- name: Host - checkout
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
fetch-depth: 0
|
||||
persist-credentials: false
|
||||
|
||||
- name: Install ninja build optional dependencies
|
||||
run: apk update && apk add -u --no-cache python3 build-base cmake re2c
|
||||
|
||||
- name: Configure ninja build
|
||||
if: matrix.build_method == 'cmake'
|
||||
run: cmake -B build -D CMAKE_BUILD_TYPE="Release" -D CMAKE_COMPILE_WARNING_AS_ERROR="ON"
|
||||
|
||||
- name: Cmake Build ninja
|
||||
if: matrix.build_method == 'cmake'
|
||||
run: cmake --build build --parallel --config Release
|
||||
|
||||
- name: Cmake test ninja
|
||||
if: matrix.build_method == 'cmake'
|
||||
run: build/ninja_test --gtest_color=yes
|
||||
|
||||
- name: Python Build ninja
|
||||
if: matrix.build_method == 'python'
|
||||
run: python3 configure.py --warnings-as-errors --bootstrap --verbose
|
||||
|
||||
- name: Python test ninja
|
||||
if: matrix.build_method == 'python'
|
||||
run: |
|
||||
./ninja all
|
||||
python3 misc/ninja_syntax_test.py
|
||||
# python3 misc/output_test.py
|
||||
|
||||
- name: Move ninja binary
|
||||
if: matrix.build_method == 'cmake'
|
||||
run: mv -f build/ninja ninja
|
||||
|
||||
- name: ninja-ninja --version
|
||||
run: ./ninja --version >> $GITHUB_STEP_SUMMARY
|
||||
|
||||
- name: binary info via file
|
||||
run: file ./ninja >> $GITHUB_STEP_SUMMARY
|
||||
@@ -0,0 +1,181 @@
|
||||
name: Linux
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
push:
|
||||
branches: ['**']
|
||||
tags-ignore: ['**'] # Don't trigger on tag pushes
|
||||
release:
|
||||
types: [published]
|
||||
|
||||
jobs:
|
||||
fedora:
|
||||
runs-on: [ubuntu-latest]
|
||||
container:
|
||||
image: fedora:40
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- name: Install dependencies
|
||||
run: dnf install -y ninja-build cmake gtest-devel re2c clang util-linux clang-tools-extra
|
||||
- name: Linting
|
||||
run: misc/ci.py
|
||||
- name: Configure with CMake
|
||||
run: cmake -Bbuild -G"Ninja Multi-Config" -DNINJA_CLANG_TIDY=1
|
||||
- name: Build debug ninja
|
||||
run: CLICOLOR_FORCE=1 ninja
|
||||
working-directory: build
|
||||
- name: Test debug ninja
|
||||
working-directory: build/Debug
|
||||
run: |
|
||||
./ninja_test --gtest_color=yes
|
||||
../../misc/output_test.py
|
||||
../../misc/jobserver_test.py
|
||||
- name: Build release ninja
|
||||
run: CLICOLOR_FORCE=1 ninja -f build-Release.ninja
|
||||
working-directory: build
|
||||
- name: Test release ninja
|
||||
working-directory: build/Release
|
||||
run: |
|
||||
./ninja_test --gtest_color=yes
|
||||
../../misc/output_test.py
|
||||
../../misc/jobserver_test.py
|
||||
|
||||
build:
|
||||
strategy:
|
||||
matrix:
|
||||
host-os: ["ubuntu-latest", "ubuntu-24.04-arm"]
|
||||
include:
|
||||
- host-os: "ubuntu-24.04-arm"
|
||||
arch: "-aarch64"
|
||||
fail-fast: false
|
||||
defaults:
|
||||
run:
|
||||
shell: bash
|
||||
runs-on: ${{ matrix.host-os }}
|
||||
container: rockylinux/rockylinux:8
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- uses: codespell-project/actions-codespell@master
|
||||
with:
|
||||
ignore_words_list: fo,wee,addin,notin
|
||||
- name: Install dependencies
|
||||
run: |
|
||||
dnf install -y make gcc-c++ libasan clang-analyzer cmake dnf-plugins-core epel-release
|
||||
dnf config-manager --set-enabled powertools
|
||||
dnf install -y gtest-devel p7zip p7zip-plugins ninja-build
|
||||
|
||||
- name: Build debug ninja
|
||||
env:
|
||||
CFLAGS: -fstack-protector-all -fsanitize=address
|
||||
CXXFLAGS: -fstack-protector-all -fsanitize=address
|
||||
run: |
|
||||
scan-build -o scanlogs cmake -GNinja -DCMAKE_BUILD_TYPE=Debug -B debug-build
|
||||
scan-build -o scanlogs cmake --build debug-build --parallel --config Debug
|
||||
|
||||
- name: Test debug ninja
|
||||
run: ASAN_OPTIONS=detect_leaks=0 ./ninja_test
|
||||
working-directory: debug-build
|
||||
|
||||
- name: Build release ninja
|
||||
run: |
|
||||
cmake -GNinja -DCMAKE_BUILD_TYPE=Release -B release-build -DCMAKE_COMPILE_WARNING_AS_ERROR=1
|
||||
cmake --build release-build --parallel --config Release
|
||||
strip release-build/ninja
|
||||
|
||||
- name: Test release ninja
|
||||
run: ./ninja_test
|
||||
working-directory: release-build
|
||||
|
||||
- name: Create ninja archive
|
||||
run: |
|
||||
mkdir artifact
|
||||
7z a artifact/ninja-linux${{ matrix.arch }}.zip ./release-build/ninja
|
||||
|
||||
# Upload ninja binary archive as an artifact
|
||||
- name: Upload artifact
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: ninja${{ matrix.arch }}-binary-archives
|
||||
path: artifact
|
||||
|
||||
- name: Upload release asset
|
||||
if: github.event.action == 'published'
|
||||
uses: actions/upload-release-asset@v1
|
||||
env:
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
with:
|
||||
upload_url: ${{ github.event.release.upload_url }}
|
||||
asset_path: ./artifact/ninja-linux${{ matrix.arch }}.zip
|
||||
asset_name: ninja-linux${{ matrix.arch }}.zip
|
||||
asset_content_type: application/zip
|
||||
|
||||
test:
|
||||
runs-on: [ubuntu-latest]
|
||||
container:
|
||||
image: ubuntu:20.04
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- name: Install dependencies
|
||||
run: |
|
||||
apt update
|
||||
apt install -y python3-pytest ninja-build python3-pip clang libgtest-dev
|
||||
pip3 install cmake==3.17.*
|
||||
- name: Configure (GCC)
|
||||
run: cmake -Bbuild-gcc -DCMAKE_BUILD_TYPE=Debug -G'Ninja Multi-Config'
|
||||
|
||||
- name: Build (GCC, Debug)
|
||||
run: cmake --build build-gcc --config Debug
|
||||
- name: Unit tests (GCC, Debug)
|
||||
run: ./build-gcc/Debug/ninja_test
|
||||
- name: Python tests (GCC, Debug)
|
||||
run: pytest-3 --color=yes ../..
|
||||
working-directory: build-gcc/Debug
|
||||
|
||||
- name: Build (GCC, Release)
|
||||
run: cmake --build build-gcc --config Release
|
||||
- name: Unit tests (GCC, Release)
|
||||
run: ./build-gcc/Release/ninja_test
|
||||
- name: Python tests (GCC, Release)
|
||||
run: pytest-3 --color=yes ../..
|
||||
working-directory: build-gcc/Release
|
||||
|
||||
- name: Configure (Clang)
|
||||
run: CC=clang CXX=clang++ cmake -Bbuild-clang -DCMAKE_BUILD_TYPE=Debug -G'Ninja Multi-Config'
|
||||
|
||||
- name: Build (Clang, Debug)
|
||||
run: cmake --build build-clang --config Debug
|
||||
- name: Unit tests (Clang, Debug)
|
||||
run: ./build-clang/Debug/ninja_test
|
||||
- name: Python tests (Clang, Debug)
|
||||
run: pytest-3 --color=yes ../..
|
||||
working-directory: build-clang/Debug
|
||||
|
||||
- name: Build (Clang, Release)
|
||||
run: cmake --build build-clang --config Release
|
||||
- name: Unit tests (Clang, Release)
|
||||
run: ./build-clang/Release/ninja_test
|
||||
- name: Python tests (Clang, Release)
|
||||
run: pytest-3 --color=yes ../..
|
||||
working-directory: build-clang/Release
|
||||
|
||||
build-with-python:
|
||||
runs-on: [ubuntu-latest]
|
||||
container:
|
||||
image: ${{ matrix.image }}
|
||||
strategy:
|
||||
matrix:
|
||||
image: ["ubuntu:20.04", "ubuntu:22.04", "ubuntu:24.04"]
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- name: Install dependencies
|
||||
run: |
|
||||
apt update
|
||||
apt install -y g++ python3
|
||||
- name: ${{ matrix.image }}
|
||||
run: |
|
||||
# Do not set --warnings-as-errors here as that triggers an irrelevant
|
||||
# compiler warnings in <stdio.h> with ubuntu:24.04. See issue #2615
|
||||
python3 configure.py --bootstrap
|
||||
./ninja all
|
||||
python3 misc/ninja_syntax_test.py
|
||||
./misc/output_test.py
|
||||
@@ -0,0 +1,55 @@
|
||||
name: macOS
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
push:
|
||||
branches: ['**']
|
||||
tags-ignore: ['**'] # Don't trigger on tag pushes
|
||||
release:
|
||||
types: [published]
|
||||
|
||||
jobs:
|
||||
build:
|
||||
runs-on: macos-13
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- name: Install dependencies
|
||||
run: brew install re2c p7zip cmake
|
||||
|
||||
- name: Build ninja
|
||||
shell: bash
|
||||
env:
|
||||
MACOSX_DEPLOYMENT_TARGET: 10.15
|
||||
run: |
|
||||
cmake -Bbuild -GXcode '-DCMAKE_OSX_ARCHITECTURES=arm64;x86_64' -DCMAKE_COMPILE_WARNING_AS_ERROR=1
|
||||
cmake --build build --config Release
|
||||
|
||||
- name: Test ninja (Release)
|
||||
run: ./ninja_test
|
||||
working-directory: build/Release
|
||||
|
||||
- name: Create ninja archive
|
||||
shell: bash
|
||||
run: |
|
||||
mkdir artifact
|
||||
7z a artifact/ninja-mac.zip ./build/Release/ninja
|
||||
|
||||
# Upload ninja binary archive as an artifact
|
||||
- name: Upload artifact
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: ninja-binary-archives
|
||||
path: artifact
|
||||
|
||||
- name: Upload release asset
|
||||
if: github.event.action == 'published'
|
||||
uses: actions/upload-release-asset@v1
|
||||
env:
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
with:
|
||||
upload_url: ${{ github.event.release.upload_url }}
|
||||
asset_path: ./artifact/ninja-mac.zip
|
||||
asset_name: ninja-mac.zip
|
||||
asset_content_type: application/zip
|
||||
@@ -0,0 +1,69 @@
|
||||
name: Windows
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
push:
|
||||
branches: ['**']
|
||||
tags-ignore: ['**'] # Don't trigger on tag pushes
|
||||
release:
|
||||
types: [published]
|
||||
|
||||
jobs:
|
||||
build:
|
||||
runs-on: windows-latest
|
||||
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
include:
|
||||
- arch: 'x64'
|
||||
suffix: ''
|
||||
- arch: 'arm64'
|
||||
suffix: 'arm64'
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- name: Install dependencies
|
||||
run: choco install re2c
|
||||
|
||||
- name: Build ninja
|
||||
shell: bash
|
||||
run: |
|
||||
cmake -Bbuild -A ${{ matrix.arch }} -DCMAKE_COMPILE_WARNING_AS_ERROR=1
|
||||
cmake --build build --parallel --config Debug
|
||||
cmake --build build --parallel --config Release
|
||||
|
||||
- name: Test ninja (Debug)
|
||||
if: matrix.arch != 'arm64'
|
||||
run: .\ninja_test.exe
|
||||
working-directory: build/Debug
|
||||
|
||||
- name: Test ninja (Release)
|
||||
if: matrix.arch != 'arm64'
|
||||
run: .\ninja_test.exe
|
||||
working-directory: build/Release
|
||||
|
||||
- name: Create ninja archive
|
||||
shell: bash
|
||||
run: |
|
||||
mkdir artifact
|
||||
7z a artifact/ninja-win${{ matrix.suffix }}.zip ./build/Release/ninja.exe
|
||||
|
||||
# Upload ninja binary archive as an artifact
|
||||
- name: Upload artifact
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: ninja-binary-archives${{ matrix.suffix }}
|
||||
path: artifact
|
||||
|
||||
- name: Upload release asset
|
||||
if: github.event.action == 'published'
|
||||
uses: actions/upload-release-asset@v1
|
||||
env:
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
with:
|
||||
upload_url: ${{ github.event.release.upload_url }}
|
||||
asset_path: ./artifact/ninja-win${{ matrix.suffix }}.zip
|
||||
asset_name: ninja-win${{ matrix.suffix }}.zip
|
||||
asset_content_type: application/zip
|
||||
@@ -0,0 +1,50 @@
|
||||
*.pyc
|
||||
*.obj
|
||||
*.exe
|
||||
*.pdb
|
||||
*.ilk
|
||||
/build*/
|
||||
/build.ninja
|
||||
/ninja
|
||||
/ninja.bootstrap
|
||||
/build_log_perftest
|
||||
/canon_perftest
|
||||
/clparser_perftest
|
||||
/elide_middle_perftest
|
||||
/depfile_parser_perftest
|
||||
/hash_collision_bench
|
||||
/ninja_test
|
||||
/manifest_parser_perftest
|
||||
/graph.png
|
||||
/doc/manual.html
|
||||
/doc/doxygen
|
||||
*.patch
|
||||
.DS_Store
|
||||
|
||||
# Eclipse project files
|
||||
.project
|
||||
.cproject
|
||||
|
||||
# SublimeText project files
|
||||
*.sublime-project
|
||||
*.sublime-workspace
|
||||
|
||||
# Ninja output
|
||||
.ninja_deps
|
||||
.ninja_log
|
||||
|
||||
# Visual Studio Code project files
|
||||
/.vscode/
|
||||
/.ccls-cache/
|
||||
|
||||
# Qt Creator project files
|
||||
/CMakeLists.txt.user
|
||||
|
||||
# clangd
|
||||
/.clangd/
|
||||
/compile_commands.json
|
||||
/.cache/
|
||||
|
||||
# Visual Studio files
|
||||
/.vs/
|
||||
/out/
|
||||
@@ -0,0 +1 @@
|
||||
v1.13.1
|
||||
@@ -0,0 +1,335 @@
|
||||
cmake_minimum_required(VERSION 3.15)
|
||||
|
||||
include(CheckSymbolExists)
|
||||
include(CheckIPOSupported)
|
||||
|
||||
option(NINJA_BUILD_BINARY "Build ninja binary" ON)
|
||||
option(NINJA_FORCE_PSELECT "Use pselect() even on platforms that provide ppoll()" OFF)
|
||||
|
||||
project(ninja CXX)
|
||||
|
||||
# --- optional link-time optimization
|
||||
check_ipo_supported(RESULT lto_supported OUTPUT error)
|
||||
|
||||
if(lto_supported)
|
||||
message(STATUS "IPO / LTO enabled")
|
||||
set(CMAKE_POLICY_DEFAULT_CMP0069 NEW)
|
||||
set(CMAKE_INTERPROCEDURAL_OPTIMIZATION_RELEASE TRUE)
|
||||
else()
|
||||
message(STATUS "IPO / LTO not supported: <${error}>")
|
||||
endif()
|
||||
|
||||
# --- compiler flags
|
||||
if(MSVC)
|
||||
set(CMAKE_MSVC_RUNTIME_LIBRARY "MultiThreaded$<$<CONFIG:Debug>:Debug>")
|
||||
string(REPLACE "/GR" "" CMAKE_CXX_FLAGS ${CMAKE_CXX_FLAGS})
|
||||
# Note that these settings are separately specified in configure.py, and
|
||||
# these lists should be kept in sync.
|
||||
add_compile_options(/W4 /wd4100 /wd4267 /wd4706 /wd4702 /wd4244 /GR- /Zc:__cplusplus)
|
||||
add_compile_definitions(_CRT_SECURE_NO_WARNINGS)
|
||||
else()
|
||||
include(CheckCXXCompilerFlag)
|
||||
check_cxx_compiler_flag(-Wno-deprecated flag_no_deprecated)
|
||||
if(flag_no_deprecated)
|
||||
add_compile_options(-Wno-deprecated)
|
||||
endif()
|
||||
if(CMAKE_VERSION VERSION_LESS 3.24)
|
||||
check_cxx_compiler_flag(-fdiagnostics-color flag_color_diag)
|
||||
if(flag_color_diag)
|
||||
add_compile_options(-fdiagnostics-color)
|
||||
endif()
|
||||
elseif(NOT DEFINED ENV{CMAKE_COLOR_DIAGNOSTICS})
|
||||
set(CMAKE_COLOR_DIAGNOSTICS ON)
|
||||
endif()
|
||||
|
||||
if(NOT NINJA_FORCE_PSELECT)
|
||||
# Check whether ppoll() is usable on the target platform.
|
||||
# Set -DUSE_PPOLL=1 if this is the case.
|
||||
#
|
||||
# NOTE: Use check_cxx_symbol_exists() instead of check_symbol_exists()
|
||||
# because on Linux, <poll.h> only exposes the symbol when _GNU_SOURCE
|
||||
# is defined.
|
||||
#
|
||||
# Both g++ and clang++ define the symbol by default, because the C++
|
||||
# standard library headers require it, but *not* gcc and clang, which
|
||||
# are used by check_symbol_exists().
|
||||
include(CheckCXXSymbolExists)
|
||||
check_cxx_symbol_exists(ppoll poll.h HAVE_PPOLL)
|
||||
if(HAVE_PPOLL)
|
||||
add_compile_definitions(USE_PPOLL=1)
|
||||
endif()
|
||||
endif()
|
||||
endif()
|
||||
|
||||
# --- optional re2c
|
||||
set(RE2C_MAJOR_VERSION 0)
|
||||
find_program(RE2C re2c)
|
||||
if(RE2C)
|
||||
execute_process(COMMAND "${RE2C}" --vernum OUTPUT_VARIABLE RE2C_RAW_VERSION)
|
||||
math(EXPR RE2C_MAJOR_VERSION "${RE2C_RAW_VERSION} / 10000")
|
||||
endif()
|
||||
if(${RE2C_MAJOR_VERSION} GREATER 1)
|
||||
# the depfile parser and ninja lexers are generated using re2c.
|
||||
function(re2c IN OUT)
|
||||
add_custom_command(DEPENDS ${IN} OUTPUT ${OUT}
|
||||
COMMAND ${RE2C} -b -i --no-generation-date --no-version -o ${OUT} ${IN}
|
||||
)
|
||||
endfunction()
|
||||
re2c(${PROJECT_SOURCE_DIR}/src/depfile_parser.in.cc ${PROJECT_BINARY_DIR}/depfile_parser.cc)
|
||||
re2c(${PROJECT_SOURCE_DIR}/src/lexer.in.cc ${PROJECT_BINARY_DIR}/lexer.cc)
|
||||
add_library(libninja-re2c OBJECT ${PROJECT_BINARY_DIR}/depfile_parser.cc ${PROJECT_BINARY_DIR}/lexer.cc)
|
||||
else()
|
||||
message(WARNING "re2c 2 or later was not found; changes to src/*.in.cc will not affect your build.")
|
||||
add_library(libninja-re2c OBJECT src/depfile_parser.cc src/lexer.cc)
|
||||
endif()
|
||||
target_include_directories(libninja-re2c PRIVATE src)
|
||||
|
||||
# --- Check for 'browse' mode support
|
||||
function(check_platform_supports_browse_mode RESULT)
|
||||
# Make sure the inline.sh script works on this platform.
|
||||
# It uses the shell commands such as 'od', which may not be available.
|
||||
|
||||
execute_process(
|
||||
COMMAND sh -c "echo 'TEST' | src/inline.sh var"
|
||||
WORKING_DIRECTORY ${PROJECT_SOURCE_DIR}
|
||||
RESULT_VARIABLE inline_result
|
||||
OUTPUT_QUIET
|
||||
ERROR_QUIET
|
||||
)
|
||||
if(NOT inline_result EQUAL "0")
|
||||
# The inline script failed, so browse mode is not supported.
|
||||
set(${RESULT} "0" PARENT_SCOPE)
|
||||
if(NOT WIN32)
|
||||
message(WARNING "browse feature omitted due to inline script failure")
|
||||
endif()
|
||||
return()
|
||||
endif()
|
||||
|
||||
# Now check availability of the unistd header
|
||||
check_symbol_exists(fork "unistd.h" HAVE_FORK)
|
||||
check_symbol_exists(pipe "unistd.h" HAVE_PIPE)
|
||||
set(browse_supported 0)
|
||||
if (HAVE_FORK AND HAVE_PIPE)
|
||||
set(browse_supported 1)
|
||||
endif ()
|
||||
set(${RESULT} "${browse_supported}" PARENT_SCOPE)
|
||||
if(NOT browse_supported)
|
||||
message(WARNING "browse feature omitted due to missing `fork` and `pipe` functions")
|
||||
endif()
|
||||
|
||||
endfunction()
|
||||
|
||||
set(NINJA_PYTHON "python" CACHE STRING "Python interpreter to use for the browse tool")
|
||||
|
||||
check_platform_supports_browse_mode(platform_supports_ninja_browse)
|
||||
|
||||
# Core source files all build into ninja library.
|
||||
add_library(libninja OBJECT
|
||||
src/build_log.cc
|
||||
src/build.cc
|
||||
src/clean.cc
|
||||
src/clparser.cc
|
||||
src/dyndep.cc
|
||||
src/dyndep_parser.cc
|
||||
src/debug_flags.cc
|
||||
src/deps_log.cc
|
||||
src/disk_interface.cc
|
||||
src/edit_distance.cc
|
||||
src/elide_middle.cc
|
||||
src/eval_env.cc
|
||||
src/graph.cc
|
||||
src/graphviz.cc
|
||||
src/jobserver.cc
|
||||
src/json.cc
|
||||
src/line_printer.cc
|
||||
src/manifest_parser.cc
|
||||
src/metrics.cc
|
||||
src/missing_deps.cc
|
||||
src/parser.cc
|
||||
src/real_command_runner.cc
|
||||
src/state.cc
|
||||
src/status_printer.cc
|
||||
src/string_piece_util.cc
|
||||
src/util.cc
|
||||
src/version.cc
|
||||
)
|
||||
if(WIN32)
|
||||
target_sources(libninja PRIVATE
|
||||
src/subprocess-win32.cc
|
||||
src/includes_normalize-win32.cc
|
||||
src/jobserver-win32.cc
|
||||
src/msvc_helper-win32.cc
|
||||
src/msvc_helper_main-win32.cc
|
||||
src/getopt.c
|
||||
src/minidump-win32.cc
|
||||
)
|
||||
# Build getopt.c, which can be compiled as either C or C++, as C++
|
||||
# so that build environments which lack a C compiler, but have a C++
|
||||
# compiler may build ninja.
|
||||
set_source_files_properties(src/getopt.c PROPERTIES LANGUAGE CXX)
|
||||
|
||||
# windows.h defines min() and max() which conflict with std::min()
|
||||
# and std::max(), which both might be used in sources. Avoid compile
|
||||
# errors by telling windows.h to not define those two.
|
||||
add_compile_definitions(NOMINMAX)
|
||||
else()
|
||||
target_sources(libninja PRIVATE
|
||||
src/jobserver-posix.cc
|
||||
src/subprocess-posix.cc
|
||||
)
|
||||
if(CMAKE_SYSTEM_NAME STREQUAL "OS400" OR CMAKE_SYSTEM_NAME STREQUAL "AIX")
|
||||
target_sources(libninja PRIVATE src/getopt.c)
|
||||
# Build getopt.c, which can be compiled as either C or C++, as C++
|
||||
# so that build environments which lack a C compiler, but have a C++
|
||||
# compiler may build ninja.
|
||||
set_source_files_properties(src/getopt.c PROPERTIES LANGUAGE CXX)
|
||||
endif()
|
||||
|
||||
# Needed for perfstat_cpu_total
|
||||
if(CMAKE_SYSTEM_NAME STREQUAL "AIX")
|
||||
target_link_libraries(libninja PUBLIC "-lperfstat")
|
||||
endif()
|
||||
endif()
|
||||
|
||||
target_compile_features(libninja PUBLIC cxx_std_11)
|
||||
target_compile_features(libninja-re2c PUBLIC cxx_std_11)
|
||||
|
||||
#Fixes GetActiveProcessorCount on MinGW
|
||||
if(MINGW)
|
||||
target_compile_definitions(libninja PRIVATE _WIN32_WINNT=0x0601 __USE_MINGW_ANSI_STDIO=1)
|
||||
endif()
|
||||
|
||||
# On IBM i (identified as "OS400" for compatibility reasons) and AIX, this fixes missing
|
||||
# PRId64 (and others) at compile time in C++ sources
|
||||
if(CMAKE_SYSTEM_NAME STREQUAL "OS400" OR CMAKE_SYSTEM_NAME STREQUAL "AIX")
|
||||
add_compile_definitions(__STDC_FORMAT_MACROS)
|
||||
endif()
|
||||
|
||||
# Main executable is library plus main() function.
|
||||
if(NINJA_BUILD_BINARY)
|
||||
add_executable(ninja src/ninja.cc)
|
||||
target_link_libraries(ninja PRIVATE libninja libninja-re2c)
|
||||
|
||||
if(WIN32)
|
||||
target_sources(ninja PRIVATE windows/ninja.manifest)
|
||||
endif()
|
||||
|
||||
option(NINJA_CLANG_TIDY "Run clang-tidy on source files" OFF)
|
||||
if(NINJA_CLANG_TIDY)
|
||||
set_target_properties(libninja PROPERTIES CXX_CLANG_TIDY "clang-tidy;--use-color")
|
||||
set_target_properties(ninja PROPERTIES CXX_CLANG_TIDY "clang-tidy;--use-color")
|
||||
endif()
|
||||
endif()
|
||||
|
||||
# Adds browse mode into the ninja binary if it's supported by the host platform.
|
||||
if(platform_supports_ninja_browse)
|
||||
# Inlines src/browse.py into the browse_py.h header, so that it can be included
|
||||
# by src/browse.cc
|
||||
add_custom_command(
|
||||
OUTPUT build/browse_py.h
|
||||
MAIN_DEPENDENCY src/browse.py
|
||||
DEPENDS src/inline.sh
|
||||
COMMAND ${CMAKE_COMMAND} -E make_directory ${PROJECT_BINARY_DIR}/build
|
||||
COMMAND src/inline.sh kBrowsePy
|
||||
< src/browse.py
|
||||
> ${PROJECT_BINARY_DIR}/build/browse_py.h
|
||||
WORKING_DIRECTORY ${PROJECT_SOURCE_DIR}
|
||||
VERBATIM
|
||||
)
|
||||
|
||||
if(NINJA_BUILD_BINARY)
|
||||
target_compile_definitions(ninja PRIVATE NINJA_HAVE_BROWSE)
|
||||
target_sources(ninja PRIVATE src/browse.cc)
|
||||
endif()
|
||||
set_source_files_properties(src/browse.cc
|
||||
PROPERTIES
|
||||
OBJECT_DEPENDS "${PROJECT_BINARY_DIR}/build/browse_py.h"
|
||||
INCLUDE_DIRECTORIES "${PROJECT_BINARY_DIR}"
|
||||
COMPILE_DEFINITIONS NINJA_PYTHON="${NINJA_PYTHON}"
|
||||
)
|
||||
endif()
|
||||
|
||||
include(CTest)
|
||||
if(BUILD_TESTING)
|
||||
|
||||
# Can be removed if cmake min version is >=3.24
|
||||
if (POLICY CMP0135)
|
||||
cmake_policy(SET CMP0135 NEW)
|
||||
endif()
|
||||
|
||||
find_package(GTest)
|
||||
if(NOT GTest_FOUND)
|
||||
include(FetchContent)
|
||||
FetchContent_Declare(
|
||||
googletest
|
||||
# GoogleTest v1.17.0 requires at least C++17 and cmake 3.16 which are above ninja project minimum requirements.
|
||||
# GoogleTest v1.16.0 requires at least C++14 and cmake 3.13 which are within the ninja project minimum requirements.
|
||||
URL https://github.com/google/googletest/releases/download/v1.16.0/googletest-1.16.0.tar.gz
|
||||
URL_HASH SHA256=78c676fc63881529bf97bf9d45948d905a66833fbfa5318ea2cd7478cb98f399
|
||||
)
|
||||
FetchContent_MakeAvailable(googletest)
|
||||
endif()
|
||||
|
||||
# Tests all build into ninja_test executable.
|
||||
add_executable(ninja_test
|
||||
src/build_log_test.cc
|
||||
src/build_test.cc
|
||||
src/clean_test.cc
|
||||
src/clparser_test.cc
|
||||
src/depfile_parser_test.cc
|
||||
src/deps_log_test.cc
|
||||
src/disk_interface_test.cc
|
||||
src/dyndep_parser_test.cc
|
||||
src/edit_distance_test.cc
|
||||
src/elide_middle_test.cc
|
||||
src/explanations_test.cc
|
||||
src/graph_test.cc
|
||||
src/jobserver_test.cc
|
||||
src/json_test.cc
|
||||
src/lexer_test.cc
|
||||
src/manifest_parser_test.cc
|
||||
src/missing_deps_test.cc
|
||||
src/ninja_test.cc
|
||||
src/state_test.cc
|
||||
src/string_piece_util_test.cc
|
||||
src/subprocess_test.cc
|
||||
src/test.cc
|
||||
src/util_test.cc
|
||||
)
|
||||
if(WIN32)
|
||||
target_sources(ninja_test PRIVATE src/includes_normalize_test.cc src/msvc_helper_test.cc
|
||||
windows/ninja.manifest)
|
||||
|
||||
if(MSVC)
|
||||
# Silence warnings about using unlink rather than _unlink
|
||||
target_compile_definitions(ninja_test PRIVATE _CRT_NONSTDC_NO_DEPRECATE)
|
||||
endif()
|
||||
endif()
|
||||
find_package(Threads REQUIRED)
|
||||
target_link_libraries(ninja_test PRIVATE libninja libninja-re2c GTest::gtest Threads::Threads)
|
||||
|
||||
foreach(perftest
|
||||
build_log_perftest
|
||||
canon_perftest
|
||||
clparser_perftest
|
||||
depfile_parser_perftest
|
||||
elide_middle_perftest
|
||||
hash_collision_bench
|
||||
manifest_parser_perftest
|
||||
)
|
||||
add_executable(${perftest} src/${perftest}.cc)
|
||||
target_link_libraries(${perftest} PRIVATE libninja libninja-re2c)
|
||||
endforeach()
|
||||
|
||||
if(CMAKE_SYSTEM_NAME STREQUAL "AIX" AND CMAKE_SIZEOF_VOID_P EQUAL 4)
|
||||
# These tests require more memory than will fit in the standard AIX shared stack/heap (256M)
|
||||
target_link_options(hash_collision_bench PRIVATE "-Wl,-bmaxdata:0x80000000")
|
||||
target_link_options(manifest_parser_perftest PRIVATE "-Wl,-bmaxdata:0x80000000")
|
||||
endif()
|
||||
|
||||
add_test(NAME NinjaTest COMMAND ninja_test)
|
||||
endif()
|
||||
|
||||
if(NINJA_BUILD_BINARY)
|
||||
install(TARGETS ninja)
|
||||
endif()
|
||||
@@ -0,0 +1,30 @@
|
||||
# How to successfully make changes to Ninja
|
||||
|
||||
We're very wary of changes that increase the complexity of Ninja (in particular,
|
||||
new build file syntax or command-line flags) or increase the maintenance burden
|
||||
of Ninja. Ninja is already successfully used by hundreds of developers for large
|
||||
projects and it already achieves (most of) the goals we set out for it to do.
|
||||
It's probably best to discuss new feature ideas on the
|
||||
[mailing list](https://groups.google.com/forum/#!forum/ninja-build) or in an
|
||||
issue before creating a PR.
|
||||
|
||||
## Coding guidelines
|
||||
|
||||
Generally it's the
|
||||
[Google C++ Style Guide](https://google.github.io/styleguide/cppguide.html) with
|
||||
a few additions:
|
||||
|
||||
* We have used `using namespace std;` a lot in the past. For new contributions,
|
||||
please try to avoid relying on it and instead whenever possible use `std::`.
|
||||
However, please do not change existing code simply to add `std::` unless your
|
||||
contribution already needs to change that line of code anyway.
|
||||
* Use `///` for [Doxygen](http://www.doxygen.nl/) (use `\a` to refer to
|
||||
arguments).
|
||||
* It's not necessary to document each argument, especially when they're
|
||||
relatively self-evident (e.g. in
|
||||
`CanonicalizePath(string* path, string* err)`, the arguments are hopefully
|
||||
obvious).
|
||||
|
||||
If you're unsure about code formatting, please use
|
||||
[clang-format](https://clang.llvm.org/docs/ClangFormat.html). However, please do
|
||||
not format code that is not otherwise part of your contribution.
|
||||
@@ -0,0 +1,202 @@
|
||||
|
||||
Apache License
|
||||
Version 2.0, January 2010
|
||||
http://www.apache.org/licenses/
|
||||
|
||||
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
|
||||
|
||||
1. Definitions.
|
||||
|
||||
"License" shall mean the terms and conditions for use, reproduction,
|
||||
and distribution as defined by Sections 1 through 9 of this document.
|
||||
|
||||
"Licensor" shall mean the copyright owner or entity authorized by
|
||||
the copyright owner that is granting the License.
|
||||
|
||||
"Legal Entity" shall mean the union of the acting entity and all
|
||||
other entities that control, are controlled by, or are under common
|
||||
control with that entity. For the purposes of this definition,
|
||||
"control" means (i) the power, direct or indirect, to cause the
|
||||
direction or management of such entity, whether by contract or
|
||||
otherwise, or (ii) ownership of fifty percent (50%) or more of the
|
||||
outstanding shares, or (iii) beneficial ownership of such entity.
|
||||
|
||||
"You" (or "Your") shall mean an individual or Legal Entity
|
||||
exercising permissions granted by this License.
|
||||
|
||||
"Source" form shall mean the preferred form for making modifications,
|
||||
including but not limited to software source code, documentation
|
||||
source, and configuration files.
|
||||
|
||||
"Object" form shall mean any form resulting from mechanical
|
||||
transformation or translation of a Source form, including but
|
||||
not limited to compiled object code, generated documentation,
|
||||
and conversions to other media types.
|
||||
|
||||
"Work" shall mean the work of authorship, whether in Source or
|
||||
Object form, made available under the License, as indicated by a
|
||||
copyright notice that is included in or attached to the work
|
||||
(an example is provided in the Appendix below).
|
||||
|
||||
"Derivative Works" shall mean any work, whether in Source or Object
|
||||
form, that is based on (or derived from) the Work and for which the
|
||||
editorial revisions, annotations, elaborations, or other modifications
|
||||
represent, as a whole, an original work of authorship. For the purposes
|
||||
of this License, Derivative Works shall not include works that remain
|
||||
separable from, or merely link (or bind by name) to the interfaces of,
|
||||
the Work and Derivative Works thereof.
|
||||
|
||||
"Contribution" shall mean any work of authorship, including
|
||||
the original version of the Work and any modifications or additions
|
||||
to that Work or Derivative Works thereof, that is intentionally
|
||||
submitted to Licensor for inclusion in the Work by the copyright owner
|
||||
or by an individual or Legal Entity authorized to submit on behalf of
|
||||
the copyright owner. For the purposes of this definition, "submitted"
|
||||
means any form of electronic, verbal, or written communication sent
|
||||
to the Licensor or its representatives, including but not limited to
|
||||
communication on electronic mailing lists, source code control systems,
|
||||
and issue tracking systems that are managed by, or on behalf of, the
|
||||
Licensor for the purpose of discussing and improving the Work, but
|
||||
excluding communication that is conspicuously marked or otherwise
|
||||
designated in writing by the copyright owner as "Not a Contribution."
|
||||
|
||||
"Contributor" shall mean Licensor and any individual or Legal Entity
|
||||
on behalf of whom a Contribution has been received by Licensor and
|
||||
subsequently incorporated within the Work.
|
||||
|
||||
2. Grant of Copyright License. Subject to the terms and conditions of
|
||||
this License, each Contributor hereby grants to You a perpetual,
|
||||
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
||||
copyright license to reproduce, prepare Derivative Works of,
|
||||
publicly display, publicly perform, sublicense, and distribute the
|
||||
Work and such Derivative Works in Source or Object form.
|
||||
|
||||
3. Grant of Patent License. Subject to the terms and conditions of
|
||||
this License, each Contributor hereby grants to You a perpetual,
|
||||
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
||||
(except as stated in this section) patent license to make, have made,
|
||||
use, offer to sell, sell, import, and otherwise transfer the Work,
|
||||
where such license applies only to those patent claims licensable
|
||||
by such Contributor that are necessarily infringed by their
|
||||
Contribution(s) alone or by combination of their Contribution(s)
|
||||
with the Work to which such Contribution(s) was submitted. If You
|
||||
institute patent litigation against any entity (including a
|
||||
cross-claim or counterclaim in a lawsuit) alleging that the Work
|
||||
or a Contribution incorporated within the Work constitutes direct
|
||||
or contributory patent infringement, then any patent licenses
|
||||
granted to You under this License for that Work shall terminate
|
||||
as of the date such litigation is filed.
|
||||
|
||||
4. Redistribution. You may reproduce and distribute copies of the
|
||||
Work or Derivative Works thereof in any medium, with or without
|
||||
modifications, and in Source or Object form, provided that You
|
||||
meet the following conditions:
|
||||
|
||||
(a) You must give any other recipients of the Work or
|
||||
Derivative Works a copy of this License; and
|
||||
|
||||
(b) You must cause any modified files to carry prominent notices
|
||||
stating that You changed the files; and
|
||||
|
||||
(c) You must retain, in the Source form of any Derivative Works
|
||||
that You distribute, all copyright, patent, trademark, and
|
||||
attribution notices from the Source form of the Work,
|
||||
excluding those notices that do not pertain to any part of
|
||||
the Derivative Works; and
|
||||
|
||||
(d) If the Work includes a "NOTICE" text file as part of its
|
||||
distribution, then any Derivative Works that You distribute must
|
||||
include a readable copy of the attribution notices contained
|
||||
within such NOTICE file, excluding those notices that do not
|
||||
pertain to any part of the Derivative Works, in at least one
|
||||
of the following places: within a NOTICE text file distributed
|
||||
as part of the Derivative Works; within the Source form or
|
||||
documentation, if provided along with the Derivative Works; or,
|
||||
within a display generated by the Derivative Works, if and
|
||||
wherever such third-party notices normally appear. The contents
|
||||
of the NOTICE file are for informational purposes only and
|
||||
do not modify the License. You may add Your own attribution
|
||||
notices within Derivative Works that You distribute, alongside
|
||||
or as an addendum to the NOTICE text from the Work, provided
|
||||
that such additional attribution notices cannot be construed
|
||||
as modifying the License.
|
||||
|
||||
You may add Your own copyright statement to Your modifications and
|
||||
may provide additional or different license terms and conditions
|
||||
for use, reproduction, or distribution of Your modifications, or
|
||||
for any such Derivative Works as a whole, provided Your use,
|
||||
reproduction, and distribution of the Work otherwise complies with
|
||||
the conditions stated in this License.
|
||||
|
||||
5. Submission of Contributions. Unless You explicitly state otherwise,
|
||||
any Contribution intentionally submitted for inclusion in the Work
|
||||
by You to the Licensor shall be under the terms and conditions of
|
||||
this License, without any additional terms or conditions.
|
||||
Notwithstanding the above, nothing herein shall supersede or modify
|
||||
the terms of any separate license agreement you may have executed
|
||||
with Licensor regarding such Contributions.
|
||||
|
||||
6. Trademarks. This License does not grant permission to use the trade
|
||||
names, trademarks, service marks, or product names of the Licensor,
|
||||
except as required for reasonable and customary use in describing the
|
||||
origin of the Work and reproducing the content of the NOTICE file.
|
||||
|
||||
7. Disclaimer of Warranty. Unless required by applicable law or
|
||||
agreed to in writing, Licensor provides the Work (and each
|
||||
Contributor provides its Contributions) on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
|
||||
implied, including, without limitation, any warranties or conditions
|
||||
of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
|
||||
PARTICULAR PURPOSE. You are solely responsible for determining the
|
||||
appropriateness of using or redistributing the Work and assume any
|
||||
risks associated with Your exercise of permissions under this License.
|
||||
|
||||
8. Limitation of Liability. In no event and under no legal theory,
|
||||
whether in tort (including negligence), contract, or otherwise,
|
||||
unless required by applicable law (such as deliberate and grossly
|
||||
negligent acts) or agreed to in writing, shall any Contributor be
|
||||
liable to You for damages, including any direct, indirect, special,
|
||||
incidental, or consequential damages of any character arising as a
|
||||
result of this License or out of the use or inability to use the
|
||||
Work (including but not limited to damages for loss of goodwill,
|
||||
work stoppage, computer failure or malfunction, or any and all
|
||||
other commercial damages or losses), even if such Contributor
|
||||
has been advised of the possibility of such damages.
|
||||
|
||||
9. Accepting Warranty or Additional Liability. While redistributing
|
||||
the Work or Derivative Works thereof, You may choose to offer,
|
||||
and charge a fee for, acceptance of support, warranty, indemnity,
|
||||
or other liability obligations and/or rights consistent with this
|
||||
License. However, in accepting such obligations, You may act only
|
||||
on Your own behalf and on Your sole responsibility, not on behalf
|
||||
of any other Contributor, and only if You agree to indemnify,
|
||||
defend, and hold each Contributor harmless for any liability
|
||||
incurred by, or claims asserted against, such Contributor by reason
|
||||
of your accepting any such warranty or additional liability.
|
||||
|
||||
END OF TERMS AND CONDITIONS
|
||||
|
||||
APPENDIX: How to apply the Apache License to your work.
|
||||
|
||||
To apply the Apache License to your work, attach the following
|
||||
boilerplate notice, with the fields enclosed by brackets "[]"
|
||||
replaced with your own identifying information. (Don't include
|
||||
the brackets!) The text should be enclosed in the appropriate
|
||||
comment syntax for the file format. We also recommend that a
|
||||
file or class name and description of purpose be included on the
|
||||
same "printed page" as the copyright notice for easier
|
||||
identification within third-party archives.
|
||||
|
||||
Copyright [yyyy] [name of copyright owner]
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
@@ -0,0 +1,102 @@
|
||||
# Ninja
|
||||
|
||||
Ninja is a small build system with a focus on speed.
|
||||
https://ninja-build.org/
|
||||
|
||||
See [the manual](https://ninja-build.org/manual.html) or
|
||||
`doc/manual.asciidoc` included in the distribution for background
|
||||
and more details.
|
||||
|
||||
Binaries for Linux, Mac and Windows are available on
|
||||
[GitHub](https://github.com/ninja-build/ninja/releases).
|
||||
Run `./ninja -h` for Ninja help.
|
||||
|
||||
Installation is not necessary because the only required file is the
|
||||
resulting ninja binary. However, to enable features like Bash
|
||||
completion and Emacs and Vim editing modes, some files in misc/ must be
|
||||
copied to appropriate locations.
|
||||
|
||||
If you're interested in making changes to Ninja, read
|
||||
[CONTRIBUTING.md](CONTRIBUTING.md) first.
|
||||
|
||||
## Building Ninja itself
|
||||
|
||||
You can either build Ninja via the custom generator script written in Python or
|
||||
via CMake. For more details see
|
||||
[the wiki](https://github.com/ninja-build/ninja/wiki).
|
||||
|
||||
### Python
|
||||
|
||||
```
|
||||
./configure.py --bootstrap
|
||||
```
|
||||
|
||||
This will generate the `ninja` binary and a `build.ninja` file you can now use
|
||||
to build Ninja with itself.
|
||||
|
||||
If you have a GoogleTest source directory, you can build the tests
|
||||
by passing its path with `--gtest-source-dir=PATH` option, or the
|
||||
`GTEST_SOURCE_DIR` environment variable, e.g.:
|
||||
|
||||
```
|
||||
./configure.py --bootstrap --gtest-source-dir=/path/to/googletest
|
||||
./ninja all # build ninja_test and other auxiliary binaries
|
||||
./ninja_test` # run the unit-test suite.
|
||||
```
|
||||
|
||||
Use the CMake build below if you want to use a preinstalled binary
|
||||
version of the library.
|
||||
|
||||
### CMake
|
||||
|
||||
To build the ninja binary without building the unit tests, disable test building by setting `BUILD_TESTING` to `OFF`:
|
||||
|
||||
```
|
||||
cmake -Bbuild-cmake -DBUILD_TESTING=OFF
|
||||
cmake --build build-cmake
|
||||
```
|
||||
|
||||
The `ninja` binary will now be inside the `build-cmake` directory (you can
|
||||
choose any other name you like).
|
||||
|
||||
To run the unit tests, omit the `-DBUILD_TESTING=OFF` option, and after building, run:
|
||||
|
||||
```
|
||||
./build-cmake/ninja_test
|
||||
```
|
||||
|
||||
## Generating documentation
|
||||
|
||||
### Ninja Manual
|
||||
|
||||
You must have `asciidoc` and `xsltproc` in your PATH, then do:
|
||||
|
||||
```
|
||||
./configure.py
|
||||
ninja manual doc/manual.html
|
||||
```
|
||||
|
||||
Which will generate `doc/manual.html`.
|
||||
|
||||
To generate the PDF version of the manual, you must have `dblatext` in your PATH then do:
|
||||
|
||||
```
|
||||
./configure.py # only if you didn't do it previously.
|
||||
ninja doc/manual.pdf
|
||||
```
|
||||
|
||||
Which will generate `doc/manual.pdf`.
|
||||
|
||||
### Doxygen documentation
|
||||
|
||||
If you have `doxygen` installed, you can build documentation extracted from C++
|
||||
declarations and comments to help you navigate the code. Note that Ninja is a standalone
|
||||
executable, not a library, so there is no public API, all details exposed here are
|
||||
internal.
|
||||
|
||||
```
|
||||
./configure.py # if needed
|
||||
ninja doxygen
|
||||
```
|
||||
|
||||
Then open `doc/doxygen/html/index.html` in a browser to look at it.
|
||||
@@ -0,0 +1,41 @@
|
||||
Notes to myself on all the steps to make for a Ninja release.
|
||||
|
||||
### Push new release branch:
|
||||
1. Run afl-fuzz for a day or so and run ninja_test
|
||||
2. Consider sending a heads-up to the ninja-build mailing list first
|
||||
3. Make sure branches 'master' and 'release' are synced up locally
|
||||
4. Update src/version.cc with new version (with ".git"), then
|
||||
```
|
||||
git commit -am 'mark this 1.5.0.git'
|
||||
```
|
||||
5. git checkout release; git merge master
|
||||
6. Fix version number in src/version.cc (it will likely conflict in the above)
|
||||
7. Fix version in doc/manual.asciidoc (exists only on release branch)
|
||||
8. commit, tag, push (don't forget to push --tags)
|
||||
```
|
||||
git commit -am v1.5.0; git push origin release
|
||||
git tag v1.5.0; git push --tags
|
||||
# Push the 1.5.0.git change on master too:
|
||||
git checkout master; git push origin master
|
||||
```
|
||||
9. Construct release notes from prior notes
|
||||
|
||||
credits: `git shortlog -s --no-merges REV..`
|
||||
|
||||
|
||||
### Release on GitHub:
|
||||
1. Go to [Tags](https://github.com/ninja-build/ninja/tags)
|
||||
2. Open the newly created tag and select "Create release from tag"
|
||||
3. Create the release which will trigger a build which automatically attaches
|
||||
the binaries
|
||||
|
||||
### Make announcement on mailing list:
|
||||
1. copy old mail
|
||||
|
||||
### Update website:
|
||||
1. Make sure your ninja checkout is on the v1.5.0 tag
|
||||
2. Clone https://github.com/ninja-build/ninja-build.github.io
|
||||
3. In that repo, `./update-docs.sh`
|
||||
4. Update index.html with newest version and link to release notes
|
||||
5. `git commit -m 'run update-docs.sh, 1.5.0 release'`
|
||||
6. `git push origin master`
|
||||
@@ -0,0 +1,40 @@
|
||||
version: 1.0.{build}
|
||||
image:
|
||||
- Visual Studio 2017
|
||||
- Ubuntu2204
|
||||
|
||||
environment:
|
||||
CLICOLOR_FORCE: 1
|
||||
CHERE_INVOKING: 1 # Tell Bash to inherit the current working directory
|
||||
matrix:
|
||||
- MSYSTEM: MINGW64
|
||||
- MSYSTEM: LINUX
|
||||
|
||||
matrix:
|
||||
exclude:
|
||||
- image: Visual Studio 2017
|
||||
MSYSTEM: LINUX
|
||||
- image: Ubuntu2204
|
||||
MSYSTEM: MINGW64
|
||||
|
||||
for:
|
||||
-
|
||||
matrix:
|
||||
only:
|
||||
- MSYSTEM: MINGW64
|
||||
build_script:
|
||||
ps: "C:\\msys64\\usr\\bin\\bash -lc @\"\n
|
||||
pacman -S --quiet --noconfirm --needed re2c 2>&1\n
|
||||
./configure.py --bootstrap --platform mingw 2>&1\n
|
||||
./ninja all\n
|
||||
./misc/ninja_syntax_test.py 2>&1\n\"@"
|
||||
- matrix:
|
||||
only:
|
||||
- image: Ubuntu2204
|
||||
build_script:
|
||||
- ./configure.py --bootstrap
|
||||
- ./ninja all
|
||||
- misc/ninja_syntax_test.py
|
||||
- misc/output_test.py
|
||||
|
||||
test: off
|
||||
+810
@@ -0,0 +1,810 @@
|
||||
#!/usr/bin/env python3
|
||||
#
|
||||
# Copyright 2001 Google Inc. All Rights Reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
"""Script that generates the build.ninja for ninja itself.
|
||||
|
||||
Projects that use ninja themselves should either write a similar script
|
||||
or use a meta-build system that supports Ninja output."""
|
||||
|
||||
from optparse import OptionParser
|
||||
import os
|
||||
import shlex
|
||||
import subprocess
|
||||
import sys
|
||||
from typing import Optional, Union, Dict, List, Any, TYPE_CHECKING
|
||||
|
||||
sourcedir = os.path.dirname(os.path.realpath(__file__))
|
||||
sys.path.insert(0, os.path.join(sourcedir, 'misc'))
|
||||
if TYPE_CHECKING:
|
||||
import misc.ninja_syntax as ninja_syntax
|
||||
else:
|
||||
import ninja_syntax
|
||||
|
||||
|
||||
class Platform(object):
|
||||
"""Represents a host/target platform and its specific build attributes."""
|
||||
def __init__(self, platform: Optional[str]) -> None:
|
||||
self._platform = platform
|
||||
if self._platform is not None:
|
||||
return
|
||||
self._platform = sys.platform
|
||||
if self._platform.startswith('linux'):
|
||||
self._platform = 'linux'
|
||||
elif self._platform.startswith('freebsd'):
|
||||
self._platform = 'freebsd'
|
||||
elif self._platform.startswith('gnukfreebsd'):
|
||||
self._platform = 'freebsd'
|
||||
elif self._platform.startswith('openbsd'):
|
||||
self._platform = 'openbsd'
|
||||
elif self._platform.startswith('solaris') or self._platform == 'sunos5':
|
||||
self._platform = 'solaris'
|
||||
elif self._platform.startswith('mingw'):
|
||||
self._platform = 'mingw'
|
||||
elif self._platform.startswith('win'):
|
||||
self._platform = 'msvc'
|
||||
elif self._platform.startswith('bitrig'):
|
||||
self._platform = 'bitrig'
|
||||
elif self._platform.startswith('netbsd'):
|
||||
self._platform = 'netbsd'
|
||||
elif self._platform.startswith('aix'):
|
||||
self._platform = 'aix'
|
||||
elif self._platform.startswith('os400'):
|
||||
self._platform = 'os400'
|
||||
elif self._platform.startswith('dragonfly'):
|
||||
self._platform = 'dragonfly'
|
||||
|
||||
@staticmethod
|
||||
def known_platforms() -> List[str]:
|
||||
return ['linux', 'darwin', 'freebsd', 'openbsd', 'solaris', 'sunos5',
|
||||
'mingw', 'msvc', 'gnukfreebsd', 'bitrig', 'netbsd', 'aix',
|
||||
'dragonfly']
|
||||
|
||||
def platform(self) -> str:
|
||||
return self._platform # type: ignore # Incompatible return value type
|
||||
|
||||
def is_linux(self) -> bool:
|
||||
return self._platform == 'linux'
|
||||
|
||||
def is_mingw(self) -> bool:
|
||||
return self._platform == 'mingw'
|
||||
|
||||
def is_msvc(self) -> bool:
|
||||
return self._platform == 'msvc'
|
||||
|
||||
def msvc_needs_fs(self) -> bool:
|
||||
popen = subprocess.Popen(['cl', '/nologo', '/help'],
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.PIPE)
|
||||
out, err = popen.communicate()
|
||||
return b'/FS' in out
|
||||
|
||||
def is_windows(self) -> bool:
|
||||
return self.is_mingw() or self.is_msvc()
|
||||
|
||||
def is_solaris(self) -> bool:
|
||||
return self._platform == 'solaris'
|
||||
|
||||
def is_aix(self) -> bool:
|
||||
return self._platform == 'aix'
|
||||
|
||||
def is_os400_pase(self) -> bool:
|
||||
return self._platform == 'os400' or os.uname().sysname.startswith('OS400') # type: ignore # Module has no attribute "uname"
|
||||
|
||||
def uses_usr_local(self) -> bool:
|
||||
return self._platform in ('freebsd', 'openbsd', 'bitrig', 'dragonfly', 'netbsd')
|
||||
|
||||
def supports_ppoll(self) -> bool:
|
||||
return self._platform in ('freebsd', 'linux', 'openbsd', 'bitrig',
|
||||
'dragonfly')
|
||||
|
||||
def supports_ninja_browse(self) -> bool:
|
||||
return (not self.is_windows()
|
||||
and not self.is_solaris()
|
||||
and not self.is_aix())
|
||||
|
||||
def can_rebuild_in_place(self) -> bool:
|
||||
return not (self.is_windows() or self.is_aix())
|
||||
|
||||
class Bootstrap:
|
||||
"""API shim for ninja_syntax.Writer that instead runs the commands.
|
||||
|
||||
Used to bootstrap Ninja from scratch. In --bootstrap mode this
|
||||
class is used to execute all the commands to build an executable.
|
||||
It also proxies all calls to an underlying ninja_syntax.Writer, to
|
||||
behave like non-bootstrap mode.
|
||||
"""
|
||||
def __init__(self, writer: ninja_syntax.Writer, verbose: bool = False) -> None:
|
||||
self.writer = writer
|
||||
self.verbose = verbose
|
||||
# Map of variable name => expanded variable value.
|
||||
self.vars: Dict[str, str] = {}
|
||||
# Map of rule name => dict of rule attributes.
|
||||
self.rules: Dict[str, Dict[str, Any]] = {
|
||||
'phony': {}
|
||||
}
|
||||
|
||||
def comment(self, text: str) -> None:
|
||||
return self.writer.comment(text)
|
||||
|
||||
def newline(self) -> None:
|
||||
return self.writer.newline()
|
||||
|
||||
def variable(self, key: str, val: str) -> None:
|
||||
# In bootstrap mode, we have no ninja process to catch /showIncludes
|
||||
# output.
|
||||
self.vars[key] = self._expand(val).replace('/showIncludes', '')
|
||||
return self.writer.variable(key, val)
|
||||
|
||||
def rule(self, name: str, **kwargs: Any) -> None:
|
||||
self.rules[name] = kwargs
|
||||
return self.writer.rule(name, **kwargs)
|
||||
|
||||
def build(
|
||||
self,
|
||||
outputs: Union[str, List[str]],
|
||||
rule: str,
|
||||
inputs: Optional[Union[str, List[str]]] = None,
|
||||
**kwargs: Any
|
||||
) -> List[str]:
|
||||
ruleattr = self.rules[rule]
|
||||
cmd = ruleattr.get('command')
|
||||
if cmd is None: # A phony rule, for example.
|
||||
return # type: ignore # Return value expected
|
||||
|
||||
# Implement just enough of Ninja variable expansion etc. to
|
||||
# make the bootstrap build work.
|
||||
local_vars = {
|
||||
'in': self._expand_paths(inputs),
|
||||
'out': self._expand_paths(outputs)
|
||||
}
|
||||
for key, val in kwargs.get('variables', []):
|
||||
local_vars[key] = ' '.join(ninja_syntax.as_list(val))
|
||||
|
||||
self._run_command(self._expand(cmd, local_vars))
|
||||
|
||||
return self.writer.build(outputs, rule, inputs, **kwargs)
|
||||
|
||||
def default(self, paths: Union[str, List[str]]) -> None:
|
||||
return self.writer.default(paths)
|
||||
|
||||
def _expand_paths(self, paths: Optional[Union[str, List[str]]]) -> str:
|
||||
"""Expand $vars in an array of paths, e.g. from a 'build' block."""
|
||||
paths = ninja_syntax.as_list(paths)
|
||||
return ' '.join(map(self._shell_escape, (map(self._expand, paths))))
|
||||
|
||||
def _expand(self, str: str, local_vars: Dict[str, str] = {}) -> str:
|
||||
"""Expand $vars in a string."""
|
||||
return ninja_syntax.expand(str, self.vars, local_vars)
|
||||
|
||||
def _shell_escape(self, path: str) -> str:
|
||||
"""Quote paths containing spaces."""
|
||||
return '"%s"' % path if ' ' in path else path
|
||||
|
||||
def _run_command(self, cmdline: str) -> None:
|
||||
"""Run a subcommand, quietly. Prints the full command on error."""
|
||||
try:
|
||||
if self.verbose:
|
||||
print(cmdline)
|
||||
subprocess.check_call(cmdline, shell=True)
|
||||
except subprocess.CalledProcessError:
|
||||
print('when running: ', cmdline)
|
||||
raise
|
||||
|
||||
|
||||
parser = OptionParser()
|
||||
profilers = ['gmon', 'pprof']
|
||||
parser.add_option('--bootstrap', action='store_true',
|
||||
help='bootstrap a ninja binary from nothing')
|
||||
parser.add_option('--verbose', action='store_true',
|
||||
help='enable verbose build')
|
||||
parser.add_option('--platform',
|
||||
help='target platform (' +
|
||||
'/'.join(Platform.known_platforms()) + ')',
|
||||
choices=Platform.known_platforms())
|
||||
parser.add_option('--host',
|
||||
help='host platform (' +
|
||||
'/'.join(Platform.known_platforms()) + ')',
|
||||
choices=Platform.known_platforms())
|
||||
parser.add_option('--debug', action='store_true',
|
||||
help='enable debugging extras',)
|
||||
parser.add_option('--warnings-as-errors', action='store_true',
|
||||
help='convert warnings into errors during build',)
|
||||
parser.add_option('--profile', metavar='TYPE',
|
||||
choices=profilers,
|
||||
help='enable profiling (' + '/'.join(profilers) + ')',)
|
||||
parser.add_option('--gtest-source-dir', metavar='PATH',
|
||||
help='Path to GoogleTest source directory. If not provided ' +
|
||||
'GTEST_SOURCE_DIR will be probed in the environment. ' +
|
||||
'Tests will not be built without a value.')
|
||||
parser.add_option('--with-python', metavar='EXE',
|
||||
help='use EXE as the Python interpreter',
|
||||
default=os.path.basename(sys.executable))
|
||||
parser.add_option('--force-pselect', action='store_true',
|
||||
help='ppoll() is used by default where available, '
|
||||
'but some platforms may need to use pselect instead',)
|
||||
(options, args) = parser.parse_args()
|
||||
if args:
|
||||
print('ERROR: extra unparsed command-line arguments:', args)
|
||||
sys.exit(1)
|
||||
|
||||
platform = Platform(options.platform)
|
||||
if options.host:
|
||||
host = Platform(options.host)
|
||||
else:
|
||||
host = platform
|
||||
|
||||
BUILD_FILENAME = 'build.ninja'
|
||||
ninja_writer = ninja_syntax.Writer(open(BUILD_FILENAME, 'w'))
|
||||
n: Union[ninja_syntax.Writer, Bootstrap] = ninja_writer
|
||||
|
||||
if options.bootstrap:
|
||||
# Make the build directory.
|
||||
try:
|
||||
os.mkdir('build')
|
||||
except OSError:
|
||||
pass
|
||||
# Wrap ninja_writer with the Bootstrapper, which also executes the
|
||||
# commands.
|
||||
print('bootstrapping ninja...')
|
||||
n = Bootstrap(n, verbose=options.verbose) # type: ignore # Incompatible types in assignment
|
||||
|
||||
n.comment('This file is used to build ninja itself.')
|
||||
n.comment('It is generated by ' + os.path.basename(__file__) + '.')
|
||||
n.newline()
|
||||
|
||||
n.variable('ninja_required_version', '1.3')
|
||||
n.newline()
|
||||
|
||||
n.comment('The arguments passed to configure.py, for rerunning it.')
|
||||
configure_args = sys.argv[1:]
|
||||
if '--bootstrap' in configure_args:
|
||||
configure_args.remove('--bootstrap')
|
||||
n.variable('configure_args', ' '.join(configure_args))
|
||||
env_keys = set(['CXX', 'AR', 'CFLAGS', 'CXXFLAGS', 'LDFLAGS'])
|
||||
configure_env = dict((k, os.environ[k]) for k in os.environ if k in env_keys)
|
||||
if configure_env:
|
||||
config_str = ' '.join([k + '=' + shlex.quote(configure_env[k])
|
||||
for k in configure_env])
|
||||
n.variable('configure_env', config_str + '$ ')
|
||||
n.newline()
|
||||
|
||||
CXX = configure_env.get('CXX', 'c++')
|
||||
objext = '.o'
|
||||
if platform.is_msvc():
|
||||
CXX = 'cl'
|
||||
objext = '.obj'
|
||||
|
||||
def src(filename: str) -> str:
|
||||
return os.path.join('$root', 'src', filename)
|
||||
def built(filename: str) -> str:
|
||||
return os.path.join('$builddir', filename)
|
||||
def doc(filename: str) -> str:
|
||||
return os.path.join('$root', 'doc', filename)
|
||||
def cc(name: str, **kwargs: Any) -> List[str]:
|
||||
return n.build(built(name + objext), 'cxx', src(name + '.c'), **kwargs)
|
||||
def cxx(name: str, **kwargs: Any) -> List[str]:
|
||||
return n.build(built(name + objext), 'cxx', src(name + '.cc'), **kwargs)
|
||||
def binary(name: str) -> str:
|
||||
if platform.is_windows():
|
||||
exe = name + '.exe'
|
||||
n.build(name, 'phony', exe)
|
||||
return exe
|
||||
return name
|
||||
|
||||
root = sourcedir
|
||||
if root == os.getcwd():
|
||||
# In the common case where we're building directly in the source
|
||||
# tree, simplify all the paths to just be cwd-relative.
|
||||
root = '.'
|
||||
n.variable('root', root)
|
||||
n.variable('builddir', 'build')
|
||||
n.variable('cxx', CXX)
|
||||
if platform.is_msvc():
|
||||
n.variable('ar', 'link')
|
||||
else:
|
||||
n.variable('ar', configure_env.get('AR', 'ar'))
|
||||
|
||||
def search_system_path(file_name: str) -> Optional[str]: # type: ignore # Missing return statement
|
||||
"""Find a file in the system path."""
|
||||
for dir in os.environ['path'].split(';'):
|
||||
path = os.path.join(dir, file_name)
|
||||
if os.path.exists(path):
|
||||
return path
|
||||
|
||||
# Note that build settings are separately specified in CMakeLists.txt and
|
||||
# these lists should be kept in sync.
|
||||
if platform.is_msvc():
|
||||
if not search_system_path('cl.exe'):
|
||||
raise Exception('cl.exe not found. Run again from the Developer Command Prompt for VS')
|
||||
cflags = ['/showIncludes',
|
||||
'/nologo', # Don't print startup banner.
|
||||
'/utf-8',
|
||||
'/Zi', # Create pdb with debug info.
|
||||
'/W4', # Highest warning level.
|
||||
'/WX', # Warnings as errors.
|
||||
'/wd4530', '/wd4100', '/wd4706', '/wd4244',
|
||||
'/wd4512', '/wd4800', '/wd4702',
|
||||
# Disable warnings about constant conditional expressions.
|
||||
'/wd4127',
|
||||
# Disable warnings about passing "this" during initialization.
|
||||
'/wd4355',
|
||||
# Disable warnings about ignored typedef in DbgHelp.h
|
||||
'/wd4091',
|
||||
'/GR-', # Disable RTTI.
|
||||
'/Zc:__cplusplus',
|
||||
# Disable size_t -> int truncation warning.
|
||||
# We never have strings or arrays larger than 2**31.
|
||||
'/wd4267',
|
||||
'/DNOMINMAX', '/D_CRT_SECURE_NO_WARNINGS',
|
||||
'/D_HAS_EXCEPTIONS=0',
|
||||
'/DNINJA_PYTHON="%s"' % options.with_python]
|
||||
if options.warnings_as_errors:
|
||||
cflags.append('/WX')
|
||||
if platform.msvc_needs_fs():
|
||||
cflags.append('/FS')
|
||||
ldflags = ['/DEBUG', '/libpath:$builddir']
|
||||
if not options.debug:
|
||||
cflags += ['/Ox', '/DNDEBUG', '/GL']
|
||||
ldflags += ['/LTCG', '/OPT:REF', '/OPT:ICF']
|
||||
else:
|
||||
cflags = ['-g', '-Wall', '-Wextra',
|
||||
'-Wno-deprecated',
|
||||
'-Wno-missing-field-initializers',
|
||||
'-Wno-unused-parameter',
|
||||
'-fno-rtti',
|
||||
'-fno-exceptions',
|
||||
'-std=c++14',
|
||||
'-fvisibility=hidden', '-pipe',
|
||||
'-DNINJA_PYTHON="%s"' % options.with_python]
|
||||
if options.warnings_as_errors:
|
||||
cflags += [ "-Werror" ]
|
||||
if options.debug:
|
||||
cflags += ['-D_GLIBCXX_DEBUG', '-D_GLIBCXX_DEBUG_PEDANTIC']
|
||||
cflags.remove('-fno-rtti') # Needed for above pedanticness.
|
||||
else:
|
||||
cflags += ['-O2', '-DNDEBUG']
|
||||
try:
|
||||
proc = subprocess.Popen(
|
||||
[CXX, '-fdiagnostics-color', '-c', '-x', 'c++', '/dev/null',
|
||||
'-o', '/dev/null'],
|
||||
stdout=open(os.devnull, 'wb'), stderr=subprocess.STDOUT)
|
||||
if proc.wait() == 0:
|
||||
cflags += ['-fdiagnostics-color']
|
||||
except:
|
||||
pass
|
||||
if platform.is_mingw():
|
||||
cflags += ['-D_WIN32_WINNT=0x0601', '-D__USE_MINGW_ANSI_STDIO=1']
|
||||
ldflags = ['-L$builddir']
|
||||
if platform.uses_usr_local():
|
||||
cflags.append('-I/usr/local/include')
|
||||
ldflags.append('-L/usr/local/lib')
|
||||
if platform.is_aix():
|
||||
# printf formats for int64_t, uint64_t; large file support
|
||||
cflags.append('-D__STDC_FORMAT_MACROS')
|
||||
cflags.append('-D_LARGE_FILES')
|
||||
|
||||
|
||||
libs = []
|
||||
|
||||
if platform.is_mingw():
|
||||
cflags.remove('-fvisibility=hidden');
|
||||
ldflags.append('-static')
|
||||
elif platform.is_solaris():
|
||||
cflags.remove('-fvisibility=hidden')
|
||||
elif platform.is_aix():
|
||||
cflags.remove('-fvisibility=hidden')
|
||||
elif platform.is_msvc():
|
||||
pass
|
||||
else:
|
||||
if options.profile == 'gmon':
|
||||
cflags.append('-pg')
|
||||
ldflags.append('-pg')
|
||||
elif options.profile == 'pprof':
|
||||
cflags.append('-fno-omit-frame-pointer')
|
||||
libs.extend(['-Wl,--no-as-needed', '-lprofiler'])
|
||||
|
||||
if platform.supports_ppoll() and not options.force_pselect:
|
||||
cflags.append('-DUSE_PPOLL')
|
||||
if platform.supports_ninja_browse():
|
||||
cflags.append('-DNINJA_HAVE_BROWSE')
|
||||
|
||||
# Search for generated headers relative to build dir.
|
||||
cflags.append('-I.')
|
||||
|
||||
def shell_escape(str: str) -> str:
|
||||
"""Escape str such that it's interpreted as a single argument by
|
||||
the shell."""
|
||||
|
||||
# This isn't complete, but it's just enough to make NINJA_PYTHON work.
|
||||
if platform.is_windows():
|
||||
return str
|
||||
if '"' in str:
|
||||
return "'%s'" % str.replace("'", "\\'")
|
||||
return str
|
||||
|
||||
if 'CFLAGS' in configure_env:
|
||||
cflags.append(configure_env['CFLAGS'])
|
||||
ldflags.append(configure_env['CFLAGS'])
|
||||
if 'CXXFLAGS' in configure_env:
|
||||
cflags.append(configure_env['CXXFLAGS'])
|
||||
ldflags.append(configure_env['CXXFLAGS'])
|
||||
n.variable('cflags', ' '.join(shell_escape(flag) for flag in cflags))
|
||||
if 'LDFLAGS' in configure_env:
|
||||
ldflags.append(configure_env['LDFLAGS'])
|
||||
n.variable('ldflags', ' '.join(shell_escape(flag) for flag in ldflags))
|
||||
|
||||
n.newline()
|
||||
|
||||
if platform.is_msvc():
|
||||
n.rule('cxx',
|
||||
command='$cxx $cflags -c $in /Fo$out /Fd' + built('$pdb'),
|
||||
description='CXX $out',
|
||||
deps='msvc' # /showIncludes is included in $cflags.
|
||||
)
|
||||
else:
|
||||
n.rule('cxx',
|
||||
command='$cxx -MMD -MT $out -MF $out.d $cflags -c $in -o $out',
|
||||
depfile='$out.d',
|
||||
deps='gcc',
|
||||
description='CXX $out')
|
||||
n.newline()
|
||||
|
||||
if host.is_msvc():
|
||||
n.rule('ar',
|
||||
command='lib /nologo /ltcg /out:$out $in',
|
||||
description='LIB $out')
|
||||
elif host.is_mingw():
|
||||
n.rule('ar',
|
||||
command='$ar crs $out $in',
|
||||
description='AR $out')
|
||||
else:
|
||||
n.rule('ar',
|
||||
command='rm -f $out && $ar crs $out $in',
|
||||
description='AR $out')
|
||||
n.newline()
|
||||
|
||||
if platform.is_msvc():
|
||||
n.rule('link',
|
||||
command='$cxx $in $libs /nologo /link $ldflags /out:$out',
|
||||
description='LINK $out')
|
||||
else:
|
||||
n.rule('link',
|
||||
command='$cxx $ldflags -o $out $in $libs',
|
||||
description='LINK $out')
|
||||
n.newline()
|
||||
|
||||
objs = []
|
||||
|
||||
if platform.supports_ninja_browse():
|
||||
n.comment('browse_py.h is used to inline browse.py.')
|
||||
n.rule('inline',
|
||||
command='"%s"' % src('inline.sh') + ' $varname < $in > $out',
|
||||
description='INLINE $out')
|
||||
n.build(built('browse_py.h'), 'inline', src('browse.py'),
|
||||
implicit=src('inline.sh'),
|
||||
variables=[('varname', 'kBrowsePy')])
|
||||
n.newline()
|
||||
|
||||
objs += cxx('browse', order_only=built('browse_py.h'))
|
||||
n.newline()
|
||||
|
||||
n.comment('the depfile parser and ninja lexers are generated using re2c.')
|
||||
def has_re2c() -> bool:
|
||||
try:
|
||||
proc = subprocess.Popen(['re2c', '-V'], stdout=subprocess.PIPE)
|
||||
return int(proc.communicate()[0], 10) >= 1503
|
||||
except OSError:
|
||||
return False
|
||||
if has_re2c():
|
||||
n.rule('re2c',
|
||||
command='re2c -b -i --no-generation-date --no-version -o $out $in',
|
||||
description='RE2C $out')
|
||||
# Generate the .cc files in the source directory so we can check them in.
|
||||
n.build(src('depfile_parser.cc'), 're2c', src('depfile_parser.in.cc'))
|
||||
n.build(src('lexer.cc'), 're2c', src('lexer.in.cc'))
|
||||
else:
|
||||
print("warning: A compatible version of re2c (>= 0.15.3) was not found; "
|
||||
"changes to src/*.in.cc will not affect your build.")
|
||||
n.newline()
|
||||
|
||||
cxxvariables = []
|
||||
if platform.is_msvc():
|
||||
cxxvariables = [('pdb', 'ninja.pdb')]
|
||||
|
||||
n.comment('Generate a library for `ninja-re2c`.')
|
||||
re2c_objs = []
|
||||
for name in ['depfile_parser', 'lexer']:
|
||||
re2c_objs += cxx(name, variables=cxxvariables)
|
||||
if platform.is_msvc():
|
||||
n.build(built('ninja-re2c.lib'), 'ar', re2c_objs)
|
||||
else:
|
||||
n.build(built('libninja-re2c.a'), 'ar', re2c_objs)
|
||||
n.newline()
|
||||
|
||||
n.comment('Core source files all build into ninja library.')
|
||||
objs.extend(re2c_objs)
|
||||
for name in ['build',
|
||||
'build_log',
|
||||
'clean',
|
||||
'clparser',
|
||||
'debug_flags',
|
||||
'deps_log',
|
||||
'disk_interface',
|
||||
'dyndep',
|
||||
'dyndep_parser',
|
||||
'edit_distance',
|
||||
'elide_middle',
|
||||
'eval_env',
|
||||
'graph',
|
||||
'graphviz',
|
||||
'jobserver',
|
||||
'json',
|
||||
'line_printer',
|
||||
'manifest_parser',
|
||||
'metrics',
|
||||
'missing_deps',
|
||||
'parser',
|
||||
'real_command_runner',
|
||||
'state',
|
||||
'status_printer',
|
||||
'string_piece_util',
|
||||
'util',
|
||||
'version']:
|
||||
objs += cxx(name, variables=cxxvariables)
|
||||
if platform.is_windows():
|
||||
for name in ['subprocess-win32',
|
||||
'includes_normalize-win32',
|
||||
'jobserver-win32',
|
||||
'msvc_helper-win32',
|
||||
'msvc_helper_main-win32']:
|
||||
objs += cxx(name, variables=cxxvariables)
|
||||
if platform.is_msvc():
|
||||
objs += cxx('minidump-win32', variables=cxxvariables)
|
||||
objs += cc('getopt')
|
||||
else:
|
||||
for name in ['jobserver-posix',
|
||||
'subprocess-posix']:
|
||||
objs += cxx(name, variables=cxxvariables)
|
||||
if platform.is_aix():
|
||||
objs += cc('getopt')
|
||||
if platform.is_msvc():
|
||||
ninja_lib = n.build(built('ninja.lib'), 'ar', objs)
|
||||
else:
|
||||
ninja_lib = n.build(built('libninja.a'), 'ar', objs)
|
||||
n.newline()
|
||||
|
||||
if platform.is_msvc():
|
||||
libs.append('ninja.lib')
|
||||
else:
|
||||
libs.append('-lninja')
|
||||
|
||||
if platform.is_aix() and not platform.is_os400_pase():
|
||||
libs.append('-lperfstat')
|
||||
|
||||
all_targets = []
|
||||
|
||||
n.comment('Main executable is library plus main() function.')
|
||||
objs = cxx('ninja', variables=cxxvariables)
|
||||
ninja = n.build(binary('ninja'), 'link', objs, implicit=ninja_lib,
|
||||
variables=[('libs', libs)])
|
||||
n.newline()
|
||||
all_targets += ninja
|
||||
|
||||
if options.bootstrap:
|
||||
# We've built the ninja binary. Don't run any more commands
|
||||
# through the bootstrap executor, but continue writing the
|
||||
# build.ninja file.
|
||||
n = ninja_writer
|
||||
|
||||
# Build the ninja_test executable only if the GTest source directory
|
||||
# is provided explicitly. Either from the environment with GTEST_SOURCE_DIR
|
||||
# or with the --gtest-source-dir command-line option.
|
||||
#
|
||||
# Do not try to look for an installed binary version, and link against it
|
||||
# because doing so properly is platform-specific (use the CMake build for
|
||||
# this).
|
||||
if options.gtest_source_dir:
|
||||
gtest_src_dir = options.gtest_source_dir
|
||||
else:
|
||||
gtest_src_dir = os.environ.get('GTEST_SOURCE_DIR')
|
||||
|
||||
if gtest_src_dir:
|
||||
# Verify GoogleTest source directory, and add its include directory
|
||||
# to the global include search path (even for non-test sources) to
|
||||
# keep the build plan generation simple.
|
||||
gtest_all_cc = os.path.join(gtest_src_dir, 'googletest', 'src', 'gtest-all.cc')
|
||||
if not os.path.exists(gtest_all_cc):
|
||||
print('ERROR: Missing GoogleTest source file: %s' % gtest_all_cc)
|
||||
sys.exit(1)
|
||||
|
||||
n.comment('Tests all build into ninja_test executable.')
|
||||
|
||||
# Test-specific version of cflags, must include the GoogleTest
|
||||
# include directory.
|
||||
test_cflags = cflags.copy()
|
||||
test_cflags.append('-I' + os.path.join(gtest_src_dir, 'googletest', 'include'))
|
||||
|
||||
test_variables = [('cflags', test_cflags)]
|
||||
if platform.is_msvc():
|
||||
test_variables += [('pdb', 'ninja_test.pdb')]
|
||||
|
||||
test_names = [
|
||||
'build_log_test',
|
||||
'build_test',
|
||||
'clean_test',
|
||||
'clparser_test',
|
||||
'depfile_parser_test',
|
||||
'deps_log_test',
|
||||
'disk_interface_test',
|
||||
'dyndep_parser_test',
|
||||
'edit_distance_test',
|
||||
'elide_middle_test',
|
||||
'explanations_test',
|
||||
'graph_test',
|
||||
'jobserver_test',
|
||||
'json_test',
|
||||
'lexer_test',
|
||||
'manifest_parser_test',
|
||||
'ninja_test',
|
||||
'state_test',
|
||||
'string_piece_util_test',
|
||||
'subprocess_test',
|
||||
'test',
|
||||
'util_test',
|
||||
]
|
||||
if platform.is_windows():
|
||||
test_names += [
|
||||
'includes_normalize_test',
|
||||
'msvc_helper_test',
|
||||
]
|
||||
|
||||
objs = []
|
||||
for name in test_names:
|
||||
objs += cxx(name, variables=test_variables)
|
||||
|
||||
# Build GTest as a monolithic source file.
|
||||
# This requires one extra include search path, so replace the
|
||||
# value of 'cflags' in our list.
|
||||
gtest_all_variables = test_variables[1:] + [
|
||||
('cflags', test_cflags + ['-I' + os.path.join(gtest_src_dir, 'googletest') ]),
|
||||
]
|
||||
# Do not use cxx() directly to ensure the object file is under $builddir.
|
||||
objs += n.build(built('gtest_all' + objext), 'cxx', gtest_all_cc, variables=gtest_all_variables)
|
||||
|
||||
ninja_test = n.build(binary('ninja_test'), 'link', objs, implicit=ninja_lib,
|
||||
variables=[('libs', libs)])
|
||||
n.newline()
|
||||
all_targets += ninja_test
|
||||
|
||||
n.comment('Ancillary executables.')
|
||||
|
||||
if platform.is_aix() and '-maix64' not in ldflags:
|
||||
# Both hash_collision_bench and manifest_parser_perftest require more
|
||||
# memory than will fit in the standard 32-bit AIX shared stack/heap (256M)
|
||||
libs.append('-Wl,-bmaxdata:0x80000000')
|
||||
|
||||
for name in ['build_log_perftest',
|
||||
'canon_perftest',
|
||||
'elide_middle_perftest',
|
||||
'depfile_parser_perftest',
|
||||
'hash_collision_bench',
|
||||
'manifest_parser_perftest',
|
||||
'clparser_perftest']:
|
||||
if platform.is_msvc():
|
||||
cxxvariables = [('pdb', name + '.pdb')]
|
||||
objs = cxx(name, variables=cxxvariables)
|
||||
all_targets += n.build(binary(name), 'link', objs,
|
||||
implicit=ninja_lib, variables=[('libs', libs)])
|
||||
|
||||
n.newline()
|
||||
|
||||
n.comment('Generate a graph using the "graph" tool.')
|
||||
n.rule('gendot',
|
||||
command='./ninja -t graph all > $out')
|
||||
n.rule('gengraph',
|
||||
command='dot -Tpng $in > $out')
|
||||
dot = n.build(built('graph.dot'), 'gendot', ['ninja', 'build.ninja'])
|
||||
n.build('graph.png', 'gengraph', dot)
|
||||
n.newline()
|
||||
|
||||
n.comment('Generate the manual using asciidoc.')
|
||||
n.rule('asciidoc',
|
||||
command='asciidoc -b docbook -d book -o $out $in',
|
||||
description='ASCIIDOC $out')
|
||||
n.rule('xsltproc',
|
||||
command='xsltproc --nonet doc/docbook.xsl $in > $out',
|
||||
description='XSLTPROC $out')
|
||||
docbookxml = n.build(built('manual.xml'), 'asciidoc', doc('manual.asciidoc'))
|
||||
manual = n.build(doc('manual.html'), 'xsltproc', docbookxml,
|
||||
implicit=[doc('style.css'), doc('docbook.xsl')])
|
||||
n.build('manual', 'phony',
|
||||
order_only=manual)
|
||||
n.newline()
|
||||
|
||||
n.rule('dblatex',
|
||||
command='dblatex -q -o $out -p doc/dblatex.xsl $in',
|
||||
description='DBLATEX $out')
|
||||
n.build(doc('manual.pdf'), 'dblatex', docbookxml,
|
||||
implicit=[doc('dblatex.xsl')])
|
||||
|
||||
n.comment('Generate Doxygen.')
|
||||
n.rule('doxygen',
|
||||
command='doxygen $in',
|
||||
description='DOXYGEN $in')
|
||||
n.variable('doxygen_mainpage_generator',
|
||||
src('gen_doxygen_mainpage.sh'))
|
||||
n.rule('doxygen_mainpage',
|
||||
command='$doxygen_mainpage_generator $in > $out',
|
||||
description='DOXYGEN_MAINPAGE $out')
|
||||
mainpage = n.build(built('doxygen_mainpage'), 'doxygen_mainpage',
|
||||
['README.md', 'COPYING'],
|
||||
implicit=['$doxygen_mainpage_generator'])
|
||||
n.build('doxygen', 'doxygen', doc('doxygen.config'),
|
||||
implicit=mainpage)
|
||||
n.newline()
|
||||
|
||||
if not host.is_mingw():
|
||||
n.comment('Regenerate build files if build script changes.')
|
||||
n.rule('configure',
|
||||
command='${configure_env}%s $root/configure.py $configure_args' %
|
||||
options.with_python,
|
||||
generator=True)
|
||||
n.build('build.ninja', 'configure',
|
||||
implicit=['$root/configure.py',
|
||||
os.path.normpath('$root/misc/ninja_syntax.py')])
|
||||
n.newline()
|
||||
|
||||
n.default(ninja)
|
||||
n.newline()
|
||||
|
||||
if host.is_linux():
|
||||
n.comment('Packaging')
|
||||
n.rule('rpmbuild',
|
||||
command="misc/packaging/rpmbuild.sh",
|
||||
description='Building rpms..')
|
||||
n.build('rpm', 'rpmbuild')
|
||||
n.newline()
|
||||
|
||||
n.build('all', 'phony', all_targets)
|
||||
|
||||
n.close() # type: ignore # Item "Bootstrap" of "Writer | Bootstrap" has no attribute "close"
|
||||
print('wrote %s.' % BUILD_FILENAME)
|
||||
|
||||
if options.bootstrap:
|
||||
print('bootstrap complete. rebuilding...')
|
||||
|
||||
rebuild_args = []
|
||||
|
||||
if platform.can_rebuild_in_place():
|
||||
rebuild_args.append('./ninja')
|
||||
else:
|
||||
if platform.is_windows():
|
||||
bootstrap_exe = 'ninja.bootstrap.exe'
|
||||
final_exe = 'ninja.exe'
|
||||
else:
|
||||
bootstrap_exe = './ninja.bootstrap'
|
||||
final_exe = './ninja'
|
||||
|
||||
if os.path.exists(bootstrap_exe):
|
||||
os.unlink(bootstrap_exe)
|
||||
os.rename(final_exe, bootstrap_exe)
|
||||
|
||||
rebuild_args.append(bootstrap_exe)
|
||||
|
||||
if options.verbose:
|
||||
rebuild_args.append('-v')
|
||||
|
||||
subprocess.check_call(rebuild_args)
|
||||
@@ -0,0 +1,11 @@
|
||||
This directory contains the Ninja manual and support files used in
|
||||
building it. Here's a brief overview of how it works.
|
||||
|
||||
The source text, `manual.asciidoc`, is written in the AsciiDoc format.
|
||||
AsciiDoc can generate HTML but it doesn't look great; instead, we use
|
||||
AsciiDoc to generate the Docbook XML format and then provide our own
|
||||
Docbook XSL tweaks to produce HTML from that.
|
||||
|
||||
In theory using AsciiDoc and DocBook allows us to produce nice PDF
|
||||
documentation etc. In reality it's not clear anyone wants that, but the
|
||||
build rules are in place to generate it if you install dblatex.
|
||||
@@ -0,0 +1,7 @@
|
||||
<!-- This custom XSL tweaks the dblatex XML settings. -->
|
||||
<xsl:stylesheet xmlns:xsl='http://www.w3.org/1999/XSL/Transform' version='1.0'>
|
||||
<!-- These parameters disable the list of collaborators and revisions.
|
||||
Together remove a useless page from the front matter. -->
|
||||
<xsl:param name='doc.collab.show'>0</xsl:param>
|
||||
<xsl:param name='latex.output.revhistory'>0</xsl:param>
|
||||
</xsl:stylesheet>
|
||||
@@ -0,0 +1,34 @@
|
||||
<!-- This custom XSL tweaks the DocBook XML -> HTML settings to produce
|
||||
an OK-looking manual. -->
|
||||
<!DOCTYPE xsl:stylesheet [
|
||||
<!ENTITY css SYSTEM "style.css">
|
||||
]>
|
||||
<xsl:stylesheet xmlns:xsl="http://www.w3.org/1999/XSL/Transform"
|
||||
version='1.0'>
|
||||
<xsl:import href="http://docbook.sourceforge.net/release/xsl/current/html/docbook.xsl"/>
|
||||
|
||||
<!-- Embed our stylesheet as the user-provided <head> content. -->
|
||||
<xsl:template name="user.head.content"><style>&css;</style></xsl:template>
|
||||
|
||||
<!-- Remove the body.attributes block, which specifies a bunch of
|
||||
useless bgcolor etc. attrs on the <body> tag. -->
|
||||
<xsl:template name="body.attributes"></xsl:template>
|
||||
|
||||
<!-- Specify that in "book" form (which we're using), we only want a
|
||||
single table of contents at the beginning of the document. -->
|
||||
<xsl:param name="generate.toc">book toc</xsl:param>
|
||||
|
||||
<!-- Don't put the "Chapter 1." prefix on the "chapters". -->
|
||||
<xsl:param name="chapter.autolabel">0</xsl:param>
|
||||
|
||||
<!-- Make builds reproducible by generating the same IDs from the same inputs -->
|
||||
<xsl:param name="generate.consistent.ids">1</xsl:param>
|
||||
|
||||
<!-- Use <ul> for the table of contents. By default DocBook uses a
|
||||
<dl>, which makes no semantic sense. I imagine they just did
|
||||
it because it looks nice? -->
|
||||
<xsl:param name="toc.list.type">ul</xsl:param>
|
||||
|
||||
<xsl:output method="html" encoding="utf-8" indent="no"
|
||||
doctype-public=""/>
|
||||
</xsl:stylesheet>
|
||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,68 @@
|
||||
:root {
|
||||
color-scheme: light dark;
|
||||
}
|
||||
|
||||
body {
|
||||
margin: 5ex 10ex;
|
||||
max-width: 80ex;
|
||||
line-height: 1.5;
|
||||
font-family: sans-serif;
|
||||
}
|
||||
|
||||
h1, h2, h3 {
|
||||
font-weight: normal;
|
||||
}
|
||||
|
||||
pre, code {
|
||||
font-family: x, monospace;
|
||||
}
|
||||
|
||||
pre {
|
||||
padding: 1ex;
|
||||
background: #eee;
|
||||
border: solid 1px #ddd;
|
||||
min-width: 0;
|
||||
font-size: 90%;
|
||||
}
|
||||
@media (prefers-color-scheme: dark) {
|
||||
pre {
|
||||
background: #333;
|
||||
border: solid 1px #444;
|
||||
}
|
||||
}
|
||||
|
||||
code {
|
||||
color: #007;
|
||||
}
|
||||
@media (prefers-color-scheme: dark) {
|
||||
code {
|
||||
color: #a7cec8;
|
||||
}
|
||||
}
|
||||
|
||||
div.chapter {
|
||||
margin-top: 4em;
|
||||
border-top: solid 2px black;
|
||||
}
|
||||
@media (prefers-color-scheme: dark) {
|
||||
div.chapter {
|
||||
border-top: solid 2px white;
|
||||
}
|
||||
}
|
||||
|
||||
p {
|
||||
margin-top: 0;
|
||||
}
|
||||
|
||||
/* The following applies to the left column of a [horizontal] labeled list: */
|
||||
table.horizontal > tbody > tr > td:nth-child(1) {
|
||||
|
||||
/* prevent the insertion of a line-break in the middle of a label: */
|
||||
white-space: nowrap;
|
||||
|
||||
/* insert a little horizontal padding between the two columns: */
|
||||
padding-right: 1.5em;
|
||||
|
||||
/* right-justify labels: */
|
||||
text-align: end;
|
||||
}
|
||||
@@ -0,0 +1 @@
|
||||
build
|
||||
@@ -0,0 +1 @@
|
||||
default
|
||||
@@ -0,0 +1 @@
|
||||
include
|
||||
@@ -0,0 +1 @@
|
||||
pool
|
||||
@@ -0,0 +1 @@
|
||||
rule
|
||||
@@ -0,0 +1 @@
|
||||
subninja
|
||||
@@ -0,0 +1 @@
|
||||
a
|
||||
@@ -0,0 +1 @@
|
||||
b
|
||||
@@ -0,0 +1 @@
|
||||
:
|
||||
@@ -0,0 +1 @@
|
||||
$
|
||||
@@ -0,0 +1 @@
|
||||
$
|
||||
@@ -0,0 +1 @@
|
||||
=
|
||||
@@ -0,0 +1 @@
|
||||
|
|
||||
@@ -0,0 +1 @@
|
||||
||
|
||||
@@ -0,0 +1,5 @@
|
||||
rule b
|
||||
command = clang -MMD -MF $out.d -o $out -c $in
|
||||
description = building $out
|
||||
|
||||
build a.o: b a.c
|
||||
@@ -0,0 +1,57 @@
|
||||
# Copyright 2011 Google Inc. All Rights Reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
# Add the following to your .bashrc to tab-complete ninja targets
|
||||
# . path/to/ninja/misc/bash-completion
|
||||
|
||||
_ninja_target() {
|
||||
local cur prev targets dir line targets_command OPTIND
|
||||
|
||||
# When available, use bash_completion to:
|
||||
# 1) Complete words when the cursor is in the middle of the word
|
||||
# 2) Complete paths with files or directories, as appropriate
|
||||
if _get_comp_words_by_ref cur prev &>/dev/null ; then
|
||||
case $prev in
|
||||
-f)
|
||||
_filedir
|
||||
return 0
|
||||
;;
|
||||
-C)
|
||||
_filedir -d
|
||||
return 0
|
||||
;;
|
||||
esac
|
||||
else
|
||||
cur="${COMP_WORDS[COMP_CWORD]}"
|
||||
fi
|
||||
|
||||
if [[ "$cur" == "--"* ]]; then
|
||||
# there is currently only one argument that takes --
|
||||
COMPREPLY=($(compgen -P '--' -W 'version' -- "${cur:2}"))
|
||||
else
|
||||
dir="."
|
||||
line=$(echo ${COMP_LINE} | cut -d" " -f 2-)
|
||||
# filter out all non relevant arguments but keep C for dirs
|
||||
while getopts :C:f:j:l:k:nvd:t: opt $line; do
|
||||
case $opt in
|
||||
# eval for tilde expansion
|
||||
C) eval dir="$OPTARG" ;;
|
||||
esac
|
||||
done;
|
||||
targets_command="eval ninja -C \"${dir}\" -t targets all 2>/dev/null | cut -d: -f1"
|
||||
COMPREPLY=($(compgen -W '`${targets_command}`' -- "$cur"))
|
||||
fi
|
||||
return
|
||||
}
|
||||
complete -F _ninja_target ninja
|
||||
+46
@@ -0,0 +1,46 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
import os
|
||||
|
||||
ignores = [
|
||||
'.git/',
|
||||
'misc/afl-fuzz-tokens/',
|
||||
'src/depfile_parser.cc',
|
||||
'src/lexer.cc',
|
||||
]
|
||||
|
||||
error_count = 0
|
||||
|
||||
def error(path: str, msg: str) -> None:
|
||||
global error_count
|
||||
error_count += 1
|
||||
print('\x1b[1;31m{}\x1b[0;31m{}\x1b[0m'.format(path, msg))
|
||||
|
||||
try:
|
||||
import git
|
||||
repo = git.Repo('.')
|
||||
except:
|
||||
repo = None
|
||||
|
||||
for root, directory, filenames in os.walk('.'):
|
||||
for filename in filenames:
|
||||
path = os.path.join(root, filename)[2:]
|
||||
if any([path.startswith(x) for x in ignores]) or (repo is not None and repo.ignored(path)):
|
||||
continue
|
||||
with open(path, 'rb') as file:
|
||||
line_nr = 1
|
||||
try:
|
||||
for line in [x.decode() for x in file.readlines()]:
|
||||
if len(line) == 0 or line[-1] != '\n':
|
||||
error(path, ' missing newline at end of file.')
|
||||
if len(line) > 1:
|
||||
if line[-2] == '\r':
|
||||
error(path, ' has Windows line endings.')
|
||||
break
|
||||
if line[-2] == ' ' or line[-2] == '\t':
|
||||
error(path, ':{} has trailing whitespace.'.format(line_nr))
|
||||
line_nr += 1
|
||||
except UnicodeError:
|
||||
pass # binary file
|
||||
|
||||
exit(error_count)
|
||||
@@ -0,0 +1,23 @@
|
||||
# This build file prints out a list of open file descriptors in
|
||||
# Ninja subprocesses, to help verify we don't accidentally leak
|
||||
# any.
|
||||
|
||||
# Because one fd leak was in the code managing multiple subprocesses,
|
||||
# this test brings up multiple subprocesses and then dumps the fd
|
||||
# table of the last one.
|
||||
|
||||
# Use like: ./ninja -f misc/inherited-fds.ninja
|
||||
|
||||
rule sleep
|
||||
command = sleep 10000
|
||||
|
||||
rule dump
|
||||
command = sleep 1; ls -l /proc/self/fd; exit 1
|
||||
|
||||
build all: phony a b c d e
|
||||
|
||||
build a: sleep
|
||||
build b: sleep
|
||||
build c: sleep
|
||||
build d: sleep
|
||||
build e: dump
|
||||
+340
@@ -0,0 +1,340 @@
|
||||
#!/usr/bin/env python3
|
||||
# Copyright 2024 Google Inc. All Rights Reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
"""Setup a GNU Make Jobserver jobs pool then launch a command with it.
|
||||
|
||||
On Windows, this only supports the semaphore-based scheme.
|
||||
On Posix, this uses a fifo by default, use --pipe for pipe mode.
|
||||
|
||||
On exit, this script verifies that all job slots were returned to
|
||||
the pool, and will print an error message if this is not the case.
|
||||
|
||||
This is useful to catch the use of broken protocol clients.
|
||||
Use the `--no-check` flag to disable this.
|
||||
|
||||
See --help-usage for usage examples.
|
||||
"""
|
||||
import argparse
|
||||
import os
|
||||
import platform
|
||||
import subprocess
|
||||
import sys
|
||||
import typing as T
|
||||
|
||||
# Technical note about the MAKEFLAGS values set by this script.
|
||||
#
|
||||
# All the MAKEFLAGS values created by this script begin
|
||||
# with " -j{count} ", i.e. an initial space, the "-j" characters
|
||||
# followed by a job slot count then another space.
|
||||
#
|
||||
# The initial space is only there to mimic what GNU Make 4.3
|
||||
# does. Other pool implementations do not use one and thus
|
||||
# clients should not expect it (even GNU Make doesn't seem
|
||||
# to care when used as a jobserver client).
|
||||
#
|
||||
# The {count} value is also not available in many pool
|
||||
# implementations, but is useful to better debug multi-builds
|
||||
# using this script (i.e. to verify that the pool has the
|
||||
# expected size). Protocol clients should not depend on it
|
||||
# though.
|
||||
|
||||
_DEFAULT_NAME = "jobserver_pool"
|
||||
_IS_WINDOWS = sys.platform in ("win32", "cygwin")
|
||||
|
||||
if _IS_WINDOWS:
|
||||
|
||||
try:
|
||||
# This requires pywin32 to be installed.
|
||||
import pywintypes
|
||||
import win32event
|
||||
import win32api
|
||||
except ModuleNotFoundError:
|
||||
print(
|
||||
"\nERROR: Could not import Win32 API, please install pywin32, e.g. `python -m pip install pywin32`.\n",
|
||||
file=sys.stderr,
|
||||
)
|
||||
raise
|
||||
|
||||
# It seems impossible to import a proper mypy-compatible type definition for PyHANDLE
|
||||
# 'from pywintypes import PyHANDLE' fails stating there is no such name.
|
||||
# 'from pywintypes import HANDLE as PyHANDLE' fails because HANDLE is a function, not a type.
|
||||
PyHandle: T.TypeAlias = T.Any
|
||||
|
||||
def create_sem(
|
||||
sem_name: str, jobs_count: int
|
||||
) -> T.Tuple[PyHandle, T.Dict[str, str]]:
|
||||
"""Create and initialize Win32 semaphore."""
|
||||
assert jobs_count > 0, f"Jobs count must be strictly positive"
|
||||
# The win32event documentation states that the first argument to CreateSemaphore()
|
||||
# can be None to indicate default security attributes, but mypy only wants
|
||||
# a PySECURITY_ATTRIBUTES for some reason.
|
||||
handle = win32event.CreateSemaphore(
|
||||
None, # type: ignore
|
||||
jobs_count - 1,
|
||||
jobs_count - 1,
|
||||
sem_name,
|
||||
)
|
||||
assert bool(handle), f"Error creating Win32 semaphore {win32api.GetLastError()}"
|
||||
# See technical note above about MAKEFLAGS format.
|
||||
env = dict(os.environ)
|
||||
env["MAKEFLAGS"] = f" -j{jobs_count} --jobserver-auth=" + sem_name
|
||||
return handle, env
|
||||
|
||||
def check_sem_count(handle: PyHandle, jobs_count: int) -> int:
|
||||
if jobs_count <= 1:
|
||||
# Nothing to check here.
|
||||
return 0
|
||||
|
||||
expected_count = jobs_count - 1
|
||||
|
||||
read_count = win32event.ReleaseSemaphore(handle, 1)
|
||||
if read_count < expected_count:
|
||||
print(
|
||||
f"ERROR: {expected_count - read_count} were missing from the jobs pool (got {read_count}, expected {expected_count})",
|
||||
file=sys.stderr,
|
||||
)
|
||||
return 1
|
||||
if read_count > expected_count:
|
||||
print(
|
||||
f"ERROR: {read_count - expected_count} extra tokens were released to the jobs pool (got {read_count}, expected {expected_count})",
|
||||
file=sys.stderr,
|
||||
)
|
||||
return 1
|
||||
|
||||
return 0
|
||||
|
||||
def print_usage() -> int:
|
||||
print(
|
||||
r"""Example usage:
|
||||
|
||||
# Start <command> after setting the server to provide as many jobs
|
||||
# as available CPUs (the default)
|
||||
python \path\to\jobserver_pool.py <command>
|
||||
|
||||
# Start <command> with a fixed number of job slots.
|
||||
python \path\to\jobserver_pool.py -j10 <command>
|
||||
|
||||
# Disable the feature with a non-positive count. This is equivalent
|
||||
# to running <command> directly.
|
||||
python \path\to\jobserver_pool.py -j0 <command>
|
||||
|
||||
# Use a specific semaphore name
|
||||
python \path\to\jobserver_pool.py --name=my_build_jobs <command>
|
||||
|
||||
# Setup jobserver then start new interactive PowerShell
|
||||
# session, print MAKEFLAGS value, build stuff, then exit.
|
||||
python \path\to\jobserver_pool.py powershell.exe
|
||||
$env:MAKEFLAGS
|
||||
... build stuff ...
|
||||
exit
|
||||
"""
|
||||
)
|
||||
return 0
|
||||
|
||||
else: # !_IS_WINDOWS
|
||||
|
||||
def create_pipe(jobs_count: int) -> T.Tuple[int, int, T.Dict[str, str]]:
|
||||
"""Create and fill Posix PIPE."""
|
||||
read_fd, write_fd = os.pipe()
|
||||
os.set_inheritable(read_fd, True)
|
||||
os.set_inheritable(write_fd, True)
|
||||
assert jobs_count > 0, f"Token count must be strictly positive"
|
||||
os.write(write_fd, (jobs_count - 1) * b"x")
|
||||
# See technical note above about MAKEFLAGS format.
|
||||
env = dict(os.environ)
|
||||
env["MAKEFLAGS"] = (
|
||||
f" -j{jobs_count} --jobserver-fds={read_fd},{write_fd} --jobserver-auth={read_fd},{write_fd}"
|
||||
)
|
||||
return read_fd, write_fd, env
|
||||
|
||||
def create_fifo(path: str, jobs_count: int) -> T.Tuple[int, int, T.Dict[str, str]]:
|
||||
"""Create and fill Posix FIFO."""
|
||||
if os.path.exists(path):
|
||||
os.remove(path)
|
||||
|
||||
# mypy complains that this does not exit on Windows.
|
||||
os.mkfifo(path) # type: ignore
|
||||
|
||||
read_fd = os.open(path, os.O_RDONLY | os.O_NONBLOCK)
|
||||
write_fd = os.open(path, os.O_WRONLY | os.O_NONBLOCK)
|
||||
assert jobs_count > 0, f"Token count must be strictly positive"
|
||||
os.write(write_fd, (jobs_count - 1) * b"x")
|
||||
# See technical note above about MAKEFLAGS format.
|
||||
env = dict(os.environ)
|
||||
env["MAKEFLAGS"] = f" -j{jobs_count} --jobserver-auth=fifo:" + path
|
||||
return read_fd, write_fd, env
|
||||
|
||||
def print_usage() -> int:
|
||||
print(
|
||||
r"""Example usage:
|
||||
|
||||
# Start <command> after setting the job pool to provide as many jobs
|
||||
# as available CPUs (the default)
|
||||
/path/to/jobserver_pool.py <command>
|
||||
|
||||
# Start <command> with a fixed number of jobs
|
||||
/path/to/jobserver_pool.py -j10 <command>
|
||||
|
||||
# Disable the feature with a non-positive count. This is equivalent
|
||||
# to running <command> directly.
|
||||
/path/to/jobserver_pool.py -j0 <command>
|
||||
|
||||
# Use a specific FIFO path
|
||||
/path/to/jobserver_pool.py --fifo=/tmp/my_build_jobs <command>
|
||||
|
||||
# Setup jobserver then start new interactive Bash shell
|
||||
# session, print MAKEFLAGS value, build stuff, then exit.
|
||||
/path/to/jobserver_pool.py bash -i
|
||||
echo "$MAKEFLAGS"
|
||||
... build stuff ...
|
||||
exit
|
||||
"""
|
||||
)
|
||||
return 0
|
||||
|
||||
def check_pipe_tokens(read_fd: int, jobs_count: int) -> int:
|
||||
if jobs_count <= 1: # Nothing to check
|
||||
return 0
|
||||
|
||||
# Remove implicit token from the expected count.
|
||||
expected_count = jobs_count - 1
|
||||
os.set_blocking(read_fd, False)
|
||||
read_count = 0
|
||||
while True:
|
||||
try:
|
||||
token = os.read(read_fd, 1)
|
||||
if len(token) == 0: # End of pipe?
|
||||
break
|
||||
read_count += 1
|
||||
except BlockingIOError:
|
||||
break
|
||||
|
||||
if read_count < expected_count:
|
||||
print(
|
||||
f"ERROR: {expected_count - read_count} tokens were missing from the jobs pool (got {read_count}, expected {expected_count})",
|
||||
file=sys.stderr,
|
||||
)
|
||||
return 1
|
||||
if read_count > expected_count:
|
||||
print(
|
||||
f"ERROR: {read_count - expected_count} extra tokens were released to the jobs pool (got {read_count}, expected {expected_count})",
|
||||
file=sys.stderr,
|
||||
)
|
||||
return 1
|
||||
|
||||
return 0
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(
|
||||
description=__doc__, formatter_class=argparse.RawTextHelpFormatter
|
||||
)
|
||||
if _IS_WINDOWS:
|
||||
parser.add_argument(
|
||||
"--name",
|
||||
help=f"Specify semaphore name, default is {_DEFAULT_NAME}",
|
||||
default=_DEFAULT_NAME,
|
||||
)
|
||||
else:
|
||||
mutex_group = parser.add_mutually_exclusive_group()
|
||||
mutex_group.add_argument(
|
||||
"--pipe",
|
||||
action="store_true",
|
||||
help="Implement the pool with a Unix pipe (default is FIFO).",
|
||||
)
|
||||
mutex_group.add_argument(
|
||||
"--fifo",
|
||||
default=_DEFAULT_NAME,
|
||||
help=f"Specify pool FIFO file path (default ./{_DEFAULT_NAME})",
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
"--no-check",
|
||||
action="store_true",
|
||||
help="Disable the final check that verifies that all job slots were returned to the pool on exit.",
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
"--help-usage", action="store_true", help="Print usage examples."
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
"-j",
|
||||
"--jobs",
|
||||
action="store",
|
||||
metavar="COUNT",
|
||||
dest="jobs_count",
|
||||
type=int,
|
||||
default=os.cpu_count(),
|
||||
help="Set job slots ccount, default is available CPUs count",
|
||||
)
|
||||
|
||||
parser.add_argument("command", nargs=argparse.REMAINDER, help="Command to run.")
|
||||
args = parser.parse_args()
|
||||
|
||||
if args.help_usage:
|
||||
return print_usage()
|
||||
|
||||
if not args.command:
|
||||
parser.error("This script requires at least one command argument!")
|
||||
|
||||
jobs_count = args.jobs_count
|
||||
if jobs_count <= 0:
|
||||
# Disable the feature.
|
||||
ret = subprocess.run(args.command)
|
||||
exit_code = ret.returncode
|
||||
elif _IS_WINDOWS:
|
||||
# Run with a Window semaphore.
|
||||
try:
|
||||
handle, env = create_sem(args.name, jobs_count)
|
||||
ret = subprocess.run(args.command, env=env)
|
||||
exit_code = ret.returncode
|
||||
|
||||
if exit_code == 0 and not args.no_check:
|
||||
exit_code = check_sem_count(handle, jobs_count)
|
||||
|
||||
finally:
|
||||
win32api.CloseHandle(handle)
|
||||
else:
|
||||
# Run with pipe descriptors by default, or a FIFO if --fifo is used.
|
||||
exit_code = 0
|
||||
fifo_path = ""
|
||||
try:
|
||||
if not args.pipe:
|
||||
fifo_path = os.path.abspath(args.fifo)
|
||||
read_fd, write_fd, env = create_fifo(fifo_path, args.jobs_count)
|
||||
ret = subprocess.run(args.command, env=env)
|
||||
else:
|
||||
read_fd, write_fd, env = create_pipe(args.jobs_count)
|
||||
ret = subprocess.run(
|
||||
args.command, env=env, pass_fds=(read_fd, write_fd)
|
||||
)
|
||||
|
||||
exit_code = ret.returncode
|
||||
if exit_code == 0 and not args.no_check:
|
||||
exit_code = check_pipe_tokens(read_fd, jobs_count)
|
||||
|
||||
finally:
|
||||
os.close(read_fd)
|
||||
os.close(write_fd)
|
||||
|
||||
if fifo_path:
|
||||
os.remove(fifo_path)
|
||||
|
||||
return exit_code
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
@@ -0,0 +1,174 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
"""Regression tests for the jobserver_pool.py script."""
|
||||
|
||||
import os
|
||||
import re
|
||||
import platform
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
import unittest
|
||||
import typing as T
|
||||
|
||||
_SCRIPT_DIR = os.path.dirname(__file__)
|
||||
_JOBSERVER_SCRIPT = os.path.join(_SCRIPT_DIR, "jobserver_pool.py")
|
||||
_JOBSERVER_CMD = [sys.executable, _JOBSERVER_SCRIPT]
|
||||
|
||||
_IS_WINDOWS = sys.platform == "win32"
|
||||
|
||||
# This is only here to avoid depending on the non-standard
|
||||
# scanf package which does the job properly :-)
|
||||
|
||||
|
||||
def _simple_scanf(pattern: str, input: str) -> T.Sequence[T.Any]:
|
||||
"""Extract values from input using a scanf-like pattern.
|
||||
|
||||
This is very basic and only used to avoid depending on the
|
||||
non-standard scanf package which does the job properly.
|
||||
Only supports %d, %s and %%, does not support any fancy
|
||||
escaping.
|
||||
"""
|
||||
re_pattern = ""
|
||||
groups = ""
|
||||
from_pos = 0
|
||||
|
||||
# Just in case.
|
||||
assert "." not in pattern, f"Dots in pattern not supported."
|
||||
assert "?" not in pattern, f"Question marks in pattern not supported."
|
||||
|
||||
while True:
|
||||
next_percent = pattern.find("%", from_pos)
|
||||
if next_percent < 0 or next_percent + 1 >= len(pattern):
|
||||
re_pattern += pattern[from_pos:]
|
||||
break
|
||||
|
||||
re_pattern += pattern[from_pos:next_percent]
|
||||
|
||||
from_pos = next_percent + 2
|
||||
formatter = pattern[next_percent + 1]
|
||||
if formatter == "%":
|
||||
re_pattern += "%"
|
||||
elif formatter == "d":
|
||||
groups += formatter
|
||||
re_pattern += "(\\d+)"
|
||||
elif formatter == "s":
|
||||
groups += formatter
|
||||
re_pattern += "(\\S+)"
|
||||
else:
|
||||
assert False, f"Unsupported scanf formatter: %{formatter}"
|
||||
|
||||
m = re.match(re_pattern, input)
|
||||
if not m:
|
||||
return None
|
||||
|
||||
result = []
|
||||
for group_index, formatter in enumerate(groups, start=1):
|
||||
if formatter == "d":
|
||||
result.append(int(m.group(group_index)))
|
||||
elif formatter == "s":
|
||||
result.append(m.group(group_index))
|
||||
else:
|
||||
assert False, f"Unsupported formatter {formatter}"
|
||||
|
||||
return result
|
||||
|
||||
|
||||
class JobserverPool(unittest.TestCase):
|
||||
def _run_jobserver_echo_MAKEFLAGS(
|
||||
self, cmd_args_prefix
|
||||
) -> "subprocess.CompletedProcess[str]":
|
||||
if _IS_WINDOWS:
|
||||
cmd_args = cmd_args_prefix + ["cmd.exe", "/c", "echo %MAKEFLAGS%"]
|
||||
else:
|
||||
cmd_args = cmd_args_prefix + ["sh", "-c", 'echo "$MAKEFLAGS"']
|
||||
|
||||
ret = subprocess.run(
|
||||
cmd_args,
|
||||
text=True,
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.PIPE,
|
||||
)
|
||||
ret.check_returncode()
|
||||
return ret
|
||||
|
||||
def _test_echo_MAKEFLAGS(self, cmd_args_prefix, expected_core_count: int):
|
||||
ret = self._run_jobserver_echo_MAKEFLAGS(cmd_args_prefix)
|
||||
makeflags = ret.stdout.rstrip()
|
||||
|
||||
if expected_core_count == 0:
|
||||
if _IS_WINDOWS:
|
||||
# On Windows, echo %FOO% prints "%FOO%" if FOO is not defined!
|
||||
self.assertEqual(makeflags.strip(), "%MAKEFLAGS%")
|
||||
else:
|
||||
self.assertEqual(makeflags.strip(), "")
|
||||
|
||||
else: # expected_core_count > 0
|
||||
if _IS_WINDOWS:
|
||||
expected_format = " -j%d --jobserver-auth=%s"
|
||||
else:
|
||||
expected_format = " -j%d --jobserver-auth=fifo:%s"
|
||||
|
||||
m = _simple_scanf(expected_format, makeflags)
|
||||
self.assertTrue(
|
||||
m,
|
||||
f"Invalid MAKEFLAGS value, expected format [{expected_format}], got: [{makeflags}]",
|
||||
)
|
||||
|
||||
if _IS_WINDOWS:
|
||||
sem_name = m[1]
|
||||
self.assertEqual(
|
||||
sem_name,
|
||||
"jobserver_pool",
|
||||
f"Invalid semaphore name in MAKEFLAGS value [{makeflags}]",
|
||||
)
|
||||
else:
|
||||
fifo_name = os.path.basename(m[1])
|
||||
self.assertEqual(
|
||||
fifo_name,
|
||||
"jobserver_pool",
|
||||
f"Invalid fifo name in MAKEFLAGS value [{makeflags}]",
|
||||
)
|
||||
|
||||
core_count = m[0]
|
||||
self.assertEqual(
|
||||
core_count,
|
||||
expected_core_count,
|
||||
f"Invalid core count {core_count}, expected {expected_core_count}",
|
||||
)
|
||||
|
||||
def test_MAKEFLAGS_default(self):
|
||||
self._test_echo_MAKEFLAGS(_JOBSERVER_CMD, os.cpu_count())
|
||||
|
||||
def test_MAKEFLAGS_with_10_jobs(self):
|
||||
self._test_echo_MAKEFLAGS(_JOBSERVER_CMD + ["-j10"], 10)
|
||||
self._test_echo_MAKEFLAGS(_JOBSERVER_CMD + ["--jobs=10"], 10)
|
||||
self._test_echo_MAKEFLAGS(_JOBSERVER_CMD + ["--jobs", "10"], 10)
|
||||
|
||||
def test_MAKEFLAGS_with_no_jobs(self):
|
||||
self._test_echo_MAKEFLAGS(_JOBSERVER_CMD + ["-j0"], 0)
|
||||
self._test_echo_MAKEFLAGS(_JOBSERVER_CMD + ["--jobs=0"], 0)
|
||||
self._test_echo_MAKEFLAGS(_JOBSERVER_CMD + ["--jobs", "0"], 0)
|
||||
|
||||
@unittest.skipIf(_IS_WINDOWS, "--fifo is not supported on Windows")
|
||||
def test_MAKEFLAGS_with_fifo(self):
|
||||
fifo_name = "test_fifo"
|
||||
fifo_path = os.path.abspath(fifo_name)
|
||||
ret = self._run_jobserver_echo_MAKEFLAGS(
|
||||
_JOBSERVER_CMD + ["-j10", "--fifo", fifo_name]
|
||||
)
|
||||
makeflags = ret.stdout.rstrip()
|
||||
self.assertEqual(makeflags, " -j10 --jobserver-auth=fifo:" + fifo_path)
|
||||
|
||||
@unittest.skipIf(not _IS_WINDOWS, "--name is not supported on Posix")
|
||||
def test_MAKEFLAGS_with_name(self):
|
||||
sem_name = "test_semaphore"
|
||||
ret = self._run_jobserver_echo_MAKEFLAGS(
|
||||
_JOBSERVER_CMD + ["-j10", "--name", sem_name]
|
||||
)
|
||||
makeflags = ret.stdout.rstrip()
|
||||
self.assertEqual(makeflags, " -j10 --jobserver-auth=" + sem_name)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
+324
@@ -0,0 +1,324 @@
|
||||
#!/usr/bin/env python3
|
||||
# Copyright 2024 Google Inc. All Rights Reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
from textwrap import dedent
|
||||
import os
|
||||
import platform
|
||||
import subprocess
|
||||
import tempfile
|
||||
import typing as T
|
||||
import shlex
|
||||
import sys
|
||||
import unittest
|
||||
|
||||
_SCRIPT_DIR = os.path.realpath(os.path.dirname(__file__))
|
||||
_JOBSERVER_POOL_SCRIPT = os.path.join(_SCRIPT_DIR, "jobserver_pool.py")
|
||||
_JOBSERVER_TEST_HELPER_SCRIPT = os.path.join(_SCRIPT_DIR, "jobserver_test_helper.py")
|
||||
|
||||
_PLATFORM_IS_WINDOWS = platform.system() == "Windows"
|
||||
|
||||
# Set this to True to debug command invocations.
|
||||
_DEBUG = False
|
||||
|
||||
default_env = dict(os.environ)
|
||||
default_env.pop("NINJA_STATUS", None)
|
||||
default_env.pop("MAKEFLAGS", None)
|
||||
default_env["TERM"] = "dumb"
|
||||
NINJA_PATH = os.path.abspath("./ninja")
|
||||
|
||||
|
||||
class BuildDir:
|
||||
def __init__(self, build_ninja: str):
|
||||
self.build_ninja = dedent(build_ninja)
|
||||
self.d: T.Optional[tempfile.TemporaryDirectory] = None
|
||||
|
||||
def __enter__(self):
|
||||
self.d = tempfile.TemporaryDirectory()
|
||||
with open(os.path.join(self.d.name, "build.ninja"), "w") as f:
|
||||
f.write(self.build_ninja)
|
||||
return self
|
||||
|
||||
def __exit__(self, exc_type, exc_val, exc_tb):
|
||||
self.d.cleanup()
|
||||
|
||||
@property
|
||||
def path(self) -> str:
|
||||
assert self.d
|
||||
return self.d.name
|
||||
|
||||
def run(
|
||||
self,
|
||||
cmd_flags: T.Sequence[str] = [],
|
||||
env: T.Dict[str, str] = default_env,
|
||||
) -> None:
|
||||
"""Run a command, raise exception on error. Do not capture outputs."""
|
||||
ret = subprocess.run(cmd_flags, env=env)
|
||||
ret.check_returncode()
|
||||
|
||||
def ninja_run(
|
||||
self,
|
||||
ninja_args: T.List[str],
|
||||
prefix_args: T.List[str] = [],
|
||||
extra_env: T.Dict[str, str] = {},
|
||||
) -> "subprocess.CompletedProcess[str]":
|
||||
ret = self.ninja_spawn(
|
||||
ninja_args,
|
||||
prefix_args=prefix_args,
|
||||
extra_env=extra_env,
|
||||
capture_output=False,
|
||||
)
|
||||
ret.check_returncode()
|
||||
return ret
|
||||
|
||||
def ninja_clean(self) -> None:
|
||||
self.ninja_run(["-t", "clean"])
|
||||
|
||||
def ninja_spawn(
|
||||
self,
|
||||
ninja_args: T.List[str],
|
||||
prefix_args: T.List[str] = [],
|
||||
extra_env: T.Dict[str, str] = {},
|
||||
capture_output: bool = True,
|
||||
) -> "subprocess.CompletedProcess[str]":
|
||||
"""Run Ninja command and capture outputs."""
|
||||
cmd_args = prefix_args + [NINJA_PATH, "-C", self.path] + ninja_args
|
||||
if _DEBUG:
|
||||
cmd_str = " ".join(shlex.quote(c) for c in cmd_args)
|
||||
print(f"CMD [{cmd_str}]", file=sys.stderr)
|
||||
return subprocess.run(
|
||||
cmd_args,
|
||||
text=True,
|
||||
stdout=subprocess.PIPE if capture_output else None,
|
||||
stderr=subprocess.PIPE if capture_output else None,
|
||||
env={**default_env, **extra_env},
|
||||
)
|
||||
|
||||
|
||||
def span_output_file(span_n: int) -> str:
|
||||
return "out%02d" % span_n
|
||||
|
||||
|
||||
def generate_build_plan(command_count: int) -> str:
|
||||
"""Generate a Ninja build plan for |command_count| parallel tasks.
|
||||
|
||||
Each task calls the test helper script which waits for 50ms
|
||||
then writes its own start and end time to its output file.
|
||||
"""
|
||||
result = f"""
|
||||
rule span
|
||||
command = {sys.executable} -S {_JOBSERVER_TEST_HELPER_SCRIPT} --duration-ms=50 $out
|
||||
|
||||
"""
|
||||
|
||||
for n in range(command_count):
|
||||
result += "build %s: span\n" % span_output_file(n)
|
||||
|
||||
result += "build all: phony %s\n" % " ".join(
|
||||
[span_output_file(n) for n in range(command_count)]
|
||||
)
|
||||
return result
|
||||
|
||||
|
||||
def compute_max_overlapped_spans(build_dir: str, command_count: int) -> int:
|
||||
"""Compute the maximum number of overlapped spanned tasks.
|
||||
|
||||
This reads the output files from |build_dir| and look at their start and end times
|
||||
to compute the maximum number of tasks that were run in parallel.
|
||||
"""
|
||||
# Read the output files.
|
||||
if command_count < 2:
|
||||
return 0
|
||||
|
||||
spans: T.List[T.Tuple[int, int]] = []
|
||||
for n in range(command_count):
|
||||
with open(os.path.join(build_dir, span_output_file(n)), "rb") as f:
|
||||
content = f.read().decode("utf-8")
|
||||
lines = content.splitlines()
|
||||
assert len(lines) == 2, f"Unexpected output file content: [{content}]"
|
||||
spans.append((int(lines[0]), int(lines[1])))
|
||||
|
||||
# Stupid but simple, for each span, count the number of other spans that overlap it.
|
||||
max_overlaps = 1
|
||||
for n in range(command_count):
|
||||
cur_start, cur_end = spans[n]
|
||||
cur_overlaps = 1
|
||||
for m in range(command_count):
|
||||
other_start, other_end = spans[m]
|
||||
if n != m and other_end > cur_start and other_start < cur_end:
|
||||
cur_overlaps += 1
|
||||
|
||||
if cur_overlaps > max_overlaps:
|
||||
max_overlaps = cur_overlaps
|
||||
|
||||
return max_overlaps
|
||||
|
||||
|
||||
class JobserverTest(unittest.TestCase):
|
||||
|
||||
def test_no_jobserver_client(self):
|
||||
task_count = 4
|
||||
build_plan = generate_build_plan(task_count)
|
||||
with BuildDir(build_plan) as b:
|
||||
output = b.run([NINJA_PATH, "-C", b.path, f"-j{task_count}", "all"])
|
||||
|
||||
max_overlaps = compute_max_overlapped_spans(b.path, task_count)
|
||||
self.assertEqual(max_overlaps, task_count)
|
||||
|
||||
b.ninja_clean()
|
||||
output = b.run([NINJA_PATH, "-C", b.path, "-j1", "all"])
|
||||
|
||||
max_overlaps = compute_max_overlapped_spans(b.path, task_count)
|
||||
self.assertEqual(max_overlaps, 1)
|
||||
|
||||
def _run_client_test(self, jobserver_args: T.List[str]) -> None:
|
||||
task_count = 4
|
||||
build_plan = generate_build_plan(task_count)
|
||||
with BuildDir(build_plan) as b:
|
||||
# First, run the full tasks with with {task_count} tokens, this should allow all
|
||||
# tasks to run in parallel.
|
||||
ret = b.ninja_run(
|
||||
ninja_args=["all"],
|
||||
prefix_args=jobserver_args + [f"--jobs={task_count}"],
|
||||
)
|
||||
max_overlaps = compute_max_overlapped_spans(b.path, task_count)
|
||||
self.assertEqual(max_overlaps, task_count)
|
||||
|
||||
# Second, use 2 tokens only, and verify that this was enforced by Ninja.
|
||||
b.ninja_clean()
|
||||
b.ninja_run(
|
||||
["all"],
|
||||
prefix_args=jobserver_args + ["--jobs=2"],
|
||||
)
|
||||
max_overlaps = compute_max_overlapped_spans(b.path, task_count)
|
||||
self.assertEqual(max_overlaps, 2)
|
||||
|
||||
# Third, verify that --jobs=1 serializes all tasks.
|
||||
b.ninja_clean()
|
||||
b.ninja_run(
|
||||
["all"],
|
||||
prefix_args=jobserver_args + ["--jobs=1"],
|
||||
)
|
||||
max_overlaps = compute_max_overlapped_spans(b.path, task_count)
|
||||
self.assertEqual(max_overlaps, 1)
|
||||
|
||||
# Finally, verify that -j1 overrides the pool.
|
||||
b.ninja_clean()
|
||||
b.ninja_run(
|
||||
["-j1", "all"],
|
||||
prefix_args=jobserver_args + [f"--jobs={task_count}"],
|
||||
)
|
||||
max_overlaps = compute_max_overlapped_spans(b.path, task_count)
|
||||
self.assertEqual(max_overlaps, 1)
|
||||
|
||||
# On Linux, use taskset to limit the number of available cores to 1
|
||||
# and verify that the jobserver overrides the default Ninja parallelism
|
||||
# and that {task_count} tasks are still spawned in parallel.
|
||||
if platform.system() == "Linux":
|
||||
# First, run without a jobserver, with a single CPU, Ninja will
|
||||
# use a parallelism of 2 in this case (GuessParallelism() in ninja.cc)
|
||||
b.ninja_clean()
|
||||
b.ninja_run(
|
||||
["all"],
|
||||
prefix_args=["taskset", "-c", "0"],
|
||||
)
|
||||
max_overlaps = compute_max_overlapped_spans(b.path, task_count)
|
||||
self.assertEqual(max_overlaps, 2)
|
||||
|
||||
# Now with a jobserver with {task_count} tasks.
|
||||
b.ninja_clean()
|
||||
b.ninja_run(
|
||||
["all"],
|
||||
prefix_args=jobserver_args
|
||||
+ [f"--jobs={task_count}"]
|
||||
+ ["taskset", "-c", "0"],
|
||||
)
|
||||
max_overlaps = compute_max_overlapped_spans(b.path, task_count)
|
||||
self.assertEqual(max_overlaps, task_count)
|
||||
|
||||
@unittest.skipIf(_PLATFORM_IS_WINDOWS, "These test methods do not work on Windows")
|
||||
def test_jobserver_client_with_posix_fifo(self):
|
||||
self._run_client_test([sys.executable, "-S", _JOBSERVER_POOL_SCRIPT])
|
||||
|
||||
@unittest.skipIf(_PLATFORM_IS_WINDOWS, "These test methods do not work on Windows")
|
||||
def test_jobserver_client_with_posix_pipe(self):
|
||||
# Verify that setting up a --pipe server does not make Ninja exit with an error.
|
||||
# Instead, a warning is printed.
|
||||
task_count = 4
|
||||
build_plan = generate_build_plan(task_count)
|
||||
with BuildDir(build_plan) as b:
|
||||
|
||||
prefix_args = [
|
||||
sys.executable,
|
||||
"-S",
|
||||
_JOBSERVER_POOL_SCRIPT,
|
||||
"--pipe",
|
||||
f"--jobs={task_count}",
|
||||
]
|
||||
|
||||
def run_ninja_with_jobserver_pipe(args):
|
||||
ret = b.ninja_spawn(args, prefix_args=prefix_args)
|
||||
ret.check_returncode()
|
||||
return ret.stdout, ret.stderr
|
||||
|
||||
output, error = run_ninja_with_jobserver_pipe(["all"])
|
||||
if _DEBUG:
|
||||
print(f"OUTPUT [{output}]\nERROR [{error}]\n", file=sys.stderr)
|
||||
self.assertTrue(error.find("Pipe-based protocol is not supported!") >= 0)
|
||||
|
||||
max_overlaps = compute_max_overlapped_spans(b.path, task_count)
|
||||
self.assertEqual(max_overlaps, task_count)
|
||||
|
||||
# Using an explicit -j<N> ignores the jobserver pool.
|
||||
b.ninja_clean()
|
||||
output, error = run_ninja_with_jobserver_pipe(["-j1", "all"])
|
||||
if _DEBUG:
|
||||
print(f"OUTPUT [{output}]\nERROR [{error}]\n", file=sys.stderr)
|
||||
self.assertFalse(error.find("Pipe-based protocol is not supported!") >= 0)
|
||||
|
||||
max_overlaps = compute_max_overlapped_spans(b.path, task_count)
|
||||
self.assertEqual(max_overlaps, 1)
|
||||
|
||||
def _test_MAKEFLAGS_value(
|
||||
self, ninja_args: T.List[str] = [], prefix_args: T.List[str] = []
|
||||
):
|
||||
build_plan = r"""
|
||||
rule print
|
||||
command = echo MAKEFLAGS="[$$MAKEFLAGS]"
|
||||
|
||||
build all: print
|
||||
"""
|
||||
with BuildDir(build_plan) as b:
|
||||
ret = b.ninja_spawn(
|
||||
ninja_args + ["--quiet", "all"], prefix_args=prefix_args
|
||||
)
|
||||
self.assertEqual(ret.returncode, 0)
|
||||
output = ret.stdout.strip()
|
||||
pos = output.find("MAKEFLAGS=[")
|
||||
self.assertNotEqual(pos, -1, "Could not find MAKEFLAGS in output!")
|
||||
makeflags, sep, _ = output[pos + len("MAKEFLAGS=[") :].partition("]")
|
||||
self.assertEqual(sep, "]", "Missing ] in output!: " + output)
|
||||
self.assertTrue(
|
||||
"--jobserver-auth=" in makeflags,
|
||||
f"Missing --jobserver-auth from MAKEFLAGS [{makeflags}]\nSTDOUT [{ret.stdout}]\nSTDERR [{ret.stderr}]",
|
||||
)
|
||||
|
||||
def test_client_passes_MAKEFLAGS(self):
|
||||
self._test_MAKEFLAGS_value(
|
||||
prefix_args=[sys.executable, "-S", _JOBSERVER_POOL_SCRIPT]
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,42 @@
|
||||
#!/usr/bin/env python3
|
||||
# Copyright 2024 Google Inc. All Rights Reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
"""Simple utility used by the jobserver test. Wait for specific time, then write start/stop times to output file."""
|
||||
|
||||
import argparse
|
||||
import time
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument(
|
||||
"--duration-ms",
|
||||
default="50",
|
||||
help="sleep duration in milliseconds (default 50)",
|
||||
)
|
||||
parser.add_argument("output_file", type=Path, help="output file name.")
|
||||
args = parser.parse_args()
|
||||
|
||||
now_time_ns = time.time_ns()
|
||||
time.sleep(int(args.duration_ms) / 1000.0)
|
||||
args.output_file.write_text(f"{now_time_ns}\n{time.time_ns()}\n")
|
||||
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
@@ -0,0 +1,38 @@
|
||||
# An input file for running a "slow" build.
|
||||
# Use like: ninja -f misc/long-slow-build.ninja all
|
||||
|
||||
rule sleep
|
||||
command = sleep 1
|
||||
description = SLEEP $out
|
||||
|
||||
build 0: sleep README
|
||||
build 1: sleep README
|
||||
build 2: sleep README
|
||||
build 3: sleep README
|
||||
build 4: sleep README
|
||||
build 5: sleep README
|
||||
build 6: sleep README
|
||||
build 7: sleep README
|
||||
build 8: sleep README
|
||||
build 9: sleep README
|
||||
build 10: sleep 0
|
||||
build 11: sleep 1
|
||||
build 12: sleep 2
|
||||
build 13: sleep 3
|
||||
build 14: sleep 4
|
||||
build 15: sleep 5
|
||||
build 16: sleep 6
|
||||
build 17: sleep 7
|
||||
build 18: sleep 8
|
||||
build 19: sleep 9
|
||||
build 20: sleep 10
|
||||
build 21: sleep 11
|
||||
build 22: sleep 12
|
||||
build 23: sleep 13
|
||||
build 24: sleep 14
|
||||
build 25: sleep 15
|
||||
build 26: sleep 16
|
||||
build 27: sleep 17
|
||||
build 28: sleep 18
|
||||
build 29: sleep 19
|
||||
build all: phony 20 21 22 23 24 25 26 27 28 29
|
||||
@@ -0,0 +1,41 @@
|
||||
// Copyright 2020 Google Inc. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#include "stdint.h"
|
||||
#include <string>
|
||||
#include "disk_interface.h"
|
||||
#include "state.h"
|
||||
#include "manifest_parser.h"
|
||||
#include <filesystem>
|
||||
|
||||
extern "C" int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size)
|
||||
{
|
||||
char build_file[256];
|
||||
sprintf(build_file, "/tmp/build.ninja");
|
||||
FILE *fp = fopen(build_file, "wb");
|
||||
if (!fp)
|
||||
return 0;
|
||||
fwrite(data, size, 1, fp);
|
||||
fclose(fp);
|
||||
|
||||
std::string err;
|
||||
RealDiskInterface disk_interface;
|
||||
State state;
|
||||
ManifestParser parser(&state, &disk_interface);
|
||||
|
||||
parser.Load("/tmp/build.ninja", &err);
|
||||
|
||||
std::__fs::filesystem::remove_all("/tmp/build.ninja");
|
||||
return 0;
|
||||
}
|
||||
+55
@@ -0,0 +1,55 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
# Copyright 2011 Google Inc. All Rights Reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
"""measure the runtime of a command by repeatedly running it.
|
||||
"""
|
||||
|
||||
import time
|
||||
import subprocess
|
||||
import sys
|
||||
from typing import Union, List
|
||||
|
||||
devnull = open('/dev/null', 'w')
|
||||
|
||||
def run(cmd: Union[str, List[str]], repeat: int = 10) -> None:
|
||||
print('sampling:', end=' ')
|
||||
sys.stdout.flush()
|
||||
|
||||
samples = []
|
||||
for _ in range(repeat):
|
||||
start = time.time()
|
||||
subprocess.call(cmd, stdout=devnull, stderr=devnull)
|
||||
end = time.time()
|
||||
dt = (end - start) * 1000
|
||||
print('%dms' % int(dt), end=' ')
|
||||
sys.stdout.flush()
|
||||
samples.append(dt)
|
||||
print()
|
||||
|
||||
# We're interested in the 'pure' runtime of the code, which is
|
||||
# conceptually the smallest time we'd see if we ran it enough times
|
||||
# such that it got the perfect time slices / disk cache hits.
|
||||
best = min(samples)
|
||||
# Also print how varied the outputs were in an attempt to make it
|
||||
# more obvious if something has gone terribly wrong.
|
||||
err = sum(s - best for s in samples) / float(len(samples))
|
||||
print('estimate: %dms (mean err %.1fms)' % (best, err))
|
||||
|
||||
if __name__ == '__main__':
|
||||
if len(sys.argv) < 2:
|
||||
print('usage: measure.py command args...')
|
||||
sys.exit(1)
|
||||
run(cmd=sys.argv[1:])
|
||||
@@ -0,0 +1,87 @@
|
||||
" ninja build file syntax.
|
||||
" Language: ninja build file as described at
|
||||
" http://ninja-build.org/manual.html
|
||||
" Version: 1.5
|
||||
" Last Change: 2018/04/05
|
||||
" Maintainer: Nicolas Weber <nicolasweber@gmx.de>
|
||||
" Version 1.4 of this script is in the upstream vim repository and will be
|
||||
" included in the next vim release. If you change this, please send your change
|
||||
" upstream.
|
||||
|
||||
" ninja lexer and parser are at
|
||||
" https://github.com/ninja-build/ninja/blob/master/src/lexer.in.cc
|
||||
" https://github.com/ninja-build/ninja/blob/master/src/manifest_parser.cc
|
||||
|
||||
if exists("b:current_syntax")
|
||||
finish
|
||||
endif
|
||||
|
||||
let s:cpo_save = &cpo
|
||||
set cpo&vim
|
||||
|
||||
syn case match
|
||||
|
||||
" Comments are only matched when the # is at the beginning of the line (with
|
||||
" optional whitespace), as long as the prior line didn't end with a $
|
||||
" continuation.
|
||||
syn match ninjaComment /\(\$\n\)\@<!\_^\s*#.*$/ contains=@Spell
|
||||
|
||||
" Toplevel statements are the ones listed here and
|
||||
" toplevel variable assignments (ident '=' value).
|
||||
" lexer.in.cc, ReadToken() and manifest_parser.cc, Parse()
|
||||
syn match ninjaKeyword "^build\>"
|
||||
syn match ninjaKeyword "^rule\>"
|
||||
syn match ninjaKeyword "^pool\>"
|
||||
syn match ninjaKeyword "^default\>"
|
||||
syn match ninjaKeyword "^include\>"
|
||||
syn match ninjaKeyword "^subninja\>"
|
||||
|
||||
" Both 'build' and 'rule' begin a variable scope that ends
|
||||
" on the first line without indent. 'rule' allows only a
|
||||
" limited set of magic variables, 'build' allows general
|
||||
" let assignments.
|
||||
" manifest_parser.cc, ParseRule()
|
||||
syn region ninjaRule start="^rule" end="^\ze\S" contains=TOP transparent
|
||||
syn keyword ninjaRuleCommand contained containedin=ninjaRule command
|
||||
\ deps depfile description generator
|
||||
\ pool restat rspfile rspfile_content
|
||||
|
||||
syn region ninjaPool start="^pool" end="^\ze\S" contains=TOP transparent
|
||||
syn keyword ninjaPoolCommand contained containedin=ninjaPool depth
|
||||
|
||||
" Strings are parsed as follows:
|
||||
" lexer.in.cc, ReadEvalString()
|
||||
" simple_varname = [a-zA-Z0-9_-]+;
|
||||
" varname = [a-zA-Z0-9_.-]+;
|
||||
" $$ -> $
|
||||
" $\n -> line continuation
|
||||
" '$ ' -> escaped space
|
||||
" $simple_varname -> variable
|
||||
" ${varname} -> variable
|
||||
|
||||
syn match ninjaDollar "\$\$"
|
||||
syn match ninjaWrapLineOperator "\$$"
|
||||
syn match ninjaSimpleVar "\$[a-zA-Z0-9_-]\+"
|
||||
syn match ninjaVar "\${[a-zA-Z0-9_.-]\+}"
|
||||
|
||||
" operators are:
|
||||
" variable assignment =
|
||||
" rule definition :
|
||||
" implicit dependency |
|
||||
" order-only dependency ||
|
||||
syn match ninjaOperator "\(=\|:\||\|||\)\ze\s"
|
||||
|
||||
hi def link ninjaComment Comment
|
||||
hi def link ninjaKeyword Keyword
|
||||
hi def link ninjaRuleCommand Statement
|
||||
hi def link ninjaPoolCommand Statement
|
||||
hi def link ninjaDollar ninjaOperator
|
||||
hi def link ninjaWrapLineOperator ninjaOperator
|
||||
hi def link ninjaOperator Operator
|
||||
hi def link ninjaSimpleVar ninjaVar
|
||||
hi def link ninjaVar Identifier
|
||||
|
||||
let b:current_syntax = "ninja"
|
||||
|
||||
let &cpo = s:cpo_save
|
||||
unlet s:cpo_save
|
||||
@@ -0,0 +1,231 @@
|
||||
#!/usr/bin/python
|
||||
|
||||
# Copyright 2011 Google Inc. All Rights Reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
"""Python module for generating .ninja files.
|
||||
|
||||
Note that this is emphatically not a required piece of Ninja; it's
|
||||
just a helpful utility for build-file-generation systems that already
|
||||
use Python.
|
||||
"""
|
||||
|
||||
import re
|
||||
import textwrap
|
||||
from io import TextIOWrapper
|
||||
from typing import Dict, List, Match, Optional, Tuple, Union
|
||||
|
||||
def escape_path(word: str) -> str:
|
||||
return word.replace('$ ', '$$ ').replace(' ', '$ ').replace(':', '$:')
|
||||
|
||||
class Writer(object):
|
||||
def __init__(self, output: TextIOWrapper, width: int = 78) -> None:
|
||||
self.output = output
|
||||
self.width = width
|
||||
|
||||
def newline(self) -> None:
|
||||
self.output.write('\n')
|
||||
|
||||
def comment(self, text: str) -> None:
|
||||
for line in textwrap.wrap(text, self.width - 2, break_long_words=False,
|
||||
break_on_hyphens=False):
|
||||
self.output.write('# ' + line + '\n')
|
||||
|
||||
def variable(
|
||||
self,
|
||||
key: str,
|
||||
value: Optional[Union[bool, int, float, str, List[str]]],
|
||||
indent: int = 0,
|
||||
) -> None:
|
||||
if value is None:
|
||||
return
|
||||
if isinstance(value, list):
|
||||
value = ' '.join(filter(None, value)) # Filter out empty strings.
|
||||
self._line('%s = %s' % (key, value), indent)
|
||||
|
||||
def pool(self, name: str, depth: int) -> None:
|
||||
self._line('pool %s' % name)
|
||||
self.variable('depth', depth, indent=1)
|
||||
|
||||
def rule(
|
||||
self,
|
||||
name: str,
|
||||
command: str,
|
||||
description: Optional[str] = None,
|
||||
depfile: Optional[str] = None,
|
||||
generator: bool = False,
|
||||
pool: Optional[str] = None,
|
||||
restat: bool = False,
|
||||
rspfile: Optional[str] = None,
|
||||
rspfile_content: Optional[str] = None,
|
||||
deps: Optional[Union[str, List[str]]] = None,
|
||||
) -> None:
|
||||
self._line('rule %s' % name)
|
||||
self.variable('command', command, indent=1)
|
||||
if description:
|
||||
self.variable('description', description, indent=1)
|
||||
if depfile:
|
||||
self.variable('depfile', depfile, indent=1)
|
||||
if generator:
|
||||
self.variable('generator', '1', indent=1)
|
||||
if pool:
|
||||
self.variable('pool', pool, indent=1)
|
||||
if restat:
|
||||
self.variable('restat', '1', indent=1)
|
||||
if rspfile:
|
||||
self.variable('rspfile', rspfile, indent=1)
|
||||
if rspfile_content:
|
||||
self.variable('rspfile_content', rspfile_content, indent=1)
|
||||
if deps:
|
||||
self.variable('deps', deps, indent=1)
|
||||
|
||||
def build(
|
||||
self,
|
||||
outputs: Union[str, List[str]],
|
||||
rule: str,
|
||||
inputs: Optional[Union[str, List[str]]] = None,
|
||||
implicit: Optional[Union[str, List[str]]] = None,
|
||||
order_only: Optional[Union[str, List[str]]] = None,
|
||||
variables: Optional[
|
||||
Union[
|
||||
List[Tuple[str, Optional[Union[str, List[str]]]]],
|
||||
Dict[str, Optional[Union[str, List[str]]]],
|
||||
]
|
||||
] = None,
|
||||
implicit_outputs: Optional[Union[str, List[str]]] = None,
|
||||
pool: Optional[str] = None,
|
||||
dyndep: Optional[str] = None,
|
||||
) -> List[str]:
|
||||
outputs = as_list(outputs)
|
||||
out_outputs = [escape_path(x) for x in outputs]
|
||||
all_inputs = [escape_path(x) for x in as_list(inputs)]
|
||||
|
||||
if implicit:
|
||||
implicit = [escape_path(x) for x in as_list(implicit)]
|
||||
all_inputs.append('|')
|
||||
all_inputs.extend(implicit)
|
||||
if order_only:
|
||||
order_only = [escape_path(x) for x in as_list(order_only)]
|
||||
all_inputs.append('||')
|
||||
all_inputs.extend(order_only)
|
||||
if implicit_outputs:
|
||||
implicit_outputs = [escape_path(x)
|
||||
for x in as_list(implicit_outputs)]
|
||||
out_outputs.append('|')
|
||||
out_outputs.extend(implicit_outputs)
|
||||
|
||||
self._line('build %s: %s' % (' '.join(out_outputs),
|
||||
' '.join([rule] + all_inputs)))
|
||||
if pool is not None:
|
||||
self._line(' pool = %s' % pool)
|
||||
if dyndep is not None:
|
||||
self._line(' dyndep = %s' % dyndep)
|
||||
|
||||
if variables:
|
||||
if isinstance(variables, dict):
|
||||
iterator = iter(variables.items())
|
||||
else:
|
||||
iterator = iter(variables)
|
||||
|
||||
for key, val in iterator:
|
||||
self.variable(key, val, indent=1)
|
||||
|
||||
return outputs
|
||||
|
||||
def include(self, path: str) -> None:
|
||||
self._line('include %s' % path)
|
||||
|
||||
def subninja(self, path: str) -> None:
|
||||
self._line('subninja %s' % path)
|
||||
|
||||
def default(self, paths: Union[str, List[str]]) -> None:
|
||||
self._line('default %s' % ' '.join(as_list(paths)))
|
||||
|
||||
def _count_dollars_before_index(self, s: str, i: int) -> int:
|
||||
"""Returns the number of '$' characters right in front of s[i]."""
|
||||
dollar_count = 0
|
||||
dollar_index = i - 1
|
||||
while dollar_index > 0 and s[dollar_index] == '$':
|
||||
dollar_count += 1
|
||||
dollar_index -= 1
|
||||
return dollar_count
|
||||
|
||||
def _line(self, text: str, indent: int = 0) -> None:
|
||||
"""Write 'text' word-wrapped at self.width characters."""
|
||||
leading_space = ' ' * indent
|
||||
while len(leading_space) + len(text) > self.width:
|
||||
# The text is too wide; wrap if possible.
|
||||
|
||||
# Find the rightmost space that would obey our width constraint and
|
||||
# that's not an escaped space.
|
||||
available_space = self.width - len(leading_space) - len(' $')
|
||||
space = available_space
|
||||
while True:
|
||||
space = text.rfind(' ', 0, space)
|
||||
if (space < 0 or
|
||||
self._count_dollars_before_index(text, space) % 2 == 0):
|
||||
break
|
||||
|
||||
if space < 0:
|
||||
# No such space; just use the first unescaped space we can find.
|
||||
space = available_space - 1
|
||||
while True:
|
||||
space = text.find(' ', space + 1)
|
||||
if (space < 0 or
|
||||
self._count_dollars_before_index(text, space) % 2 == 0):
|
||||
break
|
||||
if space < 0:
|
||||
# Give up on breaking.
|
||||
break
|
||||
|
||||
self.output.write(leading_space + text[0:space] + ' $\n')
|
||||
text = text[space+1:]
|
||||
|
||||
# Subsequent lines are continuations, so indent them.
|
||||
leading_space = ' ' * (indent+2)
|
||||
|
||||
self.output.write(leading_space + text + '\n')
|
||||
|
||||
def close(self) -> None:
|
||||
self.output.close()
|
||||
|
||||
|
||||
def as_list(input: Optional[Union[str, List[str]]]) -> List[str]:
|
||||
if input is None:
|
||||
return []
|
||||
if isinstance(input, list):
|
||||
return input
|
||||
return [input]
|
||||
|
||||
|
||||
def escape(string: str) -> str:
|
||||
"""Escape a string such that it can be embedded into a Ninja file without
|
||||
further interpretation."""
|
||||
assert '\n' not in string, 'Ninja syntax does not allow newlines'
|
||||
# We only have one special metacharacter: '$'.
|
||||
return string.replace('$', '$$')
|
||||
|
||||
|
||||
def expand(string: str, vars: Dict[str, str], local_vars: Dict[str, str] = {}) -> str:
|
||||
"""Expand a string containing $vars as Ninja would.
|
||||
|
||||
Note: doesn't handle the full Ninja variable syntax, but it's enough
|
||||
to make configure.py's use of it work.
|
||||
"""
|
||||
def exp(m: Match[str]) -> str:
|
||||
var = m.group(1)
|
||||
if var == '$':
|
||||
return '$'
|
||||
return local_vars.get(var, vars.get(var, ''))
|
||||
return re.sub(r'\$(\$|\w*)', exp, string)
|
||||
@@ -0,0 +1,192 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
# Copyright 2011 Google Inc. All Rights Reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
import unittest
|
||||
from typing import Dict
|
||||
|
||||
try:
|
||||
from StringIO import StringIO
|
||||
except ImportError:
|
||||
from io import StringIO
|
||||
|
||||
import ninja_syntax
|
||||
|
||||
LONGWORD = 'a' * 10
|
||||
LONGWORDWITHSPACES = 'a'*5 + '$ ' + 'a'*5
|
||||
INDENT = ' '
|
||||
|
||||
class TestLineWordWrap(unittest.TestCase):
|
||||
def setUp(self) -> None:
|
||||
self.out = StringIO()
|
||||
self.n = ninja_syntax.Writer(self.out, width=8)
|
||||
|
||||
def test_single_long_word(self) -> None:
|
||||
# We shouldn't wrap a single long word.
|
||||
self.n._line(LONGWORD)
|
||||
self.assertEqual(LONGWORD + '\n', self.out.getvalue())
|
||||
|
||||
def test_few_long_words(self) -> None:
|
||||
# We should wrap a line where the second word is overlong.
|
||||
self.n._line(' '.join(['x', LONGWORD, 'y']))
|
||||
self.assertEqual(' $\n'.join(['x',
|
||||
INDENT + LONGWORD,
|
||||
INDENT + 'y']) + '\n',
|
||||
self.out.getvalue())
|
||||
|
||||
def test_comment_wrap(self) -> None:
|
||||
# Filenames should not be wrapped
|
||||
self.n.comment('Hello /usr/local/build-tools/bin')
|
||||
self.assertEqual('# Hello\n# /usr/local/build-tools/bin\n',
|
||||
self.out.getvalue())
|
||||
|
||||
def test_short_words_indented(self) -> None:
|
||||
# Test that indent is taking into account when breaking subsequent lines.
|
||||
# The second line should not be ' to tree', as that's longer than the
|
||||
# test layout width of 8.
|
||||
self.n._line('line_one to tree')
|
||||
self.assertEqual('''\
|
||||
line_one $
|
||||
to $
|
||||
tree
|
||||
''',
|
||||
self.out.getvalue())
|
||||
|
||||
def test_few_long_words_indented(self) -> None:
|
||||
# Check wrapping in the presence of indenting.
|
||||
self.n._line(' '.join(['x', LONGWORD, 'y']), indent=1)
|
||||
self.assertEqual(' $\n'.join([' ' + 'x',
|
||||
' ' + INDENT + LONGWORD,
|
||||
' ' + INDENT + 'y']) + '\n',
|
||||
self.out.getvalue())
|
||||
|
||||
def test_escaped_spaces(self) -> None:
|
||||
self.n._line(' '.join(['x', LONGWORDWITHSPACES, 'y']))
|
||||
self.assertEqual(' $\n'.join(['x',
|
||||
INDENT + LONGWORDWITHSPACES,
|
||||
INDENT + 'y']) + '\n',
|
||||
self.out.getvalue())
|
||||
|
||||
def test_fit_many_words(self) -> None:
|
||||
self.n = ninja_syntax.Writer(self.out, width=78)
|
||||
self.n._line('command = cd ../../chrome; python ../tools/grit/grit/format/repack.py ../out/Debug/obj/chrome/chrome_dll.gen/repack/theme_resources_large.pak ../out/Debug/gen/chrome/theme_resources_large.pak', 1)
|
||||
self.assertEqual('''\
|
||||
command = cd ../../chrome; python ../tools/grit/grit/format/repack.py $
|
||||
../out/Debug/obj/chrome/chrome_dll.gen/repack/theme_resources_large.pak $
|
||||
../out/Debug/gen/chrome/theme_resources_large.pak
|
||||
''',
|
||||
self.out.getvalue())
|
||||
|
||||
def test_leading_space(self) -> None:
|
||||
self.n = ninja_syntax.Writer(self.out, width=14) # force wrapping
|
||||
self.n.variable('foo', ['', '-bar', '-somethinglong'], 0)
|
||||
self.assertEqual('''\
|
||||
foo = -bar $
|
||||
-somethinglong
|
||||
''',
|
||||
self.out.getvalue())
|
||||
|
||||
def test_embedded_dollar_dollar(self) -> None:
|
||||
self.n = ninja_syntax.Writer(self.out, width=15) # force wrapping
|
||||
self.n.variable('foo', ['a$$b', '-somethinglong'], 0)
|
||||
self.assertEqual('''\
|
||||
foo = a$$b $
|
||||
-somethinglong
|
||||
''',
|
||||
self.out.getvalue())
|
||||
|
||||
def test_two_embedded_dollar_dollars(self) -> None:
|
||||
self.n = ninja_syntax.Writer(self.out, width=17) # force wrapping
|
||||
self.n.variable('foo', ['a$$b', '-somethinglong'], 0)
|
||||
self.assertEqual('''\
|
||||
foo = a$$b $
|
||||
-somethinglong
|
||||
''',
|
||||
self.out.getvalue())
|
||||
|
||||
def test_leading_dollar_dollar(self) -> None:
|
||||
self.n = ninja_syntax.Writer(self.out, width=14) # force wrapping
|
||||
self.n.variable('foo', ['$$b', '-somethinglong'], 0)
|
||||
self.assertEqual('''\
|
||||
foo = $$b $
|
||||
-somethinglong
|
||||
''',
|
||||
self.out.getvalue())
|
||||
|
||||
def test_trailing_dollar_dollar(self) -> None:
|
||||
self.n = ninja_syntax.Writer(self.out, width=14) # force wrapping
|
||||
self.n.variable('foo', ['a$$', '-somethinglong'], 0)
|
||||
self.assertEqual('''\
|
||||
foo = a$$ $
|
||||
-somethinglong
|
||||
''',
|
||||
self.out.getvalue())
|
||||
|
||||
class TestBuild(unittest.TestCase):
|
||||
def setUp(self) -> None:
|
||||
self.out = StringIO()
|
||||
self.n = ninja_syntax.Writer(self.out)
|
||||
|
||||
def test_variables_dict(self) -> None:
|
||||
self.n.build('out', 'cc', 'in', variables={'name': 'value'})
|
||||
self.assertEqual('''\
|
||||
build out: cc in
|
||||
name = value
|
||||
''',
|
||||
self.out.getvalue())
|
||||
|
||||
def test_variables_list(self) -> None:
|
||||
self.n.build('out', 'cc', 'in', variables=[('name', 'value')])
|
||||
self.assertEqual('''\
|
||||
build out: cc in
|
||||
name = value
|
||||
''',
|
||||
self.out.getvalue())
|
||||
|
||||
def test_implicit_outputs(self) -> None:
|
||||
self.n.build('o', 'cc', 'i', implicit_outputs='io')
|
||||
self.assertEqual('''\
|
||||
build o | io: cc i
|
||||
''',
|
||||
self.out.getvalue())
|
||||
|
||||
class TestExpand(unittest.TestCase):
|
||||
def test_basic(self) -> None:
|
||||
vars = {'x': 'X'}
|
||||
self.assertEqual('foo', ninja_syntax.expand('foo', vars))
|
||||
|
||||
def test_var(self) -> None:
|
||||
vars = {'xyz': 'XYZ'}
|
||||
self.assertEqual('fooXYZ', ninja_syntax.expand('foo$xyz', vars))
|
||||
|
||||
def test_vars(self) -> None:
|
||||
vars = {'x': 'X', 'y': 'YYY'}
|
||||
self.assertEqual('XYYY', ninja_syntax.expand('$x$y', vars))
|
||||
|
||||
def test_space(self) -> None:
|
||||
vars: Dict[str, str] = {}
|
||||
self.assertEqual('x y z', ninja_syntax.expand('x$ y$ z', vars))
|
||||
|
||||
def test_locals(self) -> None:
|
||||
vars = {'x': 'a'}
|
||||
local_vars = {'x': 'b'}
|
||||
self.assertEqual('a', ninja_syntax.expand('$x', vars))
|
||||
self.assertEqual('b', ninja_syntax.expand('$x', vars, local_vars))
|
||||
|
||||
def test_double(self) -> None:
|
||||
self.assertEqual('a b$c', ninja_syntax.expand('a$ b$$c', {}))
|
||||
|
||||
if __name__ == '__main__':
|
||||
unittest.main()
|
||||
@@ -0,0 +1,29 @@
|
||||
#!/bin/bash -eu
|
||||
# Copyright 2020 Google Inc.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
#
|
||||
################################################################################
|
||||
|
||||
cmake -Bbuild-cmake -H.
|
||||
cmake --build build-cmake
|
||||
|
||||
cd $SRC/ninja/misc
|
||||
|
||||
$CXX $CXXFLAGS -fdiagnostics-color -I/src/ninja/src -o fuzzer.o -c manifest_fuzzer.cc
|
||||
|
||||
find .. -name "*.o" -exec ar rcs fuzz_lib.a {} \;
|
||||
|
||||
$CXX $CXXFLAGS $LIB_FUZZING_ENGINE fuzzer.o -o $OUT/fuzzer fuzz_lib.a
|
||||
|
||||
zip $OUT/fuzzer_seed_corpus.zip $SRC/sample_ninja_build
|
||||
@@ -0,0 +1,14 @@
|
||||
# build.ninja
|
||||
cc = clang
|
||||
cflags = -Weverything
|
||||
|
||||
rule compile
|
||||
command = $cc $cflags -c $in -o $out
|
||||
|
||||
rule link
|
||||
command = $cc $in -o $out
|
||||
|
||||
build hello.o: compile hello.c
|
||||
build hello: link hello.o
|
||||
|
||||
default hello
|
||||
+635
@@ -0,0 +1,635 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
"""Runs ./ninja and checks if the output is correct.
|
||||
|
||||
In order to simulate a smart terminal it uses the 'script' command.
|
||||
"""
|
||||
|
||||
import os
|
||||
import platform
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
import unittest
|
||||
from textwrap import dedent
|
||||
import typing as T
|
||||
|
||||
default_env = dict(os.environ)
|
||||
default_env.pop('NINJA_STATUS', None)
|
||||
default_env.pop('CLICOLOR_FORCE', None)
|
||||
default_env['TERM'] = ''
|
||||
NINJA_PATH = os.path.abspath('./ninja')
|
||||
|
||||
def remove_non_visible_lines(raw_output: bytes) -> str:
|
||||
# When running in a smart terminal, Ninja uses CR (\r) to
|
||||
# return the cursor to the start of the current line, prints
|
||||
# something, then uses `\x1b[K` to clear everything until
|
||||
# the end of the line.
|
||||
#
|
||||
# Thus printing 'FOO', 'BAR', 'ZOO' on the same line, then
|
||||
# jumping to the next one results in the following output
|
||||
# on Posix:
|
||||
#
|
||||
# '\rFOO\x1b[K\rBAR\x1b[K\rZOO\x1b[K\r\n'
|
||||
#
|
||||
# The following splits the output at both \r, \n and \r\n
|
||||
# boundaries, which gives:
|
||||
#
|
||||
# [ '\r', 'FOO\x1b[K\r', 'BAR\x1b[K\r', 'ZOO\x1b[K\r\n' ]
|
||||
#
|
||||
decoded_lines = raw_output.decode('utf-8').splitlines(True)
|
||||
|
||||
# Remove any item that ends with a '\r' as this means its
|
||||
# content will be overwritten by the next item in the list.
|
||||
# For the previous example, this gives:
|
||||
#
|
||||
# [ 'ZOO\x1b[K\r\n' ]
|
||||
#
|
||||
final_lines = [ l for l in decoded_lines if not l.endswith('\r') ]
|
||||
|
||||
# Return a single string that concatenates all filtered lines
|
||||
# while removing any remaining \r in it. Needed to transform
|
||||
# \r\n into \n.
|
||||
#
|
||||
# "ZOO\x1b[K\n'
|
||||
#
|
||||
return ''.join(final_lines).replace('\r', '')
|
||||
|
||||
class BuildDir:
|
||||
def __init__(self, build_ninja: str):
|
||||
self.build_ninja = dedent(build_ninja)
|
||||
self.d = None
|
||||
|
||||
def __enter__(self):
|
||||
self.d = tempfile.TemporaryDirectory()
|
||||
with open(os.path.join(self.d.name, 'build.ninja'), 'w') as f:
|
||||
f.write(self.build_ninja)
|
||||
f.flush()
|
||||
return self
|
||||
|
||||
def __exit__(self, exc_type, exc_val, exc_tb):
|
||||
self.d.cleanup()
|
||||
|
||||
@property
|
||||
def path(self) -> str:
|
||||
return os.path.realpath(self.d.name)
|
||||
|
||||
|
||||
def run(
|
||||
self,
|
||||
flags: T.Optional[str] = None,
|
||||
pipe: bool = False,
|
||||
raw_output: bool = False,
|
||||
env: T.Dict[str, str] = default_env,
|
||||
print_err_output = True,
|
||||
) -> str:
|
||||
"""Run Ninja command, and get filtered output.
|
||||
|
||||
Args:
|
||||
flags: Extra arguments passed to Ninja.
|
||||
|
||||
pipe: set to True to run Ninja in a non-interactive terminal.
|
||||
If False (the default), this runs Ninja in a pty to simulate
|
||||
a smart terminal (this feature cannot work on Windows!).
|
||||
|
||||
raw_output: set to True to return the raw, unfiltered command
|
||||
output.
|
||||
|
||||
env: Optional environment dictionary to run the command in.
|
||||
|
||||
print_err_output: set to False if the test expects ninja to print
|
||||
something to stderr. (Otherwise, an error message from Ninja
|
||||
probably represents a failed test.)
|
||||
|
||||
Returns:
|
||||
A UTF-8 string corresponding to the output (stdout only) of the
|
||||
Ninja command. By default, partial lines that were overwritten
|
||||
are removed according to the rules described in the comments
|
||||
below.
|
||||
"""
|
||||
ninja_cmd = '{} {}'.format(NINJA_PATH, flags if flags else '')
|
||||
try:
|
||||
if pipe:
|
||||
output = subprocess.check_output(
|
||||
[ninja_cmd], shell=True, cwd=self.d.name, env=env)
|
||||
elif platform.system() == 'Darwin':
|
||||
output = subprocess.check_output(['script', '-q', '/dev/null', 'bash', '-c', ninja_cmd],
|
||||
cwd=self.d.name, env=env)
|
||||
else:
|
||||
output = subprocess.check_output(['script', '-qfec', ninja_cmd, '/dev/null'],
|
||||
cwd=self.d.name, env=env)
|
||||
except subprocess.CalledProcessError as err:
|
||||
if print_err_output:
|
||||
sys.stdout.buffer.write(err.output)
|
||||
err.cooked_output = remove_non_visible_lines(err.output)
|
||||
raise err
|
||||
|
||||
if raw_output:
|
||||
return output.decode('utf-8')
|
||||
return remove_non_visible_lines(output)
|
||||
|
||||
def run(
|
||||
build_ninja: str,
|
||||
flags: T.Optional[str] = None,
|
||||
pipe: bool = False,
|
||||
raw_output: bool = False,
|
||||
env: T.Dict[str, str] = default_env,
|
||||
print_err_output = True,
|
||||
) -> str:
|
||||
"""Run Ninja with a given build plan in a temporary directory.
|
||||
"""
|
||||
with BuildDir(build_ninja) as b:
|
||||
return b.run(flags, pipe, raw_output, env, print_err_output)
|
||||
|
||||
@unittest.skipIf(platform.system() == 'Windows', 'These test methods do not work on Windows')
|
||||
class Output(unittest.TestCase):
|
||||
BUILD_SIMPLE_ECHO = '\n'.join((
|
||||
'rule echo',
|
||||
' command = printf "do thing"',
|
||||
' description = echo $out',
|
||||
'',
|
||||
'build a: echo',
|
||||
'',
|
||||
))
|
||||
|
||||
def _test_expected_error(self, plan: str, flags: T.Optional[str],expected: str,
|
||||
*args, exit_code: T.Optional[int]=None, **kwargs)->None:
|
||||
"""Run Ninja with a given plan and flags, and verify its cooked output against an expected content.
|
||||
All *args and **kwargs are passed to the `run` function
|
||||
"""
|
||||
actual = ''
|
||||
kwargs['print_err_output'] = False
|
||||
with self.assertRaises(subprocess.CalledProcessError) as cm:
|
||||
run(plan, flags, *args, **kwargs)
|
||||
actual = cm.exception.cooked_output
|
||||
if exit_code is not None:
|
||||
self.assertEqual(cm.exception.returncode, exit_code)
|
||||
self.assertEqual(expected, actual)
|
||||
|
||||
def test_issue_1418(self) -> None:
|
||||
self.assertEqual(run(
|
||||
'''rule echo
|
||||
command = sleep $delay && echo $out
|
||||
description = echo $out
|
||||
|
||||
build a: echo
|
||||
delay = 3
|
||||
build b: echo
|
||||
delay = 2
|
||||
build c: echo
|
||||
delay = 1
|
||||
''', '-j3'),
|
||||
'''[1/3] echo c\x1b[K
|
||||
c
|
||||
[2/3] echo b\x1b[K
|
||||
b
|
||||
[3/3] echo a\x1b[K
|
||||
a
|
||||
''')
|
||||
|
||||
def test_issue_1214(self) -> None:
|
||||
print_red = '''rule echo
|
||||
command = printf '\x1b[31mred\x1b[0m'
|
||||
description = echo $out
|
||||
|
||||
build a: echo
|
||||
'''
|
||||
# Only strip color when ninja's output is piped.
|
||||
self.assertEqual(run(print_red),
|
||||
'''[1/1] echo a\x1b[K
|
||||
\x1b[31mred\x1b[0m
|
||||
''')
|
||||
self.assertEqual(run(print_red, pipe=True),
|
||||
'''[1/1] echo a
|
||||
red
|
||||
''')
|
||||
# Even in verbose mode, colors should still only be stripped when piped.
|
||||
self.assertEqual(run(print_red, flags='-v'),
|
||||
'''[1/1] printf '\x1b[31mred\x1b[0m'
|
||||
\x1b[31mred\x1b[0m
|
||||
''')
|
||||
self.assertEqual(run(print_red, flags='-v', pipe=True),
|
||||
'''[1/1] printf '\x1b[31mred\x1b[0m'
|
||||
red
|
||||
''')
|
||||
|
||||
# CLICOLOR_FORCE=1 can be used to disable escape code stripping.
|
||||
env = default_env.copy()
|
||||
env['CLICOLOR_FORCE'] = '1'
|
||||
self.assertEqual(run(print_red, pipe=True, env=env),
|
||||
'''[1/1] echo a
|
||||
\x1b[31mred\x1b[0m
|
||||
''')
|
||||
|
||||
def test_issue_1966(self) -> None:
|
||||
self.assertEqual(run(
|
||||
'''rule cat
|
||||
command = cat $rspfile $rspfile > $out
|
||||
rspfile = cat.rsp
|
||||
rspfile_content = a b c
|
||||
|
||||
build a: cat
|
||||
''', '-j3'),
|
||||
'''[1/1] cat cat.rsp cat.rsp > a\x1b[K
|
||||
''')
|
||||
|
||||
def test_issue_2499(self) -> None:
|
||||
# This verifies that Ninja prints its status line updates on a single
|
||||
# line when running in a smart terminal, and when commands do not have
|
||||
# any output. Get the raw command output which includes CR (\r) codes
|
||||
# and all content that was printed by Ninja.
|
||||
self.assertEqual(run(
|
||||
'''rule touch
|
||||
command = touch $out
|
||||
|
||||
build foo: touch
|
||||
build bar: touch foo
|
||||
build zoo: touch bar
|
||||
''', flags='-j1 zoo', raw_output=True).split('\r'),
|
||||
[
|
||||
'',
|
||||
'[0/3] touch foo\x1b[K',
|
||||
'[1/3] touch foo\x1b[K',
|
||||
'[1/3] touch bar\x1b[K',
|
||||
'[2/3] touch bar\x1b[K',
|
||||
'[2/3] touch zoo\x1b[K',
|
||||
'[3/3] touch zoo\x1b[K',
|
||||
'\n',
|
||||
])
|
||||
|
||||
def test_pr_1685(self) -> None:
|
||||
# Running those tools without .ninja_deps and .ninja_log shouldn't fail.
|
||||
self.assertEqual(run('', flags='-t recompact'), '')
|
||||
self.assertEqual(run('', flags='-t restat'), '')
|
||||
|
||||
def test_issue_2048(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as d:
|
||||
with open(os.path.join(d, 'build.ninja'), 'w'):
|
||||
pass
|
||||
|
||||
with open(os.path.join(d, '.ninja_log'), 'w') as f:
|
||||
f.write('# ninja log v4\n')
|
||||
|
||||
try:
|
||||
output = subprocess.check_output([NINJA_PATH, '-t', 'recompact'],
|
||||
cwd=d,
|
||||
env=default_env,
|
||||
stderr=subprocess.STDOUT,
|
||||
text=True
|
||||
)
|
||||
|
||||
self.assertEqual(
|
||||
output.strip(),
|
||||
"ninja: warning: build log version is too old; starting over"
|
||||
)
|
||||
except subprocess.CalledProcessError as err:
|
||||
self.fail("non-zero exit code with: " + err.output)
|
||||
|
||||
def test_pr_2540(self)->None:
|
||||
py = sys.executable
|
||||
plan = f'''\
|
||||
rule CUSTOM_COMMAND
|
||||
command = $COMMAND
|
||||
|
||||
build 124: CUSTOM_COMMAND
|
||||
COMMAND = {py} -c 'exit(124)'
|
||||
|
||||
build 127: CUSTOM_COMMAND
|
||||
COMMAND = {py} -c 'exit(127)'
|
||||
|
||||
build 130: CUSTOM_COMMAND
|
||||
COMMAND = {py} -c 'exit(130)'
|
||||
|
||||
build 137: CUSTOM_COMMAND
|
||||
COMMAND = {py} -c 'exit(137)'
|
||||
|
||||
build success: CUSTOM_COMMAND
|
||||
COMMAND = sleep 0.3; echo success
|
||||
'''
|
||||
# Disable colors
|
||||
env = default_env.copy()
|
||||
env['TERM'] = 'dumb'
|
||||
self._test_expected_error(
|
||||
plan, '124',
|
||||
f'''[1/1] {py} -c 'exit(124)'
|
||||
FAILED: [code=124] 124 \n{py} -c 'exit(124)'
|
||||
ninja: build stopped: subcommand failed.
|
||||
''',
|
||||
exit_code=124, env=env,
|
||||
)
|
||||
self._test_expected_error(
|
||||
plan, '127',
|
||||
f'''[1/1] {py} -c 'exit(127)'
|
||||
FAILED: [code=127] 127 \n{py} -c 'exit(127)'
|
||||
ninja: build stopped: subcommand failed.
|
||||
''',
|
||||
exit_code=127, env=env,
|
||||
)
|
||||
self._test_expected_error(
|
||||
plan, '130',
|
||||
'ninja: build stopped: interrupted by user.\n',
|
||||
exit_code=130, env=env,
|
||||
)
|
||||
self._test_expected_error(
|
||||
plan, '137',
|
||||
f'''[1/1] {py} -c 'exit(137)'
|
||||
FAILED: [code=137] 137 \n{py} -c 'exit(137)'
|
||||
ninja: build stopped: subcommand failed.
|
||||
''',
|
||||
exit_code=137, env=env,
|
||||
)
|
||||
self._test_expected_error(
|
||||
plan, 'non-existent-target',
|
||||
"ninja: error: unknown target 'non-existent-target'\n",
|
||||
exit_code=1, env=env,
|
||||
)
|
||||
self._test_expected_error(
|
||||
plan, '-j2 success 127',
|
||||
f'''[1/2] {py} -c 'exit(127)'
|
||||
FAILED: [code=127] 127 \n{py} -c 'exit(127)'
|
||||
[2/2] sleep 0.3; echo success
|
||||
success
|
||||
ninja: build stopped: subcommand failed.
|
||||
''',
|
||||
exit_code=127, env=env,
|
||||
)
|
||||
|
||||
def test_depfile_directory_creation(self) -> None:
|
||||
b = BuildDir('''\
|
||||
rule touch
|
||||
command = touch $out && echo "$out: extra" > $depfile
|
||||
|
||||
build somewhere/out: touch
|
||||
depfile = somewhere_else/out.d
|
||||
''')
|
||||
with b:
|
||||
self.assertEqual(b.run('', pipe=True), dedent('''\
|
||||
[1/1] touch somewhere/out && echo "somewhere/out: extra" > somewhere_else/out.d
|
||||
'''))
|
||||
self.assertTrue(os.path.isfile(os.path.join(b.d.name, "somewhere", "out")))
|
||||
self.assertTrue(os.path.isfile(os.path.join(b.d.name, "somewhere_else", "out.d")))
|
||||
|
||||
def test_status(self) -> None:
|
||||
self.assertEqual(run(''), 'ninja: no work to do.\n')
|
||||
self.assertEqual(run('', pipe=True), 'ninja: no work to do.\n')
|
||||
self.assertEqual(run('', flags='--quiet'), '')
|
||||
|
||||
def test_ninja_status_default(self) -> None:
|
||||
'Do we show the default status by default?'
|
||||
self.assertEqual(run(Output.BUILD_SIMPLE_ECHO), '[1/1] echo a\x1b[K\ndo thing\n')
|
||||
|
||||
def test_ninja_status_quiet(self) -> None:
|
||||
'Do we suppress the status information when --quiet is specified?'
|
||||
output = run(Output.BUILD_SIMPLE_ECHO, flags='--quiet')
|
||||
self.assertEqual(output, 'do thing\n')
|
||||
|
||||
def test_entering_directory_on_stdout(self) -> None:
|
||||
output = run(Output.BUILD_SIMPLE_ECHO, flags='-C$PWD', pipe=True)
|
||||
self.assertEqual(output.splitlines()[0][:25], "ninja: Entering directory")
|
||||
|
||||
def test_tool_inputs(self) -> None:
|
||||
plan = '''
|
||||
rule cat
|
||||
command = cat $in $out
|
||||
build out1 : cat in1
|
||||
build out2 : cat in2 out1
|
||||
build out3 : cat out2 out1 | implicit || order_only
|
||||
'''
|
||||
self.assertEqual(run(plan, flags='-t inputs out3'),
|
||||
'''implicit
|
||||
in1
|
||||
in2
|
||||
order_only
|
||||
out1
|
||||
out2
|
||||
''')
|
||||
|
||||
self.assertEqual(run(plan, flags='-t inputs --dependency-order out3'),
|
||||
'''in2
|
||||
in1
|
||||
out1
|
||||
out2
|
||||
implicit
|
||||
order_only
|
||||
''')
|
||||
|
||||
# Verify that results are shell-escaped by default, unless --no-shell-escape
|
||||
# is used. Also verify that phony outputs are never part of the results.
|
||||
quote = '"' if platform.system() == "Windows" else "'"
|
||||
|
||||
plan = '''
|
||||
rule cat
|
||||
command = cat $in $out
|
||||
build out1 : cat in1
|
||||
build out$ 2 : cat out1
|
||||
build out$ 3 : phony out$ 2
|
||||
build all: phony out$ 3
|
||||
'''
|
||||
|
||||
# Quoting changes the order of results when sorting alphabetically.
|
||||
self.assertEqual(run(plan, flags='-t inputs all'),
|
||||
f'''{quote}out 2{quote}
|
||||
in1
|
||||
out1
|
||||
''')
|
||||
|
||||
self.assertEqual(run(plan, flags='-t inputs --no-shell-escape all'),
|
||||
'''in1
|
||||
out 2
|
||||
out1
|
||||
''')
|
||||
|
||||
# But not when doing dependency order.
|
||||
self.assertEqual(
|
||||
run(
|
||||
plan,
|
||||
flags='-t inputs --dependency-order all'
|
||||
),
|
||||
f'''in1
|
||||
out1
|
||||
{quote}out 2{quote}
|
||||
''')
|
||||
|
||||
self.assertEqual(
|
||||
run(
|
||||
plan,
|
||||
flags='-t inputs --dependency-order --no-shell-escape all'
|
||||
),
|
||||
f'''in1
|
||||
out1
|
||||
out 2
|
||||
''')
|
||||
|
||||
self.assertEqual(
|
||||
run(
|
||||
plan,
|
||||
flags='-t inputs --dependency-order --no-shell-escape --print0 all'
|
||||
),
|
||||
f'''in1\0out1\0out 2\0'''
|
||||
)
|
||||
|
||||
|
||||
def test_tool_compdb_targets(self) -> None:
|
||||
plan = '''
|
||||
rule cat
|
||||
command = cat $in $out
|
||||
build out1 : cat in1
|
||||
build out2 : cat in2 out1
|
||||
build out3 : cat out2 out1
|
||||
build out4 : cat in4
|
||||
'''
|
||||
|
||||
|
||||
self._test_expected_error(plan, '-t compdb-targets',
|
||||
'''ninja: error: compdb-targets expects the name of at least one target
|
||||
usage: ninja -t compdb [-hx] target [targets]
|
||||
|
||||
options:
|
||||
-h display this help message
|
||||
-x expand @rspfile style response file invocations
|
||||
''')
|
||||
|
||||
self._test_expected_error(plan, '-t compdb-targets in1',
|
||||
"ninja: fatal: 'in1' is not a target (i.e. it is not an output of any `build` statement)\n")
|
||||
|
||||
self._test_expected_error(plan, '-t compdb-targets nonexistent_target',
|
||||
"ninja: fatal: unknown target 'nonexistent_target'\n")
|
||||
|
||||
|
||||
with BuildDir(plan) as b:
|
||||
actual = b.run(flags='-t compdb-targets out3')
|
||||
expected = f'''[
|
||||
{{
|
||||
"directory": "{b.path}",
|
||||
"command": "cat in1 out1",
|
||||
"file": "in1",
|
||||
"output": "out1"
|
||||
}},
|
||||
{{
|
||||
"directory": "{b.path}",
|
||||
"command": "cat in2 out1 out2",
|
||||
"file": "in2",
|
||||
"output": "out2"
|
||||
}},
|
||||
{{
|
||||
"directory": "{b.path}",
|
||||
"command": "cat out2 out1 out3",
|
||||
"file": "out2",
|
||||
"output": "out3"
|
||||
}}
|
||||
]
|
||||
'''
|
||||
self.assertEqual(expected, actual)
|
||||
|
||||
|
||||
def test_tool_multi_inputs(self) -> None:
|
||||
plan = '''
|
||||
rule cat
|
||||
command = cat $in $out
|
||||
build out1 : cat in1
|
||||
build out2 : cat in1 in2
|
||||
build out3 : cat in1 in2 in3
|
||||
'''
|
||||
self.assertEqual(run(plan, flags='-t multi-inputs out1'),
|
||||
'''out1<TAB>in1
|
||||
'''.replace("<TAB>", "\t"))
|
||||
|
||||
self.assertEqual(run(plan, flags='-t multi-inputs out1 out2 out3'),
|
||||
'''out1<TAB>in1
|
||||
out2<TAB>in1
|
||||
out2<TAB>in2
|
||||
out3<TAB>in1
|
||||
out3<TAB>in2
|
||||
out3<TAB>in3
|
||||
'''.replace("<TAB>", "\t"))
|
||||
|
||||
self.assertEqual(run(plan, flags='-t multi-inputs -d: out1'),
|
||||
'''out1:in1
|
||||
''')
|
||||
|
||||
self.assertEqual(
|
||||
run(
|
||||
plan,
|
||||
flags='-t multi-inputs -d, --print0 out1 out2'
|
||||
),
|
||||
'''out1,in1\0out2,in1\0out2,in2\0'''
|
||||
)
|
||||
|
||||
|
||||
def test_explain_output(self):
|
||||
b = BuildDir('''\
|
||||
build .FORCE: phony
|
||||
rule create_if_non_exist
|
||||
command = [ -e $out ] || touch $out
|
||||
restat = true
|
||||
rule write
|
||||
command = cp $in $out
|
||||
build input : create_if_non_exist .FORCE
|
||||
build mid : write input
|
||||
build output : write mid
|
||||
default output
|
||||
''')
|
||||
with b:
|
||||
# The explain output is shown just before the relevant build:
|
||||
self.assertEqual(b.run('-v -d explain'), dedent('''\
|
||||
ninja explain: .FORCE is dirty
|
||||
[1/3] [ -e input ] || touch input
|
||||
ninja explain: input is dirty
|
||||
[2/3] cp input mid
|
||||
ninja explain: mid is dirty
|
||||
[3/3] cp mid output
|
||||
'''))
|
||||
# Don't print "ninja explain: XXX is dirty" for inputs that are
|
||||
# pruned from the graph by an earlier restat.
|
||||
self.assertEqual(b.run('-v -d explain'), dedent('''\
|
||||
ninja explain: .FORCE is dirty
|
||||
[1/3] [ -e input ] || touch input
|
||||
'''))
|
||||
|
||||
def test_issue_2586(self):
|
||||
"""This shouldn't hang"""
|
||||
plan = '''rule echo
|
||||
command = echo echo
|
||||
build dep: echo
|
||||
build console1: echo dep
|
||||
pool = console
|
||||
build console2: echo
|
||||
pool = console
|
||||
build all: phony console1 console2
|
||||
default all
|
||||
'''
|
||||
self.assertEqual(run(plan, flags='-j2', env={'NINJA_STATUS':''}), '''echo echo
|
||||
echo
|
||||
echo echo
|
||||
echo
|
||||
echo echo
|
||||
echo
|
||||
''')
|
||||
|
||||
def test_issue_2621(self):
|
||||
"""Should result in "multiple rules generate" error"""
|
||||
plan = r"""rule dd
|
||||
command = printf 'ninja_dyndep_version = 1\nbuild stamp-$n | out: dyndep\n' > $out
|
||||
rule touch
|
||||
command = touch stamp-$n out
|
||||
dyndep = dd-$n
|
||||
build dd-1: dd
|
||||
n = 1
|
||||
build dd-2: dd
|
||||
n = 2
|
||||
build stamp-1: touch || dd-1
|
||||
n = 1
|
||||
build stamp-2: touch || dd-2
|
||||
n = 2
|
||||
"""
|
||||
self._test_expected_error(
|
||||
plan,
|
||||
"-v",
|
||||
r"""[1/4] printf 'ninja_dyndep_version = 1\nbuild stamp-1 | out: dyndep\n' > dd-1
|
||||
[2/4] printf 'ninja_dyndep_version = 1\nbuild stamp-2 | out: dyndep\n' > dd-2
|
||||
ninja: build stopped: multiple rules generate out.
|
||||
""",
|
||||
)
|
||||
|
||||
if __name__ == '__main__':
|
||||
unittest.main()
|
||||
@@ -0,0 +1,42 @@
|
||||
Summary: Ninja is a small build system with a focus on speed.
|
||||
Name: ninja
|
||||
Version: %{ver}
|
||||
Release: %{rel}%{?dist}
|
||||
Group: Development/Tools
|
||||
License: Apache 2.0
|
||||
URL: https://github.com/ninja-build/ninja
|
||||
Source0: %{name}-%{version}-%{rel}.tar.gz
|
||||
BuildRoot: %{_tmppath}/%{name}-%{version}-%{rel}
|
||||
|
||||
BuildRequires: asciidoc
|
||||
|
||||
%description
|
||||
Ninja is yet another build system. It takes as input the interdependencies of files (typically source code and output executables) and
|
||||
orchestrates building them, quickly.
|
||||
|
||||
Ninja joins a sea of other build systems. Its distinguishing goal is to be fast. It is born from my work on the Chromium browser project,
|
||||
which has over 30,000 source files and whose other build systems (including one built from custom non-recursive Makefiles) can take ten
|
||||
seconds to start building after changing one file. Ninja is under a second.
|
||||
|
||||
%prep
|
||||
%setup -q -n %{name}-%{version}-%{rel}
|
||||
|
||||
%build
|
||||
echo Building..
|
||||
./configure.py --bootstrap
|
||||
./ninja manual
|
||||
|
||||
%install
|
||||
mkdir -p %{buildroot}%{_bindir} %{buildroot}%{_docdir}
|
||||
cp -p ninja %{buildroot}%{_bindir}/
|
||||
|
||||
%files
|
||||
%defattr(-, root, root)
|
||||
%doc COPYING README.md doc/manual.html
|
||||
%{_bindir}/*
|
||||
|
||||
%clean
|
||||
rm -rf %{buildroot}
|
||||
|
||||
#The changelog is built automatically from Git history
|
||||
%changelog
|
||||
@@ -0,0 +1,29 @@
|
||||
#!/bin/bash
|
||||
|
||||
echo Building ninja RPMs..
|
||||
GITROOT=$(git rev-parse --show-toplevel)
|
||||
cd $GITROOT
|
||||
|
||||
VER=1.0
|
||||
REL=$(git rev-parse --short HEAD)git
|
||||
RPMTOPDIR=$GITROOT/rpm-build
|
||||
echo "Ver: $VER, Release: $REL"
|
||||
|
||||
# Create tarball
|
||||
mkdir -p $RPMTOPDIR/{SOURCES,SPECS}
|
||||
git archive --format=tar --prefix=ninja-${VER}-${REL}/ HEAD | gzip -c > $RPMTOPDIR/SOURCES/ninja-${VER}-${REL}.tar.gz
|
||||
|
||||
# Convert git log to RPM's ChangeLog format (shown with rpm -qp --changelog <rpm file>)
|
||||
sed -e "s/%{ver}/$VER/" -e "s/%{rel}/$REL/" misc/packaging/ninja.spec > $RPMTOPDIR/SPECS/ninja.spec
|
||||
git log --format="* %cd %aN%n- (%h) %s%d%n" --date=local | sed -r 's/[0-9]+:[0-9]+:[0-9]+ //' >> $RPMTOPDIR/SPECS/ninja.spec
|
||||
|
||||
# Build SRC and binary RPMs
|
||||
rpmbuild --quiet \
|
||||
--define "_topdir $RPMTOPDIR" \
|
||||
--define "_rpmdir $PWD" \
|
||||
--define "_srcrpmdir $PWD" \
|
||||
--define '_rpmfilename %%{NAME}-%%{VERSION}-%%{RELEASE}.%%{ARCH}.rpm' \
|
||||
-ba $RPMTOPDIR/SPECS/ninja.spec &&
|
||||
|
||||
rm -rf $RPMTOPDIR &&
|
||||
echo Done
|
||||
@@ -0,0 +1,273 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
"""Writes large manifest files, for manifest parser performance testing.
|
||||
|
||||
The generated manifest files are (eerily) similar in appearance and size to the
|
||||
ones used in the Chromium project.
|
||||
|
||||
Usage:
|
||||
python misc/write_fake_manifests.py outdir # Will run for about 5s.
|
||||
|
||||
The program contains a hardcoded random seed, so it will generate the same
|
||||
output every time it runs. By changing the seed, it's easy to generate many
|
||||
different sets of manifest files.
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import contextlib
|
||||
import os
|
||||
import random
|
||||
import sys
|
||||
from typing import Generator, Optional, Tuple, List, Set
|
||||
|
||||
import ninja_syntax
|
||||
|
||||
|
||||
def paretoint(avg: float, alpha: float) -> int:
|
||||
"""Returns a random integer that's avg on average, following a power law.
|
||||
alpha determines the shape of the power curve. alpha has to be larger
|
||||
than 1. The closer alpha is to 1, the higher the variation of the returned
|
||||
numbers."""
|
||||
return int(random.paretovariate(alpha) * avg / (alpha / (alpha - 1)))
|
||||
|
||||
|
||||
# Based on http://neugierig.org/software/chromium/class-name-generator.html
|
||||
def moar(avg_options: float, p_suffix: float) -> str:
|
||||
kStart = ['render', 'web', 'browser', 'tab', 'content', 'extension', 'url',
|
||||
'file', 'sync', 'content', 'http', 'profile']
|
||||
kOption = ['view', 'host', 'holder', 'container', 'impl', 'ref',
|
||||
'delegate', 'widget', 'proxy', 'stub', 'context',
|
||||
'manager', 'master', 'watcher', 'service', 'file', 'data',
|
||||
'resource', 'device', 'info', 'provider', 'internals', 'tracker',
|
||||
'api', 'layer']
|
||||
kOS = ['win', 'mac', 'aura', 'linux', 'android', 'unittest', 'browsertest']
|
||||
num_options = min(paretoint(avg_options, alpha=4), 5)
|
||||
# The original allows kOption to repeat as long as no consecutive options
|
||||
# repeat. This version doesn't allow any option repetition.
|
||||
name = [random.choice(kStart)] + random.sample(kOption, num_options)
|
||||
if random.random() < p_suffix:
|
||||
name.append(random.choice(kOS))
|
||||
return '_'.join(name)
|
||||
|
||||
|
||||
class GenRandom(object):
|
||||
def __init__(self, src_dir: str) -> None:
|
||||
self.seen_names: Set[Optional[str]] = set([None])
|
||||
self.seen_defines: Set[Optional[str]] = set([None])
|
||||
self.src_dir = src_dir
|
||||
|
||||
def _unique_string(self, seen: Set[Optional[str]], avg_options: float = 1.3, p_suffix: float = 0.1) -> str:
|
||||
s = None
|
||||
while s in seen:
|
||||
s = moar(avg_options, p_suffix)
|
||||
seen.add(s)
|
||||
return s # type: ignore # Incompatible return value type
|
||||
|
||||
def _n_unique_strings(self, n: int) -> List[str]:
|
||||
seen: Set[Optional[str]] = set([None])
|
||||
return [self._unique_string(seen, avg_options=3, p_suffix=0.4)
|
||||
for _ in range(n)]
|
||||
|
||||
def target_name(self) -> str:
|
||||
return self._unique_string(p_suffix=0, seen=self.seen_names)
|
||||
|
||||
def path(self) -> str:
|
||||
return os.path.sep.join([
|
||||
self._unique_string(self.seen_names, avg_options=1, p_suffix=0)
|
||||
for _ in range(1 + paretoint(0.6, alpha=4))])
|
||||
|
||||
def src_obj_pairs(self, path: str, name: str) -> List[Tuple[str, str]]:
|
||||
num_sources = paretoint(55, alpha=2) + 1
|
||||
return [(os.path.join(self.src_dir, path, s + '.cc'),
|
||||
os.path.join('obj', path, '%s.%s.o' % (name, s)))
|
||||
for s in self._n_unique_strings(num_sources)]
|
||||
|
||||
def defines(self) -> List[str]:
|
||||
return [
|
||||
'-DENABLE_' + self._unique_string(self.seen_defines).upper()
|
||||
for _ in range(paretoint(20, alpha=3))]
|
||||
|
||||
|
||||
LIB, EXE = 0, 1
|
||||
class Target(object):
|
||||
def __init__(self, gen: GenRandom, kind: int) -> None:
|
||||
self.name = gen.target_name()
|
||||
self.dir_path = gen.path()
|
||||
self.ninja_file_path = os.path.join(
|
||||
'obj', self.dir_path, self.name + '.ninja')
|
||||
self.src_obj_pairs = gen.src_obj_pairs(self.dir_path, self.name)
|
||||
if kind == LIB:
|
||||
self.output = os.path.join('lib' + self.name + '.a')
|
||||
elif kind == EXE:
|
||||
self.output = os.path.join(self.name)
|
||||
self.defines = gen.defines()
|
||||
self.deps: List[Target] = []
|
||||
self.kind = kind
|
||||
self.has_compile_depends = random.random() < 0.4
|
||||
|
||||
|
||||
def write_target_ninja(ninja: ninja_syntax.Writer, target: Target, src_dir: str) -> None:
|
||||
compile_depends = None
|
||||
if target.has_compile_depends:
|
||||
compile_depends = os.path.join(
|
||||
'obj', target.dir_path, target.name + '.stamp')
|
||||
ninja.build(compile_depends, 'stamp', target.src_obj_pairs[0][0])
|
||||
ninja.newline()
|
||||
|
||||
ninja.variable('defines', target.defines)
|
||||
ninja.variable('includes', '-I' + src_dir)
|
||||
ninja.variable('cflags', ['-Wall', '-fno-rtti', '-fno-exceptions'])
|
||||
ninja.newline()
|
||||
|
||||
for src, obj in target.src_obj_pairs:
|
||||
ninja.build(obj, 'cxx', src, implicit=compile_depends)
|
||||
ninja.newline()
|
||||
|
||||
deps = [dep.output for dep in target.deps]
|
||||
libs = [dep.output for dep in target.deps if dep.kind == LIB]
|
||||
if target.kind == EXE:
|
||||
ninja.variable('libs', libs)
|
||||
if sys.platform == "darwin":
|
||||
ninja.variable('ldflags', '-Wl,-pie')
|
||||
link = { LIB: 'alink', EXE: 'link'}[target.kind]
|
||||
ninja.build(target.output, link, [obj for _, obj in target.src_obj_pairs],
|
||||
implicit=deps)
|
||||
|
||||
|
||||
def write_sources(target: Target, root_dir: str) -> None:
|
||||
need_main = target.kind == EXE
|
||||
|
||||
includes = []
|
||||
|
||||
# Include siblings.
|
||||
for cc_filename, _ in target.src_obj_pairs:
|
||||
h_filename = os.path.basename(os.path.splitext(cc_filename)[0] + '.h')
|
||||
includes.append(h_filename)
|
||||
|
||||
# Include deps.
|
||||
for dep in target.deps:
|
||||
for cc_filename, _ in dep.src_obj_pairs:
|
||||
h_filename = os.path.basename(
|
||||
os.path.splitext(cc_filename)[0] + '.h')
|
||||
includes.append("%s/%s" % (dep.dir_path, h_filename))
|
||||
|
||||
for cc_filename, _ in target.src_obj_pairs:
|
||||
cc_path = os.path.join(root_dir, cc_filename)
|
||||
h_path = os.path.splitext(cc_path)[0] + '.h'
|
||||
namespace = os.path.basename(target.dir_path)
|
||||
class_ = os.path.splitext(os.path.basename(cc_filename))[0]
|
||||
try:
|
||||
os.makedirs(os.path.dirname(cc_path))
|
||||
except OSError:
|
||||
pass
|
||||
|
||||
with open(h_path, 'w') as f:
|
||||
f.write('namespace %s { struct %s { %s(); }; }' % (namespace,
|
||||
class_, class_))
|
||||
with open(cc_path, 'w') as f:
|
||||
for include in includes:
|
||||
f.write('#include "%s"\n' % include)
|
||||
f.write('\n')
|
||||
f.write('namespace %s { %s::%s() {} }' % (namespace,
|
||||
class_, class_))
|
||||
|
||||
if need_main:
|
||||
f.write('int main(int argc, char **argv) {}\n')
|
||||
need_main = False
|
||||
|
||||
def write_master_ninja(master_ninja: ninja_syntax.Writer, targets: List[Target]) -> None:
|
||||
"""Writes master build.ninja file, referencing all given subninjas."""
|
||||
master_ninja.variable('cxx', 'c++')
|
||||
master_ninja.variable('ld', '$cxx')
|
||||
if sys.platform == 'darwin':
|
||||
master_ninja.variable('alink', 'libtool -static')
|
||||
else:
|
||||
master_ninja.variable('alink', 'ar rcs')
|
||||
master_ninja.newline()
|
||||
|
||||
master_ninja.pool('link_pool', depth=4)
|
||||
master_ninja.newline()
|
||||
|
||||
master_ninja.rule('cxx', description='CXX $out',
|
||||
command='$cxx -MMD -MF $out.d $defines $includes $cflags -c $in -o $out',
|
||||
depfile='$out.d', deps='gcc')
|
||||
master_ninja.rule('alink', description='ARCHIVE $out',
|
||||
command='rm -f $out && $alink -o $out $in')
|
||||
master_ninja.rule('link', description='LINK $out', pool='link_pool',
|
||||
command='$ld $ldflags -o $out $in $libs')
|
||||
master_ninja.rule('stamp', description='STAMP $out', command='touch $out')
|
||||
master_ninja.newline()
|
||||
|
||||
for target in targets:
|
||||
master_ninja.subninja(target.ninja_file_path)
|
||||
master_ninja.newline()
|
||||
|
||||
master_ninja.comment('Short names for targets.')
|
||||
for target in targets:
|
||||
if target.name != target.output:
|
||||
master_ninja.build(target.name, 'phony', target.output)
|
||||
master_ninja.newline()
|
||||
|
||||
master_ninja.build('all', 'phony', [target.output for target in targets])
|
||||
master_ninja.default('all')
|
||||
|
||||
|
||||
@contextlib.contextmanager
|
||||
def FileWriter(path: str) -> Generator[ninja_syntax.Writer, None, None]:
|
||||
"""Context manager for a ninja_syntax object writing to a file."""
|
||||
try:
|
||||
os.makedirs(os.path.dirname(path))
|
||||
except OSError:
|
||||
pass
|
||||
f = open(path, 'w')
|
||||
yield ninja_syntax.Writer(f)
|
||||
f.close()
|
||||
|
||||
|
||||
def random_targets(num_targets: int, src_dir: str) -> List[Target]:
|
||||
gen = GenRandom(src_dir)
|
||||
|
||||
# N-1 static libraries, and 1 executable depending on all of them.
|
||||
targets = [Target(gen, LIB) for i in range(num_targets - 1)]
|
||||
for i in range(len(targets)):
|
||||
targets[i].deps = [t for t in targets[0:i] if random.random() < 0.05]
|
||||
|
||||
last_target = Target(gen, EXE)
|
||||
last_target.deps = targets[:]
|
||||
last_target.src_obj_pairs = last_target.src_obj_pairs[0:10] # Trim.
|
||||
targets.append(last_target)
|
||||
return targets
|
||||
|
||||
|
||||
def main() -> None:
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument('-s', '--sources', nargs="?", const="src",
|
||||
help='write sources to directory (relative to output directory)')
|
||||
parser.add_argument('-t', '--targets', type=int, default=1500,
|
||||
help='number of targets (default: 1500)')
|
||||
parser.add_argument('-S', '--seed', type=int, help='random seed',
|
||||
default=12345)
|
||||
parser.add_argument('outdir', help='output directory')
|
||||
args = parser.parse_args()
|
||||
root_dir = args.outdir
|
||||
|
||||
random.seed(args.seed)
|
||||
|
||||
do_write_sources = args.sources is not None
|
||||
src_dir = args.sources if do_write_sources else "src"
|
||||
|
||||
targets = random_targets(args.targets, src_dir)
|
||||
for target in targets:
|
||||
with FileWriter(os.path.join(root_dir, target.ninja_file_path)) as n:
|
||||
write_target_ninja(n, target, src_dir)
|
||||
|
||||
if do_write_sources:
|
||||
write_sources(target, root_dir)
|
||||
|
||||
with FileWriter(os.path.join(root_dir, 'build.ninja')) as master_ninja:
|
||||
master_ninja.width = 120
|
||||
write_master_ninja(master_ninja, targets)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
sys.exit(main()) # type: ignore # "main" does not return a value
|
||||
@@ -0,0 +1,75 @@
|
||||
#compdef ninja
|
||||
# Copyright 2011 Google Inc. All Rights Reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
# Add the following to your .zshrc to tab-complete ninja targets
|
||||
# fpath=(path/to/ninja/misc/zsh-completion $fpath)
|
||||
|
||||
(( $+functions[_ninja-get-targets] )) || _ninja-get-targets() {
|
||||
dir="."
|
||||
if [ -n "${opt_args[-C]}" ];
|
||||
then
|
||||
eval dir="${opt_args[-C]}"
|
||||
fi
|
||||
file="build.ninja"
|
||||
if [ -n "${opt_args[-f]}" ];
|
||||
then
|
||||
eval file="${opt_args[-f]}"
|
||||
fi
|
||||
targets_command="ninja -f \"${file}\" -C \"${dir}\" -t targets all"
|
||||
eval ${targets_command} 2>/dev/null | cut -d: -f1
|
||||
}
|
||||
|
||||
(( $+functions[_ninja-get-tools] )) || _ninja-get-tools() {
|
||||
# remove the first line; remove the leading spaces; replace spaces with colon
|
||||
ninja -t list 2> /dev/null | sed -e '1d;s/^ *//;s/ \+/:/'
|
||||
}
|
||||
|
||||
(( $+functions[_ninja-get-modes] )) || _ninja-get-modes() {
|
||||
# remove the first line; remove the last line; remove the leading spaces; replace spaces with colon
|
||||
ninja -d list 2> /dev/null | sed -e '1d;$d;s/^ *//;s/ \+/:/'
|
||||
}
|
||||
|
||||
(( $+functions[_ninja-modes] )) || _ninja-modes() {
|
||||
local -a modes
|
||||
modes=(${(fo)"$(_ninja-get-modes)"})
|
||||
_describe 'modes' modes
|
||||
}
|
||||
|
||||
(( $+functions[_ninja-tools] )) || _ninja-tools() {
|
||||
local -a tools
|
||||
tools=(${(fo)"$(_ninja-get-tools)"})
|
||||
_describe 'tools' tools
|
||||
}
|
||||
|
||||
(( $+functions[_ninja-targets] )) || _ninja-targets() {
|
||||
local -a targets
|
||||
targets=(${(fo)"$(_ninja-get-targets)"})
|
||||
_describe 'targets' targets
|
||||
}
|
||||
|
||||
_arguments \
|
||||
'(- *)'{-h,--help}'[Show help]' \
|
||||
'(- *)--version[Print ninja version]' \
|
||||
'-C+[Change to directory before doing anything else]:directories:_directories' \
|
||||
'-f+[Specify input build file (default=build.ninja)]:files:_files' \
|
||||
'-j+[Run N jobs in parallel (default=number of CPUs available)]:number of jobs' \
|
||||
'-l+[Do not start new jobs if the load average is greater than N]:number of jobs' \
|
||||
'-k+[Keep going until N jobs fail (default=1)]:number of jobs' \
|
||||
'-n[Dry run (do not run commands but act like they succeeded)]' \
|
||||
'(-v --verbose --quiet)'{-v,--verbose}'[Show all command lines while building]' \
|
||||
"(-v --verbose --quiet)--quiet[Don't show progress status, just command output]" \
|
||||
'-d+[Enable debugging (use -d list to list modes)]:modes:_ninja-modes' \
|
||||
'-t+[Run a subtool (use -t list to list subtools)]:tools:_ninja-tools' \
|
||||
'*::targets:_ninja-targets'
|
||||
@@ -0,0 +1,85 @@
|
||||
// Copyright 2011 Google Inc. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#include "browse.h"
|
||||
|
||||
#include <errno.h>
|
||||
#include <stdio.h>
|
||||
#include <stdlib.h>
|
||||
#include <unistd.h>
|
||||
#include <vector>
|
||||
|
||||
#include "build/browse_py.h"
|
||||
|
||||
using namespace std;
|
||||
|
||||
void RunBrowsePython(State* state, const char* ninja_command,
|
||||
const char* input_file, int argc, char* argv[]) {
|
||||
// Fork off a Python process and have it run our code via its stdin.
|
||||
// (Actually the Python process becomes the parent.)
|
||||
int pipefd[2];
|
||||
if (pipe(pipefd) < 0) {
|
||||
perror("ninja: pipe");
|
||||
return;
|
||||
}
|
||||
|
||||
pid_t pid = fork();
|
||||
if (pid < 0) {
|
||||
perror("ninja: fork");
|
||||
return;
|
||||
}
|
||||
|
||||
if (pid > 0) { // Parent.
|
||||
close(pipefd[1]);
|
||||
do {
|
||||
if (dup2(pipefd[0], 0) < 0) {
|
||||
perror("ninja: dup2");
|
||||
break;
|
||||
}
|
||||
|
||||
std::vector<const char *> command;
|
||||
command.push_back(NINJA_PYTHON);
|
||||
command.push_back("-");
|
||||
command.push_back("--ninja-command");
|
||||
command.push_back(ninja_command);
|
||||
command.push_back("-f");
|
||||
command.push_back(input_file);
|
||||
for (int i = 0; i < argc; i++) {
|
||||
command.push_back(argv[i]);
|
||||
}
|
||||
command.push_back(NULL);
|
||||
execvp(command[0], const_cast<char**>(&command[0]));
|
||||
if (errno == ENOENT) {
|
||||
printf("ninja: %s is required for the browse tool\n", NINJA_PYTHON);
|
||||
} else {
|
||||
perror("ninja: execvp");
|
||||
}
|
||||
} while (false);
|
||||
_exit(1);
|
||||
} else { // Child.
|
||||
close(pipefd[0]);
|
||||
|
||||
// Write the script file into the stdin of the Python process.
|
||||
// Only write n - 1 bytes, because Python 3.11 does not allow null
|
||||
// bytes in source code anymore, so avoid writing the null string
|
||||
// terminator.
|
||||
// See https://github.com/python/cpython/issues/96670
|
||||
auto kBrowsePyLength = sizeof(kBrowsePy) - 1;
|
||||
ssize_t len = write(pipefd[1], kBrowsePy, kBrowsePyLength);
|
||||
if (len < (ssize_t)kBrowsePyLength)
|
||||
perror("ninja: write");
|
||||
close(pipefd[1]);
|
||||
exit(0);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,28 @@
|
||||
// Copyright 2011 Google Inc. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#ifndef NINJA_BROWSE_H_
|
||||
#define NINJA_BROWSE_H_
|
||||
|
||||
struct State;
|
||||
|
||||
/// Run in "browse" mode, which execs a Python webserver.
|
||||
/// \a ninja_command is the command used to invoke ninja.
|
||||
/// \a args are the number of arguments to be passed to the Python script.
|
||||
/// \a argv are arguments to be passed to the Python script.
|
||||
/// This function does not return if it runs successfully.
|
||||
void RunBrowsePython(State* state, const char* ninja_command,
|
||||
const char* input_file, int argc, char* argv[]);
|
||||
|
||||
#endif // NINJA_BROWSE_H_
|
||||
+232
@@ -0,0 +1,232 @@
|
||||
#!/usr/bin/env python3
|
||||
#
|
||||
# Copyright 2001 Google Inc. All Rights Reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
"""Simple web server for browsing dependency graph data.
|
||||
|
||||
This script is inlined into the final executable and spawned by
|
||||
it when needed.
|
||||
"""
|
||||
|
||||
try:
|
||||
import http.server as httpserver
|
||||
import socketserver
|
||||
except ImportError:
|
||||
import BaseHTTPServer as httpserver # type: ignore # Name "httpserver" already defined
|
||||
import SocketServer as socketserver # type: ignore # Name "socketserver" already defined
|
||||
import argparse
|
||||
import os
|
||||
import socket
|
||||
import subprocess
|
||||
import sys
|
||||
import webbrowser
|
||||
if sys.version_info >= (3, 2):
|
||||
from html import escape
|
||||
else:
|
||||
from cgi import escape
|
||||
try:
|
||||
from urllib.request import unquote # type: ignore # Module "urllib.request" has no attribute "unquote"
|
||||
except ImportError:
|
||||
from urllib2 import unquote
|
||||
from collections import namedtuple
|
||||
from typing import Tuple, Any
|
||||
|
||||
Node = namedtuple('Node', ['inputs', 'rule', 'target', 'outputs'])
|
||||
|
||||
# Ideally we'd allow you to navigate to a build edge or a build node,
|
||||
# with appropriate views for each. But there's no way to *name* a build
|
||||
# edge so we can only display nodes.
|
||||
#
|
||||
# For a given node, it has at most one input edge, which has n
|
||||
# different inputs. This becomes node.inputs. (We leave out the
|
||||
# outputs of the input edge due to what follows.) The node can have
|
||||
# multiple dependent output edges. Rather than attempting to display
|
||||
# those, they are summarized by taking the union of all their outputs.
|
||||
#
|
||||
# This means there's no single view that shows you all inputs and outputs
|
||||
# of an edge. But I think it's less confusing than alternatives.
|
||||
|
||||
def match_strip(line: str, prefix: str) -> Tuple[bool, str]:
|
||||
if not line.startswith(prefix):
|
||||
return (False, line)
|
||||
return (True, line[len(prefix):])
|
||||
|
||||
def html_escape(text: str) -> str:
|
||||
return escape(text, quote=True)
|
||||
|
||||
def parse(text: str) -> Node:
|
||||
lines = iter(text.split('\n'))
|
||||
|
||||
target = None
|
||||
rule = None
|
||||
inputs = []
|
||||
outputs = []
|
||||
|
||||
try:
|
||||
target = next(lines)[:-1] # strip trailing colon
|
||||
|
||||
line = next(lines)
|
||||
(match, rule) = match_strip(line, ' input: ')
|
||||
if match:
|
||||
(match, line) = match_strip(next(lines), ' ')
|
||||
while match:
|
||||
type = ""
|
||||
(match, line) = match_strip(line, '| ')
|
||||
if match:
|
||||
type = 'implicit'
|
||||
(match, line) = match_strip(line, '|| ')
|
||||
if match:
|
||||
type = 'order-only'
|
||||
inputs.append((line, type))
|
||||
(match, line) = match_strip(next(lines), ' ')
|
||||
|
||||
match, _ = match_strip(line, ' outputs:')
|
||||
if match:
|
||||
(match, line) = match_strip(next(lines), ' ')
|
||||
while match:
|
||||
outputs.append(line)
|
||||
(match, line) = match_strip(next(lines), ' ')
|
||||
except StopIteration:
|
||||
pass
|
||||
|
||||
return Node(inputs, rule, target, outputs)
|
||||
|
||||
def create_page(body: str) -> str:
|
||||
return '''<!DOCTYPE html>
|
||||
<style>
|
||||
body {
|
||||
font-family: sans;
|
||||
font-size: 0.8em;
|
||||
margin: 4ex;
|
||||
}
|
||||
h1 {
|
||||
font-weight: normal;
|
||||
font-size: 140%;
|
||||
text-align: center;
|
||||
margin: 0;
|
||||
}
|
||||
h2 {
|
||||
font-weight: normal;
|
||||
font-size: 120%;
|
||||
}
|
||||
tt {
|
||||
font-family: WebKitHack, monospace;
|
||||
white-space: nowrap;
|
||||
}
|
||||
.filelist {
|
||||
-webkit-columns: auto 2;
|
||||
}
|
||||
</style>
|
||||
''' + body
|
||||
|
||||
def generate_html(node: Node) -> str:
|
||||
document = ['<h1><tt>%s</tt></h1>' % html_escape(node.target)]
|
||||
|
||||
if node.inputs:
|
||||
document.append('<h2>target is built using rule <tt>%s</tt> of</h2>' %
|
||||
html_escape(node.rule))
|
||||
if len(node.inputs) > 0:
|
||||
document.append('<div class=filelist>')
|
||||
for input, type in sorted(node.inputs):
|
||||
extra = ''
|
||||
if type:
|
||||
extra = ' (%s)' % html_escape(type)
|
||||
document.append('<tt><a href="?%s">%s</a>%s</tt><br>' %
|
||||
(html_escape(input), html_escape(input), extra))
|
||||
document.append('</div>')
|
||||
|
||||
if node.outputs:
|
||||
document.append('<h2>dependent edges build:</h2>')
|
||||
document.append('<div class=filelist>')
|
||||
for output in sorted(node.outputs):
|
||||
document.append('<tt><a href="?%s">%s</a></tt><br>' %
|
||||
(html_escape(output), html_escape(output)))
|
||||
document.append('</div>')
|
||||
|
||||
return '\n'.join(document)
|
||||
|
||||
def ninja_dump(target: str) -> Tuple[str, str, int]:
|
||||
cmd = [args.ninja_command, '-f', args.f, '-t', 'query', target]
|
||||
proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE,
|
||||
universal_newlines=True)
|
||||
return proc.communicate() + (proc.returncode,)
|
||||
|
||||
class RequestHandler(httpserver.BaseHTTPRequestHandler):
|
||||
def do_GET(self) -> None:
|
||||
assert self.path[0] == '/'
|
||||
target = unquote(self.path[1:])
|
||||
|
||||
if target == '':
|
||||
self.send_response(302)
|
||||
self.send_header('Location', '?' + args.initial_target)
|
||||
self.end_headers()
|
||||
return
|
||||
|
||||
if not target.startswith('?'):
|
||||
self.send_response(404)
|
||||
self.end_headers()
|
||||
return
|
||||
target = target[1:]
|
||||
|
||||
ninja_output, ninja_error, exit_code = ninja_dump(target)
|
||||
if exit_code == 0:
|
||||
page_body = generate_html(parse(ninja_output.strip()))
|
||||
else:
|
||||
# Relay ninja's error message.
|
||||
page_body = '<h1><tt>%s</tt></h1>' % html_escape(ninja_error)
|
||||
|
||||
self.send_response(200)
|
||||
self.end_headers()
|
||||
self.wfile.write(create_page(page_body).encode('utf-8'))
|
||||
|
||||
def log_message(self, format: str, *args: Any) -> None:
|
||||
pass # Swallow console spam.
|
||||
|
||||
parser = argparse.ArgumentParser(prog='ninja -t browse')
|
||||
parser.add_argument('--port', '-p', default=8000, type=int,
|
||||
help='Port number to use (default %(default)d)')
|
||||
parser.add_argument('--hostname', '-a', default='localhost', type=str,
|
||||
help='Hostname to bind to (default %(default)s)')
|
||||
parser.add_argument('--no-browser', action='store_true',
|
||||
help='Do not open a webbrowser on startup.')
|
||||
|
||||
parser.add_argument('--ninja-command', default='ninja',
|
||||
help='Path to ninja binary (default %(default)s)')
|
||||
parser.add_argument('-f', default='build.ninja',
|
||||
help='Path to build.ninja file (default %(default)s)')
|
||||
parser.add_argument('initial_target', default='all', nargs='?',
|
||||
help='Initial target to show (default %(default)s)')
|
||||
|
||||
class HTTPServer(socketserver.ThreadingMixIn, httpserver.HTTPServer):
|
||||
# terminate server immediately when Python exits.
|
||||
daemon_threads = True
|
||||
|
||||
args = parser.parse_args()
|
||||
port = args.port
|
||||
hostname = args.hostname
|
||||
httpd = HTTPServer((hostname,port), RequestHandler)
|
||||
try:
|
||||
if hostname == "":
|
||||
hostname = socket.gethostname()
|
||||
print('Web server running on %s:%d, ctl-C to abort...' % (hostname,port) )
|
||||
print('Web server pid %d' % os.getpid(), file=sys.stderr )
|
||||
if not args.no_browser:
|
||||
webbrowser.open_new('http://%s:%s' % (hostname, port) )
|
||||
httpd.serve_forever()
|
||||
except KeyboardInterrupt:
|
||||
print()
|
||||
pass # Swallow console spam.
|
||||
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,278 @@
|
||||
// Copyright 2011 Google Inc. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#ifndef NINJA_BUILD_H_
|
||||
#define NINJA_BUILD_H_
|
||||
|
||||
#include <cstdio>
|
||||
#include <map>
|
||||
#include <memory>
|
||||
#include <string>
|
||||
#include <vector>
|
||||
|
||||
#include "depfile_parser.h"
|
||||
#include "exit_status.h"
|
||||
#include "graph.h"
|
||||
#include "jobserver.h"
|
||||
#include "util.h" // int64_t
|
||||
|
||||
struct BuildLog;
|
||||
struct Builder;
|
||||
struct DiskInterface;
|
||||
struct Edge;
|
||||
struct Explanations;
|
||||
struct Node;
|
||||
struct State;
|
||||
struct Status;
|
||||
|
||||
/// Plan stores the state of a build plan: what we intend to build,
|
||||
/// which steps we're ready to execute.
|
||||
struct Plan {
|
||||
Plan(Builder* builder = NULL);
|
||||
|
||||
/// Add a target to our plan (including all its dependencies).
|
||||
/// Returns false if we don't need to build this target; may
|
||||
/// fill in |err| with an error message if there's a problem.
|
||||
bool AddTarget(const Node* target, std::string* err);
|
||||
|
||||
// Pop a ready edge off the queue of edges to build.
|
||||
// Returns NULL if there's no work to do.
|
||||
Edge* FindWork();
|
||||
|
||||
/// Returns true if there's more work to be done.
|
||||
bool more_to_do() const { return wanted_edges_ > 0 && command_edges_ > 0; }
|
||||
|
||||
/// Dumps the current state of the plan.
|
||||
void Dump() const;
|
||||
|
||||
enum EdgeResult {
|
||||
kEdgeFailed,
|
||||
kEdgeSucceeded
|
||||
};
|
||||
|
||||
/// Mark an edge as done building (whether it succeeded or failed).
|
||||
/// If any of the edge's outputs are dyndep bindings of their dependents,
|
||||
/// this loads dynamic dependencies from the nodes' paths.
|
||||
/// Returns 'false' if loading dyndep info fails and 'true' otherwise.
|
||||
bool EdgeFinished(Edge* edge, EdgeResult result, std::string* err);
|
||||
|
||||
/// Clean the given node during the build.
|
||||
/// Return false on error.
|
||||
bool CleanNode(DependencyScan* scan, Node* node, std::string* err);
|
||||
|
||||
/// Number of edges with commands to run.
|
||||
int command_edge_count() const { return command_edges_; }
|
||||
|
||||
/// Reset state. Clears want and ready sets.
|
||||
void Reset();
|
||||
|
||||
// After all targets have been added, prepares the ready queue for find work.
|
||||
void PrepareQueue();
|
||||
|
||||
/// Update the build plan to account for modifications made to the graph
|
||||
/// by information loaded from a dyndep file.
|
||||
bool DyndepsLoaded(DependencyScan* scan, const Node* node,
|
||||
const DyndepFile& ddf, std::string* err);
|
||||
|
||||
/// Enumerate possible steps we want for an edge.
|
||||
enum Want
|
||||
{
|
||||
/// We do not want to build the edge, but we might want to build one of
|
||||
/// its dependents.
|
||||
kWantNothing,
|
||||
/// We want to build the edge, but have not yet scheduled it.
|
||||
kWantToStart,
|
||||
/// We want to build the edge, have scheduled it, and are waiting
|
||||
/// for it to complete.
|
||||
kWantToFinish
|
||||
};
|
||||
|
||||
private:
|
||||
void ComputeCriticalPath();
|
||||
bool RefreshDyndepDependents(DependencyScan* scan, const Node* node, std::string* err);
|
||||
void UnmarkDependents(const Node* node, std::set<Node*>* dependents);
|
||||
bool AddSubTarget(const Node* node, const Node* dependent, std::string* err,
|
||||
std::set<Edge*>* dyndep_walk);
|
||||
|
||||
// Add edges that kWantToStart into the ready queue
|
||||
// Must be called after ComputeCriticalPath and before FindWork
|
||||
void ScheduleInitialEdges();
|
||||
|
||||
/// Update plan with knowledge that the given node is up to date.
|
||||
/// If the node is a dyndep binding on any of its dependents, this
|
||||
/// loads dynamic dependencies from the node's path.
|
||||
/// Returns 'false' if loading dyndep info fails and 'true' otherwise.
|
||||
bool NodeFinished(Node* node, std::string* err);
|
||||
|
||||
void EdgeWanted(const Edge* edge);
|
||||
bool EdgeMaybeReady(std::map<Edge*, Want>::iterator want_e, std::string* err);
|
||||
|
||||
/// Submits a ready edge as a candidate for execution.
|
||||
/// The edge may be delayed from running, for example if it's a member of a
|
||||
/// currently-full pool.
|
||||
void ScheduleWork(std::map<Edge*, Want>::iterator want_e);
|
||||
|
||||
/// Keep track of which edges we want to build in this plan. If this map does
|
||||
/// not contain an entry for an edge, we do not want to build the entry or its
|
||||
/// dependents. If it does contain an entry, the enumeration indicates what
|
||||
/// we want for the edge.
|
||||
std::map<Edge*, Want> want_;
|
||||
|
||||
EdgePriorityQueue ready_;
|
||||
|
||||
Builder* builder_;
|
||||
/// user provided targets in build order, earlier one have higher priority
|
||||
std::vector<const Node*> targets_;
|
||||
|
||||
/// Total number of edges that have commands (not phony).
|
||||
int command_edges_;
|
||||
|
||||
/// Total remaining number of wanted edges.
|
||||
int wanted_edges_;
|
||||
};
|
||||
|
||||
struct BuildConfig;
|
||||
|
||||
/// CommandRunner is an interface that wraps running the build
|
||||
/// subcommands. This allows tests to abstract out running commands.
|
||||
/// RealCommandRunner is an implementation that actually runs commands.
|
||||
struct CommandRunner {
|
||||
virtual ~CommandRunner() {}
|
||||
virtual size_t CanRunMore() const = 0;
|
||||
virtual bool StartCommand(Edge* edge) = 0;
|
||||
|
||||
/// The result of waiting for a command.
|
||||
struct Result {
|
||||
Result() : edge(NULL) {}
|
||||
Edge* edge;
|
||||
ExitStatus status;
|
||||
std::string output;
|
||||
bool success() const { return status == ExitSuccess; }
|
||||
};
|
||||
/// Wait for a command to complete, or return false if interrupted.
|
||||
virtual bool WaitForCommand(Result* result) = 0;
|
||||
|
||||
virtual std::vector<Edge*> GetActiveEdges() { return std::vector<Edge*>(); }
|
||||
virtual void Abort() {}
|
||||
|
||||
/// Creates the RealCommandRunner. \arg jobserver can be nullptr if there
|
||||
/// is no jobserver pool to use.
|
||||
static CommandRunner* factory(const BuildConfig& config,
|
||||
Jobserver::Client* jobserver);
|
||||
};
|
||||
|
||||
/// Options (e.g. verbosity, parallelism) passed to a build.
|
||||
struct BuildConfig {
|
||||
BuildConfig() = default;
|
||||
|
||||
enum Verbosity {
|
||||
QUIET, // No output -- used when testing.
|
||||
NO_STATUS_UPDATE, // just regular output but suppress status update
|
||||
NORMAL, // regular output and status update
|
||||
VERBOSE
|
||||
};
|
||||
Verbosity verbosity = NORMAL;
|
||||
bool dry_run = false;
|
||||
int parallelism = 1;
|
||||
bool disable_jobserver_client = false;
|
||||
int failures_allowed = 1;
|
||||
/// The maximum load average we must not exceed. A negative value
|
||||
/// means that we do not have any limit.
|
||||
double max_load_average = -0.0f;
|
||||
DepfileParserOptions depfile_parser_options;
|
||||
};
|
||||
|
||||
/// Builder wraps the build process: starting commands, updating status.
|
||||
struct Builder {
|
||||
Builder(State* state, const BuildConfig& config, BuildLog* build_log,
|
||||
DepsLog* deps_log, DiskInterface* disk_interface, Status* status,
|
||||
int64_t start_time_millis);
|
||||
~Builder();
|
||||
|
||||
/// Set Jobserver client instance for this builder.
|
||||
void SetJobserverClient(std::unique_ptr<Jobserver::Client> jobserver_client) {
|
||||
jobserver_ = std::move(jobserver_client);
|
||||
}
|
||||
|
||||
/// Clean up after interrupted commands by deleting output files.
|
||||
void Cleanup();
|
||||
|
||||
Node* AddTarget(const std::string& name, std::string* err);
|
||||
|
||||
/// Add a target to the build, scanning dependencies.
|
||||
/// @return false on error.
|
||||
bool AddTarget(Node* target, std::string* err);
|
||||
|
||||
/// Returns true if the build targets are already up to date.
|
||||
bool AlreadyUpToDate() const;
|
||||
|
||||
/// Run the build. Returns ExitStatus or the exit code of the last failed job.
|
||||
/// It is an error to call this function when AlreadyUpToDate() is true.
|
||||
ExitStatus Build(std::string* err);
|
||||
|
||||
bool StartEdge(Edge* edge, std::string* err);
|
||||
|
||||
/// Update status ninja logs following a command termination.
|
||||
/// @return false if the build can not proceed further due to a fatal error.
|
||||
bool FinishCommand(CommandRunner::Result* result, std::string* err);
|
||||
|
||||
/// Used for tests.
|
||||
void SetBuildLog(BuildLog* log) {
|
||||
scan_.set_build_log(log);
|
||||
}
|
||||
|
||||
/// Load the dyndep information provided by the given node.
|
||||
bool LoadDyndeps(Node* node, std::string* err);
|
||||
|
||||
State* state_;
|
||||
const BuildConfig& config_;
|
||||
Plan plan_;
|
||||
std::unique_ptr<Jobserver::Client> jobserver_;
|
||||
std::unique_ptr<CommandRunner> command_runner_;
|
||||
Status* status_;
|
||||
|
||||
/// Returns ExitStatus or the exit code of the last failed job
|
||||
/// (doesn't need to be an enum value of ExitStatus)
|
||||
ExitStatus GetExitCode() const { return exit_code_; }
|
||||
|
||||
private:
|
||||
bool ExtractDeps(CommandRunner::Result* result, const std::string& deps_type,
|
||||
const std::string& deps_prefix,
|
||||
std::vector<Node*>* deps_nodes, std::string* err);
|
||||
|
||||
/// Map of running edge to time the edge started running.
|
||||
typedef std::map<const Edge*, int> RunningEdgeMap;
|
||||
RunningEdgeMap running_edges_;
|
||||
|
||||
/// Time the build started.
|
||||
int64_t start_time_millis_;
|
||||
|
||||
std::string lock_file_path_;
|
||||
DiskInterface* disk_interface_;
|
||||
|
||||
// Only create an Explanations class if '-d explain' is used.
|
||||
std::unique_ptr<Explanations> explanations_;
|
||||
|
||||
DependencyScan scan_;
|
||||
|
||||
/// Keep the global exit code for the build
|
||||
ExitStatus exit_code_ = ExitSuccess;
|
||||
void SetFailureCode(ExitStatus code);
|
||||
|
||||
// Unimplemented copy ctor and operator= ensure we don't copy the auto_ptr.
|
||||
Builder(const Builder &other); // DO NOT IMPLEMENT
|
||||
void operator=(const Builder &other); // DO NOT IMPLEMENT
|
||||
};
|
||||
|
||||
#endif // NINJA_BUILD_H_
|
||||
@@ -0,0 +1,444 @@
|
||||
// Copyright 2011 Google Inc. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
// On AIX, inttypes.h gets indirectly included by build_log.h.
|
||||
// It's easiest just to ask for the printf format macros right away.
|
||||
#ifndef _WIN32
|
||||
#ifndef __STDC_FORMAT_MACROS
|
||||
#define __STDC_FORMAT_MACROS
|
||||
#endif
|
||||
#endif
|
||||
|
||||
#include "build_log.h"
|
||||
#include "disk_interface.h"
|
||||
|
||||
#include <cassert>
|
||||
#include <errno.h>
|
||||
#include <stdlib.h>
|
||||
#include <string.h>
|
||||
|
||||
#ifndef _WIN32
|
||||
#include <inttypes.h>
|
||||
#include <unistd.h>
|
||||
#endif
|
||||
|
||||
#include "build.h"
|
||||
#include "graph.h"
|
||||
#include "metrics.h"
|
||||
#include "util.h"
|
||||
#if defined(_MSC_VER) && (_MSC_VER < 1800)
|
||||
#define strtoll _strtoi64
|
||||
#endif
|
||||
|
||||
// Implementation details:
|
||||
// Each run's log appends to the log file.
|
||||
// To load, we run through all log entries in series, throwing away
|
||||
// older runs.
|
||||
// Once the number of redundant entries exceeds a threshold, we write
|
||||
// out a new file and replace the existing one with it.
|
||||
|
||||
namespace {
|
||||
|
||||
const char kFileSignature[] = "# ninja log v%d\n";
|
||||
const int kOldestSupportedVersion = 7;
|
||||
const int kCurrentVersion = 7;
|
||||
|
||||
} // namespace
|
||||
|
||||
// static
|
||||
uint64_t BuildLog::LogEntry::HashCommand(StringPiece command) {
|
||||
return rapidhash(command.str_, command.len_);
|
||||
}
|
||||
|
||||
BuildLog::LogEntry::LogEntry(std::string output) : output(std::move(output)) {}
|
||||
|
||||
BuildLog::LogEntry::LogEntry(const std::string& output, uint64_t command_hash,
|
||||
int start_time, int end_time, TimeStamp mtime)
|
||||
: output(output), command_hash(command_hash), start_time(start_time),
|
||||
end_time(end_time), mtime(mtime) {}
|
||||
|
||||
BuildLog::BuildLog() = default;
|
||||
|
||||
BuildLog::~BuildLog() {
|
||||
Close();
|
||||
}
|
||||
|
||||
bool BuildLog::OpenForWrite(const std::string& path, const BuildLogUser& user,
|
||||
std::string* err) {
|
||||
if (needs_recompaction_) {
|
||||
if (!Recompact(path, user, err))
|
||||
return false;
|
||||
}
|
||||
|
||||
assert(!log_file_);
|
||||
log_file_path_ = path; // we don't actually open the file right now, but will
|
||||
// do so on the first write attempt
|
||||
return true;
|
||||
}
|
||||
|
||||
bool BuildLog::RecordCommand(Edge* edge, int start_time, int end_time,
|
||||
TimeStamp mtime) {
|
||||
std::string command = edge->EvaluateCommand(true);
|
||||
uint64_t command_hash = LogEntry::HashCommand(command);
|
||||
for (std::vector<Node*>::iterator out = edge->outputs_.begin();
|
||||
out != edge->outputs_.end(); ++out) {
|
||||
const std::string& path = (*out)->path();
|
||||
Entries::iterator i = entries_.find(path);
|
||||
LogEntry* log_entry;
|
||||
if (i != entries_.end()) {
|
||||
log_entry = i->second.get();
|
||||
} else {
|
||||
log_entry = new LogEntry(path);
|
||||
// Passes ownership of |log_entry| to the map, but keeps the pointer valid.
|
||||
entries_.emplace(log_entry->output, std::unique_ptr<LogEntry>(log_entry));
|
||||
}
|
||||
log_entry->command_hash = command_hash;
|
||||
log_entry->start_time = start_time;
|
||||
log_entry->end_time = end_time;
|
||||
log_entry->mtime = mtime;
|
||||
|
||||
if (!OpenForWriteIfNeeded()) {
|
||||
return false;
|
||||
}
|
||||
if (log_file_) {
|
||||
if (!WriteEntry(log_file_, *log_entry))
|
||||
return false;
|
||||
if (fflush(log_file_) != 0) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
void BuildLog::Close() {
|
||||
OpenForWriteIfNeeded(); // create the file even if nothing has been recorded
|
||||
if (log_file_)
|
||||
fclose(log_file_);
|
||||
log_file_ = NULL;
|
||||
}
|
||||
|
||||
bool BuildLog::OpenForWriteIfNeeded() {
|
||||
if (log_file_ || log_file_path_.empty()) {
|
||||
return true;
|
||||
}
|
||||
log_file_ = fopen(log_file_path_.c_str(), "ab");
|
||||
if (!log_file_) {
|
||||
return false;
|
||||
}
|
||||
if (setvbuf(log_file_, NULL, _IOLBF, BUFSIZ) != 0) {
|
||||
return false;
|
||||
}
|
||||
SetCloseOnExec(fileno(log_file_));
|
||||
|
||||
// Opening a file in append mode doesn't set the file pointer to the file's
|
||||
// end on Windows. Do that explicitly.
|
||||
fseek(log_file_, 0, SEEK_END);
|
||||
|
||||
if (ftell(log_file_) == 0) {
|
||||
if (fprintf(log_file_, kFileSignature, kCurrentVersion) < 0) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
struct LineReader {
|
||||
explicit LineReader(FILE* file)
|
||||
: file_(file), buf_end_(buf_), line_start_(buf_), line_end_(NULL) {
|
||||
memset(buf_, 0, sizeof(buf_));
|
||||
}
|
||||
|
||||
// Reads a \n-terminated line from the file passed to the constructor.
|
||||
// On return, *line_start points to the beginning of the next line, and
|
||||
// *line_end points to the \n at the end of the line. If no newline is seen
|
||||
// in a fixed buffer size, *line_end is set to NULL. Returns false on EOF.
|
||||
bool ReadLine(char** line_start, char** line_end) {
|
||||
if (line_start_ >= buf_end_ || !line_end_) {
|
||||
// Buffer empty, refill.
|
||||
size_t size_read = fread(buf_, 1, sizeof(buf_), file_);
|
||||
if (!size_read)
|
||||
return false;
|
||||
line_start_ = buf_;
|
||||
buf_end_ = buf_ + size_read;
|
||||
} else {
|
||||
// Advance to next line in buffer.
|
||||
line_start_ = line_end_ + 1;
|
||||
}
|
||||
|
||||
line_end_ = static_cast<char*>(memchr(line_start_, '\n', buf_end_ - line_start_));
|
||||
if (!line_end_) {
|
||||
// No newline. Move rest of data to start of buffer, fill rest.
|
||||
size_t already_consumed = line_start_ - buf_;
|
||||
size_t size_rest = (buf_end_ - buf_) - already_consumed;
|
||||
memmove(buf_, line_start_, size_rest);
|
||||
|
||||
size_t read = fread(buf_ + size_rest, 1, sizeof(buf_) - size_rest, file_);
|
||||
buf_end_ = buf_ + size_rest + read;
|
||||
line_start_ = buf_;
|
||||
line_end_ = static_cast<char*>(memchr(line_start_, '\n', buf_end_ - line_start_));
|
||||
}
|
||||
|
||||
*line_start = line_start_;
|
||||
*line_end = line_end_;
|
||||
return true;
|
||||
}
|
||||
|
||||
private:
|
||||
FILE* file_;
|
||||
char buf_[256 << 10];
|
||||
char* buf_end_; // Points one past the last valid byte in |buf_|.
|
||||
|
||||
char* line_start_;
|
||||
// Points at the next \n in buf_ after line_start, or NULL.
|
||||
char* line_end_;
|
||||
};
|
||||
|
||||
LoadStatus BuildLog::Load(const std::string& path, std::string* err) {
|
||||
METRIC_RECORD(".ninja_log load");
|
||||
FILE* file = fopen(path.c_str(), "r");
|
||||
if (!file) {
|
||||
if (errno == ENOENT)
|
||||
return LOAD_NOT_FOUND;
|
||||
*err = strerror(errno);
|
||||
return LOAD_ERROR;
|
||||
}
|
||||
|
||||
int log_version = 0;
|
||||
int unique_entry_count = 0;
|
||||
int total_entry_count = 0;
|
||||
|
||||
LineReader reader(file);
|
||||
char* line_start = 0;
|
||||
char* line_end = 0;
|
||||
while (reader.ReadLine(&line_start, &line_end)) {
|
||||
if (!log_version) {
|
||||
sscanf(line_start, kFileSignature, &log_version);
|
||||
|
||||
bool invalid_log_version = false;
|
||||
if (log_version < kOldestSupportedVersion) {
|
||||
invalid_log_version = true;
|
||||
*err = "build log version is too old; starting over";
|
||||
|
||||
} else if (log_version > kCurrentVersion) {
|
||||
invalid_log_version = true;
|
||||
*err = "build log version is too new; starting over";
|
||||
}
|
||||
if (invalid_log_version) {
|
||||
fclose(file);
|
||||
platformAwareUnlink(path.c_str());
|
||||
// Don't report this as a failure. A missing build log will cause
|
||||
// us to rebuild the outputs anyway.
|
||||
return LOAD_NOT_FOUND;
|
||||
}
|
||||
}
|
||||
|
||||
// If no newline was found in this chunk, read the next.
|
||||
if (!line_end)
|
||||
continue;
|
||||
|
||||
const char kFieldSeparator = '\t';
|
||||
|
||||
char* start = line_start;
|
||||
char* end = static_cast<char*>(memchr(start, kFieldSeparator, line_end - start));
|
||||
if (!end)
|
||||
continue;
|
||||
*end = 0;
|
||||
|
||||
int start_time = 0, end_time = 0;
|
||||
TimeStamp mtime = 0;
|
||||
|
||||
start_time = atoi(start);
|
||||
start = end + 1;
|
||||
|
||||
end = static_cast<char*>(memchr(start, kFieldSeparator, line_end - start));
|
||||
if (!end)
|
||||
continue;
|
||||
*end = 0;
|
||||
end_time = atoi(start);
|
||||
start = end + 1;
|
||||
|
||||
end = static_cast<char*>(memchr(start, kFieldSeparator, line_end - start));
|
||||
if (!end)
|
||||
continue;
|
||||
*end = 0;
|
||||
mtime = strtoll(start, NULL, 10);
|
||||
start = end + 1;
|
||||
|
||||
end = static_cast<char*>(memchr(start, kFieldSeparator, line_end - start));
|
||||
if (!end)
|
||||
continue;
|
||||
std::string output(start, end - start);
|
||||
|
||||
start = end + 1;
|
||||
end = line_end;
|
||||
|
||||
LogEntry* entry;
|
||||
Entries::iterator i = entries_.find(output);
|
||||
if (i != entries_.end()) {
|
||||
entry = i->second.get();
|
||||
} else {
|
||||
entry = new LogEntry(std::move(output));
|
||||
// Passes ownership of |entry| to the map, but keeps the pointer valid.
|
||||
entries_.emplace(entry->output, std::unique_ptr<LogEntry>(entry));
|
||||
++unique_entry_count;
|
||||
}
|
||||
++total_entry_count;
|
||||
|
||||
entry->start_time = start_time;
|
||||
entry->end_time = end_time;
|
||||
entry->mtime = mtime;
|
||||
char c = *end; *end = '\0';
|
||||
entry->command_hash = (uint64_t)strtoull(start, NULL, 16);
|
||||
*end = c;
|
||||
}
|
||||
fclose(file);
|
||||
|
||||
if (!line_start) {
|
||||
return LOAD_SUCCESS; // file was empty
|
||||
}
|
||||
|
||||
// Decide whether it's time to rebuild the log:
|
||||
// - if we're upgrading versions
|
||||
// - if it's getting large
|
||||
int kMinCompactionEntryCount = 100;
|
||||
int kCompactionRatio = 3;
|
||||
if (log_version < kCurrentVersion) {
|
||||
needs_recompaction_ = true;
|
||||
} else if (total_entry_count > kMinCompactionEntryCount &&
|
||||
total_entry_count > unique_entry_count * kCompactionRatio) {
|
||||
needs_recompaction_ = true;
|
||||
}
|
||||
|
||||
return LOAD_SUCCESS;
|
||||
}
|
||||
|
||||
BuildLog::LogEntry* BuildLog::LookupByOutput(const std::string& path) {
|
||||
Entries::iterator i = entries_.find(path);
|
||||
if (i != entries_.end())
|
||||
return i->second.get();
|
||||
return NULL;
|
||||
}
|
||||
|
||||
bool BuildLog::WriteEntry(FILE* f, const LogEntry& entry) {
|
||||
return fprintf(f, "%d\t%d\t%" PRId64 "\t%s\t%" PRIx64 "\n",
|
||||
entry.start_time, entry.end_time, entry.mtime,
|
||||
entry.output.c_str(), entry.command_hash) > 0;
|
||||
}
|
||||
|
||||
bool BuildLog::Recompact(const std::string& path, const BuildLogUser& user,
|
||||
std::string* err) {
|
||||
METRIC_RECORD(".ninja_log recompact");
|
||||
|
||||
Close();
|
||||
std::string temp_path = path + ".recompact";
|
||||
FILE* f = fopen(temp_path.c_str(), "wb");
|
||||
if (!f) {
|
||||
*err = strerror(errno);
|
||||
return false;
|
||||
}
|
||||
|
||||
if (fprintf(f, kFileSignature, kCurrentVersion) < 0) {
|
||||
*err = strerror(errno);
|
||||
fclose(f);
|
||||
return false;
|
||||
}
|
||||
|
||||
std::vector<StringPiece> dead_outputs;
|
||||
for (const auto& pair : entries_) {
|
||||
if (user.IsPathDead(pair.first)) {
|
||||
dead_outputs.push_back(pair.first);
|
||||
continue;
|
||||
}
|
||||
|
||||
if (!WriteEntry(f, *pair.second)) {
|
||||
*err = strerror(errno);
|
||||
fclose(f);
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
for (StringPiece output : dead_outputs)
|
||||
entries_.erase(output);
|
||||
|
||||
fclose(f);
|
||||
if (platformAwareUnlink(path.c_str()) < 0) {
|
||||
*err = strerror(errno);
|
||||
return false;
|
||||
}
|
||||
|
||||
if (rename(temp_path.c_str(), path.c_str()) < 0) {
|
||||
*err = strerror(errno);
|
||||
return false;
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
bool BuildLog::Restat(const StringPiece path,
|
||||
const DiskInterface& disk_interface,
|
||||
const int output_count, char** outputs,
|
||||
std::string* const err) {
|
||||
METRIC_RECORD(".ninja_log restat");
|
||||
|
||||
Close();
|
||||
std::string temp_path = path.AsString() + ".restat";
|
||||
FILE* f = fopen(temp_path.c_str(), "wb");
|
||||
if (!f) {
|
||||
*err = strerror(errno);
|
||||
return false;
|
||||
}
|
||||
|
||||
if (fprintf(f, kFileSignature, kCurrentVersion) < 0) {
|
||||
*err = strerror(errno);
|
||||
fclose(f);
|
||||
return false;
|
||||
}
|
||||
for (auto& pair : entries_) {
|
||||
bool skip = output_count > 0;
|
||||
for (int j = 0; j < output_count; ++j) {
|
||||
if (pair.second->output == outputs[j]) {
|
||||
skip = false;
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (!skip) {
|
||||
const TimeStamp mtime = disk_interface.Stat(pair.second->output, err);
|
||||
if (mtime == -1) {
|
||||
fclose(f);
|
||||
return false;
|
||||
}
|
||||
pair.second->mtime = mtime;
|
||||
}
|
||||
|
||||
if (!WriteEntry(f, *pair.second)) {
|
||||
*err = strerror(errno);
|
||||
fclose(f);
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
fclose(f);
|
||||
if (platformAwareUnlink(path.str_) < 0) {
|
||||
*err = strerror(errno);
|
||||
return false;
|
||||
}
|
||||
|
||||
if (rename(temp_path.c_str(), path.str_) < 0) {
|
||||
*err = strerror(errno);
|
||||
return false;
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
@@ -0,0 +1,109 @@
|
||||
// Copyright 2011 Google Inc. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#ifndef NINJA_BUILD_LOG_H_
|
||||
#define NINJA_BUILD_LOG_H_
|
||||
|
||||
#include <stdio.h>
|
||||
|
||||
#include <memory>
|
||||
#include <string>
|
||||
|
||||
#include "hash_map.h"
|
||||
#include "load_status.h"
|
||||
#include "timestamp.h"
|
||||
#include "util.h" // uint64_t
|
||||
|
||||
struct DiskInterface;
|
||||
struct Edge;
|
||||
|
||||
/// Can answer questions about the manifest for the BuildLog.
|
||||
struct BuildLogUser {
|
||||
/// Return if a given output is no longer part of the build manifest.
|
||||
/// This is only called during recompaction and doesn't have to be fast.
|
||||
virtual bool IsPathDead(StringPiece s) const = 0;
|
||||
};
|
||||
|
||||
/// Store a log of every command ran for every build.
|
||||
/// It has a few uses:
|
||||
///
|
||||
/// 1) (hashes of) command lines for existing output files, so we know
|
||||
/// when we need to rebuild due to the command changing
|
||||
/// 2) timing information, perhaps for generating reports
|
||||
/// 3) restat information
|
||||
struct BuildLog {
|
||||
BuildLog();
|
||||
~BuildLog();
|
||||
|
||||
/// Prepares writing to the log file without actually opening it - that will
|
||||
/// happen when/if it's needed
|
||||
bool OpenForWrite(const std::string& path, const BuildLogUser& user,
|
||||
std::string* err);
|
||||
bool RecordCommand(Edge* edge, int start_time, int end_time,
|
||||
TimeStamp mtime = 0);
|
||||
void Close();
|
||||
|
||||
/// Load the on-disk log.
|
||||
LoadStatus Load(const std::string& path, std::string* err);
|
||||
|
||||
struct LogEntry {
|
||||
std::string output;
|
||||
uint64_t command_hash = 0;
|
||||
int start_time = 0;
|
||||
int end_time = 0;
|
||||
TimeStamp mtime = 0;
|
||||
|
||||
static uint64_t HashCommand(StringPiece command);
|
||||
|
||||
// Used by tests.
|
||||
bool operator==(const LogEntry& o) const {
|
||||
return output == o.output && command_hash == o.command_hash &&
|
||||
start_time == o.start_time && end_time == o.end_time &&
|
||||
mtime == o.mtime;
|
||||
}
|
||||
|
||||
explicit LogEntry(std::string output);
|
||||
LogEntry(const std::string& output, uint64_t command_hash,
|
||||
int start_time, int end_time, TimeStamp mtime);
|
||||
};
|
||||
|
||||
/// Lookup a previously-run command by its output path.
|
||||
LogEntry* LookupByOutput(const std::string& path);
|
||||
|
||||
/// Serialize an entry into a log file.
|
||||
bool WriteEntry(FILE* f, const LogEntry& entry);
|
||||
|
||||
/// Rewrite the known log entries, throwing away old data.
|
||||
bool Recompact(const std::string& path, const BuildLogUser& user,
|
||||
std::string* err);
|
||||
|
||||
/// Restat all outputs in the log
|
||||
bool Restat(StringPiece path, const DiskInterface& disk_interface,
|
||||
int output_count, char** outputs, std::string* err);
|
||||
|
||||
typedef ExternalStringHashMap<std::unique_ptr<LogEntry>>::Type Entries;
|
||||
const Entries& entries() const { return entries_; }
|
||||
|
||||
private:
|
||||
/// Should be called before using log_file_. When false is returned, errno
|
||||
/// will be set.
|
||||
bool OpenForWriteIfNeeded();
|
||||
|
||||
Entries entries_;
|
||||
FILE* log_file_ = nullptr;
|
||||
std::string log_file_path_;
|
||||
bool needs_recompaction_ = false;
|
||||
};
|
||||
|
||||
#endif // NINJA_BUILD_LOG_H_
|
||||
@@ -0,0 +1,150 @@
|
||||
// Copyright 2012 Google Inc. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#include <stdio.h>
|
||||
#include <stdlib.h>
|
||||
|
||||
#include "build_log.h"
|
||||
#include "graph.h"
|
||||
#include "manifest_parser.h"
|
||||
#include "state.h"
|
||||
#include "util.h"
|
||||
#include "metrics.h"
|
||||
|
||||
#ifndef _WIN32
|
||||
#include <unistd.h>
|
||||
#endif
|
||||
|
||||
using namespace std;
|
||||
|
||||
const char kTestFilename[] = "BuildLogPerfTest-tempfile";
|
||||
|
||||
struct NoDeadPaths : public BuildLogUser {
|
||||
virtual bool IsPathDead(StringPiece) const { return false; }
|
||||
};
|
||||
|
||||
bool WriteTestData(string* err) {
|
||||
BuildLog log;
|
||||
|
||||
NoDeadPaths no_dead_paths;
|
||||
if (!log.OpenForWrite(kTestFilename, no_dead_paths, err))
|
||||
return false;
|
||||
|
||||
/*
|
||||
A histogram of command lengths in chromium. For example, 407 builds,
|
||||
1.4% of all builds, had commands longer than 32 bytes but shorter than 64.
|
||||
32 407 1.4%
|
||||
64 183 0.6%
|
||||
128 1461 5.1%
|
||||
256 791 2.8%
|
||||
512 1314 4.6%
|
||||
1024 6114 21.3%
|
||||
2048 11759 41.0%
|
||||
4096 2056 7.2%
|
||||
8192 4567 15.9%
|
||||
16384 13 0.0%
|
||||
32768 4 0.0%
|
||||
65536 5 0.0%
|
||||
The average command length is 4.1 kB and there were 28674 commands in total,
|
||||
which makes for a total log size of ~120 MB (also counting output filenames).
|
||||
|
||||
Based on this, write 30000 many 4 kB long command lines.
|
||||
*/
|
||||
|
||||
// ManifestParser is the only object allowed to create Rules.
|
||||
const size_t kRuleSize = 4000;
|
||||
string long_rule_command = "gcc ";
|
||||
for (int i = 0; long_rule_command.size() < kRuleSize; ++i) {
|
||||
char buf[80];
|
||||
sprintf(buf, "-I../../and/arbitrary/but/fairly/long/path/suffixed/%d ", i);
|
||||
long_rule_command += buf;
|
||||
}
|
||||
long_rule_command += "$in -o $out\n";
|
||||
|
||||
State state;
|
||||
ManifestParser parser(&state, NULL);
|
||||
if (!parser.ParseTest("rule cxx\n command = " + long_rule_command, err))
|
||||
return false;
|
||||
|
||||
// Create build edges. Using ManifestParser is as fast as using the State api
|
||||
// for edge creation, so just use that.
|
||||
const int kNumCommands = 30000;
|
||||
string build_rules;
|
||||
for (int i = 0; i < kNumCommands; ++i) {
|
||||
char buf[80];
|
||||
sprintf(buf, "build input%d.o: cxx input%d.cc\n", i, i);
|
||||
build_rules += buf;
|
||||
}
|
||||
|
||||
if (!parser.ParseTest(build_rules, err))
|
||||
return false;
|
||||
|
||||
for (int i = 0; i < kNumCommands; ++i) {
|
||||
log.RecordCommand(state.edges_[i],
|
||||
/*start_time=*/100 * i,
|
||||
/*end_time=*/100 * i + 1,
|
||||
/*mtime=*/0);
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
int main() {
|
||||
vector<int> times;
|
||||
string err;
|
||||
|
||||
if (!WriteTestData(&err)) {
|
||||
fprintf(stderr, "Failed to write test data: %s\n", err.c_str());
|
||||
return 1;
|
||||
}
|
||||
|
||||
{
|
||||
// Read once to warm up disk cache.
|
||||
BuildLog log;
|
||||
if (log.Load(kTestFilename, &err) == LOAD_ERROR) {
|
||||
fprintf(stderr, "Failed to read test data: %s\n", err.c_str());
|
||||
return 1;
|
||||
}
|
||||
}
|
||||
const int kNumRepetitions = 5;
|
||||
for (int i = 0; i < kNumRepetitions; ++i) {
|
||||
int64_t start = GetTimeMillis();
|
||||
BuildLog log;
|
||||
if (log.Load(kTestFilename, &err) == LOAD_ERROR) {
|
||||
fprintf(stderr, "Failed to read test data: %s\n", err.c_str());
|
||||
return 1;
|
||||
}
|
||||
int delta = (int)(GetTimeMillis() - start);
|
||||
printf("%dms\n", delta);
|
||||
times.push_back(delta);
|
||||
}
|
||||
|
||||
int min = times[0];
|
||||
int max = times[0];
|
||||
float total = 0;
|
||||
for (size_t i = 0; i < times.size(); ++i) {
|
||||
total += times[i];
|
||||
if (times[i] < min)
|
||||
min = times[i];
|
||||
else if (times[i] > max)
|
||||
max = times[i];
|
||||
}
|
||||
|
||||
printf("min %dms max %dms avg %.1fms\n",
|
||||
min, max, total / times.size());
|
||||
|
||||
platformAwareUnlink(kTestFilename);
|
||||
|
||||
return 0;
|
||||
}
|
||||
@@ -0,0 +1,368 @@
|
||||
// Copyright 2011 Google Inc. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#include "build_log.h"
|
||||
|
||||
#include "util.h"
|
||||
#include "test.h"
|
||||
|
||||
#include <sys/stat.h>
|
||||
#ifdef _WIN32
|
||||
#include <fcntl.h>
|
||||
#include <share.h>
|
||||
#else
|
||||
#include <sys/types.h>
|
||||
#include <unistd.h>
|
||||
#endif
|
||||
#include <cassert>
|
||||
|
||||
namespace {
|
||||
|
||||
const char kTestFilename[] = "BuildLogTest-tempfile";
|
||||
|
||||
struct BuildLogTest : public StateTestWithBuiltinRules, public BuildLogUser {
|
||||
virtual void SetUp() {
|
||||
// In case a crashing test left a stale file behind.
|
||||
platformAwareUnlink(kTestFilename);
|
||||
}
|
||||
virtual void TearDown() {
|
||||
platformAwareUnlink(kTestFilename);
|
||||
}
|
||||
virtual bool IsPathDead(StringPiece s) const { return false; }
|
||||
};
|
||||
|
||||
TEST_F(BuildLogTest, WriteRead) {
|
||||
AssertParse(&state_,
|
||||
"build out: cat mid\n"
|
||||
"build mid: cat in\n");
|
||||
|
||||
BuildLog log1;
|
||||
std::string err;
|
||||
EXPECT_TRUE(log1.OpenForWrite(kTestFilename, *this, &err));
|
||||
ASSERT_EQ("", err);
|
||||
log1.RecordCommand(state_.edges_[0], 15, 18);
|
||||
log1.RecordCommand(state_.edges_[1], 20, 25);
|
||||
log1.Close();
|
||||
|
||||
BuildLog log2;
|
||||
EXPECT_TRUE(log2.Load(kTestFilename, &err));
|
||||
ASSERT_EQ("", err);
|
||||
|
||||
ASSERT_EQ(2u, log1.entries().size());
|
||||
ASSERT_EQ(2u, log2.entries().size());
|
||||
BuildLog::LogEntry* e1 = log1.LookupByOutput("out");
|
||||
ASSERT_TRUE(e1);
|
||||
BuildLog::LogEntry* e2 = log2.LookupByOutput("out");
|
||||
ASSERT_TRUE(e2);
|
||||
ASSERT_TRUE(*e1 == *e2);
|
||||
ASSERT_EQ(15, e1->start_time);
|
||||
ASSERT_EQ("out", e1->output);
|
||||
}
|
||||
|
||||
TEST_F(BuildLogTest, FirstWriteAddsSignature) {
|
||||
const char kExpectedVersion[] = "# ninja log vX\n";
|
||||
const size_t kVersionPos = strlen(kExpectedVersion) - 2; // Points at 'X'.
|
||||
|
||||
BuildLog log;
|
||||
std::string contents, err;
|
||||
|
||||
EXPECT_TRUE(log.OpenForWrite(kTestFilename, *this, &err));
|
||||
ASSERT_EQ("", err);
|
||||
log.Close();
|
||||
|
||||
ASSERT_EQ(0, ReadFile(kTestFilename, &contents, &err));
|
||||
ASSERT_EQ("", err);
|
||||
if (contents.size() >= kVersionPos)
|
||||
contents[kVersionPos] = 'X';
|
||||
EXPECT_EQ(kExpectedVersion, contents);
|
||||
|
||||
// Opening the file anew shouldn't add a second version string.
|
||||
EXPECT_TRUE(log.OpenForWrite(kTestFilename, *this, &err));
|
||||
ASSERT_EQ("", err);
|
||||
log.Close();
|
||||
|
||||
contents.clear();
|
||||
ASSERT_EQ(0, ReadFile(kTestFilename, &contents, &err));
|
||||
ASSERT_EQ("", err);
|
||||
if (contents.size() >= kVersionPos)
|
||||
contents[kVersionPos] = 'X';
|
||||
EXPECT_EQ(kExpectedVersion, contents);
|
||||
}
|
||||
|
||||
TEST_F(BuildLogTest, DoubleEntry) {
|
||||
FILE* f = fopen(kTestFilename, "wb");
|
||||
fprintf(f, "# ninja log v7\n");
|
||||
fprintf(f, "0\t1\t2\tout\t%" PRIx64 "\n",
|
||||
BuildLog::LogEntry::HashCommand("command abc"));
|
||||
fprintf(f, "0\t1\t2\tout\t%" PRIx64 "\n",
|
||||
BuildLog::LogEntry::HashCommand("command def"));
|
||||
fclose(f);
|
||||
|
||||
std::string err;
|
||||
BuildLog log;
|
||||
EXPECT_TRUE(log.Load(kTestFilename, &err));
|
||||
ASSERT_EQ("", err);
|
||||
|
||||
BuildLog::LogEntry* e = log.LookupByOutput("out");
|
||||
ASSERT_TRUE(e);
|
||||
ASSERT_NO_FATAL_FAILURE(AssertHash("command def", e->command_hash));
|
||||
}
|
||||
|
||||
TEST_F(BuildLogTest, Truncate) {
|
||||
AssertParse(&state_,
|
||||
"build out: cat mid\n"
|
||||
"build mid: cat in\n");
|
||||
|
||||
{
|
||||
BuildLog log1;
|
||||
std::string err;
|
||||
EXPECT_TRUE(log1.OpenForWrite(kTestFilename, *this, &err));
|
||||
ASSERT_EQ("", err);
|
||||
log1.RecordCommand(state_.edges_[0], 15, 18);
|
||||
log1.RecordCommand(state_.edges_[1], 20, 25);
|
||||
log1.Close();
|
||||
}
|
||||
#ifdef __USE_LARGEFILE64
|
||||
struct stat64 statbuf;
|
||||
ASSERT_EQ(0, stat64(kTestFilename, &statbuf));
|
||||
#else
|
||||
struct stat statbuf;
|
||||
ASSERT_EQ(0, stat(kTestFilename, &statbuf));
|
||||
#endif
|
||||
ASSERT_GT(statbuf.st_size, 0);
|
||||
|
||||
// For all possible truncations of the input file, assert that we don't
|
||||
// crash when parsing.
|
||||
for (off_t size = statbuf.st_size; size > 0; --size) {
|
||||
BuildLog log2;
|
||||
std::string err;
|
||||
EXPECT_TRUE(log2.OpenForWrite(kTestFilename, *this, &err));
|
||||
ASSERT_EQ("", err);
|
||||
log2.RecordCommand(state_.edges_[0], 15, 18);
|
||||
log2.RecordCommand(state_.edges_[1], 20, 25);
|
||||
log2.Close();
|
||||
|
||||
ASSERT_TRUE(Truncate(kTestFilename, size, &err));
|
||||
|
||||
BuildLog log3;
|
||||
err.clear();
|
||||
ASSERT_TRUE(log3.Load(kTestFilename, &err) == LOAD_SUCCESS || !err.empty());
|
||||
}
|
||||
}
|
||||
|
||||
TEST_F(BuildLogTest, ObsoleteOldVersion) {
|
||||
FILE* f = fopen(kTestFilename, "wb");
|
||||
fprintf(f, "# ninja log v3\n");
|
||||
fprintf(f, "123 456 0 out command\n");
|
||||
fclose(f);
|
||||
|
||||
std::string err;
|
||||
BuildLog log;
|
||||
EXPECT_TRUE(log.Load(kTestFilename, &err));
|
||||
ASSERT_NE(err.find("version"), std::string::npos);
|
||||
}
|
||||
|
||||
TEST_F(BuildLogTest, SpacesInOutput) {
|
||||
FILE* f = fopen(kTestFilename, "wb");
|
||||
fprintf(f, "# ninja log v7\n");
|
||||
fprintf(f, "123\t456\t456\tout with space\t%" PRIx64 "\n",
|
||||
BuildLog::LogEntry::HashCommand("command"));
|
||||
fclose(f);
|
||||
|
||||
std::string err;
|
||||
BuildLog log;
|
||||
EXPECT_TRUE(log.Load(kTestFilename, &err));
|
||||
ASSERT_EQ("", err);
|
||||
|
||||
BuildLog::LogEntry* e = log.LookupByOutput("out with space");
|
||||
ASSERT_TRUE(e);
|
||||
ASSERT_EQ(123, e->start_time);
|
||||
ASSERT_EQ(456, e->end_time);
|
||||
ASSERT_EQ(456, e->mtime);
|
||||
ASSERT_NO_FATAL_FAILURE(AssertHash("command", e->command_hash));
|
||||
}
|
||||
|
||||
TEST_F(BuildLogTest, DuplicateVersionHeader) {
|
||||
// Old versions of ninja accidentally wrote multiple version headers to the
|
||||
// build log on Windows. This shouldn't crash, and the second version header
|
||||
// should be ignored.
|
||||
FILE* f = fopen(kTestFilename, "wb");
|
||||
fprintf(f, "# ninja log v7\n");
|
||||
fprintf(f, "123\t456\t456\tout\t%" PRIx64 "\n",
|
||||
BuildLog::LogEntry::HashCommand("command"));
|
||||
fprintf(f, "# ninja log v7\n");
|
||||
fprintf(f, "456\t789\t789\tout2\t%" PRIx64 "\n",
|
||||
BuildLog::LogEntry::HashCommand("command2"));
|
||||
fclose(f);
|
||||
|
||||
std::string err;
|
||||
BuildLog log;
|
||||
EXPECT_TRUE(log.Load(kTestFilename, &err));
|
||||
ASSERT_EQ("", err);
|
||||
|
||||
BuildLog::LogEntry* e = log.LookupByOutput("out");
|
||||
ASSERT_TRUE(e);
|
||||
ASSERT_EQ(123, e->start_time);
|
||||
ASSERT_EQ(456, e->end_time);
|
||||
ASSERT_EQ(456, e->mtime);
|
||||
ASSERT_NO_FATAL_FAILURE(AssertHash("command", e->command_hash));
|
||||
|
||||
e = log.LookupByOutput("out2");
|
||||
ASSERT_TRUE(e);
|
||||
ASSERT_EQ(456, e->start_time);
|
||||
ASSERT_EQ(789, e->end_time);
|
||||
ASSERT_EQ(789, e->mtime);
|
||||
ASSERT_NO_FATAL_FAILURE(AssertHash("command2", e->command_hash));
|
||||
}
|
||||
|
||||
struct TestDiskInterface : public DiskInterface {
|
||||
TimeStamp Stat(const std::string& path, std::string* err) const override {
|
||||
return 4;
|
||||
}
|
||||
bool WriteFile(const std::string& path, const std::string& contents,
|
||||
bool crlf_on_windows) override {
|
||||
assert(false);
|
||||
return true;
|
||||
}
|
||||
bool MakeDir(const std::string& path) override {
|
||||
assert(false);
|
||||
return false;
|
||||
}
|
||||
Status ReadFile(const std::string& path, std::string* contents,
|
||||
std::string* err) override {
|
||||
assert(false);
|
||||
return NotFound;
|
||||
}
|
||||
int RemoveFile(const std::string& path) override {
|
||||
assert(false);
|
||||
return 0;
|
||||
}
|
||||
};
|
||||
|
||||
TEST_F(BuildLogTest, Restat) {
|
||||
FILE* f = fopen(kTestFilename, "wb");
|
||||
fprintf(f, "# ninja log v7\n"
|
||||
"1\t2\t3\tout\tcommand\n");
|
||||
fclose(f);
|
||||
std::string err;
|
||||
BuildLog log;
|
||||
EXPECT_TRUE(log.Load(kTestFilename, &err));
|
||||
ASSERT_EQ("", err);
|
||||
BuildLog::LogEntry* e = log.LookupByOutput("out");
|
||||
ASSERT_EQ(3, e->mtime);
|
||||
|
||||
TestDiskInterface testDiskInterface;
|
||||
char out2[] = { 'o', 'u', 't', '2', 0 };
|
||||
char* filter2[] = { out2 };
|
||||
EXPECT_TRUE(log.Restat(kTestFilename, testDiskInterface, 1, filter2, &err));
|
||||
ASSERT_EQ("", err);
|
||||
e = log.LookupByOutput("out");
|
||||
ASSERT_EQ(3, e->mtime); // unchanged, since the filter doesn't match
|
||||
|
||||
EXPECT_TRUE(log.Restat(kTestFilename, testDiskInterface, 0, NULL, &err));
|
||||
ASSERT_EQ("", err);
|
||||
e = log.LookupByOutput("out");
|
||||
ASSERT_EQ(4, e->mtime);
|
||||
}
|
||||
|
||||
TEST_F(BuildLogTest, VeryLongInputLine) {
|
||||
// Ninja's build log buffer is currently 256kB. Lines longer than that are
|
||||
// silently ignored, but don't affect parsing of other lines.
|
||||
FILE* f = fopen(kTestFilename, "wb");
|
||||
fprintf(f, "# ninja log v7\n");
|
||||
fprintf(f, "123\t456\t456\tout\tcommand start");
|
||||
for (size_t i = 0; i < (512 << 10) / strlen(" more_command"); ++i)
|
||||
fputs(" more_command", f);
|
||||
fprintf(f, "\n");
|
||||
fprintf(f, "456\t789\t789\tout2\t%" PRIx64 "\n",
|
||||
BuildLog::LogEntry::HashCommand("command2"));
|
||||
fclose(f);
|
||||
|
||||
std::string err;
|
||||
BuildLog log;
|
||||
EXPECT_TRUE(log.Load(kTestFilename, &err));
|
||||
ASSERT_EQ("", err);
|
||||
|
||||
BuildLog::LogEntry* e = log.LookupByOutput("out");
|
||||
ASSERT_EQ(NULL, e);
|
||||
|
||||
e = log.LookupByOutput("out2");
|
||||
ASSERT_TRUE(e);
|
||||
ASSERT_EQ(456, e->start_time);
|
||||
ASSERT_EQ(789, e->end_time);
|
||||
ASSERT_EQ(789, e->mtime);
|
||||
ASSERT_NO_FATAL_FAILURE(AssertHash("command2", e->command_hash));
|
||||
}
|
||||
|
||||
TEST_F(BuildLogTest, MultiTargetEdge) {
|
||||
AssertParse(&state_,
|
||||
"build out out.d: cat\n");
|
||||
|
||||
BuildLog log;
|
||||
log.RecordCommand(state_.edges_[0], 21, 22);
|
||||
|
||||
ASSERT_EQ(2u, log.entries().size());
|
||||
BuildLog::LogEntry* e1 = log.LookupByOutput("out");
|
||||
ASSERT_TRUE(e1);
|
||||
BuildLog::LogEntry* e2 = log.LookupByOutput("out.d");
|
||||
ASSERT_TRUE(e2);
|
||||
ASSERT_EQ("out", e1->output);
|
||||
ASSERT_EQ("out.d", e2->output);
|
||||
ASSERT_EQ(21, e1->start_time);
|
||||
ASSERT_EQ(21, e2->start_time);
|
||||
ASSERT_EQ(22, e2->end_time);
|
||||
ASSERT_EQ(22, e2->end_time);
|
||||
}
|
||||
|
||||
struct BuildLogRecompactTest : public BuildLogTest {
|
||||
virtual bool IsPathDead(StringPiece s) const { return s == "out2"; }
|
||||
};
|
||||
|
||||
TEST_F(BuildLogRecompactTest, Recompact) {
|
||||
AssertParse(&state_,
|
||||
"build out: cat in\n"
|
||||
"build out2: cat in\n");
|
||||
|
||||
BuildLog log1;
|
||||
std::string err;
|
||||
EXPECT_TRUE(log1.OpenForWrite(kTestFilename, *this, &err));
|
||||
ASSERT_EQ("", err);
|
||||
// Record the same edge several times, to trigger recompaction
|
||||
// the next time the log is opened.
|
||||
for (int i = 0; i < 200; ++i)
|
||||
log1.RecordCommand(state_.edges_[0], 15, 18 + i);
|
||||
log1.RecordCommand(state_.edges_[1], 21, 22);
|
||||
log1.Close();
|
||||
|
||||
// Load...
|
||||
BuildLog log2;
|
||||
EXPECT_TRUE(log2.Load(kTestFilename, &err));
|
||||
ASSERT_EQ("", err);
|
||||
ASSERT_EQ(2u, log2.entries().size());
|
||||
ASSERT_TRUE(log2.LookupByOutput("out"));
|
||||
ASSERT_TRUE(log2.LookupByOutput("out2"));
|
||||
// ...and force a recompaction.
|
||||
EXPECT_TRUE(log2.OpenForWrite(kTestFilename, *this, &err));
|
||||
log2.Close();
|
||||
|
||||
// "out2" is dead, it should've been removed.
|
||||
BuildLog log3;
|
||||
EXPECT_TRUE(log2.Load(kTestFilename, &err));
|
||||
ASSERT_EQ("", err);
|
||||
ASSERT_EQ(1u, log2.entries().size());
|
||||
ASSERT_TRUE(log2.LookupByOutput("out"));
|
||||
ASSERT_FALSE(log2.LookupByOutput("out2"));
|
||||
}
|
||||
|
||||
} // anonymous namespace
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,58 @@
|
||||
// Copyright 2012 Google Inc. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#include <stdio.h>
|
||||
#include <string.h>
|
||||
|
||||
#include "util.h"
|
||||
#include "metrics.h"
|
||||
|
||||
using namespace std;
|
||||
|
||||
const char kPath[] =
|
||||
"../../third_party/WebKit/Source/WebCore/"
|
||||
"platform/leveldb/LevelDBWriteBatch.cpp";
|
||||
|
||||
int main() {
|
||||
vector<int> times;
|
||||
|
||||
char buf[200];
|
||||
size_t len = strlen(kPath);
|
||||
strcpy(buf, kPath);
|
||||
|
||||
for (int j = 0; j < 5; ++j) {
|
||||
const int kNumRepetitions = 2000000;
|
||||
int64_t start = GetTimeMillis();
|
||||
uint64_t slash_bits;
|
||||
for (int i = 0; i < kNumRepetitions; ++i) {
|
||||
CanonicalizePath(buf, &len, &slash_bits);
|
||||
}
|
||||
int delta = (int)(GetTimeMillis() - start);
|
||||
times.push_back(delta);
|
||||
}
|
||||
|
||||
int min = times[0];
|
||||
int max = times[0];
|
||||
float total = 0;
|
||||
for (size_t i = 0; i < times.size(); ++i) {
|
||||
total += times[i];
|
||||
if (times[i] < min)
|
||||
min = times[i];
|
||||
else if (times[i] > max)
|
||||
max = times[i];
|
||||
}
|
||||
|
||||
printf("min %dms max %dms avg %.1fms\n",
|
||||
min, max, total / times.size());
|
||||
}
|
||||
@@ -0,0 +1,304 @@
|
||||
// Copyright 2011 Google Inc. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#include "clean.h"
|
||||
|
||||
#include <assert.h>
|
||||
#include <stdio.h>
|
||||
|
||||
#include "disk_interface.h"
|
||||
#include "graph.h"
|
||||
#include "state.h"
|
||||
#include "util.h"
|
||||
|
||||
using namespace std;
|
||||
|
||||
Cleaner::Cleaner(State* state,
|
||||
const BuildConfig& config,
|
||||
DiskInterface* disk_interface)
|
||||
: state_(state),
|
||||
config_(config),
|
||||
dyndep_loader_(state, disk_interface),
|
||||
cleaned_files_count_(0),
|
||||
disk_interface_(disk_interface),
|
||||
status_(0) {
|
||||
}
|
||||
|
||||
int Cleaner::RemoveFile(const string& path) {
|
||||
return disk_interface_->RemoveFile(path);
|
||||
}
|
||||
|
||||
bool Cleaner::FileExists(const string& path) {
|
||||
string err;
|
||||
TimeStamp mtime = disk_interface_->Stat(path, &err);
|
||||
if (mtime == -1)
|
||||
Error("%s", err.c_str());
|
||||
return mtime > 0; // Treat Stat() errors as "file does not exist".
|
||||
}
|
||||
|
||||
void Cleaner::Report(const string& path) {
|
||||
++cleaned_files_count_;
|
||||
if (IsVerbose())
|
||||
printf("Remove %s\n", path.c_str());
|
||||
}
|
||||
|
||||
void Cleaner::Remove(const string& path) {
|
||||
if (!IsAlreadyRemoved(path)) {
|
||||
removed_.insert(path);
|
||||
if (config_.dry_run) {
|
||||
if (FileExists(path))
|
||||
Report(path);
|
||||
} else {
|
||||
int ret = RemoveFile(path);
|
||||
if (ret == 0)
|
||||
Report(path);
|
||||
else if (ret == -1)
|
||||
status_ = 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
bool Cleaner::IsAlreadyRemoved(const string& path) {
|
||||
set<string>::iterator i = removed_.find(path);
|
||||
return (i != removed_.end());
|
||||
}
|
||||
|
||||
void Cleaner::RemoveEdgeFiles(Edge* edge) {
|
||||
string depfile = edge->GetUnescapedDepfile();
|
||||
if (!depfile.empty())
|
||||
Remove(depfile);
|
||||
|
||||
string rspfile = edge->GetUnescapedRspfile();
|
||||
if (!rspfile.empty())
|
||||
Remove(rspfile);
|
||||
}
|
||||
|
||||
void Cleaner::PrintHeader() {
|
||||
if (config_.verbosity == BuildConfig::QUIET)
|
||||
return;
|
||||
printf("Cleaning...");
|
||||
if (IsVerbose())
|
||||
printf("\n");
|
||||
else
|
||||
printf(" ");
|
||||
fflush(stdout);
|
||||
}
|
||||
|
||||
void Cleaner::PrintFooter() {
|
||||
if (config_.verbosity == BuildConfig::QUIET)
|
||||
return;
|
||||
printf("%d files.\n", cleaned_files_count_);
|
||||
}
|
||||
|
||||
int Cleaner::CleanAll(bool generator) {
|
||||
Reset();
|
||||
PrintHeader();
|
||||
LoadDyndeps();
|
||||
for (vector<Edge*>::iterator e = state_->edges_.begin();
|
||||
e != state_->edges_.end(); ++e) {
|
||||
// Do not try to remove phony targets
|
||||
if ((*e)->is_phony())
|
||||
continue;
|
||||
// Do not remove generator's files unless generator specified.
|
||||
if (!generator && (*e)->GetBindingBool("generator"))
|
||||
continue;
|
||||
for (vector<Node*>::iterator out_node = (*e)->outputs_.begin();
|
||||
out_node != (*e)->outputs_.end(); ++out_node) {
|
||||
Remove((*out_node)->path());
|
||||
}
|
||||
|
||||
RemoveEdgeFiles(*e);
|
||||
}
|
||||
PrintFooter();
|
||||
return status_;
|
||||
}
|
||||
|
||||
int Cleaner::CleanDead(const BuildLog::Entries& entries) {
|
||||
Reset();
|
||||
PrintHeader();
|
||||
LoadDyndeps();
|
||||
for (BuildLog::Entries::const_iterator i = entries.begin(); i != entries.end(); ++i) {
|
||||
Node* n = state_->LookupNode(i->first);
|
||||
// Detecting stale outputs works as follows:
|
||||
//
|
||||
// - If it has no Node, it is not in the build graph, or the deps log
|
||||
// anymore, hence is stale.
|
||||
//
|
||||
// - If it isn't an output or input for any edge, it comes from a stale
|
||||
// entry in the deps log, but no longer referenced from the build
|
||||
// graph.
|
||||
//
|
||||
if (!n || (!n->in_edge() && n->out_edges().empty())) {
|
||||
Remove(i->first.AsString());
|
||||
}
|
||||
}
|
||||
PrintFooter();
|
||||
return status_;
|
||||
}
|
||||
|
||||
void Cleaner::DoCleanTarget(Node* target) {
|
||||
if (Edge* e = target->in_edge()) {
|
||||
// Do not try to remove phony targets
|
||||
if (!e->is_phony()) {
|
||||
Remove(target->path());
|
||||
RemoveEdgeFiles(e);
|
||||
}
|
||||
for (vector<Node*>::iterator n = e->inputs_.begin(); n != e->inputs_.end();
|
||||
++n) {
|
||||
Node* next = *n;
|
||||
// call DoCleanTarget recursively if this node has not been visited
|
||||
if (cleaned_.count(next) == 0) {
|
||||
DoCleanTarget(next);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// mark this target to be cleaned already
|
||||
cleaned_.insert(target);
|
||||
}
|
||||
|
||||
int Cleaner::CleanTarget(Node* target) {
|
||||
assert(target);
|
||||
|
||||
Reset();
|
||||
PrintHeader();
|
||||
LoadDyndeps();
|
||||
DoCleanTarget(target);
|
||||
PrintFooter();
|
||||
return status_;
|
||||
}
|
||||
|
||||
int Cleaner::CleanTarget(const char* target) {
|
||||
assert(target);
|
||||
|
||||
Reset();
|
||||
Node* node = state_->LookupNode(target);
|
||||
if (node) {
|
||||
CleanTarget(node);
|
||||
} else {
|
||||
Error("unknown target '%s'", target);
|
||||
status_ = 1;
|
||||
}
|
||||
return status_;
|
||||
}
|
||||
|
||||
int Cleaner::CleanTargets(int target_count, char* targets[]) {
|
||||
Reset();
|
||||
PrintHeader();
|
||||
LoadDyndeps();
|
||||
for (int i = 0; i < target_count; ++i) {
|
||||
string target_name = targets[i];
|
||||
if (target_name.empty()) {
|
||||
Error("failed to canonicalize '': empty path");
|
||||
status_ = 1;
|
||||
continue;
|
||||
}
|
||||
uint64_t slash_bits;
|
||||
CanonicalizePath(&target_name, &slash_bits);
|
||||
Node* target = state_->LookupNode(target_name);
|
||||
if (target) {
|
||||
if (IsVerbose())
|
||||
printf("Target %s\n", target_name.c_str());
|
||||
DoCleanTarget(target);
|
||||
} else {
|
||||
Error("unknown target '%s'", target_name.c_str());
|
||||
status_ = 1;
|
||||
}
|
||||
}
|
||||
PrintFooter();
|
||||
return status_;
|
||||
}
|
||||
|
||||
void Cleaner::DoCleanRule(const Rule* rule) {
|
||||
assert(rule);
|
||||
|
||||
for (vector<Edge*>::iterator e = state_->edges_.begin();
|
||||
e != state_->edges_.end(); ++e) {
|
||||
if ((*e)->rule().name() == rule->name()) {
|
||||
for (vector<Node*>::iterator out_node = (*e)->outputs_.begin();
|
||||
out_node != (*e)->outputs_.end(); ++out_node) {
|
||||
Remove((*out_node)->path());
|
||||
RemoveEdgeFiles(*e);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
int Cleaner::CleanRule(const Rule* rule) {
|
||||
assert(rule);
|
||||
|
||||
Reset();
|
||||
PrintHeader();
|
||||
LoadDyndeps();
|
||||
DoCleanRule(rule);
|
||||
PrintFooter();
|
||||
return status_;
|
||||
}
|
||||
|
||||
int Cleaner::CleanRule(const char* rule) {
|
||||
assert(rule);
|
||||
|
||||
Reset();
|
||||
const Rule* r = state_->bindings_.LookupRule(rule);
|
||||
if (r) {
|
||||
CleanRule(r);
|
||||
} else {
|
||||
Error("unknown rule '%s'", rule);
|
||||
status_ = 1;
|
||||
}
|
||||
return status_;
|
||||
}
|
||||
|
||||
int Cleaner::CleanRules(int rule_count, char* rules[]) {
|
||||
assert(rules);
|
||||
|
||||
Reset();
|
||||
PrintHeader();
|
||||
LoadDyndeps();
|
||||
for (int i = 0; i < rule_count; ++i) {
|
||||
const char* rule_name = rules[i];
|
||||
const Rule* rule = state_->bindings_.LookupRule(rule_name);
|
||||
if (rule) {
|
||||
if (IsVerbose())
|
||||
printf("Rule %s\n", rule_name);
|
||||
DoCleanRule(rule);
|
||||
} else {
|
||||
Error("unknown rule '%s'", rule_name);
|
||||
status_ = 1;
|
||||
}
|
||||
}
|
||||
PrintFooter();
|
||||
return status_;
|
||||
}
|
||||
|
||||
void Cleaner::Reset() {
|
||||
status_ = 0;
|
||||
cleaned_files_count_ = 0;
|
||||
removed_.clear();
|
||||
cleaned_.clear();
|
||||
}
|
||||
|
||||
void Cleaner::LoadDyndeps() {
|
||||
// Load dyndep files that exist, before they are cleaned.
|
||||
for (vector<Edge*>::iterator e = state_->edges_.begin();
|
||||
e != state_->edges_.end(); ++e) {
|
||||
Node* dyndep;
|
||||
if ((dyndep = (*e)->dyndep_) && dyndep->dyndep_pending()) {
|
||||
// Capture and ignore errors loading the dyndep file.
|
||||
// We clean as much of the graph as we know.
|
||||
std::string err;
|
||||
dyndep_loader_.LoadDyndeps(dyndep, &err);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,111 @@
|
||||
// Copyright 2011 Google Inc. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#ifndef NINJA_CLEAN_H_
|
||||
#define NINJA_CLEAN_H_
|
||||
|
||||
#include <set>
|
||||
#include <string>
|
||||
|
||||
#include "build.h"
|
||||
#include "dyndep.h"
|
||||
#include "build_log.h"
|
||||
|
||||
struct State;
|
||||
struct Node;
|
||||
struct Rule;
|
||||
struct DiskInterface;
|
||||
|
||||
struct Cleaner {
|
||||
/// Build a cleaner object with the given @a disk_interface
|
||||
Cleaner(State* state,
|
||||
const BuildConfig& config,
|
||||
DiskInterface* disk_interface);
|
||||
|
||||
/// Clean the given @a target and all the file built for it.
|
||||
/// @return non-zero if an error occurs.
|
||||
int CleanTarget(Node* target);
|
||||
/// Clean the given target @a target.
|
||||
/// @return non-zero if an error occurs.
|
||||
int CleanTarget(const char* target);
|
||||
/// Clean the given target @a targets.
|
||||
/// @return non-zero if an error occurs.
|
||||
int CleanTargets(int target_count, char* targets[]);
|
||||
|
||||
/// Clean all built files, except for files created by generator rules.
|
||||
/// @param generator If set, also clean files created by generator rules.
|
||||
/// @return non-zero if an error occurs.
|
||||
int CleanAll(bool generator = false);
|
||||
|
||||
/// Clean all the file built with the given rule @a rule.
|
||||
/// @return non-zero if an error occurs.
|
||||
int CleanRule(const Rule* rule);
|
||||
/// Clean the file produced by the given @a rule.
|
||||
/// @return non-zero if an error occurs.
|
||||
int CleanRule(const char* rule);
|
||||
/// Clean the file produced by the given @a rules.
|
||||
/// @return non-zero if an error occurs.
|
||||
int CleanRules(int rule_count, char* rules[]);
|
||||
/// Clean the files produced by previous builds that are no longer in the
|
||||
/// manifest.
|
||||
/// @return non-zero if an error occurs.
|
||||
int CleanDead(const BuildLog::Entries& entries);
|
||||
|
||||
/// @return the number of file cleaned.
|
||||
int cleaned_files_count() const {
|
||||
return cleaned_files_count_;
|
||||
}
|
||||
|
||||
/// @return whether the cleaner is in verbose mode.
|
||||
bool IsVerbose() const {
|
||||
return (config_.verbosity != BuildConfig::QUIET
|
||||
&& (config_.verbosity == BuildConfig::VERBOSE || config_.dry_run));
|
||||
}
|
||||
|
||||
private:
|
||||
/// Remove the file @a path.
|
||||
/// @return whether the file has been removed.
|
||||
int RemoveFile(const std::string& path);
|
||||
/// @returns whether the file @a path exists.
|
||||
bool FileExists(const std::string& path);
|
||||
void Report(const std::string& path);
|
||||
|
||||
/// Remove the given @a path file only if it has not been already removed.
|
||||
void Remove(const std::string& path);
|
||||
/// @return whether the given @a path has already been removed.
|
||||
bool IsAlreadyRemoved(const std::string& path);
|
||||
/// Remove the depfile and rspfile for an Edge.
|
||||
void RemoveEdgeFiles(Edge* edge);
|
||||
|
||||
/// Helper recursive method for CleanTarget().
|
||||
void DoCleanTarget(Node* target);
|
||||
void PrintHeader();
|
||||
void PrintFooter();
|
||||
void DoCleanRule(const Rule* rule);
|
||||
void Reset();
|
||||
|
||||
/// Load dependencies from dyndep bindings.
|
||||
void LoadDyndeps();
|
||||
|
||||
State* state_;
|
||||
const BuildConfig& config_;
|
||||
DyndepLoader dyndep_loader_;
|
||||
std::set<std::string> removed_;
|
||||
std::set<Node*> cleaned_;
|
||||
int cleaned_files_count_;
|
||||
DiskInterface* disk_interface_;
|
||||
int status_;
|
||||
};
|
||||
|
||||
#endif // NINJA_CLEAN_H_
|
||||
@@ -0,0 +1,601 @@
|
||||
// Copyright 2011 Google Inc. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#include "clean.h"
|
||||
#include "build.h"
|
||||
|
||||
#include "util.h"
|
||||
#include "test.h"
|
||||
|
||||
#ifndef _WIN32
|
||||
#include <unistd.h>
|
||||
#endif
|
||||
|
||||
using namespace std;
|
||||
|
||||
namespace {
|
||||
|
||||
const char kTestFilename[] = "CleanTest-tempfile";
|
||||
|
||||
struct CleanTest : public StateTestWithBuiltinRules {
|
||||
VirtualFileSystem fs_;
|
||||
BuildConfig config_;
|
||||
virtual void SetUp() {
|
||||
config_.verbosity = BuildConfig::QUIET;
|
||||
}
|
||||
};
|
||||
|
||||
TEST_F(CleanTest, CleanAll) {
|
||||
ASSERT_NO_FATAL_FAILURE(AssertParse(&state_,
|
||||
"build in1: cat src1\n"
|
||||
"build out1: cat in1\n"
|
||||
"build in2: cat src2\n"
|
||||
"build out2: cat in2\n"));
|
||||
fs_.Create("in1", "");
|
||||
fs_.Create("out1", "");
|
||||
fs_.Create("in2", "");
|
||||
fs_.Create("out2", "");
|
||||
|
||||
Cleaner cleaner(&state_, config_, &fs_);
|
||||
|
||||
ASSERT_EQ(0, cleaner.cleaned_files_count());
|
||||
EXPECT_EQ(0, cleaner.CleanAll());
|
||||
EXPECT_EQ(4, cleaner.cleaned_files_count());
|
||||
EXPECT_EQ(4u, fs_.files_removed_.size());
|
||||
|
||||
// Check they are removed.
|
||||
string err;
|
||||
EXPECT_EQ(0, fs_.Stat("in1", &err));
|
||||
EXPECT_EQ(0, fs_.Stat("out1", &err));
|
||||
EXPECT_EQ(0, fs_.Stat("in2", &err));
|
||||
EXPECT_EQ(0, fs_.Stat("out2", &err));
|
||||
fs_.files_removed_.clear();
|
||||
|
||||
EXPECT_EQ(0, cleaner.CleanAll());
|
||||
EXPECT_EQ(0, cleaner.cleaned_files_count());
|
||||
EXPECT_EQ(0u, fs_.files_removed_.size());
|
||||
}
|
||||
|
||||
TEST_F(CleanTest, CleanAllDryRun) {
|
||||
ASSERT_NO_FATAL_FAILURE(AssertParse(&state_,
|
||||
"build in1: cat src1\n"
|
||||
"build out1: cat in1\n"
|
||||
"build in2: cat src2\n"
|
||||
"build out2: cat in2\n"));
|
||||
fs_.Create("in1", "");
|
||||
fs_.Create("out1", "");
|
||||
fs_.Create("in2", "");
|
||||
fs_.Create("out2", "");
|
||||
|
||||
config_.dry_run = true;
|
||||
Cleaner cleaner(&state_, config_, &fs_);
|
||||
|
||||
ASSERT_EQ(0, cleaner.cleaned_files_count());
|
||||
EXPECT_EQ(0, cleaner.CleanAll());
|
||||
EXPECT_EQ(4, cleaner.cleaned_files_count());
|
||||
EXPECT_EQ(0u, fs_.files_removed_.size());
|
||||
|
||||
// Check they are not removed.
|
||||
string err;
|
||||
EXPECT_LT(0, fs_.Stat("in1", &err));
|
||||
EXPECT_LT(0, fs_.Stat("out1", &err));
|
||||
EXPECT_LT(0, fs_.Stat("in2", &err));
|
||||
EXPECT_LT(0, fs_.Stat("out2", &err));
|
||||
fs_.files_removed_.clear();
|
||||
|
||||
EXPECT_EQ(0, cleaner.CleanAll());
|
||||
EXPECT_EQ(4, cleaner.cleaned_files_count());
|
||||
EXPECT_EQ(0u, fs_.files_removed_.size());
|
||||
}
|
||||
|
||||
TEST_F(CleanTest, CleanTarget) {
|
||||
ASSERT_NO_FATAL_FAILURE(AssertParse(&state_,
|
||||
"build in1: cat src1\n"
|
||||
"build out1: cat in1\n"
|
||||
"build in2: cat src2\n"
|
||||
"build out2: cat in2\n"));
|
||||
fs_.Create("in1", "");
|
||||
fs_.Create("out1", "");
|
||||
fs_.Create("in2", "");
|
||||
fs_.Create("out2", "");
|
||||
|
||||
Cleaner cleaner(&state_, config_, &fs_);
|
||||
|
||||
ASSERT_EQ(0, cleaner.cleaned_files_count());
|
||||
ASSERT_EQ(0, cleaner.CleanTarget("out1"));
|
||||
EXPECT_EQ(2, cleaner.cleaned_files_count());
|
||||
EXPECT_EQ(2u, fs_.files_removed_.size());
|
||||
|
||||
// Check they are removed.
|
||||
string err;
|
||||
EXPECT_EQ(0, fs_.Stat("in1", &err));
|
||||
EXPECT_EQ(0, fs_.Stat("out1", &err));
|
||||
EXPECT_LT(0, fs_.Stat("in2", &err));
|
||||
EXPECT_LT(0, fs_.Stat("out2", &err));
|
||||
fs_.files_removed_.clear();
|
||||
|
||||
ASSERT_EQ(0, cleaner.CleanTarget("out1"));
|
||||
EXPECT_EQ(0, cleaner.cleaned_files_count());
|
||||
EXPECT_EQ(0u, fs_.files_removed_.size());
|
||||
}
|
||||
|
||||
TEST_F(CleanTest, CleanTargetDryRun) {
|
||||
ASSERT_NO_FATAL_FAILURE(AssertParse(&state_,
|
||||
"build in1: cat src1\n"
|
||||
"build out1: cat in1\n"
|
||||
"build in2: cat src2\n"
|
||||
"build out2: cat in2\n"));
|
||||
fs_.Create("in1", "");
|
||||
fs_.Create("out1", "");
|
||||
fs_.Create("in2", "");
|
||||
fs_.Create("out2", "");
|
||||
|
||||
config_.dry_run = true;
|
||||
Cleaner cleaner(&state_, config_, &fs_);
|
||||
|
||||
ASSERT_EQ(0, cleaner.cleaned_files_count());
|
||||
ASSERT_EQ(0, cleaner.CleanTarget("out1"));
|
||||
EXPECT_EQ(2, cleaner.cleaned_files_count());
|
||||
EXPECT_EQ(0u, fs_.files_removed_.size());
|
||||
|
||||
// Check they are not removed.
|
||||
string err;
|
||||
EXPECT_LT(0, fs_.Stat("in1", &err));
|
||||
EXPECT_LT(0, fs_.Stat("out1", &err));
|
||||
EXPECT_LT(0, fs_.Stat("in2", &err));
|
||||
EXPECT_LT(0, fs_.Stat("out2", &err));
|
||||
fs_.files_removed_.clear();
|
||||
|
||||
ASSERT_EQ(0, cleaner.CleanTarget("out1"));
|
||||
EXPECT_EQ(2, cleaner.cleaned_files_count());
|
||||
EXPECT_EQ(0u, fs_.files_removed_.size());
|
||||
}
|
||||
|
||||
TEST_F(CleanTest, CleanRule) {
|
||||
ASSERT_NO_FATAL_FAILURE(AssertParse(&state_,
|
||||
"rule cat_e\n"
|
||||
" command = cat -e $in > $out\n"
|
||||
"build in1: cat_e src1\n"
|
||||
"build out1: cat in1\n"
|
||||
"build in2: cat_e src2\n"
|
||||
"build out2: cat in2\n"));
|
||||
fs_.Create("in1", "");
|
||||
fs_.Create("out1", "");
|
||||
fs_.Create("in2", "");
|
||||
fs_.Create("out2", "");
|
||||
|
||||
Cleaner cleaner(&state_, config_, &fs_);
|
||||
|
||||
ASSERT_EQ(0, cleaner.cleaned_files_count());
|
||||
ASSERT_EQ(0, cleaner.CleanRule("cat_e"));
|
||||
EXPECT_EQ(2, cleaner.cleaned_files_count());
|
||||
EXPECT_EQ(2u, fs_.files_removed_.size());
|
||||
|
||||
// Check they are removed.
|
||||
string err;
|
||||
EXPECT_EQ(0, fs_.Stat("in1", &err));
|
||||
EXPECT_LT(0, fs_.Stat("out1", &err));
|
||||
EXPECT_EQ(0, fs_.Stat("in2", &err));
|
||||
EXPECT_LT(0, fs_.Stat("out2", &err));
|
||||
fs_.files_removed_.clear();
|
||||
|
||||
ASSERT_EQ(0, cleaner.CleanRule("cat_e"));
|
||||
EXPECT_EQ(0, cleaner.cleaned_files_count());
|
||||
EXPECT_EQ(0u, fs_.files_removed_.size());
|
||||
}
|
||||
|
||||
TEST_F(CleanTest, CleanRuleDryRun) {
|
||||
ASSERT_NO_FATAL_FAILURE(AssertParse(&state_,
|
||||
"rule cat_e\n"
|
||||
" command = cat -e $in > $out\n"
|
||||
"build in1: cat_e src1\n"
|
||||
"build out1: cat in1\n"
|
||||
"build in2: cat_e src2\n"
|
||||
"build out2: cat in2\n"));
|
||||
fs_.Create("in1", "");
|
||||
fs_.Create("out1", "");
|
||||
fs_.Create("in2", "");
|
||||
fs_.Create("out2", "");
|
||||
|
||||
config_.dry_run = true;
|
||||
Cleaner cleaner(&state_, config_, &fs_);
|
||||
|
||||
ASSERT_EQ(0, cleaner.cleaned_files_count());
|
||||
ASSERT_EQ(0, cleaner.CleanRule("cat_e"));
|
||||
EXPECT_EQ(2, cleaner.cleaned_files_count());
|
||||
EXPECT_EQ(0u, fs_.files_removed_.size());
|
||||
|
||||
// Check they are not removed.
|
||||
string err;
|
||||
EXPECT_LT(0, fs_.Stat("in1", &err));
|
||||
EXPECT_LT(0, fs_.Stat("out1", &err));
|
||||
EXPECT_LT(0, fs_.Stat("in2", &err));
|
||||
EXPECT_LT(0, fs_.Stat("out2", &err));
|
||||
fs_.files_removed_.clear();
|
||||
|
||||
ASSERT_EQ(0, cleaner.CleanRule("cat_e"));
|
||||
EXPECT_EQ(2, cleaner.cleaned_files_count());
|
||||
EXPECT_EQ(0u, fs_.files_removed_.size());
|
||||
}
|
||||
|
||||
TEST_F(CleanTest, CleanRuleGenerator) {
|
||||
ASSERT_NO_FATAL_FAILURE(AssertParse(&state_,
|
||||
"rule regen\n"
|
||||
" command = cat $in > $out\n"
|
||||
" generator = 1\n"
|
||||
"build out1: cat in1\n"
|
||||
"build out2: regen in2\n"));
|
||||
fs_.Create("out1", "");
|
||||
fs_.Create("out2", "");
|
||||
|
||||
Cleaner cleaner(&state_, config_, &fs_);
|
||||
EXPECT_EQ(0, cleaner.CleanAll());
|
||||
EXPECT_EQ(1, cleaner.cleaned_files_count());
|
||||
EXPECT_EQ(1u, fs_.files_removed_.size());
|
||||
|
||||
fs_.Create("out1", "");
|
||||
|
||||
EXPECT_EQ(0, cleaner.CleanAll(/*generator=*/true));
|
||||
EXPECT_EQ(2, cleaner.cleaned_files_count());
|
||||
EXPECT_EQ(2u, fs_.files_removed_.size());
|
||||
}
|
||||
|
||||
TEST_F(CleanTest, CleanDepFile) {
|
||||
ASSERT_NO_FATAL_FAILURE(AssertParse(&state_,
|
||||
"rule cc\n"
|
||||
" command = cc $in > $out\n"
|
||||
" depfile = $out.d\n"
|
||||
"build out1: cc in1\n"));
|
||||
fs_.Create("out1", "");
|
||||
fs_.Create("out1.d", "");
|
||||
|
||||
Cleaner cleaner(&state_, config_, &fs_);
|
||||
EXPECT_EQ(0, cleaner.CleanAll());
|
||||
EXPECT_EQ(2, cleaner.cleaned_files_count());
|
||||
EXPECT_EQ(2u, fs_.files_removed_.size());
|
||||
}
|
||||
|
||||
TEST_F(CleanTest, CleanDepFileOnCleanTarget) {
|
||||
ASSERT_NO_FATAL_FAILURE(AssertParse(&state_,
|
||||
"rule cc\n"
|
||||
" command = cc $in > $out\n"
|
||||
" depfile = $out.d\n"
|
||||
"build out1: cc in1\n"));
|
||||
fs_.Create("out1", "");
|
||||
fs_.Create("out1.d", "");
|
||||
|
||||
Cleaner cleaner(&state_, config_, &fs_);
|
||||
EXPECT_EQ(0, cleaner.CleanTarget("out1"));
|
||||
EXPECT_EQ(2, cleaner.cleaned_files_count());
|
||||
EXPECT_EQ(2u, fs_.files_removed_.size());
|
||||
}
|
||||
|
||||
TEST_F(CleanTest, CleanDepFileOnCleanRule) {
|
||||
ASSERT_NO_FATAL_FAILURE(AssertParse(&state_,
|
||||
"rule cc\n"
|
||||
" command = cc $in > $out\n"
|
||||
" depfile = $out.d\n"
|
||||
"build out1: cc in1\n"));
|
||||
fs_.Create("out1", "");
|
||||
fs_.Create("out1.d", "");
|
||||
|
||||
Cleaner cleaner(&state_, config_, &fs_);
|
||||
EXPECT_EQ(0, cleaner.CleanRule("cc"));
|
||||
EXPECT_EQ(2, cleaner.cleaned_files_count());
|
||||
EXPECT_EQ(2u, fs_.files_removed_.size());
|
||||
}
|
||||
|
||||
TEST_F(CleanTest, CleanDyndep) {
|
||||
// Verify that a dyndep file can be loaded to discover a new output
|
||||
// to be cleaned.
|
||||
ASSERT_NO_FATAL_FAILURE(AssertParse(&state_,
|
||||
"build out: cat in || dd\n"
|
||||
" dyndep = dd\n"
|
||||
));
|
||||
fs_.Create("in", "");
|
||||
fs_.Create("dd",
|
||||
"ninja_dyndep_version = 1\n"
|
||||
"build out | out.imp: dyndep\n"
|
||||
);
|
||||
fs_.Create("out", "");
|
||||
fs_.Create("out.imp", "");
|
||||
|
||||
Cleaner cleaner(&state_, config_, &fs_);
|
||||
|
||||
ASSERT_EQ(0, cleaner.cleaned_files_count());
|
||||
EXPECT_EQ(0, cleaner.CleanAll());
|
||||
EXPECT_EQ(2, cleaner.cleaned_files_count());
|
||||
EXPECT_EQ(2u, fs_.files_removed_.size());
|
||||
|
||||
string err;
|
||||
EXPECT_EQ(0, fs_.Stat("out", &err));
|
||||
EXPECT_EQ(0, fs_.Stat("out.imp", &err));
|
||||
}
|
||||
|
||||
TEST_F(CleanTest, CleanDyndepMissing) {
|
||||
// Verify that a missing dyndep file is tolerated.
|
||||
ASSERT_NO_FATAL_FAILURE(AssertParse(&state_,
|
||||
"build out: cat in || dd\n"
|
||||
" dyndep = dd\n"
|
||||
));
|
||||
fs_.Create("in", "");
|
||||
fs_.Create("out", "");
|
||||
fs_.Create("out.imp", "");
|
||||
|
||||
Cleaner cleaner(&state_, config_, &fs_);
|
||||
|
||||
ASSERT_EQ(0, cleaner.cleaned_files_count());
|
||||
EXPECT_EQ(0, cleaner.CleanAll());
|
||||
EXPECT_EQ(1, cleaner.cleaned_files_count());
|
||||
EXPECT_EQ(1u, fs_.files_removed_.size());
|
||||
|
||||
string err;
|
||||
EXPECT_EQ(0, fs_.Stat("out", &err));
|
||||
EXPECT_EQ(1, fs_.Stat("out.imp", &err));
|
||||
}
|
||||
|
||||
TEST_F(CleanTest, CleanRspFile) {
|
||||
ASSERT_NO_FATAL_FAILURE(AssertParse(&state_,
|
||||
"rule cc\n"
|
||||
" command = cc $in > $out\n"
|
||||
" rspfile = $rspfile\n"
|
||||
" rspfile_content=$in\n"
|
||||
"build out1: cc in1\n"
|
||||
" rspfile = cc1.rsp\n"));
|
||||
fs_.Create("out1", "");
|
||||
fs_.Create("cc1.rsp", "");
|
||||
|
||||
Cleaner cleaner(&state_, config_, &fs_);
|
||||
EXPECT_EQ(0, cleaner.CleanAll());
|
||||
EXPECT_EQ(2, cleaner.cleaned_files_count());
|
||||
EXPECT_EQ(2u, fs_.files_removed_.size());
|
||||
}
|
||||
|
||||
TEST_F(CleanTest, CleanRsp) {
|
||||
ASSERT_NO_FATAL_FAILURE(AssertParse(&state_,
|
||||
"rule cat_rsp \n"
|
||||
" command = cat $rspfile > $out\n"
|
||||
" rspfile = $rspfile\n"
|
||||
" rspfile_content = $in\n"
|
||||
"build in1: cat src1\n"
|
||||
"build out1: cat in1\n"
|
||||
"build in2: cat_rsp src2\n"
|
||||
" rspfile=in2.rsp\n"
|
||||
"build out2: cat_rsp in2\n"
|
||||
" rspfile=out2.rsp\n"
|
||||
));
|
||||
fs_.Create("in1", "");
|
||||
fs_.Create("out1", "");
|
||||
fs_.Create("in2.rsp", "");
|
||||
fs_.Create("out2.rsp", "");
|
||||
fs_.Create("in2", "");
|
||||
fs_.Create("out2", "");
|
||||
|
||||
Cleaner cleaner(&state_, config_, &fs_);
|
||||
ASSERT_EQ(0, cleaner.cleaned_files_count());
|
||||
ASSERT_EQ(0, cleaner.CleanTarget("out1"));
|
||||
EXPECT_EQ(2, cleaner.cleaned_files_count());
|
||||
ASSERT_EQ(0, cleaner.CleanTarget("in2"));
|
||||
EXPECT_EQ(2, cleaner.cleaned_files_count());
|
||||
ASSERT_EQ(0, cleaner.CleanRule("cat_rsp"));
|
||||
EXPECT_EQ(2, cleaner.cleaned_files_count());
|
||||
|
||||
EXPECT_EQ(6u, fs_.files_removed_.size());
|
||||
|
||||
// Check they are removed.
|
||||
string err;
|
||||
EXPECT_EQ(0, fs_.Stat("in1", &err));
|
||||
EXPECT_EQ(0, fs_.Stat("out1", &err));
|
||||
EXPECT_EQ(0, fs_.Stat("in2", &err));
|
||||
EXPECT_EQ(0, fs_.Stat("out2", &err));
|
||||
EXPECT_EQ(0, fs_.Stat("in2.rsp", &err));
|
||||
EXPECT_EQ(0, fs_.Stat("out2.rsp", &err));
|
||||
}
|
||||
|
||||
TEST_F(CleanTest, CleanFailure) {
|
||||
ASSERT_NO_FATAL_FAILURE(AssertParse(&state_,
|
||||
"build dir: cat src1\n"));
|
||||
fs_.MakeDir("dir");
|
||||
Cleaner cleaner(&state_, config_, &fs_);
|
||||
EXPECT_NE(0, cleaner.CleanAll());
|
||||
}
|
||||
|
||||
TEST_F(CleanTest, CleanPhony) {
|
||||
string err;
|
||||
ASSERT_NO_FATAL_FAILURE(AssertParse(&state_,
|
||||
"build phony: phony t1 t2\n"
|
||||
"build t1: cat\n"
|
||||
"build t2: cat\n"));
|
||||
|
||||
fs_.Create("phony", "");
|
||||
fs_.Create("t1", "");
|
||||
fs_.Create("t2", "");
|
||||
|
||||
// Check that CleanAll does not remove "phony".
|
||||
Cleaner cleaner(&state_, config_, &fs_);
|
||||
EXPECT_EQ(0, cleaner.CleanAll());
|
||||
EXPECT_EQ(2, cleaner.cleaned_files_count());
|
||||
EXPECT_LT(0, fs_.Stat("phony", &err));
|
||||
|
||||
fs_.Create("t1", "");
|
||||
fs_.Create("t2", "");
|
||||
|
||||
// Check that CleanTarget does not remove "phony".
|
||||
EXPECT_EQ(0, cleaner.CleanTarget("phony"));
|
||||
EXPECT_EQ(2, cleaner.cleaned_files_count());
|
||||
EXPECT_LT(0, fs_.Stat("phony", &err));
|
||||
}
|
||||
|
||||
TEST_F(CleanTest, CleanDepFileAndRspFileWithSpaces) {
|
||||
ASSERT_NO_FATAL_FAILURE(AssertParse(&state_,
|
||||
"rule cc_dep\n"
|
||||
" command = cc $in > $out\n"
|
||||
" depfile = $out.d\n"
|
||||
"rule cc_rsp\n"
|
||||
" command = cc $in > $out\n"
|
||||
" rspfile = $out.rsp\n"
|
||||
" rspfile_content = $in\n"
|
||||
"build out$ 1: cc_dep in$ 1\n"
|
||||
"build out$ 2: cc_rsp in$ 1\n"
|
||||
));
|
||||
fs_.Create("out 1", "");
|
||||
fs_.Create("out 2", "");
|
||||
fs_.Create("out 1.d", "");
|
||||
fs_.Create("out 2.rsp", "");
|
||||
|
||||
Cleaner cleaner(&state_, config_, &fs_);
|
||||
EXPECT_EQ(0, cleaner.CleanAll());
|
||||
EXPECT_EQ(4, cleaner.cleaned_files_count());
|
||||
EXPECT_EQ(4u, fs_.files_removed_.size());
|
||||
|
||||
string err;
|
||||
EXPECT_EQ(0, fs_.Stat("out 1", &err));
|
||||
EXPECT_EQ(0, fs_.Stat("out 2", &err));
|
||||
EXPECT_EQ(0, fs_.Stat("out 1.d", &err));
|
||||
EXPECT_EQ(0, fs_.Stat("out 2.rsp", &err));
|
||||
}
|
||||
|
||||
struct CleanDeadTest : public CleanTest, public BuildLogUser{
|
||||
virtual void SetUp() {
|
||||
// In case a crashing test left a stale file behind.
|
||||
platformAwareUnlink(kTestFilename);
|
||||
CleanTest::SetUp();
|
||||
}
|
||||
virtual void TearDown() {
|
||||
platformAwareUnlink(kTestFilename);
|
||||
}
|
||||
virtual bool IsPathDead(StringPiece) const { return false; }
|
||||
};
|
||||
|
||||
TEST_F(CleanDeadTest, CleanDead) {
|
||||
State state;
|
||||
ASSERT_NO_FATAL_FAILURE(AssertParse(&state,
|
||||
"rule cat\n"
|
||||
" command = cat $in > $out\n"
|
||||
"build out1: cat in\n"
|
||||
"build out2: cat in\n"
|
||||
));
|
||||
ASSERT_NO_FATAL_FAILURE(AssertParse(&state_,
|
||||
"build out2: cat in\n"
|
||||
));
|
||||
fs_.Create("in", "");
|
||||
fs_.Create("out1", "");
|
||||
fs_.Create("out2", "");
|
||||
|
||||
BuildLog log1;
|
||||
string err;
|
||||
EXPECT_TRUE(log1.OpenForWrite(kTestFilename, *this, &err));
|
||||
ASSERT_EQ("", err);
|
||||
log1.RecordCommand(state.edges_[0], 15, 18);
|
||||
log1.RecordCommand(state.edges_[1], 20, 25);
|
||||
log1.Close();
|
||||
|
||||
BuildLog log2;
|
||||
EXPECT_TRUE(log2.Load(kTestFilename, &err));
|
||||
ASSERT_EQ("", err);
|
||||
ASSERT_EQ(2u, log2.entries().size());
|
||||
ASSERT_TRUE(log2.LookupByOutput("out1"));
|
||||
ASSERT_TRUE(log2.LookupByOutput("out2"));
|
||||
|
||||
// First use the manifest that describe how to build out1.
|
||||
Cleaner cleaner1(&state, config_, &fs_);
|
||||
EXPECT_EQ(0, cleaner1.CleanDead(log2.entries()));
|
||||
EXPECT_EQ(0, cleaner1.cleaned_files_count());
|
||||
EXPECT_EQ(0u, fs_.files_removed_.size());
|
||||
EXPECT_NE(0, fs_.Stat("in", &err));
|
||||
EXPECT_NE(0, fs_.Stat("out1", &err));
|
||||
EXPECT_NE(0, fs_.Stat("out2", &err));
|
||||
|
||||
// Then use the manifest that does not build out1 anymore.
|
||||
Cleaner cleaner2(&state_, config_, &fs_);
|
||||
EXPECT_EQ(0, cleaner2.CleanDead(log2.entries()));
|
||||
EXPECT_EQ(1, cleaner2.cleaned_files_count());
|
||||
EXPECT_EQ(1u, fs_.files_removed_.size());
|
||||
EXPECT_EQ("out1", *(fs_.files_removed_.begin()));
|
||||
EXPECT_NE(0, fs_.Stat("in", &err));
|
||||
EXPECT_EQ(0, fs_.Stat("out1", &err));
|
||||
EXPECT_NE(0, fs_.Stat("out2", &err));
|
||||
|
||||
// Nothing to do now.
|
||||
EXPECT_EQ(0, cleaner2.CleanDead(log2.entries()));
|
||||
EXPECT_EQ(0, cleaner2.cleaned_files_count());
|
||||
EXPECT_EQ(1u, fs_.files_removed_.size());
|
||||
EXPECT_EQ("out1", *(fs_.files_removed_.begin()));
|
||||
EXPECT_NE(0, fs_.Stat("in", &err));
|
||||
EXPECT_EQ(0, fs_.Stat("out1", &err));
|
||||
EXPECT_NE(0, fs_.Stat("out2", &err));
|
||||
log2.Close();
|
||||
}
|
||||
|
||||
TEST_F(CleanDeadTest, CleanDeadPreservesInputs) {
|
||||
State state;
|
||||
ASSERT_NO_FATAL_FAILURE(AssertParse(&state,
|
||||
"rule cat\n"
|
||||
" command = cat $in > $out\n"
|
||||
"build out1: cat in\n"
|
||||
"build out2: cat in\n"
|
||||
));
|
||||
// This manifest does not build out1 anymore, but makes
|
||||
// it an implicit input. CleanDead should detect this
|
||||
// and preserve it.
|
||||
ASSERT_NO_FATAL_FAILURE(AssertParse(&state_,
|
||||
"build out2: cat in | out1\n"
|
||||
));
|
||||
fs_.Create("in", "");
|
||||
fs_.Create("out1", "");
|
||||
fs_.Create("out2", "");
|
||||
|
||||
BuildLog log1;
|
||||
string err;
|
||||
EXPECT_TRUE(log1.OpenForWrite(kTestFilename, *this, &err));
|
||||
ASSERT_EQ("", err);
|
||||
log1.RecordCommand(state.edges_[0], 15, 18);
|
||||
log1.RecordCommand(state.edges_[1], 20, 25);
|
||||
log1.Close();
|
||||
|
||||
BuildLog log2;
|
||||
EXPECT_TRUE(log2.Load(kTestFilename, &err));
|
||||
ASSERT_EQ("", err);
|
||||
ASSERT_EQ(2u, log2.entries().size());
|
||||
ASSERT_TRUE(log2.LookupByOutput("out1"));
|
||||
ASSERT_TRUE(log2.LookupByOutput("out2"));
|
||||
|
||||
// First use the manifest that describe how to build out1.
|
||||
Cleaner cleaner1(&state, config_, &fs_);
|
||||
EXPECT_EQ(0, cleaner1.CleanDead(log2.entries()));
|
||||
EXPECT_EQ(0, cleaner1.cleaned_files_count());
|
||||
EXPECT_EQ(0u, fs_.files_removed_.size());
|
||||
EXPECT_NE(0, fs_.Stat("in", &err));
|
||||
EXPECT_NE(0, fs_.Stat("out1", &err));
|
||||
EXPECT_NE(0, fs_.Stat("out2", &err));
|
||||
|
||||
// Then use the manifest that does not build out1 anymore.
|
||||
Cleaner cleaner2(&state_, config_, &fs_);
|
||||
EXPECT_EQ(0, cleaner2.CleanDead(log2.entries()));
|
||||
EXPECT_EQ(0, cleaner2.cleaned_files_count());
|
||||
EXPECT_EQ(0u, fs_.files_removed_.size());
|
||||
EXPECT_NE(0, fs_.Stat("in", &err));
|
||||
EXPECT_NE(0, fs_.Stat("out1", &err));
|
||||
EXPECT_NE(0, fs_.Stat("out2", &err));
|
||||
|
||||
// Nothing to do now.
|
||||
EXPECT_EQ(0, cleaner2.CleanDead(log2.entries()));
|
||||
EXPECT_EQ(0, cleaner2.cleaned_files_count());
|
||||
EXPECT_EQ(0u, fs_.files_removed_.size());
|
||||
EXPECT_NE(0, fs_.Stat("in", &err));
|
||||
EXPECT_NE(0, fs_.Stat("out1", &err));
|
||||
EXPECT_NE(0, fs_.Stat("out2", &err));
|
||||
log2.Close();
|
||||
}
|
||||
} // anonymous namespace
|
||||
@@ -0,0 +1,130 @@
|
||||
// Copyright 2015 Google Inc. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#include "clparser.h"
|
||||
|
||||
#include <algorithm>
|
||||
#include <assert.h>
|
||||
#include <string.h>
|
||||
|
||||
#include "metrics.h"
|
||||
#include "string_piece_util.h"
|
||||
|
||||
#ifdef _WIN32
|
||||
#include "includes_normalize.h"
|
||||
#include "string_piece.h"
|
||||
#else
|
||||
#include "util.h"
|
||||
#endif
|
||||
|
||||
using namespace std;
|
||||
|
||||
namespace {
|
||||
|
||||
/// Return true if \a input ends with \a needle.
|
||||
bool EndsWith(const string& input, const string& needle) {
|
||||
return (input.size() >= needle.size() &&
|
||||
input.substr(input.size() - needle.size()) == needle);
|
||||
}
|
||||
|
||||
} // anonymous namespace
|
||||
|
||||
// static
|
||||
string CLParser::FilterShowIncludes(const string& line,
|
||||
const string& deps_prefix) {
|
||||
const string kDepsPrefixEnglish = "Note: including file: ";
|
||||
const char* in = line.c_str();
|
||||
const char* end = in + line.size();
|
||||
const string& prefix = deps_prefix.empty() ? kDepsPrefixEnglish : deps_prefix;
|
||||
if (end - in > (int)prefix.size() &&
|
||||
memcmp(in, prefix.c_str(), (int)prefix.size()) == 0) {
|
||||
in += prefix.size();
|
||||
while (*in == ' ')
|
||||
++in;
|
||||
return line.substr(in - line.c_str());
|
||||
}
|
||||
return "";
|
||||
}
|
||||
|
||||
// static
|
||||
bool CLParser::IsSystemInclude(string path) {
|
||||
transform(path.begin(), path.end(), path.begin(), ToLowerASCII);
|
||||
// TODO: this is a heuristic, perhaps there's a better way?
|
||||
return (path.find("program files") != string::npos ||
|
||||
path.find("microsoft visual studio") != string::npos);
|
||||
}
|
||||
|
||||
// static
|
||||
bool CLParser::FilterInputFilename(string line) {
|
||||
transform(line.begin(), line.end(), line.begin(), ToLowerASCII);
|
||||
// TODO: other extensions, like .asm?
|
||||
return EndsWith(line, ".c") ||
|
||||
EndsWith(line, ".cc") ||
|
||||
EndsWith(line, ".cxx") ||
|
||||
EndsWith(line, ".cpp") ||
|
||||
EndsWith(line, ".c++");
|
||||
}
|
||||
|
||||
// static
|
||||
bool CLParser::Parse(const string& output, const string& deps_prefix,
|
||||
string* filtered_output, string* err) {
|
||||
METRIC_RECORD("CLParser::Parse");
|
||||
|
||||
// Loop over all lines in the output to process them.
|
||||
assert(&output != filtered_output);
|
||||
size_t start = 0;
|
||||
bool seen_show_includes = false;
|
||||
#ifdef _WIN32
|
||||
IncludesNormalize normalizer(".");
|
||||
#endif
|
||||
|
||||
while (start < output.size()) {
|
||||
size_t end = output.find_first_of("\r\n", start);
|
||||
if (end == string::npos)
|
||||
end = output.size();
|
||||
string line = output.substr(start, end - start);
|
||||
|
||||
string include = FilterShowIncludes(line, deps_prefix);
|
||||
if (!include.empty()) {
|
||||
seen_show_includes = true;
|
||||
string normalized;
|
||||
#ifdef _WIN32
|
||||
if (!normalizer.Normalize(include, &normalized, err))
|
||||
return false;
|
||||
#else
|
||||
// TODO: should this make the path relative to cwd?
|
||||
normalized = include;
|
||||
uint64_t slash_bits;
|
||||
CanonicalizePath(&normalized, &slash_bits);
|
||||
#endif
|
||||
if (!IsSystemInclude(normalized))
|
||||
includes_.insert(normalized);
|
||||
} else if (!seen_show_includes && FilterInputFilename(line)) {
|
||||
// Drop it.
|
||||
// TODO: if we support compiling multiple output files in a single
|
||||
// cl.exe invocation, we should stash the filename.
|
||||
} else {
|
||||
filtered_output->append(line);
|
||||
filtered_output->append("\n");
|
||||
}
|
||||
|
||||
if (end < output.size() && output[end] == '\r')
|
||||
++end;
|
||||
if (end < output.size() && output[end] == '\n')
|
||||
++end;
|
||||
start = end;
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
@@ -0,0 +1,51 @@
|
||||
// Copyright 2015 Google Inc. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#ifndef NINJA_CLPARSER_H_
|
||||
#define NINJA_CLPARSER_H_
|
||||
|
||||
#include <set>
|
||||
#include <string>
|
||||
|
||||
/// Visual Studio's cl.exe requires some massaging to work with Ninja;
|
||||
/// for example, it emits include information on stderr in a funny
|
||||
/// format when building with /showIncludes. This class parses this
|
||||
/// output.
|
||||
struct CLParser {
|
||||
/// Parse a line of cl.exe output and extract /showIncludes info.
|
||||
/// If a dependency is extracted, returns a nonempty string.
|
||||
/// Exposed for testing.
|
||||
static std::string FilterShowIncludes(const std::string& line,
|
||||
const std::string& deps_prefix);
|
||||
|
||||
/// Return true if a mentioned include file is a system path.
|
||||
/// Filtering these out reduces dependency information considerably.
|
||||
static bool IsSystemInclude(std::string path);
|
||||
|
||||
/// Parse a line of cl.exe output and return true if it looks like
|
||||
/// it's printing an input filename. This is a heuristic but it appears
|
||||
/// to be the best we can do.
|
||||
/// Exposed for testing.
|
||||
static bool FilterInputFilename(std::string line);
|
||||
|
||||
/// Parse the full output of cl, filling filtered_output with the text that
|
||||
/// should be printed (if any). Returns true on success, or false with err
|
||||
/// filled. output must not be the same object as filtered_object.
|
||||
bool Parse(const std::string& output, const std::string& deps_prefix,
|
||||
std::string* filtered_output, std::string* err);
|
||||
|
||||
std::set<std::string> includes_;
|
||||
};
|
||||
|
||||
#endif // NINJA_CLPARSER_H_
|
||||
@@ -0,0 +1,159 @@
|
||||
// Copyright 2017 Google Inc. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#include <stdio.h>
|
||||
#include <stdlib.h>
|
||||
|
||||
#include "clparser.h"
|
||||
#include "metrics.h"
|
||||
|
||||
using namespace std;
|
||||
|
||||
int main(int argc, char* argv[]) {
|
||||
// Output of /showIncludes from #include <iostream>
|
||||
string perf_testdata =
|
||||
"Note: including file: C:\\Program Files (x86)\\Microsoft Visual Studio 14.0\\VC\\INCLUDE\\iostream\r\n"
|
||||
"Note: including file: C:\\Program Files (x86)\\Microsoft Visual Studio 14.0\\VC\\INCLUDE\\istream\r\n"
|
||||
"Note: including file: C:\\Program Files (x86)\\Microsoft Visual Studio 14.0\\VC\\INCLUDE\\ostream\r\n"
|
||||
"Note: including file: C:\\Program Files (x86)\\Microsoft Visual Studio 14.0\\VC\\INCLUDE\\ios\r\n"
|
||||
"Note: including file: C:\\Program Files (x86)\\Microsoft Visual Studio 14.0\\VC\\INCLUDE\\xlocnum\r\n"
|
||||
"Note: including file: C:\\Program Files (x86)\\Microsoft Visual Studio 14.0\\VC\\INCLUDE\\climits\r\n"
|
||||
"Note: including file: C:\\Program Files (x86)\\Microsoft Visual Studio 14.0\\VC\\INCLUDE\\yvals.h\r\n"
|
||||
"Note: including file: C:\\Program Files (x86)\\Microsoft Visual Studio 14.0\\VC\\INCLUDE\\xkeycheck.h\r\n"
|
||||
"Note: including file: C:\\Program Files (x86)\\Microsoft Visual Studio 14.0\\VC\\INCLUDE\\crtdefs.h\r\n"
|
||||
"Note: including file: C:\\Program Files (x86)\\Microsoft Visual Studio 14.0\\VC\\INCLUDE\\vcruntime.h\r\n"
|
||||
"Note: including file: C:\\Program Files (x86)\\Microsoft Visual Studio 14.0\\VC\\INCLUDE\\sal.h\r\n"
|
||||
"Note: including file: C:\\Program Files (x86)\\Microsoft Visual Studio 14.0\\VC\\INCLUDE\\ConcurrencySal.h\r\n"
|
||||
"Note: including file: C:\\Program Files (x86)\\Microsoft Visual Studio 14.0\\VC\\INCLUDE\\vadefs.h\r\n"
|
||||
"Note: including file: C:\\Program Files (x86)\\Windows Kits\\10\\include\\10.0.10240.0\\ucrt\\corecrt.h\r\n"
|
||||
"Note: including file: C:\\Program Files (x86)\\Microsoft Visual Studio 14.0\\VC\\INCLUDE\\vcruntime.h\r\n"
|
||||
"Note: including file: C:\\Program Files (x86)\\Microsoft Visual Studio 14.0\\VC\\INCLUDE\\use_ansi.h\r\n"
|
||||
"Note: including file: C:\\Program Files (x86)\\Microsoft Visual Studio 14.0\\VC\\INCLUDE\\limits.h\r\n"
|
||||
"Note: including file: C:\\Program Files (x86)\\Microsoft Visual Studio 14.0\\VC\\INCLUDE\\vcruntime.h\r\n"
|
||||
"Note: including file: C:\\Program Files (x86)\\Microsoft Visual Studio 14.0\\VC\\INCLUDE\\cmath\r\n"
|
||||
"Note: including file: C:\\Program Files (x86)\\Windows Kits\\10\\include\\10.0.10240.0\\ucrt\\math.h\r\n"
|
||||
"Note: including file: C:\\Program Files (x86)\\Microsoft Visual Studio 14.0\\VC\\INCLUDE\\xtgmath.h\r\n"
|
||||
"Note: including file: C:\\Program Files (x86)\\Microsoft Visual Studio 14.0\\VC\\INCLUDE\\xtr1common\r\n"
|
||||
"Note: including file: C:\\Program Files (x86)\\Microsoft Visual Studio 14.0\\VC\\INCLUDE\\cstdlib\r\n"
|
||||
"Note: including file: C:\\Program Files (x86)\\Windows Kits\\10\\include\\10.0.10240.0\\ucrt\\stdlib.h\r\n"
|
||||
"Note: including file: C:\\Program Files (x86)\\Windows Kits\\10\\include\\10.0.10240.0\\ucrt\\corecrt_malloc.h\r\n"
|
||||
"Note: including file: C:\\Program Files (x86)\\Windows Kits\\10\\include\\10.0.10240.0\\ucrt\\corecrt_search.h\r\n"
|
||||
"Note: including file: C:\\Program Files (x86)\\Windows Kits\\10\\include\\10.0.10240.0\\ucrt\\stddef.h\r\n"
|
||||
"Note: including file: C:\\Program Files (x86)\\Windows Kits\\10\\include\\10.0.10240.0\\ucrt\\corecrt_wstdlib.h\r\n"
|
||||
"Note: including file: C:\\Program Files (x86)\\Microsoft Visual Studio 14.0\\VC\\INCLUDE\\cstdio\r\n"
|
||||
"Note: including file: C:\\Program Files (x86)\\Windows Kits\\10\\include\\10.0.10240.0\\ucrt\\stdio.h\r\n"
|
||||
"Note: including file: C:\\Program Files (x86)\\Windows Kits\\10\\include\\10.0.10240.0\\ucrt\\corecrt_wstdio.h\r\n"
|
||||
"Note: including file: C:\\Program Files (x86)\\Windows Kits\\10\\include\\10.0.10240.0\\ucrt\\corecrt_stdio_config.h\r\n"
|
||||
"Note: including file: C:\\Program Files (x86)\\Microsoft Visual Studio 14.0\\VC\\INCLUDE\\streambuf\r\n"
|
||||
"Note: including file: C:\\Program Files (x86)\\Microsoft Visual Studio 14.0\\VC\\INCLUDE\\xiosbase\r\n"
|
||||
"Note: including file: C:\\Program Files (x86)\\Microsoft Visual Studio 14.0\\VC\\INCLUDE\\xlocale\r\n"
|
||||
"Note: including file: C:\\Program Files (x86)\\Microsoft Visual Studio 14.0\\VC\\INCLUDE\\cstring\r\n"
|
||||
"Note: including file: C:\\Program Files (x86)\\Windows Kits\\10\\include\\10.0.10240.0\\ucrt\\string.h\r\n"
|
||||
"Note: including file: C:\\Program Files (x86)\\Windows Kits\\10\\include\\10.0.10240.0\\ucrt\\corecrt_memory.h\r\n"
|
||||
"Note: including file: C:\\Program Files (x86)\\Windows Kits\\10\\include\\10.0.10240.0\\ucrt\\corecrt_memcpy_s.h\r\n"
|
||||
"Note: including file: C:\\Program Files (x86)\\Windows Kits\\10\\include\\10.0.10240.0\\ucrt\\errno.h\r\n"
|
||||
"Note: including file: C:\\Program Files (x86)\\Microsoft Visual Studio 14.0\\VC\\INCLUDE\\vcruntime_string.h\r\n"
|
||||
"Note: including file: C:\\Program Files (x86)\\Microsoft Visual Studio 14.0\\VC\\INCLUDE\\vcruntime.h\r\n"
|
||||
"Note: including file: C:\\Program Files (x86)\\Windows Kits\\10\\include\\10.0.10240.0\\ucrt\\corecrt_wstring.h\r\n"
|
||||
"Note: including file: C:\\Program Files (x86)\\Microsoft Visual Studio 14.0\\VC\\INCLUDE\\stdexcept\r\n"
|
||||
"Note: including file: C:\\Program Files (x86)\\Microsoft Visual Studio 14.0\\VC\\INCLUDE\\exception\r\n"
|
||||
"Note: including file: C:\\Program Files (x86)\\Microsoft Visual Studio 14.0\\VC\\INCLUDE\\type_traits\r\n"
|
||||
"Note: including file: C:\\Program Files (x86)\\Microsoft Visual Studio 14.0\\VC\\INCLUDE\\xstddef\r\n"
|
||||
"Note: including file: C:\\Program Files (x86)\\Microsoft Visual Studio 14.0\\VC\\INCLUDE\\cstddef\r\n"
|
||||
"Note: including file: C:\\Program Files (x86)\\Microsoft Visual Studio 14.0\\VC\\INCLUDE\\initializer_list\r\n"
|
||||
"Note: including file: C:\\Program Files (x86)\\Windows Kits\\10\\include\\10.0.10240.0\\ucrt\\malloc.h\r\n"
|
||||
"Note: including file: C:\\Program Files (x86)\\Microsoft Visual Studio 14.0\\VC\\INCLUDE\\vcruntime_exception.h\r\n"
|
||||
"Note: including file: C:\\Program Files (x86)\\Microsoft Visual Studio 14.0\\VC\\INCLUDE\\eh.h\r\n"
|
||||
"Note: including file: C:\\Program Files (x86)\\Windows Kits\\10\\include\\10.0.10240.0\\ucrt\\corecrt_terminate.h\r\n"
|
||||
"Note: including file: C:\\Program Files (x86)\\Microsoft Visual Studio 14.0\\VC\\INCLUDE\\xstring\r\n"
|
||||
"Note: including file: C:\\Program Files (x86)\\Microsoft Visual Studio 14.0\\VC\\INCLUDE\\xmemory0\r\n"
|
||||
"Note: including file: C:\\Program Files (x86)\\Microsoft Visual Studio 14.0\\VC\\INCLUDE\\cstdint\r\n"
|
||||
"Note: including file: C:\\Program Files (x86)\\Microsoft Visual Studio 14.0\\VC\\INCLUDE\\stdint.h\r\n"
|
||||
"Note: including file: C:\\Program Files (x86)\\Microsoft Visual Studio 14.0\\VC\\INCLUDE\\vcruntime.h\r\n"
|
||||
"Note: including file: C:\\Program Files (x86)\\Microsoft Visual Studio 14.0\\VC\\INCLUDE\\limits\r\n"
|
||||
"Note: including file: C:\\Program Files (x86)\\Microsoft Visual Studio 14.0\\VC\\INCLUDE\\ymath.h\r\n"
|
||||
"Note: including file: C:\\Program Files (x86)\\Microsoft Visual Studio 14.0\\VC\\INCLUDE\\cfloat\r\n"
|
||||
"Note: including file: C:\\Program Files (x86)\\Windows Kits\\10\\include\\10.0.10240.0\\ucrt\\float.h\r\n"
|
||||
"Note: including file: C:\\Program Files (x86)\\Microsoft Visual Studio 14.0\\VC\\INCLUDE\\cwchar\r\n"
|
||||
"Note: including file: C:\\Program Files (x86)\\Windows Kits\\10\\include\\10.0.10240.0\\ucrt\\wchar.h\r\n"
|
||||
"Note: including file: C:\\Program Files (x86)\\Windows Kits\\10\\include\\10.0.10240.0\\ucrt\\corecrt_wconio.h\r\n"
|
||||
"Note: including file: C:\\Program Files (x86)\\Windows Kits\\10\\include\\10.0.10240.0\\ucrt\\corecrt_wctype.h\r\n"
|
||||
"Note: including file: C:\\Program Files (x86)\\Windows Kits\\10\\include\\10.0.10240.0\\ucrt\\corecrt_wdirect.h\r\n"
|
||||
"Note: including file: C:\\Program Files (x86)\\Windows Kits\\10\\include\\10.0.10240.0\\ucrt\\corecrt_wio.h\r\n"
|
||||
"Note: including file: C:\\Program Files (x86)\\Windows Kits\\10\\include\\10.0.10240.0\\ucrt\\corecrt_share.h\r\n"
|
||||
"Note: including file: C:\\Program Files (x86)\\Windows Kits\\10\\include\\10.0.10240.0\\ucrt\\corecrt_wprocess.h\r\n"
|
||||
"Note: including file: C:\\Program Files (x86)\\Windows Kits\\10\\include\\10.0.10240.0\\ucrt\\corecrt_wtime.h\r\n"
|
||||
"Note: including file: C:\\Program Files (x86)\\Windows Kits\\10\\include\\10.0.10240.0\\ucrt\\sys/stat.h\r\n"
|
||||
"Note: including file: C:\\Program Files (x86)\\Windows Kits\\10\\include\\10.0.10240.0\\ucrt\\sys/types.h\r\n"
|
||||
"Note: including file: C:\\Program Files (x86)\\Microsoft Visual Studio 14.0\\VC\\INCLUDE\\new\r\n"
|
||||
"Note: including file: C:\\Program Files (x86)\\Microsoft Visual Studio 14.0\\VC\\INCLUDE\\vcruntime_new.h\r\n"
|
||||
"Note: including file: C:\\Program Files (x86)\\Microsoft Visual Studio 14.0\\VC\\INCLUDE\\vcruntime.h\r\n"
|
||||
"Note: including file: C:\\Program Files (x86)\\Microsoft Visual Studio 14.0\\VC\\INCLUDE\\xutility\r\n"
|
||||
"Note: including file: C:\\Program Files (x86)\\Microsoft Visual Studio 14.0\\VC\\INCLUDE\\utility\r\n"
|
||||
"Note: including file: C:\\Program Files (x86)\\Microsoft Visual Studio 14.0\\VC\\INCLUDE\\iosfwd\r\n"
|
||||
"Note: including file: C:\\Program Files (x86)\\Windows Kits\\10\\include\\10.0.10240.0\\ucrt\\crtdbg.h\r\n"
|
||||
"Note: including file: C:\\Program Files (x86)\\Microsoft Visual Studio 14.0\\VC\\INCLUDE\\vcruntime_new_debug.h\r\n"
|
||||
"Note: including file: C:\\Program Files (x86)\\Microsoft Visual Studio 14.0\\VC\\INCLUDE\\xatomic0.h\r\n"
|
||||
"Note: including file: C:\\Program Files (x86)\\Microsoft Visual Studio 14.0\\VC\\INCLUDE\\intrin.h\r\n"
|
||||
"Note: including file: C:\\Program Files (x86)\\Microsoft Visual Studio 14.0\\VC\\INCLUDE\\vcruntime.h\r\n"
|
||||
"Note: including file: C:\\Program Files (x86)\\Microsoft Visual Studio 14.0\\VC\\INCLUDE\\setjmp.h\r\n"
|
||||
"Note: including file: C:\\Program Files (x86)\\Microsoft Visual Studio 14.0\\VC\\INCLUDE\\vcruntime.h\r\n"
|
||||
"Note: including file: C:\\Program Files (x86)\\Microsoft Visual Studio 14.0\\VC\\INCLUDE\\immintrin.h\r\n"
|
||||
"Note: including file: C:\\Program Files (x86)\\Microsoft Visual Studio 14.0\\VC\\INCLUDE\\wmmintrin.h\r\n"
|
||||
"Note: including file: C:\\Program Files (x86)\\Microsoft Visual Studio 14.0\\VC\\INCLUDE\\nmmintrin.h\r\n"
|
||||
"Note: including file: C:\\Program Files (x86)\\Microsoft Visual Studio 14.0\\VC\\INCLUDE\\smmintrin.h\r\n"
|
||||
"Note: including file: C:\\Program Files (x86)\\Microsoft Visual Studio 14.0\\VC\\INCLUDE\\tmmintrin.h\r\n"
|
||||
"Note: including file: C:\\Program Files (x86)\\Microsoft Visual Studio 14.0\\VC\\INCLUDE\\pmmintrin.h\r\n"
|
||||
"Note: including file: C:\\Program Files (x86)\\Microsoft Visual Studio 14.0\\VC\\INCLUDE\\emmintrin.h\r\n"
|
||||
"Note: including file: C:\\Program Files (x86)\\Microsoft Visual Studio 14.0\\VC\\INCLUDE\\xmmintrin.h\r\n"
|
||||
"Note: including file: C:\\Program Files (x86)\\Microsoft Visual Studio 14.0\\VC\\INCLUDE\\mmintrin.h\r\n"
|
||||
"Note: including file: C:\\Program Files (x86)\\Microsoft Visual Studio 14.0\\VC\\INCLUDE\\ammintrin.h\r\n"
|
||||
"Note: including file: C:\\Program Files (x86)\\Microsoft Visual Studio 14.0\\VC\\INCLUDE\\mm3dnow.h\r\n"
|
||||
"Note: including file: C:\\Program Files (x86)\\Microsoft Visual Studio 14.0\\VC\\INCLUDE\\vcruntime.h\r\n"
|
||||
"Note: including file: C:\\Program Files (x86)\\Microsoft Visual Studio 14.0\\VC\\INCLUDE\\typeinfo\r\n"
|
||||
"Note: including file: C:\\Program Files (x86)\\Microsoft Visual Studio 14.0\\VC\\INCLUDE\\vcruntime_typeinfo.h\r\n"
|
||||
"Note: including file: C:\\Program Files (x86)\\Microsoft Visual Studio 14.0\\VC\\INCLUDE\\vcruntime.h\r\n"
|
||||
"Note: including file: C:\\Program Files (x86)\\Microsoft Visual Studio 14.0\\VC\\INCLUDE\\xlocinfo\r\n"
|
||||
"Note: including file: C:\\Program Files (x86)\\Microsoft Visual Studio 14.0\\VC\\INCLUDE\\xlocinfo.h\r\n"
|
||||
"Note: including file: C:\\Program Files (x86)\\Windows Kits\\10\\include\\10.0.10240.0\\ucrt\\ctype.h\r\n"
|
||||
"Note: including file: C:\\Program Files (x86)\\Windows Kits\\10\\include\\10.0.10240.0\\ucrt\\locale.h\r\n"
|
||||
"Note: including file: C:\\Program Files (x86)\\Microsoft Visual Studio 14.0\\VC\\INCLUDE\\xfacet\r\n"
|
||||
"Note: including file: C:\\Program Files (x86)\\Microsoft Visual Studio 14.0\\VC\\INCLUDE\\system_error\r\n"
|
||||
"Note: including file: C:\\Program Files (x86)\\Microsoft Visual Studio 14.0\\VC\\INCLUDE\\cerrno\r\n"
|
||||
"Note: including file: C:\\Program Files (x86)\\Windows Kits\\10\\include\\10.0.10240.0\\ucrt\\share.h\r\n";
|
||||
|
||||
for (int limit = 1 << 10; limit < (1<<20); limit *= 2) {
|
||||
int64_t start = GetTimeMillis();
|
||||
for (int rep = 0; rep < limit; ++rep) {
|
||||
string output;
|
||||
string err;
|
||||
|
||||
CLParser parser;
|
||||
if (!parser.Parse(perf_testdata, "", &output, &err)) {
|
||||
printf("%s\n", err.c_str());
|
||||
return 1;
|
||||
}
|
||||
}
|
||||
int64_t end = GetTimeMillis();
|
||||
|
||||
if (end - start > 2000) {
|
||||
int delta_ms = (int)(end - start);
|
||||
printf("Parse %d times in %dms avg %.1fus\n",
|
||||
limit, delta_ms, float(delta_ms * 1000) / limit);
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
@@ -0,0 +1,130 @@
|
||||
// Copyright 2011 Google Inc. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#include "clparser.h"
|
||||
|
||||
#include "test.h"
|
||||
#include "util.h"
|
||||
|
||||
using namespace std;
|
||||
|
||||
TEST(CLParserTest, ShowIncludes) {
|
||||
ASSERT_EQ("", CLParser::FilterShowIncludes("", ""));
|
||||
|
||||
ASSERT_EQ("", CLParser::FilterShowIncludes("Sample compiler output", ""));
|
||||
ASSERT_EQ("c:\\Some Files\\foobar.h",
|
||||
CLParser::FilterShowIncludes("Note: including file: "
|
||||
"c:\\Some Files\\foobar.h", ""));
|
||||
ASSERT_EQ("c:\\initspaces.h",
|
||||
CLParser::FilterShowIncludes("Note: including file: "
|
||||
"c:\\initspaces.h", ""));
|
||||
ASSERT_EQ("c:\\initspaces.h",
|
||||
CLParser::FilterShowIncludes("Non-default prefix: inc file: "
|
||||
"c:\\initspaces.h",
|
||||
"Non-default prefix: inc file:"));
|
||||
}
|
||||
|
||||
TEST(CLParserTest, FilterInputFilename) {
|
||||
ASSERT_TRUE(CLParser::FilterInputFilename("foobar.cc"));
|
||||
ASSERT_TRUE(CLParser::FilterInputFilename("foo bar.cc"));
|
||||
ASSERT_TRUE(CLParser::FilterInputFilename("baz.c"));
|
||||
ASSERT_TRUE(CLParser::FilterInputFilename("FOOBAR.CC"));
|
||||
|
||||
ASSERT_FALSE(CLParser::FilterInputFilename(
|
||||
"src\\cl_helper.cc(166) : fatal error C1075: end "
|
||||
"of file found ..."));
|
||||
}
|
||||
|
||||
TEST(CLParserTest, ParseSimple) {
|
||||
CLParser parser;
|
||||
string output, err;
|
||||
ASSERT_TRUE(parser.Parse(
|
||||
"foo\r\n"
|
||||
"Note: inc file prefix: foo.h\r\n"
|
||||
"bar\r\n",
|
||||
"Note: inc file prefix:", &output, &err));
|
||||
|
||||
ASSERT_EQ("foo\nbar\n", output);
|
||||
ASSERT_EQ(1u, parser.includes_.size());
|
||||
ASSERT_EQ("foo.h", *parser.includes_.begin());
|
||||
}
|
||||
|
||||
TEST(CLParserTest, ParseFilenameFilter) {
|
||||
CLParser parser;
|
||||
string output, err;
|
||||
ASSERT_TRUE(parser.Parse(
|
||||
"foo.cc\r\n"
|
||||
"cl: warning\r\n",
|
||||
"", &output, &err));
|
||||
ASSERT_EQ("cl: warning\n", output);
|
||||
}
|
||||
|
||||
TEST(CLParserTest, NoFilenameFilterAfterShowIncludes) {
|
||||
CLParser parser;
|
||||
string output, err;
|
||||
ASSERT_TRUE(parser.Parse(
|
||||
"foo.cc\r\n"
|
||||
"Note: including file: foo.h\r\n"
|
||||
"something something foo.cc\r\n",
|
||||
"", &output, &err));
|
||||
ASSERT_EQ("something something foo.cc\n", output);
|
||||
}
|
||||
|
||||
TEST(CLParserTest, ParseSystemInclude) {
|
||||
CLParser parser;
|
||||
string output, err;
|
||||
ASSERT_TRUE(parser.Parse(
|
||||
"Note: including file: c:\\Program Files\\foo.h\r\n"
|
||||
"Note: including file: d:\\Microsoft Visual Studio\\bar.h\r\n"
|
||||
"Note: including file: path.h\r\n",
|
||||
"", &output, &err));
|
||||
// We should have dropped the first two includes because they look like
|
||||
// system headers.
|
||||
ASSERT_EQ("", output);
|
||||
ASSERT_EQ(1u, parser.includes_.size());
|
||||
ASSERT_EQ("path.h", *parser.includes_.begin());
|
||||
}
|
||||
|
||||
TEST(CLParserTest, DuplicatedHeader) {
|
||||
CLParser parser;
|
||||
string output, err;
|
||||
ASSERT_TRUE(parser.Parse(
|
||||
"Note: including file: foo.h\r\n"
|
||||
"Note: including file: bar.h\r\n"
|
||||
"Note: including file: foo.h\r\n",
|
||||
"", &output, &err));
|
||||
// We should have dropped one copy of foo.h.
|
||||
ASSERT_EQ("", output);
|
||||
ASSERT_EQ(2u, parser.includes_.size());
|
||||
}
|
||||
|
||||
TEST(CLParserTest, DuplicatedHeaderPathConverted) {
|
||||
CLParser parser;
|
||||
string output, err;
|
||||
|
||||
// This isn't inline in the Parse() call below because the #ifdef in
|
||||
// a macro expansion would confuse MSVC2013's preprocessor.
|
||||
const char kInput[] =
|
||||
"Note: including file: sub/./foo.h\r\n"
|
||||
"Note: including file: bar.h\r\n"
|
||||
#ifdef _WIN32
|
||||
"Note: including file: sub\\foo.h\r\n";
|
||||
#else
|
||||
"Note: including file: sub/foo.h\r\n";
|
||||
#endif
|
||||
ASSERT_TRUE(parser.Parse(kInput, "", &output, &err));
|
||||
// We should have dropped one copy of foo.h.
|
||||
ASSERT_EQ("", output);
|
||||
ASSERT_EQ(2u, parser.includes_.size());
|
||||
}
|
||||
@@ -0,0 +1,65 @@
|
||||
// Copyright 2024 Google Inc. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#ifndef NINJA_COMMAND_COLLECTOR_H_
|
||||
#define NINJA_COMMAND_COLLECTOR_H_
|
||||
|
||||
#include <cassert>
|
||||
#include <unordered_set>
|
||||
#include <vector>
|
||||
|
||||
#include "graph.h"
|
||||
|
||||
/// Collects the transitive set of edges that lead into a given set
|
||||
/// of starting nodes. Used to implement the `compdb-targets` tool.
|
||||
///
|
||||
/// When collecting inputs, the outputs of phony edges are always ignored
|
||||
/// from the result, but are followed by the dependency walk.
|
||||
///
|
||||
/// Usage is:
|
||||
/// - Create instance.
|
||||
/// - Call CollectFrom() for each root node to collect edges from.
|
||||
/// - Call TakeResult() to retrieve the list of edges.
|
||||
///
|
||||
struct CommandCollector {
|
||||
void CollectFrom(const Node* node) {
|
||||
assert(node);
|
||||
|
||||
if (!visited_nodes_.insert(node).second)
|
||||
return;
|
||||
|
||||
Edge* edge = node->in_edge();
|
||||
if (!edge || !visited_edges_.insert(edge).second)
|
||||
return;
|
||||
|
||||
for (Node* input_node : edge->inputs_)
|
||||
CollectFrom(input_node);
|
||||
|
||||
if (!edge->is_phony())
|
||||
in_edges.push_back(edge);
|
||||
}
|
||||
|
||||
private:
|
||||
std::unordered_set<const Node*> visited_nodes_;
|
||||
std::unordered_set<Edge*> visited_edges_;
|
||||
|
||||
/// we use a vector to preserve order from requisites to their dependents.
|
||||
/// This may help LSP server performance in languages that support modules,
|
||||
/// but it also ensures that the output of `-t compdb-targets foo` is
|
||||
/// consistent, which is useful in regression tests.
|
||||
public:
|
||||
std::vector<Edge*> in_edges;
|
||||
};
|
||||
|
||||
#endif // NINJA_COMMAND_COLLECTOR_H_
|
||||
@@ -0,0 +1,28 @@
|
||||
// Copyright 2012 Google Inc. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#include <stdio.h>
|
||||
#include <map>
|
||||
#include <vector>
|
||||
#include <string>
|
||||
|
||||
#include "graph.h"
|
||||
|
||||
bool g_explaining = false;
|
||||
|
||||
bool g_keep_depfile = false;
|
||||
|
||||
bool g_keep_rsp = false;
|
||||
|
||||
bool g_experimental_statcache = true;
|
||||
@@ -0,0 +1,31 @@
|
||||
// Copyright 2012 Google Inc. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#ifndef NINJA_EXPLAIN_H_
|
||||
#define NINJA_EXPLAIN_H_
|
||||
|
||||
#include <stdio.h>
|
||||
|
||||
struct Edge;
|
||||
struct Node;
|
||||
|
||||
extern bool g_explaining;
|
||||
|
||||
extern bool g_keep_depfile;
|
||||
|
||||
extern bool g_keep_rsp;
|
||||
|
||||
extern bool g_experimental_statcache;
|
||||
|
||||
#endif // NINJA_EXPLAIN_H_
|
||||
@@ -0,0 +1,373 @@
|
||||
/* Generated by re2c */
|
||||
// Copyright 2011 Google Inc. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#include "depfile_parser.h"
|
||||
#include "util.h"
|
||||
|
||||
#include <algorithm>
|
||||
|
||||
using namespace std;
|
||||
|
||||
DepfileParser::DepfileParser(DepfileParserOptions options)
|
||||
: options_(options)
|
||||
{
|
||||
}
|
||||
|
||||
// A note on backslashes in Makefiles, from reading the docs:
|
||||
// Backslash-newline is the line continuation character.
|
||||
// Backslash-# escapes a # (otherwise meaningful as a comment start).
|
||||
// Backslash-% escapes a % (otherwise meaningful as a special).
|
||||
// Finally, quoting the GNU manual, "Backslashes that are not in danger
|
||||
// of quoting ‘%’ characters go unmolested."
|
||||
// How do you end a line with a backslash? The netbsd Make docs suggest
|
||||
// reading the result of a shell command echoing a backslash!
|
||||
//
|
||||
// Rather than implement all of above, we follow what GCC/Clang produces:
|
||||
// Backslashes escape a space or hash sign.
|
||||
// When a space is preceded by 2N+1 backslashes, it is represents N backslashes
|
||||
// followed by space.
|
||||
// When a space is preceded by 2N backslashes, it represents 2N backslashes at
|
||||
// the end of a filename.
|
||||
// A hash sign is escaped by a single backslash. All other backslashes remain
|
||||
// unchanged.
|
||||
//
|
||||
// If anyone actually has depfiles that rely on the more complicated
|
||||
// behavior we can adjust this.
|
||||
bool DepfileParser::Parse(string* content, string* err) {
|
||||
// in: current parser input point.
|
||||
// end: end of input.
|
||||
// parsing_targets: whether we are parsing targets or dependencies.
|
||||
char* in = &(*content)[0];
|
||||
char* end = in + content->size();
|
||||
bool have_target = false;
|
||||
bool parsing_targets = true;
|
||||
bool poisoned_input = false;
|
||||
bool is_empty = true;
|
||||
while (in < end) {
|
||||
bool have_newline = false;
|
||||
// out: current output point (typically same as in, but can fall behind
|
||||
// as we de-escape backslashes).
|
||||
char* out = in;
|
||||
// filename: start of the current parsed filename.
|
||||
char* filename = out;
|
||||
for (;;) {
|
||||
// start: beginning of the current parsed span.
|
||||
const char* start = in;
|
||||
char* yymarker = NULL;
|
||||
|
||||
{
|
||||
unsigned char yych;
|
||||
static const unsigned char yybm[] = {
|
||||
0, 0, 0, 0, 0, 0, 0, 0,
|
||||
0, 0, 0, 0, 0, 0, 0, 0,
|
||||
0, 0, 0, 0, 0, 0, 0, 0,
|
||||
0, 0, 0, 0, 0, 0, 0, 0,
|
||||
0, 128, 128, 0, 0, 128, 128, 128,
|
||||
128, 128, 0, 128, 128, 128, 128, 128,
|
||||
128, 128, 128, 128, 128, 128, 128, 128,
|
||||
128, 128, 128, 0, 0, 128, 0, 128,
|
||||
128, 128, 128, 128, 128, 128, 128, 128,
|
||||
128, 128, 128, 128, 128, 128, 128, 128,
|
||||
128, 128, 128, 128, 128, 128, 128, 128,
|
||||
128, 128, 128, 128, 0, 128, 0, 128,
|
||||
0, 128, 128, 128, 128, 128, 128, 128,
|
||||
128, 128, 128, 128, 128, 128, 128, 128,
|
||||
128, 128, 128, 128, 128, 128, 128, 128,
|
||||
128, 128, 128, 128, 0, 128, 128, 0,
|
||||
128, 128, 128, 128, 128, 128, 128, 128,
|
||||
128, 128, 128, 128, 128, 128, 128, 128,
|
||||
128, 128, 128, 128, 128, 128, 128, 128,
|
||||
128, 128, 128, 128, 128, 128, 128, 128,
|
||||
128, 128, 128, 128, 128, 128, 128, 128,
|
||||
128, 128, 128, 128, 128, 128, 128, 128,
|
||||
128, 128, 128, 128, 128, 128, 128, 128,
|
||||
128, 128, 128, 128, 128, 128, 128, 128,
|
||||
128, 128, 128, 128, 128, 128, 128, 128,
|
||||
128, 128, 128, 128, 128, 128, 128, 128,
|
||||
128, 128, 128, 128, 128, 128, 128, 128,
|
||||
128, 128, 128, 128, 128, 128, 128, 128,
|
||||
128, 128, 128, 128, 128, 128, 128, 128,
|
||||
128, 128, 128, 128, 128, 128, 128, 128,
|
||||
128, 128, 128, 128, 128, 128, 128, 128,
|
||||
128, 128, 128, 128, 128, 128, 128, 128,
|
||||
};
|
||||
yych = *in;
|
||||
if (yybm[0+yych] & 128) {
|
||||
goto yy5;
|
||||
}
|
||||
if (yych <= '\r') {
|
||||
if (yych <= '\t') {
|
||||
if (yych >= 0x01) goto yy1;
|
||||
} else {
|
||||
if (yych <= '\n') goto yy3;
|
||||
if (yych <= '\f') goto yy1;
|
||||
goto yy4;
|
||||
}
|
||||
} else {
|
||||
if (yych <= '$') {
|
||||
if (yych <= '#') goto yy1;
|
||||
goto yy7;
|
||||
} else {
|
||||
if (yych <= '>') goto yy1;
|
||||
if (yych <= '\\') goto yy8;
|
||||
goto yy1;
|
||||
}
|
||||
}
|
||||
++in;
|
||||
{
|
||||
break;
|
||||
}
|
||||
yy1:
|
||||
++in;
|
||||
yy2:
|
||||
{
|
||||
// For any other character (e.g. whitespace), swallow it here,
|
||||
// allowing the outer logic to loop around again.
|
||||
break;
|
||||
}
|
||||
yy3:
|
||||
++in;
|
||||
{
|
||||
// A newline ends the current file name and the current rule.
|
||||
have_newline = true;
|
||||
break;
|
||||
}
|
||||
yy4:
|
||||
yych = *++in;
|
||||
if (yych == '\n') goto yy3;
|
||||
goto yy2;
|
||||
yy5:
|
||||
yych = *++in;
|
||||
if (yybm[0+yych] & 128) {
|
||||
goto yy5;
|
||||
}
|
||||
yy6:
|
||||
{
|
||||
// Got a span of plain text.
|
||||
int len = (int)(in - start);
|
||||
// Need to shift it over if we're overwriting backslashes.
|
||||
if (out < start)
|
||||
memmove(out, start, len);
|
||||
out += len;
|
||||
continue;
|
||||
}
|
||||
yy7:
|
||||
yych = *++in;
|
||||
if (yych == '$') goto yy9;
|
||||
goto yy2;
|
||||
yy8:
|
||||
yych = *(yymarker = ++in);
|
||||
if (yych <= ' ') {
|
||||
if (yych <= '\n') {
|
||||
if (yych <= 0x00) goto yy2;
|
||||
if (yych <= '\t') goto yy10;
|
||||
goto yy11;
|
||||
} else {
|
||||
if (yych == '\r') goto yy12;
|
||||
if (yych <= 0x1F) goto yy10;
|
||||
goto yy13;
|
||||
}
|
||||
} else {
|
||||
if (yych <= '9') {
|
||||
if (yych == '#') goto yy14;
|
||||
goto yy10;
|
||||
} else {
|
||||
if (yych <= ':') goto yy15;
|
||||
if (yych == '\\') goto yy17;
|
||||
goto yy10;
|
||||
}
|
||||
}
|
||||
yy9:
|
||||
++in;
|
||||
{
|
||||
// De-escape dollar character.
|
||||
*out++ = '$';
|
||||
continue;
|
||||
}
|
||||
yy10:
|
||||
++in;
|
||||
goto yy6;
|
||||
yy11:
|
||||
++in;
|
||||
{
|
||||
// A line continuation ends the current file name.
|
||||
break;
|
||||
}
|
||||
yy12:
|
||||
yych = *++in;
|
||||
if (yych == '\n') goto yy11;
|
||||
in = yymarker;
|
||||
goto yy2;
|
||||
yy13:
|
||||
++in;
|
||||
{
|
||||
// 2N+1 backslashes plus space -> N backslashes plus space.
|
||||
int len = (int)(in - start);
|
||||
int n = len / 2 - 1;
|
||||
if (out < start)
|
||||
memset(out, '\\', n);
|
||||
out += n;
|
||||
*out++ = ' ';
|
||||
continue;
|
||||
}
|
||||
yy14:
|
||||
++in;
|
||||
{
|
||||
// De-escape hash sign, but preserve other leading backslashes.
|
||||
int len = (int)(in - start);
|
||||
if (len > 2 && out < start)
|
||||
memset(out, '\\', len - 2);
|
||||
out += len - 2;
|
||||
*out++ = '#';
|
||||
continue;
|
||||
}
|
||||
yy15:
|
||||
yych = *++in;
|
||||
if (yych <= '\f') {
|
||||
if (yych <= 0x00) goto yy18;
|
||||
if (yych <= 0x08) goto yy16;
|
||||
if (yych <= '\n') goto yy18;
|
||||
} else {
|
||||
if (yych <= '\r') goto yy18;
|
||||
if (yych == ' ') goto yy18;
|
||||
}
|
||||
yy16:
|
||||
{
|
||||
// De-escape colon sign, but preserve other leading backslashes.
|
||||
// Regular expression uses lookahead to make sure that no whitespace
|
||||
// nor EOF follows. In that case it'd be the : at the end of a target
|
||||
int len = (int)(in - start);
|
||||
if (len > 2 && out < start)
|
||||
memset(out, '\\', len - 2);
|
||||
out += len - 2;
|
||||
*out++ = ':';
|
||||
continue;
|
||||
}
|
||||
yy17:
|
||||
yych = *++in;
|
||||
if (yych <= ' ') {
|
||||
if (yych <= '\n') {
|
||||
if (yych <= 0x00) goto yy6;
|
||||
if (yych <= '\t') goto yy10;
|
||||
goto yy6;
|
||||
} else {
|
||||
if (yych == '\r') goto yy6;
|
||||
if (yych <= 0x1F) goto yy10;
|
||||
goto yy19;
|
||||
}
|
||||
} else {
|
||||
if (yych <= '9') {
|
||||
if (yych == '#') goto yy14;
|
||||
goto yy10;
|
||||
} else {
|
||||
if (yych <= ':') goto yy15;
|
||||
if (yych == '\\') goto yy20;
|
||||
goto yy10;
|
||||
}
|
||||
}
|
||||
yy18:
|
||||
++in;
|
||||
{
|
||||
// Backslash followed by : and whitespace.
|
||||
// It is therefore normal text and not an escaped colon
|
||||
int len = (int)(in - start - 1);
|
||||
// Need to shift it over if we're overwriting backslashes.
|
||||
if (out < start)
|
||||
memmove(out, start, len);
|
||||
out += len;
|
||||
if (*(in - 1) == '\n')
|
||||
have_newline = true;
|
||||
break;
|
||||
}
|
||||
yy19:
|
||||
++in;
|
||||
{
|
||||
// 2N backslashes plus space -> 2N backslashes, end of filename.
|
||||
int len = (int)(in - start);
|
||||
if (out < start)
|
||||
memset(out, '\\', len - 1);
|
||||
out += len - 1;
|
||||
break;
|
||||
}
|
||||
yy20:
|
||||
yych = *++in;
|
||||
if (yych <= ' ') {
|
||||
if (yych <= '\n') {
|
||||
if (yych <= 0x00) goto yy6;
|
||||
if (yych <= '\t') goto yy10;
|
||||
goto yy6;
|
||||
} else {
|
||||
if (yych == '\r') goto yy6;
|
||||
if (yych <= 0x1F) goto yy10;
|
||||
goto yy13;
|
||||
}
|
||||
} else {
|
||||
if (yych <= '9') {
|
||||
if (yych == '#') goto yy14;
|
||||
goto yy10;
|
||||
} else {
|
||||
if (yych <= ':') goto yy15;
|
||||
if (yych == '\\') goto yy17;
|
||||
goto yy10;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
int len = (int)(out - filename);
|
||||
const bool is_dependency = !parsing_targets;
|
||||
if (len > 0 && filename[len - 1] == ':') {
|
||||
len--; // Strip off trailing colon, if any.
|
||||
parsing_targets = false;
|
||||
have_target = true;
|
||||
}
|
||||
|
||||
if (len > 0) {
|
||||
is_empty = false;
|
||||
StringPiece piece = StringPiece(filename, len);
|
||||
// If we've seen this as an input before, skip it.
|
||||
std::vector<StringPiece>::iterator pos = std::find(ins_.begin(), ins_.end(), piece);
|
||||
if (pos == ins_.end()) {
|
||||
if (is_dependency) {
|
||||
if (poisoned_input) {
|
||||
*err = "inputs may not also have inputs";
|
||||
return false;
|
||||
}
|
||||
// New input.
|
||||
ins_.push_back(piece);
|
||||
} else {
|
||||
// Check for a new output.
|
||||
if (std::find(outs_.begin(), outs_.end(), piece) == outs_.end())
|
||||
outs_.push_back(piece);
|
||||
}
|
||||
} else if (!is_dependency) {
|
||||
// We've passed an input on the left side; reject new inputs.
|
||||
poisoned_input = true;
|
||||
}
|
||||
}
|
||||
|
||||
if (have_newline) {
|
||||
// A newline ends a rule so the next filename will be a new target.
|
||||
parsing_targets = true;
|
||||
poisoned_input = false;
|
||||
}
|
||||
}
|
||||
if (!have_target && !is_empty) {
|
||||
*err = "expected ':' in depfile";
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
@@ -0,0 +1,42 @@
|
||||
// Copyright 2011 Google Inc. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#ifndef NINJA_DEPFILE_PARSER_H_
|
||||
#define NINJA_DEPFILE_PARSER_H_
|
||||
|
||||
#include <string>
|
||||
#include <vector>
|
||||
|
||||
#include "string_piece.h"
|
||||
|
||||
struct DepfileParserOptions {
|
||||
DepfileParserOptions() {}
|
||||
};
|
||||
|
||||
/// Parser for the dependency information emitted by gcc's -M flags.
|
||||
struct DepfileParser {
|
||||
explicit DepfileParser(DepfileParserOptions options =
|
||||
DepfileParserOptions());
|
||||
|
||||
/// Parse an input file. Input must be NUL-terminated.
|
||||
/// Warning: may mutate the content in-place and parsed StringPieces are
|
||||
/// pointers within it.
|
||||
bool Parse(std::string* content, std::string* err);
|
||||
|
||||
std::vector<StringPiece> outs_;
|
||||
std::vector<StringPiece> ins_;
|
||||
DepfileParserOptions options_;
|
||||
};
|
||||
|
||||
#endif // NINJA_DEPFILE_PARSER_H_
|
||||
@@ -0,0 +1,209 @@
|
||||
// Copyright 2011 Google Inc. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#include "depfile_parser.h"
|
||||
#include "util.h"
|
||||
|
||||
#include <algorithm>
|
||||
|
||||
using namespace std;
|
||||
|
||||
DepfileParser::DepfileParser(DepfileParserOptions options)
|
||||
: options_(options)
|
||||
{
|
||||
}
|
||||
|
||||
// A note on backslashes in Makefiles, from reading the docs:
|
||||
// Backslash-newline is the line continuation character.
|
||||
// Backslash-# escapes a # (otherwise meaningful as a comment start).
|
||||
// Backslash-% escapes a % (otherwise meaningful as a special).
|
||||
// Finally, quoting the GNU manual, "Backslashes that are not in danger
|
||||
// of quoting ‘%’ characters go unmolested."
|
||||
// How do you end a line with a backslash? The netbsd Make docs suggest
|
||||
// reading the result of a shell command echoing a backslash!
|
||||
//
|
||||
// Rather than implement all of above, we follow what GCC/Clang produces:
|
||||
// Backslashes escape a space or hash sign.
|
||||
// When a space is preceded by 2N+1 backslashes, it is represents N backslashes
|
||||
// followed by space.
|
||||
// When a space is preceded by 2N backslashes, it represents 2N backslashes at
|
||||
// the end of a filename.
|
||||
// A hash sign is escaped by a single backslash. All other backslashes remain
|
||||
// unchanged.
|
||||
//
|
||||
// If anyone actually has depfiles that rely on the more complicated
|
||||
// behavior we can adjust this.
|
||||
bool DepfileParser::Parse(string* content, string* err) {
|
||||
// in: current parser input point.
|
||||
// end: end of input.
|
||||
// parsing_targets: whether we are parsing targets or dependencies.
|
||||
char* in = &(*content)[0];
|
||||
char* end = in + content->size();
|
||||
bool have_target = false;
|
||||
bool parsing_targets = true;
|
||||
bool poisoned_input = false;
|
||||
bool is_empty = true;
|
||||
while (in < end) {
|
||||
bool have_newline = false;
|
||||
// out: current output point (typically same as in, but can fall behind
|
||||
// as we de-escape backslashes).
|
||||
char* out = in;
|
||||
// filename: start of the current parsed filename.
|
||||
char* filename = out;
|
||||
for (;;) {
|
||||
// start: beginning of the current parsed span.
|
||||
const char* start = in;
|
||||
char* yymarker = NULL;
|
||||
/*!re2c
|
||||
re2c:define:YYCTYPE = "unsigned char";
|
||||
re2c:define:YYCURSOR = in;
|
||||
re2c:define:YYLIMIT = end;
|
||||
re2c:define:YYMARKER = yymarker;
|
||||
|
||||
re2c:yyfill:enable = 0;
|
||||
|
||||
re2c:indent:top = 2;
|
||||
re2c:indent:string = " ";
|
||||
|
||||
nul = "\000";
|
||||
newline = '\r'?'\n';
|
||||
|
||||
'\\\\'* '\\ ' {
|
||||
// 2N+1 backslashes plus space -> N backslashes plus space.
|
||||
int len = (int)(in - start);
|
||||
int n = len / 2 - 1;
|
||||
if (out < start)
|
||||
memset(out, '\\', n);
|
||||
out += n;
|
||||
*out++ = ' ';
|
||||
continue;
|
||||
}
|
||||
'\\\\'+ ' ' {
|
||||
// 2N backslashes plus space -> 2N backslashes, end of filename.
|
||||
int len = (int)(in - start);
|
||||
if (out < start)
|
||||
memset(out, '\\', len - 1);
|
||||
out += len - 1;
|
||||
break;
|
||||
}
|
||||
'\\'+ '#' {
|
||||
// De-escape hash sign, but preserve other leading backslashes.
|
||||
int len = (int)(in - start);
|
||||
if (len > 2 && out < start)
|
||||
memset(out, '\\', len - 2);
|
||||
out += len - 2;
|
||||
*out++ = '#';
|
||||
continue;
|
||||
}
|
||||
'\\'+ ':' [\x00\x20\r\n\t] {
|
||||
// Backslash followed by : and whitespace.
|
||||
// It is therefore normal text and not an escaped colon
|
||||
int len = (int)(in - start - 1);
|
||||
// Need to shift it over if we're overwriting backslashes.
|
||||
if (out < start)
|
||||
memmove(out, start, len);
|
||||
out += len;
|
||||
if (*(in - 1) == '\n')
|
||||
have_newline = true;
|
||||
break;
|
||||
}
|
||||
'\\'+ ':' {
|
||||
// De-escape colon sign, but preserve other leading backslashes.
|
||||
// Regular expression uses lookahead to make sure that no whitespace
|
||||
// nor EOF follows. In that case it'd be the : at the end of a target
|
||||
int len = (int)(in - start);
|
||||
if (len > 2 && out < start)
|
||||
memset(out, '\\', len - 2);
|
||||
out += len - 2;
|
||||
*out++ = ':';
|
||||
continue;
|
||||
}
|
||||
'$$' {
|
||||
// De-escape dollar character.
|
||||
*out++ = '$';
|
||||
continue;
|
||||
}
|
||||
'\\'+ [^\000\r\n] | [a-zA-Z0-9+?"'&,/_:.~()}{%=@\x5B\x5D!\x80-\xFF-]+ {
|
||||
// Got a span of plain text.
|
||||
int len = (int)(in - start);
|
||||
// Need to shift it over if we're overwriting backslashes.
|
||||
if (out < start)
|
||||
memmove(out, start, len);
|
||||
out += len;
|
||||
continue;
|
||||
}
|
||||
nul {
|
||||
break;
|
||||
}
|
||||
'\\' newline {
|
||||
// A line continuation ends the current file name.
|
||||
break;
|
||||
}
|
||||
newline {
|
||||
// A newline ends the current file name and the current rule.
|
||||
have_newline = true;
|
||||
break;
|
||||
}
|
||||
[^] {
|
||||
// For any other character (e.g. whitespace), swallow it here,
|
||||
// allowing the outer logic to loop around again.
|
||||
break;
|
||||
}
|
||||
*/
|
||||
}
|
||||
|
||||
int len = (int)(out - filename);
|
||||
const bool is_dependency = !parsing_targets;
|
||||
if (len > 0 && filename[len - 1] == ':') {
|
||||
len--; // Strip off trailing colon, if any.
|
||||
parsing_targets = false;
|
||||
have_target = true;
|
||||
}
|
||||
|
||||
if (len > 0) {
|
||||
is_empty = false;
|
||||
StringPiece piece = StringPiece(filename, len);
|
||||
// If we've seen this as an input before, skip it.
|
||||
std::vector<StringPiece>::iterator pos = std::find(ins_.begin(), ins_.end(), piece);
|
||||
if (pos == ins_.end()) {
|
||||
if (is_dependency) {
|
||||
if (poisoned_input) {
|
||||
*err = "inputs may not also have inputs";
|
||||
return false;
|
||||
}
|
||||
// New input.
|
||||
ins_.push_back(piece);
|
||||
} else {
|
||||
// Check for a new output.
|
||||
if (std::find(outs_.begin(), outs_.end(), piece) == outs_.end())
|
||||
outs_.push_back(piece);
|
||||
}
|
||||
} else if (!is_dependency) {
|
||||
// We've passed an input on the left side; reject new inputs.
|
||||
poisoned_input = true;
|
||||
}
|
||||
}
|
||||
|
||||
if (have_newline) {
|
||||
// A newline ends a rule so the next filename will be a new target.
|
||||
parsing_targets = true;
|
||||
poisoned_input = false;
|
||||
}
|
||||
}
|
||||
if (!have_target && !is_empty) {
|
||||
*err = "expected ':' in depfile";
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
@@ -0,0 +1,79 @@
|
||||
// Copyright 2011 Google Inc. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#include <stdio.h>
|
||||
#include <stdlib.h>
|
||||
|
||||
#include "depfile_parser.h"
|
||||
#include "util.h"
|
||||
#include "metrics.h"
|
||||
|
||||
using namespace std;
|
||||
|
||||
int main(int argc, char* argv[]) {
|
||||
if (argc < 2) {
|
||||
printf("usage: %s <file1> <file2...>\n", argv[0]);
|
||||
return 1;
|
||||
}
|
||||
|
||||
vector<float> times;
|
||||
for (int i = 1; i < argc; ++i) {
|
||||
const char* filename = argv[i];
|
||||
|
||||
for (int limit = 1 << 10; limit < (1<<20); limit *= 2) {
|
||||
int64_t start = GetTimeMillis();
|
||||
for (int rep = 0; rep < limit; ++rep) {
|
||||
string buf;
|
||||
string err;
|
||||
if (ReadFile(filename, &buf, &err) < 0) {
|
||||
printf("%s: %s\n", filename, err.c_str());
|
||||
return 1;
|
||||
}
|
||||
|
||||
DepfileParser parser;
|
||||
if (!parser.Parse(&buf, &err)) {
|
||||
printf("%s: %s\n", filename, err.c_str());
|
||||
return 1;
|
||||
}
|
||||
}
|
||||
int64_t end = GetTimeMillis();
|
||||
|
||||
if (end - start > 100) {
|
||||
int delta = (int)(end - start);
|
||||
float time = delta*1000 / (float)limit;
|
||||
printf("%s: %.1fus\n", filename, time);
|
||||
times.push_back(time);
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (!times.empty()) {
|
||||
float min = times[0];
|
||||
float max = times[0];
|
||||
float total = 0;
|
||||
for (size_t i = 0; i < times.size(); ++i) {
|
||||
total += times[i];
|
||||
if (times[i] < min)
|
||||
min = times[i];
|
||||
else if (times[i] > max)
|
||||
max = times[i];
|
||||
}
|
||||
|
||||
printf("min %.1fus max %.1fus avg %.1fus\n",
|
||||
min, max, total / times.size());
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
@@ -0,0 +1,425 @@
|
||||
// Copyright 2011 Google Inc. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#include "depfile_parser.h"
|
||||
|
||||
#include "test.h"
|
||||
|
||||
using namespace std;
|
||||
|
||||
struct DepfileParserTest : public testing::Test {
|
||||
bool Parse(const char* input, string* err);
|
||||
|
||||
DepfileParser parser_;
|
||||
string input_;
|
||||
};
|
||||
|
||||
bool DepfileParserTest::Parse(const char* input, string* err) {
|
||||
input_ = input;
|
||||
return parser_.Parse(&input_, err);
|
||||
}
|
||||
|
||||
TEST_F(DepfileParserTest, Basic) {
|
||||
string err;
|
||||
EXPECT_TRUE(Parse(
|
||||
"build/ninja.o: ninja.cc ninja.h eval_env.h manifest_parser.h\n",
|
||||
&err));
|
||||
ASSERT_EQ("", err);
|
||||
ASSERT_EQ(1u, parser_.outs_.size());
|
||||
EXPECT_EQ("build/ninja.o", parser_.outs_[0].AsString());
|
||||
EXPECT_EQ(4u, parser_.ins_.size());
|
||||
}
|
||||
|
||||
TEST_F(DepfileParserTest, EarlyNewlineAndWhitespace) {
|
||||
string err;
|
||||
EXPECT_TRUE(Parse(
|
||||
" \\\n"
|
||||
" out: in\n",
|
||||
&err));
|
||||
ASSERT_EQ("", err);
|
||||
}
|
||||
|
||||
TEST_F(DepfileParserTest, Continuation) {
|
||||
string err;
|
||||
EXPECT_TRUE(Parse(
|
||||
"foo.o: \\\n"
|
||||
" bar.h baz.h\n",
|
||||
&err));
|
||||
ASSERT_EQ("", err);
|
||||
ASSERT_EQ(1u, parser_.outs_.size());
|
||||
EXPECT_EQ("foo.o", parser_.outs_[0].AsString());
|
||||
EXPECT_EQ(2u, parser_.ins_.size());
|
||||
}
|
||||
|
||||
TEST_F(DepfileParserTest, WindowsDrivePaths) {
|
||||
string err;
|
||||
EXPECT_TRUE(Parse("foo.o: //?/c:/bar.h\n", &err));
|
||||
ASSERT_EQ("", err);
|
||||
ASSERT_EQ(1u, parser_.outs_.size());
|
||||
EXPECT_EQ("foo.o", parser_.outs_[0].AsString());
|
||||
EXPECT_EQ(1u, parser_.ins_.size());
|
||||
EXPECT_EQ("//?/c:/bar.h", parser_.ins_[0].AsString());
|
||||
}
|
||||
|
||||
TEST_F(DepfileParserTest, AmpersandsAndQuotes) {
|
||||
string err;
|
||||
EXPECT_TRUE(Parse("foo&bar.o foo'bar.o foo\"bar.o: foo&bar.h foo'bar.h foo\"bar.h\n", &err));
|
||||
ASSERT_EQ("", err);
|
||||
ASSERT_EQ(3u, parser_.outs_.size());
|
||||
EXPECT_EQ("foo&bar.o", parser_.outs_[0].AsString());
|
||||
EXPECT_EQ("foo'bar.o", parser_.outs_[1].AsString());
|
||||
EXPECT_EQ("foo\"bar.o", parser_.outs_[2].AsString());
|
||||
EXPECT_EQ(3u, parser_.ins_.size());
|
||||
EXPECT_EQ("foo&bar.h", parser_.ins_[0].AsString());
|
||||
EXPECT_EQ("foo'bar.h", parser_.ins_[1].AsString());
|
||||
EXPECT_EQ("foo\"bar.h", parser_.ins_[2].AsString());
|
||||
}
|
||||
|
||||
TEST_F(DepfileParserTest, CarriageReturnContinuation) {
|
||||
string err;
|
||||
EXPECT_TRUE(Parse(
|
||||
"foo.o: \\\r\n"
|
||||
" bar.h baz.h\r\n",
|
||||
&err));
|
||||
ASSERT_EQ("", err);
|
||||
ASSERT_EQ(1u, parser_.outs_.size());
|
||||
EXPECT_EQ("foo.o", parser_.outs_[0].AsString());
|
||||
EXPECT_EQ(2u, parser_.ins_.size());
|
||||
}
|
||||
|
||||
TEST_F(DepfileParserTest, BackSlashes) {
|
||||
string err;
|
||||
EXPECT_TRUE(Parse(
|
||||
"Project\\Dir\\Build\\Release8\\Foo\\Foo.res : \\\n"
|
||||
" Dir\\Library\\Foo.rc \\\n"
|
||||
" Dir\\Library\\Version\\Bar.h \\\n"
|
||||
" Dir\\Library\\Foo.ico \\\n"
|
||||
" Project\\Thing\\Bar.tlb \\\n",
|
||||
&err));
|
||||
ASSERT_EQ("", err);
|
||||
ASSERT_EQ(1u, parser_.outs_.size());
|
||||
EXPECT_EQ("Project\\Dir\\Build\\Release8\\Foo\\Foo.res",
|
||||
parser_.outs_[0].AsString());
|
||||
EXPECT_EQ(4u, parser_.ins_.size());
|
||||
}
|
||||
|
||||
TEST_F(DepfileParserTest, Spaces) {
|
||||
string err;
|
||||
EXPECT_TRUE(Parse(
|
||||
"a\\ bc\\ def: a\\ b c d",
|
||||
&err));
|
||||
ASSERT_EQ("", err);
|
||||
ASSERT_EQ(1u, parser_.outs_.size());
|
||||
EXPECT_EQ("a bc def",
|
||||
parser_.outs_[0].AsString());
|
||||
ASSERT_EQ(3u, parser_.ins_.size());
|
||||
EXPECT_EQ("a b",
|
||||
parser_.ins_[0].AsString());
|
||||
EXPECT_EQ("c",
|
||||
parser_.ins_[1].AsString());
|
||||
EXPECT_EQ("d",
|
||||
parser_.ins_[2].AsString());
|
||||
}
|
||||
|
||||
TEST_F(DepfileParserTest, MultipleBackslashes) {
|
||||
// Successive 2N+1 backslashes followed by space (' ') are replaced by N >= 0
|
||||
// backslashes and the space. A single backslash before hash sign is removed.
|
||||
// Other backslashes remain untouched (including 2N backslashes followed by
|
||||
// space).
|
||||
string err;
|
||||
EXPECT_TRUE(Parse(
|
||||
"a\\ b\\#c.h: \\\\\\\\\\ \\\\\\\\ \\\\share\\info\\\\#1",
|
||||
&err));
|
||||
ASSERT_EQ("", err);
|
||||
ASSERT_EQ(1u, parser_.outs_.size());
|
||||
EXPECT_EQ("a b#c.h",
|
||||
parser_.outs_[0].AsString());
|
||||
ASSERT_EQ(3u, parser_.ins_.size());
|
||||
EXPECT_EQ("\\\\ ",
|
||||
parser_.ins_[0].AsString());
|
||||
EXPECT_EQ("\\\\\\\\",
|
||||
parser_.ins_[1].AsString());
|
||||
EXPECT_EQ("\\\\share\\info\\#1",
|
||||
parser_.ins_[2].AsString());
|
||||
}
|
||||
|
||||
TEST_F(DepfileParserTest, Escapes) {
|
||||
// Put backslashes before a variety of characters, see which ones make
|
||||
// it through.
|
||||
string err;
|
||||
EXPECT_TRUE(Parse(
|
||||
"\\!\\@\\#$$\\%\\^\\&\\[\\]\\\\:",
|
||||
&err));
|
||||
ASSERT_EQ("", err);
|
||||
ASSERT_EQ(1u, parser_.outs_.size());
|
||||
EXPECT_EQ("\\!\\@#$\\%\\^\\&\\[\\]\\\\",
|
||||
parser_.outs_[0].AsString());
|
||||
ASSERT_EQ(0u, parser_.ins_.size());
|
||||
}
|
||||
|
||||
TEST_F(DepfileParserTest, EscapedColons)
|
||||
{
|
||||
std::string err;
|
||||
// Tests for correct parsing of depfiles produced on Windows
|
||||
// by both Clang, GCC pre 10 and GCC 10
|
||||
EXPECT_TRUE(Parse(
|
||||
"c\\:\\gcc\\x86_64-w64-mingw32\\include\\stddef.o: \\\n"
|
||||
" c:\\gcc\\x86_64-w64-mingw32\\include\\stddef.h \n",
|
||||
&err));
|
||||
ASSERT_EQ("", err);
|
||||
ASSERT_EQ(1u, parser_.outs_.size());
|
||||
EXPECT_EQ("c:\\gcc\\x86_64-w64-mingw32\\include\\stddef.o",
|
||||
parser_.outs_[0].AsString());
|
||||
ASSERT_EQ(1u, parser_.ins_.size());
|
||||
EXPECT_EQ("c:\\gcc\\x86_64-w64-mingw32\\include\\stddef.h",
|
||||
parser_.ins_[0].AsString());
|
||||
}
|
||||
|
||||
TEST_F(DepfileParserTest, EscapedTargetColon)
|
||||
{
|
||||
std::string err;
|
||||
EXPECT_TRUE(Parse(
|
||||
"foo1\\: x\n"
|
||||
"foo1\\:\n"
|
||||
"foo1\\:\r\n"
|
||||
"foo1\\:\t\n"
|
||||
"foo1\\:",
|
||||
&err));
|
||||
ASSERT_EQ("", err);
|
||||
ASSERT_EQ(1u, parser_.outs_.size());
|
||||
EXPECT_EQ("foo1\\", parser_.outs_[0].AsString());
|
||||
ASSERT_EQ(1u, parser_.ins_.size());
|
||||
EXPECT_EQ("x", parser_.ins_[0].AsString());
|
||||
}
|
||||
|
||||
TEST_F(DepfileParserTest, SpecialChars) {
|
||||
// See filenames like istreambuf.iterator_op!= in
|
||||
// https://github.com/google/libcxx/tree/master/test/iterators/stream.iterators/istreambuf.iterator/
|
||||
string err;
|
||||
EXPECT_TRUE(Parse(
|
||||
"C:/Program\\ Files\\ (x86)/Microsoft\\ crtdefs.h: \\\n"
|
||||
" en@quot.header~ t+t-x!=1 \\\n"
|
||||
" openldap/slapd.d/cn=config/cn=schema/cn={0}core.ldif\\\n"
|
||||
" Fu\303\244ball\\\n"
|
||||
" a[1]b@2%c",
|
||||
&err));
|
||||
ASSERT_EQ("", err);
|
||||
ASSERT_EQ(1u, parser_.outs_.size());
|
||||
EXPECT_EQ("C:/Program Files (x86)/Microsoft crtdefs.h",
|
||||
parser_.outs_[0].AsString());
|
||||
ASSERT_EQ(5u, parser_.ins_.size());
|
||||
EXPECT_EQ("en@quot.header~",
|
||||
parser_.ins_[0].AsString());
|
||||
EXPECT_EQ("t+t-x!=1",
|
||||
parser_.ins_[1].AsString());
|
||||
EXPECT_EQ("openldap/slapd.d/cn=config/cn=schema/cn={0}core.ldif",
|
||||
parser_.ins_[2].AsString());
|
||||
EXPECT_EQ("Fu\303\244ball",
|
||||
parser_.ins_[3].AsString());
|
||||
EXPECT_EQ("a[1]b@2%c",
|
||||
parser_.ins_[4].AsString());
|
||||
}
|
||||
|
||||
TEST_F(DepfileParserTest, UnifyMultipleOutputs) {
|
||||
// check that multiple duplicate targets are properly unified
|
||||
string err;
|
||||
EXPECT_TRUE(Parse("foo foo: x y z", &err));
|
||||
ASSERT_EQ(1u, parser_.outs_.size());
|
||||
ASSERT_EQ("foo", parser_.outs_[0].AsString());
|
||||
ASSERT_EQ(3u, parser_.ins_.size());
|
||||
EXPECT_EQ("x", parser_.ins_[0].AsString());
|
||||
EXPECT_EQ("y", parser_.ins_[1].AsString());
|
||||
EXPECT_EQ("z", parser_.ins_[2].AsString());
|
||||
}
|
||||
|
||||
TEST_F(DepfileParserTest, MultipleDifferentOutputs) {
|
||||
// check that multiple different outputs are accepted by the parser
|
||||
string err;
|
||||
EXPECT_TRUE(Parse("foo bar: x y z", &err));
|
||||
ASSERT_EQ(2u, parser_.outs_.size());
|
||||
ASSERT_EQ("foo", parser_.outs_[0].AsString());
|
||||
ASSERT_EQ("bar", parser_.outs_[1].AsString());
|
||||
ASSERT_EQ(3u, parser_.ins_.size());
|
||||
EXPECT_EQ("x", parser_.ins_[0].AsString());
|
||||
EXPECT_EQ("y", parser_.ins_[1].AsString());
|
||||
EXPECT_EQ("z", parser_.ins_[2].AsString());
|
||||
}
|
||||
|
||||
TEST_F(DepfileParserTest, MultipleEmptyRules) {
|
||||
string err;
|
||||
EXPECT_TRUE(Parse("foo: x\n"
|
||||
"foo: \n"
|
||||
"foo:\n", &err));
|
||||
ASSERT_EQ(1u, parser_.outs_.size());
|
||||
ASSERT_EQ("foo", parser_.outs_[0].AsString());
|
||||
ASSERT_EQ(1u, parser_.ins_.size());
|
||||
EXPECT_EQ("x", parser_.ins_[0].AsString());
|
||||
}
|
||||
|
||||
TEST_F(DepfileParserTest, UnifyMultipleRulesLF) {
|
||||
string err;
|
||||
EXPECT_TRUE(Parse("foo: x\n"
|
||||
"foo: y\n"
|
||||
"foo \\\n"
|
||||
"foo: z\n", &err));
|
||||
ASSERT_EQ(1u, parser_.outs_.size());
|
||||
ASSERT_EQ("foo", parser_.outs_[0].AsString());
|
||||
ASSERT_EQ(3u, parser_.ins_.size());
|
||||
EXPECT_EQ("x", parser_.ins_[0].AsString());
|
||||
EXPECT_EQ("y", parser_.ins_[1].AsString());
|
||||
EXPECT_EQ("z", parser_.ins_[2].AsString());
|
||||
}
|
||||
|
||||
TEST_F(DepfileParserTest, UnifyMultipleRulesCRLF) {
|
||||
string err;
|
||||
EXPECT_TRUE(Parse("foo: x\r\n"
|
||||
"foo: y\r\n"
|
||||
"foo \\\r\n"
|
||||
"foo: z\r\n", &err));
|
||||
ASSERT_EQ(1u, parser_.outs_.size());
|
||||
ASSERT_EQ("foo", parser_.outs_[0].AsString());
|
||||
ASSERT_EQ(3u, parser_.ins_.size());
|
||||
EXPECT_EQ("x", parser_.ins_[0].AsString());
|
||||
EXPECT_EQ("y", parser_.ins_[1].AsString());
|
||||
EXPECT_EQ("z", parser_.ins_[2].AsString());
|
||||
}
|
||||
|
||||
TEST_F(DepfileParserTest, UnifyMixedRulesLF) {
|
||||
string err;
|
||||
EXPECT_TRUE(Parse("foo: x\\\n"
|
||||
" y\n"
|
||||
"foo \\\n"
|
||||
"foo: z\n", &err));
|
||||
ASSERT_EQ(1u, parser_.outs_.size());
|
||||
ASSERT_EQ("foo", parser_.outs_[0].AsString());
|
||||
ASSERT_EQ(3u, parser_.ins_.size());
|
||||
EXPECT_EQ("x", parser_.ins_[0].AsString());
|
||||
EXPECT_EQ("y", parser_.ins_[1].AsString());
|
||||
EXPECT_EQ("z", parser_.ins_[2].AsString());
|
||||
}
|
||||
|
||||
TEST_F(DepfileParserTest, UnifyMixedRulesCRLF) {
|
||||
string err;
|
||||
EXPECT_TRUE(Parse("foo: x\\\r\n"
|
||||
" y\r\n"
|
||||
"foo \\\r\n"
|
||||
"foo: z\r\n", &err));
|
||||
ASSERT_EQ(1u, parser_.outs_.size());
|
||||
ASSERT_EQ("foo", parser_.outs_[0].AsString());
|
||||
ASSERT_EQ(3u, parser_.ins_.size());
|
||||
EXPECT_EQ("x", parser_.ins_[0].AsString());
|
||||
EXPECT_EQ("y", parser_.ins_[1].AsString());
|
||||
EXPECT_EQ("z", parser_.ins_[2].AsString());
|
||||
}
|
||||
|
||||
TEST_F(DepfileParserTest, IndentedRulesLF) {
|
||||
string err;
|
||||
EXPECT_TRUE(Parse(" foo: x\n"
|
||||
" foo: y\n"
|
||||
" foo: z\n", &err));
|
||||
ASSERT_EQ(1u, parser_.outs_.size());
|
||||
ASSERT_EQ("foo", parser_.outs_[0].AsString());
|
||||
ASSERT_EQ(3u, parser_.ins_.size());
|
||||
EXPECT_EQ("x", parser_.ins_[0].AsString());
|
||||
EXPECT_EQ("y", parser_.ins_[1].AsString());
|
||||
EXPECT_EQ("z", parser_.ins_[2].AsString());
|
||||
}
|
||||
|
||||
TEST_F(DepfileParserTest, IndentedRulesCRLF) {
|
||||
string err;
|
||||
EXPECT_TRUE(Parse(" foo: x\r\n"
|
||||
" foo: y\r\n"
|
||||
" foo: z\r\n", &err));
|
||||
ASSERT_EQ(1u, parser_.outs_.size());
|
||||
ASSERT_EQ("foo", parser_.outs_[0].AsString());
|
||||
ASSERT_EQ(3u, parser_.ins_.size());
|
||||
EXPECT_EQ("x", parser_.ins_[0].AsString());
|
||||
EXPECT_EQ("y", parser_.ins_[1].AsString());
|
||||
EXPECT_EQ("z", parser_.ins_[2].AsString());
|
||||
}
|
||||
|
||||
TEST_F(DepfileParserTest, TolerateMP) {
|
||||
string err;
|
||||
EXPECT_TRUE(Parse("foo: x y z\n"
|
||||
"x:\n"
|
||||
"y:\n"
|
||||
"z:\n", &err));
|
||||
ASSERT_EQ(1u, parser_.outs_.size());
|
||||
ASSERT_EQ("foo", parser_.outs_[0].AsString());
|
||||
ASSERT_EQ(3u, parser_.ins_.size());
|
||||
EXPECT_EQ("x", parser_.ins_[0].AsString());
|
||||
EXPECT_EQ("y", parser_.ins_[1].AsString());
|
||||
EXPECT_EQ("z", parser_.ins_[2].AsString());
|
||||
}
|
||||
|
||||
TEST_F(DepfileParserTest, MultipleRulesTolerateMP) {
|
||||
string err;
|
||||
EXPECT_TRUE(Parse("foo: x\n"
|
||||
"x:\n"
|
||||
"foo: y\n"
|
||||
"y:\n"
|
||||
"foo: z\n"
|
||||
"z:\n", &err));
|
||||
ASSERT_EQ(1u, parser_.outs_.size());
|
||||
ASSERT_EQ("foo", parser_.outs_[0].AsString());
|
||||
ASSERT_EQ(3u, parser_.ins_.size());
|
||||
EXPECT_EQ("x", parser_.ins_[0].AsString());
|
||||
EXPECT_EQ("y", parser_.ins_[1].AsString());
|
||||
EXPECT_EQ("z", parser_.ins_[2].AsString());
|
||||
}
|
||||
|
||||
TEST_F(DepfileParserTest, MultipleRulesDifferentOutputs) {
|
||||
// check that multiple different outputs are accepted by the parser
|
||||
// when spread across multiple rules
|
||||
string err;
|
||||
EXPECT_TRUE(Parse("foo: x y\n"
|
||||
"bar: y z\n", &err));
|
||||
ASSERT_EQ(2u, parser_.outs_.size());
|
||||
ASSERT_EQ("foo", parser_.outs_[0].AsString());
|
||||
ASSERT_EQ("bar", parser_.outs_[1].AsString());
|
||||
ASSERT_EQ(3u, parser_.ins_.size());
|
||||
EXPECT_EQ("x", parser_.ins_[0].AsString());
|
||||
EXPECT_EQ("y", parser_.ins_[1].AsString());
|
||||
EXPECT_EQ("z", parser_.ins_[2].AsString());
|
||||
}
|
||||
|
||||
TEST_F(DepfileParserTest, BuggyMP) {
|
||||
std::string err;
|
||||
EXPECT_FALSE(Parse("foo: x y z\n"
|
||||
"x: alsoin\n"
|
||||
"y:\n"
|
||||
"z:\n", &err));
|
||||
ASSERT_EQ("inputs may not also have inputs", err);
|
||||
}
|
||||
|
||||
TEST_F(DepfileParserTest, EmptyFile) {
|
||||
std::string err;
|
||||
EXPECT_TRUE(Parse("", &err));
|
||||
ASSERT_EQ(0u, parser_.outs_.size());
|
||||
ASSERT_EQ(0u, parser_.ins_.size());
|
||||
}
|
||||
|
||||
TEST_F(DepfileParserTest, EmptyLines) {
|
||||
std::string err;
|
||||
EXPECT_TRUE(Parse("\n\n", &err));
|
||||
ASSERT_EQ(0u, parser_.outs_.size());
|
||||
ASSERT_EQ(0u, parser_.ins_.size());
|
||||
}
|
||||
|
||||
TEST_F(DepfileParserTest, MissingColon) {
|
||||
// The file is not empty but is missing a colon separator.
|
||||
std::string err;
|
||||
EXPECT_FALSE(Parse("foo.o foo.c\n", &err));
|
||||
EXPECT_EQ("expected ':' in depfile", err);
|
||||
}
|
||||
@@ -0,0 +1,466 @@
|
||||
// Copyright 2012 Google Inc. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#include "deps_log.h"
|
||||
|
||||
#include <assert.h>
|
||||
#include <errno.h>
|
||||
#include <stdint.h>
|
||||
#include <stdio.h>
|
||||
#include <string.h>
|
||||
#ifndef _WIN32
|
||||
#include <unistd.h>
|
||||
#elif defined(_MSC_VER) && (_MSC_VER < 1900)
|
||||
typedef __int32 int32_t;
|
||||
typedef unsigned __int32 uint32_t;
|
||||
#endif
|
||||
|
||||
#include "graph.h"
|
||||
#include "metrics.h"
|
||||
#include "state.h"
|
||||
#include "util.h"
|
||||
|
||||
using namespace std;
|
||||
|
||||
// The version is stored as 4 bytes after the signature and also serves as a
|
||||
// byte order mark. Signature and version combined are 16 bytes long.
|
||||
static const char kFileSignature[] = "# ninjadeps\n";
|
||||
static const size_t kFileSignatureSize = sizeof(kFileSignature) - 1u;
|
||||
|
||||
static const int32_t kCurrentVersion = 4;
|
||||
|
||||
// Record size is currently limited to less than the full 32 bit, due to
|
||||
// internal buffers having to have this size.
|
||||
static constexpr size_t kMaxRecordSize = (1 << 19) - 1;
|
||||
|
||||
DepsLog::~DepsLog() {
|
||||
Close();
|
||||
}
|
||||
|
||||
bool DepsLog::OpenForWrite(const string& path, string* err) {
|
||||
if (needs_recompaction_) {
|
||||
if (!Recompact(path, err))
|
||||
return false;
|
||||
}
|
||||
|
||||
assert(!file_);
|
||||
file_path_ = path; // we don't actually open the file right now, but will do
|
||||
// so on the first write attempt
|
||||
return true;
|
||||
}
|
||||
|
||||
bool DepsLog::RecordDeps(Node* node, TimeStamp mtime,
|
||||
const vector<Node*>& nodes) {
|
||||
return RecordDeps(node, mtime, static_cast<int>(nodes.size()), nodes.data());
|
||||
}
|
||||
|
||||
bool DepsLog::RecordDeps(Node* node, TimeStamp mtime, int node_count,
|
||||
Node* const* nodes) {
|
||||
// Track whether there's any new data to be recorded.
|
||||
bool made_change = false;
|
||||
|
||||
// Assign ids to all nodes that are missing one.
|
||||
if (node->id() < 0) {
|
||||
if (!RecordId(node))
|
||||
return false;
|
||||
made_change = true;
|
||||
}
|
||||
for (int i = 0; i < node_count; ++i) {
|
||||
if (nodes[i]->id() < 0) {
|
||||
if (!RecordId(nodes[i]))
|
||||
return false;
|
||||
made_change = true;
|
||||
}
|
||||
}
|
||||
|
||||
// See if the new data is different than the existing data, if any.
|
||||
if (!made_change) {
|
||||
Deps* deps = GetDeps(node);
|
||||
if (!deps ||
|
||||
deps->mtime != mtime ||
|
||||
deps->node_count != node_count) {
|
||||
made_change = true;
|
||||
} else {
|
||||
for (int i = 0; i < node_count; ++i) {
|
||||
if (deps->nodes[i] != nodes[i]) {
|
||||
made_change = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Don't write anything if there's no new info.
|
||||
if (!made_change)
|
||||
return true;
|
||||
|
||||
// Update on-disk representation.
|
||||
unsigned size = 4 * (1 + 2 + node_count);
|
||||
if (size > kMaxRecordSize) {
|
||||
errno = ERANGE;
|
||||
return false;
|
||||
}
|
||||
|
||||
if (!OpenForWriteIfNeeded()) {
|
||||
return false;
|
||||
}
|
||||
size |= 0x80000000; // Deps record: set high bit.
|
||||
if (fwrite(&size, 4, 1, file_) < 1)
|
||||
return false;
|
||||
int id = node->id();
|
||||
if (fwrite(&id, 4, 1, file_) < 1)
|
||||
return false;
|
||||
uint32_t mtime_part = static_cast<uint32_t>(mtime & 0xffffffff);
|
||||
if (fwrite(&mtime_part, 4, 1, file_) < 1)
|
||||
return false;
|
||||
mtime_part = static_cast<uint32_t>((mtime >> 32) & 0xffffffff);
|
||||
if (fwrite(&mtime_part, 4, 1, file_) < 1)
|
||||
return false;
|
||||
for (int i = 0; i < node_count; ++i) {
|
||||
id = nodes[i]->id();
|
||||
if (fwrite(&id, 4, 1, file_) < 1)
|
||||
return false;
|
||||
}
|
||||
if (fflush(file_) != 0)
|
||||
return false;
|
||||
|
||||
// Update in-memory representation.
|
||||
Deps* deps = new Deps(mtime, node_count);
|
||||
for (int i = 0; i < node_count; ++i)
|
||||
deps->nodes[i] = nodes[i];
|
||||
UpdateDeps(node->id(), deps);
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
void DepsLog::Close() {
|
||||
OpenForWriteIfNeeded(); // create the file even if nothing has been recorded
|
||||
if (file_)
|
||||
fclose(file_);
|
||||
file_ = NULL;
|
||||
}
|
||||
|
||||
LoadStatus DepsLog::Load(const string& path, State* state, string* err) {
|
||||
METRIC_RECORD(".ninja_deps load");
|
||||
char buf[kMaxRecordSize + 1];
|
||||
FILE* f = fopen(path.c_str(), "rb");
|
||||
if (!f) {
|
||||
if (errno == ENOENT)
|
||||
return LOAD_NOT_FOUND;
|
||||
*err = strerror(errno);
|
||||
return LOAD_ERROR;
|
||||
}
|
||||
|
||||
bool valid_header = fread(buf, kFileSignatureSize, 1, f) == 1 &&
|
||||
!memcmp(buf, kFileSignature, kFileSignatureSize);
|
||||
|
||||
int32_t version = 0;
|
||||
bool valid_version =
|
||||
fread(&version, 4, 1, f) == 1 && version == kCurrentVersion;
|
||||
|
||||
// Note: For version differences, this should migrate to the new format.
|
||||
// But the v1 format could sometimes (rarely) end up with invalid data, so
|
||||
// don't migrate v1 to v3 to force a rebuild. (v2 only existed for a few days,
|
||||
// and there was no release with it, so pretend that it never happened.)
|
||||
if (!valid_header || !valid_version) {
|
||||
if (version == 1)
|
||||
*err = "deps log version change; rebuilding";
|
||||
else
|
||||
*err = "bad deps log signature or version; starting over";
|
||||
fclose(f);
|
||||
platformAwareUnlink(path.c_str());
|
||||
// Don't report this as a failure. An empty deps log will cause
|
||||
// us to rebuild the outputs anyway.
|
||||
return LOAD_SUCCESS;
|
||||
}
|
||||
|
||||
long offset = ftell(f);
|
||||
bool read_failed = false;
|
||||
int unique_dep_record_count = 0;
|
||||
int total_dep_record_count = 0;
|
||||
for (;;) {
|
||||
unsigned size;
|
||||
if (fread(&size, sizeof(size), 1, f) < 1) {
|
||||
if (!feof(f))
|
||||
read_failed = true;
|
||||
break;
|
||||
}
|
||||
bool is_deps = (size >> 31) != 0;
|
||||
size = size & 0x7FFFFFFF;
|
||||
|
||||
if (size > kMaxRecordSize || fread(buf, size, 1, f) < 1) {
|
||||
read_failed = true;
|
||||
break;
|
||||
}
|
||||
offset += size + sizeof(size);
|
||||
|
||||
if (is_deps) {
|
||||
if ((size % 4) != 0) {
|
||||
read_failed = true;
|
||||
break;
|
||||
}
|
||||
int* deps_data = reinterpret_cast<int*>(buf);
|
||||
int out_id = deps_data[0];
|
||||
TimeStamp mtime;
|
||||
mtime = (TimeStamp)(((uint64_t)(unsigned int)deps_data[2] << 32) |
|
||||
(uint64_t)(unsigned int)deps_data[1]);
|
||||
deps_data += 3;
|
||||
int deps_count = (size / 4) - 3;
|
||||
|
||||
for (int i = 0; i < deps_count; ++i) {
|
||||
int node_id = deps_data[i];
|
||||
if (node_id >= (int)nodes_.size() || !nodes_[node_id]) {
|
||||
read_failed = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (read_failed)
|
||||
break;
|
||||
|
||||
Deps* deps = new Deps(mtime, deps_count);
|
||||
for (int i = 0; i < deps_count; ++i) {
|
||||
deps->nodes[i] = nodes_[deps_data[i]];
|
||||
}
|
||||
|
||||
total_dep_record_count++;
|
||||
if (!UpdateDeps(out_id, deps))
|
||||
++unique_dep_record_count;
|
||||
} else {
|
||||
int path_size = size - 4;
|
||||
if (path_size <= 0) {
|
||||
read_failed = true;
|
||||
break;
|
||||
}
|
||||
// There can be up to 3 bytes of padding.
|
||||
if (buf[path_size - 1] == '\0') --path_size;
|
||||
if (buf[path_size - 1] == '\0') --path_size;
|
||||
if (buf[path_size - 1] == '\0') --path_size;
|
||||
StringPiece subpath(buf, path_size);
|
||||
// It is not necessary to pass in a correct slash_bits here. It will
|
||||
// either be a Node that's in the manifest (in which case it will already
|
||||
// have a correct slash_bits that GetNode will look up), or it is an
|
||||
// implicit dependency from a .d which does not affect the build command
|
||||
// (and so need not have its slashes maintained).
|
||||
Node* node = state->GetNode(subpath, 0);
|
||||
|
||||
// Check that the expected index matches the actual index. This can only
|
||||
// happen if two ninja processes write to the same deps log concurrently.
|
||||
// (This uses unary complement to make the checksum look less like a
|
||||
// dependency record entry.)
|
||||
unsigned checksum = *reinterpret_cast<unsigned*>(buf + size - 4);
|
||||
int expected_id = ~checksum;
|
||||
int id = static_cast<int>(nodes_.size());
|
||||
if (id != expected_id || node->id() >= 0) {
|
||||
read_failed = true;
|
||||
break;
|
||||
}
|
||||
node->set_id(id);
|
||||
nodes_.push_back(node);
|
||||
}
|
||||
}
|
||||
|
||||
if (read_failed) {
|
||||
// An error occurred while loading; try to recover by truncating the
|
||||
// file to the last fully-read record.
|
||||
if (ferror(f)) {
|
||||
*err = strerror(ferror(f));
|
||||
} else {
|
||||
*err = "premature end of file";
|
||||
}
|
||||
fclose(f);
|
||||
|
||||
if (!Truncate(path, offset, err))
|
||||
return LOAD_ERROR;
|
||||
|
||||
// The truncate succeeded; we'll just report the load error as a
|
||||
// warning because the build can proceed.
|
||||
*err += "; recovering";
|
||||
return LOAD_SUCCESS;
|
||||
}
|
||||
|
||||
fclose(f);
|
||||
|
||||
// Rebuild the log if there are too many dead records.
|
||||
int kMinCompactionEntryCount = 1000;
|
||||
int kCompactionRatio = 3;
|
||||
if (total_dep_record_count > kMinCompactionEntryCount &&
|
||||
total_dep_record_count > unique_dep_record_count * kCompactionRatio) {
|
||||
needs_recompaction_ = true;
|
||||
}
|
||||
|
||||
return LOAD_SUCCESS;
|
||||
}
|
||||
|
||||
DepsLog::Deps* DepsLog::GetDeps(Node* node) {
|
||||
// Abort if the node has no id (never referenced in the deps) or if
|
||||
// there's no deps recorded for the node.
|
||||
if (node->id() < 0 || node->id() >= (int)deps_.size())
|
||||
return NULL;
|
||||
return deps_[node->id()];
|
||||
}
|
||||
|
||||
Node* DepsLog::GetFirstReverseDepsNode(Node* node) {
|
||||
for (size_t id = 0; id < deps_.size(); ++id) {
|
||||
Deps* deps = deps_[id];
|
||||
if (!deps)
|
||||
continue;
|
||||
for (int i = 0; i < deps->node_count; ++i) {
|
||||
if (deps->nodes[i] == node)
|
||||
return nodes_[id];
|
||||
}
|
||||
}
|
||||
return NULL;
|
||||
}
|
||||
|
||||
bool DepsLog::Recompact(const string& path, string* err) {
|
||||
METRIC_RECORD(".ninja_deps recompact");
|
||||
|
||||
Close();
|
||||
string temp_path = path + ".recompact";
|
||||
|
||||
// OpenForWrite() opens for append. Make sure it's not appending to a
|
||||
// left-over file from a previous recompaction attempt that crashed somehow.
|
||||
platformAwareUnlink(temp_path.c_str());
|
||||
|
||||
DepsLog new_log;
|
||||
if (!new_log.OpenForWrite(temp_path, err))
|
||||
return false;
|
||||
|
||||
// Clear all known ids so that new ones can be reassigned. The new indices
|
||||
// will refer to the ordering in new_log, not in the current log.
|
||||
for (vector<Node*>::iterator i = nodes_.begin(); i != nodes_.end(); ++i)
|
||||
(*i)->set_id(-1);
|
||||
|
||||
// Write out all deps again.
|
||||
for (int old_id = 0; old_id < (int)deps_.size(); ++old_id) {
|
||||
Deps* deps = deps_[old_id];
|
||||
if (!deps) continue; // If nodes_[old_id] is a leaf, it has no deps.
|
||||
|
||||
if (!IsDepsEntryLiveFor(nodes_[old_id]))
|
||||
continue;
|
||||
|
||||
if (!new_log.RecordDeps(nodes_[old_id], deps->mtime,
|
||||
deps->node_count, deps->nodes)) {
|
||||
new_log.Close();
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
new_log.Close();
|
||||
|
||||
// All nodes now have ids that refer to new_log, so steal its data.
|
||||
deps_.swap(new_log.deps_);
|
||||
nodes_.swap(new_log.nodes_);
|
||||
|
||||
if (platformAwareUnlink(path.c_str()) < 0) {
|
||||
*err = strerror(errno);
|
||||
return false;
|
||||
}
|
||||
|
||||
if (rename(temp_path.c_str(), path.c_str()) < 0) {
|
||||
*err = strerror(errno);
|
||||
return false;
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
bool DepsLog::IsDepsEntryLiveFor(const Node* node) {
|
||||
// Skip entries that don't have in-edges or whose edges don't have a
|
||||
// "deps" attribute. They were in the deps log from previous builds, but
|
||||
// the the files they were for were removed from the build and their deps
|
||||
// entries are no longer needed.
|
||||
// (Without the check for "deps", a chain of two or more nodes that each
|
||||
// had deps wouldn't be collected in a single recompaction.)
|
||||
return node->in_edge() && !node->in_edge()->GetBinding("deps").empty();
|
||||
}
|
||||
|
||||
bool DepsLog::UpdateDeps(int out_id, Deps* deps) {
|
||||
if (out_id >= (int)deps_.size())
|
||||
deps_.resize(out_id + 1);
|
||||
|
||||
bool delete_old = deps_[out_id] != NULL;
|
||||
if (delete_old)
|
||||
delete deps_[out_id];
|
||||
deps_[out_id] = deps;
|
||||
return delete_old;
|
||||
}
|
||||
|
||||
bool DepsLog::RecordId(Node* node) {
|
||||
int path_size = static_cast<int>(node->path().size());
|
||||
assert(path_size > 0 && "Trying to record empty path Node!");
|
||||
int padding = (4 - path_size % 4) % 4; // Pad path to 4 byte boundary.
|
||||
|
||||
unsigned size = path_size + padding + 4;
|
||||
if (size > kMaxRecordSize) {
|
||||
errno = ERANGE;
|
||||
return false;
|
||||
}
|
||||
|
||||
if (!OpenForWriteIfNeeded()) {
|
||||
return false;
|
||||
}
|
||||
if (fwrite(&size, 4, 1, file_) < 1)
|
||||
return false;
|
||||
if (fwrite(node->path().data(), path_size, 1, file_) < 1) {
|
||||
return false;
|
||||
}
|
||||
if (padding && fwrite("\0\0", padding, 1, file_) < 1)
|
||||
return false;
|
||||
int id = static_cast<int>(nodes_.size());
|
||||
unsigned checksum = ~(unsigned)id;
|
||||
if (fwrite(&checksum, 4, 1, file_) < 1)
|
||||
return false;
|
||||
if (fflush(file_) != 0)
|
||||
return false;
|
||||
|
||||
node->set_id(id);
|
||||
nodes_.push_back(node);
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
bool DepsLog::OpenForWriteIfNeeded() {
|
||||
if (file_path_.empty()) {
|
||||
return true;
|
||||
}
|
||||
file_ = fopen(file_path_.c_str(), "ab");
|
||||
if (!file_) {
|
||||
return false;
|
||||
}
|
||||
// Set the buffer size to this and flush the file buffer after every record
|
||||
// to make sure records aren't written partially.
|
||||
if (setvbuf(file_, NULL, _IOFBF, kMaxRecordSize + 1) != 0) {
|
||||
return false;
|
||||
}
|
||||
SetCloseOnExec(fileno(file_));
|
||||
|
||||
// Opening a file in append mode doesn't set the file pointer to the file's
|
||||
// end on Windows. Do that explicitly.
|
||||
fseek(file_, 0, SEEK_END);
|
||||
|
||||
if (ftell(file_) == 0) {
|
||||
if (fwrite(kFileSignature, sizeof(kFileSignature) - 1, 1, file_) < 1) {
|
||||
return false;
|
||||
}
|
||||
if (fwrite(&kCurrentVersion, 4, 1, file_) < 1) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
if (fflush(file_) != 0) {
|
||||
return false;
|
||||
}
|
||||
file_path_.clear();
|
||||
return true;
|
||||
}
|
||||
@@ -0,0 +1,130 @@
|
||||
// Copyright 2012 Google Inc. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#ifndef NINJA_DEPS_LOG_H_
|
||||
#define NINJA_DEPS_LOG_H_
|
||||
|
||||
#include <string>
|
||||
#include <vector>
|
||||
|
||||
#include <stdio.h>
|
||||
|
||||
#include "load_status.h"
|
||||
#include "timestamp.h"
|
||||
|
||||
struct Node;
|
||||
struct State;
|
||||
|
||||
/// As build commands run they can output extra dependency information
|
||||
/// (e.g. header dependencies for C source) dynamically. DepsLog collects
|
||||
/// that information at build time and uses it for subsequent builds.
|
||||
///
|
||||
/// The on-disk format is based on two primary design constraints:
|
||||
/// - it must be written to as a stream (during the build, which may be
|
||||
/// interrupted);
|
||||
/// - it can be read all at once on startup. (Alternative designs, where
|
||||
/// it contains indexing information, were considered and discarded as
|
||||
/// too complicated to implement; if the file is small than reading it
|
||||
/// fully on startup is acceptable.)
|
||||
/// Here are some stats from the Windows Chrome dependency files, to
|
||||
/// help guide the design space. The total text in the files sums to
|
||||
/// 90mb so some compression is warranted to keep load-time fast.
|
||||
/// There's about 10k files worth of dependencies that reference about
|
||||
/// 40k total paths totalling 2mb of unique strings.
|
||||
///
|
||||
/// Based on these stats, here's the current design.
|
||||
/// The file is structured as version header followed by a sequence of records.
|
||||
/// Each record is either a path string or a dependency list.
|
||||
/// Numbering the path strings in file order gives them dense integer ids.
|
||||
/// A dependency list maps an output id to a list of input ids.
|
||||
///
|
||||
/// Concretely, a record is:
|
||||
/// four bytes record length, high bit indicates record type
|
||||
/// (but max record sizes are capped at 512kB)
|
||||
/// path records contain the string name of the path, followed by up to 3
|
||||
/// padding bytes to align on 4 byte boundaries, followed by the
|
||||
/// one's complement of the expected index of the record (to detect
|
||||
/// concurrent writes of multiple ninja processes to the log).
|
||||
/// dependency records are an array of 4-byte integers
|
||||
/// [output path id,
|
||||
/// output path mtime (lower 4 bytes), output path mtime (upper 4 bytes),
|
||||
/// input path id, input path id...]
|
||||
/// (The mtime is compared against the on-disk output path mtime
|
||||
/// to verify the stored data is up-to-date.)
|
||||
/// If two records reference the same output the latter one in the file
|
||||
/// wins, allowing updates to just be appended to the file. A separate
|
||||
/// repacking step can run occasionally to remove dead records.
|
||||
struct DepsLog {
|
||||
DepsLog() : needs_recompaction_(false), file_(NULL) {}
|
||||
~DepsLog();
|
||||
|
||||
// Writing (build-time) interface.
|
||||
bool OpenForWrite(const std::string& path, std::string* err);
|
||||
bool RecordDeps(Node* node, TimeStamp mtime, const std::vector<Node*>& nodes);
|
||||
bool RecordDeps(Node* node, TimeStamp mtime, int node_count,
|
||||
Node* const* nodes);
|
||||
void Close();
|
||||
|
||||
// Reading (startup-time) interface.
|
||||
struct Deps {
|
||||
Deps(int64_t mtime, int node_count)
|
||||
: mtime(mtime), node_count(node_count), nodes(new Node*[node_count]) {}
|
||||
~Deps() { delete [] nodes; }
|
||||
TimeStamp mtime;
|
||||
int node_count;
|
||||
Node** nodes;
|
||||
};
|
||||
LoadStatus Load(const std::string& path, State* state, std::string* err);
|
||||
Deps* GetDeps(Node* node);
|
||||
Node* GetFirstReverseDepsNode(Node* node);
|
||||
|
||||
/// Rewrite the known log entries, throwing away old data.
|
||||
bool Recompact(const std::string& path, std::string* err);
|
||||
|
||||
/// Returns if the deps entry for a node is still reachable from the manifest.
|
||||
///
|
||||
/// The deps log can contain deps entries for files that were built in the
|
||||
/// past but are no longer part of the manifest. This function returns if
|
||||
/// this is the case for a given node. This function is slow, don't call
|
||||
/// it from code that runs on every build.
|
||||
static bool IsDepsEntryLiveFor(const Node* node);
|
||||
|
||||
/// Used for tests.
|
||||
const std::vector<Node*>& nodes() const { return nodes_; }
|
||||
const std::vector<Deps*>& deps() const { return deps_; }
|
||||
|
||||
private:
|
||||
// Updates the in-memory representation. Takes ownership of |deps|.
|
||||
// Returns true if a prior deps record was deleted.
|
||||
bool UpdateDeps(int out_id, Deps* deps);
|
||||
// Write a node name record, assigning it an id.
|
||||
bool RecordId(Node* node);
|
||||
|
||||
/// Should be called before using file_. When false is returned, errno will
|
||||
/// be set.
|
||||
bool OpenForWriteIfNeeded();
|
||||
|
||||
bool needs_recompaction_;
|
||||
FILE* file_;
|
||||
std::string file_path_;
|
||||
|
||||
/// Maps id -> Node.
|
||||
std::vector<Node*> nodes_;
|
||||
/// Maps id -> deps of that id.
|
||||
std::vector<Deps*> deps_;
|
||||
|
||||
friend struct DepsLogTest;
|
||||
};
|
||||
|
||||
#endif // NINJA_DEPS_LOG_H_
|
||||
@@ -0,0 +1,679 @@
|
||||
// Copyright 2012 Google Inc. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#include "deps_log.h"
|
||||
|
||||
#include <sys/stat.h>
|
||||
#ifndef _WIN32
|
||||
#include <unistd.h>
|
||||
#endif
|
||||
|
||||
#include "disk_interface.h"
|
||||
#include "graph.h"
|
||||
#include "util.h"
|
||||
#include "test.h"
|
||||
|
||||
using namespace std;
|
||||
|
||||
namespace {
|
||||
|
||||
const char kTestFilename[] = "DepsLogTest-tempfile";
|
||||
|
||||
struct DepsLogTest : public testing::Test {
|
||||
virtual void SetUp() {
|
||||
// In case a crashing test left a stale file behind.
|
||||
platformAwareUnlink(kTestFilename);
|
||||
}
|
||||
virtual void TearDown() { platformAwareUnlink(kTestFilename); }
|
||||
};
|
||||
|
||||
TEST_F(DepsLogTest, WriteRead) {
|
||||
State state1;
|
||||
DepsLog log1;
|
||||
string err;
|
||||
EXPECT_TRUE(log1.OpenForWrite(kTestFilename, &err));
|
||||
ASSERT_EQ("", err);
|
||||
|
||||
{
|
||||
vector<Node*> deps;
|
||||
deps.push_back(state1.GetNode("foo.h", 0));
|
||||
deps.push_back(state1.GetNode("bar.h", 0));
|
||||
log1.RecordDeps(state1.GetNode("out.o", 0), 1, deps);
|
||||
|
||||
deps.clear();
|
||||
deps.push_back(state1.GetNode("foo.h", 0));
|
||||
deps.push_back(state1.GetNode("bar2.h", 0));
|
||||
log1.RecordDeps(state1.GetNode("out2.o", 0), 2, deps);
|
||||
|
||||
DepsLog::Deps* log_deps = log1.GetDeps(state1.GetNode("out.o", 0));
|
||||
ASSERT_TRUE(log_deps);
|
||||
ASSERT_EQ(1, log_deps->mtime);
|
||||
ASSERT_EQ(2, log_deps->node_count);
|
||||
ASSERT_EQ("foo.h", log_deps->nodes[0]->path());
|
||||
ASSERT_EQ("bar.h", log_deps->nodes[1]->path());
|
||||
}
|
||||
|
||||
log1.Close();
|
||||
|
||||
State state2;
|
||||
DepsLog log2;
|
||||
EXPECT_TRUE(log2.Load(kTestFilename, &state2, &err));
|
||||
ASSERT_EQ("", err);
|
||||
|
||||
ASSERT_EQ(log1.nodes().size(), log2.nodes().size());
|
||||
for (int i = 0; i < (int)log1.nodes().size(); ++i) {
|
||||
Node* node1 = log1.nodes()[i];
|
||||
Node* node2 = log2.nodes()[i];
|
||||
ASSERT_EQ(i, node1->id());
|
||||
ASSERT_EQ(node1->id(), node2->id());
|
||||
}
|
||||
|
||||
// Spot-check the entries in log2.
|
||||
DepsLog::Deps* log_deps = log2.GetDeps(state2.GetNode("out2.o", 0));
|
||||
ASSERT_TRUE(log_deps);
|
||||
ASSERT_EQ(2, log_deps->mtime);
|
||||
ASSERT_EQ(2, log_deps->node_count);
|
||||
ASSERT_EQ("foo.h", log_deps->nodes[0]->path());
|
||||
ASSERT_EQ("bar2.h", log_deps->nodes[1]->path());
|
||||
}
|
||||
|
||||
TEST_F(DepsLogTest, LotsOfDeps) {
|
||||
const int kNumDeps = 100000; // More than 64k.
|
||||
|
||||
State state1;
|
||||
DepsLog log1;
|
||||
string err;
|
||||
EXPECT_TRUE(log1.OpenForWrite(kTestFilename, &err));
|
||||
ASSERT_EQ("", err);
|
||||
|
||||
{
|
||||
vector<Node*> deps;
|
||||
for (int i = 0; i < kNumDeps; ++i) {
|
||||
char buf[32];
|
||||
sprintf(buf, "file%d.h", i);
|
||||
deps.push_back(state1.GetNode(buf, 0));
|
||||
}
|
||||
log1.RecordDeps(state1.GetNode("out.o", 0), 1, deps);
|
||||
|
||||
DepsLog::Deps* log_deps = log1.GetDeps(state1.GetNode("out.o", 0));
|
||||
ASSERT_EQ(kNumDeps, log_deps->node_count);
|
||||
}
|
||||
|
||||
log1.Close();
|
||||
|
||||
State state2;
|
||||
DepsLog log2;
|
||||
EXPECT_TRUE(log2.Load(kTestFilename, &state2, &err));
|
||||
ASSERT_EQ("", err);
|
||||
|
||||
DepsLog::Deps* log_deps = log2.GetDeps(state2.GetNode("out.o", 0));
|
||||
ASSERT_EQ(kNumDeps, log_deps->node_count);
|
||||
}
|
||||
|
||||
// Verify that adding the same deps twice doesn't grow the file.
|
||||
TEST_F(DepsLogTest, DoubleEntry) {
|
||||
// Write some deps to the file and grab its size.
|
||||
int file_size;
|
||||
{
|
||||
State state;
|
||||
DepsLog log;
|
||||
string err;
|
||||
EXPECT_TRUE(log.OpenForWrite(kTestFilename, &err));
|
||||
ASSERT_EQ("", err);
|
||||
|
||||
vector<Node*> deps;
|
||||
deps.push_back(state.GetNode("foo.h", 0));
|
||||
deps.push_back(state.GetNode("bar.h", 0));
|
||||
log.RecordDeps(state.GetNode("out.o", 0), 1, deps);
|
||||
log.Close();
|
||||
#ifdef __USE_LARGEFILE64
|
||||
struct stat64 st;
|
||||
ASSERT_EQ(0, stat64(kTestFilename, &st));
|
||||
#else
|
||||
struct stat st;
|
||||
ASSERT_EQ(0, stat(kTestFilename, &st));
|
||||
#endif
|
||||
file_size = (int)st.st_size;
|
||||
ASSERT_GT(file_size, 0);
|
||||
}
|
||||
|
||||
// Now reload the file, and read the same deps.
|
||||
{
|
||||
State state;
|
||||
DepsLog log;
|
||||
string err;
|
||||
EXPECT_TRUE(log.Load(kTestFilename, &state, &err));
|
||||
|
||||
EXPECT_TRUE(log.OpenForWrite(kTestFilename, &err));
|
||||
ASSERT_EQ("", err);
|
||||
|
||||
vector<Node*> deps;
|
||||
deps.push_back(state.GetNode("foo.h", 0));
|
||||
deps.push_back(state.GetNode("bar.h", 0));
|
||||
log.RecordDeps(state.GetNode("out.o", 0), 1, deps);
|
||||
log.Close();
|
||||
#ifdef __USE_LARGEFILE64
|
||||
struct stat64 st;
|
||||
ASSERT_EQ(0, stat64(kTestFilename, &st));
|
||||
#else
|
||||
struct stat st;
|
||||
ASSERT_EQ(0, stat(kTestFilename, &st));
|
||||
#endif
|
||||
int file_size_2 = (int)st.st_size;
|
||||
ASSERT_EQ(file_size, file_size_2);
|
||||
}
|
||||
}
|
||||
|
||||
// Verify that adding the new deps works and can be compacted away.
|
||||
TEST_F(DepsLogTest, Recompact) {
|
||||
const char kManifest[] =
|
||||
"rule cc\n"
|
||||
" command = cc\n"
|
||||
" deps = gcc\n"
|
||||
"build out.o: cc\n"
|
||||
"build other_out.o: cc\n";
|
||||
|
||||
// Write some deps to the file and grab its size.
|
||||
int file_size;
|
||||
{
|
||||
State state;
|
||||
ASSERT_NO_FATAL_FAILURE(AssertParse(&state, kManifest));
|
||||
DepsLog log;
|
||||
string err;
|
||||
ASSERT_TRUE(log.OpenForWrite(kTestFilename, &err));
|
||||
ASSERT_EQ("", err);
|
||||
|
||||
vector<Node*> deps;
|
||||
deps.push_back(state.GetNode("foo.h", 0));
|
||||
deps.push_back(state.GetNode("bar.h", 0));
|
||||
log.RecordDeps(state.GetNode("out.o", 0), 1, deps);
|
||||
|
||||
deps.clear();
|
||||
deps.push_back(state.GetNode("foo.h", 0));
|
||||
deps.push_back(state.GetNode("baz.h", 0));
|
||||
log.RecordDeps(state.GetNode("other_out.o", 0), 1, deps);
|
||||
|
||||
log.Close();
|
||||
#ifdef __USE_LARGEFILE64
|
||||
struct stat64 st;
|
||||
ASSERT_EQ(0, stat64(kTestFilename, &st));
|
||||
#else
|
||||
struct stat st;
|
||||
ASSERT_EQ(0, stat(kTestFilename, &st));
|
||||
#endif
|
||||
file_size = (int)st.st_size;
|
||||
ASSERT_GT(file_size, 0);
|
||||
}
|
||||
|
||||
// Now reload the file, and add slightly different deps.
|
||||
int file_size_2;
|
||||
{
|
||||
State state;
|
||||
ASSERT_NO_FATAL_FAILURE(AssertParse(&state, kManifest));
|
||||
DepsLog log;
|
||||
string err;
|
||||
ASSERT_TRUE(log.Load(kTestFilename, &state, &err));
|
||||
|
||||
ASSERT_TRUE(log.OpenForWrite(kTestFilename, &err));
|
||||
ASSERT_EQ("", err);
|
||||
|
||||
vector<Node*> deps;
|
||||
deps.push_back(state.GetNode("foo.h", 0));
|
||||
log.RecordDeps(state.GetNode("out.o", 0), 1, deps);
|
||||
log.Close();
|
||||
|
||||
#ifdef __USE_LARGEFILE64
|
||||
struct stat64 st;
|
||||
ASSERT_EQ(0, stat64(kTestFilename, &st));
|
||||
#else
|
||||
struct stat st;
|
||||
ASSERT_EQ(0, stat(kTestFilename, &st));
|
||||
#endif
|
||||
file_size_2 = (int)st.st_size;
|
||||
// The file should grow to record the new deps.
|
||||
ASSERT_GT(file_size_2, file_size);
|
||||
}
|
||||
|
||||
// Now reload the file, verify the new deps have replaced the old, then
|
||||
// recompact.
|
||||
int file_size_3;
|
||||
{
|
||||
State state;
|
||||
ASSERT_NO_FATAL_FAILURE(AssertParse(&state, kManifest));
|
||||
DepsLog log;
|
||||
string err;
|
||||
ASSERT_TRUE(log.Load(kTestFilename, &state, &err));
|
||||
|
||||
Node* out = state.GetNode("out.o", 0);
|
||||
DepsLog::Deps* deps = log.GetDeps(out);
|
||||
ASSERT_TRUE(deps);
|
||||
ASSERT_EQ(1, deps->mtime);
|
||||
ASSERT_EQ(1, deps->node_count);
|
||||
ASSERT_EQ("foo.h", deps->nodes[0]->path());
|
||||
|
||||
Node* other_out = state.GetNode("other_out.o", 0);
|
||||
deps = log.GetDeps(other_out);
|
||||
ASSERT_TRUE(deps);
|
||||
ASSERT_EQ(1, deps->mtime);
|
||||
ASSERT_EQ(2, deps->node_count);
|
||||
ASSERT_EQ("foo.h", deps->nodes[0]->path());
|
||||
ASSERT_EQ("baz.h", deps->nodes[1]->path());
|
||||
|
||||
ASSERT_TRUE(log.Recompact(kTestFilename, &err));
|
||||
|
||||
// The in-memory deps graph should still be valid after recompaction.
|
||||
deps = log.GetDeps(out);
|
||||
ASSERT_TRUE(deps);
|
||||
ASSERT_EQ(1, deps->mtime);
|
||||
ASSERT_EQ(1, deps->node_count);
|
||||
ASSERT_EQ("foo.h", deps->nodes[0]->path());
|
||||
ASSERT_EQ(out, log.nodes()[out->id()]);
|
||||
|
||||
deps = log.GetDeps(other_out);
|
||||
ASSERT_TRUE(deps);
|
||||
ASSERT_EQ(1, deps->mtime);
|
||||
ASSERT_EQ(2, deps->node_count);
|
||||
ASSERT_EQ("foo.h", deps->nodes[0]->path());
|
||||
ASSERT_EQ("baz.h", deps->nodes[1]->path());
|
||||
ASSERT_EQ(other_out, log.nodes()[other_out->id()]);
|
||||
|
||||
// The file should have shrunk a bit for the smaller deps.
|
||||
#ifdef __USE_LARGEFILE64
|
||||
struct stat64 st;
|
||||
ASSERT_EQ(0, stat64(kTestFilename, &st));
|
||||
#else
|
||||
struct stat st;
|
||||
ASSERT_EQ(0, stat(kTestFilename, &st));
|
||||
#endif
|
||||
file_size_3 = (int)st.st_size;
|
||||
ASSERT_LT(file_size_3, file_size_2);
|
||||
}
|
||||
|
||||
// Now reload the file and recompact with an empty manifest. The previous
|
||||
// entries should be removed.
|
||||
{
|
||||
State state;
|
||||
// Intentionally not parsing kManifest here.
|
||||
DepsLog log;
|
||||
string err;
|
||||
ASSERT_TRUE(log.Load(kTestFilename, &state, &err));
|
||||
|
||||
Node* out = state.GetNode("out.o", 0);
|
||||
DepsLog::Deps* deps = log.GetDeps(out);
|
||||
ASSERT_TRUE(deps);
|
||||
ASSERT_EQ(1, deps->mtime);
|
||||
ASSERT_EQ(1, deps->node_count);
|
||||
ASSERT_EQ("foo.h", deps->nodes[0]->path());
|
||||
|
||||
Node* other_out = state.GetNode("other_out.o", 0);
|
||||
deps = log.GetDeps(other_out);
|
||||
ASSERT_TRUE(deps);
|
||||
ASSERT_EQ(1, deps->mtime);
|
||||
ASSERT_EQ(2, deps->node_count);
|
||||
ASSERT_EQ("foo.h", deps->nodes[0]->path());
|
||||
ASSERT_EQ("baz.h", deps->nodes[1]->path());
|
||||
|
||||
ASSERT_TRUE(log.Recompact(kTestFilename, &err));
|
||||
|
||||
// The previous entries should have been removed.
|
||||
deps = log.GetDeps(out);
|
||||
ASSERT_FALSE(deps);
|
||||
|
||||
deps = log.GetDeps(other_out);
|
||||
ASSERT_FALSE(deps);
|
||||
|
||||
// The .h files pulled in via deps should no longer have ids either.
|
||||
ASSERT_EQ(-1, state.LookupNode("foo.h")->id());
|
||||
ASSERT_EQ(-1, state.LookupNode("baz.h")->id());
|
||||
|
||||
// The file should have shrunk more.
|
||||
#ifdef __USE_LARGEFILE64
|
||||
struct stat64 st;
|
||||
ASSERT_EQ(0, stat64(kTestFilename, &st));
|
||||
#else
|
||||
struct stat st;
|
||||
ASSERT_EQ(0, stat(kTestFilename, &st));
|
||||
#endif
|
||||
int file_size_4 = (int)st.st_size;
|
||||
ASSERT_LT(file_size_4, file_size_3);
|
||||
}
|
||||
}
|
||||
|
||||
// Verify that invalid file headers cause a new build.
|
||||
TEST_F(DepsLogTest, InvalidHeader) {
|
||||
const char *kInvalidHeaders[] = {
|
||||
"", // Empty file.
|
||||
"# ninjad", // Truncated first line.
|
||||
"# ninjadeps\n", // No version int.
|
||||
"# ninjadeps\n\001\002", // Truncated version int.
|
||||
"# ninjadeps\n\001\002\003\004" // Invalid version int.
|
||||
};
|
||||
for (size_t i = 0; i < sizeof(kInvalidHeaders) / sizeof(kInvalidHeaders[0]);
|
||||
++i) {
|
||||
FILE* deps_log = fopen(kTestFilename, "wb");
|
||||
ASSERT_TRUE(deps_log != NULL);
|
||||
ASSERT_EQ(
|
||||
strlen(kInvalidHeaders[i]),
|
||||
fwrite(kInvalidHeaders[i], 1, strlen(kInvalidHeaders[i]), deps_log));
|
||||
ASSERT_EQ(0 ,fclose(deps_log));
|
||||
|
||||
string err;
|
||||
DepsLog log;
|
||||
State state;
|
||||
ASSERT_TRUE(log.Load(kTestFilename, &state, &err));
|
||||
EXPECT_EQ("bad deps log signature or version; starting over", err);
|
||||
}
|
||||
}
|
||||
|
||||
// Simulate what happens when loading a truncated log file.
|
||||
TEST_F(DepsLogTest, Truncated) {
|
||||
// Create a file with some entries.
|
||||
{
|
||||
State state;
|
||||
DepsLog log;
|
||||
string err;
|
||||
EXPECT_TRUE(log.OpenForWrite(kTestFilename, &err));
|
||||
ASSERT_EQ("", err);
|
||||
|
||||
vector<Node*> deps;
|
||||
deps.push_back(state.GetNode("foo.h", 0));
|
||||
deps.push_back(state.GetNode("bar.h", 0));
|
||||
log.RecordDeps(state.GetNode("out.o", 0), 1, deps);
|
||||
|
||||
deps.clear();
|
||||
deps.push_back(state.GetNode("foo.h", 0));
|
||||
deps.push_back(state.GetNode("bar2.h", 0));
|
||||
log.RecordDeps(state.GetNode("out2.o", 0), 2, deps);
|
||||
|
||||
log.Close();
|
||||
}
|
||||
|
||||
// Get the file size.
|
||||
#ifdef __USE_LARGEFILE64
|
||||
struct stat64 st;
|
||||
ASSERT_EQ(0, stat64(kTestFilename, &st));
|
||||
#else
|
||||
struct stat st;
|
||||
ASSERT_EQ(0, stat(kTestFilename, &st));
|
||||
#endif
|
||||
|
||||
// Try reloading at truncated sizes.
|
||||
// Track how many nodes/deps were found; they should decrease with
|
||||
// smaller sizes.
|
||||
int node_count = 5;
|
||||
int deps_count = 2;
|
||||
for (int size = (int)st.st_size; size > 0; --size) {
|
||||
string err;
|
||||
ASSERT_TRUE(Truncate(kTestFilename, size, &err));
|
||||
|
||||
State state;
|
||||
DepsLog log;
|
||||
EXPECT_TRUE(log.Load(kTestFilename, &state, &err));
|
||||
if (!err.empty()) {
|
||||
// At some point the log will be so short as to be unparsable.
|
||||
break;
|
||||
}
|
||||
|
||||
ASSERT_GE(node_count, (int)log.nodes().size());
|
||||
node_count = static_cast<int>(log.nodes().size());
|
||||
|
||||
// Count how many non-NULL deps entries there are.
|
||||
int new_deps_count = 0;
|
||||
for (vector<DepsLog::Deps*>::const_iterator i = log.deps().begin();
|
||||
i != log.deps().end(); ++i) {
|
||||
if (*i)
|
||||
++new_deps_count;
|
||||
}
|
||||
ASSERT_GE(deps_count, new_deps_count);
|
||||
deps_count = new_deps_count;
|
||||
}
|
||||
}
|
||||
|
||||
// Run the truncation-recovery logic.
|
||||
TEST_F(DepsLogTest, TruncatedRecovery) {
|
||||
// Create a file with some entries.
|
||||
{
|
||||
State state;
|
||||
DepsLog log;
|
||||
string err;
|
||||
EXPECT_TRUE(log.OpenForWrite(kTestFilename, &err));
|
||||
ASSERT_EQ("", err);
|
||||
|
||||
vector<Node*> deps;
|
||||
deps.push_back(state.GetNode("foo.h", 0));
|
||||
deps.push_back(state.GetNode("bar.h", 0));
|
||||
log.RecordDeps(state.GetNode("out.o", 0), 1, deps);
|
||||
|
||||
deps.clear();
|
||||
deps.push_back(state.GetNode("foo.h", 0));
|
||||
deps.push_back(state.GetNode("bar2.h", 0));
|
||||
log.RecordDeps(state.GetNode("out2.o", 0), 2, deps);
|
||||
|
||||
log.Close();
|
||||
}
|
||||
|
||||
// Shorten the file, corrupting the last record.
|
||||
{
|
||||
#ifdef __USE_LARGEFILE64
|
||||
struct stat64 st;
|
||||
ASSERT_EQ(0, stat64(kTestFilename, &st));
|
||||
#else
|
||||
struct stat st;
|
||||
ASSERT_EQ(0, stat(kTestFilename, &st));
|
||||
#endif
|
||||
string err;
|
||||
ASSERT_TRUE(Truncate(kTestFilename, st.st_size - 2, &err));
|
||||
}
|
||||
|
||||
// Load the file again, add an entry.
|
||||
{
|
||||
State state;
|
||||
DepsLog log;
|
||||
string err;
|
||||
EXPECT_TRUE(log.Load(kTestFilename, &state, &err));
|
||||
ASSERT_EQ("premature end of file; recovering", err);
|
||||
err.clear();
|
||||
|
||||
// The truncated entry should've been discarded.
|
||||
EXPECT_EQ(NULL, log.GetDeps(state.GetNode("out2.o", 0)));
|
||||
|
||||
EXPECT_TRUE(log.OpenForWrite(kTestFilename, &err));
|
||||
ASSERT_EQ("", err);
|
||||
|
||||
// Add a new entry.
|
||||
vector<Node*> deps;
|
||||
deps.push_back(state.GetNode("foo.h", 0));
|
||||
deps.push_back(state.GetNode("bar2.h", 0));
|
||||
log.RecordDeps(state.GetNode("out2.o", 0), 3, deps);
|
||||
|
||||
log.Close();
|
||||
}
|
||||
|
||||
// Load the file a third time to verify appending after a mangled
|
||||
// entry doesn't break things.
|
||||
{
|
||||
State state;
|
||||
DepsLog log;
|
||||
string err;
|
||||
EXPECT_TRUE(log.Load(kTestFilename, &state, &err));
|
||||
|
||||
// The truncated entry should exist.
|
||||
DepsLog::Deps* deps = log.GetDeps(state.GetNode("out2.o", 0));
|
||||
ASSERT_TRUE(deps);
|
||||
}
|
||||
}
|
||||
|
||||
TEST_F(DepsLogTest, ReverseDepsNodes) {
|
||||
State state;
|
||||
DepsLog log;
|
||||
string err;
|
||||
EXPECT_TRUE(log.OpenForWrite(kTestFilename, &err));
|
||||
ASSERT_EQ("", err);
|
||||
|
||||
vector<Node*> deps;
|
||||
deps.push_back(state.GetNode("foo.h", 0));
|
||||
deps.push_back(state.GetNode("bar.h", 0));
|
||||
log.RecordDeps(state.GetNode("out.o", 0), 1, deps);
|
||||
|
||||
deps.clear();
|
||||
deps.push_back(state.GetNode("foo.h", 0));
|
||||
deps.push_back(state.GetNode("bar2.h", 0));
|
||||
log.RecordDeps(state.GetNode("out2.o", 0), 2, deps);
|
||||
|
||||
log.Close();
|
||||
|
||||
Node* rev_deps = log.GetFirstReverseDepsNode(state.GetNode("foo.h", 0));
|
||||
EXPECT_TRUE(rev_deps == state.GetNode("out.o", 0) ||
|
||||
rev_deps == state.GetNode("out2.o", 0));
|
||||
|
||||
rev_deps = log.GetFirstReverseDepsNode(state.GetNode("bar.h", 0));
|
||||
EXPECT_TRUE(rev_deps == state.GetNode("out.o", 0));
|
||||
}
|
||||
|
||||
TEST_F(DepsLogTest, MalformedDepsLog) {
|
||||
std::string err;
|
||||
{
|
||||
State state;
|
||||
DepsLog log;
|
||||
EXPECT_TRUE(log.OpenForWrite(kTestFilename, &err));
|
||||
ASSERT_EQ("", err);
|
||||
|
||||
// First, create a valid log file.
|
||||
std::vector<Node*> deps;
|
||||
deps.push_back(state.GetNode("foo.hh", 0));
|
||||
deps.push_back(state.GetNode("bar.hpp", 0));
|
||||
log.RecordDeps(state.GetNode("out.o", 0), 1, deps);
|
||||
log.Close();
|
||||
}
|
||||
|
||||
// Now read its value, validate it a little.
|
||||
RealDiskInterface disk;
|
||||
|
||||
std::string original_contents;
|
||||
ASSERT_EQ(FileReader::Okay, disk.ReadFile(kTestFilename,
|
||||
&original_contents, &err));
|
||||
|
||||
const size_t version_offset = 12;
|
||||
ASSERT_EQ("# ninjadeps\n", original_contents.substr(0, version_offset));
|
||||
ASSERT_EQ('\x04', original_contents[version_offset + 0]);
|
||||
ASSERT_EQ('\x00', original_contents[version_offset + 1]);
|
||||
ASSERT_EQ('\x00', original_contents[version_offset + 2]);
|
||||
ASSERT_EQ('\x00', original_contents[version_offset + 3]);
|
||||
|
||||
// clang-format off
|
||||
static const uint8_t kFirstRecord[] = {
|
||||
// size field == 0x0000000c
|
||||
0x0c, 0x00, 0x00, 0x00,
|
||||
// name field = 'out.o' + 3 bytes of padding.
|
||||
'o', 'u', 't', '.', 'o', 0x00, 0x00, 0x00,
|
||||
// checksum = ~0
|
||||
0xff, 0xff, 0xff, 0xff,
|
||||
};
|
||||
// clang-format on
|
||||
const size_t kFirstRecordLen = sizeof(kFirstRecord);
|
||||
const size_t first_offset = version_offset + 4;
|
||||
|
||||
#define COMPARE_RECORD(start_pos, reference, len) \
|
||||
ASSERT_EQ(original_contents.substr(start_pos, len), std::string(reinterpret_cast<const char*>(reference), len))
|
||||
|
||||
COMPARE_RECORD(first_offset, kFirstRecord, kFirstRecordLen);
|
||||
|
||||
const size_t second_offset = first_offset + kFirstRecordLen;
|
||||
// clang-format off
|
||||
static const uint8_t kSecondRecord[] = {
|
||||
// size field == 0x0000000c
|
||||
0x0c, 0x00, 0x00, 0x00,
|
||||
// name field = 'foo.hh' + 2 bytes of padding.
|
||||
'f', 'o', 'o', '.', 'h', 'h', 0x00, 0x00,
|
||||
// checksum = ~1
|
||||
0xfe, 0xff, 0xff, 0xff,
|
||||
};
|
||||
// clang-format on
|
||||
const size_t kSecondRecordLen = sizeof(kSecondRecord);
|
||||
COMPARE_RECORD(second_offset, kSecondRecord, kSecondRecordLen);
|
||||
|
||||
// Then start generating corrupted versions and trying to load them.
|
||||
const char kBadLogFile[] = "DepsLogTest-corrupted.tempfile";
|
||||
|
||||
// Helper lambda to rewrite the bad log file with new content.
|
||||
auto write_bad_log_file =
|
||||
[&disk, &kBadLogFile](const std::string& bad_contents) -> bool {
|
||||
(void)disk.RemoveFile(kBadLogFile);
|
||||
return disk.WriteFile(kBadLogFile, bad_contents, false);
|
||||
};
|
||||
|
||||
// First, corrupt the header.
|
||||
std::string bad_contents = original_contents;
|
||||
bad_contents[0] = '@';
|
||||
|
||||
ASSERT_TRUE(write_bad_log_file(bad_contents)) << strerror(errno);
|
||||
{
|
||||
State state;
|
||||
DepsLog log;
|
||||
err.clear();
|
||||
ASSERT_EQ(LOAD_SUCCESS, log.Load(kBadLogFile, &state, &err));
|
||||
ASSERT_EQ("bad deps log signature or version; starting over", err);
|
||||
}
|
||||
|
||||
// Second, truncate the version.
|
||||
bad_contents = original_contents.substr(0, version_offset + 3);
|
||||
ASSERT_TRUE(write_bad_log_file(bad_contents)) << strerror(errno);
|
||||
{
|
||||
State state;
|
||||
DepsLog log;
|
||||
err.clear();
|
||||
ASSERT_EQ(LOAD_SUCCESS, log.Load(kBadLogFile, &state, &err));
|
||||
ASSERT_EQ("bad deps log signature or version; starting over", err);
|
||||
}
|
||||
|
||||
// Truncate first record's |size| field. The loader should recover.
|
||||
bad_contents = original_contents.substr(0, version_offset + 4 + 3);
|
||||
ASSERT_TRUE(write_bad_log_file(bad_contents)) << strerror(errno);
|
||||
{
|
||||
State state;
|
||||
DepsLog log;
|
||||
err.clear();
|
||||
ASSERT_EQ(LOAD_SUCCESS, log.Load(kBadLogFile, &state, &err));
|
||||
ASSERT_EQ("", err);
|
||||
}
|
||||
|
||||
// Corrupt first record |size| value.
|
||||
bad_contents = original_contents;
|
||||
bad_contents[first_offset + 0] = '\x55';
|
||||
bad_contents[first_offset + 1] = '\xaa';
|
||||
bad_contents[first_offset + 2] = '\xff';
|
||||
bad_contents[first_offset + 3] = '\xff';
|
||||
ASSERT_TRUE(write_bad_log_file(bad_contents)) << strerror(errno);
|
||||
{
|
||||
State state;
|
||||
DepsLog log;
|
||||
err.clear();
|
||||
ASSERT_EQ(LOAD_SUCCESS, log.Load(kBadLogFile, &state, &err));
|
||||
ASSERT_EQ("premature end of file; recovering", err);
|
||||
}
|
||||
|
||||
// Make first record |size| less than 4.
|
||||
bad_contents = original_contents;
|
||||
bad_contents[first_offset] = '\x01';
|
||||
ASSERT_TRUE(write_bad_log_file(bad_contents)) << strerror(errno);
|
||||
{
|
||||
State state;
|
||||
DepsLog log;
|
||||
err.clear();
|
||||
ASSERT_EQ(LOAD_SUCCESS, log.Load(kBadLogFile, &state, &err));
|
||||
ASSERT_EQ("premature end of file; recovering", err);
|
||||
}
|
||||
}
|
||||
|
||||
} // anonymous namespace
|
||||
@@ -0,0 +1,367 @@
|
||||
// Copyright 2011 Google Inc. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#include "disk_interface.h"
|
||||
|
||||
#include <algorithm>
|
||||
|
||||
#include <errno.h>
|
||||
#include <stdio.h>
|
||||
#include <string.h>
|
||||
#include <sys/stat.h>
|
||||
#include <sys/types.h>
|
||||
|
||||
#ifdef _WIN32
|
||||
#include <direct.h> // _mkdir
|
||||
#include <windows.h>
|
||||
|
||||
#include <sstream>
|
||||
#else
|
||||
#include <unistd.h>
|
||||
#endif
|
||||
|
||||
#include "metrics.h"
|
||||
#include "util.h"
|
||||
|
||||
using namespace std;
|
||||
|
||||
namespace {
|
||||
|
||||
string DirName(const string& path) {
|
||||
#ifdef _WIN32
|
||||
static const char kPathSeparators[] = "\\/";
|
||||
#else
|
||||
static const char kPathSeparators[] = "/";
|
||||
#endif
|
||||
static const char* const kEnd = kPathSeparators + sizeof(kPathSeparators) - 1;
|
||||
|
||||
string::size_type slash_pos = path.find_last_of(kPathSeparators);
|
||||
if (slash_pos == string::npos)
|
||||
return string(); // Nothing to do.
|
||||
while (slash_pos > 0 &&
|
||||
std::find(kPathSeparators, kEnd, path[slash_pos - 1]) != kEnd)
|
||||
--slash_pos;
|
||||
return path.substr(0, slash_pos);
|
||||
}
|
||||
|
||||
int MakeDir(const string& path) {
|
||||
#ifdef _WIN32
|
||||
return _mkdir(path.c_str());
|
||||
#else
|
||||
return mkdir(path.c_str(), 0777);
|
||||
#endif
|
||||
}
|
||||
|
||||
#ifdef _WIN32
|
||||
TimeStamp TimeStampFromFileTime(const FILETIME& filetime) {
|
||||
// FILETIME is in 100-nanosecond increments since the Windows epoch.
|
||||
// We don't much care about epoch correctness but we do want the
|
||||
// resulting value to fit in a 64-bit integer.
|
||||
uint64_t mtime = ((uint64_t)filetime.dwHighDateTime << 32) |
|
||||
((uint64_t)filetime.dwLowDateTime);
|
||||
// 1600 epoch -> 2000 epoch (subtract 400 years).
|
||||
return (TimeStamp)mtime - 12622770400LL * (1000000000LL / 100);
|
||||
}
|
||||
|
||||
TimeStamp StatSingleFile(const string& path, string* err) {
|
||||
WIN32_FILE_ATTRIBUTE_DATA attrs;
|
||||
if (!GetFileAttributesExA(path.c_str(), GetFileExInfoStandard, &attrs)) {
|
||||
DWORD win_err = GetLastError();
|
||||
if (win_err == ERROR_FILE_NOT_FOUND || win_err == ERROR_PATH_NOT_FOUND)
|
||||
return 0;
|
||||
*err = "GetFileAttributesEx(" + path + "): " + GetLastErrorString();
|
||||
return -1;
|
||||
}
|
||||
return TimeStampFromFileTime(attrs.ftLastWriteTime);
|
||||
}
|
||||
|
||||
bool IsWindows7OrLater() {
|
||||
OSVERSIONINFOEX version_info =
|
||||
{ sizeof(OSVERSIONINFOEX), 6, 1, 0, 0, {0}, 0, 0, 0, 0, 0};
|
||||
DWORDLONG comparison = 0;
|
||||
VER_SET_CONDITION(comparison, VER_MAJORVERSION, VER_GREATER_EQUAL);
|
||||
VER_SET_CONDITION(comparison, VER_MINORVERSION, VER_GREATER_EQUAL);
|
||||
return VerifyVersionInfo(
|
||||
&version_info, VER_MAJORVERSION | VER_MINORVERSION, comparison);
|
||||
}
|
||||
|
||||
bool StatAllFilesInDir(const string& dir, map<string, TimeStamp>* stamps,
|
||||
string* err) {
|
||||
// FindExInfoBasic is 30% faster than FindExInfoStandard.
|
||||
static bool can_use_basic_info = IsWindows7OrLater();
|
||||
// This is not in earlier SDKs.
|
||||
const FINDEX_INFO_LEVELS kFindExInfoBasic =
|
||||
static_cast<FINDEX_INFO_LEVELS>(1);
|
||||
FINDEX_INFO_LEVELS level =
|
||||
can_use_basic_info ? kFindExInfoBasic : FindExInfoStandard;
|
||||
WIN32_FIND_DATAA ffd;
|
||||
HANDLE find_handle = FindFirstFileExA((dir + "\\*").c_str(), level, &ffd,
|
||||
FindExSearchNameMatch, NULL, 0);
|
||||
|
||||
if (find_handle == INVALID_HANDLE_VALUE) {
|
||||
DWORD win_err = GetLastError();
|
||||
if (win_err == ERROR_FILE_NOT_FOUND || win_err == ERROR_PATH_NOT_FOUND ||
|
||||
win_err == ERROR_DIRECTORY)
|
||||
return true;
|
||||
*err = "FindFirstFileExA(" + dir + "): " + GetLastErrorString();
|
||||
return false;
|
||||
}
|
||||
do {
|
||||
string lowername = ffd.cFileName;
|
||||
if (lowername == "..") {
|
||||
// Seems to just copy the timestamp for ".." from ".", which is wrong.
|
||||
// This is the case at least on NTFS under Windows 7.
|
||||
continue;
|
||||
}
|
||||
transform(lowername.begin(), lowername.end(), lowername.begin(), ::tolower);
|
||||
stamps->insert(make_pair(lowername,
|
||||
TimeStampFromFileTime(ffd.ftLastWriteTime)));
|
||||
} while (FindNextFileA(find_handle, &ffd));
|
||||
FindClose(find_handle);
|
||||
return true;
|
||||
}
|
||||
#endif // _WIN32
|
||||
|
||||
} // namespace
|
||||
|
||||
// DiskInterface ---------------------------------------------------------------
|
||||
|
||||
bool DiskInterface::MakeDirs(const string& path) {
|
||||
string dir = DirName(path);
|
||||
if (dir.empty())
|
||||
return true; // Reached root; assume it's there.
|
||||
string err;
|
||||
TimeStamp mtime = Stat(dir, &err);
|
||||
if (mtime < 0) {
|
||||
Error("%s", err.c_str());
|
||||
return false;
|
||||
}
|
||||
if (mtime > 0)
|
||||
return true; // Exists already; we're done.
|
||||
|
||||
// Directory doesn't exist. Try creating its parent first.
|
||||
bool success = MakeDirs(dir);
|
||||
if (!success)
|
||||
return false;
|
||||
return MakeDir(dir);
|
||||
}
|
||||
|
||||
// RealDiskInterface -----------------------------------------------------------
|
||||
RealDiskInterface::RealDiskInterface()
|
||||
#ifdef _WIN32
|
||||
: use_cache_(false), long_paths_enabled_(false) {
|
||||
// Probe ntdll.dll for RtlAreLongPathsEnabled, and call it if it exists.
|
||||
HINSTANCE ntdll_lib = ::GetModuleHandleW(L"ntdll");
|
||||
if (ntdll_lib) {
|
||||
typedef BOOLEAN(WINAPI FunctionType)();
|
||||
auto* func_ptr = FunctionCast<FunctionType*>(
|
||||
::GetProcAddress(ntdll_lib, "RtlAreLongPathsEnabled"));
|
||||
if (func_ptr) {
|
||||
long_paths_enabled_ = (*func_ptr)();
|
||||
}
|
||||
}
|
||||
}
|
||||
#else
|
||||
{}
|
||||
#endif
|
||||
|
||||
TimeStamp RealDiskInterface::Stat(const string& path, string* err) const {
|
||||
METRIC_RECORD("node stat");
|
||||
#ifdef _WIN32
|
||||
// MSDN: "Naming Files, Paths, and Namespaces"
|
||||
// http://msdn.microsoft.com/en-us/library/windows/desktop/aa365247(v=vs.85).aspx
|
||||
if (!path.empty() && !AreLongPathsEnabled() && path[0] != '\\' &&
|
||||
path.size() > MAX_PATH) {
|
||||
ostringstream err_stream;
|
||||
err_stream << "Stat(" << path << "): Filename longer than " << MAX_PATH
|
||||
<< " characters";
|
||||
*err = err_stream.str();
|
||||
return -1;
|
||||
}
|
||||
if (!use_cache_)
|
||||
return StatSingleFile(path, err);
|
||||
|
||||
string dir = DirName(path);
|
||||
string base(path.substr(dir.size() ? dir.size() + 1 : 0));
|
||||
if (base == "..") {
|
||||
// StatAllFilesInDir does not report any information for base = "..".
|
||||
base = ".";
|
||||
dir = path;
|
||||
}
|
||||
|
||||
string dir_lowercase = dir;
|
||||
transform(dir.begin(), dir.end(), dir_lowercase.begin(), ::tolower);
|
||||
transform(base.begin(), base.end(), base.begin(), ::tolower);
|
||||
|
||||
Cache::iterator ci = cache_.find(dir_lowercase);
|
||||
if (ci == cache_.end()) {
|
||||
ci = cache_.insert(make_pair(dir_lowercase, DirCache())).first;
|
||||
if (!StatAllFilesInDir(dir.empty() ? "." : dir, &ci->second, err)) {
|
||||
cache_.erase(ci);
|
||||
return -1;
|
||||
}
|
||||
}
|
||||
DirCache::iterator di = ci->second.find(base);
|
||||
return di != ci->second.end() ? di->second : 0;
|
||||
#else
|
||||
#ifdef __USE_LARGEFILE64
|
||||
struct stat64 st;
|
||||
if (stat64(path.c_str(), &st) < 0) {
|
||||
#else
|
||||
struct stat st;
|
||||
if (stat(path.c_str(), &st) < 0) {
|
||||
#endif
|
||||
if (errno == ENOENT || errno == ENOTDIR)
|
||||
return 0;
|
||||
*err = "stat(" + path + "): " + strerror(errno);
|
||||
return -1;
|
||||
}
|
||||
// Some users (Flatpak) set mtime to 0, this should be harmless
|
||||
// and avoids conflicting with our return value of 0 meaning
|
||||
// that it doesn't exist.
|
||||
if (st.st_mtime == 0)
|
||||
return 1;
|
||||
#if defined(_AIX)
|
||||
return (int64_t)st.st_mtime * 1000000000LL + st.st_mtime_n;
|
||||
#elif defined(__APPLE__)
|
||||
return ((int64_t)st.st_mtimespec.tv_sec * 1000000000LL +
|
||||
st.st_mtimespec.tv_nsec);
|
||||
#elif defined(st_mtime) // A macro, so we're likely on modern POSIX.
|
||||
return (int64_t)st.st_mtim.tv_sec * 1000000000LL + st.st_mtim.tv_nsec;
|
||||
#else
|
||||
return (int64_t)st.st_mtime * 1000000000LL + st.st_mtimensec;
|
||||
#endif
|
||||
#endif
|
||||
}
|
||||
|
||||
bool RealDiskInterface::WriteFile(const string& path, const string& contents,
|
||||
bool crlf_on_windows) {
|
||||
FILE* fp = fopen(path.c_str(),
|
||||
#ifdef _WIN32
|
||||
crlf_on_windows ? "w" : "wb");
|
||||
#else
|
||||
"wb");
|
||||
(void)crlf_on_windows;
|
||||
#endif
|
||||
if (fp == NULL) {
|
||||
Error("WriteFile(%s): Unable to create file. %s",
|
||||
path.c_str(), strerror(errno));
|
||||
return false;
|
||||
}
|
||||
|
||||
if (fwrite(contents.data(), 1, contents.length(), fp) < contents.length()) {
|
||||
Error("WriteFile(%s): Unable to write to the file. %s",
|
||||
path.c_str(), strerror(errno));
|
||||
fclose(fp);
|
||||
return false;
|
||||
}
|
||||
|
||||
if (fclose(fp) == EOF) {
|
||||
Error("WriteFile(%s): Unable to close the file. %s",
|
||||
path.c_str(), strerror(errno));
|
||||
return false;
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
bool RealDiskInterface::MakeDir(const string& path) {
|
||||
if (::MakeDir(path) < 0) {
|
||||
if (errno == EEXIST) {
|
||||
return true;
|
||||
}
|
||||
Error("mkdir(%s): %s", path.c_str(), strerror(errno));
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
FileReader::Status RealDiskInterface::ReadFile(const string& path,
|
||||
string* contents,
|
||||
string* err) {
|
||||
switch (::ReadFile(path, contents, err)) {
|
||||
case 0: return Okay;
|
||||
case -ENOENT: return NotFound;
|
||||
default: return OtherError;
|
||||
}
|
||||
}
|
||||
|
||||
int RealDiskInterface::RemoveFile(const string& path) {
|
||||
#ifdef _WIN32
|
||||
DWORD attributes = GetFileAttributesA(path.c_str());
|
||||
if (attributes == INVALID_FILE_ATTRIBUTES) {
|
||||
DWORD win_err = GetLastError();
|
||||
if (win_err == ERROR_FILE_NOT_FOUND || win_err == ERROR_PATH_NOT_FOUND) {
|
||||
return 1;
|
||||
}
|
||||
} else if (attributes & FILE_ATTRIBUTE_READONLY) {
|
||||
// On non-Windows systems, remove() will happily delete read-only files.
|
||||
// On Windows Ninja should behave the same:
|
||||
// https://github.com/ninja-build/ninja/issues/1886
|
||||
// Skip error checking. If this fails, accept whatever happens below.
|
||||
SetFileAttributesA(path.c_str(), attributes & ~FILE_ATTRIBUTE_READONLY);
|
||||
}
|
||||
if (attributes & FILE_ATTRIBUTE_DIRECTORY) {
|
||||
// remove() deletes both files and directories. On Windows we have to
|
||||
// select the correct function (DeleteFile will yield Permission Denied when
|
||||
// used on a directory)
|
||||
// This fixes the behavior of ninja -t clean in some cases
|
||||
// https://github.com/ninja-build/ninja/issues/828
|
||||
if (!RemoveDirectoryA(path.c_str())) {
|
||||
DWORD win_err = GetLastError();
|
||||
if (win_err == ERROR_FILE_NOT_FOUND || win_err == ERROR_PATH_NOT_FOUND) {
|
||||
return 1;
|
||||
}
|
||||
// Report remove(), not RemoveDirectory(), for cross-platform consistency.
|
||||
Error("remove(%s): %s", path.c_str(), GetLastErrorString().c_str());
|
||||
return -1;
|
||||
}
|
||||
} else {
|
||||
if (!DeleteFileA(path.c_str())) {
|
||||
DWORD win_err = GetLastError();
|
||||
if (win_err == ERROR_FILE_NOT_FOUND || win_err == ERROR_PATH_NOT_FOUND) {
|
||||
return 1;
|
||||
}
|
||||
// Report as remove(), not DeleteFile(), for cross-platform consistency.
|
||||
Error("remove(%s): %s", path.c_str(), GetLastErrorString().c_str());
|
||||
return -1;
|
||||
}
|
||||
}
|
||||
#else
|
||||
if (remove(path.c_str()) < 0) {
|
||||
switch (errno) {
|
||||
case ENOENT:
|
||||
return 1;
|
||||
default:
|
||||
Error("remove(%s): %s", path.c_str(), strerror(errno));
|
||||
return -1;
|
||||
}
|
||||
}
|
||||
#endif
|
||||
return 0;
|
||||
}
|
||||
|
||||
void RealDiskInterface::AllowStatCache(bool allow) {
|
||||
#ifdef _WIN32
|
||||
use_cache_ = allow;
|
||||
if (!use_cache_)
|
||||
cache_.clear();
|
||||
#endif
|
||||
}
|
||||
|
||||
#ifdef _WIN32
|
||||
bool RealDiskInterface::AreLongPathsEnabled(void) const {
|
||||
return long_paths_enabled_;
|
||||
}
|
||||
#endif
|
||||
@@ -0,0 +1,108 @@
|
||||
// Copyright 2011 Google Inc. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#ifndef NINJA_DISK_INTERFACE_H_
|
||||
#define NINJA_DISK_INTERFACE_H_
|
||||
|
||||
#include <map>
|
||||
#include <string>
|
||||
|
||||
#include "timestamp.h"
|
||||
|
||||
/// Interface for reading files from disk. See DiskInterface for details.
|
||||
/// This base offers the minimum interface needed just to read files.
|
||||
struct FileReader {
|
||||
virtual ~FileReader() {}
|
||||
|
||||
/// Result of ReadFile.
|
||||
enum Status {
|
||||
Okay,
|
||||
NotFound,
|
||||
OtherError
|
||||
};
|
||||
|
||||
/// Read and store in given string. On success, return Okay.
|
||||
/// On error, return another Status and fill |err|.
|
||||
virtual Status ReadFile(const std::string& path, std::string* contents,
|
||||
std::string* err) = 0;
|
||||
};
|
||||
|
||||
/// Interface for accessing the disk.
|
||||
///
|
||||
/// Abstract so it can be mocked out for tests. The real implementation
|
||||
/// is RealDiskInterface.
|
||||
struct DiskInterface: public FileReader {
|
||||
/// stat() a file, returning the mtime, or 0 if missing and -1 on
|
||||
/// other errors.
|
||||
virtual TimeStamp Stat(const std::string& path, std::string* err) const = 0;
|
||||
|
||||
/// Create a directory, returning false on failure.
|
||||
virtual bool MakeDir(const std::string& path) = 0;
|
||||
|
||||
/// Create a file, with the specified name and contents
|
||||
/// If \a crlf_on_windows is true, \n will be converted to \r\n (only on
|
||||
/// Windows builds of Ninja).
|
||||
/// Returns true on success, false on failure
|
||||
virtual bool WriteFile(const std::string& path, const std::string& contents,
|
||||
bool crlf_on_windows) = 0;
|
||||
|
||||
/// Remove the file named @a path. It behaves like 'rm -f path' so no errors
|
||||
/// are reported if it does not exists.
|
||||
/// @returns 0 if the file has been removed,
|
||||
/// 1 if the file does not exist, and
|
||||
/// -1 if an error occurs.
|
||||
virtual int RemoveFile(const std::string& path) = 0;
|
||||
|
||||
/// Create all the parent directories for path; like mkdir -p
|
||||
/// `basename path`.
|
||||
bool MakeDirs(const std::string& path);
|
||||
};
|
||||
|
||||
/// Implementation of DiskInterface that actually hits the disk.
|
||||
struct RealDiskInterface : public DiskInterface {
|
||||
RealDiskInterface();
|
||||
virtual ~RealDiskInterface() {}
|
||||
TimeStamp Stat(const std::string& path, std::string* err) const override;
|
||||
bool MakeDir(const std::string& path) override;
|
||||
bool WriteFile(const std::string& path, const std::string& contents,
|
||||
bool crlf_on_windows) override;
|
||||
Status ReadFile(const std::string& path, std::string* contents,
|
||||
std::string* err) override;
|
||||
int RemoveFile(const std::string& path) override;
|
||||
|
||||
/// Whether stat information can be cached. Only has an effect on Windows.
|
||||
void AllowStatCache(bool allow);
|
||||
|
||||
#ifdef _WIN32
|
||||
/// Whether long paths are enabled. Only has an effect on Windows.
|
||||
bool AreLongPathsEnabled() const;
|
||||
#endif
|
||||
|
||||
private:
|
||||
#ifdef _WIN32
|
||||
/// Whether stat information can be cached.
|
||||
bool use_cache_;
|
||||
|
||||
/// Whether long paths are enabled.
|
||||
bool long_paths_enabled_;
|
||||
|
||||
typedef std::map<std::string, TimeStamp> DirCache;
|
||||
// TODO: Neither a map nor a hashmap seems ideal here. If the statcache
|
||||
// works out, come up with a better data structure.
|
||||
typedef std::map<std::string, DirCache> Cache;
|
||||
mutable Cache cache_;
|
||||
#endif
|
||||
};
|
||||
|
||||
#endif // NINJA_DISK_INTERFACE_H_
|
||||
@@ -0,0 +1,369 @@
|
||||
// Copyright 2011 Google Inc. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#include <assert.h>
|
||||
#include <stdio.h>
|
||||
#ifdef _WIN32
|
||||
#include <io.h>
|
||||
#include <windows.h>
|
||||
#include <direct.h>
|
||||
#endif
|
||||
|
||||
#include "disk_interface.h"
|
||||
#include "graph.h"
|
||||
#include "test.h"
|
||||
|
||||
using namespace std;
|
||||
|
||||
namespace {
|
||||
|
||||
struct DiskInterfaceTest : public testing::Test {
|
||||
virtual void SetUp() {
|
||||
// These tests do real disk accesses, so create a temp dir.
|
||||
temp_dir_.CreateAndEnter("Ninja-DiskInterfaceTest");
|
||||
}
|
||||
|
||||
virtual void TearDown() {
|
||||
temp_dir_.Cleanup();
|
||||
}
|
||||
|
||||
bool Touch(const char* path) {
|
||||
FILE *f = fopen(path, "w");
|
||||
if (!f)
|
||||
return false;
|
||||
return fclose(f) == 0;
|
||||
}
|
||||
|
||||
ScopedTempDir temp_dir_;
|
||||
RealDiskInterface disk_;
|
||||
};
|
||||
|
||||
TEST_F(DiskInterfaceTest, StatMissingFile) {
|
||||
string err;
|
||||
EXPECT_EQ(0, disk_.Stat("nosuchfile", &err));
|
||||
EXPECT_EQ("", err);
|
||||
|
||||
// On Windows, the errno for a file in a nonexistent directory
|
||||
// is different.
|
||||
EXPECT_EQ(0, disk_.Stat("nosuchdir/nosuchfile", &err));
|
||||
EXPECT_EQ("", err);
|
||||
|
||||
// On POSIX systems, the errno is different if a component of the
|
||||
// path prefix is not a directory.
|
||||
ASSERT_TRUE(Touch("notadir"));
|
||||
EXPECT_EQ(0, disk_.Stat("notadir/nosuchfile", &err));
|
||||
EXPECT_EQ("", err);
|
||||
}
|
||||
|
||||
TEST_F(DiskInterfaceTest, StatMissingFileWithCache) {
|
||||
disk_.AllowStatCache(true);
|
||||
string err;
|
||||
|
||||
// On Windows, the errno for FindFirstFileExA, which is used when the stat
|
||||
// cache is enabled, is different when the directory name is not a directory.
|
||||
ASSERT_TRUE(Touch("notadir"));
|
||||
EXPECT_EQ(0, disk_.Stat("notadir/nosuchfile", &err));
|
||||
EXPECT_EQ("", err);
|
||||
}
|
||||
|
||||
TEST_F(DiskInterfaceTest, StatBadPath) {
|
||||
string err;
|
||||
#ifdef _WIN32
|
||||
string bad_path("cc:\\foo");
|
||||
EXPECT_EQ(-1, disk_.Stat(bad_path, &err));
|
||||
EXPECT_NE("", err);
|
||||
#else
|
||||
string too_long_name(512, 'x');
|
||||
EXPECT_EQ(-1, disk_.Stat(too_long_name, &err));
|
||||
EXPECT_NE("", err);
|
||||
#endif
|
||||
}
|
||||
|
||||
TEST_F(DiskInterfaceTest, StatExistingFile) {
|
||||
string err;
|
||||
ASSERT_TRUE(Touch("file"));
|
||||
EXPECT_GT(disk_.Stat("file", &err), 1);
|
||||
EXPECT_EQ("", err);
|
||||
}
|
||||
|
||||
#ifdef _WIN32
|
||||
TEST_F(DiskInterfaceTest, StatExistingFileWithLongPath) {
|
||||
string err;
|
||||
char currentdir[32767];
|
||||
_getcwd(currentdir, sizeof(currentdir));
|
||||
const string filename = string(currentdir) +
|
||||
"\\filename_with_256_characters_xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx\
|
||||
xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx\
|
||||
xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx\
|
||||
xxxxxxxxxxxxxxxxxxxxx";
|
||||
const string prefixed = "\\\\?\\" + filename;
|
||||
ASSERT_TRUE(Touch(prefixed.c_str()));
|
||||
EXPECT_GT(disk_.Stat(disk_.AreLongPathsEnabled() ?
|
||||
filename : prefixed, &err), 1);
|
||||
EXPECT_EQ("", err);
|
||||
}
|
||||
#endif
|
||||
|
||||
TEST_F(DiskInterfaceTest, StatExistingDir) {
|
||||
string err;
|
||||
ASSERT_TRUE(disk_.MakeDir("subdir"));
|
||||
ASSERT_TRUE(disk_.MakeDir("subdir/subsubdir"));
|
||||
EXPECT_GT(disk_.Stat("..", &err), 1);
|
||||
EXPECT_EQ("", err);
|
||||
EXPECT_GT(disk_.Stat(".", &err), 1);
|
||||
EXPECT_EQ("", err);
|
||||
EXPECT_GT(disk_.Stat("subdir", &err), 1);
|
||||
EXPECT_EQ("", err);
|
||||
EXPECT_GT(disk_.Stat("subdir/subsubdir", &err), 1);
|
||||
EXPECT_EQ("", err);
|
||||
|
||||
EXPECT_EQ(disk_.Stat("subdir", &err),
|
||||
disk_.Stat("subdir/.", &err));
|
||||
EXPECT_EQ(disk_.Stat("subdir", &err),
|
||||
disk_.Stat("subdir/subsubdir/..", &err));
|
||||
EXPECT_EQ(disk_.Stat("subdir/subsubdir", &err),
|
||||
disk_.Stat("subdir/subsubdir/.", &err));
|
||||
}
|
||||
|
||||
#ifdef _WIN32
|
||||
TEST_F(DiskInterfaceTest, StatCache) {
|
||||
string err;
|
||||
|
||||
ASSERT_TRUE(Touch("file1"));
|
||||
ASSERT_TRUE(Touch("fiLE2"));
|
||||
ASSERT_TRUE(disk_.MakeDir("subdir"));
|
||||
ASSERT_TRUE(disk_.MakeDir("subdir/subsubdir"));
|
||||
ASSERT_TRUE(Touch("subdir\\subfile1"));
|
||||
ASSERT_TRUE(Touch("subdir\\SUBFILE2"));
|
||||
ASSERT_TRUE(Touch("subdir\\SUBFILE3"));
|
||||
|
||||
disk_.AllowStatCache(false);
|
||||
TimeStamp parent_stat_uncached = disk_.Stat("..", &err);
|
||||
disk_.AllowStatCache(true);
|
||||
|
||||
EXPECT_GT(disk_.Stat("FIle1", &err), 1);
|
||||
EXPECT_EQ("", err);
|
||||
EXPECT_GT(disk_.Stat("file1", &err), 1);
|
||||
EXPECT_EQ("", err);
|
||||
|
||||
EXPECT_GT(disk_.Stat("subdir/subfile2", &err), 1);
|
||||
EXPECT_EQ("", err);
|
||||
EXPECT_GT(disk_.Stat("sUbdir\\suBFile1", &err), 1);
|
||||
EXPECT_EQ("", err);
|
||||
|
||||
EXPECT_GT(disk_.Stat("..", &err), 1);
|
||||
EXPECT_EQ("", err);
|
||||
EXPECT_GT(disk_.Stat(".", &err), 1);
|
||||
EXPECT_EQ("", err);
|
||||
EXPECT_GT(disk_.Stat("subdir", &err), 1);
|
||||
EXPECT_EQ("", err);
|
||||
EXPECT_GT(disk_.Stat("subdir/subsubdir", &err), 1);
|
||||
EXPECT_EQ("", err);
|
||||
|
||||
#ifndef _MSC_VER // TODO: Investigate why. Also see https://github.com/ninja-build/ninja/pull/1423
|
||||
EXPECT_EQ(disk_.Stat("subdir", &err),
|
||||
disk_.Stat("subdir/.", &err));
|
||||
EXPECT_EQ("", err);
|
||||
EXPECT_EQ(disk_.Stat("subdir", &err),
|
||||
disk_.Stat("subdir/subsubdir/..", &err));
|
||||
#endif
|
||||
EXPECT_EQ("", err);
|
||||
EXPECT_EQ(disk_.Stat("..", &err), parent_stat_uncached);
|
||||
EXPECT_EQ("", err);
|
||||
EXPECT_EQ(disk_.Stat("subdir/subsubdir", &err),
|
||||
disk_.Stat("subdir/subsubdir/.", &err));
|
||||
EXPECT_EQ("", err);
|
||||
|
||||
// Test error cases.
|
||||
string bad_path("cc:\\foo");
|
||||
EXPECT_EQ(-1, disk_.Stat(bad_path, &err));
|
||||
EXPECT_NE("", err); err.clear();
|
||||
EXPECT_EQ(-1, disk_.Stat(bad_path, &err));
|
||||
EXPECT_NE("", err); err.clear();
|
||||
EXPECT_EQ(0, disk_.Stat("nosuchfile", &err));
|
||||
EXPECT_EQ("", err);
|
||||
EXPECT_EQ(0, disk_.Stat("nosuchdir/nosuchfile", &err));
|
||||
EXPECT_EQ("", err);
|
||||
}
|
||||
#endif
|
||||
|
||||
TEST_F(DiskInterfaceTest, ReadFile) {
|
||||
string err;
|
||||
std::string content;
|
||||
ASSERT_EQ(DiskInterface::NotFound,
|
||||
disk_.ReadFile("foobar", &content, &err));
|
||||
EXPECT_EQ("", content);
|
||||
EXPECT_NE("", err); // actual value is platform-specific
|
||||
err.clear();
|
||||
|
||||
const char* kTestFile = "testfile";
|
||||
FILE* f = fopen(kTestFile, "wb");
|
||||
ASSERT_TRUE(f);
|
||||
const char* kTestContent = "test content\nok";
|
||||
fprintf(f, "%s", kTestContent);
|
||||
ASSERT_EQ(0, fclose(f));
|
||||
|
||||
ASSERT_EQ(DiskInterface::Okay,
|
||||
disk_.ReadFile(kTestFile, &content, &err));
|
||||
EXPECT_EQ(kTestContent, content);
|
||||
EXPECT_EQ("", err);
|
||||
}
|
||||
|
||||
TEST_F(DiskInterfaceTest, MakeDirs) {
|
||||
string path = "path/with/double//slash/";
|
||||
EXPECT_TRUE(disk_.MakeDirs(path));
|
||||
FILE* f = fopen((path + "a_file").c_str(), "w");
|
||||
EXPECT_TRUE(f);
|
||||
EXPECT_EQ(0, fclose(f));
|
||||
#ifdef _WIN32
|
||||
string path2 = "another\\with\\back\\\\slashes\\";
|
||||
EXPECT_TRUE(disk_.MakeDirs(path2));
|
||||
FILE* f2 = fopen((path2 + "a_file").c_str(), "w");
|
||||
EXPECT_TRUE(f2);
|
||||
EXPECT_EQ(0, fclose(f2));
|
||||
#endif
|
||||
}
|
||||
|
||||
TEST_F(DiskInterfaceTest, RemoveFile) {
|
||||
const char* kFileName = "file-to-remove";
|
||||
ASSERT_TRUE(Touch(kFileName));
|
||||
EXPECT_EQ(0, disk_.RemoveFile(kFileName));
|
||||
EXPECT_EQ(1, disk_.RemoveFile(kFileName));
|
||||
EXPECT_EQ(1, disk_.RemoveFile("does not exist"));
|
||||
#ifdef _WIN32
|
||||
ASSERT_TRUE(Touch(kFileName));
|
||||
EXPECT_EQ(0, system((std::string("attrib +R ") + kFileName).c_str()));
|
||||
EXPECT_EQ(0, disk_.RemoveFile(kFileName));
|
||||
EXPECT_EQ(1, disk_.RemoveFile(kFileName));
|
||||
#endif
|
||||
}
|
||||
|
||||
TEST_F(DiskInterfaceTest, RemoveDirectory) {
|
||||
const char* kDirectoryName = "directory-to-remove";
|
||||
EXPECT_TRUE(disk_.MakeDir(kDirectoryName));
|
||||
EXPECT_EQ(0, disk_.RemoveFile(kDirectoryName));
|
||||
EXPECT_EQ(1, disk_.RemoveFile(kDirectoryName));
|
||||
EXPECT_EQ(1, disk_.RemoveFile("does not exist"));
|
||||
}
|
||||
|
||||
struct StatTest : public StateTestWithBuiltinRules,
|
||||
public DiskInterface {
|
||||
StatTest() : scan_(&state_, NULL, NULL, this, NULL, NULL) {}
|
||||
|
||||
// DiskInterface implementation.
|
||||
TimeStamp Stat(const string& path, string* err) const override;
|
||||
bool WriteFile(const string& path, const string& contents,
|
||||
bool /*crlf_on_windows*/) override {
|
||||
assert(false);
|
||||
return true;
|
||||
}
|
||||
bool MakeDir(const string& path) override {
|
||||
assert(false);
|
||||
return false;
|
||||
}
|
||||
Status ReadFile(const string& path, string* contents, string* err) override {
|
||||
assert(false);
|
||||
return NotFound;
|
||||
}
|
||||
int RemoveFile(const string& path) override {
|
||||
assert(false);
|
||||
return 0;
|
||||
}
|
||||
|
||||
DependencyScan scan_;
|
||||
map<string, TimeStamp> mtimes_;
|
||||
mutable vector<string> stats_;
|
||||
};
|
||||
|
||||
TimeStamp StatTest::Stat(const string& path, string* err) const {
|
||||
stats_.push_back(path);
|
||||
map<string, TimeStamp>::const_iterator i = mtimes_.find(path);
|
||||
if (i == mtimes_.end())
|
||||
return 0; // File not found.
|
||||
return i->second;
|
||||
}
|
||||
|
||||
TEST_F(StatTest, Simple) {
|
||||
ASSERT_NO_FATAL_FAILURE(AssertParse(&state_,
|
||||
"build out: cat in\n"));
|
||||
|
||||
Node* out = GetNode("out");
|
||||
string err;
|
||||
EXPECT_TRUE(out->Stat(this, &err));
|
||||
EXPECT_EQ("", err);
|
||||
ASSERT_EQ(1u, stats_.size());
|
||||
scan_.RecomputeDirty(out, NULL, NULL);
|
||||
ASSERT_EQ(2u, stats_.size());
|
||||
ASSERT_EQ("out", stats_[0]);
|
||||
ASSERT_EQ("in", stats_[1]);
|
||||
}
|
||||
|
||||
TEST_F(StatTest, TwoStep) {
|
||||
ASSERT_NO_FATAL_FAILURE(AssertParse(&state_,
|
||||
"build out: cat mid\n"
|
||||
"build mid: cat in\n"));
|
||||
|
||||
Node* out = GetNode("out");
|
||||
string err;
|
||||
EXPECT_TRUE(out->Stat(this, &err));
|
||||
EXPECT_EQ("", err);
|
||||
ASSERT_EQ(1u, stats_.size());
|
||||
scan_.RecomputeDirty(out, NULL, NULL);
|
||||
ASSERT_EQ(3u, stats_.size());
|
||||
ASSERT_EQ("out", stats_[0]);
|
||||
ASSERT_TRUE(GetNode("out")->dirty());
|
||||
ASSERT_EQ("mid", stats_[1]);
|
||||
ASSERT_TRUE(GetNode("mid")->dirty());
|
||||
ASSERT_EQ("in", stats_[2]);
|
||||
}
|
||||
|
||||
TEST_F(StatTest, Tree) {
|
||||
ASSERT_NO_FATAL_FAILURE(AssertParse(&state_,
|
||||
"build out: cat mid1 mid2\n"
|
||||
"build mid1: cat in11 in12\n"
|
||||
"build mid2: cat in21 in22\n"));
|
||||
|
||||
Node* out = GetNode("out");
|
||||
string err;
|
||||
EXPECT_TRUE(out->Stat(this, &err));
|
||||
EXPECT_EQ("", err);
|
||||
ASSERT_EQ(1u, stats_.size());
|
||||
scan_.RecomputeDirty(out, NULL, NULL);
|
||||
ASSERT_EQ(1u + 6u, stats_.size());
|
||||
ASSERT_EQ("mid1", stats_[1]);
|
||||
ASSERT_TRUE(GetNode("mid1")->dirty());
|
||||
ASSERT_EQ("in11", stats_[2]);
|
||||
}
|
||||
|
||||
TEST_F(StatTest, Middle) {
|
||||
ASSERT_NO_FATAL_FAILURE(AssertParse(&state_,
|
||||
"build out: cat mid\n"
|
||||
"build mid: cat in\n"));
|
||||
|
||||
mtimes_["in"] = 1;
|
||||
mtimes_["mid"] = 0; // missing
|
||||
mtimes_["out"] = 1;
|
||||
|
||||
Node* out = GetNode("out");
|
||||
string err;
|
||||
EXPECT_TRUE(out->Stat(this, &err));
|
||||
EXPECT_EQ("", err);
|
||||
ASSERT_EQ(1u, stats_.size());
|
||||
scan_.RecomputeDirty(out, NULL, NULL);
|
||||
ASSERT_FALSE(GetNode("in")->dirty());
|
||||
ASSERT_TRUE(GetNode("mid")->dirty());
|
||||
ASSERT_TRUE(GetNode("out")->dirty());
|
||||
}
|
||||
|
||||
} // namespace
|
||||
@@ -0,0 +1,122 @@
|
||||
// Copyright 2015 Google Inc. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#include "dyndep.h"
|
||||
|
||||
#include <assert.h>
|
||||
#include <stdio.h>
|
||||
|
||||
#include "debug_flags.h"
|
||||
#include "disk_interface.h"
|
||||
#include "dyndep_parser.h"
|
||||
#include "explanations.h"
|
||||
#include "graph.h"
|
||||
#include "state.h"
|
||||
#include "util.h"
|
||||
|
||||
using namespace std;
|
||||
|
||||
bool DyndepLoader::LoadDyndeps(Node* node, std::string* err) const {
|
||||
DyndepFile ddf;
|
||||
return LoadDyndeps(node, &ddf, err);
|
||||
}
|
||||
|
||||
bool DyndepLoader::LoadDyndeps(Node* node, DyndepFile* ddf,
|
||||
std::string* err) const {
|
||||
// We are loading the dyndep file now so it is no longer pending.
|
||||
node->set_dyndep_pending(false);
|
||||
|
||||
// Load the dyndep information from the file.
|
||||
explanations_.Record(node, "loading dyndep file '%s'", node->path().c_str());
|
||||
|
||||
if (!LoadDyndepFile(node, ddf, err))
|
||||
return false;
|
||||
|
||||
// Update each edge that specified this node as its dyndep binding.
|
||||
std::vector<Edge*> const& out_edges = node->out_edges();
|
||||
for (Edge* edge : out_edges) {
|
||||
if (edge->dyndep_ != node)
|
||||
continue;
|
||||
|
||||
DyndepFile::iterator ddi = ddf->find(edge);
|
||||
if (ddi == ddf->end()) {
|
||||
*err = ("'" + edge->outputs_[0]->path() + "' "
|
||||
"not mentioned in its dyndep file "
|
||||
"'" + node->path() + "'");
|
||||
return false;
|
||||
}
|
||||
|
||||
ddi->second.used_ = true;
|
||||
Dyndeps const& dyndeps = ddi->second;
|
||||
if (!UpdateEdge(edge, &dyndeps, err)) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
// Reject extra outputs in dyndep file.
|
||||
for (const auto& dyndep_output : *ddf) {
|
||||
if (!dyndep_output.second.used_) {
|
||||
Edge* const edge = dyndep_output.first;
|
||||
*err = ("dyndep file '" + node->path() + "' mentions output "
|
||||
"'" + edge->outputs_[0]->path() + "' whose build statement "
|
||||
"does not have a dyndep binding for the file");
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
bool DyndepLoader::UpdateEdge(Edge* edge, Dyndeps const* dyndeps,
|
||||
std::string* err) const {
|
||||
// Add dyndep-discovered bindings to the edge.
|
||||
// We know the edge already has its own binding
|
||||
// scope because it has a "dyndep" binding.
|
||||
if (dyndeps->restat_)
|
||||
edge->env_->AddBinding("restat", "1");
|
||||
|
||||
// Add the dyndep-discovered outputs to the edge.
|
||||
edge->outputs_.insert(edge->outputs_.end(),
|
||||
dyndeps->implicit_outputs_.begin(),
|
||||
dyndeps->implicit_outputs_.end());
|
||||
edge->implicit_outs_ += dyndeps->implicit_outputs_.size();
|
||||
|
||||
// Add this edge as incoming to each new output.
|
||||
for (Node* node : dyndeps->implicit_outputs_) {
|
||||
if (node->in_edge()) {
|
||||
// This node already has an edge producing it.
|
||||
*err = "multiple rules generate " + node->path();
|
||||
return false;
|
||||
}
|
||||
node->set_in_edge(edge);
|
||||
}
|
||||
|
||||
// Add the dyndep-discovered inputs to the edge.
|
||||
edge->inputs_.insert(edge->inputs_.end() - edge->order_only_deps_,
|
||||
dyndeps->implicit_inputs_.begin(),
|
||||
dyndeps->implicit_inputs_.end());
|
||||
edge->implicit_deps_ += dyndeps->implicit_inputs_.size();
|
||||
|
||||
// Add this edge as outgoing from each new input.
|
||||
for (Node* node : dyndeps->implicit_inputs_)
|
||||
node->AddOutEdge(edge);
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
bool DyndepLoader::LoadDyndepFile(Node* file, DyndepFile* ddf,
|
||||
std::string* err) const {
|
||||
DyndepParser parser(state_, disk_interface_, ddf);
|
||||
return parser.Load(file->path(), err);
|
||||
}
|
||||
@@ -0,0 +1,69 @@
|
||||
// Copyright 2015 Google Inc. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#ifndef NINJA_DYNDEP_LOADER_H_
|
||||
#define NINJA_DYNDEP_LOADER_H_
|
||||
|
||||
#include <map>
|
||||
#include <string>
|
||||
#include <vector>
|
||||
|
||||
#include "explanations.h"
|
||||
|
||||
struct DiskInterface;
|
||||
struct Edge;
|
||||
struct Node;
|
||||
struct State;
|
||||
|
||||
/// Store dynamically-discovered dependency information for one edge.
|
||||
struct Dyndeps {
|
||||
Dyndeps() : used_(false), restat_(false) {}
|
||||
bool used_;
|
||||
bool restat_;
|
||||
std::vector<Node*> implicit_inputs_;
|
||||
std::vector<Node*> implicit_outputs_;
|
||||
};
|
||||
|
||||
/// Store data loaded from one dyndep file. Map from an edge
|
||||
/// to its dynamically-discovered dependency information.
|
||||
/// This is a struct rather than a typedef so that we can
|
||||
/// forward-declare it in other headers.
|
||||
struct DyndepFile: public std::map<Edge*, Dyndeps> {};
|
||||
|
||||
/// DyndepLoader loads dynamically discovered dependencies, as
|
||||
/// referenced via the "dyndep" attribute in build files.
|
||||
struct DyndepLoader {
|
||||
DyndepLoader(State* state, DiskInterface* disk_interface,
|
||||
Explanations* explanations = nullptr)
|
||||
: state_(state), disk_interface_(disk_interface),
|
||||
explanations_(explanations) {}
|
||||
|
||||
/// Load a dyndep file from the given node's path and update the
|
||||
/// build graph with the new information. One overload accepts
|
||||
/// a caller-owned 'DyndepFile' object in which to store the
|
||||
/// information loaded from the dyndep file.
|
||||
bool LoadDyndeps(Node* node, std::string* err) const;
|
||||
bool LoadDyndeps(Node* node, DyndepFile* ddf, std::string* err) const;
|
||||
|
||||
private:
|
||||
bool LoadDyndepFile(Node* file, DyndepFile* ddf, std::string* err) const;
|
||||
|
||||
bool UpdateEdge(Edge* edge, Dyndeps const* dyndeps, std::string* err) const;
|
||||
|
||||
State* state_;
|
||||
DiskInterface* disk_interface_;
|
||||
mutable OptionalExplanations explanations_;
|
||||
};
|
||||
|
||||
#endif // NINJA_DYNDEP_LOADER_H_
|
||||
@@ -0,0 +1,220 @@
|
||||
// Copyright 2015 Google Inc. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#include "dyndep_parser.h"
|
||||
|
||||
#include <vector>
|
||||
|
||||
#include "dyndep.h"
|
||||
#include "graph.h"
|
||||
#include "state.h"
|
||||
#include "util.h"
|
||||
#include "version.h"
|
||||
|
||||
using namespace std;
|
||||
|
||||
DyndepParser::DyndepParser(State* state, FileReader* file_reader,
|
||||
DyndepFile* dyndep_file)
|
||||
: Parser(state, file_reader)
|
||||
, dyndep_file_(dyndep_file) {
|
||||
}
|
||||
|
||||
bool DyndepParser::Parse(const string& filename, const string& input,
|
||||
string* err) {
|
||||
lexer_.Start(filename, input);
|
||||
|
||||
// Require a supported ninja_dyndep_version value immediately so
|
||||
// we can exit before encountering any syntactic surprises.
|
||||
bool haveDyndepVersion = false;
|
||||
|
||||
for (;;) {
|
||||
Lexer::Token token = lexer_.ReadToken();
|
||||
switch (token) {
|
||||
case Lexer::BUILD: {
|
||||
if (!haveDyndepVersion)
|
||||
return lexer_.Error("expected 'ninja_dyndep_version = ...'", err);
|
||||
if (!ParseEdge(err))
|
||||
return false;
|
||||
break;
|
||||
}
|
||||
case Lexer::IDENT: {
|
||||
lexer_.UnreadToken();
|
||||
if (haveDyndepVersion)
|
||||
return lexer_.Error(string("unexpected ") + Lexer::TokenName(token),
|
||||
err);
|
||||
if (!ParseDyndepVersion(err))
|
||||
return false;
|
||||
haveDyndepVersion = true;
|
||||
break;
|
||||
}
|
||||
case Lexer::ERROR:
|
||||
return lexer_.Error(lexer_.DescribeLastError(), err);
|
||||
case Lexer::TEOF:
|
||||
if (!haveDyndepVersion)
|
||||
return lexer_.Error("expected 'ninja_dyndep_version = ...'", err);
|
||||
return true;
|
||||
case Lexer::NEWLINE:
|
||||
break;
|
||||
default:
|
||||
return lexer_.Error(string("unexpected ") + Lexer::TokenName(token),
|
||||
err);
|
||||
}
|
||||
}
|
||||
return false; // not reached
|
||||
}
|
||||
|
||||
bool DyndepParser::ParseDyndepVersion(string* err) {
|
||||
string name;
|
||||
EvalString let_value;
|
||||
if (!ParseLet(&name, &let_value, err))
|
||||
return false;
|
||||
if (name != "ninja_dyndep_version") {
|
||||
return lexer_.Error("expected 'ninja_dyndep_version = ...'", err);
|
||||
}
|
||||
string version = let_value.Evaluate(&env_);
|
||||
int major, minor;
|
||||
ParseVersion(version, &major, &minor);
|
||||
if (major != 1 || minor != 0) {
|
||||
return lexer_.Error(
|
||||
string("unsupported 'ninja_dyndep_version = ") + version + "'", err);
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
bool DyndepParser::ParseLet(string* key, EvalString* value, string* err) {
|
||||
if (!lexer_.ReadIdent(key))
|
||||
return lexer_.Error("expected variable name", err);
|
||||
return (ExpectToken(Lexer::EQUALS, err) && lexer_.ReadVarValue(value, err));
|
||||
}
|
||||
|
||||
bool DyndepParser::ParseEdge(string* err) {
|
||||
// Parse one explicit output. We expect it to already have an edge.
|
||||
// We will record its dynamically-discovered dependency information.
|
||||
Dyndeps* dyndeps = NULL;
|
||||
{
|
||||
EvalString out0;
|
||||
if (!lexer_.ReadPath(&out0, err))
|
||||
return false;
|
||||
if (out0.empty())
|
||||
return lexer_.Error("expected path", err);
|
||||
|
||||
string path = out0.Evaluate(&env_);
|
||||
if (path.empty())
|
||||
return lexer_.Error("empty path", err);
|
||||
uint64_t slash_bits;
|
||||
CanonicalizePath(&path, &slash_bits);
|
||||
Node* node = state_->LookupNode(path);
|
||||
if (!node || !node->in_edge())
|
||||
return lexer_.Error("no build statement exists for '" + path + "'", err);
|
||||
Edge* edge = node->in_edge();
|
||||
std::pair<DyndepFile::iterator, bool> res =
|
||||
dyndep_file_->insert(DyndepFile::value_type(edge, Dyndeps()));
|
||||
if (!res.second)
|
||||
return lexer_.Error("multiple statements for '" + path + "'", err);
|
||||
dyndeps = &res.first->second;
|
||||
}
|
||||
|
||||
// Disallow explicit outputs.
|
||||
{
|
||||
EvalString out;
|
||||
if (!lexer_.ReadPath(&out, err))
|
||||
return false;
|
||||
if (!out.empty())
|
||||
return lexer_.Error("explicit outputs not supported", err);
|
||||
}
|
||||
|
||||
// Parse implicit outputs, if any.
|
||||
vector<EvalString> outs;
|
||||
if (lexer_.PeekToken(Lexer::PIPE)) {
|
||||
for (;;) {
|
||||
EvalString out;
|
||||
if (!lexer_.ReadPath(&out, err))
|
||||
return err;
|
||||
if (out.empty())
|
||||
break;
|
||||
outs.push_back(out);
|
||||
}
|
||||
}
|
||||
|
||||
if (!ExpectToken(Lexer::COLON, err))
|
||||
return false;
|
||||
|
||||
string rule_name;
|
||||
if (!lexer_.ReadIdent(&rule_name) || rule_name != "dyndep")
|
||||
return lexer_.Error("expected build command name 'dyndep'", err);
|
||||
|
||||
// Disallow explicit inputs.
|
||||
{
|
||||
EvalString in;
|
||||
if (!lexer_.ReadPath(&in, err))
|
||||
return false;
|
||||
if (!in.empty())
|
||||
return lexer_.Error("explicit inputs not supported", err);
|
||||
}
|
||||
|
||||
// Parse implicit inputs, if any.
|
||||
vector<EvalString> ins;
|
||||
if (lexer_.PeekToken(Lexer::PIPE)) {
|
||||
for (;;) {
|
||||
EvalString in;
|
||||
if (!lexer_.ReadPath(&in, err))
|
||||
return err;
|
||||
if (in.empty())
|
||||
break;
|
||||
ins.push_back(in);
|
||||
}
|
||||
}
|
||||
|
||||
// Disallow order-only inputs.
|
||||
if (lexer_.PeekToken(Lexer::PIPE2))
|
||||
return lexer_.Error("order-only inputs not supported", err);
|
||||
|
||||
if (!ExpectToken(Lexer::NEWLINE, err))
|
||||
return false;
|
||||
|
||||
if (lexer_.PeekToken(Lexer::INDENT)) {
|
||||
string key;
|
||||
EvalString val;
|
||||
if (!ParseLet(&key, &val, err))
|
||||
return false;
|
||||
if (key != "restat")
|
||||
return lexer_.Error("binding is not 'restat'", err);
|
||||
string value = val.Evaluate(&env_);
|
||||
dyndeps->restat_ = !value.empty();
|
||||
}
|
||||
|
||||
dyndeps->implicit_inputs_.reserve(ins.size());
|
||||
for (const EvalString& in : ins) {
|
||||
string path = in.Evaluate(&env_);
|
||||
if (path.empty())
|
||||
return lexer_.Error("empty path", err);
|
||||
uint64_t slash_bits;
|
||||
CanonicalizePath(&path, &slash_bits);
|
||||
Node* n = state_->GetNode(path, slash_bits);
|
||||
dyndeps->implicit_inputs_.push_back(n);
|
||||
}
|
||||
|
||||
dyndeps->implicit_outputs_.reserve(outs.size());
|
||||
for (const EvalString& out : outs) {
|
||||
string path = out.Evaluate(&env_);
|
||||
if (path.empty())
|
||||
return lexer_.Error("empty path", err);
|
||||
uint64_t slash_bits;
|
||||
CanonicalizePath(&path, &slash_bits);
|
||||
Node* n = state_->GetNode(path, slash_bits);
|
||||
dyndeps->implicit_outputs_.push_back(n);
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
@@ -0,0 +1,47 @@
|
||||
// Copyright 2015 Google Inc. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#ifndef NINJA_DYNDEP_PARSER_H_
|
||||
#define NINJA_DYNDEP_PARSER_H_
|
||||
|
||||
#include "eval_env.h"
|
||||
#include "parser.h"
|
||||
|
||||
struct DyndepFile;
|
||||
struct EvalString;
|
||||
|
||||
/// Parses dyndep files.
|
||||
struct DyndepParser: public Parser {
|
||||
DyndepParser(State* state, FileReader* file_reader,
|
||||
DyndepFile* dyndep_file);
|
||||
|
||||
/// Parse a text string of input. Used by tests.
|
||||
bool ParseTest(const std::string& input, std::string* err) {
|
||||
return Parse("input", input, err);
|
||||
}
|
||||
|
||||
private:
|
||||
/// Parse a file, given its contents as a string.
|
||||
bool Parse(const std::string& filename, const std::string& input,
|
||||
std:: string* err);
|
||||
|
||||
bool ParseDyndepVersion(std::string* err);
|
||||
bool ParseLet(std::string* key, EvalString* val, std::string* err);
|
||||
bool ParseEdge(std::string* err);
|
||||
|
||||
DyndepFile* dyndep_file_;
|
||||
BindingEnv env_;
|
||||
};
|
||||
|
||||
#endif // NINJA_DYNDEP_PARSER_H_
|
||||
@@ -0,0 +1,514 @@
|
||||
// Copyright 2015 Google Inc. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#include "dyndep_parser.h"
|
||||
|
||||
#include <map>
|
||||
#include <vector>
|
||||
|
||||
#include "dyndep.h"
|
||||
#include "graph.h"
|
||||
#include "state.h"
|
||||
#include "test.h"
|
||||
|
||||
using namespace std;
|
||||
|
||||
struct DyndepParserTest : public testing::Test {
|
||||
void AssertParse(const char* input) {
|
||||
DyndepParser parser(&state_, &fs_, &dyndep_file_);
|
||||
string err;
|
||||
EXPECT_TRUE(parser.ParseTest(input, &err));
|
||||
ASSERT_EQ("", err);
|
||||
}
|
||||
|
||||
virtual void SetUp() {
|
||||
::AssertParse(&state_,
|
||||
"rule touch\n"
|
||||
" command = touch $out\n"
|
||||
"build out otherout: touch\n");
|
||||
}
|
||||
|
||||
State state_;
|
||||
VirtualFileSystem fs_;
|
||||
DyndepFile dyndep_file_;
|
||||
};
|
||||
|
||||
TEST_F(DyndepParserTest, Empty) {
|
||||
const char kInput[] =
|
||||
"";
|
||||
DyndepParser parser(&state_, &fs_, &dyndep_file_);
|
||||
string err;
|
||||
EXPECT_FALSE(parser.ParseTest(kInput, &err));
|
||||
EXPECT_EQ("input:1: expected 'ninja_dyndep_version = ...'\n", err);
|
||||
}
|
||||
|
||||
TEST_F(DyndepParserTest, Version1) {
|
||||
ASSERT_NO_FATAL_FAILURE(AssertParse(
|
||||
"ninja_dyndep_version = 1\n"));
|
||||
}
|
||||
|
||||
TEST_F(DyndepParserTest, Version1Extra) {
|
||||
ASSERT_NO_FATAL_FAILURE(AssertParse(
|
||||
"ninja_dyndep_version = 1-extra\n"));
|
||||
}
|
||||
|
||||
TEST_F(DyndepParserTest, Version1_0) {
|
||||
ASSERT_NO_FATAL_FAILURE(AssertParse(
|
||||
"ninja_dyndep_version = 1.0\n"));
|
||||
}
|
||||
|
||||
TEST_F(DyndepParserTest, Version1_0Extra) {
|
||||
ASSERT_NO_FATAL_FAILURE(AssertParse(
|
||||
"ninja_dyndep_version = 1.0-extra\n"));
|
||||
}
|
||||
|
||||
TEST_F(DyndepParserTest, CommentVersion) {
|
||||
ASSERT_NO_FATAL_FAILURE(AssertParse(
|
||||
"# comment\n"
|
||||
"ninja_dyndep_version = 1\n"));
|
||||
}
|
||||
|
||||
TEST_F(DyndepParserTest, BlankLineVersion) {
|
||||
ASSERT_NO_FATAL_FAILURE(AssertParse(
|
||||
"\n"
|
||||
"ninja_dyndep_version = 1\n"));
|
||||
}
|
||||
|
||||
TEST_F(DyndepParserTest, VersionCRLF) {
|
||||
ASSERT_NO_FATAL_FAILURE(AssertParse(
|
||||
"ninja_dyndep_version = 1\r\n"));
|
||||
}
|
||||
|
||||
TEST_F(DyndepParserTest, CommentVersionCRLF) {
|
||||
ASSERT_NO_FATAL_FAILURE(AssertParse(
|
||||
"# comment\r\n"
|
||||
"ninja_dyndep_version = 1\r\n"));
|
||||
}
|
||||
|
||||
TEST_F(DyndepParserTest, BlankLineVersionCRLF) {
|
||||
ASSERT_NO_FATAL_FAILURE(AssertParse(
|
||||
"\r\n"
|
||||
"ninja_dyndep_version = 1\r\n"));
|
||||
}
|
||||
|
||||
TEST_F(DyndepParserTest, VersionUnexpectedEOF) {
|
||||
const char kInput[] =
|
||||
"ninja_dyndep_version = 1.0";
|
||||
DyndepParser parser(&state_, &fs_, &dyndep_file_);
|
||||
string err;
|
||||
EXPECT_FALSE(parser.ParseTest(kInput, &err));
|
||||
EXPECT_EQ("input:1: unexpected EOF\n"
|
||||
"ninja_dyndep_version = 1.0\n"
|
||||
" ^ near here", err);
|
||||
}
|
||||
|
||||
TEST_F(DyndepParserTest, UnsupportedVersion0) {
|
||||
const char kInput[] =
|
||||
"ninja_dyndep_version = 0\n";
|
||||
DyndepParser parser(&state_, &fs_, &dyndep_file_);
|
||||
string err;
|
||||
EXPECT_FALSE(parser.ParseTest(kInput, &err));
|
||||
EXPECT_EQ("input:1: unsupported 'ninja_dyndep_version = 0'\n"
|
||||
"ninja_dyndep_version = 0\n"
|
||||
" ^ near here", err);
|
||||
}
|
||||
|
||||
TEST_F(DyndepParserTest, UnsupportedVersion1_1) {
|
||||
const char kInput[] =
|
||||
"ninja_dyndep_version = 1.1\n";
|
||||
DyndepParser parser(&state_, &fs_, &dyndep_file_);
|
||||
string err;
|
||||
EXPECT_FALSE(parser.ParseTest(kInput, &err));
|
||||
EXPECT_EQ("input:1: unsupported 'ninja_dyndep_version = 1.1'\n"
|
||||
"ninja_dyndep_version = 1.1\n"
|
||||
" ^ near here", err);
|
||||
}
|
||||
|
||||
TEST_F(DyndepParserTest, DuplicateVersion) {
|
||||
const char kInput[] =
|
||||
"ninja_dyndep_version = 1\n"
|
||||
"ninja_dyndep_version = 1\n";
|
||||
DyndepParser parser(&state_, &fs_, &dyndep_file_);
|
||||
string err;
|
||||
EXPECT_FALSE(parser.ParseTest(kInput, &err));
|
||||
EXPECT_EQ("input:2: unexpected identifier\n", err);
|
||||
}
|
||||
|
||||
TEST_F(DyndepParserTest, MissingVersionOtherVar) {
|
||||
const char kInput[] =
|
||||
"not_ninja_dyndep_version = 1\n";
|
||||
DyndepParser parser(&state_, &fs_, &dyndep_file_);
|
||||
string err;
|
||||
EXPECT_FALSE(parser.ParseTest(kInput, &err));
|
||||
EXPECT_EQ("input:1: expected 'ninja_dyndep_version = ...'\n"
|
||||
"not_ninja_dyndep_version = 1\n"
|
||||
" ^ near here", err);
|
||||
}
|
||||
|
||||
TEST_F(DyndepParserTest, MissingVersionBuild) {
|
||||
const char kInput[] =
|
||||
"build out: dyndep\n";
|
||||
DyndepParser parser(&state_, &fs_, &dyndep_file_);
|
||||
string err;
|
||||
EXPECT_FALSE(parser.ParseTest(kInput, &err));
|
||||
EXPECT_EQ("input:1: expected 'ninja_dyndep_version = ...'\n", err);
|
||||
}
|
||||
|
||||
TEST_F(DyndepParserTest, UnexpectedEqual) {
|
||||
const char kInput[] =
|
||||
"= 1\n";
|
||||
DyndepParser parser(&state_, &fs_, &dyndep_file_);
|
||||
string err;
|
||||
EXPECT_FALSE(parser.ParseTest(kInput, &err));
|
||||
EXPECT_EQ("input:1: unexpected '='\n", err);
|
||||
}
|
||||
|
||||
TEST_F(DyndepParserTest, UnexpectedIndent) {
|
||||
const char kInput[] =
|
||||
" = 1\n";
|
||||
DyndepParser parser(&state_, &fs_, &dyndep_file_);
|
||||
string err;
|
||||
EXPECT_FALSE(parser.ParseTest(kInput, &err));
|
||||
EXPECT_EQ("input:1: unexpected indent\n", err);
|
||||
}
|
||||
|
||||
TEST_F(DyndepParserTest, OutDuplicate) {
|
||||
const char kInput[] =
|
||||
"ninja_dyndep_version = 1\n"
|
||||
"build out: dyndep\n"
|
||||
"build out: dyndep\n";
|
||||
DyndepParser parser(&state_, &fs_, &dyndep_file_);
|
||||
string err;
|
||||
EXPECT_FALSE(parser.ParseTest(kInput, &err));
|
||||
EXPECT_EQ("input:3: multiple statements for 'out'\n"
|
||||
"build out: dyndep\n"
|
||||
" ^ near here", err);
|
||||
}
|
||||
|
||||
TEST_F(DyndepParserTest, OutDuplicateThroughOther) {
|
||||
const char kInput[] =
|
||||
"ninja_dyndep_version = 1\n"
|
||||
"build out: dyndep\n"
|
||||
"build otherout: dyndep\n";
|
||||
DyndepParser parser(&state_, &fs_, &dyndep_file_);
|
||||
string err;
|
||||
EXPECT_FALSE(parser.ParseTest(kInput, &err));
|
||||
EXPECT_EQ("input:3: multiple statements for 'otherout'\n"
|
||||
"build otherout: dyndep\n"
|
||||
" ^ near here", err);
|
||||
}
|
||||
|
||||
TEST_F(DyndepParserTest, NoOutEOF) {
|
||||
const char kInput[] =
|
||||
"ninja_dyndep_version = 1\n"
|
||||
"build";
|
||||
DyndepParser parser(&state_, &fs_, &dyndep_file_);
|
||||
string err;
|
||||
EXPECT_FALSE(parser.ParseTest(kInput, &err));
|
||||
EXPECT_EQ("input:2: unexpected EOF\n"
|
||||
"build\n"
|
||||
" ^ near here", err);
|
||||
}
|
||||
|
||||
TEST_F(DyndepParserTest, NoOutColon) {
|
||||
const char kInput[] =
|
||||
"ninja_dyndep_version = 1\n"
|
||||
"build :\n";
|
||||
DyndepParser parser(&state_, &fs_, &dyndep_file_);
|
||||
string err;
|
||||
EXPECT_FALSE(parser.ParseTest(kInput, &err));
|
||||
EXPECT_EQ("input:2: expected path\n"
|
||||
"build :\n"
|
||||
" ^ near here", err);
|
||||
}
|
||||
|
||||
TEST_F(DyndepParserTest, OutNoStatement) {
|
||||
const char kInput[] =
|
||||
"ninja_dyndep_version = 1\n"
|
||||
"build missing: dyndep\n";
|
||||
DyndepParser parser(&state_, &fs_, &dyndep_file_);
|
||||
string err;
|
||||
EXPECT_FALSE(parser.ParseTest(kInput, &err));
|
||||
EXPECT_EQ("input:2: no build statement exists for 'missing'\n"
|
||||
"build missing: dyndep\n"
|
||||
" ^ near here", err);
|
||||
}
|
||||
|
||||
TEST_F(DyndepParserTest, OutEOF) {
|
||||
const char kInput[] =
|
||||
"ninja_dyndep_version = 1\n"
|
||||
"build out";
|
||||
DyndepParser parser(&state_, &fs_, &dyndep_file_);
|
||||
string err;
|
||||
EXPECT_FALSE(parser.ParseTest(kInput, &err));
|
||||
EXPECT_EQ("input:2: unexpected EOF\n"
|
||||
"build out\n"
|
||||
" ^ near here", err);
|
||||
}
|
||||
|
||||
TEST_F(DyndepParserTest, OutNoRule) {
|
||||
const char kInput[] =
|
||||
"ninja_dyndep_version = 1\n"
|
||||
"build out:";
|
||||
DyndepParser parser(&state_, &fs_, &dyndep_file_);
|
||||
string err;
|
||||
EXPECT_FALSE(parser.ParseTest(kInput, &err));
|
||||
EXPECT_EQ("input:2: expected build command name 'dyndep'\n"
|
||||
"build out:\n"
|
||||
" ^ near here", err);
|
||||
}
|
||||
|
||||
TEST_F(DyndepParserTest, OutBadRule) {
|
||||
const char kInput[] =
|
||||
"ninja_dyndep_version = 1\n"
|
||||
"build out: touch";
|
||||
DyndepParser parser(&state_, &fs_, &dyndep_file_);
|
||||
string err;
|
||||
EXPECT_FALSE(parser.ParseTest(kInput, &err));
|
||||
EXPECT_EQ("input:2: expected build command name 'dyndep'\n"
|
||||
"build out: touch\n"
|
||||
" ^ near here", err);
|
||||
}
|
||||
|
||||
TEST_F(DyndepParserTest, BuildEOF) {
|
||||
const char kInput[] =
|
||||
"ninja_dyndep_version = 1\n"
|
||||
"build out: dyndep";
|
||||
DyndepParser parser(&state_, &fs_, &dyndep_file_);
|
||||
string err;
|
||||
EXPECT_FALSE(parser.ParseTest(kInput, &err));
|
||||
EXPECT_EQ("input:2: unexpected EOF\n"
|
||||
"build out: dyndep\n"
|
||||
" ^ near here", err);
|
||||
}
|
||||
|
||||
TEST_F(DyndepParserTest, ExplicitOut) {
|
||||
const char kInput[] =
|
||||
"ninja_dyndep_version = 1\n"
|
||||
"build out exp: dyndep\n";
|
||||
DyndepParser parser(&state_, &fs_, &dyndep_file_);
|
||||
string err;
|
||||
EXPECT_FALSE(parser.ParseTest(kInput, &err));
|
||||
EXPECT_EQ("input:2: explicit outputs not supported\n"
|
||||
"build out exp: dyndep\n"
|
||||
" ^ near here", err);
|
||||
}
|
||||
|
||||
TEST_F(DyndepParserTest, ExplicitIn) {
|
||||
const char kInput[] =
|
||||
"ninja_dyndep_version = 1\n"
|
||||
"build out: dyndep exp\n";
|
||||
DyndepParser parser(&state_, &fs_, &dyndep_file_);
|
||||
string err;
|
||||
EXPECT_FALSE(parser.ParseTest(kInput, &err));
|
||||
EXPECT_EQ("input:2: explicit inputs not supported\n"
|
||||
"build out: dyndep exp\n"
|
||||
" ^ near here", err);
|
||||
}
|
||||
|
||||
TEST_F(DyndepParserTest, OrderOnlyIn) {
|
||||
const char kInput[] =
|
||||
"ninja_dyndep_version = 1\n"
|
||||
"build out: dyndep ||\n";
|
||||
DyndepParser parser(&state_, &fs_, &dyndep_file_);
|
||||
string err;
|
||||
EXPECT_FALSE(parser.ParseTest(kInput, &err));
|
||||
EXPECT_EQ("input:2: order-only inputs not supported\n"
|
||||
"build out: dyndep ||\n"
|
||||
" ^ near here", err);
|
||||
}
|
||||
|
||||
TEST_F(DyndepParserTest, BadBinding) {
|
||||
const char kInput[] =
|
||||
"ninja_dyndep_version = 1\n"
|
||||
"build out: dyndep\n"
|
||||
" not_restat = 1\n";
|
||||
DyndepParser parser(&state_, &fs_, &dyndep_file_);
|
||||
string err;
|
||||
EXPECT_FALSE(parser.ParseTest(kInput, &err));
|
||||
EXPECT_EQ("input:3: binding is not 'restat'\n"
|
||||
" not_restat = 1\n"
|
||||
" ^ near here", err);
|
||||
}
|
||||
|
||||
TEST_F(DyndepParserTest, RestatTwice) {
|
||||
const char kInput[] =
|
||||
"ninja_dyndep_version = 1\n"
|
||||
"build out: dyndep\n"
|
||||
" restat = 1\n"
|
||||
" restat = 1\n";
|
||||
DyndepParser parser(&state_, &fs_, &dyndep_file_);
|
||||
string err;
|
||||
EXPECT_FALSE(parser.ParseTest(kInput, &err));
|
||||
EXPECT_EQ("input:4: unexpected indent\n", err);
|
||||
}
|
||||
|
||||
TEST_F(DyndepParserTest, NoImplicit) {
|
||||
ASSERT_NO_FATAL_FAILURE(AssertParse(
|
||||
"ninja_dyndep_version = 1\n"
|
||||
"build out: dyndep\n"));
|
||||
|
||||
EXPECT_EQ(1u, dyndep_file_.size());
|
||||
DyndepFile::iterator i = dyndep_file_.find(state_.edges_[0]);
|
||||
ASSERT_NE(i, dyndep_file_.end());
|
||||
EXPECT_EQ(false, i->second.restat_);
|
||||
EXPECT_EQ(0u, i->second.implicit_outputs_.size());
|
||||
EXPECT_EQ(0u, i->second.implicit_inputs_.size());
|
||||
}
|
||||
|
||||
TEST_F(DyndepParserTest, EmptyImplicit) {
|
||||
ASSERT_NO_FATAL_FAILURE(AssertParse(
|
||||
"ninja_dyndep_version = 1\n"
|
||||
"build out | : dyndep |\n"));
|
||||
|
||||
EXPECT_EQ(1u, dyndep_file_.size());
|
||||
DyndepFile::iterator i = dyndep_file_.find(state_.edges_[0]);
|
||||
ASSERT_NE(i, dyndep_file_.end());
|
||||
EXPECT_EQ(false, i->second.restat_);
|
||||
EXPECT_EQ(0u, i->second.implicit_outputs_.size());
|
||||
EXPECT_EQ(0u, i->second.implicit_inputs_.size());
|
||||
}
|
||||
|
||||
TEST_F(DyndepParserTest, ImplicitIn) {
|
||||
ASSERT_NO_FATAL_FAILURE(AssertParse(
|
||||
"ninja_dyndep_version = 1\n"
|
||||
"build out: dyndep | impin\n"));
|
||||
|
||||
EXPECT_EQ(1u, dyndep_file_.size());
|
||||
DyndepFile::iterator i = dyndep_file_.find(state_.edges_[0]);
|
||||
ASSERT_NE(i, dyndep_file_.end());
|
||||
EXPECT_EQ(false, i->second.restat_);
|
||||
EXPECT_EQ(0u, i->second.implicit_outputs_.size());
|
||||
ASSERT_EQ(1u, i->second.implicit_inputs_.size());
|
||||
EXPECT_EQ("impin", i->second.implicit_inputs_[0]->path());
|
||||
}
|
||||
|
||||
TEST_F(DyndepParserTest, ImplicitIns) {
|
||||
ASSERT_NO_FATAL_FAILURE(AssertParse(
|
||||
"ninja_dyndep_version = 1\n"
|
||||
"build out: dyndep | impin1 impin2\n"));
|
||||
|
||||
EXPECT_EQ(1u, dyndep_file_.size());
|
||||
DyndepFile::iterator i = dyndep_file_.find(state_.edges_[0]);
|
||||
ASSERT_NE(i, dyndep_file_.end());
|
||||
EXPECT_EQ(false, i->second.restat_);
|
||||
EXPECT_EQ(0u, i->second.implicit_outputs_.size());
|
||||
ASSERT_EQ(2u, i->second.implicit_inputs_.size());
|
||||
EXPECT_EQ("impin1", i->second.implicit_inputs_[0]->path());
|
||||
EXPECT_EQ("impin2", i->second.implicit_inputs_[1]->path());
|
||||
}
|
||||
|
||||
TEST_F(DyndepParserTest, ImplicitOut) {
|
||||
ASSERT_NO_FATAL_FAILURE(AssertParse(
|
||||
"ninja_dyndep_version = 1\n"
|
||||
"build out | impout: dyndep\n"));
|
||||
|
||||
EXPECT_EQ(1u, dyndep_file_.size());
|
||||
DyndepFile::iterator i = dyndep_file_.find(state_.edges_[0]);
|
||||
ASSERT_NE(i, dyndep_file_.end());
|
||||
EXPECT_EQ(false, i->second.restat_);
|
||||
ASSERT_EQ(1u, i->second.implicit_outputs_.size());
|
||||
EXPECT_EQ("impout", i->second.implicit_outputs_[0]->path());
|
||||
EXPECT_EQ(0u, i->second.implicit_inputs_.size());
|
||||
}
|
||||
|
||||
TEST_F(DyndepParserTest, ImplicitOuts) {
|
||||
ASSERT_NO_FATAL_FAILURE(AssertParse(
|
||||
"ninja_dyndep_version = 1\n"
|
||||
"build out | impout1 impout2 : dyndep\n"));
|
||||
|
||||
EXPECT_EQ(1u, dyndep_file_.size());
|
||||
DyndepFile::iterator i = dyndep_file_.find(state_.edges_[0]);
|
||||
ASSERT_NE(i, dyndep_file_.end());
|
||||
EXPECT_EQ(false, i->second.restat_);
|
||||
ASSERT_EQ(2u, i->second.implicit_outputs_.size());
|
||||
EXPECT_EQ("impout1", i->second.implicit_outputs_[0]->path());
|
||||
EXPECT_EQ("impout2", i->second.implicit_outputs_[1]->path());
|
||||
EXPECT_EQ(0u, i->second.implicit_inputs_.size());
|
||||
}
|
||||
|
||||
TEST_F(DyndepParserTest, ImplicitInsAndOuts) {
|
||||
ASSERT_NO_FATAL_FAILURE(AssertParse(
|
||||
"ninja_dyndep_version = 1\n"
|
||||
"build out | impout1 impout2: dyndep | impin1 impin2\n"));
|
||||
|
||||
EXPECT_EQ(1u, dyndep_file_.size());
|
||||
DyndepFile::iterator i = dyndep_file_.find(state_.edges_[0]);
|
||||
ASSERT_NE(i, dyndep_file_.end());
|
||||
EXPECT_EQ(false, i->second.restat_);
|
||||
ASSERT_EQ(2u, i->second.implicit_outputs_.size());
|
||||
EXPECT_EQ("impout1", i->second.implicit_outputs_[0]->path());
|
||||
EXPECT_EQ("impout2", i->second.implicit_outputs_[1]->path());
|
||||
ASSERT_EQ(2u, i->second.implicit_inputs_.size());
|
||||
EXPECT_EQ("impin1", i->second.implicit_inputs_[0]->path());
|
||||
EXPECT_EQ("impin2", i->second.implicit_inputs_[1]->path());
|
||||
}
|
||||
|
||||
TEST_F(DyndepParserTest, Restat) {
|
||||
ASSERT_NO_FATAL_FAILURE(AssertParse(
|
||||
"ninja_dyndep_version = 1\n"
|
||||
"build out: dyndep\n"
|
||||
" restat = 1\n"));
|
||||
|
||||
EXPECT_EQ(1u, dyndep_file_.size());
|
||||
DyndepFile::iterator i = dyndep_file_.find(state_.edges_[0]);
|
||||
ASSERT_NE(i, dyndep_file_.end());
|
||||
EXPECT_EQ(true, i->second.restat_);
|
||||
EXPECT_EQ(0u, i->second.implicit_outputs_.size());
|
||||
EXPECT_EQ(0u, i->second.implicit_inputs_.size());
|
||||
}
|
||||
|
||||
TEST_F(DyndepParserTest, OtherOutput) {
|
||||
ASSERT_NO_FATAL_FAILURE(AssertParse(
|
||||
"ninja_dyndep_version = 1\n"
|
||||
"build otherout: dyndep\n"));
|
||||
|
||||
EXPECT_EQ(1u, dyndep_file_.size());
|
||||
DyndepFile::iterator i = dyndep_file_.find(state_.edges_[0]);
|
||||
ASSERT_NE(i, dyndep_file_.end());
|
||||
EXPECT_EQ(false, i->second.restat_);
|
||||
EXPECT_EQ(0u, i->second.implicit_outputs_.size());
|
||||
EXPECT_EQ(0u, i->second.implicit_inputs_.size());
|
||||
}
|
||||
|
||||
TEST_F(DyndepParserTest, MultipleEdges) {
|
||||
::AssertParse(&state_,
|
||||
"build out2: touch\n");
|
||||
ASSERT_EQ(2u, state_.edges_.size());
|
||||
ASSERT_EQ(1u, state_.edges_[1]->outputs_.size());
|
||||
EXPECT_EQ("out2", state_.edges_[1]->outputs_[0]->path());
|
||||
EXPECT_EQ(0u, state_.edges_[0]->inputs_.size());
|
||||
|
||||
ASSERT_NO_FATAL_FAILURE(AssertParse(
|
||||
"ninja_dyndep_version = 1\n"
|
||||
"build out: dyndep\n"
|
||||
"build out2: dyndep\n"
|
||||
" restat = 1\n"));
|
||||
|
||||
EXPECT_EQ(2u, dyndep_file_.size());
|
||||
{
|
||||
DyndepFile::iterator i = dyndep_file_.find(state_.edges_[0]);
|
||||
ASSERT_NE(i, dyndep_file_.end());
|
||||
EXPECT_EQ(false, i->second.restat_);
|
||||
EXPECT_EQ(0u, i->second.implicit_outputs_.size());
|
||||
EXPECT_EQ(0u, i->second.implicit_inputs_.size());
|
||||
}
|
||||
{
|
||||
DyndepFile::iterator i = dyndep_file_.find(state_.edges_[1]);
|
||||
ASSERT_NE(i, dyndep_file_.end());
|
||||
EXPECT_EQ(true, i->second.restat_);
|
||||
EXPECT_EQ(0u, i->second.implicit_outputs_.size());
|
||||
EXPECT_EQ(0u, i->second.implicit_inputs_.size());
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,69 @@
|
||||
// Copyright 2011 Google Inc. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#include "edit_distance.h"
|
||||
|
||||
#include <algorithm>
|
||||
#include <vector>
|
||||
|
||||
int EditDistance(const StringPiece& s1,
|
||||
const StringPiece& s2,
|
||||
bool allow_replacements,
|
||||
int max_edit_distance) {
|
||||
// The algorithm implemented below is the "classic"
|
||||
// dynamic-programming algorithm for computing the Levenshtein
|
||||
// distance, which is described here:
|
||||
//
|
||||
// http://en.wikipedia.org/wiki/Levenshtein_distance
|
||||
//
|
||||
// Although the algorithm is typically described using an m x n
|
||||
// array, only one row plus one element are used at a time, so this
|
||||
// implementation just keeps one vector for the row. To update one entry,
|
||||
// only the entries to the left, top, and top-left are needed. The left
|
||||
// entry is in row[x-1], the top entry is what's in row[x] from the last
|
||||
// iteration, and the top-left entry is stored in previous.
|
||||
int m = static_cast<int>(s1.len_);
|
||||
int n = static_cast<int>(s2.len_);
|
||||
|
||||
std::vector<int> row(n + 1);
|
||||
for (int i = 1; i <= n; ++i)
|
||||
row[i] = i;
|
||||
|
||||
for (int y = 1; y <= m; ++y) {
|
||||
row[0] = y;
|
||||
int best_this_row = row[0];
|
||||
|
||||
int previous = y - 1;
|
||||
for (int x = 1; x <= n; ++x) {
|
||||
int old_row = row[x];
|
||||
if (allow_replacements) {
|
||||
row[x] = std::min(previous + (s1.str_[y - 1] == s2.str_[x - 1] ? 0 : 1),
|
||||
std::min(row[x - 1], row[x]) + 1);
|
||||
}
|
||||
else {
|
||||
if (s1.str_[y - 1] == s2.str_[x - 1])
|
||||
row[x] = previous;
|
||||
else
|
||||
row[x] = std::min(row[x - 1], row[x]) + 1;
|
||||
}
|
||||
previous = old_row;
|
||||
best_this_row = std::min(best_this_row, row[x]);
|
||||
}
|
||||
|
||||
if (max_edit_distance && best_this_row > max_edit_distance)
|
||||
return max_edit_distance + 1;
|
||||
}
|
||||
|
||||
return row[n];
|
||||
}
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user