diff options
Diffstat (limited to 'third_party/aom/tools')
| -rw-r--r-- | third_party/aom/tools/aggregate_entropy_stats.py | 39 | ||||
| -rw-r--r-- | third_party/aom/tools/aom_entropy_optimizer.c | 758 | ||||
| -rwxr-xr-x | third_party/aom/tools/cpplint.py | 4756 | ||||
| -rw-r--r-- | third_party/aom/tools/diff.py | 132 | ||||
| -rw-r--r-- | third_party/aom/tools/dump_obu.cc | 164 | ||||
| -rwxr-xr-x | third_party/aom/tools/gen_authors.sh | 10 | ||||
| -rwxr-xr-x | third_party/aom/tools/gen_constrained_tokenset.py | 120 | ||||
| -rw-r--r-- | third_party/aom/tools/inspect-cli.js | 39 | ||||
| -rw-r--r-- | third_party/aom/tools/inspect-post.js | 1 | ||||
| -rwxr-xr-x | third_party/aom/tools/intersect-diffs.py | 78 | ||||
| -rwxr-xr-x | third_party/aom/tools/lint-hunks.py | 146 | ||||
| -rw-r--r-- | third_party/aom/tools/obu_parser.cc | 190 | ||||
| -rw-r--r-- | third_party/aom/tools/obu_parser.h | 27 | ||||
| -rw-r--r-- | third_party/aom/tools/txfm_analyzer/txfm_gen_code.cc | 580 | ||||
| -rw-r--r-- | third_party/aom/tools/txfm_analyzer/txfm_graph.cc | 943 | ||||
| -rw-r--r-- | third_party/aom/tools/txfm_analyzer/txfm_graph.h | 161 | ||||
| -rwxr-xr-x | third_party/aom/tools/wrap-commit-msg.py | 72 |
17 files changed, 0 insertions, 8216 deletions
diff --git a/third_party/aom/tools/aggregate_entropy_stats.py b/third_party/aom/tools/aggregate_entropy_stats.py deleted file mode 100644 index 7cb4d18e1..000000000 --- a/third_party/aom/tools/aggregate_entropy_stats.py +++ /dev/null @@ -1,39 +0,0 @@ -#!/usr/bin/env python -## Copyright (c) 2017, Alliance for Open Media. All rights reserved -## -## This source code is subject to the terms of the BSD 2 Clause License and -## the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License -## was not distributed with this source code in the LICENSE file, you can -## obtain it at www.aomedia.org/license/software. If the Alliance for Open -## Media Patent License 1.0 was not distributed with this source code in the -## PATENTS file, you can obtain it at www.aomedia.org/license/patent. -## -"""Aggregate multiple entropy stats output which is written in 32-bit int. - -python ./aggregate_entropy_stats.py [dir of stats files] [keyword of filenames] - [filename of final stats] -""" - -__author__ = "yuec@google.com" - -import os -import sys -import numpy as np - -def main(): - dir = sys.argv[1] - sum = [] - for fn in os.listdir(dir): - if sys.argv[2] in fn: - stats = np.fromfile(dir + fn, dtype=np.int32) - if len(sum) == 0: - sum = stats - else: - sum = np.add(sum, stats) - if len(sum) == 0: - print("No stats file is found. Double-check directory and keyword?") - else: - sum.tofile(dir+sys.argv[3]) - -if __name__ == '__main__': - main() diff --git a/third_party/aom/tools/aom_entropy_optimizer.c b/third_party/aom/tools/aom_entropy_optimizer.c deleted file mode 100644 index 551adf4f2..000000000 --- a/third_party/aom/tools/aom_entropy_optimizer.c +++ /dev/null @@ -1,758 +0,0 @@ -/* - * Copyright (c) 2017, Alliance for Open Media. All rights reserved - * - * This source code is subject to the terms of the BSD 2 Clause License and - * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License - * was not distributed with this source code in the LICENSE file, you can - * obtain it at www.aomedia.org/license/software. If the Alliance for Open - * Media Patent License 1.0 was not distributed with this source code in the - * PATENTS file, you can obtain it at www.aomedia.org/license/patent. - */ - -// This tool is a gadget for offline probability training. -// A binary executable aom_entropy_optimizer will be generated in tools/. It -// parses a binary file consisting of counts written in the format of -// FRAME_COUNTS in entropymode.h, and computes optimized probability tables -// and CDF tables, which will be written to a new c file optimized_probs.c -// according to format in the codebase. -// -// Command line: ./aom_entropy_optimizer [directory of the count file] -// -// The input file can either be generated by encoding a single clip by -// turning on entropy_stats experiment, or be collected at a larger scale at -// which a python script which will be provided soon can be used to aggregate -// multiple stats output. - -#include <assert.h> -#include <stdio.h> - -#include "config/aom_config.h" - -#include "av1/encoder/encoder.h" - -#define SPACES_PER_TAB 2 -#define CDF_MAX_SIZE 16 - -typedef unsigned int aom_count_type; -// A log file recording parsed counts -static FILE *logfile; // TODO(yuec): make it a command line option - -static void counts_to_cdf(const aom_count_type *counts, aom_cdf_prob *cdf, - int modes) { - int64_t csum[CDF_MAX_SIZE]; - assert(modes <= CDF_MAX_SIZE); - - csum[0] = counts[0] + 1; - for (int i = 1; i < modes; ++i) csum[i] = counts[i] + 1 + csum[i - 1]; - - for (int i = 0; i < modes; ++i) fprintf(logfile, "%d ", counts[i]); - fprintf(logfile, "\n"); - - int64_t sum = csum[modes - 1]; - const int64_t round_shift = sum >> 1; - for (int i = 0; i < modes; ++i) { - cdf[i] = (csum[i] * CDF_PROB_TOP + round_shift) / sum; - cdf[i] = AOMMIN(cdf[i], CDF_PROB_TOP - (modes - 1 + i) * 4); - cdf[i] = (i == 0) ? AOMMAX(cdf[i], 4) : AOMMAX(cdf[i], cdf[i - 1] + 4); - } -} - -static int parse_counts_for_cdf_opt(aom_count_type **ct_ptr, - FILE *const probsfile, int tabs, - int dim_of_cts, int *cts_each_dim) { - if (dim_of_cts < 1) { - fprintf(stderr, "The dimension of a counts vector should be at least 1!\n"); - return 1; - } - const int total_modes = cts_each_dim[0]; - if (dim_of_cts == 1) { - assert(total_modes <= CDF_MAX_SIZE); - aom_cdf_prob cdfs[CDF_MAX_SIZE]; - aom_count_type *counts1d = *ct_ptr; - - counts_to_cdf(counts1d, cdfs, total_modes); - (*ct_ptr) += total_modes; - - if (tabs > 0) fprintf(probsfile, "%*c", tabs * SPACES_PER_TAB, ' '); - fprintf(probsfile, "AOM_CDF%d(", total_modes); - for (int k = 0; k < total_modes - 1; ++k) { - fprintf(probsfile, "%d", cdfs[k]); - if (k < total_modes - 2) fprintf(probsfile, ", "); - } - fprintf(probsfile, ")"); - } else { - for (int k = 0; k < total_modes; ++k) { - int tabs_next_level; - - if (dim_of_cts == 2) - fprintf(probsfile, "%*c{ ", tabs * SPACES_PER_TAB, ' '); - else - fprintf(probsfile, "%*c{\n", tabs * SPACES_PER_TAB, ' '); - tabs_next_level = dim_of_cts == 2 ? 0 : tabs + 1; - - if (parse_counts_for_cdf_opt(ct_ptr, probsfile, tabs_next_level, - dim_of_cts - 1, cts_each_dim + 1)) { - return 1; - } - - if (dim_of_cts == 2) { - if (k == total_modes - 1) - fprintf(probsfile, " }\n"); - else - fprintf(probsfile, " },\n"); - } else { - if (k == total_modes - 1) - fprintf(probsfile, "%*c}\n", tabs * SPACES_PER_TAB, ' '); - else - fprintf(probsfile, "%*c},\n", tabs * SPACES_PER_TAB, ' '); - } - } - } - return 0; -} - -static void optimize_cdf_table(aom_count_type *counts, FILE *const probsfile, - int dim_of_cts, int *cts_each_dim, - char *prefix) { - aom_count_type *ct_ptr = counts; - - fprintf(probsfile, "%s = {\n", prefix); - fprintf(logfile, "%s\n", prefix); - if (parse_counts_for_cdf_opt(&ct_ptr, probsfile, 1, dim_of_cts, - cts_each_dim)) { - fprintf(probsfile, "Optimizer failed!\n"); - } - fprintf(probsfile, "};\n\n"); - fprintf(logfile, "============================\n"); -} - -static void optimize_uv_mode(aom_count_type *counts, FILE *const probsfile, - int dim_of_cts, int *cts_each_dim, char *prefix) { - aom_count_type *ct_ptr = counts; - - fprintf(probsfile, "%s = {\n", prefix); - fprintf(probsfile, "%*c{\n", SPACES_PER_TAB, ' '); - fprintf(logfile, "%s\n", prefix); - cts_each_dim[2] = UV_INTRA_MODES - 1; - for (int k = 0; k < cts_each_dim[1]; ++k) { - fprintf(probsfile, "%*c{ ", 2 * SPACES_PER_TAB, ' '); - parse_counts_for_cdf_opt(&ct_ptr, probsfile, 0, dim_of_cts - 2, - cts_each_dim + 2); - if (k + 1 == cts_each_dim[1]) { - fprintf(probsfile, " }\n"); - } else { - fprintf(probsfile, " },\n"); - } - ++ct_ptr; - } - fprintf(probsfile, "%*c},\n", SPACES_PER_TAB, ' '); - fprintf(probsfile, "%*c{\n", SPACES_PER_TAB, ' '); - cts_each_dim[2] = UV_INTRA_MODES; - parse_counts_for_cdf_opt(&ct_ptr, probsfile, 2, dim_of_cts - 1, - cts_each_dim + 1); - fprintf(probsfile, "%*c}\n", SPACES_PER_TAB, ' '); - fprintf(probsfile, "};\n\n"); - fprintf(logfile, "============================\n"); -} - -static void optimize_cdf_table_var_modes_2d(aom_count_type *counts, - FILE *const probsfile, - int dim_of_cts, int *cts_each_dim, - int *modes_each_ctx, char *prefix) { - aom_count_type *ct_ptr = counts; - - assert(dim_of_cts == 2); - (void)dim_of_cts; - - fprintf(probsfile, "%s = {\n", prefix); - fprintf(logfile, "%s\n", prefix); - - for (int d0_idx = 0; d0_idx < cts_each_dim[0]; ++d0_idx) { - int num_of_modes = modes_each_ctx[d0_idx]; - - if (num_of_modes > 0) { - fprintf(probsfile, "%*c{ ", SPACES_PER_TAB, ' '); - parse_counts_for_cdf_opt(&ct_ptr, probsfile, 0, 1, &num_of_modes); - ct_ptr += cts_each_dim[1] - num_of_modes; - fprintf(probsfile, " },\n"); - } else { - fprintf(probsfile, "%*c{ 0 },\n", SPACES_PER_TAB, ' '); - fprintf(logfile, "dummy cdf, no need to optimize\n"); - ct_ptr += cts_each_dim[1]; - } - } - fprintf(probsfile, "};\n\n"); - fprintf(logfile, "============================\n"); -} - -static void optimize_cdf_table_var_modes_3d(aom_count_type *counts, - FILE *const probsfile, - int dim_of_cts, int *cts_each_dim, - int *modes_each_ctx, char *prefix) { - aom_count_type *ct_ptr = counts; - - assert(dim_of_cts == 3); - (void)dim_of_cts; - - fprintf(probsfile, "%s = {\n", prefix); - fprintf(logfile, "%s\n", prefix); - - for (int d0_idx = 0; d0_idx < cts_each_dim[0]; ++d0_idx) { - fprintf(probsfile, "%*c{\n", SPACES_PER_TAB, ' '); - for (int d1_idx = 0; d1_idx < cts_each_dim[1]; ++d1_idx) { - int num_of_modes = modes_each_ctx[d0_idx]; - - if (num_of_modes > 0) { - fprintf(probsfile, "%*c{ ", 2 * SPACES_PER_TAB, ' '); - parse_counts_for_cdf_opt(&ct_ptr, probsfile, 0, 1, &num_of_modes); - ct_ptr += cts_each_dim[2] - num_of_modes; - fprintf(probsfile, " },\n"); - } else { - fprintf(probsfile, "%*c{ 0 },\n", 2 * SPACES_PER_TAB, ' '); - fprintf(logfile, "dummy cdf, no need to optimize\n"); - ct_ptr += cts_each_dim[2]; - } - } - fprintf(probsfile, "%*c},\n", SPACES_PER_TAB, ' '); - } - fprintf(probsfile, "};\n\n"); - fprintf(logfile, "============================\n"); -} - -static void optimize_cdf_table_var_modes_4d(aom_count_type *counts, - FILE *const probsfile, - int dim_of_cts, int *cts_each_dim, - int *modes_each_ctx, char *prefix) { - aom_count_type *ct_ptr = counts; - - assert(dim_of_cts == 4); - (void)dim_of_cts; - - fprintf(probsfile, "%s = {\n", prefix); - fprintf(logfile, "%s\n", prefix); - - for (int d0_idx = 0; d0_idx < cts_each_dim[0]; ++d0_idx) { - fprintf(probsfile, "%*c{\n", SPACES_PER_TAB, ' '); - for (int d1_idx = 0; d1_idx < cts_each_dim[1]; ++d1_idx) { - fprintf(probsfile, "%*c{\n", 2 * SPACES_PER_TAB, ' '); - for (int d2_idx = 0; d2_idx < cts_each_dim[2]; ++d2_idx) { - int num_of_modes = modes_each_ctx[d0_idx]; - - if (num_of_modes > 0) { - fprintf(probsfile, "%*c{ ", 3 * SPACES_PER_TAB, ' '); - parse_counts_for_cdf_opt(&ct_ptr, probsfile, 0, 1, &num_of_modes); - ct_ptr += cts_each_dim[3] - num_of_modes; - fprintf(probsfile, " },\n"); - } else { - fprintf(probsfile, "%*c{ 0 },\n", 3 * SPACES_PER_TAB, ' '); - fprintf(logfile, "dummy cdf, no need to optimize\n"); - ct_ptr += cts_each_dim[3]; - } - } - fprintf(probsfile, "%*c},\n", 2 * SPACES_PER_TAB, ' '); - } - fprintf(probsfile, "%*c},\n", SPACES_PER_TAB, ' '); - } - fprintf(probsfile, "};\n\n"); - fprintf(logfile, "============================\n"); -} - -int main(int argc, const char **argv) { - if (argc < 2) { - fprintf(stderr, "Please specify the input stats file!\n"); - exit(EXIT_FAILURE); - } - - FILE *const statsfile = fopen(argv[1], "rb"); - if (statsfile == NULL) { - fprintf(stderr, "Failed to open input file!\n"); - exit(EXIT_FAILURE); - } - - FRAME_COUNTS fc; - const size_t bytes = fread(&fc, sizeof(FRAME_COUNTS), 1, statsfile); - if (!bytes) return 1; - - FILE *const probsfile = fopen("optimized_probs.c", "w"); - if (probsfile == NULL) { - fprintf(stderr, - "Failed to create output file for optimized entropy tables!\n"); - exit(EXIT_FAILURE); - } - - logfile = fopen("aom_entropy_optimizer_parsed_counts.log", "w"); - if (logfile == NULL) { - fprintf(stderr, "Failed to create log file for parsed counts!\n"); - exit(EXIT_FAILURE); - } - - int cts_each_dim[10]; - - /* Intra mode (keyframe luma) */ - cts_each_dim[0] = KF_MODE_CONTEXTS; - cts_each_dim[1] = KF_MODE_CONTEXTS; - cts_each_dim[2] = INTRA_MODES; - optimize_cdf_table(&fc.kf_y_mode[0][0][0], probsfile, 3, cts_each_dim, - "const aom_cdf_prob\n" - "default_kf_y_mode_cdf[KF_MODE_CONTEXTS][KF_MODE_CONTEXTS]" - "[CDF_SIZE(INTRA_MODES)]"); - - cts_each_dim[0] = DIRECTIONAL_MODES; - cts_each_dim[1] = 2 * MAX_ANGLE_DELTA + 1; - optimize_cdf_table(&fc.angle_delta[0][0], probsfile, 2, cts_each_dim, - "static const aom_cdf_prob default_angle_delta_cdf" - "[DIRECTIONAL_MODES][CDF_SIZE(2 * MAX_ANGLE_DELTA + 1)]"); - - /* Intra mode (non-keyframe luma) */ - cts_each_dim[0] = BLOCK_SIZE_GROUPS; - cts_each_dim[1] = INTRA_MODES; - optimize_cdf_table( - &fc.y_mode[0][0], probsfile, 2, cts_each_dim, - "static const aom_cdf_prob\n" - "default_if_y_mode_cdf[BLOCK_SIZE_GROUPS][CDF_SIZE(INTRA_MODES)]"); - - /* Intra mode (chroma) */ - cts_each_dim[0] = CFL_ALLOWED_TYPES; - cts_each_dim[1] = INTRA_MODES; - cts_each_dim[2] = UV_INTRA_MODES; - optimize_uv_mode(&fc.uv_mode[0][0][0], probsfile, 3, cts_each_dim, - "static const aom_cdf_prob\n" - "default_uv_mode_cdf[CFL_ALLOWED_TYPES][INTRA_MODES]" - "[CDF_SIZE(UV_INTRA_MODES)]"); - - /* block partition */ - cts_each_dim[0] = PARTITION_CONTEXTS; - cts_each_dim[1] = EXT_PARTITION_TYPES; - int part_types_each_ctx[PARTITION_CONTEXTS] = { - 4, 4, 4, 4, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 8, 8, 8, 8 - }; - optimize_cdf_table_var_modes_2d( - &fc.partition[0][0], probsfile, 2, cts_each_dim, part_types_each_ctx, - "static const aom_cdf_prob default_partition_cdf[PARTITION_CONTEXTS]" - "[CDF_SIZE(EXT_PARTITION_TYPES)]"); - - /* tx type */ - cts_each_dim[0] = EXT_TX_SETS_INTRA; - cts_each_dim[1] = EXT_TX_SIZES; - cts_each_dim[2] = INTRA_MODES; - cts_each_dim[3] = TX_TYPES; - int intra_ext_tx_types_each_ctx[EXT_TX_SETS_INTRA] = { 0, 7, 5 }; - optimize_cdf_table_var_modes_4d( - &fc.intra_ext_tx[0][0][0][0], probsfile, 4, cts_each_dim, - intra_ext_tx_types_each_ctx, - "static const aom_cdf_prob default_intra_ext_tx_cdf[EXT_TX_SETS_INTRA]" - "[EXT_TX_SIZES][INTRA_MODES][CDF_SIZE(TX_TYPES)]"); - - cts_each_dim[0] = EXT_TX_SETS_INTER; - cts_each_dim[1] = EXT_TX_SIZES; - cts_each_dim[2] = TX_TYPES; - int inter_ext_tx_types_each_ctx[EXT_TX_SETS_INTER] = { 0, 16, 12, 2 }; - optimize_cdf_table_var_modes_3d( - &fc.inter_ext_tx[0][0][0], probsfile, 3, cts_each_dim, - inter_ext_tx_types_each_ctx, - "static const aom_cdf_prob default_inter_ext_tx_cdf[EXT_TX_SETS_INTER]" - "[EXT_TX_SIZES][CDF_SIZE(TX_TYPES)]"); - - /* Chroma from Luma */ - cts_each_dim[0] = CFL_JOINT_SIGNS; - optimize_cdf_table(&fc.cfl_sign[0], probsfile, 1, cts_each_dim, - "static const aom_cdf_prob\n" - "default_cfl_sign_cdf[CDF_SIZE(CFL_JOINT_SIGNS)]"); - cts_each_dim[0] = CFL_ALPHA_CONTEXTS; - cts_each_dim[1] = CFL_ALPHABET_SIZE; - optimize_cdf_table(&fc.cfl_alpha[0][0], probsfile, 2, cts_each_dim, - "static const aom_cdf_prob\n" - "default_cfl_alpha_cdf[CFL_ALPHA_CONTEXTS]" - "[CDF_SIZE(CFL_ALPHABET_SIZE)]"); - - /* Interpolation filter */ - cts_each_dim[0] = SWITCHABLE_FILTER_CONTEXTS; - cts_each_dim[1] = SWITCHABLE_FILTERS; - optimize_cdf_table(&fc.switchable_interp[0][0], probsfile, 2, cts_each_dim, - "static const aom_cdf_prob\n" - "default_switchable_interp_cdf[SWITCHABLE_FILTER_CONTEXTS]" - "[CDF_SIZE(SWITCHABLE_FILTERS)]"); - - /* Motion vector referencing */ - cts_each_dim[0] = NEWMV_MODE_CONTEXTS; - cts_each_dim[1] = 2; - optimize_cdf_table(&fc.newmv_mode[0][0], probsfile, 2, cts_each_dim, - "static const aom_cdf_prob " - "default_newmv_cdf[NEWMV_MODE_CONTEXTS][CDF_SIZE(2)]"); - - cts_each_dim[0] = GLOBALMV_MODE_CONTEXTS; - cts_each_dim[1] = 2; - optimize_cdf_table(&fc.zeromv_mode[0][0], probsfile, 2, cts_each_dim, - "static const aom_cdf_prob " - "default_zeromv_cdf[GLOBALMV_MODE_CONTEXTS][CDF_SIZE(2)]"); - - cts_each_dim[0] = REFMV_MODE_CONTEXTS; - cts_each_dim[1] = 2; - optimize_cdf_table(&fc.refmv_mode[0][0], probsfile, 2, cts_each_dim, - "static const aom_cdf_prob " - "default_refmv_cdf[REFMV_MODE_CONTEXTS][CDF_SIZE(2)]"); - - cts_each_dim[0] = DRL_MODE_CONTEXTS; - cts_each_dim[1] = 2; - optimize_cdf_table(&fc.drl_mode[0][0], probsfile, 2, cts_each_dim, - "static const aom_cdf_prob " - "default_drl_cdf[DRL_MODE_CONTEXTS][CDF_SIZE(2)]"); - - /* ext_inter experiment */ - /* New compound mode */ - cts_each_dim[0] = INTER_MODE_CONTEXTS; - cts_each_dim[1] = INTER_COMPOUND_MODES; - optimize_cdf_table(&fc.inter_compound_mode[0][0], probsfile, 2, cts_each_dim, - "static const aom_cdf_prob\n" - "default_inter_compound_mode_cdf[INTER_MODE_CONTEXTS][CDF_" - "SIZE(INTER_COMPOUND_MODES)]"); - - /* Interintra */ - cts_each_dim[0] = BLOCK_SIZE_GROUPS; - cts_each_dim[1] = 2; - optimize_cdf_table(&fc.interintra[0][0], probsfile, 2, cts_each_dim, - "static const aom_cdf_prob " - "default_interintra_cdf[BLOCK_SIZE_GROUPS][CDF_SIZE(2)]"); - - cts_each_dim[0] = BLOCK_SIZE_GROUPS; - cts_each_dim[1] = INTERINTRA_MODES; - optimize_cdf_table(&fc.interintra_mode[0][0], probsfile, 2, cts_each_dim, - "static const aom_cdf_prob\n" - "default_interintra_mode_cdf[BLOCK_SIZE_GROUPS][CDF_SIZE(" - "INTERINTRA_MODES)]"); - - cts_each_dim[0] = BLOCK_SIZES_ALL; - cts_each_dim[1] = 2; - optimize_cdf_table( - &fc.wedge_interintra[0][0], probsfile, 2, cts_each_dim, - "static const aom_cdf_prob\n" - "default_wedge_interintra_cdf[BLOCK_SIZES_ALL][CDF_SIZE(2)]"); - - /* Compound type */ - cts_each_dim[0] = BLOCK_SIZES_ALL; - cts_each_dim[1] = COMPOUND_TYPES - 1; - optimize_cdf_table(&fc.compound_type[0][0], probsfile, 2, cts_each_dim, - "static const aom_cdf_prob default_compound_type_cdf" - "[BLOCK_SIZES_ALL][CDF_SIZE(COMPOUND_TYPES - 1)]"); - - cts_each_dim[0] = BLOCK_SIZES_ALL; - cts_each_dim[1] = 16; - optimize_cdf_table(&fc.wedge_idx[0][0], probsfile, 2, cts_each_dim, - "static const aom_cdf_prob " - "default_wedge_idx_cdf[BLOCK_SIZES_ALL][CDF_SIZE(16)]"); - - /* motion_var and warped_motion experiments */ - cts_each_dim[0] = BLOCK_SIZES_ALL; - cts_each_dim[1] = MOTION_MODES; - optimize_cdf_table( - &fc.motion_mode[0][0], probsfile, 2, cts_each_dim, - "static const aom_cdf_prob\n" - "default_motion_mode_cdf[BLOCK_SIZES_ALL][CDF_SIZE(MOTION_MODES)]"); - cts_each_dim[0] = BLOCK_SIZES_ALL; - cts_each_dim[1] = 2; - optimize_cdf_table(&fc.obmc[0][0], probsfile, 2, cts_each_dim, - "static const aom_cdf_prob " - "default_obmc_cdf[BLOCK_SIZES_ALL][CDF_SIZE(2)]"); - - /* Intra/inter flag */ - cts_each_dim[0] = INTRA_INTER_CONTEXTS; - cts_each_dim[1] = 2; - optimize_cdf_table( - &fc.intra_inter[0][0], probsfile, 2, cts_each_dim, - "static const aom_cdf_prob\n" - "default_intra_inter_cdf[INTRA_INTER_CONTEXTS][CDF_SIZE(2)]"); - - /* Single/comp ref flag */ - cts_each_dim[0] = COMP_INTER_CONTEXTS; - cts_each_dim[1] = 2; - optimize_cdf_table( - &fc.comp_inter[0][0], probsfile, 2, cts_each_dim, - "static const aom_cdf_prob\n" - "default_comp_inter_cdf[COMP_INTER_CONTEXTS][CDF_SIZE(2)]"); - - /* ext_comp_refs experiment */ - cts_each_dim[0] = COMP_REF_TYPE_CONTEXTS; - cts_each_dim[1] = 2; - optimize_cdf_table( - &fc.comp_ref_type[0][0], probsfile, 2, cts_each_dim, - "static const aom_cdf_prob\n" - "default_comp_ref_type_cdf[COMP_REF_TYPE_CONTEXTS][CDF_SIZE(2)]"); - - cts_each_dim[0] = UNI_COMP_REF_CONTEXTS; - cts_each_dim[1] = UNIDIR_COMP_REFS - 1; - cts_each_dim[2] = 2; - optimize_cdf_table(&fc.uni_comp_ref[0][0][0], probsfile, 3, cts_each_dim, - "static const aom_cdf_prob\n" - "default_uni_comp_ref_cdf[UNI_COMP_REF_CONTEXTS][UNIDIR_" - "COMP_REFS - 1][CDF_SIZE(2)]"); - - /* Reference frame (single ref) */ - cts_each_dim[0] = REF_CONTEXTS; - cts_each_dim[1] = SINGLE_REFS - 1; - cts_each_dim[2] = 2; - optimize_cdf_table( - &fc.single_ref[0][0][0], probsfile, 3, cts_each_dim, - "static const aom_cdf_prob\n" - "default_single_ref_cdf[REF_CONTEXTS][SINGLE_REFS - 1][CDF_SIZE(2)]"); - - /* ext_refs experiment */ - cts_each_dim[0] = REF_CONTEXTS; - cts_each_dim[1] = FWD_REFS - 1; - cts_each_dim[2] = 2; - optimize_cdf_table( - &fc.comp_ref[0][0][0], probsfile, 3, cts_each_dim, - "static const aom_cdf_prob\n" - "default_comp_ref_cdf[REF_CONTEXTS][FWD_REFS - 1][CDF_SIZE(2)]"); - - cts_each_dim[0] = REF_CONTEXTS; - cts_each_dim[1] = BWD_REFS - 1; - cts_each_dim[2] = 2; - optimize_cdf_table( - &fc.comp_bwdref[0][0][0], probsfile, 3, cts_each_dim, - "static const aom_cdf_prob\n" - "default_comp_bwdref_cdf[REF_CONTEXTS][BWD_REFS - 1][CDF_SIZE(2)]"); - - /* palette */ - cts_each_dim[0] = PALATTE_BSIZE_CTXS; - cts_each_dim[1] = PALETTE_SIZES; - optimize_cdf_table(&fc.palette_y_size[0][0], probsfile, 2, cts_each_dim, - "const aom_cdf_prob default_palette_y_size_cdf" - "[PALATTE_BSIZE_CTXS][CDF_SIZE(PALETTE_SIZES)]"); - - cts_each_dim[0] = PALATTE_BSIZE_CTXS; - cts_each_dim[1] = PALETTE_SIZES; - optimize_cdf_table(&fc.palette_uv_size[0][0], probsfile, 2, cts_each_dim, - "const aom_cdf_prob default_palette_uv_size_cdf" - "[PALATTE_BSIZE_CTXS][CDF_SIZE(PALETTE_SIZES)]"); - - cts_each_dim[0] = PALATTE_BSIZE_CTXS; - cts_each_dim[1] = PALETTE_Y_MODE_CONTEXTS; - cts_each_dim[2] = 2; - optimize_cdf_table(&fc.palette_y_mode[0][0][0], probsfile, 3, cts_each_dim, - "const aom_cdf_prob default_palette_y_mode_cdf" - "[PALATTE_BSIZE_CTXS][PALETTE_Y_MODE_CONTEXTS]" - "[CDF_SIZE(2)]"); - - cts_each_dim[0] = PALETTE_UV_MODE_CONTEXTS; - cts_each_dim[1] = 2; - optimize_cdf_table(&fc.palette_uv_mode[0][0], probsfile, 2, cts_each_dim, - "const aom_cdf_prob default_palette_uv_mode_cdf" - "[PALETTE_UV_MODE_CONTEXTS][CDF_SIZE(2)]"); - - cts_each_dim[0] = PALETTE_SIZES; - cts_each_dim[1] = PALETTE_COLOR_INDEX_CONTEXTS; - cts_each_dim[2] = PALETTE_COLORS; - int palette_color_indexes_each_ctx[PALETTE_SIZES] = { 2, 3, 4, 5, 6, 7, 8 }; - optimize_cdf_table_var_modes_3d( - &fc.palette_y_color_index[0][0][0], probsfile, 3, cts_each_dim, - palette_color_indexes_each_ctx, - "const aom_cdf_prob default_palette_y_color_index_cdf[PALETTE_SIZES]" - "[PALETTE_COLOR_INDEX_CONTEXTS][CDF_SIZE(PALETTE_COLORS)]"); - - cts_each_dim[0] = PALETTE_SIZES; - cts_each_dim[1] = PALETTE_COLOR_INDEX_CONTEXTS; - cts_each_dim[2] = PALETTE_COLORS; - optimize_cdf_table_var_modes_3d( - &fc.palette_uv_color_index[0][0][0], probsfile, 3, cts_each_dim, - palette_color_indexes_each_ctx, - "const aom_cdf_prob default_palette_uv_color_index_cdf[PALETTE_SIZES]" - "[PALETTE_COLOR_INDEX_CONTEXTS][CDF_SIZE(PALETTE_COLORS)]"); - - /* Transform size */ - cts_each_dim[0] = TXFM_PARTITION_CONTEXTS; - cts_each_dim[1] = 2; - optimize_cdf_table( - &fc.txfm_partition[0][0], probsfile, 2, cts_each_dim, - "static const aom_cdf_prob\n" - "default_txfm_partition_cdf[TXFM_PARTITION_CONTEXTS][CDF_SIZE(2)]"); - - /* Skip flag */ - cts_each_dim[0] = SKIP_CONTEXTS; - cts_each_dim[1] = 2; - optimize_cdf_table(&fc.skip[0][0], probsfile, 2, cts_each_dim, - "static const aom_cdf_prob " - "default_skip_cdfs[SKIP_CONTEXTS][CDF_SIZE(2)]"); - - /* Skip mode flag */ - cts_each_dim[0] = SKIP_MODE_CONTEXTS; - cts_each_dim[1] = 2; - optimize_cdf_table(&fc.skip_mode[0][0], probsfile, 2, cts_each_dim, - "static const aom_cdf_prob " - "default_skip_mode_cdfs[SKIP_MODE_CONTEXTS][CDF_SIZE(2)]"); - - /* joint compound flag */ - cts_each_dim[0] = COMP_INDEX_CONTEXTS; - cts_each_dim[1] = 2; - optimize_cdf_table(&fc.compound_index[0][0], probsfile, 2, cts_each_dim, - "static const aom_cdf_prob default_compound_idx_cdfs" - "[COMP_INDEX_CONTEXTS][CDF_SIZE(2)]"); - - cts_each_dim[0] = COMP_GROUP_IDX_CONTEXTS; - cts_each_dim[1] = 2; - optimize_cdf_table(&fc.comp_group_idx[0][0], probsfile, 2, cts_each_dim, - "static const aom_cdf_prob default_comp_group_idx_cdfs" - "[COMP_GROUP_IDX_CONTEXTS][CDF_SIZE(2)]"); - - /* intrabc */ - cts_each_dim[0] = 2; - optimize_cdf_table( - &fc.intrabc[0], probsfile, 1, cts_each_dim, - "static const aom_cdf_prob default_intrabc_cdf[CDF_SIZE(2)]"); - - /* filter_intra experiment */ - cts_each_dim[0] = FILTER_INTRA_MODES; - optimize_cdf_table( - &fc.filter_intra_mode[0], probsfile, 1, cts_each_dim, - "static const aom_cdf_prob " - "default_filter_intra_mode_cdf[CDF_SIZE(FILTER_INTRA_MODES)]"); - - cts_each_dim[0] = BLOCK_SIZES_ALL; - cts_each_dim[1] = 2; - optimize_cdf_table(&fc.filter_intra[0][0], probsfile, 2, cts_each_dim, - "static const aom_cdf_prob " - "default_filter_intra_cdfs[BLOCK_SIZES_ALL][CDF_SIZE(2)]"); - - /* restoration type */ - cts_each_dim[0] = RESTORE_SWITCHABLE_TYPES; - optimize_cdf_table(&fc.switchable_restore[0], probsfile, 1, cts_each_dim, - "static const aom_cdf_prob default_switchable_restore_cdf" - "[CDF_SIZE(RESTORE_SWITCHABLE_TYPES)]"); - - cts_each_dim[0] = 2; - optimize_cdf_table(&fc.wiener_restore[0], probsfile, 1, cts_each_dim, - "static const aom_cdf_prob default_wiener_restore_cdf" - "[CDF_SIZE(2)]"); - - cts_each_dim[0] = 2; - optimize_cdf_table(&fc.sgrproj_restore[0], probsfile, 1, cts_each_dim, - "static const aom_cdf_prob default_sgrproj_restore_cdf" - "[CDF_SIZE(2)]"); - - /* intra tx size */ - cts_each_dim[0] = MAX_TX_CATS; - cts_each_dim[1] = TX_SIZE_CONTEXTS; - cts_each_dim[2] = MAX_TX_DEPTH + 1; - int intra_tx_sizes_each_ctx[MAX_TX_CATS] = { 2, 3, 3, 3 }; - optimize_cdf_table_var_modes_3d( - &fc.intra_tx_size[0][0][0], probsfile, 3, cts_each_dim, - intra_tx_sizes_each_ctx, - "static const aom_cdf_prob default_tx_size_cdf" - "[MAX_TX_CATS][TX_SIZE_CONTEXTS][CDF_SIZE(MAX_TX_DEPTH + 1)]"); - - /* transform coding */ - cts_each_dim[0] = TOKEN_CDF_Q_CTXS; - cts_each_dim[1] = TX_SIZES; - cts_each_dim[2] = TXB_SKIP_CONTEXTS; - cts_each_dim[3] = 2; - optimize_cdf_table(&fc.txb_skip[0][0][0][0], probsfile, 4, cts_each_dim, - "static const aom_cdf_prob " - "av1_default_txb_skip_cdfs[TOKEN_CDF_Q_CTXS][TX_SIZES]" - "[TXB_SKIP_CONTEXTS][CDF_SIZE(2)]"); - - cts_each_dim[0] = TOKEN_CDF_Q_CTXS; - cts_each_dim[1] = TX_SIZES; - cts_each_dim[2] = PLANE_TYPES; - cts_each_dim[3] = EOB_COEF_CONTEXTS; - cts_each_dim[4] = 2; - optimize_cdf_table( - &fc.eob_extra[0][0][0][0][0], probsfile, 5, cts_each_dim, - "static const aom_cdf_prob av1_default_eob_extra_cdfs " - "[TOKEN_CDF_Q_CTXS][TX_SIZES][PLANE_TYPES][EOB_COEF_CONTEXTS]" - "[CDF_SIZE(2)]"); - - cts_each_dim[0] = TOKEN_CDF_Q_CTXS; - cts_each_dim[1] = PLANE_TYPES; - cts_each_dim[2] = 2; - cts_each_dim[3] = 5; - optimize_cdf_table(&fc.eob_multi16[0][0][0][0], probsfile, 4, cts_each_dim, - "static const aom_cdf_prob av1_default_eob_multi16_cdfs" - "[TOKEN_CDF_Q_CTXS][PLANE_TYPES][2][CDF_SIZE(5)]"); - - cts_each_dim[0] = TOKEN_CDF_Q_CTXS; - cts_each_dim[1] = PLANE_TYPES; - cts_each_dim[2] = 2; - cts_each_dim[3] = 6; - optimize_cdf_table(&fc.eob_multi32[0][0][0][0], probsfile, 4, cts_each_dim, - "static const aom_cdf_prob av1_default_eob_multi32_cdfs" - "[TOKEN_CDF_Q_CTXS][PLANE_TYPES][2][CDF_SIZE(6)]"); - - cts_each_dim[0] = TOKEN_CDF_Q_CTXS; - cts_each_dim[1] = PLANE_TYPES; - cts_each_dim[2] = 2; - cts_each_dim[3] = 7; - optimize_cdf_table(&fc.eob_multi64[0][0][0][0], probsfile, 4, cts_each_dim, - "static const aom_cdf_prob av1_default_eob_multi64_cdfs" - "[TOKEN_CDF_Q_CTXS][PLANE_TYPES][2][CDF_SIZE(7)]"); - - cts_each_dim[0] = TOKEN_CDF_Q_CTXS; - cts_each_dim[1] = PLANE_TYPES; - cts_each_dim[2] = 2; - cts_each_dim[3] = 8; - optimize_cdf_table(&fc.eob_multi128[0][0][0][0], probsfile, 4, cts_each_dim, - "static const aom_cdf_prob av1_default_eob_multi128_cdfs" - "[TOKEN_CDF_Q_CTXS][PLANE_TYPES][2][CDF_SIZE(8)]"); - - cts_each_dim[0] = TOKEN_CDF_Q_CTXS; - cts_each_dim[1] = PLANE_TYPES; - cts_each_dim[2] = 2; - cts_each_dim[3] = 9; - optimize_cdf_table(&fc.eob_multi256[0][0][0][0], probsfile, 4, cts_each_dim, - "static const aom_cdf_prob av1_default_eob_multi256_cdfs" - "[TOKEN_CDF_Q_CTXS][PLANE_TYPES][2][CDF_SIZE(9)]"); - - cts_each_dim[0] = TOKEN_CDF_Q_CTXS; - cts_each_dim[1] = PLANE_TYPES; - cts_each_dim[2] = 2; - cts_each_dim[3] = 10; - optimize_cdf_table(&fc.eob_multi512[0][0][0][0], probsfile, 4, cts_each_dim, - "static const aom_cdf_prob av1_default_eob_multi512_cdfs" - "[TOKEN_CDF_Q_CTXS][PLANE_TYPES][2][CDF_SIZE(10)]"); - - cts_each_dim[0] = TOKEN_CDF_Q_CTXS; - cts_each_dim[1] = PLANE_TYPES; - cts_each_dim[2] = 2; - cts_each_dim[3] = 11; - optimize_cdf_table(&fc.eob_multi1024[0][0][0][0], probsfile, 4, cts_each_dim, - "static const aom_cdf_prob av1_default_eob_multi1024_cdfs" - "[TOKEN_CDF_Q_CTXS][PLANE_TYPES][2][CDF_SIZE(11)]"); - - cts_each_dim[0] = TOKEN_CDF_Q_CTXS; - cts_each_dim[1] = TX_SIZES; - cts_each_dim[2] = PLANE_TYPES; - cts_each_dim[3] = LEVEL_CONTEXTS; - cts_each_dim[4] = BR_CDF_SIZE; - optimize_cdf_table(&fc.coeff_lps_multi[0][0][0][0][0], probsfile, 5, - cts_each_dim, - "static const aom_cdf_prob " - "av1_default_coeff_lps_multi_cdfs[TOKEN_CDF_Q_CTXS]" - "[TX_SIZES][PLANE_TYPES][LEVEL_CONTEXTS]" - "[CDF_SIZE(BR_CDF_SIZE)]"); - - cts_each_dim[0] = TOKEN_CDF_Q_CTXS; - cts_each_dim[1] = TX_SIZES; - cts_each_dim[2] = PLANE_TYPES; - cts_each_dim[3] = SIG_COEF_CONTEXTS; - cts_each_dim[4] = NUM_BASE_LEVELS + 2; - optimize_cdf_table( - &fc.coeff_base_multi[0][0][0][0][0], probsfile, 5, cts_each_dim, - "static const aom_cdf_prob av1_default_coeff_base_multi_cdfs" - "[TOKEN_CDF_Q_CTXS][TX_SIZES][PLANE_TYPES][SIG_COEF_CONTEXTS]" - "[CDF_SIZE(NUM_BASE_LEVELS + 2)]"); - - cts_each_dim[0] = TOKEN_CDF_Q_CTXS; - cts_each_dim[1] = TX_SIZES; - cts_each_dim[2] = PLANE_TYPES; - cts_each_dim[3] = SIG_COEF_CONTEXTS_EOB; - cts_each_dim[4] = NUM_BASE_LEVELS + 1; - optimize_cdf_table( - &fc.coeff_base_eob_multi[0][0][0][0][0], probsfile, 5, cts_each_dim, - "static const aom_cdf_prob av1_default_coeff_base_eob_multi_cdfs" - "[TOKEN_CDF_Q_CTXS][TX_SIZES][PLANE_TYPES][SIG_COEF_CONTEXTS_EOB]" - "[CDF_SIZE(NUM_BASE_LEVELS + 1)]"); - - fclose(statsfile); - fclose(logfile); - fclose(probsfile); - - return 0; -} diff --git a/third_party/aom/tools/cpplint.py b/third_party/aom/tools/cpplint.py deleted file mode 100755 index 25fbef73d..000000000 --- a/third_party/aom/tools/cpplint.py +++ /dev/null @@ -1,4756 +0,0 @@ -#!/usr/bin/python -# -# Copyright (c) 2009 Google Inc. All rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions are -# met: -# -# * Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# * Redistributions in binary form must reproduce the above -# copyright notice, this list of conditions and the following disclaimer -# in the documentation and/or other materials provided with the -# distribution. -# * Neither the name of Google Inc. nor the names of its -# contributors may be used to endorse or promote products derived from -# this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -"""Does google-lint on c++ files. - -The goal of this script is to identify places in the code that *may* -be in non-compliance with google style. It does not attempt to fix -up these problems -- the point is to educate. It does also not -attempt to find all problems, or to ensure that everything it does -find is legitimately a problem. - -In particular, we can get very confused by /* and // inside strings! -We do a small hack, which is to ignore //'s with "'s after them on the -same line, but it is far from perfect (in either direction). -""" - -import codecs -import copy -import getopt -import math # for log -import os -import re -import sre_compile -import string -import sys -import unicodedata - - -_USAGE = """ -Syntax: cpplint.py [--verbose=#] [--output=vs7] [--filter=-x,+y,...] - [--counting=total|toplevel|detailed] [--root=subdir] - [--linelength=digits] - <file> [file] ... - - The style guidelines this tries to follow are those in - http://google-styleguide.googlecode.com/svn/trunk/cppguide.xml - - Every problem is given a confidence score from 1-5, with 5 meaning we are - certain of the problem, and 1 meaning it could be a legitimate construct. - This will miss some errors, and is not a substitute for a code review. - - To suppress false-positive errors of a certain category, add a - 'NOLINT(category)' comment to the line. NOLINT or NOLINT(*) - suppresses errors of all categories on that line. - - The files passed in will be linted; at least one file must be provided. - Default linted extensions are .cc, .cpp, .cu, .cuh and .h. Change the - extensions with the --extensions flag. - - Flags: - - output=vs7 - By default, the output is formatted to ease emacs parsing. Visual Studio - compatible output (vs7) may also be used. Other formats are unsupported. - - verbose=# - Specify a number 0-5 to restrict errors to certain verbosity levels. - - filter=-x,+y,... - Specify a comma-separated list of category-filters to apply: only - error messages whose category names pass the filters will be printed. - (Category names are printed with the message and look like - "[whitespace/indent]".) Filters are evaluated left to right. - "-FOO" and "FOO" means "do not print categories that start with FOO". - "+FOO" means "do print categories that start with FOO". - - Examples: --filter=-whitespace,+whitespace/braces - --filter=whitespace,runtime/printf,+runtime/printf_format - --filter=-,+build/include_what_you_use - - To see a list of all the categories used in cpplint, pass no arg: - --filter= - - counting=total|toplevel|detailed - The total number of errors found is always printed. If - 'toplevel' is provided, then the count of errors in each of - the top-level categories like 'build' and 'whitespace' will - also be printed. If 'detailed' is provided, then a count - is provided for each category like 'build/class'. - - root=subdir - The root directory used for deriving header guard CPP variable. - By default, the header guard CPP variable is calculated as the relative - path to the directory that contains .git, .hg, or .svn. When this flag - is specified, the relative path is calculated from the specified - directory. If the specified directory does not exist, this flag is - ignored. - - Examples: - Assuing that src/.git exists, the header guard CPP variables for - src/chrome/browser/ui/browser.h are: - - No flag => CHROME_BROWSER_UI_BROWSER_H_ - --root=chrome => BROWSER_UI_BROWSER_H_ - --root=chrome/browser => UI_BROWSER_H_ - - linelength=digits - This is the allowed line length for the project. The default value is - 80 characters. - - Examples: - --linelength=120 - - extensions=extension,extension,... - The allowed file extensions that cpplint will check - - Examples: - --extensions=hpp,cpp -""" - -# We categorize each error message we print. Here are the categories. -# We want an explicit list so we can list them all in cpplint --filter=. -# If you add a new error message with a new category, add it to the list -# here! cpplint_unittest.py should tell you if you forget to do this. -_ERROR_CATEGORIES = [ - 'build/class', - 'build/deprecated', - 'build/endif_comment', - 'build/explicit_make_pair', - 'build/forward_decl', - 'build/header_guard', - 'build/include', - 'build/include_alpha', - 'build/include_order', - 'build/include_what_you_use', - 'build/namespaces', - 'build/printf_format', - 'build/storage_class', - 'legal/copyright', - 'readability/alt_tokens', - 'readability/braces', - 'readability/casting', - 'readability/check', - 'readability/constructors', - 'readability/fn_size', - 'readability/function', - 'readability/multiline_comment', - 'readability/multiline_string', - 'readability/namespace', - 'readability/nolint', - 'readability/nul', - 'readability/streams', - 'readability/todo', - 'readability/utf8', - 'runtime/arrays', - 'runtime/casting', - 'runtime/explicit', - 'runtime/int', - 'runtime/init', - 'runtime/invalid_increment', - 'runtime/member_string_references', - 'runtime/memset', - 'runtime/operator', - 'runtime/printf', - 'runtime/printf_format', - 'runtime/references', - 'runtime/sizeof', - 'runtime/string', - 'runtime/threadsafe_fn', - 'runtime/vlog', - 'whitespace/blank_line', - 'whitespace/braces', - 'whitespace/comma', - 'whitespace/comments', - 'whitespace/empty_conditional_body', - 'whitespace/empty_loop_body', - 'whitespace/end_of_line', - 'whitespace/ending_newline', - 'whitespace/forcolon', - 'whitespace/indent', - 'whitespace/line_length', - 'whitespace/newline', - 'whitespace/operators', - 'whitespace/parens', - 'whitespace/semicolon', - 'whitespace/tab', - 'whitespace/todo' - ] - -# The default state of the category filter. This is overrided by the --filter= -# flag. By default all errors are on, so only add here categories that should be -# off by default (i.e., categories that must be enabled by the --filter= flags). -# All entries here should start with a '-' or '+', as in the --filter= flag. -_DEFAULT_FILTERS = ['-build/include_alpha'] - -# We used to check for high-bit characters, but after much discussion we -# decided those were OK, as long as they were in UTF-8 and didn't represent -# hard-coded international strings, which belong in a separate i18n file. - - -# C++ headers -_CPP_HEADERS = frozenset([ - # Legacy - 'algobase.h', - 'algo.h', - 'alloc.h', - 'builtinbuf.h', - 'bvector.h', - 'complex.h', - 'defalloc.h', - 'deque.h', - 'editbuf.h', - 'fstream.h', - 'function.h', - 'hash_map', - 'hash_map.h', - 'hash_set', - 'hash_set.h', - 'hashtable.h', - 'heap.h', - 'indstream.h', - 'iomanip.h', - 'iostream.h', - 'istream.h', - 'iterator.h', - 'list.h', - 'map.h', - 'multimap.h', - 'multiset.h', - 'ostream.h', - 'pair.h', - 'parsestream.h', - 'pfstream.h', - 'procbuf.h', - 'pthread_alloc', - 'pthread_alloc.h', - 'rope', - 'rope.h', - 'ropeimpl.h', - 'set.h', - 'slist', - 'slist.h', - 'stack.h', - 'stdiostream.h', - 'stl_alloc.h', - 'stl_relops.h', - 'streambuf.h', - 'stream.h', - 'strfile.h', - 'strstream.h', - 'tempbuf.h', - 'tree.h', - 'type_traits.h', - 'vector.h', - # 17.6.1.2 C++ library headers - 'algorithm', - 'array', - 'atomic', - 'bitset', - 'chrono', - 'codecvt', - 'complex', - 'condition_variable', - 'deque', - 'exception', - 'forward_list', - 'fstream', - 'functional', - 'future', - 'initializer_list', - 'iomanip', - 'ios', - 'iosfwd', - 'iostream', - 'istream', - 'iterator', - 'limits', - 'list', - 'locale', - 'map', - 'memory', - 'mutex', - 'new', - 'numeric', - 'ostream', - 'queue', - 'random', - 'ratio', - 'regex', - 'set', - 'sstream', - 'stack', - 'stdexcept', - 'streambuf', - 'string', - 'strstream', - 'system_error', - 'thread', - 'tuple', - 'typeindex', - 'typeinfo', - 'type_traits', - 'unordered_map', - 'unordered_set', - 'utility', - 'valarray', - 'vector', - # 17.6.1.2 C++ headers for C library facilities - 'cassert', - 'ccomplex', - 'cctype', - 'cerrno', - 'cfenv', - 'cfloat', - 'cinttypes', - 'ciso646', - 'climits', - 'clocale', - 'cmath', - 'csetjmp', - 'csignal', - 'cstdalign', - 'cstdarg', - 'cstdbool', - 'cstddef', - 'cstdint', - 'cstdio', - 'cstdlib', - 'cstring', - 'ctgmath', - 'ctime', - 'cuchar', - 'cwchar', - 'cwctype', - ]) - -# Assertion macros. These are defined in base/logging.h and -# testing/base/gunit.h. Note that the _M versions need to come first -# for substring matching to work. -_CHECK_MACROS = [ - 'DCHECK', 'CHECK', - 'EXPECT_TRUE_M', 'EXPECT_TRUE', - 'ASSERT_TRUE_M', 'ASSERT_TRUE', - 'EXPECT_FALSE_M', 'EXPECT_FALSE', - 'ASSERT_FALSE_M', 'ASSERT_FALSE', - ] - -# Replacement macros for CHECK/DCHECK/EXPECT_TRUE/EXPECT_FALSE -_CHECK_REPLACEMENT = dict([(m, {}) for m in _CHECK_MACROS]) - -for op, replacement in [('==', 'EQ'), ('!=', 'NE'), - ('>=', 'GE'), ('>', 'GT'), - ('<=', 'LE'), ('<', 'LT')]: - _CHECK_REPLACEMENT['DCHECK'][op] = 'DCHECK_%s' % replacement - _CHECK_REPLACEMENT['CHECK'][op] = 'CHECK_%s' % replacement - _CHECK_REPLACEMENT['EXPECT_TRUE'][op] = 'EXPECT_%s' % replacement - _CHECK_REPLACEMENT['ASSERT_TRUE'][op] = 'ASSERT_%s' % replacement - _CHECK_REPLACEMENT['EXPECT_TRUE_M'][op] = 'EXPECT_%s_M' % replacement - _CHECK_REPLACEMENT['ASSERT_TRUE_M'][op] = 'ASSERT_%s_M' % replacement - -for op, inv_replacement in [('==', 'NE'), ('!=', 'EQ'), - ('>=', 'LT'), ('>', 'LE'), - ('<=', 'GT'), ('<', 'GE')]: - _CHECK_REPLACEMENT['EXPECT_FALSE'][op] = 'EXPECT_%s' % inv_replacement - _CHECK_REPLACEMENT['ASSERT_FALSE'][op] = 'ASSERT_%s' % inv_replacement - _CHECK_REPLACEMENT['EXPECT_FALSE_M'][op] = 'EXPECT_%s_M' % inv_replacement - _CHECK_REPLACEMENT['ASSERT_FALSE_M'][op] = 'ASSERT_%s_M' % inv_replacement - -# Alternative tokens and their replacements. For full list, see section 2.5 -# Alternative tokens [lex.digraph] in the C++ standard. -# -# Digraphs (such as '%:') are not included here since it's a mess to -# match those on a word boundary. -_ALT_TOKEN_REPLACEMENT = { - 'and': '&&', - 'bitor': '|', - 'or': '||', - 'xor': '^', - 'compl': '~', - 'bitand': '&', - 'and_eq': '&=', - 'or_eq': '|=', - 'xor_eq': '^=', - 'not': '!', - 'not_eq': '!=' - } - -# Compile regular expression that matches all the above keywords. The "[ =()]" -# bit is meant to avoid matching these keywords outside of boolean expressions. -# -# False positives include C-style multi-line comments and multi-line strings -# but those have always been troublesome for cpplint. -_ALT_TOKEN_REPLACEMENT_PATTERN = re.compile( - r'[ =()](' + ('|'.join(_ALT_TOKEN_REPLACEMENT.keys())) + r')(?=[ (]|$)') - - -# These constants define types of headers for use with -# _IncludeState.CheckNextIncludeOrder(). -_C_SYS_HEADER = 1 -_CPP_SYS_HEADER = 2 -_LIKELY_MY_HEADER = 3 -_POSSIBLE_MY_HEADER = 4 -_OTHER_HEADER = 5 - -# These constants define the current inline assembly state -_NO_ASM = 0 # Outside of inline assembly block -_INSIDE_ASM = 1 # Inside inline assembly block -_END_ASM = 2 # Last line of inline assembly block -_BLOCK_ASM = 3 # The whole block is an inline assembly block - -# Match start of assembly blocks -_MATCH_ASM = re.compile(r'^\s*(?:asm|_asm|__asm|__asm__)' - r'(?:\s+(volatile|__volatile__))?' - r'\s*[{(]') - - -_regexp_compile_cache = {} - -# Finds occurrences of NOLINT or NOLINT(...). -_RE_SUPPRESSION = re.compile(r'\bNOLINT\b(\([^)]*\))?') - -# {str, set(int)}: a map from error categories to sets of linenumbers -# on which those errors are expected and should be suppressed. -_error_suppressions = {} - -# The root directory used for deriving header guard CPP variable. -# This is set by --root flag. -_root = None - -# The allowed line length of files. -# This is set by --linelength flag. -_line_length = 80 - -# The allowed extensions for file names -# This is set by --extensions flag. -_valid_extensions = set(['cc', 'h', 'cpp', 'cu', 'cuh']) - -def ParseNolintSuppressions(filename, raw_line, linenum, error): - """Updates the global list of error-suppressions. - - Parses any NOLINT comments on the current line, updating the global - error_suppressions store. Reports an error if the NOLINT comment - was malformed. - - Args: - filename: str, the name of the input file. - raw_line: str, the line of input text, with comments. - linenum: int, the number of the current line. - error: function, an error handler. - """ - # FIXME(adonovan): "NOLINT(" is misparsed as NOLINT(*). - matched = _RE_SUPPRESSION.search(raw_line) - if matched: - category = matched.group(1) - if category in (None, '(*)'): # => "suppress all" - _error_suppressions.setdefault(None, set()).add(linenum) - else: - if category.startswith('(') and category.endswith(')'): - category = category[1:-1] - if category in _ERROR_CATEGORIES: - _error_suppressions.setdefault(category, set()).add(linenum) - else: - error(filename, linenum, 'readability/nolint', 5, - 'Unknown NOLINT error category: %s' % category) - - -def ResetNolintSuppressions(): - "Resets the set of NOLINT suppressions to empty." - _error_suppressions.clear() - - -def IsErrorSuppressedByNolint(category, linenum): - """Returns true if the specified error category is suppressed on this line. - - Consults the global error_suppressions map populated by - ParseNolintSuppressions/ResetNolintSuppressions. - - Args: - category: str, the category of the error. - linenum: int, the current line number. - Returns: - bool, True iff the error should be suppressed due to a NOLINT comment. - """ - return (linenum in _error_suppressions.get(category, set()) or - linenum in _error_suppressions.get(None, set())) - -def Match(pattern, s): - """Matches the string with the pattern, caching the compiled regexp.""" - # The regexp compilation caching is inlined in both Match and Search for - # performance reasons; factoring it out into a separate function turns out - # to be noticeably expensive. - if pattern not in _regexp_compile_cache: - _regexp_compile_cache[pattern] = sre_compile.compile(pattern) - return _regexp_compile_cache[pattern].match(s) - - -def ReplaceAll(pattern, rep, s): - """Replaces instances of pattern in a string with a replacement. - - The compiled regex is kept in a cache shared by Match and Search. - - Args: - pattern: regex pattern - rep: replacement text - s: search string - - Returns: - string with replacements made (or original string if no replacements) - """ - if pattern not in _regexp_compile_cache: - _regexp_compile_cache[pattern] = sre_compile.compile(pattern) - return _regexp_compile_cache[pattern].sub(rep, s) - - -def Search(pattern, s): - """Searches the string for the pattern, caching the compiled regexp.""" - if pattern not in _regexp_compile_cache: - _regexp_compile_cache[pattern] = sre_compile.compile(pattern) - return _regexp_compile_cache[pattern].search(s) - - -class _IncludeState(dict): - """Tracks line numbers for includes, and the order in which includes appear. - - As a dict, an _IncludeState object serves as a mapping between include - filename and line number on which that file was included. - - Call CheckNextIncludeOrder() once for each header in the file, passing - in the type constants defined above. Calls in an illegal order will - raise an _IncludeError with an appropriate error message. - - """ - # self._section will move monotonically through this set. If it ever - # needs to move backwards, CheckNextIncludeOrder will raise an error. - _INITIAL_SECTION = 0 - _MY_H_SECTION = 1 - _C_SECTION = 2 - _CPP_SECTION = 3 - _OTHER_H_SECTION = 4 - - _TYPE_NAMES = { - _C_SYS_HEADER: 'C system header', - _CPP_SYS_HEADER: 'C++ system header', - _LIKELY_MY_HEADER: 'header this file implements', - _POSSIBLE_MY_HEADER: 'header this file may implement', - _OTHER_HEADER: 'other header', - } - _SECTION_NAMES = { - _INITIAL_SECTION: "... nothing. (This can't be an error.)", - _MY_H_SECTION: 'a header this file implements', - _C_SECTION: 'C system header', - _CPP_SECTION: 'C++ system header', - _OTHER_H_SECTION: 'other header', - } - - def __init__(self): - dict.__init__(self) - self.ResetSection() - - def ResetSection(self): - # The name of the current section. - self._section = self._INITIAL_SECTION - # The path of last found header. - self._last_header = '' - - def SetLastHeader(self, header_path): - self._last_header = header_path - - def CanonicalizeAlphabeticalOrder(self, header_path): - """Returns a path canonicalized for alphabetical comparison. - - - replaces "-" with "_" so they both cmp the same. - - removes '-inl' since we don't require them to be after the main header. - - lowercase everything, just in case. - - Args: - header_path: Path to be canonicalized. - - Returns: - Canonicalized path. - """ - return header_path.replace('-inl.h', '.h').replace('-', '_').lower() - - def IsInAlphabeticalOrder(self, clean_lines, linenum, header_path): - """Check if a header is in alphabetical order with the previous header. - - Args: - clean_lines: A CleansedLines instance containing the file. - linenum: The number of the line to check. - header_path: Canonicalized header to be checked. - - Returns: - Returns true if the header is in alphabetical order. - """ - # If previous section is different from current section, _last_header will - # be reset to empty string, so it's always less than current header. - # - # If previous line was a blank line, assume that the headers are - # intentionally sorted the way they are. - if (self._last_header > header_path and - not Match(r'^\s*$', clean_lines.elided[linenum - 1])): - return False - return True - - def CheckNextIncludeOrder(self, header_type): - """Returns a non-empty error message if the next header is out of order. - - This function also updates the internal state to be ready to check - the next include. - - Args: - header_type: One of the _XXX_HEADER constants defined above. - - Returns: - The empty string if the header is in the right order, or an - error message describing what's wrong. - - """ - error_message = ('Found %s after %s' % - (self._TYPE_NAMES[header_type], - self._SECTION_NAMES[self._section])) - - last_section = self._section - - if header_type == _C_SYS_HEADER: - if self._section <= self._C_SECTION: - self._section = self._C_SECTION - else: - self._last_header = '' - return error_message - elif header_type == _CPP_SYS_HEADER: - if self._section <= self._CPP_SECTION: - self._section = self._CPP_SECTION - else: - self._last_header = '' - return error_message - elif header_type == _LIKELY_MY_HEADER: - if self._section <= self._MY_H_SECTION: - self._section = self._MY_H_SECTION - else: - self._section = self._OTHER_H_SECTION - elif header_type == _POSSIBLE_MY_HEADER: - if self._section <= self._MY_H_SECTION: - self._section = self._MY_H_SECTION - else: - # This will always be the fallback because we're not sure - # enough that the header is associated with this file. - self._section = self._OTHER_H_SECTION - else: - assert header_type == _OTHER_HEADER - self._section = self._OTHER_H_SECTION - - if last_section != self._section: - self._last_header = '' - - return '' - - -class _CppLintState(object): - """Maintains module-wide state..""" - - def __init__(self): - self.verbose_level = 1 # global setting. - self.error_count = 0 # global count of reported errors - # filters to apply when emitting error messages - self.filters = _DEFAULT_FILTERS[:] - self.counting = 'total' # In what way are we counting errors? - self.errors_by_category = {} # string to int dict storing error counts - - # output format: - # "emacs" - format that emacs can parse (default) - # "vs7" - format that Microsoft Visual Studio 7 can parse - self.output_format = 'emacs' - - def SetOutputFormat(self, output_format): - """Sets the output format for errors.""" - self.output_format = output_format - - def SetVerboseLevel(self, level): - """Sets the module's verbosity, and returns the previous setting.""" - last_verbose_level = self.verbose_level - self.verbose_level = level - return last_verbose_level - - def SetCountingStyle(self, counting_style): - """Sets the module's counting options.""" - self.counting = counting_style - - def SetFilters(self, filters): - """Sets the error-message filters. - - These filters are applied when deciding whether to emit a given - error message. - - Args: - filters: A string of comma-separated filters (eg "+whitespace/indent"). - Each filter should start with + or -; else we die. - - Raises: - ValueError: The comma-separated filters did not all start with '+' or '-'. - E.g. "-,+whitespace,-whitespace/indent,whitespace/badfilter" - """ - # Default filters always have less priority than the flag ones. - self.filters = _DEFAULT_FILTERS[:] - for filt in filters.split(','): - clean_filt = filt.strip() - if clean_filt: - self.filters.append(clean_filt) - for filt in self.filters: - if not (filt.startswith('+') or filt.startswith('-')): - raise ValueError('Every filter in --filters must start with + or -' - ' (%s does not)' % filt) - - def ResetErrorCounts(self): - """Sets the module's error statistic back to zero.""" - self.error_count = 0 - self.errors_by_category = {} - - def IncrementErrorCount(self, category): - """Bumps the module's error statistic.""" - self.error_count += 1 - if self.counting in ('toplevel', 'detailed'): - if self.counting != 'detailed': - category = category.split('/')[0] - if category not in self.errors_by_category: - self.errors_by_category[category] = 0 - self.errors_by_category[category] += 1 - - def PrintErrorCounts(self): - """Print a summary of errors by category, and the total.""" - for category, count in self.errors_by_category.iteritems(): - sys.stderr.write('Category \'%s\' errors found: %d\n' % - (category, count)) - sys.stderr.write('Total errors found: %d\n' % self.error_count) - -_cpplint_state = _CppLintState() - - -def _OutputFormat(): - """Gets the module's output format.""" - return _cpplint_state.output_format - - -def _SetOutputFormat(output_format): - """Sets the module's output format.""" - _cpplint_state.SetOutputFormat(output_format) - - -def _VerboseLevel(): - """Returns the module's verbosity setting.""" - return _cpplint_state.verbose_level - - -def _SetVerboseLevel(level): - """Sets the module's verbosity, and returns the previous setting.""" - return _cpplint_state.SetVerboseLevel(level) - - -def _SetCountingStyle(level): - """Sets the module's counting options.""" - _cpplint_state.SetCountingStyle(level) - - -def _Filters(): - """Returns the module's list of output filters, as a list.""" - return _cpplint_state.filters - - -def _SetFilters(filters): - """Sets the module's error-message filters. - - These filters are applied when deciding whether to emit a given - error message. - - Args: - filters: A string of comma-separated filters (eg "whitespace/indent"). - Each filter should start with + or -; else we die. - """ - _cpplint_state.SetFilters(filters) - - -class _FunctionState(object): - """Tracks current function name and the number of lines in its body.""" - - _NORMAL_TRIGGER = 250 # for --v=0, 500 for --v=1, etc. - _TEST_TRIGGER = 400 # about 50% more than _NORMAL_TRIGGER. - - def __init__(self): - self.in_a_function = False - self.lines_in_function = 0 - self.current_function = '' - - def Begin(self, function_name): - """Start analyzing function body. - - Args: - function_name: The name of the function being tracked. - """ - self.in_a_function = True - self.lines_in_function = 0 - self.current_function = function_name - - def Count(self): - """Count line in current function body.""" - if self.in_a_function: - self.lines_in_function += 1 - - def Check(self, error, filename, linenum): - """Report if too many lines in function body. - - Args: - error: The function to call with any errors found. - filename: The name of the current file. - linenum: The number of the line to check. - """ - if Match(r'T(EST|est)', self.current_function): - base_trigger = self._TEST_TRIGGER - else: - base_trigger = self._NORMAL_TRIGGER - trigger = base_trigger * 2**_VerboseLevel() - - if self.lines_in_function > trigger: - error_level = int(math.log(self.lines_in_function / base_trigger, 2)) - # 50 => 0, 100 => 1, 200 => 2, 400 => 3, 800 => 4, 1600 => 5, ... - if error_level > 5: - error_level = 5 - error(filename, linenum, 'readability/fn_size', error_level, - 'Small and focused functions are preferred:' - ' %s has %d non-comment lines' - ' (error triggered by exceeding %d lines).' % ( - self.current_function, self.lines_in_function, trigger)) - - def End(self): - """Stop analyzing function body.""" - self.in_a_function = False - - -class _IncludeError(Exception): - """Indicates a problem with the include order in a file.""" - pass - - -class FileInfo: - """Provides utility functions for filenames. - - FileInfo provides easy access to the components of a file's path - relative to the project root. - """ - - def __init__(self, filename): - self._filename = filename - - def FullName(self): - """Make Windows paths like Unix.""" - return os.path.abspath(self._filename).replace('\\', '/') - - def RepositoryName(self): - """FullName after removing the local path to the repository. - - If we have a real absolute path name here we can try to do something smart: - detecting the root of the checkout and truncating /path/to/checkout from - the name so that we get header guards that don't include things like - "C:\Documents and Settings\..." or "/home/username/..." in them and thus - people on different computers who have checked the source out to different - locations won't see bogus errors. - """ - fullname = self.FullName() - - if os.path.exists(fullname): - project_dir = os.path.dirname(fullname) - - if os.path.exists(os.path.join(project_dir, ".svn")): - # If there's a .svn file in the current directory, we recursively look - # up the directory tree for the top of the SVN checkout - root_dir = project_dir - one_up_dir = os.path.dirname(root_dir) - while os.path.exists(os.path.join(one_up_dir, ".svn")): - root_dir = os.path.dirname(root_dir) - one_up_dir = os.path.dirname(one_up_dir) - - prefix = os.path.commonprefix([root_dir, project_dir]) - return fullname[len(prefix) + 1:] - - # Not SVN <= 1.6? Try to find a git, hg, or svn top level directory by - # searching up from the current path. - root_dir = os.path.dirname(fullname) - while (root_dir != os.path.dirname(root_dir) and - not os.path.exists(os.path.join(root_dir, ".git")) and - not os.path.exists(os.path.join(root_dir, ".hg")) and - not os.path.exists(os.path.join(root_dir, ".svn"))): - root_dir = os.path.dirname(root_dir) - - if (os.path.exists(os.path.join(root_dir, ".git")) or - os.path.exists(os.path.join(root_dir, ".hg")) or - os.path.exists(os.path.join(root_dir, ".svn"))): - prefix = os.path.commonprefix([root_dir, project_dir]) - return fullname[len(prefix) + 1:] - - # Don't know what to do; header guard warnings may be wrong... - return fullname - - def Split(self): - """Splits the file into the directory, basename, and extension. - - For 'chrome/browser/browser.cc', Split() would - return ('chrome/browser', 'browser', '.cc') - - Returns: - A tuple of (directory, basename, extension). - """ - - googlename = self.RepositoryName() - project, rest = os.path.split(googlename) - return (project,) + os.path.splitext(rest) - - def BaseName(self): - """File base name - text after the final slash, before the final period.""" - return self.Split()[1] - - def Extension(self): - """File extension - text following the final period.""" - return self.Split()[2] - - def NoExtension(self): - """File has no source file extension.""" - return '/'.join(self.Split()[0:2]) - - def IsSource(self): - """File has a source file extension.""" - return self.Extension()[1:] in ('c', 'cc', 'cpp', 'cxx') - - -def _ShouldPrintError(category, confidence, linenum): - """If confidence >= verbose, category passes filter and is not suppressed.""" - - # There are three ways we might decide not to print an error message: - # a "NOLINT(category)" comment appears in the source, - # the verbosity level isn't high enough, or the filters filter it out. - if IsErrorSuppressedByNolint(category, linenum): - return False - if confidence < _cpplint_state.verbose_level: - return False - - is_filtered = False - for one_filter in _Filters(): - if one_filter.startswith('-'): - if category.startswith(one_filter[1:]): - is_filtered = True - elif one_filter.startswith('+'): - if category.startswith(one_filter[1:]): - is_filtered = False - else: - assert False # should have been checked for in SetFilter. - if is_filtered: - return False - - return True - - -def Error(filename, linenum, category, confidence, message): - """Logs the fact we've found a lint error. - - We log where the error was found, and also our confidence in the error, - that is, how certain we are this is a legitimate style regression, and - not a misidentification or a use that's sometimes justified. - - False positives can be suppressed by the use of - "cpplint(category)" comments on the offending line. These are - parsed into _error_suppressions. - - Args: - filename: The name of the file containing the error. - linenum: The number of the line containing the error. - category: A string used to describe the "category" this bug - falls under: "whitespace", say, or "runtime". Categories - may have a hierarchy separated by slashes: "whitespace/indent". - confidence: A number from 1-5 representing a confidence score for - the error, with 5 meaning that we are certain of the problem, - and 1 meaning that it could be a legitimate construct. - message: The error message. - """ - if _ShouldPrintError(category, confidence, linenum): - _cpplint_state.IncrementErrorCount(category) - if _cpplint_state.output_format == 'vs7': - sys.stderr.write('%s(%s): %s [%s] [%d]\n' % ( - filename, linenum, message, category, confidence)) - elif _cpplint_state.output_format == 'eclipse': - sys.stderr.write('%s:%s: warning: %s [%s] [%d]\n' % ( - filename, linenum, message, category, confidence)) - else: - sys.stderr.write('%s:%s: %s [%s] [%d]\n' % ( - filename, linenum, message, category, confidence)) - - -# Matches standard C++ escape sequences per 2.13.2.3 of the C++ standard. -_RE_PATTERN_CLEANSE_LINE_ESCAPES = re.compile( - r'\\([abfnrtv?"\\\']|\d+|x[0-9a-fA-F]+)') -# Matches strings. Escape codes should already be removed by ESCAPES. -_RE_PATTERN_CLEANSE_LINE_DOUBLE_QUOTES = re.compile(r'"[^"]*"') -# Matches characters. Escape codes should already be removed by ESCAPES. -_RE_PATTERN_CLEANSE_LINE_SINGLE_QUOTES = re.compile(r"'.'") -# Matches multi-line C++ comments. -# This RE is a little bit more complicated than one might expect, because we -# have to take care of space removals tools so we can handle comments inside -# statements better. -# The current rule is: We only clear spaces from both sides when we're at the -# end of the line. Otherwise, we try to remove spaces from the right side, -# if this doesn't work we try on left side but only if there's a non-character -# on the right. -_RE_PATTERN_CLEANSE_LINE_C_COMMENTS = re.compile( - r"""(\s*/\*.*\*/\s*$| - /\*.*\*/\s+| - \s+/\*.*\*/(?=\W)| - /\*.*\*/)""", re.VERBOSE) - - -def IsCppString(line): - """Does line terminate so, that the next symbol is in string constant. - - This function does not consider single-line nor multi-line comments. - - Args: - line: is a partial line of code starting from the 0..n. - - Returns: - True, if next character appended to 'line' is inside a - string constant. - """ - - line = line.replace(r'\\', 'XX') # after this, \\" does not match to \" - return ((line.count('"') - line.count(r'\"') - line.count("'\"'")) & 1) == 1 - - -def CleanseRawStrings(raw_lines): - """Removes C++11 raw strings from lines. - - Before: - static const char kData[] = R"( - multi-line string - )"; - - After: - static const char kData[] = "" - (replaced by blank line) - ""; - - Args: - raw_lines: list of raw lines. - - Returns: - list of lines with C++11 raw strings replaced by empty strings. - """ - - delimiter = None - lines_without_raw_strings = [] - for line in raw_lines: - if delimiter: - # Inside a raw string, look for the end - end = line.find(delimiter) - if end >= 0: - # Found the end of the string, match leading space for this - # line and resume copying the original lines, and also insert - # a "" on the last line. - leading_space = Match(r'^(\s*)\S', line) - line = leading_space.group(1) + '""' + line[end + len(delimiter):] - delimiter = None - else: - # Haven't found the end yet, append a blank line. - line = '' - - else: - # Look for beginning of a raw string. - # See 2.14.15 [lex.string] for syntax. - matched = Match(r'^(.*)\b(?:R|u8R|uR|UR|LR)"([^\s\\()]*)\((.*)$', line) - if matched: - delimiter = ')' + matched.group(2) + '"' - - end = matched.group(3).find(delimiter) - if end >= 0: - # Raw string ended on same line - line = (matched.group(1) + '""' + - matched.group(3)[end + len(delimiter):]) - delimiter = None - else: - # Start of a multi-line raw string - line = matched.group(1) + '""' - - lines_without_raw_strings.append(line) - - # TODO(unknown): if delimiter is not None here, we might want to - # emit a warning for unterminated string. - return lines_without_raw_strings - - -def FindNextMultiLineCommentStart(lines, lineix): - """Find the beginning marker for a multiline comment.""" - while lineix < len(lines): - if lines[lineix].strip().startswith('/*'): - # Only return this marker if the comment goes beyond this line - if lines[lineix].strip().find('*/', 2) < 0: - return lineix - lineix += 1 - return len(lines) - - -def FindNextMultiLineCommentEnd(lines, lineix): - """We are inside a comment, find the end marker.""" - while lineix < len(lines): - if lines[lineix].strip().endswith('*/'): - return lineix - lineix += 1 - return len(lines) - - -def RemoveMultiLineCommentsFromRange(lines, begin, end): - """Clears a range of lines for multi-line comments.""" - # Having // dummy comments makes the lines non-empty, so we will not get - # unnecessary blank line warnings later in the code. - for i in range(begin, end): - lines[i] = '// dummy' - - -def RemoveMultiLineComments(filename, lines, error): - """Removes multiline (c-style) comments from lines.""" - lineix = 0 - while lineix < len(lines): - lineix_begin = FindNextMultiLineCommentStart(lines, lineix) - if lineix_begin >= len(lines): - return - lineix_end = FindNextMultiLineCommentEnd(lines, lineix_begin) - if lineix_end >= len(lines): - error(filename, lineix_begin + 1, 'readability/multiline_comment', 5, - 'Could not find end of multi-line comment') - return - RemoveMultiLineCommentsFromRange(lines, lineix_begin, lineix_end + 1) - lineix = lineix_end + 1 - - -def CleanseComments(line): - """Removes //-comments and single-line C-style /* */ comments. - - Args: - line: A line of C++ source. - - Returns: - The line with single-line comments removed. - """ - commentpos = line.find('//') - if commentpos != -1 and not IsCppString(line[:commentpos]): - line = line[:commentpos].rstrip() - # get rid of /* ... */ - return _RE_PATTERN_CLEANSE_LINE_C_COMMENTS.sub('', line) - - -class CleansedLines(object): - """Holds 3 copies of all lines with different preprocessing applied to them. - - 1) elided member contains lines without strings and comments, - 2) lines member contains lines without comments, and - 3) raw_lines member contains all the lines without processing. - All these three members are of <type 'list'>, and of the same length. - """ - - def __init__(self, lines): - self.elided = [] - self.lines = [] - self.raw_lines = lines - self.num_lines = len(lines) - self.lines_without_raw_strings = CleanseRawStrings(lines) - for linenum in range(len(self.lines_without_raw_strings)): - self.lines.append(CleanseComments( - self.lines_without_raw_strings[linenum])) - elided = self._CollapseStrings(self.lines_without_raw_strings[linenum]) - self.elided.append(CleanseComments(elided)) - - def NumLines(self): - """Returns the number of lines represented.""" - return self.num_lines - - @staticmethod - def _CollapseStrings(elided): - """Collapses strings and chars on a line to simple "" or '' blocks. - - We nix strings first so we're not fooled by text like '"http://"' - - Args: - elided: The line being processed. - - Returns: - The line with collapsed strings. - """ - if not _RE_PATTERN_INCLUDE.match(elided): - # Remove escaped characters first to make quote/single quote collapsing - # basic. Things that look like escaped characters shouldn't occur - # outside of strings and chars. - elided = _RE_PATTERN_CLEANSE_LINE_ESCAPES.sub('', elided) - elided = _RE_PATTERN_CLEANSE_LINE_SINGLE_QUOTES.sub("''", elided) - elided = _RE_PATTERN_CLEANSE_LINE_DOUBLE_QUOTES.sub('""', elided) - return elided - - -def FindEndOfExpressionInLine(line, startpos, depth, startchar, endchar): - """Find the position just after the matching endchar. - - Args: - line: a CleansedLines line. - startpos: start searching at this position. - depth: nesting level at startpos. - startchar: expression opening character. - endchar: expression closing character. - - Returns: - On finding matching endchar: (index just after matching endchar, 0) - Otherwise: (-1, new depth at end of this line) - """ - for i in xrange(startpos, len(line)): - if line[i] == startchar: - depth += 1 - elif line[i] == endchar: - depth -= 1 - if depth == 0: - return (i + 1, 0) - return (-1, depth) - - -def CloseExpression(clean_lines, linenum, pos): - """If input points to ( or { or [ or <, finds the position that closes it. - - If lines[linenum][pos] points to a '(' or '{' or '[' or '<', finds the - linenum/pos that correspond to the closing of the expression. - - Args: - clean_lines: A CleansedLines instance containing the file. - linenum: The number of the line to check. - pos: A position on the line. - - Returns: - A tuple (line, linenum, pos) pointer *past* the closing brace, or - (line, len(lines), -1) if we never find a close. Note we ignore - strings and comments when matching; and the line we return is the - 'cleansed' line at linenum. - """ - - line = clean_lines.elided[linenum] - startchar = line[pos] - if startchar not in '({[<': - return (line, clean_lines.NumLines(), -1) - if startchar == '(': endchar = ')' - if startchar == '[': endchar = ']' - if startchar == '{': endchar = '}' - if startchar == '<': endchar = '>' - - # Check first line - (end_pos, num_open) = FindEndOfExpressionInLine( - line, pos, 0, startchar, endchar) - if end_pos > -1: - return (line, linenum, end_pos) - - # Continue scanning forward - while linenum < clean_lines.NumLines() - 1: - linenum += 1 - line = clean_lines.elided[linenum] - (end_pos, num_open) = FindEndOfExpressionInLine( - line, 0, num_open, startchar, endchar) - if end_pos > -1: - return (line, linenum, end_pos) - - # Did not find endchar before end of file, give up - return (line, clean_lines.NumLines(), -1) - - -def FindStartOfExpressionInLine(line, endpos, depth, startchar, endchar): - """Find position at the matching startchar. - - This is almost the reverse of FindEndOfExpressionInLine, but note - that the input position and returned position differs by 1. - - Args: - line: a CleansedLines line. - endpos: start searching at this position. - depth: nesting level at endpos. - startchar: expression opening character. - endchar: expression closing character. - - Returns: - On finding matching startchar: (index at matching startchar, 0) - Otherwise: (-1, new depth at beginning of this line) - """ - for i in xrange(endpos, -1, -1): - if line[i] == endchar: - depth += 1 - elif line[i] == startchar: - depth -= 1 - if depth == 0: - return (i, 0) - return (-1, depth) - - -def ReverseCloseExpression(clean_lines, linenum, pos): - """If input points to ) or } or ] or >, finds the position that opens it. - - If lines[linenum][pos] points to a ')' or '}' or ']' or '>', finds the - linenum/pos that correspond to the opening of the expression. - - Args: - clean_lines: A CleansedLines instance containing the file. - linenum: The number of the line to check. - pos: A position on the line. - - Returns: - A tuple (line, linenum, pos) pointer *at* the opening brace, or - (line, 0, -1) if we never find the matching opening brace. Note - we ignore strings and comments when matching; and the line we - return is the 'cleansed' line at linenum. - """ - line = clean_lines.elided[linenum] - endchar = line[pos] - if endchar not in ')}]>': - return (line, 0, -1) - if endchar == ')': startchar = '(' - if endchar == ']': startchar = '[' - if endchar == '}': startchar = '{' - if endchar == '>': startchar = '<' - - # Check last line - (start_pos, num_open) = FindStartOfExpressionInLine( - line, pos, 0, startchar, endchar) - if start_pos > -1: - return (line, linenum, start_pos) - - # Continue scanning backward - while linenum > 0: - linenum -= 1 - line = clean_lines.elided[linenum] - (start_pos, num_open) = FindStartOfExpressionInLine( - line, len(line) - 1, num_open, startchar, endchar) - if start_pos > -1: - return (line, linenum, start_pos) - - # Did not find startchar before beginning of file, give up - return (line, 0, -1) - - -def CheckForCopyright(filename, lines, error): - """Logs an error if no Copyright message appears at the top of the file.""" - - # We'll say it should occur by line 10. Don't forget there's a - # dummy line at the front. - for line in xrange(1, min(len(lines), 11)): - if re.search(r'Copyright', lines[line], re.I): break - else: # means no copyright line was found - error(filename, 0, 'legal/copyright', 5, - 'No copyright message found. ' - 'You should have a line: "Copyright [year] <Copyright Owner>"') - - -def GetHeaderGuardCPPVariable(filename): - """Returns the CPP variable that should be used as a header guard. - - Args: - filename: The name of a C++ header file. - - Returns: - The CPP variable that should be used as a header guard in the - named file. - - """ - - # Restores original filename in case that cpplint is invoked from Emacs's - # flymake. - filename = re.sub(r'_flymake\.h$', '.h', filename) - filename = re.sub(r'/\.flymake/([^/]*)$', r'/\1', filename) - - fileinfo = FileInfo(filename) - file_path_from_root = fileinfo.RepositoryName() - if _root: - file_path_from_root = re.sub('^' + _root + os.sep, '', file_path_from_root) - return re.sub(r'[-./\s]', '_', file_path_from_root).upper() + '_' - - -def CheckForHeaderGuard(filename, lines, error): - """Checks that the file contains a header guard. - - Logs an error if no #ifndef header guard is present. For other - headers, checks that the full pathname is used. - - Args: - filename: The name of the C++ header file. - lines: An array of strings, each representing a line of the file. - error: The function to call with any errors found. - """ - - cppvar = GetHeaderGuardCPPVariable(filename) - - ifndef = None - ifndef_linenum = 0 - define = None - endif = None - endif_linenum = 0 - for linenum, line in enumerate(lines): - linesplit = line.split() - if len(linesplit) >= 2: - # find the first occurrence of #ifndef and #define, save arg - if not ifndef and linesplit[0] == '#ifndef': - # set ifndef to the header guard presented on the #ifndef line. - ifndef = linesplit[1] - ifndef_linenum = linenum - if not define and linesplit[0] == '#define': - define = linesplit[1] - # find the last occurrence of #endif, save entire line - if line.startswith('#endif'): - endif = line - endif_linenum = linenum - - if not ifndef: - error(filename, 0, 'build/header_guard', 5, - 'No #ifndef header guard found, suggested CPP variable is: %s' % - cppvar) - return - - if not define: - error(filename, 0, 'build/header_guard', 5, - 'No #define header guard found, suggested CPP variable is: %s' % - cppvar) - return - - # The guard should be PATH_FILE_H_, but we also allow PATH_FILE_H__ - # for backward compatibility. - if ifndef != cppvar: - error_level = 0 - if ifndef != cppvar + '_': - error_level = 5 - - ParseNolintSuppressions(filename, lines[ifndef_linenum], ifndef_linenum, - error) - error(filename, ifndef_linenum, 'build/header_guard', error_level, - '#ifndef header guard has wrong style, please use: %s' % cppvar) - - if define != ifndef: - error(filename, 0, 'build/header_guard', 5, - '#ifndef and #define don\'t match, suggested CPP variable is: %s' % - cppvar) - return - - if endif != ('#endif // %s' % cppvar): - error_level = 0 - if endif != ('#endif // %s' % (cppvar + '_')): - error_level = 5 - - ParseNolintSuppressions(filename, lines[endif_linenum], endif_linenum, - error) - error(filename, endif_linenum, 'build/header_guard', error_level, - '#endif line should be "#endif // %s"' % cppvar) - - -def CheckForBadCharacters(filename, lines, error): - """Logs an error for each line containing bad characters. - - Two kinds of bad characters: - - 1. Unicode replacement characters: These indicate that either the file - contained invalid UTF-8 (likely) or Unicode replacement characters (which - it shouldn't). Note that it's possible for this to throw off line - numbering if the invalid UTF-8 occurred adjacent to a newline. - - 2. NUL bytes. These are problematic for some tools. - - Args: - filename: The name of the current file. - lines: An array of strings, each representing a line of the file. - error: The function to call with any errors found. - """ - for linenum, line in enumerate(lines): - if u'\ufffd' in line: - error(filename, linenum, 'readability/utf8', 5, - 'Line contains invalid UTF-8 (or Unicode replacement character).') - if '\0' in line: - error(filename, linenum, 'readability/nul', 5, 'Line contains NUL byte.') - - -def CheckForNewlineAtEOF(filename, lines, error): - """Logs an error if there is no newline char at the end of the file. - - Args: - filename: The name of the current file. - lines: An array of strings, each representing a line of the file. - error: The function to call with any errors found. - """ - - # The array lines() was created by adding two newlines to the - # original file (go figure), then splitting on \n. - # To verify that the file ends in \n, we just have to make sure the - # last-but-two element of lines() exists and is empty. - if len(lines) < 3 or lines[-2]: - error(filename, len(lines) - 2, 'whitespace/ending_newline', 5, - 'Could not find a newline character at the end of the file.') - - -def CheckForMultilineCommentsAndStrings(filename, clean_lines, linenum, error): - """Logs an error if we see /* ... */ or "..." that extend past one line. - - /* ... */ comments are legit inside macros, for one line. - Otherwise, we prefer // comments, so it's ok to warn about the - other. Likewise, it's ok for strings to extend across multiple - lines, as long as a line continuation character (backslash) - terminates each line. Although not currently prohibited by the C++ - style guide, it's ugly and unnecessary. We don't do well with either - in this lint program, so we warn about both. - - Args: - filename: The name of the current file. - clean_lines: A CleansedLines instance containing the file. - linenum: The number of the line to check. - error: The function to call with any errors found. - """ - line = clean_lines.elided[linenum] - - # Remove all \\ (escaped backslashes) from the line. They are OK, and the - # second (escaped) slash may trigger later \" detection erroneously. - line = line.replace('\\\\', '') - - if line.count('/*') > line.count('*/'): - error(filename, linenum, 'readability/multiline_comment', 5, - 'Complex multi-line /*...*/-style comment found. ' - 'Lint may give bogus warnings. ' - 'Consider replacing these with //-style comments, ' - 'with #if 0...#endif, ' - 'or with more clearly structured multi-line comments.') - - if (line.count('"') - line.count('\\"')) % 2: - error(filename, linenum, 'readability/multiline_string', 5, - 'Multi-line string ("...") found. This lint script doesn\'t ' - 'do well with such strings, and may give bogus warnings. ' - 'Use C++11 raw strings or concatenation instead.') - - -threading_list = ( - ('asctime(', 'asctime_r('), - ('ctime(', 'ctime_r('), - ('getgrgid(', 'getgrgid_r('), - ('getgrnam(', 'getgrnam_r('), - ('getlogin(', 'getlogin_r('), - ('getpwnam(', 'getpwnam_r('), - ('getpwuid(', 'getpwuid_r('), - ('gmtime(', 'gmtime_r('), - ('localtime(', 'localtime_r('), - ('rand(', 'rand_r('), - ('strtok(', 'strtok_r('), - ('ttyname(', 'ttyname_r('), - ) - - -def CheckPosixThreading(filename, clean_lines, linenum, error): - """Checks for calls to thread-unsafe functions. - - Much code has been originally written without consideration of - multi-threading. Also, engineers are relying on their old experience; - they have learned posix before threading extensions were added. These - tests guide the engineers to use thread-safe functions (when using - posix directly). - - Args: - filename: The name of the current file. - clean_lines: A CleansedLines instance containing the file. - linenum: The number of the line to check. - error: The function to call with any errors found. - """ - line = clean_lines.elided[linenum] - for single_thread_function, multithread_safe_function in threading_list: - ix = line.find(single_thread_function) - # Comparisons made explicit for clarity -- pylint: disable=g-explicit-bool-comparison - if ix >= 0 and (ix == 0 or (not line[ix - 1].isalnum() and - line[ix - 1] not in ('_', '.', '>'))): - error(filename, linenum, 'runtime/threadsafe_fn', 2, - 'Consider using ' + multithread_safe_function + - '...) instead of ' + single_thread_function + - '...) for improved thread safety.') - - -def CheckVlogArguments(filename, clean_lines, linenum, error): - """Checks that VLOG() is only used for defining a logging level. - - For example, VLOG(2) is correct. VLOG(INFO), VLOG(WARNING), VLOG(ERROR), and - VLOG(FATAL) are not. - - Args: - filename: The name of the current file. - clean_lines: A CleansedLines instance containing the file. - linenum: The number of the line to check. - error: The function to call with any errors found. - """ - line = clean_lines.elided[linenum] - if Search(r'\bVLOG\((INFO|ERROR|WARNING|DFATAL|FATAL)\)', line): - error(filename, linenum, 'runtime/vlog', 5, - 'VLOG() should be used with numeric verbosity level. ' - 'Use LOG() if you want symbolic severity levels.') - - -# Matches invalid increment: *count++, which moves pointer instead of -# incrementing a value. -_RE_PATTERN_INVALID_INCREMENT = re.compile( - r'^\s*\*\w+(\+\+|--);') - - -def CheckInvalidIncrement(filename, clean_lines, linenum, error): - """Checks for invalid increment *count++. - - For example following function: - void increment_counter(int* count) { - *count++; - } - is invalid, because it effectively does count++, moving pointer, and should - be replaced with ++*count, (*count)++ or *count += 1. - - Args: - filename: The name of the current file. - clean_lines: A CleansedLines instance containing the file. - linenum: The number of the line to check. - error: The function to call with any errors found. - """ - line = clean_lines.elided[linenum] - if _RE_PATTERN_INVALID_INCREMENT.match(line): - error(filename, linenum, 'runtime/invalid_increment', 5, - 'Changing pointer instead of value (or unused value of operator*).') - - -class _BlockInfo(object): - """Stores information about a generic block of code.""" - - def __init__(self, seen_open_brace): - self.seen_open_brace = seen_open_brace - self.open_parentheses = 0 - self.inline_asm = _NO_ASM - - def CheckBegin(self, filename, clean_lines, linenum, error): - """Run checks that applies to text up to the opening brace. - - This is mostly for checking the text after the class identifier - and the "{", usually where the base class is specified. For other - blocks, there isn't much to check, so we always pass. - - Args: - filename: The name of the current file. - clean_lines: A CleansedLines instance containing the file. - linenum: The number of the line to check. - error: The function to call with any errors found. - """ - pass - - def CheckEnd(self, filename, clean_lines, linenum, error): - """Run checks that applies to text after the closing brace. - - This is mostly used for checking end of namespace comments. - - Args: - filename: The name of the current file. - clean_lines: A CleansedLines instance containing the file. - linenum: The number of the line to check. - error: The function to call with any errors found. - """ - pass - - -class _ClassInfo(_BlockInfo): - """Stores information about a class.""" - - def __init__(self, name, class_or_struct, clean_lines, linenum): - _BlockInfo.__init__(self, False) - self.name = name - self.starting_linenum = linenum - self.is_derived = False - if class_or_struct == 'struct': - self.access = 'public' - self.is_struct = True - else: - self.access = 'private' - self.is_struct = False - - # Remember initial indentation level for this class. Using raw_lines here - # instead of elided to account for leading comments. - initial_indent = Match(r'^( *)\S', clean_lines.raw_lines[linenum]) - if initial_indent: - self.class_indent = len(initial_indent.group(1)) - else: - self.class_indent = 0 - - # Try to find the end of the class. This will be confused by things like: - # class A { - # } *x = { ... - # - # But it's still good enough for CheckSectionSpacing. - self.last_line = 0 - depth = 0 - for i in range(linenum, clean_lines.NumLines()): - line = clean_lines.elided[i] - depth += line.count('{') - line.count('}') - if not depth: - self.last_line = i - break - - def CheckBegin(self, filename, clean_lines, linenum, error): - # Look for a bare ':' - if Search('(^|[^:]):($|[^:])', clean_lines.elided[linenum]): - self.is_derived = True - - def CheckEnd(self, filename, clean_lines, linenum, error): - # Check that closing brace is aligned with beginning of the class. - # Only do this if the closing brace is indented by only whitespaces. - # This means we will not check single-line class definitions. - indent = Match(r'^( *)\}', clean_lines.elided[linenum]) - if indent and len(indent.group(1)) != self.class_indent: - if self.is_struct: - parent = 'struct ' + self.name - else: - parent = 'class ' + self.name - error(filename, linenum, 'whitespace/indent', 3, - 'Closing brace should be aligned with beginning of %s' % parent) - - -class _NamespaceInfo(_BlockInfo): - """Stores information about a namespace.""" - - def __init__(self, name, linenum): - _BlockInfo.__init__(self, False) - self.name = name or '' - self.starting_linenum = linenum - - def CheckEnd(self, filename, clean_lines, linenum, error): - """Check end of namespace comments.""" - line = clean_lines.raw_lines[linenum] - - # Check how many lines is enclosed in this namespace. Don't issue - # warning for missing namespace comments if there aren't enough - # lines. However, do apply checks if there is already an end of - # namespace comment and it's incorrect. - # - # TODO(unknown): We always want to check end of namespace comments - # if a namespace is large, but sometimes we also want to apply the - # check if a short namespace contained nontrivial things (something - # other than forward declarations). There is currently no logic on - # deciding what these nontrivial things are, so this check is - # triggered by namespace size only, which works most of the time. - if (linenum - self.starting_linenum < 10 - and not Match(r'};*\s*(//|/\*).*\bnamespace\b', line)): - return - - # Look for matching comment at end of namespace. - # - # Note that we accept C style "/* */" comments for terminating - # namespaces, so that code that terminate namespaces inside - # preprocessor macros can be cpplint clean. - # - # We also accept stuff like "// end of namespace <name>." with the - # period at the end. - # - # Besides these, we don't accept anything else, otherwise we might - # get false negatives when existing comment is a substring of the - # expected namespace. - if self.name: - # Named namespace - if not Match((r'};*\s*(//|/\*).*\bnamespace\s+' + re.escape(self.name) + - r'[\*/\.\\\s]*$'), - line): - error(filename, linenum, 'readability/namespace', 5, - 'Namespace should be terminated with "// namespace %s"' % - self.name) - else: - # Anonymous namespace - if not Match(r'};*\s*(//|/\*).*\bnamespace[\*/\.\\\s]*$', line): - error(filename, linenum, 'readability/namespace', 5, - 'Namespace should be terminated with "// namespace"') - - -class _PreprocessorInfo(object): - """Stores checkpoints of nesting stacks when #if/#else is seen.""" - - def __init__(self, stack_before_if): - # The entire nesting stack before #if - self.stack_before_if = stack_before_if - - # The entire nesting stack up to #else - self.stack_before_else = [] - - # Whether we have already seen #else or #elif - self.seen_else = False - - -class _NestingState(object): - """Holds states related to parsing braces.""" - - def __init__(self): - # Stack for tracking all braces. An object is pushed whenever we - # see a "{", and popped when we see a "}". Only 3 types of - # objects are possible: - # - _ClassInfo: a class or struct. - # - _NamespaceInfo: a namespace. - # - _BlockInfo: some other type of block. - self.stack = [] - - # Stack of _PreprocessorInfo objects. - self.pp_stack = [] - - def SeenOpenBrace(self): - """Check if we have seen the opening brace for the innermost block. - - Returns: - True if we have seen the opening brace, False if the innermost - block is still expecting an opening brace. - """ - return (not self.stack) or self.stack[-1].seen_open_brace - - def InNamespaceBody(self): - """Check if we are currently one level inside a namespace body. - - Returns: - True if top of the stack is a namespace block, False otherwise. - """ - return self.stack and isinstance(self.stack[-1], _NamespaceInfo) - - def UpdatePreprocessor(self, line): - """Update preprocessor stack. - - We need to handle preprocessors due to classes like this: - #ifdef SWIG - struct ResultDetailsPageElementExtensionPoint { - #else - struct ResultDetailsPageElementExtensionPoint : public Extension { - #endif - - We make the following assumptions (good enough for most files): - - Preprocessor condition evaluates to true from #if up to first - #else/#elif/#endif. - - - Preprocessor condition evaluates to false from #else/#elif up - to #endif. We still perform lint checks on these lines, but - these do not affect nesting stack. - - Args: - line: current line to check. - """ - if Match(r'^\s*#\s*(if|ifdef|ifndef)\b', line): - # Beginning of #if block, save the nesting stack here. The saved - # stack will allow us to restore the parsing state in the #else case. - self.pp_stack.append(_PreprocessorInfo(copy.deepcopy(self.stack))) - elif Match(r'^\s*#\s*(else|elif)\b', line): - # Beginning of #else block - if self.pp_stack: - if not self.pp_stack[-1].seen_else: - # This is the first #else or #elif block. Remember the - # whole nesting stack up to this point. This is what we - # keep after the #endif. - self.pp_stack[-1].seen_else = True - self.pp_stack[-1].stack_before_else = copy.deepcopy(self.stack) - - # Restore the stack to how it was before the #if - self.stack = copy.deepcopy(self.pp_stack[-1].stack_before_if) - else: - # TODO(unknown): unexpected #else, issue warning? - pass - elif Match(r'^\s*#\s*endif\b', line): - # End of #if or #else blocks. - if self.pp_stack: - # If we saw an #else, we will need to restore the nesting - # stack to its former state before the #else, otherwise we - # will just continue from where we left off. - if self.pp_stack[-1].seen_else: - # Here we can just use a shallow copy since we are the last - # reference to it. - self.stack = self.pp_stack[-1].stack_before_else - # Drop the corresponding #if - self.pp_stack.pop() - else: - # TODO(unknown): unexpected #endif, issue warning? - pass - - def Update(self, filename, clean_lines, linenum, error): - """Update nesting state with current line. - - Args: - filename: The name of the current file. - clean_lines: A CleansedLines instance containing the file. - linenum: The number of the line to check. - error: The function to call with any errors found. - """ - line = clean_lines.elided[linenum] - - # Update pp_stack first - self.UpdatePreprocessor(line) - - # Count parentheses. This is to avoid adding struct arguments to - # the nesting stack. - if self.stack: - inner_block = self.stack[-1] - depth_change = line.count('(') - line.count(')') - inner_block.open_parentheses += depth_change - - # Also check if we are starting or ending an inline assembly block. - if inner_block.inline_asm in (_NO_ASM, _END_ASM): - if (depth_change != 0 and - inner_block.open_parentheses == 1 and - _MATCH_ASM.match(line)): - # Enter assembly block - inner_block.inline_asm = _INSIDE_ASM - else: - # Not entering assembly block. If previous line was _END_ASM, - # we will now shift to _NO_ASM state. - inner_block.inline_asm = _NO_ASM - elif (inner_block.inline_asm == _INSIDE_ASM and - inner_block.open_parentheses == 0): - # Exit assembly block - inner_block.inline_asm = _END_ASM - - # Consume namespace declaration at the beginning of the line. Do - # this in a loop so that we catch same line declarations like this: - # namespace proto2 { namespace bridge { class MessageSet; } } - while True: - # Match start of namespace. The "\b\s*" below catches namespace - # declarations even if it weren't followed by a whitespace, this - # is so that we don't confuse our namespace checker. The - # missing spaces will be flagged by CheckSpacing. - namespace_decl_match = Match(r'^\s*namespace\b\s*([:\w]+)?(.*)$', line) - if not namespace_decl_match: - break - - new_namespace = _NamespaceInfo(namespace_decl_match.group(1), linenum) - self.stack.append(new_namespace) - - line = namespace_decl_match.group(2) - if line.find('{') != -1: - new_namespace.seen_open_brace = True - line = line[line.find('{') + 1:] - - # Look for a class declaration in whatever is left of the line - # after parsing namespaces. The regexp accounts for decorated classes - # such as in: - # class LOCKABLE API Object { - # }; - # - # Templates with class arguments may confuse the parser, for example: - # template <class T - # class Comparator = less<T>, - # class Vector = vector<T> > - # class HeapQueue { - # - # Because this parser has no nesting state about templates, by the - # time it saw "class Comparator", it may think that it's a new class. - # Nested templates have a similar problem: - # template < - # typename ExportedType, - # typename TupleType, - # template <typename, typename> class ImplTemplate> - # - # To avoid these cases, we ignore classes that are followed by '=' or '>' - class_decl_match = Match( - r'\s*(template\s*<[\w\s<>,:]*>\s*)?' - r'(class|struct)\s+([A-Z_]+\s+)*(\w+(?:::\w+)*)' - r'(([^=>]|<[^<>]*>|<[^<>]*<[^<>]*>\s*>)*)$', line) - if (class_decl_match and - (not self.stack or self.stack[-1].open_parentheses == 0)): - self.stack.append(_ClassInfo( - class_decl_match.group(4), class_decl_match.group(2), - clean_lines, linenum)) - line = class_decl_match.group(5) - - # If we have not yet seen the opening brace for the innermost block, - # run checks here. - if not self.SeenOpenBrace(): - self.stack[-1].CheckBegin(filename, clean_lines, linenum, error) - - # Update access control if we are inside a class/struct - if self.stack and isinstance(self.stack[-1], _ClassInfo): - classinfo = self.stack[-1] - access_match = Match( - r'^(.*)\b(public|private|protected|signals)(\s+(?:slots\s*)?)?' - r':(?:[^:]|$)', - line) - if access_match: - classinfo.access = access_match.group(2) - - # Check that access keywords are indented +1 space. Skip this - # check if the keywords are not preceded by whitespaces. - indent = access_match.group(1) - if (len(indent) != classinfo.class_indent + 1 and - Match(r'^\s*$', indent)): - if classinfo.is_struct: - parent = 'struct ' + classinfo.name - else: - parent = 'class ' + classinfo.name - slots = '' - if access_match.group(3): - slots = access_match.group(3) - error(filename, linenum, 'whitespace/indent', 3, - '%s%s: should be indented +1 space inside %s' % ( - access_match.group(2), slots, parent)) - - # Consume braces or semicolons from what's left of the line - while True: - # Match first brace, semicolon, or closed parenthesis. - matched = Match(r'^[^{;)}]*([{;)}])(.*)$', line) - if not matched: - break - - token = matched.group(1) - if token == '{': - # If namespace or class hasn't seen a opening brace yet, mark - # namespace/class head as complete. Push a new block onto the - # stack otherwise. - if not self.SeenOpenBrace(): - self.stack[-1].seen_open_brace = True - else: - self.stack.append(_BlockInfo(True)) - if _MATCH_ASM.match(line): - self.stack[-1].inline_asm = _BLOCK_ASM - elif token == ';' or token == ')': - # If we haven't seen an opening brace yet, but we already saw - # a semicolon, this is probably a forward declaration. Pop - # the stack for these. - # - # Similarly, if we haven't seen an opening brace yet, but we - # already saw a closing parenthesis, then these are probably - # function arguments with extra "class" or "struct" keywords. - # Also pop these stack for these. - if not self.SeenOpenBrace(): - self.stack.pop() - else: # token == '}' - # Perform end of block checks and pop the stack. - if self.stack: - self.stack[-1].CheckEnd(filename, clean_lines, linenum, error) - self.stack.pop() - line = matched.group(2) - - def InnermostClass(self): - """Get class info on the top of the stack. - - Returns: - A _ClassInfo object if we are inside a class, or None otherwise. - """ - for i in range(len(self.stack), 0, -1): - classinfo = self.stack[i - 1] - if isinstance(classinfo, _ClassInfo): - return classinfo - return None - - def CheckCompletedBlocks(self, filename, error): - """Checks that all classes and namespaces have been completely parsed. - - Call this when all lines in a file have been processed. - Args: - filename: The name of the current file. - error: The function to call with any errors found. - """ - # Note: This test can result in false positives if #ifdef constructs - # get in the way of brace matching. See the testBuildClass test in - # cpplint_unittest.py for an example of this. - for obj in self.stack: - if isinstance(obj, _ClassInfo): - error(filename, obj.starting_linenum, 'build/class', 5, - 'Failed to find complete declaration of class %s' % - obj.name) - elif isinstance(obj, _NamespaceInfo): - error(filename, obj.starting_linenum, 'build/namespaces', 5, - 'Failed to find complete declaration of namespace %s' % - obj.name) - - -def CheckForNonStandardConstructs(filename, clean_lines, linenum, - nesting_state, error): - r"""Logs an error if we see certain non-ANSI constructs ignored by gcc-2. - - Complain about several constructs which gcc-2 accepts, but which are - not standard C++. Warning about these in lint is one way to ease the - transition to new compilers. - - put storage class first (e.g. "static const" instead of "const static"). - - "%lld" instead of %qd" in printf-type functions. - - "%1$d" is non-standard in printf-type functions. - - "\%" is an undefined character escape sequence. - - text after #endif is not allowed. - - invalid inner-style forward declaration. - - >? and <? operators, and their >?= and <?= cousins. - - Additionally, check for constructor/destructor style violations and reference - members, as it is very convenient to do so while checking for - gcc-2 compliance. - - Args: - filename: The name of the current file. - clean_lines: A CleansedLines instance containing the file. - linenum: The number of the line to check. - nesting_state: A _NestingState instance which maintains information about - the current stack of nested blocks being parsed. - error: A callable to which errors are reported, which takes 4 arguments: - filename, line number, error level, and message - """ - - # Remove comments from the line, but leave in strings for now. - line = clean_lines.lines[linenum] - - if Search(r'printf\s*\(.*".*%[-+ ]?\d*q', line): - error(filename, linenum, 'runtime/printf_format', 3, - '%q in format strings is deprecated. Use %ll instead.') - - if Search(r'printf\s*\(.*".*%\d+\$', line): - error(filename, linenum, 'runtime/printf_format', 2, - '%N$ formats are unconventional. Try rewriting to avoid them.') - - # Remove escaped backslashes before looking for undefined escapes. - line = line.replace('\\\\', '') - - if Search(r'("|\').*\\(%|\[|\(|{)', line): - error(filename, linenum, 'build/printf_format', 3, - '%, [, (, and { are undefined character escapes. Unescape them.') - - # For the rest, work with both comments and strings removed. - line = clean_lines.elided[linenum] - - if Search(r'\b(const|volatile|void|char|short|int|long' - r'|float|double|signed|unsigned' - r'|schar|u?int8|u?int16|u?int32|u?int64)' - r'\s+(register|static|extern|typedef)\b', - line): - error(filename, linenum, 'build/storage_class', 5, - 'Storage class (static, extern, typedef, etc) should be first.') - - if Match(r'\s*#\s*endif\s*[^/\s]+', line): - error(filename, linenum, 'build/endif_comment', 5, - 'Uncommented text after #endif is non-standard. Use a comment.') - - if Match(r'\s*class\s+(\w+\s*::\s*)+\w+\s*;', line): - error(filename, linenum, 'build/forward_decl', 5, - 'Inner-style forward declarations are invalid. Remove this line.') - - if Search(r'(\w+|[+-]?\d+(\.\d*)?)\s*(<|>)\?=?\s*(\w+|[+-]?\d+)(\.\d*)?', - line): - error(filename, linenum, 'build/deprecated', 3, - '>? and <? (max and min) operators are non-standard and deprecated.') - - if Search(r'^\s*const\s*string\s*&\s*\w+\s*;', line): - # TODO(unknown): Could it be expanded safely to arbitrary references, - # without triggering too many false positives? The first - # attempt triggered 5 warnings for mostly benign code in the regtest, hence - # the restriction. - # Here's the original regexp, for the reference: - # type_name = r'\w+((\s*::\s*\w+)|(\s*<\s*\w+?\s*>))?' - # r'\s*const\s*' + type_name + '\s*&\s*\w+\s*;' - error(filename, linenum, 'runtime/member_string_references', 2, - 'const string& members are dangerous. It is much better to use ' - 'alternatives, such as pointers or simple constants.') - - # Everything else in this function operates on class declarations. - # Return early if the top of the nesting stack is not a class, or if - # the class head is not completed yet. - classinfo = nesting_state.InnermostClass() - if not classinfo or not classinfo.seen_open_brace: - return - - # The class may have been declared with namespace or classname qualifiers. - # The constructor and destructor will not have those qualifiers. - base_classname = classinfo.name.split('::')[-1] - - # Look for single-argument constructors that aren't marked explicit. - # Technically a valid construct, but against style. - args = Match(r'\s+(?:inline\s+)?%s\s*\(([^,()]+)\)' - % re.escape(base_classname), - line) - if (args and - args.group(1) != 'void' and - not Match(r'(const\s+)?%s(\s+const)?\s*(?:<\w+>\s*)?&' - % re.escape(base_classname), args.group(1).strip())): - error(filename, linenum, 'runtime/explicit', 5, - 'Single-argument constructors should be marked explicit.') - - -def CheckSpacingForFunctionCall(filename, line, linenum, error): - """Checks for the correctness of various spacing around function calls. - - Args: - filename: The name of the current file. - line: The text of the line to check. - linenum: The number of the line to check. - error: The function to call with any errors found. - """ - - # Since function calls often occur inside if/for/while/switch - # expressions - which have their own, more liberal conventions - we - # first see if we should be looking inside such an expression for a - # function call, to which we can apply more strict standards. - fncall = line # if there's no control flow construct, look at whole line - for pattern in (r'\bif\s*\((.*)\)\s*{', - r'\bfor\s*\((.*)\)\s*{', - r'\bwhile\s*\((.*)\)\s*[{;]', - r'\bswitch\s*\((.*)\)\s*{'): - match = Search(pattern, line) - if match: - fncall = match.group(1) # look inside the parens for function calls - break - - # Except in if/for/while/switch, there should never be space - # immediately inside parens (eg "f( 3, 4 )"). We make an exception - # for nested parens ( (a+b) + c ). Likewise, there should never be - # a space before a ( when it's a function argument. I assume it's a - # function argument when the char before the whitespace is legal in - # a function name (alnum + _) and we're not starting a macro. Also ignore - # pointers and references to arrays and functions coz they're too tricky: - # we use a very simple way to recognize these: - # " (something)(maybe-something)" or - # " (something)(maybe-something," or - # " (something)[something]" - # Note that we assume the contents of [] to be short enough that - # they'll never need to wrap. - if ( # Ignore control structures. - not Search(r'\b(if|for|while|switch|return|new|delete|catch|sizeof)\b', - fncall) and - # Ignore pointers/references to functions. - not Search(r' \([^)]+\)\([^)]*(\)|,$)', fncall) and - # Ignore pointers/references to arrays. - not Search(r' \([^)]+\)\[[^\]]+\]', fncall)): - if Search(r'\w\s*\(\s(?!\s*\\$)', fncall): # a ( used for a fn call - error(filename, linenum, 'whitespace/parens', 4, - 'Extra space after ( in function call') - elif Search(r'\(\s+(?!(\s*\\)|\()', fncall): - error(filename, linenum, 'whitespace/parens', 2, - 'Extra space after (') - if (Search(r'\w\s+\(', fncall) and - not Search(r'#\s*define|typedef', fncall) and - not Search(r'\w\s+\((\w+::)*\*\w+\)\(', fncall)): - error(filename, linenum, 'whitespace/parens', 4, - 'Extra space before ( in function call') - # If the ) is followed only by a newline or a { + newline, assume it's - # part of a control statement (if/while/etc), and don't complain - if Search(r'[^)]\s+\)\s*[^{\s]', fncall): - # If the closing parenthesis is preceded by only whitespaces, - # try to give a more descriptive error message. - if Search(r'^\s+\)', fncall): - error(filename, linenum, 'whitespace/parens', 2, - 'Closing ) should be moved to the previous line') - else: - error(filename, linenum, 'whitespace/parens', 2, - 'Extra space before )') - - -def IsBlankLine(line): - """Returns true if the given line is blank. - - We consider a line to be blank if the line is empty or consists of - only white spaces. - - Args: - line: A line of a string. - - Returns: - True, if the given line is blank. - """ - return not line or line.isspace() - - -def CheckForFunctionLengths(filename, clean_lines, linenum, - function_state, error): - """Reports for long function bodies. - - For an overview why this is done, see: - http://google-styleguide.googlecode.com/svn/trunk/cppguide.xml#Write_Short_Functions - - Uses a simplistic algorithm assuming other style guidelines - (especially spacing) are followed. - Only checks unindented functions, so class members are unchecked. - Trivial bodies are unchecked, so constructors with huge initializer lists - may be missed. - Blank/comment lines are not counted so as to avoid encouraging the removal - of vertical space and comments just to get through a lint check. - NOLINT *on the last line of a function* disables this check. - - Args: - filename: The name of the current file. - clean_lines: A CleansedLines instance containing the file. - linenum: The number of the line to check. - function_state: Current function name and lines in body so far. - error: The function to call with any errors found. - """ - lines = clean_lines.lines - line = lines[linenum] - raw = clean_lines.raw_lines - raw_line = raw[linenum] - joined_line = '' - - starting_func = False - regexp = r'(\w(\w|::|\*|\&|\s)*)\(' # decls * & space::name( ... - match_result = Match(regexp, line) - if match_result: - # If the name is all caps and underscores, figure it's a macro and - # ignore it, unless it's TEST or TEST_F. - function_name = match_result.group(1).split()[-1] - if function_name == 'TEST' or function_name == 'TEST_F' or ( - not Match(r'[A-Z_]+$', function_name)): - starting_func = True - - if starting_func: - body_found = False - for start_linenum in xrange(linenum, clean_lines.NumLines()): - start_line = lines[start_linenum] - joined_line += ' ' + start_line.lstrip() - if Search(r'(;|})', start_line): # Declarations and trivial functions - body_found = True - break # ... ignore - elif Search(r'{', start_line): - body_found = True - function = Search(r'((\w|:)*)\(', line).group(1) - if Match(r'TEST', function): # Handle TEST... macros - parameter_regexp = Search(r'(\(.*\))', joined_line) - if parameter_regexp: # Ignore bad syntax - function += parameter_regexp.group(1) - else: - function += '()' - function_state.Begin(function) - break - if not body_found: - # No body for the function (or evidence of a non-function) was found. - error(filename, linenum, 'readability/fn_size', 5, - 'Lint failed to find start of function body.') - elif Match(r'^\}\s*$', line): # function end - function_state.Check(error, filename, linenum) - function_state.End() - elif not Match(r'^\s*$', line): - function_state.Count() # Count non-blank/non-comment lines. - - -_RE_PATTERN_TODO = re.compile(r'^//(\s*)TODO(\(.+?\))?:?(\s|$)?') - - -def CheckComment(comment, filename, linenum, error): - """Checks for common mistakes in TODO comments. - - Args: - comment: The text of the comment from the line in question. - filename: The name of the current file. - linenum: The number of the line to check. - error: The function to call with any errors found. - """ - match = _RE_PATTERN_TODO.match(comment) - if match: - # One whitespace is correct; zero whitespace is handled elsewhere. - leading_whitespace = match.group(1) - if len(leading_whitespace) > 1: - error(filename, linenum, 'whitespace/todo', 2, - 'Too many spaces before TODO') - - username = match.group(2) - if not username: - error(filename, linenum, 'readability/todo', 2, - 'Missing username in TODO; it should look like ' - '"// TODO(my_username): Stuff."') - - middle_whitespace = match.group(3) - # Comparisons made explicit for correctness -- pylint: disable=g-explicit-bool-comparison - if middle_whitespace != ' ' and middle_whitespace != '': - error(filename, linenum, 'whitespace/todo', 2, - 'TODO(my_username) should be followed by a space') - -def CheckAccess(filename, clean_lines, linenum, nesting_state, error): - """Checks for improper use of DISALLOW* macros. - - Args: - filename: The name of the current file. - clean_lines: A CleansedLines instance containing the file. - linenum: The number of the line to check. - nesting_state: A _NestingState instance which maintains information about - the current stack of nested blocks being parsed. - error: The function to call with any errors found. - """ - line = clean_lines.elided[linenum] # get rid of comments and strings - - matched = Match((r'\s*(DISALLOW_COPY_AND_ASSIGN|' - r'DISALLOW_EVIL_CONSTRUCTORS|' - r'DISALLOW_IMPLICIT_CONSTRUCTORS)'), line) - if not matched: - return - if nesting_state.stack and isinstance(nesting_state.stack[-1], _ClassInfo): - if nesting_state.stack[-1].access != 'private': - error(filename, linenum, 'readability/constructors', 3, - '%s must be in the private: section' % matched.group(1)) - - else: - # Found DISALLOW* macro outside a class declaration, or perhaps it - # was used inside a function when it should have been part of the - # class declaration. We could issue a warning here, but it - # probably resulted in a compiler error already. - pass - - -def FindNextMatchingAngleBracket(clean_lines, linenum, init_suffix): - """Find the corresponding > to close a template. - - Args: - clean_lines: A CleansedLines instance containing the file. - linenum: Current line number. - init_suffix: Remainder of the current line after the initial <. - - Returns: - True if a matching bracket exists. - """ - line = init_suffix - nesting_stack = ['<'] - while True: - # Find the next operator that can tell us whether < is used as an - # opening bracket or as a less-than operator. We only want to - # warn on the latter case. - # - # We could also check all other operators and terminate the search - # early, e.g. if we got something like this "a<b+c", the "<" is - # most likely a less-than operator, but then we will get false - # positives for default arguments and other template expressions. - match = Search(r'^[^<>(),;\[\]]*([<>(),;\[\]])(.*)$', line) - if match: - # Found an operator, update nesting stack - operator = match.group(1) - line = match.group(2) - - if nesting_stack[-1] == '<': - # Expecting closing angle bracket - if operator in ('<', '(', '['): - nesting_stack.append(operator) - elif operator == '>': - nesting_stack.pop() - if not nesting_stack: - # Found matching angle bracket - return True - elif operator == ',': - # Got a comma after a bracket, this is most likely a template - # argument. We have not seen a closing angle bracket yet, but - # it's probably a few lines later if we look for it, so just - # return early here. - return True - else: - # Got some other operator. - return False - - else: - # Expecting closing parenthesis or closing bracket - if operator in ('<', '(', '['): - nesting_stack.append(operator) - elif operator in (')', ']'): - # We don't bother checking for matching () or []. If we got - # something like (] or [), it would have been a syntax error. - nesting_stack.pop() - - else: - # Scan the next line - linenum += 1 - if linenum >= len(clean_lines.elided): - break - line = clean_lines.elided[linenum] - - # Exhausted all remaining lines and still no matching angle bracket. - # Most likely the input was incomplete, otherwise we should have - # seen a semicolon and returned early. - return True - - -def FindPreviousMatchingAngleBracket(clean_lines, linenum, init_prefix): - """Find the corresponding < that started a template. - - Args: - clean_lines: A CleansedLines instance containing the file. - linenum: Current line number. - init_prefix: Part of the current line before the initial >. - - Returns: - True if a matching bracket exists. - """ - line = init_prefix - nesting_stack = ['>'] - while True: - # Find the previous operator - match = Search(r'^(.*)([<>(),;\[\]])[^<>(),;\[\]]*$', line) - if match: - # Found an operator, update nesting stack - operator = match.group(2) - line = match.group(1) - - if nesting_stack[-1] == '>': - # Expecting opening angle bracket - if operator in ('>', ')', ']'): - nesting_stack.append(operator) - elif operator == '<': - nesting_stack.pop() - if not nesting_stack: - # Found matching angle bracket - return True - elif operator == ',': - # Got a comma before a bracket, this is most likely a - # template argument. The opening angle bracket is probably - # there if we look for it, so just return early here. - return True - else: - # Got some other operator. - return False - - else: - # Expecting opening parenthesis or opening bracket - if operator in ('>', ')', ']'): - nesting_stack.append(operator) - elif operator in ('(', '['): - nesting_stack.pop() - - else: - # Scan the previous line - linenum -= 1 - if linenum < 0: - break - line = clean_lines.elided[linenum] - - # Exhausted all earlier lines and still no matching angle bracket. - return False - - -def CheckSpacing(filename, clean_lines, linenum, nesting_state, error): - """Checks for the correctness of various spacing issues in the code. - - Things we check for: spaces around operators, spaces after - if/for/while/switch, no spaces around parens in function calls, two - spaces between code and comment, don't start a block with a blank - line, don't end a function with a blank line, don't add a blank line - after public/protected/private, don't have too many blank lines in a row. - - Args: - filename: The name of the current file. - clean_lines: A CleansedLines instance containing the file. - linenum: The number of the line to check. - nesting_state: A _NestingState instance which maintains information about - the current stack of nested blocks being parsed. - error: The function to call with any errors found. - """ - - # Don't use "elided" lines here, otherwise we can't check commented lines. - # Don't want to use "raw" either, because we don't want to check inside C++11 - # raw strings, - raw = clean_lines.lines_without_raw_strings - line = raw[linenum] - - # Before nixing comments, check if the line is blank for no good - # reason. This includes the first line after a block is opened, and - # blank lines at the end of a function (ie, right before a line like '}' - # - # Skip all the blank line checks if we are immediately inside a - # namespace body. In other words, don't issue blank line warnings - # for this block: - # namespace { - # - # } - # - # A warning about missing end of namespace comments will be issued instead. - if IsBlankLine(line) and not nesting_state.InNamespaceBody(): - elided = clean_lines.elided - prev_line = elided[linenum - 1] - prevbrace = prev_line.rfind('{') - # TODO(unknown): Don't complain if line before blank line, and line after, - # both start with alnums and are indented the same amount. - # This ignores whitespace at the start of a namespace block - # because those are not usually indented. - if prevbrace != -1 and prev_line[prevbrace:].find('}') == -1: - # OK, we have a blank line at the start of a code block. Before we - # complain, we check if it is an exception to the rule: The previous - # non-empty line has the parameters of a function header that are indented - # 4 spaces (because they did not fit in a 80 column line when placed on - # the same line as the function name). We also check for the case where - # the previous line is indented 6 spaces, which may happen when the - # initializers of a constructor do not fit into a 80 column line. - exception = False - if Match(r' {6}\w', prev_line): # Initializer list? - # We are looking for the opening column of initializer list, which - # should be indented 4 spaces to cause 6 space indentation afterwards. - search_position = linenum-2 - while (search_position >= 0 - and Match(r' {6}\w', elided[search_position])): - search_position -= 1 - exception = (search_position >= 0 - and elided[search_position][:5] == ' :') - else: - # Search for the function arguments or an initializer list. We use a - # simple heuristic here: If the line is indented 4 spaces; and we have a - # closing paren, without the opening paren, followed by an opening brace - # or colon (for initializer lists) we assume that it is the last line of - # a function header. If we have a colon indented 4 spaces, it is an - # initializer list. - exception = (Match(r' {4}\w[^\(]*\)\s*(const\s*)?(\{\s*$|:)', - prev_line) - or Match(r' {4}:', prev_line)) - - if not exception: - error(filename, linenum, 'whitespace/blank_line', 2, - 'Redundant blank line at the start of a code block ' - 'should be deleted.') - # Ignore blank lines at the end of a block in a long if-else - # chain, like this: - # if (condition1) { - # // Something followed by a blank line - # - # } else if (condition2) { - # // Something else - # } - if linenum + 1 < clean_lines.NumLines(): - next_line = raw[linenum + 1] - if (next_line - and Match(r'\s*}', next_line) - and next_line.find('} else ') == -1): - error(filename, linenum, 'whitespace/blank_line', 3, - 'Redundant blank line at the end of a code block ' - 'should be deleted.') - - matched = Match(r'\s*(public|protected|private):', prev_line) - if matched: - error(filename, linenum, 'whitespace/blank_line', 3, - 'Do not leave a blank line after "%s:"' % matched.group(1)) - - # Next, we complain if there's a comment too near the text - commentpos = line.find('//') - if commentpos != -1: - # Check if the // may be in quotes. If so, ignore it - # Comparisons made explicit for clarity -- pylint: disable=g-explicit-bool-comparison - if (line.count('"', 0, commentpos) - - line.count('\\"', 0, commentpos)) % 2 == 0: # not in quotes - # Allow one space for new scopes, two spaces otherwise: - if (not Match(r'^\s*{ //', line) and - ((commentpos >= 1 and - line[commentpos-1] not in string.whitespace) or - (commentpos >= 2 and - line[commentpos-2] not in string.whitespace))): - error(filename, linenum, 'whitespace/comments', 2, - 'At least two spaces is best between code and comments') - # There should always be a space between the // and the comment - commentend = commentpos + 2 - if commentend < len(line) and not line[commentend] == ' ': - # but some lines are exceptions -- e.g. if they're big - # comment delimiters like: - # //---------------------------------------------------------- - # or are an empty C++ style Doxygen comment, like: - # /// - # or C++ style Doxygen comments placed after the variable: - # ///< Header comment - # //!< Header comment - # or they begin with multiple slashes followed by a space: - # //////// Header comment - match = (Search(r'[=/-]{4,}\s*$', line[commentend:]) or - Search(r'^/$', line[commentend:]) or - Search(r'^!< ', line[commentend:]) or - Search(r'^/< ', line[commentend:]) or - Search(r'^/+ ', line[commentend:])) - if not match: - error(filename, linenum, 'whitespace/comments', 4, - 'Should have a space between // and comment') - CheckComment(line[commentpos:], filename, linenum, error) - - line = clean_lines.elided[linenum] # get rid of comments and strings - - # Don't try to do spacing checks for operator methods - line = re.sub(r'operator(==|!=|<|<<|<=|>=|>>|>)\(', 'operator\(', line) - - # We allow no-spaces around = within an if: "if ( (a=Foo()) == 0 )". - # Otherwise not. Note we only check for non-spaces on *both* sides; - # sometimes people put non-spaces on one side when aligning ='s among - # many lines (not that this is behavior that I approve of...) - if Search(r'[\w.]=[\w.]', line) and not Search(r'\b(if|while) ', line): - error(filename, linenum, 'whitespace/operators', 4, - 'Missing spaces around =') - - # It's ok not to have spaces around binary operators like + - * /, but if - # there's too little whitespace, we get concerned. It's hard to tell, - # though, so we punt on this one for now. TODO. - - # You should always have whitespace around binary operators. - # - # Check <= and >= first to avoid false positives with < and >, then - # check non-include lines for spacing around < and >. - match = Search(r'[^<>=!\s](==|!=|<=|>=)[^<>=!\s]', line) - if match: - error(filename, linenum, 'whitespace/operators', 3, - 'Missing spaces around %s' % match.group(1)) - # We allow no-spaces around << when used like this: 10<<20, but - # not otherwise (particularly, not when used as streams) - # Also ignore using ns::operator<<; - match = Search(r'(operator|\S)(?:L|UL|ULL|l|ul|ull)?<<(\S)', line) - if (match and - not (match.group(1).isdigit() and match.group(2).isdigit()) and - not (match.group(1) == 'operator' and match.group(2) == ';')): - error(filename, linenum, 'whitespace/operators', 3, - 'Missing spaces around <<') - elif not Match(r'#.*include', line): - # Avoid false positives on -> - reduced_line = line.replace('->', '') - - # Look for < that is not surrounded by spaces. This is only - # triggered if both sides are missing spaces, even though - # technically should should flag if at least one side is missing a - # space. This is done to avoid some false positives with shifts. - match = Search(r'[^\s<]<([^\s=<].*)', reduced_line) - if (match and - not FindNextMatchingAngleBracket(clean_lines, linenum, match.group(1))): - error(filename, linenum, 'whitespace/operators', 3, - 'Missing spaces around <') - - # Look for > that is not surrounded by spaces. Similar to the - # above, we only trigger if both sides are missing spaces to avoid - # false positives with shifts. - match = Search(r'^(.*[^\s>])>[^\s=>]', reduced_line) - if (match and - not FindPreviousMatchingAngleBracket(clean_lines, linenum, - match.group(1))): - error(filename, linenum, 'whitespace/operators', 3, - 'Missing spaces around >') - - # We allow no-spaces around >> for almost anything. This is because - # C++11 allows ">>" to close nested templates, which accounts for - # most cases when ">>" is not followed by a space. - # - # We still warn on ">>" followed by alpha character, because that is - # likely due to ">>" being used for right shifts, e.g.: - # value >> alpha - # - # When ">>" is used to close templates, the alphanumeric letter that - # follows would be part of an identifier, and there should still be - # a space separating the template type and the identifier. - # type<type<type>> alpha - match = Search(r'>>[a-zA-Z_]', line) - if match: - error(filename, linenum, 'whitespace/operators', 3, - 'Missing spaces around >>') - - # There shouldn't be space around unary operators - match = Search(r'(!\s|~\s|[\s]--[\s;]|[\s]\+\+[\s;])', line) - if match: - error(filename, linenum, 'whitespace/operators', 4, - 'Extra space for operator %s' % match.group(1)) - - # A pet peeve of mine: no spaces after an if, while, switch, or for - match = Search(r' (if\(|for\(|while\(|switch\()', line) - if match: - error(filename, linenum, 'whitespace/parens', 5, - 'Missing space before ( in %s' % match.group(1)) - - # For if/for/while/switch, the left and right parens should be - # consistent about how many spaces are inside the parens, and - # there should either be zero or one spaces inside the parens. - # We don't want: "if ( foo)" or "if ( foo )". - # Exception: "for ( ; foo; bar)" and "for (foo; bar; )" are allowed. - match = Search(r'\b(if|for|while|switch)\s*' - r'\(([ ]*)(.).*[^ ]+([ ]*)\)\s*{\s*$', - line) - if match: - if len(match.group(2)) != len(match.group(4)): - if not (match.group(3) == ';' and - len(match.group(2)) == 1 + len(match.group(4)) or - not match.group(2) and Search(r'\bfor\s*\(.*; \)', line)): - error(filename, linenum, 'whitespace/parens', 5, - 'Mismatching spaces inside () in %s' % match.group(1)) - if len(match.group(2)) not in [0, 1]: - error(filename, linenum, 'whitespace/parens', 5, - 'Should have zero or one spaces inside ( and ) in %s' % - match.group(1)) - - # You should always have a space after a comma (either as fn arg or operator) - # - # This does not apply when the non-space character following the - # comma is another comma, since the only time when that happens is - # for empty macro arguments. - # - # We run this check in two passes: first pass on elided lines to - # verify that lines contain missing whitespaces, second pass on raw - # lines to confirm that those missing whitespaces are not due to - # elided comments. - if Search(r',[^,\s]', line) and Search(r',[^,\s]', raw[linenum]): - error(filename, linenum, 'whitespace/comma', 3, - 'Missing space after ,') - - # You should always have a space after a semicolon - # except for few corner cases - # TODO(unknown): clarify if 'if (1) { return 1;}' is requires one more - # space after ; - if Search(r';[^\s};\\)/]', line): - error(filename, linenum, 'whitespace/semicolon', 3, - 'Missing space after ;') - - # Next we will look for issues with function calls. - CheckSpacingForFunctionCall(filename, line, linenum, error) - - # Except after an opening paren, or after another opening brace (in case of - # an initializer list, for instance), you should have spaces before your - # braces. And since you should never have braces at the beginning of a line, - # this is an easy test. - match = Match(r'^(.*[^ ({]){', line) - if match: - # Try a bit harder to check for brace initialization. This - # happens in one of the following forms: - # Constructor() : initializer_list_{} { ... } - # Constructor{}.MemberFunction() - # Type variable{}; - # FunctionCall(type{}, ...); - # LastArgument(..., type{}); - # LOG(INFO) << type{} << " ..."; - # map_of_type[{...}] = ...; - # - # We check for the character following the closing brace, and - # silence the warning if it's one of those listed above, i.e. - # "{.;,)<]". - # - # To account for nested initializer list, we allow any number of - # closing braces up to "{;,)<". We can't simply silence the - # warning on first sight of closing brace, because that would - # cause false negatives for things that are not initializer lists. - # Silence this: But not this: - # Outer{ if (...) { - # Inner{...} if (...){ // Missing space before { - # }; } - # - # There is a false negative with this approach if people inserted - # spurious semicolons, e.g. "if (cond){};", but we will catch the - # spurious semicolon with a separate check. - (endline, endlinenum, endpos) = CloseExpression( - clean_lines, linenum, len(match.group(1))) - trailing_text = '' - if endpos > -1: - trailing_text = endline[endpos:] - for offset in xrange(endlinenum + 1, - min(endlinenum + 3, clean_lines.NumLines() - 1)): - trailing_text += clean_lines.elided[offset] - if not Match(r'^[\s}]*[{.;,)<\]]', trailing_text): - error(filename, linenum, 'whitespace/braces', 5, - 'Missing space before {') - - # Make sure '} else {' has spaces. - if Search(r'}else', line): - error(filename, linenum, 'whitespace/braces', 5, - 'Missing space before else') - - # You shouldn't have spaces before your brackets, except maybe after - # 'delete []' or 'new char * []'. - if Search(r'\w\s+\[', line) and not Search(r'delete\s+\[', line): - error(filename, linenum, 'whitespace/braces', 5, - 'Extra space before [') - - # You shouldn't have a space before a semicolon at the end of the line. - # There's a special case for "for" since the style guide allows space before - # the semicolon there. - if Search(r':\s*;\s*$', line): - error(filename, linenum, 'whitespace/semicolon', 5, - 'Semicolon defining empty statement. Use {} instead.') - elif Search(r'^\s*;\s*$', line): - error(filename, linenum, 'whitespace/semicolon', 5, - 'Line contains only semicolon. If this should be an empty statement, ' - 'use {} instead.') - elif (Search(r'\s+;\s*$', line) and - not Search(r'\bfor\b', line)): - error(filename, linenum, 'whitespace/semicolon', 5, - 'Extra space before last semicolon. If this should be an empty ' - 'statement, use {} instead.') - - # In range-based for, we wanted spaces before and after the colon, but - # not around "::" tokens that might appear. - if (Search('for *\(.*[^:]:[^: ]', line) or - Search('for *\(.*[^: ]:[^:]', line)): - error(filename, linenum, 'whitespace/forcolon', 2, - 'Missing space around colon in range-based for loop') - - -def CheckSectionSpacing(filename, clean_lines, class_info, linenum, error): - """Checks for additional blank line issues related to sections. - - Currently the only thing checked here is blank line before protected/private. - - Args: - filename: The name of the current file. - clean_lines: A CleansedLines instance containing the file. - class_info: A _ClassInfo objects. - linenum: The number of the line to check. - error: The function to call with any errors found. - """ - # Skip checks if the class is small, where small means 25 lines or less. - # 25 lines seems like a good cutoff since that's the usual height of - # terminals, and any class that can't fit in one screen can't really - # be considered "small". - # - # Also skip checks if we are on the first line. This accounts for - # classes that look like - # class Foo { public: ... }; - # - # If we didn't find the end of the class, last_line would be zero, - # and the check will be skipped by the first condition. - if (class_info.last_line - class_info.starting_linenum <= 24 or - linenum <= class_info.starting_linenum): - return - - matched = Match(r'\s*(public|protected|private):', clean_lines.lines[linenum]) - if matched: - # Issue warning if the line before public/protected/private was - # not a blank line, but don't do this if the previous line contains - # "class" or "struct". This can happen two ways: - # - We are at the beginning of the class. - # - We are forward-declaring an inner class that is semantically - # private, but needed to be public for implementation reasons. - # Also ignores cases where the previous line ends with a backslash as can be - # common when defining classes in C macros. - prev_line = clean_lines.lines[linenum - 1] - if (not IsBlankLine(prev_line) and - not Search(r'\b(class|struct)\b', prev_line) and - not Search(r'\\$', prev_line)): - # Try a bit harder to find the beginning of the class. This is to - # account for multi-line base-specifier lists, e.g.: - # class Derived - # : public Base { - end_class_head = class_info.starting_linenum - for i in range(class_info.starting_linenum, linenum): - if Search(r'\{\s*$', clean_lines.lines[i]): - end_class_head = i - break - if end_class_head < linenum - 1: - error(filename, linenum, 'whitespace/blank_line', 3, - '"%s:" should be preceded by a blank line' % matched.group(1)) - - -def GetPreviousNonBlankLine(clean_lines, linenum): - """Return the most recent non-blank line and its line number. - - Args: - clean_lines: A CleansedLines instance containing the file contents. - linenum: The number of the line to check. - - Returns: - A tuple with two elements. The first element is the contents of the last - non-blank line before the current line, or the empty string if this is the - first non-blank line. The second is the line number of that line, or -1 - if this is the first non-blank line. - """ - - prevlinenum = linenum - 1 - while prevlinenum >= 0: - prevline = clean_lines.elided[prevlinenum] - if not IsBlankLine(prevline): # if not a blank line... - return (prevline, prevlinenum) - prevlinenum -= 1 - return ('', -1) - - -def CheckBraces(filename, clean_lines, linenum, error): - """Looks for misplaced braces (e.g. at the end of line). - - Args: - filename: The name of the current file. - clean_lines: A CleansedLines instance containing the file. - linenum: The number of the line to check. - error: The function to call with any errors found. - """ - - line = clean_lines.elided[linenum] # get rid of comments and strings - - if Match(r'\s*{\s*$', line): - # We allow an open brace to start a line in the case where someone is using - # braces in a block to explicitly create a new scope, which is commonly used - # to control the lifetime of stack-allocated variables. Braces are also - # used for brace initializers inside function calls. We don't detect this - # perfectly: we just don't complain if the last non-whitespace character on - # the previous non-blank line is ',', ';', ':', '(', '{', or '}', or if the - # previous line starts a preprocessor block. - prevline = GetPreviousNonBlankLine(clean_lines, linenum)[0] - if (not Search(r'[,;:}{(]\s*$', prevline) and - not Match(r'\s*#', prevline)): - error(filename, linenum, 'whitespace/braces', 4, - '{ should almost always be at the end of the previous line') - - # An else clause should be on the same line as the preceding closing brace. - if Match(r'\s*else\s*', line): - prevline = GetPreviousNonBlankLine(clean_lines, linenum)[0] - if Match(r'\s*}\s*$', prevline): - error(filename, linenum, 'whitespace/newline', 4, - 'An else should appear on the same line as the preceding }') - - # If braces come on one side of an else, they should be on both. - # However, we have to worry about "else if" that spans multiple lines! - if Search(r'}\s*else[^{]*$', line) or Match(r'[^}]*else\s*{', line): - if Search(r'}\s*else if([^{]*)$', line): # could be multi-line if - # find the ( after the if - pos = line.find('else if') - pos = line.find('(', pos) - if pos > 0: - (endline, _, endpos) = CloseExpression(clean_lines, linenum, pos) - if endline[endpos:].find('{') == -1: # must be brace after if - error(filename, linenum, 'readability/braces', 5, - 'If an else has a brace on one side, it should have it on both') - else: # common case: else not followed by a multi-line if - error(filename, linenum, 'readability/braces', 5, - 'If an else has a brace on one side, it should have it on both') - - # Likewise, an else should never have the else clause on the same line - if Search(r'\belse [^\s{]', line) and not Search(r'\belse if\b', line): - error(filename, linenum, 'whitespace/newline', 4, - 'Else clause should never be on same line as else (use 2 lines)') - - # In the same way, a do/while should never be on one line - if Match(r'\s*do [^\s{]', line): - error(filename, linenum, 'whitespace/newline', 4, - 'do/while clauses should not be on a single line') - - # Block bodies should not be followed by a semicolon. Due to C++11 - # brace initialization, there are more places where semicolons are - # required than not, so we use a whitelist approach to check these - # rather than a blacklist. These are the places where "};" should - # be replaced by just "}": - # 1. Some flavor of block following closing parenthesis: - # for (;;) {}; - # while (...) {}; - # switch (...) {}; - # Function(...) {}; - # if (...) {}; - # if (...) else if (...) {}; - # - # 2. else block: - # if (...) else {}; - # - # 3. const member function: - # Function(...) const {}; - # - # 4. Block following some statement: - # x = 42; - # {}; - # - # 5. Block at the beginning of a function: - # Function(...) { - # {}; - # } - # - # Note that naively checking for the preceding "{" will also match - # braces inside multi-dimensional arrays, but this is fine since - # that expression will not contain semicolons. - # - # 6. Block following another block: - # while (true) {} - # {}; - # - # 7. End of namespaces: - # namespace {}; - # - # These semicolons seems far more common than other kinds of - # redundant semicolons, possibly due to people converting classes - # to namespaces. For now we do not warn for this case. - # - # Try matching case 1 first. - match = Match(r'^(.*\)\s*)\{', line) - if match: - # Matched closing parenthesis (case 1). Check the token before the - # matching opening parenthesis, and don't warn if it looks like a - # macro. This avoids these false positives: - # - macro that defines a base class - # - multi-line macro that defines a base class - # - macro that defines the whole class-head - # - # But we still issue warnings for macros that we know are safe to - # warn, specifically: - # - TEST, TEST_F, TEST_P, MATCHER, MATCHER_P - # - TYPED_TEST - # - INTERFACE_DEF - # - EXCLUSIVE_LOCKS_REQUIRED, SHARED_LOCKS_REQUIRED, LOCKS_EXCLUDED: - # - # We implement a whitelist of safe macros instead of a blacklist of - # unsafe macros, even though the latter appears less frequently in - # google code and would have been easier to implement. This is because - # the downside for getting the whitelist wrong means some extra - # semicolons, while the downside for getting the blacklist wrong - # would result in compile errors. - # - # In addition to macros, we also don't want to warn on compound - # literals. - closing_brace_pos = match.group(1).rfind(')') - opening_parenthesis = ReverseCloseExpression( - clean_lines, linenum, closing_brace_pos) - if opening_parenthesis[2] > -1: - line_prefix = opening_parenthesis[0][0:opening_parenthesis[2]] - macro = Search(r'\b([A-Z_]+)\s*$', line_prefix) - if ((macro and - macro.group(1) not in ( - 'TEST', 'TEST_F', 'MATCHER', 'MATCHER_P', 'TYPED_TEST', - 'EXCLUSIVE_LOCKS_REQUIRED', 'SHARED_LOCKS_REQUIRED', - 'LOCKS_EXCLUDED', 'INTERFACE_DEF')) or - Search(r'\s+=\s*$', line_prefix)): - match = None - - else: - # Try matching cases 2-3. - match = Match(r'^(.*(?:else|\)\s*const)\s*)\{', line) - if not match: - # Try matching cases 4-6. These are always matched on separate lines. - # - # Note that we can't simply concatenate the previous line to the - # current line and do a single match, otherwise we may output - # duplicate warnings for the blank line case: - # if (cond) { - # // blank line - # } - prevline = GetPreviousNonBlankLine(clean_lines, linenum)[0] - if prevline and Search(r'[;{}]\s*$', prevline): - match = Match(r'^(\s*)\{', line) - - # Check matching closing brace - if match: - (endline, endlinenum, endpos) = CloseExpression( - clean_lines, linenum, len(match.group(1))) - if endpos > -1 and Match(r'^\s*;', endline[endpos:]): - # Current {} pair is eligible for semicolon check, and we have found - # the redundant semicolon, output warning here. - # - # Note: because we are scanning forward for opening braces, and - # outputting warnings for the matching closing brace, if there are - # nested blocks with trailing semicolons, we will get the error - # messages in reversed order. - error(filename, endlinenum, 'readability/braces', 4, - "You don't need a ; after a }") - - -def CheckEmptyBlockBody(filename, clean_lines, linenum, error): - """Look for empty loop/conditional body with only a single semicolon. - - Args: - filename: The name of the current file. - clean_lines: A CleansedLines instance containing the file. - linenum: The number of the line to check. - error: The function to call with any errors found. - """ - - # Search for loop keywords at the beginning of the line. Because only - # whitespaces are allowed before the keywords, this will also ignore most - # do-while-loops, since those lines should start with closing brace. - # - # We also check "if" blocks here, since an empty conditional block - # is likely an error. - line = clean_lines.elided[linenum] - matched = Match(r'\s*(for|while|if)\s*\(', line) - if matched: - # Find the end of the conditional expression - (end_line, end_linenum, end_pos) = CloseExpression( - clean_lines, linenum, line.find('(')) - - # Output warning if what follows the condition expression is a semicolon. - # No warning for all other cases, including whitespace or newline, since we - # have a separate check for semicolons preceded by whitespace. - if end_pos >= 0 and Match(r';', end_line[end_pos:]): - if matched.group(1) == 'if': - error(filename, end_linenum, 'whitespace/empty_conditional_body', 5, - 'Empty conditional bodies should use {}') - else: - error(filename, end_linenum, 'whitespace/empty_loop_body', 5, - 'Empty loop bodies should use {} or continue') - - -def CheckCheck(filename, clean_lines, linenum, error): - """Checks the use of CHECK and EXPECT macros. - - Args: - filename: The name of the current file. - clean_lines: A CleansedLines instance containing the file. - linenum: The number of the line to check. - error: The function to call with any errors found. - """ - - # Decide the set of replacement macros that should be suggested - lines = clean_lines.elided - check_macro = None - start_pos = -1 - for macro in _CHECK_MACROS: - i = lines[linenum].find(macro) - if i >= 0: - check_macro = macro - - # Find opening parenthesis. Do a regular expression match here - # to make sure that we are matching the expected CHECK macro, as - # opposed to some other macro that happens to contain the CHECK - # substring. - matched = Match(r'^(.*\b' + check_macro + r'\s*)\(', lines[linenum]) - if not matched: - continue - start_pos = len(matched.group(1)) - break - if not check_macro or start_pos < 0: - # Don't waste time here if line doesn't contain 'CHECK' or 'EXPECT' - return - - # Find end of the boolean expression by matching parentheses - (last_line, end_line, end_pos) = CloseExpression( - clean_lines, linenum, start_pos) - if end_pos < 0: - return - if linenum == end_line: - expression = lines[linenum][start_pos + 1:end_pos - 1] - else: - expression = lines[linenum][start_pos + 1:] - for i in xrange(linenum + 1, end_line): - expression += lines[i] - expression += last_line[0:end_pos - 1] - - # Parse expression so that we can take parentheses into account. - # This avoids false positives for inputs like "CHECK((a < 4) == b)", - # which is not replaceable by CHECK_LE. - lhs = '' - rhs = '' - operator = None - while expression: - matched = Match(r'^\s*(<<|<<=|>>|>>=|->\*|->|&&|\|\||' - r'==|!=|>=|>|<=|<|\()(.*)$', expression) - if matched: - token = matched.group(1) - if token == '(': - # Parenthesized operand - expression = matched.group(2) - (end, _) = FindEndOfExpressionInLine(expression, 0, 1, '(', ')') - if end < 0: - return # Unmatched parenthesis - lhs += '(' + expression[0:end] - expression = expression[end:] - elif token in ('&&', '||'): - # Logical and/or operators. This means the expression - # contains more than one term, for example: - # CHECK(42 < a && a < b); - # - # These are not replaceable with CHECK_LE, so bail out early. - return - elif token in ('<<', '<<=', '>>', '>>=', '->*', '->'): - # Non-relational operator - lhs += token - expression = matched.group(2) - else: - # Relational operator - operator = token - rhs = matched.group(2) - break - else: - # Unparenthesized operand. Instead of appending to lhs one character - # at a time, we do another regular expression match to consume several - # characters at once if possible. Trivial benchmark shows that this - # is more efficient when the operands are longer than a single - # character, which is generally the case. - matched = Match(r'^([^-=!<>()&|]+)(.*)$', expression) - if not matched: - matched = Match(r'^(\s*\S)(.*)$', expression) - if not matched: - break - lhs += matched.group(1) - expression = matched.group(2) - - # Only apply checks if we got all parts of the boolean expression - if not (lhs and operator and rhs): - return - - # Check that rhs do not contain logical operators. We already know - # that lhs is fine since the loop above parses out && and ||. - if rhs.find('&&') > -1 or rhs.find('||') > -1: - return - - # At least one of the operands must be a constant literal. This is - # to avoid suggesting replacements for unprintable things like - # CHECK(variable != iterator) - # - # The following pattern matches decimal, hex integers, strings, and - # characters (in that order). - lhs = lhs.strip() - rhs = rhs.strip() - match_constant = r'^([-+]?(\d+|0[xX][0-9a-fA-F]+)[lLuU]{0,3}|".*"|\'.*\')$' - if Match(match_constant, lhs) or Match(match_constant, rhs): - # Note: since we know both lhs and rhs, we can provide a more - # descriptive error message like: - # Consider using CHECK_EQ(x, 42) instead of CHECK(x == 42) - # Instead of: - # Consider using CHECK_EQ instead of CHECK(a == b) - # - # We are still keeping the less descriptive message because if lhs - # or rhs gets long, the error message might become unreadable. - error(filename, linenum, 'readability/check', 2, - 'Consider using %s instead of %s(a %s b)' % ( - _CHECK_REPLACEMENT[check_macro][operator], - check_macro, operator)) - - -def CheckAltTokens(filename, clean_lines, linenum, error): - """Check alternative keywords being used in boolean expressions. - - Args: - filename: The name of the current file. - clean_lines: A CleansedLines instance containing the file. - linenum: The number of the line to check. - error: The function to call with any errors found. - """ - line = clean_lines.elided[linenum] - - # Avoid preprocessor lines - if Match(r'^\s*#', line): - return - - # Last ditch effort to avoid multi-line comments. This will not help - # if the comment started before the current line or ended after the - # current line, but it catches most of the false positives. At least, - # it provides a way to workaround this warning for people who use - # multi-line comments in preprocessor macros. - # - # TODO(unknown): remove this once cpplint has better support for - # multi-line comments. - if line.find('/*') >= 0 or line.find('*/') >= 0: - return - - for match in _ALT_TOKEN_REPLACEMENT_PATTERN.finditer(line): - error(filename, linenum, 'readability/alt_tokens', 2, - 'Use operator %s instead of %s' % ( - _ALT_TOKEN_REPLACEMENT[match.group(1)], match.group(1))) - - -def GetLineWidth(line): - """Determines the width of the line in column positions. - - Args: - line: A string, which may be a Unicode string. - - Returns: - The width of the line in column positions, accounting for Unicode - combining characters and wide characters. - """ - if isinstance(line, unicode): - width = 0 - for uc in unicodedata.normalize('NFC', line): - if unicodedata.east_asian_width(uc) in ('W', 'F'): - width += 2 - elif not unicodedata.combining(uc): - width += 1 - return width - else: - return len(line) - - -def CheckStyle(filename, clean_lines, linenum, file_extension, nesting_state, - error): - """Checks rules from the 'C++ style rules' section of cppguide.html. - - Most of these rules are hard to test (naming, comment style), but we - do what we can. In particular we check for 2-space indents, line lengths, - tab usage, spaces inside code, etc. - - Args: - filename: The name of the current file. - clean_lines: A CleansedLines instance containing the file. - linenum: The number of the line to check. - file_extension: The extension (without the dot) of the filename. - nesting_state: A _NestingState instance which maintains information about - the current stack of nested blocks being parsed. - error: The function to call with any errors found. - """ - - # Don't use "elided" lines here, otherwise we can't check commented lines. - # Don't want to use "raw" either, because we don't want to check inside C++11 - # raw strings, - raw_lines = clean_lines.lines_without_raw_strings - line = raw_lines[linenum] - - if line.find('\t') != -1: - error(filename, linenum, 'whitespace/tab', 1, - 'Tab found; better to use spaces') - - # One or three blank spaces at the beginning of the line is weird; it's - # hard to reconcile that with 2-space indents. - # NOTE: here are the conditions rob pike used for his tests. Mine aren't - # as sophisticated, but it may be worth becoming so: RLENGTH==initial_spaces - # if(RLENGTH > 20) complain = 0; - # if(match($0, " +(error|private|public|protected):")) complain = 0; - # if(match(prev, "&& *$")) complain = 0; - # if(match(prev, "\\|\\| *$")) complain = 0; - # if(match(prev, "[\",=><] *$")) complain = 0; - # if(match($0, " <<")) complain = 0; - # if(match(prev, " +for \\(")) complain = 0; - # if(prevodd && match(prevprev, " +for \\(")) complain = 0; - initial_spaces = 0 - cleansed_line = clean_lines.elided[linenum] - while initial_spaces < len(line) and line[initial_spaces] == ' ': - initial_spaces += 1 - if line and line[-1].isspace(): - error(filename, linenum, 'whitespace/end_of_line', 4, - 'Line ends in whitespace. Consider deleting these extra spaces.') - # There are certain situations we allow one space, notably for section labels - elif ((initial_spaces == 1 or initial_spaces == 3) and - not Match(r'\s*\w+\s*:\s*$', cleansed_line)): - error(filename, linenum, 'whitespace/indent', 3, - 'Weird number of spaces at line-start. ' - 'Are you using a 2-space indent?') - - # Check if the line is a header guard. - is_header_guard = False - if file_extension == 'h': - cppvar = GetHeaderGuardCPPVariable(filename) - if (line.startswith('#ifndef %s' % cppvar) or - line.startswith('#define %s' % cppvar) or - line.startswith('#endif // %s' % cppvar)): - is_header_guard = True - # #include lines and header guards can be long, since there's no clean way to - # split them. - # - # URLs can be long too. It's possible to split these, but it makes them - # harder to cut&paste. - # - # The "$Id:...$" comment may also get very long without it being the - # developers fault. - if (not line.startswith('#include') and not is_header_guard and - not Match(r'^\s*//.*http(s?)://\S*$', line) and - not Match(r'^// \$Id:.*#[0-9]+ \$$', line)): - line_width = GetLineWidth(line) - extended_length = int((_line_length * 1.25)) - if line_width > extended_length: - error(filename, linenum, 'whitespace/line_length', 4, - 'Lines should very rarely be longer than %i characters' % - extended_length) - elif line_width > _line_length: - error(filename, linenum, 'whitespace/line_length', 2, - 'Lines should be <= %i characters long' % _line_length) - - if (cleansed_line.count(';') > 1 and - # for loops are allowed two ;'s (and may run over two lines). - cleansed_line.find('for') == -1 and - (GetPreviousNonBlankLine(clean_lines, linenum)[0].find('for') == -1 or - GetPreviousNonBlankLine(clean_lines, linenum)[0].find(';') != -1) and - # It's ok to have many commands in a switch case that fits in 1 line - not ((cleansed_line.find('case ') != -1 or - cleansed_line.find('default:') != -1) and - cleansed_line.find('break;') != -1)): - error(filename, linenum, 'whitespace/newline', 0, - 'More than one command on the same line') - - # Some more style checks - CheckBraces(filename, clean_lines, linenum, error) - CheckEmptyBlockBody(filename, clean_lines, linenum, error) - CheckAccess(filename, clean_lines, linenum, nesting_state, error) - CheckSpacing(filename, clean_lines, linenum, nesting_state, error) - CheckCheck(filename, clean_lines, linenum, error) - CheckAltTokens(filename, clean_lines, linenum, error) - classinfo = nesting_state.InnermostClass() - if classinfo: - CheckSectionSpacing(filename, clean_lines, classinfo, linenum, error) - - -_RE_PATTERN_INCLUDE_NEW_STYLE = re.compile(r'#include +"[^/]+\.h"') -_RE_PATTERN_INCLUDE = re.compile(r'^\s*#\s*include\s*([<"])([^>"]*)[>"].*$') -# Matches the first component of a filename delimited by -s and _s. That is: -# _RE_FIRST_COMPONENT.match('foo').group(0) == 'foo' -# _RE_FIRST_COMPONENT.match('foo.cc').group(0) == 'foo' -# _RE_FIRST_COMPONENT.match('foo-bar_baz.cc').group(0) == 'foo' -# _RE_FIRST_COMPONENT.match('foo_bar-baz.cc').group(0) == 'foo' -_RE_FIRST_COMPONENT = re.compile(r'^[^-_.]+') - - -def _DropCommonSuffixes(filename): - """Drops common suffixes like _test.cc or -inl.h from filename. - - For example: - >>> _DropCommonSuffixes('foo/foo-inl.h') - 'foo/foo' - >>> _DropCommonSuffixes('foo/bar/foo.cc') - 'foo/bar/foo' - >>> _DropCommonSuffixes('foo/foo_internal.h') - 'foo/foo' - >>> _DropCommonSuffixes('foo/foo_unusualinternal.h') - 'foo/foo_unusualinternal' - - Args: - filename: The input filename. - - Returns: - The filename with the common suffix removed. - """ - for suffix in ('test.cc', 'regtest.cc', 'unittest.cc', - 'inl.h', 'impl.h', 'internal.h'): - if (filename.endswith(suffix) and len(filename) > len(suffix) and - filename[-len(suffix) - 1] in ('-', '_')): - return filename[:-len(suffix) - 1] - return os.path.splitext(filename)[0] - - -def _IsTestFilename(filename): - """Determines if the given filename has a suffix that identifies it as a test. - - Args: - filename: The input filename. - - Returns: - True if 'filename' looks like a test, False otherwise. - """ - if (filename.endswith('_test.cc') or - filename.endswith('_unittest.cc') or - filename.endswith('_regtest.cc')): - return True - else: - return False - - -def _ClassifyInclude(fileinfo, include, is_system): - """Figures out what kind of header 'include' is. - - Args: - fileinfo: The current file cpplint is running over. A FileInfo instance. - include: The path to a #included file. - is_system: True if the #include used <> rather than "". - - Returns: - One of the _XXX_HEADER constants. - - For example: - >>> _ClassifyInclude(FileInfo('foo/foo.cc'), 'stdio.h', True) - _C_SYS_HEADER - >>> _ClassifyInclude(FileInfo('foo/foo.cc'), 'string', True) - _CPP_SYS_HEADER - >>> _ClassifyInclude(FileInfo('foo/foo.cc'), 'foo/foo.h', False) - _LIKELY_MY_HEADER - >>> _ClassifyInclude(FileInfo('foo/foo_unknown_extension.cc'), - ... 'bar/foo_other_ext.h', False) - _POSSIBLE_MY_HEADER - >>> _ClassifyInclude(FileInfo('foo/foo.cc'), 'foo/bar.h', False) - _OTHER_HEADER - """ - # This is a list of all standard c++ header files, except - # those already checked for above. - is_cpp_h = include in _CPP_HEADERS - - if is_system: - if is_cpp_h: - return _CPP_SYS_HEADER - else: - return _C_SYS_HEADER - - # If the target file and the include we're checking share a - # basename when we drop common extensions, and the include - # lives in . , then it's likely to be owned by the target file. - target_dir, target_base = ( - os.path.split(_DropCommonSuffixes(fileinfo.RepositoryName()))) - include_dir, include_base = os.path.split(_DropCommonSuffixes(include)) - if target_base == include_base and ( - include_dir == target_dir or - include_dir == os.path.normpath(target_dir + '/../public')): - return _LIKELY_MY_HEADER - - # If the target and include share some initial basename - # component, it's possible the target is implementing the - # include, so it's allowed to be first, but we'll never - # complain if it's not there. - target_first_component = _RE_FIRST_COMPONENT.match(target_base) - include_first_component = _RE_FIRST_COMPONENT.match(include_base) - if (target_first_component and include_first_component and - target_first_component.group(0) == - include_first_component.group(0)): - return _POSSIBLE_MY_HEADER - - return _OTHER_HEADER - - - -def CheckIncludeLine(filename, clean_lines, linenum, include_state, error): - """Check rules that are applicable to #include lines. - - Strings on #include lines are NOT removed from elided line, to make - certain tasks easier. However, to prevent false positives, checks - applicable to #include lines in CheckLanguage must be put here. - - Args: - filename: The name of the current file. - clean_lines: A CleansedLines instance containing the file. - linenum: The number of the line to check. - include_state: An _IncludeState instance in which the headers are inserted. - error: The function to call with any errors found. - """ - fileinfo = FileInfo(filename) - - line = clean_lines.lines[linenum] - - # "include" should use the new style "foo/bar.h" instead of just "bar.h" - if _RE_PATTERN_INCLUDE_NEW_STYLE.search(line): - error(filename, linenum, 'build/include', 4, - 'Include the directory when naming .h files') - - # we shouldn't include a file more than once. actually, there are a - # handful of instances where doing so is okay, but in general it's - # not. - match = _RE_PATTERN_INCLUDE.search(line) - if match: - include = match.group(2) - is_system = (match.group(1) == '<') - if include in include_state: - error(filename, linenum, 'build/include', 4, - '"%s" already included at %s:%s' % - (include, filename, include_state[include])) - else: - include_state[include] = linenum - - # We want to ensure that headers appear in the right order: - # 1) for foo.cc, foo.h (preferred location) - # 2) c system files - # 3) cpp system files - # 4) for foo.cc, foo.h (deprecated location) - # 5) other google headers - # - # We classify each include statement as one of those 5 types - # using a number of techniques. The include_state object keeps - # track of the highest type seen, and complains if we see a - # lower type after that. - error_message = include_state.CheckNextIncludeOrder( - _ClassifyInclude(fileinfo, include, is_system)) - if error_message: - error(filename, linenum, 'build/include_order', 4, - '%s. Should be: %s.h, c system, c++ system, other.' % - (error_message, fileinfo.BaseName())) - canonical_include = include_state.CanonicalizeAlphabeticalOrder(include) - if not include_state.IsInAlphabeticalOrder( - clean_lines, linenum, canonical_include): - error(filename, linenum, 'build/include_alpha', 4, - 'Include "%s" not in alphabetical order' % include) - include_state.SetLastHeader(canonical_include) - - # Look for any of the stream classes that are part of standard C++. - match = _RE_PATTERN_INCLUDE.match(line) - if match: - include = match.group(2) - if Match(r'(f|ind|io|i|o|parse|pf|stdio|str|)?stream$', include): - # Many unit tests use cout, so we exempt them. - if not _IsTestFilename(filename): - error(filename, linenum, 'readability/streams', 3, - 'Streams are highly discouraged.') - - -def _GetTextInside(text, start_pattern): - r"""Retrieves all the text between matching open and close parentheses. - - Given a string of lines and a regular expression string, retrieve all the text - following the expression and between opening punctuation symbols like - (, [, or {, and the matching close-punctuation symbol. This properly nested - occurrences of the punctuations, so for the text like - printf(a(), b(c())); - a call to _GetTextInside(text, r'printf\(') will return 'a(), b(c())'. - start_pattern must match string having an open punctuation symbol at the end. - - Args: - text: The lines to extract text. Its comments and strings must be elided. - It can be single line and can span multiple lines. - start_pattern: The regexp string indicating where to start extracting - the text. - Returns: - The extracted text. - None if either the opening string or ending punctuation could not be found. - """ - # TODO(sugawarayu): Audit cpplint.py to see what places could be profitably - # rewritten to use _GetTextInside (and use inferior regexp matching today). - - # Give opening punctuations to get the matching close-punctuations. - matching_punctuation = {'(': ')', '{': '}', '[': ']'} - closing_punctuation = set(matching_punctuation.itervalues()) - - # Find the position to start extracting text. - match = re.search(start_pattern, text, re.M) - if not match: # start_pattern not found in text. - return None - start_position = match.end(0) - - assert start_position > 0, ( - 'start_pattern must ends with an opening punctuation.') - assert text[start_position - 1] in matching_punctuation, ( - 'start_pattern must ends with an opening punctuation.') - # Stack of closing punctuations we expect to have in text after position. - punctuation_stack = [matching_punctuation[text[start_position - 1]]] - position = start_position - while punctuation_stack and position < len(text): - if text[position] == punctuation_stack[-1]: - punctuation_stack.pop() - elif text[position] in closing_punctuation: - # A closing punctuation without matching opening punctuations. - return None - elif text[position] in matching_punctuation: - punctuation_stack.append(matching_punctuation[text[position]]) - position += 1 - if punctuation_stack: - # Opening punctuations left without matching close-punctuations. - return None - # punctuations match. - return text[start_position:position - 1] - - -# Patterns for matching call-by-reference parameters. -# -# Supports nested templates up to 2 levels deep using this messy pattern: -# < (?: < (?: < [^<>]* -# > -# | [^<>] )* -# > -# | [^<>] )* -# > -_RE_PATTERN_IDENT = r'[_a-zA-Z]\w*' # =~ [[:alpha:]][[:alnum:]]* -_RE_PATTERN_TYPE = ( - r'(?:const\s+)?(?:typename\s+|class\s+|struct\s+|union\s+|enum\s+)?' - r'(?:\w|' - r'\s*<(?:<(?:<[^<>]*>|[^<>])*>|[^<>])*>|' - r'::)+') -# A call-by-reference parameter ends with '& identifier'. -_RE_PATTERN_REF_PARAM = re.compile( - r'(' + _RE_PATTERN_TYPE + r'(?:\s*(?:\bconst\b|[*]))*\s*' - r'&\s*' + _RE_PATTERN_IDENT + r')\s*(?:=[^,()]+)?[,)]') -# A call-by-const-reference parameter either ends with 'const& identifier' -# or looks like 'const type& identifier' when 'type' is atomic. -_RE_PATTERN_CONST_REF_PARAM = ( - r'(?:.*\s*\bconst\s*&\s*' + _RE_PATTERN_IDENT + - r'|const\s+' + _RE_PATTERN_TYPE + r'\s*&\s*' + _RE_PATTERN_IDENT + r')') - - -def CheckLanguage(filename, clean_lines, linenum, file_extension, - include_state, nesting_state, error): - """Checks rules from the 'C++ language rules' section of cppguide.html. - - Some of these rules are hard to test (function overloading, using - uint32 inappropriately), but we do the best we can. - - Args: - filename: The name of the current file. - clean_lines: A CleansedLines instance containing the file. - linenum: The number of the line to check. - file_extension: The extension (without the dot) of the filename. - include_state: An _IncludeState instance in which the headers are inserted. - nesting_state: A _NestingState instance which maintains information about - the current stack of nested blocks being parsed. - error: The function to call with any errors found. - """ - # If the line is empty or consists of entirely a comment, no need to - # check it. - line = clean_lines.elided[linenum] - if not line: - return - - match = _RE_PATTERN_INCLUDE.search(line) - if match: - CheckIncludeLine(filename, clean_lines, linenum, include_state, error) - return - - # Reset include state across preprocessor directives. This is meant - # to silence warnings for conditional includes. - if Match(r'^\s*#\s*(?:ifdef|elif|else|endif)\b', line): - include_state.ResetSection() - - # Make Windows paths like Unix. - fullname = os.path.abspath(filename).replace('\\', '/') - - # TODO(unknown): figure out if they're using default arguments in fn proto. - - # Check to see if they're using an conversion function cast. - # I just try to capture the most common basic types, though there are more. - # Parameterless conversion functions, such as bool(), are allowed as they are - # probably a member operator declaration or default constructor. - match = Search( - r'(\bnew\s+)?\b' # Grab 'new' operator, if it's there - r'(int|float|double|bool|char|int32|uint32|int64|uint64)' - r'(\([^)].*)', line) - if match: - matched_new = match.group(1) - matched_type = match.group(2) - matched_funcptr = match.group(3) - - # gMock methods are defined using some variant of MOCK_METHODx(name, type) - # where type may be float(), int(string), etc. Without context they are - # virtually indistinguishable from int(x) casts. Likewise, gMock's - # MockCallback takes a template parameter of the form return_type(arg_type), - # which looks much like the cast we're trying to detect. - # - # std::function<> wrapper has a similar problem. - # - # Return types for function pointers also look like casts if they - # don't have an extra space. - if (matched_new is None and # If new operator, then this isn't a cast - not (Match(r'^\s*MOCK_(CONST_)?METHOD\d+(_T)?\(', line) or - Search(r'\bMockCallback<.*>', line) or - Search(r'\bstd::function<.*>', line)) and - not (matched_funcptr and - Match(r'\((?:[^() ]+::\s*\*\s*)?[^() ]+\)\s*\(', - matched_funcptr))): - # Try a bit harder to catch gmock lines: the only place where - # something looks like an old-style cast is where we declare the - # return type of the mocked method, and the only time when we - # are missing context is if MOCK_METHOD was split across - # multiple lines. The missing MOCK_METHOD is usually one or two - # lines back, so scan back one or two lines. - # - # It's not possible for gmock macros to appear in the first 2 - # lines, since the class head + section name takes up 2 lines. - if (linenum < 2 or - not (Match(r'^\s*MOCK_(?:CONST_)?METHOD\d+(?:_T)?\((?:\S+,)?\s*$', - clean_lines.elided[linenum - 1]) or - Match(r'^\s*MOCK_(?:CONST_)?METHOD\d+(?:_T)?\(\s*$', - clean_lines.elided[linenum - 2]))): - error(filename, linenum, 'readability/casting', 4, - 'Using deprecated casting style. ' - 'Use static_cast<%s>(...) instead' % - matched_type) - - CheckCStyleCast(filename, linenum, line, clean_lines.raw_lines[linenum], - 'static_cast', - r'\((int|float|double|bool|char|u?int(16|32|64))\)', error) - - # This doesn't catch all cases. Consider (const char * const)"hello". - # - # (char *) "foo" should always be a const_cast (reinterpret_cast won't - # compile). - if CheckCStyleCast(filename, linenum, line, clean_lines.raw_lines[linenum], - 'const_cast', r'\((char\s?\*+\s?)\)\s*"', error): - pass - else: - # Check pointer casts for other than string constants - CheckCStyleCast(filename, linenum, line, clean_lines.raw_lines[linenum], - 'reinterpret_cast', r'\((\w+\s?\*+\s?)\)', error) - - # In addition, we look for people taking the address of a cast. This - # is dangerous -- casts can assign to temporaries, so the pointer doesn't - # point where you think. - match = Search( - r'(?:&\(([^)]+)\)[\w(])|' - r'(?:&(static|dynamic|down|reinterpret)_cast\b)', line) - if match and match.group(1) != '*': - error(filename, linenum, 'runtime/casting', 4, - ('Are you taking an address of a cast? ' - 'This is dangerous: could be a temp var. ' - 'Take the address before doing the cast, rather than after')) - - # Create an extended_line, which is the concatenation of the current and - # next lines, for more effective checking of code that may span more than one - # line. - if linenum + 1 < clean_lines.NumLines(): - extended_line = line + clean_lines.elided[linenum + 1] - else: - extended_line = line - - # Check for people declaring static/global STL strings at the top level. - # This is dangerous because the C++ language does not guarantee that - # globals with constructors are initialized before the first access. - match = Match( - r'((?:|static +)(?:|const +))string +([a-zA-Z0-9_:]+)\b(.*)', - line) - # Make sure it's not a function. - # Function template specialization looks like: "string foo<Type>(...". - # Class template definitions look like: "string Foo<Type>::Method(...". - # - # Also ignore things that look like operators. These are matched separately - # because operator names cross non-word boundaries. If we change the pattern - # above, we would decrease the accuracy of matching identifiers. - if (match and - not Search(r'\boperator\W', line) and - not Match(r'\s*(<.*>)?(::[a-zA-Z0-9_]+)?\s*\(([^"]|$)', match.group(3))): - error(filename, linenum, 'runtime/string', 4, - 'For a static/global string constant, use a C style string instead: ' - '"%schar %s[]".' % - (match.group(1), match.group(2))) - - if Search(r'\b([A-Za-z0-9_]*_)\(\1\)', line): - error(filename, linenum, 'runtime/init', 4, - 'You seem to be initializing a member variable with itself.') - - if file_extension == 'h': - # TODO(unknown): check that 1-arg constructors are explicit. - # How to tell it's a constructor? - # (handled in CheckForNonStandardConstructs for now) - # TODO(unknown): check that classes have DISALLOW_EVIL_CONSTRUCTORS - # (level 1 error) - pass - - # Check if people are using the verboten C basic types. The only exception - # we regularly allow is "unsigned short port" for port. - if Search(r'\bshort port\b', line): - if not Search(r'\bunsigned short port\b', line): - error(filename, linenum, 'runtime/int', 4, - 'Use "unsigned short" for ports, not "short"') - else: - match = Search(r'\b(short|long(?! +double)|long long)\b', line) - if match: - error(filename, linenum, 'runtime/int', 4, - 'Use int16/int64/etc, rather than the C type %s' % match.group(1)) - - # When snprintf is used, the second argument shouldn't be a literal. - match = Search(r'snprintf\s*\(([^,]*),\s*([0-9]*)\s*,', line) - if match and match.group(2) != '0': - # If 2nd arg is zero, snprintf is used to calculate size. - error(filename, linenum, 'runtime/printf', 3, - 'If you can, use sizeof(%s) instead of %s as the 2nd arg ' - 'to snprintf.' % (match.group(1), match.group(2))) - - # Check if some verboten C functions are being used. - if Search(r'\bsprintf\b', line): - error(filename, linenum, 'runtime/printf', 5, - 'Never use sprintf. Use snprintf instead.') - match = Search(r'\b(strcpy|strcat)\b', line) - if match: - error(filename, linenum, 'runtime/printf', 4, - 'Almost always, snprintf is better than %s' % match.group(1)) - - # Check if some verboten operator overloading is going on - # TODO(unknown): catch out-of-line unary operator&: - # class X {}; - # int operator&(const X& x) { return 42; } // unary operator& - # The trick is it's hard to tell apart from binary operator&: - # class Y { int operator&(const Y& x) { return 23; } }; // binary operator& - if Search(r'\boperator\s*&\s*\(\s*\)', line): - error(filename, linenum, 'runtime/operator', 4, - 'Unary operator& is dangerous. Do not use it.') - - # Check for suspicious usage of "if" like - # } if (a == b) { - if Search(r'\}\s*if\s*\(', line): - error(filename, linenum, 'readability/braces', 4, - 'Did you mean "else if"? If not, start a new line for "if".') - - # Check for potential format string bugs like printf(foo). - # We constrain the pattern not to pick things like DocidForPrintf(foo). - # Not perfect but it can catch printf(foo.c_str()) and printf(foo->c_str()) - # TODO(sugawarayu): Catch the following case. Need to change the calling - # convention of the whole function to process multiple line to handle it. - # printf( - # boy_this_is_a_really_long_variable_that_cannot_fit_on_the_prev_line); - printf_args = _GetTextInside(line, r'(?i)\b(string)?printf\s*\(') - if printf_args: - match = Match(r'([\w.\->()]+)$', printf_args) - if match and match.group(1) != '__VA_ARGS__': - function_name = re.search(r'\b((?:string)?printf)\s*\(', - line, re.I).group(1) - error(filename, linenum, 'runtime/printf', 4, - 'Potential format string bug. Do %s("%%s", %s) instead.' - % (function_name, match.group(1))) - - # Check for potential memset bugs like memset(buf, sizeof(buf), 0). - match = Search(r'memset\s*\(([^,]*),\s*([^,]*),\s*0\s*\)', line) - if match and not Match(r"^''|-?[0-9]+|0x[0-9A-Fa-f]$", match.group(2)): - error(filename, linenum, 'runtime/memset', 4, - 'Did you mean "memset(%s, 0, %s)"?' - % (match.group(1), match.group(2))) - - if Search(r'\busing namespace\b', line): - error(filename, linenum, 'build/namespaces', 5, - 'Do not use namespace using-directives. ' - 'Use using-declarations instead.') - - # Detect variable-length arrays. - match = Match(r'\s*(.+::)?(\w+) [a-z]\w*\[(.+)];', line) - if (match and match.group(2) != 'return' and match.group(2) != 'delete' and - match.group(3).find(']') == -1): - # Split the size using space and arithmetic operators as delimiters. - # If any of the resulting tokens are not compile time constants then - # report the error. - tokens = re.split(r'\s|\+|\-|\*|\/|<<|>>]', match.group(3)) - is_const = True - skip_next = False - for tok in tokens: - if skip_next: - skip_next = False - continue - - if Search(r'sizeof\(.+\)', tok): continue - if Search(r'arraysize\(\w+\)', tok): continue - - tok = tok.lstrip('(') - tok = tok.rstrip(')') - if not tok: continue - if Match(r'\d+', tok): continue - if Match(r'0[xX][0-9a-fA-F]+', tok): continue - if Match(r'k[A-Z0-9]\w*', tok): continue - if Match(r'(.+::)?k[A-Z0-9]\w*', tok): continue - if Match(r'(.+::)?[A-Z][A-Z0-9_]*', tok): continue - # A catch all for tricky sizeof cases, including 'sizeof expression', - # 'sizeof(*type)', 'sizeof(const type)', 'sizeof(struct StructName)' - # requires skipping the next token because we split on ' ' and '*'. - if tok.startswith('sizeof'): - skip_next = True - continue - is_const = False - break - if not is_const: - error(filename, linenum, 'runtime/arrays', 1, - 'Do not use variable-length arrays. Use an appropriately named ' - "('k' followed by CamelCase) compile-time constant for the size.") - - # If DISALLOW_EVIL_CONSTRUCTORS, DISALLOW_COPY_AND_ASSIGN, or - # DISALLOW_IMPLICIT_CONSTRUCTORS is present, then it should be the last thing - # in the class declaration. - match = Match( - (r'\s*' - r'(DISALLOW_(EVIL_CONSTRUCTORS|COPY_AND_ASSIGN|IMPLICIT_CONSTRUCTORS))' - r'\(.*\);$'), - line) - if match and linenum + 1 < clean_lines.NumLines(): - next_line = clean_lines.elided[linenum + 1] - # We allow some, but not all, declarations of variables to be present - # in the statement that defines the class. The [\w\*,\s]* fragment of - # the regular expression below allows users to declare instances of - # the class or pointers to instances, but not less common types such - # as function pointers or arrays. It's a tradeoff between allowing - # reasonable code and avoiding trying to parse more C++ using regexps. - if not Search(r'^\s*}[\w\*,\s]*;', next_line): - error(filename, linenum, 'readability/constructors', 3, - match.group(1) + ' should be the last thing in the class') - - # Check for use of unnamed namespaces in header files. Registration - # macros are typically OK, so we allow use of "namespace {" on lines - # that end with backslashes. - if (file_extension == 'h' - and Search(r'\bnamespace\s*{', line) - and line[-1] != '\\'): - error(filename, linenum, 'build/namespaces', 4, - 'Do not use unnamed namespaces in header files. See ' - 'http://google-styleguide.googlecode.com/svn/trunk/cppguide.xml#Namespaces' - ' for more information.') - -def CheckForNonConstReference(filename, clean_lines, linenum, - nesting_state, error): - """Check for non-const references. - - Separate from CheckLanguage since it scans backwards from current - line, instead of scanning forward. - - Args: - filename: The name of the current file. - clean_lines: A CleansedLines instance containing the file. - linenum: The number of the line to check. - nesting_state: A _NestingState instance which maintains information about - the current stack of nested blocks being parsed. - error: The function to call with any errors found. - """ - # Do nothing if there is no '&' on current line. - line = clean_lines.elided[linenum] - if '&' not in line: - return - - # Long type names may be broken across multiple lines, usually in one - # of these forms: - # LongType - # ::LongTypeContinued &identifier - # LongType:: - # LongTypeContinued &identifier - # LongType< - # ...>::LongTypeContinued &identifier - # - # If we detected a type split across two lines, join the previous - # line to current line so that we can match const references - # accordingly. - # - # Note that this only scans back one line, since scanning back - # arbitrary number of lines would be expensive. If you have a type - # that spans more than 2 lines, please use a typedef. - if linenum > 1: - previous = None - if Match(r'\s*::(?:[\w<>]|::)+\s*&\s*\S', line): - # previous_line\n + ::current_line - previous = Search(r'\b((?:const\s*)?(?:[\w<>]|::)+[\w<>])\s*$', - clean_lines.elided[linenum - 1]) - elif Match(r'\s*[a-zA-Z_]([\w<>]|::)+\s*&\s*\S', line): - # previous_line::\n + current_line - previous = Search(r'\b((?:const\s*)?(?:[\w<>]|::)+::)\s*$', - clean_lines.elided[linenum - 1]) - if previous: - line = previous.group(1) + line.lstrip() - else: - # Check for templated parameter that is split across multiple lines - endpos = line.rfind('>') - if endpos > -1: - (_, startline, startpos) = ReverseCloseExpression( - clean_lines, linenum, endpos) - if startpos > -1 and startline < linenum: - # Found the matching < on an earlier line, collect all - # pieces up to current line. - line = '' - for i in xrange(startline, linenum + 1): - line += clean_lines.elided[i].strip() - - # Check for non-const references in function parameters. A single '&' may - # found in the following places: - # inside expression: binary & for bitwise AND - # inside expression: unary & for taking the address of something - # inside declarators: reference parameter - # We will exclude the first two cases by checking that we are not inside a - # function body, including one that was just introduced by a trailing '{'. - # TODO(unknwon): Doesn't account for preprocessor directives. - # TODO(unknown): Doesn't account for 'catch(Exception& e)' [rare]. - check_params = False - if not nesting_state.stack: - check_params = True # top level - elif (isinstance(nesting_state.stack[-1], _ClassInfo) or - isinstance(nesting_state.stack[-1], _NamespaceInfo)): - check_params = True # within class or namespace - elif Match(r'.*{\s*$', line): - if (len(nesting_state.stack) == 1 or - isinstance(nesting_state.stack[-2], _ClassInfo) or - isinstance(nesting_state.stack[-2], _NamespaceInfo)): - check_params = True # just opened global/class/namespace block - # We allow non-const references in a few standard places, like functions - # called "swap()" or iostream operators like "<<" or ">>". Do not check - # those function parameters. - # - # We also accept & in static_assert, which looks like a function but - # it's actually a declaration expression. - whitelisted_functions = (r'(?:[sS]wap(?:<\w:+>)?|' - r'operator\s*[<>][<>]|' - r'static_assert|COMPILE_ASSERT' - r')\s*\(') - if Search(whitelisted_functions, line): - check_params = False - elif not Search(r'\S+\([^)]*$', line): - # Don't see a whitelisted function on this line. Actually we - # didn't see any function name on this line, so this is likely a - # multi-line parameter list. Try a bit harder to catch this case. - for i in xrange(2): - if (linenum > i and - Search(whitelisted_functions, clean_lines.elided[linenum - i - 1])): - check_params = False - break - - if check_params: - decls = ReplaceAll(r'{[^}]*}', ' ', line) # exclude function body - for parameter in re.findall(_RE_PATTERN_REF_PARAM, decls): - if not Match(_RE_PATTERN_CONST_REF_PARAM, parameter): - error(filename, linenum, 'runtime/references', 2, - 'Is this a non-const reference? ' - 'If so, make const or use a pointer: ' + - ReplaceAll(' *<', '<', parameter)) - - -def CheckCStyleCast(filename, linenum, line, raw_line, cast_type, pattern, - error): - """Checks for a C-style cast by looking for the pattern. - - Args: - filename: The name of the current file. - linenum: The number of the line to check. - line: The line of code to check. - raw_line: The raw line of code to check, with comments. - cast_type: The string for the C++ cast to recommend. This is either - reinterpret_cast, static_cast, or const_cast, depending. - pattern: The regular expression used to find C-style casts. - error: The function to call with any errors found. - - Returns: - True if an error was emitted. - False otherwise. - """ - match = Search(pattern, line) - if not match: - return False - - # e.g., sizeof(int) - sizeof_match = Match(r'.*sizeof\s*$', line[0:match.start(1) - 1]) - if sizeof_match: - error(filename, linenum, 'runtime/sizeof', 1, - 'Using sizeof(type). Use sizeof(varname) instead if possible') - return True - - # operator++(int) and operator--(int) - if (line[0:match.start(1) - 1].endswith(' operator++') or - line[0:match.start(1) - 1].endswith(' operator--')): - return False - - # A single unnamed argument for a function tends to look like old - # style cast. If we see those, don't issue warnings for deprecated - # casts, instead issue warnings for unnamed arguments where - # appropriate. - # - # These are things that we want warnings for, since the style guide - # explicitly require all parameters to be named: - # Function(int); - # Function(int) { - # ConstMember(int) const; - # ConstMember(int) const { - # ExceptionMember(int) throw (...); - # ExceptionMember(int) throw (...) { - # PureVirtual(int) = 0; - # - # These are functions of some sort, where the compiler would be fine - # if they had named parameters, but people often omit those - # identifiers to reduce clutter: - # (FunctionPointer)(int); - # (FunctionPointer)(int) = value; - # Function((function_pointer_arg)(int)) - # <TemplateArgument(int)>; - # <(FunctionPointerTemplateArgument)(int)>; - remainder = line[match.end(0):] - if Match(r'^\s*(?:;|const\b|throw\b|=|>|\{|\))', remainder): - # Looks like an unnamed parameter. - - # Don't warn on any kind of template arguments. - if Match(r'^\s*>', remainder): - return False - - # Don't warn on assignments to function pointers, but keep warnings for - # unnamed parameters to pure virtual functions. Note that this pattern - # will also pass on assignments of "0" to function pointers, but the - # preferred values for those would be "nullptr" or "NULL". - matched_zero = Match(r'^\s=\s*(\S+)\s*;', remainder) - if matched_zero and matched_zero.group(1) != '0': - return False - - # Don't warn on function pointer declarations. For this we need - # to check what came before the "(type)" string. - if Match(r'.*\)\s*$', line[0:match.start(0)]): - return False - - # Don't warn if the parameter is named with block comments, e.g.: - # Function(int /*unused_param*/); - if '/*' in raw_line: - return False - - # Passed all filters, issue warning here. - error(filename, linenum, 'readability/function', 3, - 'All parameters should be named in a function') - return True - - # At this point, all that should be left is actual casts. - error(filename, linenum, 'readability/casting', 4, - 'Using C-style cast. Use %s<%s>(...) instead' % - (cast_type, match.group(1))) - - return True - - -_HEADERS_CONTAINING_TEMPLATES = ( - ('<deque>', ('deque',)), - ('<functional>', ('unary_function', 'binary_function', - 'plus', 'minus', 'multiplies', 'divides', 'modulus', - 'negate', - 'equal_to', 'not_equal_to', 'greater', 'less', - 'greater_equal', 'less_equal', - 'logical_and', 'logical_or', 'logical_not', - 'unary_negate', 'not1', 'binary_negate', 'not2', - 'bind1st', 'bind2nd', - 'pointer_to_unary_function', - 'pointer_to_binary_function', - 'ptr_fun', - 'mem_fun_t', 'mem_fun', 'mem_fun1_t', 'mem_fun1_ref_t', - 'mem_fun_ref_t', - 'const_mem_fun_t', 'const_mem_fun1_t', - 'const_mem_fun_ref_t', 'const_mem_fun1_ref_t', - 'mem_fun_ref', - )), - ('<limits>', ('numeric_limits',)), - ('<list>', ('list',)), - ('<map>', ('map', 'multimap',)), - ('<memory>', ('allocator',)), - ('<queue>', ('queue', 'priority_queue',)), - ('<set>', ('set', 'multiset',)), - ('<stack>', ('stack',)), - ('<string>', ('char_traits', 'basic_string',)), - ('<utility>', ('pair',)), - ('<vector>', ('vector',)), - - # gcc extensions. - # Note: std::hash is their hash, ::hash is our hash - ('<hash_map>', ('hash_map', 'hash_multimap',)), - ('<hash_set>', ('hash_set', 'hash_multiset',)), - ('<slist>', ('slist',)), - ) - -_RE_PATTERN_STRING = re.compile(r'\bstring\b') - -_re_pattern_algorithm_header = [] -for _template in ('copy', 'max', 'min', 'min_element', 'sort', 'swap', - 'transform'): - # Match max<type>(..., ...), max(..., ...), but not foo->max, foo.max or - # type::max(). - _re_pattern_algorithm_header.append( - (re.compile(r'[^>.]\b' + _template + r'(<.*?>)?\([^\)]'), - _template, - '<algorithm>')) - -_re_pattern_templates = [] -for _header, _templates in _HEADERS_CONTAINING_TEMPLATES: - for _template in _templates: - _re_pattern_templates.append( - (re.compile(r'(\<|\b)' + _template + r'\s*\<'), - _template + '<>', - _header)) - - -def FilesBelongToSameModule(filename_cc, filename_h): - """Check if these two filenames belong to the same module. - - The concept of a 'module' here is a as follows: - foo.h, foo-inl.h, foo.cc, foo_test.cc and foo_unittest.cc belong to the - same 'module' if they are in the same directory. - some/path/public/xyzzy and some/path/internal/xyzzy are also considered - to belong to the same module here. - - If the filename_cc contains a longer path than the filename_h, for example, - '/absolute/path/to/base/sysinfo.cc', and this file would include - 'base/sysinfo.h', this function also produces the prefix needed to open the - header. This is used by the caller of this function to more robustly open the - header file. We don't have access to the real include paths in this context, - so we need this guesswork here. - - Known bugs: tools/base/bar.cc and base/bar.h belong to the same module - according to this implementation. Because of this, this function gives - some false positives. This should be sufficiently rare in practice. - - Args: - filename_cc: is the path for the .cc file - filename_h: is the path for the header path - - Returns: - Tuple with a bool and a string: - bool: True if filename_cc and filename_h belong to the same module. - string: the additional prefix needed to open the header file. - """ - - if not filename_cc.endswith('.cc'): - return (False, '') - filename_cc = filename_cc[:-len('.cc')] - if filename_cc.endswith('_unittest'): - filename_cc = filename_cc[:-len('_unittest')] - elif filename_cc.endswith('_test'): - filename_cc = filename_cc[:-len('_test')] - filename_cc = filename_cc.replace('/public/', '/') - filename_cc = filename_cc.replace('/internal/', '/') - - if not filename_h.endswith('.h'): - return (False, '') - filename_h = filename_h[:-len('.h')] - if filename_h.endswith('-inl'): - filename_h = filename_h[:-len('-inl')] - filename_h = filename_h.replace('/public/', '/') - filename_h = filename_h.replace('/internal/', '/') - - files_belong_to_same_module = filename_cc.endswith(filename_h) - common_path = '' - if files_belong_to_same_module: - common_path = filename_cc[:-len(filename_h)] - return files_belong_to_same_module, common_path - - -def UpdateIncludeState(filename, include_state, io=codecs): - """Fill up the include_state with new includes found from the file. - - Args: - filename: the name of the header to read. - include_state: an _IncludeState instance in which the headers are inserted. - io: The io factory to use to read the file. Provided for testability. - - Returns: - True if a header was succesfully added. False otherwise. - """ - headerfile = None - try: - headerfile = io.open(filename, 'r', 'utf8', 'replace') - except IOError: - return False - linenum = 0 - for line in headerfile: - linenum += 1 - clean_line = CleanseComments(line) - match = _RE_PATTERN_INCLUDE.search(clean_line) - if match: - include = match.group(2) - # The value formatting is cute, but not really used right now. - # What matters here is that the key is in include_state. - include_state.setdefault(include, '%s:%d' % (filename, linenum)) - return True - - -def CheckForIncludeWhatYouUse(filename, clean_lines, include_state, error, - io=codecs): - """Reports for missing stl includes. - - This function will output warnings to make sure you are including the headers - necessary for the stl containers and functions that you use. We only give one - reason to include a header. For example, if you use both equal_to<> and - less<> in a .h file, only one (the latter in the file) of these will be - reported as a reason to include the <functional>. - - Args: - filename: The name of the current file. - clean_lines: A CleansedLines instance containing the file. - include_state: An _IncludeState instance. - error: The function to call with any errors found. - io: The IO factory to use to read the header file. Provided for unittest - injection. - """ - required = {} # A map of header name to linenumber and the template entity. - # Example of required: { '<functional>': (1219, 'less<>') } - - for linenum in xrange(clean_lines.NumLines()): - line = clean_lines.elided[linenum] - if not line or line[0] == '#': - continue - - # String is special -- it is a non-templatized type in STL. - matched = _RE_PATTERN_STRING.search(line) - if matched: - # Don't warn about strings in non-STL namespaces: - # (We check only the first match per line; good enough.) - prefix = line[:matched.start()] - if prefix.endswith('std::') or not prefix.endswith('::'): - required['<string>'] = (linenum, 'string') - - for pattern, template, header in _re_pattern_algorithm_header: - if pattern.search(line): - required[header] = (linenum, template) - - # The following function is just a speed up, no semantics are changed. - if not '<' in line: # Reduces the cpu time usage by skipping lines. - continue - - for pattern, template, header in _re_pattern_templates: - if pattern.search(line): - required[header] = (linenum, template) - - # The policy is that if you #include something in foo.h you don't need to - # include it again in foo.cc. Here, we will look at possible includes. - # Let's copy the include_state so it is only messed up within this function. - include_state = include_state.copy() - - # Did we find the header for this file (if any) and succesfully load it? - header_found = False - - # Use the absolute path so that matching works properly. - abs_filename = FileInfo(filename).FullName() - - # For Emacs's flymake. - # If cpplint is invoked from Emacs's flymake, a temporary file is generated - # by flymake and that file name might end with '_flymake.cc'. In that case, - # restore original file name here so that the corresponding header file can be - # found. - # e.g. If the file name is 'foo_flymake.cc', we should search for 'foo.h' - # instead of 'foo_flymake.h' - abs_filename = re.sub(r'_flymake\.cc$', '.cc', abs_filename) - - # include_state is modified during iteration, so we iterate over a copy of - # the keys. - header_keys = include_state.keys() - for header in header_keys: - (same_module, common_path) = FilesBelongToSameModule(abs_filename, header) - fullpath = common_path + header - if same_module and UpdateIncludeState(fullpath, include_state, io): - header_found = True - - # If we can't find the header file for a .cc, assume it's because we don't - # know where to look. In that case we'll give up as we're not sure they - # didn't include it in the .h file. - # TODO(unknown): Do a better job of finding .h files so we are confident that - # not having the .h file means there isn't one. - if filename.endswith('.cc') and not header_found: - return - - # All the lines have been processed, report the errors found. - for required_header_unstripped in required: - template = required[required_header_unstripped][1] - if required_header_unstripped.strip('<>"') not in include_state: - error(filename, required[required_header_unstripped][0], - 'build/include_what_you_use', 4, - 'Add #include ' + required_header_unstripped + ' for ' + template) - - -_RE_PATTERN_EXPLICIT_MAKEPAIR = re.compile(r'\bmake_pair\s*<') - - -def CheckMakePairUsesDeduction(filename, clean_lines, linenum, error): - """Check that make_pair's template arguments are deduced. - - G++ 4.6 in C++0x mode fails badly if make_pair's template arguments are - specified explicitly, and such use isn't intended in any case. - - Args: - filename: The name of the current file. - clean_lines: A CleansedLines instance containing the file. - linenum: The number of the line to check. - error: The function to call with any errors found. - """ - line = clean_lines.elided[linenum] - match = _RE_PATTERN_EXPLICIT_MAKEPAIR.search(line) - if match: - error(filename, linenum, 'build/explicit_make_pair', - 4, # 4 = high confidence - 'For C++11-compatibility, omit template arguments from make_pair' - ' OR use pair directly OR if appropriate, construct a pair directly') - - -def ProcessLine(filename, file_extension, clean_lines, line, - include_state, function_state, nesting_state, error, - extra_check_functions=[]): - """Processes a single line in the file. - - Args: - filename: Filename of the file that is being processed. - file_extension: The extension (dot not included) of the file. - clean_lines: An array of strings, each representing a line of the file, - with comments stripped. - line: Number of line being processed. - include_state: An _IncludeState instance in which the headers are inserted. - function_state: A _FunctionState instance which counts function lines, etc. - nesting_state: A _NestingState instance which maintains information about - the current stack of nested blocks being parsed. - error: A callable to which errors are reported, which takes 4 arguments: - filename, line number, error level, and message - extra_check_functions: An array of additional check functions that will be - run on each source line. Each function takes 4 - arguments: filename, clean_lines, line, error - """ - raw_lines = clean_lines.raw_lines - ParseNolintSuppressions(filename, raw_lines[line], line, error) - nesting_state.Update(filename, clean_lines, line, error) - if nesting_state.stack and nesting_state.stack[-1].inline_asm != _NO_ASM: - return - CheckForFunctionLengths(filename, clean_lines, line, function_state, error) - CheckForMultilineCommentsAndStrings(filename, clean_lines, line, error) - CheckStyle(filename, clean_lines, line, file_extension, nesting_state, error) - CheckLanguage(filename, clean_lines, line, file_extension, include_state, - nesting_state, error) - CheckForNonConstReference(filename, clean_lines, line, nesting_state, error) - CheckForNonStandardConstructs(filename, clean_lines, line, - nesting_state, error) - CheckVlogArguments(filename, clean_lines, line, error) - CheckPosixThreading(filename, clean_lines, line, error) - CheckInvalidIncrement(filename, clean_lines, line, error) - CheckMakePairUsesDeduction(filename, clean_lines, line, error) - for check_fn in extra_check_functions: - check_fn(filename, clean_lines, line, error) - -def ProcessFileData(filename, file_extension, lines, error, - extra_check_functions=[]): - """Performs lint checks and reports any errors to the given error function. - - Args: - filename: Filename of the file that is being processed. - file_extension: The extension (dot not included) of the file. - lines: An array of strings, each representing a line of the file, with the - last element being empty if the file is terminated with a newline. - error: A callable to which errors are reported, which takes 4 arguments: - filename, line number, error level, and message - extra_check_functions: An array of additional check functions that will be - run on each source line. Each function takes 4 - arguments: filename, clean_lines, line, error - """ - lines = (['// marker so line numbers and indices both start at 1'] + lines + - ['// marker so line numbers end in a known way']) - - include_state = _IncludeState() - function_state = _FunctionState() - nesting_state = _NestingState() - - ResetNolintSuppressions() - - CheckForCopyright(filename, lines, error) - - if file_extension == 'h': - CheckForHeaderGuard(filename, lines, error) - - RemoveMultiLineComments(filename, lines, error) - clean_lines = CleansedLines(lines) - for line in xrange(clean_lines.NumLines()): - ProcessLine(filename, file_extension, clean_lines, line, - include_state, function_state, nesting_state, error, - extra_check_functions) - nesting_state.CheckCompletedBlocks(filename, error) - - CheckForIncludeWhatYouUse(filename, clean_lines, include_state, error) - - # We check here rather than inside ProcessLine so that we see raw - # lines rather than "cleaned" lines. - CheckForBadCharacters(filename, lines, error) - - CheckForNewlineAtEOF(filename, lines, error) - -def ProcessFile(filename, vlevel, extra_check_functions=[]): - """Does google-lint on a single file. - - Args: - filename: The name of the file to parse. - - vlevel: The level of errors to report. Every error of confidence - >= verbose_level will be reported. 0 is a good default. - - extra_check_functions: An array of additional check functions that will be - run on each source line. Each function takes 4 - arguments: filename, clean_lines, line, error - """ - - _SetVerboseLevel(vlevel) - - try: - # Support the UNIX convention of using "-" for stdin. Note that - # we are not opening the file with universal newline support - # (which codecs doesn't support anyway), so the resulting lines do - # contain trailing '\r' characters if we are reading a file that - # has CRLF endings. - # If after the split a trailing '\r' is present, it is removed - # below. If it is not expected to be present (i.e. os.linesep != - # '\r\n' as in Windows), a warning is issued below if this file - # is processed. - - if filename == '-': - lines = codecs.StreamReaderWriter(sys.stdin, - codecs.getreader('utf8'), - codecs.getwriter('utf8'), - 'replace').read().split('\n') - else: - lines = codecs.open(filename, 'r', 'utf8', 'replace').read().split('\n') - - carriage_return_found = False - # Remove trailing '\r'. - for linenum in range(len(lines)): - if lines[linenum].endswith('\r'): - lines[linenum] = lines[linenum].rstrip('\r') - carriage_return_found = True - - except IOError: - sys.stderr.write( - "Skipping input '%s': Can't open for reading\n" % filename) - return - - # Note, if no dot is found, this will give the entire filename as the ext. - file_extension = filename[filename.rfind('.') + 1:] - - # When reading from stdin, the extension is unknown, so no cpplint tests - # should rely on the extension. - if filename != '-' and file_extension not in _valid_extensions: - sys.stderr.write('Ignoring %s; not a valid file name ' - '(%s)\n' % (filename, ', '.join(_valid_extensions))) - else: - ProcessFileData(filename, file_extension, lines, Error, - extra_check_functions) - if carriage_return_found and os.linesep != '\r\n': - # Use 0 for linenum since outputting only one error for potentially - # several lines. - Error(filename, 0, 'whitespace/newline', 1, - 'One or more unexpected \\r (^M) found;' - 'better to use only a \\n') - - sys.stderr.write('Done processing %s\n' % filename) - - -def PrintUsage(message): - """Prints a brief usage string and exits, optionally with an error message. - - Args: - message: The optional error message. - """ - sys.stderr.write(_USAGE) - if message: - sys.exit('\nFATAL ERROR: ' + message) - else: - sys.exit(1) - - -def PrintCategories(): - """Prints a list of all the error-categories used by error messages. - - These are the categories used to filter messages via --filter. - """ - sys.stderr.write(''.join(' %s\n' % cat for cat in _ERROR_CATEGORIES)) - sys.exit(0) - - -def ParseArguments(args): - """Parses the command line arguments. - - This may set the output format and verbosity level as side-effects. - - Args: - args: The command line arguments: - - Returns: - The list of filenames to lint. - """ - try: - (opts, filenames) = getopt.getopt(args, '', ['help', 'output=', 'verbose=', - 'counting=', - 'filter=', - 'root=', - 'linelength=', - 'extensions=']) - except getopt.GetoptError: - PrintUsage('Invalid arguments.') - - verbosity = _VerboseLevel() - output_format = _OutputFormat() - filters = '' - counting_style = '' - - for (opt, val) in opts: - if opt == '--help': - PrintUsage(None) - elif opt == '--output': - if val not in ('emacs', 'vs7', 'eclipse'): - PrintUsage('The only allowed output formats are emacs, vs7 and eclipse.') - output_format = val - elif opt == '--verbose': - verbosity = int(val) - elif opt == '--filter': - filters = val - if not filters: - PrintCategories() - elif opt == '--counting': - if val not in ('total', 'toplevel', 'detailed'): - PrintUsage('Valid counting options are total, toplevel, and detailed') - counting_style = val - elif opt == '--root': - global _root - _root = val - elif opt == '--linelength': - global _line_length - try: - _line_length = int(val) - except ValueError: - PrintUsage('Line length must be digits.') - elif opt == '--extensions': - global _valid_extensions - try: - _valid_extensions = set(val.split(',')) - except ValueError: - PrintUsage('Extensions must be comma seperated list.') - - if not filenames: - PrintUsage('No files were specified.') - - _SetOutputFormat(output_format) - _SetVerboseLevel(verbosity) - _SetFilters(filters) - _SetCountingStyle(counting_style) - - return filenames - - -def main(): - filenames = ParseArguments(sys.argv[1:]) - - # Change stderr to write with replacement characters so we don't die - # if we try to print something containing non-ASCII characters. - sys.stderr = codecs.StreamReaderWriter(sys.stderr, - codecs.getreader('utf8'), - codecs.getwriter('utf8'), - 'replace') - - _cpplint_state.ResetErrorCounts() - for filename in filenames: - ProcessFile(filename, _cpplint_state.verbose_level) - _cpplint_state.PrintErrorCounts() - - sys.exit(_cpplint_state.error_count > 0) - - -if __name__ == '__main__': - main() diff --git a/third_party/aom/tools/diff.py b/third_party/aom/tools/diff.py deleted file mode 100644 index bac6aabdc..000000000 --- a/third_party/aom/tools/diff.py +++ /dev/null @@ -1,132 +0,0 @@ -#!/usr/bin/env python -## -## Copyright (c) 2016, Alliance for Open Media. All rights reserved -## -## This source code is subject to the terms of the BSD 2 Clause License and -## the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License -## was not distributed with this source code in the LICENSE file, you can -## obtain it at www.aomedia.org/license/software. If the Alliance for Open -## Media Patent License 1.0 was not distributed with this source code in the -## PATENTS file, you can obtain it at www.aomedia.org/license/patent. -## -"""Classes for representing diff pieces.""" - -__author__ = "jkoleszar@google.com" - -import re - - -class DiffLines(object): - """A container for one half of a diff.""" - - def __init__(self, filename, offset, length): - self.filename = filename - self.offset = offset - self.length = length - self.lines = [] - self.delta_line_nums = [] - - def Append(self, line): - l = len(self.lines) - if line[0] != " ": - self.delta_line_nums.append(self.offset + l) - self.lines.append(line[1:]) - assert l+1 <= self.length - - def Complete(self): - return len(self.lines) == self.length - - def __contains__(self, item): - return item >= self.offset and item <= self.offset + self.length - 1 - - -class DiffHunk(object): - """A container for one diff hunk, consisting of two DiffLines.""" - - def __init__(self, header, file_a, file_b, start_a, len_a, start_b, len_b): - self.header = header - self.left = DiffLines(file_a, start_a, len_a) - self.right = DiffLines(file_b, start_b, len_b) - self.lines = [] - - def Append(self, line): - """Adds a line to the DiffHunk and its DiffLines children.""" - if line[0] == "-": - self.left.Append(line) - elif line[0] == "+": - self.right.Append(line) - elif line[0] == " ": - self.left.Append(line) - self.right.Append(line) - elif line[0] == "\\": - # Ignore newline messages from git diff. - pass - else: - assert False, ("Unrecognized character at start of diff line " - "%r" % line[0]) - self.lines.append(line) - - def Complete(self): - return self.left.Complete() and self.right.Complete() - - def __repr__(self): - return "DiffHunk(%s, %s, len %d)" % ( - self.left.filename, self.right.filename, - max(self.left.length, self.right.length)) - - -def ParseDiffHunks(stream): - """Walk a file-like object, yielding DiffHunks as they're parsed.""" - - file_regex = re.compile(r"(\+\+\+|---) (\S+)") - range_regex = re.compile(r"@@ -(\d+)(,(\d+))? \+(\d+)(,(\d+))?") - hunk = None - while True: - line = stream.readline() - if not line: - break - - if hunk is None: - # Parse file names - diff_file = file_regex.match(line) - if diff_file: - if line.startswith("---"): - a_line = line - a = diff_file.group(2) - continue - if line.startswith("+++"): - b_line = line - b = diff_file.group(2) - continue - - # Parse offset/lengths - diffrange = range_regex.match(line) - if diffrange: - if diffrange.group(2): - start_a = int(diffrange.group(1)) - len_a = int(diffrange.group(3)) - else: - start_a = 1 - len_a = int(diffrange.group(1)) - - if diffrange.group(5): - start_b = int(diffrange.group(4)) - len_b = int(diffrange.group(6)) - else: - start_b = 1 - len_b = int(diffrange.group(4)) - - header = [a_line, b_line, line] - hunk = DiffHunk(header, a, b, start_a, len_a, start_b, len_b) - else: - # Add the current line to the hunk - hunk.Append(line) - - # See if the whole hunk has been parsed. If so, yield it and prepare - # for the next hunk. - if hunk.Complete(): - yield hunk - hunk = None - - # Partial hunks are a parse error - assert hunk is None diff --git a/third_party/aom/tools/dump_obu.cc b/third_party/aom/tools/dump_obu.cc deleted file mode 100644 index 30ee5e7a1..000000000 --- a/third_party/aom/tools/dump_obu.cc +++ /dev/null @@ -1,164 +0,0 @@ -/* - * Copyright (c) 2017, Alliance for Open Media. All rights reserved - * - * This source code is subject to the terms of the BSD 2 Clause License and - * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License - * was not distributed with this source code in the LICENSE file, you can - * obtain it at www.aomedia.org/license/software. If the Alliance for Open - * Media Patent License 1.0 was not distributed with this source code in the - * PATENTS file, you can obtain it at www.aomedia.org/license/patent. - */ - -#include <stdlib.h> -#include <string.h> - -#include <memory> -#include <string> - -#include "config/aom_config.h" - -#include "common/ivfdec.h" -#include "common/obudec.h" -#include "common/tools_common.h" -#include "common/webmdec.h" -#include "tools/obu_parser.h" - -namespace { - -const size_t kInitialBufferSize = 100 * 1024; - -struct InputContext { - InputContext() = default; - ~InputContext() { free(unit_buffer); } - - void Init() { - memset(avx_ctx, 0, sizeof(*avx_ctx)); - memset(obu_ctx, 0, sizeof(*obu_ctx)); - obu_ctx->avx_ctx = avx_ctx; -#if CONFIG_WEBM_IO - memset(webm_ctx, 0, sizeof(*webm_ctx)); -#endif - } - - AvxInputContext *avx_ctx = nullptr; - ObuDecInputContext *obu_ctx = nullptr; -#if CONFIG_WEBM_IO - WebmInputContext *webm_ctx = nullptr; -#endif - uint8_t *unit_buffer = nullptr; - size_t unit_buffer_size = 0; -}; - -void PrintUsage() { - printf("Libaom OBU dump.\nUsage: dump_obu <input_file>\n"); -} - -VideoFileType GetFileType(InputContext *ctx) { - if (file_is_ivf(ctx->avx_ctx)) return FILE_TYPE_IVF; - if (file_is_obu(ctx->obu_ctx)) return FILE_TYPE_OBU; -#if CONFIG_WEBM_IO - if (file_is_webm(ctx->webm_ctx, ctx->avx_ctx)) return FILE_TYPE_WEBM; -#endif - return FILE_TYPE_RAW; -} - -bool ReadTemporalUnit(InputContext *ctx, size_t *unit_size) { - const VideoFileType file_type = ctx->avx_ctx->file_type; - switch (file_type) { - case FILE_TYPE_IVF: { - if (ivf_read_frame(ctx->avx_ctx->file, &ctx->unit_buffer, unit_size, - &ctx->unit_buffer_size, NULL)) { - return false; - } - break; - } - case FILE_TYPE_OBU: { - if (obudec_read_temporal_unit(ctx->obu_ctx, &ctx->unit_buffer, unit_size, - &ctx->unit_buffer_size)) { - return false; - } - break; - } -#if CONFIG_WEBM_IO - case FILE_TYPE_WEBM: { - if (webm_read_frame(ctx->webm_ctx, &ctx->unit_buffer, unit_size, - &ctx->unit_buffer_size)) { - return false; - } - break; - } -#endif - default: - // TODO(tomfinegan): Abuse FILE_TYPE_RAW for AV1/OBU elementary streams? - fprintf(stderr, "Error: Unsupported file type.\n"); - return false; - } - - return true; -} - -} // namespace - -int main(int argc, const char *argv[]) { - // TODO(tomfinegan): Could do with some params for verbosity. - if (argc < 2) { - PrintUsage(); - return EXIT_SUCCESS; - } - - const std::string filename = argv[1]; - - using FilePtr = std::unique_ptr<FILE, decltype(&fclose)>; - FilePtr input_file(fopen(filename.c_str(), "rb"), &fclose); - if (input_file.get() == nullptr) { - input_file.release(); - fprintf(stderr, "Error: Cannot open input file.\n"); - return EXIT_FAILURE; - } - - AvxInputContext avx_ctx; - InputContext input_ctx; - input_ctx.avx_ctx = &avx_ctx; - ObuDecInputContext obu_ctx; - input_ctx.obu_ctx = &obu_ctx; -#if CONFIG_WEBM_IO - WebmInputContext webm_ctx; - input_ctx.webm_ctx = &webm_ctx; -#endif - - input_ctx.Init(); - avx_ctx.file = input_file.get(); - avx_ctx.file_type = GetFileType(&input_ctx); - - // Note: the reader utilities will realloc the buffer using realloc() etc. - // Can't have nice things like unique_ptr wrappers with that type of - // behavior underneath the function calls. - input_ctx.unit_buffer = - reinterpret_cast<uint8_t *>(calloc(kInitialBufferSize, 1)); - if (!input_ctx.unit_buffer) { - fprintf(stderr, "Error: No memory, can't alloc input buffer.\n"); - return EXIT_FAILURE; - } - input_ctx.unit_buffer_size = kInitialBufferSize; - - size_t unit_size = 0; - int unit_number = 0; - int64_t obu_overhead_bytes_total = 0; - while (ReadTemporalUnit(&input_ctx, &unit_size)) { - printf("Temporal unit %d\n", unit_number); - - int obu_overhead_current_unit = 0; - if (!aom_tools::DumpObu(input_ctx.unit_buffer, static_cast<int>(unit_size), - &obu_overhead_current_unit)) { - fprintf(stderr, "Error: Temporal Unit parse failed on unit number %d.\n", - unit_number); - return EXIT_FAILURE; - } - printf(" OBU overhead: %d\n", obu_overhead_current_unit); - ++unit_number; - obu_overhead_bytes_total += obu_overhead_current_unit; - } - - printf("File total OBU overhead: %" PRId64 "\n", obu_overhead_bytes_total); - return EXIT_SUCCESS; -} diff --git a/third_party/aom/tools/gen_authors.sh b/third_party/aom/tools/gen_authors.sh deleted file mode 100755 index 5def8bc89..000000000 --- a/third_party/aom/tools/gen_authors.sh +++ /dev/null @@ -1,10 +0,0 @@ -#!/bin/bash - -# Add organization names manually. - -cat <<EOF -# This file is automatically generated from the git commit history -# by tools/gen_authors.sh. - -$(git log --pretty=format:"%aN <%aE>" | sort | uniq | grep -v "corp.google\|clang-format") -EOF diff --git a/third_party/aom/tools/gen_constrained_tokenset.py b/third_party/aom/tools/gen_constrained_tokenset.py deleted file mode 100755 index 5d12ee1ef..000000000 --- a/third_party/aom/tools/gen_constrained_tokenset.py +++ /dev/null @@ -1,120 +0,0 @@ -#!/usr/bin/python -## -## Copyright (c) 2016, Alliance for Open Media. All rights reserved -## -## This source code is subject to the terms of the BSD 2 Clause License and -## the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License -## was not distributed with this source code in the LICENSE file, you can -## obtain it at www.aomedia.org/license/software. If the Alliance for Open -## Media Patent License 1.0 was not distributed with this source code in the -## PATENTS file, you can obtain it at www.aomedia.org/license/patent. -## -"""Generate the probability model for the constrained token set. - -Model obtained from a 2-sided zero-centered distribution derived -from a Pareto distribution. The cdf of the distribution is: -cdf(x) = 0.5 + 0.5 * sgn(x) * [1 - {alpha/(alpha + |x|)} ^ beta] - -For a given beta and a given probability of the 1-node, the alpha -is first solved, and then the {alpha, beta} pair is used to generate -the probabilities for the rest of the nodes. -""" - -import heapq -import sys -import numpy as np -import scipy.optimize -import scipy.stats - - -def cdf_spareto(x, xm, beta): - p = 1 - (xm / (np.abs(x) + xm))**beta - p = 0.5 + 0.5 * np.sign(x) * p - return p - - -def get_spareto(p, beta): - cdf = cdf_spareto - - def func(x): - return ((cdf(1.5, x, beta) - cdf(0.5, x, beta)) / - (1 - cdf(0.5, x, beta)) - p)**2 - - alpha = scipy.optimize.fminbound(func, 1e-12, 10000, xtol=1e-12) - parray = np.zeros(11) - parray[0] = 2 * (cdf(0.5, alpha, beta) - 0.5) - parray[1] = (2 * (cdf(1.5, alpha, beta) - cdf(0.5, alpha, beta))) - parray[2] = (2 * (cdf(2.5, alpha, beta) - cdf(1.5, alpha, beta))) - parray[3] = (2 * (cdf(3.5, alpha, beta) - cdf(2.5, alpha, beta))) - parray[4] = (2 * (cdf(4.5, alpha, beta) - cdf(3.5, alpha, beta))) - parray[5] = (2 * (cdf(6.5, alpha, beta) - cdf(4.5, alpha, beta))) - parray[6] = (2 * (cdf(10.5, alpha, beta) - cdf(6.5, alpha, beta))) - parray[7] = (2 * (cdf(18.5, alpha, beta) - cdf(10.5, alpha, beta))) - parray[8] = (2 * (cdf(34.5, alpha, beta) - cdf(18.5, alpha, beta))) - parray[9] = (2 * (cdf(66.5, alpha, beta) - cdf(34.5, alpha, beta))) - parray[10] = 2 * (1. - cdf(66.5, alpha, beta)) - return parray - - -def quantize_probs(p, save_first_bin, bits): - """Quantize probability precisely. - - Quantize probabilities minimizing dH (Kullback-Leibler divergence) - approximated by: sum (p_i-q_i)^2/p_i. - References: - https://en.wikipedia.org/wiki/Kullback%E2%80%93Leibler_divergence - https://github.com/JarekDuda/AsymmetricNumeralSystemsToolkit - """ - num_sym = p.size - p = np.clip(p, 1e-16, 1) - L = 2**bits - pL = p * L - ip = 1. / p # inverse probability - q = np.clip(np.round(pL), 1, L + 1 - num_sym) - quant_err = (pL - q)**2 * ip - sgn = np.sign(L - q.sum()) # direction of correction - if sgn != 0: # correction is needed - v = [] # heap of adjustment results (adjustment err, index) of each symbol - for i in range(1 if save_first_bin else 0, num_sym): - q_adj = q[i] + sgn - if q_adj > 0 and q_adj < L: - adj_err = (pL[i] - q_adj)**2 * ip[i] - quant_err[i] - heapq.heappush(v, (adj_err, i)) - while q.sum() != L: - # apply lowest error adjustment - (adj_err, i) = heapq.heappop(v) - quant_err[i] += adj_err - q[i] += sgn - # calculate the cost of adjusting this symbol again - q_adj = q[i] + sgn - if q_adj > 0 and q_adj < L: - adj_err = (pL[i] - q_adj)**2 * ip[i] - quant_err[i] - heapq.heappush(v, (adj_err, i)) - return q - - -def get_quantized_spareto(p, beta, bits, first_token): - parray = get_spareto(p, beta) - parray = parray[1:] / (1 - parray[0]) - # CONFIG_NEW_TOKENSET - if first_token > 1: - parray = parray[1:] / (1 - parray[0]) - qarray = quantize_probs(parray, first_token == 1, bits) - return qarray.astype(np.int) - - -def main(bits=15, first_token=1): - beta = 8 - for q in range(1, 256): - parray = get_quantized_spareto(q / 256., beta, bits, first_token) - assert parray.sum() == 2**bits - print '{', ', '.join('%d' % i for i in parray), '},' - - -if __name__ == '__main__': - if len(sys.argv) > 2: - main(int(sys.argv[1]), int(sys.argv[2])) - elif len(sys.argv) > 1: - main(int(sys.argv[1])) - else: - main() diff --git a/third_party/aom/tools/inspect-cli.js b/third_party/aom/tools/inspect-cli.js deleted file mode 100644 index a14c08111..000000000 --- a/third_party/aom/tools/inspect-cli.js +++ /dev/null @@ -1,39 +0,0 @@ -/** - * This tool lets you test if the compiled Javascript decoder is functioning properly. You'll - * need to download a SpiderMonkey js-shell to run this script. - * https://archive.mozilla.org/pub/firefox/nightly/latest-mozilla-central/ - * - * Example: - * js-shell inspect-cli.js video.ivf - */ -load("inspect.js"); -var buffer = read(scriptArgs[0], "binary"); -var Module = { - noExitRuntime: true, - noInitialRun: true, - preInit: [], - preRun: [], - postRun: [function () { - printErr(`Loaded Javascript Decoder OK`); - }], - memoryInitializerPrefixURL: "bin/", - arguments: ['input.ivf', 'output.raw'], - on_frame_decoded_json: function (jsonString) { - let json = JSON.parse("[" + Module.UTF8ToString(jsonString) + "null]"); - json.forEach(frame => { - if (frame) { - print(frame.frame); - } - }); - } -}; -DecoderModule(Module); -Module.FS.writeFile("/tmp/input.ivf", buffer, { encoding: "binary" }); -Module._open_file(); -Module._set_layers(0xFFFFFFFF); // Set this to zero if you want to benchmark decoding. -while(true) { - printErr("Decoding Frame ..."); - if (Module._read_frame()) { - break; - } -} diff --git a/third_party/aom/tools/inspect-post.js b/third_party/aom/tools/inspect-post.js deleted file mode 100644 index 31c40bb82..000000000 --- a/third_party/aom/tools/inspect-post.js +++ /dev/null @@ -1 +0,0 @@ -Module["FS"] = FS; diff --git a/third_party/aom/tools/intersect-diffs.py b/third_party/aom/tools/intersect-diffs.py deleted file mode 100755 index df13c4ef7..000000000 --- a/third_party/aom/tools/intersect-diffs.py +++ /dev/null @@ -1,78 +0,0 @@ -#!/usr/bin/env python -## -## Copyright (c) 2016, Alliance for Open Media. All rights reserved -## -## This source code is subject to the terms of the BSD 2 Clause License and -## the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License -## was not distributed with this source code in the LICENSE file, you can -## obtain it at www.aomedia.org/license/software. If the Alliance for Open -## Media Patent License 1.0 was not distributed with this source code in the -## PATENTS file, you can obtain it at www.aomedia.org/license/patent. -## -"""Calculates the "intersection" of two unified diffs. - -Given two diffs, A and B, it finds all hunks in B that had non-context lines -in A and prints them to stdout. This is useful to determine the hunks in B that -are relevant to A. The resulting file can be applied with patch(1) on top of A. -""" - -__author__ = "jkoleszar@google.com" - -import sys - -import diff - - -def FormatDiffHunks(hunks): - """Re-serialize a list of DiffHunks.""" - r = [] - last_header = None - for hunk in hunks: - this_header = hunk.header[0:2] - if last_header != this_header: - r.extend(hunk.header) - last_header = this_header - else: - r.extend(hunk.header[2]) - r.extend(hunk.lines) - r.append("\n") - return "".join(r) - - -def ZipHunks(rhs_hunks, lhs_hunks): - """Join two hunk lists on filename.""" - for rhs_hunk in rhs_hunks: - rhs_file = rhs_hunk.right.filename.split("/")[1:] - - for lhs_hunk in lhs_hunks: - lhs_file = lhs_hunk.left.filename.split("/")[1:] - if lhs_file != rhs_file: - continue - yield (rhs_hunk, lhs_hunk) - - -def main(): - old_hunks = [x for x in diff.ParseDiffHunks(open(sys.argv[1], "r"))] - new_hunks = [x for x in diff.ParseDiffHunks(open(sys.argv[2], "r"))] - out_hunks = [] - - # Join the right hand side of the older diff with the left hand side of the - # newer diff. - for old_hunk, new_hunk in ZipHunks(old_hunks, new_hunks): - if new_hunk in out_hunks: - continue - old_lines = old_hunk.right - new_lines = new_hunk.left - - # Determine if this hunk overlaps any non-context line from the other - for i in old_lines.delta_line_nums: - if i in new_lines: - out_hunks.append(new_hunk) - break - - if out_hunks: - print FormatDiffHunks(out_hunks) - sys.exit(1) - -if __name__ == "__main__": - main() diff --git a/third_party/aom/tools/lint-hunks.py b/third_party/aom/tools/lint-hunks.py deleted file mode 100755 index d02bee16c..000000000 --- a/third_party/aom/tools/lint-hunks.py +++ /dev/null @@ -1,146 +0,0 @@ -#!/usr/bin/python -## -## Copyright (c) 2016, Alliance for Open Media. All rights reserved -## -## This source code is subject to the terms of the BSD 2 Clause License and -## the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License -## was not distributed with this source code in the LICENSE file, you can -## obtain it at www.aomedia.org/license/software. If the Alliance for Open -## Media Patent License 1.0 was not distributed with this source code in the -## PATENTS file, you can obtain it at www.aomedia.org/license/patent. -## -"""Performs style checking on each diff hunk.""" -import getopt -import os -import StringIO -import subprocess -import sys - -import diff - - -SHORT_OPTIONS = "h" -LONG_OPTIONS = ["help"] - -TOPLEVEL_CMD = ["git", "rev-parse", "--show-toplevel"] -DIFF_CMD = ["git", "diff"] -DIFF_INDEX_CMD = ["git", "diff-index", "-u", "HEAD", "--"] -SHOW_CMD = ["git", "show"] -CPPLINT_FILTERS = ["-readability/casting"] - - -class Usage(Exception): - pass - - -class SubprocessException(Exception): - def __init__(self, args): - msg = "Failed to execute '%s'"%(" ".join(args)) - super(SubprocessException, self).__init__(msg) - - -class Subprocess(subprocess.Popen): - """Adds the notion of an expected returncode to Popen.""" - - def __init__(self, args, expected_returncode=0, **kwargs): - self._args = args - self._expected_returncode = expected_returncode - super(Subprocess, self).__init__(args, **kwargs) - - def communicate(self, *args, **kwargs): - result = super(Subprocess, self).communicate(*args, **kwargs) - if self._expected_returncode is not None: - try: - ok = self.returncode in self._expected_returncode - except TypeError: - ok = self.returncode == self._expected_returncode - if not ok: - raise SubprocessException(self._args) - return result - - -def main(argv=None): - if argv is None: - argv = sys.argv - try: - try: - opts, args = getopt.getopt(argv[1:], SHORT_OPTIONS, LONG_OPTIONS) - except getopt.error, msg: - raise Usage(msg) - - # process options - for o, _ in opts: - if o in ("-h", "--help"): - print __doc__ - sys.exit(0) - - if args and len(args) > 1: - print __doc__ - sys.exit(0) - - # Find the fully qualified path to the root of the tree - tl = Subprocess(TOPLEVEL_CMD, stdout=subprocess.PIPE) - tl = tl.communicate()[0].strip() - - # See if we're working on the index or not. - if args: - diff_cmd = DIFF_CMD + [args[0] + "^!"] - else: - diff_cmd = DIFF_INDEX_CMD - - # Build the command line to execute cpplint - cpplint_cmd = [os.path.join(tl, "tools", "cpplint.py"), - "--filter=" + ",".join(CPPLINT_FILTERS), - "-"] - - # Get a list of all affected lines - file_affected_line_map = {} - p = Subprocess(diff_cmd, stdout=subprocess.PIPE) - stdout = p.communicate()[0] - for hunk in diff.ParseDiffHunks(StringIO.StringIO(stdout)): - filename = hunk.right.filename[2:] - if filename not in file_affected_line_map: - file_affected_line_map[filename] = set() - file_affected_line_map[filename].update(hunk.right.delta_line_nums) - - # Run each affected file through cpplint - lint_failed = False - for filename, affected_lines in file_affected_line_map.iteritems(): - if filename.split(".")[-1] not in ("c", "h", "cc"): - continue - - if args: - # File contents come from git - show_cmd = SHOW_CMD + [args[0] + ":" + filename] - show = Subprocess(show_cmd, stdout=subprocess.PIPE) - lint = Subprocess(cpplint_cmd, expected_returncode=(0, 1), - stdin=show.stdout, stderr=subprocess.PIPE) - lint_out = lint.communicate()[1] - else: - # File contents come from the working tree - lint = Subprocess(cpplint_cmd, expected_returncode=(0, 1), - stdin=subprocess.PIPE, stderr=subprocess.PIPE) - stdin = open(os.path.join(tl, filename)).read() - lint_out = lint.communicate(stdin)[1] - - for line in lint_out.split("\n"): - fields = line.split(":") - if fields[0] != "-": - continue - warning_line_num = int(fields[1]) - if warning_line_num in affected_lines: - print "%s:%d:%s"%(filename, warning_line_num, - ":".join(fields[2:])) - lint_failed = True - - # Set exit code if any relevant lint errors seen - if lint_failed: - return 1 - - except Usage, err: - print >>sys.stderr, err - print >>sys.stderr, "for help use --help" - return 2 - -if __name__ == "__main__": - sys.exit(main()) diff --git a/third_party/aom/tools/obu_parser.cc b/third_party/aom/tools/obu_parser.cc deleted file mode 100644 index 7d71386ce..000000000 --- a/third_party/aom/tools/obu_parser.cc +++ /dev/null @@ -1,190 +0,0 @@ -/* - * Copyright (c) 2017, Alliance for Open Media. All rights reserved - * - * This source code is subject to the terms of the BSD 2 Clause License and - * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License - * was not distributed with this source code in the LICENSE file, you can - * obtain it at www.aomedia.org/license/software. If the Alliance for Open - * Media Patent License 1.0 was not distributed with this source code in the - * PATENTS file, you can obtain it at www.aomedia.org/license/patent. - */ -#include <string.h> - -#include <cstdio> -#include <string> - -#include "aom/aom_codec.h" -#include "aom/aom_integer.h" -#include "aom_ports/mem_ops.h" -#include "av1/common/obu_util.h" -#include "tools/obu_parser.h" - -namespace aom_tools { - -// Basic OBU syntax -// 8 bits: Header -// 7 -// forbidden bit -// 6,5,4,3 -// type bits -// 2 -// extension flag bit -// 1 -// has size field bit -// 0 -// reserved bit -const uint32_t kObuForbiddenBitMask = 0x1; -const uint32_t kObuForbiddenBitShift = 7; -const uint32_t kObuTypeBitsMask = 0xF; -const uint32_t kObuTypeBitsShift = 3; -const uint32_t kObuExtensionFlagBitMask = 0x1; -const uint32_t kObuExtensionFlagBitShift = 2; -const uint32_t kObuHasSizeFieldBitMask = 0x1; -const uint32_t kObuHasSizeFieldBitShift = 1; - -// When extension flag bit is set: -// 8 bits: extension header -// 7,6,5 -// temporal ID -// 4,3 -// spatial ID -// 2,1,0 -// reserved bits -const uint32_t kObuExtTemporalIdBitsMask = 0x7; -const uint32_t kObuExtTemporalIdBitsShift = 5; -const uint32_t kObuExtSpatialIdBitsMask = 0x3; -const uint32_t kObuExtSpatialIdBitsShift = 3; - -bool ValidObuType(int obu_type) { - switch (obu_type) { - case OBU_SEQUENCE_HEADER: - case OBU_TEMPORAL_DELIMITER: - case OBU_FRAME_HEADER: - case OBU_TILE_GROUP: - case OBU_METADATA: - case OBU_FRAME: - case OBU_REDUNDANT_FRAME_HEADER: - case OBU_TILE_LIST: - case OBU_PADDING: return true; - } - return false; -} - -bool ParseObuHeader(uint8_t obu_header_byte, ObuHeader *obu_header) { - const int forbidden_bit = - (obu_header_byte >> kObuForbiddenBitShift) & kObuForbiddenBitMask; - if (forbidden_bit) { - fprintf(stderr, "Invalid OBU, forbidden bit set.\n"); - return false; - } - - obu_header->type = static_cast<OBU_TYPE>( - (obu_header_byte >> kObuTypeBitsShift) & kObuTypeBitsMask); - if (!ValidObuType(obu_header->type)) { - fprintf(stderr, "Invalid OBU type: %d.\n", obu_header->type); - return false; - } - - obu_header->has_extension = - (obu_header_byte >> kObuExtensionFlagBitShift) & kObuExtensionFlagBitMask; - obu_header->has_size_field = - (obu_header_byte >> kObuHasSizeFieldBitShift) & kObuHasSizeFieldBitMask; - return true; -} - -bool ParseObuExtensionHeader(uint8_t ext_header_byte, ObuHeader *obu_header) { - obu_header->temporal_layer_id = - (ext_header_byte >> kObuExtTemporalIdBitsShift) & - kObuExtTemporalIdBitsMask; - obu_header->spatial_layer_id = - (ext_header_byte >> kObuExtSpatialIdBitsShift) & kObuExtSpatialIdBitsMask; - - return true; -} - -void PrintObuHeader(const ObuHeader *header) { - printf( - " OBU type: %s\n" - " extension: %s\n", - aom_obu_type_to_string(static_cast<OBU_TYPE>(header->type)), - header->has_extension ? "yes" : "no"); - if (header->has_extension) { - printf( - " temporal_id: %d\n" - " spatial_id: %d\n", - header->temporal_layer_id, header->temporal_layer_id); - } -} - -bool DumpObu(const uint8_t *data, int length, int *obu_overhead_bytes) { - const int kObuHeaderSizeBytes = 1; - const int kMinimumBytesRequired = 1 + kObuHeaderSizeBytes; - int consumed = 0; - int obu_overhead = 0; - ObuHeader obu_header; - while (consumed < length) { - const int remaining = length - consumed; - if (remaining < kMinimumBytesRequired) { - fprintf(stderr, - "OBU parse error. Did not consume all data, %d bytes remain.\n", - remaining); - return false; - } - - int obu_header_size = 0; - - memset(&obu_header, 0, sizeof(obu_header)); - const uint8_t obu_header_byte = *(data + consumed); - if (!ParseObuHeader(obu_header_byte, &obu_header)) { - fprintf(stderr, "OBU parsing failed at offset %d.\n", consumed); - return false; - } - - ++obu_overhead; - ++obu_header_size; - - if (obu_header.has_extension) { - const uint8_t obu_ext_header_byte = - *(data + consumed + kObuHeaderSizeBytes); - if (!ParseObuExtensionHeader(obu_ext_header_byte, &obu_header)) { - fprintf(stderr, "OBU extension parsing failed at offset %d.\n", - consumed + kObuHeaderSizeBytes); - return false; - } - - ++obu_overhead; - ++obu_header_size; - } - - PrintObuHeader(&obu_header); - - uint64_t obu_size = 0; - size_t length_field_size = 0; - if (aom_uleb_decode(data + consumed + obu_header_size, - remaining - obu_header_size, &obu_size, - &length_field_size) != 0) { - fprintf(stderr, "OBU size parsing failed at offset %d.\n", - consumed + obu_header_size); - return false; - } - int current_obu_length = static_cast<int>(obu_size); - if (obu_header_size + static_cast<int>(length_field_size) + - current_obu_length > - remaining) { - fprintf(stderr, "OBU parsing failed: not enough OBU data.\n"); - return false; - } - consumed += obu_header_size + static_cast<int>(length_field_size) + - current_obu_length; - printf(" length: %d\n", - static_cast<int>(obu_header_size + length_field_size + - current_obu_length)); - } - - if (obu_overhead_bytes != nullptr) *obu_overhead_bytes = obu_overhead; - printf(" TU size: %d\n", consumed); - - return true; -} - -} // namespace aom_tools diff --git a/third_party/aom/tools/obu_parser.h b/third_party/aom/tools/obu_parser.h deleted file mode 100644 index 1d7d2d794..000000000 --- a/third_party/aom/tools/obu_parser.h +++ /dev/null @@ -1,27 +0,0 @@ -/* - * Copyright (c) 2017, Alliance for Open Media. All rights reserved - * - * This source code is subject to the terms of the BSD 2 Clause License and - * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License - * was not distributed with this source code in the LICENSE file, you can - * obtain it at www.aomedia.org/license/software. If the Alliance for Open - * Media Patent License 1.0 was not distributed with this source code in the - * PATENTS file, you can obtain it at www.aomedia.org/license/patent. - */ - -#ifndef AOM_TOOLS_OBU_PARSER_H_ -#define AOM_TOOLS_OBU_PARSER_H_ - -#include <cstdint> - -namespace aom_tools { - -// Print information obtained from OBU(s) in data until data is exhausted or an -// error occurs. Returns true when all data is consumed successfully, and -// optionally reports OBU storage overhead via obu_overhead_bytes when the -// pointer is non-null. -bool DumpObu(const uint8_t *data, int length, int *obu_overhead_bytes); - -} // namespace aom_tools - -#endif // AOM_TOOLS_OBU_PARSER_H_ diff --git a/third_party/aom/tools/txfm_analyzer/txfm_gen_code.cc b/third_party/aom/tools/txfm_analyzer/txfm_gen_code.cc deleted file mode 100644 index 7c5400b91..000000000 --- a/third_party/aom/tools/txfm_analyzer/txfm_gen_code.cc +++ /dev/null @@ -1,580 +0,0 @@ -/* - * Copyright (c) 2018, Alliance for Open Media. All rights reserved - * - * This source code is subject to the terms of the BSD 2 Clause License and - * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License - * was not distributed with this source code in the LICENSE file, you can - * obtain it at www.aomedia.org/license/software. If the Alliance for Open - * Media Patent License 1.0 was not distributed with this source code in the - * PATENTS file, you can obtain it at www.aomedia.org/license/patent. - */ - -#include <stdio.h> -#include <stdlib.h> -#include <math.h> -#include <float.h> -#include <string.h> - -#include "tools/txfm_analyzer/txfm_graph.h" - -typedef enum CODE_TYPE { - CODE_TYPE_C, - CODE_TYPE_SSE2, - CODE_TYPE_SSE4_1 -} CODE_TYPE; - -int get_cos_idx(double value, int mod) { - return round(acos(fabs(value)) / PI * mod); -} - -char *cos_text_arr(double value, int mod, char *text, int size) { - int num = get_cos_idx(value, mod); - if (value < 0) { - snprintf(text, size, "-cospi[%2d]", num); - } else { - snprintf(text, size, " cospi[%2d]", num); - } - - if (num == 0) - printf("v: %f -> %d/%d v==-1 is %d\n", value, num, mod, value == -1); - - return text; -} - -char *cos_text_sse2(double w0, double w1, int mod, char *text, int size) { - int idx0 = get_cos_idx(w0, mod); - int idx1 = get_cos_idx(w1, mod); - char p[] = "p"; - char n[] = "m"; - char *sgn0 = w0 < 0 ? n : p; - char *sgn1 = w1 < 0 ? n : p; - snprintf(text, size, "cospi_%s%02d_%s%02d", sgn0, idx0, sgn1, idx1); - return text; -} - -char *cos_text_sse4_1(double w, int mod, char *text, int size) { - int idx = get_cos_idx(w, mod); - char p[] = "p"; - char n[] = "m"; - char *sgn = w < 0 ? n : p; - snprintf(text, size, "cospi_%s%02d", sgn, idx); - return text; -} - -void node_to_code_c(Node *node, const char *buf0, const char *buf1) { - int cnt = 0; - for (int i = 0; i < 2; i++) { - if (fabs(node->inWeight[i]) == 1 || fabs(node->inWeight[i]) == 0) cnt++; - } - if (cnt == 2) { - int cnt2 = 0; - printf(" %s[%d] =", buf1, node->nodeIdx); - for (int i = 0; i < 2; i++) { - if (fabs(node->inWeight[i]) == 1) { - cnt2++; - } - } - if (cnt2 == 2) { - printf(" apply_value("); - } - int cnt1 = 0; - for (int i = 0; i < 2; i++) { - if (node->inWeight[i] == 1) { - if (cnt1 > 0) - printf(" + %s[%d]", buf0, node->inNodeIdx[i]); - else - printf(" %s[%d]", buf0, node->inNodeIdx[i]); - cnt1++; - } else if (node->inWeight[i] == -1) { - if (cnt1 > 0) - printf(" - %s[%d]", buf0, node->inNodeIdx[i]); - else - printf("-%s[%d]", buf0, node->inNodeIdx[i]); - cnt1++; - } - } - if (cnt2 == 2) { - printf(", stage_range[stage])"); - } - printf(";\n"); - } else { - char w0[100]; - char w1[100]; - printf( - " %s[%d] = half_btf(%s, %s[%d], %s, %s[%d], " - "cos_bit);\n", - buf1, node->nodeIdx, cos_text_arr(node->inWeight[0], COS_MOD, w0, 100), - buf0, node->inNodeIdx[0], - cos_text_arr(node->inWeight[1], COS_MOD, w1, 100), buf0, - node->inNodeIdx[1]); - } -} - -void gen_code_c(Node *node, int stage_num, int node_num, TYPE_TXFM type) { - char *fun_name = new char[100]; - get_fun_name(fun_name, 100, type, node_num); - - printf("\n"); - printf( - "void av1_%s(const int32_t *input, int32_t *output, int8_t cos_bit, " - "const int8_t* stage_range) " - "{\n", - fun_name); - printf(" assert(output != input);\n"); - printf(" const int32_t size = %d;\n", node_num); - printf(" const int32_t *cospi = cospi_arr(cos_bit);\n"); - printf("\n"); - - printf(" int32_t stage = 0;\n"); - printf(" int32_t *bf0, *bf1;\n"); - printf(" int32_t step[%d];\n", node_num); - - const char *buf0 = "bf0"; - const char *buf1 = "bf1"; - const char *input = "input"; - - int si = 0; - printf("\n"); - printf(" // stage %d;\n", si); - printf(" apply_range(stage, input, %s, size, stage_range[stage]);\n", input); - - si = 1; - printf("\n"); - printf(" // stage %d;\n", si); - printf(" stage++;\n"); - if (si % 2 == (stage_num - 1) % 2) { - printf(" %s = output;\n", buf1); - } else { - printf(" %s = step;\n", buf1); - } - - for (int ni = 0; ni < node_num; ni++) { - int idx = get_idx(si, ni, node_num); - node_to_code_c(node + idx, input, buf1); - } - - printf(" range_check_buf(stage, input, bf1, size, stage_range[stage]);\n"); - - for (int si = 2; si < stage_num; si++) { - printf("\n"); - printf(" // stage %d\n", si); - printf(" stage++;\n"); - if (si % 2 == (stage_num - 1) % 2) { - printf(" %s = step;\n", buf0); - printf(" %s = output;\n", buf1); - } else { - printf(" %s = output;\n", buf0); - printf(" %s = step;\n", buf1); - } - - // computation code - for (int ni = 0; ni < node_num; ni++) { - int idx = get_idx(si, ni, node_num); - node_to_code_c(node + idx, buf0, buf1); - } - - if (si != stage_num - 1) { - printf( - " range_check_buf(stage, input, bf1, size, stage_range[stage]);\n"); - } - } - printf(" apply_range(stage, input, output, size, stage_range[stage]);\n"); - printf("}\n"); -} - -void single_node_to_code_sse2(Node *node, const char *buf0, const char *buf1) { - printf(" %s[%2d] =", buf1, node->nodeIdx); - if (node->inWeight[0] == 1 && node->inWeight[1] == 1) { - printf(" _mm_adds_epi16(%s[%d], %s[%d])", buf0, node->inNodeIdx[0], buf0, - node->inNodeIdx[1]); - } else if (node->inWeight[0] == 1 && node->inWeight[1] == -1) { - printf(" _mm_subs_epi16(%s[%d], %s[%d])", buf0, node->inNodeIdx[0], buf0, - node->inNodeIdx[1]); - } else if (node->inWeight[0] == -1 && node->inWeight[1] == 1) { - printf(" _mm_subs_epi16(%s[%d], %s[%d])", buf0, node->inNodeIdx[1], buf0, - node->inNodeIdx[0]); - } else if (node->inWeight[0] == 1 && node->inWeight[1] == 0) { - printf(" %s[%d]", buf0, node->inNodeIdx[0]); - } else if (node->inWeight[0] == 0 && node->inWeight[1] == 1) { - printf(" %s[%d]", buf0, node->inNodeIdx[1]); - } else if (node->inWeight[0] == -1 && node->inWeight[1] == 0) { - printf(" _mm_subs_epi16(__zero, %s[%d])", buf0, node->inNodeIdx[0]); - } else if (node->inWeight[0] == 0 && node->inWeight[1] == -1) { - printf(" _mm_subs_epi16(__zero, %s[%d])", buf0, node->inNodeIdx[1]); - } - printf(";\n"); -} - -void pair_node_to_code_sse2(Node *node, Node *partnerNode, const char *buf0, - const char *buf1) { - char temp0[100]; - char temp1[100]; - // btf_16_sse2_type0(w0, w1, in0, in1, out0, out1) - if (node->inNodeIdx[0] != partnerNode->inNodeIdx[0]) - printf(" btf_16_sse2(%s, %s, %s[%d], %s[%d], %s[%d], %s[%d]);\n", - cos_text_sse2(node->inWeight[0], node->inWeight[1], COS_MOD, temp0, - 100), - cos_text_sse2(partnerNode->inWeight[1], partnerNode->inWeight[0], - COS_MOD, temp1, 100), - buf0, node->inNodeIdx[0], buf0, node->inNodeIdx[1], buf1, - node->nodeIdx, buf1, partnerNode->nodeIdx); - else - printf(" btf_16_sse2(%s, %s, %s[%d], %s[%d], %s[%d], %s[%d]);\n", - cos_text_sse2(node->inWeight[0], node->inWeight[1], COS_MOD, temp0, - 100), - cos_text_sse2(partnerNode->inWeight[0], partnerNode->inWeight[1], - COS_MOD, temp1, 100), - buf0, node->inNodeIdx[0], buf0, node->inNodeIdx[1], buf1, - node->nodeIdx, buf1, partnerNode->nodeIdx); -} - -Node *get_partner_node(Node *node) { - int diff = node->inNode[1]->nodeIdx - node->nodeIdx; - return node + diff; -} - -void node_to_code_sse2(Node *node, const char *buf0, const char *buf1) { - int cnt = 0; - int cnt1 = 0; - if (node->visited == 0) { - node->visited = 1; - for (int i = 0; i < 2; i++) { - if (fabs(node->inWeight[i]) == 1 || fabs(node->inWeight[i]) == 0) cnt++; - if (fabs(node->inWeight[i]) == 1) cnt1++; - } - if (cnt == 2) { - if (cnt1 == 2) { - // has a partner - Node *partnerNode = get_partner_node(node); - partnerNode->visited = 1; - single_node_to_code_sse2(node, buf0, buf1); - single_node_to_code_sse2(partnerNode, buf0, buf1); - } else { - single_node_to_code_sse2(node, buf0, buf1); - } - } else { - Node *partnerNode = get_partner_node(node); - partnerNode->visited = 1; - pair_node_to_code_sse2(node, partnerNode, buf0, buf1); - } - } -} - -void gen_cospi_list_sse2(Node *node, int stage_num, int node_num) { - int visited[65][65][2][2]; - memset(visited, 0, sizeof(visited)); - char text[100]; - char text1[100]; - char text2[100]; - int size = 100; - printf("\n"); - for (int si = 1; si < stage_num; si++) { - for (int ni = 0; ni < node_num; ni++) { - int idx = get_idx(si, ni, node_num); - int cnt = 0; - Node *node0 = node + idx; - if (node0->visited == 0) { - node0->visited = 1; - for (int i = 0; i < 2; i++) { - if (fabs(node0->inWeight[i]) == 1 || fabs(node0->inWeight[i]) == 0) - cnt++; - } - if (cnt != 2) { - { - double w0 = node0->inWeight[0]; - double w1 = node0->inWeight[1]; - int idx0 = get_cos_idx(w0, COS_MOD); - int idx1 = get_cos_idx(w1, COS_MOD); - int sgn0 = w0 < 0 ? 1 : 0; - int sgn1 = w1 < 0 ? 1 : 0; - - if (!visited[idx0][idx1][sgn0][sgn1]) { - visited[idx0][idx1][sgn0][sgn1] = 1; - printf(" __m128i %s = pair_set_epi16(%s, %s);\n", - cos_text_sse2(w0, w1, COS_MOD, text, size), - cos_text_arr(w0, COS_MOD, text1, size), - cos_text_arr(w1, COS_MOD, text2, size)); - } - } - Node *node1 = get_partner_node(node0); - node1->visited = 1; - if (node1->inNode[0]->nodeIdx != node0->inNode[0]->nodeIdx) { - double w0 = node1->inWeight[0]; - double w1 = node1->inWeight[1]; - int idx0 = get_cos_idx(w0, COS_MOD); - int idx1 = get_cos_idx(w1, COS_MOD); - int sgn0 = w0 < 0 ? 1 : 0; - int sgn1 = w1 < 0 ? 1 : 0; - - if (!visited[idx1][idx0][sgn1][sgn0]) { - visited[idx1][idx0][sgn1][sgn0] = 1; - printf(" __m128i %s = pair_set_epi16(%s, %s);\n", - cos_text_sse2(w1, w0, COS_MOD, text, size), - cos_text_arr(w1, COS_MOD, text1, size), - cos_text_arr(w0, COS_MOD, text2, size)); - } - } else { - double w0 = node1->inWeight[0]; - double w1 = node1->inWeight[1]; - int idx0 = get_cos_idx(w0, COS_MOD); - int idx1 = get_cos_idx(w1, COS_MOD); - int sgn0 = w0 < 0 ? 1 : 0; - int sgn1 = w1 < 0 ? 1 : 0; - - if (!visited[idx0][idx1][sgn0][sgn1]) { - visited[idx0][idx1][sgn0][sgn1] = 1; - printf(" __m128i %s = pair_set_epi16(%s, %s);\n", - cos_text_sse2(w0, w1, COS_MOD, text, size), - cos_text_arr(w0, COS_MOD, text1, size), - cos_text_arr(w1, COS_MOD, text2, size)); - } - } - } - } - } - } -} - -void gen_code_sse2(Node *node, int stage_num, int node_num, TYPE_TXFM type) { - char *fun_name = new char[100]; - get_fun_name(fun_name, 100, type, node_num); - - printf("\n"); - printf( - "void %s_sse2(const __m128i *input, __m128i *output, int8_t cos_bit) " - "{\n", - fun_name); - - printf(" const int32_t* cospi = cospi_arr(cos_bit);\n"); - printf(" const __m128i __zero = _mm_setzero_si128();\n"); - printf(" const __m128i __rounding = _mm_set1_epi32(1 << (cos_bit - 1));\n"); - - graph_reset_visited(node, stage_num, node_num); - gen_cospi_list_sse2(node, stage_num, node_num); - graph_reset_visited(node, stage_num, node_num); - for (int si = 1; si < stage_num; si++) { - char in[100]; - char out[100]; - printf("\n"); - printf(" // stage %d\n", si); - if (si == 1) - snprintf(in, 100, "%s", "input"); - else - snprintf(in, 100, "x%d", si - 1); - if (si == stage_num - 1) { - snprintf(out, 100, "%s", "output"); - } else { - snprintf(out, 100, "x%d", si); - printf(" __m128i %s[%d];\n", out, node_num); - } - // computation code - for (int ni = 0; ni < node_num; ni++) { - int idx = get_idx(si, ni, node_num); - node_to_code_sse2(node + idx, in, out); - } - } - - printf("}\n"); -} -void gen_cospi_list_sse4_1(Node *node, int stage_num, int node_num) { - int visited[65][2]; - memset(visited, 0, sizeof(visited)); - char text[100]; - char text1[100]; - int size = 100; - printf("\n"); - for (int si = 1; si < stage_num; si++) { - for (int ni = 0; ni < node_num; ni++) { - int idx = get_idx(si, ni, node_num); - Node *node0 = node + idx; - if (node0->visited == 0) { - int cnt = 0; - node0->visited = 1; - for (int i = 0; i < 2; i++) { - if (fabs(node0->inWeight[i]) == 1 || fabs(node0->inWeight[i]) == 0) - cnt++; - } - if (cnt != 2) { - for (int i = 0; i < 2; i++) { - if (fabs(node0->inWeight[i]) != 1 && - fabs(node0->inWeight[i]) != 0) { - double w = node0->inWeight[i]; - int idx = get_cos_idx(w, COS_MOD); - int sgn = w < 0 ? 1 : 0; - - if (!visited[idx][sgn]) { - visited[idx][sgn] = 1; - printf(" __m128i %s = _mm_set1_epi32(%s);\n", - cos_text_sse4_1(w, COS_MOD, text, size), - cos_text_arr(w, COS_MOD, text1, size)); - } - } - } - Node *node1 = get_partner_node(node0); - node1->visited = 1; - } - } - } - } -} - -void single_node_to_code_sse4_1(Node *node, const char *buf0, - const char *buf1) { - printf(" %s[%2d] =", buf1, node->nodeIdx); - if (node->inWeight[0] == 1 && node->inWeight[1] == 1) { - printf(" _mm_add_epi32(%s[%d], %s[%d])", buf0, node->inNodeIdx[0], buf0, - node->inNodeIdx[1]); - } else if (node->inWeight[0] == 1 && node->inWeight[1] == -1) { - printf(" _mm_sub_epi32(%s[%d], %s[%d])", buf0, node->inNodeIdx[0], buf0, - node->inNodeIdx[1]); - } else if (node->inWeight[0] == -1 && node->inWeight[1] == 1) { - printf(" _mm_sub_epi32(%s[%d], %s[%d])", buf0, node->inNodeIdx[1], buf0, - node->inNodeIdx[0]); - } else if (node->inWeight[0] == 1 && node->inWeight[1] == 0) { - printf(" %s[%d]", buf0, node->inNodeIdx[0]); - } else if (node->inWeight[0] == 0 && node->inWeight[1] == 1) { - printf(" %s[%d]", buf0, node->inNodeIdx[1]); - } else if (node->inWeight[0] == -1 && node->inWeight[1] == 0) { - printf(" _mm_sub_epi32(__zero, %s[%d])", buf0, node->inNodeIdx[0]); - } else if (node->inWeight[0] == 0 && node->inWeight[1] == -1) { - printf(" _mm_sub_epi32(__zero, %s[%d])", buf0, node->inNodeIdx[1]); - } - printf(";\n"); -} - -void pair_node_to_code_sse4_1(Node *node, Node *partnerNode, const char *buf0, - const char *buf1) { - char temp0[100]; - char temp1[100]; - if (node->inWeight[0] * partnerNode->inWeight[0] < 0) { - /* type0 - * cos sin - * sin -cos - */ - // btf_32_sse2_type0(w0, w1, in0, in1, out0, out1) - // out0 = w0*in0 + w1*in1 - // out1 = -w0*in1 + w1*in0 - printf( - " btf_32_type0_sse4_1_new(%s, %s, %s[%d], %s[%d], %s[%d], %s[%d], " - "__rounding, cos_bit);\n", - cos_text_sse4_1(node->inWeight[0], COS_MOD, temp0, 100), - cos_text_sse4_1(node->inWeight[1], COS_MOD, temp1, 100), buf0, - node->inNodeIdx[0], buf0, node->inNodeIdx[1], buf1, node->nodeIdx, buf1, - partnerNode->nodeIdx); - } else { - /* type1 - * cos sin - * -sin cos - */ - // btf_32_sse2_type1(w0, w1, in0, in1, out0, out1) - // out0 = w0*in0 + w1*in1 - // out1 = w0*in1 - w1*in0 - printf( - " btf_32_type1_sse4_1_new(%s, %s, %s[%d], %s[%d], %s[%d], %s[%d], " - "__rounding, cos_bit);\n", - cos_text_sse4_1(node->inWeight[0], COS_MOD, temp0, 100), - cos_text_sse4_1(node->inWeight[1], COS_MOD, temp1, 100), buf0, - node->inNodeIdx[0], buf0, node->inNodeIdx[1], buf1, node->nodeIdx, buf1, - partnerNode->nodeIdx); - } -} - -void node_to_code_sse4_1(Node *node, const char *buf0, const char *buf1) { - int cnt = 0; - int cnt1 = 0; - if (node->visited == 0) { - node->visited = 1; - for (int i = 0; i < 2; i++) { - if (fabs(node->inWeight[i]) == 1 || fabs(node->inWeight[i]) == 0) cnt++; - if (fabs(node->inWeight[i]) == 1) cnt1++; - } - if (cnt == 2) { - if (cnt1 == 2) { - // has a partner - Node *partnerNode = get_partner_node(node); - partnerNode->visited = 1; - single_node_to_code_sse4_1(node, buf0, buf1); - single_node_to_code_sse4_1(partnerNode, buf0, buf1); - } else { - single_node_to_code_sse2(node, buf0, buf1); - } - } else { - Node *partnerNode = get_partner_node(node); - partnerNode->visited = 1; - pair_node_to_code_sse4_1(node, partnerNode, buf0, buf1); - } - } -} - -void gen_code_sse4_1(Node *node, int stage_num, int node_num, TYPE_TXFM type) { - char *fun_name = new char[100]; - get_fun_name(fun_name, 100, type, node_num); - - printf("\n"); - printf( - "void %s_sse4_1(const __m128i *input, __m128i *output, int8_t cos_bit) " - "{\n", - fun_name); - - printf(" const int32_t* cospi = cospi_arr(cos_bit);\n"); - printf(" const __m128i __zero = _mm_setzero_si128();\n"); - printf(" const __m128i __rounding = _mm_set1_epi32(1 << (cos_bit - 1));\n"); - - graph_reset_visited(node, stage_num, node_num); - gen_cospi_list_sse4_1(node, stage_num, node_num); - graph_reset_visited(node, stage_num, node_num); - for (int si = 1; si < stage_num; si++) { - char in[100]; - char out[100]; - printf("\n"); - printf(" // stage %d\n", si); - if (si == 1) - snprintf(in, 100, "%s", "input"); - else - snprintf(in, 100, "x%d", si - 1); - if (si == stage_num - 1) { - snprintf(out, 100, "%s", "output"); - } else { - snprintf(out, 100, "x%d", si); - printf(" __m128i %s[%d];\n", out, node_num); - } - // computation code - for (int ni = 0; ni < node_num; ni++) { - int idx = get_idx(si, ni, node_num); - node_to_code_sse4_1(node + idx, in, out); - } - } - - printf("}\n"); -} - -void gen_hybrid_code(CODE_TYPE code_type, TYPE_TXFM txfm_type, int node_num) { - int stage_num = get_hybrid_stage_num(txfm_type, node_num); - - Node *node = new Node[node_num * stage_num]; - init_graph(node, stage_num, node_num); - - gen_hybrid_graph_1d(node, stage_num, node_num, 0, 0, node_num, txfm_type); - - switch (code_type) { - case CODE_TYPE_C: gen_code_c(node, stage_num, node_num, txfm_type); break; - case CODE_TYPE_SSE2: - gen_code_sse2(node, stage_num, node_num, txfm_type); - break; - case CODE_TYPE_SSE4_1: - gen_code_sse4_1(node, stage_num, node_num, txfm_type); - break; - } - - delete[] node; -} - -int main(int argc, char **argv) { - CODE_TYPE code_type = CODE_TYPE_SSE4_1; - for (int txfm_type = TYPE_DCT; txfm_type < TYPE_LAST; txfm_type++) { - for (int node_num = 4; node_num <= 64; node_num *= 2) { - gen_hybrid_code(code_type, (TYPE_TXFM)txfm_type, node_num); - } - } - return 0; -} diff --git a/third_party/aom/tools/txfm_analyzer/txfm_graph.cc b/third_party/aom/tools/txfm_analyzer/txfm_graph.cc deleted file mode 100644 index a24906100..000000000 --- a/third_party/aom/tools/txfm_analyzer/txfm_graph.cc +++ /dev/null @@ -1,943 +0,0 @@ -/* - * Copyright (c) 2018, Alliance for Open Media. All rights reserved - * - * This source code is subject to the terms of the BSD 2 Clause License and - * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License - * was not distributed with this source code in the LICENSE file, you can - * obtain it at www.aomedia.org/license/software. If the Alliance for Open - * Media Patent License 1.0 was not distributed with this source code in the - * PATENTS file, you can obtain it at www.aomedia.org/license/patent. - */ - -#include "tools/txfm_analyzer/txfm_graph.h" - -#include <stdio.h> -#include <stdlib.h> -#include <math.h> - -typedef struct Node Node; - -void get_fun_name(char *str_fun_name, int str_buf_size, const TYPE_TXFM type, - const int txfm_size) { - if (type == TYPE_DCT) - snprintf(str_fun_name, str_buf_size, "fdct%d_new", txfm_size); - else if (type == TYPE_ADST) - snprintf(str_fun_name, str_buf_size, "fadst%d_new", txfm_size); - else if (type == TYPE_IDCT) - snprintf(str_fun_name, str_buf_size, "idct%d_new", txfm_size); - else if (type == TYPE_IADST) - snprintf(str_fun_name, str_buf_size, "iadst%d_new", txfm_size); -} - -void get_txfm_type_name(char *str_fun_name, int str_buf_size, - const TYPE_TXFM type, const int txfm_size) { - if (type == TYPE_DCT) - snprintf(str_fun_name, str_buf_size, "TXFM_TYPE_DCT%d", txfm_size); - else if (type == TYPE_ADST) - snprintf(str_fun_name, str_buf_size, "TXFM_TYPE_ADST%d", txfm_size); - else if (type == TYPE_IDCT) - snprintf(str_fun_name, str_buf_size, "TXFM_TYPE_DCT%d", txfm_size); - else if (type == TYPE_IADST) - snprintf(str_fun_name, str_buf_size, "TXFM_TYPE_ADST%d", txfm_size); -} - -void get_hybrid_2d_type_name(char *buf, int buf_size, const TYPE_TXFM type0, - const TYPE_TXFM type1, const int txfm_size0, - const int txfm_size1) { - if (type0 == TYPE_DCT && type1 == TYPE_DCT) - snprintf(buf, buf_size, "_dct_dct_%dx%d", txfm_size1, txfm_size0); - else if (type0 == TYPE_DCT && type1 == TYPE_ADST) - snprintf(buf, buf_size, "_dct_adst_%dx%d", txfm_size1, txfm_size0); - else if (type0 == TYPE_ADST && type1 == TYPE_ADST) - snprintf(buf, buf_size, "_adst_adst_%dx%d", txfm_size1, txfm_size0); - else if (type0 == TYPE_ADST && type1 == TYPE_DCT) - snprintf(buf, buf_size, "_adst_dct_%dx%d", txfm_size1, txfm_size0); -} - -TYPE_TXFM get_inv_type(TYPE_TXFM type) { - if (type == TYPE_DCT) - return TYPE_IDCT; - else if (type == TYPE_ADST) - return TYPE_IADST; - else if (type == TYPE_IDCT) - return TYPE_DCT; - else if (type == TYPE_IADST) - return TYPE_ADST; - else - return TYPE_LAST; -} - -void reference_dct_1d(double *in, double *out, int size) { - const double kInvSqrt2 = 0.707106781186547524400844362104; - for (int k = 0; k < size; k++) { - out[k] = 0; // initialize out[k] - for (int n = 0; n < size; n++) { - out[k] += in[n] * cos(PI * (2 * n + 1) * k / (2 * size)); - } - if (k == 0) out[k] = out[k] * kInvSqrt2; - } -} - -void reference_dct_2d(double *in, double *out, int size) { - double *tempOut = new double[size * size]; - // dct each row: in -> out - for (int r = 0; r < size; r++) { - reference_dct_1d(in + r * size, out + r * size, size); - } - - for (int r = 0; r < size; r++) { - // out ->tempOut - for (int c = 0; c < size; c++) { - tempOut[r * size + c] = out[c * size + r]; - } - } - for (int r = 0; r < size; r++) { - reference_dct_1d(tempOut + r * size, out + r * size, size); - } - delete[] tempOut; -} - -void reference_adst_1d(double *in, double *out, int size) { - for (int k = 0; k < size; k++) { - out[k] = 0; // initialize out[k] - for (int n = 0; n < size; n++) { - out[k] += in[n] * sin(PI * (2 * n + 1) * (2 * k + 1) / (4 * size)); - } - } -} - -void reference_hybrid_2d(double *in, double *out, int size, int type0, - int type1) { - double *tempOut = new double[size * size]; - // dct each row: in -> out - for (int r = 0; r < size; r++) { - if (type0 == TYPE_DCT) - reference_dct_1d(in + r * size, out + r * size, size); - else - reference_adst_1d(in + r * size, out + r * size, size); - } - - for (int r = 0; r < size; r++) { - // out ->tempOut - for (int c = 0; c < size; c++) { - tempOut[r * size + c] = out[c * size + r]; - } - } - for (int r = 0; r < size; r++) { - if (type1 == TYPE_DCT) - reference_dct_1d(tempOut + r * size, out + r * size, size); - else - reference_adst_1d(tempOut + r * size, out + r * size, size); - } - delete[] tempOut; -} - -void reference_hybrid_2d_new(double *in, double *out, int size0, int size1, - int type0, int type1) { - double *tempOut = new double[size0 * size1]; - // dct each row: in -> out - for (int r = 0; r < size1; r++) { - if (type0 == TYPE_DCT) - reference_dct_1d(in + r * size0, out + r * size0, size0); - else - reference_adst_1d(in + r * size0, out + r * size0, size0); - } - - for (int r = 0; r < size1; r++) { - // out ->tempOut - for (int c = 0; c < size0; c++) { - tempOut[c * size1 + r] = out[r * size0 + c]; - } - } - for (int r = 0; r < size0; r++) { - if (type1 == TYPE_DCT) - reference_dct_1d(tempOut + r * size1, out + r * size1, size1); - else - reference_adst_1d(tempOut + r * size1, out + r * size1, size1); - } - delete[] tempOut; -} - -unsigned int get_max_bit(unsigned int x) { - int max_bit = -1; - while (x) { - x = x >> 1; - max_bit++; - } - return max_bit; -} - -unsigned int bitwise_reverse(unsigned int x, int max_bit) { - x = ((x >> 16) & 0x0000ffff) | ((x & 0x0000ffff) << 16); - x = ((x >> 8) & 0x00ff00ff) | ((x & 0x00ff00ff) << 8); - x = ((x >> 4) & 0x0f0f0f0f) | ((x & 0x0f0f0f0f) << 4); - x = ((x >> 2) & 0x33333333) | ((x & 0x33333333) << 2); - x = ((x >> 1) & 0x55555555) | ((x & 0x55555555) << 1); - x = x >> (31 - max_bit); - return x; -} - -int get_idx(int ri, int ci, int cSize) { return ri * cSize + ci; } - -void add_node(Node *node, int stage_num, int node_num, int stage_idx, - int node_idx, int in, double w) { - int outIdx = get_idx(stage_idx, node_idx, node_num); - int inIdx = get_idx(stage_idx - 1, in, node_num); - int idx = node[outIdx].inNodeNum; - if (idx < 2) { - node[outIdx].inNode[idx] = &node[inIdx]; - node[outIdx].inNodeIdx[idx] = in; - node[outIdx].inWeight[idx] = w; - idx++; - node[outIdx].inNodeNum = idx; - } else { - printf("Error: inNode is full"); - } -} - -void connect_node(Node *node, int stage_num, int node_num, int stage_idx, - int node_idx, int in0, double w0, int in1, double w1) { - int outIdx = get_idx(stage_idx, node_idx, node_num); - int inIdx0 = get_idx(stage_idx - 1, in0, node_num); - int inIdx1 = get_idx(stage_idx - 1, in1, node_num); - - int idx = 0; - // if(w0 != 0) { - node[outIdx].inNode[idx] = &node[inIdx0]; - node[outIdx].inNodeIdx[idx] = in0; - node[outIdx].inWeight[idx] = w0; - idx++; - //} - - // if(w1 != 0) { - node[outIdx].inNode[idx] = &node[inIdx1]; - node[outIdx].inNodeIdx[idx] = in1; - node[outIdx].inWeight[idx] = w1; - idx++; - //} - - node[outIdx].inNodeNum = idx; -} - -void propagate(Node *node, int stage_num, int node_num, int stage_idx) { - for (int ni = 0; ni < node_num; ni++) { - int outIdx = get_idx(stage_idx, ni, node_num); - node[outIdx].value = 0; - for (int k = 0; k < node[outIdx].inNodeNum; k++) { - node[outIdx].value += - node[outIdx].inNode[k]->value * node[outIdx].inWeight[k]; - } - } -} - -int64_t round_shift(int64_t value, int bit) { - if (bit > 0) { - if (value < 0) { - return -round_shift(-value, bit); - } else { - return (value + (1 << (bit - 1))) >> bit; - } - } else { - return value << (-bit); - } -} - -void round_shift_array(int32_t *arr, int size, int bit) { - if (bit == 0) { - return; - } else { - for (int i = 0; i < size; i++) { - arr[i] = round_shift(arr[i], bit); - } - } -} - -void graph_reset_visited(Node *node, int stage_num, int node_num) { - for (int si = 0; si < stage_num; si++) { - for (int ni = 0; ni < node_num; ni++) { - int idx = get_idx(si, ni, node_num); - node[idx].visited = 0; - } - } -} - -void estimate_value(Node *node, int stage_num, int node_num, int stage_idx, - int node_idx, int estimate_bit) { - if (stage_idx > 0) { - int outIdx = get_idx(stage_idx, node_idx, node_num); - int64_t out = 0; - node[outIdx].value = 0; - for (int k = 0; k < node[outIdx].inNodeNum; k++) { - int64_t w = round(node[outIdx].inWeight[k] * (1 << estimate_bit)); - int64_t v = round(node[outIdx].inNode[k]->value); - out += v * w; - } - node[outIdx].value = round_shift(out, estimate_bit); - } -} - -void amplify_value(Node *node, int stage_num, int node_num, int stage_idx, - int node_idx, int amplify_bit) { - int outIdx = get_idx(stage_idx, node_idx, node_num); - node[outIdx].value = round_shift(round(node[outIdx].value), -amplify_bit); -} - -void propagate_estimate_amlify(Node *node, int stage_num, int node_num, - int stage_idx, int amplify_bit, - int estimate_bit) { - for (int ni = 0; ni < node_num; ni++) { - estimate_value(node, stage_num, node_num, stage_idx, ni, estimate_bit); - amplify_value(node, stage_num, node_num, stage_idx, ni, amplify_bit); - } -} - -void init_graph(Node *node, int stage_num, int node_num) { - for (int si = 0; si < stage_num; si++) { - for (int ni = 0; ni < node_num; ni++) { - int outIdx = get_idx(si, ni, node_num); - node[outIdx].stageIdx = si; - node[outIdx].nodeIdx = ni; - node[outIdx].value = 0; - node[outIdx].inNodeNum = 0; - if (si >= 1) { - connect_node(node, stage_num, node_num, si, ni, ni, 1, ni, 0); - } - } - } -} - -void gen_B_graph(Node *node, int stage_num, int node_num, int stage_idx, - int node_idx, int N, int star) { - for (int i = 0; i < N / 2; i++) { - int out = node_idx + i; - int in1 = node_idx + N - 1 - i; - if (star == 1) { - connect_node(node, stage_num, node_num, stage_idx + 1, out, out, -1, in1, - 1); - } else { - connect_node(node, stage_num, node_num, stage_idx + 1, out, out, 1, in1, - 1); - } - } - for (int i = N / 2; i < N; i++) { - int out = node_idx + i; - int in1 = node_idx + N - 1 - i; - if (star == 1) { - connect_node(node, stage_num, node_num, stage_idx + 1, out, out, 1, in1, - 1); - } else { - connect_node(node, stage_num, node_num, stage_idx + 1, out, out, -1, in1, - 1); - } - } -} - -void gen_P_graph(Node *node, int stage_num, int node_num, int stage_idx, - int node_idx, int N) { - int max_bit = get_max_bit(N - 1); - for (int i = 0; i < N; i++) { - int out = node_idx + bitwise_reverse(i, max_bit); - int in = node_idx + i; - connect_node(node, stage_num, node_num, stage_idx + 1, out, in, 1, in, 0); - } -} - -void gen_type1_graph(Node *node, int stage_num, int node_num, int stage_idx, - int node_idx, int N) { - int max_bit = get_max_bit(N); - for (int ni = 0; ni < N / 2; ni++) { - int ai = bitwise_reverse(N + ni, max_bit); - int out = node_idx + ni; - int in1 = node_idx + N - ni - 1; - connect_node(node, stage_num, node_num, stage_idx + 1, out, out, - sin(PI * ai / (2 * 2 * N)), in1, cos(PI * ai / (2 * 2 * N))); - } - for (int ni = N / 2; ni < N; ni++) { - int ai = bitwise_reverse(N + ni, max_bit); - int out = node_idx + ni; - int in1 = node_idx + N - ni - 1; - connect_node(node, stage_num, node_num, stage_idx + 1, out, out, - cos(PI * ai / (2 * 2 * N)), in1, -sin(PI * ai / (2 * 2 * N))); - } -} - -void gen_type2_graph(Node *node, int stage_num, int node_num, int stage_idx, - int node_idx, int N) { - for (int ni = 0; ni < N / 4; ni++) { - int out = node_idx + ni; - connect_node(node, stage_num, node_num, stage_idx + 1, out, out, 1, out, 0); - } - - for (int ni = N / 4; ni < N / 2; ni++) { - int out = node_idx + ni; - int in1 = node_idx + N - ni - 1; - connect_node(node, stage_num, node_num, stage_idx + 1, out, out, - -cos(PI / 4), in1, cos(-PI / 4)); - } - - for (int ni = N / 2; ni < N * 3 / 4; ni++) { - int out = node_idx + ni; - int in1 = node_idx + N - ni - 1; - connect_node(node, stage_num, node_num, stage_idx + 1, out, out, - cos(-PI / 4), in1, cos(PI / 4)); - } - - for (int ni = N * 3 / 4; ni < N; ni++) { - int out = node_idx + ni; - connect_node(node, stage_num, node_num, stage_idx + 1, out, out, 1, out, 0); - } -} - -void gen_type3_graph(Node *node, int stage_num, int node_num, int stage_idx, - int node_idx, int idx, int N) { - // TODO(angiebird): Simplify and clarify this function - - int i = 2 * N / (1 << (idx / 2)); - int max_bit = - get_max_bit(i / 2) - 1; // the max_bit counts on i/2 instead of N here - int N_over_i = 2 << (idx / 2); - - for (int nj = 0; nj < N / 2; nj += N_over_i) { - int j = nj / (N_over_i); - int kj = bitwise_reverse(i / 4 + j, max_bit); - // printf("kj = %d\n", kj); - - // I_N/2i --- 0 - int offset = nj; - for (int ni = 0; ni < N_over_i / 4; ni++) { - int out = node_idx + offset + ni; - int in = out; - connect_node(node, stage_num, node_num, stage_idx + 1, out, in, 1, in, 0); - } - - // -C_Kj/i --- S_Kj/i - offset += N_over_i / 4; - for (int ni = 0; ni < N_over_i / 4; ni++) { - int out = node_idx + offset + ni; - int in0 = out; - double w0 = -cos(kj * PI / i); - int in1 = N - (offset + ni) - 1 + node_idx; - double w1 = sin(kj * PI / i); - connect_node(node, stage_num, node_num, stage_idx + 1, out, in0, w0, in1, - w1); - } - - // S_kj/i --- -C_Kj/i - offset += N_over_i / 4; - for (int ni = 0; ni < N_over_i / 4; ni++) { - int out = node_idx + offset + ni; - int in0 = out; - double w0 = -sin(kj * PI / i); - int in1 = N - (offset + ni) - 1 + node_idx; - double w1 = -cos(kj * PI / i); - connect_node(node, stage_num, node_num, stage_idx + 1, out, in0, w0, in1, - w1); - } - - // I_N/2i --- 0 - offset += N_over_i / 4; - for (int ni = 0; ni < N_over_i / 4; ni++) { - int out = node_idx + offset + ni; - int in = out; - connect_node(node, stage_num, node_num, stage_idx + 1, out, in, 1, in, 0); - } - } - - for (int nj = N / 2; nj < N; nj += N_over_i) { - int j = nj / N_over_i; - int kj = bitwise_reverse(i / 4 + j, max_bit); - - // I_N/2i --- 0 - int offset = nj; - for (int ni = 0; ni < N_over_i / 4; ni++) { - int out = node_idx + offset + ni; - int in = out; - connect_node(node, stage_num, node_num, stage_idx + 1, out, in, 1, in, 0); - } - - // C_kj/i --- -S_Kj/i - offset += N_over_i / 4; - for (int ni = 0; ni < N_over_i / 4; ni++) { - int out = node_idx + offset + ni; - int in0 = out; - double w0 = cos(kj * PI / i); - int in1 = N - (offset + ni) - 1 + node_idx; - double w1 = -sin(kj * PI / i); - connect_node(node, stage_num, node_num, stage_idx + 1, out, in0, w0, in1, - w1); - } - - // S_kj/i --- C_Kj/i - offset += N_over_i / 4; - for (int ni = 0; ni < N_over_i / 4; ni++) { - int out = node_idx + offset + ni; - int in0 = out; - double w0 = sin(kj * PI / i); - int in1 = N - (offset + ni) - 1 + node_idx; - double w1 = cos(kj * PI / i); - connect_node(node, stage_num, node_num, stage_idx + 1, out, in0, w0, in1, - w1); - } - - // I_N/2i --- 0 - offset += N_over_i / 4; - for (int ni = 0; ni < N_over_i / 4; ni++) { - int out = node_idx + offset + ni; - int in = out; - connect_node(node, stage_num, node_num, stage_idx + 1, out, in, 1, in, 0); - } - } -} - -void gen_type4_graph(Node *node, int stage_num, int node_num, int stage_idx, - int node_idx, int idx, int N) { - int B_size = 1 << ((idx + 1) / 2); - for (int ni = 0; ni < N; ni += B_size) { - gen_B_graph(node, stage_num, node_num, stage_idx, node_idx + ni, B_size, - (ni / B_size) % 2); - } -} - -void gen_R_graph(Node *node, int stage_num, int node_num, int stage_idx, - int node_idx, int N) { - int max_idx = 2 * (get_max_bit(N) + 1) - 3; - for (int idx = 0; idx < max_idx; idx++) { - int s = stage_idx + max_idx - idx - 1; - if (idx == 0) { - // type 1 - gen_type1_graph(node, stage_num, node_num, s, node_idx, N); - } else if (idx == max_idx - 1) { - // type 2 - gen_type2_graph(node, stage_num, node_num, s, node_idx, N); - } else if ((idx + 1) % 2 == 0) { - // type 4 - gen_type4_graph(node, stage_num, node_num, s, node_idx, idx, N); - } else if ((idx + 1) % 2 == 1) { - // type 3 - gen_type3_graph(node, stage_num, node_num, s, node_idx, idx, N); - } else { - printf("check gen_R_graph()\n"); - } - } -} - -void gen_DCT_graph(Node *node, int stage_num, int node_num, int stage_idx, - int node_idx, int N) { - if (N > 2) { - gen_B_graph(node, stage_num, node_num, stage_idx, node_idx, N, 0); - gen_DCT_graph(node, stage_num, node_num, stage_idx + 1, node_idx, N / 2); - gen_R_graph(node, stage_num, node_num, stage_idx + 1, node_idx + N / 2, - N / 2); - } else { - // generate dct_2 - connect_node(node, stage_num, node_num, stage_idx + 1, node_idx, node_idx, - cos(PI / 4), node_idx + 1, cos(PI / 4)); - connect_node(node, stage_num, node_num, stage_idx + 1, node_idx + 1, - node_idx + 1, -cos(PI / 4), node_idx, cos(PI / 4)); - } -} - -int get_dct_stage_num(int size) { return 2 * get_max_bit(size); } - -void gen_DCT_graph_1d(Node *node, int stage_num, int node_num, int stage_idx, - int node_idx, int dct_node_num) { - gen_DCT_graph(node, stage_num, node_num, stage_idx, node_idx, dct_node_num); - int dct_stage_num = get_dct_stage_num(dct_node_num); - gen_P_graph(node, stage_num, node_num, stage_idx + dct_stage_num - 2, - node_idx, dct_node_num); -} - -void gen_adst_B_graph(Node *node, int stage_num, int node_num, int stage_idx, - int node_idx, int adst_idx) { - int size = 1 << (adst_idx + 1); - for (int ni = 0; ni < size / 2; ni++) { - int nOut = node_idx + ni; - int nIn = nOut + size / 2; - connect_node(node, stage_num, node_num, stage_idx + 1, nOut, nOut, 1, nIn, - 1); - // printf("nOut: %d nIn: %d\n", nOut, nIn); - } - for (int ni = size / 2; ni < size; ni++) { - int nOut = node_idx + ni; - int nIn = nOut - size / 2; - connect_node(node, stage_num, node_num, stage_idx + 1, nOut, nOut, -1, nIn, - 1); - // printf("ndctOut: %d nIn: %d\n", nOut, nIn); - } -} - -void gen_adst_U_graph(Node *node, int stage_num, int node_num, int stage_idx, - int node_idx, int adst_idx, int adst_node_num) { - int size = 1 << (adst_idx + 1); - for (int ni = 0; ni < adst_node_num; ni += size) { - gen_adst_B_graph(node, stage_num, node_num, stage_idx, node_idx + ni, - adst_idx); - } -} - -void gen_adst_T_graph(Node *node, int stage_num, int node_num, int stage_idx, - int node_idx, double freq) { - connect_node(node, stage_num, node_num, stage_idx + 1, node_idx, node_idx, - cos(freq * PI), node_idx + 1, sin(freq * PI)); - connect_node(node, stage_num, node_num, stage_idx + 1, node_idx + 1, - node_idx + 1, -cos(freq * PI), node_idx, sin(freq * PI)); -} - -void gen_adst_E_graph(Node *node, int stage_num, int node_num, int stage_idx, - int node_idx, int adst_idx) { - int size = 1 << (adst_idx); - for (int i = 0; i < size / 2; i++) { - int ni = i * 2; - double fi = (1 + 4 * i) * 1.0 / (1 << (adst_idx + 1)); - gen_adst_T_graph(node, stage_num, node_num, stage_idx, node_idx + ni, fi); - } -} - -void gen_adst_V_graph(Node *node, int stage_num, int node_num, int stage_idx, - int node_idx, int adst_idx, int adst_node_num) { - int size = 1 << (adst_idx); - for (int i = 0; i < adst_node_num / size; i++) { - if (i % 2 == 1) { - int ni = i * size; - gen_adst_E_graph(node, stage_num, node_num, stage_idx, node_idx + ni, - adst_idx); - } - } -} -void gen_adst_VJ_graph(Node *node, int stage_num, int node_num, int stage_idx, - int node_idx, int adst_node_num) { - for (int i = 0; i < adst_node_num / 2; i++) { - int ni = i * 2; - double fi = (1 + 4 * i) * 1.0 / (4 * adst_node_num); - gen_adst_T_graph(node, stage_num, node_num, stage_idx, node_idx + ni, fi); - } -} -void gen_adst_Q_graph(Node *node, int stage_num, int node_num, int stage_idx, - int node_idx, int adst_node_num) { - // reverse order when idx is 1, 3, 5, 7 ... - // example of adst_node_num = 8: - // 0 1 2 3 4 5 6 7 - // --> 0 7 2 5 4 3 6 1 - for (int ni = 0; ni < adst_node_num; ni++) { - if (ni % 2 == 0) { - int out = node_idx + ni; - connect_node(node, stage_num, node_num, stage_idx + 1, out, out, 1, out, - 0); - } else { - int out = node_idx + ni; - int in = node_idx + adst_node_num - ni; - connect_node(node, stage_num, node_num, stage_idx + 1, out, in, 1, in, 0); - } - } -} -void gen_adst_Ibar_graph(Node *node, int stage_num, int node_num, int stage_idx, - int node_idx, int adst_node_num) { - // reverse order - // 0 1 2 3 --> 3 2 1 0 - for (int ni = 0; ni < adst_node_num; ni++) { - int out = node_idx + ni; - int in = node_idx + adst_node_num - ni - 1; - connect_node(node, stage_num, node_num, stage_idx + 1, out, in, 1, in, 0); - } -} - -int get_Q_out2in(int adst_node_num, int out) { - int in; - if (out % 2 == 0) { - in = out; - } else { - in = adst_node_num - out; - } - return in; -} - -int get_Ibar_out2in(int adst_node_num, int out) { - return adst_node_num - out - 1; -} - -void gen_adst_IbarQ_graph(Node *node, int stage_num, int node_num, - int stage_idx, int node_idx, int adst_node_num) { - // in -> Ibar -> Q -> out - for (int ni = 0; ni < adst_node_num; ni++) { - int out = node_idx + ni; - int in = node_idx + - get_Ibar_out2in(adst_node_num, get_Q_out2in(adst_node_num, ni)); - connect_node(node, stage_num, node_num, stage_idx + 1, out, in, 1, in, 0); - } -} - -void gen_adst_D_graph(Node *node, int stage_num, int node_num, int stage_idx, - int node_idx, int adst_node_num) { - // reverse order - for (int ni = 0; ni < adst_node_num; ni++) { - int out = node_idx + ni; - int in = out; - if (ni % 2 == 0) { - connect_node(node, stage_num, node_num, stage_idx + 1, out, in, 1, in, 0); - } else { - connect_node(node, stage_num, node_num, stage_idx + 1, out, in, -1, in, - 0); - } - } -} - -int get_hadamard_idx(int x, int adst_node_num) { - int max_bit = get_max_bit(adst_node_num - 1); - x = bitwise_reverse(x, max_bit); - - // gray code - int c = x & 1; - int p = x & 1; - int y = c; - - for (int i = 1; i <= max_bit; i++) { - p = c; - c = (x >> i) & 1; - y += (c ^ p) << i; - } - return y; -} - -void gen_adst_Ht_graph(Node *node, int stage_num, int node_num, int stage_idx, - int node_idx, int adst_node_num) { - for (int ni = 0; ni < adst_node_num; ni++) { - int out = node_idx + ni; - int in = node_idx + get_hadamard_idx(ni, adst_node_num); - connect_node(node, stage_num, node_num, stage_idx + 1, out, in, 1, in, 0); - } -} - -void gen_adst_HtD_graph(Node *node, int stage_num, int node_num, int stage_idx, - int node_idx, int adst_node_num) { - for (int ni = 0; ni < adst_node_num; ni++) { - int out = node_idx + ni; - int in = node_idx + get_hadamard_idx(ni, adst_node_num); - double inW; - if (ni % 2 == 0) - inW = 1; - else - inW = -1; - connect_node(node, stage_num, node_num, stage_idx + 1, out, in, inW, in, 0); - } -} - -int get_adst_stage_num(int adst_node_num) { - return 2 * get_max_bit(adst_node_num) + 2; -} - -int gen_iadst_graph(Node *node, int stage_num, int node_num, int stage_idx, - int node_idx, int adst_node_num) { - int max_bit = get_max_bit(adst_node_num); - int si = 0; - gen_adst_IbarQ_graph(node, stage_num, node_num, stage_idx + si, node_idx, - adst_node_num); - si++; - gen_adst_VJ_graph(node, stage_num, node_num, stage_idx + si, node_idx, - adst_node_num); - si++; - for (int adst_idx = max_bit - 1; adst_idx >= 1; adst_idx--) { - gen_adst_U_graph(node, stage_num, node_num, stage_idx + si, node_idx, - adst_idx, adst_node_num); - si++; - gen_adst_V_graph(node, stage_num, node_num, stage_idx + si, node_idx, - adst_idx, adst_node_num); - si++; - } - gen_adst_HtD_graph(node, stage_num, node_num, stage_idx + si, node_idx, - adst_node_num); - si++; - return si + 1; -} - -int gen_adst_graph(Node *node, int stage_num, int node_num, int stage_idx, - int node_idx, int adst_node_num) { - int hybrid_stage_num = get_hybrid_stage_num(TYPE_ADST, adst_node_num); - // generate a adst tempNode - Node *tempNode = new Node[hybrid_stage_num * adst_node_num]; - init_graph(tempNode, hybrid_stage_num, adst_node_num); - int si = gen_iadst_graph(tempNode, hybrid_stage_num, adst_node_num, 0, 0, - adst_node_num); - - // tempNode's inverse graph to node[stage_idx][node_idx] - gen_inv_graph(tempNode, hybrid_stage_num, adst_node_num, node, stage_num, - node_num, stage_idx, node_idx); - delete[] tempNode; - return si; -} - -void connect_layer_2d(Node *node, int stage_num, int node_num, int stage_idx, - int node_idx, int dct_node_num) { - for (int first = 0; first < dct_node_num; first++) { - for (int second = 0; second < dct_node_num; second++) { - // int sIn = stage_idx; - int sOut = stage_idx + 1; - int nIn = node_idx + first * dct_node_num + second; - int nOut = node_idx + second * dct_node_num + first; - - // printf("sIn: %d nIn: %d sOut: %d nOut: %d\n", sIn, nIn, sOut, nOut); - - connect_node(node, stage_num, node_num, sOut, nOut, nIn, 1, nIn, 0); - } - } -} - -void connect_layer_2d_new(Node *node, int stage_num, int node_num, - int stage_idx, int node_idx, int dct_node_num0, - int dct_node_num1) { - for (int i = 0; i < dct_node_num1; i++) { - for (int j = 0; j < dct_node_num0; j++) { - // int sIn = stage_idx; - int sOut = stage_idx + 1; - int nIn = node_idx + i * dct_node_num0 + j; - int nOut = node_idx + j * dct_node_num1 + i; - - // printf("sIn: %d nIn: %d sOut: %d nOut: %d\n", sIn, nIn, sOut, nOut); - - connect_node(node, stage_num, node_num, sOut, nOut, nIn, 1, nIn, 0); - } - } -} - -void gen_DCT_graph_2d(Node *node, int stage_num, int node_num, int stage_idx, - int node_idx, int dct_node_num) { - int dct_stage_num = get_dct_stage_num(dct_node_num); - // put 2 layers of dct_node_num DCTs on the graph - for (int ni = 0; ni < dct_node_num; ni++) { - gen_DCT_graph_1d(node, stage_num, node_num, stage_idx, - node_idx + ni * dct_node_num, dct_node_num); - gen_DCT_graph_1d(node, stage_num, node_num, stage_idx + dct_stage_num, - node_idx + ni * dct_node_num, dct_node_num); - } - // connect first layer and second layer - connect_layer_2d(node, stage_num, node_num, stage_idx + dct_stage_num - 1, - node_idx, dct_node_num); -} - -int get_hybrid_stage_num(int type, int hybrid_node_num) { - if (type == TYPE_DCT || type == TYPE_IDCT) { - return get_dct_stage_num(hybrid_node_num); - } else if (type == TYPE_ADST || type == TYPE_IADST) { - return get_adst_stage_num(hybrid_node_num); - } - return 0; -} - -int get_hybrid_2d_stage_num(int type0, int type1, int hybrid_node_num) { - int stage_num = 0; - stage_num += get_hybrid_stage_num(type0, hybrid_node_num); - stage_num += get_hybrid_stage_num(type1, hybrid_node_num); - return stage_num; -} - -int get_hybrid_2d_stage_num_new(int type0, int type1, int hybrid_node_num0, - int hybrid_node_num1) { - int stage_num = 0; - stage_num += get_hybrid_stage_num(type0, hybrid_node_num0); - stage_num += get_hybrid_stage_num(type1, hybrid_node_num1); - return stage_num; -} - -int get_hybrid_amplify_factor(int type, int hybrid_node_num) { - return get_max_bit(hybrid_node_num) - 1; -} - -void gen_hybrid_graph_1d(Node *node, int stage_num, int node_num, int stage_idx, - int node_idx, int hybrid_node_num, int type) { - if (type == TYPE_DCT) { - gen_DCT_graph_1d(node, stage_num, node_num, stage_idx, node_idx, - hybrid_node_num); - } else if (type == TYPE_ADST) { - gen_adst_graph(node, stage_num, node_num, stage_idx, node_idx, - hybrid_node_num); - } else if (type == TYPE_IDCT) { - int hybrid_stage_num = get_hybrid_stage_num(type, hybrid_node_num); - // generate a dct tempNode - Node *tempNode = new Node[hybrid_stage_num * hybrid_node_num]; - init_graph(tempNode, hybrid_stage_num, hybrid_node_num); - gen_DCT_graph_1d(tempNode, hybrid_stage_num, hybrid_node_num, 0, 0, - hybrid_node_num); - - // tempNode's inverse graph to node[stage_idx][node_idx] - gen_inv_graph(tempNode, hybrid_stage_num, hybrid_node_num, node, stage_num, - node_num, stage_idx, node_idx); - delete[] tempNode; - } else if (type == TYPE_IADST) { - int hybrid_stage_num = get_hybrid_stage_num(type, hybrid_node_num); - // generate a adst tempNode - Node *tempNode = new Node[hybrid_stage_num * hybrid_node_num]; - init_graph(tempNode, hybrid_stage_num, hybrid_node_num); - gen_adst_graph(tempNode, hybrid_stage_num, hybrid_node_num, 0, 0, - hybrid_node_num); - - // tempNode's inverse graph to node[stage_idx][node_idx] - gen_inv_graph(tempNode, hybrid_stage_num, hybrid_node_num, node, stage_num, - node_num, stage_idx, node_idx); - delete[] tempNode; - } -} - -void gen_hybrid_graph_2d(Node *node, int stage_num, int node_num, int stage_idx, - int node_idx, int hybrid_node_num, int type0, - int type1) { - int hybrid_stage_num = get_hybrid_stage_num(type0, hybrid_node_num); - - for (int ni = 0; ni < hybrid_node_num; ni++) { - gen_hybrid_graph_1d(node, stage_num, node_num, stage_idx, - node_idx + ni * hybrid_node_num, hybrid_node_num, - type0); - gen_hybrid_graph_1d(node, stage_num, node_num, stage_idx + hybrid_stage_num, - node_idx + ni * hybrid_node_num, hybrid_node_num, - type1); - } - - // connect first layer and second layer - connect_layer_2d(node, stage_num, node_num, stage_idx + hybrid_stage_num - 1, - node_idx, hybrid_node_num); -} - -void gen_hybrid_graph_2d_new(Node *node, int stage_num, int node_num, - int stage_idx, int node_idx, int hybrid_node_num0, - int hybrid_node_num1, int type0, int type1) { - int hybrid_stage_num0 = get_hybrid_stage_num(type0, hybrid_node_num0); - - for (int ni = 0; ni < hybrid_node_num1; ni++) { - gen_hybrid_graph_1d(node, stage_num, node_num, stage_idx, - node_idx + ni * hybrid_node_num0, hybrid_node_num0, - type0); - } - for (int ni = 0; ni < hybrid_node_num0; ni++) { - gen_hybrid_graph_1d( - node, stage_num, node_num, stage_idx + hybrid_stage_num0, - node_idx + ni * hybrid_node_num1, hybrid_node_num1, type1); - } - - // connect first layer and second layer - connect_layer_2d_new(node, stage_num, node_num, - stage_idx + hybrid_stage_num0 - 1, node_idx, - hybrid_node_num0, hybrid_node_num1); -} - -void gen_inv_graph(Node *node, int stage_num, int node_num, Node *invNode, - int inv_stage_num, int inv_node_num, int inv_stage_idx, - int inv_node_idx) { - // clean up inNodeNum in invNode because of add_node - for (int si = 1 + inv_stage_idx; si < inv_stage_idx + stage_num; si++) { - for (int ni = inv_node_idx; ni < inv_node_idx + node_num; ni++) { - int idx = get_idx(si, ni, inv_node_num); - invNode[idx].inNodeNum = 0; - } - } - // generate inverse graph of node on invNode - for (int si = 1; si < stage_num; si++) { - for (int ni = 0; ni < node_num; ni++) { - int invSi = stage_num - si; - int idx = get_idx(si, ni, node_num); - for (int k = 0; k < node[idx].inNodeNum; k++) { - int invNi = node[idx].inNodeIdx[k]; - add_node(invNode, inv_stage_num, inv_node_num, invSi + inv_stage_idx, - invNi + inv_node_idx, ni + inv_node_idx, - node[idx].inWeight[k]); - } - } - } -} diff --git a/third_party/aom/tools/txfm_analyzer/txfm_graph.h b/third_party/aom/tools/txfm_analyzer/txfm_graph.h deleted file mode 100644 index 2e3c9551e..000000000 --- a/third_party/aom/tools/txfm_analyzer/txfm_graph.h +++ /dev/null @@ -1,161 +0,0 @@ -/* - * Copyright (c) 2018, Alliance for Open Media. All rights reserved - * - * This source code is subject to the terms of the BSD 2 Clause License and - * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License - * was not distributed with this source code in the LICENSE file, you can - * obtain it at www.aomedia.org/license/software. If the Alliance for Open - * Media Patent License 1.0 was not distributed with this source code in the - * PATENTS file, you can obtain it at www.aomedia.org/license/patent. - */ - -#ifndef AOM_TOOLS_TXFM_ANALYZER_TXFM_GRAPH_H_ -#define AOM_TOOLS_TXFM_ANALYZER_TXFM_GRAPH_H_ - -struct Node { - Node *inNode[2]; - int inNodeNum; - int inNodeIdx[2]; - double inWeight[2]; - double value; - int nodeIdx; - int stageIdx; - int visited; -}; - -#define PI (3.141592653589793238462643383279502884) -#define STAGENUM (10) -#define NODENUM (32) -#define COS_MOD (128) - -typedef enum { - TYPE_DCT = 0, - TYPE_ADST, - TYPE_IDCT, - TYPE_IADST, - TYPE_LAST -} TYPE_TXFM; - -TYPE_TXFM get_inv_type(TYPE_TXFM type); -void get_fun_name(char *str_fun_name, int str_buf_size, const TYPE_TXFM type, - const int txfm_size); - -void get_txfm_type_name(char *str_fun_name, int str_buf_size, - const TYPE_TXFM type, const int txfm_size); -void get_hybrid_2d_type_name(char *buf, int buf_size, const TYPE_TXFM type0, - const TYPE_TXFM type1, const int txfm_size0, - const int txfm_size1); -unsigned int get_max_bit(unsigned int x); -unsigned int bitwise_reverse(unsigned int x, int max_bit); -int get_idx(int ri, int ci, int cSize); - -int get_dct_stage_num(int size); -void reference_dct_1d(double *in, double *out, int size); -void reference_dct_2d(double *in, double *out, int size); -void connect_node(Node *node, int stage_num, int node_num, int stage_idx, - int node_idx, int in0, double w0, int in1, double w1); -void propagate(Node *node, int stage_num, int node_num, int stage); -void init_graph(Node *node, int stage_num, int node_num); -void graph_reset_visited(Node *node, int stage_num, int node_num); -void gen_B_graph(Node *node, int stage_num, int node_num, int stage_idx, - int node_idx, int N, int star); -void gen_P_graph(Node *node, int stage_num, int node_num, int stage_idx, - int node_idx, int N); - -void gen_type1_graph(Node *node, int stage_num, int node_num, int stage_idx, - int node_idx, int N); -void gen_type2_graph(Node *node, int stage_num, int node_num, int stage_idx, - int node_idx, int N); -void gen_type3_graph(Node *node, int stage_num, int node_num, int stage_idx, - int node_idx, int idx, int N); -void gen_type4_graph(Node *node, int stage_num, int node_num, int stage_idx, - int node_idx, int idx, int N); - -void gen_R_graph(Node *node, int stage_num, int node_num, int stage_idx, - int node_idx, int N); - -void gen_DCT_graph(Node *node, int stage_num, int node_num, int stage_idx, - int node_idx, int N); - -void gen_DCT_graph_1d(Node *node, int stage_num, int node_num, int stage_idx, - int node_idx, int dct_node_num); -void connect_layer_2d(Node *node, int stage_num, int node_num, int stage_idx, - int node_idx, int dct_node_num); - -void gen_DCT_graph_2d(Node *node, int stage_num, int node_num, int stage_idx, - int node_idx, int dct_node_num); - -void gen_adst_B_graph(Node *node, int stage_num, int node_num, int stage_idx, - int node_idx, int adst_idx); - -void gen_adst_U_graph(Node *node, int stage_num, int node_num, int stage_idx, - int node_idx, int adst_idx, int adst_node_num); -void gen_adst_T_graph(Node *node, int stage_num, int node_num, int stage_idx, - int node_idx, double freq); - -void gen_adst_E_graph(Node *node, int stage_num, int node_num, int stage_idx, - int node_idx, int adst_idx); - -void gen_adst_V_graph(Node *node, int stage_num, int node_num, int stage_idx, - int node_idx, int adst_idx, int adst_node_num); - -void gen_adst_VJ_graph(Node *node, int stage_num, int node_num, int stage_idx, - int node_idx, int adst_node_num); -void gen_adst_Q_graph(Node *node, int stage_num, int node_num, int stage_idx, - int node_idx, int adst_node_num); -void gen_adst_Ibar_graph(Node *node, int stage_num, int node_num, int stage_idx, - int node_idx, int adst_node_num); - -void gen_adst_D_graph(Node *node, int stage_num, int node_num, int stage_idx, - int node_idx, int adst_node_num); - -int get_hadamard_idx(int x, int adst_node_num); -void gen_adst_Ht_graph(Node *node, int stage_num, int node_num, int stage_idx, - int node_idx, int adst_node_num); - -int gen_adst_graph(Node *node, int stage_num, int node_num, int stage_idx, - int node_idx, int adst_node_num); -int gen_iadst_graph(Node *node, int stage_num, int node_num, int stage_idx, - int node_idx, int adst_node_num); -void reference_adst_1d(double *in, double *out, int size); - -int get_adst_stage_num(int adst_node_num); -int get_hybrid_stage_num(int type, int hybrid_node_num); -int get_hybrid_2d_stage_num(int type0, int type1, int hybrid_node_num); -int get_hybrid_2d_stage_num_new(int type0, int type1, int hybrid_node_num0, - int hybrid_node_num1); -int get_hybrid_amplify_factor(int type, int hybrid_node_num); -void gen_hybrid_graph_1d(Node *node, int stage_num, int node_num, int stage_idx, - int node_idx, int hybrid_node_num, int type); -void gen_hybrid_graph_2d(Node *node, int stage_num, int node_num, int stage_idx, - int node_idx, int hybrid_node_num, int type0, - int type1); -void gen_hybrid_graph_2d_new(Node *node, int stage_num, int node_num, - int stage_idx, int node_idx, int hybrid_node_num0, - int hybrid_node_num1, int type0, int type1); - -void reference_hybrid_2d(double *in, double *out, int size, int type0, - int type1); - -void reference_hybrid_2d_new(double *in, double *out, int size0, int size1, - int type0, int type1); -void reference_adst_dct_2d(double *in, double *out, int size); - -void gen_code(Node *node, int stage_num, int node_num, TYPE_TXFM type); - -void gen_inv_graph(Node *node, int stage_num, int node_num, Node *invNode, - int inv_stage_num, int inv_node_num, int inv_stage_idx, - int inv_node_idx); - -TYPE_TXFM hybrid_char_to_int(char ctype); - -int64_t round_shift(int64_t value, int bit); -void round_shift_array(int32_t *arr, int size, int bit); -void estimate_value(Node *node, int stage_num, int node_num, int stage_idx, - int node_idx, int estimate_bit); -void amplify_value(Node *node, int stage_num, int node_num, int stage_idx, - int node_idx, int estimate_bit); -void propagate_estimate_amlify(Node *node, int stage_num, int node_num, - int stage_idx, int amplify_bit, - int estimate_bit); -#endif // AOM_TOOLS_TXFM_ANALYZER_TXFM_GRAPH_H_ diff --git a/third_party/aom/tools/wrap-commit-msg.py b/third_party/aom/tools/wrap-commit-msg.py deleted file mode 100755 index 1c7882443..000000000 --- a/third_party/aom/tools/wrap-commit-msg.py +++ /dev/null @@ -1,72 +0,0 @@ -#!/usr/bin/env python -## -## Copyright (c) 2016, Alliance for Open Media. All rights reserved -## -## This source code is subject to the terms of the BSD 2 Clause License and -## the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License -## was not distributed with this source code in the LICENSE file, you can -## obtain it at www.aomedia.org/license/software. If the Alliance for Open -## Media Patent License 1.0 was not distributed with this source code in the -## PATENTS file, you can obtain it at www.aomedia.org/license/patent. -## -"""Wraps paragraphs of text, preserving manual formatting - -This is like fold(1), but has the special convention of not modifying lines -that start with whitespace. This allows you to intersperse blocks with -special formatting, like code blocks, with written prose. The prose will -be wordwrapped, and the manual formatting will be preserved. - - * This won't handle the case of a bulleted (or ordered) list specially, so - manual wrapping must be done. - -Occasionally it's useful to put something with explicit formatting that -doesn't look at all like a block of text inline. - - indicator = has_leading_whitespace(line); - if (indicator) - preserve_formatting(line); - -The intent is that this docstring would make it through the transform -and still be legible and presented as it is in the source. If additional -cases are handled, update this doc to describe the effect. -""" - -__author__ = "jkoleszar@google.com" -import textwrap -import sys - -def wrap(text): - if text: - return textwrap.fill(text, break_long_words=False) + '\n' - return "" - - -def main(fileobj): - text = "" - output = "" - while True: - line = fileobj.readline() - if not line: - break - - if line.lstrip() == line: - text += line - else: - output += wrap(text) - text="" - output += line - output += wrap(text) - - # Replace the file or write to stdout. - if fileobj == sys.stdin: - fileobj = sys.stdout - else: - fileobj.seek(0) - fileobj.truncate(0) - fileobj.write(output) - -if __name__ == "__main__": - if len(sys.argv) > 1: - main(open(sys.argv[1], "r+")) - else: - main(sys.stdin) |