diff options
Diffstat (limited to 'Tools/c-analyzer/c_analyzer')
| -rw-r--r-- | Tools/c-analyzer/c_analyzer/__init__.py | 8 | ||||
| -rw-r--r-- | Tools/c-analyzer/c_analyzer/__main__.py | 104 | ||||
| -rw-r--r-- | Tools/c-analyzer/c_analyzer/analyze.py | 6 | ||||
| -rw-r--r-- | Tools/c-analyzer/c_analyzer/datafiles.py | 3 | ||||
| -rw-r--r-- | Tools/c-analyzer/c_analyzer/info.py | 42 | ||||
| -rw-r--r-- | Tools/c-analyzer/c_analyzer/match.py | 212 |
6 files changed, 296 insertions, 79 deletions
diff --git a/Tools/c-analyzer/c_analyzer/__init__.py b/Tools/c-analyzer/c_analyzer/__init__.py index 4a01cd396f5..171fa25102b 100644 --- a/Tools/c-analyzer/c_analyzer/__init__.py +++ b/Tools/c-analyzer/c_analyzer/__init__.py @@ -4,10 +4,12 @@ from c_parser import ( from c_parser.info import ( KIND, TypeDeclaration, - filter_by_kind, - collate_by_kind_group, resolve_parsed, ) +from c_parser.match import ( + filter_by_kind, + group_by_kinds, +) from . import ( analyze as _analyze, datafiles as _datafiles, @@ -55,7 +57,7 @@ def analyze_decls(decls, known, *, ) decls = list(decls) - collated = collate_by_kind_group(decls) + collated = group_by_kinds(decls) types = {decl: None for decl in collated['type']} typespecs = _analyze.get_typespecs(types) diff --git a/Tools/c-analyzer/c_analyzer/__main__.py b/Tools/c-analyzer/c_analyzer/__main__.py index 1fd45b985d9..4cff1d4efb5 100644 --- a/Tools/c-analyzer/c_analyzer/__main__.py +++ b/Tools/c-analyzer/c_analyzer/__main__.py @@ -1,5 +1,6 @@ import io import logging +import os import os.path import re import sys @@ -9,6 +10,7 @@ from c_common.scriptutil import ( add_verbosity_cli, add_traceback_cli, add_sepval_cli, + add_progress_cli, add_files_cli, add_commands_cli, process_args_by_key, @@ -17,11 +19,13 @@ from c_common.scriptutil import ( filter_filenames, iter_marks, ) -from c_parser.info import KIND, is_type_decl +from c_parser.info import KIND +from c_parser.match import is_type_decl +from .match import filter_forward from . import ( analyze as _analyze, - check_all as _check_all, datafiles as _datafiles, + check_all as _check_all, ) @@ -44,7 +48,7 @@ logger = logging.getLogger(__name__) TABLE_SECTIONS = { 'types': ( ['kind', 'name', 'data', 'file'], - is_type_decl, + KIND.is_type_decl, (lambda v: (v.kind.value, v.filename or '', v.name)), ), 'typedefs': 'types', @@ -167,9 +171,7 @@ def _get_check_handlers(fmt, printer, verbosity=VERBOSITY): print(f'{data.filename}:{name} - {failure}') elif fmt == 'summary': def handle_failure(failure, data): - parent = data.parent or '' - funcname = parent if isinstance(parent, str) else parent.name - print(f'{data.filename:35}\t{funcname or "-":35}\t{data.name:40}\t{failure}') + print(_fmt_one_summary(data, failure)) elif fmt == 'full': div = '' def handle_failure(failure, data): @@ -230,6 +232,15 @@ def fmt_summary(analysis): yield f'grand total: {total}' +def _fmt_one_summary(item, extra=None): + parent = item.parent or '' + funcname = parent if isinstance(parent, str) else parent.name + if extra: + return f'{item.filename:35}\t{funcname or "-":35}\t{item.name:40}\t{extra}' + else: + return f'{item.filename:35}\t{funcname or "-":35}\t{item.name}' + + def fmt_full(analysis): # XXX Support sorting. items = sorted(analysis, key=lambda v: v.key) @@ -272,10 +283,12 @@ def _cli_check(parser, checks=None, **kwargs): args.checks = [check] else: process_checks = add_checks_cli(parser, checks=checks) + process_progress = add_progress_cli(parser) process_output = add_output_cli(parser, default=None) process_files = add_files_cli(parser, **kwargs) return [ process_checks, + process_progress, process_output, process_files, ] @@ -288,6 +301,7 @@ def cmd_check(filenames, *, relroot=None, failfast=False, iter_filenames=None, + track_progress=None, verbosity=VERBOSITY, _analyze=_analyze, _CHECKS=CHECKS, @@ -304,36 +318,53 @@ def cmd_check(filenames, *, ) = _get_check_handlers(fmt, printer, verbosity) filenames = filter_filenames(filenames, iter_filenames) + if track_progress: + filenames = track_progress(filenames) - logger.info('analyzing...') + logger.info('analyzing files...') analyzed = _analyze(filenames, **kwargs) if relroot: analyzed.fix_filenames(relroot) + decls = filter_forward(analyzed, markpublic=True) - logger.info('checking...') - numfailed = 0 - for data, failure in _check_all(analyzed, checks, failfast=failfast): + logger.info('checking analysis results...') + failed = [] + for data, failure in _check_all(decls, checks, failfast=failfast): if data is None: printer.info('stopping after one failure') break - if div is not None and numfailed > 0: + if div is not None and len(failed) > 0: printer.info(div) - numfailed += 1 + failed.append(data) handle_failure(failure, data) handle_after() printer.info('-------------------------') - logger.info(f'total failures: {numfailed}') + logger.info(f'total failures: {len(failed)}') logger.info('done checking') - if numfailed > 0: - sys.exit(numfailed) + if fmt == 'summary': + print('Categorized by storage:') + print() + from .match import group_by_storage + grouped = group_by_storage(failed, ignore_non_match=False) + for group, decls in grouped.items(): + print() + print(group) + for decl in decls: + print(' ', _fmt_one_summary(decl)) + print(f'subtotal: {len(decls)}') + + if len(failed) > 0: + sys.exit(len(failed)) def _cli_analyze(parser, **kwargs): + process_progress = add_progress_cli(parser) process_output = add_output_cli(parser) process_files = add_files_cli(parser, **kwargs) return [ + process_progress, process_output, process_files, ] @@ -343,6 +374,7 @@ def _cli_analyze(parser, **kwargs): def cmd_analyze(filenames, *, fmt=None, iter_filenames=None, + track_progress=None, verbosity=None, _analyze=_analyze, formats=FORMATS, @@ -356,49 +388,46 @@ def cmd_analyze(filenames, *, raise ValueError(f'unsupported fmt {fmt!r}') filenames = filter_filenames(filenames, iter_filenames) - if verbosity == 2: - def iter_filenames(filenames=filenames): - marks = iter_marks() - for filename in filenames: - print(next(marks), end='') - yield filename - filenames = iter_filenames() - elif verbosity > 2: - def iter_filenames(filenames=filenames): - for filename in filenames: - print(f'<{filename}>') - yield filename - filenames = iter_filenames() - - logger.info('analyzing...') + if track_progress: + filenames = track_progress(filenames) + + logger.info('analyzing files...') analyzed = _analyze(filenames, **kwargs) + decls = filter_forward(analyzed, markpublic=True) - for line in do_fmt(analyzed): + for line in do_fmt(decls): print(line) def _cli_data(parser, filenames=None, known=None): ArgumentParser = type(parser) common = ArgumentParser(add_help=False) - if filenames is None: - common.add_argument('filenames', metavar='FILE', nargs='+') + # These flags will get processed by the top-level parse_args(). + add_verbosity_cli(common) + add_traceback_cli(common) subs = parser.add_subparsers(dest='datacmd') sub = subs.add_parser('show', parents=[common]) if known is None: sub.add_argument('--known', required=True) + if filenames is None: + sub.add_argument('filenames', metavar='FILE', nargs='+') - sub = subs.add_parser('dump') + sub = subs.add_parser('dump', parents=[common]) if known is None: sub.add_argument('--known') sub.add_argument('--show', action='store_true') + process_progress = add_progress_cli(sub) - sub = subs.add_parser('check') + sub = subs.add_parser('check', parents=[common]) if known is None: sub.add_argument('--known', required=True) - return None + def process_args(args): + if args.datacmd == 'dump': + process_progress(args) + return process_args def cmd_data(datacmd, filenames, known=None, *, @@ -406,6 +435,7 @@ def cmd_data(datacmd, filenames, known=None, *, formats=FORMATS, extracolumns=None, relroot=None, + track_progress=None, **kwargs ): kwargs.pop('verbosity', None) @@ -417,6 +447,8 @@ def cmd_data(datacmd, filenames, known=None, *, for line in do_fmt(known): print(line) elif datacmd == 'dump': + if track_progress: + filenames = track_progress(filenames) analyzed = _analyze(filenames, **kwargs) if known is None or usestdout: outfile = io.StringIO() diff --git a/Tools/c-analyzer/c_analyzer/analyze.py b/Tools/c-analyzer/c_analyzer/analyze.py index d8ae915e420..267d058e07a 100644 --- a/Tools/c-analyzer/c_analyzer/analyze.py +++ b/Tools/c-analyzer/c_analyzer/analyze.py @@ -3,15 +3,19 @@ from c_parser.info import ( TypeDeclaration, POTSType, FuncPtr, +) +from c_parser.match import ( is_pots, is_funcptr, ) from .info import ( IGNORED, UNKNOWN, - is_system_type, SystemType, ) +from .match import ( + is_system_type, +) def get_typespecs(typedecls): diff --git a/Tools/c-analyzer/c_analyzer/datafiles.py b/Tools/c-analyzer/c_analyzer/datafiles.py index 0de438cce47..d37a4eefe35 100644 --- a/Tools/c-analyzer/c_analyzer/datafiles.py +++ b/Tools/c-analyzer/c_analyzer/datafiles.py @@ -1,5 +1,6 @@ import c_common.tables as _tables import c_parser.info as _info +import c_parser.match as _match import c_parser.datafiles as _parser from . import analyze as _analyze @@ -17,7 +18,7 @@ def analyze_known(known, *, handle_unresolved=True, ): knowntypes = knowntypespecs = {} - collated = _info.collate_by_kind_group(known) + collated = _match.group_by_kinds(known) types = {decl: None for decl in collated['type']} typespecs = _analyze.get_typespecs(types) def analyze_decl(decl): diff --git a/Tools/c-analyzer/c_analyzer/info.py b/Tools/c-analyzer/c_analyzer/info.py index 23d77611a4c..be9281502d2 100644 --- a/Tools/c-analyzer/c_analyzer/info.py +++ b/Tools/c-analyzer/c_analyzer/info.py @@ -7,7 +7,11 @@ from c_parser.info import ( HighlevelParsedItem, Declaration, TypeDeclaration, +) +from c_parser.match import ( is_type_decl, +) +from .match import ( is_process_global, ) @@ -16,44 +20,6 @@ IGNORED = _misc.Labeled('IGNORED') UNKNOWN = _misc.Labeled('UNKNOWN') -# XXX Use known.tsv for these? -SYSTEM_TYPES = { - 'int8_t', - 'uint8_t', - 'int16_t', - 'uint16_t', - 'int32_t', - 'uint32_t', - 'int64_t', - 'uint64_t', - 'size_t', - 'ssize_t', - 'intptr_t', - 'uintptr_t', - 'wchar_t', - '', - # OS-specific - 'pthread_cond_t', - 'pthread_mutex_t', - 'pthread_key_t', - 'atomic_int', - 'atomic_uintptr_t', - '', - # lib-specific - 'WINDOW', # curses - 'XML_LChar', - 'XML_Size', - 'XML_Parser', - 'enum XML_Error', - 'enum XML_Status', - '', -} - - -def is_system_type(typespec): - return typespec in SYSTEM_TYPES - - class SystemType(TypeDeclaration): def __init__(self, name): diff --git a/Tools/c-analyzer/c_analyzer/match.py b/Tools/c-analyzer/c_analyzer/match.py new file mode 100644 index 00000000000..5c27e4a224a --- /dev/null +++ b/Tools/c-analyzer/c_analyzer/match.py @@ -0,0 +1,212 @@ +import os.path + +from c_parser import ( + info as _info, + match as _match, +) + + +_KIND = _info.KIND + + +# XXX Use known.tsv for these? +SYSTEM_TYPES = { + 'int8_t', + 'uint8_t', + 'int16_t', + 'uint16_t', + 'int32_t', + 'uint32_t', + 'int64_t', + 'uint64_t', + 'size_t', + 'ssize_t', + 'intptr_t', + 'uintptr_t', + 'wchar_t', + '', + # OS-specific + 'pthread_cond_t', + 'pthread_mutex_t', + 'pthread_key_t', + 'atomic_int', + 'atomic_uintptr_t', + '', + # lib-specific + 'WINDOW', # curses + 'XML_LChar', + 'XML_Size', + 'XML_Parser', + 'enum XML_Error', + 'enum XML_Status', + '', +} + + +def is_system_type(typespec): + return typespec in SYSTEM_TYPES + + +################################## +# decl matchers + +def is_public(decl): + if not decl.filename.endswith('.h'): + return False + if 'Include' not in decl.filename.split(os.path.sep): + return False + return True + + +def is_process_global(vardecl): + kind, storage, _, _, _ = _info.get_parsed_vartype(vardecl) + if kind is not _KIND.VARIABLE: + raise NotImplementedError(vardecl) + if 'static' in (storage or ''): + return True + + if hasattr(vardecl, 'parent'): + parent = vardecl.parent + else: + parent = vardecl.get('parent') + return not parent + + +def is_fixed_type(vardecl): + if not vardecl: + return None + _, _, _, typespec, abstract = _info.get_parsed_vartype(vardecl) + if 'typeof' in typespec: + raise NotImplementedError(vardecl) + elif not abstract: + return True + + if '*' not in abstract: + # XXX What about []? + return True + elif _match._is_funcptr(abstract): + return True + else: + for after in abstract.split('*')[1:]: + if not after.lstrip().startswith('const'): + return False + else: + return True + + +def is_immutable(vardecl): + if not vardecl: + return None + if not is_fixed_type(vardecl): + return False + _, _, typequal, _, _ = _info.get_parsed_vartype(vardecl) + # If there, it can only be "const" or "volatile". + return typequal == 'const' + + +def is_public_api(decl): + if not is_public(decl): + return False + if decl.kind is _KIND.TYPEDEF: + return True + elif _match.is_type_decl(decl): + return not _match.is_forward_decl(decl) + else: + return _match.is_external_reference(decl) + + +def is_public_declaration(decl): + if not is_public(decl): + return False + if decl.kind is _KIND.TYPEDEF: + return True + elif _match.is_type_decl(decl): + return _match.is_forward_decl(decl) + else: + return _match.is_external_reference(decl) + + +def is_public_definition(decl): + if not is_public(decl): + return False + if decl.kind is _KIND.TYPEDEF: + return True + elif _match.is_type_decl(decl): + return not _match.is_forward_decl(decl) + else: + return not _match.is_external_reference(decl) + + +def is_public_impl(decl): + if not _KIND.is_decl(decl.kind): + return False + # See filter_forward() about "is_public". + return getattr(decl, 'is_public', False) + + +def is_module_global_decl(decl): + if is_public_impl(decl): + return False + if _match.is_forward_decl(decl): + return False + return not _match.is_local_var(decl) + + +################################## +# filtering with matchers + +def filter_forward(items, *, markpublic=False): + if markpublic: + public = set() + actual = [] + for item in items: + if is_public_api(item): + public.add(item.id) + elif not _match.is_forward_decl(item): + actual.append(item) + else: + # non-public duplicate! + # XXX + raise Exception(item) + for item in actual: + _info.set_flag(item, 'is_public', item.id in public) + yield item + else: + for item in items: + if _match.is_forward_decl(item): + continue + yield item + + +################################## +# grouping with matchers + +def group_by_storage(decls, **kwargs): + def is_module_global(decl): + if not is_module_global_decl(decl): + return False + if decl.kind == _KIND.VARIABLE: + if _info.get_effective_storage(decl) == 'static': + # This is covered by is_static_module_global(). + return False + return True + def is_static_module_global(decl): + if not _match.is_global_var(decl): + return False + return _info.get_effective_storage(decl) == 'static' + def is_static_local(decl): + if not _match.is_local_var(decl): + return False + return _info.get_effective_storage(decl) == 'static' + #def is_local(decl): + # if not _match.is_local_var(decl): + # return False + # return _info.get_effective_storage(decl) != 'static' + categories = { + #'extern': is_extern, + 'published': is_public_impl, + 'module-global': is_module_global, + 'static-module-global': is_static_module_global, + 'static-local': is_static_local, + } + return _match.group_by_category(decls, categories, **kwargs) |