#! /usr/bin/python3

#   Copyright (c) MediaTek USA Inc., 2020-2026
#
#   This program is free software;  you can redistribute it and/or modify
#   it under the terms of the GNU General Public License as published by
#   the Free Software Foundation; either version 2 of the License, or (at
#   your option) any later version.
#
#   This program is distributed in the hope that it will be useful, but
#   WITHOUT ANY WARRANTY;  without even the implied warranty of
#   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
#   General Public License for more details.
#
#   You should have received a copy of the GNU General Public License
#   along with this program;  if not, see
#   <http://www.gnu.org/licenses/>.
#
#
# This script traverses XML coverage data in
# https://raw.githubusercontent.com/cobertura/web/master/htdocs/xml/coverage-04.dtd
# format (e.g., generated by Cobertura or the Coverage.py module) and translates it
# into LCOV .info format.
#
# It also reads the JaCoCo XML report format
# https://www.jacoco.org/jacoco/trunk/coverage/report.dtd
# ('-//JACOCO//DTD Report 1.1//EN') - which is a quite different schema:  see
# the comments on the '_process_jacoco' method below for the differences which
# matter to the translation.
#
# We distinguish the two from their root element, 'coverage' vs. 'report'.
#
# Arguably, this should be written in Perl so we could use lcovutil module
# utilities.
# In the meantime:  suggested use model is to translate the XML using this
# utility, then read it back into lcov for additional processing.
#
# @todo figure out how to characterize branch expressions in XML data

import os
import os.path
import sys
import re
import xml.etree.ElementTree as ET
import fnmatch
import subprocess
import copy
import base64
import hashlib

# Windows path handling.  See comments in py2lcov and xml2lcovutil.py, for reasoning
drive_mounts = ['', '/cygdrive']
# var used for testing
if 'LCOV_DRIVE_MOUNT' in os.environ:
    drive_mounts.insert(0, os.environ['LCOV_DRIVE_MOUNT'])
translate_windows_names = ('LCOV_DRIVE_MOUNT' in os.environ or
                           sys.platform in ('cygwin', 'msys'))
drive_letter_re = re.compile(r'^([A-Za-z]):/(.*)$')


def posix_path(path):
    """The name this python can use for the file the argument names:  the
    argument itself, unless it is a Windows path name and we are a python which
    cannot read one."""

    if not path or not translate_windows_names:
        return path
    # the separator first, and everywhere:  a backslash is the Windows
    #   separator, whether the path is absolute, relative or a UNC name, and
    #   whether or not it is written consistently - 'Z:\dir/other\file' is a
    #   path a Windows tool might pass us.  The result of this alone is usable
    #   for a relative name, and for a UNC name, which is '//host/share/...'
    #   here
    path = path.replace('\\', '/')
    m = drive_letter_re.match(path)
    if not m:
        return path
    drive = m.group(1).lower()
    for mount in drive_mounts:
        if os.path.isdir('%s/%s' % (mount, drive)):
            return '%s/%s/%s' % (mount, drive, m.group(2))
    # a drive this installation has not mounted:  there is no name for it to be
    #   translated to, so leave it as is and let whatever needs the file report
    #   the drive the user named
    return path


def posix_paths(args, *names):
    """'posix_path' applied in place to the named attributes of 'args' - each of
    them either one path or a list of them.  This is how a tool translates
    everything it was handed in one place, before any of it is looked at."""

    for name in names:
        value = getattr(args, name, None)
        if isinstance(value, list):
            setattr(args, name, [posix_path(p) for p in value])
        elif value:
            setattr(args, name, posix_path(value))


def line_hash(line: str) -> str:
    """Produce a hash of a source line for use in the LCOV file."""
    hashed = hashlib.md5(line.encode("utf-8")).digest()
    return base64.b64encode(hashed).decode("ascii").rstrip("=")


# The coverage status of a JaCoCo line, as JaCoCo itself computes it in
#  LineImpl.getStatus():  the instruction status and the branch status of the
#  line are OR'ed together, so PARTLY_COVERED is also what you get when one of
#  the two says NOT_COVERED and the other says FULLY_COVERED.
JACOCO_EMPTY = 0
JACOCO_NOT_COVERED = 1
JACOCO_FULLY_COVERED = 2
JACOCO_PARTLY_COVERED = 3


def jacoco_status(missed: int, covered: int) -> int:
    """The status bits a JaCoCo (missed, covered) counter pair stands for."""
    if missed + covered == 0:
        return JACOCO_EMPTY
    if missed > 0:
        return JACOCO_PARTLY_COVERED if covered > 0 else JACOCO_NOT_COVERED
    return JACOCO_FULLY_COVERED


def jacoco_counter(node, kind: str):
    """The (missed, covered) pair of the 'kind' counter of a JaCoCo element:
    (0, 0) if the element has no counter of that kind, and None if it has one
    which cannot be read."""
    for counter in node.findall('counter'):
        if counter.attrib.get('type') == kind:
            try:
                return (int(counter.attrib['missed']),
                        int(counter.attrib['covered']))
            except (KeyError, ValueError):
                return None
    return (0, 0)


class ProcessFile:
    """Expected/support scriptArgs:
    args.input     : name of XML file
    args.outf      : output FILE handle (written to)
    args.testName  : LCOV test name (optional) - see 'man lcov'
    args.excludePatterns :
                     comma-separated list of glob patterns
    args.verbose   : verbosity
    args.version   : version script callback
    args.checksum  : compute base64 checksum for each line - see 'man lcov'
    args.isPython  : input XML file came from Coverage.py - so apply certain
                     Python-specific derivations.
    args.deriveFunctions :
                     derive function coverpoints (primarily useful for Python -
                     see 'py2lcov --help' and the Coverage.py documentation
    args.tabWidth  : tab width to assume when deriving information from indentation -
                     used during Python function derivation.
    args.keepGoing : do not stop when error or inconsistency is detected
    args.format    : 'auto', 'cobertura' or 'jacoco' - which schema the input
                     XML is expected to use.  Optional:  'auto' if unset.
    args.sourceDirectories :
                     list of directories to search for source files, used after
                     the directories the XML itself names (Cobertura) and as the
                     entire search path for JaCoCo XML, which names none.
                     Optional:  empty if unset.

    """

    usageNote = """

Note that the XML coverage data format does not contain enough information
to deduce exactly which branch expressions have been taken or not taken.

It reports the total number of branch expressions associated with a particular
line, and the number of those which have been taken. There is no way to know
(except, possibly by inspection of surrounding code and/or some understanding
of your implementation) exactly which ones.

This is a problem in at least 2 ways:

  - It is not straightforward to use the result to improve your regression
    suite because you don't really know what was exercised/not exercised.

  - Coverage data merge is problematic.

      o For example: you have two testcase XML files, each of which hit
        4 of 8 branches on some line.

      o  Does that mean you hit 4 of them (both tests exercised the same
         code), all 8 (tests exercised disjoint subsets), or some number
         between?

    This implementation assumes that the first M branches are the ones
    which are hit and the remaining N-M were not hit, in each testcase.
    Thus, the combined result in the above example would claim 4 of 8
    branches hit.
    This definition turns out to be a lower bound.

    If your data comes from JaCoCo, this merge problem is avoidable:
    merge before you translate rather than after - see 'Merging JaCoCo
    data' below.
"""

    jacocoNote = """
JaCoCo XML input:

JaCoCo does not name the directories its source files came from - there is no
equivalent of the Cobertura 'sources' element - so use '--source-directory' to
tell xml2lcov where to look.  A source file which is not found in the search
path is not translated:  a coverage report keyed by a name which does not
resolve to a file is of no use to anybody.  Source files of classes compiled
without debug information are not found at all, and are not translated either.

JaCoCo counts instructions and branches - it does not count executions.  As a
result:

  - The DA: count of a line is the number of branches taken on that line, or 1
    if the line is covered and has no branches.  A line which JaCoCo considers
    covered never reports a count of 0, but the count is not the number of
    times the line was executed.

  - The FNA: count of a function is its covered cyclomatic complexity, which is
    0 exactly when the function was not entered.

A JaCoCo 'method' element names the line the method starts on, but not the line
it ends on, so the FNL: extent is derived:  each method claims lines from its
source file's line list, starting at its own first line, until the covered and
missed quotas of its own LINE counter are filled.  Every function which begins
on a particular line then reports the largest extent found for that line,
because LCOV identifies a function by its file and its begin line.

MC/DC is not supported:  JaCoCo reports how many branches on a line were taken,
but not which condition, and not which sense of it, was left uncovered - which
is precisely what an MCDC: record has to say.

Merging JaCoCo data:

If you have coverage data from more than one test run, combine it on the JaCoCo
side and translate the single combined report - either by handing all of the
execution data files to one 'jacococli.jar report' run, or by combining them
first with 'jacococli.jar merge'.  Do not translate each report separately and
then merge the resulting .info files with 'lcov -a'.

The reason is branch data.  JaCoCo knows which branch of a line is which, so its
own merge combines the per-branch results exactly:  a line whose first branch was
taken in one run and whose second branch was taken in another is reported with
both branches taken.  xml2lcov cannot do that, because the XML no longer says
which branch is which - only how many of them were taken - so it assigns the
taken ones by position, as described above.  Merging translated data therefore
merges those lower bounds, and the result understates branch coverage whenever
two runs took different branches on the same line.

Line and function coverage do not have this problem - they are keyed by line
number, which survives the translation - so if the reports you want to combine
are already translated, merging them with 'lcov -a' is still correct for those,
and only the branch data is pessimistic.
"""

    def __init__(self, scriptArgs):
        self._args = scriptArgs

        # every option which names a file or a directory, translated once,
        #   before any of them is used:  'excludePatterns' is deliberately not
        #   here - those are patterns which are matched against the names in
        #   the data, not names of files of ours - and neither is the callback
        #   below, whose first element is a path and whose remaining elements
        #   are the callback's own arguments.
        # 'input' and 'cover_cmd' are py2lcov's;  getattr() in posix_paths()
        #   lets both tools share this list.
        posix_paths(scriptArgs, 'output', 'input', 'inputs', 'cover_cmd',
                    'sourceDirectories')

        self._excludePatterns = scriptArgs.excludePatterns.split(',') if scriptArgs.excludePatterns else None
        self._versionScript = scriptArgs.version.split(',') if scriptArgs.version else None
        if self._versionScript:
            self._versionScript[0] = posix_path(self._versionScript[0])
            scriptArgs.version = ','.join(self._versionScript)
        self._versionModule = None
        if self._versionScript and self._versionScript[0][-3:] == ".pm":
            # hard to handle Perl module in python - so we hack it
            self._versionModule = self._versionScript
            self._versionScript = None

        self._outf = open(scriptArgs.output, "w")
        self._isPython = getattr(scriptArgs, 'isPython', False)
        self._format = getattr(scriptArgs, 'format', None) or 'auto'
        # [directory, number of files found in it] - the count is kept over all
        #  the input files, so an unused '--source-directory' is reported once
        self._searchPaths = [[d, 0] for d in
                             getattr(scriptArgs, 'sourceDirectories', None) or []]
        # things which were dropped rather than translated:
        #   key -> [count, message]
        self._dropped = {
            'unresolved' : [0, "%d source file(s) not found in the search path"],
            'unmappedClass' : [0, "%d class(es) which name no source file"],
            'unmappedMethod' : [0, "%d method(s) whose line number is unknown"],
        }

        self._outf.write("TN:%s\n" % scriptArgs.testName)

    def close(self):

        self._outf.close()

        for s in self._searchPaths:
            if s[1] == 0:
                print("Warning: source directory '%s' is unused" % (s[0]))
        for key in self._dropped:
            (count, message) = self._dropped[key]
            if count != 0:
                print("Warning: " + message % (count))

        if self._args.version and self._versionScript is None:
            # find the 'lcov' that is part of my release - should be adjacent
            # to this script
            lcov = os.path.join(os.path.split(posix_path(sys.argv[0]))[0], 'lcov')
            cmd = [
                lcov,
                "-a", self._args.output,
                "-o", self._args.output,
                "--version-script", self._args.version,
                *(["--checksum"] if self._args.checksum else []),
                "--rc", "compute_file_version=1",
                "--branch-coverage",
                "--ignore", "inconsistent",
            ]
            # '--no-function-coverage' tells that 'lcov' to throw away the
            #   function data in the file it is reading - so it is only right
            #   when the caller has none to lose.  py2lcov derives its function
            #   coverpoints and can be told not to, with '--no-functions';
            #   xml2lcov reads function data out of its input and has no such
            #   option, so a missing attribute means "there are functions
            #   here", not "there are none".  Defaulting it the other way was
            #   silently dropping every FN/FNDA/FNF/FNH record from an
            #   xml2lcov run which named a '.pm' version script.
            if not getattr(self._args, 'deriveFunctions', True):
                cmd.append("--no-function-coverage")
            try:
                x = subprocess.run(cmd, shell=False, check=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
            except subprocess.CalledProcessError as err:
                print("Error during lcov version append operation: %s" % (
                    str(err)))
                if not self._args.keepGoing:
                    sys.exit(1);


    def process_xml_file(self, xml_file):

        try:
            tree = ET.parse(xml_file)
        except (ET.ParseError, IOError, OSError) as err:
            # a file which is not there or is not XML at all:  say so, rather
            #  than dying by traceback out of the parser
            print("Error: unable to read \"%s\": %s" % (xml_file, str(err)))
            if not self._args.keepGoing:
                sys.exit(1)
            return
        root = tree.getroot()

        format = self._format
        if format == 'auto':
            # the root element is what tells the two schemas apart
            if root.tag == 'coverage':
                format = 'cobertura'
            elif root.tag == 'report':
                format = 'jacoco'
            else:
                print("Error: \"%s\": unexpected XML root element '%s': expected 'coverage' (Cobertura) or 'report' (JaCoCo).  Use --format to say which it is." % (
                    xml_file, root.tag))
                if not self._args.keepGoing:
                    sys.exit(1)
                return
        else:
            # the caller said which schema this is, so that is the reader we
            #  use - but say so if the root element is not the expected one,
            #  because the answer is then likely to be no data at all
            expected = 'coverage' if format == 'cobertura' else 'report'
            if root.tag != expected:
                print("Warning: \"%s\": --format %s expects a '%s' root element: found '%s'" % (
                    xml_file, format, expected, root.tag))
        if self._args.verbose:
            print("%s: %s format" % (xml_file, format))

        if format == 'jacoco':
            self._process_jacoco(xml_file, root)
        else:
            self._process_cobertura(xml_file, root)

    def _find_source(self, name, xmlPaths):
        """Look for 'name' in the directories the XML named, then in the
        '--source-directory' list.  Returns the path found, or None."""

        for s in xmlPaths + self._searchPaths:
            if self._args.verbose > 1:
                print("  check src_path (%s %d)" % (s[0], s[1]))
            path = os.path.join(s[0], name)
            if os.path.exists(path):
                s[1] += 1 # this source path used for something
                return path
        return None

    def _write_source_header(self, name):
        """Start the record of one source file:  its name and, if a version
        script was given, its version."""

        self._outf.write("SF:%s\n" % name)
        if self._versionScript:
            cmd = copy.deepcopy(self._versionScript)
            cmd.append(name)
            try:
                version = subprocess.check_output(cmd)
                self._outf.write("VER:%s\n" % version.strip().decode('UTF-8'))
            except Exception as err:
                print("Error: no version for %s: %s" %(
                    name, str(err)))
                if not self._args.keepGoing:
                    sys.exit(-1)

    def _write_totals(self, totals):
        """Print the LCOV totals - not used by lcov, but maybe somebody does"""

        for key in totals:
            d = totals[key]
            if d[0] == 0:
                continue
            self._outf.write("%s:%d\n" % (d[2], d[0]))
            self._outf.write("%s:%d\n" % (d[3], d[1]))

    def _process_cobertura(self, xml_file, root):

        source_paths = []

        try:
            # find the elements by name rather than by position:  the schema does
            #  not fix their order, and indexing position 0 reported a report
            #  whose elements are in the other order as having no 'sources' at all
            sources = root.find('sources')
            if sources is not None:
                for source in sources:
                    # keep track of number of times we use each source_path to find
                    #  some file.  Unused source paths are likely a problem.
                    if self._args.verbose:
                        print("source: '%s'" %(source.text))
                    # unclear why the Coverage.py version on GitHub node
                    # generates empty sources
                    if source.text == None:
                        print("skipping empty source (???)")
                        continue
                    # a report written on windows wo;; java windows-style names
                    # that we have to mung in order to be able to open on this platform
                    source_paths.append([posix_path(source.text), 0])
            elif not self._searchPaths:
                # a missing source path list leaves the search path empty -
                #  which is the same as a report whose only entries are the
                #  empty ones skipped above, and every filename in the data
                #  is still usable as is.
                # If '--source-directory' was used then the search path is not
                #  empty after all, and the caller has said where to look -
                #  so there is nothing to complain about.
                print("Error: parse xml fail: no 'sources' in %s" %(xml_file))
                if not self._args.keepGoing:
                    sys.exit(1)
            packages = root.find('packages')
            if packages is None:
                print("Error: parse xml fail: no 'packages' in %s" %(xml_file))
                if not self._args.keepGoing:
                    sys.exit(1)
                return
            if self._args.verbose:
                print("packages: " + str(packages.attrib))
        except Exception as err:
            print("Error: parse xml fail in %s: %s" % (xml_file, str(err)))
            if not self._args.keepGoing:
                sys.exit(1)
            return

        for package in packages:
            # name="." means current directory
            # name=".folder1.folder2" means external module or directory
            # name="abc" means internal module or directory
            pname = package.attrib['name']
            if self._args.verbose:
                print("package: '%s'" % (pname))
            isExternal = (pname.startswith('.') and pname != '.')
            #pdb.set_trace()
            for classes in package:
                for fileNode in classes:
                    name = fileNode.attrib['filename']
                    if self._args.excludePatterns and any([fnmatch.fnmatchcase(name, ef) for ef in self._excludePatterns]):
                        if self._args.verbose:
                            print("%s is excluded" % name)
                        continue
                    if self._args.verbose > 1:
                        print("  file: %s" % (name))
                    if not isExternal:
                        path = self._find_source(name, source_paths)
                        if path is not None:
                            name = path
                        else:
                            print("did not find %s in search path" % (name))

                    self._write_source_header(name)

                    self.process_file(fileNode, name)
                    self._outf.write("end_of_record\n")

        for s in source_paths:
            if s[1] == 0:
                print("Warning: XML file '%s': source_path '%s' is unused" %(xml_file, s[0]))


    def _formatError(self, filename, message):
        # report a malformed input.
        # All the messages are about the input XML - not about implementation
        #  invariants.  the format is Cobertura - Coverage.py is just one of
        #  the tools which write it, and a different tool might report
        #  differently.
        # An AssertionError doesn't name an input file, element, or line -
        #  so can't be waived via --keep-going.
        print("Error: \"%s\": %s" % (filename, message))
        if not self._args.keepGoing:
            sys.exit(1)

    def process_file(self, fileNode, filename):

        sourceCode = None
        if (self._args.checksum or
            (self._isPython and self._args.deriveFunctions)):
            try:
                with open(filename, 'r') as f:
                    sourceCode = f.read().split('\n')
            except (IOError, OSError) as e:
                feature = 'compute line checksum' if self._args.checksum else ''
                if self._isPython and self._args.deriveFunctions:
                   if feature != '':
                      feature += ' or'
                   feature += ' derive function data'

                print("cannot open %s - unable to %s" % (filename, feature));
                if not self._args.keepGoing:
                    sys.exit(1)

        def count(indent):
            # the column the first non-whitespace character.
            # A tab advances to the next tab stop - it does not add a whole tab
            #  width wherever it happens to be, which measured a line indented
            #  'space tab' as wider than one indented 'tab', and made the deeper
            #  of two equally indented lines the parent of the other
            count = 0
            for c in indent:
                if c == '\t':
                    count += self._args.tabWidth - (count % self._args.tabWidth)
                else:
                    # a space, or one of the rarer characters '\s' also matches
                    count += 1
            return count

        def buildFunction(functions, objStack, currentObj, lastLine):
            if currentObj and lastLine:
                currentObj['end'] = lastLine # last line
                prefix = ''
                sep = ''
                for e in objStack:
                    prefix += sep + e['name']
                    sep = "::" if e['type'] == 'class' else '.'
                if currentObj['type'] == 'def':
                    fullname = prefix + sep + currentObj['name']
                    # function might be unreachable dead code
                    hit = currentObj.get('hit', 0)
                    functions.append({'name'  : fullname,
                                      'start' : currentObj['start'],
                                      'end'   : currentObj['end'],
                                      'hit'   : hit})

        # just collect the function/class name - ignore the params
        parseLine = re.compile(r'(\s*)((def|class)\s+([^\(: \t]+))?')

        # no information about actual branch expressions/branch
        #  coverage - only the percentage and number hit/not hit
        parseCondition = re.compile(r'\d+\% \((\d+)/(\d+)\)')

        def branchCounts(attrib, where):
            # (taken, total) out of the 'condition-coverage' attribute of a
            #   '<line branch="true">' element, or None if it cannot be read
            if 'condition-coverage' not in attrib:
                self._formatError(filename,
                                  "%s: branch has no 'condition-coverage' attribute" %(where))
                return None
            m = parseCondition.search(attrib['condition-coverage'])
            if not m:
                self._formatError(filename,
                                  "%s: unable to parse condition-coverage '%s'" %(
                                      where, attrib['condition-coverage']))
                return None
            return (int(m.group(1)), int(m.group(2)))

        functions = [] # list of [functionName startLine endLine hitcount]
        for node in fileNode:

            if node.tag == 'methods':
                # build function decls...
                for method in node:
                    func = method.attrib['name']

                    # does this method contain any lines?
                    for lines in method:
                        if lines.tag != 'lines':
                            self._formatError(filename,
                                              "method '%s': expected a 'lines' element, found '%s'" %(
                                                  func, lines.tag))
                            continue
                        first = None
                        last = None
                        hit = 0
                        # might want to hang onto the method lines - and
                        #   check that the 'lines' tag we find in the parent
                        #   node contains all of the method lines we found
                        functionLines = {}
                        branches = {}
                        for l in lines:
                            lineNum = int(l.attrib['number'])
                            lineHit = int(l.attrib['hits'])
                            functionLines[lineNum] = lineHit
                            if first == None:
                                first = lineNum
                                last = lineNum
                                hit = lineHit
                            elif lineNum > last:
                                last = lineNum
                            else:
                                # the elements are expected in increasing line
                                #  order:  keep the larger number, so that the
                                #  function's end line is still its last line
                                self._formatError(filename,
                                                  "method '%s': line %d is not after line %d" %(
                                                      func, lineNum, last))
                            if 'branch' in l.attrib and 'true' == l.attrib['branch']:
                                counts = branchCounts(l.attrib,
                                                      "method '%s' line %d" %(func, lineNum))
                                if counts != None:
                                    # [taken total]
                                    branches[lineNum] = [counts[0], counts[1]]

                        if first != None:
                            functions.append({'name'  : func,
                                              'start' : first,
                                              'end'   : last,
                                              'hit'   : hit,
                                              'lines' : functionLines,
                                              'branches' : branches})
                        elif self._args.verbose:
                            # there seem to be a fair few functions
                            #  which contain no data
                            print("elided empty function %s" %(func))

                continue

            if node.tag != 'lines':
                print("not handling tag %s" %(node.tag))
                continue

            # Keep track of current function/class scope - which we use to find
            #   the first and last executable lines in each function,
            # Want to keep track of the function end line - so we can use lcov
            # function exclusions.
            #   currentObj:
            #    type:   'class' or 'def'
            #    name:   as appears in regexp
            #    indent: indent count of 'def' or 'class' statement
            #    start:  line of item (where 'def' or 'class' is found
            #    end:    last line of function
            #    hit:    whether first line of function is hit or not
            currentObj = None # {type name startIndent lineNo first end start}
            objStack = []
            prevLine = None
            totals = { 'line' : [0, 0, 'LF', 'LH'],
                       'branch' : [0, 0, 'BRF', 'BRH'],
                       'function' : [0, 0, 'FNF', 'FNH'],
            }
            # need to save the statement data and print later because Coverage.py
            # has an odd interpretation of the execution status of the function
            # decl line.
            #   - C/C++ mark it executed if the line is entered - so it
            #     is an analog of function coverage.
            #   - Coverage.py appears to mark it executed when the containing
            #     scope is executed (i.e., when a lazy interpret might compile
            #     the function).
            # However, we want to mark the decl executed only if the function
            # is executed - and we decide that the function is executed if first
            # line in the function is hit.
            #   - as a result, after seeing all the functions, we want to go back
            #     and mark the function decl line as 'not hit' if we decided that
            #     the function itself is not executed.
            lineData = {}
            for line in node:
                lineNo = int(line.attrib['number'])
                hit = int(line.attrib["hits"])
                lineData[lineNo] = hit;

                totals['line'][0] += 1
                if hit:
                    totals['line'][1] += 1
                if sourceCode and self._isPython:
                    # try to derive function names and begin/end lines in Python code
                    if lineNo <= len(sourceCode):
                        m = parseLine.search(sourceCode[lineNo-1])
                        if m:
                            indent = count(m.group(1))
                            #print(sourceCode[lineNo-1])
                            while currentObj and indent <= currentObj['indent']:
                                # lower indent - so this is a new object
                                #print("build " + currentObj['name'])
                                buildFunction(functions, objStack,
                                              currentObj, prevLine)

                                try:
                                    currentObj = objStack.pop()
                                except IndexError as err:
                                    currentObj = None
                                    break

                            if m.group(2):
                                if currentObj:
                                    objStack.append(currentObj)
                                objtype = m.group(3)
                                name = m.group(4).rstrip()
                                if (-1 != name.find('(') and
                                    ')' != name[-1]):
                                    name += '...)'
                                currentObj = { 'type':   objtype,
                                               'name':   name,
                                               'indent': indent,
                                               'start':  lineNo,
                                }
                            else:
                                # just a line - may be the first executable
                                #   line in some function:
                                if currentObj and not 'hit' in currentObj:
                                    currentObj['hit'] = hit
                                    # mark that function decl line is not
                                    #  hit if the function is not hit
                                    if 0 == hit:
                                        if currentObj['start'] in lineData:
                                            lineData[currentObj['start']] = 0

                        prevLine = lineNo
                    else:
                        print('"%s":%d: Error: out of range: file contains %d lines' % (
                            filename, lineNo, len(sourceCode)))
                        if not self._args.keepGoing:
                            sys.exit(1)

                if "branch" in line.attrib and line.attrib["branch"] == 'true':
                    # attrib is always true from xmlreport.py - but may not
                    #   be true cobertura report
                    counts = branchCounts(line.attrib, "line %d" %(lineNo))
                    if counts == None:
                        continue
                    (taken, total) = counts
                    # no information of which clause is taken or not
                    # set taken conditions start from 0 and followed by
                    #  non-taken conditions
                    # taken conditions
                    for cond in range(0,taken):
                        self._outf.write("BRDA:%d,0,%d,1\n" % (lineNo, cond))
                        totals['branch'][0] += 1
                        totals['branch'][1] += 1
                    # non-taken conditions
                    for cond in range(taken, total):
                        totals['branch'][0] += 1
                        self._outf.write("BRDA:%d,0,%d,0\n" % (lineNo, cond))

            # and build all the pending functions
            #  these were still open when we hit the end of file - e.g., because
            #  they are last elements in some package file and there are no
            #  no executable lines after the function decl.
            # There may be more than one function in the stack, if the last
            # object is nested.
            while currentObj:
                buildFunction(functions, objStack, currentObj, prevLine)

                try:
                    currentObj = objStack.pop()
                except IndexError as err:
                    currentObj = None
                    break

            # print the LCOV function data
            idx = 0
            for f in functions:
                totals['function'][0] += 1
                f['idx'] = idx
                idx += 1
                if f['hit']:
                    totals['function'][1] += 1
                self._outf.write("FNL:%(idx)d,%(start)d,%(end)d\nFNA:%(idx)d,%(hit)d,%(name)s\n" % f)
            # print the LCOV line data.
            for lineNo in sorted(lineData.keys()):
                checksum = ''
                # 'sourceCode' is None if the file could not be read
                if self._args.checksum and sourceCode != None:
                    try:
                        checksum = ',' + line_hash(sourceCode[lineNo-1])
                    except IndexError:
                        print('"%s":%d: unable to compute checksum for missing line' % (filename, lineNo))
                        if not self._args.keepGoing:
                            sys.exit(1)

                self._outf.write("DA:%d,%d%s\n" % (lineNo, lineData[lineNo], checksum));

            self._write_totals(totals)

    # ------------------------------------------------------------------------
    # JaCoCo XML
    #
    # The schema:
    #
    #   <report name="...">
    #     <sessioninfo .../>
    #     <group>*                      - optional, and can be nested
    #       <package name="a/b/c">
    #         <class name="a/b/c/Cls" sourcefilename="Cls.java">
    #           <method name="m" desc="()V" line="17">
    #             <counter type="LINE" missed="1" covered="2"/> ...
    #         <sourcefile name="Cls.java">
    #           <line nr="17" mi="0" ci="3" mb="1" cb="1"/> ...
    #
    # The differences compared to Cobertura:
    #
    #   - the line data is in the 'sourcefile' elements and the function data in
    #     the 'class' elements;  the 'sourcefilename' attribute ties the two
    #     together, and a class compiled without debug information has none
    #
    #   - the name of a source file is the package name - which is a directory
    #     path - plus the 'sourcefile' name.  Nothing anchors that anywhere in a
    #     file system, which is what '--source-directory' is for
    #
    #   - a line is described by how many of its instructions and branches were
    #     missed and covered, not by how many times it was executed, and a
    #     method reports its covered complexity rather than a call count
    #
    #   - a method says which line it starts on, and nothing about where it ends
    # ------------------------------------------------------------------------

    def _process_jacoco(self, xml_file, root):

        if not self._searchPaths:
            print("Warning: \"%s\": no --source-directory: JaCoCo XML does not say where the source files are, so their names are left relative to the package directory" % (
                xml_file))

        # 'iter' rather than 'findall':  a package may be a child of the report,
        #  or may be nested in one or more levels of 'group' element
        for package in root.iter('package'):
            pname = package.attrib.get('name', '')
            if self._args.verbose:
                print("package: '%s'" % (pname))

            # sourcefile name -> { lines, methods }.  Insertion order is
            #  document order, so the records are written in the order the
            #  report lists them.
            files = {}
            for sourcefile in package.findall('sourcefile'):
                name = sourcefile.attrib.get('name')
                if name is None:
                    self._formatError(xml_file,
                                      "package '%s': 'sourcefile' with no 'name' attribute" %(pname))
                    continue
                files[name] = {
                    'lines' : self._jacoco_lines(xml_file, sourcefile, pname, name),
                    'methods' : [],
                }

            for classNode in package.findall('class'):
                cname = classNode.attrib.get('name', '').replace('/', '.')
                sourcefilename = classNode.attrib.get('sourcefilename')
                if sourcefilename is None:
                    # a class compiled without debug information does not say
                    #  which file it came from, so there is nothing to attach
                    #  its methods to
                    self._dropped['unmappedClass'][0] += 1
                    if self._args.verbose:
                        print("  class '%s' names no source file" % (cname))
                    continue
                if sourcefilename not in files:
                    self._formatError(xml_file,
                                      "class '%s': package '%s' has no 'sourcefile' element for '%s'" %(
                                          cname, pname, sourcefilename))
                    continue
                for method in classNode.findall('method'):
                    fullname = cname + '.' + method.attrib.get('name', '') + \
                        method.attrib.get('desc', '')
                    if 'line' not in method.attrib or method.attrib['line'] == '0':
                        # no debug information for this method:  an LCOV
                        #  function record is keyed by its begin line, so
                        #  there is no way to report one
                        self._dropped['unmappedMethod'][0] += 1
                        if self._args.verbose:
                            print("  method '%s' has no line number" % (fullname))
                        continue
                    try:
                        first = int(method.attrib['line'])
                    except ValueError:
                        self._formatError(xml_file,
                                          "method '%s': unable to read line number '%s'" %(
                                              fullname, method.attrib['line']))
                        continue
                    # the LINE counter is the size of this method's share of the
                    #  file's line list, and the covered COMPLEXITY is the
                    #  nearest thing JaCoCo has to an execution count:  it is
                    #  zero exactly when the method was not entered
                    counters = {}
                    for kind in ('LINE', 'COMPLEXITY'):
                        counters[kind] = jacoco_counter(method, kind)
                        if counters[kind] is None:
                            self._formatError(xml_file,
                                              "method '%s': unable to read its %s counter" %(
                                                  fullname, kind))
                            counters[kind] = (0, 0)
                    files[sourcefilename]['methods'].append({
                        'name'     : fullname,
                        'start'    : first,
                        'nmissed'  : counters['LINE'][0],
                        'ncovered' : counters['LINE'][1],
                        'hit'      : counters['COMPLEXITY'][1],
                    })

            for (name, data) in files.items():
                relPath = os.path.join(pname, name) if pname else name
                if self._args.excludePatterns and any([fnmatch.fnmatchcase(relPath, ef) for ef in self._excludePatterns]):
                    if self._args.verbose:
                        print("%s is excluded" % relPath)
                    continue
                if self._searchPaths:
                    path = self._find_source(relPath, [])
                    if path is None:
                        # coverage data keyed by a name which does not resolve
                        #  to a file is of no use to anybody
                        self._dropped['unresolved'][0] += 1
                        print("did not find %s in search path" % (relPath))
                        continue
                else:
                    path = relPath
                if self._args.verbose > 1:
                    print("  file: %s" % (path))

                self._write_source_header(path)
                self._write_jacoco_record(path, data)
                self._outf.write("end_of_record\n")

    def _jacoco_lines(self, xml_file, sourcefile, pname, name):
        """The line data of one JaCoCo 'sourcefile' element, in line order."""

        where = "%s/%s" % (pname, name) if pname else name
        lines = []
        seen = set()
        for line in sourcefile.findall('line'):
            try:
                lineNo = int(line.attrib['nr'])
            except (KeyError, ValueError):
                self._formatError(xml_file,
                                  "%s: 'line' with no readable 'nr' attribute" %(where))
                continue
            if lineNo < 1:
                self._formatError(xml_file,
                                  "%s: 'line' with out of range 'nr' attribute %d" %(
                                      where, lineNo))
                continue
            if lineNo in seen:
                self._formatError(xml_file,
                                  "%s: line %d appears more than once" %(where, lineNo))
                continue
            # mi/ci: missed/covered instructions, mb/cb: missed/covered branches
            counts = {}
            for attrib in ('mi', 'ci', 'mb', 'cb'):
                try:
                    counts[attrib] = int(line.attrib.get(attrib, 0))
                except ValueError:
                    self._formatError(xml_file,
                                      "%s: line %d: unable to read '%s' attribute '%s'" %(
                                          where, lineNo, attrib, line.attrib[attrib]))
                    counts[attrib] = 0
            seen.add(lineNo)
            lines.append({
                'nr'      : lineNo,
                'status'  : (jacoco_status(counts['mi'], counts['ci']) |
                             jacoco_status(counts['mb'], counts['cb'])),
                'covered' : counts['cb'],
                'missed'  : counts['mb'],
            })
        # the schema does not say that the elements are in line order
        lines.sort(key=lambda l: l['nr'])
        return lines

    def _jacoco_functions(self, data):
        """The LCOV function records of one JaCoCo source file.

        A JaCoCo 'method' says which line it starts on and nothing about where
        it ends, so the extent is derived:  each method claims lines from the
        source file's line list, starting at its own first line, until the
        covered and missed quotas of its own LINE counter are filled.  A method
        need not own a contiguous range of lines - a static initializer can be
        interleaved with the field declarations which feed it - and the quotas
        are what stops the claim in that case.
        """

        lines = data['lines']
        functions = []
        for method in data['methods']:
            owned = []
            needCovered = method['ncovered']
            needMissed = method['nmissed']
            for line in lines:
                if line['nr'] < method['start']:
                    continue
                # a partly covered line has both bits set, and can fill either
                #  quota - the covered one first, as JaCoCo counts it as a
                #  covered line
                if needCovered and line['status'] & JACOCO_FULLY_COVERED:
                    owned.append(line['nr'])
                    needCovered -= 1
                elif needMissed and line['status'] & JACOCO_NOT_COVERED:
                    owned.append(line['nr'])
                    needMissed -= 1
                if not needCovered and not needMissed:
                    break
            functions.append({
                'name'  : method['name'],
                'start' : method['start'],
                'end'   : max(owned) if owned else method['start'],
                'hit'   : method['hit'],
            })
        # LCOV identifies a function by its file and its begin line, so the
        #  functions which begin on the same line have to agree about where
        #  they end:  a lambda begins on the line of the expression which
        #  creates it, which is a line of the method which contains it
        ends = {}
        for f in functions:
            ends[f['start']] = max(f['end'], ends.get(f['start'], 0))
        for f in functions:
            f['end'] = ends[f['start']]
        functions.sort(key=lambda f: (f['start'], f['name']))
        return functions

    def _write_jacoco_record(self, filename, data):
        """Write the body of the LCOV record of one JaCoCo source file."""

        sourceCode = None
        if self._args.checksum:
            try:
                with open(filename, 'r') as f:
                    sourceCode = f.read().split('\n')
            except (IOError, OSError) as e:
                print("cannot open %s - unable to compute line checksum" % (filename))
                if not self._args.keepGoing:
                    sys.exit(1)

        totals = { 'line' : [0, 0, 'LF', 'LH'],
                   'branch' : [0, 0, 'BRF', 'BRH'],
                   'function' : [0, 0, 'FNF', 'FNH'],
        }

        # branch data:  JaCoCo says how many of the branches on a line were
        #  taken, but not which ones - so the first N are the ones we call
        #  taken.  See the note in 'usageNote' above.
        for line in data['lines']:
            total = line['covered'] + line['missed']
            for branchId in range(0, total):
                taken = 1 if branchId < line['covered'] else 0
                self._outf.write("BRDA:%d,0,%d,%d\n" % (line['nr'], branchId, taken))
                totals['branch'][0] += 1
                totals['branch'][1] += taken

        # function data
        idx = 0
        for f in self._jacoco_functions(data):
            f['idx'] = idx
            idx += 1
            totals['function'][0] += 1
            if f['hit']:
                totals['function'][1] += 1
            self._outf.write("FNL:%(idx)d,%(start)d,%(end)d\nFNA:%(idx)d,%(hit)d,%(name)s\n" % f)

        # line data:  there is no execution count, so a covered line reports the
        #  number of branches taken on it, or 1 if it has none.  A line JaCoCo
        #  considers covered must not report a count of 0.
        for line in data['lines']:
            if not line['status'] & JACOCO_FULLY_COVERED:
                count = 0
            elif line['covered'] > 0:
                count = line['covered']
            else:
                count = 1
            checksum = ''
            # 'sourceCode' is None if the file could not be read
            if self._args.checksum and sourceCode != None:
                try:
                    checksum = ',' + line_hash(sourceCode[line['nr']-1])
                except IndexError:
                    print('"%s":%d: unable to compute checksum for missing line' % (
                        filename, line['nr']))
                    if not self._args.keepGoing:
                        sys.exit(1)
            totals['line'][0] += 1
            if count:
                totals['line'][1] += 1
            self._outf.write("DA:%d,%d%s\n" % (line['nr'], count, checksum))

        self._write_totals(totals)
