path: root/release-notes/tickets.py

#
# RTEMS Tools Project (http://www.rtems.org/)
# Copyright 2018 Danxue Huang (danxue.huang@gmail.com)
# Copyright 2022 Chris Johns (chris@contemporary.software)
# All rights reserved.
#
# This file is part of the RTEMS Tools package in 'rtems-tools'.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are met:
#
# 1. Redistributions of source code must retain the above copyright notice,
# this list of conditions and the following disclaimer.
#
# 2. Redistributions in binary form must reproduce the above copyright notice,
# this list of conditions and the following disclaimer in the documentation
# and/or other materials provided with the distribution.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
# POSSIBILITY OF SUCH DAMAGE.
#

import html.entities
import html.parser
import os
import sys
import time
import threading

import xml.etree.ElementTree as ElementTree

import reraise
import rtems_trac
import trac


class rss_parser(html.parser.HTMLParser):

    def __init__(self, break_p=False):
        super(rss_parser, self).__init__()
        self.trace = False
        self.tags = []
        self.text = ''
        self.div_end = 0
        self.break_p = break_p

    def __del__(self):
        if self.trace:
            print('> del: ' + str(self))

    def __str__(self):
        o = ['text: ' + self.text]
        return os.linesep.join(o)

    def _clean_data(self, data):
        leading_ws = ' ' if len(data) > 0 and data[0].isspace() else ''
        trailing_ws = ' ' if len(data) > 0 and data[-1].isspace() else ''
        data = leading_ws + data.strip() + trailing_ws
        if self.break_p:
            data = data.replace(os.linesep, '<br />')
        return data

    def _tag_attr_get(self, attrs, key):
        if attrs:
            for attr, label in attrs:
                if attr == key:
                    return label
        return None

    def _tags_parse_all(self, start, tag, attrs, extended):
        if attrs and self.trace:
            for attr in attrs:
                print("     attr:", attr)
        o = ''
        if tag == 'em':
            o = '__'
        elif tag == 'strong':
            o = '__'
        elif tag == 'br':
            if self.div_end == 0 and not start:
                o = '<br />'
        elif tag == 'p':
            if self.div_end == 0:
                if start:
                    o = '<p>'
                else:
                    o = '</p>'
            else:
                o = os.linesep
        elif tag == 'div':
            if start:
                div_class = self._tag_attr_get(attrs, 'class')
                if div_class and self.div_end == 0:
                    o = os.linesep * 2 + '<div class="' + div_class + '">' + os.linesep
                    self.div_end += 1
                elif self.div_end > 0:
                    self.div_end += 1
            else:
                if self.div_end == 1:
                    o = os.linesep + '</div>' + os.linesep
                if self.div_end > 0:
                    self.div_end -= 1
            if self.trace:
                print(' tag: start = ', start, 'dev_end =', self.div_end)
        elif tag == 'ul' and extended:
            if start:
                o = '<ul>'
            else:
                o = '</ul>'
        elif tag == 'li' and extended:
            if start:
                o = '<li>'
            else:
                o = '</li>'
        elif tag == 'pre':
            if start:
                o = '<pre class="blockquote-code">'
            else:
                o = '</pre>'
        elif tag == 'blockquote':
            bq_class = self._tag_attr_get(attrs, 'class')
            if start:
                if bq_class:
                    o = '<blockquote class="' + bq_class + '">'
                else:
                    o = '<blockquote>'
            else:
                o = '</blockquote>'
        return o

    def _tags_parse_start(self, tag, attrs, extended=True):
        return self._tags_parse_all(True, tag, attrs, extended)

    def _tags_parse_end(self, tag, extended=True):
        return self._tags_parse_all(False, tag, None, extended)

    def _tags_push(self, tag):
        self.tags.append(tag)

    def _tags_pop(self, tag):
        if len(self.tags) != 0:
            self.tags.pop()

    def _tags_path(self):
        return '/'.join(self.tags)

    def _tags_in_path(self, path):
        return self._tags_path().startswith(path)

    def handle_starttag(self, tag, attrs):
        if self.trace:
            print("> start-tag (p):", tag)
        self._tags_push(tag)
        self.text += self._tags_parse_start(tag, attrs, True)

    def handle_endtag(self, tag):
        if self.trace:
            print("> end-tag (p):", tag)
        self._tags_pop(tag)
        self.text += self._tags_parse_end(tag)

    def handle_data(self, data):
        if self.trace:
            print("> data (p) :", data)
        data = self._clean_data(data)
        self.text += data


class rss_meta_parser(rss_parser):

    def __init__(self):
        super(rss_meta_parser, self).__init__()
        self.meta_data = []
        self.meta_steps = ['ul', 'li', 'strong']
        self.meta_label = None
        self.meta_text = ''

    def __str__(self):
        o = [
            'meta_data: %r' % (self.meta_data),
            'meta_label: ' + str(self.meta_label),
            'meta_text: ' + str(self.meta_text), 'text: ' + self.text
        ]
        return os.linesep.join(o)

    def _tags_metadata(self):
        return self.meta_label and self._tags_path().startswith('ul/li')

    def _tags_meta_label(self):
        return self._tags_path() == 'ul/li/strong'

    def handle_starttag(self, tag, attrs):
        if self.trace:
            print("> start-tag (m):", tag)
        in_metadata = self._tags_metadata()
        self._tags_push(tag)
        if self._tags_metadata():
            if in_metadata:
                self.meta_text += self._tags_parse_start(tag,
                                                         attrs,
                                                         extended=False)
        elif not self._tags_meta_label():
            self.text += self._tags_parse_start(tag, attrs, extended=False)

    def handle_endtag(self, tag):
        if self.trace:
            print("> end-tag (m):", tag)
        in_metadata = self._tags_metadata()
        self._tags_pop(tag)
        if in_metadata:
            # Trailing edge detect of the metadata end
            # Ignore the meta_label eng tag
            if not self._tags_metadata():
                self.meta_data.append(
                    (self.meta_label, self.meta_text.strip()))
                self.meta_label = None
                self.meta_text = ''
            elif len(self.meta_text) > 0:
                self.meta_text += self._tags_parse_end(tag, extended=False)
        else:
            self.text += self._tags_parse_end(tag, extended=False)

    def handle_data(self, data):
        if self.trace:
            print("> data (m) :", data)
        if not self.meta_label and self._tags_meta_label():
            self.meta_label = data.strip()
        elif self._tags_metadata():
            self.meta_text += self._clean_data(data)
        else:
            super(rss_meta_parser, self).handle_data(data)


class _ticket_fetcher(object):

    ns = {'dc': '{http://purl.org/dc/elements/1.1/}'}

    def __init__(self, ticket, cache):
        self.ticket = ticket
        self.cache = cache
        self.data = None
        self.thread = None
        self.result = None

    def _parse_ticket_csv(self):
        url = rtems_trac.gen_ticket_csv_url(self.ticket_id())
        csv_rows_iter = rtems_trac.parse_csv_as_dict_iter(url, self.cache)
        return dict(next(csv_rows_iter, {}))

    @staticmethod
    def dump_element(el, indent=0):
        if isinstance(el, ElementTree.Element):
            print('%stag:' % (' ' * indent), el.tag)
            print('%stext:' % (' ' * indent), len(el.text), el.text)
            print('%stail:' % (' ' * indent), len(el.tail), el.tail.strip())
            for item in el.items():
                _ticket_fetcher.dump_element(item, indent + 1)
        else:
            print('%sitem:' % (' ' * indent), el)

    def _item_text(self, item, break_p=False):
        if item is None:
            return None
        rp = rss_parser(break_p=break_p)
        if item.text:
            rp.feed(item.text)
        if item.tail:
            rp.feed(item.tail)
        return rp.text.strip()

    def _item_meta(self, item):
        title = item.find('title')
        creator = item.find(self.ns['dc'] + 'creator')
        author = item.find('author')
        if author is not None:
            creator = author
        pub_date = item.find('pubDate')
        guid = item.find('guid')
        description = item.find('description')
        category = item.find('category')
        if title.text is None:
            actions = 'comment'
        else:
            actions = title.text
        i = {
            'tag': self._item_tag(title.text),
            'actions': actions,
            'creator': self._item_text(creator),
            'published': self._item_text(pub_date),
            'guid': self._item_text(guid),
            'category': self._item_text(category)
        }
        rp = rss_meta_parser()
        rp.feed(description.text)
        rp.feed(description.tail)
        i['meta'] = rp.meta_data
        i['description'] = rp.text.strip()
        return i

    def _item_tag(self, tag):
        if tag is not None:
            ns = {'dc': '{http://purl.org/dc/elements/1.1/}'}
            if tag == ns['dc'] + 'creator':
                tag = 'creator'
            elif tag == 'pubData':
                tag = 'published'
            elif tag.startswith('attachment'):
                tag = 'attachment'
            elif tag.startswith('description'):
                tag = 'description'
            elif tag.startswith('milestone'):
                tag = 'milestone'
        else:
            tag = 'comment'
        return tag

    def _attachment_post(self, attachment):
        for m in range(0, len(attachment['meta'])):
            meta = attachment['meta'][m]
            if meta[0] == 'attachment' and \
               meta[1].startswith('set to __') and meta[1].endswith('__'):
                set_to_len = len('set to __')
                alink = meta[1][set_to_len:-2]
                meta = (meta[0],
                        meta[1][:set_to_len - 2] + \
                        '[' + alink + '](' + attachment['guid'] + '/' + alink + ')')
            attachment['meta'][m] = meta
        return attachment

    def _parse_ticket_rss(self):
        # Read xml data as ElementTree, and parse all tags
        ticket_rss = {}
        rss_response = rtems_trac.open_ticket(self.ticket_id(),
                                              self.cache,
                                              part='rss')
        rss_root = ElementTree.parse(rss_response).getroot()
        #
        # The channel has:
        #  title
        #  link
        #  description
        #  language
        #  image
        #  generator
        #  item
        #
        # The channel/item has:
        #  dc:creator
        #  author
        #  pubDate
        #  title
        #  link
        #  guid
        #  description
        #  category
        #
        channel = rss_root.find('channel')
        title = channel.find('title')
        link = channel.find('link')
        description = channel.find('description')
        items = channel.findall('item')
        citems = [self._item_meta(item) for item in items]
        ticket_rss['title'] = self._item_text(title)
        ticket_rss['link'] = self._item_text(link)
        ticket_rss['description'] = self._item_text(description, True)
        ticket_rss['attachments'] = \
            [self._attachment_post(ci) for ci in citems if 'comment' not in ci['guid']]
        ticket_rss['comments'] = \
            sorted([ci for ci in citems if 'comment' in ci['guid']],
                   key=lambda i: int(i['guid'][i['guid'].rfind(':') + 1:]))
        return ticket_rss

    def _runner(self):
        try:
            self.data = {
                'ticket': self.ticket,
                'meta': self._parse_ticket_csv(),
                'comment_attachment': self._parse_ticket_rss()
            }
        except KeyboardInterrupt:
            pass
        except:
            self.result = sys.exc_info()

    def ticket_id(self):
        return self.ticket['id']

    def run(self):
        self.thread = threading.Thread(target=self._runner,
                                       name='ticket-%s' % (self.ticket_id()))
        self.thread.start()

    def is_alive(self):
        return self.thread and self.thread.is_alive()

    def reraise(self):
        if self.result is not None:
            print()
            print('ticket:', self.ticket_id())
            reraise.reraise(*self.result)


class tickets:
    """This class load all tickets data for a milestone."""

    def __init__(self, release, milestone, cache=None):
        self.release = release
        self.milestone = milestone
        self.lock = threading.Lock()
        self.tickets = { 'release': release, 'milestone': milestone }

    def get_ticket_ids(self):
        return self.tickets.keys()

    def _fetch_data_for_ticket(self, ticket):
        return self._parse_ticket_data(ticket)

    def _job_waiter(self, jobs, num_jobs):
        while len(jobs) >= num_jobs:
            time.sleep(0.002)
            for job in jobs:
                if not job.is_alive():
                    job.reraise()
                    self.tickets['tickets'][job.data['meta']['id']] = job.data
                    self._update_stats(job.data)
                    jobs.remove(job)

    def load(self, cache, use_cache=False):
        if use_cache:
            tickets = cache.load()
            if tickets:
                self.tickets = tickets
                return
        # Read entire trac table as DictReader (iterator)
        self._pre_process_tickets_stats()
        tickets_reader = self._get_tickets_table_as_dict(cache)
        tickets = [t for t in tickets_reader]
        num_jobs = 20
        jobs = []
        job_count = 0
        job_total = len(tickets)
        job_len = len(str(job_total))
        for ticket in tickets:
            self._job_waiter(jobs, num_jobs)
            job = _ticket_fetcher(ticket, cache)
            jobs.append(job)
            job.run()
            job_count += 1
            print('\r %*d of %d - ticket %s ' %
                  (job_len, job_count, job_total, ticket['id']),
                  end='')
        self._job_waiter(jobs, 1)
        print()
        self._post_process_ticket_stats()
        cache.unload(self.tickets)

    def _update_stats(self, ticket):
        self.tickets['overall_progress']['total'] += 1
        if ticket['meta']['status'] == 'closed':
            self.tickets['overall_progress']['closed'] += 1
        if ticket['meta']['status'] == 'assigned':
            self.tickets['overall_progress']['assigned'] += 1
        if ticket['meta']['status'] == 'new':
            self.tickets['overall_progress']['new'] += 1
        for col in rtems_trac.aggregate_cols:
            col_value = ticket['meta'][col]
            self.tickets['by_category'][col][col_value] \
                = self.tickets['by_category'][col].get(col_value, {})
            if ticket['meta']['status'] == 'closed':
                self.tickets['by_category'][col][col_value]['closed'] \
                    = self.tickets['by_category'][col][col_value] \
                          .get('closed', 0) + 1
            self.tickets['by_category'][col][col_value]['total'] \
                = self.tickets['by_category'][col][col_value].get('total', 0) + 1

    def _pre_process_tickets_stats(self):
        self.tickets['overall_progress'] = {}
        self.tickets['by_category'] = {
            col: {}
            for col in rtems_trac.aggregate_cols
        }
        self.tickets['overall_progress']['total'] = 0
        self.tickets['overall_progress']['closed'] = 0
        self.tickets['overall_progress']['in_progress'] = 0
        self.tickets['overall_progress']['new'] = 0
        self.tickets['overall_progress']['assigned'] = 0
        self.tickets['tickets'] = {}

    def _post_process_ticket_stats(self):
        # (number of closed tickets) / (number of total tickets)
        n_closed = self.tickets['overall_progress'].get('closed', 0)
        n_total = self.tickets['overall_progress'].get('total', 0)
        self.tickets['overall_progress']['percentage'] \
            = "{0:.0%}".format((n_closed / n_total) if n_total > 0 else 0.0)
        # Get progress (closed/total) for each category
        for col in self.tickets['by_category']:
            for key in self.tickets['by_category'][col]:
                closed = self.tickets['by_category'][col][key].get('closed', 0)
                if closed == 0:
                    self.tickets['by_category'][col][key]['closed'] = 0
                total = self.tickets['by_category'][col][key].get('closed', 0)
                if total == 0:
                    self.tickets['by_category'][col][key]['total'] = 0
                self.tickets['by_category'][col][key]['progress'] \
                    = '{c}/{t}'.format(c=closed, t=total)

    def _get_tickets_table_as_dict(self, cache):
        csv_url = rtems_trac.gen_trac_query_csv_url(rtems_trac.all_cols,
                                                    milestone=self.milestone)
        return rtems_trac.parse_csv_as_dict_iter(csv_url, cache=cache)