#!/usr/bin/python
 
# Copyright (C) 2007 Google Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
 
 
"""Validates a GTFS file.
 
For usage information run feedvalidator.py --help
"""
 
import bisect
import codecs
import datetime
from transitfeed.util import defaultdict
import optparse
import os
import os.path
import re
import socket
import sys
import time
import transitfeed
from transitfeed import TYPE_ERROR, TYPE_WARNING
from urllib2 import Request, urlopen, HTTPError, URLError
from transitfeed import util
import webbrowser
 
SVN_TAG_URL = 'http://googletransitdatafeed.googlecode.com/svn/tags/'
 
 
def MaybePluralizeWord(count, word):
  if count == 1:
    return word
  else:
    return word + 's'
 
 
def PrettyNumberWord(count, word):
  return '%d %s' % (count, MaybePluralizeWord(count, word))
 
 
def UnCamelCase(camel):
  return re.sub(r'([a-z])([A-Z])', r'\1 \2', camel)
 
 
def ProblemCountText(error_count, warning_count):
  results = []
  if error_count:
    results.append(PrettyNumberWord(error_count, 'error'))
  if warning_count:
    results.append(PrettyNumberWord(warning_count, 'warning'))
 
  return ' and '.join(results)
 
 
def CalendarSummary(schedule):
  today = datetime.date.today()
  summary_end_date = today + datetime.timedelta(days=60)
  start_date, end_date = schedule.GetDateRange()
 
  if not start_date or not end_date:
    return {}
  
  try:
    start_date_object = transitfeed.DateStringToDateObject(start_date)
    end_date_object = transitfeed.DateStringToDateObject(end_date)
  except ValueError:
    return {}
 
  # Get the list of trips only during the period the feed is active.
  # As such we have to check if it starts in the future and/or if
  # if it ends in less than 60 days.
  date_trips_departures = schedule.GenerateDateTripsDeparturesList(
                              max(today, start_date_object),
                              min(summary_end_date, end_date_object))
 
  if not date_trips_departures:
    return {}
 
  # Check that the dates which will be shown in summary agree with these
  # calculations. Failure implies a bug which should be fixed. It isn't good
  # for users to discover assertion failures but means it will likely be fixed.
  assert start_date <= date_trips_departures[0][0].strftime("%Y%m%d")
  assert end_date >= date_trips_departures[-1][0].strftime("%Y%m%d")
 
  # Generate a map from int number of trips in a day to a list of date objects
  # with that many trips. The list of dates is sorted.
  trips_dates = defaultdict(lambda: [])
  trips = 0
  for date, day_trips, day_departures in date_trips_departures:
    trips += day_trips
    trips_dates[day_trips].append(date)
  mean_trips = trips / len(date_trips_departures)
  max_trips = max(trips_dates.keys())
  min_trips = min(trips_dates.keys())
 
  calendar_summary = {}
  calendar_summary['mean_trips'] = mean_trips
  calendar_summary['max_trips'] = max_trips
  calendar_summary['max_trips_dates'] = FormatDateList(trips_dates[max_trips])
  calendar_summary['min_trips'] = min_trips
  calendar_summary['min_trips_dates'] = FormatDateList(trips_dates[min_trips])
  calendar_summary['date_trips_departures'] = date_trips_departures
  calendar_summary['date_summary_range'] = "%s to %s" % (
      date_trips_departures[0][0].strftime("%a %b %d"),
      date_trips_departures[-1][0].strftime("%a %b %d"))
 
  return calendar_summary
 
 
def FormatDateList(dates):
  if not dates:
    return "0 service dates"
 
  formatted = [d.strftime("%a %b %d") for d in dates[0:3]]
  if len(dates) > 3:
    formatted.append("...")
  return "%s (%s)" % (PrettyNumberWord(len(dates), "service date"),
                      ", ".join(formatted))
 
 
def MaxVersion(versions):
  versions = filter(None, versions)
  versions.sort(lambda x,y: -cmp([int(item) for item in x.split('.')],
                                 [int(item) for item in y.split('.')]))
  if len(versions) > 0:
    return versions[0]
 
 
class CountingConsoleProblemReporter(transitfeed.ProblemReporter):
  def __init__(self):
    transitfeed.ProblemReporter.__init__(self)
    self._error_count = 0
    self._warning_count = 0
 
  def _Report(self, e):
    transitfeed.ProblemReporter._Report(self, e)
    if e.IsError():
      self._error_count += 1
    else:
      self._warning_count += 1
 
  def ErrorCount(self):
    return self._error_count
 
  def WarningCount(self):
    return self._warning_count
 
  def FormatCount(self):
    return ProblemCountText(self.ErrorCount(), self.WarningCount())
 
  def HasIssues(self):
    return self.ErrorCount() or self.WarningCount()
 
 
class BoundedProblemList(object):
  """A list of one type of ExceptionWithContext objects with bounded size."""
  def __init__(self, size_bound):
    self._count = 0
    self._exceptions = []
    self._size_bound = size_bound
 
  def Add(self, e):
    self._count += 1
    try:
      bisect.insort(self._exceptions, e)
    except TypeError:
      # The base class ExceptionWithContext raises this exception in __cmp__
      # to signal that an object is not comparable. Instead of keeping the most
      # significant issue keep the first reported.
      if self._count <= self._size_bound:
        self._exceptions.append(e)
    else:
      # self._exceptions is in order. Drop the least significant if the list is
      # now too long.
      if self._count > self._size_bound:
        del self._exceptions[-1]
 
  def _GetDroppedCount(self):
    return self._count - len(self._exceptions)
 
  def __repr__(self):
    return "<BoundedProblemList %s>" % repr(self._exceptions)
 
  count = property(lambda s: s._count)
  dropped_count = property(_GetDroppedCount)
  problems = property(lambda s: s._exceptions)
 
 
class LimitPerTypeProblemReporter(transitfeed.ProblemReporter):
  def __init__(self, limit_per_type):
    transitfeed.ProblemReporter.__init__(self)
 
    # {TYPE_WARNING: {"ClassName": BoundedProblemList()}}
    self._type_to_name_to_problist = {
      TYPE_WARNING: defaultdict(lambda: BoundedProblemList(limit_per_type)),
      TYPE_ERROR: defaultdict(lambda: BoundedProblemList(limit_per_type))
    }
 
  def HasIssues(self):
    return (self._type_to_name_to_problist[TYPE_ERROR] or
            self._type_to_name_to_problist[TYPE_WARNING])
 
  def _Report(self, e):
    self._type_to_name_to_problist[e.GetType()][e.__class__.__name__].Add(e)
 
  def ErrorCount(self):
    error_sets = self._type_to_name_to_problist[TYPE_ERROR].values()
    return sum(map(lambda v: v.count, error_sets))
 
  def WarningCount(self):
    warning_sets = self._type_to_name_to_problist[TYPE_WARNING].values()
    return sum(map(lambda v: v.count, warning_sets))
 
  def ProblemList(self, problem_type, class_name):
    """Return the BoundedProblemList object for given type and class."""
    return self._type_to_name_to_problist[problem_type][class_name]
 
  def ProblemListMap(self, problem_type):
    """Return the map from class name to BoundedProblemList object."""
    return self._type_to_name_to_problist[problem_type]
 
 
class HTMLCountingProblemReporter(LimitPerTypeProblemReporter):
  def FormatType(self, f, level_name, class_problist):
    """Write the HTML dumping all problems of one type.
 
    Args:
      f: file object open for writing
      level_name: string such as "Error" or "Warning"
      class_problist: sequence of tuples (class name,
          BoundedProblemList object)
    """
    class_problist.sort()
    output = []
    for classname, problist in class_problist:
      output.append('<h4 class="issueHeader"><a name="%s%s">%s</a></h4><ul>\n' %
                    (level_name, classname, UnCamelCase(classname)))
      for e in problist.problems:
        self.FormatException(e, output)
      if problist.dropped_count:
        output.append('<li>and %d more of this type.' %
                      (problist.dropped_count))
      output.append('</ul>\n')
    f.write(''.join(output))
 
  def FormatTypeSummaryTable(self, level_name, name_to_problist):
    """Return an HTML table listing the number of problems by class name.
 
    Args:
      level_name: string such as "Error" or "Warning"
      name_to_problist: dict mapping class name to an BoundedProblemList object
 
    Returns:
      HTML in a string
    """
    output = []
    output.append('<table>')
    for classname in sorted(name_to_problist.keys()):
      problist = name_to_problist[classname]
      human_name = MaybePluralizeWord(problist.count, UnCamelCase(classname))
      output.append('<tr><td>%d</td><td><a href="#%s%s">%s</a></td></tr>\n' %
                    (problist.count, level_name, classname, human_name))
    output.append('</table>\n')
    return ''.join(output)
 
  def FormatException(self, e, output):
    """Append HTML version of e to list output."""
    d = e.GetDictToFormat()
    for k in ('file_name', 'feedname', 'column_name'):
      if k in d.keys():
        d[k] = '<code>%s</code>' % d[k]
    problem_text = e.FormatProblem(d).replace('\n', '<br>')
    output.append('<li>')
    output.append('<div class="problem">%s</div>' %
                  transitfeed.EncodeUnicode(problem_text))
    try:
      if hasattr(e, 'row_num'):
        line_str = 'line %d of ' % e.row_num
      else:
        line_str = ''
      output.append('in %s<code>%s</code><br>\n' %
                    (line_str, e.file_name))
      row = e.row
      headers = e.headers
      column_name = e.column_name
      table_header = ''  # HTML
      table_data = ''  # HTML
      for header, value in zip(headers, row):
        attributes = ''
        if header == column_name:
          attributes = ' class="problem"'
        table_header += '<th%s>%s</th>' % (attributes, header)
        table_data += '<td%s>%s</td>' % (attributes, value)
      # Make sure output is encoded into UTF-8
      output.append('<table class="dump"><tr>%s</tr>\n' %
                    transitfeed.EncodeUnicode(table_header))
      output.append('<tr>%s</tr></table>\n' %
                    transitfeed.EncodeUnicode(table_data))
    except AttributeError, e:
      pass  # Hope this was getting an attribute from e ;-)
    output.append('<br></li>\n')
 
  def FormatCount(self):
    return ProblemCountText(self.ErrorCount(), self.WarningCount())
 
  def CountTable(self):
    output = []
    output.append('<table class="count_outside">\n')
    output.append('<tr>')
    if self.ProblemListMap(TYPE_ERROR):
      output.append('<td><span class="fail">%s</span></td>' %
                    PrettyNumberWord(self.ErrorCount(), "error"))
    if self.ProblemListMap(TYPE_WARNING):
      output.append('<td><span class="fail">%s</span></td>' %
                    PrettyNumberWord(self.WarningCount(), "warning"))
    output.append('</tr>\n<tr>')
    if self.ProblemListMap(TYPE_ERROR):
      output.append('<td>\n')
      output.append(self.FormatTypeSummaryTable("Error",
                    self.ProblemListMap(TYPE_ERROR)))
      output.append('</td>\n')
    if self.ProblemListMap(TYPE_WARNING):
      output.append('<td>\n')
      output.append(self.FormatTypeSummaryTable("Warning",
                    self.ProblemListMap(TYPE_WARNING)))
      output.append('</td>\n')
    output.append('</table>')
    return ''.join(output)
 
  def WriteOutput(self, feed_location, f, schedule, other_problems):
    """Write the html output to f."""
    if self.HasIssues():
      if self.ErrorCount() + self.WarningCount() == 1:
        summary = ('<span class="fail">Found this problem:</span>\n%s' %
                   self.CountTable())
      else:
        summary = ('<span class="fail">Found these problems:</span>\n%s' %
                   self.CountTable())
    else:
      summary = '<span class="pass">feed validated successfully</span>'
    if other_problems is not None:
      summary = ('<span class="fail">\n%s</span><br><br>' %
                 other_problems) + summary
 
    basename = os.path.basename(feed_location)
    feed_path = (feed_location[:feed_location.rfind(basename)], basename)
 
    agencies = ', '.join(['<a href="%s">%s</a>' % (a.agency_url, a.agency_name)
                          for a in schedule.GetAgencyList()])
    if not agencies:
      agencies = '?'
 
    dates = "No valid service dates found"
    (start, end) = schedule.GetDateRange()
    if start and end:
      def FormatDate(yyyymmdd):
        src_format = "%Y%m%d"
        dst_format = "%B %d, %Y"
        try:
          return time.strftime(dst_format,
                               time.strptime(yyyymmdd, src_format))
        except ValueError:
          return yyyymmdd
 
      formatted_start = FormatDate(start)
      formatted_end = FormatDate(end)
      dates = "%s to %s" % (formatted_start, formatted_end)
 
    calendar_summary = CalendarSummary(schedule)
    if calendar_summary:
      calendar_summary_html = """<br>
During the upcoming service dates %(date_summary_range)s:
<table>
<tr><th class="header">Average trips per date:</th><td class="header">%(mean_trips)s</td></tr>
<tr><th class="header">Most trips on a date:</th><td class="header">%(max_trips)s, on %(max_trips_dates)s</td></tr>
<tr><th class="header">Least trips on a date:</th><td class="header">%(min_trips)s, on %(min_trips_dates)s</td></tr>
</table>""" % calendar_summary
    else:
      calendar_summary_html = ""
 
    output_prefix = """
<html>
<head>
<meta http-equiv="Content-Type" content="text/html; charset=UTF-8">
<title>FeedValidator: %(feed_file)s</title>
<style>
body {font-family: Georgia, serif; background-color: white}
.path {color: gray}
div.problem {max-width: 500px}
table.dump td,th {background-color: khaki; padding: 2px; font-family:monospace}
table.dump td.problem,th.problem {background-color: dc143c; color: white; padding: 2px; font-family:monospace}
table.count_outside td {vertical-align: top}
table.count_outside {border-spacing: 0px; }
table {border-spacing: 5px 0px; margin-top: 3px}
h3.issueHeader {padding-left: 0.5em}
h4.issueHeader {padding-left: 1em}
.pass {background-color: lightgreen}
.fail {background-color: yellow}
.pass, .fail {font-size: 16pt}
.header {background-color: white; font-family: Georgia, serif; padding: 0px}
th.header {text-align: right; font-weight: normal; color: gray}
.footer {font-size: 10pt}
</style>
</head>
<body>
GTFS validation results for feed:<br>
<code><span class="path">%(feed_dir)s</span><b>%(feed_file)s</b></code>
<br><br>
<table>
<tr><th class="header">Agencies:</th><td class="header">%(agencies)s</td></tr>
<tr><th class="header">Routes:</th><td class="header">%(routes)s</td></tr>
<tr><th class="header">Stops:</th><td class="header">%(stops)s</td></tr>
<tr><th class="header">Trips:</th><td class="