Improve/fix iso8601 timestamp parsing and generation

README.md

@@ -183,3 +183,7 @@ The [ops metametrics slidedeck](http://www.slideshare.net/jallspaw/ops-metametri
 * on graphs in dashboards, show timeframs from start to end, and start to "cause found", to "resolved" etc.
 * concurrent webserver to make sure all http requests can get served
 * better MTBF
+
+## vendored dependencies
+
+* [pyiso8601](https://bitbucket.org/micktwomey/pyiso8601/) -- forklifted 0.1.10 because project is an Hg repo; [LICENSE](https://bitbucket.org/micktwomey/pyiso8601/src/0f02cc55100a1bad23c0ea0bd0f07b8de0e3e3f0/LICENSE?at=0.1.10), [docs](http://pyiso8601.readthedocs.org/en/latest/)

anthracite-web.py

@@ -4,6 +4,8 @@
 import json
 import os
 import time
+import calendar
+import datetime
 import sys
 import types
 from view import page
@@ -184,21 +186,19 @@ def events_edit(event_id, **kwargs):
     return p(body=template('tpl/events_edit', event=event, tags=backend.get_tags()), page='edit', **kwargs)
 
 
-def local_datepick_to_unix_timestamp(datepick):
+def utc_datepick_to_unix_timestamp(datepick):
     '''
-    in: something like 12/31/2012 10:25:35 PM, which is local time.
+    in: something like 12/31/2012 22:25:35, which is local time.
     out: unix timestamp
     '''
-    import time
-    import datetime
-    return int(time.mktime(datetime.datetime.strptime(datepick, "%m/%d/%Y %I:%M:%S %p").timetuple()))
+    return int(calendar.timegm(datetime.datetime.strptime(datepick, "%m/%d/%Y %H:%M:%S").timetuple()))
 
 
 @route('/events/edit/<event_id>', method='POST')
 def events_edit_post(event_id):
     try:
         # TODO: do the same validation here as in add
-        ts = local_datepick_to_unix_timestamp(request.forms.event_datetime)
+        ts = utc_datepick_to_unix_timestamp(request.forms.event_datetime)
         # (select2 tags form field uses comma)
         tags = request.forms.event_tags.split(',')
         event = Event(timestamp=ts, desc=request.forms.event_desc, tags=tags, event_id=event_id)
@@ -220,8 +220,8 @@ def events_add(**kwargs):
 
 
 def add_post_validate_and_parse_base_attributes(request):
-    # local_datepick_to_unix_timestamp will raise exceptions if input is bad
-    ts = local_datepick_to_unix_timestamp(request.forms.event_datetime)
+    # utc_datepick_to_unix_timestamp will raise exceptions if input is bad
+    ts = utc_datepick_to_unix_timestamp(request.forms.event_datetime)
     desc = request.forms.event_desc
     if not desc:
         raise Exception("description must not be empty")
@@ -341,7 +341,7 @@ def events_add_script():
 @route('/report')
 def report(**kwargs):
     import time
-    start = local_datepick_to_unix_timestamp(config.opsreport_start)
+    start = utc_datepick_to_unix_timestamp(config.opsreport_start)
     return p(page='report', body=template('tpl/report', config=config, reportpoints=get_report_data(start, int(time.time()))), **kwargs)
 
 

backend.py

@@ -4,6 +4,7 @@
 import calendar
 import os
 import sys
+import iso8601
 
 
 class Config(dict):
@@ -105,6 +106,7 @@ def __init__(self, config=None):
         sys.path.append("%s/%s" % (os.getcwd(), 'python-dateutil'))
         sys.path.append("%s/%s" % (os.getcwd(), 'requests'))
         sys.path.append("%s/%s" % (os.getcwd(), 'rawes'))
+        sys.path.append("%s/%s" % (os.getcwd(), 'iso8601'))
         import rawes
         import requests
         from rawes.elastic_exception import ElasticException
@@ -140,7 +142,6 @@ def __init__(self, config=None):
             })
             print "created new ElasticSearch Index"
         except ElasticException as e:
-            import re
             if 'IndexAlreadyExistsException' in e.result['error']:
                 pass
             else:
@@ -160,14 +161,15 @@ def object_to_dict(self, event):
         return data
 
     def unix_timestamp_to_iso8601(self, unix_timestamp):
-        return datetime.datetime.utcfromtimestamp(unix_timestamp).isoformat()
+        ## provide "Z" as this is definitely a UTC timestamp
+        return datetime.datetime.utcfromtimestamp(unix_timestamp).isoformat() + "Z"
 
-    def iso8601_to_unix_timestamp(self, iso8601):
+    def iso8601_to_unix_timestamp(self, ts):
         '''
-            elasticsearch returns something like 2013-03-20T20:41:16
+            elasticsearch returns something like 2013-03-20T20:41:16Z
 
         '''
-        unix = calendar.timegm(datetime.datetime.strptime(iso8601, "%Y-%m-%dT%H:%M:%S").timetuple())
+        unix = calendar.timegm(iso8601.parse_date(ts).timetuple())
         return unix
 
     def hit_to_object(self, hit):

config.py

@@ -1,7 +1,6 @@
 listen_host = '0.0.0.0'  # defaults to "all interfaces"
 listen_port = 8081
-opsreport_start = '01/01/2013 12:00:00 AM'
-timezone = "America/New_York"
+opsreport_start = '01/01/2013 00:00:00'
 es_url = 'http://localhost:9200'
 es_index = 'anthracite'
 

iso8601/__init__.py

@@ -0,0 +1 @@
+from .iso8601 import *

iso8601/iso8601.py

@@ -0,0 +1,202 @@
+"""ISO 8601 date time string parsing
+
+Basic usage:
+>>> import iso8601
+>>> iso8601.parse_date("2007-01-25T12:00:00Z")
+datetime.datetime(2007, 1, 25, 12, 0, tzinfo=<iso8601.iso8601.Utc ...>)
+>>>
+
+"""
+
+from datetime import (
+    datetime,
+    timedelta,
+    tzinfo
+)
+from decimal import Decimal
+import logging
+import sys
+import re
+
+__all__ = ["parse_date", "ParseError"]
+
+LOG = logging.getLogger(__name__)
+
+if sys.version_info >= (3, 0, 0):
+    _basestring = str
+else:
+    _basestring = basestring
+
+
+# Adapted from http://delete.me.uk/2005/03/iso8601.html
+ISO8601_REGEX = re.compile(
+    r"""
+    (?P<year>[0-9]{4})
+    (
+        (
+            (-(?P<monthdash>[0-9]{1,2}))
+            |
+            (?P<month>[0-9]{2})
+            (?!$)  # Don't allow YYYYMM
+        )
+        (
+            (
+                (-(?P<daydash>[0-9]{1,2}))
+                |
+                (?P<day>[0-9]{2})
+            )
+            (
+                (
+                    (?P<separator>[ T])
+                    (?P<hour>[0-9]{2})
+                    (:{0,1}(?P<minute>[0-9]{2})){0,1}
+                    (
+                        :{0,1}(?P<second>[0-9]{1,2})
+                        (\.(?P<second_fraction>[0-9]+)){0,1}
+                    ){0,1}
+                    (?P<timezone>
+                        Z
+                        |
+                        (
+                            (?P<tz_sign>[-+])
+                            (?P<tz_hour>[0-9]{2})
+                            :{0,1}
+                            (?P<tz_minute>[0-9]{2}){0,1}
+                        )
+                    ){0,1}
+                ){0,1}
+            )
+        ){0,1}  # YYYY-MM
+    ){0,1}  # YYYY only
+    $
+    """,
+    re.VERBOSE
+)
+
+class ParseError(Exception):
+    """Raised when there is a problem parsing a date string"""
+
+# Yoinked from python docs
+ZERO = timedelta(0)
+class Utc(tzinfo):
+    """UTC
+
+    """
+    def utcoffset(self, dt):
+        return ZERO
+
+    def tzname(self, dt):
+        return "UTC"
+
+    def dst(self, dt):
+        return ZERO
+
+UTC = Utc()
+
+class FixedOffset(tzinfo):
+    """Fixed offset in hours and minutes from UTC
+
+    """
+    def __init__(self, offset_hours, offset_minutes, name):
+        self.__offset_hours = offset_hours  # Keep for later __getinitargs__
+        self.__offset_minutes = offset_minutes  # Keep for later __getinitargs__
+        self.__offset = timedelta(hours=offset_hours, minutes=offset_minutes)
+        self.__name = name
+
+    def __eq__(self, other):
+        if isinstance(other, FixedOffset):
+            return (
+                (other.__offset == self.__offset)
+                and
+                (other.__name == self.__name)
+            )
+        if isinstance(other, tzinfo):
+            return other == self
+        return False
+
+    def __getinitargs__(self):
+        return (self.__offset_hours, self.__offset_minutes, self.__name)
+
+    def utcoffset(self, dt):
+        return self.__offset
+
+    def tzname(self, dt):
+        return self.__name
+
+    def dst(self, dt):
+        return ZERO
+
+    def __repr__(self):
+        return "<FixedOffset %r %r>" % (self.__name, self.__offset)
+
+def to_int(d, key, default_to_zero=False, default=None, required=True):
+    """Pull a value from the dict and convert to int
+
+    :param default_to_zero: If the value is None or empty, treat it as zero
+    :param default: If the value is missing in the dict use this default
+
+    """
+    value = d.get(key) or default
+    LOG.debug("Got %r for %r with default %r", value, key, default)
+    if (value in ["", None]) and default_to_zero:
+        return 0
+    if value is None:
+        if required:
+            raise ParseError("Unable to read %s from %s" % (key, d))
+    else:
+        return int(value)
+
+def parse_timezone(matches, default_timezone=UTC):
+    """Parses ISO 8601 time zone specs into tzinfo offsets
+
+    """
+
+    if matches["timezone"] == "Z":
+        return UTC
+    # This isn't strictly correct, but it's common to encounter dates without
+    # timezones so I'll assume the default (which defaults to UTC).
+    # Addresses issue 4.
+    if matches["timezone"] is None:
+        return default_timezone
+    sign = matches["tz_sign"]
+    hours = to_int(matches, "tz_hour")
+    minutes = to_int(matches, "tz_minute", default_to_zero=True)
+    description = "%s%02d:%02d" % (sign, hours, minutes)
+    if sign == "-":
+        hours = -hours
+        minutes = -minutes
+    return FixedOffset(hours, minutes, description)
+
+def parse_date(datestring, default_timezone=UTC):
+    """Parses ISO 8601 dates into datetime objects
+
+    The timezone is parsed from the date string. However it is quite common to
+    have dates without a timezone (not strictly correct). In this case the
+    default timezone specified in default_timezone is used. This is UTC by
+    default.
+    """
+    if not isinstance(datestring, _basestring):
+        raise ParseError("Expecting a string %r" % datestring)
+    m = ISO8601_REGEX.match(datestring)
+    if not m:
+        raise ParseError("Unable to parse date string %r" % datestring)
+    groups = m.groupdict()
+    LOG.debug("Parsed %s into %s with default timezone %s", datestring, groups, default_timezone)
+
+    tz = parse_timezone(groups, default_timezone=default_timezone)
+
+    groups["second_fraction"] = int(Decimal("0.%s" % (groups["second_fraction"] or 0)) * Decimal("1000000.0"))
+
+    try:
+        return datetime(
+            year=to_int(groups, "year"),
+            month=to_int(groups, "month", default=to_int(groups, "monthdash", required=False, default=1)),
+            day=to_int(groups, "day", default=to_int(groups, "daydash", required=False, default=1)),
+            hour=to_int(groups, "hour", default_to_zero=True),
+            minute=to_int(groups, "minute", default_to_zero=True),
+            second=to_int(groups, "second", default_to_zero=True),
+            microsecond=groups["second_fraction"],
+            tzinfo=tz,
+        )
+    except Exception as e:
+        raise ParseError(e)