diff --git a/CHANGELOG b/CHANGELOG
index 9eb9d29..475ce1d 100644
--- a/CHANGELOG
+++ b/CHANGELOG
@@ -1,3 +1,152 @@
+0.10.1
+======
+
+Bulk inserts into Redis batch TTL calls
+
+0.10.0
+======
+
+Added parameter "duration" to custom transforms to facilitate rate
+calculations. This is a breaking change. Added more documentation on
+custom transforms.
+
+0.9.2
+=====
+
+Implemented support for creating a Timeseries using a URL
+https://github.com/agoragames/kairos/pull/45
+
+0.9.1
+=====
+
+Fixed calculation of relative TTLs
+https://github.com/agoragames/kairos/pull/44
+
+0.9.0
+=====
+
+Replace print statements with warnings for unsupported backends from @akabos
+https://github.com/agoragames/kairos/pull/40
+
+Added `Timeseries.bulk_insert` with optimized Redis and Mongo implementations
+https://github.com/agoragames/kairos/pull/41
+
+Fixed relative TTLs for Redis and Cassandra, fixng #14
+https://github.com/agoragames/kairos/pull/42
+
+0.8.1
+=====
+
+Implemented gevent-compatible Cassandra connection pooling.
+
+Changed the API for Cassandra custom fetch functions.
+
+Any value inserted into a timeseries of ascii, text, blob type will be
+auto-quoted with single quote.
+
+Fixed mongo escaping of values with periods by replacing with "U+FFFF"
+unicode character.
+
+0.8.0
+=====
+
+Added support for Cassandra storage.
+
+Added limits when querying properties of a SQL stat.
+
+Changed behavior of sets to include storage of all datapoints, positive
+or negative.
+
+Added table name option to SQL stores.
+
+0.7.1
+=====
+
+Added configuration options for columns in SQL stores.
+
+Added `Timeseries.delete_all()` to delete all stats in a timeseries.
+
+0.7.0
+=====
+
+Added `Timeseries.list()` to get all of the stat names in a store.
+
+Added `Timeseries.properties()` to get some stats about a stat.
+
+Added `Timeseries.iterate()` to iterate over all the data in a stat.
+
+Added `Timeseries.expire()` to manually expire SQL data.
+
+Changed data type to integer for gregorian time stamps. This will possibly
+require updates for any gregorian data already stored in mongo.
+
+Tests can be skipped using environment variables.
+
+SQL tests can target different host and driver types through SQL_HOST
+environment variable.
+
+SQL string columns (VARCHAR) have a fixed length of 255.
+
+0.6.3
+=====
+
+Alpha support for SQL storage.
+
+0.6.2
+=====
+
+Fix reference to interval when using a custom fetch function in mongo.
+
+0.6.1
+=====
+
+Fix cases where keyword arguments are defined with "None".
+
+Fix case where series() is called with with steps=0.
+
+0.6.0
+=====
+
+New API to fully support customized data reading.
+
+0.5.0
+=====
+
+Provide callers with the ability to customize the fetching of data from a timeseries.
+
+0.4.3
+=====
+
+pin monthdelta requirement, pull request #22 from titusz
+https://github.com/agoragames/kairos/pull/22
+
+0.4.2
+=====
+
+Fixed insert on count timeseries not supporting the intervals keyword argument.
+
+0.4.1
+=====
+
+Adding support for inserting data into multiple intervals with a single
+call to `insert()`.
+
+Added support for set types when using redis storage.
+
+0.4.0
+=====
+
+Added `collapse` keyword argument to `series()` for collapsing an entire
+timeseries into a single row. Transforms are performed after the collapse,
+allowing easy calculation of aggregates across an entire timeseries.
+
+Fix inconsistency in gauge type when collapsing or condensing gauge data.
+Value for an interval is the last recorded non-null gauge value now.
+
+Added support in `get()` and `series()` for joining multiple timeseries
+into a single result by calling with a list of timeseries names rather
+than a single string.
+
0.3.0
=====
diff --git a/LICENSE.txt b/LICENSE.txt
index f07dac7..efd73b4 100644
--- a/LICENSE.txt
+++ b/LICENSE.txt
@@ -1,4 +1,4 @@
-Copyright (c) 2012-2013, Agora Games, LLC All rights reserved.
+Copyright (c) 2012-2017, Agora Games, LLC All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions
diff --git a/README.rst b/README.rst
index bd0853c..4a32347 100644
--- a/README.rst
+++ b/README.rst
@@ -1,11 +1,11 @@
-====================================================
-Kairos - Time series data storage in Redis and Mongo
-====================================================
+=================================
+Kairos - Time series data storage
+=================================
-:Version: 0.3.0
+:Version: 0.10.1
:Download: http://pypi.python.org/pypi/kairos
:Source: https://github.com/agoragames/kairos
-:Keywords: python, redis, mongo, time, timeseries, rrd, gevent, statistics
+:Keywords: python, redis, mongo, sql, mysql, sqlite, postgresql, cassandra, timeseries, rrd, gevent, statistics
.. contents::
:local:
@@ -15,29 +15,154 @@ Kairos - Time series data storage in Redis and Mongo
Overview
========
-Kairos provides time series storage using Redis or Mongo backends. Kairos is
-intended to replace RRD and Whisper in situations where the scale and
-flexibility of Redis or Mongo is required. It works with
-`gevent `_ out of the box.
+Kairos provides time series storage using Redis, Mongo, SQL or Cassandra
+backends. Kairos is intended to replace RRD and Whisper in situations where
+the scale and flexibility of other data stores is required. It works with
+`gevent `_ and is the library on which
+`torus `_ is built.
Recommended for python 2.7 and later, it can work with previous versions if you
install `OrderedDict `_.
+Kairos provides a consistent API for a variety of timeseries types and the
+storage engines they're implemented in. Each timestamp is resolved to a
+consistent bucket identifier ("interval") based on the number of whole seconds
+since epoch, or a number corresponding to the Gregorian date associated with
+the relative intervals ``[daily, weekly, monthly, yearly]`` (e.g ``19991231``).
+Within that, data can optionally be stored at resolutions (e.g. "daily,
+in 1 hour increments"). Multiple intervals can be tracked within a timeseries,
+each with its own resolution and optional TTL (e.g. 60 days of daily data,
+lifetime monthly data).
+
+In data stores that support it, TTLs can be set for automatically deleting
+data after a set number of intervals; other data stores expose an ``expire()``
+method for deleting data programmatically.
+
+Within each interval (or resolution), data is stored according to the type of
+the timeseries and what each backend supports. The values tracked in each
+timeseries can be loosely typed for backends that support it, else the type
+will be whatever is set in the timeseries constructor. Even when loosely typed,
+it should be assumed that the value should be a string or number.
+
+series
+ All data will be stored in the order in which it arrives. Uses data store
+ list types where supported, else it will be timestamped records that
+ come as close as possible to the order in which they were written. Queries
+ will return list objects.
+
+histogram
+ A hash of unique values to the number of its occurrences within an interval.
+ Uses data store dictionaries where supported, else it will be separate
+ records for each unique value and timestamp. Queries will return dictionary
+ objects. Built-in transforms assume that keys are real numbers.
+
+count
+ A simple counter will be maintained for each interval, starting at 0 for
+ an interval. Queries will return an integer.
+
+gauge
+ Stores the last-written value for each interval. Queries will return whatever
+ the value type was.
+
+set
+ Stores all the unique values within an interval. Uses data store sets where
+ supported, else it will be separate records for each unique value. Queries
+ will return set objects.
+
Usage
=====
-Kairos supports redis and mongo storage using the same API.
+Kairos supports all storage engines using the same API. The constructor will
+return a Timeseries instance tailored for the type of data and the storage
+engine, and the API for updating and querying the timeseries is consistent
+for all combinations of data type and storage engine.
-Redis
------
+Constructor
+-----------
+
+The first argument is a handle to a supported storage engine or a URL (see
+below), and the rest of the keyword arguments configure the timeseries. The
+keyword arguments supported by all storage engines are:
+
+type
+ Required, defines the type of the timeseries. Raises a ``NotImplementedError``
+ if the backend does not support the type.
+
+read_func
+ Optional, is a function applied to all values read back from the
+ database. Without it, values will be strings for Redis, whatever
+ `write_func` defined for Mongo. Must accept a string value for Redis
+ (empty string for no data) and can return anything.
-::
+write_func
+ Optional, is a function applied to all values when writing. Can be
+ used for histogram resolution, converting an object into an id, etc.
+ Must accept whatever can be inserted into a timeseries and return an
+ object which can be saved according to the rules of the storage engine.
+
+intervals
+ Required, a dictionary of interval configurations in the form of: ::
+
+ {
+ # interval name, used in keys and should conform to best
+ # practices according to the storage engine.
+ minute: {
+
+ # Required. The number of seconds that the interval will cover,
+ # or a supported Gregorian interval.
+ step: 60,
+
+ # Optional. The maximum number of intervals to maintain. If supplied,
+ # will use Redis and Mongo expiration to delete old intervals, else
+ # intervals exist in perpetuity. If the storage engine doesn't support
+ # expiry, will be used to implement the expire() call.
+ steps: 240,
+
+ # Optional. Defines the resolution of the data, i.e. the number of
+ # seconds in which data is assumed to have occurred "at the same time".
+ # So if you're tracking a month-long time series, you may only need
+ # resolution down to the day, or resolution=86400. Defaults to same
+ # value as "step". Can also be a Gregorian interval.
+ resolution: 60,
+ }
+ }
+
+ In addition to specifying ``step`` and ``resolution`` in terms of seconds,
+ kairos also supports a simplified format for larger time intervals. For
+ hours (h), days (d), weeks (w), months (m) and years (y), you can use
+ the format ``30d`` to represent 30 days, for example.
+
+ As of ``0.3.0``, kairos also supports the Gregorian calendar for ``step``
+ and ``resolution``. Either or both parameters can use the terms ``[daily,
+ weekly, monthly, yearly]`` to describe an interval. You can also mix these
+ terms between ``step`` and ``resolution`` (e.g. ``daily`` in
+ ``1h`` resolutions). The expiration time for Gregorian dates is still defined
+ in terms of seconds and may not match the varying month lengths, leap years,
+ etc. Gregorian dates are translated into ``strptime``- and ``strftime``-compatible
+ keys (as integers) and so may be easier to use in raw form or with any
+ external tools. The ``duration`` parameter to transforms run on gregorian
+ series will be seconds in whole number of days (where a day is 86400 seconds).
+
+Storage Engines
+---------------
+
+Each of the supported storage engines also supports a set of keyword arguments
+to configure their behavior. When intializing with a URL, the keyword argument
+``client_config`` can optionally be a dictionary which will be passed as
+keyword arguments to the constructor for the client associated with the URL.
+If kairos implements any custom keyword arguments from ``client_config`` those
+are documented below.
+
+Redis (redis://)
+****************
+
+An example timeseries stored in Redis: ::
from kairos import Timeseries
import redis
client = redis.Redis('localhost', 6379)
- t = Timeseries(client, type='histogram', read_func=int, intervals={
+ t = Timeseries(client, type='histogram', read_func=float, intervals={
'minute':{
'step':60, # 60 seconds
'steps':120, # last 2 hours
@@ -48,16 +173,31 @@ Redis
t.insert('example', 2.71828)
print t.get('example', 'minute')
-Mongo
------
+Additional keyword arguments are: ::
+
+ prefix
+ Optional, is a prefix for all keys in this timeseries. If
+ supplied and it doesn't end with ":", it will be automatically appended.
+
+Supported URL `formats`__: ::
-::
+ redis://localhost
+ redis://localhost/3
+
+__ https://github.com/andymccurdy/redis-py/blob/master/redis/client.py#L332
+
+All `supported `_ configuration options can be passed in ``client_config``.
+
+Mongo (mongodb://)
+******************
+
+An example timeseries stored in Mongo: ::
from kairos import Timeseries
import pymongo
client = pymongo.MongoClient('localhost')
- t = Timeseries(client, type='histogram', read_func=int, intervals={
+ t = Timeseries(client, type='histogram', read_func=float, intervals={
'minute':{
'step':60, # 60 seconds
'steps':120, # last 2 hours
@@ -68,94 +208,213 @@ Mongo
t.insert('example', 2.71828)
print t.get('example', 'minute')
-Each Timeseries will store data according to one of the supported types. The
-keyword arguments to the constructor are: ::
+Additional keyword arguments are: ::
- type
- One of (series, histogram, count). Optional, defaults to "series".
+ escape_character
+ Optional, defines the character used to escape periods. Defaults to the
+ unicode character "U+FFFF".
- series - each interval will append values to a list
- histogram - each interval will track count of unique values
- count - each interval will maintain a single counter
- gauge - each interval will store the most recent data point
+Supported URL `formats`__: ::
- prefix
- Optional, Redis only, is a prefix for all keys in this timeseries. If
- supplied and it doesn't end with ":", it will be automatically appended.
+ mongodb://localhost
+ mongodb://localhost:27018/timeseries
+ mongodb://guest:host@localhost/authed_db
- read_func
- Optional, is a function applied to all values read back from the
- database. Without it, values will be strings for Redis, whatever
- `write_func` defined for Mongo. Must accept a string value for Redis
- (empty string for no data) and can return anything.
+__ http://docs.mongodb.org/manual/reference/connection-string/
- write_func
- Optional, is a function applied to all values when writing. Can be
- used for histogram resolution, converting an object into an id, etc.
- Must accept whatever can be inserted into a timeseries and return an
- object which can be saved according to the rules of Redis or Mongo.
+All `supported`__ configuration arguments can be passed in ``client_config``, in addition to: ::
- intervals
- Required, a dictionary of interval configurations in the form of:
+ database
+ The name of the database to use. Defaults to 'kairos'. Required if using
+ an auth database. Overrides any database provided in the URL.
- {
- # interval name, used in Redis and Mongo keys and should conform to best
- # practices according to the backend type.
- minute: {
-
- # Required. The number of seconds that the interval will cover,
- # or a supported Gregorian interval.
- step: 60,
-
- # Optional. The maximum number of intervals to maintain. If supplied,
- # will use Redis and Mongo expiration to delete old intervals, else
- # intervals exist in perpetuity.
- steps: 240,
-
- # Optional. Defines the resolution of the data, i.e. the number of
- # seconds in which data is assumed to have occurred "at the same time".
- # So if you're tracking a month-long time series, you may only need
- # resolution down to the day, or resolution=86400. Defaults to same
- # value as "step". Can also be a Gregorian interval.
- resolution: 60,
- }
+__ http://api.mongodb.org/python/current/api/pymongo/mongo_client.html
+
+SQL (\*sql\*://)
+****************
+
+An example timeseries stored in a SQLite memory store: ::
+
+ from kairos import Timeseries
+ from sqlalchemy import create_engine
+
+ client = create_engine('sqlite:///:memory:')
+ t = Timeseries(client, type='histogram', read_func=int, intervals={
+ 'minute':{
+ 'step':60, # 60 seconds
+ 'steps':120, # last 2 hours
}
+ })
+
+ t.insert('example', 3.14159)
+ t.insert('example', 2.71828)
+ print t.get('example', 'minute')
+
+Additional keyword arguments are: ::
+
+ string_length
+ Optional, configures the length of strings (VARCHARs). Defaults to 255.
+ All tables have at least 2 string columns, and the size of these columns
+ may impact usability of the SQL storage engine.
+
+ text_length
+ Optional, configures the length of TEXT and BLOB columns. Defaults to
+ 32Kbytes. Only matters if value_type is a text or blob.
+
+ table_name
+ Optional, overrides the default table name for a timeseries type.
+
+ value_type
+ Optional, defines the type of value to be stored in the timeseries.
+ Defaults to float. Can be a string, a Python type or a SQLAlchemy type
+ or instance.
+
+ 'blob'
+ 'bool'
+
+ 'boolean'
+ 'clob'
+ 'date'
+
+ 'datetime'
+
+ 'decimal'
+
+ 'float'
+
+ 'int'
+ 'int64'
+ 'integer'
+
+ 'long'
+
+ 'str'
+ 'string'
+
+ 'text'
+ 'time'
+
+ 'unicode'
+
+
+Supported URL `formats `_ are many and varied. A few examples: ::
+
+ sqlite:///:memory:
+ postgresql://scott:tiger@localhost/mydatabase
+ mysql+mysqldb://scott:tiger@localhost/foo
+ oracle://scott:tiger@127.0.0.1:1521/sidname
+
+All `supported`__ constructor arguments can be used in ``client_config``.
+
+__ http://docs.sqlalchemy.org/en/rel_0_9/core/engines.html#sqlalchemy.create_engine
+
+Cassandra (cassandra://, cql://)
+********************************
+
+An example timeseries stored in Cassandra: ::
-In addition to specifying ``step`` and ``resolution`` in terms of seconds,
-kairos also supports a simplified format for larger time intervals. For
-hours (h), days (d), weeks (w), months (m) and years (y), you can use
-the format ``30d`` to represent 30 days, for example.
+ from kairos import Timeseries
+ import cql
+
+ client = cql.connect('localhost', 9160, 'keyspace', cql_version='3.0.0')
+ t = Timeseries(client, type='histogram', read_func=int, intervals={
+ 'minute':{
+ 'step':60, # 60 seconds
+ 'steps':120, # last 2 hours
+ }
+ })
-As of ``0.3.0``, kairos also supports the Gregorian calendar for ``step``
-and ``resolution``. Either or both parameters can use the terms ``[daily,
-weekly, monthly, yearly]`` to describe an interval. You can also mix these
-terms with the time periods defined in terms of seconds (e.g. ``daily`` in
-``1h`` resolutions). The expiration time for Gregorian dates is still defined
-in terms of seconds and may not match the varying month lengths, leap years,
-etc. Gregorian dates are translated into ``strptime``- and ``strftime``-compatible
-keys are so may be easier to use in raw form or any integrated tools.
+ t.insert('example', 3.14159)
+ t.insert('example', 2.71828)
+ print t.get('example', 'minute')
-Each retrieval function will by default return an ordered dictionary, though
-condensed results are also available. Run ``script/example`` to see standard
-output; ``watch -n 4 script/example`` is a useful tool as well.
+Additional keyword arguments are: ::
+
+ table_name
+ Optional, overrides the default table name for a timeseries type.
+
+ pool_size
+ Optional, set a cap on the pool size. Defines the maximum number of
+ connections to maintain in the pool. Defaults to 0 for no maximum.
+
+ value_type
+ Optional, defines the type of value to be stored in the timeseries.
+ Defaults to float. Can be a string or a Python type.
+
+
+ string
+ decimal
+
+ int
+ double
+ unicode
+ float
+ long
+
+
+ boolean
+ int64
+ str
+ text
+ blob
+ clob
+ integer
+ bool
+
+
+ inet
+
+Supported URL formats are: ::
+
+ cql://
+ cassandra://localhost:9160
+ cassandra://localhost/database
+
+All `supported`__ constructor arguments can be used in ``client_config``.
+
+__ https://code.google.com/a/apache-extras.org/p/cassandra-dbapi2/source/browse/cql/connection.py
+
+kairos requires `cql `_ as it supports
+`CQL3 `_ and gevent. This
+requires that the keyspace be created before the connection, and the keyword
+argument ``cql_version='3.0.0'`` must be used.
+
+A notable downside of this library is that it does not support a list of
+endpoints to connect to, so is missing key High Availability features.
+
+It is likely that future versions of kairos will require
+`cassandra-driver `_ when it
+is ready.
+
+Cassandra counters can only store integers, and cannot be used for a
+running total of floating point numbers.
+
+Kairos implements a connection pooling mechanism on top of `cql`. The pool
+is a simple soft-cap on the number of connections maintained in the pool,
+but not necessarily the total number of connections at a time. An optional
+hard cap may be implemented in a future release.
Inserting Data
--------------
-There is one method to insert data, ``Timeseries.insert`` which takes the
-followng arguments:
+There are two methods to insert data, ``Timeseries.insert`` and ``Timeseries.bulk_insert``.
+insert
+******
* **name** The name of the statistic
-* **value** The value of the statistic (optional for count timeseries)
-* **timestamp** `(optional)` The timestamp of the statstic, defaults to ``time.time()`` if not supplied
+* **value** The value of the statistic (optional for count timeseries), or a list of values
+* **timestamp** `(optional)` The timestamp of the statistic, defaults to ``time.time()`` if not supplied
+* **intervals** `(optional)` The number of time intervals before (<0) or after (>0) ``timestamp`` to copy the data
+* **\*\*kwargs** `(optional)` Any additional keyword arguments supported by a backend, see below
For ``series`` and ``histogram`` timeseries types, ``value`` can be whatever
you'd like, optionally processed through the ``write_func`` method before being
written to storage. Depending on your needs, ``value`` (or the output of
``write_func``) does not have to be a number, and can be used to track such
things as unique occurances of a string or references to other objects, such
-as MongoDB ObjectIds.
+as MongoDB ObjectIds. Note that many of the aggregate functions in ``histogram``
+expect the data to be real numbers.
For the ``count`` type, ``value`` is optional and should be a float or integer
representing the amount by which to increment or decrement ``name``; it defaults
@@ -163,58 +422,173 @@ to ``1``.
For the ``gauge`` type, ``value`` can be anything and it will be stored as-is.
-Data for all timeseries is stored in "buckets", where any Unix timestamp will
-resolve a consistent bucket name according to the ``step`` and ``resolution``
-attributes of a schema. A bucket will contain the following data structures for
-the corresponding series type.
+For all timeseries types, if ``value`` is one of ``(list,tuple,set)``, will
+call ``bulk_insert``.
+
+The ``intervals`` option allows the caller to simulate the value appearing in
+time periods before or after the ``timestamp``. This is useful for creating
+fast trending (e.g. "count over last seven days"). It is important to note
+that, because the time periods are simulated, resolution is lost for the
+the simulated timestamps.
+
+Redis
+#####
+
+Redis supports an additional keyword argument, ``pipeline``, to give the caller
+control over batches of commands. If ``pipeline`` is supplied, the ``execute``
+method will not be called and it is up to the caller to do so.
+
+bulk_insert
+***********
+
+* **inserts** The structure of inserts (see below)
+* **intervals** `(optional)` The number of time intervals before (<0) or after (>0) ``timestamp`` to copy the data
+* **\*\*kwargs** `(optional)` Any additional keyword arguments supported by a backend, see below
+
+The ``inserts`` field must take the following form: ::
+
+ {
+ timestamp : {
+ name: [ value, ... ],
+ ...
+ },
+ ...
+ }
+
+The meaning of ``timestamp``, ``name`` and ``value`` are identical to those
+parameters in ``insert``. The caller can insert any number of timestamps,
+statistic names and values, and the backend will optimize the insert where
+possible. See details on the different backends below. Where a backend does
+not support an optimized bulk insert, the data structure will be processed
+such that each value will be passed to ``insert``.
+
+The ``inserts`` structure can be a ``dict`` or ``OrderedDict``. If you need
+the insert order preserved, such as when inserting into a ``series`` or
+``gauge``, you should use ``OrderedDict``.
+
+If ``timestamp`` is unknown, use ``None`` for the key and it will be set to
+the current value of ``time.time()``. Note that this may alter ordering if
+``inserts`` is an ``OrderedDict``.
+
+**NOTE** bulk inserts will increase memory usage of the client process.
+
+Redis
+#####
+
+Redis bulk inserts are implemented by using a single pipeline (without
+transactions) and committing the pipeline after all bulk inserts have been
+executed. The bulk insert also supports the ``pipeline`` argument, with the
+same rules as ``insert``.
+
+Mongo
+#####
+
+Mongo bulk inserts are implemented by joining all of the data together into
+a condensed set of queries and updates. As the configuration of a timeseries
+may result in multiple timestamps resolving to the same record (e.g. per-day
+data), this could result in significant performance gains when the timeseries
+is a ``count``, ``histogram`` or ``gauge``.
+
+SQL
+###
+
+There is no optimization for bulk inserts in SQL due to the lack of
+native update-or-insert support. The generic SQL implementation requires an
+attempted update to be committed before kairos can determine if an insert is
+required. Future versions may have optimized implementations for specific
+SQL servers which support such a feature, at which time bulk inserts may be
+optimized for those specific backends.
+
+Cassandra
+#########
+
+The ``cql`` library has no support for transactions, grouping, etc.
+
+Meta Data
+---------
+
+There are two methods to query meta data about a Timeseries.
+
+list
+****
+
+There are no arguments. Returns a list of all of the stat names stored
+in the Timeseries.
+
+properties
+**********
+
+Takes a single argument, the name of the timeseries. Returns a dictionary
+with the following fields: ::
+
+ { interval : { 'first' : timestamp, 'last' : timestamp } }
+
+``interval`` will be the named interval, such as "minute". For each interval,
+there is a dictionary of properties. ``first`` is the timestamp of the first
+data point in the timeseries, and ``last`` is the last data point in the
+timeseries.
-* **series** list
-* **histogram** dictionary (map)
-* **count** integer or float
Reading Data
------------
-There are two methods to read data, ``Timeseries.get`` and ``Timeseries.series``.
-``get`` will return data from a single bucket, and ``series`` will return data
-from several buckets.
+There are three methods to read data, ``Timeseries.get``, ``Timeseries.series``
+and ``Timeseries.iterate``. ``get`` will return data from a single bucket,
+and ``series`` will return data from several buckets. ``iterate`` will use
+the ``Timeseries.properties`` method to determine the date range of the data,
+and return a generator that calls ``get`` for every possible interval in
+the date range.
get
***
-Supports the following parameters:
+Supports the following parameters. All optional parameters are keyword arguments.
-* **name** The name of the statistic
+* **name** The name of the statistic, or a list of names whose data will be joined together.
* **interval** The named interval to read from
* **timestamp** `(optional)` The timestamp to read, defaults to ``time.time()``
-* **condensed** `(optional)` If using resolutions, ``True`` will collapse the resolution data into a single row
-* **transform** `(optional)` Optionally process each row of data. Supports ``[mean, count, min, max, sum]``, or any callable that accepts datapoints according to the type of series (e.g histograms are dictionaries, counts are integers, etc). Transforms are called after ``read_func`` has cast the data type and after resolution data is optionally condensed.
+* **condensed** `(optional)` **DEPRECATED** Use ``condense`` instead. Support for this will be removed entirely in a future release.
+* **transform** `(optional)` Optionally process each row of data. Supports ``[mean, count, min, max, sum]``, or any callable that accepts datapoints according to the type of series (e.g histograms are dictionaries, counts are integers, etc). Transforms are called after ``read_func`` has cast the data type and after resolution data is optionally condensed. If ``transform`` is one of ``(list,tuple,set)``, will load the data once and run all the transforms on that data set. If ``transform`` is a ``dict`` of the form ``{ transform_name : transform_func }``, will run all of the transform functions on the data set. See `Customized Reads`_ for more on custom transforms.
+* **fetch** `(optional)` Function to use instead of the built-in implementations for fetching data. See `Customized Reads`_.
+* **process_row** `(optional)` Can be a callable to implement `Customized Reads`_.
+* **condense** `(optional)` If using resolutions, ``True`` will collapse the resolution data into a single row. Can be a callable to implement `Customized Reads`_.
+* **join_rows** `(optional)` Can be a callable to implement `Customized Reads`_.
Returns a dictionary of ``{ timestamp : data }``, where ``timestamp`` is a Unix timestamp
and ``data`` is a data structure corresponding to the type of series, or whatever
-``transform`` returns. If not using resolutions or ``condensed=True``, the length
+``transform`` returns. If not using resolutions or ``condense=True``, the length
of the dictionary is 1, else it will be the number of resolution buckets within
-the interval that contained data.
+the interval that contained data. If ``transform`` is a list, ``data`` will be a
+dictionary of ``{ transform_func : transformed_data }``. If ``transform`` is a ``dict``,
+``data`` will be a dictionary of ``{ transform_name : transformed_data }``.
series
******
-Almost identical to ``get``, supports the following parameters:
+Almost identical to ``get``, supports the following parameters. All optional parameters are keyword arguments.
-* **name** The name of the statistic
+* **name** The name of the statistic, or a list of names whose data will be joined together.
* **interval** The named interval to read from
* **start** `(optional)` The timestamp which should be in the first interval of the returned data.
* **end** `(optional)` The timestamp which should be in the last interval of the returned data.
* **steps** `(optional)` The number of steps in the interval to read, defaults to either ``steps`` in the configuration or 1. Ignored if both ``start`` and ``end`` are defined. If either ``start`` or ``end`` are defined, ``steps`` is inclusive of whatever interval that timestamp falls into.
-* **condensed** `(optional)` If using resolutions, ``True`` will collapse the resolution data into a single row
-* **transform** `(optional)` Optionally process each row of data. Supports ``[mean, count, min, max, sum]``, or any callable that accepts a list of datapoints according to the type of series (e.g histograms are dictionaries, counts are integers, etc). Transforms are called after ``read_func`` has cast the data type and after resolution data is optionally condensed.
+* **condensed** `(optional)` **DEPRECATED** Use ``condense`` instead. Support for this will be removed entirely in a future release.
+* **transform** `(optional)` Optionally process each row of data. Supports ``[mean, count, min, max, sum]``, or any callable that accepts a list of datapoints according to the type of series (e.g histograms are dictionaries, counts are integers, etc). Transforms are called after ``read_func`` has cast the data type and after resolution data is optionally condensed. If ``transform`` is one of ``(list,tuple,set)``, will load the data once and run all the transforms on that data set. If ``transform`` is a ``dict`` of the form ``{ transform_name : transform_func }``, will run all of the transform functions on the data set. See `Customized Reads`_ for more on custom transforms.
+* **fetch** `(optional)` Function to use instead of the built-in implementations for fetching data. See `Customized Reads`_.
+* **process_row** `(optional)` Can be a callable to implement `Customized Reads`_.
+* **condense** `(optional)` If using resolutions, ``True`` will collapse the resolution data into a single row. Can be a callable to implement `Customized Reads`_.
+* **join_rows** `(optional)` Can be a callable to implement `Customized Reads`_.
+* **collapse** `(optional)` ``True`` will collapse all of the data in the date range into a single result. Can be a callable to implement `Customized Reads`_.
Returns an ordered dictionary of ``{ interval_timestamp : { resolution_timestamp: data } }``,
where ``interval_timestamp`` and ``resolution_timestamp`` are Unix timestamps
and ``data`` is a data structure corresponding to the type of series, or whatever
-``transform`` returns. If not using resolutions or ``condensed=True``, the dictionary
+``transform`` returns. If not using resolutions or ``condense=True``, the dictionary
will be of the form ``{ interval_timestamp : data }``.
+All variations of ``transform`` and the resulting format of ``data`` are the same
+as in ``get``.
+
If both ``start`` and ``end`` are defined, the returned data will start and end
on intervals including those timestamps. If only ``start`` is defined, then the
return data will start with an interval that includes that timestamp, with the
@@ -227,17 +601,295 @@ not necessarily match ``start`` or ``end``. This is because of the consistent
hashing scheme that kairos uses, such that ``start`` and ``end`` will be
translated into the bucket in which it can be found.
+iterate
+*******
+
+Almost identical to ``get`` except it does not accept a ``timestamp`` argument.
+
+* **name** The name of the statistic, or a list of names whose data will be joined together.
+* **interval** The named interval to read from
+* **transform** `(optional)` Optionally process each row of data. Supports ``[mean, count, min, max, sum]``, or any callable that accepts datapoints according to the type of series (e.g histograms are dictionaries, counts are integers, etc). Transforms are called after ``read_func`` has cast the data type and after resolution data is optionally condensed. If ``transform`` is one of ``(list,tuple,set)``, will load the data once and run all the transforms on that data set. If ``transform`` is a ``dict`` of the form ``{ transform_name : transform_func }``, will run all of the transform functions on the data set. See `Customized Reads`_ for more on custom transforms.
+* **fetch** `(optional)` Function to use instead of the built-in implementations for fetching data. See `Customized Reads`_.
+* **process_row** `(optional)` Can be a callable to implement `Customized Reads`_.
+* **condense** `(optional)` If using resolutions, ``True`` will collapse the resolution data into a single row. Can be a callable to implement `Customized Reads`_.
+* **join_rows** `(optional)` Can be a callable to implement `Customized Reads`_.
+
+Returns a generator which iterates over ``( timestamp, data )`` tuples, where
+``timestamp`` is a Unix timestamp and ``data`` corresponds to the rules
+documented in ``get``. Yields a tuple for each potential timestamp in the
+entire date range of the timeseries, even if there is no data.
+
+
+Customized Reads
+----------------
+
+**ALPHA** This feature is still being explored and the API may change significantly.
+
+There are times when the data in a timeseries requires processing to
+be pushed onto the datastore.
+
+There are times when one needs custom control over the reading and processing
+of data in a timeseries. As there is no good way to do this generically,
+the ``get`` and ``series`` API supports several keyword arguments to customize
+access to the data. Common use cases are to handle large sets of data that
+can be processed in the datastore, and situations where one wants to implement
+cutom analysis of the dataset such as calculating variance.
+
+General
+*******
+
+The following functions can be overloaded with keyword parameters to ``get`` and
+``series`` (``collapse`` being only used for a series).
+
+fetch
+#####
+
+A customized database read function. The usage varies depending on the backends
+which are described in detail below.
+**IMPORTANT** You are welcome to change the type of the return value, but be
+wary that transforms, condense and collapse functionality may not work
+properly with the changed data types.
+
+process_row
+###########
+
+The function which handles the type casting of the data read from the backend
+and also calling the ``read_func`` if it has been defined for the time series.
+It is required that you define this function if you overload ``fetch`` such
+that the returned data type is not the same as the time series' native format
+(``dict`` for histogram, ``list`` for series, etc).
+
+The function must be in the form of ``process_row(data)``, where:
+
+* **data** The row data generated by the native or ``fetch`` implementation, not
+ including any time stamps.
+
+The function may return any data type, but if it's not the native format of the
+time series, additional downstream functions may have to be overloaded.
+
+condense
+########
+
+If the ``condense`` argument is a callable, the caller can override how resolution
+data is collapsed (reduced) into a single interval. The argument will always be
+in the form of: ::
+
+ {
+ 'resolution_t0' : ,
+ 'resolution_t1' : ,
+ ...
+ 'resolution_tN' : ,
+ }
+
+Where ```` is the data returned from the native or ``fetch``
+implementation and passed through the native or custom ``process_row``
+implementation.
+
+The function should return a single value, optionally in the same format as
+````, but this method could also be used for calculating such
+things as rate of change or variance within a time interval.
+
+join_rows
+#########
+
+If the ``join_rows`` argument is a callable and the ``name`` parameter to ``get``
+or ``series`` is one of ``(list,tuple,set)``, this method will be called to join
+the data from several named timeseries into a single result. The argument will
+always be in the form of: ::
+
+ [
+ ,
+ ,
+ ...
+
+ ]
+
+Where ```` will be the data within a single timestamp window in
+the series' native format or whatever was generated by custom implementations
+of ``fetch``, ``process_row`` and/or ``condense``. It is important to note
+that not every series will contain data points within a given time interval.
+
+In addition to reducing multiple time series' worth of data within an interval
+into a single result, this method could be used to implement cross-series
+analytics such as unions, intersections and differentials.
+
+collapse
+########
+
+If the ``collapse`` argument is a callable, the caller can override how interval
+data is collapsed (reduced) into a single result. The native implementation is to
+call the ``condense`` function implemented by a time series. The arguments are
+the same as a custom ``condense`` function, as-is the expected return value.
+
+It's important to note that if ``collapse`` is defined, the series will
+automatically be condensed as well, so if ``fetch`` is overloaded to return a
+custom data type, then ``condense`` must also be defined. If ``collapse`` is
+``True``, the custom ``condense`` function will be used if defined.
+
+In addition to collapsing the result of a time series into a single data set,
+this method could also be used to calculate data across a time series, such as
+variance.
+
+transform
+#########
+
+As noted previously, ``transform`` can be any callable, list of names or callables,
+or a named map of transform names or callables. The transforms will be processed
+after all previous native or custom read functions, including ``collapse``.
+
+Transforms must accept 2 parameters, the data and the time interval in seconds
+over which that data was captured. The ``data`` parameter will be of a type
+corresponding to the timeseries type, or whatever was generated by a custom
+``fetch``. For example: ::
+
+ def custom_rate_for_counts(data, duration):
+ return float(data) / float(duration)
+
+For gregorian timeseries, ``duration`` will seconds in terms of the whole
+number of days over which the data was captured, where a day is 86400 seconds.
+
+Redis
+*****
+
+The function must be in the form of ``fetch(handle, key)``, where:
+
+* **handle** Either a Redis client or pipeline instance
+* **key** The key for the timeseries data
+
+The return value should correspond to the data type of timeseries, e.g. ``dict``
+for a histogram. One should always assume that ``handle`` is both a pipeline
+`and` a client, and ``fetch`` should return the result of, e.g.
+``handle.hlen(...)``, but that it cannot be used to return a literal, such
+as ``lambda: h,k: { 'foo' : h.hlen(k) }``
+
+Mongo
+*****
+
+The function must be in the form of ``fetch(handle, **kwargs)``, where:
+
+* **handle** A PyMongo ``Collection``
+* **spec** The (suggested) query specification
+* **sort** The (suggested) sort definition for the query
+* **method** The suggested method to use on the ``handle``
+
+The required return value depends on the value of ``method``.
+
+* **find_one** Should return a hash in the form ``{ value : }``, where
+ ```` should correspond to the data type of the timeseries, e.g. ``list``
+ for a series. May directly return a result from ``pymongo.collection.find_one``.
+* **find** Should return an iterable in the form ``[ { value: }, ... ]``,
+ where ```` follows the same rules as ``find_one``.
+
+Re-implementing the default functionality would look like: ::
+
+ def mongo_fetch(handle, spec, sort, method):
+ if method=='find':
+ return handle.find( spec=spec, sort=sort )
+ elif method=='find_one':
+ return handle.find_one( spec )
+
+SQL
+***
+
+The function must be in the form
+``fetch(connection, table, name, interval, i_start, i_end)``, where:
+
+* **connection** A SQLAlchemy ``Connection``
+* **table** A SQLAlchemy ``Table``
+* **name** The name of the stat to fetch
+* **interval** The interval of the stat to fetch
+* **i_start** The interval timestamp (starting) key
+* **i_end** (optional) For a series, the ending timestamp key
+
+The return value should be in the form of ::
+
+ {
+ 'interval_t0' : {
+ 'resolution_t0t0' : ,
+ 'resolution_t0t1' : ,
+ ...
+ 'resolution_t0tN' :
+ },
+ 'interval_t1' : { ... },
+ ...
+ 'interval_tN' : { ... },
+ }
+
+If the series doesn't use resolutions, then ``resolution_tNtN`` should be
+``None``, and so each interval will be in the form
+``{ 'interval_tN: { None : } }``. This is inherent in the way that
+data is stored within the tables.
+
+If ``i_end`` is supplied, the query should be over the range
+``i_time >= i_start AND i_time <= i_end``, else the query should be for
+the interval ``i_time = i_start``.
+
+Cassandra
+*********
+
+The function must be in the form
+``fetch(connection, table, name, interval, i_start, i_end)``, where:
+
+* **cursor** A ``cql`` ``Connection``
+* **table** The name of the table
+* **name** The name of the stat to fetch
+* **interval** The interval of the stat to fetch
+* **intervals** The list of interval timestamps
+
+The return value should be in the form of ::
+
+ {
+ 'interval_t0' : {
+ 'resolution_t0t0' : ,
+ 'resolution_t0t1' : ,
+ ...
+ 'resolution_t0tN' :
+ },
+ 'interval_t1' : { ... },
+ ...
+ 'interval_tN' : { ... },
+ }
+
+If the series doesn't use resolutions, then ``resolution_tNtN`` should be
+``None``, and so each interval will be in the form
+``{ 'interval_tN: { None : } }`` and can be determined when a row
+has an ``r_time`` of ``-1``.
+
+If ``intervals`` is a list of 1, it's effectively a ``get`` query where
+``i_time = intervals[0]``, else it's ``i_time >= intervals[0] AND
+i_time <= intervals[-1]``. The full list of intervals is supplied to workaround
+Cassandra's lack of grouping support in situations where an aggregate per
+``i_time`` is desired.
+
Deleting Data
-------------
-To delete the data, call ``Timeseries.delete`` with the name of your statistic,
-and all values in all intervals will be deleted.
+delete
+******
+
+Takes a single argument, the name of the timeseries. Will delete all data for
+that timeseries in all intervals.
+
+delete_all
+**********
+
+Deletes every timeseries for all intervals. This method may be fast in data
+stores that support optimized deletes, else it will have to delete for each
+timeseries returned in ``list``.
+
+expire
+******
+
+Takes a single argument, the name of the timeseries. For storage engines that
+do not support expiry, such as SQL, will delete expired data from intervals
+for which ``steps`` is defined. All other storage engines will raise the
+``NotImplementedError`` exception.
Dragons!
--------
-Kairos achieves its efficiency by using Redis or Mongo TTLs and data structures
+Kairos achieves its efficiency by using TTLs and data structures
in combination with a key naming scheme that generates consistent keys based on
any timestamp relative to epoch. However, just like
`RRDtool `_, changing any attribute of the
@@ -245,11 +897,15 @@ timeseries means that new data will be stored differently than old data. For
this reason it's best to completely delete all data in an old time series
before creating or querying using a new configuration.
+If you want to migrate data, there are tools in
+`torus `_ that can help.
+
Installation
============
-Kairos is available on `pypi `_ and can be installed using ``pip`` ::
+Kairos is available on `pypi `_ and can be
+installed using ``pip`` ::
pip install kairos
@@ -264,9 +920,8 @@ If installing from source:
pip install -r requirements.pip
-Note that kairos does not by default require the redis package, nor does
-it require `hiredis `_ though it is
-strongly recommended.
+Note that kairos does not install packages for any of the supported backends,
+and that you must do this yourself.
Tests
=====
@@ -275,13 +930,34 @@ Use `nose `_ to run the test suite. ::
$ nosetests
+The test suite can be controlled through several environment variables, all
+defaulting to ``true``.
+
+* **TEST_REDIS** *true*
+* **TEST_MONGO** *true*
+* **TEST_SQL** *true*
+* **TEST_CASSANDRA** *true*
+* **TEST_SERIES** *true*
+* **TEST_HISTOGRAM** *true*
+* **TEST_COUNT** *true*
+* **TEST_GAUGE** *true*
+* **TEST_SET** *true*
+* **SQL_HOST** *sqlite:///:memory:*
+* **CASSANDRA_KEYSPACE** *kairos*
+
+Roadmap
+=======
-Future
-======
-
+* Round-robbin intervals for datastores without TTLs
+* Round-robbin databases: memcache (and compatible, e.g. ElastiCache), Riak,
+ DynamoDB, SimpleDB, GDBM, Berkeley DB, and more
* Redis optimizations
+* Capped collection support for mongo
+* Python 3 support
+* InfluxDB support
* Bloom filters
-* "Native" transforms that leverage data store features (e.g. "length")
+* Joined series populate a data structure at query time
+* Joined series support concurrency "runner"
License
=======
diff --git a/development.pip b/development.pip
index 5ceaf85..75adcf8 100644
--- a/development.pip
+++ b/development.pip
@@ -3,12 +3,15 @@
-r requirements.pip
# Required redis
-redis==2.4.11
+redis==2.8
hiredis>=0.1.0
# Required pymongo
pymongo>=2.5
+# Required cql
+cql==1.4.0
+
# mocking framework
chai>=0.3.1
diff --git a/kairos/__init__.py b/kairos/__init__.py
index 33c5aee..8e7123a 100644
--- a/kairos/__init__.py
+++ b/kairos/__init__.py
@@ -1,9 +1,10 @@
'''
-Copyright (c) 2012-2013, Agora Games, LLC All rights reserved.
+Copyright (c) 2012-2017, Agora Games, LLC All rights reserved.
https://github.com/agoragames/kairos/blob/master/LICENSE.txt
'''
-__version__ = "0.3.0"
+from __future__ import absolute_import
+__version__ = "0.10.1"
-from timeseries import Timeseries
-from exceptions import *
+from .timeseries import Timeseries
+from .exceptions import *
diff --git a/kairos/cassandra_backend.py b/kairos/cassandra_backend.py
new file mode 100644
index 0000000..71f2987
--- /dev/null
+++ b/kairos/cassandra_backend.py
@@ -0,0 +1,692 @@
+'''
+Copyright (c) 2012-2017, Agora Games, LLC All rights reserved.
+
+https://github.com/agoragames/kairos/blob/master/LICENSE.txt
+'''
+from .exceptions import *
+from .timeseries import *
+
+import cql
+
+import time
+from datetime import date, datetime
+from datetime import time as time_type
+from decimal import Decimal
+from Queue import Queue, Empty, Full
+import re
+from urlparse import *
+
+# Test python3 compatibility
+try:
+ x = long(1)
+except NameError:
+ long = int
+try:
+ x = unicode('foo')
+except NameError:
+ unicode = str
+
+TYPE_MAP = {
+ str : 'ascii',
+ 'str' : 'ascii',
+ 'string' : 'ascii',
+
+ unicode : 'text', # works for py3 too
+ 'unicode' : 'text',
+
+ float : 'float',
+ 'float' : 'float',
+
+ 'double' : 'double',
+
+ int : 'int',
+ 'int' : 'int',
+ 'integer' : 'int',
+
+ long : 'varint', # works for py3 too
+ 'long' : 'varint',
+ 'int64' : 'bigint',
+
+ 'decimal' : 'decimal',
+
+ bool : 'boolean',
+ 'bool' : 'boolean',
+ 'boolean' : 'boolean',
+
+ 'text' : 'text',
+ 'clob' : 'blob',
+ 'blob' : 'blob',
+
+ 'inet' : 'inet',
+}
+
+QUOTE_TYPES = set(['ascii','text','blob'])
+QUOTE_MATCH = re.compile("^'.*'$")
+
+def scoped_connection(func):
+ '''
+ Decorator that gives out connections.
+ '''
+ def _with(series, *args, **kwargs):
+ connection = None
+ try:
+ connection = series._connection()
+ return func(series, connection, *args, **kwargs)
+ finally:
+ series._return( connection )
+ return _with
+
+class CassandraBackend(Timeseries):
+
+ def __new__(cls, *args, **kwargs):
+ if cls==CassandraBackend:
+ ttype = kwargs.pop('type', None)
+ if ttype=='series':
+ return CassandraSeries.__new__(CassandraSeries, *args, **kwargs)
+ elif ttype=='histogram':
+ return CassandraHistogram.__new__(CassandraHistogram, *args, **kwargs)
+ elif ttype=='count':
+ return CassandraCount.__new__(CassandraCount, *args, **kwargs)
+ elif ttype=='gauge':
+ return CassandraGauge.__new__(CassandraGauge, *args, **kwargs)
+ elif ttype=='set':
+ return CassandraSet.__new__(CassandraSet, *args, **kwargs)
+ raise NotImplementedError("No implementation for %s types"%(ttype))
+ return Timeseries.__new__(cls, *args, **kwargs)
+
+ @classmethod
+ def url_parse(self, url, **kwargs):
+ location = urlparse(url)
+ if location.scheme in ('cassandra','cql'):
+ host = location.netloc or "localhost:9160"
+ if re.search(":[0-9]+$", host):
+ ip,port = host.split(':')
+ else:
+ ip = host
+ port = 9160
+
+ keyspace = location.path[1:] or kwargs.get('database', 'kairos')
+ if '?' in keyspace:
+ keyspace,params = keyspace.split('?')
+
+ return cql.connect(ip, int(port), keyspace, cql_version='3.0.0', **kwargs)
+
+
+ def __init__(self, client, **kwargs):
+ '''
+ Initialize the sql backend after timeseries has processed the configuration.
+ '''
+ # Only CQL3 is supported
+ if client.cql_major_version != 3:
+ raise TypeError("Only CQL3 is supported")
+
+ vtype = kwargs.get('value_type', float)
+ if vtype in TYPE_MAP:
+ self._value_type = TYPE_MAP[vtype]
+ else:
+ raise TypeError("Unsupported type '%s'"%(vtype))
+
+ self._table = kwargs.get('table_name', self._table)
+
+ # copy internal variables of the connection for poor-mans pooling
+ self._host = client.host
+ self._port = client.port
+ self._keyspace = client.keyspace
+ self._cql_version = client.cql_version
+ self._compression = client.compression
+ self._consistency_level = client.consistency_level
+ self._transport = client.transport
+ self._credentials = client.credentials
+ self._pool = Queue(kwargs.get('pool_size',0))
+ self._pool.put( client )
+
+ super(CassandraBackend,self).__init__(client, **kwargs)
+
+ def _connection(self):
+ '''
+ Return a connection from the pool
+ '''
+ try:
+ return self._pool.get(False)
+ except Empty:
+ args = [
+ self._host, self._port, self._keyspace
+ ]
+ kwargs = {
+ 'user' : None,
+ 'password' : None,
+ 'cql_version' : self._cql_version,
+ 'compression' : self._compression,
+ 'consistency_level' : self._consistency_level,
+ 'transport' : self._transport,
+ }
+ if self._credentials:
+ kwargs['user'] = self._credentials['user']
+ kwargs['password'] = self._credentials['password']
+ return cql.connect(*args, **kwargs)
+
+ def _return(self, connection):
+ try:
+ self._pool.put(connection, False)
+ except Full:
+ # do not return connection to the pool.
+ pass
+
+ def _insert(self, name, value, timestamp, intervals, **kwargs):
+ '''
+ Insert the new value.
+ '''
+ if self._value_type in QUOTE_TYPES and not QUOTE_MATCH.match(value):
+ value = "'%s'"%(value)
+
+ for interval,config in self._intervals.items():
+ timestamps = self._normalize_timestamps(timestamp, intervals, config)
+ for tstamp in timestamps:
+ self._insert_data(name, value, tstamp, interval, config, **kwargs)
+
+ @scoped_connection
+ def _insert_data(self, connection, name, value, timestamp, interval, config):
+ '''Helper to insert data into cql.'''
+ cursor = connection.cursor()
+ try:
+ stmt = self._insert_stmt(name, value, timestamp, interval, config)
+ if stmt:
+ cursor.execute(stmt)
+ finally:
+ cursor.close()
+
+ @scoped_connection
+ def _get(self, connection, name, interval, config, timestamp, **kws):
+ '''
+ Get the interval.
+ '''
+ i_bucket = config['i_calc'].to_bucket(timestamp)
+ fetch = kws.get('fetch')
+ process_row = kws.get('process_row') or self._process_row
+
+ rval = OrderedDict()
+ if fetch:
+ data = fetch( connection, self._table, name, interval, [i_bucket] )
+ else:
+ data = self._type_get(name, interval, i_bucket)
+
+ if config['coarse']:
+ if data:
+ rval[ config['i_calc'].from_bucket(i_bucket) ] = process_row(data.values()[0][None])
+ else:
+ rval[ config['i_calc'].from_bucket(i_bucket) ] = self._type_no_value()
+ else:
+ for r_bucket,row_data in data.values()[0].items():
+ rval[ config['r_calc'].from_bucket(r_bucket) ] = process_row(row_data)
+
+ return rval
+
+ @scoped_connection
+ def _series(self, connection, name, interval, config, buckets, **kws):
+ '''
+ Fetch a series of buckets.
+ '''
+ fetch = kws.get('fetch')
+ process_row = kws.get('process_row') or self._process_row
+
+ rval = OrderedDict()
+
+ if fetch:
+ data = fetch( connection, self._table, name, interval, buckets )
+ else:
+ data = self._type_get(name, interval, buckets[0], buckets[-1])
+
+ if config['coarse']:
+ for i_bucket in buckets:
+ i_key = config['i_calc'].from_bucket(i_bucket)
+ i_data = data.get( i_bucket )
+ if i_data:
+ rval[ i_key ] = process_row( i_data[None] )
+ else:
+ rval[ i_key ] = self._type_no_value()
+ else:
+ if data:
+ for i_bucket, i_data in data.items():
+ i_key = config['i_calc'].from_bucket(i_bucket)
+ rval[i_key] = OrderedDict()
+ for r_bucket, r_data in i_data.items():
+ r_key = config['r_calc'].from_bucket(r_bucket)
+ if r_data:
+ rval[i_key][r_key] = process_row(r_data)
+ else:
+ rval[i_key][r_key] = self._type_no_value()
+
+ return rval
+
+ @scoped_connection
+ def delete(self, connection, name):
+ cursor = connection.cursor()
+ try:
+ cursor.execute("DELETE FROM %s WHERE name='%s';"%(self._table,name))
+ finally:
+ cursor.close()
+
+ @scoped_connection
+ def delete_all(self, connection):
+ cursor = connection.cursor()
+ try:
+ cursor.execute("TRUNCATE %s"%(self._table))
+ finally:
+ cursor.close()
+
+ @scoped_connection
+ def list(self, connection):
+ cursor = connection.cursor()
+ rval = set()
+
+ try:
+ cursor.execute('SELECT name FROM %s'%(self._table))
+ for row in cursor:
+ rval.add(row[0])
+ finally:
+ cursor.close()
+ return list(rval)
+
+ @scoped_connection
+ def properties(self, connection, name):
+ cursor = connection.cursor()
+ rval = {}
+
+ try:
+ for interval,config in self._intervals.items():
+ rval.setdefault(interval, {})
+
+ cursor.execute('''SELECT i_time
+ FROM %s
+ WHERE name = '%s' AND interval = '%s'
+ ORDER BY interval ASC, i_time ASC
+ LIMIT 1'''%(self._table, name, interval))
+
+ rval[interval]['first'] = config['i_calc'].from_bucket(
+ cursor.fetchone()[0] )
+
+ cursor.execute('''SELECT i_time
+ FROM %s
+ WHERE name = '%s' AND interval = '%s'
+ ORDER BY interval DESC, i_time DESC
+ LIMIT 1'''%(self._table, name, interval))
+ rval[interval]['last'] = config['i_calc'].from_bucket(
+ cursor.fetchone()[0] )
+ finally:
+ cursor.close()
+
+ return rval
+
+class CassandraSeries(CassandraBackend, Series):
+
+ def __init__(self, *a, **kwargs):
+ self._table = 'series'
+ super(CassandraSeries,self).__init__(*a, **kwargs)
+
+ cursor = self._client.cursor()
+ # TODO: support other value types
+ # TODO: use varint for [ir]_time?
+ try:
+ res = cursor.execute('''CREATE TABLE IF NOT EXISTS %s (
+ name text,
+ interval text,
+ i_time bigint,
+ r_time bigint,
+ value list<%s>,
+ PRIMARY KEY(name, interval, i_time, r_time)
+ )'''%(self._table, self._value_type))
+ except cql.ProgrammingError as pe:
+ if 'existing' not in str(pe):
+ raise
+ finally:
+ cursor.close()
+
+ def _insert_stmt(self, name, value, timestamp, interval, config):
+ '''Helper to generate the insert statement.'''
+ # Calculate the TTL and abort if inserting into the past
+ expire, ttl = config['expire'], config['ttl'](timestamp)
+ if expire and not ttl:
+ return None
+
+ i_time = config['i_calc'].to_bucket(timestamp)
+ if not config['coarse']:
+ r_time = config['r_calc'].to_bucket(timestamp)
+ else:
+ r_time = -1
+
+ # TODO: figure out escaping rules of CQL
+ table_spec = self._table
+ if ttl:
+ table_spec += " USING TTL %s "%(ttl)
+ stmt = '''UPDATE %s SET value = value + [%s]
+ WHERE name = '%s'
+ AND interval = '%s'
+ AND i_time = %s
+ AND r_time = %s'''%(table_spec, value, name, interval, i_time, r_time)
+ return stmt
+
+ @scoped_connection
+ def _type_get(self, connection, name, interval, i_bucket, i_end=None):
+ rval = OrderedDict()
+
+ # TODO: more efficient creation of query string
+ stmt = '''SELECT i_time, r_time, value
+ FROM %s
+ WHERE name = '%s' AND interval = '%s'
+ '''%(self._table, name, interval)
+ if i_end :
+ stmt += ' AND i_time >= %s AND i_time <= %s'%(i_bucket, i_end)
+ else:
+ stmt += ' AND i_time = %s'%(i_bucket)
+ stmt += ' ORDER BY interval, i_time, r_time'
+
+ cursor = connection.cursor()
+ try:
+ cursor.execute(stmt)
+ for row in cursor:
+ i_time, r_time, value = row
+ if r_time==-1:
+ r_time = None
+ rval.setdefault(i_time,OrderedDict())[r_time] = value
+ finally:
+ cursor.close()
+ return rval
+
+class CassandraHistogram(CassandraBackend, Histogram):
+
+ def __init__(self, *a, **kwargs):
+ self._table = 'histogram'
+ super(CassandraHistogram,self).__init__(*a, **kwargs)
+
+ # TODO: use varint for [ir]_time?
+ # TODO: support other value types
+ cursor = self._client.cursor()
+ try:
+ res = cursor.execute('''CREATE TABLE IF NOT EXISTS %s (
+ name text,
+ interval text,
+ i_time bigint,
+ r_time bigint,
+ value %s,
+ count counter,
+ PRIMARY KEY(name, interval, i_time, r_time, value)
+ )'''%(self._table, self._value_type))
+ except cql.ProgrammingError as pe:
+ if 'existing' not in str(pe):
+ raise
+ finally:
+ cursor.close()
+
+ def _insert_stmt(self, name, value, timestamp, interval, config):
+ '''Helper to generate the insert statement.'''
+ # Calculate the TTL and abort if inserting into the past
+ expire, ttl = config['expire'], config['ttl'](timestamp)
+ if expire and not ttl:
+ return None
+
+ i_time = config['i_calc'].to_bucket(timestamp)
+ if not config['coarse']:
+ r_time = config['r_calc'].to_bucket(timestamp)
+ else:
+ r_time = -1
+
+ # TODO: figure out escaping rules of CQL
+ table_spec = self._table
+ if ttl:
+ table_spec += " USING TTL %s "%(ttl)
+ stmt = '''UPDATE %s SET count = count + 1
+ WHERE name = '%s'
+ AND interval = '%s'
+ AND i_time = %s
+ AND r_time = %s
+ AND value = %s'''%(table_spec, name, interval, i_time, r_time, value)
+ return stmt
+
+ @scoped_connection
+ def _type_get(self, connection, name, interval, i_bucket, i_end=None):
+ rval = OrderedDict()
+
+ # TODO: more efficient creation of query string
+ stmt = '''SELECT i_time, r_time, value, count
+ FROM %s
+ WHERE name = '%s' AND interval = '%s'
+ '''%(self._table, name, interval)
+ if i_end :
+ stmt += ' AND i_time >= %s AND i_time <= %s'%(i_bucket, i_end)
+ else:
+ stmt += ' AND i_time = %s'%(i_bucket)
+ stmt += ' ORDER BY interval, i_time, r_time'
+
+ cursor = connection.cursor()
+ try:
+ cursor.execute(stmt)
+ for row in cursor:
+ i_time, r_time, value, count = row
+ if r_time==-1:
+ r_time = None
+ rval.setdefault(i_time,OrderedDict()).setdefault(r_time,{})[value] = count
+ finally:
+ cursor.close()
+ return rval
+
+class CassandraCount(CassandraBackend, Count):
+
+ def __init__(self, *a, **kwargs):
+ self._table = 'count'
+ super(CassandraCount,self).__init__(*a, **kwargs)
+
+ # TODO: use varint for [ir]_time?
+ # TODO: support other value types
+ cursor = self._client.cursor()
+ try:
+ res = cursor.execute('''CREATE TABLE IF NOT EXISTS %s (
+ name text,
+ interval text,
+ i_time bigint,
+ r_time bigint,
+ count counter,
+ PRIMARY KEY(name, interval, i_time, r_time)
+ )'''%(self._table))
+ except cql.ProgrammingError as pe:
+ if 'existing' not in str(pe):
+ raise
+ finally:
+ cursor.close()
+
+ def _insert_stmt(self, name, value, timestamp, interval, config):
+ '''Helper to generate the insert statement.'''
+ # Calculate the TTL and abort if inserting into the past
+ expire, ttl = config['expire'], config['ttl'](timestamp)
+ if expire and not ttl:
+ return None
+
+ i_time = config['i_calc'].to_bucket(timestamp)
+ if not config['coarse']:
+ r_time = config['r_calc'].to_bucket(timestamp)
+ else:
+ r_time = -1
+
+ # TODO: figure out escaping rules of CQL
+ table_spec = self._table
+ if ttl:
+ table_spec += " USING TTL %s "%(ttl)
+ stmt = '''UPDATE %s SET count = count + %s
+ WHERE name = '%s'
+ AND interval = '%s'
+ AND i_time = %s
+ AND r_time = %s'''%(table_spec, value, name, interval, i_time, r_time)
+ return stmt
+
+ @scoped_connection
+ def _type_get(self, connection, name, interval, i_bucket, i_end=None):
+ rval = OrderedDict()
+
+ # TODO: more efficient creation of query string
+ stmt = '''SELECT i_time, r_time, count
+ FROM %s
+ WHERE name = '%s' AND interval = '%s'
+ '''%(self._table, name, interval)
+ if i_end :
+ stmt += ' AND i_time >= %s AND i_time <= %s'%(i_bucket, i_end)
+ else:
+ stmt += ' AND i_time = %s'%(i_bucket)
+ stmt += ' ORDER BY interval, i_time, r_time'
+
+ cursor = connection.cursor()
+ try:
+ cursor.execute(stmt)
+ for row in cursor:
+ i_time, r_time, count = row
+ if r_time==-1:
+ r_time = None
+ rval.setdefault(i_time,OrderedDict())[r_time] = count
+ finally:
+ cursor.close()
+ return rval
+
+class CassandraGauge(CassandraBackend, Gauge):
+
+ def __init__(self, *a, **kwargs):
+ self._table = 'gauge'
+ super(CassandraGauge,self).__init__(*a, **kwargs)
+
+ # TODO: use varint for [ir]_time?
+ # TODO: support other value types
+ cursor = self._client.cursor()
+ try:
+ res = cursor.execute('''CREATE TABLE IF NOT EXISTS %s (
+ name text,
+ interval text,
+ i_time bigint,
+ r_time bigint,
+ value %s,
+ PRIMARY KEY(name, interval, i_time, r_time)
+ )'''%(self._table, self._value_type))
+ except cql.ProgrammingError as pe:
+ if 'existing' not in str(pe):
+ raise
+ finally:
+ cursor.close()
+
+ def _insert_stmt(self, name, value, timestamp, interval, config):
+ '''Helper to generate the insert statement.'''
+ # Calculate the TTL and abort if inserting into the past
+ expire, ttl = config['expire'], config['ttl'](timestamp)
+ if expire and not ttl:
+ return None
+
+ i_time = config['i_calc'].to_bucket(timestamp)
+ if not config['coarse']:
+ r_time = config['r_calc'].to_bucket(timestamp)
+ else:
+ r_time = -1
+
+ # TODO: figure out escaping rules of CQL
+ table_spec = self._table
+ if ttl:
+ table_spec += " USING TTL %s "%(ttl)
+ stmt = '''UPDATE %s SET value = %s
+ WHERE name = '%s'
+ AND interval = '%s'
+ AND i_time = %s
+ AND r_time = %s'''%(table_spec, value, name, interval, i_time, r_time)
+ return stmt
+
+ @scoped_connection
+ def _type_get(self, connection, name, interval, i_bucket, i_end=None):
+ rval = OrderedDict()
+
+ # TODO: more efficient creation of query string
+ stmt = '''SELECT i_time, r_time, value
+ FROM %s
+ WHERE name = '%s' AND interval = '%s'
+ '''%(self._table, name, interval)
+ if i_end :
+ stmt += ' AND i_time >= %s AND i_time <= %s'%(i_bucket, i_end)
+ else:
+ stmt += ' AND i_time = %s'%(i_bucket)
+ stmt += ' ORDER BY interval, i_time, r_time'
+
+ cursor = connection.cursor()
+ try:
+ cursor.execute(stmt)
+ for row in cursor:
+ i_time, r_time, value = row
+ if r_time==-1:
+ r_time = None
+ rval.setdefault(i_time,OrderedDict())[r_time] = value
+ finally:
+ cursor.close()
+ return rval
+
+class CassandraSet(CassandraBackend, Set):
+
+ def __init__(self, *a, **kwargs):
+ self._table = 'sets'
+ super(CassandraSet,self).__init__(*a, **kwargs)
+
+ # TODO: use varint for [ir]_time?
+ # TODO: support other value types
+ cursor = self._client.cursor()
+ try:
+ res = cursor.execute('''CREATE TABLE IF NOT EXISTS %s (
+ name text,
+ interval text,
+ i_time bigint,
+ r_time bigint,
+ value %s,
+ PRIMARY KEY(name, interval, i_time, r_time, value)
+ )'''%(self._table, self._value_type))
+ except cql.ProgrammingError as pe:
+ if 'existing' not in str(pe):
+ raise
+ finally:
+ cursor.close()
+
+ def _insert_stmt(self, name, value, timestamp, interval, config):
+ '''Helper to generate the insert statement.'''
+ # Calculate the TTL and abort if inserting into the past
+ expire, ttl = config['expire'], config['ttl'](timestamp)
+ if expire and not ttl:
+ return None
+
+ i_time = config['i_calc'].to_bucket(timestamp)
+ if not config['coarse']:
+ r_time = config['r_calc'].to_bucket(timestamp)
+ else:
+ r_time = -1
+
+ # TODO: figure out escaping rules of CQL
+ stmt = '''INSERT INTO %s (name, interval, i_time, r_time, value)
+ VALUES ('%s', '%s', %s, %s, %s)'''%(self._table, name, interval, i_time, r_time, value)
+ expire = config['expire']
+ if ttl:
+ stmt += " USING TTL %s"%(ttl)
+ return stmt
+
+ @scoped_connection
+ def _type_get(self, connection, name, interval, i_bucket, i_end=None):
+ rval = OrderedDict()
+
+ # TODO: more efficient creation of query string
+ stmt = '''SELECT i_time, r_time, value
+ FROM %s
+ WHERE name = '%s' AND interval = '%s'
+ '''%(self._table, name, interval)
+ if i_end :
+ stmt += ' AND i_time >= %s AND i_time <= %s'%(i_bucket, i_end)
+ else:
+ stmt += ' AND i_time = %s'%(i_bucket)
+ stmt += ' ORDER BY interval, i_time, r_time'
+
+ cursor = connection.cursor()
+ try:
+ cursor.execute(stmt)
+ for row in cursor:
+ i_time, r_time, value = row
+ if r_time==-1:
+ r_time = None
+ rval.setdefault(i_time,OrderedDict()).setdefault(r_time,set()).add( value )
+ finally:
+ cursor.close()
+ return rval
diff --git a/kairos/exceptions.py b/kairos/exceptions.py
index f262d61..89d2eb8 100644
--- a/kairos/exceptions.py
+++ b/kairos/exceptions.py
@@ -1,5 +1,5 @@
'''
-Copyright (c) 2012-2013, Agora Games, LLC All rights reserved.
+Copyright (c) 2012-2017, Agora Games, LLC All rights reserved.
https://github.com/agoragames/kairos/blob/master/LICENSE.txt
'''
diff --git a/kairos/mongo_backend.py b/kairos/mongo_backend.py
index aacd9cd..baa7ee4 100644
--- a/kairos/mongo_backend.py
+++ b/kairos/mongo_backend.py
@@ -1,9 +1,10 @@
'''
-Copyright (c) 2012-2013, Agora Games, LLC All rights reserved.
+Copyright (c) 2012-2017, Agora Games, LLC All rights reserved.
https://github.com/agoragames/kairos/blob/master/LICENSE.txt
'''
-from exceptions import *
+from .exceptions import *
+from .timeseries import *
import operator
import sys
@@ -12,8 +13,7 @@
import pymongo
from pymongo import ASCENDING, DESCENDING
from datetime import datetime
-
-from timeseries import *
+from urlparse import *
class MongoBackend(Timeseries):
'''
@@ -31,6 +31,7 @@ def __new__(cls, *args, **kwargs):
return MongoCount.__new__(MongoCount, *args, **kwargs)
elif ttype=='gauge':
return MongoGauge.__new__(MongoGauge, *args, **kwargs)
+ raise NotImplementedError("No implementation for %s types"%(ttype))
return Timeseries.__new__(cls, *args, **kwargs)
def __init__(self, client, **kwargs):
@@ -42,35 +43,126 @@ def __init__(self, client, **kwargs):
elif not isinstance(client, pymongo.database.Database):
raise TypeError('Mongo handle must be MongoClient or database instance')
+ self._escape_character = kwargs.get('escape_character', u"\U0000FFFF")
super(MongoBackend,self).__init__(client, **kwargs)
-
+
# Define the indices for lookups and TTLs
- for interval,config in self._intervals.iteritems():
+ for interval,config in self._intervals.items():
# TODO: the interval+(resolution+)name combination should be unique,
# but should the index be defined as such? Consider performance vs.
- # correctness tradeoff. Also will need to determine if we need to
+ # correctness tradeoff. Also will need to determine if we need to
# maintain this multikey index or if there's a better way to implement
# all the features. Lastly, determine if it's better to add 'value'
# to the index and spec the fields in get() and series() so that we
# get covered indices. There are reasons why that might be a
# configuration option (performance vs. memory tradeoff)
if config['coarse']:
- self._client[interval].ensure_index(
+ self._client[interval].ensure_index(
[('interval',ASCENDING),('name',ASCENDING)], background=True )
else:
- self._client[interval].ensure_index(
+ self._client[interval].ensure_index(
[('interval',ASCENDING),('resolution',ASCENDING),('name',ASCENDING)],
background=True )
if config['expire']:
- self._client[interval].ensure_index(
+ self._client[interval].ensure_index(
[('expire_from',ASCENDING)], expireAfterSeconds=config['expire'], background=True )
- def _insert(self, name, value, timestamp):
+ @classmethod
+ def url_parse(self, url, **kwargs):
+ location = urlparse(url)
+ if location.scheme == 'mongodb':
+ client = pymongo.MongoClient( url, **kwargs )
+
+ # Stupid urlparse has a "does this scheme use queries" registrar,
+ # so copy that work here. Then pull out the optional database name.
+ path = location.path
+ if '?' in path:
+ path = path.split('?',1)[0]
+ path = re.search('[/]*([\w]*)', path).groups()[0] or kwargs.get('database','kairos')
+
+ return client[ path ]
+
+ # A very ugly way to capture histogram updates
+ @property
+ def _single_value(self):
+ return True
+
+ def _unescape(self, value):
+ '''
+ Recursively unescape values. Though slower, this doesn't require the user to
+ know anything about the escaping when writing their own custom fetch functions.
+ '''
+ if isinstance(value, (str,unicode)):
+ return value.replace(self._escape_character, '.')
+ elif isinstance(value, dict):
+ return { self._unescape(k) : self._unescape(v) for k,v in value.items() }
+ elif isinstance(value, list):
+ return [ self._unescape(v) for v in value ]
+ return value
+
+ def list(self):
+ rval = set()
+ for interval,config in self._intervals.items():
+ rval.update( self._client.command({'distinct':interval, 'key':'name'})['values'] )
+ return list(rval)
+
+ def properties(self, name):
+ rval = {}
+ for interval,config in self._intervals.items():
+ rval.setdefault(interval, {})
+ query = {'name':name}
+ res = self._client[interval].find_one(query, sort=[('interval',ASCENDING)])
+ rval[interval]['first'] = config['i_calc'].from_bucket(res['interval'])
+ res = self._client[interval].find_one(query, sort=[('interval',DESCENDING)])
+ rval[interval]['last'] = config['i_calc'].from_bucket(res['interval'])
+
+ return rval
+
+ def _batch_key(self, query):
+ '''
+ Get a unique id from a query.
+ '''
+ return ''.join( ['%s%s'%(k,v) for k,v in sorted(query.items())] )
+
+ def _batch_insert(self, inserts, intervals, **kwargs):
+ '''
+ Batch insert implementation.
+ '''
+ updates = {}
+ # TODO support flush interval
+ for interval,config in self._intervals.items():
+ for timestamp,names in inserts.iteritems():
+ timestamps = self._normalize_timestamps(timestamp, intervals, config)
+ for name,values in names.iteritems():
+ for value in values:
+ for tstamp in timestamps:
+ query,insert = self._insert_data(
+ name, value, tstamp, interval, config, dry_run=True)
+
+ batch_key = self._batch_key(query)
+ updates.setdefault(batch_key, {'query':query, 'interval':interval})
+ new_insert = self._batch(insert, updates[batch_key].get('insert'))
+ updates[batch_key]['insert'] = new_insert
+
+ # now that we've collected a bunch of updates, flush them out
+ for spec in updates.values():
+ self._client[ spec['interval'] ].update(
+ spec['query'], spec['insert'], upsert=True, check_keys=False )
+
+ def _insert(self, name, value, timestamp, intervals, **kwargs):
'''
Insert the new value.
'''
+ # TODO: confirm that this is in fact using the indices correctly.
+ for interval,config in self._intervals.items():
+ timestamps = self._normalize_timestamps(timestamp, intervals, config)
+ for tstamp in timestamps:
+ self._insert_data(name, value, tstamp, interval, config, **kwargs)
+
+ def _insert_data(self, name, value, timestamp, interval, config, **kwargs):
+ '''Helper to insert data into mongo.'''
# Mongo does not allow mixing atomic modifiers and non-$set sets in the
- # same update, so the choice is to either run the first upsert on
+ # same update, so the choice is to either run the first upsert on
# {'_id':id} to ensure the record is in place followed by an atomic update
# based on the series type, or use $set for all of the keys to be used in
# creating the record, which then disallows setting our own _id because
@@ -81,66 +173,91 @@ def _insert(self, name, value, timestamp):
# overhead of that index match vs. the presumably-faster match on _id
# * Yet another index for the would-be _id of i_key or r_key, where each
# index has a notable impact on write performance.
- # For now, choosing to go with matching on the tuple until performance
+ # For now, choosing to go with matching on the tuple until performance
# testing can be done. Even then, there may be a variety of factors which
# make the results situation-dependent.
- # TODO: confirm that this is in fact using the indices correctly.
- for interval,config in self._intervals.iteritems():
- insert = {'name':name, 'interval':config['i_calc'].to_bucket(timestamp)}
- if not config['coarse']:
- insert['resolution'] = config['r_calc'].to_bucket(timestamp)
- query = insert.copy()
+ insert = {'name':name, 'interval':config['i_calc'].to_bucket(timestamp)}
+ if not config['coarse']:
+ insert['resolution'] = config['r_calc'].to_bucket(timestamp)
+ # copy the query before expire_from as that is not indexed
+ query = insert.copy()
- if config['expire']:
- insert['expire_from'] = datetime.utcfromtimestamp( timestamp )
+ if config['expire']:
+ insert['expire_from'] = datetime.utcfromtimestamp( timestamp )
- # switch to atomic updates
- insert = {'$set':insert.copy()}
- self._insert_type( insert, value )
-
- # TODO: use write preference settings if we have them
+ # switch to atomic updates
+ insert = {'$set':insert.copy()}
+
+ # need to hide the period of any values. best option seems to be to pick
+ # a character that "no one" uses.
+ if isinstance(value, (str,unicode)):
+ value = value.replace('.', self._escape_character)
+ elif isinstance(value, float):
+ value = str(value).replace('.', self._escape_character)
+
+ self._insert_type( insert, value )
+
+ # TODO: use write preference settings if we have them
+ if not kwargs.get('dry_run',False):
self._client[interval].update( query, insert, upsert=True, check_keys=False )
+ return query, insert
- def _get(self, name, interval, config, timestamp):
+ def _get(self, name, interval, config, timestamp, **kws):
'''
Get the interval.
'''
i_bucket = config['i_calc'].to_bucket(timestamp)
+ fetch = kws.get('fetch')
+ process_row = kws.get('process_row') or self._process_row
rval = OrderedDict()
query = {'name':name, 'interval':i_bucket}
if config['coarse']:
- record = self._client[interval].find_one( query )
+ if fetch:
+ record = fetch( self._client[interval], spec=query, method='find_one' )
+ else:
+ record = self._client[interval].find_one( query )
+
if record:
- data = self._process_row( record['value'] )
+ data = process_row( self._unescape(record['value']) )
rval[ config['i_calc'].from_bucket(i_bucket) ] = data
else:
rval[ config['i_calc'].from_bucket(i_bucket) ] = self._type_no_value()
else:
sort = [('interval', ASCENDING), ('resolution', ASCENDING) ]
- cursor = self._client[interval].find( spec=query, sort=sort )
+ if fetch:
+ cursor = fetch( self._client[interval], spec=query, sort=sort, method='find' )
+ else:
+ cursor = self._client[interval].find( spec=query, sort=sort )
idx = 0
for record in cursor:
rval[ config['r_calc'].from_bucket(record['resolution']) ] = \
- self._process_row(record['value'])
+ process_row(record['value'])
return rval
- def _series(self, name, interval, config, buckets):
+ def _series(self, name, interval, config, buckets, **kws):
'''
Fetch a series of buckets.
'''
+ # make a copy of the buckets because we're going to mutate it
+ buckets = list(buckets)
rval = OrderedDict()
step = config['step']
resolution = config.get('resolution',step)
-
+ fetch = kws.get('fetch')
+ process_row = kws.get('process_row', self._process_row)
+
query = { 'name':name, 'interval':{'$gte':buckets[0], '$lte':buckets[-1]} }
sort = [('interval', ASCENDING)]
if not config['coarse']:
sort.append( ('resolution', ASCENDING) )
-
- cursor = self._client[interval].find( spec=query, sort=sort )
+
+ if fetch:
+ cursor = fetch( self._client[interval], spec=query, sort=sort, method='find' )
+ else:
+ cursor = self._client[interval].find( spec=query, sort=sort )
for record in cursor:
while buckets and buckets[0] < record['interval']:
rval[ config['i_calc'].from_bucket(buckets.pop(0)) ] = self._type_no_value()
@@ -148,15 +265,19 @@ def _series(self, name, interval, config, buckets):
buckets.pop(0)
i_key = config['i_calc'].from_bucket(record['interval'])
- data = self._process_row( record['value'] )
+ data = process_row( record['value'] )
if config['coarse']:
rval[ i_key ] = data
else:
rval.setdefault( i_key, OrderedDict() )
rval[ i_key ][ config['r_calc'].from_bucket(record['resolution']) ] = data
+ # are there buckets at the end for which we received no data?
+ while buckets:
+ rval[ config['i_calc'].from_bucket(buckets.pop(0)) ] = self._type_no_value()
+
return rval
-
+
def delete(self, name):
'''
Delete time series by name across all intervals. Returns the number of
@@ -165,40 +286,57 @@ def delete(self, name):
# TODO: confirm that this does not use the combo index and determine
# performance implications.
num_deleted = 0
- for interval,config in self._intervals.iteritems():
+ for interval,config in self._intervals.items():
# TODO: use write preference settings if we have them
num_deleted += self._client[interval].remove( {'name':name} )['n']
return num_deleted
class MongoSeries(MongoBackend, Series):
-
+
+ def _batch(self, insert, existing):
+ if not existing:
+ insert['$push'] = {'value':{'$each':[ insert['$push']['value'] ]}}
+ return insert
+
+ existing['$push']['value']['$each'].append( insert.pop('$push')['value'] )
+ return existing
+
def _insert_type(self, spec, value):
spec['$push'] = {'value':value}
- def _type_no_value(self):
- return []
-
class MongoHistogram(MongoBackend, Histogram):
-
+
+ def _batch(self, insert, existing):
+ if not existing:
+ return insert
+
+ for value,incr in insert['$inc'].iteritems():
+ existing['$inc'][value] = existing['$inc'].get(value,0)+incr
+ return existing
+
def _insert_type(self, spec, value):
spec['$inc'] = {'value.%s'%(value): 1}
- def _type_no_value(self):
- return {}
-
class MongoCount(MongoBackend, Count):
-
+
+ def _batch(self, insert, existing):
+ if not existing:
+ return insert
+
+ existing['$inc']['value'] += insert['$inc']['value']
+ return existing
+
def _insert_type(self, spec, value):
spec['$inc'] = {'value':value}
- def _type_no_value(self):
- return 0
-
class MongoGauge(MongoBackend, Gauge):
-
+
+ def _batch(self, insert, existing):
+ if not existing:
+ return insert
+
+ existing['$set']['value'] = insert['$set']['value']
+ return existing
+
def _insert_type(self, spec, value):
spec['$set']['value'] = value
-
- def _type_no_value(self):
- # TODO: resolve this disconnect with redis backend
- return 0
diff --git a/kairos/redis_backend.py b/kairos/redis_backend.py
index 9130ba8..b3b4ee2 100644
--- a/kairos/redis_backend.py
+++ b/kairos/redis_backend.py
@@ -1,16 +1,17 @@
'''
-Copyright (c) 2012-2013, Agora Games, LLC All rights reserved.
+Copyright (c) 2012-2017, Agora Games, LLC All rights reserved.
https://github.com/agoragames/kairos/blob/master/LICENSE.txt
'''
-from exceptions import *
+from .exceptions import *
+from .timeseries import *
import operator
import sys
import time
import re
-
-from timeseries import *
+from urlparse import *
+from redis import Redis
class RedisBackend(Timeseries):
'''
@@ -28,6 +29,9 @@ def __new__(cls, *args, **kwargs):
return RedisCount.__new__(RedisCount, *args, **kwargs)
elif ttype=='gauge':
return RedisGauge.__new__(RedisGauge, *args, **kwargs)
+ elif ttype=='set':
+ return RedisSet.__new__(RedisSet, *args, **kwargs)
+ raise NotImplementedError("No implementation for %s types"%(ttype))
return Timeseries.__new__(cls, *args, **kwargs)
def __init__(self, client, **kwargs):
@@ -38,6 +42,12 @@ def __init__(self, client, **kwargs):
super(RedisBackend,self).__init__( client, **kwargs )
+ @classmethod
+ def url_parse(self, url, **kwargs):
+ location = urlparse(url)
+ if location.scheme == 'redis':
+ return Redis.from_url( url, **kwargs )
+
def _calc_keys(self, config, name, timestamp):
'''
Calculate keys given a stat name and timestamp.
@@ -50,33 +60,117 @@ def _calc_keys(self, config, name, timestamp):
return i_bucket, r_bucket, i_key, r_key
- def _insert(self, name, value, timestamp):
+ def list(self):
+ keys = self._client.keys()
+ rval = set()
+ for key in keys:
+ key = key[len(self._prefix):]
+ rval.add( key.split(':')[0] )
+ return list(rval)
+
+ def properties(self, name):
+ prefix = '%s%s:'%(self._prefix,name)
+ keys = self._client.keys('%s*'%(prefix))
+ rval = {}
+
+ for key in keys:
+ key = key[len(prefix):].split(':')
+ rval.setdefault( key[0], {} )
+ if 'first' in rval[key[0]]:
+ rval[key[0]]['first'] = min(rval[key[0]]['first'], int(key[1]))
+ else:
+ rval[key[0]]['first'] = int(key[1])
+ if 'last' in rval[key[0]]:
+ rval[key[0]]['last'] = max(rval[key[0]]['last'], int(key[1]))
+ else:
+ rval[key[0]]['last'] = int(key[1])
+
+ for interval, properties in rval.items():
+ # It's possible that there is data stored for which an interval
+ # is not defined.
+ if interval in self._intervals:
+ config = self._intervals[interval]
+ properties['first'] = config['i_calc'].from_bucket( properties['first'] )
+ properties['last'] = config['i_calc'].from_bucket( properties['last'] )
+ else:
+ rval.pop(interval)
+
+ return rval
+
+ def _batch_insert(self, inserts, intervals, **kwargs):
+ '''
+ Specialized batch insert
+ '''
+ if 'pipeline' in kwargs:
+ pipe = kwargs.get('pipeline')
+ own_pipe = False
+ else:
+ pipe = self._client.pipeline(transaction=False)
+ kwargs['pipeline'] = pipe
+ own_pipe = True
+
+ ttl_batch = set()
+ for timestamp,names in inserts.iteritems():
+ for name,values in names.iteritems():
+ for value in values:
+ # TODO: support config param to flush the pipe every X inserts
+ self._insert( name, value, timestamp, intervals, ttl_batch=ttl_batch, **kwargs )
+
+ for ttl_args in ttl_batch:
+ pipe.expire(*ttl_args)
+
+ if own_pipe:
+ kwargs['pipeline'].execute()
+
+ def _insert(self, name, value, timestamp, intervals, **kwargs):
'''
Insert the value.
'''
- pipe = self._client.pipeline(transaction=False)
- # TODO: apply the prefix if we're using one.
+ if 'pipeline' in kwargs:
+ pipe = kwargs.get('pipeline')
+ else:
+ pipe = self._client.pipeline(transaction=False)
for interval,config in self._intervals.iteritems():
- i_bucket, r_bucket, i_key, r_key = self._calc_keys(config, name, timestamp)
-
- if config['coarse']:
- self._type_insert(pipe, i_key, value)
- else:
- # Add the resolution bucket to the interval. This allows us to easily
- # discover the resolution intervals within the larger interval, and
- # if there is a cap on the number of steps, it will go out of scope
- # along with the rest of the data
- pipe.sadd(i_key, r_bucket)
- self._type_insert(pipe, r_key, value)
-
- expire = config['expire']
- if expire:
- pipe.expire(i_key, expire)
- if not config['coarse']:
- pipe.expire(r_key, expire)
+ timestamps = self._normalize_timestamps(timestamp, intervals, config)
+ for tstamp in timestamps:
+ self._insert_data(name, value, tstamp, interval, config, pipe,
+ ttl_batch=kwargs.get('ttl_batch'))
- pipe.execute()
+ if 'pipeline' not in kwargs:
+ pipe.execute()
+
+ def _insert_data(self, name, value, timestamp, interval, config, pipe, ttl_batch=None):
+ '''Helper to insert data into redis'''
+ # Calculate the TTL and abort if inserting into the past
+ expire, ttl = config['expire'], config['ttl'](timestamp)
+ if expire and not ttl:
+ return
+
+ i_bucket, r_bucket, i_key, r_key = self._calc_keys(config, name, timestamp)
+
+ if config['coarse']:
+ self._type_insert(pipe, i_key, value)
+ else:
+ # Add the resolution bucket to the interval. This allows us to easily
+ # discover the resolution intervals within the larger interval, and
+ # if there is a cap on the number of steps, it will go out of scope
+ # along with the rest of the data
+ pipe.sadd(i_key, r_bucket)
+ self._type_insert(pipe, r_key, value)
+
+ if expire:
+ ttl_args = (i_key, ttl)
+ if ttl_batch is not None:
+ ttl_batch.add(ttl_args)
+ else:
+ pipe.expire(*ttl_args)
+ if not config['coarse']:
+ ttl_args = (r_key, ttl)
+ if ttl_batch is not None:
+ ttl_batch.add(ttl_args)
+ else:
+ pipe.expire(*ttl_args)
def delete(self, name):
'''
@@ -93,15 +187,17 @@ def delete(self, name):
# enough approximation
return len(keys)
- def _get(self, name, interval, config, timestamp):
+ def _get(self, name, interval, config, timestamp, **kws):
'''
Fetch a single interval from redis.
'''
i_bucket, r_bucket, i_key, r_key = self._calc_keys(config, name, timestamp)
-
- rval = OrderedDict()
+ fetch = kws.get('fetch') or self._type_get
+ process_row = kws.get('process_row') or self._process_row
+
+ rval = OrderedDict()
if config['coarse']:
- data = self._process_row( self._type_get(self._client, i_key) )
+ data = process_row( fetch(self._client, i_key) )
rval[ config['i_calc'].from_bucket(i_bucket) ] = data
else:
# First fetch all of the resolution buckets for this set.
@@ -112,29 +208,31 @@ def _get(self, name, interval, config, timestamp):
pipe = self._client.pipeline(transaction=False)
for bucket in resolution_buckets:
r_key = '%s:%s'%(i_key, bucket) # TODO: make this the "resolution_bucket" closure?
- self._type_get(pipe, r_key)
+ fetch(pipe, r_key)
res = pipe.execute()
for idx,data in enumerate(res):
- data = self._process_row(data)
+ data = process_row(data)
rval[ config['r_calc'].from_bucket(resolution_buckets[idx]) ] = data
return rval
- def _series(self, name, interval, config, buckets):
+ def _series(self, name, interval, config, buckets, **kws):
'''
Fetch a series of buckets.
'''
pipe = self._client.pipeline(transaction=False)
step = config['step']
resolution = config.get('resolution',step)
+ fetch = kws.get('fetch') or self._type_get
+ process_row = kws.get('process_row') or self._process_row
rval = OrderedDict()
for interval_bucket in buckets:
i_key = '%s%s:%s:%s'%(self._prefix, name, interval, interval_bucket)
if config['coarse']:
- self._type_get(pipe, i_key)
+ fetch(pipe, i_key)
else:
pipe.smembers(i_key)
res = pipe.execute()
@@ -146,7 +244,7 @@ def _series(self, name, interval, config, buckets):
interval_key = '%s%s:%s:%s'%(self._prefix, name, interval, interval_bucket)
if config['coarse']:
- data = self._process_row( data )
+ data = process_row( data )
rval[ config['i_calc'].from_bucket(interval_bucket) ] = data
else:
rval[ config['i_calc'].from_bucket(interval_bucket) ] = OrderedDict()
@@ -155,13 +253,13 @@ def _series(self, name, interval, config, buckets):
for bucket in resolution_buckets:
# TODO: use closures on the config for generating this resolution key
resolution_key = '%s:%s'%(interval_key, bucket)
- self._type_get(pipe, resolution_key)
-
+ fetch(pipe, resolution_key)
+
resolution_res = pipe.execute()
for x,data in enumerate(resolution_res):
i_t = config['i_calc'].from_bucket(interval_bucket)
r_t = config['r_calc'].from_bucket(resolution_buckets[x])
- rval[ i_t ][ r_t ] = self._process_row(data)
+ rval[ i_t ][ r_t ] = process_row(data)
return rval
@@ -201,17 +299,28 @@ def _type_insert(self, handle, key, value):
handle.incrbyfloat(key, value)
else:
handle.incr(key,value)
-
+
def _type_get(self, handle, key):
return handle.get(key)
class RedisGauge(RedisBackend, Gauge):
-
+
def _type_insert(self, handle, key, value):
'''
Insert the value into the series.
'''
handle.set(key, value)
-
+
def _type_get(self, handle, key):
return handle.get(key)
+
+class RedisSet(RedisBackend, Set):
+
+ def _type_insert(self, handle, key, value):
+ '''
+ Insert the value into the series.
+ '''
+ handle.sadd(key, value)
+
+ def _type_get(self, handle, key):
+ return handle.smembers(key)
diff --git a/kairos/sql_backend.py b/kairos/sql_backend.py
new file mode 100644
index 0000000..d0695fd
--- /dev/null
+++ b/kairos/sql_backend.py
@@ -0,0 +1,571 @@
+'''
+Copyright (c) 2012-2017, Agora Games, LLC All rights reserved.
+
+https://github.com/agoragames/kairos/blob/master/LICENSE.txt
+'''
+from .exceptions import *
+from .timeseries import *
+
+from sqlalchemy.types import TypeEngine
+from sqlalchemy import Table, Column, BigInteger, Integer, String, Unicode, Text, LargeBinary, Float, Boolean, Time, Date, DateTime, Numeric, MetaData, UniqueConstraint, create_engine
+from sqlalchemy.sql import select, update, insert, distinct, asc, desc, and_, or_, not_
+
+import time
+from datetime import date, datetime
+from datetime import time as time_type
+from decimal import Decimal
+from urlparse import *
+
+# Test python3 compatibility
+try:
+ x = long(1)
+except NameError:
+ long = int
+try:
+ x = unicode('foo')
+except NameError:
+ unicode = str
+
+TYPE_MAP = {
+ str : String,
+ 'str' : String,
+ 'string' : String,
+
+ unicode : Unicode, # works for py3 too
+ 'unicode' : Unicode,
+
+ float : Float,
+ 'float' : Float,
+
+ int : Integer,
+ 'int' : Integer,
+ 'integer' : Integer,
+
+ long : BigInteger, # works for py3 too
+ 'long' : BigInteger,
+ 'int64' : BigInteger,
+
+ bool : Boolean,
+ 'bool' : Boolean,
+ 'boolean' : Boolean,
+
+ date : Date,
+ 'date' : Date,
+ datetime : DateTime,
+ 'datetime' : DateTime,
+ time_type : Time,
+ 'time' : Time,
+
+ Decimal : Numeric,
+ 'decimal' : Numeric,
+
+ 'text' : Text,
+ 'clob' : Text,
+ 'blob' : LargeBinary,
+}
+
+class SqlBackend(Timeseries):
+
+ def __new__(cls, *args, **kwargs):
+ if cls==SqlBackend:
+ ttype = kwargs.pop('type', None)
+ if ttype=='series':
+ return SqlSeries.__new__(SqlSeries, *args, **kwargs)
+ elif ttype=='histogram':
+ return SqlHistogram.__new__(SqlHistogram, *args, **kwargs)
+ elif ttype=='count':
+ return SqlCount.__new__(SqlCount, *args, **kwargs)
+ elif ttype=='gauge':
+ return SqlGauge.__new__(SqlGauge, *args, **kwargs)
+ raise NotImplementedError("No implementation for %s types"%(ttype))
+ return Timeseries.__new__(cls, *args, **kwargs)
+
+ @classmethod
+ def url_parse(self, url, **kwargs):
+ location = urlparse(url)
+ if 'sql' in location.scheme:
+ return create_engine( url, **kwargs )
+
+ def __init__(self, client, **kwargs):
+ '''
+ Initialize the sql backend after timeseries has processed the configuration.
+ '''
+ self._metadata = MetaData()
+ self._str_length = kwargs.get('string_length',255)
+ self._txt_length = kwargs.get('text_length', 32*1024)
+
+ vtype = kwargs.get('value_type', float)
+ if vtype in TYPE_MAP:
+ self._value_type = TYPE_MAP[vtype]
+ if self._value_type == String:
+ self._value_type = String(self._str_length)
+ elif self._value_type == Text:
+ self._value_type = Text(self._txt_length)
+ elif self._value_type == LargeBinary:
+ self._value_type = LargeBinary(self._txt_length)
+
+ elif issubclass(vtype, TypeEngine):
+ if vtype == String:
+ self._value_type = String(self._str_length)
+ elif vtype == Text:
+ self._value_type = Text(self._txt_length)
+ elif vtype == LargeBinary:
+ self._value_type = LargeBinary(self._txt_length)
+
+ elif isinstance(vtype, TypeEngine):
+ self._value_type = vtype
+
+ else:
+ raise ValueError("Unsupported type '%s'"%(vtype))
+
+ self._table_name = kwargs.get('table_name', self._table_name)
+
+ super(SqlBackend,self).__init__(client, **kwargs)
+
+ def list(self):
+ connection = self._client.connect()
+ rval = set()
+ stmt = select([distinct(self._table.c.name)])
+
+ for row in connection.execute(stmt):
+ rval.add(row['name'])
+ return list(rval)
+
+ def properties(self, name):
+ connection = self._client.connect()
+ rval = {}
+
+ for interval,config in self._intervals.items():
+ rval.setdefault(interval, {})
+
+ stmt = select([self._table.c.i_time]).where(
+ and_(
+ self._table.c.name==name,
+ self._table.c.interval==interval
+ )
+ ).order_by( asc(self._table.c.i_time) ).limit(1)
+ rval[interval]['first'] = config['i_calc'].from_bucket(
+ connection.execute(stmt).first()['i_time'] )
+
+ stmt = select([self._table.c.i_time]).where(
+ and_(
+ self._table.c.name==name,
+ self._table.c.interval==interval
+ )
+ ).order_by( desc(self._table.c.i_time) ).limit(1)
+ rval[interval]['last'] = config['i_calc'].from_bucket(
+ connection.execute(stmt).first()['i_time'] )
+
+ return rval
+
+ def expire(self, name):
+ '''
+ Expire all the data.
+ '''
+ for interval,config in self._intervals.items():
+ if config['expire']:
+ # Because we're storing the bucket time, expiry has the same
+ # "skew" as whatever the buckets are.
+ expire_from = config['i_calc'].to_bucket(time.time() - config['expire'])
+ conn = self._client.connect()
+
+ conn.execute( self._table.delete().where(
+ and_(
+ self._table.c.name==name,
+ self._table.c.interval==interval,
+ self._table.c.i_time<=expire_from
+ )
+ ))
+
+ def _insert(self, name, value, timestamp, intervals, **kwargs):
+ '''
+ Insert the new value.
+ '''
+ for interval,config in self._intervals.items():
+ timestamps = self._normalize_timestamps(timestamp, intervals, config)
+ for tstamp in timestamps:
+ self._insert_data(name, value, tstamp, interval, config, **kwargs)
+
+ def _get(self, name, interval, config, timestamp, **kws):
+ '''
+ Get the interval.
+ '''
+ i_bucket = config['i_calc'].to_bucket(timestamp)
+ fetch = kws.get('fetch')
+ process_row = kws.get('process_row') or self._process_row
+
+ rval = OrderedDict()
+ if fetch:
+ data = fetch( self._client.connect(), self._table, name, interval, i_bucket )
+ else:
+ data = self._type_get(name, interval, i_bucket)
+
+ if config['coarse']:
+ if data:
+ rval[ config['i_calc'].from_bucket(i_bucket) ] = process_row(data.values()[0][None])
+ else:
+ rval[ config['i_calc'].from_bucket(i_bucket) ] = self._type_no_value()
+ else:
+ for r_bucket,row_data in data.values()[0].items():
+ rval[ config['r_calc'].from_bucket(r_bucket) ] = process_row(row_data)
+
+ return rval
+
+ def _series(self, name, interval, config, buckets, **kws):
+ '''
+ Fetch a series of buckets.
+ '''
+ fetch = kws.get('fetch')
+ process_row = kws.get('process_row') or self._process_row
+
+ rval = OrderedDict()
+
+ if fetch:
+ data = fetch( self._client.connect(), self._table, name, interval, buckets[0], buckets[-1] )
+ else:
+ data = self._type_get(name, interval, buckets[0], buckets[-1])
+
+ if config['coarse']:
+ for i_bucket in buckets:
+ i_key = config['i_calc'].from_bucket(i_bucket)
+ i_data = data.get( i_bucket )
+ if i_data:
+ rval[ i_key ] = process_row( i_data[None] )
+ else:
+ rval[ i_key ] = self._type_no_value()
+ else:
+ if data:
+ for i_bucket, i_data in data.items():
+ i_key = config['i_calc'].from_bucket(i_bucket)
+ rval[i_key] = OrderedDict()
+ for r_bucket, r_data in i_data.items():
+ r_key = config['r_calc'].from_bucket(r_bucket)
+ if r_data:
+ rval[i_key][r_key] = process_row(r_data)
+ else:
+ rval[i_key][r_key] = self._type_no_value()
+
+ return rval
+
+ def delete(self, name):
+ '''
+ Delete time series by name across all intervals. Returns the number of
+ records deleted.
+ '''
+ conn = self._client.connect()
+ conn.execute( self._table.delete().where(self._table.c.name==name) )
+
+class SqlSeries(SqlBackend, Series):
+
+ def __init__(self, *a, **kwargs):
+ # TODO: define indices
+ # TODO: optionally create separate tables for each interval, like mongo?
+ self._table_name = 'series'
+ super(SqlSeries,self).__init__(*a, **kwargs)
+ self._table = Table(self._table_name, self._metadata,
+ Column('name', String(self._str_length), nullable=False), # stat name
+ Column('interval', String(self._str_length), nullable=False), # interval name
+ Column('insert_time', Float, nullable=False), # to preserve order
+ Column('i_time', Integer, nullable=False), # interval timestamp
+ Column('r_time', Integer, nullable=True), # resolution timestamp
+ Column('value', self._value_type, nullable=False) # datas
+ )
+ self._metadata.create_all(self._client)
+
+ def _insert_data(self, name, value, timestamp, interval, config, **kwargs):
+ '''Helper to insert data into sql.'''
+ kwargs = {
+ 'name' : name,
+ 'interval' : interval,
+ 'insert_time' : time.time(),
+ 'i_time' : config['i_calc'].to_bucket(timestamp),
+ 'value' : value
+ }
+ if not config['coarse']:
+ kwargs['r_time'] = config['r_calc'].to_bucket(timestamp)
+ stmt = self._table.insert().values(**kwargs)
+ conn = self._client.connect()
+ result = conn.execute(stmt)
+
+ def _type_get(self, name, interval, i_bucket, i_end=None):
+ connection = self._client.connect()
+ rval = OrderedDict()
+ stmt = self._table.select()
+
+ if i_end:
+ stmt = stmt.where(
+ and_(
+ self._table.c.name==name,
+ self._table.c.interval==interval,
+ self._table.c.i_time>=i_bucket,
+ self._table.c.i_time<=i_end,
+ )
+ )
+ else:
+ stmt = stmt.where(
+ and_(
+ self._table.c.name==name,
+ self._table.c.interval==interval,
+ self._table.c.i_time==i_bucket,
+ )
+ )
+ stmt = stmt.order_by( self._table.c.r_time, self._table.c.insert_time )
+
+ for row in connection.execute(stmt):
+ rval.setdefault(row['i_time'],OrderedDict()).setdefault(row['r_time'],[]).append( row['value'] )
+ return rval
+
+class SqlHistogram(SqlBackend, Histogram):
+
+ def __init__(self, *a, **kwargs):
+ # TODO: define indices
+ # TODO: optionally create separate tables for each interval, like mongo?
+ self._table_name = 'histogram'
+ super(SqlHistogram,self).__init__(*a, **kwargs)
+ self._table = Table(self._table_name, self._metadata,
+ Column('name', String(self._str_length), nullable=False), # stat name
+ Column('interval', String(self._str_length), nullable=False), # interval name
+ Column('i_time', Integer, nullable=False), # interval timestamp
+ Column('r_time', Integer, nullable=True), # resolution timestamp
+ Column('value', self._value_type, nullable=False), # histogram keys
+ Column('count', Integer, nullable=False), # key counts
+
+ # Use a constraint for transaction-less insert vs update
+ UniqueConstraint('name', 'interval', 'i_time', 'r_time', 'value', name='unique_value')
+ )
+ self._metadata.create_all(self._client)
+
+ def _insert_data(self, name, value, timestamp, interval, config, **kwargs):
+ '''Helper to insert data into sql.'''
+ conn = self._client.connect()
+ if not self._update_data(name, value, timestamp, interval, config, conn):
+ try:
+ kwargs = {
+ 'name' : name,
+ 'interval' : interval,
+ 'i_time' : config['i_calc'].to_bucket(timestamp),
+ 'value' : value,
+ 'count' : 1
+ }
+ if not config['coarse']:
+ kwargs['r_time'] = config['r_calc'].to_bucket(timestamp)
+ stmt = self._table.insert().values(**kwargs)
+ result = conn.execute(stmt)
+ except:
+ # TODO: only catch IntegrityError
+ if not self._update_data(name, value, timestamp, interval, config, conn):
+ raise
+
+ def _update_data(self, name, value, timestamp, interval, config, conn):
+ '''Support function for insert. Should be called within a transaction'''
+ i_time = config['i_calc'].to_bucket(timestamp)
+ if not config['coarse']:
+ r_time = config['r_calc'].to_bucket(timestamp)
+ else:
+ r_time = None
+ stmt = self._table.update().where(
+ and_(
+ self._table.c.name==name,
+ self._table.c.interval==interval,
+ self._table.c.i_time==i_time,
+ self._table.c.r_time==r_time,
+ self._table.c.value==value)
+ ).values({self._table.c.count: self._table.c.count + 1})
+ rval = conn.execute( stmt )
+ return rval.rowcount
+
+ def _type_get(self, name, interval, i_bucket, i_end=None):
+ connection = self._client.connect()
+ rval = OrderedDict()
+ stmt = self._table.select()
+
+ if i_end:
+ stmt = stmt.where(
+ and_(
+ self._table.c.name==name,
+ self._table.c.interval==interval,
+ self._table.c.i_time>=i_bucket,
+ self._table.c.i_time<=i_end,
+ )
+ )
+ else:
+ stmt = stmt.where(
+ and_(
+ self._table.c.name==name,
+ self._table.c.interval==interval,
+ self._table.c.i_time==i_bucket,
+ )
+ )
+ stmt = stmt.order_by( self._table.c.r_time )
+
+ for row in connection.execute(stmt):
+ rval.setdefault(row['i_time'],OrderedDict()).setdefault(row['r_time'],{})[row['value']] = row['count']
+ return rval
+
+class SqlCount(SqlBackend, Count):
+
+ def __init__(self, *a, **kwargs):
+ # TODO: define indices
+ # TODO: optionally create separate tables for each interval, like mongo?
+ self._table_name = 'count'
+ super(SqlCount,self).__init__(*a, **kwargs)
+ self._table = Table(self._table_name, self._metadata,
+ Column('name', String(self._str_length), nullable=False), # stat name
+ Column('interval', String(self._str_length), nullable=False), # interval name
+ Column('i_time', Integer, nullable=False), # interval timestamp
+ Column('r_time', Integer, nullable=True), # resolution timestamp
+ Column('count', Integer, nullable=False), # key counts
+
+ # Use a constraint for transaction-less insert vs update
+ UniqueConstraint('name', 'interval', 'i_time', 'r_time', name='unique_count')
+ )
+ self._metadata.create_all(self._client)
+
+ def _insert_data(self, name, value, timestamp, interval, config, **kwargs):
+ '''Helper to insert data into sql.'''
+ conn = self._client.connect()
+ if not self._update_data(name, value, timestamp, interval, config, conn):
+ try:
+ kwargs = {
+ 'name' : name,
+ 'interval' : interval,
+ 'i_time' : config['i_calc'].to_bucket(timestamp),
+ 'count' : value
+ }
+ if not config['coarse']:
+ kwargs['r_time'] = config['r_calc'].to_bucket(timestamp)
+ stmt = self._table.insert().values(**kwargs)
+ result = conn.execute(stmt)
+ except:
+ # TODO: only catch IntegrityError
+ if not self._update_data(name, value, timestamp, interval, config, conn):
+ raise
+
+ def _update_data(self, name, value, timestamp, interval, config, conn):
+ '''Support function for insert. Should be called within a transaction'''
+ i_time = config['i_calc'].to_bucket(timestamp)
+ if not config['coarse']:
+ r_time = config['r_calc'].to_bucket(timestamp)
+ else:
+ r_time = None
+ stmt = self._table.update().where(
+ and_(
+ self._table.c.name==name,
+ self._table.c.interval==interval,
+ self._table.c.i_time==i_time,
+ self._table.c.r_time==r_time)
+ ).values({self._table.c.count: self._table.c.count + value})
+ rval = conn.execute( stmt )
+ return rval.rowcount
+
+ def _type_get(self, name, interval, i_bucket, i_end=None):
+ connection = self._client.connect()
+ rval = OrderedDict()
+ stmt = self._table.select()
+
+ if i_end:
+ stmt = stmt.where(
+ and_(
+ self._table.c.name==name,
+ self._table.c.interval==interval,
+ self._table.c.i_time>=i_bucket,
+ self._table.c.i_time<=i_end,
+ )
+ )
+ else:
+ stmt = stmt.where(
+ and_(
+ self._table.c.name==name,
+ self._table.c.interval==interval,
+ self._table.c.i_time==i_bucket,
+ )
+ )
+ stmt = stmt.order_by( self._table.c.r_time )
+
+ for row in connection.execute(stmt):
+ rval.setdefault(row['i_time'],OrderedDict())[row['r_time']] = row['count']
+ return rval
+
+class SqlGauge(SqlBackend, Gauge):
+
+ def __init__(self, *a, **kwargs):
+ # TODO: define indices
+ # TODO: optionally create separate tables for each interval, like mongo?
+ self._table_name = 'gauge'
+ super(SqlGauge,self).__init__(*a, **kwargs)
+ self._table = Table(self._table_name, self._metadata,
+ Column('name', String(self._str_length), nullable=False), # stat name
+ Column('interval', String(self._str_length), nullable=False), # interval name
+ Column('i_time', Integer, nullable=False), # interval timestamp
+ Column('r_time', Integer, nullable=True), # resolution timestamp
+ Column('value', self._value_type, nullable=False), # key counts
+
+ # Use a constraint for transaction-less insert vs update
+ UniqueConstraint('name', 'interval', 'i_time', 'r_time', name='unique_count')
+ )
+ self._metadata.create_all(self._client)
+
+ def _insert_data(self, name, value, timestamp, interval, config, **kwargs):
+ '''Helper to insert data into sql.'''
+ conn = self._client.connect()
+ if not self._update_data(name, value, timestamp, interval, config, conn):
+ try:
+ kwargs = {
+ 'name' : name,
+ 'interval' : interval,
+ 'i_time' : config['i_calc'].to_bucket(timestamp),
+ 'value' : value
+ }
+ if not config['coarse']:
+ kwargs['r_time'] = config['r_calc'].to_bucket(timestamp)
+ stmt = self._table.insert().values(**kwargs)
+ result = conn.execute(stmt)
+ except:
+ # TODO: only catch IntegrityError
+ if not self._update_data(name, value, timestamp, interval, config, conn):
+ raise
+
+ def _update_data(self, name, value, timestamp, interval, config, conn):
+ '''Support function for insert. Should be called within a transaction'''
+ i_time = config['i_calc'].to_bucket(timestamp)
+ if not config['coarse']:
+ r_time = config['r_calc'].to_bucket(timestamp)
+ else:
+ r_time = None
+ stmt = self._table.update().where(
+ and_(
+ self._table.c.name==name,
+ self._table.c.interval==interval,
+ self._table.c.i_time==i_time,
+ self._table.c.r_time==r_time)
+ ).values({self._table.c.value: value})
+ rval = conn.execute( stmt )
+ return rval.rowcount
+
+ def _type_get(self, name, interval, i_bucket, i_end=None):
+ connection = self._client.connect()
+ rval = OrderedDict()
+ stmt = self._table.select()
+
+ if i_end:
+ stmt = stmt.where(
+ and_(
+ self._table.c.name==name,
+ self._table.c.interval==interval,
+ self._table.c.i_time>=i_bucket,
+ self._table.c.i_time<=i_end,
+ )
+ )
+ else:
+ stmt = stmt.where(
+ and_(
+ self._table.c.name==name,
+ self._table.c.interval==interval,
+ self._table.c.i_time==i_bucket,
+ )
+ )
+ stmt = stmt.order_by( self._table.c.r_time )
+
+ for row in connection.execute(stmt):
+ rval.setdefault(row['i_time'],OrderedDict())[row['r_time']] = row['value']
+ return rval
diff --git a/kairos/timeseries.py b/kairos/timeseries.py
index 254610d..4667cc7 100644
--- a/kairos/timeseries.py
+++ b/kairos/timeseries.py
@@ -1,15 +1,17 @@
'''
-Copyright (c) 2012-2013, Agora Games, LLC All rights reserved.
+Copyright (c) 2012-2017, Agora Games, LLC All rights reserved.
https://github.com/agoragames/kairos/blob/master/LICENSE.txt
'''
-from exceptions import *
-from datetime import datetime, timedelta
+from .exceptions import *
+from datetime import datetime, timedelta
import operator
import sys
import time
import re
+import warnings
+import functools
if sys.version_info[:2] > (2, 6):
from collections import OrderedDict
@@ -33,6 +35,12 @@
GREGORIAN_TIMES = set(['daily', 'weekly', 'monthly', 'yearly'])
+# Test python3 compatibility
+try:
+ x = long(1)
+except NameError:
+ long = int
+
def _resolve_time(value):
'''
Resolve the time in seconds of a configuration value.
@@ -62,6 +70,20 @@ class RelativeTime(object):
def __init__(self, step=1):
self._step = step
+ def step_size(self, t0=None, t1=None):
+ '''
+ Return the time in seconds of a step. If a begin and end timestamp,
+ return the time in seconds between them after adjusting for what buckets
+ they alias to. If t1 and t0 resolve to the same bucket,
+ '''
+ if t0!=None and t1!=None:
+ tb0 = self.to_bucket( t0 )
+ tb1 = self.to_bucket( t1, steps=1 ) # NOTE: "end" of second bucket
+ if tb0==tb1:
+ return self._step
+ return self.from_bucket( tb1 ) - self.from_bucket( tb0 )
+ return self._step
+
def to_bucket(self, timestamp, steps=0):
'''
Calculate the bucket from a timestamp, optionally including a step offset.
@@ -80,21 +102,32 @@ def buckets(self, start, end):
'''
start_bucket = self.to_bucket(start)
end_bucket = self.to_bucket(end)
- return range(start_bucket, end_bucket+1)
+ return range(start_bucket, end_bucket+1)
- def normalize(self, timestamp):
+ def normalize(self, timestamp, steps=0):
'''
Normalize a timestamp according to the interval configuration. Can
- optionally determine an offset
+ optionally determine an offset.
'''
- return self.from_bucket( self.to_bucket(timestamp) )
+ return self.from_bucket( self.to_bucket(timestamp, steps) )
- def ttl(self, steps):
+ def ttl(self, steps, relative_time=None):
'''
Return the ttl given the number of steps, None if steps is not defined
- or we're otherwise unable to calculate one.
+ or we're otherwise unable to calculate one. If relative_time is defined,
+ then return a ttl that is the number of seconds from now that the
+ record should be expired.
'''
if steps:
+ if relative_time:
+ rtime = self.to_bucket(relative_time)
+ ntime = self.to_bucket(time.time())
+ # The relative time is beyond our TTL cutoff
+ if (ntime - rtime) > steps:
+ return 0
+ # The relative time is in "recent" past or future
+ else:
+ return (steps + rtime - ntime) * self._step
return steps * self._step
return None
@@ -115,10 +148,26 @@ class GregorianTime(object):
'monthly' : '%Y%m',
'yearly' : '%Y'
}
-
+
def __init__(self, step='daily'):
self._step = step
+ def step_size(self, t0, t1=None):
+ '''
+ Return the time in seconds for each step. Requires that we know a time
+ relative to which we should calculate to account for variable length
+ intervals (e.g. February)
+ '''
+ tb0 = self.to_bucket( t0 )
+ if t1:
+ tb1 = self.to_bucket( t1, steps=1 ) # NOTE: "end" of second bucket
+ else:
+ tb1 = self.to_bucket( t0, steps=1 )
+
+ # Calculate the difference in days, then multiply by simple scalar
+ days = (self.from_bucket(tb1, native=True) - self.from_bucket(tb0, native=True)).days
+ return days * SIMPLE_TIMES['d']
+
def to_bucket(self, timestamp, steps=0):
'''
Calculate the bucket from a timestamp.
@@ -137,17 +186,23 @@ def to_bucket(self, timestamp, steps=0):
year += steps
dt = datetime(year=year, month=1, day=1)
- return dt.strftime( self.FORMATS[self._step] )
+ return int(dt.strftime( self.FORMATS[self._step] ))
- def from_bucket(self, bucket):
+ def from_bucket(self, bucket, native=False):
'''
Calculate the timestamp given a bucket.
'''
+ # NOTE: this is due to a bug somewhere in strptime that does not process
+ # the week number of '%Y%U' correctly. That bug could be very specific to
+ # the combination of python and ubuntu that I was testing.
+ bucket = str(bucket)
if self._step == 'weekly':
year, week = bucket[:4], bucket[4:]
normal = datetime(year=int(year), month=1, day=1) + timedelta(weeks=int(week))
else:
normal = datetime.strptime(bucket, self.FORMATS[self._step])
+ if native:
+ return normal
return long(time.mktime( normal.timetuple() ))
def buckets(self, start, end):
@@ -162,53 +217,89 @@ def buckets(self, start, end):
while True:
bucket = self.to_bucket(start, step)
bucket_time = self.from_bucket( bucket )
- if bucket_time > end:
+ if bucket_time >= end:
+ if bucket_time==end:
+ rval.append( bucket )
break
rval.append( bucket )
step += 1
- return rval
+ return rval
- def normalize(self, timestamp):
+ def normalize(self, timestamp, steps=0):
'''
- Normalize a timestamp according to the interval configuration.
+ Normalize a timestamp according to the interval configuration. Optionally
+ can be used to calculate the timestamp N steps away.
'''
# So far, the only commonality with RelativeTime
- return self.from_bucket( self.to_bucket(timestamp) )
+ return self.from_bucket( self.to_bucket(timestamp, steps) )
- def ttl(self, steps):
+ def ttl(self, steps, relative_time=None):
'''
Return the ttl given the number of steps, None if steps is not defined
- or we're otherwise unable to calculate one.
+ or we're otherwise unable to calculate one. If relative_time is defined,
+ then return a ttl that is the number of seconds from now that the
+ record should be expired.
'''
if steps:
- # Approximate the ttl based on number of seconds, since it's
- # "close enough"
+ # Approximate the ttl based on number of seconds, since it's
+ # "close enough"
+ if relative_time:
+ rtime = self.to_bucket(relative_time)
+ ntime = self.to_bucket(time.time())
+
+ # Convert to number of days
+ day_diff = (self.from_bucket(ntime, native=True) - self.from_bucket(rtime, native=True)).days
+ # Convert steps to number of days as well
+ step_diff = (steps*SIMPLE_TIMES[self._step[0]]) / SIMPLE_TIMES['d']
+
+ # The relative time is beyond our TTL cutoff
+ if day_diff > step_diff:
+ return 0
+ # The relative time is in "recent" past or future
+ else:
+ return (step_diff - day_diff) * SIMPLE_TIMES['d']
return steps * SIMPLE_TIMES[ self._step[0] ]
return None
+class TimeseriesMeta(type):
+ '''
+ Meta class for URL parsing
+ '''
+ def __call__(cls, client, **kwargs):
+ if isinstance(client, (str,unicode)):
+ for backend in BACKENDS.values():
+ handle = backend.url_parse(client, **kwargs.pop('client_config',{}))
+ if handle:
+ client = handle
+ break
+ if isinstance(client, (str,unicode)):
+ raise ImportError("Unsupported or unknown client type for %s", client)
+ return type.__call__(cls, client, **kwargs)
+
class Timeseries(object):
'''
Base class of all time series. Also acts as a factory to return the correct
subclass if "type=" keyword argument supplied.
'''
-
+ __metaclass__ = TimeseriesMeta
+
def __new__(cls, client, **kwargs):
if cls==Timeseries:
- # load a backend based on the name of the client module
+ # load a backend based on the name of the client module
client_module = client.__module__.split('.')[0]
backend = BACKENDS.get( client_module )
if backend:
return backend( client, **kwargs )
- else:
- raise ImportError("Unsupported or unknown client type %s", client_module)
+
+ raise ImportError("Unsupported or unknown client type %s", client_module)
return object.__new__(cls, client, **kwargs)
def __init__(self, client, **kwargs):
'''
Create a time series using a given redis client and keyword arguments
- defining the series configuration.
+ defining the series configuration.
Optionally provide a prefix for all keys. If prefix length>0 and it
doesn't end with ":", it will be automatically appended. Redis only.
@@ -243,13 +334,13 @@ def __init__(self, client, **kwargs):
object which can be cast to a string.
intervals
- Required, a dictionary of interval configurations in the form of:
+ Required, a dictionary of interval configurations in the form of:
{
# interval name, used in redis keys and should conform to best practices
# and not include ":"
minute: {
-
+
# Required. The number of seconds that the interval will cover
step: 60,
@@ -258,16 +349,16 @@ def __init__(self, client, **kwargs):
# exist in perpetuity.
steps: 240,
- # Optional. Defines the resolution of the data, i.e. the number of
+ # Optional. Defines the resolution of the data, i.e. the number of
# seconds in which data is assumed to have occurred "at the same time".
- # So if you're tracking a month long time series, you may only need
+ # So if you're tracking a month long time series, you may only need
# resolution down to the day, or resolution=86400. Defaults to same
# value as "step".
resolution: 60,
}
}
'''
- # Process the configuration first so that the backends can use that to
+ # Process the configuration first so that the backends can use that to
# complete their setup.
# Prefix is determined by the backend implementation.
self._client = client
@@ -276,12 +367,12 @@ def __init__(self, client, **kwargs):
self._intervals = kwargs.get('intervals', {})
# Preprocess the intervals
- for interval,config in self._intervals.iteritems():
+ for interval,config in self._intervals.items():
# Copy the interval name into the configuration, needed for redis
config['interval'] = interval
step = config['step'] = _resolve_time( config['step'] ) # Required
steps = config.get('steps',None) # Optional
- resolution = config['resolution'] = _resolve_time(
+ resolution = config['resolution'] = _resolve_time(
config.get('resolution',config['step']) ) # Optional
if step in GREGORIAN_TIMES:
@@ -293,39 +384,119 @@ def __init__(self, client, **kwargs):
resolution_calc = GregorianTime(resolution)
else:
resolution_calc = RelativeTime(resolution)
-
- expire = False
- if steps: expire = step*steps
-
+
config['i_calc'] = interval_calc
config['r_calc'] = resolution_calc
-
+
config['expire'] = interval_calc.ttl( steps )
+ config['ttl'] = functools.partial( interval_calc.ttl, steps )
config['coarse'] = (resolution==step)
- def insert(self, name, value, timestamp=None):
+ def list(self):
'''
- Insert a value for the timeseries "name". For each interval in the
+ List all of the stat names that are stored.
+ '''
+ raise NotImplementedError()
+
+ def properties(self, name):
+ '''
+ Get the properties of a stat.
+ '''
+ raise NotImplementedError()
+
+ def expire(self, name):
+ '''
+ Manually expire data for storage engines that do not support auto expiry.
+ '''
+ raise NotImplementedError()
+
+ def bulk_insert(self, inserts, intervals=0, **kwargs):
+ '''
+ Perform a bulk insert. The format of the inserts must be:
+
+ {
+ timestamp : {
+ name: [ values ],
+ ...
+ },
+ ...
+ }
+
+ If the timestamp should be auto-generated, then "timestamp" should be None.
+
+ Backends can implement this in any number of ways, the default being to
+ perform a single insert for each value, with no specialized locking,
+ transactions or batching.
+ '''
+ if None in inserts:
+ inserts[ time.time() ] = inserts.pop(None)
+ if self._write_func:
+ for timestamp,names in inserts.iteritems():
+ for name,values in names.iteritems():
+ names[name] = [ self._write_func(v) for v in values ]
+ self._batch_insert(inserts, intervals, **kwargs)
+
+ def insert(self, name, value, timestamp=None, intervals=0, **kwargs):
+ '''
+ Insert a value for the timeseries "name". For each interval in the
configuration, will insert the value into a bucket for the interval
"timestamp". If time is not supplied, will default to time.time(), else it
should be a floating point value.
+ If "intervals" is less than 0, inserts the value into timestamps
+ "abs(intervals)" preceeding "timestamp" (i.e. "-1" inserts one extra value).
+ If "intervals" is greater than 0, inserts the value into that many more
+ intervals after "timestamp". The default behavior is to insert for a single
+ timestamp.
+
This supports the public methods of the same name in the subclasses. The
- value is expected to already be converted
+ value is expected to already be converted.
'''
if not timestamp:
timestamp = time.time()
+
+ if isinstance(value, (list,tuple,set)):
+ if self._write_func:
+ value = [ self._write_func(v) for v in value ]
+ return self._batch_insert({timestamp:{name:value}}, intervals, **kwargs)
+
if self._write_func:
value = self._write_func(value)
# TODO: document acceptable names
# TODO: document what types values are supported
- # TODO: document behavior when time is outside the bounds of step*steps
+ # TODO: document behavior when time is outside the bounds of TTLed config
# TODO: document how the data is stored.
+ # TODO: better abstraction for "intervals" processing rather than in each implementation
- self._insert( name, value, timestamp )
+ self._insert( name, value, timestamp, intervals, **kwargs )
- def _insert(self, name, value, timestamp):
+ def _batch_insert(self, inserts, intervals, **kwargs):
+ '''
+ Support for batch insert. Default implementation is non-optimized and
+ is a simple loop over values.
+ '''
+ for timestamp,names in inserts.iteritems():
+ for name,values in names.iteritems():
+ for value in values:
+ self._insert( name, value, timestamp, intervals, **kwargs )
+
+ def _normalize_timestamps(self, timestamp, intervals, config):
+ '''
+ Helper for the subclasses to generate a list of timestamps.
+ '''
+ rval = [timestamp]
+ if intervals<0:
+ while intervals<0:
+ rval.append( config['i_calc'].normalize(timestamp, intervals) )
+ intervals += 1
+ elif intervals>0:
+ while intervals>0:
+ rval.append( config['i_calc'].normalize(timestamp, intervals) )
+ intervals -= 1
+ return rval
+
+ def _insert(self, name, value, timestamp, intervals, **kwargs):
'''
Support for the insert per type of series.
'''
@@ -333,63 +504,119 @@ def _insert(self, name, value, timestamp):
def delete(self, name):
'''
- Delete all data in a timeseries. Subclasses are responsible for
+ Delete all data in a timeseries. Subclasses are responsible for
implementing this.
'''
raise NotImplementedError()
- def get(self, name, interval, timestamp=None, condensed=False, transform=None):
+ def delete_all(self):
+ '''
+ Deletes all data in every timeseries. Default implementation is to walk
+ all of the names and call delete(stat), storage implementations are welcome
+ to optimize this.
+ '''
+ for name in self.list():
+ self.delete(name)
+
+ def iterate(self, name, interval, **kwargs):
+ '''
+ Returns a generator that iterates over all the intervals and returns
+ data for various timestamps, in the form:
+
+ ( unix_timestamp, data )
+
+ This will check for all timestamp buckets that might exist between
+ the first and last timestamp in a series. Each timestamp bucket will
+ be fetched separately to keep this memory efficient, at the cost of
+ extra trips to the data store.
+
+ Keyword arguments are the same as get().
+ '''
+ config = self._intervals.get(interval)
+ if not config:
+ raise UnknownInterval(interval)
+ properties = self.properties(name)[interval]
+
+ i_buckets = config['i_calc'].buckets(properties['first'], properties['last'])
+ for i_bucket in i_buckets:
+ data = self.get(name, interval,
+ timestamp=config['i_calc'].from_bucket(i_bucket), **kwargs)
+ for timestamp,row in data.items():
+ yield (timestamp,row)
+
+ def get(self, name, interval, **kwargs):
'''
Get the set of values for a named timeseries and interval. If timestamp
supplied, will fetch data for the period of time in which that timestamp
- would have fallen, else returns data for "now". If the timeseries
+ would have fallen, else returns data for "now". If the timeseries
resolution was not defined, then returns a simple list of values for the
- interval, else returns an ordered dict where the keys define the resolution
- interval and the values are the time series data in that (sub)interval.
+ interval, else returns an ordered dict where the keys define the resolution
+ interval and the values are the time series data in that (sub)interval.
This allows the user to interpolate sparse data sets.
If transform is defined, will utilize one of `[mean, count, min, max, sum]`
to process each row of data returned. If the transform is a callable, will
pass an array of data to the function. Note that the transform will be run
after the data is condensed. If the transform is a list, then each row will
- return a hash of the form { transform_name_or_func : transformed_data }.
- If the transform is a hash, then it should be of the form
+ return a hash of the form { transform_name_or_func : transformed_data }.
+ If the transform is a hash, then it should be of the form
{ transform_name : transform_func } and will return the same structure as
a list argument.
- Raises UnknownInterval if `interval` is not one of the configured
+ Raises UnknownInterval if `interval` is not one of the configured
intervals.
TODO: Fix this method doc
'''
- # TODO: support negative values of timestamp as "-N intervals", i.e.
- # -1 on a day interval is yesterday
- if not timestamp:
- timestamp = time.time()
-
config = self._intervals.get(interval)
if not config:
raise UnknownInterval(interval)
- rval = self._get( name, interval, config, timestamp )
+ timestamp = kwargs.get('timestamp', time.time())
+ fetch = kwargs.get('fetch')
+ process_row = kwargs.get('process_row') or self._process_row
+ condense = kwargs.get('condense', False)
+ join_rows = kwargs.get('join_rows') or self._join
+ transform = kwargs.get('transform')
+
+ # DEPRECATED handle the deprecated version of condense
+ condense = kwargs.get('condensed',condense)
+
+ # If name is a list, then join all of results. It is more efficient to
+ # use a single data structure and join "in-line" but that requires a major
+ # refactor of the backends, so trying this solution to start with. At a
+ # minimum we'd have to rebuild the results anyway because of the potential
+ # for sparse data points would result in an out-of-order result.
+ if isinstance(name, (list,tuple,set)):
+ results = [ self._get(x, interval, config, timestamp, fetch=fetch, process_row=process_row) for x in name ]
+ # Even resolution data is "coarse" in that it's not nested
+ rval = self._join_results( results, True, join_rows )
+ else:
+ rval = self._get( name, interval, config, timestamp, fetch=fetch, process_row=process_row )
+
+ # If condensed, collapse the result into a single row. Adjust the step_size
+ # calculation to match.
+ if config['coarse']:
+ step_size = config['i_calc'].step_size(timestamp)
+ else:
+ step_size = config['r_calc'].step_size(timestamp)
+ if condense and not config['coarse']:
+ condense = condense if callable(condense) else self._condense
+ rval = { config['i_calc'].normalize(timestamp) : condense(rval) }
+ step_size = config['i_calc'].step_size(timestamp)
- # If condensed, collapse the result into a single row
- # TODO: figure out how to not recalculate this value that subclasses will
- # also be calculating
- if condensed and not config['coarse']:
- rval = { config['i_calc'].normalize(timestamp) : self._condense(rval) }
if transform:
- for k,v in rval.iteritems():
- rval[k] = self._process_transform(v, transform)
+ for k,v in rval.items():
+ rval[k] = self._process_transform(v, transform, step_size)
return rval
- def _get(self, name, interval, config, timestamp):
+ def _get(self, name, interval, config, timestamp, fetch):
'''
Support for the insert per type of series.
'''
raise NotImplementedError()
-
- def series(self, name, interval, steps=None, condensed=False, start=None, end=None, transform=None):
+
+ def series(self, name, interval, **kwargs):
'''
Return all the data in a named time series for a given interval. If steps
not defined and there are none in the config, defaults to 1.
@@ -407,61 +634,127 @@ def series(self, name, interval, steps=None, condensed=False, start=None, end=No
config = self._intervals.get(interval)
if not config:
raise UnknownInterval(interval)
- steps = steps if steps else config.get('steps',1)
- # Fugly range determination, all to get ourselves a start and end
+ start = kwargs.get('start')
+ end = kwargs.get('end')
+ steps = kwargs.get('steps') or config.get('steps',1)
+
+ fetch = kwargs.get('fetch')
+ process_row = kwargs.get('process_row') or self._process_row
+ condense = kwargs.get('condense', False)
+ join_rows = kwargs.get('join_rows') or self._join
+ collapse = kwargs.get('collapse', False)
+ transform = kwargs.get('transform')
+ # DEPRECATED handle the deprecated version of condense
+ condense = kwargs.get('condensed',condense)
+
+ # If collapse, also condense
+ if collapse: condense = condense or True
+
+ # Fugly range determination, all to get ourselves a start and end
# timestamp. Adjust steps argument to include the anchoring date.
- if not end:
- if start:
- start_bucket = config['i_calc'].to_bucket( start )
- end_bucket = config['i_calc'].to_bucket( start, steps-1 )
- else:
+ if end is None:
+ if start is None:
end = time.time()
end_bucket = config['i_calc'].to_bucket( end )
start_bucket = config['i_calc'].to_bucket( end, (-steps+1) )
+ else:
+ start_bucket = config['i_calc'].to_bucket( start )
+ end_bucket = config['i_calc'].to_bucket( start, steps-1 )
else:
end_bucket = config['i_calc'].to_bucket( end )
- if start:
- start_bucket = config['i_calc'].to_bucket( start )
- else:
+ if start is None:
start_bucket = config['i_calc'].to_bucket( end, (-steps+1) )
-
+ else:
+ start_bucket = config['i_calc'].to_bucket( start )
+
# Now that we have start and end buckets, convert them back to normalized
# time stamps and then back to buckets. :)
start = config['i_calc'].from_bucket( start_bucket )
end = config['i_calc'].from_bucket( end_bucket )
-
+ if start > end: end = start
+
interval_buckets = config['i_calc'].buckets(start, end)
- rval = self._series(name, interval, config, interval_buckets)
- if config['coarse'] and transform:
- for key,data in rval.iteritems():
- rval[key] = self._process_transform(data, transform)
+ # If name is a list, then join all of results. It is more efficient to
+ # use a single data structure and join "in-line" but that requires a major
+ # refactor of the backends, so trying this solution to start with. At a
+ # minimum we'd have to rebuild the results anyway because of the potential
+ # for sparse data points would result in an out-of-order result.
+ if isinstance(name, (list,tuple,set)):
+ results = [ self._series(x, interval, config, interval_buckets, fetch=fetch, process_row=process_row) for x in name ]
+ rval = self._join_results( results, config['coarse'], join_rows )
+ else:
+ rval = self._series(name, interval, config, interval_buckets, fetch=fetch, process_row=process_row)
+
+ # If fine-grained, first do the condensed pass so that it's easier to do
+ # the collapse afterwards. Be careful not to run the transform if there's
+ # going to be another pass at condensing the data.
if not config['coarse']:
- if condensed:
+ if condense:
+ condense = condense if callable(condense) else self._condense
for key in rval.iterkeys():
- data = self._condense( rval[key] )
- if transform:
- data = self._process_transform(data, transform)
+ data = condense( rval[key] )
+ if transform and not collapse:
+ data = self._process_transform(data, transform, config['i_calc'].step_size(key))
rval[key] = data
elif transform:
- for interval,resolutions in rval.iteritems():
+ for interval,resolutions in rval.items():
for key in resolutions.iterkeys():
- resolutions[key] = self._process_transform(resolutions[key], transform)
-
+ resolutions[key] = self._process_transform(resolutions[key], transform, config['r_calc'].step_size(key))
+
+ if config['coarse'] or collapse:
+ if collapse:
+ collapse = collapse if callable(collapse) else condense if callable(condense) else self._condense
+ data = collapse(rval)
+ if transform:
+ rval = { rval.keys()[0] : self._process_transform(data, transform, config['i_calc'].step_size(rval.keys()[0], rval.keys()[-1])) }
+ else:
+ rval = { rval.keys()[0] : data }
+
+ elif transform:
+ for key,data in rval.items():
+ rval[key] = self._process_transform(data, transform, config['i_calc'].step_size(key))
+
+ return rval
+
+ def _series(self, name, interval, config, buckets, **kws):
+ '''
+ Subclasses must implement fetching a series.
+ '''
+ raise NotImplementedError()
+
+ def _join_results(self, results, coarse, join):
+ '''
+ Join a list of results. Supports both get and series.
+ '''
+ rval = OrderedDict()
+ i_keys = set()
+ for res in results:
+ i_keys.update( res.keys() )
+ for i_key in sorted(i_keys):
+ if coarse:
+ rval[i_key] = join( [res.get(i_key) for res in results] )
+ else:
+ rval[i_key] = OrderedDict()
+ r_keys = set()
+ for res in results:
+ r_keys.update( res.get(i_key,{}).keys() )
+ for r_key in sorted(r_keys):
+ rval[i_key][r_key] = join( [res.get(i_key,{}).get(r_key) for res in results] )
return rval
- def _process_transform(self, data, transform):
+ def _process_transform(self, data, transform, step_size):
'''
Process transforms on the data.
'''
if isinstance(transform, (list,tuple,set)):
- return { t : self._transform(data,t) for t in transform }
+ return { t : self._transform(data,t,step_size) for t in transform }
elif isinstance(transform, dict):
- return { tn : self._transform(data,tf) for tn,tf in transform.iteritems() }
- return self._transform(data, transform)
+ return { tn : self._transform(data,tf,step_size) for tn,tf in transform.items() }
+ return self._transform(data,transform,step_size)
- def _transform(self, data, transform):
+ def _transform(self, data, transform, step_size):
'''
Transform the data. If the transform is not supported by this series,
returns the data unaltered.
@@ -473,12 +766,6 @@ def _insert(self, handle, key, value):
Subclasses must implement inserting a value for a key.
'''
raise NotImplementedError()
-
- def _series(self, name, interval, config, buckets):
- '''
- Subclasses must implement fetching a series.
- '''
- raise NotImplementedError()
def _process_row(self, data):
'''
@@ -489,19 +776,28 @@ def _process_row(self, data):
def _condense(self, data):
'''
- Condense a mapping of timestamps and associated data into a single
+ Condense a mapping of timestamps and associated data into a single
object/value which will be mapped back to a timestamp that covers all
of the data.
'''
raise NotImplementedError()
+ def _join(self, rows):
+ '''
+ Join multiple rows worth of data into a single result.
+ '''
+ raise NotImplementedError()
+
class Series(Timeseries):
'''
Simple time series where all data is stored in a list for each interval.
'''
- def _transform(self, data, transform):
+ def _type_no_value(self):
+ return []
+
+ def _transform(self, data, transform, step_size):
'''
Transform the data. If the transform is not supported by this series,
returns the data unaltered.
@@ -518,8 +814,10 @@ def _transform(self, data, transform):
data = max( data or [0])
elif transform=='sum':
data = sum( data )
+ elif transform=='rate':
+ data = len( data ) / float(step_size)
elif callable(transform):
- data = transform(data)
+ data = transform(data, step_size)
return data
def _process_row(self, data):
@@ -535,6 +833,15 @@ def _condense(self, data):
return reduce(operator.add, data.values())
return []
+ def _join(self, rows):
+ '''
+ Join multiple rows worth of data into a single result.
+ '''
+ rval = []
+ for row in rows:
+ if row: rval.extend( row )
+ return rval
+
class Histogram(Timeseries):
'''
Data for each interval is stored in a hash, counting occurrances of the
@@ -542,13 +849,16 @@ class Histogram(Timeseries):
and distribution of the data points within the histogram.
'''
- def _transform(self, data, transform):
+ def _type_no_value(self):
+ return {}
+
+ def _transform(self, data, transform, step_size):
'''
Transform the data. If the transform is not supported by this series,
returns the data unaltered.
'''
if transform=='mean':
- total = sum( k*v for k,v in data.iteritems() )
+ total = sum( k*v for k,v in data.items() )
count = sum( data.values() )
data = float(total)/float(count) if count>0 else 0
elif transform=='count':
@@ -558,44 +868,62 @@ def _transform(self, data, transform):
elif transform=='max':
data = max(data.keys() or [0])
elif transform=='sum':
- data = sum( k*v for k,v in data.iteritems() )
+ data = sum( k*v for k,v in data.items() )
+ elif transform=='rate':
+ data = { k:v/float(step_size) for k,v in data.items() }
elif callable(transform):
- data = transform(data)
+ data = transform(data, step_size)
return data
def _process_row(self, data):
rval = {}
- for value,count in data.iteritems():
+ for value,count in data.items():
if self._read_func: value = self._read_func(value)
rval[ value ] = int(count)
return rval
-
+
def _condense(self, data):
'''
Condense by adding together all of the lists.
'''
rval = {}
- for resolution,histogram in data.iteritems():
- for value,count in histogram.iteritems():
+ for resolution,histogram in data.items():
+ for value,count in histogram.items():
rval[ value ] = count + rval.get(value,0)
return rval
+ def _join(self, rows):
+ '''
+ Join multiple rows worth of data into a single result.
+ '''
+ rval = {}
+ for row in rows:
+ if row:
+ for value,count in row.items():
+ rval[ value ] = count + rval.get(value,0)
+ return rval
+
class Count(Timeseries):
'''
Time series that simply increments within each interval.
'''
- def _transform(self, data, transform):
+ def _type_no_value(self):
+ return 0
+
+ def _transform(self, data, transform, step_size):
'''
Transform the data. If the transform is not supported by this series,
returns the data unaltered.
'''
- if callable(transform):
- data = transform(data)
+ if transform=='rate':
+ data = data / float(step_size)
+ elif callable(transform):
+ data = transform(data, step_size)
return data
-
- def insert(self, name, value=1, timestamp=None):
- super(Count,self).insert(name, value, timestamp)
+
+ def insert(self, name, value=1, timestamp=None, **kwargs):
+ super(Count,self).insert(name, value, timestamp, **kwargs)
def _process_row(self, data):
return int(data) if data else 0
@@ -608,18 +936,31 @@ def _condense(self, data):
return sum(data.values())
return 0
+ def _join(self, rows):
+ '''
+ Join multiple rows worth of data into a single result.
+ '''
+ rval = 0
+ for row in rows:
+ if row: rval += row
+ return rval
+
class Gauge(Timeseries):
'''
Time series that stores the last value.
'''
- def _transform(self, data, transform):
+ def _type_no_value(self):
+ # TODO: resolve this disconnect with redis backend
+ return 0
+
+ def _transform(self, data, transform, step_size):
'''
Transform the data. If the transform is not supported by this series,
returns the data unaltered.
'''
if callable(transform):
- data = transform(data)
+ data = transform(data, step_size)
return data
def _process_row(self, data):
@@ -629,21 +970,97 @@ def _process_row(self, data):
def _condense(self, data):
'''
- Condense by adding together all of the lists.
+ Condense by returning the last real value of the gauge.
'''
if data:
- return data.values()
- return []
+ data = filter(None,data.values())
+ if data:
+ return data[-1]
+ return None
+
+ def _join(self, rows):
+ '''
+ Join multiple rows worth of data into a single result.
+ '''
+ rval = None
+ for row in rows:
+ if row: rval = row
+ return rval
+
+class Set(Timeseries):
+ '''
+ Time series that manages sets.
+ '''
+
+ def _type_no_value(self):
+ return set()
+
+ def _transform(self, data, transform, step_size):
+ '''
+ Transform the data. If the transform is not supported by this series,
+ returns the data unaltered.
+ '''
+ if transform=='mean':
+ total = sum( data )
+ count = len( data )
+ data = float(total)/float(count) if count>0 else 0
+ elif transform=='count':
+ data = len(data)
+ elif transform=='min':
+ data = min(data or [0])
+ elif transform=='max':
+ data = max(data or [0])
+ elif transform=='sum':
+ data = sum(data)
+ elif transform=='rate':
+ data = len(data) / float(step_size)
+ elif callable(transform):
+ data = transform(data)
+ return data
+
+ def _process_row(self, data):
+ if self._read_func:
+ return set( (self._read_func(d) for d in data) )
+ return data
+
+ def _condense(self, data):
+ '''
+ Condense by or-ing all of the sets.
+ '''
+ if data:
+ return reduce(operator.ior, data.values())
+ return set()
+
+ def _join(self, rows):
+ '''
+ Join multiple rows worth of data into a single result.
+ '''
+ rval = set()
+ for row in rows:
+ if row: rval |= row
+ return rval
# Load the backends after all the timeseries had been defined.
try:
- from redis_backend import RedisBackend
+ from .redis_backend import RedisBackend
BACKENDS['redis'] = RedisBackend
except ImportError as e:
- print 'Redis backend not loaded,', e
+ warnings.warn('Redis backend not loaded, {}'.format(e))
try:
- from mongo_backend import MongoBackend
+ from .mongo_backend import MongoBackend
BACKENDS['pymongo'] = MongoBackend
except ImportError as e:
- print 'Mongo backend not loaded,', e
+ warnings.warn('Mongo backend not loaded, {}'.format(e))
+
+try:
+ from .sql_backend import SqlBackend
+ BACKENDS['sqlalchemy'] = SqlBackend
+except ImportError as e:
+ warnings.warn('SQL backend not loaded, {}'.format(e))
+
+try:
+ from .cassandra_backend import CassandraBackend
+ BACKENDS['cql'] = CassandraBackend
+except ImportError as e:
+ warnings.warn('Cassandra backend not loaded, {}'.format(e))
diff --git a/requirements.pip b/requirements.pip
index 6c624dc..92402a2 100644
--- a/requirements.pip
+++ b/requirements.pip
@@ -1,2 +1,2 @@
# For calculating month deltas in gregorian systems
-monthdelta
+MonthDelta==1.0b
diff --git a/setup.py b/setup.py
index 138c971..7031f1b 100644
--- a/setup.py
+++ b/setup.py
@@ -20,9 +20,9 @@
install_requires = requirements,
url='https://github.com/agoragames/kairos',
license="LICENSE.txt",
- description='Time series data storage using Redis and Mongo',
+ description='Time series data storage in Redis, Mongo, SQL and Cassandra',
long_description=open('README.rst').read(),
- keywords=['python', 'redis', 'mongo', 'time', 'timeseries', 'rrd', 'gevent', 'statistics'],
+ keywords=['python', 'redis', 'mongo', 'sql', 'mysql', 'sqlite', 'postgresql', 'cassandra', 'time', 'timeseries', 'rrd', 'gevent', 'statistics'],
classifiers=[
'Development Status :: 3 - Alpha',
'License :: OSI Approved :: BSD License',
@@ -33,7 +33,6 @@
"Topic :: Software Development :: Libraries :: Python Modules",
'Programming Language :: Python',
'Programming Language :: Python :: 2.7',
- 'Programming Language :: Python :: 3',
'Topic :: Software Development :: Libraries'
]
)
diff --git a/test/functional/api_helper.py b/test/functional/api_helper.py
new file mode 100644
index 0000000..a49d489
--- /dev/null
+++ b/test/functional/api_helper.py
@@ -0,0 +1,81 @@
+from helper_helper import *
+from helper_helper import _time
+
+@unittest.skipUnless( os.environ.get('TEST_API','true').lower()=='true', 'skipping api' )
+class ApiHelper(Chai):
+
+ def setUp(self):
+ super(ApiHelper,self).setUp()
+
+ self.series = Timeseries(self.client, type='series', prefix='kairos',
+ read_func=int, #write_func=str,
+ intervals={
+ 'minute' : {
+ 'step' : 60,
+ 'steps' : 5,
+ },
+ 'hour' : {
+ 'step' : 3600,
+ 'resolution' : 60,
+ },
+ 'bulk-hour' : {
+ 'step' : 3600,
+ }
+ } )
+ self.series.delete_all()
+
+ def tearDown(self):
+ self.series.delete_all()
+
+ def test_list(self):
+ self.series.insert( 'test', 32, timestamp=_time(0) )
+ self.series.insert( 'test1', 32, timestamp=_time(0) )
+ self.series.insert( 'test2', 32, timestamp=_time(0) )
+ self.series.insert( 'test', 32, timestamp=_time(0) )
+
+ res = sorted(self.series.list())
+ assert_equals( ['test', 'test1', 'test2'], res )
+
+ self.series.delete('test')
+ self.series.delete('test1')
+ self.series.delete('test2')
+
+ def test_properties(self):
+ self.series.insert( 'test', 32, timestamp=_time(0) )
+ self.series.insert( 'test', 32, timestamp=_time(60) )
+ self.series.insert( 'test', 32, timestamp=_time(600) )
+
+ res = self.series.properties('test')
+ assert_equals( _time(0), res['minute']['first'] )
+ assert_equals( _time(600), res['minute']['last'] )
+ assert_equals( _time(0), res['hour']['first'] )
+ assert_equals( _time(0), res['hour']['last'] )
+
+ self.series.delete('test')
+
+ def test_iterate(self):
+ self.series.insert( 'test', 32, timestamp=_time(0) )
+ self.series.insert( 'test', 42, timestamp=_time(60) )
+ self.series.insert( 'test', 52, timestamp=_time(600) )
+
+ # There should be a result for every possible step between first and last
+ res = list(self.series.iterate('test','minute'))
+ assert_equals( 11, len(res) )
+ assert_equals( (_time(0),[32]), res[0] )
+ assert_equals( (_time(60),[42]), res[1] )
+ assert_equals( (_time(120),[]), res[2] )
+ assert_equals( (_time(600),[52]), res[-1] )
+
+ # With resolutions, there should be a result only for where there's data
+ res = list(self.series.iterate('test','hour'))
+ assert_equals( 3, len(res) )
+ assert_equals( (_time(0),[32]), res[0] )
+ assert_equals( (_time(60),[42]), res[1] )
+ assert_equals( (_time(600),[52]), res[2] )
+
+ # Without resolutions, there should be a single result
+ res = list(self.series.iterate('test','bulk-hour'))
+ assert_equals( 1, len(res) )
+ assert_equals( (_time(0),[32,42,52]), res[0] )
+
+ self.series.delete('test')
diff --git a/test/functional/cassandra_timeseries_test.py b/test/functional/cassandra_timeseries_test.py
new file mode 100644
index 0000000..028cf39
--- /dev/null
+++ b/test/functional/cassandra_timeseries_test.py
@@ -0,0 +1,66 @@
+'''
+Functional tests for cassandra timeseries
+'''
+import time
+import datetime
+import os
+
+import cql
+
+from . import helpers
+from .helpers import unittest, os, Timeseries
+
+@unittest.skipUnless( os.environ.get('TEST_CASSANDRA','true').lower()=='true', 'skipping cassandra' )
+class CassandraApiTest(helpers.ApiHelper):
+
+ def setUp(self):
+ self.client = cql.connect('localhost', 9160, os.environ.get('CASSANDRA_KEYSPACE','kairos'), cql_version='3.0.0')
+ super(CassandraApiTest,self).setUp()
+
+ def test_url_parse(self):
+ assert_equals( 'CassandraSeries',
+ Timeseries( 'cql://localhost', type='series' ).__class__.__name__ )
+
+# Not running gregorian tests because they run in the "far future" where long
+# TTLs are not supported.
+#@unittest.skipUnless( os.environ.get('TEST_CASSANDRA','true').lower()=='true', 'skipping cassandra' )
+#class CassandraGregorianTest(helpers.GregorianHelper):
+
+ #def setUp(self):
+ #self.client = cql.connect('localhost', 9160, os.environ.get('CASSANDRA_KEYSPACE','kairos'), cql_version='3.0.0')
+ #super(CassandraGregorianTest,self).setUp()
+
+@unittest.skipUnless( os.environ.get('TEST_CASSANDRA','true').lower()=='true', 'skipping cassandra' )
+class CassandraSeriesTest(helpers.SeriesHelper):
+
+ def setUp(self):
+ self.client = cql.connect('localhost', 9160, os.environ.get('CASSANDRA_KEYSPACE','kairos'), cql_version='3.0.0')
+ super(CassandraSeriesTest,self).setUp()
+
+@unittest.skipUnless( os.environ.get('TEST_CASSANDRA','true').lower()=='true', 'skipping cassandra' )
+class CassandraHistogramTest(helpers.HistogramHelper):
+
+ def setUp(self):
+ self.client = cql.connect('localhost', 9160, os.environ.get('CASSANDRA_KEYSPACE','kairos'), cql_version='3.0.0')
+ super(CassandraHistogramTest,self).setUp()
+
+@unittest.skipUnless( os.environ.get('TEST_CASSANDRA','true').lower()=='true', 'skipping cassandra' )
+class CassandraCountTest(helpers.CountHelper):
+
+ def setUp(self):
+ self.client = cql.connect('localhost', 9160, os.environ.get('CASSANDRA_KEYSPACE','kairos'), cql_version='3.0.0')
+ super(CassandraCountTest,self).setUp()
+
+@unittest.skipUnless( os.environ.get('TEST_CASSANDRA','true').lower()=='true', 'skipping cassandra' )
+class CassandraGaugeTest(helpers.GaugeHelper):
+
+ def setUp(self):
+ self.client = cql.connect('localhost', 9160, os.environ.get('CASSANDRA_KEYSPACE','kairos'), cql_version='3.0.0')
+ super(CassandraGaugeTest,self).setUp()
+
+@unittest.skipUnless( os.environ.get('TEST_CASSANDRA','true').lower()=='true', 'skipping cassandra' )
+class CassandraSetTest(helpers.SetHelper):
+
+ def setUp(self):
+ self.client = cql.connect('localhost', 9160, os.environ.get('CASSANDRA_KEYSPACE','kairos'), cql_version='3.0.0')
+ super(CassandraSetTest,self).setUp()
diff --git a/test/functional/count_helper.py b/test/functional/count_helper.py
new file mode 100644
index 0000000..8c1b5c5
--- /dev/null
+++ b/test/functional/count_helper.py
@@ -0,0 +1,314 @@
+from helper_helper import *
+from helper_helper import _time
+
+from collections import OrderedDict
+
+@unittest.skipUnless( os.environ.get('TEST_COUNT','true').lower()=='true', 'skipping counts' )
+class CountHelper(Chai):
+
+ def setUp(self):
+ super(CountHelper,self).setUp()
+
+ self.series = Timeseries(self.client, type='count', prefix='kairos',
+ read_func=int,
+ intervals={
+ 'minute' : {
+ 'step' : 60,
+ 'steps' : 5,
+ },
+ 'hour' : {
+ 'step' : 3600,
+ 'resolution' : 60,
+ }
+ } )
+ self.series.delete_all()
+
+ def tearDown(self):
+ self.series.delete_all()
+
+ def test_bulk_insert(self):
+ inserts = {
+ None : { 'test1':[1,2,3], 'test2':[4,5,6] },
+ _time(0) : { 'test1':[1,2,3], 'test2':[4,5,6], 'test3':[7,8,9] },
+ _time(30) : { 'test1':[1,2,3], 'test2':[4,5,6] },
+ _time(60) : { 'test1':[1,2,3], 'test3':[7,8,9] }
+ }
+ self.series.bulk_insert( inserts )
+
+ t1_i1 = self.series.get('test1', 'minute', timestamp=_time(0))
+ assert_equals( 1+2+3+1+2+3, t1_i1[_time(0)] )
+
+ t2_i1 = self.series.get('test2', 'minute', timestamp=_time(0))
+ assert_equals( 4+5+6+4+5+6, t2_i1[_time(0)] )
+
+ t3_i1 = self.series.get('test3', 'minute', timestamp=_time(0))
+ assert_equals( 7+8+9, t3_i1[_time(0)] )
+
+ t1_i2 = self.series.get('test1', 'minute', timestamp=_time(60))
+ assert_equals( 1+2+3, t1_i2[_time(60)] )
+
+ def test_bulk_insert_intervals_after(self):
+ a,b,c,d,e,f = 10,11,12,13,14,15
+ inserts = OrderedDict( (
+ (None , { 'test1':[1,2,3], 'test2':[4,5,6] } ),
+ (_time(0) , { 'test1':[1,2,3], 'test2':[4,5,6], 'test3':[7,8,9] } ),
+ (_time(30), { 'test1':[1,2,3], 'test2':[4,5,6] } ),
+ (_time(60), { 'test1':[a,b,c], 'test3':[d,e,f] })
+ ) )
+ self.series.bulk_insert( inserts, intervals=3 )
+
+ t1_i1 = self.series.get('test1', 'minute', timestamp=_time(0))
+ assert_equals( 1+2+3+1+2+3, t1_i1[_time(0)] )
+
+ t2_i1 = self.series.get('test2', 'minute', timestamp=_time(0))
+ assert_equals( 4+5+6+4+5+6, t2_i1[_time(0)] )
+
+ t3_i1 = self.series.get('test3', 'minute', timestamp=_time(0))
+ assert_equals( 7+8+9, t3_i1[_time(0)] )
+
+ t1_i2 = self.series.get('test1', 'minute', timestamp=_time(60))
+ assert_equals( 1+2+3+1+2+3+a+b+c, t1_i2[_time(60)] )
+
+ t3_i3 = self.series.get('test3', 'minute', timestamp=_time(120))
+ assert_equals( 7+8+9+d+e+f, t3_i3[_time(120)] )
+
+ t3_i4 = self.series.get('test3', 'minute', timestamp=_time(180))
+ assert_equals( 7+8+9+d+e+f, t3_i4[_time(180)] )
+
+ def test_bulk_insert_intervals_before(self):
+ a,b,c,d,e,f = 10,11,12,13,14,15
+ inserts = OrderedDict( (
+ (None , { 'test1':[1,2,3], 'test2':[4,5,6] } ),
+ (_time(0) , { 'test1':[1,2,3], 'test2':[4,5,6], 'test3':[7,8,9] } ),
+ (_time(30), { 'test1':[1,2,3], 'test2':[4,5,6] } ),
+ (_time(60), { 'test1':[a,b,c], 'test3':[d,e,f] })
+ ) )
+ self.series.bulk_insert( inserts, intervals=-3 )
+
+ t1_i1 = self.series.get('test1', 'minute', timestamp=_time(0))
+ assert_equals( 1+2+3+1+2+3+a+b+c, t1_i1[_time(0)] )
+
+ t2_i1 = self.series.get('test2', 'minute', timestamp=_time(0))
+ assert_equals( 4+5+6+4+5+6, t2_i1[_time(0)] )
+
+ t3_i1 = self.series.get('test3', 'minute', timestamp=_time(0))
+ assert_equals( 7+8+9+d+e+f, t3_i1[_time(0)] )
+
+ t1_i2 = self.series.get('test1', 'minute', timestamp=_time(-60))
+ assert_equals( 1+2+3+1+2+3+a+b+c, t1_i2[_time(-60)] )
+
+ t3_i3 = self.series.get('test3', 'minute', timestamp=_time(-120))
+ assert_equals( 7+8+9+d+e+f, t3_i3[_time(-120)] )
+
+ t3_i4 = self.series.get('test3', 'minute', timestamp=_time(-180))
+ assert_equals( 7+8+9, t3_i4[_time(-180)] )
+
+ def test_get(self):
+ # 2 hours worth of data
+ for t in xrange(1, 7200):
+ self.series.insert( 'test', 1, timestamp=_time(t) )
+
+ ###
+ ### no resolution, condensed has no impact
+ ###
+ # middle of an interval
+ interval = self.series.get( 'test', 'minute', timestamp=_time(100) )
+ assert_equals( map(_time, [60]), interval.keys() )
+ assert_equals( 60, interval[_time(60)] )
+
+ # no matching interval, returns no with empty value list
+ interval = self.series.get( 'test', 'minute' )
+ assert_equals( 1, len(interval) )
+ assert_equals( 0, interval.values()[0] )
+
+ ###
+ ### with resolution, optionally condensed
+ ###
+ interval = self.series.get( 'test', 'hour', timestamp=_time(100) )
+ assert_equals( 60, len(interval) )
+ assert_equals( 60, interval[_time(60)] )
+
+ interval = self.series.get( 'test', 'hour', timestamp=_time(100), transform='rate' )
+ assert_equals( 1.0, interval[_time(60)] )
+
+ interval = self.series.get( 'test', 'hour', timestamp=_time(100), condensed=True )
+ assert_equals( 1, len(interval) )
+ assert_equals( 3599, interval[_time(0)] )
+
+ interval = self.series.get( 'test', 'hour', timestamp=_time(4000), condensed=True )
+ assert_equals( 1, len(interval) )
+ assert_equals( 3600, interval[_time(3600)] )
+
+ interval = self.series.get( 'test', 'hour', timestamp=_time(4000), condensed=True, transform='rate' )
+ assert_equals( 1, len(interval) )
+ assert_equals( 1.0, interval[_time(3600)] )
+
+ def test_get_joined(self):
+ # put some data in the first minutes of each hour for test1, and then for
+ # a few more minutes in test2
+ for t in xrange(1, 120):
+ self.series.insert( 'test1', t, timestamp=_time(t) )
+ self.series.insert( 'test2', t, timestamp=_time(t) )
+ for t in xrange(3600, 3720):
+ self.series.insert( 'test1', t, timestamp=_time(t) )
+ self.series.insert( 'test2', t, timestamp=_time(t) )
+ for t in xrange(120, 240):
+ self.series.insert( 'test1', t, timestamp=_time(t) )
+ for t in xrange(3721, 3840):
+ self.series.insert( 'test1', t, timestamp=_time(t) )
+
+ ###
+ ### no resolution, condensed has no impact
+ ###
+ # interval with 2 series worth of data
+ interval = self.series.get( ['test1','test2'], 'minute', timestamp=_time(100) )
+ assert_equals( [_time(60)], interval.keys() )
+ assert_equals( 2*sum(range(60,120)), interval[_time(60)] )
+
+ interval = self.series.get( ['test1','test2'], 'minute', timestamp=_time(100), transform='rate' )
+ assert_equals( (2*sum(range(60,120)))/60.0, interval[_time(60)] )
+
+ # interval with 1 series worth of data
+ interval = self.series.get( ['test1','test2'], 'minute', timestamp=_time(122) )
+ assert_equals( [_time(120)], interval.keys() )
+ assert_equals( sum(range(120,180)), interval[_time(120)] )
+ interval = self.series.get( ['test1','test2'], 'minute', timestamp=_time(122), transform='rate' )
+ assert_equals( (sum(range(120,180)))/60.0, interval[_time(120)] )
+
+ # no matching interval, returns no with empty value list
+ interval = self.series.get( ['test1','test2'], 'minute' )
+ assert_equals( 1, len(interval) )
+ assert_equals( 0, interval.values()[0] )
+ interval = self.series.get( ['test1','test2'], 'minute', transform='rate' )
+
+ ###
+ ### with resolution, optionally condensed
+ ###
+ interval = self.series.get( ['test1','test2'], 'hour', timestamp=_time(100) )
+ assert_equals( map(_time,[0,60,120,180]), interval.keys() )
+ assert_equals( 2*sum(range(1,60)), interval[_time(0)] )
+ assert_equals( 2*sum(range(60,120)), interval[_time(60)] )
+ assert_equals( sum(range(120,180)), interval[_time(120)] )
+ assert_equals( sum(range(180,240)), interval[_time(180)] )
+
+ interval = self.series.get( ['test1','test2'], 'hour', timestamp=_time(100), condensed=True )
+ assert_equals( [_time(0)], interval.keys() )
+ assert_equals( 2*sum(range(1,120)) + sum(range(120,240)), interval[_time(0)] )
+
+ interval = self.series.get( ['test1','test2'], 'hour', timestamp=_time(100), condensed=True, transform='rate' )
+ assert_equals( (2*sum(range(1,120)) + sum(range(120,240)))/3600.0, interval[_time(0)] )
+
+ def test_series(self):
+ # 2 hours worth of data
+ for t in xrange(1, 7200):
+ self.series.insert( 'test', 1, timestamp=_time(t) )
+
+ ###
+ ### no resolution, condensed has no impact
+ ###
+ interval = self.series.series( 'test', 'minute', end=_time(250) )
+ assert_equals( map(_time, [0,60,120,180,240]), interval.keys() )
+ assert_equals( 59, interval[_time(0)] )
+ assert_equals( 60, interval[_time(60)] )
+
+ interval = self.series.series( 'test', 'minute', steps=2, end=_time(250) )
+ assert_equals( map(_time, [180,240]), interval.keys() )
+ assert_equals( 60, interval[_time(240)] )
+
+ interval = self.series.series( 'test', 'minute', steps=2, end=_time(250), transform='rate' )
+ assert_equals( 1.0, interval[_time(240)] )
+
+ # with collapse
+ interval = self.series.series( 'test', 'minute', end=_time(250), collapse=True )
+ assert_equals( map(_time, [0]), interval.keys() )
+ assert_equals( 299, interval[_time(0)] )
+
+ ###
+ ### with resolution
+ ###
+ interval = self.series.series( 'test', 'hour', end=_time(250) )
+ assert_equals( 1, len(interval) )
+ assert_equals( 60, len(interval[_time(0)]) )
+ assert_equals( 59, interval[_time(0)][_time(0)] )
+ assert_equals( 60, interval[_time(0)][_time(60)] )
+
+ # single step, last one
+ interval = self.series.series( 'test', 'hour', condensed=True, end=_time(4200) )
+ assert_equals( 1, len(interval) )
+ assert_equals( 3600, interval[_time(3600)] )
+ interval = self.series.series( 'test', 'hour', condensed=True, end=_time(4200), transform='rate' )
+ assert_equals( 1.0, interval[_time(3600)] )
+
+ interval = self.series.series( 'test', 'hour', condensed=True, end=_time(4200), steps=2 )
+ assert_equals( map(_time, [0,3600]), interval.keys() )
+ assert_equals( 3599, interval[_time(0)] )
+ assert_equals( 3600, interval[_time(3600)] )
+
+ # with collapse
+ interval = self.series.series( 'test', 'hour', condensed=True, end=_time(4200), steps=2, collapse=True )
+ assert_equals( map(_time, [0]), interval.keys() )
+ assert_equals( 7199, interval[_time(0)] )
+
+ def test_series_joined(self):
+ # put some data in the first minutes of each hour for test1, and then for
+ # a few more minutes in test2
+ for t in xrange(1, 120):
+ self.series.insert( 'test1', t, timestamp=_time(t) )
+ self.series.insert( 'test2', t, timestamp=_time(t) )
+ for t in xrange(3600, 3720):
+ self.series.insert( 'test1', t, timestamp=_time(t) )
+ self.series.insert( 'test2', t, timestamp=_time(t) )
+ for t in xrange(120, 240):
+ self.series.insert( 'test1', t, timestamp=_time(t) )
+ for t in xrange(3720, 3840):
+ self.series.insert( 'test1', t, timestamp=_time(t) )
+
+ ###
+ ### no resolution, condensed has no impact
+ ###
+ interval = self.series.series( ['test1','test2'], 'minute', end=_time(250) )
+ assert_equals( map(_time,[0,60,120,180,240]), interval.keys() )
+ assert_equals( 2*sum(range(1,60)), interval[_time(0)] )
+ assert_equals( 2*sum(range(60,120)), interval[_time(60)] )
+ assert_equals( sum(range(120,180)), interval[_time(120)] )
+ assert_equals( sum(range(180,240)), interval[_time(180)] )
+ assert_equals( 0, interval[_time(240)] )
+ interval = self.series.series( ['test1','test2'], 'minute', end=_time(250), transform='rate' )
+ assert_equals( sum(range(120,180))/60.0, interval[_time(120)] )
+
+ # no matching interval, returns no with empty value list
+ interval = self.series.series( ['test1','test2'], 'minute', start=time.time(), steps=2 )
+ assert_equals( 2, len(interval) )
+ assert_equals( 0, interval.values()[0] )
+
+ # with collapsed
+ interval = self.series.series( ['test1','test2'], 'minute', end=_time(250), collapse=True )
+ assert_equals( [_time(0)], interval.keys() )
+ assert_equals( 2*sum(range(1,120))+sum(range(120,240)), interval[_time(0)] )
+
+ ###
+ ### with resolution, optionally condensed
+ ###
+ interval = self.series.series( ['test1','test2'], 'hour', end=_time(250) )
+ assert_equals( 1, len(interval) )
+ assert_equals( map(_time,[0,60,120,180]), interval[_time(0)].keys() )
+ assert_equals( 4, len(interval[_time(0)]) )
+ assert_equals( 2*sum(range(1,60)), interval[_time(0)][_time(0)] )
+ assert_equals( 2*sum(range(60,120)), interval[_time(0)][_time(60)] )
+ assert_equals( sum(range(120,180)), interval[_time(0)][_time(120)] )
+ assert_equals( sum(range(180,240)), interval[_time(0)][_time(180)] )
+
+ interval = self.series.series( ['test1','test2'], 'hour', end=_time(250), transform='rate' )
+ assert_equals( sum(range(180,240)) / 60.0, interval[_time(0)][_time(180)] )
+
+ # condensed
+ interval = self.series.series( ['test1','test2'], 'hour', end=_time(250), condensed=True )
+ assert_equals( [_time(0)], interval.keys() )
+ assert_equals( 2*sum(range(1,120))+sum(range(120,240)), interval[_time(0)] )
+
+ # with collapsed
+ interval = self.series.series( ['test1','test2'], 'hour', condensed=True, end=_time(4200), steps=2, collapse=True )
+ assert_equals( map(_time, [0]), interval.keys() )
+ assert_equals(
+ 2*sum(range(1,120))+sum(range(120,240))+2*sum(range(3600,3720))+sum(range(3720,3840)),
+ interval[_time(0)] )
diff --git a/test/functional/gauge_helper.py b/test/functional/gauge_helper.py
new file mode 100644
index 0000000..1810035
--- /dev/null
+++ b/test/functional/gauge_helper.py
@@ -0,0 +1,284 @@
+from helper_helper import *
+from helper_helper import _time
+
+@unittest.skipUnless( os.environ.get('TEST_GAUGE','true').lower()=='true', 'skipping gauges' )
+class GaugeHelper(Chai):
+
+ def setUp(self):
+ super(GaugeHelper,self).setUp()
+
+ self.series = Timeseries(self.client, type='gauge', prefix='kairos',
+ read_func=lambda v: int(v or 0),
+ intervals={
+ 'minute' : {
+ 'step' : 60,
+ 'steps' : 5,
+ },
+ 'hour' : {
+ 'step' : 3600,
+ 'resolution' : 60,
+ }
+ } )
+ self.series.delete_all()
+
+ def tearDown(self):
+ self.series.delete_all()
+
+ def test_bulk_insert(self):
+ inserts = {
+ None : { 'test1':[1,2,3], 'test2':[4,5,6] },
+ _time(0) : { 'test1':[1,2,3], 'test2':[4,5,6], 'test3':[7,8,9] },
+ _time(30) : { 'test1':[1,2,3], 'test2':[4,5,6] },
+ _time(60) : { 'test1':[1,2,3], 'test3':[7,8,9] }
+ }
+ self.series.bulk_insert( inserts )
+
+ t1_i1 = self.series.get('test1', 'minute', timestamp=_time(0))
+ assert_equals( 3, t1_i1[_time(0)] )
+
+ t2_i1 = self.series.get('test2', 'minute', timestamp=_time(0))
+ assert_equals( 6, t2_i1[_time(0)] )
+
+ t3_i1 = self.series.get('test3', 'minute', timestamp=_time(0))
+ assert_equals( 9, t3_i1[_time(0)] )
+
+ t1_i2 = self.series.get('test1', 'minute', timestamp=_time(60))
+ assert_equals( 3, t1_i2[_time(60)] )
+
+ def test_bulk_insert_intervals_after(self):
+ a,b,c,d,e,f = 10,11,12,13,14,15
+ inserts = OrderedDict( (
+ (None , { 'test1':[1,2,3], 'test2':[4,5,6] } ),
+ (_time(0) , { 'test1':[1,2,3], 'test2':[4,5,6], 'test3':[7,8,9] } ),
+ (_time(30), { 'test1':[1,2,3], 'test2':[4,5,6] } ),
+ (_time(60), { 'test1':[a,b,c], 'test3':[d,e,f] })
+ ) )
+ self.series.bulk_insert( inserts, intervals=3 )
+
+ t1_i1 = self.series.get('test1', 'minute', timestamp=_time(0))
+ assert_equals( 3, t1_i1[_time(0)] )
+
+ t2_i1 = self.series.get('test2', 'minute', timestamp=_time(0))
+ assert_equals( 6, t2_i1[_time(0)] )
+
+ t3_i1 = self.series.get('test3', 'minute', timestamp=_time(0))
+ assert_equals( 9, t3_i1[_time(0)] )
+
+ t1_i2 = self.series.get('test1', 'minute', timestamp=_time(60))
+ assert_equals( c, t1_i2[_time(60)] )
+
+ t3_i3 = self.series.get('test3', 'minute', timestamp=_time(120))
+ assert_equals( f, t3_i3[_time(120)] )
+
+ t3_i4 = self.series.get('test3', 'minute', timestamp=_time(180))
+ assert_equals( f, t3_i4[_time(180)] )
+
+ def test_bulk_insert_intervals_before(self):
+ a,b,c,d,e,f = 10,11,12,13,14,15
+ inserts = OrderedDict( (
+ (None , { 'test1':[1,2,3], 'test2':[4,5,6] } ),
+ (_time(0) , { 'test1':[1,2,3], 'test2':[4,5,6], 'test3':[7,8,9] } ),
+ (_time(30), { 'test1':[1,2,3], 'test2':[4,5,6] } ),
+ (_time(60), { 'test1':[a,b,c], 'test3':[d,e,f] })
+ ) )
+ self.series.bulk_insert( inserts, intervals=-3 )
+
+ t1_i1 = self.series.get('test1', 'minute', timestamp=_time(0))
+ assert_equals( c, t1_i1[_time(0)] )
+
+ t2_i1 = self.series.get('test2', 'minute', timestamp=_time(0))
+ assert_equals( 6, t2_i1[_time(0)] )
+
+ t3_i1 = self.series.get('test3', 'minute', timestamp=_time(0))
+ assert_equals( f, t3_i1[_time(0)] )
+
+ t1_i2 = self.series.get('test1', 'minute', timestamp=_time(-60))
+ assert_equals( c, t1_i2[_time(-60)] )
+
+ t3_i3 = self.series.get('test3', 'minute', timestamp=_time(-120))
+ assert_equals( f, t3_i3[_time(-120)] )
+
+ t3_i4 = self.series.get('test3', 'minute', timestamp=_time(-180))
+ assert_equals( 9, t3_i4[_time(-180)] )
+
+ def test_get(self):
+ # 2 hours worth of data
+ for t in xrange(1, 7200):
+ self.series.insert( 'test', t, timestamp=_time(t) )
+
+ ###
+ ### no resolution, condensed has no impact
+ ###
+ # middle of an interval
+ interval = self.series.get( 'test', 'minute', timestamp=_time(100) )
+ assert_equals( map(_time, [60]), interval.keys() )
+ assert_equals( 119, interval[_time(60)] )
+
+ # no matching interval, returns no with empty value list
+ interval = self.series.get( 'test', 'minute' )
+ assert_equals( 1, len(interval) )
+ assert_equals( 0, interval.values()[0] )
+
+ ###
+ ### with resolution, optionally condensed
+ ###
+ interval = self.series.get( 'test', 'hour', timestamp=_time(100) )
+ assert_equals( 60, len(interval) )
+ assert_equals( 119, interval[_time(60)] )
+
+ interval = self.series.get( 'test', 'hour', timestamp=_time(100), condensed=True )
+ assert_equals( 1, len(interval) )
+ assert_equals( 3599, interval[_time(0)] )
+
+ interval = self.series.get( 'test', 'hour', timestamp=_time(4000), condensed=True )
+ assert_equals( 1, len(interval) )
+ assert_equals( 7199, interval[_time(3600)] )
+
+ def test_get_joined(self):
+ # put some data in the first minutes of each hour for test1, and then for
+ # a few more minutes in test2
+ for t in xrange(1, 120):
+ self.series.insert( 'test1', t, timestamp=_time(t) )
+ self.series.insert( 'test2', t, timestamp=_time(t) )
+ for t in xrange(3600, 3720):
+ self.series.insert( 'test1', t, timestamp=_time(t) )
+ self.series.insert( 'test2', t, timestamp=_time(t) )
+ for t in xrange(120, 240):
+ self.series.insert( 'test1', t, timestamp=_time(t) )
+ for t in xrange(3721, 3840):
+ self.series.insert( 'test1', t, timestamp=_time(t) )
+
+ ###
+ ### no resolution, condensed has no impact
+ ###
+ # interval with 2 series worth of data
+ interval = self.series.get( ['test1','test2'], 'minute', timestamp=_time(100) )
+ assert_equals( [_time(60)], interval.keys() )
+ assert_equals( 119, interval[_time(60)] )
+
+ # interval with 1 series worth of data
+ interval = self.series.get( ['test1','test2'], 'minute', timestamp=_time(122) )
+ assert_equals( [_time(120)], interval.keys() )
+ assert_equals( 179, interval[_time(120)] )
+
+ # no matching interval, returns no with empty value list
+ interval = self.series.get( ['test1','test2'], 'minute' )
+ assert_equals( 1, len(interval) )
+ assert_equals( None, interval.values()[0] )
+
+ ###
+ ### with resolution, optionally condensed
+ ###
+ interval = self.series.get( ['test1','test2'], 'hour', timestamp=_time(100) )
+ assert_equals( map(_time,[0,60,120,180]), interval.keys() )
+ assert_equals( 59, interval[_time(0)] )
+ assert_equals( 119, interval[_time(60)] )
+ assert_equals( 179, interval[_time(120)] )
+ assert_equals( 239, interval[_time(180)] )
+
+ interval = self.series.get( ['test1','test2'], 'hour', timestamp=_time(100), condensed=True )
+ assert_equals( [_time(0)], interval.keys() )
+ assert_equals( 239, interval[_time(0)] )
+
+ def test_series(self):
+ # 2 hours worth of data
+ for t in xrange(1, 7200):
+ self.series.insert( 'test', t, timestamp=_time(t) )
+
+ ###
+ ### no resolution, condensed has no impact
+ ###
+ interval = self.series.series( 'test', 'minute', end=_time(250) )
+ assert_equals( map(_time, [0,60,120,180,240]), interval.keys() )
+ assert_equals( 59, interval[_time(0)] )
+ assert_equals( 119, interval[_time(60)] )
+
+ interval = self.series.series( 'test', 'minute', steps=2, end=_time(250) )
+ assert_equals( map(_time, [180,240]), interval.keys() )
+ assert_equals( 299, interval[_time(240)] )
+
+ # with collapse
+ interval = self.series.series( 'test', 'minute', end=_time(250), collapse=True )
+ assert_equals( map(_time, [0]), interval.keys() )
+ assert_equals( 299, interval[_time(0)] )
+
+ ###
+ ### with resolution
+ ###
+ interval = self.series.series( 'test', 'hour', end=_time(250) )
+ assert_equals( 1, len(interval) )
+ assert_equals( 60, len(interval[_time(0)]) )
+ assert_equals( 59, interval[_time(0)][_time(0)] )
+ assert_equals( 119, interval[_time(0)][_time(60)] )
+
+ # single step, last one
+ interval = self.series.series( 'test', 'hour', condensed=True, end=_time(4200) )
+ assert_equals( 1, len(interval) )
+ assert_equals( 7199, interval[_time(3600)] )
+
+ interval = self.series.series( 'test', 'hour', condensed=True, end=_time(4200), steps=2 )
+ assert_equals( map(_time, [0,3600]), interval.keys() )
+ assert_equals( 3599, interval[_time(0)] )
+ assert_equals( 7199, interval[_time(3600)] )
+
+ # with collapse
+ interval = self.series.series( 'test', 'hour', condensed=True, end=_time(4200), steps=2, collapse=True )
+ assert_equals( map(_time, [0]), interval.keys() )
+ assert_equals( 7199, interval[_time(0)] )
+
+ def test_series_joined(self):
+ # put some data in the first minutes of each hour for test1, and then for
+ # a few more minutes in test2
+ for t in xrange(1, 120):
+ self.series.insert( 'test1', t, timestamp=_time(t) )
+ self.series.insert( 'test2', t, timestamp=_time(t) )
+ for t in xrange(3600, 3720):
+ self.series.insert( 'test1', t, timestamp=_time(t) )
+ self.series.insert( 'test2', t, timestamp=_time(t) )
+ for t in xrange(120, 240):
+ self.series.insert( 'test1', t, timestamp=_time(t) )
+ for t in xrange(3720, 3840):
+ self.series.insert( 'test1', t, timestamp=_time(t) )
+
+ ###
+ ### no resolution, condensed has no impact
+ ###
+ interval = self.series.series( ['test1','test2'], 'minute', end=_time(250) )
+ assert_equals( map(_time,[0,60,120,180,240]), interval.keys() )
+ assert_equals( 59, interval[_time(0)] )
+ assert_equals( 119, interval[_time(60)] )
+ assert_equals( 179, interval[_time(120)] )
+ assert_equals( 239, interval[_time(180)] )
+ assert_equals( None, interval[_time(240)] )
+
+ # no matching interval, returns no with empty value list
+ interval = self.series.series( ['test1','test2'], 'minute', start=time.time(), steps=2 )
+ assert_equals( 2, len(interval) )
+ assert_equals( None, interval.values()[0] )
+
+ # with collapsed
+ interval = self.series.series( ['test1','test2'], 'minute', end=_time(250), collapse=True )
+ assert_equals( [_time(0)], interval.keys() )
+ assert_equals( 239, interval[_time(0)] )
+
+ ###
+ ### with resolution, optionally condensed
+ ###
+ interval = self.series.series( ['test1','test2'], 'hour', end=_time(250) )
+ assert_equals( 1, len(interval) )
+ assert_equals( map(_time,[0,60,120,180]), interval[_time(0)].keys() )
+ assert_equals( 4, len(interval[_time(0)]) )
+ assert_equals( 59, interval[_time(0)][_time(0)] )
+ assert_equals( 119, interval[_time(0)][_time(60)] )
+ assert_equals( 179, interval[_time(0)][_time(120)] )
+ assert_equals( 239, interval[_time(0)][_time(180)] )
+
+ # condensed
+ interval = self.series.series( ['test1','test2'], 'hour', end=_time(250), condensed=True )
+ assert_equals( [_time(0)], interval.keys() )
+ assert_equals( 239, interval[_time(0)] )
+
+ # with collapsed
+ interval = self.series.series( ['test1','test2'], 'hour', condensed=True, end=_time(4200), steps=2, collapse=True )
+ assert_equals( map(_time, [0]), interval.keys() )
+ assert_equals( 3839, interval[_time(0)] )
diff --git a/test/functional/gregorian_helper.py b/test/functional/gregorian_helper.py
new file mode 100644
index 0000000..ce3863d
--- /dev/null
+++ b/test/functional/gregorian_helper.py
@@ -0,0 +1,148 @@
+from helper_helper import *
+from helper_helper import _time
+
+@unittest.skipUnless( os.environ.get('TEST_GREGORIAN','true').lower()=='true', 'skipping gregorian' )
+class GregorianHelper(Chai):
+ '''Test that Gregorian data is working right.'''
+ def setUp(self):
+ super(GregorianHelper,self).setUp()
+
+ self.series = Timeseries(self.client, type='series', prefix='kairos',
+ read_func=int, #write_func=str,
+ intervals={
+ 'daily' : {
+ 'step' : 'daily',
+ 'steps' : 5,
+ },
+ 'weekly' : {
+ 'step' : 'weekly',
+ 'resolution' : 60,
+ },
+ 'monthly' : {
+ 'step' : 'monthly',
+ },
+ 'yearly' : {
+ 'step' : 'yearly',
+ }
+ } )
+ self.series.delete_all()
+
+ def tearDown(self):
+ self.series.delete_all()
+
+ def test_insert_multiple_intervals_after(self):
+ ts1 = _time(0)
+ ts2 = self.series._intervals['weekly']['i_calc'].normalize(ts1, 1)
+ ts3 = self.series._intervals['weekly']['i_calc'].normalize(ts1, 2)
+ assert_not_equals( ts1, ts2 )
+
+ self.series.insert( 'test', 32, timestamp=ts1, intervals=1 )
+
+ interval_1 = self.series.get( 'test', 'weekly', timestamp=ts1 )
+ assert_equals( [32], interval_1[ts1] )
+
+ interval_2 = self.series.get( 'test', 'weekly', timestamp=ts2 )
+ assert_equals( [32], interval_2[ts2] )
+
+ self.series.insert( 'test', 42, timestamp=ts1, intervals=2 )
+
+ interval_1 = self.series.get( 'test', 'weekly', timestamp=ts1 )
+ assert_equals( [32,42], interval_1[ts1] )
+ interval_2 = self.series.get( 'test', 'weekly', timestamp=ts2 )
+ assert_equals( [32,42], interval_2[ts2] )
+ interval_3 = self.series.get( 'test', 'weekly', timestamp=ts3 )
+ assert_equals( [42], interval_3[ts3] )
+
+ def test_insert_multiple_intervals_before(self):
+ ts1 = _time(0)
+ ts2 = self.series._intervals['weekly']['i_calc'].normalize(ts1, -1)
+ ts3 = self.series._intervals['weekly']['i_calc'].normalize(ts1, -2)
+ assert_not_equals( ts1, ts2 )
+
+ self.series.insert( 'test', 32, timestamp=ts1, intervals=-1 )
+
+ interval_1 = self.series.get( 'test', 'weekly', timestamp=ts1 )
+ assert_equals( [32], interval_1[ts1] )
+
+ interval_2 = self.series.get( 'test', 'weekly', timestamp=ts2 )
+ assert_equals( [32], interval_2[ts2] )
+
+ self.series.insert( 'test', 42, timestamp=ts1, intervals=-2 )
+
+ interval_1 = self.series.get( 'test', 'weekly', timestamp=ts1 )
+ assert_equals( [32,42], interval_1[ts1] )
+ interval_2 = self.series.get( 'test', 'weekly', timestamp=ts2 )
+ assert_equals( [32,42], interval_2[ts2] )
+ interval_3 = self.series.get( 'test', 'weekly', timestamp=ts3 )
+ assert_equals( [42], interval_3[ts3] )
+
+ def test_get(self):
+ for day in range(0,365):
+ d = datetime(year=2038, month=1, day=1) + timedelta(days=day)
+ t = time.mktime( d.timetuple() )
+ self.series.insert( 'test', 1, t )
+ feb1 = long( time.mktime( datetime(year=2038,month=2,day=1).timetuple() ) )
+
+ data = self.series.get('test', 'daily', timestamp=feb1)
+ assert_equals( [1], data[feb1] )
+
+ data = self.series.get('test', 'weekly', timestamp=feb1)
+ assert_equals( 7, len(data) )
+ assert_equals( [1], data.values()[0] )
+
+ data = self.series.get('test', 'weekly', timestamp=feb1, condensed=True)
+ assert_equals( 1, len(data) )
+ assert_equals( 7*[1], data.values()[0] )
+
+ data = self.series.get('test', 'monthly', timestamp=feb1)
+ assert_equals( 28, len(data[feb1]) )
+
+ data = self.series.get('test', 'yearly', timestamp=feb1)
+ assert_equals( 365, len(data.items()[0][1]) )
+
+ def test_series(self):
+ for day in range(0,2*365):
+ d = datetime(year=2038, month=1, day=1) + timedelta(days=day)
+ t = time.mktime( d.timetuple() )
+ self.series.insert( 'test', 1, t )
+
+ start = long( time.mktime( datetime(year=2038,month=1,day=1).timetuple() ) )
+ end = long( time.mktime( datetime(year=2038,month=12,day=31).timetuple() ) )
+
+ data = self.series.series('test', 'daily', start=start, end=end)
+ assert_equals( 365, len(data) )
+ assert_equals( [1], data.values()[0] )
+ assert_equals( [1], data.values()[-1] )
+
+ data = self.series.series('test', 'weekly', start=start, end=end)
+ assert_equals( 53, len(data) )
+ assert_equals( 2, len(data.values()[0]) )
+ assert_equals( 7, len(data.values()[1]) )
+ assert_equals( 6, len(data.values()[-1]) )
+ assert_equals( [1], data.values()[0].values()[0] )
+ assert_equals( [1], data.values()[-1].values()[0] )
+
+ data = self.series.series('test', 'weekly', start=start, end=end, condensed=True)
+ assert_equals( 53, len(data) )
+ assert_equals( 2*[1], data.values()[0] )
+ assert_equals( 7*[1], data.values()[1] )
+ assert_equals( 6*[1], data.values()[-1] )
+
+ data = self.series.series('test', 'monthly', start=start, end=end)
+ assert_equals( 12, len(data) )
+ assert_equals( 31, len(data.values()[0]) ) # jan
+ assert_equals( 28, len(data.values()[1]) ) # feb
+ assert_equals( 30, len(data.values()[3]) ) # april
+
+ data = self.series.series('test', 'yearly', start=start, end=end)
+ assert_equals( 1, len(data) )
+ assert_equals( 365, len(data.values()[0]) )
+
+ data = self.series.series('test', 'yearly', start=start, steps=2)
+ assert_equals( 2, len(data) )
+ assert_equals( 365, len(data.values()[0]) )
+
+ data = self.series.series('test', 'yearly', end=end, steps=2)
+ assert_equals( 2, len(data) )
+ assert_equals( [], data.values()[0] )
+ assert_equals( 365, len(data.values()[1]) )
diff --git a/test/functional/helper_helper.py b/test/functional/helper_helper.py
new file mode 100644
index 0000000..b022eb8
--- /dev/null
+++ b/test/functional/helper_helper.py
@@ -0,0 +1,13 @@
+import os
+import time
+import unittest
+from datetime import *
+
+from chai import Chai
+
+from kairos.timeseries import *
+
+# mongo expiry requires absolute time vs. redis ttls, so adjust it in whole hours
+def _time(t):
+ return (500000*3600)+t
+
diff --git a/test/functional/helpers.py b/test/functional/helpers.py
index e26fada..5b9a45d 100644
--- a/test/functional/helpers.py
+++ b/test/functional/helpers.py
@@ -1,499 +1,12 @@
'''
Implementation of functional tests independent of backend
'''
-import time
-from datetime import *
-from chai import Chai
-
-from kairos.timeseries import *
-
-# mongo expiry requires absolute time vs. redis ttls, so adjust it in whole hours
-def _time(t):
- return (500000*3600)+t
-
-class GregorianTest(Chai):
- '''Test that Gregorian data is working right.'''
- def setUp(self):
- super(GregorianTest,self).setUp()
-
- self.series = Timeseries(self.client, type='series', prefix='kairos',
- read_func=int, #write_func=str,
- intervals={
- 'daily' : {
- 'step' : 'daily',
- 'steps' : 5,
- },
- 'weekly' : {
- 'step' : 'weekly',
- 'resolution' : 60,
- },
- 'monthly' : {
- 'step' : 'monthly',
- },
- 'yearly' : {
- 'step' : 'yearly',
- }
- } )
- self.series.delete('test')
-
- def test_get(self):
- for day in range(0,365):
- d = datetime(year=2038, month=1, day=1) + timedelta(days=day)
- t = time.mktime( d.timetuple() )
- self.series.insert( 'test', 1, t )
- feb1 = long( time.mktime( datetime(year=2038,month=2,day=1).timetuple() ) )
-
- data = self.series.get('test', 'daily', timestamp=feb1)
- assert_equals( [1], data[feb1] )
-
- data = self.series.get('test', 'weekly', timestamp=feb1)
- assert_equals( 7, len(data) )
- assert_equals( [1], data.values()[0] )
-
- data = self.series.get('test', 'weekly', timestamp=feb1, condensed=True)
- assert_equals( 1, len(data) )
- assert_equals( 7*[1], data.values()[0] )
-
- data = self.series.get('test', 'monthly', timestamp=feb1)
- assert_equals( 28, len(data[feb1]) )
-
- data = self.series.get('test', 'yearly', timestamp=feb1)
- assert_equals( 365, len(data.items()[0][1]) )
-
- def test_series(self):
- for day in range(0,2*365):
- d = datetime(year=2038, month=1, day=1) + timedelta(days=day)
- t = time.mktime( d.timetuple() )
- self.series.insert( 'test', 1, t )
-
- start = long( time.mktime( datetime(year=2038,month=1,day=1).timetuple() ) )
- end = long( time.mktime( datetime(year=2038,month=12,day=31).timetuple() ) )
-
- data = self.series.series('test', 'daily', start=start, end=end)
- assert_equals( 365, len(data) )
- assert_equals( [1], data.values()[0] )
- assert_equals( [1], data.values()[-1] )
-
- data = self.series.series('test', 'weekly', start=start, end=end)
- assert_equals( 53, len(data) )
- assert_equals( 2, len(data.values()[0]) )
- assert_equals( 7, len(data.values()[1]) )
- assert_equals( 6, len(data.values()[-1]) )
- assert_equals( [1], data.values()[0].values()[0] )
- assert_equals( [1], data.values()[-1].values()[0] )
-
- data = self.series.series('test', 'weekly', start=start, end=end, condensed=True)
- assert_equals( 53, len(data) )
- assert_equals( 2*[1], data.values()[0] )
- assert_equals( 7*[1], data.values()[1] )
- assert_equals( 6*[1], data.values()[-1] )
-
- data = self.series.series('test', 'monthly', start=start, end=end)
- assert_equals( 12, len(data) )
- assert_equals( 31, len(data.values()[0]) ) # jan
- assert_equals( 28, len(data.values()[1]) ) # feb
- assert_equals( 30, len(data.values()[3]) ) # april
-
- data = self.series.series('test', 'yearly', start=start, end=end)
- assert_equals( 1, len(data) )
- assert_equals( 365, len(data.values()[0]) )
-
- data = self.series.series('test', 'yearly', start=start, steps=2)
- assert_equals( 2, len(data) )
- assert_equals( 365, len(data.values()[0]) )
-
- data = self.series.series('test', 'yearly', end=end, steps=2)
- assert_equals( 2, len(data) )
- assert_equals( [], data.values()[0] )
- assert_equals( 365, len(data.values()[1]) )
-
-class SeriesTest(Chai):
-
- def setUp(self):
- super(SeriesTest,self).setUp()
-
- self.series = Timeseries(self.client, type='series', prefix='kairos',
- read_func=int, #write_func=str,
- intervals={
- 'minute' : {
- 'step' : 60,
- 'steps' : 5,
- },
- 'hour' : {
- 'step' : 3600,
- 'resolution' : 60,
- }
- } )
- self.series.delete('test')
-
- def test_get(self):
- # 2 hours worth of data, value is same asV timestamp
- for t in xrange(1, 7200):
- self.series.insert( 'test', t, timestamp=_time(t) )
-
- ###
- ### no resolution, condensed has no impact
- ###
- # middle of an interval
- interval = self.series.get( 'test', 'minute', timestamp=_time(100) )
- assert_equals( [_time(60)], interval.keys() )
- assert_equals( list(range(60,120)), interval[_time(60)] )
-
- # end of an interval
- interval = self.series.get( 'test', 'minute', timestamp=_time(59) )
- assert_equals( [_time(0)], interval.keys() )
- assert_equals( list(range(1,60)), interval[_time(0)] )
-
- # no matching interval, returns no with empty value list
- interval = self.series.get( 'test', 'minute' )
- assert_equals( 1, len(interval) )
- assert_equals( 0, len(interval.values()[0]) )
-
- # with transforms
- interval = self.series.get( 'test', 'minute', timestamp=_time(100), transform='count' )
- assert_equals( 60, interval[_time(60)] )
-
- interval = self.series.get( 'test', 'minute', timestamp=_time(100), transform=['min','max'] )
- assert_equals( {'min':60, 'max':119}, interval[_time(60)] )
-
- ###
- ### with resolution, optionally condensed
- ###
- interval = self.series.get( 'test', 'hour', timestamp=_time(100) )
- assert_equals( 60, len(interval) )
- assert_equals( list(range(60,120)), interval[_time(60)] )
-
- interval = self.series.get( 'test', 'hour', timestamp=_time(100), condensed=True )
- assert_equals( 1, len(interval) )
- assert_equals( list(range(1,3600)), interval[_time(0)] )
-
- # with transforms
- interval = self.series.get( 'test', 'hour', timestamp=_time(100), transform='count' )
- assert_equals( 60, interval[_time(60)] )
-
- interval = self.series.get( 'test', 'hour', timestamp=_time(100), transform=['min','max'], condensed=True )
- assert_equals( {'min':1, 'max':3599}, interval[_time(0)] )
-
- def test_series(self):
- # 2 hours worth of data, value is same asV timestamp
- for t in xrange(1, 7200):
- self.series.insert( 'test', t, timestamp=_time(t) )
-
- ###
- ### no resolution, condensed has no impact
- ###
- interval = self.series.series( 'test', 'minute', end=_time(250) )
- assert_equals( map(_time,[0,60,120,180,240]), interval.keys() )
- assert_equals( list(range(1,60)), interval[_time(0)] )
- assert_equals( list(range(240,300)), interval[_time(240)] )
-
- interval = self.series.series( 'test', 'minute', steps=2, end=_time(250) )
- assert_equals( map(_time,[180,240]), interval.keys() )
- assert_equals( list(range(240,300)), interval[_time(240)] )
-
- # with transforms
- interval = self.series.series( 'test', 'minute', end=_time(250), transform=['min','count'] )
- assert_equals( map(_time,[0,60,120,180,240]), interval.keys() )
- assert_equals( {'min':1, 'count':59}, interval[_time(0)] )
- assert_equals( {'min':240, 'count':60}, interval[_time(240)] )
-
- ###
- ### with resolution
- ###
- interval = self.series.series( 'test', 'hour', end=_time(250) )
- assert_equals( 1, len(interval) )
- assert_equals( 60, len(interval[_time(0)]) )
- assert_equals( list(range(1,60)), interval[_time(0)][_time(0)] )
-
- interval = self.series.series( 'test', 'hour', end=_time(250), transform=['count','max'] )
- assert_equals( 1, len(interval) )
- assert_equals( 60, len(interval[_time(0)]) )
- assert_equals( {'max':59, 'count':59}, interval[_time(0)][_time(0)] )
-
- # single step, last one
- interval = self.series.series( 'test', 'hour', condensed=True, end=_time(4200) )
- assert_equals( 1, len(interval) )
- assert_equals( 3600, len(interval[_time(3600)]) )
- assert_equals( list(range(3600,7200)), interval[_time(3600)] )
-
- interval = self.series.series( 'test', 'hour', condensed=True, end=_time(4200), steps=2 )
- assert_equals( map(_time, [0,3600]), interval.keys() )
- assert_equals( 3599, len(interval[_time(0)]) )
- assert_equals( 3600, len(interval[_time(3600)]) )
- assert_equals( list(range(3600,7200)), interval[_time(3600)] )
-
- # with transforms
- interval = self.series.series( 'test', 'hour', condensed=True, end=_time(4200), transform=['min','max'] )
- assert_equals( 1, len(interval) )
- assert_equals( {'min':3600, 'max':7199}, interval[_time(3600)] )
-
-class HistogramTest(Chai):
-
- def setUp(self):
- super(HistogramTest,self).setUp()
-
- self.series = Timeseries(self.client, type='histogram', prefix='kairos',
- read_func=int,
- intervals={
- 'minute' : {
- 'step' : 60,
- 'steps' : 5,
- },
- 'hour' : {
- 'step' : 3600,
- 'resolution' : 60,
- }
- } )
- self.series.delete('test')
-
- def test_get(self):
- # 2 hours worth of data, value is same asV timestamp
- for t in xrange(1, 7200):
- self.series.insert( 'test', t/2, timestamp=_time(t) )
-
- ###
- ### no resolution, condensed has no impact
- ###
- # middle of an interval
- interval = self.series.get( 'test', 'minute', timestamp=_time(100) )
- assert_equals( [_time(60)], interval.keys() )
- keys = list(range(30,60))
- assert_equals( keys, interval[_time(60)].keys() )
- for k in keys:
- assert_equals( 2, interval[_time(60)][k] )
-
- # no matching interval, returns no with empty value list
- interval = self.series.get( 'test', 'minute' )
- assert_equals( 1, len(interval) )
- assert_equals( 0, len(interval.values()[0]) )
-
- ###
- ### with resolution, optionally condensed
- ###
- interval = self.series.get( 'test', 'hour', timestamp=_time(100) )
- keys = list(range(30,60))
- assert_equals( 60, len(interval) )
- assert_equals( keys, interval[_time(60)].keys() )
-
- interval = self.series.get( 'test', 'hour', timestamp=_time(100), condensed=True )
- assert_equals( 1, len(interval) )
- assert_equals( list(range(0,1800)), interval[_time(0)].keys() )
-
- def test_series(self):
- # 2 hours worth of data, value is same asV timestamp
- for t in xrange(1, 7200):
- self.series.insert( 'test', t/2, timestamp=_time(t) )
-
- ###
- ### no resolution, condensed has no impact
- ###
- interval = self.series.series( 'test', 'minute', end=_time(250) )
- assert_equals( map(_time, [0,60,120,180,240]), interval.keys() )
- assert_equals( list(range(0,30)), sorted(interval[_time(0)].keys()) )
- assert_equals( 1, interval[_time(0)][0] )
- for k in xrange(1,30):
- assert_equals(2, interval[_time(0)][k])
- assert_equals( list(range(120,150)), sorted(interval[_time(240)].keys()) )
- for k in xrange(120,150):
- assert_equals(2, interval[_time(240)][k])
-
- interval = self.series.series( 'test', 'minute', steps=2, end=_time(250) )
- assert_equals( map(_time, [180,240]), interval.keys() )
- assert_equals( list(range(120,150)), sorted(interval[_time(240)].keys()) )
-
- ###
- ### with resolution
- ###
- interval = self.series.series( 'test', 'hour', end=_time(250) )
- assert_equals( 1, len(interval) )
- assert_equals( 60, len(interval[_time(0)]) )
- assert_equals( list(range(0,30)), sorted(interval[_time(0)][_time(0)].keys()) )
-
- # single step, last one
- interval = self.series.series( 'test', 'hour', condensed=True, end=_time(4200) )
- assert_equals( 1, len(interval) )
- assert_equals( 1800, len(interval[_time(3600)]) )
- assert_equals( list(range(1800,3600)), sorted(interval[_time(3600)].keys()) )
-
- interval = self.series.series( 'test', 'hour', condensed=True, end=_time(4200), steps=2 )
- assert_equals( map(_time, [0,3600]), interval.keys() )
- assert_equals( 1800, len(interval[_time(0)]) )
- assert_equals( 1800, len(interval[_time(3600)]) )
- assert_equals( list(range(1800,3600)), sorted(interval[_time(3600)].keys()) )
-
-class CountTest(Chai):
-
- def setUp(self):
- super(CountTest,self).setUp()
-
- self.series = Timeseries(self.client, type='count', prefix='kairos',
- read_func=int,
- intervals={
- 'minute' : {
- 'step' : 60,
- 'steps' : 5,
- },
- 'hour' : {
- 'step' : 3600,
- 'resolution' : 60,
- }
- } )
- self.series.delete('test')
-
- def test_get(self):
- # 2 hours worth of data
- for t in xrange(1, 7200):
- self.series.insert( 'test', 1, timestamp=_time(t) )
-
- ###
- ### no resolution, condensed has no impact
- ###
- # middle of an interval
- interval = self.series.get( 'test', 'minute', timestamp=_time(100) )
- assert_equals( map(_time, [60]), interval.keys() )
- assert_equals( 60, interval[_time(60)] )
-
- # no matching interval, returns no with empty value list
- interval = self.series.get( 'test', 'minute' )
- assert_equals( 1, len(interval) )
- assert_equals( 0, interval.values()[0] )
-
- ###
- ### with resolution, optionally condensed
- ###
- interval = self.series.get( 'test', 'hour', timestamp=_time(100) )
- assert_equals( 60, len(interval) )
- assert_equals( 60, interval[_time(60)] )
-
- interval = self.series.get( 'test', 'hour', timestamp=_time(100), condensed=True )
- assert_equals( 1, len(interval) )
- assert_equals( 3599, interval[_time(0)] )
-
- interval = self.series.get( 'test', 'hour', timestamp=_time(4000), condensed=True )
- assert_equals( 1, len(interval) )
- assert_equals( 3600, interval[_time(3600)] )
-
- def test_series(self):
- # 2 hours worth of data
- for t in xrange(1, 7200):
- self.series.insert( 'test', 1, timestamp=_time(t) )
-
- ###
- ### no resolution, condensed has no impact
- ###
- interval = self.series.series( 'test', 'minute', end=_time(250) )
- assert_equals( map(_time, [0,60,120,180,240]), interval.keys() )
- assert_equals( 59, interval[_time(0)] )
- assert_equals( 60, interval[_time(60)] )
-
- interval = self.series.series( 'test', 'minute', steps=2, end=_time(250) )
- assert_equals( map(_time, [180,240]), interval.keys() )
- assert_equals( 60, interval[_time(240)] )
-
- ###
- ### with resolution
- ###
- interval = self.series.series( 'test', 'hour', end=_time(250) )
- assert_equals( 1, len(interval) )
- assert_equals( 60, len(interval[_time(0)]) )
- assert_equals( 59, interval[_time(0)][_time(0)] )
- assert_equals( 60, interval[_time(0)][_time(60)] )
-
- # single step, last one
- interval = self.series.series( 'test', 'hour', condensed=True, end=_time(4200) )
- assert_equals( 1, len(interval) )
- assert_equals( 3600, interval[_time(3600)] )
-
- interval = self.series.series( 'test', 'hour', condensed=True, end=_time(4200), steps=2 )
- assert_equals( map(_time, [0,3600]), interval.keys() )
- assert_equals( 3599, interval[_time(0)] )
- assert_equals( 3600, interval[_time(3600)] )
-
-class GaugeTest(Chai):
-
- def setUp(self):
- super(GaugeTest,self).setUp()
-
- self.series = Timeseries(self.client, type='gauge', prefix='kairos',
- read_func=lambda v: int(v or 0),
- intervals={
- 'minute' : {
- 'step' : 60,
- 'steps' : 5,
- },
- 'hour' : {
- 'step' : 3600,
- 'resolution' : 60,
- }
- } )
- self.series.delete('test')
-
- def test_get(self):
- # 2 hours worth of data
- for t in xrange(1, 7200):
- self.series.insert( 'test', t, timestamp=_time(t) )
-
- ###
- ### no resolution, condensed has no impact
- ###
- # middle of an interval
- interval = self.series.get( 'test', 'minute', timestamp=_time(100) )
- assert_equals( map(_time, [60]), interval.keys() )
- assert_equals( 119, interval[_time(60)] )
-
- # no matching interval, returns no with empty value list
- interval = self.series.get( 'test', 'minute' )
- assert_equals( 1, len(interval) )
- assert_equals( 0, interval.values()[0] )
-
- ###
- ### with resolution, optionally condensed
- ###
- interval = self.series.get( 'test', 'hour', timestamp=_time(100) )
- assert_equals( 60, len(interval) )
- assert_equals( 119, interval[_time(60)] )
-
- interval = self.series.get( 'test', 'hour', timestamp=_time(100), condensed=True )
- assert_equals( 1, len(interval) )
- assert_equals( range(59,3600,60), interval[_time(0)] )
-
- interval = self.series.get( 'test', 'hour', timestamp=_time(4000), condensed=True )
- assert_equals( 1, len(interval) )
- assert_equals( range(3659,7200,60), interval[_time(3600)] )
-
- def test_series(self):
- # 2 hours worth of data
- for t in xrange(1, 7200):
- self.series.insert( 'test', t, timestamp=_time(t) )
-
- ###
- ### no resolution, condensed has no impact
- ###
- interval = self.series.series( 'test', 'minute', end=_time(250) )
- assert_equals( map(_time, [0,60,120,180,240]), interval.keys() )
- assert_equals( 59, interval[_time(0)] )
- assert_equals( 119, interval[_time(60)] )
-
- interval = self.series.series( 'test', 'minute', steps=2, end=_time(250) )
- assert_equals( map(_time, [180,240]), interval.keys() )
- assert_equals( 299, interval[_time(240)] )
-
- ###
- ### with resolution
- ###
- interval = self.series.series( 'test', 'hour', end=_time(250) )
- assert_equals( 1, len(interval) )
- assert_equals( 60, len(interval[_time(0)]) )
- assert_equals( 59, interval[_time(0)][_time(0)] )
- assert_equals( 119, interval[_time(0)][_time(60)] )
-
- # single step, last one
- interval = self.series.series( 'test', 'hour', condensed=True, end=_time(4200) )
- assert_equals( 1, len(interval) )
- assert_equals( range(3659,7200,60), interval[_time(3600)] )
-
- interval = self.series.series( 'test', 'hour', condensed=True, end=_time(4200), steps=2 )
- assert_equals( map(_time, [0,3600]), interval.keys() )
- assert_equals( range(59,3600,60), interval[_time(0)] )
- assert_equals( range(3659,7200,60), interval[_time(3600)] )
+from .api_helper import ApiHelper
+from .gregorian_helper import GregorianHelper
+from .series_helper import SeriesHelper
+from .histogram_helper import HistogramHelper
+from .count_helper import CountHelper
+from .set_helper import SetHelper
+from .gauge_helper import GaugeHelper
+from .helper_helper import *
diff --git a/test/functional/histogram_helper.py b/test/functional/histogram_helper.py
new file mode 100644
index 0000000..1e06603
--- /dev/null
+++ b/test/functional/histogram_helper.py
@@ -0,0 +1,331 @@
+from helper_helper import *
+from helper_helper import _time
+
+from collections import OrderedDict
+
+@unittest.skipUnless( os.environ.get('TEST_HISTOGRAM','true').lower()=='true', 'skipping histogram' )
+class HistogramHelper(Chai):
+
+ def setUp(self):
+ super(HistogramHelper,self).setUp()
+
+ self.series = Timeseries(self.client, type='histogram', prefix='kairos',
+ read_func=int,
+ intervals={
+ 'minute' : {
+ 'step' : 60,
+ 'steps' : 5,
+ },
+ 'hour' : {
+ 'step' : 3600,
+ 'resolution' : 60,
+ }
+ } )
+ self.series.delete_all()
+
+ def tearDown(self):
+ self.series.delete_all()
+
+ def test_bulk_insert(self):
+ inserts = {
+ None : { 'test1':[1,2,3], 'test2':[4,5,6] },
+ _time(0) : { 'test1':[1,2,3], 'test2':[4,5,6], 'test3':[7,8,9] },
+ _time(30) : { 'test1':[1,2,3], 'test2':[4,5,6] },
+ _time(60) : { 'test1':[1,2,3], 'test3':[7,8,9] }
+ }
+ self.series.bulk_insert( inserts )
+
+ t1_i1 = self.series.get('test1', 'minute', timestamp=_time(0))
+ assert_equals( {1:2, 2:2, 3:2}, t1_i1[_time(0)] )
+
+ t2_i1 = self.series.get('test2', 'minute', timestamp=_time(0))
+ assert_equals( {4:2, 5:2, 6:2}, t2_i1[_time(0)] )
+
+ t3_i1 = self.series.get('test3', 'minute', timestamp=_time(0))
+ assert_equals( {7:1, 8:1, 9:1}, t3_i1[_time(0)] )
+
+ t1_i2 = self.series.get('test1', 'minute', timestamp=_time(60))
+ assert_equals( {1:1, 2:1, 3:1}, t1_i2[_time(60)] )
+
+ def test_bulk_insert_intervals_after(self):
+ a,b,c,d,e,f = 10,11,12,13,14,15
+ inserts = OrderedDict( (
+ (None , { 'test1':[1,2,3], 'test2':[4,5,6] } ),
+ (_time(0) , { 'test1':[1,2,3], 'test2':[4,5,6], 'test3':[7,8,9] } ),
+ (_time(30), { 'test1':[1,2,3], 'test2':[4,5,6] } ),
+ (_time(60), { 'test1':[a,b,c], 'test3':[d,e,f] })
+ ) )
+ self.series.bulk_insert( inserts, intervals=3 )
+
+ t1_i1 = self.series.get('test1', 'minute', timestamp=_time(0))
+ assert_equals( {1:2, 2:2, 3:2}, t1_i1[_time(0)] )
+
+ t2_i1 = self.series.get('test2', 'minute', timestamp=_time(0))
+ assert_equals( {4:2, 5:2, 6:2}, t2_i1[_time(0)] )
+
+ t3_i1 = self.series.get('test3', 'minute', timestamp=_time(0))
+ assert_equals( {7:1, 8:1, 9:1}, t3_i1[_time(0)] )
+
+ t1_i2 = self.series.get('test1', 'minute', timestamp=_time(60))
+ assert_equals( {1:2, 2:2, 3:2, a:1, b:1, c:1}, t1_i2[_time(60)] )
+
+ t3_i3 = self.series.get('test3', 'minute', timestamp=_time(120))
+ assert_equals( {7:1, 8:1, 9:1, d:1, e:1, f:1}, t3_i3[_time(120)] )
+
+ t3_i4 = self.series.get('test3', 'minute', timestamp=_time(180))
+ assert_equals( {7:1, 8:1, 9:1, d:1, e:1, f:1}, t3_i4[_time(180)] )
+
+ def test_bulk_insert_intervals_before(self):
+ a,b,c,d,e,f = 10,11,12,13,14,15
+ inserts = OrderedDict( (
+ (None , { 'test1':[1,2,3], 'test2':[4,5,6] } ),
+ (_time(0) , { 'test1':[1,2,3], 'test2':[4,5,6], 'test3':[7,8,9] } ),
+ (_time(30), { 'test1':[1,2,3], 'test2':[4,5,6] } ),
+ (_time(60), { 'test1':[a,b,c], 'test3':[d,e,f] })
+ ) )
+ self.series.bulk_insert( inserts, intervals=-3 )
+
+ t1_i1 = self.series.get('test1', 'minute', timestamp=_time(0))
+ assert_equals( {1:2, 2:2, 3:2, a:1, b:1, c:1}, t1_i1[_time(0)] )
+
+ t2_i1 = self.series.get('test2', 'minute', timestamp=_time(0))
+ assert_equals( {4:2, 5:2, 6:2}, t2_i1[_time(0)] )
+
+ t3_i1 = self.series.get('test3', 'minute', timestamp=_time(0))
+ assert_equals( {7:1, 8:1, 9:1, d:1, e:1, f:1}, t3_i1[_time(0)] )
+
+ t1_i2 = self.series.get('test1', 'minute', timestamp=_time(-60))
+ assert_equals( {1:2, 2:2, 3:2, a:1, b:1, c:1}, t1_i2[_time(-60)] )
+
+ t3_i3 = self.series.get('test3', 'minute', timestamp=_time(-120))
+ assert_equals( {7:1, 8:1, 9:1, d:1, e:1, f:1}, t3_i3[_time(-120)] )
+
+ t3_i4 = self.series.get('test3', 'minute', timestamp=_time(-180))
+ assert_equals( {7:1, 8:1, 9:1}, t3_i4[_time(-180)] )
+
+ def test_get(self):
+ # 2 hours worth of data, value is same asV timestamp
+ for t in xrange(1, 7200):
+ self.series.insert( 'test', t/2, timestamp=_time(t) )
+
+ ###
+ ### no resolution, condensed has no impact
+ ###
+ # middle of an interval
+ interval = self.series.get( 'test', 'minute', timestamp=_time(100) )
+ assert_equals( [_time(60)], interval.keys() )
+ keys = list(range(30,60))
+ assert_equals( keys, interval[_time(60)].keys() )
+ for k in keys:
+ assert_equals( 2, interval[_time(60)][k] )
+
+ # no matching interval, returns no with empty value list
+ interval = self.series.get( 'test', 'minute' )
+ assert_equals( 1, len(interval) )
+ assert_equals( 0, len(interval.values()[0]) )
+
+ ###
+ ### with resolution, optionally condensed
+ ###
+ interval = self.series.get( 'test', 'hour', timestamp=_time(100) )
+ keys = list(range(30,60))
+ assert_equals( 60, len(interval) )
+ assert_equals( keys, interval[_time(60)].keys() )
+
+ interval = self.series.get( 'test', 'hour', timestamp=_time(100), condensed=True )
+ assert_equals( 1, len(interval) )
+ assert_equals( list(range(0,1800)), interval[_time(0)].keys() )
+
+ def test_get_joined(self):
+ # put some data in the first minutes of each hour for test1, and then for
+ # a few more minutes in test2
+ for t in xrange(1, 120):
+ self.series.insert( 'test1', t, timestamp=_time(t) )
+ self.series.insert( 'test2', t, timestamp=_time(t) )
+ for t in xrange(3600, 3720):
+ self.series.insert( 'test1', t, timestamp=_time(t) )
+ self.series.insert( 'test2', t, timestamp=_time(t) )
+ for t in xrange(120, 240):
+ self.series.insert( 'test1', t, timestamp=_time(t) )
+ for t in xrange(3721, 3840):
+ self.series.insert( 'test1', t, timestamp=_time(t) )
+
+ ###
+ ### no resolution, condensed has no impact
+ ###
+ # interval with 2 series worth of data
+ interval = self.series.get( ['test1','test2'], 'minute', timestamp=_time(100) )
+ assert_equals( [_time(60)], interval.keys() )
+ assert_equals( dict.fromkeys(range(60,120),2), interval[_time(60)] )
+
+ # interval with 1 series worth of data
+ interval = self.series.get( ['test1','test2'], 'minute', timestamp=_time(122) )
+ assert_equals( [_time(120)], interval.keys() )
+ assert_equals( dict.fromkeys(range(120,180),1), interval[_time(120)] )
+
+ # no matching interval, returns no with empty value list
+ interval = self.series.get( ['test1','test2'], 'minute' )
+ assert_equals( 1, len(interval) )
+ assert_equals( 0, len(interval.values()[0]) )
+
+ ###
+ ### with resolution, optionally condensed
+ ###
+ interval = self.series.get( ['test1','test2'], 'hour', timestamp=_time(100) )
+ assert_equals( map(_time,[0,60,120,180]), interval.keys() )
+ assert_equals( dict.fromkeys(range(1,60), 2), interval[_time(0)] )
+ assert_equals( dict.fromkeys(range(60,120), 2), interval[_time(60)] )
+ assert_equals( dict.fromkeys(range(120,180), 1), interval[_time(120)] )
+ assert_equals( dict.fromkeys(range(180,240), 1), interval[_time(180)] )
+
+ data = dict.fromkeys(range(1,120), 2)
+ data.update( dict.fromkeys(range(120,240),1) )
+ interval = self.series.get( ['test1','test2'], 'hour', timestamp=_time(100), condensed=True )
+ assert_equals( [_time(0)], interval.keys() )
+ assert_equals( data, interval[_time(0)] )
+
+ # with transforms
+ interval = self.series.get( ['test1','test2'], 'hour', timestamp=_time(100), transform='count' )
+ assert_equals( 120, interval[_time(60)] )
+
+ interval = self.series.get( ['test1','test2'], 'hour', timestamp=_time(100), transform=['min','max','count'], condensed=True )
+ assert_equals( {'min':1, 'max':239, 'count':358}, interval[_time(0)] )
+
+ def test_series(self):
+ # 2 hours worth of data, value is same asV timestamp
+ for t in xrange(1, 7200):
+ self.series.insert( 'test', t/2, timestamp=_time(t) )
+
+ ###
+ ### no resolution, condensed has no impact
+ ###
+ interval = self.series.series( 'test', 'minute', end=_time(250) )
+ assert_equals( map(_time, [0,60,120,180,240]), interval.keys() )
+ assert_equals( list(range(0,30)), sorted(interval[_time(0)].keys()) )
+ assert_equals( 1, interval[_time(0)][0] )
+ for k in xrange(1,30):
+ assert_equals(2, interval[_time(0)][k])
+ assert_equals( list(range(120,150)), sorted(interval[_time(240)].keys()) )
+ for k in xrange(120,150):
+ assert_equals(2, interval[_time(240)][k])
+
+ interval = self.series.series( 'test', 'minute', steps=2, end=_time(250) )
+ assert_equals( map(_time, [180,240]), interval.keys() )
+ assert_equals( list(range(120,150)), sorted(interval[_time(240)].keys()) )
+
+ # with collapsed
+ interval = self.series.series( 'test', 'minute', end=_time(250), collapse=True )
+ assert_equals( map(_time, [0]), interval.keys() )
+ assert_equals( list(range(0,150)), sorted(interval[_time(0)].keys()) )
+ for k in xrange(1,150):
+ assert_equals(2, interval[_time(0)][k])
+
+ ###
+ ### with resolution
+ ###
+ interval = self.series.series( 'test', 'hour', end=_time(250) )
+ assert_equals( 1, len(interval) )
+ assert_equals( 60, len(interval[_time(0)]) )
+ assert_equals( list(range(0,30)), sorted(interval[_time(0)][_time(0)].keys()) )
+
+ # single step, last one
+ interval = self.series.series( 'test', 'hour', condensed=True, end=_time(4200) )
+ assert_equals( 1, len(interval) )
+ assert_equals( 1800, len(interval[_time(3600)]) )
+ assert_equals( list(range(1800,3600)), sorted(interval[_time(3600)].keys()) )
+
+ interval = self.series.series( 'test', 'hour', condensed=True, end=_time(4200), steps=2 )
+ assert_equals( map(_time, [0,3600]), interval.keys() )
+ assert_equals( 1800, len(interval[_time(0)]) )
+ assert_equals( 1800, len(interval[_time(3600)]) )
+ assert_equals( list(range(1800,3600)), sorted(interval[_time(3600)].keys()) )
+
+ # with collapsed
+ interval = self.series.series( 'test', 'hour', condensed=True, end=_time(4200), steps=2, collapse=True )
+ assert_equals( map(_time, [0]), interval.keys() )
+ assert_equals( 3600, len(interval[_time(0)]) )
+ assert_equals( list(range(0,3600)), sorted(interval[_time(0)].keys()) )
+
+ def test_series_joined(self):
+ # put some data in the first minutes of each hour for test1, and then for
+ # a few more minutes in test2
+ for t in xrange(1, 120):
+ self.series.insert( 'test1', t, timestamp=_time(t) )
+ self.series.insert( 'test2', t, timestamp=_time(t) )
+ for t in xrange(3600, 3720):
+ self.series.insert( 'test1', t, timestamp=_time(t) )
+ self.series.insert( 'test2', t, timestamp=_time(t) )
+ for t in xrange(120, 240):
+ self.series.insert( 'test1', t, timestamp=_time(t) )
+ for t in xrange(3720, 3840):
+ self.series.insert( 'test1', t, timestamp=_time(t) )
+
+ ###
+ ### no resolution, condensed has no impact
+ ###
+ interval = self.series.series( ['test1','test2'], 'minute', end=_time(250) )
+ assert_equals( map(_time,[0,60,120,180,240]), interval.keys() )
+ assert_equals( dict.fromkeys(range(1,60), 2), interval[_time(0)] )
+ assert_equals( dict.fromkeys(range(60,120), 2), interval[_time(60)] )
+ assert_equals( dict.fromkeys(range(120,180), 1), interval[_time(120)] )
+ assert_equals( dict.fromkeys(range(180,240), 1), interval[_time(180)] )
+ assert_equals( {}, interval[_time(240)] )
+
+ # no matching interval, returns no with empty value list
+ interval = self.series.series( ['test1','test2'], 'minute', start=time.time(), steps=2 )
+ assert_equals( 2, len(interval) )
+ assert_equals( {}, interval.values()[0] )
+
+ # with transforms
+ interval = self.series.series( ['test1','test2'], 'minute', end=_time(250), transform=['min','count'] )
+ assert_equals( map(_time,[0,60,120,180,240]), interval.keys() )
+ assert_equals( {'min':1, 'count':118}, interval[_time(0)] )
+ assert_equals( {'min':60, 'count':120}, interval[_time(60)] )
+ assert_equals( {'min':120, 'count':60}, interval[_time(120)] )
+ assert_equals( {'min':180, 'count':60}, interval[_time(180)] )
+ assert_equals( {'min':0, 'count':0}, interval[_time(240)] )
+
+ # with collapsed
+ data = dict.fromkeys(range(1,120), 2)
+ data.update( dict.fromkeys(range(120,240), 1) )
+ interval = self.series.series( ['test1','test2'], 'minute', end=_time(250), collapse=True )
+ assert_equals( [_time(0)], interval.keys() )
+ assert_equals( data, interval[_time(0)] )
+
+ # with tranforms and collapsed
+ interval = self.series.series( ['test1','test2'], 'minute', end=_time(250), transform=['min','max', 'count'], collapse=True )
+ assert_equals( [_time(0)], interval.keys() )
+ assert_equals( {'min':1, 'max':239, 'count':358}, interval[_time(0)] )
+
+ ###
+ ### with resolution, optionally condensed
+ ###
+ interval = self.series.series( ['test1','test2'], 'hour', end=_time(250) )
+ assert_equals( 1, len(interval) )
+ assert_equals( map(_time,[0,60,120,180]), interval[_time(0)].keys() )
+ assert_equals( 4, len(interval[_time(0)]) )
+ assert_equals( dict.fromkeys(range(1,60), 2), interval[_time(0)][_time(0)] )
+ assert_equals( dict.fromkeys(range(60,120), 2), interval[_time(0)][_time(60)] )
+ assert_equals( dict.fromkeys(range(120,180), 1), interval[_time(0)][_time(120)] )
+ assert_equals( dict.fromkeys(range(180,240), 1), interval[_time(0)][_time(180)] )
+
+ # condensed
+ data = dict.fromkeys(range(1,120), 2)
+ data.update( dict.fromkeys(range(120,240), 1) )
+ interval = self.series.series( ['test1','test2'], 'hour', end=_time(250), condensed=True )
+ assert_equals( [_time(0)], interval.keys() )
+ assert_equals( data, interval[_time(0)] )
+
+ # with collapsed across multiple intervals
+ data = dict.fromkeys(range(1,120), 2)
+ data.update( dict.fromkeys(range(120,240), 1) )
+ data.update( dict.fromkeys(range(3600,3720), 2) )
+ data.update( dict.fromkeys(range(3720,3840), 1) )
+ interval = self.series.series( ['test1','test2'], 'hour', condensed=True, end=_time(4200), steps=2, collapse=True )
+ assert_equals( map(_time, [0]), interval.keys() )
+ assert_equals( data, interval[_time(0)] )
+
+ # with transforms collapsed
+ interval = self.series.series( ['test1','test2'], 'hour', condensed=True, end=_time(4200), steps=2, collapse=True, transform=['min','max','count'] )
+ assert_equals( map(_time, [0]), interval.keys() )
+ assert_equals( {'min':1,'max':3839,'count':718}, interval[_time(0)] )
diff --git a/test/functional/mongo_timeseries_test.py b/test/functional/mongo_timeseries_test.py
index dccda74..95865e9 100644
--- a/test/functional/mongo_timeseries_test.py
+++ b/test/functional/mongo_timeseries_test.py
@@ -7,34 +7,51 @@
from pymongo import *
from chai import Chai
-import helpers
+from . import helpers
+from .helpers import unittest, os, Timeseries
-class MongoSeriesTest(helpers.SeriesTest):
+@unittest.skipUnless( os.environ.get('TEST_MONGO','true').lower()=='true', 'skipping mongo' )
+class MongoApiTest(helpers.ApiHelper):
def setUp(self):
self.client = MongoClient('localhost')
- super(MongoSeriesTest,self).setUp()
+ super(MongoApiTest,self).setUp()
+
+ def test_url_parse(self):
+ assert_equals( 'MongoSeries',
+ Timeseries('mongodb://localhost/kairos', type='series').__class__.__name__ )
-class MongoHistogramTest(helpers.HistogramTest):
+@unittest.skipUnless( os.environ.get('TEST_MONGO','true').lower()=='true', 'skipping mongo' )
+class MongoGregorianTest(helpers.GregorianHelper):
def setUp(self):
self.client = MongoClient('localhost')
- super(MongoHistogramTest,self).setUp()
+ super(MongoGregorianTest,self).setUp()
-class MongoCountTest(helpers.CountTest):
+@unittest.skipUnless( os.environ.get('TEST_MONGO','true').lower()=='true', 'skipping mongo' )
+class MongoSeriesTest(helpers.SeriesHelper):
def setUp(self):
self.client = MongoClient('localhost')
- super(MongoCountTest,self).setUp()
+ super(MongoSeriesTest,self).setUp()
-class MongoGaugeTest(helpers.GaugeTest):
+@unittest.skipUnless( os.environ.get('TEST_MONGO','true').lower()=='true', 'skipping mongo' )
+class MongoHistogramTest(helpers.HistogramHelper):
def setUp(self):
self.client = MongoClient('localhost')
- super(MongoGaugeTest,self).setUp()
+ super(MongoHistogramTest,self).setUp()
-class MongoGregorianTest(helpers.GregorianTest):
+@unittest.skipUnless( os.environ.get('TEST_MONGO','true').lower()=='true', 'skipping mongo' )
+class MongoCountTest(helpers.CountHelper):
def setUp(self):
self.client = MongoClient('localhost')
- super(MongoGregorianTest,self).setUp()
+ super(MongoCountTest,self).setUp()
+
+@unittest.skipUnless( os.environ.get('TEST_MONGO','true').lower()=='true', 'skipping mongo' )
+class MongoGaugeTest(helpers.GaugeHelper):
+
+ def setUp(self):
+ self.client = MongoClient('localhost')
+ super(MongoGaugeTest,self).setUp()
diff --git a/test/functional/redis_timeseries_test.py b/test/functional/redis_timeseries_test.py
index 0e7b182..9bf0ddb 100644
--- a/test/functional/redis_timeseries_test.py
+++ b/test/functional/redis_timeseries_test.py
@@ -7,34 +7,58 @@
import redis
from chai import Chai
-import helpers
+from . import helpers
+from .helpers import unittest, os, Timeseries
-class RedisSeriesTest(helpers.SeriesTest):
+@unittest.skipUnless( os.environ.get('TEST_REDIS','true').lower()=='true', 'skipping redis' )
+class RedisApiTest(helpers.ApiHelper):
+
+ def setUp(self):
+ self.client = redis.Redis('localhost')
+ super(RedisApiTest,self).setUp()
+
+ def test_url_parse(self):
+ assert_equals( 'RedisSeries',
+ Timeseries('redis://', type='series').__class__.__name__ )
+
+@unittest.skipUnless( os.environ.get('TEST_REDIS','true').lower()=='true', 'skipping redis' )
+class RedisGregorianTest(helpers.GregorianHelper):
+
+ def setUp(self):
+ self.client = redis.Redis('localhost')
+ super(RedisGregorianTest,self).setUp()
+
+@unittest.skipUnless( os.environ.get('TEST_REDIS','true').lower()=='true', 'skipping redis' )
+class RedisSeriesTest(helpers.SeriesHelper):
def setUp(self):
self.client = redis.Redis('localhost')
super(RedisSeriesTest,self).setUp()
-class RedisHistogramTest(helpers.HistogramTest):
+@unittest.skipUnless( os.environ.get('TEST_REDIS','true').lower()=='true', 'skipping redis' )
+class RedisHistogramTest(helpers.HistogramHelper):
def setUp(self):
self.client = redis.Redis('localhost')
super(RedisHistogramTest,self).setUp()
-class RedisCountTest(helpers.CountTest):
+@unittest.skipUnless( os.environ.get('TEST_REDIS','true').lower()=='true', 'skipping redis' )
+class RedisCountTest(helpers.CountHelper):
def setUp(self):
self.client = redis.Redis('localhost')
super(RedisCountTest,self).setUp()
-class RedisGaugeTest(helpers.GaugeTest):
+@unittest.skipUnless( os.environ.get('TEST_REDIS','true').lower()=='true', 'skipping redis' )
+class RedisGaugeTest(helpers.GaugeHelper):
def setUp(self):
self.client = redis.Redis('localhost')
super(RedisGaugeTest,self).setUp()
-class RedisGregorianTest(helpers.GregorianTest):
+@unittest.skipUnless( os.environ.get('TEST_REDIS','true').lower()=='true', 'skipping redis' )
+class RedisSetTest(helpers.SetHelper):
def setUp(self):
self.client = redis.Redis('localhost')
- super(RedisGregorianTest,self).setUp()
+ super(RedisSetTest,self).setUp()
diff --git a/test/functional/series_helper.py b/test/functional/series_helper.py
new file mode 100644
index 0000000..b571d43
--- /dev/null
+++ b/test/functional/series_helper.py
@@ -0,0 +1,419 @@
+from helper_helper import *
+from helper_helper import _time
+
+from collections import OrderedDict
+
+@unittest.skipUnless( os.environ.get('TEST_SERIES','true').lower()=='true', 'skipping series' )
+class SeriesHelper(Chai):
+
+ def setUp(self):
+ super(SeriesHelper,self).setUp()
+
+ self.series = Timeseries(self.client, type='series', prefix='kairos',
+ read_func=int, #write_func=str,
+ intervals={
+ 'minute' : {
+ 'step' : 60,
+ 'steps' : 5,
+ },
+ 'hour' : {
+ 'step' : 3600,
+ 'resolution' : 60,
+ }
+ } )
+ self.series.delete_all()
+
+ def tearDown(self):
+ self.series.delete_all()
+
+ def test_bulk_insert(self):
+ inserts = {
+ None : { 'test1':[1,2,3], 'test2':[4,5,6] },
+ _time(0) : { 'test1':[1,2,3], 'test2':[4,5,6], 'test3':[7,8,9] },
+ _time(30) : { 'test1':[1,2,3], 'test2':[4,5,6] },
+ _time(60) : { 'test1':[1,2,3], 'test3':[7,8,9] }
+ }
+ self.series.bulk_insert( inserts )
+
+ t1_i1 = self.series.get('test1', 'minute', timestamp=_time(0))
+ assert_equals( [1,2,3,1,2,3], t1_i1[_time(0)] )
+
+ t2_i1 = self.series.get('test2', 'minute', timestamp=_time(0))
+ assert_equals( [4,5,6,4,5,6], t2_i1[_time(0)] )
+
+ t3_i1 = self.series.get('test3', 'minute', timestamp=_time(0))
+ assert_equals( [7,8,9], t3_i1[_time(0)] )
+
+ t1_i2 = self.series.get('test1', 'minute', timestamp=_time(60))
+ assert_equals( [1,2,3], t1_i2[_time(60)] )
+
+ def test_bulk_insert_intervals_after(self):
+ a,b,c,d,e,f = 10,11,12,13,14,15
+ inserts = OrderedDict( (
+ (None , { 'test1':[1,2,3], 'test2':[4,5,6] } ),
+ (_time(0) , { 'test1':[1,2,3], 'test2':[4,5,6], 'test3':[7,8,9] } ),
+ (_time(30), { 'test1':[1,2,3], 'test2':[4,5,6] } ),
+ (_time(60), { 'test1':[a,b,c], 'test3':[d,e,f] })
+ ) )
+ self.series.bulk_insert( inserts, intervals=3 )
+
+ t1_i1 = self.series.get('test1', 'minute', timestamp=_time(0))
+ assert_equals( [1,2,3,1,2,3], t1_i1[_time(0)] )
+
+ t2_i1 = self.series.get('test2', 'minute', timestamp=_time(0))
+ assert_equals( [4,5,6,4,5,6], t2_i1[_time(0)] )
+
+ t3_i1 = self.series.get('test3', 'minute', timestamp=_time(0))
+ assert_equals( [7,8,9], t3_i1[_time(0)] )
+
+ t1_i2 = self.series.get('test1', 'minute', timestamp=_time(60))
+ assert_equals( [1,2,3,1,2,3,a,b,c], t1_i2[_time(60)] )
+
+ t3_i3 = self.series.get('test3', 'minute', timestamp=_time(120))
+ assert_equals( [7,8,9,d,e,f], t3_i3[_time(120)] )
+
+ t3_i4 = self.series.get('test3', 'minute', timestamp=_time(180))
+ assert_equals( [7,8,9,d,e,f], t3_i4[_time(180)] )
+
+ def test_bulk_insert_intervals_before(self):
+ a,b,c,d,e,f = 10,11,12,13,14,15
+ inserts = OrderedDict( (
+ (None , { 'test1':[1,2,3], 'test2':[4,5,6] } ),
+ (_time(0) , { 'test1':[1,2,3], 'test2':[4,5,6], 'test3':[7,8,9] } ),
+ (_time(30), { 'test1':[1,2,3], 'test2':[4,5,6] } ),
+ (_time(60), { 'test1':[a,b,c], 'test3':[d,e,f] })
+ ) )
+ self.series.bulk_insert( inserts, intervals=-3 )
+
+ t1_i1 = self.series.get('test1', 'minute', timestamp=_time(0))
+ assert_equals( [1,2,3,1,2,3,a,b,c], t1_i1[_time(0)] )
+
+ t2_i1 = self.series.get('test2', 'minute', timestamp=_time(0))
+ assert_equals( [4,5,6,4,5,6], t2_i1[_time(0)] )
+
+ t3_i1 = self.series.get('test3', 'minute', timestamp=_time(0))
+ assert_equals( [7,8,9,d,e,f], t3_i1[_time(0)] )
+
+ t1_i2 = self.series.get('test1', 'minute', timestamp=_time(-60))
+ assert_equals( [1,2,3,1,2,3,a,b,c], t1_i2[_time(-60)] )
+
+ t3_i3 = self.series.get('test3', 'minute', timestamp=_time(-120))
+ assert_equals( [7,8,9,d,e,f], t3_i3[_time(-120)] )
+
+ t3_i4 = self.series.get('test3', 'minute', timestamp=_time(-180))
+ assert_equals( [7,8,9], t3_i4[_time(-180)] )
+
+ def test_insert_multiple_intervals_after(self):
+ ts1 = _time(0)
+ ts2 = self.series._intervals['minute']['i_calc'].normalize(ts1, 1)
+ ts3 = self.series._intervals['minute']['i_calc'].normalize(ts1, 2)
+ assert_not_equals( ts1, ts2 )
+
+ self.series.insert( 'test', 32, timestamp=ts1, intervals=1 )
+
+ interval_1 = self.series.get( 'test', 'minute', timestamp=ts1 )
+ assert_equals( [32], interval_1[ts1] )
+
+ interval_2 = self.series.get( 'test', 'minute', timestamp=ts2 )
+ assert_equals( [32], interval_2[ts2] )
+
+ self.series.insert( 'test', 42, timestamp=ts1, intervals=2 )
+
+ interval_1 = self.series.get( 'test', 'minute', timestamp=ts1 )
+ assert_equals( [32,42], interval_1[ts1] )
+ interval_2 = self.series.get( 'test', 'minute', timestamp=ts2 )
+ assert_equals( [32,42], interval_2[ts2] )
+ interval_3 = self.series.get( 'test', 'minute', timestamp=ts3 )
+ assert_equals( [42], interval_3[ts3] )
+
+ def test_insert_multiple_intervals_before(self):
+ ts1 = _time(0)
+ ts2 = self.series._intervals['minute']['i_calc'].normalize(ts1, -1)
+ ts3 = self.series._intervals['minute']['i_calc'].normalize(ts1, -2)
+ assert_not_equals( ts1, ts2 )
+
+ self.series.insert( 'test', 32, timestamp=ts1, intervals=-1 )
+
+ interval_1 = self.series.get( 'test', 'minute', timestamp=ts1 )
+ assert_equals( [32], interval_1[ts1] )
+
+ interval_2 = self.series.get( 'test', 'minute', timestamp=ts2 )
+ assert_equals( [32], interval_2[ts2] )
+
+ self.series.insert( 'test', 42, timestamp=ts1, intervals=-2 )
+
+ interval_1 = self.series.get( 'test', 'minute', timestamp=ts1 )
+ assert_equals( [32,42], interval_1[ts1] )
+ interval_2 = self.series.get( 'test', 'minute', timestamp=ts2 )
+ assert_equals( [32,42], interval_2[ts2] )
+ interval_3 = self.series.get( 'test', 'minute', timestamp=ts3 )
+ assert_equals( [42], interval_3[ts3] )
+
+ def test_get(self):
+ # 2 hours worth of data, value is same asV timestamp
+ for t in xrange(1, 7200):
+ self.series.insert( 'test', t, timestamp=_time(t) )
+
+ ###
+ ### no resolution, condensed has no impact
+ ###
+ # middle of an interval
+ interval = self.series.get( 'test', 'minute', timestamp=_time(100) )
+ assert_equals( [_time(60)], interval.keys() )
+ assert_equals( list(range(60,120)), interval[_time(60)] )
+
+ # end of an interval
+ interval = self.series.get( 'test', 'minute', timestamp=_time(59) )
+ assert_equals( [_time(0)], interval.keys() )
+ assert_equals( list(range(1,60)), interval[_time(0)] )
+
+ # no matching interval, returns no with empty value list
+ interval = self.series.get( 'test', 'minute' )
+ assert_equals( 1, len(interval) )
+ assert_equals( 0, len(interval.values()[0]) )
+
+ # with transforms
+ interval = self.series.get( 'test', 'minute', timestamp=_time(100), transform='count' )
+ assert_equals( 60, interval[_time(60)] )
+
+ interval = self.series.get( 'test', 'minute', timestamp=_time(100), transform=['min','max'] )
+ assert_equals( {'min':60, 'max':119}, interval[_time(60)] )
+
+ ###
+ ### with resolution, optionally condensed
+ ###
+ interval = self.series.get( 'test', 'hour', timestamp=_time(100) )
+ assert_equals( 60, len(interval) )
+ assert_equals( list(range(60,120)), interval[_time(60)] )
+
+ interval = self.series.get( 'test', 'hour', timestamp=_time(100), condensed=True )
+ assert_equals( 1, len(interval) )
+ assert_equals( list(range(1,3600)), interval[_time(0)] )
+
+ # with transforms
+ interval = self.series.get( 'test', 'hour', timestamp=_time(100), transform='count' )
+ assert_equals( 60, interval[_time(60)] )
+
+ interval = self.series.get( 'test', 'hour', timestamp=_time(100), transform=['min','max'], condensed=True )
+ assert_equals( {'min':1, 'max':3599}, interval[_time(0)] )
+
+ def test_get_joined(self):
+ # put some data in the first minutes of each hour for test1, and then for
+ # a few more minutes in test2
+ self.series.delete('test1')
+ self.series.delete('test2')
+ for t in xrange(1, 120):
+ self.series.insert( 'test1', t, timestamp=_time(t) )
+ self.series.insert( 'test2', t, timestamp=_time(t) )
+ for t in xrange(3600, 3720):
+ self.series.insert( 'test1', t, timestamp=_time(t) )
+ self.series.insert( 'test2', t, timestamp=_time(t) )
+ for t in xrange(120, 240):
+ self.series.insert( 'test1', t, timestamp=_time(t) )
+ for t in xrange(3721, 3840):
+ self.series.insert( 'test1', t, timestamp=_time(t) )
+
+ ###
+ ### no resolution, condensed has no impact
+ ###
+ # interval with 2 series worth of data
+ interval = self.series.get( ['test1','test2'], 'minute', timestamp=_time(100) )
+ assert_equals( [_time(60)], interval.keys() )
+ assert_equals( list(range(60,120))+list(range(60,120)), interval[_time(60)] )
+
+ # interval with 1 series worth of data
+ interval = self.series.get( ['test1','test2'], 'minute', timestamp=_time(122) )
+ assert_equals( [_time(120)], interval.keys() )
+ assert_equals( list(range(120,180)), interval[_time(120)] )
+
+ # no matching interval, returns no with empty value list
+ interval = self.series.get( ['test1','test2'], 'minute' )
+ assert_equals( 1, len(interval) )
+ assert_equals( 0, len(interval.values()[0]) )
+
+ ###
+ ### with resolution, optionally condensed
+ ###
+ interval = self.series.get( ['test1','test2'], 'hour', timestamp=_time(100) )
+ assert_equals( map(_time,[0,60,120,180]), interval.keys() )
+ assert_equals( list(range(1,60))+list(range(1,60)), interval[_time(0)] )
+ assert_equals( list(range(60,120))+list(range(60,120)), interval[_time(60)] )
+ assert_equals( list(range(120,180)), interval[_time(120)] )
+ assert_equals( list(range(180,240)), interval[_time(180)] )
+
+ interval = self.series.get( ['test1','test2'], 'hour', timestamp=_time(100), condensed=True )
+ assert_equals( [_time(0)], interval.keys() )
+ assert_equals(
+ list(range(1,60))+list(range(1,60))+list(range(60,120))+list(range(60,120))+\
+ list(range(120,180))+list(range(180,240)),
+ interval[_time(0)] )
+
+ # with transforms
+ interval = self.series.get( ['test1','test2'], 'hour', timestamp=_time(100), transform='count' )
+ assert_equals( 120, interval[_time(60)] )
+
+ interval = self.series.get( ['test1','test2'], 'hour', timestamp=_time(100), transform=['min','max','count'], condensed=True )
+ assert_equals( {'min':1, 'max':239, 'count':358}, interval[_time(0)] )
+
+ def test_series(self):
+ # 2 hours worth of data, value is same asV timestamp
+ for t in xrange(1, 7200):
+ self.series.insert( 'test', t, timestamp=_time(t) )
+
+ ###
+ ### no resolution, condensed has no impact
+ ###
+ interval = self.series.series( 'test', 'minute', end=_time(250) )
+ assert_equals( map(_time,[0,60,120,180,240]), interval.keys() )
+ assert_equals( list(range(1,60)), interval[_time(0)] )
+ assert_equals( list(range(240,300)), interval[_time(240)] )
+
+ interval = self.series.series( 'test', 'minute', steps=2, end=_time(250) )
+ assert_equals( map(_time,[180,240]), interval.keys() )
+ assert_equals( list(range(240,300)), interval[_time(240)] )
+
+ # with transforms
+ interval = self.series.series( 'test', 'minute', end=_time(250), transform=['min','count'] )
+ assert_equals( map(_time,[0,60,120,180,240]), interval.keys() )
+ assert_equals( {'min':1, 'count':59}, interval[_time(0)] )
+ assert_equals( {'min':240, 'count':60}, interval[_time(240)] )
+
+ # with collapsed
+ interval = self.series.series( 'test', 'minute', end=_time(250), collapse=True )
+ assert_equals( map(_time,[0]), interval.keys() )
+ assert_equals( list(range(1,300)), interval[_time(0)] )
+
+ # with transforms and collapsed
+ interval = self.series.series( 'test', 'minute', end=_time(250), transform=['min','count'], collapse=True )
+ assert_equals( map(_time,[0]), interval.keys() )
+ assert_equals( {'min':1, 'count':299}, interval[_time(0)] )
+
+ ###
+ ### with resolution
+ ###
+ interval = self.series.series( 'test', 'hour', end=_time(250) )
+ assert_equals( 1, len(interval) )
+ assert_equals( 60, len(interval[_time(0)]) )
+ assert_equals( list(range(1,60)), interval[_time(0)][_time(0)] )
+
+ interval = self.series.series( 'test', 'hour', end=_time(250), transform=['count','max'] )
+ assert_equals( 1, len(interval) )
+ assert_equals( 60, len(interval[_time(0)]) )
+ assert_equals( {'max':59, 'count':59}, interval[_time(0)][_time(0)] )
+
+ # single step, last one
+ interval = self.series.series( 'test', 'hour', condensed=True, end=_time(4200) )
+ assert_equals( 1, len(interval) )
+ assert_equals( 3600, len(interval[_time(3600)]) )
+ assert_equals( list(range(3600,7200)), interval[_time(3600)] )
+
+ interval = self.series.series( 'test', 'hour', condensed=True, end=_time(4200), steps=2 )
+ assert_equals( map(_time, [0,3600]), interval.keys() )
+ assert_equals( 3599, len(interval[_time(0)]) )
+ assert_equals( 3600, len(interval[_time(3600)]) )
+ assert_equals( list(range(3600,7200)), interval[_time(3600)] )
+
+ # with transforms
+ interval = self.series.series( 'test', 'hour', condensed=True, end=_time(4200), transform=['min','max'] )
+ assert_equals( 1, len(interval) )
+ assert_equals( {'min':3600, 'max':7199}, interval[_time(3600)] )
+
+ # with collapsed
+ interval = self.series.series( 'test', 'hour', condensed=True, end=_time(4200), steps=2, collapse=True )
+ assert_equals( map(_time, [0]), interval.keys() )
+ assert_equals( 7199, len(interval[_time(0)]) )
+ assert_equals( list(range(1,7200)), interval[_time(0)] )
+
+ # with transforms and collapsed
+ interval = self.series.series( 'test', 'hour', condensed=True, end=_time(4200), steps=2, collapse=True, transform=['min','count','max'] )
+ assert_equals( map(_time, [0]), interval.keys() )
+ assert_equals( {'min':1, 'max':7199, 'count':7199}, interval[_time(0)] )
+
+ def test_series_joined(self):
+ # put some data in the first minutes of each hour for test1, and then for
+ # a few more minutes in test2
+ self.series.delete('test1')
+ self.series.delete('test2')
+ for t in xrange(1, 120):
+ self.series.insert( 'test1', t, timestamp=_time(t) )
+ self.series.insert( 'test2', t, timestamp=_time(t) )
+ for t in xrange(3600, 3720):
+ self.series.insert( 'test1', t, timestamp=_time(t) )
+ self.series.insert( 'test2', t, timestamp=_time(t) )
+ for t in xrange(120, 240):
+ self.series.insert( 'test1', t, timestamp=_time(t) )
+ for t in xrange(3720, 3840):
+ self.series.insert( 'test1', t, timestamp=_time(t) )
+
+ ###
+ ### no resolution, condensed has no impact
+ ###
+ interval = self.series.series( ['test1','test2'], 'minute', end=_time(250) )
+ assert_equals( map(_time,[0,60,120,180,240]), interval.keys() )
+ assert_equals( list(range(1,60))+list(range(1,60)), interval[_time(0)] )
+ assert_equals( list(range(60,120))+list(range(60,120)), interval[_time(60)] )
+ assert_equals( list(range(120,180)), interval[_time(120)] )
+ assert_equals( list(range(180,240)), interval[_time(180)] )
+ assert_equals( [], interval[_time(240)] )
+
+ # no matching interval, returns no with empty value list
+ interval = self.series.series( ['test1','test2'], 'minute', start=time.time(), steps=2 )
+ assert_equals( 2, len(interval) )
+ assert_equals( [], interval.values()[0] )
+
+ # with transforms
+ interval = self.series.series( ['test1','test2'], 'minute', end=_time(250), transform=['min','count'] )
+ assert_equals( map(_time,[0,60,120,180,240]), interval.keys() )
+ assert_equals( {'min':1, 'count':118}, interval[_time(0)] )
+ assert_equals( {'min':60, 'count':120}, interval[_time(60)] )
+ assert_equals( {'min':120, 'count':60}, interval[_time(120)] )
+ assert_equals( {'min':180, 'count':60}, interval[_time(180)] )
+ assert_equals( {'min':0, 'count':0}, interval[_time(240)] )
+
+ # with collapsed
+ interval = self.series.series( ['test1','test2'], 'minute', end=_time(250), collapse=True )
+ assert_equals( [_time(0)], interval.keys() )
+ assert_equals(
+ list(range(1,60))+list(range(1,60))+list(range(60,120))+list(range(60,120))+\
+ list(range(120,180))+list(range(180,240)),
+ interval[_time(0)] )
+
+ # with tranforms and collapsed
+ interval = self.series.series( ['test1','test2'], 'minute', end=_time(250), transform=['min','max', 'count'], collapse=True )
+ assert_equals( [_time(0)], interval.keys() )
+ assert_equals( {'min':1, 'max':239, 'count':358}, interval[_time(0)] )
+
+ ###
+ ### with resolution, optionally condensed
+ ###
+ interval = self.series.series( ['test1','test2'], 'hour', end=_time(250) )
+ assert_equals( 1, len(interval) )
+ assert_equals( map(_time,[0,60,120,180]), interval[_time(0)].keys() )
+ assert_equals( 4, len(interval[_time(0)]) )
+ assert_equals( list(range(1,60))+list(range(1,60)), interval[_time(0)][_time(0)] )
+ assert_equals( list(range(60,120))+list(range(60,120)), interval[_time(0)][_time(60)] )
+ assert_equals( list(range(120,180)), interval[_time(0)][_time(120)] )
+ assert_equals( list(range(180,240)), interval[_time(0)][_time(180)] )
+
+ # condensed
+ interval = self.series.series( ['test1','test2'], 'hour', end=_time(250), condensed=True )
+ assert_equals( [_time(0)], interval.keys() )
+ assert_equals(
+ list(range(1,60))+list(range(1,60))+list(range(60,120))+list(range(60,120))+\
+ list(range(120,180))+list(range(180,240)),
+ interval[_time(0)] )
+
+ # with collapsed
+ interval = self.series.series( ['test1','test2'], 'hour', condensed=True, end=_time(4200), steps=2, collapse=True )
+ assert_equals( map(_time, [0]), interval.keys() )
+ assert_equals(
+ list(range(1,60))+list(range(1,60))+list(range(60,120))+list(range(60,120))+\
+ list(range(120,180))+list(range(180,240))+\
+ list(range(3600,3660))+list(range(3600,3660))+list(range(3660,3720))+list(range(3660,3720))+\
+ list(range(3720,3780))+list(range(3780,3840)),
+ interval[_time(0)] )
+
+ # with transforms collapsed
+ interval = self.series.series( ['test1','test2'], 'hour', condensed=True, end=_time(4200), steps=2, collapse=True, transform=['min','max','count'] )
+ assert_equals( map(_time, [0]), interval.keys() )
+ assert_equals( {'min':1,'max':3839,'count':718}, interval[_time(0)] )
diff --git a/test/functional/set_helper.py b/test/functional/set_helper.py
new file mode 100644
index 0000000..e4c78cd
--- /dev/null
+++ b/test/functional/set_helper.py
@@ -0,0 +1,240 @@
+from helper_helper import *
+from helper_helper import _time
+
+@unittest.skipUnless( os.environ.get('TEST_SET','true').lower()=='true', 'skipping sets' )
+class SetHelper(Chai):
+
+ def setUp(self):
+ super(SetHelper,self).setUp()
+
+ self.series = Timeseries(self.client, type='set', prefix='kairos',
+ read_func=lambda v: int(v or 0),
+ intervals={
+ 'minute' : {
+ 'step' : 60,
+ 'steps' : 5,
+ },
+ 'hour' : {
+ 'step' : 3600,
+ 'resolution' : 60,
+ }
+ } )
+ self.series.delete_all()
+
+ def tearDown(self):
+ self.series.delete_all()
+
+ def test_bulk_insert(self):
+ inserts = {
+ None : { 'test1':[1,2,3], 'test2':[4,5,6] },
+ _time(0) : { 'test1':[1,2,3], 'test2':[4,5,6], 'test3':[7,8,9] },
+ _time(30) : { 'test1':[1,2,3], 'test2':[4,5,6] },
+ _time(60) : { 'test1':[1,2,3], 'test3':[7,8,9] }
+ }
+ self.series.bulk_insert( inserts )
+
+ t1_i1 = self.series.get('test1', 'minute', timestamp=_time(0))
+ assert_equals( {1,2,3}, t1_i1[_time(0)] )
+
+ t2_i1 = self.series.get('test2', 'minute', timestamp=_time(0))
+ assert_equals( {4,5,6}, t2_i1[_time(0)] )
+
+ t3_i1 = self.series.get('test3', 'minute', timestamp=_time(0))
+ assert_equals( {7,8,9}, t3_i1[_time(0)] )
+
+ t1_i2 = self.series.get('test1', 'minute', timestamp=_time(60))
+ assert_equals( {1,2,3}, t1_i2[_time(60)] )
+
+ def test_bulk_insert_intervals_after(self):
+ a,b,c,d,e,f = 10,11,12,13,14,15
+ inserts = OrderedDict( (
+ (None , { 'test1':[1,2,3], 'test2':[4,5,6] } ),
+ (_time(0) , { 'test1':[1,2,3], 'test2':[4,5,6], 'test3':[7,8,9] } ),
+ (_time(30), { 'test1':[1,2,3], 'test2':[4,5,6] } ),
+ (_time(60), { 'test1':[a,b,c], 'test3':[d,e,f] })
+ ) )
+ self.series.bulk_insert( inserts, intervals=3 )
+
+ t1_i1 = self.series.get('test1', 'minute', timestamp=_time(0))
+ assert_equals( {1,2,3}, t1_i1[_time(0)] )
+
+ t2_i1 = self.series.get('test2', 'minute', timestamp=_time(0))
+ assert_equals( {4,5,6}, t2_i1[_time(0)] )
+
+ t3_i1 = self.series.get('test3', 'minute', timestamp=_time(0))
+ assert_equals( {7,8,9}, t3_i1[_time(0)] )
+
+ t1_i2 = self.series.get('test1', 'minute', timestamp=_time(60))
+ assert_equals( {1,2,3,a,b,c}, t1_i2[_time(60)] )
+
+ t3_i3 = self.series.get('test3', 'minute', timestamp=_time(120))
+ assert_equals( {7,8,9,d,e,f}, t3_i3[_time(120)] )
+
+ t3_i4 = self.series.get('test3', 'minute', timestamp=_time(180))
+ assert_equals( {7,8,9,d,e,f}, t3_i4[_time(180)] )
+
+ def test_bulk_insert_intervals_before(self):
+ a,b,c,d,e,f = 10,11,12,13,14,15
+ inserts = OrderedDict( (
+ (None , { 'test1':[1,2,3], 'test2':[4,5,6] } ),
+ (_time(0) , { 'test1':[1,2,3], 'test2':[4,5,6], 'test3':[7,8,9] } ),
+ (_time(30), { 'test1':[1,2,3], 'test2':[4,5,6] } ),
+ (_time(60), { 'test1':[a,b,c], 'test3':[d,e,f] })
+ ) )
+ self.series.bulk_insert( inserts, intervals=-3 )
+
+ t1_i1 = self.series.get('test1', 'minute', timestamp=_time(0))
+ assert_equals( {1,2,3,a,b,c}, t1_i1[_time(0)] )
+
+ t2_i1 = self.series.get('test2', 'minute', timestamp=_time(0))
+ assert_equals( {4,5,6}, t2_i1[_time(0)] )
+
+ t3_i1 = self.series.get('test3', 'minute', timestamp=_time(0))
+ assert_equals( {7,8,9,d,e,f}, t3_i1[_time(0)] )
+
+ t1_i2 = self.series.get('test1', 'minute', timestamp=_time(-60))
+ assert_equals( {1,2,3,a,b,c}, t1_i2[_time(-60)] )
+
+ t3_i3 = self.series.get('test3', 'minute', timestamp=_time(-120))
+ assert_equals( {7,8,9,d,e,f}, t3_i3[_time(-120)] )
+
+ t3_i4 = self.series.get('test3', 'minute', timestamp=_time(-180))
+ assert_equals( {7,8,9}, t3_i4[_time(-180)] )
+
+ def test_get(self):
+ # 2 hours worth of data. Trim some bits from data resolution to assert
+ # proper set behavior
+ for t in xrange(1, 7200):
+ self.series.insert( 'test', t/15, timestamp=_time(t) )
+
+ ###
+ ### no resolution, condensed has no impact
+ ###
+ # middle of an interval
+ interval = self.series.get( 'test', 'minute', timestamp=_time(100) )
+ assert_equals( map(_time, [60]), interval.keys() )
+ assert_equals( set([4,5,6,7]), interval[_time(60)] )
+
+ # no matching interval, returns no with empty value list
+ interval = self.series.get( 'test', 'minute' )
+ assert_equals( 1, len(interval) )
+ assert_equals( set(), interval.values()[0] )
+
+ ###
+ ### with resolution, optionally condensed
+ ###
+ interval = self.series.get( 'test', 'hour', timestamp=_time(100) )
+ assert_equals( 60, len(interval) )
+ assert_equals( set([4,5,6,7]), interval[_time(60)] )
+
+ interval = self.series.get( 'test', 'hour', timestamp=_time(100), condensed=True )
+ assert_equals( 1, len(interval) )
+ assert_equals( set(range(0,240)), interval[_time(0)] )
+
+ interval = self.series.get( 'test', 'hour', timestamp=_time(4000), condensed=True )
+ assert_equals( 1, len(interval) )
+ assert_equals( set(range(240,480)), interval[_time(3600)] )
+
+ def test_series(self):
+ # 2 hours worth of data. Trim some bits from data resolution to assert
+ # proper set behavior
+ for t in xrange(1, 7200):
+ self.series.insert( 'test', t/15, timestamp=_time(t) )
+
+ ###
+ ### no resolution, condensed has no impact
+ ###
+ interval = self.series.series( 'test', 'minute', end=_time(250) )
+ assert_equals( map(_time, [0,60,120,180,240]), interval.keys() )
+ assert_equals( set([0,1,2,3]), interval[_time(0)] )
+ assert_equals( set([4,5,6,7]), interval[_time(60)] )
+
+ interval = self.series.series( 'test', 'minute', steps=2, end=_time(250) )
+ assert_equals( map(_time, [180,240]), interval.keys() )
+ assert_equals( set([16,17,18,19]), interval[_time(240)] )
+
+ # with collapse
+ interval = self.series.series( 'test', 'minute', end=_time(250), collapse=True )
+ assert_equals( map(_time, [0]), interval.keys() )
+ assert_equals( set(range(0,20)), interval[_time(0)] )
+
+ ###
+ ### with resolution
+ ###
+ interval = self.series.series( 'test', 'hour', end=_time(250) )
+ assert_equals( 1, len(interval) )
+ assert_equals( 60, len(interval[_time(0)]) )
+ assert_equals( set([0,1,2,3]), interval[_time(0)][_time(0)] )
+ assert_equals( set([4,5,6,7]), interval[_time(0)][_time(60)] )
+
+ # single step, last one
+ interval = self.series.series( 'test', 'hour', condensed=True, end=_time(4200) )
+ assert_equals( 1, len(interval) )
+ assert_equals( set(range(240,480)), interval[_time(3600)] )
+
+ interval = self.series.series( 'test', 'hour', condensed=True, end=_time(4200), steps=2 )
+ assert_equals( map(_time, [0,3600]), interval.keys() )
+ assert_equals( set(range(0,240)), interval[_time(0)] )
+ assert_equals( set(range(240,480)), interval[_time(3600)] )
+
+ # with collapse
+ interval = self.series.series( 'test', 'hour', condensed=True, end=_time(4200), steps=2, collapse=True )
+ assert_equals( map(_time, [0]), interval.keys() )
+ assert_equals( set(range(0,480)), interval[_time(0)] )
+
+ def test_series_joined(self):
+ # put some data in the first minutes of each hour for test1, and then for
+ # a few more minutes in test2
+ for t in xrange(1, 120):
+ self.series.insert( 'test1', t/15, timestamp=_time(t) )
+ self.series.insert( 'test2', t/15, timestamp=_time(t) )
+ for t in xrange(3600, 3720):
+ self.series.insert( 'test1', t/15, timestamp=_time(t) )
+ self.series.insert( 'test2', t/15, timestamp=_time(t) )
+ for t in xrange(120, 240):
+ self.series.insert( 'test1', t/15, timestamp=_time(t) )
+ for t in xrange(3720, 3840):
+ self.series.insert( 'test1', t/15, timestamp=_time(t) )
+
+ ###
+ ### no resolution, condensed has no impact
+ ###
+ interval = self.series.series( ['test1','test2'], 'minute', end=_time(250) )
+ assert_equals( map(_time,[0,60,120,180,240]), interval.keys() )
+ assert_equals( set([0,1,2,3]), interval[_time(0)] )
+ assert_equals( set([4,5,6,7]), interval[_time(60)] )
+ assert_equals( set([8,9,10,11]), interval[_time(120)] )
+ assert_equals( set([12,13,14,15]), interval[_time(180)] )
+ assert_equals( set(), interval[_time(240)] )
+
+ # no matching interval, returns no with empty value list
+ interval = self.series.series( ['test1','test2'], 'minute', start=time.time(), steps=2 )
+ assert_equals( 2, len(interval) )
+ assert_equals( set(), interval.values()[0] )
+
+ # with collapsed
+ interval = self.series.series( ['test1','test2'], 'minute', end=_time(250), collapse=True )
+ assert_equals( [_time(0)], interval.keys() )
+ assert_equals( set(range(0,16)), interval[_time(0)] )
+
+ ###
+ ### with resolution, optionally condensed
+ ###
+ interval = self.series.series( ['test1','test2'], 'hour', end=_time(250) )
+ assert_equals( 1, len(interval) )
+ assert_equals( map(_time,[0,60,120,180]), interval[_time(0)].keys() )
+ assert_equals( 4, len(interval[_time(0)]) )
+ assert_equals( set([0,1,2,3]), interval[_time(0)][_time(0)] )
+ assert_equals( set([4,5,6,7]), interval[_time(0)][_time(60)] )
+ assert_equals( set([8,9,10,11]), interval[_time(0)][_time(120)] )
+ assert_equals( set([12,13,14,15]), interval[_time(0)][_time(180)] )
+
+ # condensed
+ interval = self.series.series( ['test1','test2'], 'hour', end=_time(250), condensed=True )
+ assert_equals( [_time(0)], interval.keys() )
+ assert_equals( set(range(0,16)), interval[_time(0)] )
+
+ # with collapsed
+ interval = self.series.series( ['test1','test2'], 'hour', condensed=True, end=_time(4200), steps=2, collapse=True )
+ assert_equals( map(_time, [0]), interval.keys() )
+ assert_equals( set(range(0,16)) | set(range(240,256)), interval[_time(0)] )
diff --git a/test/functional/sql_timeseries_test.py b/test/functional/sql_timeseries_test.py
new file mode 100644
index 0000000..866363b
--- /dev/null
+++ b/test/functional/sql_timeseries_test.py
@@ -0,0 +1,79 @@
+'''
+Functional tests for sql timeseries
+'''
+import time
+import datetime
+import os
+
+from sqlalchemy import create_engine
+
+from . import helpers
+from .helpers import unittest, os, Timeseries
+
+SQL_HOST = os.environ.get('SQL_HOST', 'sqlite:///:memory:')
+
+@unittest.skipUnless( os.environ.get('TEST_SQL','true').lower()=='true', 'skipping sql' )
+class SqlApiTest(helpers.ApiHelper):
+
+ def setUp(self):
+ self.client = create_engine(SQL_HOST, echo=False)
+ super(SqlApiTest,self).setUp()
+
+ def test_url_parse(self):
+ assert_equals( 'SqlSeries',
+ Timeseries('sqlite:///:memory:', type='series').__class__.__name__ )
+
+ def test_expire(self):
+ cur_time = time.time()
+ self.series.insert( 'test', 1, timestamp=cur_time-600 )
+ self.series.insert( 'test', 2, timestamp=cur_time-60 )
+ self.series.insert( 'test', 3, timestamp=cur_time )
+
+ kwargs = {
+ 'condense':True,
+ 'collapse':True,
+ 'start':cur_time-600,
+ 'end':cur_time
+ }
+ assert_equals( [1,2,3], self.series.series('test', 'minute', **kwargs).values()[0] )
+ assert_equals( [1,2,3], self.series.series('test', 'hour', **kwargs).values()[0] )
+ self.series.expire('test')
+ assert_equals( [2,3], self.series.series('test', 'minute', **kwargs).values()[0] )
+ assert_equals( [1,2,3], self.series.series('test', 'hour', **kwargs).values()[0] )
+
+ self.series.delete('test')
+
+@unittest.skipUnless( os.environ.get('TEST_SQL','true').lower()=='true', 'skipping sql' )
+class SqlGregorianTest(helpers.GregorianHelper):
+
+ def setUp(self):
+ self.client = create_engine(SQL_HOST, echo=False)
+ super(SqlGregorianTest,self).setUp()
+
+@unittest.skipUnless( os.environ.get('TEST_SQL','true').lower()=='true', 'skipping sql' )
+class SqlSeriesTest(helpers.SeriesHelper):
+
+ def setUp(self):
+ self.client = create_engine(SQL_HOST, echo=False)
+ super(SqlSeriesTest,self).setUp()
+
+@unittest.skipUnless( os.environ.get('TEST_SQL','true').lower()=='true', 'skipping sql' )
+class SqlHistogramTest(helpers.HistogramHelper):
+
+ def setUp(self):
+ self.client = create_engine(SQL_HOST, echo=False)
+ super(SqlHistogramTest,self).setUp()
+
+@unittest.skipUnless( os.environ.get('TEST_SQL','true').lower()=='true', 'skipping sql' )
+class SqlCountTest(helpers.CountHelper):
+
+ def setUp(self):
+ self.client = create_engine(SQL_HOST, echo=False)
+ super(SqlCountTest,self).setUp()
+
+@unittest.skipUnless( os.environ.get('TEST_SQL','true').lower()=='true', 'skipping sql' )
+class SqlGaugeTest(helpers.GaugeHelper):
+
+ def setUp(self):
+ self.client = create_engine(SQL_HOST, echo=False)
+ super(SqlGaugeTest,self).setUp()
diff --git a/test/unit/cassandra_timeseries_test.py b/test/unit/cassandra_timeseries_test.py
new file mode 100644
index 0000000..ff622fe
--- /dev/null
+++ b/test/unit/cassandra_timeseries_test.py
@@ -0,0 +1,123 @@
+'''
+Functional tests for cassandra timeseries
+'''
+from Queue import Queue, Empty, Full
+
+import cql
+from chai import Chai
+
+from kairos.cassandra_backend import *
+
+class CassandraTest(Chai):
+
+ def setUp(self):
+ super(CassandraTest,self).setUp()
+ self.series = CassandraSeries.__new__(CassandraSeries, 'client')
+
+ def test_init(self):
+ client = mock()
+ client.cql_major_version = 3
+
+ attrs = ['host', 'port', 'keyspace', 'cql_version', 'compression',
+ 'consistency_level', 'transport', 'credentials']
+ for attr in attrs:
+ setattr(client, attr, attr.encode('hex'))
+
+ # Trap the stuff about setting things up
+ with expect( client.cursor ).returns(mock()) as c:
+ expect( c.execute )
+ expect( c.close )
+
+ self.series.__init__(client)
+ assert_equals( 'float', self.series._value_type )
+ assert_equals( 'series', self.series._table )
+
+ # connection pooling
+ for attr in attrs:
+ assert_equals( getattr(client,attr), getattr(self.series,'_'+attr) )
+ assert_equals( 1, self.series._pool.qsize() )
+ assert_equals( 0, self.series._pool.maxsize )
+ assert_equals( client, self.series._pool.get() )
+
+ def test_init_when_invalid_cql_version(self):
+ client = mock()
+ client.cql_major_version = 2
+ with assert_raises( TypeError ):
+ self.series.__init__(client)
+
+ def test_init_handles_value_type(self):
+ client = mock()
+ client.cql_major_version = 3
+ # Trap the stuff about setting things up
+ with expect( client.cursor ).returns(mock()) as c:
+ expect( c.execute )
+ expect( c.close )
+
+ self.series.__init__(client, value_type=bool)
+ assert_equals( 'boolean', self.series._value_type )
+ self.series.__init__(client, value_type='double')
+ assert_equals( 'double', self.series._value_type )
+
+ with assert_raises( TypeError ):
+ self.series.__init__(client, value_type=object())
+
+ def test_init_handles_table_name(self):
+ client = mock()
+ client.cql_major_version = 3
+ # Trap the stuff about setting things up
+ with expect( client.cursor ).returns(mock()) as c:
+ expect( c.execute )
+ expect( c.close )
+
+ self.series.__init__(client, table_name='round')
+ assert_equals( 'round', self.series._table )
+
+ def test_init_sets_pool_size(self):
+ client = mock()
+ client.cql_major_version = 3
+ # Trap the stuff about setting things up
+ with expect( client.cursor ).returns(mock()) as c:
+ expect( c.execute )
+ expect( c.close )
+
+ self.series.__init__(client, pool_size=20)
+ assert_equals( 20, self.series._pool.maxsize )
+
+ def test_connection_from_pool(self):
+ self.series._pool = Queue()
+ self.series._pool.put('conn')
+ assert_equals( 'conn', self.series._connection() )
+ assert_equals( 0, self.series._pool.qsize() )
+
+ def test_connection_when_pool_is_empty(self):
+ self.series._pool = Queue(2)
+ self.series._host = 'dotcom'
+ self.series._port = 'call'
+ self.series._keyspace = 'gatekeeper'
+ self.series._cql_version = '3.2.1'
+ self.series._compression = 'smaller'
+ self.series._consistency_level = 'most'
+ self.series._transport = 'thrifty'
+ self.series._credentials = {'user':'you', 'password':'knockknock'}
+
+ expect( cql, 'connect' ).args( 'dotcom', 'call', 'gatekeeper',
+ user='you', password='knockknock', cql_version='3.2.1',
+ compression='smaller', consistency_level='most',
+ transport='thrifty' ).returns( 'conn' )
+
+ assert_equals( 'conn', self.series._connection() )
+
+ def test_return_when_pool_not_full(self):
+ self.series._pool = Queue(1)
+ assert_equals( 0, self.series._pool.qsize() )
+ self.series._return( 'conn' )
+ assert_equals( 1, self.series._pool.qsize() )
+
+ def test_return_when_pool_full(self):
+ self.series._pool = Queue(2)
+ self.series._return( 'a' )
+ self.series._return( 'b' )
+ self.series._return( 'c' )
+ assert_equals( 2, self.series._pool.qsize() )
+ assert_equals( 'a', self.series._pool.get() )
+ assert_equals( 'b', self.series._pool.get() )
diff --git a/test/unit/timeseries_test.py b/test/unit/timeseries_test.py
index 3ddbbd3..a920b43 100644
--- a/test/unit/timeseries_test.py
+++ b/test/unit/timeseries_test.py
@@ -2,27 +2,112 @@
Functional tests for timeseries core
'''
import time
-import datetime
+from datetime import datetime
from kairos.timeseries import *
from chai import Chai
+class UrlTest(Chai):
+ def test_bad_url(self):
+ with assert_raises(ImportError):
+ Timeseries("noop://foo/bar")
+
+class RelativeTimeTest(Chai):
+
+ def test_step_size(self):
+ DAY = 60*60*24
+ rt = RelativeTime( DAY )
+ assert_equals( DAY, rt.step_size() )
+ assert_equals( DAY, rt.step_size(0, 0) )
+
+ assert_equals( DAY, rt.step_size(0, 0) )
+ assert_equals( DAY, rt.step_size(0, DAY/2) )
+ assert_equals( DAY, rt.step_size(0, DAY-1) )
+ assert_equals( 2*DAY, rt.step_size(0, DAY) )
+ assert_equals( 2*DAY, rt.step_size(0, DAY+(60*60*1)) )
+ assert_equals( 3*DAY, rt.step_size(0, 2*DAY+1) )
+ assert_equals( 2*DAY, rt.step_size(DAY+1, 2*DAY) )
+
+ def test_ttl(self):
+ DAY = 60*60*24
+ rt = RelativeTime( DAY )
+ assert_equals( 3*DAY, rt.ttl( 3 ) )
+ assert_equals( 3*DAY, rt.ttl( 3, relative_time=time.time() ) )
+ assert_equals( 4*DAY, rt.ttl( 3, relative_time=time.time()+DAY ) )
+ assert_equals( 8*DAY, rt.ttl( 3, relative_time=time.time()+(5*DAY) ) )
+ assert_equals( 2*DAY, rt.ttl( 3, relative_time=time.time()-DAY ) )
+ assert_equals( DAY, rt.ttl( 3, relative_time=time.time()-2*DAY ) )
+ assert_equals( 0, rt.ttl( 3, relative_time=time.time()-3*DAY ) )
+
class GregorianTimeTest(Chai):
def test_buckets(self):
gt = GregorianTime( 'daily' )
buckets = gt.buckets( 0, 60*60*24*42 )
- assert_equals( buckets[:3], ['19700101', '19700102', '19700103'] )
- assert_equals( buckets[-3:], ['19700209', '19700210', '19700211'] )
+ assert_equals( buckets[:3], [19700101, 19700102, 19700103] )
+ assert_equals( buckets[-3:], [19700209, 19700210, 19700211] )
gt = GregorianTime( 'weekly' )
buckets = gt.buckets( 0, 60*60*24*25 )
- assert_equals( buckets, ['197000', '197001', '197002', '197003'] )
+ assert_equals( buckets, [197000, 197001, 197002, 197003] )
gt = GregorianTime( 'monthly' )
buckets = gt.buckets( 0, 60*60*24*70 )
- assert_equals( buckets, ['197001', '197002', '197003'] )
+ assert_equals( buckets, [197001, 197002, 197003] )
gt = GregorianTime( 'yearly' )
buckets = gt.buckets( 0, 60*60*24*800 )
- assert_equals( buckets, ['1970', '1971', '1972'] )
+ assert_equals( buckets, [1970, 1971, 1972] )
+
+ def test_step_size(self):
+ DAY = 60*60*24
+ gtd = GregorianTime( 'daily' )
+ gtm = GregorianTime( 'monthly' )
+ gty = GregorianTime( 'yearly' )
+
+ # leap year
+ t0 = time.mktime( datetime(year=2012, month=1, day=1).timetuple() )
+ t1 = time.mktime( datetime(year=2012, month=1, day=5).timetuple() )
+ t2 = time.mktime( datetime(year=2012, month=2, day=13).timetuple() )
+ t3 = time.mktime( datetime(year=2012, month=2, day=29).timetuple() )
+ t4 = time.mktime( datetime(year=2012, month=3, day=5).timetuple() )
+
+ assert_equals( DAY, gtd.step_size(t0) )
+ assert_equals( 31*DAY, gtm.step_size(t0) )
+ assert_equals( 366*DAY, gty.step_size(t0) )
+
+ assert_equals( DAY, gtd.step_size(t2) )
+ assert_equals( 31*DAY, gtm.step_size(t0, t1) )
+ assert_equals( 60*DAY, gtm.step_size(t1, t2) )
+ assert_equals( 29*DAY, gtm.step_size(t2, t3) )
+ assert_equals( 91*DAY, gtm.step_size(t1, t4) )
+ assert_equals( 60*DAY, gtm.step_size(t2, t4) )
+
+ # not-leap year
+ t0 = time.mktime( datetime(year=2013, month=1, day=1).timetuple() )
+ t1 = time.mktime( datetime(year=2013, month=1, day=5).timetuple() )
+ t2 = time.mktime( datetime(year=2013, month=2, day=13).timetuple() )
+ t3 = time.mktime( datetime(year=2013, month=2, day=28).timetuple() )
+ t4 = time.mktime( datetime(year=2013, month=3, day=5).timetuple() )
+
+ assert_equals( DAY, gtd.step_size(t0) )
+ assert_equals( 31*DAY, gtm.step_size(t0) )
+ assert_equals( 365*DAY, gty.step_size(t0) )
+
+ assert_equals( DAY, gtd.step_size(t2) )
+ assert_equals( 31*DAY, gtm.step_size(t0, t1) )
+ assert_equals( 59*DAY, gtm.step_size(t1, t2) )
+ assert_equals( 28*DAY, gtm.step_size(t2, t3) )
+ assert_equals( 90*DAY, gtm.step_size(t1, t4) )
+ assert_equals( 59*DAY, gtm.step_size(t2, t4) )
+
+ def test_ttl(self):
+ DAY = 60*60*24
+ gt = GregorianTime( 'daily' )
+ assert_equals( 3*DAY, gt.ttl( 3 ) )
+ assert_equals( 3*DAY, gt.ttl( 3, relative_time=time.time() ) )
+ assert_equals( 4*DAY, gt.ttl( 3, relative_time=time.time()+DAY ) )
+ assert_equals( 8*DAY, gt.ttl( 3, relative_time=time.time()+(5*DAY) ) )
+ assert_equals( 2*DAY, gt.ttl( 3, relative_time=time.time()-DAY ) )
+ assert_equals( DAY, gt.ttl( 3, relative_time=time.time()-2*DAY ) )
+ assert_equals( 0, gt.ttl( 3, relative_time=time.time()-3*DAY ) )