| Home | Trees | Indices | Help |
|
|---|
|
|
1 # -*- coding: utf-8 -*-
2
3 """ S3 TimePlot Reports Method
4
5 @copyright: 2013-2019 (c) Sahana Software Foundation
6 @license: MIT
7
8 Permission is hereby granted, free of charge, to any person
9 obtaining a copy of this software and associated documentation
10 files (the "Software"), to deal in the Software without
11 restriction, including without limitation the rights to use,
12 copy, modify, merge, publish, distribute, sublicense, and/or sell
13 copies of the Software, and to permit persons to whom the
14 Software is furnished to do so, subject to the following
15 conditions:
16
17 The above copyright notice and this permission notice shall be
18 included in all copies or substantial portions of the Software.
19
20 THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
21 EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
22 OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
23 NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
24 HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
25 WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
26 FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
27 OTHER DEALINGS IN THE SOFTWARE.
28 """
29
30 __all__ = ("S3TimePlot",
31 "S3TimeSeries",
32 "S3TimeSeriesEvent",
33 "S3TimeSeriesEventFrame",
34 "S3TimeSeriesFact",
35 "S3TimeSeriesPeriod",
36 )
37
38 import datetime
39 import dateutil.tz
40 import json
41 import re
42 import sys
43
44 from dateutil.relativedelta import *
45 from dateutil.rrule import *
46
47 from gluon import current
48 from gluon.storage import Storage
49 from gluon.html import *
50 from gluon.validators import IS_IN_SET
51 from gluon.sqlhtml import OptionsWidget
52
53 from s3datetime import s3_decode_iso_datetime, s3_utc
54 from s3rest import S3Method
55 from s3query import FS
56 from s3report import S3Report, S3ReportForm
57 from s3utils import s3_flatlist, s3_represent_value, s3_unicode, S3MarkupStripper
58
59 tp_datetime = lambda *t: datetime.datetime(tzinfo=dateutil.tz.tzutc(), *t)
60
61 tp_tzsafe = lambda dt: dt.replace(tzinfo=dateutil.tz.tzutc()) \
62 if dt and dt.tzinfo is None else dt
63
64 # Compact JSON encoding
65 SEPARATORS = (",", ":")
66
67 DEFAULT = lambda: None
68 NUMERIC_TYPES = ("integer", "double", "id")
69
70 dt_regex = Storage(
71 YEAR = re.compile(r"\A\s*(\d{4})\s*\Z"),
72 YEAR_MONTH = re.compile(r"\A\s*(\d{4})-([0]*[1-9]|[1][12])\s*\Z"),
73 MONTH_YEAR = re.compile(r"\A\s*([0]*[1-9]|[1][12])/(\d{4})\s*\Z"),
74 DATE = re.compile(r"\A\s*(\d{4})-([0]?[1-9]|[1][12])-([012]?[1-9]|[3][01])\s*\Z"),
75 DELTA = re.compile(r"\A\s*([+-]?)\s*(\d+)\s*([ymwdh])\w*\s*\Z"),
76 )
77
78 FACT = re.compile(r"([a-zA-Z]+)\(([a-zA-Z0-9_.$:\,~]+)\),*(.*)\Z")
79 SELECTOR = re.compile(r"^[a-zA-Z0-9_.$:\~]+\Z")
80
81 # =============================================================================
82 -class S3TimePlot(S3Method):
83 """ RESTful method for time plot reports """
84
85 # -------------------------------------------------------------------------
87 """
88 Page-render entry point for REST interface.
89
90 @param r: the S3Request instance
91 @param attr: controller attributes for the request
92 """
93
94 if r.http == "GET":
95 output = self.timeplot(r, **attr)
96 else:
97 r.error(405, current.ERROR.BAD_METHOD)
98 return output
99
100 # -------------------------------------------------------------------------
102 """
103 Widget-render entry point for S3Summary.
104
105 @param r: the S3Request
106 @param method: the widget method
107 @param widget_id: the widget ID
108 @param visible: whether the widget is initially visible
109 @param attr: controller attributes
110 """
111
112 # Get the target resource
113 resource = self.get_target(r)
114
115 # Read the relevant GET vars
116 report_vars, get_vars = self.get_options(r, resource)
117
118 # Parse event timestamp option
119 timestamp = get_vars.get("timestamp")
120 event_start, event_end = self.parse_timestamp(timestamp)
121
122 # Parse fact option
123 fact = get_vars.get("fact")
124 try:
125 facts = S3TimeSeriesFact.parse(fact)
126 except SyntaxError:
127 r.error(400, sys.exc_info()[1])
128 baseline = get_vars.get("baseline")
129
130 # Parse grouping axes
131 rows = get_vars.get("rows")
132 cols = get_vars.get("cols")
133
134 # Parse event frame parameters
135 start = get_vars.get("start")
136 end = get_vars.get("end")
137 slots = get_vars.get("slots")
138
139 if visible:
140 # Create time series
141 # @todo: should become resource.timeseries()
142 ts = S3TimeSeries(resource,
143 start=start,
144 end=end,
145 slots=slots,
146 event_start=event_start,
147 event_end=event_end,
148 rows=rows,
149 cols=cols,
150 facts=facts,
151 baseline=baseline,
152 # @todo: add title
153 #title=title,
154 )
155
156 # Extract aggregated results as JSON-serializable dict
157 data = ts.as_dict()
158 else:
159 data = None
160
161 # Render output
162 if r.representation in ("html", "iframe"):
163
164 ajax_vars = Storage(r.get_vars)
165 ajax_vars.update(get_vars)
166 filter_url = r.url(method="",
167 representation="",
168 vars=ajax_vars.fromkeys((k for k in ajax_vars
169 if k not in report_vars)))
170 ajaxurl = attr.get("ajaxurl", r.url(method="timeplot",
171 representation="json",
172 vars=ajax_vars,
173 ))
174 output = S3TimePlotForm(resource).html(data,
175 get_vars = get_vars,
176 filter_widgets = None,
177 ajaxurl = ajaxurl,
178 filter_url = filter_url,
179 widget_id = widget_id,
180 )
181
182 # Detect and store theme-specific inner layout
183 view = self._view(r, "timeplot.html")
184
185 # Render inner layout (outer page layout is set by S3Summary)
186 output["title"] = None
187 output = XML(current.response.render(view, output))
188
189 else:
190 r.error(415, current.ERROR.BAD_FORMAT)
191
192 return output
193
194 # -------------------------------------------------------------------------
196 """
197 Time plot report page
198
199 @param r: the S3Request instance
200 @param attr: controller attributes for the request
201 """
202
203 output = {}
204
205 # Get the target resource
206 resource = self.get_target(r)
207 tablename = resource.tablename
208 get_config = resource.get_config
209
210 # Apply filter defaults (before rendering the data!)
211 show_filter_form = False
212 if r.representation in ("html", "iframe"):
213 filter_widgets = get_config("filter_widgets", None)
214 if filter_widgets and not self.hide_filter:
215 from s3filter import S3FilterForm
216 show_filter_form = True
217 S3FilterForm.apply_filter_defaults(r, resource)
218
219 # Read the relevant GET vars
220 report_vars, get_vars = self.get_options(r, resource)
221
222 # Parse event timestamp option
223 timestamp = get_vars.get("timestamp")
224 event_start, event_end = self.parse_timestamp(timestamp)
225
226 # Parse fact option
227 fact = get_vars.get("fact")
228 try:
229 facts = S3TimeSeriesFact.parse(fact)
230 except SyntaxError:
231 r.error(400, sys.exc_info()[1])
232 baseline = get_vars.get("baseline")
233
234 # Parse grouping axes
235 rows = get_vars.get("rows")
236 cols = get_vars.get("cols")
237
238 # Parse event frame parameters
239 start = get_vars.get("start")
240 end = get_vars.get("end")
241 slots = get_vars.get("slots")
242
243 # Create time series
244 # @todo: should become resource.timeseries()
245 ts = S3TimeSeries(resource,
246 start=start,
247 end=end,
248 slots=slots,
249 event_start=event_start,
250 event_end=event_end,
251 rows=rows,
252 cols=cols,
253 facts=facts,
254 baseline=baseline,
255 # @todo: add title
256 #title=title,
257 )
258
259 # Extract aggregated results as JSON-serializable dict
260 data = ts.as_dict()
261
262 # Widget ID
263 widget_id = "timeplot"
264
265 # Render output
266 if r.representation in ("html", "iframe"):
267 # Page load
268
269 output["title"] = self.crud_string(tablename, "title_report")
270
271 # Filter widgets
272 if show_filter_form:
273 advanced = False
274 for widget in filter_widgets:
275 if "hidden" in widget.opts and widget.opts.hidden:
276 advanced = get_config("report_advanced", True)
277 break
278 filter_formstyle = get_config("filter_formstyle", None)
279 filter_form = S3FilterForm(filter_widgets,
280 formstyle=filter_formstyle,
281 advanced=advanced,
282 submit=False,
283 _class="filter-form",
284 _id="%s-filter-form" % widget_id,
285 )
286 fresource = current.s3db.resource(tablename)
287 alias = resource.alias if resource.parent else None
288 filter_widgets = filter_form.fields(fresource,
289 r.get_vars,
290 alias=alias,
291 )
292 else:
293 # Render as empty string to avoid the exception in the view
294 filter_widgets = None
295
296 ajax_vars = Storage(r.get_vars)
297 ajax_vars.update(get_vars)
298 filter_url = r.url(method="",
299 representation="",
300 vars=ajax_vars.fromkeys((k for k in ajax_vars
301 if k not in report_vars)))
302 ajaxurl = attr.get("ajaxurl", r.url(method="timeplot",
303 representation="json",
304 vars=ajax_vars,
305 ))
306
307 output = S3TimePlotForm(resource).html(data,
308 get_vars = get_vars,
309 filter_widgets = filter_widgets,
310 ajaxurl = ajaxurl,
311 filter_url = filter_url,
312 widget_id = widget_id,
313 )
314
315 output["title"] = self.crud_string(tablename, "title_report")
316 output["report_type"] = "timeplot"
317
318 # Detect and store theme-specific inner layout
319 self._view(r, "timeplot.html")
320
321 # View
322 response = current.response
323 response.view = self._view(r, "report.html")
324
325 elif r.representation == "json":
326 # Ajax load
327 output = json.dumps(data, separators=SEPARATORS)
328
329 else:
330 r.error(415, current.ERROR.BAD_FORMAT)
331
332 return output
333
334 # -------------------------------------------------------------------------
336 """
337 Identify the target resource
338
339 @param r: the S3Request
340 """
341
342 # Fallback
343 resource = self.resource
344
345 # Read URL parameter
346 alias = r.get_vars.get("component")
347
348 # Identify target component
349 if alias and alias not in (resource.alias, "~"):
350 component = resource.components.get(alias)
351 if component:
352 resource = component
353
354 return resource
355
356 # -------------------------------------------------------------------------
357 @staticmethod
359 """
360 Read the relevant GET vars for the timeplot
361
362 @param r: the S3Request
363 @param resource: the target S3Resource
364 """
365
366 # Extract the relevant GET vars
367 report_vars = ("timestamp",
368 "start",
369 "end",
370 "slots",
371 "fact",
372 "baseline",
373 "rows",
374 "cols",
375 )
376 get_vars = dict((k, v) for k, v in r.get_vars.iteritems()
377 if k in report_vars)
378
379 # Fall back to report options defaults
380 report_options = resource.get_config("timeplot_options", {})
381 defaults = report_options.get("defaults", {})
382 if not any(k in get_vars for k in report_vars):
383 get_vars = defaults
384 else:
385 # Optional URL args always fall back to config:
386 optional = ("timestamp",
387 "fact",
388 "baseline",
389 "rows",
390 "cols",
391 )
392 for opt in optional:
393 if opt not in get_vars and opt in defaults:
394 get_vars[opt] = defaults[opt]
395
396 return report_vars, get_vars
397
398 # -------------------------------------------------------------------------
399 @staticmethod
401 """
402 Parse timestamp expression
403
404 @param timestamp: the timestamp expression
405 """
406
407 if timestamp:
408 fields = timestamp.split(",")
409 if len(fields) > 1:
410 start = fields[0].strip()
411 end = fields[1].strip()
412 else:
413 start = fields[0].strip()
414 end = None
415 else:
416 start = None
417 end = None
418
419 return start, end
420
421 # =============================================================================
422 -class S3TimePlotForm(S3ReportForm):
423 """ Helper class to render a report form """
424
428
429 # -------------------------------------------------------------------------
430 - def html(self,
431 data,
432 filter_widgets=None,
433 get_vars=None,
434 ajaxurl=None,
435 filter_url=None,
436 filter_form=None,
437 filter_tab=None,
438 widget_id=None):
439 """
440 Render the form for the report
441
442 @param get_vars: the GET vars if the request (as dict)
443 @param widget_id: the HTML element base ID for the widgets
444 """
445
446 T = current.T
447
448 # Filter options
449 if filter_widgets is not None:
450 filter_options = self._fieldset(T("Filter Options"),
451 filter_widgets,
452 _id="%s-filters" % widget_id,
453 _class="filter-form")
454 else:
455 filter_options = ""
456
457 # Report options
458 report_options = self.report_options(get_vars = get_vars,
459 widget_id = widget_id)
460
461 hidden = {"tp-data": json.dumps(data, separators=SEPARATORS)}
462
463
464 # @todo: report options
465 # @todo: chart title
466 # @todo: empty-section
467 empty = T("No data available")
468 # @todo: CSS
469
470 # Report form submit element
471 resource = self.resource
472 submit = resource.get_config("report_submit", True)
473 if submit:
474 _class = "tp-submit"
475 if submit is True:
476 label = T("Update Report")
477 elif isinstance(submit, (list, tuple)):
478 label = submit[0]
479 _class = "%s %s" % (submit[1], _class)
480 else:
481 label = submit
482 submit = TAG[""](
483 INPUT(_type="button",
484 _value=label,
485 _class=_class))
486 else:
487 submit = ""
488
489 # @todo: use view template (see S3ReportForm)
490 form = FORM(filter_options,
491 report_options,
492 submit,
493 hidden = hidden,
494 _class = "tp-form",
495 _id = "%s-tp-form" % widget_id,
496 )
497
498 # View variables
499 output = {"form": form,
500 "empty": empty,
501 "widget_id": widget_id,
502 }
503
504 # D3/Timeplot scripts (injected so that they are available for summary)
505 S3Report.inject_d3()
506 s3 = current.response.s3
507 scripts = s3.scripts
508 appname = current.request.application
509 if s3.debug:
510 script = "/%s/static/scripts/S3/s3.ui.timeplot.js" % appname
511 if script not in scripts:
512 scripts.append(script)
513 else:
514 script = "/%s/static/scripts/S3/s3.ui.timeplot.min.js" % appname
515 if script not in scripts:
516 scripts.append(script)
517
518 # Script to attach the timeplot widget
519 settings = current.deployment_settings
520 options = {
521 "ajaxURL": ajaxurl,
522 "autoSubmit": settings.get_ui_report_auto_submit(),
523 "emptyMessage": str(empty),
524 }
525 script = """$("#%(widget_id)s").timeplot(%(options)s)""" % \
526 {"widget_id": widget_id,
527 "options": json.dumps(options),
528 }
529 s3.jquery_ready.append(script)
530
531 return output
532
533 # -------------------------------------------------------------------------
535 """
536 Render the widgets for the report options form
537
538 @param get_vars: the GET vars if the request (as dict)
539 @param widget_id: the HTML element base ID for the widgets
540 """
541
542 T = current.T
543
544 timeplot_options = self.resource.get_config("timeplot_options")
545
546 selectors = []
547
548 # @todo: formstyle may not be executable => convert
549 formstyle = current.deployment_settings.get_ui_filter_formstyle()
550
551 # Report type selector
552 # @todo: implement
553 #fact_selector = self.fact_options(options = timeplot_options,
554 #get_vars = get_vars,
555 #widget_id = "%s-fact" % widget_id,
556 #)
557 #selectors.append(formstyle("%s-fact__row" % widget_id,
558 #"Label",
559 #fact_selector,
560 #None,
561 #))
562
563 # Time frame selector
564 time_selector = self.time_options(options = timeplot_options,
565 get_vars = get_vars,
566 widget_id = "%s-time" % widget_id,
567 )
568 selectors.append(formstyle("%s-time__row" % widget_id,
569 "Time Frame",
570 time_selector,
571 None,
572 ))
573
574 # Render container according to row type
575 if selectors[0].tag == "tr":
576 selectors = TABLE(selectors)
577 else:
578 selectors = TAG[""](selectors)
579
580 # Render field set
581 fieldset = self._fieldset(T("Report Options"),
582 selectors,
583 _id="%s-options" % widget_id)
584
585 return fieldset
586
587 # -------------------------------------------------------------------------
592 """
593 @todo: docstring
594 """
595
596 T = current.T
597
598 # Time options:
599 if options and "time" in options:
600 opts = options["time"]
601 else:
602 # (label, start, end, slots)
603 # If you specify a start, then end is relative to that - without start, end is relative to now
604 opts = (("All up to now", "", "", ""),
605 ("Last Year", "-1year", "", "months"),
606 ("Last 6 Months", "-6months", "", "weeks"),
607 ("Last Quarter", "-3months", "", "weeks"),
608 ("Last Month", "-1month", "", "days"),
609 ("Last Week", "-1week", "", "days"),
610 ("All/+1 Month", "", "+1month", ""),
611 ("All/+2 Month", "", "+2month", ""),
612 ("-6/+3 Months", "-6months", "+9months", "months"),
613 ("-3/+1 Months", "-3months", "+4months", "weeks"),
614 ("-4/+2 Weeks", "-4weeks", "+6weeks", "weeks"),
615 ("-2/+1 Weeks", "-2weeks", "+3weeks", "days"),
616 )
617
618 widget_opts = []
619 for opt in opts:
620 label, start, end, slots = opt
621 widget_opts.append(("|".join((start, end, slots)), T(label)))
622
623 # Get current value
624 if get_vars:
625 start = get_vars.get("start", "")
626 end = get_vars.get("end", "")
627 slots = get_vars.get("slots", "")
628 else:
629 start = end = slots = ""
630 value = "|".join((start, end, slots))
631
632 # Dummy field
633 dummy_field = Storage(name="time",
634 requires=IS_IN_SET(widget_opts, zero=None))
635
636 # Construct widget
637 return OptionsWidget.widget(dummy_field,
638 value,
639 _id=widget_id,
640 _name="time",
641 _class="tp-time",
642 )
643
644 # =============================================================================
645 -class S3TimeSeries(object):
646 """ Class representing a time series """
647
648 - def __init__(self,
649 resource,
650 start=None,
651 end=None,
652 slots=None,
653 event_start=None,
654 event_end=None,
655 rows=None,
656 cols=None,
657 facts=None,
658 baseline=None,
659 title=None):
660 """
661 Constructor
662
663 @param resource: the resource
664 @param start: the start of the series (datetime or string expression)
665 @param end: the end of the time series (datetime or string expression)
666 @param slots: the slot size (string expression)
667
668 @param event_start: the event start field (field selector)
669 @param event_end: the event end field (field selector)
670
671 @param rows: the rows axis for event grouping (field selector)
672 @param cols: the columns axis for event grouping (field selector)
673 @param facts: an array of facts (S3TimeSeriesFact)
674
675 @param baseline: the baseline field (field selector)
676
677 @param title: the time series title
678 """
679
680 self.resource = resource
681 self.rfields = {}
682
683 self.title = title
684
685 # Resolve timestamp
686 self.resolve_timestamp(event_start, event_end)
687
688 # Resolve grouping axes
689 self.resolve_axes(rows, cols)
690
691 # Resolve facts
692 if not facts:
693 facts = [S3TimeSeriesFact("count", resource._id.name)]
694 self.facts = [fact.resolve(resource) for fact in facts]
695
696 # Resolve baseline
697 self.resolve_baseline(baseline)
698
699 # Create event frame
700 self.event_frame = self._event_frame(start, end, slots)
701
702 # ...and fill it with data
703 self._select()
704
705 # -------------------------------------------------------------------------
707 """ Return the time series as JSON-serializable dict """
708
709 rfields = self.rfields
710
711 # Fact Data
712 fact_data = []
713 for fact in self.facts:
714 fact_data.append((str(fact.label),
715 fact.method,
716 fact.base,
717 fact.slope,
718 fact.interval,
719 ))
720
721 # Event start and end selectors
722 rfield = rfields.get("event_start")
723 if rfield:
724 event_start = rfield.selector
725 else:
726 event_start = None
727 rfield = rfields.get("event_end")
728 if rfield:
729 event_end = rfield.selector
730 else:
731 event_end = None
732
733 # Rows
734 rows = rfields.get("rows")
735 if rows:
736 rows_sorted = self._represent_axis(rows, self.rows_keys)
737 rows_keys = [row[0] for row in rows_sorted]
738 rows_data = {"s": rows.selector,
739 "l": str(rows.label),
740 "v": rows_sorted,
741 }
742 else:
743 rows_keys = None
744 rows_data = None
745
746 # Columns
747 cols = rfields.get("cols")
748 if cols:
749 cols_sorted = self._represent_axis(cols, self.cols_keys)
750 cols_keys = [col[0] for col in cols_sorted]
751 cols_data = {"s": cols.selector,
752 "l": str(cols.label),
753 "v": cols_sorted,
754 }
755 else:
756 cols_keys = None
757 cols_data = None
758
759 # Iterate over the event frame to collect aggregates
760 event_frame = self.event_frame
761 periods_data = []
762 append = periods_data.append
763 #fact = self.facts[0]
764 for period in event_frame:
765 # Aggregate
766 period.aggregate(self.facts)
767 # Extract
768 item = period.as_dict(rows = rows_keys,
769 cols = cols_keys,
770 )
771 append(item)
772
773 # Baseline
774 rfield = rfields.get("baseline")
775 if rfield:
776 baseline = (rfield.selector,
777 str(rfield.label),
778 event_frame.baseline,
779 )
780 else:
781 baseline = None
782
783 # Output dict
784 data = {"f": fact_data,
785 "t": (event_start, event_end),
786 "s": event_frame.slots,
787 "e": event_frame.empty,
788 "l": self.title,
789 "r": rows_data,
790 "c": cols_data,
791 "p": periods_data,
792 "z": baseline,
793 }
794
795 return data
796
797 # -------------------------------------------------------------------------
798 @staticmethod
800 """
801 Represent and sort the values of a pivot axis (rows or cols)
802
803 @param rfield: the axis rfield
804 @param values: iterable of values
805 """
806
807 if rfield.virtual:
808
809 representations = []
810 append = representations.append()
811 stripper = S3MarkupStripper()
812
813 represent = rfield.represent
814 if not represent:
815 represent = s3_unicode
816
817 for value in values:
818 if value is None:
819 append((value, "-"))
820 text = represent(value)
821 if "<" in text:
822 stripper.feed(text)
823 append((value, stripper.stripped()))
824 else:
825 append((value, text))
826 else:
827 field = rfield.field
828 represent = field.represent
829 if represent and hasattr(represent, "bulk"):
830 representations = represent.bulk(list(values),
831 list_type = False,
832 show_link = False,
833 ).items()
834 else:
835 representations = []
836 for value in values:
837 append((value, s3_represent_value(field,
838 value,
839 strip_markup = True,
840 )))
841
842 return sorted(representations, key = lambda item: item[1])
843
844 # -------------------------------------------------------------------------
846 """
847 Get the representation method for a field in the report
848
849 @param field: the field selector
850 """
851
852 rfields = self.rfields
853 default = lambda value: None
854
855 if field and field in rfields:
856
857 rfield = rfields[field]
858
859 if rfield.field:
860 def repr_method(value):
861 return s3_represent_value(rfield.field, value,
862 strip_markup=True)
863
864 elif rfield.virtual:
865 stripper = S3MarkupStripper()
866 def repr_method(val):
867 if val is None:
868 return "-"
869 text = s3_unicode(val)
870 if "<" in text:
871 stripper.feed(text)
872 return stripper.stripped() # = totally naked ;)
873 else:
874 return text
875 else:
876 repr_method = default
877 else:
878 repr_method = default
879
880 return repr_method
881
882 # -------------------------------------------------------------------------
887 """
888 Create an event frame for this report
889
890 @param start: the start date/time (string, date or datetime)
891 @param end: the end date/time (string, date or datetime)
892 @param slots: the slot length (string)
893
894 @return: the event frame
895 """
896
897 resource = self.resource
898 rfields = self.rfields
899
900 STANDARD_SLOT = "1 day"
901 now = tp_tzsafe(datetime.datetime.utcnow())
902
903 # Parse start and end time
904 dtparse = self.dtparse
905 start_dt = end_dt = None
906 if start:
907 if isinstance(start, basestring):
908 start_dt = dtparse(start, start=now)
909 else:
910 if isinstance(start, datetime.datetime):
911 start_dt = tp_tzsafe(start)
912 else:
913 # Date only => start at midnight
914 start_dt = tp_tzsafe(datetime.datetime.fromordinal(start.toordinal()))
915 if end:
916 if isinstance(end, basestring):
917 relative_to = start_dt if start_dt else now
918 end_dt = dtparse(end, start=relative_to)
919 else:
920 if isinstance(end, datetime.datetime):
921 end_dt = tp_tzsafe(end)
922 else:
923 # Date only => end at midnight
924 end_dt = tp_tzsafe(datetime.datetime.fromordinal(end.toordinal()))
925
926 # Fall back to now if end is not specified
927 if not end_dt:
928 end_dt = now
929
930 event_start = rfields["event_start"]
931 if not start_dt and event_start and event_start.field:
932 # No interval start => fall back to first event start
933 query = FS(event_start.selector) != None
934 resource.add_filter(query)
935 rows = resource.select([event_start.selector],
936 limit=1,
937 orderby=event_start.field,
938 as_rows=True)
939 # Remove the filter we just added
940 rfilter = resource.rfilter
941 rfilter.filters.pop()
942 rfilter.query = None
943 rfilter.transformed = None
944 if rows:
945 first_event = rows.first()[event_start.colname]
946 if isinstance(first_event, datetime.date):
947 first_event = tp_tzsafe(datetime.datetime.fromordinal(first_event.toordinal()))
948 start_dt = first_event
949
950 event_end = rfields["event_end"]
951 if not start_dt and event_end and event_end.field:
952 # No interval start => fall back to first event end minus
953 # one standard slot length:
954 query = FS(event_end.selector) != None
955 resource.add_filter(query)
956 rows = resource.select([event_end.selector],
957 limit=1,
958 orderby=event_end.field,
959 as_rows=True)
960 # Remove the filter we just added
961 rfilter = resource.rfilter
962 rfilter.filters.pop()
963 rfilter.query = None
964 rfilter.transformed = None
965 if rows:
966 last_event = rows.first()[event_end.colname]
967 if isinstance(last_event, datetime.date):
968 last_event = tp_tzsafe(datetime.datetime.fromordinal(last_event.toordinal()))
969 start_dt = dtparse("-%s" % STANDARD_SLOT, start=last_event)
970
971 if not start_dt:
972 # No interval start => fall back to interval end minus
973 # one slot length:
974 if not slots:
975 slots = STANDARD_SLOT
976 try:
977 start_dt = dtparse("-%s" % slots, start=end_dt)
978 except (SyntaxError, ValueError):
979 slots = STANDARD_SLOT
980 start_dt = dtparse("-%s" % slots, start=end_dt)
981
982 # Fall back for slot length
983 if not slots:
984 # No slot length specified => determine optimum automatically
985 # @todo: determine from density of events rather than from
986 # total interval length?
987 seconds = abs(end_dt - start_dt).total_seconds()
988 day = 86400
989 if seconds < day:
990 slots = "hours"
991 elif seconds < 3 * day:
992 slots = "6 hours"
993 elif seconds < 28 * day:
994 slots = "days"
995 elif seconds < 90 * day:
996 slots = "weeks"
997 elif seconds < 730 * day:
998 slots = "months"
999 elif seconds < 2190 * day:
1000 slots = "3 months"
1001 else:
1002 slots = "years"
1003
1004 # Create event frame
1005 ef = S3TimeSeriesEventFrame(start_dt, end_dt, slots)
1006
1007 return ef
1008
1009 # -------------------------------------------------------------------------
1011 """
1012 Select records from the resource and store them as events in
1013 this time series
1014 """
1015
1016 resource = self.resource
1017 rfields = self.rfields
1018
1019 # Fields to extract
1020 cumulative = False
1021 event_start = rfields.get("event_start")
1022 fields = set([event_start.selector])
1023 event_end = rfields.get("event_end")
1024 if event_end:
1025 fields.add(event_end.selector)
1026 rows_rfield = rfields.get("rows")
1027 if rows_rfield:
1028 fields.add(rows_rfield.selector)
1029 cols_rfield = rfields.get("cols")
1030 if cols_rfield:
1031 fields.add(cols_rfield.selector)
1032 fact_columns = []
1033 for fact in self.facts:
1034 if fact.method == "cumulate":
1035 cumulative = True
1036 if fact.resource is None:
1037 fact.resolve(resource)
1038 for rfield in (fact.base_rfield, fact.slope_rfield):
1039 if rfield:
1040 fact_columns.append(rfield.colname)
1041 fields.add(rfield.selector)
1042 fields.add(resource._id.name)
1043
1044 # Get event frame
1045 event_frame = self.event_frame
1046
1047 # Filter by event frame start:
1048 if not cumulative and event_end:
1049 # End date of events must be after the event frame start date
1050 end_selector = FS(event_end.selector)
1051 start = event_frame.start
1052 query = (end_selector == None) | (end_selector >= start)
1053 else:
1054 # No point if events have no end date, and wrong if
1055 # method is cumulative
1056 query = None
1057
1058 # Filter by event frame end:
1059 # Start date of events must be before event frame end date
1060 start_selector = FS(event_start.selector)
1061 end = event_frame.end
1062 q = (start_selector == None) | (start_selector <= end)
1063 query = query & q if query is not None else q
1064
1065 # Add as temporary filter
1066 resource.add_filter(query)
1067
1068 # Compute baseline
1069 value = None
1070 baseline_rfield = rfields.get("baseline")
1071 if baseline_rfield:
1072 baseline_table = current.db[baseline_rfield.tname]
1073 pkey = str(baseline_table._id)
1074 colname = baseline_rfield.colname
1075 rows = resource.select([baseline_rfield.selector],
1076 groupby = [pkey, colname],
1077 as_rows = True,
1078 )
1079 value = 0
1080 for row in rows:
1081 v = row[colname]
1082 if v is not None:
1083 value += v
1084 event_frame.baseline = value
1085
1086 # Extract the records
1087 data = resource.select(fields)
1088
1089 # Remove the filter we just added
1090 rfilter = resource.rfilter
1091 rfilter.filters.pop()
1092 rfilter.query = None
1093 rfilter.transformed = None
1094
1095 # Do we need to convert dates into datetimes?
1096 convert_start = True if event_start.ftype == "date" else False
1097 convert_end = True if event_start.ftype == "date" else False
1098 fromordinal = datetime.datetime.fromordinal
1099 convert_date = lambda d: fromordinal(d.toordinal())
1100
1101 # Column names for extractions
1102 pkey = str(resource._id)
1103 start_colname = event_start.colname
1104 end_colname = event_end.colname if event_end else None
1105 rows_colname = rows_rfield.colname if rows_rfield else None
1106 cols_colname = cols_rfield.colname if cols_rfield else None
1107
1108 # Create the events
1109 events = []
1110 add_event = events.append
1111 rows_keys = set()
1112 cols_keys = set()
1113 for row in data.rows:
1114
1115 # Extract values
1116 values = dict((colname, row[colname]) for colname in fact_columns)
1117
1118 # Extract grouping keys
1119 grouping = {}
1120 if rows_colname:
1121 grouping["row"] = row[rows_colname]
1122 if cols_colname:
1123 grouping["col"] = row[cols_colname]
1124
1125 # Extract start/end date
1126 start = row[start_colname]
1127 if convert_start and start:
1128 start = convert_date(start)
1129 end = row[end_colname] if end_colname else None
1130 if convert_end and end:
1131 end = convert_date(end)
1132
1133 # values = (base, slope)
1134 event = S3TimeSeriesEvent(row[pkey],
1135 start = start,
1136 end = end,
1137 values = values,
1138 **grouping)
1139 add_event(event)
1140 rows_keys |= event.rows
1141 cols_keys |= event.cols
1142
1143 # Extend the event frame with these events
1144 if events:
1145 event_frame.extend(events)
1146
1147 # Store the grouping keys
1148 self.rows_keys = rows_keys
1149 self.cols_keys = cols_keys
1150
1151 return data
1152
1153 # -------------------------------------------------------------------------
1155 """
1156 Resolve the event_start and event_end field selectors
1157
1158 @param event_start: the field selector for the event start field
1159 @param event_end: the field selector for the event end field
1160 """
1161
1162 resource = self.resource
1163 rfields = self.rfields
1164
1165 # Defaults
1166 if not event_start:
1167 table = resource.table
1168 for fname in ("date", "start_date", "created_on"):
1169 if fname in table.fields:
1170 event_start = fname
1171 break
1172 if event_start and not event_end:
1173 for fname in ("end_date",):
1174 if fname in table.fields:
1175 event_end = fname
1176 break
1177 if not event_start:
1178 raise SyntaxError("No time stamps found in %s" % table)
1179
1180 # Get the fields
1181 start_rfield = resource.resolve_selector(event_start)
1182 if event_end:
1183 end_rfield = resource.resolve_selector(event_end)
1184 else:
1185 end_rfield = None
1186
1187 rfields["event_start"] = start_rfield
1188 rfields["event_end"] = end_rfield
1189
1190 # -------------------------------------------------------------------------
1192 """
1193 Resolve the baseline field selector
1194
1195 @param baseline: the baseline selector
1196 """
1197
1198 resource = self.resource
1199 rfields = self.rfields
1200
1201 # Resolve baseline selector
1202 baseline_rfield = None
1203 if baseline:
1204 try:
1205 baseline_rfield = resource.resolve_selector(baseline)
1206 except (AttributeError, SyntaxError):
1207 baseline_rfield = None
1208
1209 if baseline_rfield and \
1210 baseline_rfield.ftype not in NUMERIC_TYPES:
1211 # Invalid field type - log and ignore
1212 current.log.error("Invalid field type for baseline: %s (%s)" %
1213 (baseline, baseline_rfield.ftype))
1214 baseline_rfield = None
1215
1216 rfields["baseline"] = baseline_rfield
1217
1218 # -------------------------------------------------------------------------
1220 """
1221 Resolve the grouping axes field selectors
1222
1223 @param rows: the rows field selector
1224 @param cols: the columns field selector
1225 """
1226
1227 resource = self.resource
1228 rfields = self.rfields
1229
1230 # Resolve rows selector
1231 rows_rfield = None
1232 if rows:
1233 try:
1234 rows_rfield = resource.resolve_selector(rows)
1235 except (AttributeError, SyntaxError):
1236 rows_rfield = None
1237
1238 # Resolve columns selector
1239 cols_rfield = None
1240 if cols:
1241 try:
1242 cols_rfield = resource.resolve_selector(cols)
1243 except (AttributeError, SyntaxError):
1244 cols_rfield = None
1245
1246 rfields["rows"] = rows_rfield
1247 rfields["cols"] = cols_rfield
1248
1249 # -------------------------------------------------------------------------
1250 @staticmethod
1252 """
1253 Parse a string for start/end date(time) of an interval
1254
1255 @param timestr: the time string
1256 @param start: the start datetime to relate relative times to
1257 """
1258
1259 if start is None:
1260 start = tp_tzsafe(datetime.datetime.utcnow())
1261
1262 if not timestr:
1263 return start
1264
1265 # Relative to start: [+|-]{n}[year|month|week|day|hour]s
1266 match = dt_regex.DELTA.match(timestr)
1267 if match:
1268 groups = match.groups()
1269 intervals = {"y": "years",
1270 "m": "months",
1271 "w": "weeks",
1272 "d": "days",
1273 "h": "hours"}
1274 length = intervals.get(groups[2])
1275 if not length:
1276 raise SyntaxError("Invalid date/time: %s" % timestr)
1277 num = int(groups[1])
1278 if not num:
1279 return start
1280 if groups[0] == "-":
1281 num *= -1
1282 return start + relativedelta(**{length: num})
1283
1284 # Month/Year, e.g. "5/2001"
1285 match = dt_regex.MONTH_YEAR.match(timestr)
1286 if match:
1287 groups = match.groups()
1288 year = int(groups[1])
1289 month = int(groups[0])
1290 return tp_datetime(year, month, 1, 0, 0, 0)
1291
1292 # Year-Month, e.g. "2001-05"
1293 match = dt_regex.YEAR_MONTH.match(timestr)
1294 if match:
1295 groups = match.groups()
1296 month = int(groups[1])
1297 year = int(groups[0])
1298 return tp_datetime(year, month, 1, 0, 0, 0)
1299
1300 # Year only, e.g. "1996"
1301 match = dt_regex.YEAR.match(timestr)
1302 if match:
1303 groups = match.groups()
1304 year = int(groups[0])
1305 return tp_datetime(year, 1, 1, 0, 0, 0)
1306
1307 # Date, e.g. "2013-01-04"
1308 match = dt_regex.DATE.match(timestr)
1309 if match:
1310 groups = match.groups()
1311 year = int(groups[0])
1312 month = int(groups[1])
1313 day = int(groups[2])
1314 try:
1315 return tp_datetime(year, month, day)
1316 except ValueError:
1317 # Day out of range
1318 return tp_datetime(year, month, 1) + \
1319 datetime.timedelta(days = day-1)
1320
1321 # ISO datetime
1322 dt = s3_decode_iso_datetime(str(timestr))
1323 return s3_utc(dt)
1324
1325 # =============================================================================
1326 -class S3TimeSeriesEvent(object):
1327 """ Class representing an event """
1328
1329 - def __init__(self,
1330 event_id,
1331 start=None,
1332 end=None,
1333 values=None,
1334 row=DEFAULT,
1335 col=DEFAULT):
1336 """
1337 Constructor
1338
1339 @param event_id: a unique identifier for the event (e.g. record ID)
1340 @param start: start time of the event (datetime.datetime)
1341 @param end: end time of the event (datetime.datetime)
1342 @param values: a dict of key-value pairs with the attribute
1343 values for the event
1344 @param row: the series row for this event
1345 @param col: the series column for this event
1346 """
1347
1348 self.event_id = event_id
1349
1350 self.start = tp_tzsafe(start)
1351 self.end = tp_tzsafe(end)
1352
1353 if isinstance(values, dict):
1354 self.values = values
1355 else:
1356 self.values = {}
1357
1358 self.row = row
1359 self.col = col
1360
1361 self._rows = None
1362 self._cols = None
1363
1364 # -------------------------------------------------------------------------
1365 @property
1367 """
1368 Get the set of row axis keys for this event
1369 """
1370
1371 rows = self._rows
1372 if rows is None:
1373 rows = self._rows = self.series(self.row)
1374 return rows
1375
1376 # -------------------------------------------------------------------------
1377 @property
1379 """
1380 Get the set of column axis keys for this event
1381 """
1382
1383 cols = self._cols
1384 if cols is None:
1385 cols = self._cols = self.series(self.col)
1386 return cols
1387
1388 # -------------------------------------------------------------------------
1389 @staticmethod
1391 """
1392 Convert a field value into a set of series keys
1393
1394 @param value: the field value
1395 """
1396
1397 if value is DEFAULT:
1398 series = set()
1399 elif value is None:
1400 series = set([None])
1401 elif type(value) is list:
1402 series = set(s3_flatlist(value))
1403 else:
1404 series = set([value])
1405 return series
1406
1407 # -------------------------------------------------------------------------
1409 """
1410 Access attribute values of this event
1411
1412 @param field: the attribute field name
1413 """
1414
1415 return self.values.get(field, None)
1416
1417 # -------------------------------------------------------------------------
1419 """
1420 Comparison method to allow sorting of events
1421
1422 @param other: the event to compare to
1423 """
1424
1425 this = self.start
1426 that = other.start
1427 if this is None:
1428 result = that is not None
1429 elif that is None:
1430 result = False
1431 else:
1432 result = this < that
1433 return result
1434
1435 # =============================================================================
1436 -class S3TimeSeriesFact(object):
1437 """ Class representing a fact layer """
1438
1439 #: Supported aggregation methods
1440 METHODS = {"count": "Count",
1441 "sum": "Total",
1442 "cumulate": "Cumulative Total",
1443 "min": "Minimum",
1444 "max": "Maximum",
1445 "avg": "Average",
1446 }
1447
1449 """
1450 Constructor
1451
1452 @param method: the aggregation method
1453 @param base: column name of the (base) field
1454 @param slope: column name of the slope field (for cumulate method)
1455 @param interval: time interval expression for the slope
1456 """
1457
1458 if method not in self.METHODS:
1459 raise SyntaxError("Unsupported aggregation function: %s" % method)
1460
1461 self.method = method
1462 self.base = base
1463 self.slope = slope
1464 self.interval = interval
1465
1466 self.label = label
1467
1468 self.resource = None
1469
1470 self.base_rfield = None
1471 self.base_column = base
1472
1473 self.slope_rfield = None
1474 self.slope_column = slope
1475
1476 # -------------------------------------------------------------------------
1478 """
1479 Aggregate values from events
1480
1481 @param period: the period
1482 @param events: the events
1483 """
1484
1485 values = []
1486 append = values.append
1487
1488 method = self.method
1489 base = self.base_column
1490
1491 if method == "cumulate":
1492
1493 slope = self.slope_column
1494 duration = period.duration
1495
1496 for event in events:
1497
1498 if event.start == None:
1499 continue
1500
1501 if base:
1502 base_value = event[base]
1503 else:
1504 base_value = None
1505
1506 if slope:
1507 slope_value = event[slope]
1508 else:
1509 slope_value = None
1510
1511 if base_value is None:
1512 if not slope or slope_value is None:
1513 continue
1514 else:
1515 base_value = 0
1516 elif type(base_value) is list:
1517 try:
1518 base_value = sum(base_value)
1519 except (TypeError, ValueError):
1520 continue
1521
1522 if slope_value is None:
1523 if not base or base_value is None:
1524 continue
1525 else:
1526 slope_value = 0
1527 elif type(slope_value) is list:
1528 try:
1529 slope_value = sum(slope_value)
1530 except (TypeError, ValueError):
1531 continue
1532
1533 interval = self.interval
1534 if slope_value and interval:
1535 event_duration = duration(event, interval)
1536 else:
1537 event_duration = 1
1538
1539 append((base_value, slope_value, event_duration))
1540
1541 result = self.compute(values)
1542
1543 elif base:
1544
1545 for event in events:
1546 value = event[base]
1547 if value is None:
1548 continue
1549 elif type(value) is list:
1550 values.extend([v for v in value if v is not None])
1551 else:
1552 values.append(value)
1553
1554 if method == "count":
1555 result = len(values)
1556 else:
1557 result = self.compute(values)
1558
1559 else:
1560 result = None
1561
1562 return result
1563
1564 # -------------------------------------------------------------------------
1566 """
1567 Aggregate a list of values.
1568
1569 @param values: iterable of values
1570 """
1571
1572 if values is None:
1573 return None
1574
1575 method = self.method
1576 values = [v for v in values if v != None]
1577
1578 if method == "count":
1579 return len(values)
1580 elif method == "min":
1581 try:
1582 return min(values)
1583 except (TypeError, ValueError):
1584 return None
1585 elif method == "max":
1586 try:
1587 return max(values)
1588 except (TypeError, ValueError):
1589 return None
1590 elif method == "sum":
1591 try:
1592 return sum(values)
1593 except (TypeError, ValueError):
1594 return None
1595 elif method == "avg":
1596 try:
1597 num = len(values)
1598 if num:
1599 return sum(values) / float(num)
1600 except (TypeError, ValueError):
1601 return None
1602 elif method == "cumulate":
1603 try:
1604 return sum(base + slope * duration
1605 for base, slope, duration in values)
1606 except (TypeError, ValueError):
1607 return None
1608 return None
1609
1610 # -------------------------------------------------------------------------
1611 @classmethod
1613 """
1614 Parse fact expression
1615
1616 @param fact: the fact expression
1617 """
1618
1619 if isinstance(fact, list):
1620 facts = []
1621 for f in fact:
1622 facts.extend(cls.parse(f))
1623 if not facts:
1624 raise SyntaxError("Invalid fact expression: %s" % fact)
1625 return facts
1626
1627 if isinstance(fact, tuple):
1628 label, fact = fact
1629 else:
1630 label = None
1631
1632 # Parse the fact
1633 other = None
1634 if not fact:
1635 method, parameters = "count", "id"
1636 else:
1637 match = FACT.match(fact)
1638 if match:
1639 method, parameters, other = match.groups()
1640 if other:
1641 other = cls.parse((label, other) if label else other)
1642 elif SELECTOR.match(fact):
1643 method, parameters, other = "count", fact, None
1644 else:
1645 raise SyntaxError("Invalid fact expression: %s" % fact)
1646
1647 # Validate method
1648 if method not in cls.METHODS:
1649 raise SyntaxError("Unsupported aggregation method: %s" % method)
1650
1651 # Extract parameters
1652 parameters = parameters.split(",")
1653
1654 base = parameters[0]
1655 slope = None
1656 interval = None
1657
1658 if method == "cumulate":
1659 if len(parameters) == 2:
1660 # Slope, Slots
1661 slope = base
1662 base = None
1663 interval = parameters[1]
1664 elif len(parameters) > 2:
1665 # Base, Slope, Slots
1666 slope = parameters[1]
1667 interval = parameters[2]
1668
1669 facts = [cls(method, base, slope=slope, interval=interval, label=label)]
1670 if other:
1671 facts.extend(other)
1672 return facts
1673
1674 # -------------------------------------------------------------------------
1676 """
1677 Resolve the base and slope selectors against resource
1678
1679 @param resource: the resource
1680 """
1681
1682 self.resource = None
1683
1684 base = self.base
1685 self.base_rfield = None
1686 self.base_column = base
1687
1688 slope = self.slope
1689 self.slope_rfield = None
1690 self.slope_column = slope
1691
1692 # Resolve base selector
1693 base_rfield = None
1694 if base:
1695 try:
1696 base_rfield = resource.resolve_selector(base)
1697 except (AttributeError, SyntaxError), e:
1698 base_rfield = None
1699
1700 # Resolve slope selector
1701 slope_rfield = None
1702 if slope:
1703 try:
1704 slope_rfield = resource.resolve_selector(slope)
1705 except (AttributeError, SyntaxError):
1706 slope_rfield = None
1707
1708 method = self.method
1709
1710 # At least one field parameter must be resolvable
1711 if base_rfield is None:
1712 if method != "cumulate" or slope_rfield is None:
1713 raise SyntaxError("Invalid fact parameter")
1714
1715 # All methods except count require numeric input values
1716 if method != "count":
1717 numeric_types = NUMERIC_TYPES
1718 if base_rfield and base_rfield.ftype not in numeric_types:
1719 raise SyntaxError("Fact field type not numeric: %s (%s)" %
1720 (base, base_rfield.ftype))
1721
1722 if slope_rfield and slope_rfield.ftype not in numeric_types:
1723 raise SyntaxError("Fact field type not numeric: %s (%s)" %
1724 (slope, slope_rfield.ftype))
1725
1726 if base_rfield:
1727 self.base_rfield = base_rfield
1728 self.base_column = base_rfield.colname
1729
1730 if slope_rfield:
1731 self.slope_rfield = slope_rfield
1732 self.slope_column = slope_rfield.colname
1733
1734 if not self.label:
1735 # Lookup the label from the timeplot options
1736 label = self.lookup_label(resource,
1737 method,
1738 base,
1739 slope,
1740 self.interval)
1741 if not label:
1742 # Generate a default label
1743 label = self.default_label(base_rfield, self.method)
1744 self.label = label
1745
1746 self.resource = resource
1747 return self
1748
1749 # -------------------------------------------------------------------------
1750 @classmethod
1752 """
1753 Lookup the fact label from the timeplot options of resource
1754
1755 @param resource: the resource (S3Resource)
1756 @param method: the aggregation method (string)
1757 @param base: the base field selector (string)
1758 @param slope: the slope field selector (string)
1759 @param interval: the interval expression (string)
1760 """
1761
1762 fact_opts = None
1763 if resource:
1764 config = resource.get_config("timeplot_options")
1765 if config:
1766 fact_opts = config.get("fact")
1767
1768 label = None
1769 if fact_opts:
1770 parse = cls.parse
1771 for opt in fact_opts:
1772 if isinstance(opt, tuple):
1773 title, facts = opt
1774 else:
1775 title, facts = None, opt
1776 facts = parse(facts)
1777 match = None
1778 for fact in facts:
1779 if fact.method == method and \
1780 fact.base == base and \
1781 fact.slope == slope and \
1782 fact.interval == interval:
1783 match = fact
1784 break
1785 if match:
1786 if fact.label:
1787 label = fact.label
1788 elif len(facts) == 1:
1789 label = title
1790 if label:
1791 break
1792
1793 return label
1794
1795 # -------------------------------------------------------------------------
1796 @classmethod
1798 """
1799 Generate a default fact label
1800
1801 @param rfield: the S3ResourceField (alternatively the field label)
1802 @param method: the aggregation method
1803 """
1804
1805 T = current.T
1806
1807 if hasattr(rfield, "ftype") and \
1808 rfield.ftype == "id" and \
1809 method == "count":
1810 field_label = T("Records")
1811 elif hasattr(rfield, "label"):
1812 field_label = rfield.label
1813 else:
1814 field_label = rfield
1815
1816 method_label = cls.METHODS.get(method)
1817 if not method_label:
1818 method_label = method
1819 else:
1820 method_label = T(method_label)
1821
1822 return "%s (%s)" % (field_label, method_label)
1823
1824 # =============================================================================
1825 -class S3TimeSeriesPeriod(object):
1826 """
1827 Class representing a single time period (slot) in an event frame,
1828 within which events will be grouped and facts aggregated
1829 """
1830
1832 """
1833 Constructor
1834
1835 @param start: the start of the time period (datetime)
1836 @param end: the end of the time period (datetime)
1837 """
1838
1839 self.start = tp_tzsafe(start)
1840 self.end = tp_tzsafe(end)
1841
1842 # Event sets
1843 self.pevents = {}
1844 self.cevents = {}
1845
1846 self._reset()
1847
1848 # -------------------------------------------------------------------------
1850 """ Reset the event matrix """
1851
1852 self._matrix = None
1853 self._rows = None
1854 self._cols = None
1855
1856 self._reset_aggregates()
1857
1858 # -------------------------------------------------------------------------
1860 """ Reset the aggregated values matrix """
1861
1862 self.matrix = None
1863 self.rows = None
1864 self.cols = None
1865 self.totals = None
1866
1867 # -------------------------------------------------------------------------
1869 """
1870 Add a current event to this period
1871
1872 @param event: the S3TimeSeriesEvent
1873 """
1874
1875 self.cevents[event.event_id] = event
1876
1877 # -------------------------------------------------------------------------
1879 """
1880 Add a previous event to this period
1881
1882 @param event: the S3TimeSeriesEvent
1883 """
1884
1885 self.pevents[event.event_id] = event
1886
1887 # -------------------------------------------------------------------------
1889 """
1890 Convert the aggregated results into a JSON-serializable dict
1891
1892 @param rows: the row keys for the result
1893 @param cols: the column keys for the result
1894 @param isoformat: convert datetimes into ISO-formatted strings
1895 """
1896
1897 # Start and end datetime
1898 start = self.start
1899 if start and isoformat:
1900 start = start.isoformat()
1901 end = self.end
1902 if end and isoformat:
1903 end = end.isoformat()
1904
1905 # Row totals
1906 row_totals = None
1907 if rows is not None:
1908 row_data = self.rows
1909 row_totals = [row_data.get(key) for key in rows]
1910
1911 # Column totals
1912 col_totals = None
1913 if cols is not None:
1914 col_data = self.cols
1915 col_totals = [col_data.get(key) for key in cols]
1916
1917 # Matrix
1918 matrix = None
1919 if rows is not None and cols is not None:
1920 matrix_data = self.matrix
1921 matrix = []
1922 for row in rows:
1923 matrix_row = []
1924 for col in cols:
1925 matrix_row.append(matrix_data.get((row, col)))
1926 matrix.append(matrix_row)
1927
1928 # Output
1929 return {"t": (start, end),
1930 "v": self.totals,
1931 "r": row_totals,
1932 "c": col_totals,
1933 "x": matrix,
1934 }
1935
1936 # -------------------------------------------------------------------------
1938 """
1939 Group events by their row and col axis values
1940
1941 @param cumulative: include previous events
1942 """
1943
1944 event_sets = [self.cevents]
1945 if cumulative:
1946 event_sets.append(self.pevents)
1947
1948 rows = {}
1949 cols = {}
1950 matrix = {}
1951 from itertools import product
1952 for index, events in enumerate(event_sets):
1953 for event_id, event in events.items():
1954 for key in event.rows:
1955 row = rows.get(key)
1956 if row is None:
1957 row = rows[key] = (set(), set())
1958 row[index].add(event_id)
1959 for key in event.cols:
1960 col = cols.get(key)
1961 if col is None:
1962 col = cols[key] = (set(), set())
1963 col[index].add(event_id)
1964 for key in product(event.rows, event.cols):
1965 cell = matrix.get(key)
1966 if cell is None:
1967 cell = matrix[key] = (set(), set())
1968 cell[index].add(event_id)
1969 self._rows = rows
1970 self._cols = cols
1971 self._matrix = matrix
1972
1973 # -------------------------------------------------------------------------
1975 """
1976 Group and aggregate the events in this period
1977
1978 @param facts: list of facts to aggregate
1979 """
1980
1981 # Reset
1982 self._reset()
1983
1984 rows = self.rows = {}
1985 cols = self.cols = {}
1986 matrix = self.matrix = {}
1987
1988 totals = []
1989
1990 if not isinstance(facts, (list, tuple)):
1991 facts = [facts]
1992 if any(fact.method == "cumulate" for fact in facts):
1993 self.group(cumulative=True)
1994 else:
1995 self.group()
1996
1997 for fact in facts:
1998
1999 method = fact.method
2000
2001 # Select events
2002 if method == "cumulate":
2003 events = dict(self.pevents)
2004 events.update(self.cevents)
2005 cumulative = True
2006 else:
2007 events = self.cevents
2008 cumulative = False
2009
2010 fact_aggregate = fact.aggregate
2011 aggregate = lambda items: fact_aggregate(self, items)
2012
2013 # Aggregate rows
2014 for key, event_sets in self._rows.items():
2015 event_ids = event_sets[0]
2016 if cumulative:
2017 event_ids |= event_sets[1]
2018 items = [events[event_id] for event_id in event_ids]
2019 if key not in rows:
2020 rows[key] = [aggregate(items)]
2021 else:
2022 rows[key].append(aggregate(items))
2023
2024 # Aggregate columns
2025 for key, event_sets in self._cols.items():
2026 event_ids = event_sets[0]
2027 if cumulative:
2028 event_ids |= event_sets[1]
2029 items = [events[event_id] for event_id in event_ids]
2030 if key not in cols:
2031 cols[key] = [aggregate(items)]
2032 else:
2033 cols[key].append(aggregate(items))
2034
2035 # Aggregate matrix
2036 for key, event_sets in self._matrix.items():
2037 event_ids = event_sets[0]
2038 if cumulative:
2039 event_ids |= event_sets[1]
2040 items = [events[event_id] for event_id in event_ids]
2041 if key not in matrix:
2042 matrix[key] = [aggregate(items)]
2043 else:
2044 matrix[key].append(aggregate(items))
2045
2046 # Aggregate total
2047 totals.append(aggregate(events.values()))
2048
2049 self.totals = totals
2050 return totals
2051
2052 # -------------------------------------------------------------------------
2054 """
2055 Compute the total duration of the given event before the end
2056 of this period, in number of interval
2057
2058 @param event: the S3TimeSeriesEvent
2059 @param interval: the interval expression (string)
2060 """
2061
2062 if event.end is None or event.end > self.end:
2063 end_date = self.end
2064 else:
2065 end_date = event.end
2066 if event.start is None or event.start >= end_date:
2067 result = 0
2068 else:
2069 rule = self.get_rule(event.start, end_date, interval)
2070 if rule:
2071 result = rule.count()
2072 else:
2073 result = 1
2074 return result
2075
2076 # -------------------------------------------------------------------------
2077 @staticmethod
2079 """
2080 Convert a time slot string expression into a dateutil rrule
2081 within the context of a time period
2082
2083 @param start: the start of the time period (datetime)
2084 @param end: the end of the time period (datetime)
2085 @param interval: time interval expression, like "days" or "2 weeks"
2086 """
2087
2088 match = re.match(r"\s*(\d*)\s*([hdwmy]{1}).*", interval)
2089 if match:
2090 num, delta = match.groups()
2091 deltas = {
2092 "h": HOURLY,
2093 "d": DAILY,
2094 "w": WEEKLY,
2095 "m": MONTHLY,
2096 "y": YEARLY,
2097 }
2098 if delta not in deltas:
2099 return None
2100 else:
2101 num = int(num) if num else 1
2102 return rrule(deltas[delta],
2103 dtstart=start,
2104 until=end,
2105 interval=num)
2106 else:
2107 return None
2108
2109 # =============================================================================
2110 -class S3TimeSeriesEventFrame(object):
2111 """ Class representing the whole time frame of a time plot """
2112
2114 """
2115 Constructor
2116
2117 @param start: start of the time frame (datetime.datetime)
2118 @param end: end of the time frame (datetime.datetime)
2119 @param slot: length of time slots within the event frame,
2120 format: "{n }[hour|day|week|month|year]{s}",
2121 examples: "1 week", "3 months", "years"
2122 """
2123
2124 # Start time is required
2125 if start is None:
2126 raise SyntaxError("start time required")
2127 self.start = tp_tzsafe(start)
2128
2129 # End time defaults to now
2130 if end is None:
2131 end = datetime.datetime.utcnow()
2132 self.end = tp_tzsafe(end)
2133
2134 self.empty = True
2135 self.baseline = None
2136
2137 self.slots = slots
2138 self.periods = {}
2139
2140 self.rule = self.get_rule()
2141
2142 # -------------------------------------------------------------------------
2144 """
2145 Get the recurrence rule for the periods
2146 """
2147
2148 slots = self.slots
2149 if not slots:
2150 return None
2151
2152 return S3TimeSeriesPeriod.get_rule(self.start, self.end, slots)
2153
2154 # -------------------------------------------------------------------------
2156 """
2157 Extend this time frame with events
2158
2159 @param events: iterable of events
2160
2161 @todo: integrate in constructor
2162 @todo: handle self.rule == None
2163 """
2164
2165 if not events:
2166 return
2167 empty = self.empty
2168
2169 # Order events by start datetime
2170 events = sorted(events)
2171
2172 rule = self.rule
2173 periods = self.periods
2174
2175 # No point to loop over periods before the first event:
2176 start = events[0].start
2177 if start is None or start <= self.start:
2178 first = rule[0]
2179 else:
2180 first = rule.before(start, inc=True)
2181
2182 current_events = {}
2183 previous_events = {}
2184 for start in rule.between(first, self.end, inc=True):
2185
2186 # Compute end of this period
2187 end = rule.after(start)
2188 if not end:
2189 if start < self.end:
2190 end = self.end
2191 else:
2192 # Period start is at the end of the event frame
2193 break
2194
2195 # Find all current events
2196 for index, event in enumerate(events):
2197 if event.end and event.end < start:
2198 # Event ended before this period
2199 previous_events[event.event_id] = event
2200 elif event.start is None or event.start < end:
2201 # Event starts before or during this period
2202 current_events[event.event_id] = event
2203 else:
2204 # Event starts only after this period
2205 break
2206
2207 # Add current events to current period
2208 period = periods.get(start)
2209 if period is None:
2210 period = periods[start] = S3TimeSeriesPeriod(start, end=end)
2211 for event in current_events.values():
2212 period.add_current(event)
2213 for event in previous_events.values():
2214 period.add_previous(event)
2215
2216 empty = False
2217
2218 # Remaining events
2219 events = events[index:]
2220 if not events:
2221 # No more events
2222 break
2223
2224 # Remove events which end during this period
2225 remaining = {}
2226 for event_id, event in current_events.items():
2227 if not event.end or event.end > end:
2228 remaining[event_id] = event
2229 else:
2230 previous_events[event_id] = event
2231 current_events = remaining
2232
2233 self.empty = empty
2234 return
2235
2236 # -------------------------------------------------------------------------
2238 """
2239 Iterate over all periods within this event frame
2240 """
2241
2242 periods = self.periods
2243
2244 rule = self.rule
2245 if rule:
2246 for dt in rule:
2247 if dt >= self.end:
2248 break
2249 if dt in periods:
2250 yield periods[dt]
2251 else:
2252 end = rule.after(dt)
2253 if not end:
2254 end = self.end
2255 yield S3TimeSeriesPeriod(dt, end=end)
2256 else:
2257 # @todo: continuous periods
2258 # sort actual periods and iterate over them
2259 raise NotImplementedError
2260
2261 return
2262
2263 # END =========================================================================
2264
| Home | Trees | Indices | Help |
|
|---|
| Generated by Epydoc 3.0.1 on Fri Mar 15 08:52:03 2019 | http://epydoc.sourceforge.net |