1
2
3 """ S3 Synchronization: Peer Repository Adapter
4
5 @copyright: 2014-2019 (c) Sahana Software Foundation
6 @license: MIT
7
8 Permission is hereby granted, free of charge, to any person
9 obtaining a copy of this software and associated documentation
10 files (the "Software"), to deal in the Software without
11 restriction, including without limitation the rights to use,
12 copy, modify, merge, publish, distribute, sublicense, and/or sell
13 copies of the Software, and to permit persons to whom the
14 Software is furnished to do so, subject to the following
15 conditions:
16
17 The above copyright notice and this permission notice shall be
18 included in all copies or substantial portions of the Software.
19
20 THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
21 EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
22 OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
23 NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
24 HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
25 WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
26 FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
27 OTHER DEALINGS IN THE SOFTWARE.
28 """
29
30 import json
31 import sys
32 import urllib, urllib2
33
34 try:
35 from lxml import etree
36 except ImportError:
37 sys.stderr.write("ERROR: lxml module needed for XML handling\n")
38 raise
39
40 from gluon import *
41
42 from ..s3datetime import s3_encode_iso_datetime
43 from ..s3sync import S3SyncBaseAdapter
44 from ..s3utils import s3_unicode
48 """
49 Wrike® Synchronization Adapter
50
51 http://www.wrike.com/wiki/dev/api3
52 """
53
54
56 """
57 Constructor
58 """
59
60 super(S3SyncAdapter, self).__init__(repository)
61
62 self.access_token = None
63 self.token_type = None
64
65
66
67
69 """
70 Register at the repository, in Wrike: use client ID and the
71 authorization code (site key), alternatively username and
72 password to obtain the refresh_token and store it in the
73 repository config.
74
75 @note: this invalidates the authorization code (if any), so it
76 will be set to None regardless whether this operation
77 succeeds or not
78
79 @return: True if successful, otherwise False
80 """
81
82 repository = self.repository
83
84 log = repository.log
85 success = False
86 remote = False
87 skip = False
88
89 data = None
90 site_key = repository.site_key
91 if not site_key:
92 username = repository.username
93 password = repository.password
94 if username and password:
95 data = {
96 "client_id": repository.client_id,
97 "client_secret": repository.client_secret,
98 "username": username,
99 "password": password,
100 "grant_type": "password",
101 }
102 else:
103 data = {
104 "client_id": repository.client_id,
105 "client_secret": repository.client_secret,
106 "grant_type": "authorization_code",
107 "code": site_key,
108 }
109
110 if not data:
111 if not repository.refresh_token:
112
113 result = log.WARNING
114 message = "No credentials to obtain refresh token " \
115 "with, skipping registration"
116 else:
117
118 result = log.SUCCESS
119 success = True
120 message = None
121 skip = True
122
123 else:
124 repository.refresh_token = None
125 self.access_token = None
126
127
128 response, message = self._send_request(method = "POST",
129 data = data,
130 auth = True,
131 )
132 if not response:
133 result = log.FATAL
134 remote = True
135 else:
136 refresh_token = response.get("refresh_token")
137 if not refresh_token:
138 result = log.FATAL
139 message = "No refresh token received"
140 else:
141 repository.refresh_token = refresh_token
142 self.access_token = response.get("access_token")
143 result = log.SUCCESS
144 success = True
145 message = "Registration successful"
146 self.update_refresh_token()
147
148
149 log.write(repository_id=repository.id,
150 transmission=log.OUT,
151 mode=log.PUSH,
152 action="request refresh token",
153 remote=remote,
154 result=result,
155 message=message)
156
157 if not success:
158 current.log.error(message)
159 return success if not skip else True
160
161
163 """
164 Login to the repository, in Wrike: use the client ID (username),
165 the client secret (password) and the refresh token to obtain the
166 access token for subsequent requests.
167
168 @return: None if successful, otherwise error message
169 """
170
171 repository = self.repository
172
173 log = repository.log
174 error = None
175 remote = False
176
177 refresh_token = repository.refresh_token
178 if not refresh_token:
179 result = log.FATAL
180 error = "Login failed: no refresh token available (registration failed?)"
181 else:
182 data = {
183 "client_id": repository.client_id,
184 "client_secret": repository.client_secret,
185 "grant_type": "refresh_token",
186 "refresh_token": refresh_token
187 }
188 response, message = self._send_request(method = "POST",
189 data = data,
190 auth = True,
191 )
192 if not response:
193 result = log.FATAL
194 remote = True
195 error = message
196 else:
197 access_token = response.get("access_token")
198 if not access_token:
199 result = log.FATAL
200 error = "No access token received"
201 else:
202 result = log.SUCCESS
203 self.access_token = access_token
204 self.token_type = response.get("token_type", "bearer")
205
206
207
208 log.write(repository_id=repository.id,
209 transmission=log.OUT,
210 mode=log.PUSH,
211 action="request access token",
212 remote=remote,
213 result=result,
214 message=error)
215
216 if error:
217 current.log.error(error)
218 return error
219
220
221 - def pull(self, task, onconflict=None):
222 """
223 Pull updates from this repository
224
225 @param task: the task Row
226 @param onconflict: synchronization conflict resolver
227 @return: tuple (error, mtime), with error=None if successful,
228 else error=message, and mtime=modification timestamp
229 of the youngest record received
230 """
231
232 repository = self.repository
233
234 resource_name = task.resource_name
235
236 current.log.debug("S3SyncWrike.pull(%s, %s)" %
237 (repository.url, resource_name))
238
239 xml = current.xml
240 log = repository.log
241
242
243 last_pull = task.last_pull
244 if last_pull and task.update_policy not in ("THIS", "OTHER"):
245 msince = s3_encode_iso_datetime(last_pull)
246 else:
247 msince = None
248
249
250 root = etree.Element("wrike-data")
251
252
253 accounts, error = self.fetch_accounts(root)
254 if accounts is not None:
255
256 error = None
257
258 def log_fetch_error(action, account_name, message):
259 """ Helper to log non-fatal errors during fetch """
260 action = "%s for account '%s'" % (action, account_name)
261 log.write(repository_id = repository.id,
262 resource_name = resource_name,
263 transmission = log.OUT,
264 mode = log.PULL,
265 action = action,
266 remote = True,
267 result = log.ERROR,
268 message = message)
269 return
270
271 for account_id, account_data in accounts.items():
272
273 account_name, root_folder_id, recycle_bin_id = account_data
274
275
276 response, message = self.fetch_folders(root, account_id)
277 if response is None:
278 log_fetch_error("fetch folders",
279 account_name,
280 message)
281 continue
282
283
284 response, message = self.fetch_tasks(root,
285 root_folder_id,
286 msince=msince)
287 if response is None:
288 log_fetch_error("fetch active tasks",
289 account_name,
290 message)
291
292
293 response, message = self.fetch_tasks(root,
294 recycle_bin_id,
295 msince=msince,
296 deleted = True)
297 if response is None:
298 log_fetch_error("fetch deleted tasks",
299 account_name,
300 message)
301
302 if error:
303
304 result = log.FATAL
305 remote = True
306 message = error
307 output = xml.json_message(False, 400, error)
308
309 elif len(root):
310 result = log.SUCCESS
311 remote = False
312 message = None
313 output = None
314
315
316 tree = etree.ElementTree(root)
317
318
319 strategy = task.strategy
320 update_policy = task.update_policy
321 conflict_policy = task.conflict_policy
322
323
324 folder = current.request.folder
325 import os
326 stylesheet = os.path.join(folder,
327 "static",
328 "formats",
329 "wrike",
330 "import.xsl")
331
332
333 import urlparse
334 hostname = urlparse.urlsplit(repository.url).hostname
335
336
337 resource = current.s3db.resource(resource_name)
338 if onconflict:
339 onconflict_callback = lambda item: onconflict(item,
340 repository,
341 resource)
342 else:
343 onconflict_callback = None
344
345
346 count = 0
347 try:
348 success = resource.import_xml(tree,
349 stylesheet=stylesheet,
350 ignore_errors=True,
351 strategy=strategy,
352 update_policy=update_policy,
353 conflict_policy=conflict_policy,
354 last_sync=task.last_pull,
355 onconflict=onconflict_callback,
356 site=hostname)
357 count = resource.import_count
358 except IOError:
359 success = False
360 result = log.FATAL
361 message = sys.exc_info()[1]
362 output = xml.json_message(False, 400, message)
363
364 mtime = resource.mtime
365
366
367 if resource.error_tree is not None:
368 result = log.WARNING
369 message = "%s" % resource.error
370 for element in resource.error_tree.findall("resource"):
371 for field in element.findall("data[@error]"):
372 error_msg = field.get("error", None)
373 if error_msg:
374 msg = "(UID: %s) %s.%s=%s: %s" % \
375 (element.get("uuid", None),
376 element.get("name", None),
377 field.get("field", None),
378 field.get("value", field.text),
379 field.get("error", None))
380 message = "%s, %s" % (message, msg)
381
382
383 if not success:
384 result = log.FATAL
385 if not message:
386 message = "%s" % resource.error
387 output = xml.json_message(False, 400, message)
388 mtime = None
389
390
391 elif not message:
392 message = "Data imported successfully (%s records)" % count
393 output = None
394
395 else:
396
397 result = log.WARNING
398 remote = True
399 message = "No data received from peer"
400 output = None
401
402
403 log.write(repository_id=repository.id,
404 resource_name=resource_name,
405 transmission=log.OUT,
406 mode=log.PULL,
407 action=None,
408 remote=remote,
409 result=result,
410 message=message)
411
412 current.log.debug("S3SyncWrike.pull import %s: %s" % (result, message))
413
414 return (output, mtime)
415
416
417 - def push(self, task):
418 """
419 Push data for a task
420
421 @param task: the task Row
422 @return: tuple (error, mtime), with error=None if successful,
423 else error=message, and mtime=modification timestamp
424 of the youngest record sent
425 """
426
427 error = "Wrike API push not implemented"
428 current.log.error(error)
429 return (error, None)
430
431
432
433
435 """
436 Get all accessible accounts
437
438 @return: dict {account_id: (rootFolderId, recycleBinId)}
439 """
440
441 response, message = self._send_request(path="accounts")
442 if not response:
443 return None, message
444
445 accounts = {}
446 data = response.get("data")
447 if data and type(data) is list:
448 SubElement = etree.SubElement
449 for account_data in data:
450 account_id = account_data.get("id")
451 account = SubElement(root, "account",
452 id = str(account_id))
453 account_name = account_data.get("name")
454 name = SubElement(account, "name")
455 name.text = account_name
456
457 accounts[account_id] = (account_name,
458 account_data.get("rootFolderId"),
459 account_data.get("recycleBinId"))
460 return accounts, None
461
462
464 """
465 Fetch folders from a Wrike account and add them to the
466 data tree
467
468 @param root: the root element of the data tree
469 @param account_id: the Wrike account ID
470 """
471
472 response, message = self._send_request(path="accounts/%s/folders" % account_id)
473 if not response:
474 return None, message
475
476 folders = {}
477 data = response.get("data")
478 if data and type(data) is list:
479 SubElement = etree.SubElement
480 for folder_data in data:
481 scope = folder_data.get("scope")
482 if scope not in ("WsFolder", "RbFolder"):
483 continue
484 folder_id = folder_data.get("id")
485 folder = SubElement(root, "folder",
486 id = str(folder_id))
487 folders[folder_id] = folder
488 if scope == "RbFolder":
489 folder.set("deleted", str(True))
490 else:
491 title = SubElement(folder, "title")
492 title.text = folder_data.get("title")
493 account = SubElement(folder, "accountId")
494 account.text = str(account_id)
495
496 return folders, None
497
498
499 - def fetch_tasks(self, root, folder_id, deleted=False, msince=None):
500 """
501 Fetch all tasks in a folder
502
503 @param root: the root element of the data tree
504 @param folder_id: the ID of the folder to read from
505 @param deleted: mark the tasks as deleted in the data
506 tree (when reading tasks from a recycle bin)
507 @param msince: only retrieve tasks that have been modified
508 after this date/time (ISO-formatted string)
509 """
510
511 fields = json.dumps(["parentIds", "description"])
512 args = {"descendants": "true",
513 "fields": json.dumps(["parentIds",
514 "description",
515 ]
516 ),
517 }
518 if msince is not None:
519 args["updatedDate"] = "%sZ," % msince
520 response, message = self._send_request(path = "folders/%s/tasks" % folder_id,
521 args = args,
522 )
523 if not response:
524 return None, message
525
526 details = {"title": "title",
527 "description": "description",
528 "status": "status",
529 "importance": "importance",
530 "permalink": "permalink",
531 "createdDate": "createdDate",
532 "updatedDate": "updatedDate",
533 "dates": {"due": "dueDate",
534 }
535 }
536
537 tasks = {}
538 data = response.get("data")
539 if data and type(data) is list:
540 SubElement = etree.SubElement
541 for task_data in data:
542 scope = task_data.get("scope")
543 if scope not in ("WsTask", "RbTask"):
544 continue
545 task_id = task_data.get("id")
546 task = SubElement(root, "task", id = str(task_id))
547 tasks[task_id] = task
548 deleted = scope == "RbTask"
549 if deleted:
550 task.set("deleted", str(True))
551 continue
552 parent_ids = task_data.get("parentIds")
553 if parent_ids:
554 for parent_id in parent_ids:
555 parent = SubElement(task, "parentId")
556 parent.text = str(parent_id)
557 self.add_details(task, task_data, details)
558
559 return tasks, None
560
561
562 @classmethod
564 """
565 Recursively convert the nested task details dicts into SubElements
566
567 @param task: the task Element
568 @param data: the nested dict
569 @param keys: the mapping of dict keys to SubElement names
570 """
571
572 if not isinstance(data, dict):
573 return
574 SubElement = etree.SubElement
575 for key, name in keys.items():
576 wrapper = data.get(key)
577 if wrapper is None:
578 continue
579 if isinstance(name, dict):
580 cls.add_details(task, wrapper, name)
581 else:
582 detail = SubElement(task, name)
583 detail.text = s3_unicode(wrapper)
584 return
585
586
588 """
589 Store the current refresh token in the db, also invalidated
590 the site_key (authorization code) because it can not be used
591 again.
592 """
593
594 repository = self.repository
595 repository.site_key = None
596
597 table = current.s3db.sync_repository
598 current.db(table.id == repository.id).update(
599 refresh_token = repository.refresh_token,
600 site_key = repository.site_key
601 )
602 return
603
604
605 - def _send_request(self,
606 method="GET",
607 path=None,
608 args=None,
609 data=None,
610 auth=False):
611 """
612 Send a request to the Wrike API
613
614 @param method: the HTTP method
615 @param path: the path relative to the repository URL
616 @param data: the data to send
617 @param auth: this is an authorization request
618 """
619
620 repository = self.repository
621
622
623 api = "oauth2/token" if auth else "api/v3"
624 url = "/".join((repository.url.rstrip("/"), api))
625 if path:
626 url = "/".join((url, path.lstrip("/")))
627 if args:
628 url = "?".join((url, urllib.urlencode(args)))
629
630
631 req = urllib2.Request(url=url)
632 handlers = []
633
634 if not auth:
635
636 access_token = self.access_token
637 if not access_token:
638 message = "Authorization failed: no access token"
639 current.log.error(message)
640 return None, message
641 req.add_header("Authorization", "%s %s" %
642 (self.token_type, access_token))
643
644 request_data = json.dumps(data) if data else ""
645 if request_data:
646 req.add_header("Content-Type", "application/json")
647 else:
648
649 request_data = urllib.urlencode(data) if data else ""
650
651
652 req.add_header("Accept", "application/json")
653
654
655 config = repository.config
656 proxy = repository.proxy or config.proxy or None
657 if proxy:
658 current.log.debug("using proxy=%s" % proxy)
659 proxy_handler = urllib2.ProxyHandler({"https": proxy})
660 handlers.append(proxy_handler)
661
662
663 if handlers:
664 opener = urllib2.build_opener(*handlers)
665 urllib2.install_opener(opener)
666
667
668 response = None
669 message = None
670 try:
671 if method == "POST":
672 f = urllib2.urlopen(req, data=request_data)
673 else:
674 f = urllib2.urlopen(req)
675 except urllib2.HTTPError, e:
676 message = "HTTP %s: %s" % (e.code, e.reason)
677 else:
678
679 try:
680 response = json.load(f)
681 except ValueError, e:
682 message = sys.exc_info()[1]
683
684 return response, message
685
686
687