1
2
3 """ S3 Record Merger
4
5 @requires: U{B{I{gluon}} <http://web2py.com>}
6
7 @copyright: 2012-2019 (c) Sahana Software Foundation
8 @license: MIT
9
10 Permission is hereby granted, free of charge, to any person
11 obtaining a copy of this software and associated documentation
12 files (the "Software"), to deal in the Software without
13 restriction, including without limitation the rights to use,
14 copy, modify, merge, publish, distribute, sublicense, and/or sell
15 copies of the Software, and to permit persons to whom the
16 Software is furnished to do so, subject to the following
17 conditions:
18
19 The above copyright notice and this permission notice shall be
20 included in all copies or substantial portions of the Software.
21
22 THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
23 EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
24 OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
25 NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
26 HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
27 WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
28 FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
29 OTHER DEALINGS IN THE SOFTWARE.
30 """
31
32 from gluon import current, IS_NOT_IN_DB, \
33 A, DIV, FORM, H3, INPUT, SPAN, SQLFORM, \
34 TABLE, TBODY, TD, TFOOT, TH, THEAD, TR
35 from gluon.storage import Storage
36
37 from s3dal import Field
38 from s3data import S3DataTable
39 from s3query import FS
40 from s3rest import S3Method
41 from s3utils import s3_get_foreign_key, s3_represent_value, s3_str
45 """ Interactive Record Merger """
46
47 DEDUPLICATE = "deduplicate"
48
49 ORIGINAL = "original"
50 DUPLICATE = "duplicate"
51 KEEP = Storage(o="keep_original", d="keep_duplicate")
52
53
55 """
56 Apply Merge methods
57
58 @param r: the S3Request
59 @param attr: dictionary of parameters for the method handler
60
61 @return: output object to send to the view
62 """
63
64 output = dict()
65
66 auth = current.auth
67 system_roles = auth.get_system_roles()
68 if not auth.s3_has_role(system_roles.ADMIN):
69 r.unauthorized()
70
71 if r.method == "deduplicate":
72 if r.http in ("GET", "POST"):
73 if "remove" in r.get_vars:
74 remove = r.get_vars["remove"].lower() in ("1", "true")
75 else:
76 remove = False
77 if remove:
78 output = self.unmark(r, **attr)
79 elif self.record_id:
80 if r.component and not r.component.multiple:
81 if r.http == "POST":
82 output = self.mark(r, **attr)
83 else:
84
85
86
87 output = self.duplicates(r, **attr)
88 else:
89 output = self.mark(r, **attr)
90 else:
91 output = self.duplicates(r, **attr)
92 elif r.http == "DELETE":
93 output = self.unmark(r, **attr)
94 else:
95 r.error(405, current.ERROR.BAD_METHOD)
96 else:
97 r.error(405, current.ERROR.BAD_METHOD)
98
99 return output
100
101
102 - def mark(self, r, **attr):
103 """
104 Bookmark the current record for de-duplication
105
106 @param r: the S3Request
107 @param attr: the controller parameters for the request
108 """
109
110 s3 = current.session.s3
111
112 DEDUPLICATE = self.DEDUPLICATE
113
114 if DEDUPLICATE not in s3:
115 bookmarks = s3[DEDUPLICATE] = Storage()
116 else:
117 bookmarks = s3[DEDUPLICATE]
118
119 record_id = str(self.record_id)
120 if record_id:
121 tablename = self.tablename
122 if tablename not in bookmarks:
123 records = bookmarks[tablename] = []
124 else:
125 records = bookmarks[tablename]
126 if record_id not in records:
127 records.append(record_id)
128 else:
129 return self.duplicates(r, **attr)
130
131 return current.xml.json_message()
132
133
135 """
136 Remove a record from the deduplicate list
137
138 @param r: the S3Request
139 @param attr: the controller parameters for the request
140 """
141
142 s3 = current.session.s3
143 DEDUPLICATE = self.DEDUPLICATE
144
145 success = current.xml.json_message()
146
147 if DEDUPLICATE not in s3:
148 return success
149 else:
150 bookmarks = s3[DEDUPLICATE]
151 tablename = self.tablename
152 if tablename not in bookmarks:
153 return success
154 else:
155 records = bookmarks[tablename]
156
157 record_id = str(self.record_id)
158 if record_id:
159 if record_id in records:
160 records.remove(record_id)
161 if not records:
162 bookmarks.pop(tablename)
163 else:
164 bookmarks.pop(tablename)
165 return success
166
167
168 @classmethod
169 - def bookmark(cls, r, tablename, record_id):
170 """
171 Get a bookmark link for a record in order to embed it in the
172 view, also renders a link to the duplicate bookmark list to
173 initiate the merge process from
174
175 @param r: the S3Request
176 @param tablename: the table name
177 @param record_id: the record ID
178 """
179
180 auth = current.auth
181 system_roles = auth.get_system_roles()
182 if not auth.s3_has_role(system_roles.ADMIN):
183 return ""
184 if r.component and not r.component.multiple:
185
186 return ""
187
188 s3 = current.session.s3
189 DEDUPLICATE = cls.DEDUPLICATE
190
191 remove = DEDUPLICATE in s3 and \
192 tablename in s3[DEDUPLICATE] and \
193 str(record_id) in s3[DEDUPLICATE][tablename] and \
194 True or False
195
196 mark = "mark-deduplicate action-lnk"
197 unmark = "unmark-deduplicate action-lnk"
198 deduplicate = "deduplicate action-lnk"
199
200 if remove:
201 mark += " hide"
202 else:
203 unmark += " hide"
204 deduplicate += " hide"
205
206 T = current.T
207 link = DIV(A(T("Mark as duplicate"),
208 _class=mark,
209 ),
210 A(T("Unmark as duplicate"),
211 _class=unmark,
212 ),
213 A("",
214 _href=r.url(method="deduplicate", vars={}),
215 _id="markDuplicateURL",
216 _class="hide",
217 ),
218 A(T("De-duplicate"),
219 _href=r.url(method="deduplicate", target=0, vars={}),
220 _class=deduplicate,
221 ),
222 _id="markDuplicate",
223 )
224
225 return link
226
227
229 """
230 Renders a list of all currently duplicate-bookmarked
231 records in this resource, with option to select two
232 and initiate the merge process from here
233
234 @param r: the S3Request
235 @param attr: the controller attributes for the request
236 """
237
238 s3 = current.response.s3
239 session_s3 = current.session.s3
240
241 resource = self.resource
242 tablename = self.tablename
243
244 if r.http == "POST":
245 return self.merge(r, **attr)
246
247
248 record_ids = []
249 DEDUPLICATE = self.DEDUPLICATE
250 if DEDUPLICATE in session_s3:
251 bookmarks = session_s3[DEDUPLICATE]
252 if tablename in bookmarks:
253 record_ids = bookmarks[tablename]
254 query = FS(resource._id.name).belongs(record_ids)
255 resource.add_filter(query)
256
257
258 representation = r.representation
259
260
261 list_fields = resource.list_fields()
262
263
264 get_vars = r.get_vars
265 if representation == "aadata":
266 start = get_vars.get("displayStart", None)
267 limit = get_vars.get("pageLength", None)
268 draw = int(get_vars.draw or 0)
269 else:
270 start = 0
271 limit = s3.ROWSPERPAGE
272 if limit is not None:
273 try:
274 start = int(start)
275 limit = int(limit)
276 except ValueError:
277 start = None
278 limit = None
279 else:
280 start = None
281 if s3.dataTable_pageLength:
282 display_length = s3.dataTable_pageLength
283 else:
284 display_length = 25
285 if limit is None:
286 limit = 2 * display_length
287
288
289 totalrows = None
290 if representation == "aadata":
291 searchq, orderby, left = resource.datatable_filter(list_fields,
292 get_vars)
293 if searchq is not None:
294 totalrows = resource.count()
295 resource.add_filter(searchq)
296 else:
297 dt_sorting = {"iSortingCols": "1", "sSortDir_0": "asc"}
298 if len(list_fields) > 1:
299 dt_sorting["bSortable_0"] = "false"
300 dt_sorting["iSortCol_0"] = "1"
301 else:
302 dt_sorting["bSortable_0"] = "true"
303 dt_sorting["iSortCol_0"] = "0"
304 orderby, left = resource.datatable_filter(list_fields,
305 dt_sorting)[1:]
306
307
308 data = resource.select(list_fields,
309 start=start,
310 limit=limit,
311 orderby=orderby,
312 left=left,
313 count=True,
314 represent=True)
315
316
317 displayrows = data["numrows"]
318 if totalrows is None:
319 totalrows = displayrows
320
321
322 dt = S3DataTable(data["rfields"], data["rows"])
323
324 datatable_id = "s3merge_1"
325
326 if representation == "aadata":
327 output = dt.json(totalrows,
328 displayrows,
329 datatable_id,
330 draw,
331 dt_bulk_actions = [(current.T("Merge"),
332 "merge", "pair-action")])
333
334 elif representation == "html":
335
336 T = current.T
337 output = {"title": T("De-duplicate Records")}
338
339 url = r.url(representation="aadata")
340
341
342
343
344 items = dt.html(totalrows,
345 displayrows,
346 datatable_id,
347 dt_ajax_url=url,
348 dt_bulk_actions = [(T("Merge"),
349 "merge", "pair-action")],
350 dt_pageLength=display_length,
351 )
352
353 output["items"] = items
354 s3.actions = [{"label": str(T("View")),
355 "url": r.url(target="[id]", method="read"),
356 "_class": "action-btn",
357 },
358 ]
359
360 if len(record_ids) < 2:
361 output["add_btn"] = DIV(
362 SPAN(T("You need to have at least 2 records in this list in order to merge them."),
363
364 _style="float:left;padding-right:10px;"),
365 A(T("Find more"),
366 _href=r.url(method="", id=0, component_id=0, vars={}))
367 )
368 else:
369 output["add_btn"] = DIV(
370 SPAN(T("Select 2 records from this list, then click 'Merge'.")),
371 )
372
373 s3.dataTableID = [datatable_id]
374 current.response.view = self._view(r, "list.html")
375
376 else:
377 r.error(415, current.ERROR.BAD_FORMAT)
378
379 return output
380
381
382 - def merge(self, r, **attr):
383 """
384 Merge form for two records
385
386 @param r: the S3Request
387 @param **attr: the controller attributes for the request
388
389 @note: this method can always only be POSTed, and requires
390 both "selected" and "mode" in post_vars, as well as
391 the duplicate bookmarks list in session.s3
392 """
393
394 T = current.T
395 session = current.session
396 response = current.response
397
398 output = dict()
399 tablename = self.tablename
400
401
402 s3 = session.s3
403 DEDUPLICATE = self.DEDUPLICATE
404 if DEDUPLICATE in s3:
405 bookmarks = s3[DEDUPLICATE]
406 if tablename in bookmarks:
407 record_ids = bookmarks[tablename]
408
409
410 post_vars = r.post_vars
411 mode = post_vars.get("mode")
412 selected = post_vars.get("selected", "")
413 selected = selected.split(",")
414 if mode == "Inclusive":
415 ids = selected
416 elif mode == "Exclusive":
417 ids = [i for i in record_ids if i not in selected]
418 else:
419
420 ids = []
421 if len(ids) != 2:
422 r.error(501, T("Please select exactly two records"),
423 next = r.url(id=0, vars={}))
424
425
426 table = self.table
427 query = (table._id == ids[0]) | (table._id == ids[1])
428 orderby = table.created_on if "created_on" in table else None
429 rows = current.db(query).select(orderby=orderby,
430 limitby=(0, 2))
431 if len(rows) != 2:
432 r.error(404, current.ERROR.BAD_RECORD, next = r.url(id=0, vars={}))
433 original = rows[0]
434 duplicate = rows[1]
435
436
437 formfields = [f for f in table if f.readable or f.writable]
438
439 ORIGINAL, DUPLICATE, KEEP = self.ORIGINAL, self.DUPLICATE, self.KEEP
440 keep_o = KEEP.o in post_vars and post_vars[KEEP.o]
441 keep_d = KEEP.d in post_vars and post_vars[KEEP.d]
442
443 trs = []
444 init_requires = self.init_requires
445 index = 1
446 num_fields = len(formfields)
447
448 for f in formfields:
449
450
451 oid = "%s_%s" % (ORIGINAL, f.name)
452 did = "%s_%s" % (DUPLICATE, f.name)
453 sid = "swap_%s" % f.name
454 init_requires(f, original[f], duplicate[f])
455 if keep_o or not any((keep_o, keep_d)):
456 owidget = self.widget(f, original[f], _name=oid, _id=oid, _tabindex=index)
457 else:
458 try:
459 owidget = s3_represent_value(f, value=original[f])
460 except:
461 owidget = s3_str(original[f])
462 if keep_d or not any((keep_o, keep_d)):
463 dwidget = self.widget(f, duplicate[f], _name=did, _id=did)
464 else:
465 try:
466 dwidget = s3_represent_value(f, value=duplicate[f])
467 except:
468 dwidget = s3_str(duplicate[f])
469
470
471 if not any((keep_o, keep_d)):
472 swap = INPUT(_value="<-->",
473 _class="swap-button",
474 _id=sid,
475 _type="button",
476 _tabindex = index+num_fields)
477 else:
478 swap = DIV(_class="swap-button")
479
480 if owidget is None or dwidget is None:
481 continue
482
483
484 label = f.label
485 trs.append(TR(TD(label, _class="w2p_fl"),
486 TD(),
487 TD(label, _class="w2p_fl")))
488
489
490 trs.append(TR(TD(owidget, _class="mwidget"),
491 TD(swap),
492 TD(dwidget, _class="mwidget")))
493
494 index = index + 1
495
496 if "created_on" in table:
497 original_date = original.created_on
498 duplicate_date = duplicate.created_on
499 if "created_by" in table:
500 represent = table.created_by.represent
501 original_author = represent(original.created_by)
502 duplicate_author = represent(duplicate.created_by)
503 created = T("Created on %s by %s")
504 original_created = created % (original_date, original_author)
505 duplicate_created = created % (duplicate_date, duplicate_author)
506 else:
507 created = T("Created on %s")
508 original_created = created % original_date
509 duplicate_created = created % duplicate_date
510 else:
511 original_created = ""
512 duplicate_created = ""
513
514
515 output["title"] = T("Merge records")
516
517
518
519 if keep_o or not any((keep_o, keep_d)):
520 submit_original = INPUT(_value=T("Keep Original"),
521 _type="submit", _name=KEEP.o, _id=KEEP.o)
522 else:
523 submit_original = ""
524
525 if keep_d or not any((keep_o, keep_d)):
526 submit_duplicate = INPUT(_value=T("Keep Duplicate"),
527 _type="submit", _name=KEEP.d, _id=KEEP.d)
528 else:
529 submit_duplicate = ""
530
531
532 form = FORM(TABLE(
533 THEAD(
534 TR(TH(H3(T("Original"))),
535 TH(),
536 TH(H3(T("Duplicate"))),
537 ),
538 TR(TD(original_created),
539 TD(),
540 TD(duplicate_created),
541 _class="authorinfo",
542 ),
543 ),
544 TBODY(trs),
545 TFOOT(
546 TR(TD(submit_original),
547 TD(),
548 TD(submit_duplicate),
549 ),
550 ),
551 ),
552
553 hidden = {
554 "mode": "Inclusive",
555 "selected": ",".join(ids),
556 }
557 )
558
559 output["form"] = form
560
561
562 output["reset"] = FORM(INPUT(_value=T("Reset"),
563 _type="submit",
564 _name="reset", _id="form-reset"),
565 A(T("Cancel"), _href=r.url(id=0, vars={}), _class="action-lnk"),
566 hidden = {"mode": mode,
567 "selected": ",".join(ids),
568 },
569 )
570
571
572 formname = "merge_%s_%s_%s" % (tablename,
573 original[table._id],
574 duplicate[table._id])
575 if form.accepts(post_vars, session,
576 formname=formname,
577 onvalidation=lambda form: self.onvalidation(tablename, form),
578 keepvalues=False,
579 hideerror=False):
580
581 s3db = current.s3db
582
583 if form.vars[KEEP.d]:
584 prefix = "%s_" % DUPLICATE
585 original, duplicate = duplicate, original
586 else:
587 prefix = "%s_" % ORIGINAL
588
589 data = Storage()
590 for key in form.vars:
591 if key.startswith(prefix):
592 fname = key.split("_", 1)[1]
593 data[fname] = form.vars[key]
594
595 search = False
596 resource = s3db.resource(tablename)
597 try:
598 resource.merge(original[table._id],
599 duplicate[table._id],
600 update=data)
601 except current.auth.permission.error:
602 r.unauthorized()
603 except KeyError:
604 r.error(404, current.ERROR.BAD_RECORD)
605 except Exception:
606 import sys
607 r.error(424,
608 T("Could not merge records. (Internal Error: %s)") %
609 sys.exc_info()[1],
610 next=r.url())
611 else:
612
613 if mode == "Inclusive":
614 bookmarks[tablename] = [i for i in record_ids if i not in ids]
615 if not bookmarks[tablename]:
616 del bookmarks[tablename]
617 search = True
618 elif mode == "Exclusive":
619 bookmarks[tablename].extend(ids)
620 if not selected:
621 search = True
622
623
624
625
626
627
628
629 response.confirmation = T("Records merged successfully.")
630
631
632 if search:
633 self.next = r.url(method="", id=0, vars={})
634 else:
635 self.next = r.url(id=0, vars={})
636
637
638 response.view = self._view(r, "merge.html")
639
640 return output
641
642
643 @classmethod
671
672
673 @staticmethod
675 """
676 Initialize all IS_NOT_IN_DB to allow override of
677 both original and duplicate value
678
679 @param field: the Field
680 @param o: the original value
681 @param d: the duplicate value
682 """
683
684 allowed_override = [str(o), str(d)]
685
686 requires = field.requires
687 if field.unique and not requires:
688 field.requires = IS_NOT_IN_DB(current.db, str(field),
689 allowed_override=allowed_override)
690 else:
691 if not isinstance(requires, (list, tuple)):
692 requires = [requires]
693 for r in requires:
694 if hasattr(r, "allowed_override"):
695 r.allowed_override = allowed_override
696 if hasattr(r, "other") and \
697 hasattr(r.other, "allowed_override"):
698 r.other.allowed_override = allowed_override
699 return
700
701
702 @staticmethod
758
761 """ Record Merger """
762
764 """
765 Constructor
766
767 @param resource: the resource
768 """
769
770 self.resource = resource
771
772
773 @staticmethod
775 """
776 Roll back the current transaction and raise an error
777
778 @param message: error message
779 @param error: exception class to raise
780 """
781
782 current.db.rollback()
783 raise error(msg)
784
785
802
803
805
806 s3db = current.s3db
807
808 if replaced_by is not None:
809 replaced_by = {str(record_id): replaced_by}
810 resource = s3db.resource(table, id=record_id)
811 success = resource.delete(replaced_by=replaced_by,
812 cascade=True)
813 if not success:
814 self.raise_error("Could not delete %s.%s (%s)" %
815 (resource.tablename, record_id, resource.error))
816 return success
817
818
820 """
821 Merge the realms of two person entities (update all
822 realm_entities in all records from duplicate to original)
823
824 @param table: the table original and duplicate belong to
825 @param original: the original record
826 @param duplicate: the duplicate record
827 """
828
829 if "pe_id" not in table.fields:
830 return
831
832 original_pe_id = original["pe_id"]
833 duplicate_pe_id = duplicate["pe_id"]
834
835 db = current.db
836
837 for t in db:
838 if "realm_entity" in t.fields:
839
840 query = (t.realm_entity == duplicate_pe_id)
841 if "deleted" in t.fields:
842 query &= (t.deleted != True)
843 try:
844 db(query).update(realm_entity = original_pe_id)
845 except:
846 db.rollback()
847 raise
848 return
849
850
851
853
854 fn = None
855 if "." in key:
856 alias, fn = key.split(".", 1)
857 if alias not in ("~", self.resource.alias):
858 fn = None
859 elif self.main:
860 fn = key
861 return fn
862
863
864 - def merge(self,
865 original_id,
866 duplicate_id,
867 replace=None,
868 update=None,
869 main=True):
870 """
871 Merge a duplicate record into its original and remove the
872 duplicate, updating all references in the database.
873
874 @param original_id: the ID of the original record
875 @param duplicate_id: the ID of the duplicate record
876 @param replace: list fields names for which to replace the
877 values in the original record with the values
878 of the duplicate
879 @param update: dict of {field:value} to update the final record
880 @param main: internal indicator for recursive calls
881
882 @status: work in progress
883 @todo: de-duplicate components and link table entries
884
885 @note: virtual references (i.e. non-SQL, without foreign key
886 constraints) must be declared in the table configuration
887 of the referenced table like:
888
889 s3db.configure(tablename, referenced_by=[(tablename, fieldname)])
890
891 This does not apply for list:references which will be found
892 automatically.
893
894 @note: this method can only be run from master resources (in order
895 to find all components). To merge component records, you have
896 to re-define the component as a master resource.
897
898 @note: CLI calls must db.commit()
899 """
900
901 self.main = main
902
903 db = current.db
904 resource = self.resource
905 table = resource.table
906 tablename = resource.tablename
907
908
909 if resource.parent:
910 self.raise_error("Must not merge from component", SyntaxError)
911
912
913 auth = current.auth
914 has_permission = auth.s3_has_permission
915 permitted = has_permission("update", table,
916 record_id = original_id) and \
917 has_permission("delete", table,
918 record_id = duplicate_id)
919 if not permitted:
920 self.raise_error("Operation not permitted", auth.permission.error)
921
922
923 s3db = current.s3db
924 if main:
925 s3db.load_all_models()
926 if db._lazy_tables:
927
928 for tn in db._LAZY_TABLES.keys():
929 db[tn]
930
931
932 original = None
933 duplicate = None
934 query = table._id.belongs([original_id, duplicate_id])
935 if "deleted" in table.fields:
936 query &= (table.deleted != True)
937 rows = db(query).select(table.ALL, limitby=(0, 2))
938 for row in rows:
939 record_id = row[table._id]
940 if str(record_id) == str(original_id):
941 original = row
942 original_id = row[table._id]
943 elif str(record_id) == str(duplicate_id):
944 duplicate = row
945 duplicate_id = row[table._id]
946 msg = "Record not found: %s.%s"
947 if original is None:
948 self.raise_error(msg % (tablename, original_id), KeyError)
949 if duplicate is None:
950 self.raise_error(msg % (tablename, duplicate_id), KeyError)
951
952
953
954
955
956
957
958
959 single = {}
960 hooks = s3db.get_hooks(table)[1]
961 if hooks:
962 for alias, hook in hooks.items():
963 if hook.multiple:
964 continue
965 component = resource.components.get(alias)
966 if not component:
967
968 continue
969 ctablename = component.tablename
970 if ctablename in single:
971 single[ctablename].append(component)
972 else:
973 single[ctablename] = [component]
974
975
976 is_super_entity = table._id.name != "id" and \
977 "instance_type" in table.fields
978
979
980 referenced_by = list(table._referenced_by)
981
982
983 virtual_references = s3db.get_config(tablename, "referenced_by")
984 if virtual_references:
985 referenced_by.extend(virtual_references)
986
987
988 for t in db:
989 for f in t:
990 ftype = str(f.type)
991 if ftype[:14] == "list:reference" and \
992 ftype[15:15+len(tablename)] == tablename:
993 referenced_by.append((t._tablename, f.name))
994
995 update_record = self.update_record
996 delete_record = self.delete_record
997 fieldname = self.fieldname
998
999
1000 define_resource = s3db.resource
1001 for referee in referenced_by:
1002
1003 if isinstance(referee, Field):
1004 tn, fn = referee.tablename, referee.name
1005 else:
1006 tn, fn = referee
1007
1008 se = s3db.get_config(tn, "super_entity")
1009 if is_super_entity and \
1010 (isinstance(se, (list, tuple)) and tablename in se or \
1011 se == tablename):
1012
1013 continue
1014
1015
1016 if tn not in db or fn not in db[tn].fields:
1017 continue
1018
1019 rtable = db[tn]
1020 if tn in single:
1021 for component in single[tn]:
1022
1023 if component.link is not None:
1024 component = component.link
1025
1026 if fn == component.fkey:
1027
1028
1029 join = component.get_join()
1030 pkey = component.pkey
1031 lkey = component.lkey or component.fkey
1032
1033
1034 query = (table[pkey] == original[pkey]) & join
1035 osub = db(query).select(limitby=(0, 1)).first()
1036 query = (table[pkey] == duplicate[pkey]) & join
1037 dsub = db(query).select(limitby=(0, 1)).first()
1038
1039 ctable = component.table
1040 ctable_id = ctable._id
1041
1042 if dsub is None:
1043
1044 continue
1045
1046 elif not osub:
1047
1048 dsub_id = dsub[ctable_id]
1049 data = {lkey: original[pkey]}
1050 update_record(ctable, dsub_id, dsub, data)
1051
1052 elif component.linked is not None:
1053
1054
1055 dsub_id = dsub[ctable_id]
1056 delete_record(ctable, dsub_id)
1057
1058 else:
1059
1060 osub_id = osub[ctable_id]
1061 dsub_id = dsub[ctable_id]
1062 cresource = define_resource(component.tablename)
1063 cresource.merge(osub_id, dsub_id,
1064 replace = replace,
1065 update = update,
1066 main = False,
1067 )
1068
1069
1070 rfield = rtable[fn]
1071 ktablename, key, multiple = s3_get_foreign_key(rfield)
1072 if not ktablename:
1073 if str(rfield.type) == "integer":
1074
1075 key = table._id.name
1076 else:
1077 continue
1078
1079
1080 if multiple:
1081 query = rtable[fn].contains(duplicate[key])
1082 else:
1083 query = rtable[fn] == duplicate[key]
1084 rows = db(query).select(rtable._id, rtable[fn])
1085
1086
1087 for row in rows:
1088 if not multiple:
1089 data = {fn:original[key]}
1090 else:
1091 keys = [k for k in row[fn] if k != duplicate[key]]
1092 if original[key] not in keys:
1093 keys.append(original[key])
1094 data = {fn:keys}
1095 update_record(rtable, row[rtable._id], row, data)
1096
1097
1098 super_entities = resource.get_config("super_entity")
1099 if super_entities is not None:
1100
1101 if not isinstance(super_entities, (list, tuple)):
1102 super_entities = [super_entities]
1103
1104 for super_entity in super_entities:
1105
1106 super_table = s3db.table(super_entity)
1107 if not super_table:
1108 continue
1109 superkey = super_table._id.name
1110
1111 skey_o = original[superkey]
1112 if not skey_o:
1113 msg = "No %s found in %s.%s" % (superkey,
1114 tablename,
1115 original_id)
1116 current.log.warning(msg)
1117 s3db.update_super(table, original)
1118 skey_o = original[superkey]
1119 if not skey_o:
1120 continue
1121 skey_d = duplicate[superkey]
1122 if not skey_d:
1123 msg = "No %s found in %s.%s" % (superkey,
1124 tablename,
1125 duplicate_id)
1126 current.log.warning(msg)
1127 continue
1128
1129 sresource = define_resource(super_entity)
1130 sresource.merge(skey_o, skey_d,
1131 replace=replace,
1132 update=update,
1133 main=False)
1134
1135
1136 data = Storage()
1137 if replace:
1138 for k in replace:
1139 fn = fieldname(k)
1140 if fn and fn in duplicate:
1141 data[fn] = duplicate[fn]
1142 if update:
1143 for k, v in update.items():
1144 fn = fieldname(k)
1145 if fn in table.fields:
1146 data[fn] = v
1147 if len(data):
1148 r = None
1149 p = Storage([(fn, "__deduplicate_%s__" % fn)
1150 for fn in data
1151 if table[fn].unique and \
1152 table[fn].type == "string" and \
1153 data[fn] == duplicate[fn]])
1154 if p:
1155 r = Storage([(fn, original[fn]) for fn in p])
1156 update_record(table, duplicate_id, duplicate, p)
1157 update_record(table, original_id, original, data)
1158 if r:
1159 update_record(table, duplicate_id, duplicate, r)
1160
1161
1162 if not is_super_entity:
1163 self.merge_realms(table, original, duplicate)
1164 delete_record(table, duplicate_id, replaced_by=original_id)
1165
1166
1167 return True
1168
1169
1170