Source Code for Module s3.sync_adapter.ftp

  1  # -*- coding: utf-8 -*- 
  2   
  3  """ S3 Synchronization: Peer Repository Adapter for FTP 
  4   
  5      @copyright: 2015-2019 (c) Sahana Software Foundation 
  6      @license: MIT 
  7   
  8      Permission is hereby granted, free of charge, to any person 
  9      obtaining a copy of this software and associated documentation 
 10      files (the "Software"), to deal in the Software without 
 11      restriction, including without limitation the rights to use, 
 12      copy, modify, merge, publish, distribute, sublicense, and/or sell 
 13      copies of the Software, and to permit persons to whom the 
 14      Software is furnished to do so, subject to the following 
 15      conditions: 
 16   
 17      The above copyright notice and this permission notice shall be 
 18      included in all copies or substantial portions of the Software. 
 19   
 20      THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 
 21      EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES 
 22      OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 
 23      NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT 
 24      HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, 
 25      WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 
 26      FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 
 27      OTHER DEALINGS IN THE SOFTWARE. 
 28  """ 
 29   
 30  import sys 
 31   
 32  from gluon import * 
 33   
 34  from ..s3sync import S3SyncBaseAdapter 
 35   
 36  from ..s3query import S3URLQuery, FS 
 37  from ..s3rest import S3Request 
 38  from ..s3export import S3Exporter 
 39   
 40  try: 
 41      from cStringIO import StringIO    # Faster, where available 
 42  except: 
 43      from StringIO import StringIO 
 44   
 45  # ============================================================================= 
 46 -class S3SyncAdapter(S3SyncBaseAdapter): 
 47      """ 
 48          FTP Synchronization Adapter 
 49   
 50           currently used by CAP 
 51      """ 
 52   
 53      # ------------------------------------------------------------------------- 
 54 -    def register(self): 
 55          """ 
 56              Register this site at the peer repository 
 57   
 58              @return: True to indicate success, otherwise False 
 59          """ 
 60   
 61          # No registration required 
 62          return True 
 63   
 64      # ------------------------------------------------------------------------- 
 65 -    def login(self): 
 66          """ 
 67              Login at the peer repository 
 68   
 69              @return: None if successful, otherwise the error 
 70          """ 
 71   
 72          repository = self.repository 
 73          url = repository.url 
 74          error = None 
 75   
 76          if not url: 
 77              error = "Remote URL required for FTP Push" 
 78          else: 
 79              import ftplib 
 80              try: 
 81                  ftp_connection = ftplib.FTP(url) 
 82              except ftplib.all_errors: 
 83                  error = sys.exc_info()[1] 
 84              else: 
 85                  try: 
 86                      ftp_connection.login(repository.username, repository.password) 
 87                  except ftplib.error_perm: 
 88                      error = sys.exc_info()[1] 
 89   
 90              self.ftp_connection = ftp_connection 
 91   
 92          if error: 
 93              current.log.debug(error) 
 94          return error 
 95   
 96      # ------------------------------------------------------------------------- 
 97 -    def pull(self, task, onconflict=None): 
 98          """ 
 99              Fetch updates from the repository and import them 
100              into the local database (Active Pull) 
101   
102              @param task: the task (sync_task Row) 
103          """ 
104   
105          repository = self.repository 
106   
107          # Log the operation 
108          message = "Pull from FTP currently not supported" 
109          log = repository.log 
110          log.write(repository_id = repository.id, 
111                    resource_name = task.resource_name, 
112                    transmission = log.OUT, 
113                    mode = log.PULL, 
114                    action = None, 
115                    remote = False, 
116                    result = log.FATAL, 
117                    message = message, 
118                    ) 
119   
120          return message, None 
121   
122      # ------------------------------------------------------------------------- 
123 -    def push(self, task): 
124          """ 
125              Extract new updates from the local database and send 
126              them to the peer repository (active push) 
127   
128              @param task: the synchronization task (sync_task Row) 
129   
130              @return: tuple (error, mtime), with error=None if successful, 
131                       else error=message, and mtime=modification timestamp 
132                       of the youngest record sent 
133          """ 
134   
135          repository = self.repository 
136          resource_name = task.resource_name 
137          log = repository.log 
138          remote = False 
139          output = None 
140   
141          current.log.debug("S3SyncRepository.push(%s, %s)" % (repository.url, 
142                                                               resource_name)) 
143   
144          # Define the resource 
145          resource = current.s3db.resource(resource_name, 
146                                           # FTP remote deletion is not supported yet 
147                                           #include_deleted=True, 
148                                           ) 
149   
150          # Apply sync filters for this task 
151          filters = current.sync.get_filters(task.id) 
152          table = resource.table 
153          tablename = resource.tablename 
154   
155          if filters: 
156              queries = S3URLQuery.parse(resource, filters[tablename]) 
157              [resource.add_filter(q) for a in queries for q in queries[a]] 
158   
159          # Filter to records after last push 
160          msince = task.last_push 
161   
162          if msince: 
163              strategy = task.strategy 
164              created = "create" in strategy 
165              updated = "update" in strategy 
166              if created and updated: 
167                  mtime_filter = table.modified_on > msince 
168              elif created: 
169                  mtime_filter = table.created_on > msince 
170              elif updated: 
171                  mtime_filter = (table.created_on <= msince) & \ 
172                                 (table.modified_on > msince) 
173              else: 
174                  mtime_filter = None 
175   
176              if mtime_filter: 
177                  resource.add_filter(mtime_filter) 
178   
179          mtime = resource.muntil 
180          # Get the ID of the resource after filter and msince 
181          resource_ids = resource.get_id() 
182   
183          # No Changes since last push? 
184          if resource_ids is None: 
185              message = "No Changes since last push" 
186              result = log.WARNING 
187          else: 
188              # Filename 
189              settings = current.deployment_settings 
190              # Placeholders for filename 
191              placeholders = {"systemname": settings.get_system_name(), 
192                              "systemname_short": settings.get_system_name_short(), 
193                              "resource": resource_name, 
194                              "public_url": settings.get_base_public_url(), 
195                              } 
196   
197              from string import Template 
198              filename = resource.get_config("upload_filename") 
199              if not filename: 
200                  filename = settings.get_sync_upload_filename() 
201              filename = Template(filename).safe_substitute(s="%(systemname_short)s", 
202                                                            r="%(resource)s") 
203              filename = filename % placeholders 
204              # Get Representation 
205              representation = task.representation 
206              filename = ("%s.%s") % (filename, representation) 
207   
208              # FTP Transfer 
209              remote = True 
210              import ftplib 
211              ftp_connection = self.ftp_connection 
212              if task.multiple_file: 
213                  if type(resource_ids) is not list: 
214                      resource_ids = [resource_ids] 
215   
216                  for resource_id in resource_ids: 
217                      resource.clear_query() 
218                      resource.add_filter(FS("id") == resource_id) 
219                      data = self._get_data(resource, representation) 
220   
221                      try: 
222                          ftp_connection.storbinary("STOR %s" % filename, 
223                                                    StringIO(data)) 
224                      except ftplib.error_perm: 
225                          message = sys.exc_info()[1] 
226                          result = log.ERROR 
227                          output = message 
228                      else: 
229                          message = "FTP Transfer Successful" 
230                          result = log.SUCCESS 
231   
232                      current.log.debug(message) 
233              else: 
234                  data = self._get_data(resource, representation) 
235   
236                  try: 
237                      ftp_connection.storbinary("STOR %s" % filename, 
238                                                StringIO(data)) 
239                  except ftplib.error_perm: 
240                      message = sys.exc_info()[1] 
241                      result = log.ERROR 
242                      output = message 
243                  else: 
244                      message = "FTP Transfer Successful" 
245                      result = log.SUCCESS 
246   
247                  current.log.debug(message) 
248   
249              # Quit the connection here 
250              ftp_connection.quit() 
251   
252          # Log the operation 
253          log.write(repository_id = repository.id, 
254                    resource_name = resource_name, 
255                    transmission = log.OUT, 
256                    mode = log.PUSH, 
257                    action = "send", 
258                    remote = remote, 
259                    result = result, 
260                    message = message, 
261                    ) 
262          # Returns after operation is complete 
263          if output is not None: 
264              mtime = None 
265          return output, mtime 
266   
267      # ------------------------------------------------------------------------- 
268      # Internal methods: 
269      # ------------------------------------------------------------------------- 
270 -    def _get_data(self, resource, representation): 
271          """ Returns the representation data for the resource """ 
272   
273          request = S3Request(prefix = resource.prefix, 
274                              name = resource.name, 
275                              extension = representation, 
276                              ) 
277   
278          if request.transformable(): 
279              return resource.export_xml(stylesheet = request.stylesheet(), 
280                                         pretty_print = True, 
281                                         ) 
282   
283          else: 
284              if representation == "csv": 
285                  exporter = S3Exporter().csv 
286   
287              # @ToDo use CRUD 
288              #elif representation == "html": 
289   
290              elif representation == "pdf": 
291                  exporter = S3Exporter().pdf 
292   
293              elif representation == "xls": 
294                  exporter = S3Exporter().xls 
295   
296              elif representation == "json": 
297                  exporter = S3Exporter().json 
298   
299              return exporter(resource) 
300   
301  # End ========================================================================= 
302