Source Code for Module s3.codecs.shp

  1  # -*- coding: utf-8 -*- 
  2   
  3  """ 
  4      S3 Shapefile codec 
  5   
  6      @copyright: 2013-2019 (c) Sahana Software Foundation 
  7      @license: MIT 
  8   
  9      Permission is hereby granted, free of charge, to any person 
 10      obtaining a copy of this software and associated documentation 
 11      files (the "Software"), to deal in the Software without 
 12      restriction, including without limitation the rights to use, 
 13      copy, modify, merge, publish, distribute, sublicense, and/or sell 
 14      copies of the Software, and to permit persons to whom the 
 15      Software is furnished to do so, subject to the following 
 16      conditions: 
 17   
 18      The above copyright notice and this permission notice shall be 
 19      included in all copies or substantial portions of the Software. 
 20   
 21      THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 
 22      EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES 
 23      OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 
 24      NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT 
 25      HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, 
 26      WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 
 27      FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 
 28      OTHER DEALINGS IN THE SOFTWARE. 
 29  """ 
 30   
 31  __all__ = ("S3SHP",) 
 32   
 33  import os 
 34   
 35  try: 
 36      from cStringIO import StringIO    # Faster, where available 
 37  except: 
 38      from StringIO import StringIO 
 39   
 40  from gluon import * 
 41  from gluon.contenttype import contenttype 
 42  from gluon.storage import Storage 
 43  from gluon.streamer import DEFAULT_CHUNK_SIZE 
 44   
 45  from ..s3codec import S3Codec 
 46  from ..s3utils import s3_unicode, s3_strip_markup 
 47   
 48  # ============================================================================= 
 49 -class S3SHP(S3Codec): 
 50      """ 
 51          Simple Shapefile format codec 
 52      """ 
 53   
 54      # ------------------------------------------------------------------------- 
 55 -    def __init__(self): 
 56          """ 
 57              Constructor 
 58          """ 
 59   
 60          pass 
 61   
 62      # ------------------------------------------------------------------------- 
 63 -    def extractResource(self, resource, list_fields): 
 64          """ 
 65              Extract the items from the resource 
 66   
 67              @param resource: the resource 
 68              @param list_fields: fields to include in list views 
 69          """ 
 70   
 71          title = self.crud_string(resource.tablename, "title_list") 
 72   
 73          get_vars = Storage(current.request.get_vars) 
 74          get_vars["iColumns"] = len(list_fields) 
 75          query, orderby, left = resource.datatable_filter(list_fields, get_vars) 
 76          resource.add_filter(query) 
 77   
 78          data = resource.select(list_fields, 
 79                                 left=left, 
 80                                 limit=None, 
 81                                 orderby=orderby, 
 82                                 represent=True, 
 83                                 show_links=False) 
 84   
 85          rfields = data["rfields"] 
 86          types = [] 
 87          colnames = [] 
 88          heading = {} 
 89          for rfield in rfields: 
 90              if rfield.show: 
 91                  colnames.append(rfield.colname) 
 92                  heading[rfield.colname] = rfield.label 
 93                  if rfield.virtual: 
 94                      types.append("string") 
 95                  else: 
 96                      types.append(rfield.ftype) 
 97   
 98          items = data["rows"] 
 99   
100          return (title, types, colnames, heading, items) 
101   
102      # ------------------------------------------------------------------------- 
103 -    def encode(self, data_source, **attr): 
104          """ 
105              Export data as a Shapefile 
106   
107              @param data_source: the source of the data that is to be encoded 
108                                  as a shapefile. This may be: 
109                                  resource: the resource 
110                                  item:     a list of pre-fetched values 
111                                            the headings are in the first row 
112                                            the data types are in the second row 
113              @param attr: dictionary of parameters: 
114                   * title:          The export filename 
115                   * list_fields:    Fields to include in list views 
116          """ 
117   
118          # Get the attributes 
119          title = attr.get("title") 
120   
121          # Extract the data from the data_source 
122          if isinstance(data_source, (list, tuple)): 
123              headers = data_source[0] 
124              #types = data_source[1] 
125              items = data_source[2:] 
126          else: 
127              current.s3db.gis_location.wkt.represent = None 
128              list_fields = attr.get("list_fields") 
129              if not list_fields: 
130                  list_fields = data_source.list_fields() 
131              if data_source.tablename == "gis_location": 
132                  wkt_field = "wkt" 
133              else: 
134                  wkt_field = "location_id$wkt" 
135              if wkt_field not in list_fields: 
136                  list_fields.append(wkt_field) 
137   
138              (_title, types, lfields, headers, items) = self.extractResource(data_source, 
139                                                                              list_fields) 
140              if not title: 
141                  title = _title 
142   
143          # Create the data structure 
144          output = [] 
145          oappend = output.append 
146   
147          # Header row 
148          headers["gis_location.wkt"] = "WKT" 
149          fields = [] 
150          fappend = fields.append 
151          header = [] 
152          happend = header.append 
153          for selector in lfields: 
154              h = s3_unicode(headers[selector].replace(" ", "_")) 
155              happend(h) 
156              if selector != "gis_location.wkt": 
157                  # Don't include the WKT field as an Attribute in the Shapefile 
158                  fappend(h) 
159          oappend('"%s"' % '","'.join(header)) 
160          fields = ",".join(fields) 
161   
162          for item in items: 
163              row = [] 
164              rappend = row.append 
165              for selector in lfields: 
166                  represent = s3_strip_markup(s3_unicode(item[selector])) 
167                  rappend(represent) 
168              oappend('"%s"' % '","'.join(row)) 
169   
170          # Write out as CSV 
171          import tempfile 
172          web2py_path = os.getcwd() 
173          if os.path.exists(os.path.join(web2py_path, "temp")): # use web2py/temp 
174              TEMP = os.path.join(web2py_path, "temp") 
175          else: 
176              TEMP = tempfile.gettempdir() 
177          os_handle_temp, temp_filepath = tempfile.mkstemp(dir=TEMP, suffix=".csv") 
178          with open(temp_filepath, "w") as f: 
179              for line in output: 
180                  f.write("%s\n" % line.encode("utf-8")) 
181   
182          # Convert to Shapefile 
183          # @ToDo: migrate to GDAL Python bindings 
184          # Write out VRT file 
185          temp_filename = temp_filepath.rsplit(os.path.sep, 1)[1] 
186          vrt = \ 
187  '''<OGRVRTDataSource> 
188      <OGRVRTLayer name="%s"> 
189          <SrcDataSource>%s</SrcDataSource> 
190          <GeometryType>wkbGeometryCollection</GeometryType> 
191          <TargetSRS>EPSG:4326</TargetSRS> 
192          <GeometryField encoding="WKT" field="WKT"/> 
193      </OGRVRTLayer> 
194  </OGRVRTDataSource>''' % (temp_filename.rsplit(".", 1)[0], temp_filename) 
195          os_handle_vrt, vrt_filename = tempfile.mkstemp(dir=TEMP, suffix=".vrt") 
196          with open(vrt_filename, "w") as f: 
197              f.write(vrt) 
198          # @ToDo: Check that the data exists before writing out file 
199          # Write Points 
200          os.chdir(TEMP) 
201          # Use + not %s as % within string 
202          cmd = 'ogr2ogr -a_srs "EPSG:4326" -f "ESRI Shapefile" ' + title + '_point.shp ' + vrt_filename + ' -select ' + fields + ' -skipfailures -nlt POINT -where "WKT LIKE \'%POINT%\'"' 
203          #os.system("rm %s_point.*" % title) 
204          os.system(cmd) 
205          # Write Lines 
206          cmd = 'ogr2ogr -a_srs "EPSG:4326" -f "ESRI Shapefile" ' + title + '_line.shp ' + vrt_filename + ' -select ' + fields + ' -skipfailures -nlt MULTILINESTRING -where "WKT LIKE \'%LINESTRING%\'"' 
207          #os.system("rm %s_line.*" % title) 
208          os.system(cmd) 
209          # Write Polygons 
210          cmd = 'ogr2ogr -a_srs "EPSG:4326" -f "ESRI Shapefile" ' + title + '_polygon.shp ' + vrt_filename + ' -select ' + fields + ' -skipfailures -nlt MULTIPOLYGON -where "WKT LIKE \'%POLYGON%\'"' 
211          #os.system("rm %s_polygon.*" % title) 
212          os.system(cmd) 
213          os.close(os_handle_temp) 
214          os.unlink(temp_filepath) 
215          os.close(os_handle_vrt) 
216          os.unlink(vrt_filename) 
217          # Zip up 
218          import zipfile 
219          request = current.request 
220          filename = "%s_%s.zip" % (request.env.server_name, title) 
221          fzip = zipfile.ZipFile(filename, "w") 
222          for item in ("point", "line", "polygon"): 
223              for exten in ("shp", "shx", "prj", "dbf"): 
224                  tfilename = "%s_%s.%s" % (title, item, exten) 
225                  fzip.write(tfilename) 
226                  os.unlink(tfilename) 
227          fzip.close() 
228          # Restore path 
229          os.chdir(web2py_path) 
230   
231          # Response headers 
232          disposition = "attachment; filename=\"%s\"" % filename 
233          response = current.response 
234          response.headers["Content-Type"] = contenttype(".zip") 
235          response.headers["Content-disposition"] = disposition 
236   
237          stream = open(os.path.join(TEMP, filename), "rb") 
238          return response.stream(stream, chunk_size=DEFAULT_CHUNK_SIZE, 
239                                 request=request) 
240   
241      # ------------------------------------------------------------------------- 
242 -    def decode(self, resource, source, **attr): 
243          """ 
244              Import data from a Shapefile 
245   
246              @param resource: the S3Resource 
247              @param source: the source 
248   
249              @return: an S3XML ElementTree 
250   
251              @ToDo: Handle encodings within Shapefiles other than UTF-8 
252          """ 
253   
254          # @ToDo: Complete this! 
255          # Sample code coming from this working script: 
256          # http://eden.sahanafoundation.org/wiki/BluePrint/GIS/ShapefileLayers#ImportintonativeTables 
257          # We also have sample code to read SHP from GDAL in: 
258          # gis_layer_shapefile_onaccept() & import_admin_areas() [GADM] 
259          raise NotImplementedError 
260   
261          try: 
262              from lxml import etree 
263          except ImportError: 
264              current.log.error("ERROR: lxml module needed for XML handling") 
265              raise 
266   
267          try: 
268              from osgeo import ogr 
269          except ImportError: 
270              current.log.error("ERROR: GDAL module needed for Shapefile handling") 
271              raise 
272   
273          # @ToDo: Check how this would happen 
274          shapefilename = source 
275   
276          layername = os.path.splitext(os.path.basename(shapefilename))[0] 
277   
278          # Create the datasource 
279          ds = ogr.Open(shapefilename) 
280   
281          # Open the shapefile 
282          if ds is None: 
283              # @ToDo: Bail gracefully 
284              raise 
285   
286          # Get the layer and iterate through the features 
287          lyr = ds.GetLayer(0) 
288   
289          root = etree.Element("shapefile", name=layername) 
290   
291          OFTInteger = ogr.OFTInteger 
292          OFTReal = ogr.OFTReal 
293          OFTString = ogr.OFTString 
294          for feat in lyr: 
295              featurenode = etree.SubElement(root, "feature") 
296              feat_defn = lyr.GetLayerDefn() 
297              GetFieldDefn = feat_defn.GetFieldDefn 
298              for i in range(feat_defn.GetFieldCount()): 
299                  field_defn = GetFieldDefn(i) 
300                  fieldnode = etree.SubElement(featurenode, field_defn.GetName()) 
301                  if field_defn.GetType() == OFTInteger: 
302                      fieldnode.text = str(feat.GetFieldAsInteger(i)) 
303                  elif field_defn.GetType() == OFTReal: 
304                      fieldnode.text = str(feat.GetFieldAsDouble(i)) 
305                  elif field_defn.GetType() == OFTString: 
306                      FieldString = str(feat.GetFieldAsString(i)) 
307                      # @ToDo: Don't assume UTF-8 
308                      fieldnode.text = FieldString.decode(encoding="UTF-8", 
309                                                          errors="strict") 
310   
311              wktnode = etree.SubElement(featurenode, "wkt") 
312              geom = feat.GetGeometryRef() 
313              wktnode.text = geom.ExportToWkt() 
314   
315          # @ToDo: Convert using XSLT 
316   
317          # Debug: Write out the etree 
318          #xmlString = etree.tostring(root, pretty_print=True) 
319          #f = open("test.xml","w") 
320          #f.write(xmlString) 
321   
322          return root 
323   
324  # End ========================================================================= 
325