simplify run_export

Dockerfile

@@ -10,7 +10,7 @@ RUN echo "deb http://packages.laboratoriopublico.org/publiclab/ stretch main" >
 # Obtain key
 RUN mkdir ~/.gnupg
 RUN echo "disable-ipv6" >> ~/.gnupg/dirmngr.conf
-RUN apt-key adv --keyserver fks.pgpkeys.eu --recv-keys BF26EE05EA6A68F0
+RUN apt-key adv --keyserver ipv4.pool.sks-keyservers.net --recv-keys BF26EE05EA6A68F0
 
 # Install dependencies
 RUN apt-get update -qq && apt-get install -y \

README.md

@@ -7,13 +7,17 @@ Then, use it with:
 ```ruby
 require 'mapknitter-exporter'
 
-# not sure this will work:
+# this should work:
 
 MapKnitterExporter.generate_perspectival_distort(
   scale,
-  'map',
-  nodes_array,
   1, # a unique id
+  [
+    { lat: 41.8403113680142, lon: -71.3983854668186 },
+    { lat: 41.8397358653566, lon: -71.3916477577732 },
+    { lat: 41.8351476451765, lon: -71.392699183707  },
+    { lat: 41.8377535388085, lon: -71.3981708900974 }
+  ],
   'test/fixtures/demo.png',
   image,
   height,
@@ -23,6 +27,6 @@ MapKnitterExporter.generate_perspectival_distort(
 
 ## Tests
 
-Tests are not passing completely but they are running; use `ruby test/exporter_test.rb` to run them.
+To run tests, use `ruby test/exporter_test.rb`.
 
 Tests require `minitest` which you can install with `bundle install`.

lib/mapknitterExporter.rb

@@ -10,11 +10,11 @@ def self.ulimit
   end
 
   def self.get_working_directory(path)
-    "public/warps/" + path + "-working/"
+    "public/warps/#{path}-working/"
   end
 
   def self.warps_directory(path)
-    "public/warps/" + path + "/"
+    "public/warps/#{path}/"
   end
 
   def self.delete_temp_files(path)
@@ -26,16 +26,16 @@ def self.delete_temp_files(path)
   ## Run on each image:
 
   # pixels per meter = pxperm 
-  def self.generate_perspectival_distort(pxperm, path, nodes_array, id, image_file_name, img_url, height, width, root = "https://mapknitter.org")
+  def self.generate_perspectival_distort(pxperm, id, nodes_array, image_file_name, img_url, height, width, root = "https://mapknitter.org")
     require 'net/http'
 
     # everything in -working/ can be deleted; 
     # this is just so we can use the files locally outside of s3
-    working_directory = get_working_directory(path)
+    working_directory = get_working_directory(id)
     Dir.mkdir(working_directory) unless (File.exists?(working_directory) && File.directory?(working_directory))
-    local_location = working_directory+id.to_s+'-'+image_file_name.to_s
+    local_location = "#{working_directory}#{id}-#{image_file_name}"
 
-    directory = warps_directory(path)
+    directory = warps_directory(id)
     Dir.mkdir(directory) unless (File.exists?(directory) && File.directory?(directory))
     completed_local_location = directory+id.to_s+'.png'
 
@@ -71,7 +71,7 @@ def self.generate_perspectival_distort(pxperm, path, nodes_array, id, image_file
     # puts x1.to_s+','+y1.to_s+','+x2.to_s+','+y2.to_s
 
     # should determine if it's stored in s3 or locally:
-    if (img_url[0..3] == 'http')
+    if (img_url.slice(0,4) == 'http')
       Net::HTTP.start('s3.amazonaws.com') { |http|
       #Net::HTTP.start('localhost') { |http|
         puts (img_url)
@@ -206,7 +206,7 @@ def self.generate_perspectival_distort(pxperm, path, nodes_array, id, image_file
     system(self.ulimit+gdalwarp)
 
     # deletions could happen here; do it in distinct method so we can run it independently
-    delete_temp_files(path)
+    delete_temp_files(id)
 
     [x1,y1]
   end
@@ -215,7 +215,7 @@ def self.generate_perspectival_distort(pxperm, path, nodes_array, id, image_file
   ## Run on maps:
 
   # distort all warpables, returns upper left corner coords in x,y
-  def self.distort_warpables(scale, images, export, slug)
+  def self.distort_warpables(scale, images, export, id)
 
     puts '> generating geotiffs of each warpable in GDAL'
     lowest_x = 0
@@ -233,9 +233,8 @@ def self.distort_warpables(scale, images, export, slug)
 
      img_coords = generate_perspectival_distort(
        scale,
-       slug,
+       id,
        image[:nodes_array],
-       image[:id],
        image[:filename],
        image[:url],
        image[:height],
@@ -250,9 +249,9 @@ def self.distort_warpables(scale, images, export, slug)
   end
 
   # generate a tiff from all warpable images in this set
-  def self.generate_composite_tiff(coords, origin, placed_warpables, slug, ordered)
-    directory = "public/warps/"+slug+"/"
-    composite_location = directory+slug+'-geo.tif'
+  def self.generate_composite_tiff(coords, origin, placed_warpables, id, ordered)
+    directory = "public/warps/#{id}/"
+    composite_location = directory + id.to_s + '-geo.tif'
     geotiffs = ''
     minlat = nil
     minlon = nil
@@ -275,45 +274,44 @@ def self.generate_composite_tiff(coords, origin, placed_warpables, slug, ordered
       wid = warpable[:id].to_s
       geotiffs += ' '+directory+wid+'-geo.tif'
       if first
-        gdalwarp = "gdalwarp -s_srs EPSG:3857 -te "+minlon.to_s+" "+minlat.to_s+" "+maxlon.to_s+" "+maxlat.to_s+" "+directory+wid+'-geo.tif '+directory+slug+'-geo.tif'
+        gdalwarp = "gdalwarp -s_srs EPSG:3857 -te #{minlon} #{minlat} #{maxlon} #{maxlat} #{directory}#{wid}-geo.tif #{directory}#{id}-geo.tif"
         first = false
       else
-        gdalwarp = "gdalwarp "+directory+wid+'-geo.tif '+directory+slug+'-geo.tif'
+        gdalwarp = "gdalwarp #{directory}#{wid}-geo.tif #{directory}#{id}-geo.tif"
       end
       puts gdalwarp
       system(self.ulimit+gdalwarp)
     end
     composite_location
   end
 
-  # generates a tileset at root/public/tms/<slug>/
+  # generates a tileset at root/public/tms/<id>/
   # root is something like https://mapknitter.org
-  def self.generate_tiles(key, slug, root)
+  def self.generate_tiles(key, id, root)
     key = "AIzaSyAOLUQngEmJv0_zcG1xkGq-CXIPpLQY8iQ" if key == "" # ugh, let's clean this up!
     key = key || "AIzaSyAOLUQngEmJv0_zcG1xkGq-CXIPpLQY8iQ"
-    zooms = zooms || "8-10"
-    gdal2tiles = "gdal2tiles.py -k -z #{zooms} --s_srs EPSG:3857 -t #{slug} -g #{key} public/warps/#{slug}/#{slug}-geo.tif public/tms/#{slug}/"
+    gdal2tiles = "gdal2tiles.py -k --s_srs EPSG:3857 -t #{id} -g #{key} public/warps/#{id}/#{id}-geo.tif public/tms/#{id}/"
     puts gdal2tiles
     system(self.ulimit+gdal2tiles)
   end
 
-  # zips up tiles at public/tms/<slug>.zip;
-  def self.zip_tiles(slug)
-    rmzip = 'cd public/tms/ && rm '+slug+'.zip && cd ../../'
+  # zips up tiles at public/tms/<id>.zip;
+  def self.zip_tiles(id)
+    rmzip = "cd public/tms/ && rm #{id}.zip && cd ../../"
     system(rmzip)
-    zip = "cd public/tms/ && #{self.ulimit}zip -rq #{slug}.zip #{slug}/ && cd ../../"
+    zip = "cd public/tms/ && #{self.ulimit} zip -rq #{id}.zip #{id}/ && cd ../../"
     system(zip)
   end
 
-  # generates a tileset at public/tms/<slug>/
-  def self.generate_jpg(slug, root)
-    imageMagick = "convert -background white -flatten #{root}/public/warps/#{slug}/#{slug}-geo.tif #{root}/public/warps/#{slug}/#{slug}.jpg"
+  # generates a tileset at public/tms/<id>/
+  def self.generate_jpg(id, root)
+    imageMagick = "convert -background white -flatten public/warps/#{id}/#{id}-geo.tif #{root}/public/warps/#{id}/#{id}.jpg"
     system(self.ulimit+imageMagick)
   end
 
   # runs the above map functions while maintaining a record of state in an Export model;
   # we'll be replacing the export model state with a flat status file
-  def self.run_export(user_id, resolution, export, id, slug, root, average_scale, placed_warpables, key)
+  def self.run_export(user_id, resolution, export, id, root, placed_warpables, key)
     begin
       export.user_id = user_id if user_id
       export.status = 'starting'
@@ -323,28 +321,28 @@ def self.run_export(user_id, resolution, export, id, slug, root, average_scale,
       export.jpg = false
       export.save
 
-      directory = "#{root}/public/warps/"+slug+"/"
+      directory = "#{root}/public/warps/#{id}/"
       stdin, stdout, stderr = Open3.popen3('rm -r '+directory.to_s)
       puts stdout.readlines
       puts stderr.readlines
-      stdin, stdout, stderr = Open3.popen3("rm -r #{root}/public/tms/#{slug}")
+      stdin, stdout, stderr = Open3.popen3("rm -r #{root}/public/tms/#{id}")
       puts stdout.readlines
       puts stderr.readlines
 
       puts '> averaging scales; resolution: ' + resolution.to_s
-      pxperm = 100/(resolution).to_f || average_scale # pixels per meter
+      pxperm = 100/(resolution).to_f # pixels per meter
       puts '> scale: ' + pxperm.to_s + 'pxperm'
 
       puts '> distorting warpables'
 
-      origin = self.distort_warpables(pxperm, placed_warpables, export, slug)
+      origin = self.distort_warpables(pxperm, placed_warpables, export, id)
       warpable_coords = origin.pop
 
       export.status = 'compositing'
       export.save
 
       puts '> generating composite tiff'
-      composite_location = self.generate_composite_tiff(warpable_coords,origin,placed_warpables,slug,false) # no ordering yet
+      composite_location = self.generate_composite_tiff(warpable_coords,origin,placed_warpables,id,false) # no ordering yet
 
       info = (`identify -quiet -format '%b,%w,%h' #{composite_location}`).split(',')
       puts info
@@ -360,17 +358,17 @@ def self.run_export(user_id, resolution, export, id, slug, root, average_scale,
       end
 
       puts '> generating tiles'
-      export.tms = true if self.generate_tiles(key, slug, root)
+      export.tms = true if self.generate_tiles(key, id, root)
       export.status = 'zipping tiles'
       export.save
 
       puts '> zipping tiles'
-      export.zip = true if self.zip_tiles(slug)
+      export.zip = true if self.zip_tiles(id)
       export.status = 'creating jpg'
       export.save
 
       puts '> generating jpg'
-      export.jpg = true if self.generate_jpg(slug, root)
+      export.jpg = true if self.generate_jpg(id, root)
       export.status = 'complete'
       export.save
 

test/exporter_test.rb

@@ -4,7 +4,6 @@
 class ExporterTest < Minitest::Test
   def test_all_functions # break this into separate parts
 
-    slug = "ten-forward"
     id = 1
     user_id = 1
     scale = 2
@@ -43,83 +42,84 @@ def test_all_functions # break this into separate parts
     # make a sample image
     system('mkdir -p public/system/images/1/original')
     system('cp test/fixtures/demo.png public/system/images/1/original/')
-    system("mkdir -p public/warps/#{slug}")
-    system("mkdir -p public/tms/#{slug}")
-    system("touch public/warps/#{slug}/folder")
-    assert File.exist?("public/warps/#{slug}/folder")
+    system("mkdir -p public/warps/#{id}")
+    system("mkdir -p public/tms/#{id}")
+    system("touch public/warps/#{id}/folder")
+    assert File.exist?("public/warps/#{id}/folder")
 
     coords = MapKnitterExporter.generate_perspectival_distort(
       scale, 
-      slug,
+      id,
       nodes_array,
-      id, 
       image[:filename],
       image[:url],
       image[:height],
       image[:width],
       '' # root
     )
     assert coords
-    assert MapKnitterExporter.get_working_directory(slug)
-    assert MapKnitterExporter.warps_directory(slug)
+    assert MapKnitterExporter.get_working_directory(id)
+    assert MapKnitterExporter.warps_directory(id)
 
     # get rid of existing geotiff
-    system("rm -r public/warps/#{slug}/1-geo.tif")
+    system("rm -r public/warps/#{id}/1-geo.tif")
     # make a sample image
-    system('mkdir -p public/system/images/2/original/')
-    system('cp test/fixtures/demo.png public/system/images/2/original/test.png')
+    system('mkdir -p public/system/images/1/original/')
+    system('cp test/fixtures/demo.png public/system/images/1/original/test.png')
 
     origin = MapKnitterExporter.distort_warpables(
       scale,
-      [image], # TODO: here it also expects image to have a nodes_array object
+      [image],
       export,
-      slug
+      id
     )
     lowest_x, lowest_y, warpable_coords = origin
     assert origin
     ordered = false
 
-    system("mkdir -p public/warps/#{slug}")
-    system("mkdir -p public/tms/#{slug}")
+    system("mkdir -p public/warps/#{id}")
+    system("mkdir -p public/tms/#{id}")
     # these params could be compressed - warpable coords is part of origin; are coords and origin required?
     assert MapKnitterExporter.generate_composite_tiff(
       warpable_coords,
       origin,
       [image],
-      slug,
+      id,
       ordered
     )
 
-    assert MapKnitterExporter.generate_tiles('.', slug, root)
+    assert MapKnitterExporter.generate_tiles('', id, root)
 
-    assert MapKnitterExporter.zip_tiles(slug)
+    system("mkdir -p public/tms/#{id}")
+    system("touch public/tms/#{id}/#{id}.zip")
+    assert MapKnitterExporter.zip_tiles(id)
 
-    assert MapKnitterExporter.generate_jpg(slug, '.')
+    assert MapKnitterExporter.generate_jpg(id, '.') # '.' as root
 
+    # run_export(user_id, resolution, export, id, root, placed_warpables, key)
     assert MapKnitterExporter.run_export(
       user_id,
       resolution,
       export,
       id,
-      slug,
       root,
-      scale,
       [image],
       ''
     )
 
     # test deletion of the files; they were already deleted in run_export, 
     # so let's make new dummy ones:
     # make a sample image
-    system('mkdir -p public/system/images/2/original/')
-    system('touch public/system/images/2/original/test.png')
-    system("mkdir -p public/warps/#{slug}")
-    system("mkdir -p public/tms/#{slug}")
-    system("touch public/warps/#{slug}/folder")
-    assert File.exist?("public/warps/#{slug}/folder")
-    system("mkdir -p public/warps/#{slug}-working")
-    system("touch public/warps/#{slug}/test.png")
-    assert MapKnitterExporter.delete_temp_files(slug)
+    system('mkdir -p public/system/images/1/original/')
+    system('touch public/system/images/1/original/test.png')
+    system("mkdir -p public/warps/#{id}")
+    system("mkdir -p public/tms/#{id}")
+    system("touch public/tms/#{id}/#{id}.zip")
+    system("touch public/warps/#{id}/folder")
+    assert File.exist?("public/warps/#{id}/folder")
+    system("mkdir -p public/warps/#{id}-working")
+    system("touch public/warps/#{id}/test.png")
+    assert MapKnitterExporter.delete_temp_files(id)
   end
 end