diff --git a/.travis.yml b/.travis.yml index 8444c0721..4032623a4 100644 --- a/.travis.yml +++ b/.travis.yml @@ -7,10 +7,10 @@ install: - cp config/config.yml.example config/config.yml - cp db/schema.rb.example db/schema.rb - docker-compose build - - docker-compose run web sleep 10 - - docker-compose run web bash -c "rake db:setup" - - docker-compose run web bash -c "rake db:migrate" + - docker-compose run web bash -l -c "sleep 10" + - docker-compose run web bash -l -c "rake db:setup" + - docker-compose run web bash -l -c "rake db:migrate" - docker-compose run web bower install --allow-root script: - - docker-compose run web bash -c "rake test" + - docker-compose run web bash -l -c "rake test" diff --git a/Dockerfile b/Dockerfile index 6f1a4153d..b6eadad88 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,9 +1,7 @@ # Dockerfile # Mapknitter # https://github.com/publiclab/mapknitter/ -FROM ruby:2.4.4-stretch -MAINTAINER Sebastian Silva "sebastian@fuentelibre.org" - +FROM debian:buster LABEL This image deploys Mapknitter! # Set correct environment variables. @@ -11,16 +9,35 @@ RUN mkdir -p /app ENV HOME /root # Install dependencies -RUN curl -sL https://deb.nodesource.com/setup_8.x | bash - -RUN apt-get update -qq && apt-get install -y bundler default-libmysqlclient-dev ruby-rmagick libfreeimage3 libfreeimage-dev ruby-dev gdal-bin python-gdal curl libcurl4-openssl-dev libssl-dev zip nodejs ##ALSO TRIED: ruby-pg +RUN apt-get update -qq && apt-get install -y \ + bundler ruby-rmagick libfreeimage3 \ + libfreeimage-dev zip nodejs gdal-bin \ + curl g++ gcc autoconf automake bison \ + libc6-dev libffi-dev libgdbm-dev \ + libncurses5-dev libsqlite3-dev libtool \ + libyaml-dev make pkg-config sqlite3 \ + zlib1g-dev libgmp-dev libreadline-dev libssl-dev \ + procps libmariadb-dev-compat libmariadb-dev git python-gdal \ + imagemagick + +# Ruby +RUN gpg --keyserver hkp://keys.gnupg.net --recv-keys 409B6B1796C275462A1703113804BB82D39DC0E3 7D2BAF1CF37B13E2069D6956105BD0E739499BDB && curl -sSL https://get.rvm.io | bash -s stable && usermod -a -G rvm root +RUN /bin/bash -l -c ". /etc/profile.d/rvm.sh && rvm install 2.4.4 && rvm use 2.4.4 --default" + +RUN curl -sL https://deb.nodesource.com/setup_8.x | bash - && apt-get install -y npm RUN npm install -g bower + # Install bundle of gems +SHELL [ "/bin/bash", "-l", "-c" ] WORKDIR /tmp ADD Gemfile /tmp/Gemfile ADD Gemfile.lock /tmp/Gemfile.lock RUN bundle install +# HOTFIX Workaround for mysql2 gem incompatibility with libmariadb-dev +RUN sed -i "s/ LONG_PASSWORD |//g" /usr/local/rvm/gems/ruby-*/gems/mysql2-*/lib/mysql2/client.rb + # Add the Rails app WORKDIR /app ADD . /app diff --git a/app/models/map.rb b/app/models/map.rb index a55e6a8ea..fabfaea1c 100755 --- a/app/models/map.rb +++ b/app/models/map.rb @@ -202,164 +202,23 @@ def grouped_images_histogram(binsize) hist end - def run_export(user,resolution) - begin - unless export = self.export - export = Export.new({ - :map_id => self.id - }) - end - export.user_id = user.id if user - export.status = 'starting' - export.tms = false - export.geotiff = false - export.zip = false - export.jpg = false - export.save - - directory = "#{Rails.root}/public/warps/"+self.slug+"/" - stdin, stdout, stderr = Open3.popen3('rm -r '+directory.to_s) - puts stdout.readlines - puts stderr.readlines - stdin, stdout, stderr = Open3.popen3("rm -r #{Rails.root}/public/tms/#{self.slug}") - puts stdout.readlines - puts stderr.readlines - - puts '> averaging scales' - pxperm = 100/(resolution).to_f || self.average_scale # pixels per meter - - puts '> distorting warpables' - - origin = self.distort_warpables(pxperm, self.placed_warpables, self.latest_export) - warpable_coords = origin.pop - - export = self.export - export.status = 'compositing' - export.save - - puts '> generating composite tiff' - composite_location = self.generate_composite_tiff(warpable_coords,origin) - - info = (`identify -quiet -format '%b,%w,%h' #{composite_location}`).split(',') - puts info - - export = self.export - if info[0] != '' - export.geotiff = true - export.size = info[0] - export.width = info[1] - export.height = info[2] - export.cm_per_pixel = 100.0000/pxperm - export.status = 'tiling' - export.save - end - - puts '> generating tiles' - export = self.export - export.tms = true if self.generate_tiles - export.status = 'zipping tiles' - export.save - - puts '> zipping tiles' - export = self.export - export.zip = true if self.zip_tiles - export.status = 'creating jpg' - export.save - - puts '> generating jpg' - export = self.export - export.jpg = true if self.generate_jpg - export.status = 'complete' - export.save - - rescue SystemCallError - export = self.export - export.status = 'failed' - export.save - end - return export.status - end - - # distort all warpables, returns upper left corner coords in x,y - def distort_warpables(scale, warpables, export) - - puts '> generating geotiffs of each warpable in GDAL' - lowest_x=0 - lowest_y=0 - warpable_coords = [] - current = 0 - warpables.each do |warpable| - current += 1 - - export.status = 'warping '+current.to_s+' of '+warpables.length.to_s - puts 'warping '+current.to_s+' of '+warpables.length.to_s - export.save - - my_warpable_coords = warpable.generate_perspectival_distort(scale,self.slug) - puts '- '+my_warpable_coords.to_s - warpable_coords << my_warpable_coords - lowest_x = my_warpable_coords.first if (my_warpable_coords.first < lowest_x || lowest_x == 0) - lowest_y = my_warpable_coords.last if (my_warpable_coords.last < lowest_y || lowest_y == 0) - end - [lowest_x,lowest_y,warpable_coords] - end - - # generate a tiff from all warpable images in this set - def generate_composite_tiff(coords,origin) - directory = "public/warps/"+self.slug+"/" - composite_location = directory+self.slug+'-geo.tif' - geotiffs = '' - minlat = nil - minlon = nil - maxlat = nil - maxlon = nil - self.placed_warpables.each do |warpable| - warpable.nodes_array.each do |n| - minlat = n.lat if minlat == nil || n.lat < minlat - minlon = n.lon if minlon == nil || n.lon < minlon - maxlat = n.lat if maxlat == nil || n.lat > maxlat - maxlon = n.lon if maxlon == nil || n.lon > maxlon - end - end - first = true - # sort by area; this would be overridden by a provided order - warpables = self.placed_warpables.sort{|a,b|b.poly_area <=> a.poly_area} - warpables.each do |warpable| - geotiffs += ' '+directory+warpable.id.to_s+'-geo.tif' - if first - gdalwarp = "gdalwarp -te "+minlon.to_s+" "+minlat.to_s+" "+maxlon.to_s+" "+maxlat.to_s+" "+directory+warpable.id.to_s+'-geo.tif '+directory+self.slug+'-geo.tif' - first = false - else - gdalwarp = "gdalwarp "+directory+warpable.id.to_s+'-geo.tif '+directory+self.slug+'-geo.tif' - end - puts gdalwarp - system(Gdal.ulimit+gdalwarp) + # we'll eventually replace this with a JavaScript call to initiate an external export process: + def run_export(user, resolution) + key = APP_CONFIG ? APP_CONFIG["google_maps_api_key"] : "AIzaSyAOLUQngEmJv0_zcG1xkGq-CXIPpLQY8iQ" + unless export + export = Export.new({ + :map_id => id + }) end - composite_location - end - - # generates a tileset at Rails.root.to_s/public/tms// - def generate_tiles - google_api_key = APP_CONFIG["google_maps_api_key"] - gdal2tiles = 'gdal2tiles.py -k -t "'+self.slug+'" -g "'+google_api_key+'" '+Rails.root.to_s+'/public/warps/'+self.slug+'/'+self.slug+'-geo.tif '+Rails.root.to_s+'/public/tms/'+self.slug+"/" -# puts gdal2tiles -# puts system('which gdal2tiles.py') - system(Gdal.ulimit+gdal2tiles) - end - - # zips up tiles at Rails.root/public/tms/.zip - def zip_tiles - rmzip = 'cd public/tms/ && rm '+self.slug+'.zip && cd ../../' - system(Gdal.ulimit+rmzip) - zip = 'cd public/tms/ && zip -rq '+self.slug+'.zip '+self.slug+'/ && cd ../../' - # puts zip - # puts system('which gdal2tiles.py') - system(Gdal.ulimit+zip) - end - - def generate_jpg - imageMagick = 'convert -background white -flatten '+Rails.root.to_s+'/public/warps/'+self.slug+'/'+self.slug+'-geo.tif '+Rails.root.to_s+'/public/warps/'+self.slug+'/'+self.slug+'.jpg' - system(Gdal.ulimit+imageMagick) + Exporter.run_export(user, + resolution, + self.export, + self.id, + self.slug, + Rails.root.to_s, + self.average_scale, + self.placed_warpables, + key) end def after_create diff --git a/app/models/warpable.rb b/app/models/warpable.rb index 82c77eb44..4ec8059e2 100755 --- a/app/models/warpable.rb +++ b/app/models/warpable.rb @@ -141,203 +141,9 @@ def url=(uri) self.uploaded_data = io end - # pixels per meter = pxperm - def generate_perspectival_distort(pxperm,path) - require 'net/http' - - # everything in -working/ can be deleted; - # this is just so we can use the files locally outside of s3 - working_directory = self.working_directory(path) - Dir.mkdir(working_directory) unless (File.exist?(working_directory) && File.directory?(working_directory)) - local_location = working_directory+self.id.to_s+'-'+self.image_file_name.to_s - - directory = self.warps_directory(path) - Dir.mkdir(directory) unless (File.exist?(directory) && File.directory?(directory)) - completed_local_location = directory+self.id.to_s+'.png' - - # everything -masked.png can be deleted - masked_local_location = directory+self.id.to_s+'-masked.png' - # everything -mask.png can be deleted - mask_location = directory+self.id.to_s+'-mask.png' - #completed_local_location = directory+self.id.to_s+'.tif' - # know everything -unwarped can be deleted - geotiff_location = directory+self.id.to_s+'-geo-unwarped.tif' - # everything -geo WITH AN ID could be deleted, but there is a feature request to preserve these - warped_geotiff_location = directory+self.id.to_s+'-geo.tif' - - northmost = self.nodes_array.first.lat - southmost = self.nodes_array.first.lat - westmost = self.nodes_array.first.lon - eastmost = self.nodes_array.first.lon - - self.nodes_array.each do |node| - northmost = node.lat if node.lat > northmost - southmost = node.lat if node.lat < southmost - westmost = node.lon if node.lon < westmost - eastmost = node.lon if node.lon > eastmost - end - - # puts northmost.to_s+','+southmost.to_s+','+westmost.to_s+','+eastmost.to_s - - scale = 20037508.34 - y1 = pxperm*Cartagen.spherical_mercator_lat_to_y(northmost,scale) - x1 = pxperm*Cartagen.spherical_mercator_lon_to_x(westmost,scale) - y2 = pxperm*Cartagen.spherical_mercator_lat_to_y(southmost,scale) - x2 = pxperm*Cartagen.spherical_mercator_lon_to_x(eastmost,scale) - # puts x1.to_s+','+y1.to_s+','+x2.to_s+','+y2.to_s - - # should determine if it's stored in s3 or locally: - if (self.image.url[0..3] == 'http') - Net::HTTP.start('s3.amazonaws.com') { |http| - #Net::HTTP.start('localhost') { |http| - puts (self.image.url) - resp = http.get(self.image.url) - open(local_location, "wb") { |file| - file.write(resp.body) - } - } - else - require "fileutils" - FileUtils.cp(Rails.root.to_s+'/public'+self.image.to_s,local_location) - end - - points = "" - maskpoints = "" - coordinates = "" - first = true - -#EXIF orientation values: -#Value 0th Row 0th Column -#1 top left side -#2 top right side -#3 bottom right side -#4 bottom left side -#5 left side top -#6 right side top -#7 right side bottom -#8 left side bottom - - rotation = (`identify -format %[exif:Orientation] #{local_location}`).to_i - #stdin, stdout, stderr = Open3.popen3('identify -format %[exif:Orientation] #{local_location}') - #rotation = stdout.readlines.first.to_s.to_i - #puts stderr.readlines - - if rotation == 6 - puts 'rotated CCW' - source_corners = source_corners = [[0,self.height],[0,0],[self.width,0],[self.width,self.height]] - elsif rotation == 8 - puts 'rotated CW' - source_corners = [[self.width,0],[self.width,self.height],[0,self.height],[0,0]] - elsif rotation == 3 - puts 'rotated 180 deg' - source_corners = [[self.width,self.height],[0,self.height],[0,0],[self.width,0]] - else - source_corners = [[0,0],[self.width,0],[self.width,self.height],[0,self.height]] - end - - maxdimension = 0 - - self.nodes_array.each do |node| - corner = source_corners.shift - nx1 = corner[0] - ny1 = corner[1] - nx2 = -x1+(pxperm*Cartagen.spherical_mercator_lon_to_x(node.lon,scale)) - ny2 = y1-(pxperm*Cartagen.spherical_mercator_lat_to_y(node.lat,scale)) - - points = points + ' ' unless first - maskpoints = maskpoints + ' ' unless first - points = points + nx1.to_s + ',' + ny1.to_s + ' ' + nx2.to_i.to_s + ',' + ny2.to_i.to_s - maskpoints = maskpoints + nx2.to_i.to_s + ',' + ny2.to_i.to_s - first = false - # we need to find an origin; find northwestern-most point - coordinates = coordinates+' -gcp '+nx2.to_s+', '+ny2.to_s+', '+node.lon.to_s + ', ' + node.lat.to_s - - # identify largest dimension to set canvas size for ImageMagick: - maxdimension = nx1.to_i if maxdimension < nx1.to_i - maxdimension = ny1.to_i if maxdimension < ny1.to_i - maxdimension = nx2.to_i if maxdimension < nx2.to_i - maxdimension = ny2.to_i if maxdimension < ny2.to_i - end - - # close mask polygon: - maskpoints = maskpoints + ' ' - nx2 = -x1+(pxperm*Cartagen.spherical_mercator_lon_to_x(self.nodes_array.first.lon,scale)) - ny2 = y1-(pxperm*Cartagen.spherical_mercator_lat_to_y(self.nodes_array.first.lat,scale)) - maskpoints = maskpoints + nx2.to_i.to_s + ',' + ny2.to_i.to_s - - height = (y1-y2).to_i.to_s - width = (-x1+x2).to_i.to_s - - # http://www.imagemagick.org/discourse-server/viewtopic.php?f=1&t=11319 - # http://www.imagemagick.org/discourse-server/viewtopic.php?f=3&t=8764 - # read about equalization - # -equalize - # -contrast-stretch 0 - - imageMagick = "convert " - imageMagick += "-contrast-stretch 0 " - imageMagick += local_location+" " - imageMagick += "-crop "+maxdimension.to_i.to_s+"x"+maxdimension.to_i.to_s+"+0+0! " - imageMagick += "-flatten " - imageMagick += "-distort Perspective '"+points+"' " - imageMagick += "-flatten " - if width > height - imageMagick += "-crop "+width+"x"+width+"+0+0\! " - else - imageMagick += "-crop "+height+"x"+height+"+0+0\! " - end - imageMagick += "+repage " - imageMagick += completed_local_location - puts imageMagick - system(Gdal.ulimit+imageMagick) - - # create a mask (later we can blur edges here) - imageMagick2 = 'convert +antialias ' - if width > height - imageMagick2 += "-size "+width+"x"+width+" " - else - imageMagick2 += "-size "+height+"x"+height+" " - end - # attempt at blurred edges in masking, but I've given up, as gdal_merge doesn't seem to respect variable-opacity alpha channels - imageMagick2 += ' xc:none -draw "fill black stroke red stroke-width 30 polyline ' - imageMagick2 += maskpoints + '" ' - imageMagick2 += ' -alpha set -channel A -transparent red -blur 0x8 -channel R -evaluate set 0 +channel '+mask_location - #imageMagick2 += ' xc:none -draw "fill black stroke none polyline ' - #imageMagick2 += maskpoints + '" ' - #imageMagick2 += ' '+mask_location - puts imageMagick2 - system(Gdal.ulimit+imageMagick2) - - imageMagick3 = 'composite '+mask_location+' '+completed_local_location+' -compose DstIn -alpha Set '+masked_local_location - puts imageMagick3 - system(Gdal.ulimit+imageMagick3) - - gdal_translate = "gdal_translate -of GTiff -a_srs EPSG:4326 "+coordinates+' -co "TILED=NO" '+masked_local_location+' '+geotiff_location - puts gdal_translate - system(Gdal.ulimit+gdal_translate) - - #gdalwarp = 'gdalwarp -srcnodata "255" -dstnodata 0 -cblend 30 -of GTiff -t_srs EPSG:4326 '+geotiff_location+' '+warped_geotiff_location - gdalwarp = 'gdalwarp -of GTiff -t_srs EPSG:4326 '+geotiff_location+' '+warped_geotiff_location - puts gdalwarp - system(Gdal.ulimit+gdalwarp) - - # deletions could happen here; do it in distinct method so we can run it independently - self.delete_temp_files(path) - - [x1,y1] - end - - def working_directory(path) - "public/warps/"+path+"-working/" - end - - def warps_directory(path) - "public/warps/"+path+"/" - end - - def delete_temp_files(path) - system('rm -r '+self.working_directory(path)) - system('rm '+self.warps_directory(path)+'*.png') + # TODO: simplify/reduce # of parameters needed here: + def generate_perspectival_distort(pxperm, path) + Exporter.generate_perspectival_distort(pxperm, path, nodes_array, id, image_file_name, image, height, width) end def user_id diff --git a/config/initializers/column_definition.rb b/config/initializers/column_definition.rb index ec8438adf..feca93639 100644 --- a/config/initializers/column_definition.rb +++ b/config/initializers/column_definition.rb @@ -15,7 +15,11 @@ # Read more at https://github.com/publiclab/mapknitter/pull/323 class ActiveRecord::ConnectionAdapters::ColumnDefinition - def sql_type - type.to_sym == :primary_key ? 'int(11) auto_increment PRIMARY KEY' : base.type_to_sql(type.to_sym, limit, precision, scale) rescue type + + if ActiveRecord::Base.connection.adapter_name != 'sqlite3' && ActiveRecord::Base.connection.adapter_name != 'SQLite' + def sql_type + type.to_sym == :primary_key ? 'int(11) auto_increment PRIMARY KEY' : base.type_to_sql(type.to_sym, limit, precision, scale) rescue type + end end -end \ No newline at end of file + +end diff --git a/config/initializers/mysql2_adapter.rb b/config/initializers/mysql2_adapter.rb new file mode 100644 index 000000000..801c43982 --- /dev/null +++ b/config/initializers/mysql2_adapter.rb @@ -0,0 +1,5 @@ +# https://github.com/peatio/peatio/issues/590#issuecomment-352550396 +require 'active_record/connection_adapters/mysql2_adapter' +class ActiveRecord::ConnectionAdapters::Mysql2Adapter + NATIVE_DATABASE_TYPES[:primary_key] = "int(11) auto_increment PRIMARY KEY" +end diff --git a/lib/exporter-deps.sh b/lib/exporter-deps.sh new file mode 100644 index 000000000..fb262e8ae --- /dev/null +++ b/lib/exporter-deps.sh @@ -0,0 +1 @@ +sudo apt-get install gdal-bin python-gdal curl libcurl4-openssl-dev libssl-dev zip libmysqlclient-dev imagemagick ruby-rmagick libfreeimage3 libfreeimage-dev ruby-dev libmagickcore-dev libmagickwand-dev diff --git a/lib/exporter.rb b/lib/exporter.rb new file mode 100644 index 000000000..057cdf50e --- /dev/null +++ b/lib/exporter.rb @@ -0,0 +1,370 @@ +class Exporter + + def self.ulimit + # use ulimit to restrict to 7200 CPU seconds and 5gb virtual memory, and 5gB file storage: + #"ulimit -t 7200 && ulimit -v 5000000 && ulimit -f 5000000 && " + "ulimit -t 14400 && ulimit -v 5000000 && ulimit -f 10000000 && nice -n 19 " + end + + def self.get_working_directory(path) + "public/warps/" + path + "-working/" + end + + def self.warps_directory(path) + "public/warps/" + path + "/" + end + + def self.delete_temp_files(path) + system('rm -r ' + get_working_directory(path)) + system('rm ' + warps_directory(path) + '*.png') + end + + ######################## + ## Run on each image: + + # pixels per meter = pxperm + def self.generate_perspectival_distort(pxperm, path, nodes_array, id, image_file_name, image, height, width) + require 'net/http' + + # everything in -working/ can be deleted; + # this is just so we can use the files locally outside of s3 + working_directory = get_working_directory(path) + Dir.mkdir(working_directory) unless (File.exists?(working_directory) && File.directory?(working_directory)) + local_location = working_directory+id.to_s+'-'+image_file_name.to_s + + directory = warps_directory(path) + Dir.mkdir(directory) unless (File.exists?(directory) && File.directory?(directory)) + completed_local_location = directory+id.to_s+'.png' + + # everything -masked.png can be deleted + masked_local_location = directory+id.to_s+'-masked.png' + # everything -mask.png can be deleted + mask_location = directory+id.to_s+'-mask.png' + #completed_local_location = directory+id.to_s+'.tif' + # know everything -unwarped can be deleted + geotiff_location = directory+id.to_s+'-geo-unwarped.tif' + # everything -geo WITH AN ID could be deleted, but there is a feature request to preserve these + warped_geotiff_location = directory+id.to_s+'-geo.tif' + + northmost = nodes_array.first.lat + southmost = nodes_array.first.lat + westmost = nodes_array.first.lon + eastmost = nodes_array.first.lon + + nodes_array.each do |node| + northmost = node.lat if node.lat > northmost + southmost = node.lat if node.lat < southmost + westmost = node.lon if node.lon < westmost + eastmost = node.lon if node.lon > eastmost + end + + # puts northmost.to_s+','+southmost.to_s+','+westmost.to_s+','+eastmost.to_s + + scale = 20037508.34 + y1 = pxperm*Cartagen.spherical_mercator_lat_to_y(northmost,scale) + x1 = pxperm*Cartagen.spherical_mercator_lon_to_x(westmost,scale) + y2 = pxperm*Cartagen.spherical_mercator_lat_to_y(southmost,scale) + x2 = pxperm*Cartagen.spherical_mercator_lon_to_x(eastmost,scale) + # puts x1.to_s+','+y1.to_s+','+x2.to_s+','+y2.to_s + + # should determine if it's stored in s3 or locally: + if (image.url[0..3] == 'http') + Net::HTTP.start('s3.amazonaws.com') { |http| + #Net::HTTP.start('localhost') { |http| + puts (image.url) + resp = http.get(image.url) + open(local_location, "wb") { |file| + file.write(resp.body) + } + } + else + require "fileutils" + FileUtils.cp(Rails.root.to_s+'/public'+image.to_s,local_location) + end + + points = "" + maskpoints = "" + coordinates = "" + first = true + +#EXIF orientation values: +#Value 0th Row 0th Column +#1 top left side +#2 top right side +#3 bottom right side +#4 bottom left side +#5 left side top +#6 right side top +#7 right side bottom +#8 left side bottom + + rotation = (`identify -format %[exif:Orientation] #{local_location}`).to_i + #stdin, stdout, stderr = Open3.popen3('identify -format %[exif:Orientation] #{local_location}') + #rotation = stdout.readlines.first.to_s.to_i + #puts stderr.readlines + + if rotation == 6 + puts 'rotated CCW' + source_corners = source_corners = [[0,height],[0,0],[width,0],[width,height]] + elsif rotation == 8 + puts 'rotated CW' + source_corners = [[width,0],[width,height],[0,height],[0,0]] + elsif rotation == 3 + puts 'rotated 180 deg' + source_corners = [[width,height],[0,height],[0,0],[width,0]] + else + source_corners = [[0,0],[width,0],[width,height],[0,height]] + end + + maxdimension = 0 + + nodes_array.each do |node| + corner = source_corners.shift + nx1 = corner[0] + ny1 = corner[1] + nx2 = -x1+(pxperm*Cartagen.spherical_mercator_lon_to_x(node.lon,scale)) + ny2 = y1-(pxperm*Cartagen.spherical_mercator_lat_to_y(node.lat,scale)) + + points = points + ' ' unless first + maskpoints = maskpoints + ' ' unless first + points = points + nx1.to_s + ',' + ny1.to_s + ' ' + nx2.to_i.to_s + ',' + ny2.to_i.to_s + maskpoints = maskpoints + nx2.to_i.to_s + ',' + ny2.to_i.to_s + first = false + # we need to find an origin; find northwestern-most point + coordinates = coordinates+' -gcp '+nx2.to_s+', '+ny2.to_s+', '+node.lon.to_s + ', ' + node.lat.to_s + + # identify largest dimension to set canvas size for ImageMagick: + maxdimension = nx1.to_i if maxdimension < nx1.to_i + maxdimension = ny1.to_i if maxdimension < ny1.to_i + maxdimension = nx2.to_i if maxdimension < nx2.to_i + maxdimension = ny2.to_i if maxdimension < ny2.to_i + end + + # close mask polygon: + maskpoints = maskpoints + ' ' + nx2 = -x1+(pxperm*Cartagen.spherical_mercator_lon_to_x(nodes_array.first.lon,scale)) + ny2 = y1-(pxperm*Cartagen.spherical_mercator_lat_to_y(nodes_array.first.lat,scale)) + maskpoints = maskpoints + nx2.to_i.to_s + ',' + ny2.to_i.to_s + + height = (y1-y2).to_i.to_s + width = (-x1+x2).to_i.to_s + + # http://www.imagemagick.org/discourse-server/viewtopic.php?f=1&t=11319 + # http://www.imagemagick.org/discourse-server/viewtopic.php?f=3&t=8764 + # read about equalization + # -equalize + # -contrast-stretch 0 + + imageMagick = "convert " + imageMagick += "-contrast-stretch 0 " + imageMagick += local_location+" " + imageMagick += "-crop "+maxdimension.to_i.to_s+"x"+maxdimension.to_i.to_s+"+0+0! " + imageMagick += "-flatten " + imageMagick += "-distort Perspective '"+points+"' " + imageMagick += "-flatten " + if width > height + imageMagick += "-crop "+width+"x"+width+"+0+0\! " + else + imageMagick += "-crop "+height+"x"+height+"+0+0\! " + end + imageMagick += "+repage " + imageMagick += completed_local_location + puts imageMagick + system(self.ulimit+imageMagick) + + # create a mask (later we can blur edges here) + imageMagick2 = 'convert +antialias ' + if width > height + imageMagick2 += "-size "+width+"x"+width+" " + else + imageMagick2 += "-size "+height+"x"+height+" " + end + # attempt at blurred edges in masking, but I've given up, as gdal_merge doesn't seem to respect variable-opacity alpha channels + imageMagick2 += ' xc:none -draw "fill black stroke red stroke-width 30 polyline ' + imageMagick2 += maskpoints + '" ' + imageMagick2 += ' -alpha set -channel A -transparent red -blur 0x8 -channel R -evaluate set 0 +channel '+mask_location + #imageMagick2 += ' xc:none -draw "fill black stroke none polyline ' + #imageMagick2 += maskpoints + '" ' + #imageMagick2 += ' '+mask_location + puts imageMagick2 + system(self.ulimit+imageMagick2) + + imageMagick3 = 'composite '+mask_location+' '+completed_local_location+' -compose DstIn -alpha Set '+masked_local_location + puts imageMagick3 + system(self.ulimit+imageMagick3) + + gdal_translate = "gdal_translate -of GTiff -a_srs EPSG:4326 "+coordinates+' -co "TILED=NO" '+masked_local_location+' '+geotiff_location + puts gdal_translate + system(self.ulimit+gdal_translate) + + #gdalwarp = 'gdalwarp -srcnodata "255" -dstnodata 0 -cblend 30 -of GTiff -t_srs EPSG:4326 '+geotiff_location+' '+warped_geotiff_location + gdalwarp = 'gdalwarp -of GTiff -t_srs EPSG:4326 '+geotiff_location+' '+warped_geotiff_location + puts gdalwarp + system(self.ulimit+gdalwarp) + + # deletions could happen here; do it in distinct method so we can run it independently + delete_temp_files(path) + + [x1,y1] + end + + ######################## + ## Run on maps: + + # distort all warpables, returns upper left corner coords in x,y + def self.distort_warpables(scale, warpables, export, slug) + + puts '> generating geotiffs of each warpable in GDAL' + lowest_x=0 + lowest_y=0 + warpable_coords = [] + current = 0 + warpables.each do |warpable| + current += 1 + + ## TODO: refactor to generate static status file: + export.status = 'warping '+current.to_s+' of '+warpables.length.to_s + puts 'warping '+current.to_s+' of '+warpables.length.to_s + export.save + ## + + my_warpable_coords = warpable.generate_perspectival_distort(scale,slug) + puts '- '+my_warpable_coords.to_s + warpable_coords << my_warpable_coords + lowest_x = my_warpable_coords.first if (my_warpable_coords.first < lowest_x || lowest_x == 0) + lowest_y = my_warpable_coords.last if (my_warpable_coords.last < lowest_y || lowest_y == 0) + end + [lowest_x,lowest_y,warpable_coords] + end + + # generate a tiff from all warpable images in this set + def self.generate_composite_tiff(coords, origin, placed_warpables, slug, ordered) + directory = "public/warps/"+slug+"/" + composite_location = directory+slug+'-geo.tif' + geotiffs = '' + minlat = nil + minlon = nil + maxlat = nil + maxlon = nil + placed_warpables.each do |warpable| + warpable.nodes_array.each do |n| + minlat = n.lat if minlat == nil || n.lat < minlat + minlon = n.lon if minlon == nil || n.lon < minlon + maxlat = n.lat if maxlat == nil || n.lat > maxlat + maxlon = n.lon if maxlon == nil || n.lon > maxlon + end + end + first = true + if ordered != true + # sort by area; this would be overridden by a provided order + warpables = placed_warpables.sort{|a,b|b.poly_area <=> a.poly_area} + end + warpables.each do |warpable| + geotiffs += ' '+directory+warpable.id.to_s+'-geo.tif' + if first + gdalwarp = "gdalwarp -s_srs EPSG:3857 -te "+minlon.to_s+" "+minlat.to_s+" "+maxlon.to_s+" "+maxlat.to_s+" "+directory+warpable.id.to_s+'-geo.tif '+directory+slug+'-geo.tif' + first = false + else + gdalwarp = "gdalwarp "+directory+warpable.id.to_s+'-geo.tif '+directory+slug+'-geo.tif' + end + puts gdalwarp + system(self.ulimit+gdalwarp) + end + composite_location + end + + # generates a tileset at root/public/tms// + # root is something like https://mapknitter.org + def self.generate_tiles(key, slug, root) + key = "AIzaSyAOLUQngEmJv0_zcG1xkGq-CXIPpLQY8iQ" if key == "" # ugh, let's clean this up! + key = key || "AIzaSyAOLUQngEmJv0_zcG1xkGq-CXIPpLQY8iQ" + gdal2tiles = 'gdal2tiles.py -k --s_srs EPSG:3857 -t "'+slug+'" -g "'+key+'" '+root+'/public/warps/'+slug+'/'+slug+'-geo.tif '+root+'/public/tms/'+slug+"/" + puts gdal2tiles + system(self.ulimit+gdal2tiles) + end + + # zips up tiles at root/public/tms/.zip; + def self.zip_tiles(slug) + rmzip = 'cd public/tms/ && rm '+slug+'.zip && cd ../../' + system(rmzip) + zip = 'cd public/tms/ && ' + self.ulimit + 'zip -rq '+slug+'.zip '+slug+'/ && cd ../../' + system(zip) + end + + # generates a tileset at root/public/tms// + def self.generate_jpg(slug, root) + imageMagick = 'convert -background white -flatten '+root+'/public/warps/'+slug+'/'+slug+'-geo.tif '+root+'/public/warps/'+slug+'/'+slug+'.jpg' + system(self.ulimit+imageMagick) + end + + # runs the above map functions while maintaining a record of state in an Export model; + # we'll be replacing the export model state with a flat status file + def self.run_export(user,resolution,export,id,slug,root,average_scale,placed_warpables,key) + begin + export.user_id = user.id if user + export.status = 'starting' + export.tms = false + export.geotiff = false + export.zip = false + export.jpg = false + export.save + + directory = "#{root}/public/warps/"+slug+"/" + stdin, stdout, stderr = Open3.popen3('rm -r '+directory.to_s) + puts stdout.readlines + puts stderr.readlines + stdin, stdout, stderr = Open3.popen3("rm -r #{root}/public/tms/#{slug}") + puts stdout.readlines + puts stderr.readlines + + puts '> averaging scales; resolution: ' + resolution.to_s + pxperm = 100/(resolution).to_f || average_scale # pixels per meter + puts '> scale: ' + pxperm.to_s + 'pxperm' + + puts '> distorting warpables' + + origin = self.distort_warpables(pxperm, placed_warpables, export, slug) + warpable_coords = origin.pop + + export.status = 'compositing' + export.save + + puts '> generating composite tiff' + composite_location = self.generate_composite_tiff(warpable_coords,origin,placed_warpables,slug,false) # no ordering yet + + info = (`identify -quiet -format '%b,%w,%h' #{composite_location}`).split(',') + puts info + + if info[0] != '' + export.geotiff = true + export.size = info[0] + export.width = info[1] + export.height = info[2] + export.cm_per_pixel = 100.0000/pxperm + export.status = 'tiling' + export.save + end + + puts '> generating tiles' + export.tms = true if self.generate_tiles(key, slug, root) + export.status = 'zipping tiles' + export.save + + puts '> zipping tiles' + export.zip = true if self.zip_tiles(slug) + export.status = 'creating jpg' + export.save + + puts '> generating jpg' + export.jpg = true if self.generate_jpg(slug, root) + export.status = 'complete' + export.save + + rescue SystemCallError + export.status = 'failed' + export.save + end + return export.status + end + +end diff --git a/lib/gdal.rb b/lib/gdal.rb deleted file mode 100755 index 92109f803..000000000 --- a/lib/gdal.rb +++ /dev/null @@ -1,20 +0,0 @@ -require 'open3' - -class Gdal - - def self.ulimit - # use ulimit to restrict to 7200 CPU seconds and 5gb virtual memory, and 5gB file storage: - #"ulimit -t 7200 && ulimit -v 5000000 && ulimit -f 5000000 && " - "ulimit -t 14400 && ulimit -v 5000000 && ulimit -f 10000000 && nice -n 19 " - end - - def self.raw(cmd,verbose) - # unused variable stdin review this function - stdin, stdout, stderr = Open3.popen3(self.ulimit+cmd) - if verbose - puts stderr.readlines - puts stdout.readlines - end - end - -end diff --git a/start.sh b/start.sh new file mode 100755 index 000000000..67cc29711 --- /dev/null +++ b/start.sh @@ -0,0 +1,3 @@ +#!/bin/sh +sleep 5 +bundle exec rails s -p 3000 -b '0.0.0.0' diff --git a/test/fixtures/demo.png b/test/fixtures/demo.png index 3c94776ba..e5b69fbb3 100644 Binary files a/test/fixtures/demo.png and b/test/fixtures/demo.png differ diff --git a/test/fixtures/warpables.yml b/test/fixtures/warpables.yml index 175b2c287..37dd3eefe 100644 --- a/test/fixtures/warpables.yml +++ b/test/fixtures/warpables.yml @@ -14,7 +14,7 @@ one: nodes: "1,2,3,4" locked: false deleted: false - cm_per_pixel: 99.86133660017036> + cm_per_pixel: 500 two: id: 2 diff --git a/test/unit/exporter_test.rb b/test/unit/exporter_test.rb new file mode 100644 index 000000000..d372e6688 --- /dev/null +++ b/test/unit/exporter_test.rb @@ -0,0 +1,57 @@ +require 'test_helper' + +class ExporterTest < ActiveSupport::TestCase + test "isolated exporter lib" do + + # make a sample image + system('mkdir -p public/system/images/1/original') + system('cp test/fixtures/demo.png public/system/images/1/original/') + system('mkdir -p public/warps/saugus-landfill-incinerator') + system('mkdir -p public/tms/saugus-landfill-incinerator') + system('touch public/warps/saugus-landfill-incinerator/folder') + assert File.exist?('public/warps/saugus-landfill-incinerator/folder') + + scale = 2 + + w = warpables(:one) + coords = Exporter.generate_perspectival_distort(scale, w.map.slug, w.nodes_array, w.id, w.image_file_name, w.image, w.height, w.width) + assert coords + assert Exporter.get_working_directory(w.map.slug) + assert Exporter.warps_directory(w.map.slug) + + map = Map.first + + # get rid of existing geotiff + system('rm -r public/warps/saugus-landfill-incinerator/1-geo.tif') + # make a sample image + system('mkdir -p public/system/images/2/original/') + system('cp test/fixtures/demo.png public/system/images/2/original/test.png') + origin = Exporter.distort_warpables(scale, map.warpables, map.export, map.slug) + lowest_x, lowest_y, warpable_coords = origin + assert origin + ordered = false + + system('mkdir -p public/warps/saugus-landfill-incinerator') + system('mkdir -p public/tms/saugus-landfill-incinerator') + # these params could be compressed - warpable coords is part of origin; are coords and origin required? + assert Exporter.generate_composite_tiff(warpable_coords, origin, map.placed_warpables, map.slug, ordered) + assert Exporter.generate_tiles('', map.slug, Rails.root.to_s) + assert Exporter.zip_tiles(map.slug) + assert Exporter.generate_jpg(map.slug, Rails.root.to_s) + resolution = 20 + assert Exporter.run_export(User.last, resolution, map.export, map.id, map.slug, Rails.root.to_s, map.average_scale, map.placed_warpables, '') + + # test deletion of the files; they were already deleted in run_export, so let's make sample ones: + # make a sample image + system('mkdir -p public/system/images/2/original/') + system('touch public/system/images/2/original/test.png') + system('mkdir -p public/warps/saugus-landfill-incinerator') + system('mkdir -p public/tms/saugus-landfill-incinerator') + system('touch public/warps/saugus-landfill-incinerator/folder') + assert File.exist?('public/warps/saugus-landfill-incinerator/folder') + system('mkdir -p public/warps/saugus-landfill-incinerator-working') + system('touch public/warps/saugus-landfill-incinerator/test.png') + assert Exporter.delete_temp_files(w.map.slug) + end +end + diff --git a/test/unit/map_test.rb b/test/unit/map_test.rb index b6bdfa090..8bc3d4430 100644 --- a/test/unit/map_test.rb +++ b/test/unit/map_test.rb @@ -42,7 +42,8 @@ class MapTest < ActiveSupport::TestCase assert_not_nil map.nodes assert_not_nil map.average_cm_per_pixel - assert_not_nil map.run_export(users(:quentin), map.average_cm_per_pixel) + resolution = 20 + assert_not_nil map.run_export(users(:quentin), resolution) #map.average_cm_per_pixel) # main issue will be that it creates and continuously updates an Export model. # we could shift this to a polling model, either on the client side (eliminating the Export model) diff --git a/test/unit/warpable_test.rb b/test/unit/warpable_test.rb index 481f62dca..0272105bb 100644 --- a/test/unit/warpable_test.rb +++ b/test/unit/warpable_test.rb @@ -22,17 +22,14 @@ class WarpableTest < ActiveSupport::TestCase end test "try export" do + # make a sample image system('mkdir -p public/system/images/1/original') system('cp test/fixtures/demo.png public/system/images/1/original/') system('mkdir -p public/warps/saugus-landfill-incinerator') system('touch public/warps/saugus-landfill-incinerator/folder') assert File.exist?('public/warps/saugus-landfill-incinerator/folder') - w = warpables(:one) + w = warpables(:one) assert_not_nil w.save_dimensions - assert_not_nil w.generate_perspectival_distort(10, w.map.slug) - assert_not_nil w.delete_temp_files(w.map.slug) - assert_not_nil w.working_directory(w.map.slug) - assert_not_nil w.warps_directory(w.map.slug) assert_not_nil w.user_id assert File.exist?('public/warps/saugus-landfill-incinerator/1-geo.tif') end