Changeset 9549

Timestamp:

Aug 23, 2011, 8:38:09 PM (13 years ago)

Author:

rick

Message:

• Caches are now fully-file based as memcached was not working well with large entries (like images).
• OSM django proxy is way to slow, rewrite to use apache balanced proxy members.
• Use HTTPD mod_cache to cache heavy image generation work.

Location:

src/django_gheat

Files:

• settings.py (modified) (2 diffs)
• wlheatmap-httpd.conf (added)
• wlheatmap/osm_proxy.py (modified) (1 diff)
• wlheatmap/static/heatmap.js (modified) (2 diffs)
• wlheatmap/templates/js/LayerBase.js (modified) (1 diff)
• wlheatmap/tile.py (modified) (8 diffs)
• wlheatmap/urls.py (modified) (1 diff)

src/django_gheat/settings.py

@@ -7 +7 @@
 DJANGO_BALANCERS = OSM_PROXY_BALANCERS = ['a','b','c','d','e','f']
 OSM_PROXY_CDN_DOMAINS = [d + '.osmproxy.wirelessleiden.nl' for d in DJANGO_BALANCERS]
+OSM_PREFIX = '/osm-tile-proxy'
 DJANGO_CDN_DOMAINS = [d + '.maps.wirelessleiden.nl' for d in DJANGO_BALANCERS]
 
@@ -38 +39 @@
         'LOCATION': '127.0.0.1:11211',
         'TIMEOUT': 3600 * 24 * 7,
+        'OPTIONS': {
+            'MAX_ENTRIES': 100000
+        }
+    },
+    'osm_proxy': {
+        'BACKEND': 'django.core.cache.backends.filebased.FileBasedCache',
+        'LOCATION': '/usr/local/var/django/cache/osm_proxy',
+        'TIMEOUT': 3600 * 24 * 7,
+        'OPTIONS': {
+            'MAX_ENTRIES': 100000
+        }
+    },
+    'tile_cache': {
+        'BACKEND': 'django.core.cache.backends.filebased.FileBasedCache',
+        'LOCATION': '/usr/local/var/django/cache/tile_cache',
+        'TIMEOUT': 3600 * 24,
         'OPTIONS': {
             'MAX_ENTRIES': 100000

src/django_gheat/wlheatmap/osm_proxy.py

@@ -16 +16 @@
 
 # Create your views here.
-@cache_page(60 * 60 * 24 * 7, key_prefix="osm_proxy")
+@cache_page(60 * 60 * 24 * 7, cache="osm_proxy")
 def osm_proxy(request,zoom,x,y):
   mirror = random.choice(['a','b','c'])

src/django_gheat/wlheatmap/static/heatmap.js

@@ -59 +59 @@
   map.addLayer(layerHeatmap3);
 
-//  layerHeatmap = new OpenLayers.Layer.OSM.Overlay1("Original Heatmap generator", {isBaseLayer: false, visibility: false});
-//  map.addLayer(layerHeatmap);
+  layerHeatmap = new OpenLayers.Layer.OSM.Overlay1("Original Heatmap generator", {isBaseLayer: false, visibility: false});
+  map.addLayer(layerHeatmap);
         
   var lonLat = new OpenLayers.LonLat(lon, lat).transform(new OpenLayers.Projection("EPSG:4326"), map.getProjectionObject());
@@ -73 +73 @@
    */
 
-//  $("#default").append("<span id=\"" + layerHeatmap.name + "\"><div id='layer_switcher_colour' style='background-color:rgb(" + layerHeatmap.colour + ")' /><input type=checkbox id=togglelayer /><font>" + layerHeatmap.name + " </font><br /></span>");
+  $("#default").append("<span id=\"" + layerHeatmap.name + "\"><div id='layer_switcher_colour' style='background-color:rgb(" + layerHeatmap.colour + ")' /><input type=checkbox id=togglelayer /><font>" + layerHeatmap.name + " </font><br /></span>");
 
   $("#default").append("<span id=\"" + layerHeatmap2.name + "\"><div id='layer_switcher_colour' style='background-color:rgb(" + layerHeatmap2.colour + ")' /><input type=checkbox id=togglelayer /><font>" + layerHeatmap2.name + " </font><br /></span>");

src/django_gheat/wlheatmap/templates/js/LayerBase.js

@@ -54 +54 @@
      */
     initialize: function(name, options) {
-        var url = get_balanced_urls('http://', {{ settings.OSM_PROXY_CDN_DOMAINS|safe }}, '{{ settings.DJANGO_PREFIX }}/osm-proxy/${z}/${x},${y}.png');
+        var url = get_balanced_urls('http://', {{ settings.OSM_PROXY_CDN_DOMAINS|safe }}, '{{ settings.OSM_PREFIX }}/${z}/${x}/${y}.png');
         options = OpenLayers.Util.extend({ numZoomLevels: 19 }, options);
         var newArguments = [name, url, options];

src/django_gheat/wlheatmap/tile.py

@@ -9 +9 @@
 from django.views.decorators.cache import cache_page
 from gheat.models import *
+import os
 import pygame
 import sys
@@ -36 +37 @@
     self.surf = new_surf
 
-  def write(self, fh,format='png'):
-    # XXX: How to get a PNG stream directly to the output
-    f = tempfile.NamedTemporaryFile(suffix=format)
-    pygame.image.save(self.surf,f.name)
-    f.seek(0)
-    fh.write(f.read())
+  def save_and_get_image(self,filename):
+    """ Save the file to the location and return the file """
+    basedir = os.path.dirname(filename)
+    if not os.path.isdir(basedir):
+      os.makedirs(basedir)
+    pygame.image.save(self.surf,filename)
+    return open(filename,'r').read()
 
   def get_image(self,format='png'):
@@ -48 +50 @@
     f.seek(0)
     return f.read()
+
 
 
@@ -199 +202 @@
     'longitude__gte' : nw_deg.lon
   })
-  # TODO: This is currently hard-coded to display _all_ metingen
-  metingen = Meting.objects.select_related().filter(**filter)
+  # Limit such that high level zooms does not get the whole database
+  metingen = Meting.objects.filter(**filter)[:1000].values_list('latitude', 'longitude', 'signaal')
 
   # XXX: Signal is not normalized in the database making it unknown when a
@@ -219 +222 @@
     return max(x,y) - min(x,y)
   
-  for meting in metingen:
-    lat_min = min(lat_min, meting.latitude)
-    lat_max = max(lat_max, meting.latitude)
-    lon_min = min(lon_min, meting.longitude)
-    lon_max = max(lon_max, meting.longitude)
-    xcoord = int(dif(nw_deg.lon,meting.longitude) / (resolution_deg.lon))
-    ycoord = int(dif(nw_deg.lat,meting.latitude) / (resolution_deg.lat))
+  for (latitude, longitude, signaal) in metingen:
+    lat_min = min(lat_min, latitude)
+    lat_max = max(lat_max, latitude)
+    lon_min = min(lon_min, longitude)
+    lon_max = max(lon_max, longitude)
+    xcoord = int(dif(nw_deg.lon,longitude) / (resolution_deg.lon))
+    ycoord = int(dif(nw_deg.lat,latitude) / (resolution_deg.lat))
 
     # TODO: Please note that this 'logic' technically does apply to WiFi signals,
@@ -239 +242 @@
     # with examples like this: 
     # http://stackoverflow.com/questions/3104781/inverse-distance-weighted-idw-interpolation-with-python
-    signal_normalized = MAX_RANGE - (MAX_SIGNAL - meting.signaal)
-    im.add_circle((xcoord,ycoord),float(signal_normalized) / meters_per_pixel,colour, MAX_SIGNAL - meting.signaal)
+    signal_normalized = MAX_RANGE - (MAX_SIGNAL - signaal)
+    im.add_circle((xcoord,ycoord),float(signal_normalized) / meters_per_pixel,colour, MAX_SIGNAL - signaal)
     #im.add_point((xcoord,ycoord),float(signal_normalized) / meters_per_pixel,colour, MAX_SIGNAL - meting.signaal)
   
@@ -246 +249 @@
   return im
 
-
-# Create your views here.
-@cache_page(60 * 60 * 24, key_prefix="wlheatmap_tile")
-def serve_tile(request,zoom,x,y):
+def pre_process_tile(request,zoom,x,y):
   filter = {}
   colour = (255,0,0)
@@ -259 +259 @@
   now = time.time()
   im = make_tile(int(x),int(y),int(zoom),filter=filter,colour=colour)
+  return im
+
+# Create your views here.
+@cache_page(60 * 60 * 24, cache="tile_cache")
+def serve_tile(request,zoom,x,y):
+  im = pre_process_tile(request,zoom,x,y)
   data = im.get_image('png')
-
-  response = HttpResponse(mimetype="image/png")
-  response.write(data)
-  return response
-
+  return HttpResponse(data,mimetype="image/png")
+
+def fixed_wl_only(request,zoom,x,y): 
+  """ Pre-render and save attempt """
+  im = pre_process_tile(request,zoom,x,y)
+  data = im.save_and_get_image('/usr/local/var/django/tile/fixed/wl-only/%s/%s,%s.png' % (zoom, x, y))
+  return HttpResponse(data,mimetype="image/png")

src/django_gheat/wlheatmap/urls.py

@@ -14 +14 @@
     url(r'^$', direct_to_template, {'template' : 'home.html'}),
     url(r'^js/LayerBase.js$', 'js_layer_base'),
+    url(r'^tile/fixed/wl-only/(?P<zoom>\d+)/(?P<x>\d+),(?P<y>\d+).png$', 'tile.fixed_wl_only'),
     url(r'^tile/(?P<zoom>\d+)/(?P<x>\d+),(?P<y>\d+).png$', 'tile.serve_tile'),
     url(r'^nodelist/(?P<zoom>\d+),(?P<lat>[0-9.]+),(?P<lon>[0-9.]+)$', 'nodelist.serve_nodelist'),