Hub data logs rollover daily (#42)

* Hub data logs rollover daily. closes #36 
* data files now saved under /media/data. closes #37 
* Better handling of failures writing logs to disk . closes #38 
* Removing the hub_data volume (no longer needed). Might cause data loss in existing projects

README.md

@@ -41,6 +41,15 @@ Finally, run the following command to start the server:
 
 In order to check whether the server is working, login to the admin console: http://localhost:8000/admin
 
+## Testing
+
+WIP
+
+For testing, make sure postgres is running and run `python manage.py test --settings=config.settings.test`
+NOTE: Make sure the source directory (openbadge-server) is in your PYTHONPATH
+
+Plans include enabling testing via docker
+
 ### Using a virtual machine for development
 You can also create a local Virtual Machine and provision it - https://docs.docker.com/machine/get-started/
 

compose/django/Dockerfile

@@ -27,10 +27,6 @@ RUN mkdir /www \
     && mkdir /www/static \
     && chown django /www/static
 
-RUN mkdir /data \
-    && chown django /data
-
-
 WORKDIR /app
 
 

compose/django/Dockerfile-dev

@@ -14,7 +14,6 @@ COPY ./compose/django/start-dev.sh /start-dev.sh
 RUN sed -i 's/\r//' /start-dev.sh
 RUN chmod +x /start-dev.sh
 
-
 WORKDIR /app
 
 

config/settings/local.py

@@ -57,7 +57,7 @@
 # TESTING
 # ------------------------------------------------------------------------------
 TEST_RUNNER = 'django.test.runner.DiscoverRunner'
-DATA_DIR="/data/"
+DATA_DIR="/media/data/"
 MEETINGS_DIR="../meetings/"
 
 

config/settings/production.py

@@ -30,7 +30,7 @@
 STATIC_ROOT = '/www/static'
 
 # Collected data directory:
-DATA_DIR="/data/"
+DATA_DIR="/media/data/"
 MEETINGS_DIR="data"
 
 

config/settings/test.py

@@ -12,7 +12,6 @@
 # ------------------------------------------------------------------------------
 # Turn debug off so tests run faster
 DEBUG = False
-TEMPLATES[0]['OPTIONS']['debug'] = False
 
 
 # Mail settings
@@ -46,12 +45,4 @@
     'django.contrib.auth.hashers.MD5PasswordHasher',
 )
 
-# TEMPLATE LOADERS
-# ------------------------------------------------------------------------------
-# Keep templates in memory so tests run faster
-TEMPLATES[0]['OPTIONS']['loaders'] = [
-    ('django.template.loaders.cached.Loader', [
-        'django.template.loaders.filesystem.Loader',
-        'django.template.loaders.app_directories.Loader',
-    ]),
-]
+DATA_DIR = 'testout/'

dev.yml

@@ -26,14 +26,10 @@ services:
     command: /start-dev.sh
     depends_on:
       - postgres
-    environment: 
-      - PYTHONPATH=/app/openbadge-server
-      - DJANGO_SETTINGS_MODULE=config.settings.local
-    env_file:
-      - .env
     volumes:
       - .:/app
-      - ../data:/data
+      - ../media:/media
+      - ../media/data:/media/data
     ports:
       - "8000:8000"
     links:

docker-compose.yml

@@ -1,7 +1,6 @@
 version: '2'
 
 volumes:
-  hub_data: {}
   hub_media: {}
   postgres_data: {}
   postgres_backup: {}
@@ -37,7 +36,6 @@ services:
       - redis
     volumes:
       - hub_media:/media/
-      - hub_data:/data/
       - /www/static
     command: /gunicorn.sh
 

openbadge-server/openbadge/migrations/0006_auto_20180515_1354.py

@@ -0,0 +1,27 @@
+# -*- coding: utf-8 -*-
+from __future__ import unicode_literals
+
+from django.db import models, migrations
+import datetime
+from django.utils.timezone import utc
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('openbadge', '0005_auto_20180511_2151'),
+    ]
+
+    operations = [
+        migrations.RenameField(
+            model_name='datafile',
+            old_name='last_update_timestamp',
+            new_name='last_chunk',
+        ),
+        migrations.AddField(
+            model_name='datafile',
+            name='date',
+            field=models.DateTimeField(default=datetime.datetime(2018, 5, 9, 17, 54, 40, 184045, tzinfo=utc)),
+            preserve_default=False,
+        ),
+    ]

openbadge-server/openbadge/migrations/0007_auto_20180515_1424.py

@@ -0,0 +1,19 @@
+# -*- coding: utf-8 -*-
+from __future__ import unicode_literals
+
+from django.db import models, migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('openbadge', '0006_auto_20180515_1354'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='datafile',
+            name='hub',
+            field=models.ForeignKey(related_name='data', to='openbadge.Hub', null=True),
+        ),
+    ]

openbadge-server/openbadge/migrations/0008_auto_20180516_1520.py

@@ -0,0 +1,24 @@
+# -*- coding: utf-8 -*-
+from __future__ import unicode_literals
+
+from django.db import models, migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('openbadge', '0007_auto_20180515_1424'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='datafile',
+            name='date',
+            field=models.DateField(),
+        ),
+        migrations.AlterField(
+            model_name='datafile',
+            name='filepath',
+            field=models.CharField(unique=True, max_length=128, blank=True),
+        ),
+    ]

openbadge-server/openbadge/models.py

@@ -33,7 +33,7 @@ class BaseModel(models.Model):
     id = models.AutoField(primary_key = True)
     key = models.CharField(max_length=10, unique=True, db_index=True, blank=True)
     date_created = models.DateTimeField(auto_now_add=True)
-    date_updated = models.DateTimeField(auto_now=True)  
+    date_updated = models.DateTimeField(auto_now=True)
 
     def generate_key(self, length=10):
         if not self.key:
@@ -58,7 +58,7 @@ class BaseModelMinimal(models.Model):
     """
     key = models.CharField(max_length=10, unique=True, db_index=True, blank=True)
     date_created = models.DateTimeField(auto_now_add=True)
-    date_updated = models.DateTimeField(auto_now=True)  
+    date_updated = models.DateTimeField(auto_now=True)
 
     def generate_key(self, length=10):
         if not self.key:
@@ -126,7 +126,7 @@ def get_available_name(self, name):
 
 ##########################################################################################
 def _now_as_epoch():
-    return round(Decimal(time.time()), 0) 
+    return round(Decimal(time.time()), 0)
 
 @fix_email
 class OpenBadgeUser(auth_models.AbstractUser, BaseModel):
@@ -172,7 +172,7 @@ def to_object(self):
         """for use in HTTP responses, gets the id, name, members, and a map form badge_ids to member names"""
         return {
             'project_id': self.advertisement_project_id,
-            'key': self.key,                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                    
+            'key': self.key,
             'name': self.name,
             'badge_map': {
                 member.badge: {
@@ -219,7 +219,7 @@ class Hub(BaseModel):
 
     last_seen_ts = models.DecimalField(max_digits=20, decimal_places=3, default=Decimal(0))
     """The last time the hub was seen by the server (in epoch time)"""
-    
+
     last_hub_time_ts = models.DecimalField(max_digits=20, decimal_places=3, default=Decimal(0))
     """ The clock time of the hub at the time of the last API request """
 
@@ -487,23 +487,25 @@ class DataFile(BaseModel):
     """
 
     uuid = models.CharField(max_length=64, db_index=True, unique=True)
-    """this will be a concatenation of the hub uuid and data type"""
+    """this will be a concatenation of the hub uuid, data type, and date"""
 
     data_type = models.CharField(max_length=64)
     """Is this an audio or proximity data log?"""
 
-    # at this point we don't even really care about this, maybe worth storing though
-    last_update_timestamp = models.DecimalField(
+    date = models.DateField(auto_now=False)
+    """The date this file contains data for"""
+
+    last_chunk = models.DecimalField(
         decimal_places=3,
         max_digits= 20,
         null=True,
         blank=True)
-    """Log_timestamp of the last chunk received"""
+    """Timestamp of the last chunk written to file"""
 
-    filepath = models.CharField(max_length=65, unique=True, blank=True)
+    filepath = models.CharField(max_length=128, unique=True, blank=True)
     """Local reference to log file"""
 
-    hub = models.ForeignKey(Hub, related_name="data")
+    hub = models.ForeignKey(Hub, null=True, related_name="data")
     """The Hub this DataFile belongs to"""
 
     project = models.ForeignKey(Project, null=True, related_name="data")
@@ -515,18 +517,30 @@ def __unicode__(self):
     def get_meta(self):
         """creates a json object of the metadata for this DataFile"""
         return {
-            'last_update_index': self.last_update_index,
-            'log_timestamp': self.last_update_timestamp,
-            'hub': self.hub.name
+            'date': self.date,
+            'data_type': self.data_type,
+            'hub': self.hub.name,
+            'project': self.project.name,
+            'uuid': self.uuid,
+            'last_chunk_timestamp': self.last_chunk
         }
 
     def to_object(self, file):
         """Get a representation of this object for use with HTTP responses"""
-        if file:
-            return {
-                "chunks": self.get_chunks(),
-                "metadata": self.get_meta()
-            }
-        else:
-            return { "metadata": meta }
-
+        meta = self.get_meta()
+        return {"metadata": meta}
+
+    def save(self, *args, **kwargs):
+        """
+        filepath is generated based on uuid
+        """
+        if not self.filepath:
+            filename = self.uuid + ".txt"
+            directory = "/".join((settings.DATA_DIR.rstrip('/'),
+                                  self.project.key))
+            if not os.path.exists(directory):
+                os.makedirs(directory)
+
+            self.filepath = "/".join((directory, filename))
+
+        super(DataFile, self).save(*args, **kwargs)

openbadge-server/openbadge/tests/test_auth.py

@@ -20,7 +20,7 @@
     "/project"
 """
 
-class TestPermissions(TestCase)
+class TestPermissions(TestCase):
 
     def setUp(self):
         self.factory = RequestFactory()