From 4b7738c0f2384798c4b996188f331f57af28032d Mon Sep 17 00:00:00 2001 From: Tyrel Souza Date: Thu, 1 Mar 2018 15:32:02 -0500 Subject: [PATCH] Python3 and Django 1.11 upgrade (#40) * starting py3 upgrade * b64 not base64 * more progress, some more future things, and encoding * tox * update circle.yml * that version doesnt exist yet, oops * install? * sigh * Docker me? * update readme, requirements * fix for python3 * python35 * morgan remediations * does this work? * how about this? * one more time * no blank lines? * revert a bit * add future imports --- README.md | 17 +++++++++++++++-- circle.yml | 11 +++++++++++ dbfilestorage/admin.py | 2 ++ dbfilestorage/models.py | 6 +++++- dbfilestorage/storage.py | 21 +++++++++++++++------ dbfilestorage/urls.py | 4 +++- dbfilestorage/views.py | 6 +++++- requirements-dev.txt | 4 ++++ requirements.txt | 4 +--- setup.py | 6 ++++-- tests/tests.py | 21 ++++++++++++--------- tests/urls.py | 15 +-------------- tests/wsgi.py | 9 +-------- tox.ini | 9 +++++++++ 14 files changed, 88 insertions(+), 47 deletions(-) create mode 100644 requirements-dev.txt create mode 100644 tox.ini diff --git a/README.md b/README.md index 61ad4ee..a269289 100644 --- a/README.md +++ b/README.md @@ -7,6 +7,18 @@ Easy to use for testing remote storages when you're in a transition stage betwee Intended to be used in tests, never in production. +## TESTING + +I use Pyenv and Tox to support Python2.7.13 and Python3.6.3 + +``` +pip install -r requirements-dev.txt +pyenv install 2.7.13 3.5.4 3.6.3 +tox +``` + +Or you can run individually with your shell python using `python setup.py test` + ## INSTALLATION ``` @@ -42,7 +54,7 @@ class SomeClass(models.Model): ## TODO - Test that this works on a fake model, not just the storage file. -- Different django and different python versions. +- Different django versions. ## Signing Key You can find my signing key at [TyrelSouza.com](https://tyrelsouza.com/koken/?/pages/pgp-keys/) @@ -51,8 +63,9 @@ I will sign everything with 0x769A1BC78A2DDEE2 ## CHANGELOG +- 2018-02-01 [Tyrel Souza] Bump versions to django 1.11 and testing with Python3 - 2017-05-10 [Pamela McA'Nulty] Have save overwrite existing files -- 2017-02-06 [Tyrel Souza] Set primary key to `id` not `name`, this involves a lot of migrations, so I've kept them in multiple files +- 2017-02-06 [Tyrel Souza] Set primary key to `id` not `name`, this involves a lot of migrations, so I've kept them in multiple files - 2017-01-27 [Tyrel Souza] Get rid of filehash - 2017-01-26 [Tyrel Souza] Check filehash and filename, not just hash when checking if it needs to be saved. - 2017-01-25 [Tyrel Souza] Keeping Filename on upload. diff --git a/circle.yml b/circle.yml index 8c4bef3..8cadf98 100644 --- a/circle.yml +++ b/circle.yml @@ -1,3 +1,14 @@ +version: 2 + +jobs: + build: + working_directory: ~/django-dbfilestorage + docker: + - image: themattrix/tox + steps: + - checkout + - run: tox + test: post: - bash <(curl -s https://codecov.io/bash) diff --git a/dbfilestorage/admin.py b/dbfilestorage/admin.py index 1eb5b96..1dfc87b 100644 --- a/dbfilestorage/admin.py +++ b/dbfilestorage/admin.py @@ -1,3 +1,5 @@ +from __future__ import absolute_import, print_function, unicode_literals + from django.contrib import admin from .models import DBFile diff --git a/dbfilestorage/models.py b/dbfilestorage/models.py index a1d076a..8b681a6 100644 --- a/dbfilestorage/models.py +++ b/dbfilestorage/models.py @@ -1,6 +1,10 @@ +from __future__ import absolute_import, print_function, unicode_literals + from django.db import models +from future.utils import python_2_unicode_compatible +@python_2_unicode_compatible class DBFile(models.Model): """ Model to store and access uploaded files """ @@ -11,7 +15,7 @@ class DBFile(models.Model): b64 = models.TextField() mtime = models.DateTimeField(auto_now=True) - def __unicode__(self): + def __str__(self): return u"{name} <{content_type}>".format( name=self.name, content_type=self.content_type) diff --git a/dbfilestorage/storage.py b/dbfilestorage/storage.py index e8f2bab..d27f346 100644 --- a/dbfilestorage/storage.py +++ b/dbfilestorage/storage.py @@ -1,3 +1,6 @@ +from __future__ import absolute_import, print_function, unicode_literals + +import base64 import mimetypes import logging import os @@ -6,7 +9,7 @@ from django.db.transaction import atomic from django.db.models import Q from django.core.files.base import ContentFile from django.core.files.storage import Storage -from django.core.urlresolvers import reverse +from django.core.urlresolvers import reverse_lazy from .models import DBFile @@ -29,7 +32,7 @@ class DBFileStorage(Storage): def _open(self, name, mode='rb'): the_file = _get_object(name) - return ContentFile(the_file.b64.decode('base64')) + return ContentFile(base64.b64decode(the_file.b64)) @atomic def _save(self, name, content, max_length=None): @@ -43,8 +46,13 @@ class DBFileStorage(Storage): Then it checks if the file exists and if it doesn't, it will create the entry in the database. + :param name: file name to save + :param content: Content object, where content.file is bytes + :return str: the name(md5) to look up the file by. """ + + # TODO: Make this conditional better if hasattr(content.file, "read"): read_data = content.file.read() if not read_data: @@ -52,8 +60,9 @@ class DBFileStorage(Storage): content.file.seek(0) read_data = content.file.read() else: - read_data = content.file.encode('utf8') - b64 = read_data.encode('base64') + read_data = content.file + + b64 = base64.b64encode(read_data) # USE mimetypes.guess_type as an attempt at getting the content type. ct = mimetypes.guess_type(name)[0] @@ -91,8 +100,8 @@ class DBFileStorage(Storage): def url(self, name): dbf = _get_object(name) if dbf: - return reverse('dbstorage_file', args=(dbf.name,)) - return reverse('dbstorage_file', args=(name,)) + return reverse_lazy('dbstorage_file', args=(dbf.name,)) + return reverse_lazy('dbstorage_file', args=(name,)) def modified_time(self, name): dbf = _get_object(name) diff --git a/dbfilestorage/urls.py b/dbfilestorage/urls.py index 35b7353..2292db0 100644 --- a/dbfilestorage/urls.py +++ b/dbfilestorage/urls.py @@ -1,5 +1,7 @@ +from __future__ import absolute_import, print_function, unicode_literals + from django.conf.urls import url -import views +from dbfilestorage import views urlpatterns = [ url(r'^(?P.*)$', views.show_file, name="dbstorage_file"), diff --git a/dbfilestorage/views.py b/dbfilestorage/views.py index dd8db44..4a86a64 100644 --- a/dbfilestorage/views.py +++ b/dbfilestorage/views.py @@ -1,3 +1,7 @@ +from __future__ import absolute_import, print_function, unicode_literals + +import base64 + from django.http import HttpResponse, Http404 from django.db.models import Q from django.shortcuts import get_object_or_404 @@ -15,7 +19,7 @@ def show_file(request, name): """ dbf = get_object_or_404(DBFile, name=name) response = HttpResponse( - dbf.b64.decode('base64'), + base64.b64decode(dbf.b64), content_type=dbf.content_type) response['Content-Disposition'] = 'attachment; filename="{}"'.format( dbf.name) diff --git a/requirements-dev.txt b/requirements-dev.txt new file mode 100644 index 0000000..00927f6 --- /dev/null +++ b/requirements-dev.txt @@ -0,0 +1,4 @@ +Sphinx==1.5 +coverage==4.2 +future +tox diff --git a/requirements.txt b/requirements.txt index d09cb1e..846d569 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1 @@ -Django==1.10.4 -Sphinx==1.5 -coverage==4.2 +Django==1.11.10 diff --git a/setup.py b/setup.py index 8e6f986..864f625 100644 --- a/setup.py +++ b/setup.py @@ -21,10 +21,11 @@ class CleanCommand(Command): os.system('rm -vrf ./build ./dist ./*.pyc ./*.tgz ./*.egg-info') os.system('rm -vrf htmlcov .coverage') os.system('rm -vrf .DS_Store') + os.system('rm -vrf .tox') setup( name="django-dbfilestorage", - version="0.9.2", + version="0.9.3", description="Database backed file storage for testing.", long_description="Database backed file storage for testing. Stores files as base64 encoded textfields.", author="Tyrel Souza", @@ -38,11 +39,12 @@ setup( ], include_package_data=True, install_requires=[ - "Django>=1.8.0", + "Django==1.11.10", ], tests_require=[ "nose", "coverage", + "future", ], zip_safe=False, test_suite="tests.runtests.start", diff --git a/tests/tests.py b/tests/tests.py index 1f043a2..da026f1 100644 --- a/tests/tests.py +++ b/tests/tests.py @@ -1,4 +1,6 @@ # -*- coding: utf-8 -*- +from __future__ import unicode_literals +import base64 import os from dbfilestorage.models import DBFile @@ -11,6 +13,7 @@ from django.test import TestCase, Client from django.test.utils import override_settings + PROJECT_ROOT = os.path.dirname(os.path.realpath(__file__)) DEFAULT_FILE_STORAGE = "dbfilestorage.storage.DBFileStorage" @@ -36,10 +39,10 @@ class DBFileTest(TestCase): def test_content_file(self): """ Test that this code works with ContentFile as well """ - content_file = ContentFile(u"ƊBStørage") + content_file = ContentFile(b"\\u018aBSt\\xf8rage") default_storage.save("unicode", content_file) unicode_file = DBFile.objects.get(name="unicode") - self.assertEqual(unicode(unicode_file), + self.assertEqual(str(unicode_file), "unicode ") def test_no_duplicate_upload(self): @@ -52,7 +55,7 @@ class DBFileTest(TestCase): """ Test that the DB entry matches what is expected from the file """ with open(self.filepath, 'rb') as f: dbf = DBFile.objects.get(name=self.filepath) - self.assertEqual(dbf.b64.decode("base64"), f.read()) + self.assertEqual(base64.b64decode(dbf.b64), f.read()) self.assertEqual(dbf.content_type, 'image/jpeg') def test_open(self): @@ -62,7 +65,7 @@ class DBFileTest(TestCase): self.assertEqual(dbf.read(), f.read()) def test_exists(self): - """ Test that the storage mechanism can check existance """ + """ Test that the storage mechanism can check existence """ self.assertTrue(default_storage.exists(self.filepath)) def test_delete(self): @@ -81,8 +84,8 @@ class DBFileTest(TestCase): self.assertGreater(size, 0) def test_raw_save(self): - CONTENT_DATA_1 = u"Here's some stuff! ƊBStørage - ONE" - CONTENT_DATA_2 = u"Here's some stuff! ƊBStørage - TWO" + CONTENT_DATA_1 = "Here's some stuff! ƊBStørage - ONE" + CONTENT_DATA_2 = "Here's some stuff! ƊBStørage - TWO" FILE_NAME = "saveable.txt" self.assertFalse(DBFile.objects.filter(name=FILE_NAME).exists()) @@ -142,9 +145,9 @@ class DBFileTest(TestCase): def test_listdir(self): """ Make sure listdir works, and only returns things under 'dirname' """ names = [ - u'dirname/kris.jpg', - u'dirname/kris2.jpg', - u'dirname/kris3.jpg'] + 'dirname/kris.jpg', + 'dirname/kris2.jpg', + 'dirname/kris3.jpg'] for name in names: self._upload(name=name) diff --git a/tests/urls.py b/tests/urls.py index 897d9c8..a53cb3f 100644 --- a/tests/urls.py +++ b/tests/urls.py @@ -1,18 +1,5 @@ -"""tests URL Configuration +from __future__ import absolute_import, print_function, unicode_literals -The `urlpatterns` list routes URLs to views. For more information please see: - https://docs.djangoproject.com/en/1.10/topics/http/urls/ -Examples: -Function views - 1. Add an import: from my_app import views - 2. Add a URL to urlpatterns: url(r'^$', views.home, name='home') -Class-based views - 1. Add an import: from other_app.views import Home - 2. Add a URL to urlpatterns: url(r'^$', Home.as_view(), name='home') -Including another URLconf - 1. Import the include() function: from django.conf.urls import url, include - 2. Add a URL to urlpatterns: url(r'^blog/', include('blog.urls')) -""" from django.conf.urls import url, include from django.contrib import admin diff --git a/tests/wsgi.py b/tests/wsgi.py index 0a2bd3d..16bef7c 100644 --- a/tests/wsgi.py +++ b/tests/wsgi.py @@ -1,11 +1,4 @@ -""" -WSGI config for tests project. - -It exposes the WSGI callable as a module-level variable named ``application``. - -For more information on this file, see -https://docs.djangoproject.com/en/1.10/howto/deployment/wsgi/ -""" +from __future__ import absolute_import, print_function, unicode_literals import os diff --git a/tox.ini b/tox.ini new file mode 100644 index 0000000..5c2b53e --- /dev/null +++ b/tox.ini @@ -0,0 +1,9 @@ +[tox] +envlist = py27,py35,py36 + + +[testenv] +commands=pip install -e . + pip install -r requirements.txt + python setup.py test +install_command=pip install --process-dependency-links --allow-external --allow-unverified {opts} {packages}