Django, Gunicorn, Nginx, Postgres, ошибка сервера Digital Ocean 500 при загрузке изображения

Я разрабатываю веб-сайт/блог с использованием Django и нахожусь в процессе определения правильной настройки/настройки. Я запускаю Ubuntu Server 16.04 на виртуальной машине для тестирования. Я использую стандартную настройку с Gunicorn и Nginx, а также PostgreSQL для базы данных и размещения статических и мультимедийных файлов в Digital Ocean Spaces. Я также намерен разместить сайт в Digital Ocean.

Я собрал информацию из нескольких разных руководств здесь, здесь, здесь и здесь.

Я также использую Django-Imagekit для обработки изображений (url, изменение размера и т. д.) и управляю всем в Админ Джанго.

Проблема, с которой я сталкиваюсь, заключается в том, что когда я загружаю изображение (либо непосредственно в форму изображения, либо через форму сообщения) и сохраняю объект, я получаю ошибку сервера (500). Если я обновляю страницу, то все работает нормально. Это также происходит на самом сайте (т.е. переход на домашнюю страницу, ошибка сервера, обновление, нет ошибки).

В моих логах Gunicorn и Nginx также нет абсолютно никаких ошибок.

Структура файла:

site
├── project
│   ├── gallery
│   │   ├── static
│   │   │   ├── gallery
│   │   │   │   ├── css
│   │   │   │   └── images
│   │   ├── templates
│   │   │   └── gallery
│   │   ├── admin.py
│   │   ├── models.py
│   │   ├── urls.py
│   │   └── views.py
│   ├── posts
│   │   ├── static
│   │   │   ├── posts
│   │   │   │   ├── css
│   │   │   │   └── images
│   │   ├── templates
│   │   │   └── gallery
│   │   ├── admin.py
│   │   ├── models.py
│   │   ├── urls.py
│   │   └── views.py
│   ├── project
│   │   ├── settings
│   │   │   ├── base.py
│   │   │   ├── development.py
│   │   │   ├── local.py
│   │   │   ├── production.py
│   │   │   └── testing.py
│   │   ├── urls.py
│   │   └── wsgi.py
│   ├── static
│   └── templates

галерея/models.py:

...
from imagekit.models import ImageSpecField
from imagekit.processors import ResizeToFit, ResizeToFill
...

class Watermark(object):
    def process(self, image):
        pass

class Image(models.Model):
    original = ImageField(upload_to='images/%Y/%m/%d/')
    large = ImageSpecField(source='original', processors=[Watermark(), \
        ResizeToFit(width=2000, height=2000, upscale=False)], \
        format='JPEG', options={'quality': 90})    
    medium=...
    small=...
    wide=...
    home=...
    upload_date = models.DateTimeField(null=True, editable=False)

    def save(self):
    if not self.id and not self.original:
        return

    if self.upload_date is None:
        self.upload_date = timezone.now()

    image = PIL.Image.open(self.original)
    imgFormat = image.format
    MAX_HEIGHT = 4000
    MAX_WIDTH = 4000

    # Resize image if over MAX pixels in either direction
    (width, height) = image.size
    if height > MAX_HEIGHT or width > MAX_WIDTH:
        ratio = width / height
        output = BytesIO()

        if width > height:
            width = MAX_WIDTH
            height = int(width / ratio)
        else:
            height = MAX_HEIGHT
            width = int(height * ratio)

        size = (width, height)
        image = image.resize(size, PIL.Image.ANTIALIAS)
        image.save(output, format=imgFormat, quality=100)
        self.original = InMemoryUploadedFile(output, 'ImageField', \
            self.original.name, 'images/', sys.getsizeof(output), None)
    super(Image, self).save()

posts/models.py:

class Post(models.Model):
    title = models.CharField(max_length=75)
    ...
    image = models.ForeignKey(Image, blank=True, null=True, \
        on_delete=models.SET_NULL)
    ...

settings/base.py:

import os

BASE_DIR = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))

SECRET_KEY = os.environ['SECRET_KEY']

DEBUG = True

ALLOWED_HOSTS = []

INSTALLED_APPS = [
    'django.contrib.admin',
    'django.contrib.auth',
    'django.contrib.contenttypes',
    'django.contrib.sessions',
    'django.contrib.messages',
    'django.contrib.staticfiles',
    'posts',
    'gallery',
    'taggit',
    'ckeditor',
    'storages',
    'imagekit',
    ...
]

MIDDLEWARE = [
    'django.middleware.security.SecurityMiddleware',
    'django.contrib.sessions.middleware.SessionMiddleware',
    'django.middleware.common.CommonMiddleware',
    'django.middleware.csrf.CsrfViewMiddleware',
    'django.contrib.auth.middleware.AuthenticationMiddleware',
    'django.contrib.messages.middleware.MessageMiddleware',
    'django.middleware.clickjacking.XFrameOptionsMiddleware',
]

ROOT_URLCONF = 'project.urls'

TEMPLATES = [
    {
        'BACKEND': 'django.template.backends.django.DjangoTemplates',
        'DIRS': [os.path.join(BASE_DIR, 'templates')],
        'APP_DIRS': True,
        'OPTIONS': {
            'context_processors': [
                'django.template.context_processors.debug',
                'django.template.context_processors.request',
                'django.contrib.auth.context_processors.auth',
                'django.contrib.messages.context_processors.messages',
                'django.template.context_processors.media',
            ],
        },
    },
]

WSGI_APPLICATION = 'project.wsgi.application'

AUTH_PASSWORD_VALIDATORS = [
    {
        'NAME': 'django.contrib.auth.password_validation.UserAttributeSimilarityValidator',
    },
    {
        'NAME': 'django.contrib.auth.password_validation.MinimumLengthValidator',
    },
    {
        'NAME': 'django.contrib.auth.password_validation.CommonPasswordValidator',
    },
    {
        'NAME': 'django.contrib.auth.password_validation.NumericPasswordValidator',
    },
]

LANGUAGE_CODE = 'en-us'

TIME_ZONE = 'UTC'

USE_I18N = True

USE_L10N = True

USE_TZ = True

settings/testing.py:

from project.settings.base import *

# Override base.py settings here

DEBUG = False

ALLOWED_HOSTS = ['*']

DATABASES = {
    'default': {
        'ENGINE': 'django.db.backends.postgresql_psycopg2',
        'NAME': 'test',
        'USER': 'user',
        'PASSWORD': '****',
        'HOST': 'localhost',
        'PORT': '1234',
    }
}

# DigitalOcean Spaces Settings
AWS_ACCESS_KEY_ID = '*****'
AWS_SECRET_ACCESS_KEY = '*****'
AWS_STORAGE_BUCKET_NAME = 'production-storage'
AWS_S3_ENDPOINT_URL = 'https://nyc3.digitaloceanspaces.com'
AWS_S3_OBJECT_PARAMETERS = {
    'CacheControl': 'max-age=86400'
}
AWS_LOCATION = 'static_test/'

STATICFILES_DIRS = [
    os.path.join(BASE_DIR, 'static', 'static'),
]

STATIC_URL = 'https://%s/%s/' % (AWS_S3_ENDPOINT_URL, AWS_LOCATION)
STATICFILES_STORAGE = 'storages.backends.s3boto3.S3Boto3Storage'
DEFAULT_FILE_STORAGE = 'storages.backends.s3boto3.S3Boto3Storage'
IMAGEKIT_DEFAULT_IMAGE_CACHE_BACKEND = 'imagekit.imagecache.NonValidatingImageCacheBackend'

# Needed for CKEditor to work
AWS_QUERYSTRING_AUTH = False

venv/bin/gunicorn_start:

#!/bin/bash

NAME="project"
DIR=/home/user/site/project
USER=brandon
GROUP=brandon
WORKERS=3
BIND=unix:/home/user/run/gunicorn.sock
DJANGO_SETTINGS_MODULE=project.settings.testing
DJANGO_WSGI_MODULE=project.wsgi
SECRET_KEY='*****'
LOG_LEVEL=error

cd $DIR
source ../../venv/bin/activate

export SECRET_KEY=$SECRET_KEY
export DJANGO_SETTINGS_MODULE=$DJANGO_SETTINGS_MODULE
export PYTHONPATH=$DIR:$PYTHONPATH

exec ../../venv/bin/gunicorn ${DJANGO_WSGI_MODULE}:application \
 --name $NAME \
 --workers $WORKERS \
 --user=$USER \
 --group=$GROUP \
 --bind=$BIND \
 --log-level=$LOG_LEVEL \
 --log-file=-

/etc/nginx/сайты-доступны/проект:

upstream app_server {
    server unix:/home/user/run/gunicorn.sock fail_timeout=0;
}

server {
    listen 80;

    # add here the ip address of your server
    # or a domain pointing to that ip(like example.com or www.example.com)
    server_name 192.168.1.179

    keepalive_timeout 5;
    client_max_body_size 4G;
    access_log /home/user/logs/nginx-access.log;
    error_log /home/user/logs/nginx-error.log;

    location /static/ {
        alias /home/user/site/project/static;   
    }

    # checks for static file, if not found proxy to app
    location / {
        try_files $uri @proxy_to_app;
    }

    location @proxy_to_app {
        proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
        proxy_set_header Host $http_host;
        proxy_redirect off;
        proxy_pass http://app_server;
    }
}

Любая помощь будет оценена по достоинству.

Изменить:

Установка Debug в True привела к следующим ошибкам.

Проблема, похоже, связана с Django-Imagekit

Internal Server Error: /posts/
Traceback (most recent call last):
    File "/home/user/venv/lib/python3.5/site-packages/django/template/base.py", line 882, in _resolve_lookup
    current = current[bit]
TypeError: 'ImageCacheFile' object is not subscriptable

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "/home/user/venv/lib/python3.5/site-packages/django/core/handlers/exception.py", line 41, in inner
    response = get_response(request)
  File "/home/user/venv/lib/python3.5/site-packages/django/core/handlers/base.py", line 187, in _get_response
    response = self.process_exception_by_middleware(e, request)
  File "/home/user/venv/lib/python3.5/site-packages/django/core/handlers/base.py", line 185, in _get_response
    response = wrapped_callback(request, *callback_args, **callback_kwargs)
  File "/home/user/site/project/posts/views.py", line 39, in posts
    return render(request, 'posts/posts.html', {'posts':posts, 'recentTags':recent_tags})
  File "/home/user/venv/lib/python3.5/site-packages/django/shortcuts.py", line 30, in render
    content = loader.render_to_string(template_name, context, request, using=using)
  File "/home/user/venv/lib/python3.5/site-packages/django/template/loader.py", line 68, in render_to_string
    return template.render(context, request)
  File "/home/user/venv/lib/python3.5/site-packages/django/template/backends/django.py", line 66, in render
    return self.template.render(context)
  File "/home/user/venv/lib/python3.5/site-packages/django/template/base.py", line 207, in render
    return self._render(context)
  File "/home/user/venv/lib/python3.5/site-packages/django/template/base.py", line 199, in _render
    return self.nodelist.render(context)
  File "/home/user/venv/lib/python3.5/site-packages/django/template/base.py", line 990, in render
    bit = node.render_annotated(context)
  File "/home/user/venv/lib/python3.5/site-packages/django/template/base.py", line 957, in render_annotated
    return self.render(context)
  File "/home/user/venv/lib/python3.5/site-packages/django/template/loader_tags.py", line 177, in render
    return compiled_parent._render(context)
  File "/home/user/venv/lib/python3.5/site-packages/django/template/base.py", line 199, in _render
    return self.nodelist.render(context)
  File "/home/user/venv/lib/python3.5/site-packages/django/template/base.py", line 990, in render
    bit = node.render_annotated(context)
  File "/home/user/venv/lib/python3.5/site-packages/django/template/base.py", line 957, in render_annotated
    return self.render(context)
  File "/home/user/venv/lib/python3.5/site-packages/django/template/loader_tags.py", line 72, in render
    result = block.nodelist.render(context)
  File "/home/user/venv/lib/python3.5/site-packages/django/template/base.py", line 990, in render
    bit = node.render_annotated(context)
  File "/home/user/venv/lib/python3.5/site-packages/django/template/base.py", line 957, in render_annotated
    return self.render(context)
  File "/home/user/venv/lib/python3.5/site-packages/django/template/loader_tags.py", line 216, in render
    return template.render(context)
  File "/home/user/venv/lib/python3.5/site-packages/django/template/base.py", line 209, in render
    return self._render(context)
  File "/home/user/venv/lib/python3.5/site-packages/django/template/base.py", line 199, in _render
    return self.nodelist.render(context)
  File "/home/user/venv/lib/python3.5/site-packages/django/template/base.py", line 990, in render
    bit = node.render_annotated(context)
  File "/home/user/venv/lib/python3.5/site-packages/django/template/base.py", line 957, in render_annotated
    return self.render(context)
  File "/home/user/venv/lib/python3.5/site-packages/django/template/defaulttags.py", line 216, in render
    nodelist.append(node.render_annotated(context))
  File "/home/user/venv/lib/python3.5/site-packages/django/template/base.py", line 957, in render_annotated
    return self.render(context)
  File "/home/user/venv/lib/python3.5/site-packages/django/template/defaulttags.py", line 322, in render
    return nodelist.render(context)
  File "/home/user/venv/lib/python3.5/site-packages/django/template/base.py", line 990, in render
    bit = node.render_annotated(context)
  File "/home/user/venv/lib/python3.5/site-packages/django/template/base.py", line 957, in render_annotated
    return self.render(context)
  File "/home/user/venv/lib/python3.5/site-packages/django/template/base.py", line 1040, in render
    output = self.filter_expression.resolve(context)
  File "/home/user/venv/lib/python3.5/site-packages/django/template/base.py", line 708, in resolve
    obj = self.var.resolve(context)
  File "/home/user/venv/lib/python3.5/site-packages/django/template/base.py", line 849, in resolve
    value = self._resolve_lookup(context)
  File "/home/user/venv/lib/python3.5/site-packages/django/template/base.py", line 890, in _resolve_lookup
    current = getattr(current, bit)
  File "/home/user/venv/lib/python3.5/site-packages/imagekit/cachefiles/__init__.py", line 85, in url
    return self._storage_attr('url')
  File "/home/user/venv/lib/python3.5/site-packages/imagekit/cachefiles/__init__.py", line 75, in _storage_attr
    existence_required.send(sender=self, file=self)
  File "/home/user/venv/lib/python3.5/site-packages/django/dispatch/dispatcher.py", line 193, in send
    for receiver in self._live_receivers(sender)
  File "/home/user/venv/lib/python3.5/site-packages/django/dispatch/dispatcher.py", line 193, in <listcomp>
    for receiver in self._live_receivers(sender)
  File "/home/user/venv/lib/python3.5/site-packages/imagekit/registry.py", line 53, in existence_required_receiver
    self._receive(file, 'on_existence_required')
  File "/home/user/venv/lib/python3.5/site-packages/imagekit/registry.py", line 61, in _receive
    call_strategy_method(file, callback)
  File "/home/user/venv/lib/python3.5/site-packages/imagekit/utils.py", line 166, in call_strategy_method
    fn(file)
  File "/home/user/venv/lib/python3.5/site-packages/imagekit/cachefiles/strategies.py", line 15, in on_existence_required
    file.generate()
  File "/home/user/venv/lib/python3.5/site-packages/imagekit/cachefiles/__init__.py", line 94, in generate
    self.cachefile_backend.generate(self, force)
  File "/home/user/venv/lib/python3.5/site-packages/imagekit/cachefiles/backends.py", line 109, in generate
    self.generate_now(file, force=force)
  File "/home/user/venv/lib/python3.5/site-packages/imagekit/cachefiles/backends.py", line 96, in generate_now
    file._generate()
  File "/home/user/venv/lib/python3.5/site-packages/imagekit/cachefiles/__init__.py", line 103, in _generate
    content.seek(0)
ValueError: I/O operation on closed file.

python django gunicorn digital-ocean django-imagekit
person Brandon Q    schedule 13.02.2018    source источник
comment
О чем говорит ошибка 500?   -  person petr    schedule 13.02.2018
comment
Вот в чем дело. Пишет только Ошибка сервера (500). Ничего больше.   -  person Brandon Q    schedule 14.02.2018
comment
Если я могу правильно прочитать ваши настройки, вы используете настройки тестирования, которые отключили режим отладки, и в этом случае Django скрывает информацию об отладке. Попробуйте включить режим отладки, и вы можете увидеть ошибку.   -  person petr    schedule 14.02.2018
comment
Да, это правильно, я использую настройки тестирования. Установка для отладки значения true привела к паре ошибок Gunicorn. Я добавлю трассировку стека в пост.   -  person Brandon Q    schedule 14.02.2018
comment
См. отчеты об ошибках для получения отчетов об ошибках даже с DEBUG = False   -  person kichik    schedule 14.02.2018
comment
Покажите нам представление и шаблон, который вы пытаетесь отобразить. Похоже, проблема не в Gunicorn, а в представлении или шаблоне.   -  person petr    schedule 14.02.2018

Ответы (2)


arrow_upward
0
arrow_downward

Кажется, мне удалось найти обходной путь. Проблема в том, что django-storages закрывал файл изображения после его загрузки, вызывая ошибку ввода-вывода в django-imagekit.

Я нашел обходной путь здесь.

import os
from storages.backends.s3boto3 import S3Boto3Storage
from tempfile import SpooledTemporaryFile

class CustomS3Boto3Storage(S3Boto3Storage):
"""
This is our custom version of S3Boto3Storage that fixes a bug in boto3 where the passed in file is closed upon upload.

https://github.com/boto/boto3/issues/929
https://github.com/matthewwithanm/django-imagekit/issues/391
"""

def _save_content(self, obj, content, parameters):
    """
    We create a clone of the content file as when this is passed to boto3 it wrongly closes
    the file upon upload where as the storage backend expects it to still be open
    """
    # Seek our content back to the start
    content.seek(0, os.SEEK_SET)

    # Create a temporary file that will write to disk after a specified size
    content_autoclose = SpooledTemporaryFile()

    # Write our original content into our copy that will be closed by boto3
    content_autoclose.write(content.read())

    # Upload the object which will auto close the content_autoclose instance
    super(CustomS3Boto3Storage, self)._save_content(obj, content_autoclose, parameters)

    # Cleanup if this is fixed upstream our duplicate should always close        
    if not content_autoclose.closed:
        content_autoclose.close()

Создайте файл где-нибудь в проекте и добавьте код (например, storage_backends.py). Затем в настройках установить:

DEFAULT_FILE_STORAGE='project.storage_backends.CustomS3Boto3Storage'
person Brandon Q    schedule 16.02.2018

arrow_upward
0
arrow_downward

В последней версии django-storages удален метод _save_content. Это обновленная версия этого пользовательского класса хранилища:

class CustomS3Boto3Storage(S3Boto3Storage, ABC):
    """
    This is our custom version of S3Boto3Storage that fixes a bug in
    boto3 where the passed in file is closed upon upload.
    From:
    https://github.com/matthewwithanm/django-imagekit/issues/391#issuecomment-275367006
    https://github.com/boto/boto3/issues/929
    https://github.com/matthewwithanm/django-imagekit/issues/391
    """

    def _save(self, name, content):
        """
        We create a clone of the content file as when this is passed to
        boto3 it wrongly closes the file upon upload where as the storage
        backend expects it to still be open
        """
        # Seek our content back to the start
        content.seek(0, os.SEEK_SET)

        # Create a temporary file that will write to disk after a specified
        # size
        content_autoclose = SpooledTemporaryFile()

        # Write our original content into our copy that will be closed by boto3
        content_autoclose.write(content.read())

        # Upload the object which will auto close the content_autoclose
        # instance
        super(CustomS3Boto3Storage, self)._save(name, content_autoclose)

        # Cleanup if this is fixed upstream our duplicate should always close
        if not content_autoclose.closed:
            content_autoclose.close()
person pasevin    schedule 17.02.2020