Merge branch 'master' into 287-delete-from-data

docker/Dockerfile.dev

@@ -7,6 +7,7 @@ RUN apt-get update && \
         cron \
         curl \
         dcraw \
+        file \
         git \
         gnupg \
         libatlas-base-dev \
@@ -18,6 +19,7 @@ RUN apt-get update && \
         libgl1 \
         libglib2.0-dev \
         libhdf5-dev \
+        libheif-examples \
         libimage-exiftool-perl \
         libjpeg-dev \
         liblapack-dev \

docker/Dockerfile.prd

@@ -98,11 +98,13 @@ RUN apt-get update && \
     apt-get install -y --no-install-recommends \
         cron \
         dcraw \
+        file \
         libatlas3-base \
         libfreetype6 \
         libfreetype6-dev \
         libgl1 \
         libglib2.0-dev \
+        libheif-examples \
         libimage-exiftool-perl \
         libpq-dev \
         libtiff5-dev \
@@ -139,6 +141,7 @@ COPY system/cron.d /etc/cron.d/
 RUN chmod 0644 /etc/cron.d/*
 
 ENV PYTHONPATH /srv
+ENV TF_CPP_MIN_LOG_LEVEL 3
 
 RUN DJANGO_SECRET_KEY=test python photonix/manage.py collectstatic --noinput --link
 

docker/docker-compose.dev.yml

@@ -39,6 +39,7 @@ services:
       POSTGRES_PASSWORD: password
       REDIS_HOST: redis
       ALLOWED_HOSTS: '*'
+      LOG_LEVEL: DEBUG
     volumes:
       - ../photonix:/srv/photonix
       - ../system:/srv/system

docker/docker-compose.example.yml

@@ -27,6 +27,7 @@ services:
       POSTGRES_PASSWORD: password
       REDIS_HOST: redis
       ALLOWED_HOSTS: '*'
+      # More configuration options here: https://photonix.org/docs/configuration/
     volumes:
       - ./data/photos:/data/photos
       - ./data/raw-photos-processed:/data/raw-photos-processed

photonix/accounts/schema.py

@@ -3,6 +3,7 @@
 from django.contrib.auth import get_user_model, authenticate, update_session_auth_hash
 import graphene
 from graphene_django.types import DjangoObjectType
+from graphql import GraphQLError
 from graphql_jwt.shortcuts import create_refresh_token, get_token
 import graphql_jwt
 from photonix.photos.models import Library, LibraryPath, LibraryUser
@@ -12,18 +13,12 @@
 
 
 class UserType(DjangoObjectType):
-    """Docstring for UserType."""
-
     class Meta:
         model = User
 
 
 class CreateUser(graphene.Mutation):
-    """Docstring for CreateUser."""
-
     class Arguments:
-        """Docstring for Arguments."""
-
         username = graphene.String(required=True)
         password = graphene.String(required=True)
         password1 = graphene.String(required=True)
@@ -34,13 +29,12 @@ class Arguments:
 
     @staticmethod
     def mutate(self, info, username, password, password1):
-        """Mutate method."""
         if User.objects.filter(username=username).exists():
-            raise Exception("Username already exists!")
+            raise GraphQLError('Username already exists!')
         elif len(password) < 8 and len(password1) < 8:
-            raise Exception("Password must be at least 8 characters long!")
+            raise GraphQLError('Password must be at least 8 characters long!')
         elif password != password1:
-            raise Exception("Password fields do not match!")
+            raise GraphQLError('Password fields do not match!')
         else:
             user = User(username=username)
             user.set_password(password1)
@@ -61,11 +55,11 @@ class Environment(graphene.ObjectType):
 
 
 class AfterSignup(graphene.ObjectType):
-    """Pass token for login, after signup."""
-
+    '''Pass token for login, after signup.'''
     token = graphene.String()
     refresh_token = graphene.String()
 
+
 class Query(graphene.ObjectType):
     profile = graphene.Field(UserType)
     environment = graphene.Field(Environment)
@@ -74,21 +68,20 @@ class Query(graphene.ObjectType):
     def resolve_profile(self, info):
         user = info.context.user
         if user.is_anonymous:
-            raise Exception('Not logged in')
+            raise GraphQLError('Not logged in')
         return user
 
     def resolve_environment(self, info):
         user = User.objects.first()
         if user and user.has_config_persional_info and \
             user.has_created_library and user.has_configured_importing and \
                 user.has_configured_image_analysis:
-            # raise Exception(info.context.user.is_anonymous)
             return {
                 'demo': os.environ.get('DEMO', False),
                 'first_run': False,
             }
         else:
-            if not user:
+            if not user or not user.is_authenticated:
                 return {
                     'demo': os.environ.get('DEMO', False), 'first_run': True,
                     'form': 'has_config_persional_info'}
@@ -111,29 +104,24 @@ def resolve_environment(self, info):
                 }
 
     def resolve_after_signup(self, info):
-        """To login user from frontend after finish sigunp process."""
+        '''To login user from frontend after finish sigunp process.'''
         user = info.context.user
-        if user.has_configured_image_analysis:
+        if user.is_authenticated and user.has_configured_image_analysis:
             return {'token': get_token(user), 'refresh_token': create_refresh_token(user)}
         return {'token': None, 'refresh_token': None}
 
 
 class ChangePassword(graphene.Mutation):
-    """docstring for ChangePassword."""
-
     class Arguments:
-        """docstring for Arguments."""
-
         old_password = graphene.String(required=True)
         new_password = graphene.String(required=True)
 
     ok = graphene.Boolean()
 
     @staticmethod
     def mutate(self, info, old_password, new_password):
-        """Mutate method for change password."""
         if os.environ.get('DEMO', False) and os.environ.get('ENV') != 'test':
-            raise Exception("Password cannot be changed in demo mode!")
+            raise GraphQLError('Password cannot be changed in demo mode!')
         if authenticate(username=info.context.user.username, password=old_password):
             info.context.user.set_password(new_password)
             info.context.user.save()
@@ -143,8 +131,6 @@ def mutate(self, info, old_password, new_password):
 
 
 class Mutation(graphene.ObjectType):
-    """To create objects for all mutaions."""
-
     token_auth = graphql_jwt.ObtainJSONWebToken.Field()
     verify_token = graphql_jwt.Verify.Field()
     refresh_token = graphql_jwt.Refresh.Field()

photonix/classifiers/base_model.py

@@ -9,14 +9,16 @@
 import logging
 
 import requests
-
-import redis
 from redis_lock import Lock
 
+from photonix.photos.utils.redis import redis_connection
+
+
 graph_cache = {}
 
 logger = logging.getLogger(__name__)
 
+
 class BaseModel:
     def __init__(self, model_dir=None):
         global graph_cache
@@ -50,8 +52,7 @@ def ensure_downloaded(self, lock_name=None):
         if not lock_name:
             lock_name = 'classifier_{}_download'.format(self.name)
 
-        r = redis.Redis(host=os.environ.get('REDIS_HOST', '127.0.0.1'))
-        with Lock(r, lock_name):
+        with Lock(redis_connection, lock_name):
             try:
                 with open(version_file) as f:
                     if f.read().strip() == str(self.version):

photonix/classifiers/face/model.py

@@ -9,14 +9,14 @@
 from django.utils import timezone
 import numpy as np
 from PIL import Image
-import redis
 from redis_lock import Lock
 
 from photonix.classifiers.base_model import BaseModel
 from photonix.classifiers.face.deepface import DeepFace
 from photonix.classifiers.face.mtcnn import MTCNN
 from photonix.classifiers.face.deepface.commons.distance import findEuclideanDistance
 from photonix.classifiers.face.deepface.DeepFace import build_model
+from photonix.photos.utils.redis import redis_connection
 
 
 GRAPH_FILE = os.path.join('face', 'mtcnn_weights.npy')
@@ -42,8 +42,7 @@ def __init__(self, model_dir=None, graph_file=GRAPH_FILE, library_id=None, lock_
 
 
     def load_graph(self, graph_file):
-        r = redis.Redis(host=os.environ.get('REDIS_HOST', '127.0.0.1'))
-        with Lock(r, 'classifier_{}_load_graph'.format(self.name)):
+        with Lock(redis_connection, 'classifier_{}_load_graph'.format(self.name)):
             # Load MTCNN
             mtcnn_graph = None
             mtcnn_key = '{self.graph_cache_key}:mtcnn'
@@ -98,8 +97,7 @@ def find_closest_face_tag_by_ann(self, source_embedding):
             embedding_size = 128  # FaceNet output size
             t = AnnoyIndex(embedding_size, 'euclidean')
             # Ensure ANN index, tag IDs and version files can't be updated while we are reading
-            r = redis.Redis(host=os.environ.get('REDIS_HOST', '127.0.0.1'))
-            with Lock(r, 'face_model_retrain'):
+            with Lock(redis_connection, 'face_model_retrain'):
                 self.reload_retrained_model_version()
                 t.load(str(ann_path))
                 with open(tag_ids_path) as f:
@@ -193,8 +191,7 @@ def retrain_face_similarity_index(self, training_data=None):
         t.build(3)  # Number of random forest trees
 
         # Aquire lock to save ANN, tag IDs and version files atomically
-        r = redis.Redis(host=os.environ.get('REDIS_HOST', '127.0.0.1'))
-        with Lock(r, 'face_model_retrain'):
+        with Lock(redis_connection, 'face_model_retrain'):
             # Save ANN index
             t.save(str(ann_path))
 
@@ -294,8 +291,6 @@ def run_on_photo(photo_id):
         photo.classifier_color_version = getattr(model, 'version', 0)
         photo.save()
 
-    print('Finished')
-
     return photo, results
 
 

photonix/classifiers/object/model.py

@@ -5,14 +5,14 @@
 from django.utils import timezone
 import numpy as np
 from PIL import Image
-import redis
 from redis_lock import Lock
 import tensorflow as tf
 
 from photonix.classifiers.object.utils import label_map_util
 from photonix.classifiers.base_model import BaseModel
+from photonix.photos.utils.redis import redis_connection
+
 
-r = redis.Redis(host=os.environ.get('REDIS_HOST', '127.0.0.1'))
 GRAPH_FILE = os.path.join('object', 'ssd_mobilenet_v2_oid_v4_2018_12_12_frozen_inference_graph.pb')
 LABEL_FILE = os.path.join('object', 'oid_v4_label_map.pbtxt')
 
@@ -33,7 +33,7 @@ def __init__(self, model_dir=None, graph_file=GRAPH_FILE, label_file=LABEL_FILE,
             self.labels = self.load_labels(label_file)
 
     def load_graph(self, graph_file):
-        with Lock(r, 'classifier_{}_load_graph'.format(self.name)):
+        with Lock(redis_connection, 'classifier_{}_load_graph'.format(self.name)):
             if self.graph_cache_key in self.graph_cache:
                 return self.graph_cache[self.graph_cache_key]
 

photonix/classifiers/style/model.py

@@ -4,11 +4,11 @@
 
 import numpy as np
 
-import redis
 from redis_lock import Lock
 import tensorflow as tf
 
 from photonix.classifiers.base_model import BaseModel
+from photonix.photos.utils.redis import redis_connection
 
 
 GRAPH_FILE = os.path.join('style', 'graph.pb')
@@ -34,8 +34,7 @@ def __init__(self, model_dir=None, graph_file=GRAPH_FILE, label_file=LABEL_FILE,
             self.labels = self.load_labels(label_file)
 
     def load_graph(self, graph_file):
-        r = redis.Redis(host=os.environ.get('REDIS_HOST', '127.0.0.1'))
-        with Lock(r, 'classifier_{}_load_graph'.format(self.name)):
+        with Lock(redis_connection, 'classifier_{}_load_graph'.format(self.name)):
             if self.graph_cache_key in self.graph_cache:
                 return self.graph_cache[self.graph_cache_key]
 

photonix/manage.py

@@ -2,6 +2,7 @@
 import os
 import sys
 
+
 if __name__ == "__main__":
     os.environ.setdefault("DJANGO_SETTINGS_MODULE", "photonix.web.settings")
     try:

photonix/photos/management/commands/classification_color_processor.py

@@ -1,10 +1,12 @@
 from django.core.management.base import BaseCommand
+
 # Pre-load the model graphs so it doesn't have to be done for each job
 from photonix.classifiers.color import ColorModel, run_on_photo
 from photonix.photos.utils.classification import ThreadedQueueProcessor
+from photonix.web.utils import logger
 
 
-print('Loading color model')
+logger.debug('Loading color model')
 model = ColorModel()
 
 

photonix/photos/management/commands/classification_event_processor.py

@@ -1,10 +1,12 @@
 from django.core.management.base import BaseCommand
+
 # Pre-load the model graphs so it doesn't have to be done for each job
 from photonix.classifiers.event import EventModel, run_on_photo
 from photonix.photos.utils.classification import ThreadedQueueProcessor
+from photonix.web.utils import logger
 
 
-print('Loading event model')
+logger.debug('Loading event model')
 model = EventModel()
 
 

photonix/photos/management/commands/classification_face_processor.py

@@ -1,7 +1,9 @@
 from django.core.management.base import BaseCommand
+
 # Pre-load the model graphs so it doesn't have to be done for each job
 from photonix.classifiers.face import run_on_photo
 from photonix.photos.utils.classification import ThreadedQueueProcessor
+from photonix.web.utils import logger
 
 
 model = None

photonix/photos/management/commands/classification_location_processor.py

@@ -1,10 +1,12 @@
 from django.core.management.base import BaseCommand
+
 # Pre-load the model graphs so it doesn't have to be done for each job
 from photonix.classifiers.location import LocationModel, run_on_photo
 from photonix.photos.utils.classification import ThreadedQueueProcessor
+from photonix.web.utils import logger
 
 
-print('Loading location model')
+logger.debug('Loading location model')
 model = LocationModel()
 
 

photonix/photos/management/commands/classification_object_processor.py

@@ -1,10 +1,12 @@
 from django.core.management.base import BaseCommand
+
 # Pre-load the model graphs so it doesn't have to be done for each job
 from photonix.classifiers.object import ObjectModel, run_on_photo
 from photonix.photos.utils.classification import ThreadedQueueProcessor
+from photonix.web.utils import logger
 
 
-print('Loading object classification model')
+logger.debug('Loading object classification model')
 model = ObjectModel()