Realtime support with Sockets-io (#102)

* Added socket to development envirment

* Realtime image locking

* Image locking fixes (#94)

* Realtime annotation collaberation

* Removed autosave

* Production socket server

* Fixed annotation session changing

* Username duplication errror

* Socket connection toastr

* Navbar backend status

* Fixed mobile navbar text alignment

* Tasks (#103)

* Created task model

* Tasks webpage

* Webpage updates

* Create scan task

* Realtime task progress updates and scan example task

* Formatting

* Created scanning task (#101)

* Added tasks webpage to navbar

* Delete tasks

* Created task javascript model

* Datasets javascript model

* Import task

* Task completion flag

* Fixed coco annotation importer

* Show only log warnings/errors

* Login disabled sockets

* Removed exporting of empty segmentations

* Fixed production envirment

* Added secret key to compose file

* Connection lost warning, & formatting

* Warning and error updates for tasks

* Created admin model

* Created undo model

* Added dataset name to navbar (#88)

Author: jsbroks

app/__init__.py

@@ -1,41 +1,28 @@
+import eventlet
+eventlet.monkey_patch(thread=False)
+
 from flask import Flask
-from werkzeug.contrib.fixers import ProxyFix
 from flask_cors import CORS
-from watchdog.observers import Observer
+from werkzeug.contrib.fixers import ProxyFix
 
-from .image_folder import ImageFolderHandler
-from .api import blueprint as api
-from .config import Config
 from .models import *
-from .authentication import login_manager
+from .config import Config
+from .sockets import socketio
+from .watcher import run_watcher
+from .api import blueprint as api
 from .util import query_util, color_util
+from .authentication import login_manager
 
 import threading
 import requests
 import time
 import os
 
 
-def run_watcher():
-    observer = Observer()
-    observer.schedule(ImageFolderHandler(), Config.DATASET_DIRECTORY, recursive=True)
-    observer.start()
-
-    try:
-        while True:
-            time.sleep(1)
-    except KeyboardInterrupt:
-        observer.stop()
-
-    observer.join()
-
-
 def create_app():
 
-    if os.environ.get("APP_WORKER_ID", "1") == "1" and not Config.TESTING:
-        print("Creating file watcher on PID: {}".format(os.getpid()), flush=True)
-        watcher_thread = threading.Thread(target=run_watcher)
-        watcher_thread.start()
+    if Config.FILE_WATCHER:
+        run_watcher()
 
     flask = Flask(__name__,
                   static_url_path='',
@@ -50,6 +37,11 @@ def create_app():
 
     db.init_app(flask)
     login_manager.init_app(flask)
+    socketio.init_app(flask)
+
+    # Remove all poeple who were annotating when
+    # the server shutdown
+    ImageModel.objects.update(annotating=[])
 
     return flask
 
@@ -60,9 +52,6 @@ def create_app():
 if Config.INITIALIZE_FROM_FILE:
     create_from_json(Config.INITIALIZE_FROM_FILE)
 
-if Config.LOAD_IMAGES_ON_START:
-    ImageModel.load_images(Config.DATASET_DIRECTORY)
-
 
 @app.route('/', defaults={'path': ''})
 @app.route('/<path:path>')
@@ -72,5 +61,3 @@ def index(path):
         return requests.get('http://frontend:8080/{}'.format(path)).text
 
     return app.send_static_file('index.html')
-
-

app/api/__init__.py

@@ -8,6 +8,7 @@
 from .images import api as ns_images
 from .users import api as ns_users
 from .admin import api as ns_admin
+from .tasks import api as ns_tasks
 from .undo import api as ns_undo
 from .info import api as ns_info
 
@@ -33,6 +34,7 @@
 api.add_namespace(ns_categories)
 api.add_namespace(ns_annotator)
 api.add_namespace(ns_datasets)
+api.add_namespace(ns_tasks)
 api.add_namespace(ns_undo)
 api.add_namespace(ns_admin)
 

app/api/datasets.py

@@ -170,7 +170,7 @@ def post(self, dataset_id):
 
 
 @api.route('/data')
-class Dataset(Resource):
+class DatasetData(Resource):
     @api.expect(page_data)
     @login_required
     def get(self):
@@ -234,7 +234,7 @@ def get(self, dataset_id):
             return {'message': 'Directory does not exist.'}, 400
 
         images = ImageModel.objects(dataset_id=dataset_id, path__startswith=directory, deleted=False) \
-            .order_by('file_name').only('id', 'file_name')
+            .order_by('file_name').only('id', 'file_name', 'annotating')
 
         pagination = Pagination(images.count(), limit, page)
         images = query_util.fix_ids(images[pagination.start:pagination.end])
@@ -263,12 +263,11 @@ def get(self, dataset_id):
 
 
 @api.route('/<int:dataset_id>/coco')
-class ImageCoco(Resource):
+class DatasetCoco(Resource):
 
     @login_required
     def get(self, dataset_id):
         """ Returns coco of images and annotations in the dataset """
-
         dataset = current_user.datasets.filter(id=dataset_id).first()
 
         if dataset is None:
@@ -287,16 +286,11 @@ def post(self, dataset_id):
         if dataset is None:
             return {'message': 'Invalid dataset ID'}, 400
 
-        import_id = CocoImporter.import_coco(
-            coco, dataset_id, current_user.username)
-
-        return {
-            "import_id": import_id
-        }
+        return dataset.import_coco(json.load(coco))
 
 
 @api.route('/coco/<int:import_id>')
-class ImageCocoId(Resource):
+class DatasetCocoId(Resource):
 
     @login_required
     def get(self, import_id):
@@ -311,3 +305,17 @@ def get(self, import_id):
             "progress": coco_import.progress,
             "errors": coco_import.errors
         }
+
+
+@api.route('/<int:dataset_id>/scan')
+class DatasetScan(Resource):
+    
+    @login_required
+    def get(self, dataset_id):
+
+        dataset = DatasetModel.objects(id=dataset_id).first()
+        
+        if not dataset:
+            return {'message': 'Invalid dataset ID'}, 400
+        
+        return dataset.scan()

app/api/tasks.py

@@ -0,0 +1,51 @@
+from flask_restplus import Namespace, Resource, reqparse
+from flask_login import login_required, current_user
+
+from ..util import query_util
+from ..config import Config
+from ..models import TaskModel
+
+
+api = Namespace('tasks', description='Task related operations')
+
+
+@api.route('/')
+class Task(Resource):
+    @login_required
+    def get(self):
+        """ Returns all tasks """
+        query = TaskModel.objects.only(
+            'group', 'id', 'name', 'completed', 'progress',
+            'priority', 'creator', 'desciption', 'errors',
+            'warnings'
+        ).all()
+        return query_util.fix_ids(query)
+
+
+@api.route('/<int:task_id>')
+class TaskId(Resource):
+    @login_required
+    def delete(self, task_id):
+        """ Deletes task """
+        task = TaskModel.objects(id=task_id).first()
+
+        if task is None:
+            return {"message": "Invalid task id"}, 400
+
+        if not task.completed:
+            return {"message": "Task is not completed"}, 400
+        
+        task.delete()
+        return {"success": True}
+
+
+@api.route('/<int:task_id>/logs')
+class TaskId(Resource):
+    @login_required
+    def get(self, task_id):
+        """ Deletes task """
+        task = TaskModel.objects(id=task_id).first()
+        if task is None:
+            return {"message": "Invalid task id"}, 400
+        
+        return {'logs': task.logs}

app/api/users.py

@@ -22,6 +22,7 @@
 set_password.add_argument('password', required=True, location='json')
 set_password.add_argument('new_password', required=True, location='json')
 
+from flask_socketio import SocketIO, disconnect
 
 @api.route('/')
 class User(Resource):

app/config.py

@@ -8,18 +8,21 @@ class Config:
     NAME = "COCO Annotator"
     VERSION = get_tag()
 
+    # File Watcher
+    FILE_WATCHER = os.getenv("FILE_WATCHER", False)
+    IGNORE_DIRECTORIES = ["_thumbnail", "_settings"]
+
     # Flask instance
     SWAGGER_UI_JSONEDITOR = True
     MAX_CONTENT_LENGTH = 1 * 1024 * 1024 * 1024  # 1GB
     MONGODB_HOST = os.getenv("MONGODB_HOST", "mongodb://database/flask")
-    SECRET_KEY = os.getenv('SECRET_KEY', '<--- YOUR_SECRET_FORM_KEY --->')
+    SECRET_KEY = os.getenv("SECRET_KEY", "<--- DEFAULT_SECRET_KEY --->")
 
     TESTING = os.getenv("TESTING", False)
 
     # Dataset Options
     DATASET_DIRECTORY = os.getenv("DATASET_DIRECTORY", "/datasets/")
     INITIALIZE_FROM_FILE = os.getenv("INITIALIZE_FROM_FILE")
-    LOAD_IMAGES_ON_START = os.getenv("LOAD_IMAGES_ON_START", False)
 
     # Coco Importer Options
     COCO_IMPORTER_VERBOSE = os.getenv("COCO_IMPORTER_VERBOSE", False)
@@ -30,5 +33,5 @@ class Config:
         os.getenv("COCO_IMPORTER_ANNOTATION_BATCH_SIZE", 1000))
 
     # User Options
-    LOGIN_DISABLED = os.getenv('LOGIN_DISABLED', False)
+    LOGIN_DISABLED = os.getenv("LOGIN_DISABLED", False)
     ALLOW_REGISTRATION = True