import os
import time
import shutil
import re
import tempfile

from queue import Queue
from threading import Thread, Event

from context import get_json_value
from hotfolder_modules.util import handle_exception
from hotfolder_modules.file_util import to_utf8_string, to_utf8_unicode, os_chown, os_chmod, os_remove, write_error
from hotfolder_modules.worker import HotfolderWorker


class HotfolderManager(Thread):

    def __init__(self, easydb_context, batched=True):
        Thread.__init__(self, name='hotfolder-manager')

        # See easydb-server/python/context.py
        self.easydb_context = easydb_context
        self.logger = easydb_context.get_logger('pf.plugin.base.hotfolder')
        self.connection = self.easydb_context.db_connect('hotfolder-manager')
        self.batched = batched
        config = self.easydb_context.get_config(self.connection)
        enabled = get_json_value(config, 'system.hotfolder.enabled')

        # list of files that are explicitly skipped for upload and are later deleted
        self.files_to_delete = ['Thumbs.db']

        if not enabled:
            self.logger.info('hotfolder disabled: stop')
            self.connection.close()
            raise HotfolderManagerStop()

        instance = self.easydb_context.get_instance()['name']
        hotfolder_directory = get_json_value(config, 'system.hotfolder.directory')
        self.delay = get_json_value(config, 'system.hotfolder.delay')
        number_of_workers = get_json_value(config, 'system.hotfolder.number_of_workers')
        self.upload_batch_size = get_json_value(config, 'system.hotfolder.upload_batch_size')
        self.upload_batches = get_json_value(config, 'system.hotfolder.upload_batches')
        self.connection.commit()

        if hotfolder_directory is None:
            self.logger.warn('no directory specified: stop')
            self.connection.close()
            raise HotfolderManagerStop()

        elif not os.path.isdir(hotfolder_directory):
            self.logger.error('directory does not exist: {}'.format(hotfolder_directory))
            self.connection.close()
            raise HotfolderManagerStop()

        root_directory = os.path.join(hotfolder_directory, 'collection')

        if not os.path.isdir(root_directory):
            self.logger.info('creating root directory: {}'.format(root_directory))
            os.mkdir(root_directory)
            os.chmod(root_directory, 0o711)

        self.directory = os.path.join(root_directory, instance)

        if not os.path.isdir(self.directory):
            self.logger.info('creating instance directory: {}'.format(self.directory))
            os.mkdir(self.directory)
            os.chmod(self.directory, 0o711)

        self.stop_event = Event()
        self.exception = None
        self.first_time = True

        if number_of_workers is None:
            number_of_workers = 1

        # self.threaded = number_of_workers > 1 UNTIL API BLOCK IS FIXED
        self.threaded = False

        if self.threaded:
            self.queue = Queue()
            self.workers = [HotfolderWorker(self, i)
                            for i in range(number_of_workers)]

        self.session_cache = {}

    # This Method is called after Server Start.
    # While running the while loop will repeat over and over again

    def run(self):
        try:
            self.logger.info('starting')

            if self.threaded:
                for worker in self.workers:
                    worker.daemon = True
                    worker.start()

            while not self.stop_event.is_set():
                self.session_cache = {}
                self.process_collections()
                self.first_time = False
                time.sleep(self.delay)

            self.logger.info('stopped')

            if self.threaded:
                self.queue.join()

        except Exception as e:
            self.exception = handle_exception(e, self.logger)
        try:
            self.connection.close()
        except Exception as e:
            self.logger.debug('could not close connection: {}'.format(str(e)))

    def stop(self):
        self.stop_event.set()
        try:
            self.connection.close()
        except Exception as e:
            self.logger.debug('could not close connection: {}'.format(str(e)))

    # Grabs all available Collections from PSQL and checks their Hotfolders for files

    def process_collections(self):
        datamodel = self.easydb_context.get_datamodel()

        self.connection.open_txn()
        db_cursor = self.connection.cursor()

        # SQL Query to fetch available Collections
        db_cursor.execute(sql_get_upload_collections)
        self.connection.commit()

        collections = {}
        current_collection = None

        for row in db_cursor.fetchall():
            collection_id = int(row['collection_id'])

            if current_collection is None or current_collection.id != collection_id:
                self.connection.open_txn()
                collection_js = self.easydb_context.get_collection_json(int(row['collection_id']), self.connection.connection_id)
                self.connection.commit()

                try:
                    # Create new Collection Object for every Collection in SQL-Response
                    current_collection = Collection(self.directory, datamodel, collection_js)
                except Exception as e:
                    self.logger.warn('Collection with id {0} seems to not support file upload ({1}). Skipping for now'.format(
                        collection_id, e.message))
                    continue

                collections[current_collection.uuid] = current_collection

        if current_collection is not None:
            collections[current_collection.uuid] = current_collection

        for collection in list(collections.values()):
            # Call Process Function for each Collection
            self.process_collection(collection)

        if os.path.isdir(self.directory):
            for name in os.listdir(self.directory):
                if name not in list(collections.keys()):
                    path = os.path.join(self.directory, name)

                    if os.path.isdir(path):
                        self.logger.info('delete directory {}'.format(path))
                        try:
                            shutil.rmtree(path)
                        except Exception as e:
                            self.logger.error('could not delete directory {}'.format(path))

                    else:
                        self.logger.info('delete file {}'.format(path))
                        os.remove(path)

    # Will be called the first time the Plugin is run.
    # Removes Lock-Files from previous runs

    def remove_locks(self, dir):
        for root, _, files in os.walk(to_utf8_string(dir)):
            for file in files:
                if file.endswith('.lock'):
                    os_remove(self.logger, os.path.join(root, file))

    def get_dir_size(self, dir='.'):
        size = 0
        for root, _, files in os.walk(to_utf8_string(dir)):
            for file in files:
                size += os.path.getsize(os.path.join(root, file))
        return size

    def check_for_file_ops(self, dir='.'):
        try:
            size_1 = self.get_dir_size(dir)
            time.sleep(2)
            size_2 = self.get_dir_size(dir)
            return size_2 != size_1
        except Exception as e:
            self.logger.warn('check_for_file_ops caused the following error: ' + str(e))
            self.log_event(event_name='HOTFOLDER_ERROR', json_info={
                'cause': str(e)
            })
            return True

    def check_write_delete_right(self, dir='.'):
        try:
            tmp_filename = tempfile.mktemp(dir=dir, suffix='.hf_tmp')

            with open(tmp_filename, 'w') as f:
                f.write('tmp')
                f.close()

                if not os.path.isfile(tmp_filename):
                    return False

            os.remove(tmp_filename)

            return True
        except Exception as e:
            self.logger.debug('check_write_delete_right found the following error: ' + str(e))
            self.log_event(event_name='HOTFOLDER_ERROR', json_info={
                'detail': 'Hotfolder can not write/delete files in directory {}; skip directory'.format(dir)
            })
            return False

    def upload_batch(self, collection, objects):
        if self.upload_batches:
            # upload a batch of objects in an API Call (faster)
            self.logger.info('Uploading Batch of {} objects'.format(len(objects)))
            self.process_object_list(objects, collection)

        else:
            # upload every object in a single API Call
            for obj in objects:
                obj.lock(self.logger)
                if self.threaded:
                    self.queue.put(obj)
                else:
                    self.process_object(obj)

    # For every Collection with Hotfolder a Directory is created (if there isnt one already)
    # Then checks Directory for Files to upload
    def process_collection(self, collection):

        self.logger.debug('process {}'.format(collection))

        if self.check_for_file_ops(dir=collection.directory):
            self.logger.info('Skipping {}. File Operations still underway'.format(collection))
            return

        # create collection directory
        if not os.path.isdir(collection.directory):
            self.logger.info('creating directory \'{}\' for {}'.format(collection.directory, collection))
            os.makedirs(collection.directory)
            os.chmod(collection.directory, 0o777)

        # Set startpath for naming files relatively to current collection directory
        startpath = to_utf8_string(collection.directory + '/')

        if self.first_time:
            self.remove_locks(startpath)

        if os.listdir(startpath) == []:
            # Skip Collection if its empty
            return

        objects = []

        for root, _, files in os.walk(startpath):

            # Ignoring systen-directpries and those created by webdav
            split = root.split('/')

            # Ignore all Directories with '.' as first char in name
            if split[-1].startswith('.'):
                self.logger.debug('Directories starting with \'.\' will be ignored ({})'.format(split[-1]))
                continue

            # make sure that files in the folder can be written/deleted before working on the collection
            if not self.check_write_delete_right(dir=root):
                self.logger.warn('Skipping directory: no write/delete rights in collection directory {}'.format(
                    to_utf8_unicode(root)))
                continue

            usable_files = self.get_usable_files(files, root)

            # Go through all usable files
            for x in range(len(usable_files)):

                self.logger.debug('usable_files[{0}]: \'{1}\''.format(x, usable_files[x]))

                if usable_files[x][1] == False:
                    continue

                # File-Path relative to Top-Level directory of this collection
                filename = os.path.join(root.replace(startpath, ''), usable_files[x][0])

                # basename: name without file-type-extension
                basename = '.'.join(usable_files[x][0].split('.')[:-1])
                stripped_base_name = basename
                separator = None

                if collection.recognize_series:
                    if '_nested:' in collection.eas_column_path:
                        separator = self.get_seperator(basename)
                        if separator:
                            stripped_base_name = separator.join(basename.split(separator)[:-1])

                obj = Object(collection, stripped_base_name)
                obj.add_file(basename, filename)

                # Setting False will prevent the file from being uploaded again
                usable_files[x][1] = False

                # Go through all files in the same directory again to find Serial-Pictures
                # (eg. Pic-1...Pic-22) and other versions (i.e picture.jpg and picture.raw)
                for y in range(len(usable_files)):

                    if usable_files[y][1] == False:
                        continue

                    other_filename = os.path.join(
                        root.replace(startpath, ''), usable_files[y][0])
                    other_basename = '.'.join(usable_files[y][0].split('.')[:-1])

                    # Versions
                    if collection.recognize_version:
                        if other_basename == basename and other_filename != filename:
                            obj.add_file(other_basename, other_filename)
                            usable_files[y][1] = False
                            continue

                    # Serial Pictures (Only if a Objecttype with nested field is selected in the Frontend)
                    if collection.recognize_series:
                        if '_nested:' in collection.eas_column_path and separator is not None:
                            if self.is_serial_picture(separator, stripped_base_name, other_basename):
                                obj.add_file(other_basename, other_filename)
                                usable_files[y][1] = False

                objects.append(obj)

                if len(objects) >= self.upload_batch_size:
                    self.logger.debug('Maximum Number of files per Upload Round reached [{}/{}]'.format(len(objects), self.upload_batch_size))
                    self.upload_batch(collection, objects)
                    objects = []

        if len(objects) > 0:
            self.upload_batch(collection, objects)

        self.remove_empty_directories(startpath)

    # Directly calling process function of worker.
    # This is a workaround while threading is not working.
    # TODO: If fixed: Implement Queue

    def process_object(self, obj):
        worker = HotfolderWorker(self, 0)
        worker.process(obj)

    # Is called if multithreading is deactivated (see init)

    def process_object_list(self, objs, collection):
        worker = HotfolderWorker(self, 0)
        worker.process_list(objs, collection)

    # goes through files in list files with top-level directory root and returns a list of Arrays
    # [filename, bool] boolean will later be used to mark already uplaoded files

    def get_usable_files(self, files, root):
        usable_files = []
        files_with_missing_rights = []
        files.sort()

        for file in files:
            abs_file = os.path.join(root, file)

            error_file_exists = os.path.isfile(abs_file + '.error.txt') or file.endswith('.error.txt')
            lock_file_exists = os.path.isfile(abs_file + '.lock') or file.endswith('.lock')

            # make sure that the filepath is in valid UTF8 format
            try:
                to_utf8_unicode(file)
            except Exception as e:
                if error_file_exists or lock_file_exists:
                    continue

                f = to_utf8_unicode(file, replace_errors=True)
                message = 'File {0} can not be imported: invalid filename: {1}'.format(f, str(e))
                self.logger.debug(message)

                self.log_event(event_name='HOTFOLDER_ERROR', json_info={
                    'invalid_filename': f,
                    'error': str(e)
                })

                write_error(abs_file + '.error.txt', to_utf8_string(message))

                continue

            if file.startswith('.'):
                self.logger.debug('File {} starts with \'.\', skip'.format(to_utf8_unicode(file)))
                continue

            if error_file_exists:
                self.logger.debug('File {} is skipped (error exists)'.format(to_utf8_unicode(file)))
                continue

            if lock_file_exists:
                self.logger.debug('File {} is skipped (lock exists)'.format(to_utf8_unicode(file)))
                continue

            if file in self.files_to_delete:
                self.logger.debug('File {} is skipped (to delete)'.format(to_utf8_unicode(file)))
                continue

            stinfo = os.stat(abs_file)
            if stinfo.st_size == 0:
                self.logger.debug('File {} is skipped (empty)'.format(to_utf8_unicode(file)))
                continue

            usable_files.append([file, True])

        if len(files_with_missing_rights) > 0:
            self.log_event(event_name='HOTFOLDER_ERROR', json_info={
                'missing_write_permission': files_with_missing_rights
            })

        return usable_files

    # Serial Pictures can either use ' ', '-', '_' for appending numbers to file-name.
    # Regex-Matching to find out
    @classmethod
    def get_seperator(cls, basename):

        for separator in ['_', '-', ' ']:
            pattern = re.compile(r'.*[{}][0-9]*$'.format(separator))

            if pattern.match(basename):
                return separator

        return None

    @classmethod
    def basename_without_serial(cls, basename, separator):
        if separator is None:
            return basename
        parts = basename.split(separator)
        return separator.join(parts[:-1])

    # Check wether file is part of series

    @classmethod
    def is_serial_picture(cls, separator, stripped_basename, other_basename):
        pattern = re.compile(r'.*[{}][0-9]*$'.format(separator))

        if pattern.match(other_basename):
            stripped_other_basename = separator.join(other_basename.split(separator)[:-1])
            if stripped_other_basename == stripped_basename:
                return True
        else:
            return False

    # All empty directories must be deleted after Upload
    def remove_empty_directories(self, root_dir):

        for root, dirs, files in os.walk(to_utf8_string(root_dir)):

            # do not remove the root directory itself
            if root == root_dir:
                continue

            # check if all sub directories are skipped (must start with '.')
            delete_subdirs = True
            for d in dirs:
                if not d.startswith('.'):
                    delete_subdirs = False
                    break

            # Delete empty directories / directories with only skipped sub directories
            if delete_subdirs:
                trash = True

                for file in files:
                    if not file.startswith('.') and not file in self.files_to_delete:
                        # keep the directory if it contains any file
                        # that does not start with '.' and is also none of the files to be deleted
                        trash = False
                        break

                if trash:
                    try:
                        self.logger.info('delete directory {}'.format(to_utf8_unicode(root)))
                        shutil.rmtree(root)
                    except Exception as e:
                        self.logger.warn('Could not remove directory {}: {}'.format(
                            to_utf8_unicode(root), str(e)))

    def log_event(self, event_name='HOTFOLDER_SUCCESS', json_info=None):
        db_conn = self.easydb_context.db_connect('hotfolder-manager')
        try:
            self.easydb_context.log_event(
                db_conn,
                event_name,
                json_info
            )
            db_conn.commit()
        except Exception as e:
            self.logger.warn('Could not log event {}: {}'.format(event_name, str(e)))
        finally:
            db_conn.close()


class Collection:

    def __init__(self, hotfolder_directory, datamodel, collection_js):

        self.POLICY_CREATE_VERSION = 'create_version'
        self.POLICY_CREATE_VERSION_PREFERRED = 'create_version_preferred'
        self.POLICY_REPLACE = 'replace'
        self.POLICY_REFUSE = 'refuse'

        self.MODE_INSERT = 'insert'
        self.MODE_UPDATE = 'update'
        self.MODE_UPSERT = 'upsert'

        self.datamodel = datamodel
        self.id = collection_js['collection']['_id']
        self.uuid = collection_js['collection']['uuid']
        self.user_id = collection_js['_owner']['user']['_id']
        create_object = collection_js['_create_object_compiled']
        self.objecttype = create_object['objecttype']
        self.mask_id = create_object['mask_id']
        self.pool_id = create_object['pool_id']
        self.linked_pool_id = create_object['linked_pool_id']
        self.linked_object_pools = create_object.get('linked_object_pools', {})
        self.mapping = create_object['mapping']
        self.eas_column_path = create_object['eas_field']
        self.update_mode = create_object['update_mode'] if create_object['update_mode'] != '' else None
        self.update_policy = create_object['update_policy'] if create_object['update_policy'] != '' else None
        self.update_search_field = create_object['update_search_field'] if create_object['update_search_field'] != '' else None
        self.recognize_series = create_object['recognize_series'] if 'recognize_series' in create_object else True
        self.recognize_version = create_object['recognize_version'] if 'recognize_version' in create_object else False
        self.directory = os.path.join(hotfolder_directory, self.uuid)

        for mask in datamodel['maskset']['masks']:
            if mask['mask_id'] == self.mask_id:
                self.mask_name = mask['name']
                break
        else:
            raise Exception('mask not found')

        self.tags = []

        if create_object['tags']:
            for tag in create_object['tags']:
                self.tags.append(tag['_id'])

    def __str__(self):
        return 'collection [' + ', '.join(['%s: {}' % k for k in [
            'uuid',
            'objecttype',
            'mask_id', 'mask',
            'pool',
            'linked_pool',
            'linked_object_pools',
            'column',
            'update_mode', 'update_policy', 'update_search_field',
            'mapping',
            'owner',
            'id',
            'tags',
            'recognize_series', 'recognize_version'
        ]]).format(
            self.uuid,
            self.objecttype,
            self.mask_id, self.mask_name,
            self.pool_id,
            self.linked_pool_id,
            self.linked_object_pools,
            self.eas_column_path,
            self.update_mode, self.update_policy, self.update_search_field,
            self.mapping,
            self.user_id,
            self.id,
            '[' + ','.join(list(map(str, self.tags))) + ']',
            self.recognize_series, self.recognize_version) + ']'


# Object can Hold Multiple Files. Each Object will be an Object in easydb
class Object:

    def __init__(self, collection, name):
        self.collection = collection
        self.name = name
        self.files = {}

    def add_file(self, basename, filename):
        if basename in list(self.files.keys()):
            self.files[basename].append(File(self, filename))
        else:
            self.files[basename] = [File(self, filename)]

    def lock(self, logger):
        for f in list(self.files.values()):
            for v in f:
                v.lock(logger)

    def unlock(self, logger):
        for f in list(self.files.values()):
            for v in f:
                v.unlock(logger)

    def remove(self, logger):
        for f in list(self.files.values()):
            for v in f:
                v.remove(logger)

    def __str__(self):
        return '{}-object[{}]'.format(self.collection, to_utf8_unicode(self.name))


# Every Object must hold one or more files
class File:

    def __init__(self, obj, name):
        self.obj = obj
        self.name = name

    def get_path(self):
        path = os.path.join(self.obj.collection.directory, to_utf8_unicode(self.name))
        return to_utf8_string(path)

    def get_size(self):
        return os.path.getsize(self.get_path())

    # Files are Locked, from when an Object is created until upload is finished

    def lock(self, logger):
        path = self.get_path()
        os_chown(logger, path)
        os_chmod(logger, path, 0o644)
        lock_file = '{}.lock'.format(self.get_path())
        open(lock_file, 'w').close()
        os_chmod(logger, lock_file, 0o644)

    def unlock(self, logger):
        path = self.get_path()
        lock_file = '{}.lock'.format(path)
        os_remove(logger, lock_file)
        os_chmod(logger, path, 0o666)

    def remove(self, logger):
        path = self.get_path()
        os_remove(logger, path)
        lock_file = '{}.lock'.format(path)
        os_remove(logger, lock_file)

    def get_error_file(self):
        return '{}.error.txt'.format(os.path.join(to_utf8_string(self.obj.collection.directory), self.name))

    def __str__(self):
        return 'file[{}]'.format(to_utf8_unicode(self.name))


class HotfolderManagerStop(Exception):
    pass


# Fetch Collections from Database
sql_get_upload_collections = """\
SELECT
    c."ez_collection:id" AS collection_id
FROM ez_collection c
WHERE "create_object_objecttype:id" IS NOT NULL
AND "create_object_mask:id" IS NOT NULL
AND ":owner:ez_user:id" IS NOT NULL;
"""
