require_relative '../wendelin_client'
require_relative '../dataset_utils'
require_relative '../filelogger'
require 'digest/md5'

module Embulk
  module Input

    class Fifinput < InputPlugin

      Plugin.register_input("fif", self)

      SCHEMA = [
		 {"name"=>"supplier", "type"=>"string"},
		 {"name"=>"data_set", "type"=>"string"},
		 {"name"=>"file", "type"=>"string"},
		 {"name"=>"extension", "type"=>"string"},
		 {"name"=>"data_chunk", "type"=>"string"},
		 {"name"=>"eof", "type"=>"string"},
		 {"name"=>"size", "type"=>"string"},
		 {"name"=>"hash", "type"=>"string"}
	       ]

      def self.status(task, push=FALSE)
	partial_ingestion = @dataset_utils.partialIngestionFileExist()
	staged_changes, untracked_changes = @dataset_utils.getLocalChanges(task['paths'], task['data_set'], staged=TRUE, partial_ingestion=partial_ingestion)
	staged = (not staged_changes.empty?)
	task['paths'] = staged ? staged_changes : untracked_changes
	if task['paths'].empty?
	  puts
	  @logger.info("No changes in dataset directory for dataset '#{@data_set}'.", print=TRUE)
	  puts
	  @logger.abortExecution(error=FALSE)
	end
	status = partial_ingestion ? DatasetUtils::OUTPUT_ADD : ""
	changes = partial_ingestion ? "file" : "change"
	n_changes = push ? task['paths'].length : staged_changes.length + untracked_changes.length
	@logger.info("#{n_changes} #{changes}(s) detected: ", print=TRUE)
	puts
	print_short = staged ? staged_changes.length + untracked_changes.length > 500 : task['paths'].length > 500
	message = staged ? "#{changes}(s) added for ingestion:" : "Files:"
	@dataset_utils.showChangesList(task['paths'], message, print_short, status)
	puts
	if staged and not push and not untracked_changes.empty?
	  @dataset_utils.showChangesList(untracked_changes, "Untracked changes:", print_short, status)
	  puts
	end
	if not untracked_changes.empty? and not staged
	  @logger.info("Remember that you can add specific files/changes for ingestion running 'ebulk add/remove <PATH>'", print=TRUE)
	  puts
	end
	if push and partial_ingestion and not task['data_streams'].empty?
	  self.warnOverwrite(task)
	end
      end

      def self.warnOverwrite(task)
	@logger.info("Checking posible conflicts...", print=TRUE)
	conflicts = @dataset_utils.getRemoteConflicts(task['data_streams'], task['paths'], task['data_set'])
	if not conflicts.empty?
	  puts
	  @logger.warn("Some of your local files already exist in remote dataset.", print=TRUE)
	  if task['always_yes_mode'] != "true"
	    @logger.warn("You may want to cancel this partial ingestion and download the full dataset to make local changes.", print=TRUE)
	  end
	  puts
	  @logger.warn("Current ingestion WILL OVERWRITE the following files in remote dataset:", print=TRUE)
	  if task['always_yes_mode'] != "true"
	    puts "** press key **"
	    option = gets
	  end
	  print_short = conflicts.length > 500
	  @dataset_utils.showChangesList(conflicts, "", print_short, status=DatasetUtils::OVERWRITE)
	else
	  @logger.info("All local files are new.", print=TRUE)
	end
	puts
      end

      def self.transaction(config, &control)
	begin
	  tool_dir = config.param('tool_dir', :string)
	  @logger = LogManager.instance()
	  @logger.setFilename(tool_dir, "ingestion")
	  task = { 'paths' => [] }
	  task['supplier'] = config.param('supplier', :string, default: "default")
	  task['data_set'] = config.param('data_set', :string)
	  task['chunk_size'] = config.param('chunk_size', :float, default: 0) * DatasetUtils::MEGA
	  if task['chunk_size'] == 0
	    task['chunk_size'] = DatasetUtils::CHUNK_SIZE
	  end
	  task['always_yes_mode'] = config.param('always_yes', :string)
	  @data_set = task['data_set']
	  @dataset_utils = DatasetUtils.new("")
	  paths = config.param('path_prefix', :array)
	  paths[0] = @dataset_utils.appendSlashTo(paths[0])
	  task['data_set_directory'] = paths[0]
	  @data_set_directory = task['data_set_directory']
	  @dataset_utils = DatasetUtils.new(@data_set_directory)
	  @status = config.param('status', :string, default: FALSE)
	  @status = @status == "" ? FALSE : @status
	  @dataset_utils.deleteDiscardChangesFile()
	  if @status
	    if not @dataset_utils.partialIngestionFileExist()
	      if not @dataset_utils.reportFileExist()
		puts
		@logger.error("The dataset directory does not contain a valid dataset.", print=TRUE)
		@logger.abortExecution()
	      elsif not @dataset_utils.completedFileExist()
		puts
		@logger.error("There is an interrupted download operation in dataset directory. Please resume the download first.", print=TRUE)
		@logger.abortExecution()
	      end
	    end
	  else
	    user, password = @dataset_utils.getCredentials(tool_dir)
	    @wendelin = WendelinClient.new(config.param('erp5_url', :string), user, password)
	    @new_dataset_description = @dataset_utils.datasetDescriptionPendingFileExist(@data_set, tool_dir)
	    @dataset_utils.setDatasetDescription(@data_set, tool_dir, @wendelin) if @new_dataset_description
	  end
	  @logger.info("Checking local files...", print=TRUE)
	  task['paths'] = @dataset_utils.getLocalPaths(paths)
	  if @status
	    self.status(task)
	    @logger.abortExecution(error=FALSE)
	  end
	  @logger.info("Checking remote dataset...", print=TRUE)
	  data_stream_count = @wendelin.getDataStreamCount(@data_set)
      if data_stream_count["status_code"] != 0
        @logger.error(data_stream_count["error_message"], print=TRUE)
        @logger.abortExecution()
	  end
	  data_stream_dict = @wendelin.getDataStreamList(@data_set, data_stream_count["result"], 1000)
	  if data_stream_dict["status_code"] != 0
	    @logger.error(data_stream_dict["error_message"], print=TRUE)
	    @logger.abortExecution()
	  end
	  task['data_streams'] = data_stream_dict["result"]
	  first_ingestion = task['data_streams'].length == 0
	  if not @dataset_utils.reportFileExist()
	    @dataset_utils.createPartialIngestionFile()
	    @dataset_utils.createFirstIngestionFile() if first_ingestion
	  else
	    if not @dataset_utils.partialIngestionFileExist()
	      @logger.info("Checking local dataset...", print=TRUE)
	      if not @dataset_utils.reportUpToDate(data_stream_dict, @data_set) and task['always_yes_mode'] != "true"
		puts
		@logger.error("Your current dataset is outdated. Please, run a download to update it before ingest your changes.", print=TRUE)
		puts
		@logger.abortExecution(error=FALSE)
	      end
	    end
	  end

	  @logger.info("Supplier: #{task['supplier']}")
	  @logger.info("Dataset name: #{task['data_set']}")
	  @logger.info("Chunk size set in #{task['chunk_size']/DatasetUtils::MEGA}MB")
	  if task['paths'].empty? and not @dataset_utils.reportFileExist() and not @new_dataset_description
	    @logger.error("The dataset directory '#{@data_set_directory}' is empty.", print=TRUE)
	    @logger.error("Could not find any valid file.", print=TRUE)
	    @logger.error("Please make sure your dataset directory contains files for ingestion.", print=TRUE)
	    @logger.abortExecution()
	  end

	  self.status(task, push=TRUE)
	  if task['always_yes_mode'] != "true"
	    @logger.info("Continue with ingestion? (y/n)", print=TRUE)
	    option = gets
	    option = option.chomp
	    if option == "n"
	      @logger.info("Ingestion cancelled by user.", print=TRUE)
	      @logger.abortExecution()
	    end
	  end
	  if not @dataset_utils.reportFileExist()
	    @dataset_utils.createReportFile()
	  end

	  columns = [
	    Column.new(0, "supplier", :string),
	    Column.new(1, "data_set", :string),
	    Column.new(2, "file", :string),
	    Column.new(3, "extension", :string),
	    Column.new(4, "data_chunk", :string),
	    Column.new(5, "eof", :string),
	    Column.new(6, "size", :string),
	    Column.new(7, "hash", :string)
	  ]

	  commit_reports = yield(task, columns, task['paths'].length)
	  done = commit_reports.map{|hash| hash["done"]}.flatten.compact

	  resume(task, columns, task['paths'].length, &control)
	rescue Exception => e
	  @logger.error("An error occurred during operation: " + e.to_s, print=TRUE)
	  @logger.error(e.backtrace)
	  puts "[INFO] For more detailed information, please refer to the log file."
	  @logger.abortExecution()
	end
      end

      def self.resume(task, columns, count, &control)
	@logger = LogManager.instance()
	task_reports = yield(task, columns, count)
	next_config_diff = task_reports.map{|hash| hash[DatasetUtils::RUN_DONE]}.flatten.compact
	@dataset_utils.showTaskReport(next_config_diff)
	element_output = @dataset_utils.partialIngestionFileExist() ? "new file" : "change"
	@logger.info("#{next_config_diff.length} #{element_output}(s) ingested.", print=TRUE)
	if(next_config_diff.length == count)
	  @logger.info("Dataset successfully ingested.", print=TRUE)
	  @wendelin.increaseDatasetVersion(@data_set)
	  @dataset_utils.deleteStagedFile()
	  if @dataset_utils.firstIngestionFileExist()
	    @dataset_utils.createCompletedFile()
	    @dataset_utils.deletePartialIngestionFile()
	    @dataset_utils.deleteFirstIngestionFile()
	  end
	else
	  failed_tasks = task_reports.map{|hash| hash[DatasetUtils::RUN_ERROR] || hash[DatasetUtils::RUN_ABORTED] }.flatten.compact
	  @dataset_utils.showTaskErrors(failed_tasks)
	end
	next_config_diff = {}
	return {DatasetUtils::RUN_DONE => next_config_diff}
      end

      def initialize(task, schema, index, page_builder)
	super
	@supplier = task['supplier']
	@dataset = task['data_set']
	@chunk_size = task['chunk_size']
	@data_set_directory = task['data_set_directory']
	@logger = LogManager.instance()
	@dataset_utils = DatasetUtils.new(@data_set_directory)
      end

      def run
	file_dict = task['paths'][@index]
	@logger.info("Processing file #{file_dict["path"].to_s}", print=TRUE)
	begin
	  path = file_dict["path"]
	  size = file_dict["size"]
	  hash = file_dict["hash"]
	  delete = hash == DatasetUtils::DELETE
	  rename = file_dict["status"] == DatasetUtils::STATUS_RENAMED
	  split = FALSE
	  if size == "" and hash == "" #new file
	    size = File.size(path)
	    hash = @dataset_utils.getHash(path)
	  end
	  new_filename, new_extension, new_reference = @dataset_utils.getPathInfo(file_dict["new_path"], @dataset) if rename
	  filename, extension, reference = @dataset_utils.getPathInfo(path, @dataset)
	  operation = rename ? DatasetUtils::RENAME : DatasetUtils::INGESTION
	  @dataset_utils.saveCurrentOperation(operation, reference, new_reference)
	  resume_split, large_hash = 0, ""
	  if @dataset_utils.splitOperationFileExist(reference)
	    resume_split, large_hash = @dataset_utils.getLastSplitOperation(operation, reference, hash, @chunk_size, large_hash=TRUE)
    end
	  each_chunk(path, filename, extension, size, hash, schema[1..-1].map{|elm| elm.name}, @chunk_size, delete, new_reference, resume_split) do |entry|
	    @dataset_utils.createSplitOperationControlFile(reference) if split
	    large_hash += entry[8]
	    #no need to send large hash to server
	    entry.pop()
	    @page_builder.add(entry)
	    if ! delete && ! rename && entry[5] != ""
	      split = TRUE
	      @dataset_utils.saveSplitOperation(operation, reference, entry[5], hash, @chunk_size, large_hash)
	    end
	  end
	  @page_builder.finish
	  @dataset_utils.deleteSplitOperationFile(reference) if split
	rescue java.lang.OutOfMemoryError
	  @logger.logOutOfMemoryError(path)
	  return_value = DatasetUtils::RUN_ABORTED
	rescue Exception => e
	  @logger.error("An error occurred during file ingestion: " + e.to_s, print=TRUE)
	  @logger.error(e.backtrace)
	  puts "[INFO] For more detailed information, please refer to the log file: " + @logger.getLogPath()
	  return_value = DatasetUtils::RUN_ERROR
	else
	  return_value = DatasetUtils::RUN_DONE
	end
	@dataset_utils.deleteSplitOperationControlFile(reference) if return_value != DatasetUtils::RUN_DONE
	if return_value == DatasetUtils::RUN_DONE
	  if delete
	    if @dataset_utils.reportFileExist()
	      @dataset_utils.deleteFromReport(reference, return_value)
	    end
	  else
	    if @dataset_utils.reportFileExist()
	      @dataset_utils.addToReport(reference, return_value, size, hash, large_hash, task['data_set'], new_reference)
	    end
	  end
	end
	@dataset_utils.removeCurrentOperation()
	return {return_value => path}
      end

      private

      def each_chunk(path, filename, extension, size, hash, fields, chunk_size=DatasetUtils::CHUNK_SIZE, delete=FALSE, new_reference=FALSE, resume_split=0)
	if delete
	  File.delete(path) if File.exist?(path)
	  values = [@supplier, @dataset, filename, extension, "", DatasetUtils::DELETE, "", "", ""]
	  yield(values)
	elsif new_reference
	  File.delete(path) if File.exist?(path)
	  values = [@supplier, @dataset, filename, extension, new_reference, DatasetUtils::RENAME, "", "", ""]
	  yield(values)
	else
	  file_object = File.open(path, "rb")
	  npart = 0
	  next_byte = file_object.read(1)
	  first = TRUE
	  while true
	      data = next_byte
	      if not next_byte
		if first # this means this is an empty file
		  values = [@supplier, @dataset, filename, extension, "", "", size, hash, hash]
		  yield(values)
		end
		break
	      end
	      if resume_split > 0
		@logger.info("Resuming interrupted split ingestion", print=TRUE)
		first = FALSE
		file_object.seek(chunk_size*resume_split+resume_split) # ignore ingested part
		next_byte = file_object.read(1)
		data = next_byte
		npart = resume_split
		resume_split = 0
	      end
	      data += file_object.read(chunk_size)
	      next_byte = file_object.read(1)
	      if not next_byte
		eof = first ? "" : DatasetUtils::EOF
	      else
		npart += 1
		eof = npart.to_s.rjust(3, "0")
	      end
	      content = Base64.encode64(data)
	      chunk_hash = @dataset_utils.getHashFromChunk(data)
	      values = [@supplier, @dataset, filename, extension, content, eof, size, hash, chunk_hash]
	      first = FALSE
	      yield(values)
	  end
	end
      end
    end
  end
end
