require_relative 'filelogger'
require 'digest/md5'
require 'fileutils'
require 'open-uri'

# class that handles dataset tasks report
class DatasetUtils

  EOF = "EOF"
  NONE_EXT = "none"
  REFERENCE_SEPARATOR = "/"

  DATASET_REPORT_FILE = ".dataset-task-report"
  DATASET_TEMP_REPORT_FILE = ".temp-dataset-task-report"
  DATASET_COMPLETED_FILE = ".dataset-completed"
  RESUME_OPERATION_FILE = ".resume-operation"
  PARTIAL_INGESTION_FILE = ".partial-ingestion"
  STAGED_FILE = ".staged"
  DISCARD_CHANGES_FILE = ".discard-changes"
  SPLIT_FILE = ".split-operation"
  SPLIT_CONTROL_FILE = ".control-split-operation"
  FIRST_INGESTION_FILE = ".first-ingestion"
  CREDENTIALS_FILE = ".credentials"
  CONFIG_FILE = ".config"
  DESCRIPTION_DATASET_PATH_PREFIX=".dataset-description"
  DESCRIPTION_DATASET_PENDING_PATH_PREFIX=".dataset-description-pending"

  RUN_DONE = "done"
  RUN_ERROR = "error"
  RUN_ABORTED = "aborted"
  DELETE = "DELETE"
  RENAME = "RENAME"
  INGESTION = "ingestion"
  DOWNLOAD = "download"
  ADD = "add"
  REMOVE = "remove"
  STATUS_NEW = "new"
  STATUS_RENAMED = "renamed"
  STATUS_MODIFIED = "modified"
  STATUS_DELETED = "deleted"
  STAGE_ADD="add"
  STAGE_REMOVE="remove"
  STAGE_RESET="reset"

  OPTION_UPDATE = "U"
  OPTION_RESUME = "R"
  OPTION_DOWNLOAD = "D"
  OPTION_ABORT = "A"

  OUTPUT_NEW = "new:         "
  OUTPUT_ADD = "add:         "
  OVERWRITE = "overwrite:   "
  OUTPUT_MODIFIED = "modified:    "
  OUTPUT_DELETED = "deleted:     "
  OUTPUT_RENAMED = "renamed:     "

  MEGA = 1048576
  CHUNK_SIZE = 52428800 #50MB
  RECORD_SEPARATOR = ";"
  DATE_FORMAT = "%Y-%m-%d-%H-%M-%S"

  def initialize(data_set_directory, url=FALSE)
    if url
      begin
        uri = URI(URI.escape(url + "/ERP5Site_getIngestionConstantsJson"))
        res = open(uri).read
        json = JSON.parse(res)
        @EOF = json['split_end_suffix']
        @NONE_EXT = json['none_extension']
        @REFERENCE_SEPARATOR = json['reference_separator']
      rescue
        @EOF = EOF
        @NONE_EXT = NONE_EXT
        @REFERENCE_SEPARATOR = REFERENCE_SEPARATOR
      end
    end
    @data_set_directory = appendSlashTo(data_set_directory)
    @logger = LogManager.instance()
    @task_report_file = @data_set_directory + DATASET_REPORT_FILE
    @temp_report_file = @data_set_directory + DATASET_TEMP_REPORT_FILE
    @completed_file = @data_set_directory + DATASET_COMPLETED_FILE
    @resume_operation_file = @data_set_directory + RESUME_OPERATION_FILE
    @partial_ingestion_file = @data_set_directory + PARTIAL_INGESTION_FILE
    @first_ingestion_file = @data_set_directory + FIRST_INGESTION_FILE
    @staged_file = @data_set_directory + STAGED_FILE
    @discard_changes_file = @data_set_directory + DISCARD_CHANGES_FILE
    @split_file = @data_set_directory + SPLIT_FILE
    @split_control_file = @data_set_directory + SPLIT_CONTROL_FILE
  end

  def getLocalPaths(paths)
    return paths.map {|path|
		      next [] unless Dir.exist?(path)
		      Dir[(path + '/**/*').gsub! '//', '/']
		     }.flatten.select{ |file| File.file?(file) }
  end

  def deleteDatasetDescriptionPendingFile(data_set, tool_dir)
    description_pending_path = appendSlashTo(tool_dir) + DESCRIPTION_DATASET_PENDING_PATH_PREFIX + "-" + data_set
    File.delete(description_pending_path) if File.exist?(description_pending_path)
  end

  def datasetDescriptionPendingFileExist(data_set, tool_dir)
    description_pending_path = appendSlashTo(tool_dir) + DESCRIPTION_DATASET_PENDING_PATH_PREFIX + "-" + data_set
    return File.exist?(description_pending_path)
  end

  def getDatasetDescription(data_set, tool_dir)
    description_path = appendSlashTo(tool_dir) + DESCRIPTION_DATASET_PATH_PREFIX + "-" + data_set
    description_text = ""
    File.readlines(description_path).each do |line|
      if line != "" && ! line.start_with?("#")
	description_text += line
      end
    end
    return description_text
  end

  def setDatasetDescription(data_set, tool_dir, client)
    begin
      dataset_description_text = getDatasetDescription(data_set, tool_dir)
      if dataset_description_text != ""
        client.setDatasetDescription(data_set, dataset_description_text)
        deleteDatasetDescriptionPendingFile(data_set, tool_dir)
        @logger.info("Continue with ingestion? (y/n)", print=TRUE)
        option = gets
        option = option.chomp
        if option == "n"
	  @logger.info("Ingestion cancelled by user.", print=TRUE)
	  @logger.abortExecution()
        end
      end
    rescue Exception => e
      @logger.error("An error occurred in DatasetUtils method 'setDatasetDescription':" + e.to_s)
      @logger.error(e.backtrace)
    end
  end

  def getCredentials(tool_dir)
    # using logger because it's a singleton
    # TODO: refactor datase_utils to be a singleton and keep credentials in it
    user, password = @logger.getCredentials()
    if user.nil?
      credential_path = appendSlashTo(tool_dir) + CREDENTIALS_FILE
      if File.exist?(credential_path)
        credentials = File.open(credential_path).read.chomp.split(RECORD_SEPARATOR)
        user = credentials[0]
        password = credentials[1]
        @logger.info("Using stored credentials for user '#{user}'", print=TRUE)
      else
        puts
        puts "Please, enter your ebulk user and password:"
        puts
        puts "** If you don't have a user, please feel free to register in the web site **"
        puts
        @logger.info("Remember that you can store your credentials for automatic authentication by running 'ebulk store-credentials'", print=TRUE)
        puts
        print "User:"
        user = gets.chomp
        if not /^[A-Za-z][-A-Za-z0-9_.@]*$/.match(user)
          puts
          @logger.error("Invalid user name. Did enter an invalid character?", print=TRUE)
          @logger.info("Please enter a valid user name.", print=TRUE)
          @logger.abortExecution(error=FALSE)
        end
        print "Password:"
        password = STDIN.noecho(&:gets).chomp
        puts
        puts
        @logger.info("Remember that you can store your credentials for automatic authentication by running 'ebulk store-credentials'", print=TRUE)
        puts
        sleep 1
      end
      @logger.setCredentials(user, password)
    end
    return user, password
  end

  def getLocalFiles(remove=nil)
    local_files = {}
    begin
      File.readlines(@task_report_file).each do |line|
	record = line.split(RECORD_SEPARATOR)
	if record[1].chomp == RUN_DONE
	  if (remove.nil?) || (remove != record[0])
	    local_files[record[0]] = {"size" => record[2].chomp, "hash" => record[3].chomp,
				      "status" => record[1].chomp, "modification_date" => record[4].chomp, "large_hash" => record[5].chomp }
	  end
	end
      end
    rescue Exception => e
      @logger.error("An error occurred in DatasetUtils method 'getLocalFiles':" + e.to_s)
      @logger.error(e.backtrace)
    end
    return local_files
  end

  def saveReport(local_files)
    begin
      File.delete(@temp_report_file) if File.exist?(@temp_report_file)
      if local_files.empty?
	File.open(@temp_report_file, 'w') {}
      else
	local_files.each do |key, array|
	  record = [key, array["status"], array["size"].to_s, array["hash"], array["modification_date"], array["large_hash"]].join(RECORD_SEPARATOR)
	  file = File.open(@temp_report_file, "ab")
	  file.puts(record)
	  file.close()
	end
      end
      FileUtils.cp_r(@temp_report_file, @task_report_file, :remove_destination => true)
    rescue Exception => e
      @logger.error("An error occurred in DatasetUtils method 'saveReport':" + e.to_s)
      @logger.error(e.backtrace)
    end
  end

  def removeCurrentOperation()
    if File.exist?(@resume_operation_file)
      File.delete(@resume_operation_file)
    end
  end

  def saveCurrentOperation(operation, reference, new_reference)
    if File.exist?(@resume_operation_file)
      File.delete(@resume_operation_file)
    end
    record = new_reference ? [operation, reference, new_reference].join(RECORD_SEPARATOR) : [operation, reference].join(RECORD_SEPARATOR)
    file = File.open(@resume_operation_file, "w")
    file.puts(record)
    file.close()
  end

  def getConfiguration(action, tool_dir)
    config_path = appendSlashTo(tool_dir) + CONFIG_FILE
    if File.exist?(config_path)
      config = File.open(config_path).read.chomp.split(RECORD_SEPARATOR)
      if action == OPTION_RESUME
        return config[0]
      else
        return config[1]
      end
      return OPTION_ABORT
    end
    return OPTION_ABORT
  end

  def reportUpToDate(data_stream_dict, data_set)
    begin
      # directory never downloaded -new or used for partial ingestions-
      return TRUE if not reportFileExist() and not completedFileExist()
      # download not finished
      return FALSE if reportFileExist() and not completedFileExist()
      return FALSE if data_stream_dict["status_code"] == 2
      return TRUE if data_stream_dict["status_code"] != 0
      return TRUE if data_stream_dict["result"].empty?
      changes = getRemoteChangedDataStreams(data_stream_dict["result"], data_set)
      if changes.empty?
	return TRUE
      elsif changes.length == 1
	# check if the unique detected change corresponds to an interrupted ingestion
	if File.exist?(@resume_operation_file)
	  operation=File.open(@resume_operation_file).read.chomp.split(RECORD_SEPARATOR)
	  if operation[1] == changes[0]["reference"]
	    @logger.info("File '#{operation[1]}' was detected as a change in remote dataset, but it is a false positive.")
	    @logger.info("That file operation ingestion was interrupted right after successfully ingest it but local report is outdated.")
	    @logger.info("Dataset is up to date, file operation was reported as ingested in local report.")
	    new_reference = operation[0] == RENAME ? changes[0]["new_reference"] : FALSE
	    if new_reference
	      file_path = referenceToPath(new_reference, @data_set_directory, data_set)
	    else
	      file_path = referenceToPath(changes[0]["reference"], @data_set_directory, data_set)
	    end
	    size = changes[0]["full-size"]
	    large_hash = changes[0]["large-hash"]
	    hash = getHash(file_path).to_s
	    addToReport(changes[0]["reference"], RUN_DONE, size, hash, large_hash, data_set, new_reference)
	    File.delete(@resume_operation_file)
	    return TRUE
	  end
	end
      end
      return FALSE
    rescue Exception => e
      @logger.error("An error occurred in DatasetUtils method 'reportUpToDate':" + e.to_s)
      @logger.error(e.backtrace)
      return FALSE
    end
  end

  def showChanges(changes, status)
    changes.each do |change|
      if status != ""
	status_output = status
      elsif change["status"] == STATUS_NEW
	status_output = OUTPUT_NEW
      elsif change["status"] == STATUS_MODIFIED
	status_output = OUTPUT_MODIFIED
      elsif change["status"] == STATUS_DELETED
	status_output = OUTPUT_DELETED
      elsif change["status"] == STATUS_RENAMED
	status_output = OUTPUT_RENAMED
      else
	status_output = "change:   "
      end
      path = status != OVERWRITE ? change["path"] : change
      new_path = change["status"] == STATUS_RENAMED ? " --> #{change["new_path"]}" : ""
      @logger.info("   #{status_output}#{path}#{new_path}", print=TRUE)
    end
  end

  def showChangesList(changes, message, print_short, status="")
    if not changes.empty?
      if message and message != ""
	@logger.info(message, print=TRUE)
      end
      if print_short and changes.length > 200
	limit = changes.length > 300 ? 100 : changes.length/3
	showChanges(changes[0, limit], status)
	puts "....."
	showChanges(changes[changes.length-limit, changes.length-1], status)
      else
	showChanges(changes, status)
      end
    end
  end

  def showTaskReport(task_reports)
    return if task_reports.empty?
    @logger.info("Reports:", print=TRUE)
    if task_reports.length > 15
      @logger.info(task_reports[0, 5], print=TRUE)
      puts "....."
      @logger.info(task_reports[task_reports.length-5, task_reports.length-1], print=TRUE)
      @logger.info("Full task report:")
      @logger.info(task_reports)
    else
      @logger.info(task_reports, print=TRUE)
    end
  end

  def showTaskErrors(failed_tasks, storage=FALSE)
    puts
    @logger.error("The following files could not be processed. Please check the details in the log file: " + @logger.getLogPath(), print=TRUE)
    if failed_tasks.length > 15
      @logger.error(failed_tasks[0, 5], print=TRUE)
      puts "....."
      @logger.error(failed_tasks[failed_tasks.length-5, failed_tasks.length-1], print=TRUE)
    else
      @logger.error(failed_tasks, print=TRUE)
    end
    if storage
      puts
      @logger.info("You can retry the operation for those files by running your command again with parameter --restart .", print=TRUE)
    else
      @logger.info("You can retry the operation for those files.", print=TRUE)
    end
    puts
  end

  def splitOperationControlFileExist(reference)
    file_reference = reference.gsub('/', '__')
    return File.exist?(@split_control_file + "__" + file_reference)
  end

  def createSplitOperationControlFile(reference)
    file_reference = reference.gsub('/', '__')
    File.open(@split_control_file + "__" + file_reference, 'w') {}
  end

  def deleteSplitOperationControlFile(reference=nil)
    if reference.nil?
      Dir.glob("#{@split_control_file}*").each { |file| File.delete(file) }
    else
      file_reference = reference.gsub('/', '__')
      File.delete(@split_control_file + "__" + file_reference) if File.exist?(@split_control_file + "__" + file_reference)
    end
  end

  def deleteSplitOperationFile(reference=nil)
    if reference.nil?
      Dir.glob("#{@split_file}*").each { |file| File.delete(file) }
    else
      file_reference = reference.gsub('/', '__')
      File.delete(@split_file + "__" + file_reference) if File.exist?(@split_file + "__" + file_reference)
    end
  end

  def saveSplitOperation(operation, reference, eof, hash, chunk_size, large_hash="NONE")
    file_reference = reference.gsub('/', '__')
    record = [operation, reference, eof, hash, Integer(chunk_size), large_hash].join(RECORD_SEPARATOR)
    File.open(@split_file + "__" + file_reference, 'w') { |file| file.puts(record) }
    deleteSplitOperationControlFile(reference)
  end

  def splitOperationFileExist(reference)
    file_reference = reference.gsub('/', '__')
    return File.exist?(@split_file + "__" + file_reference)
  end

  def getLastSplitOperation(operation, reference, hash, chunk_size, large_hash=FALSE)
    # returns last split operation index (and partial large hash if large_hash is TRUE)
    file_reference = reference.gsub('/', '__')
    if not File.exist?(@split_file + "__" + file_reference)
      if large_hash
        return 0, ""
      else
        return 0
      end
    end
    record = File.open(@split_file + "__" + file_reference).read.chomp.split(RECORD_SEPARATOR)
    if record[0] == operation && record[1] == reference && record[3] == hash && record[4] == Integer(chunk_size).to_s && record[2] != EOF
      # discard if user interrupted (ctrl+c) the operation
      if splitOperationControlFileExist(file_reference)
	@logger.warn("Previous split operation attempt for file #{reference} was interrupt by user (aborted tool execution), it will be restarted.", print=TRUE)
	deleteSplitOperationFile(file_reference)
	deleteSplitOperationControlFile(file_reference)
	if large_hash
    return 0, ""
  else
    return 0
  end
      end
      createSplitOperationControlFile(file_reference)
      if large_hash
        return record[2].to_i, record[5]
      else
        return record[2].to_i
      end
    end
    if large_hash
      return 0, ""
    else
      return 0
    end
  rescue Exception => e
    @logger.error("An error occurred in getLastSplitOperation method:" + e.to_s)
    @logger.error(e.backtrace)
    if large_hash
      return 0, ""
    else
      return 0
    end
  end

  def deleteDiscardChangesFile()
    File.delete(@discard_changes_file) if File.exist?(@discard_changes_file)
  end

  def discardChangesFileExist()
    return File.exist?(@discard_changes_file)
  end

  def deleteCompletedFile()
    File.delete(@completed_file) if File.exist?(@completed_file)
  end

  def createCompletedFile()
    File.open(@completed_file, 'w') {}
  end

  def completedFileExist()
    return File.exist?(@completed_file)
  end

  def createReportFile()
    File.open(@task_report_file, 'w') {}
  end

  def reportFileExist()
    return TRUE if File.exist?(@task_report_file)
    if File.exist?(@temp_report_file)
      FileUtils.cp_r(@temp_report_file, @task_report_file)
      return TRUE
    else
      return FALSE
    end
  end

  def deleteFirstIngestionFile()
    File.delete(@first_ingestion_file) if File.exist?(@first_ingestion_file)
  end

  def createFirstIngestionFile()
    File.open(@first_ingestion_file, 'w') {}
  end

  def firstIngestionFileExist()
    return File.exist?(@first_ingestion_file)
  end

  def deletePartialIngestionFile()
    File.delete(@partial_ingestion_file) if File.exist?(@partial_ingestion_file)
  end

  def createPartialIngestionFile()
    File.open(@partial_ingestion_file, 'w') {}
  end

  def partialIngestionFileExist()
    return File.exist?(@partial_ingestion_file)
  end

  def stagedFileExist()
    File.exist?(@staged_file) or File.zero?(@staged_file)
  end

  def appendSlashTo(element)
    return element.end_with?("/") ? element : element + "/"
  end

  def referenceToPath(reference, data_set_directory, dataset)
    data_set = appendSlashTo(dataset)
    file_path = data_set_directory + reference.reverse.sub(REFERENCE_SEPARATOR.reverse, ".".reverse).reverse.sub(data_set, "")
    file_path = file_path[0...-5] if file_path.end_with?(".#{NONE_EXT}")
    return file_path
  end

  def getPathInfo(path, dataset)
    extension = File.extname path
    if path.start_with?(@data_set_directory)
      filename = path.sub(@data_set_directory, "")
      filename = filename.reverse.sub(extension.reverse, "").reverse
    end
    extension.gsub! '.', ''
    extension = extension == "" ? NONE_EXT : extension
    reference = path.sub(@data_set_directory, appendSlashTo(dataset))
    if extension != NONE_EXT
      old_pattern = filename + '.' + extension
      new_pattern = filename + REFERENCE_SEPARATOR + extension
      reference = reference.reverse.sub(old_pattern.reverse, new_pattern.reverse).reverse
    else
      reference += REFERENCE_SEPARATOR + extension
    end
    return filename, extension, reference
  end

  def addToReport(reference, status, size, hash, large_hash, data_set, new_reference=FALSE)
    local_files = {}
    begin
      file_path = referenceToPath(reference, @data_set_directory, data_set)
      modification_date = File.exist?(file_path) ? File.mtime(file_path).strftime(DATE_FORMAT) : "not-modification-date"
      if not reportFileExist()
	File.open(@task_report_file, 'w') {}
      end
      new_file = TRUE
      File.readlines(@task_report_file).each do |line|
	record = line.split(RECORD_SEPARATOR)
	if reference.to_s == record[0].to_s
	  if new_reference
	    reference = new_reference
	    file_path = referenceToPath(reference, @data_set_directory, data_set)
	    modification_date = File.exist?(file_path) ? File.mtime(file_path).strftime(DATE_FORMAT) : "not-modification-date"
	    large_hash = record[5] if large_hash == ""
	  end
	  local_files[reference] = {"size" => size, "hash" => hash, "status" => status,
				    "modification_date" => modification_date, "large_hash" => large_hash }
	  new_file = FALSE
	else
	  local_files[record[0]] = {"size" => record[2].chomp, "hash" => record[3].chomp,
				    "status" => record[1].chomp, "modification_date" => record[4].chomp, "large_hash" => record[5] }
	end
      end
      if new_file
	local_files[reference] = {"size" => size, "hash" => hash, "status" => status,
				  "modification_date" => modification_date, "large_hash" => large_hash }
      end
    rescue Exception => e
      @logger.error("An error occurred in DatasetUtils method 'addToReport':" + e.to_s)
      @logger.error(e.backtrace)
    end
    saveReport(local_files)
  end

  def deleteFromReport(reference, status)
    local_files = getLocalFiles(remove=reference)
    saveReport(local_files)
  end

  def getHashFromChunk(content)
    begin
      md5 = Digest::MD5.new
      md5.update(content)
      return md5.hexdigest
    rescue Exception => e
      @logger.error("An error occurred while getting hash from chunk:" + e.to_s, print=TRUE)
      @logger.error(e.backtrace)
      raise e
    end
  end

  def getHash(file)
    begin
      raise "File doesn't exist" if ! File.exist?(file)
      chunk_size = 4 * MEGA
      md5 = Digest::MD5.new
      open(file) do |f|
	while chunk=f.read(chunk_size)
	  md5.update(chunk)
	end
      end
      return md5.hexdigest
    rescue Exception => e
      @logger.error("An error occurred while getting hash of file " + file + ":" + e.to_s, print=TRUE)
      @logger.error(e.backtrace)
      raise e
    end
  end

  def getLocalConflicts(remote_streams, data_set)
    paths = [appendSlashTo(@data_set_directory)]
    local_files = getLocalPaths(paths)
    ignore, local_changes = getLocalChanges(local_files, data_set, staged=FALSE)
    remote_changes = remote_streams.map { |remote|
	remote = referenceToPath(remote["reference"], @data_set_directory, data_set)
    }
    return local_changes.select{ |conflict| remote_changes.include? conflict["path"] }.map{ |conflict| conflict["path"] }
  end

  def getRemoteConflicts(data_streams, tasks, dataset)
    conflicts = []
    files = tasks.map{ |task| task["path"] }
    data_streams.each do |data_stream|
      file_path = referenceToPath(data_stream["reference"], @data_set_directory, dataset)
      if files.include? file_path
	conflicts.push(file_path.sub(@data_set_directory, "./"))
      end
    end
    return conflicts
  end

  def deleteStagedFile()
    File.delete(@staged_file) if File.exist?(@staged_file)
  end

  def getStagedRecords()
    # build a dictionary due to staged file could contain more than one operation per file
    staged_dict = {}
    if stagedFileExist()
      File.readlines(@staged_file).each_with_index do |line, index|
	record = line.split(RECORD_SEPARATOR)
	path = @data_set_directory + record[1].chomp
	operation = record[0].chomp
	staged_dict[appendSlashTo(path)] = {"index" => index, "operation" => operation}
      end
    end
    return staged_dict
  end

  def updateStagedDeletions(untrucked_deletions)
    untrucked_deletions.each do |path|
      path = path.sub(@data_set_directory, "")
      File.open(@staged_file, 'ab') { |file| file.puts(STAGE_RESET+RECORD_SEPARATOR+path) }
    end
  end

  def isLocalChangeRenamed(local_file_path, deleted_file_dict_list)
    # checking if local_file_path marked as new is also a deleted file, so it's in fact a renamed
    size = File.size(local_file_path).to_s if File.exist?(local_file_path)
    deleted_file_dict_list.each do |path, deleted_file_dict|
      if size == deleted_file_dict["size"].to_s
        hash = getHash(local_file_path).to_s
        if hash == deleted_file_dict["hash"]
          old_path = path
          return {"key" => old_path, "size" => size, "hash" => hash}
        end
      end
    end
    return FALSE
  end

  def isRemoteChangeRenamed(new_remote_file_dict_list, local_report_file_dict)
    #checking if report local file missing in remote dataset is one of the remote files marked as new/modified, so it's in fact a remote rename
    hash = local_report_file_dict["large-hash"]
    size = local_report_file_dict["size"]
    new_remote_file_dict_list.each do |path, new_remote_file_dict|
      if size == new_remote_file_dict["full-size"].to_s
        hash = hash != "" ? hash : getHash(file).to_s
        if hash == new_remote_file_dict["large-hash"]
          old_path = path
          return {"key" => old_path, "size" => size, "hash" => hash}
        end
      end
    end
    return FALSE
  end

  def isStaged(path, staged_dict, status)
    return FALSE if staged_dict.nil? || staged_dict.empty?
    staged_status = {"index" => -1, "status" => ""}
    original_file_path = path
    while appendSlashTo(path) != @data_set_directory
      key = appendSlashTo(path) if staged_dict.key?(appendSlashTo(path))
      if ! key.nil?
	if staged_dict[key]["index"] > staged_status["index"]
	  # ignore files marked as added that not exists in filesystem
	  if not (staged_dict[key]["operation"] == STAGE_ADD and not File.exist?(original_file_path))
	    staged_status = {"index" => staged_dict[key]["index"], "status" => staged_dict[key]["operation"]}
	  end
	end
      end
      path = appendSlashTo(path).split("/")[0...-1].join("/")
    end
    return FALSE if staged_status["index"] == -1
    return staged_status["status"] == status
  end

  def checkRenamed(path, deleted_file_dict_list, change)
    renamed_dict = isLocalChangeRenamed(path, deleted_file_dict_list)
    if renamed_dict
      deleted_file_dict_list.delete(renamed_dict["key"])
      change["status"] = STATUS_RENAMED
      change["new_path"] = change["path"]
      change["path"] = renamed_dict["key"]
      change["size"] = renamed_dict["size"]
      change["hash"] = renamed_dict["hash"]
    end
    return change
  end

  def getLocalChanges(files, data_set, staged, partial_ingestion=FALSE)
    staged_changes, untracked_changes = [], []
    staged_dict = getStagedRecords() if staged
    deleted_files = {}
    begin
      if reportFileExist()
	File.readlines(@task_report_file).each do |line|
	  record = line.split(RECORD_SEPARATOR)
	  if record[1].chomp == RUN_DONE
	    file_path = referenceToPath(record[0], @data_set_directory, data_set)
	    if files.include? file_path
	      modification_date = File.mtime(file_path).strftime(DATE_FORMAT)
	      if staged && isStaged(file_path, staged_dict, STAGE_REMOVE)
		staged_changes.push({"path" => file_path, "size" => "", "hash" => DELETE, "status" => STATUS_DELETED })
	      elsif modification_date != record[4].chomp
		size = File.size(file_path).to_s
		hash = getHash(file_path).to_s
		change = {"path" => file_path, "size" => size, "hash" => hash, "status" => STATUS_MODIFIED }
		if size == record[2].to_s
		  if hash != record[3].chomp
		    staged && isStaged(file_path, staged_dict, STAGE_ADD) ? staged_changes.push(change) : untracked_changes.push(change)
		  end
		else
		  staged && isStaged(file_path, staged_dict, STAGE_ADD) ? staged_changes.push(change) : untracked_changes.push(change)
		end
	      end
	      files.delete(file_path)
	    else
	      if not partial_ingestion
		deleted_files[file_path] = {"size" => record[2].to_s, "hash" => record[3].chomp}
	      end
	    end
	  end
	end
      end
      untrucked_deletions = []
      files.each do |path|
	change = {"path" => path, "size" => "", "hash" => "", "status" => STATUS_NEW }
	if staged
	  # if an untracked new file was staged as deletion, delete it
	  if isStaged(path, staged_dict, STAGE_REMOVE)
	    if File.exist?(path)
	      # check scenario where new files were created inside a directory staged as deleted
	      if File.mtime(path) > File.mtime(@staged_file)
		change = checkRenamed(path, deleted_files, change)
		untracked_changes.push(change)
	      else
		File.delete(path)
		@logger.info("Local file added for remove deleted: " + path, print=TRUE)
	      end
	    end
	    untrucked_deletions.push(path)
	  else
	    change = checkRenamed(path, deleted_files, change)
	    isStaged(path, staged_dict, STAGE_ADD) ? staged_changes.push(change) : untracked_changes.push(change)
	  end
	else
	  change = checkRenamed(path, deleted_files, change)
	  untracked_changes.push(change)
	end
      end
      deleted_files.each do |deleted_path, dict|
	change = {"path" => deleted_path, "size" => "", "hash" => DELETE, "status" => STATUS_DELETED }
	staged && isStaged(deleted_path, staged_dict, STAGE_REMOVE) ? staged_changes.push(change) : untracked_changes.push(change)
      end
      updateStagedDeletions(untrucked_deletions) if staged
    rescue Exception => e
      @logger.error("An error occurred in DatasetUtils method 'getLocalChanges':" + e.to_s)
      @logger.error(e.backtrace)
    end
    return staged_changes, untracked_changes
  end

  def dirEmpty(dir)
    Dir[(dir + '/**/*').gsub! '//', '/'].each do |path|
      return FALSE if File.file?(path)
    end
    return TRUE
  end

  def getRemoteFileListForDiscardLocalChanges(data_streams, data_set, check_changes=FALSE, changes=[])
    remotes_for_discard_local = []
    if changes.empty?
      paths = [appendSlashTo(@data_set_directory)]
      local_files = getLocalPaths(paths)
      ignore, local_changes = getLocalChanges(local_files, data_set, staged=FALSE)
    else
      local_changes = changes
    end
    return local_changes if check_changes
    local_changes.each do |change|
      f, e, reference = getPathInfo(change["path"], data_set)
      case change["status"]
	when STATUS_NEW
	  @logger.info("Discarding local change on '#{change["path"]}'", print=TRUE)
	  @logger.info("New file removed.", print=TRUE)
	  puts
	  File.delete(change["path"]) if File.exist?(change["path"])
	when STATUS_RENAMED
	  @logger.info("Discarding local change on '#{change["new_path"]}'", print=TRUE)
	  @logger.info("File moved/renamed back to '#{change["path"]}'.", print=TRUE)
	  puts
	  new_reference = reference
	  f, e, reference = getPathInfo(change["new_path"], data_set)
	  unless File.directory?(File.dirname(change["path"]))
	    FileUtils.mkdir_p(File.dirname(change["path"]))
	  end
	  FileUtils.mv(change["new_path"], change["path"]) if File.exist?(change["new_path"])
	when STATUS_MODIFIED
	  data_stream = data_streams.find{|data_stream| data_stream["reference"] == reference }
	  if data_stream
	    data_stream["path"] = change["path"]
	    remotes_for_discard_local.push(data_stream)
	  end
	when STATUS_DELETED
	  data_stream = data_streams.find{|data_stream| data_stream["reference"] == reference }
	  if data_stream
	    data_stream["path"] = change["path"]
	    remotes_for_discard_local.push(data_stream)
	  end
      end
    end
    return remotes_for_discard_local
  end

  def getRemoteChangedDataStreams(data_streams, data_set)
    changed_data_streams = []
    new_changed_files = {}
    begin
      if reportFileExist()
	local_files = {}
	remote_files = []
	File.readlines(@task_report_file).each do |line|
	  record = line.split(RECORD_SEPARATOR)
	  if record[1].chomp == RUN_DONE
	    local_files[record[0]] = {"size" => record[2].chomp, "hash" => record[3].chomp, "large-hash" => record[5].chomp, }
	  end
	end
	data_streams.each do |data_stream|
	  reference = data_stream["reference"]
	  remote_files.push(reference)
	  pending = TRUE
	  if local_files.has_key? reference
	    size = local_files[reference]["size"]
	    if size.to_s == data_stream["full-size"].to_s
	      hash = local_files[reference]["large-hash"]
	      if hash == data_stream["large-hash"] or data_stream["large-hash"] == ""
		pending = FALSE
	      end
	    end
	  end
	  if pending
	    local_files.delete(reference)
	    file_path = referenceToPath(reference, @data_set_directory, data_set)
	    new_changed_files[file_path] = data_stream
	  end
	end
	local_files.each do |reference, file_dict|
	  if not remote_files.include? reference
	    changed_data_stream = {"reference" => reference, "large-hash" => DELETE }
	    file_path = referenceToPath(reference, @data_set_directory, data_set)
	    renamed_dict = isRemoteChangeRenamed(new_changed_files, file_dict)
	    if renamed_dict
	      changed_data_stream = {"reference" => reference, "id" => new_changed_files[renamed_dict["key"]]["id"],
				     "new_reference" => new_changed_files[renamed_dict["key"]]["reference"], "status" => STATUS_RENAMED,
				     "full-size" => renamed_dict["size"], "large-hash" => renamed_dict["hash"] }
	      new_changed_files.delete(renamed_dict["key"])
	    end
	    changed_data_streams.push(changed_data_stream)
	  end
	end
	new_changed_files.each do |path, data_stream|
	  changed_data_streams.push(data_stream)
	end
      end
    rescue Exception => e
      @logger.error("An error occurred in DatasetUtils method 'getRemoteChangedDataStreams':" + e.to_s)
      @logger.error(e.backtrace)
    end
    return changed_data_streams
  end
end
