ebulk: discard changes features

- --discard-changes parameter on command pull
- new help and example files
- several fixes
parent f7de6621
...@@ -4,6 +4,10 @@ DOWN_URL='https://softinst104003.host.vifib.net/erp5/' ...@@ -4,6 +4,10 @@ DOWN_URL='https://softinst104003.host.vifib.net/erp5/'
ING_URL='https://softinst104003.host.vifib.net/erp5/portal_ingestion_policies/wendelin_embulk' ING_URL='https://softinst104003.host.vifib.net/erp5/portal_ingestion_policies/wendelin_embulk'
EBULK_DATA_PATH=~/.ebulk EBULK_DATA_PATH=~/.ebulk
EBULK_DATASET_FILE_NAME="/.ebulk_dataset"
DATASET_REPORT_FILE_NAME="/.dataset-task-report"
DATASET_COMPLETE_FILE_NAME="/.dataset-completed"
DISCARD_CHANGES_FILE_NAME="/.discard-changes"
LOG_DIR="$EBULK_DATA_PATH/logs" LOG_DIR="$EBULK_DATA_PATH/logs"
TOOL_PATH="$(dirname "$0")/ebulk-data" TOOL_PATH="$(dirname "$0")/ebulk-data"
DOWN_FILE="$EBULK_DATA_PATH/download-config.yml" DOWN_FILE="$EBULK_DATA_PATH/download-config.yml"
...@@ -46,6 +50,11 @@ function checkParameters { ...@@ -46,6 +50,11 @@ function checkParameters {
fi fi
if [ "$STORAGE" = "" ] ; then if [ "$STORAGE" = "" ] ; then
if [ ! -d "$DATASET_DIR" ]; then if [ ! -d "$DATASET_DIR" ]; then
if [ "$STATUS" ]; then
echo
echo -e "${ORANGE}[ERROR] ${GREEN}'$DATASET_DIR'${ORANGE} is not a dataset directory.${NC}"
echo >&2; return 1
fi
echo echo
mkdir "$DATASET_DIR" 2>/dev/null mkdir "$DATASET_DIR" 2>/dev/null
if [ ! $? -eq 0 ]; then if [ ! $? -eq 0 ]; then
...@@ -56,16 +65,29 @@ function checkParameters { ...@@ -56,16 +65,29 @@ function checkParameters {
helpReadme >&2; return 1 helpReadme >&2; return 1
fi fi
fi fi
EBULK_DATASET_FILE="$DATASET_DIR/.ebulk_dataset" EBULK_DATASET_FILE="$DATASET_DIR$EBULK_DATASET_FILE_NAME"
if [[ $DATASET_DIR != $REFERENCE ]]; then if [[ $DATASET_DIR != $REFERENCE ]]; then
if [ "$REFERENCE" = "." ] ; then if [ "$REFERENCE" = "." ] ; then
REFERENCE=$(basename "$DATASET_DIR") REFERENCE=$(basename "$DATASET_DIR")
fi fi
DATA_SET=$REFERENCE DATA_SET=$REFERENCE
if [ -f "$EBULK_DATASET_FILE" ]; then
PREVIOUS_DATA_SET=$(cat "$EBULK_DATASET_FILE" 2>/dev/null)
if [[ "$PREVIOUS_DATA_SET" != "$REFERENCE" ]]; then
DATASET_REPORT_FILE="$DATASET_DIR$DATASET_REPORT_FILE_NAME"
if [ -f "$DATASET_REPORT_FILE" ]; then
rm -f ${DATASET_REPORT_FILE}
fi
DATASET_COMPLETE_FILE="$DATASET_DIR$DATASET_COMPLETE_FILE_NAME"
if [ -f "$DATASET_COMPLETE_FILE" ]; then
rm -f ${DATASET_COMPLETE_FILE}
fi
fi
fi
echo $REFERENCE > "$EBULK_DATASET_FILE" 2>/dev/null echo $REFERENCE > "$EBULK_DATASET_FILE" 2>/dev/null
else else
if [ -f "$EBULK_DATASET_FILE" ]; then if [ -f "$EBULK_DATASET_FILE" ]; then
DATA_SET=$(cat "$DATASET_DIR/.ebulk_dataset" 2>/dev/null) DATA_SET=$(cat "$EBULK_DATASET_FILE" 2>/dev/null)
else else
DATA_SET=$(basename "$DATASET_DIR") DATA_SET=$(basename "$DATASET_DIR")
if [ "$DATA_SET" != "." ] ; then if [ "$DATA_SET" != "." ] ; then
...@@ -403,7 +425,7 @@ function askS3parameters { ...@@ -403,7 +425,7 @@ function askS3parameters {
} }
function stage { function stage {
EBULK_DATASET_FILE="./.ebulk_dataset" EBULK_DATASET_FILE=".$EBULK_DATASET_FILE_NAME"
if [ ! -f "$EBULK_DATASET_FILE" ]; then if [ ! -f "$EBULK_DATASET_FILE" ]; then
echo echo
echo -e "${ORANGE}[ERROR] You are not in a dataset directory." echo -e "${ORANGE}[ERROR] You are not in a dataset directory."
...@@ -461,6 +483,8 @@ while [ "$1" != "" ]; do ...@@ -461,6 +483,8 @@ while [ "$1" != "" ]; do
;; ;;
-a | --advanced ) ADVANCED=true -a | --advanced ) ADVANCED=true
;; ;;
-dc | --discard-changes ) DISCARD_CHANGES=true
;;
-c | --chunk ) shift -c | --chunk ) shift
CHUNK=$1 CHUNK=$1
;; ;;
...@@ -490,7 +514,7 @@ while [ "$1" != "" ]; do ...@@ -490,7 +514,7 @@ while [ "$1" != "" ]; do
shift shift
done done
for ELEMENT in '' '-d' '--directory' '-s' '--storage' '-cs' '--custom-storage' '-a' '--advanced' '-c' '--chunk'; do for ELEMENT in '' '-d' '--directory' '-s' '--storage' '-cs' '--custom-storage' '-a' '--advanced' '-c' '--chunk' '-dc' '--discard-changes'; do
if [ "$ELEMENT" = "$REFERENCE" ]; then if [ "$ELEMENT" = "$REFERENCE" ]; then
REFERENCE="." REFERENCE="."
fi fi
...@@ -554,7 +578,13 @@ case $OPERATION in ...@@ -554,7 +578,13 @@ case $OPERATION in
fi fi
echo "### DATASET DOWNLOAD ###" echo "### DATASET DOWNLOAD ###"
echo echo
if [ "$DISCARD_CHANGES" != "" ] ; then
DISCARD_CHANGES_FILE="$DATASET_DIR$DISCARD_CHANGES_FILE_NAME"
touch "$DISCARD_CHANGES_FILE" 2>/dev/null
echo -e "** Discard all local changes in directory: $DATASET_DIR"
else
echo -e "** The dataset will be downloaded in the specified directory: $DATASET_DIR" echo -e "** The dataset will be downloaded in the specified directory: $DATASET_DIR"
fi
echo echo
read -n 1 -s -r -p "Press any key to continue" read -n 1 -s -r -p "Press any key to continue"
echo echo
...@@ -588,8 +618,9 @@ case $OPERATION in ...@@ -588,8 +618,9 @@ case $OPERATION in
PARAMETER_FUNCTION=askFTPparameters PARAMETER_FUNCTION=askFTPparameters
STORAGE_GEM=embulk-input-ftp STORAGE_GEM=embulk-input-ftp
;; ;;
*) echo -e "${ORANGE}[ERROR] '$STORAGE' storage is not available in ebulk tool yet.${NC}" *) echo -e "${ORANGE}[ERROR] '$STORAGE' storage is not available in ebulk tool yet or it is not a valid storage.${NC}"
echo "[INFO] If you want to configure yourself this storage, you can run the tool with parameter --custom-storage" echo "[INFO] If you want to configure yourself this storage, you can run the tool with parameter --custom-storage"
echo "[INFO] Current Ebulk version has the following storages available: ftp, http, s3."
echo echo
exit exit
esac esac
......
...@@ -11,6 +11,7 @@ in: ...@@ -11,6 +11,7 @@ in:
user: $USER user: $USER
password: $pwd password: $pwd
tool_dir: $TOOL_DIR tool_dir: $TOOL_DIR
status: $STATUS
out: out:
type: wendelin type: wendelin
......
...@@ -94,6 +94,7 @@ module Embulk ...@@ -94,6 +94,7 @@ module Embulk
@dataset_utils = DatasetUtils.new(@data_set_directory) @dataset_utils = DatasetUtils.new(@data_set_directory)
@status = config.param('status', :string, default: FALSE) @status = config.param('status', :string, default: FALSE)
@status = @status == "" ? FALSE : @status @status = @status == "" ? FALSE : @status
@dataset_utils.deleteDiscardChangesFile()
if @status if @status
if not @dataset_utils.initialIngestionFileExist() if not @dataset_utils.initialIngestionFileExist()
if not @dataset_utils.reportFileExist() if not @dataset_utils.reportFileExist()
...@@ -102,7 +103,7 @@ module Embulk ...@@ -102,7 +103,7 @@ module Embulk
@logger.abortExecution() @logger.abortExecution()
elsif not @dataset_utils.completedFileExist() elsif not @dataset_utils.completedFileExist()
puts puts
@logger.error("There is an interrumped download operation in dataset directory. Please resume the download first.", print=TRUE) @logger.error("There is an interrupted download operation in dataset directory. Please resume the download first.", print=TRUE)
@logger.abortExecution() @logger.abortExecution()
end end
end end
...@@ -128,7 +129,7 @@ module Embulk ...@@ -128,7 +129,7 @@ module Embulk
else else
if not @dataset_utils.initialIngestionFileExist() if not @dataset_utils.initialIngestionFileExist()
@logger.info("Checking local dataset...", print=TRUE) @logger.info("Checking local dataset...", print=TRUE)
if not @dataset_utils.reportUpToDate(data_stream_dict) if not @dataset_utils.reportUpToDate(data_stream_dict, @data_set)
puts puts
@logger.error("Your current dataset is outdated. Please, run a download to update it before ingest your changes.", print=TRUE) @logger.error("Your current dataset is outdated. Please, run a download to update it before ingest your changes.", print=TRUE)
puts puts
...@@ -185,8 +186,8 @@ module Embulk ...@@ -185,8 +186,8 @@ module Embulk
def self.resume(task, columns, count, &control) def self.resume(task, columns, count, &control)
@logger = LogManager.instance() @logger = LogManager.instance()
task_reports = yield(task, columns, count) task_reports = yield(task, columns, count)
@dataset_utils.showTaskReport(task_reports) next_config_diff = task_reports.map{|hash| hash[DatasetUtils::RUN_DONE]}.flatten.compact
next_config_diff = task_reports.map{|hash| hash["done"]}.flatten.compact @dataset_utils.showTaskReport(next_config_diff)
element_output = @dataset_utils.initialIngestionFileExist() ? "new file" : "change" element_output = @dataset_utils.initialIngestionFileExist() ? "new file" : "change"
@logger.info("#{next_config_diff.length} #{element_output}(s) ingested.", print=TRUE) @logger.info("#{next_config_diff.length} #{element_output}(s) ingested.", print=TRUE)
if(next_config_diff.length == count) if(next_config_diff.length == count)
...@@ -194,7 +195,7 @@ module Embulk ...@@ -194,7 +195,7 @@ module Embulk
@wendelin.increaseDatasetVersion(@data_set) @wendelin.increaseDatasetVersion(@data_set)
@dataset_utils.deleteStagedFile() @dataset_utils.deleteStagedFile()
else else
failed_tasks = task_reports.map{|hash| hash["error"]}.flatten.compact failed_tasks = task_reports.map{|hash| hash[DatasetUtils::RUN_ERROR] || hash[DatasetUtils::RUN_ABORTED] }.flatten.compact
@dataset_utils.showTaskErrors(failed_tasks) @dataset_utils.showTaskErrors(failed_tasks)
end end
next_config_diff = {} next_config_diff = {}
...@@ -219,13 +220,16 @@ module Embulk ...@@ -219,13 +220,16 @@ module Embulk
size = file_dict["size"] size = file_dict["size"]
hash = file_dict["hash"] hash = file_dict["hash"]
delete = hash == DatasetUtils::DELETE delete = hash == DatasetUtils::DELETE
rename = file_dict["status"] == DatasetUtils::STATUS_RENAMED
if size == "" and hash == "" #new file if size == "" and hash == "" #new file
size = File.size(path) size = File.size(path)
hash = @dataset_utils.getHash(path) hash = @dataset_utils.getHash(path)
end end
new_filename, new_extension, new_reference = @dataset_utils.getPathInfo(file_dict["new_path"], @dataset) if rename
filename, extension, reference = @dataset_utils.getPathInfo(path, @dataset) filename, extension, reference = @dataset_utils.getPathInfo(path, @dataset)
@dataset_utils.saveCurrentOperation(DatasetUtils::INGESTION, reference) operation = rename ? DatasetUtils::RENAME : DatasetUtils::INGESTION
each_chunk(path, filename, extension, size, hash, schema[1..-1].map{|elm| elm.name}, @chunk_size, delete) do |entry| @dataset_utils.saveCurrentOperation(operation, reference, new_reference)
each_chunk(path, filename, extension, size, hash, schema[1..-1].map{|elm| elm.name}, @chunk_size, delete, new_reference) do |entry|
@page_builder.add(entry) @page_builder.add(entry)
end end
@page_builder.finish @page_builder.finish
...@@ -247,7 +251,7 @@ module Embulk ...@@ -247,7 +251,7 @@ module Embulk
end end
else else
if @dataset_utils.reportFileExist() if @dataset_utils.reportFileExist()
@dataset_utils.addToReport(reference, return_value, size, hash, task['data_set']) @dataset_utils.addToReport(reference, return_value, size, hash, task['data_set'], new_reference)
end end
end end
end end
...@@ -257,11 +261,15 @@ module Embulk ...@@ -257,11 +261,15 @@ module Embulk
private private
def each_chunk(path, filename, extension, size, hash, fields, chunk_size=DatasetUtils::CHUNK_SIZE, delete=FALSE) def each_chunk(path, filename, extension, size, hash, fields, chunk_size=DatasetUtils::CHUNK_SIZE, delete=FALSE, new_reference=FALSE)
if delete if delete
File.delete(path) if File.exist?(path) File.delete(path) if File.exist?(path)
values = [@supplier, @dataset, filename, extension, "", DatasetUtils::DELETE, "", ""] values = [@supplier, @dataset, filename, extension, "", DatasetUtils::DELETE, "", ""]
yield(values) yield(values)
elsif new_reference
File.delete(path) if File.exist?(path)
values = [@supplier, @dataset, filename, extension, new_reference, DatasetUtils::RENAME, "", ""]
yield(values)
else else
file_object = File.open(path, "rb") file_object = File.open(path, "rb")
npart = 0 npart = 0
......
...@@ -61,7 +61,7 @@ module Embulk ...@@ -61,7 +61,7 @@ module Embulk
case option case option
when action when action
@logger.info("Checking remote changes and posible local conflicts...", print=TRUE) if action != RESUME @logger.info("Checking remote changes and posible local conflicts...", print=TRUE) if action != RESUME
task['data_streams'] = @dataset_utils.getRemoteChangedDataStreams(task['data_streams']) task['data_streams'] = @dataset_utils.getRemoteChangedDataStreams(task['data_streams'], @data_set)
self.warnConflicts(task['data_streams'], task['data_set']) if action != RESUME self.warnConflicts(task['data_streams'], task['data_set']) if action != RESUME
@dataset_utils.deleteCompletedFile() @dataset_utils.deleteCompletedFile()
if task['data_streams'].empty? if task['data_streams'].empty?
...@@ -95,7 +95,6 @@ module Embulk ...@@ -95,7 +95,6 @@ module Embulk
@logger.abortExecution() @logger.abortExecution()
end end
@wendelin = WendelinClient.new(@erp5_url, @user, @password) @wendelin = WendelinClient.new(@erp5_url, @user, @password)
task = { task = {
'erp5_url' => @erp5_url, 'erp5_url' => @erp5_url,
'data_set' => @data_set, 'data_set' => @data_set,
...@@ -113,6 +112,17 @@ module Embulk ...@@ -113,6 +112,17 @@ module Embulk
task['data_set_directory'] = @dataset_utils.appendSlashTo(@output_path) task['data_set_directory'] = @dataset_utils.appendSlashTo(@output_path)
@data_set_directory = task['data_set_directory'] @data_set_directory = task['data_set_directory']
@dataset_utils = DatasetUtils.new(@data_set_directory) @dataset_utils = DatasetUtils.new(@data_set_directory)
if @dataset_utils.reportFileExist() && @dataset_utils.completedFileExist() && @dataset_utils.discardChangesFileExist() && ! @dataset_utils.initialIngestionFileExist()
task['discard_changes'] = @dataset_utils.discardChangesFileExist()
local_changes = @dataset_utils.getRemoteFileListForDiscardLocalChanges([], @data_set, check_changes=TRUE)
if local_changes.empty?
puts
@logger.info("No local changes to discard.", print=TRUE)
puts
@dataset_utils.deleteDiscardChangesFile()
@logger.abortExecution(error=FALSE)
end
end
@logger.info("Getting remote file list from dataset '#{@data_set}'...", print=TRUE) @logger.info("Getting remote file list from dataset '#{@data_set}'...", print=TRUE)
data_stream_list = @wendelin.getDataStreams(@data_set) data_stream_list = @wendelin.getDataStreams(@data_set)
if data_stream_list["status_code"] == 0 if data_stream_list["status_code"] == 0
...@@ -129,10 +139,26 @@ module Embulk ...@@ -129,10 +139,26 @@ module Embulk
if @dataset_utils.reportFileExist() if @dataset_utils.reportFileExist()
if @dataset_utils.completedFileExist() if @dataset_utils.completedFileExist()
if task['discard_changes']
puts
@logger.warn("All your local changes will be discarded.", print=TRUE)
@logger.warn("Do you want to continue? (y/n)", print=TRUE)
option = gets
option = option.chomp
if option == "n" or option == "N"
@logger.info("Download cancelled by user.", print=TRUE)
@dataset_utils.deleteDiscardChangesFile()
@logger.abortExecution(error=FALSE)
end
@dataset_utils.deleteStagedFile()
task['data_streams'] = @dataset_utils.getRemoteFileListForDiscardLocalChanges(task['data_streams'], @data_set,
check_changes=FALSE, changes=local_changes)
else
puts puts
@logger.info("This dataset was already downloaded. What do you want to do?", print=TRUE) @logger.info("This dataset was already downloaded. What do you want to do?", print=TRUE)
puts puts
self.askUserForAction(task, action=UPDATE) self.askUserForAction(task, action=UPDATE)
end
elsif not @dataset_utils.initialIngestionFileExist() elsif not @dataset_utils.initialIngestionFileExist()
puts puts
@logger.info("There was a previous attempt to download this dataset but it did not finish successfully.", print=TRUE) @logger.info("There was a previous attempt to download this dataset but it did not finish successfully.", print=TRUE)
...@@ -140,10 +166,20 @@ module Embulk ...@@ -140,10 +166,20 @@ module Embulk
puts puts
self.askUserForAction(task, action=RESUME) self.askUserForAction(task, action=RESUME)
else else
if @dataset_utils.discardChangesFileExist()
puts
@logger.info("Discard changes feature do not apply in current dataset directory status.", print=TRUE)
@logger.info("Continuing with dataset download.", print=TRUE)
end
puts puts
self.askUserForAction(task, action=UPDATE) self.askUserForAction(task, action=UPDATE)
end end
else else
if @dataset_utils.discardChangesFileExist()
puts
@logger.info("Discard changes feature do not apply in current dataset directory status.", print=TRUE)
@logger.info("Continuing with dataset download.", print=TRUE)
end
if not @dataset_utils.dirEmpty(@data_set_directory) if not @dataset_utils.dirEmpty(@data_set_directory)
puts puts
@logger.info("Dataset download directory is not empty! Its files could be overwritten: " + @data_set_directory, print=TRUE) @logger.info("Dataset download directory is not empty! Its files could be overwritten: " + @data_set_directory, print=TRUE)
...@@ -160,11 +196,13 @@ module Embulk ...@@ -160,11 +196,13 @@ module Embulk
@dataset_utils.createReportFile() @dataset_utils.createReportFile()
end end
@dataset_utils.deleteInitialIngestionFile() @dataset_utils.deleteInitialIngestionFile()
@dataset_utils.deleteDiscardChangesFile()
columns = [ columns = [
Column.new(0, "reference", :string), Column.new(0, "reference", :string),
Column.new(1, "data_chunk", :string), Column.new(1, "data_chunk", :string),
Column.new(2, "data_set", :string), Column.new(2, "data_set", :string),
Column.new(3, "mode", :string) Column.new(3, "mode", :string),
Column.new(4, "rename", :string)
] ]
resume(task, columns, task['data_streams'].length, &control) resume(task, columns, task['data_streams'].length, &control)
rescue Exception => e rescue Exception => e
...@@ -178,14 +216,18 @@ module Embulk ...@@ -178,14 +216,18 @@ module Embulk
def self.resume(task, columns, count, &control) def self.resume(task, columns, count, &control)
@logger = LogManager.instance() @logger = LogManager.instance()
task_reports = yield(task, columns, count) task_reports = yield(task, columns, count)
@dataset_utils.showTaskReport(task_reports) @dataset_utils.showTaskReport(task_reports) if not task['discard_changes']
next_config_diff = task_reports.map{|hash| hash[DatasetUtils::RUN_DONE]}.flatten.compact next_config_diff = task_reports.map{|hash| hash[DatasetUtils::RUN_DONE]}.flatten.compact
if(next_config_diff.length == count) if(next_config_diff.length == count)
if(count > 0) if(count > 0)
if task['discard_changes']
@logger.info("All local changes were discarded.", print=TRUE)
else
@logger.info("Dataset successfully downloaded.", print=TRUE) @logger.info("Dataset successfully downloaded.", print=TRUE)
@logger.info("#{count} files processed.", print=TRUE) @logger.info("#{count} files processed.", print=TRUE)
@logger.info("Dataset files are in dataset directory: " + @data_set_directory, print=TRUE) @logger.info("Dataset files are in dataset directory: " + @data_set_directory, print=TRUE)
end end
end
@dataset_utils.createCompletedFile() @dataset_utils.createCompletedFile()
else else
if(count > 0) if(count > 0)
...@@ -215,18 +257,24 @@ module Embulk ...@@ -215,18 +257,24 @@ module Embulk
ref = data_stream["reference"] ref = data_stream["reference"]
size = data_stream["size"] size = data_stream["size"]
hash = data_stream["hash"] hash = data_stream["hash"]
renamed = data_stream["status"] == DatasetUtils::STATUS_RENAMED
deleted = hash.to_s == DatasetUtils::DELETE
begin begin
if hash.to_s == DatasetUtils::DELETE if deleted
@logger.info("Deleting #{ref}", print=TRUE) entry = [ref, "", @data_set, DatasetUtils::DELETE, renamed]
entry = [ref, "", @data_set, hash.to_s] page_builder.add(entry)
elsif renamed
new_reference = data_stream["new_reference"]
entry = [ref, new_reference, @data_set, TRUE, renamed]
page_builder.add(entry) page_builder.add(entry)
else else
@logger.info("Discarding local change on '#{data_stream["path"]}'", print=TRUE) if task['discard_changes']
@logger.info("Getting content from remote #{ref}", print=TRUE) @logger.info("Getting content from remote #{ref}", print=TRUE)
n_chunk = 0 n_chunk = 0
@wendelin.eachDataStreamContentChunk(id, @chunk_size) do |chunk| @wendelin.eachDataStreamContentChunk(id, @chunk_size) do |chunk|
content = chunk.nil? || chunk.empty? ? "" : Base64.encode64(chunk) content = chunk.nil? || chunk.empty? ? "" : Base64.encode64(chunk)
begin_of_file = n_chunk == 0 begin_of_file = n_chunk == 0
entry = [ref, content, @data_set, begin_of_file] entry = [ref, content, @data_set, begin_of_file, renamed]
page_builder.add(entry) page_builder.add(entry)
n_chunk += 1 n_chunk += 1
end end
...@@ -244,10 +292,10 @@ module Embulk ...@@ -244,10 +292,10 @@ module Embulk
return_value = DatasetUtils::RUN_DONE return_value = DatasetUtils::RUN_DONE
end end
if return_value == DatasetUtils::RUN_DONE if return_value == DatasetUtils::RUN_DONE
if hash.to_s == DatasetUtils::DELETE if deleted
@dataset_utils.deleteFromReport(ref, return_value) @dataset_utils.deleteFromReport(ref, return_value)
else else
@dataset_utils.addToReport(ref, return_value, size, hash, task['data_set']) @dataset_utils.addToReport(ref, return_value, size, hash, task['data_set'], new_reference)
end end
end end
return {return_value => ref} return {return_value => ref}
......
...@@ -34,14 +34,22 @@ module Embulk ...@@ -34,14 +34,22 @@ module Embulk
begin begin
page.each do |record| page.each do |record|
reference = record[0] reference = record[0]
data_chunk = Base64.decode64(record[1])
@dataset_utils = DatasetUtils.new("") @dataset_utils = DatasetUtils.new("")
data_set_directory = @dataset_utils.appendSlashTo(@output_path) data_set_directory = @dataset_utils.appendSlashTo(@output_path)
file_path = @dataset_utils.referenceToPath(reference, data_set_directory, record[2]) file_path = @dataset_utils.referenceToPath(reference, data_set_directory, record[2])
write_mode = 'ab' write_mode = 'ab'
if record[3] == DatasetUtils::DELETE if record[3] == DatasetUtils::DELETE
@logger.info("Deleting '#{file_path}'", print=TRUE)
File.delete(file_path) if File.exist?(file_path) File.delete(file_path) if File.exist?(file_path)
elsif record[4] == TRUE.to_s # if renamed
new_file_path = @dataset_utils.referenceToPath(record[1], data_set_directory, record[2])
@logger.info("Renaming '#{file_path}' to '#{new_file_path}'", print=TRUE)
unless File.directory?(File.dirname(new_file_path))
FileUtils.mkdir_p(File.dirname(new_file_path))
end
FileUtils.mv(file_path, new_file_path) if File.exist?(file_path)
else else
data_chunk = Base64.decode64(record[1])
if record[3] == TRUE.to_s if record[3] == TRUE.to_s
write_mode = 'w' write_mode = 'w'
end end
......
...@@ -48,6 +48,9 @@ module Embulk ...@@ -48,6 +48,9 @@ module Embulk
if eof == DatasetUtils::DELETE if eof == DatasetUtils::DELETE
reference = [dataset, filename, extension].join(DatasetUtils::REFERENCE_SEPARATOR) reference = [dataset, filename, extension].join(DatasetUtils::REFERENCE_SEPARATOR)
@wendelin.delete(reference) @wendelin.delete(reference)
elsif eof == DatasetUtils::RENAME
reference = [dataset, filename, extension].join(DatasetUtils::REFERENCE_SEPARATOR)
@wendelin.rename(reference, record[4].to_s)
else else
reference = [supplier, dataset, filename, extension, eof, size, hash].join(DatasetUtils::REFERENCE_SEPARATOR) reference = [supplier, dataset, filename, extension, eof, size, hash].join(DatasetUtils::REFERENCE_SEPARATOR)
split = eof != "" split = eof != ""
......
...@@ -16,8 +16,15 @@ class WendelinClient ...@@ -16,8 +16,15 @@ class WendelinClient
@last_ingestion = Time.new - 2 @last_ingestion = Time.new - 2
end end
def checkReferenceChars(reference)
if ["&", ";", "#", "%", '"', "+"].any? { |char| reference.include?(char) }
raise "invalid char in filename. Following chars are not allowed for filenames: \& \; \% \" \+ \# Please rename it."
end
end
def exists(reference) def exists(reference)
uri = URI("#{@erp5_url}/ingestionReferenceExists?reference=#{reference}") checkReferenceChars(reference)
uri = URI(URI.escape("#{@erp5_url}/ingestionReferenceExists?reference=#{reference}"))
begin begin
res = open(uri, http_basic_authentication: [@user, @password]).read res = open(uri, http_basic_authentication: [@user, @password]).read
rescue Exception => e rescue Exception => e
...@@ -34,11 +41,25 @@ class WendelinClient ...@@ -34,11 +41,25 @@ class WendelinClient
def delete(reference) def delete(reference)
@logger.info("Deletion requested for reference #{reference}", print=TRUE) @logger.info("Deletion requested for reference #{reference}", print=TRUE)
uri = URI("#{@erp5_url}/ERP5Site_invalidateIngestionObjects?reference=#{reference}") checkReferenceChars(reference)
uri = URI(URI.escape("#{@erp5_url}/ERP5Site_invalidateIngestionObjects?reference=#{reference}"))
res = handleRequest(uri)
if res == FALSE
@logger.abortExecution()
end
@logger.info("Remote file successfully ingested.", print=TRUE)
end
def rename(reference, new_reference)
@logger.info("Rename requested for reference #{reference}, new reference #{new_reference}", print=TRUE)
checkReferenceChars(reference)
checkReferenceChars(new_reference)
uri = URI(URI.escape("#{@erp5_url}/ERP5Site_renameIngestion?reference=#{reference}&new_reference=#{new_reference}"))
res = handleRequest(uri) res = handleRequest(uri)
if res == FALSE if res == FALSE
@logger.abortExecution() @logger.abortExecution()
end end
@logger.info("Remote file successfully renamed.", print=TRUE)
end end
def increaseDatasetVersion(reference) def increaseDatasetVersion(reference)
...@@ -46,7 +67,7 @@ class WendelinClient ...@@ -46,7 +67,7 @@ class WendelinClient
@logger.warn("Could not increase data set version because dataset reference is empty.") @logger.warn("Could not increase data set version because dataset reference is empty.")
else else
@logger.info("Increasing dataset version") @logger.info("Increasing dataset version")
uri = URI("#{@erp5_url}/ERP5Site_increaseDatasetVersion?reference=#{reference}") uri = URI(URI.escape("#{@erp5_url}/ERP5Site_increaseDatasetVersion?reference=#{reference}"))
begin begin
res = open(uri, http_basic_authentication: [@user, @password]).read res = open(uri, http_basic_authentication: [@user, @password]).read
rescue Exception => e rescue Exception => e
...@@ -68,16 +89,8 @@ class WendelinClient ...@@ -68,16 +89,8 @@ class WendelinClient
@logger.info("Rename your file or download the full dataset to make local changes.", print=TRUE) @logger.info("Rename your file or download the full dataset to make local changes.", print=TRUE)
return FALSE return FALSE
end end
if reference.include? "#" or reference.include? "+" checkReferenceChars(reference)
raise "invalid chars in file name. Please rename it." uri = URI(URI.escape("#{@erp5_url}/ingest?reference=#{reference}"))
end
begin
uri = URI("#{@erp5_url}/ingest?reference=#{reference}")
rescue Exception => e
@logger.error("An error occurred while generating url: " + e.to_s)
@logger.error(e.backtrace)
raise "invalid chars in file name. Please rename it."
end
response = handleRequest(uri, reference, data_chunk) response = handleRequest(uri, reference, data_chunk)
if response == FALSE if response == FALSE
return FALSE return FALSE
...@@ -88,7 +101,7 @@ class WendelinClient ...@@ -88,7 +101,7 @@ class WendelinClient
end end
def eachDataStreamContentChunk(id, chunk_size) def eachDataStreamContentChunk(id, chunk_size)
uri = URI("#{@erp5_url}#{id}/getData") uri = URI(URI.escape("#{@erp5_url}#{id}/getData"))
@logger.info("Downloading...", print=TRUE) @logger.info("Downloading...", print=TRUE)
first = TRUE first = TRUE
res = open(uri, http_basic_authentication: [@user, @password]) { res = open(uri, http_basic_authentication: [@user, @password]) {
...@@ -109,7 +122,7 @@ class WendelinClient ...@@ -109,7 +122,7 @@ class WendelinClient
end end
def getDataStreams(data_set_reference) def getDataStreams(data_set_reference)
uri = URI("#{@erp5_url}getDataStreamList?data_set_reference=#{data_set_reference}") uri = URI(URI.escape("#{@erp5_url}getDataStreamList?data_set_reference=#{data_set_reference}"))
str = handleRequest(uri) str = handleRequest(uri)
if str == FALSE if str == FALSE
@logger.abortExecution() @logger.abortExecution()
......
ebulk ingest-download tool examples
Basic ingestion/download
ebulk pull <DATASET>
* downloads the content of target dataset
ebulk push <DATASET>
* ingests files into the target dataset
ebulk pull <DATASET> -d <PATH>
* downloads the content of target dataset in target PATH
* future operations on PATH directory will use the DATASET reference implicitly
ebulk push <DATASET> -c 20
* ingests files into the <DATASET> splitting them in chunks of 20MB
ebulk push <DATASET> -s <STORAGE>
* ingests the content of the input storage [http, ftp, s3] into the target dataset
ebulk push <DATASET> -s <STORAGE> --advanced
* allows the user to edit the configuration file of the selected storage
ebulk push <DATASET> --custom-storage
* user can install and configure a new input plugin storage
Manage local changes
ebulk status <DATASET>
* checks local changes of target dataset
ebulk add <PATH>
* marks files in path for ingestion
ebulk remove <PATH>
* marks files in path for deletion
ebulk reset <PATH>
* resets marked files in path
ebulk pull --discard-changes
* discards local changes by checking the remote dataset
ebulk ingest-download tool help ebulk ingest-download tool help
usage: ebulk <command> <dataset> [options...] ebulk [-h|--help] [-r|--readme] [-e|--examples] <command> [<args>]
[-d|--directory <path>] [-c|--chunk <size>]
[-s|--storage <storage>] [-cs|--custom-storage]
[-a|--advanced] [-dc|--discard-changes]
commands: commands:
pull <dataset> Downloads the content of the target dataset from the site into the output folder pull [<dataset>] Downloads the content of the target dataset from the site into the output location
push <dataset> Ingests the content of the input folder into a target dataset on the site push [<dataset>] Ingests the content of the input location into a target dataset on the site
status [<dataset>] Lists the local changes of target dataset
add <path> Marks new or modified files in path for ingestion
remove <path> Marks files in path for removal
reset <path> Resets marked files in path
-h, --help Tool help -h, --help Tool help
-r, --readme Opens README file -r, --readme Opens README file
-e, --examples Shows some tool usage examples
argument: argument:
dataset Mandatory. Unique reference for the target dataset dataset argument Unique reference for the target dataset
If empty, current directory will be used as dataset directory and reference
It must start with a letter, and only alphanumerics, dots ( . ), underscores ( _ ) and hyphens ( - ) are allowed It must start with a letter, and only alphanumerics, dots ( . ), underscores ( _ ) and hyphens ( - ) are allowed
* For download, the reference must be one of the available datasets on the site * For download, the reference must be one of the available datasets on the site
* For ingestion, an existing reference will append the files to the corresponding dataset * For ingestion, an existing reference will append the files to the corresponding dataset
* A new reference will create a new dataset on the site * A new reference will create a new dataset on the site
It could be a path, then the last directory will be interpreted as the reference It could be a path, then that directory will be used as dataset reference
e.g. pull my_directory/sample/ --> dataset reference will be "sample" e.g. pull my_directory/sample/ --> dataset reference will be "sample"
options: options:
-d, --directory <path> Besides the dataset reference, sets the dataset directory and it links that location to the reference -d, --directory <path> Besides the dataset reference, sets the dataset directory and it links that location to the reference
-c, --chunk <chunk> Sets the chunk size (in megabytes) to split large files -c, --chunk <size> Sets the chunk size (in megabytes) to split large files
-s, --storage <storage> Uses the selected input storage from this set: [http, ftp, s3] -s, --storage <storage> Uses the selected input storage from this set: [http, ftp, s3]
-cs, --custom-storage Allows user to set a new input storage. -cs, --custom-storage Allows user to set a new input storage.
-a, --advanced Allows to edit the Embulk cofiguration file of the input storage -a, --advanced Allows to edit the Embulk cofiguration file of the input storage
-dc, --discard-changes Discards local changes by checking the remote dataset
examples:
ebulk pull <DATASET>
* downloads the content of target dataset
ebulk push <DATASET>
* ingests files into the target dataset
ebulk pull <DATASET> -d <PATH>
* downloads the content of target dataset in target PATH
* future operations on PATH directory will use the DATASET reference implicitly
ebulk push <DATASET> -c 20
* ingests files into the <DATASET> splitting them in chunks of 20MB
ebulk push <DATASET> -s <STORAGE>
* ingests the content of the input storage [http, ftp, s3] into the target dataset
ebulk push <DATASET> -s <STORAGE> --advanced
* allows the user to edit the configuration file of the selected storage
ebulk push <DATASET> --custom-storage
* user can install and configure a new input plugin storage
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment