Commit 0ee18a35 authored by Stan Hu's avatar Stan Hu

Merge branch 'revert-733ce4aa' into 'master'

Revert merge request !46321

See merge request gitlab-org/gitlab!46409
parents 8a6c04d1 2d8dc0c2
.tests-metadata-state: .tests-metadata-state:
variables:
TESTS_METADATA_S3_BUCKET: "gitlab-ce-cache"
before_script: before_script:
- source scripts/utils.sh - source scripts/utils.sh
artifacts: artifacts:
......
...@@ -6,7 +6,7 @@ Our current CI parallelization setup is as follows: ...@@ -6,7 +6,7 @@ Our current CI parallelization setup is as follows:
1. The `retrieve-tests-metadata` job in the `prepare` stage ensures we have a 1. The `retrieve-tests-metadata` job in the `prepare` stage ensures we have a
`knapsack/report-master.json` file: `knapsack/report-master.json` file:
- The `knapsack/report-master.json` file is fetched from the latest `master` artifacts, if it's not here - The `knapsack/report-master.json` file is fetched from S3, if it's not here
we initialize the file with `{}`. we initialize the file with `{}`.
1. Each `[rspec|rspec-ee] [unit|integration|system|geo] n m` job are run with 1. Each `[rspec|rspec-ee] [unit|integration|system|geo] n m` job are run with
`knapsack rspec` and should have an evenly distributed share of tests: `knapsack rspec` and should have an evenly distributed share of tests:
...@@ -19,7 +19,7 @@ Our current CI parallelization setup is as follows: ...@@ -19,7 +19,7 @@ Our current CI parallelization setup is as follows:
1. The `update-tests-metadata` job (which only runs on scheduled pipelines for 1. The `update-tests-metadata` job (which only runs on scheduled pipelines for
[the canonical project](https://gitlab.com/gitlab-org/gitlab) takes all the [the canonical project](https://gitlab.com/gitlab-org/gitlab) takes all the
`knapsack/rspec*_pg_*.json` files and merge them all together into a single `knapsack/rspec*_pg_*.json` files and merge them all together into a single
`knapsack/report-master.json` file that is saved as artifact. `knapsack/report-master.json` file that is then uploaded to S3.
After that, the next pipeline will use the up-to-date `knapsack/report-master.json` file. After that, the next pipeline will use the up-to-date `knapsack/report-master.json` file.
......
...@@ -4,11 +4,11 @@ function retrieve_tests_metadata() { ...@@ -4,11 +4,11 @@ function retrieve_tests_metadata() {
mkdir -p knapsack/ rspec_flaky/ rspec_profiling/ mkdir -p knapsack/ rspec_flaky/ rspec_profiling/
if [[ ! -f "${KNAPSACK_RSPEC_SUITE_REPORT_PATH}" ]]; then if [[ ! -f "${KNAPSACK_RSPEC_SUITE_REPORT_PATH}" ]]; then
wget -O "${KNAPSACK_RSPEC_SUITE_REPORT_PATH}" "https://gitlab.com/gitlab-org/gitlab/-/jobs/artifacts/master/raw/${KNAPSACK_RSPEC_SUITE_REPORT_PATH}?job=retrieve-tests-metadata" || echo "{}" > "${KNAPSACK_RSPEC_SUITE_REPORT_PATH}" wget -O "${KNAPSACK_RSPEC_SUITE_REPORT_PATH}" "http://${TESTS_METADATA_S3_BUCKET}.s3.amazonaws.com/${KNAPSACK_RSPEC_SUITE_REPORT_PATH}" || echo "{}" > "${KNAPSACK_RSPEC_SUITE_REPORT_PATH}"
fi fi
if [[ ! -f "${FLAKY_RSPEC_SUITE_REPORT_PATH}" ]]; then if [[ ! -f "${FLAKY_RSPEC_SUITE_REPORT_PATH}" ]]; then
wget -O "${FLAKY_RSPEC_SUITE_REPORT_PATH}" "https://gitlab.com/gitlab-org/gitlab/-/jobs/artifacts/master/raw/${FLAKY_RSPEC_SUITE_REPORT_PATH}?job=retrieve-tests-metadata" || echo "{}" > "${FLAKY_RSPEC_SUITE_REPORT_PATH}" wget -O "${FLAKY_RSPEC_SUITE_REPORT_PATH}" "http://${TESTS_METADATA_S3_BUCKET}.s3.amazonaws.com/${FLAKY_RSPEC_SUITE_REPORT_PATH}" || echo "{}" > "${FLAKY_RSPEC_SUITE_REPORT_PATH}"
fi fi
} }
...@@ -16,11 +16,29 @@ function update_tests_metadata() { ...@@ -16,11 +16,29 @@ function update_tests_metadata() {
echo "{}" > "${KNAPSACK_RSPEC_SUITE_REPORT_PATH}" echo "{}" > "${KNAPSACK_RSPEC_SUITE_REPORT_PATH}"
scripts/merge-reports "${KNAPSACK_RSPEC_SUITE_REPORT_PATH}" knapsack/rspec*.json scripts/merge-reports "${KNAPSACK_RSPEC_SUITE_REPORT_PATH}" knapsack/rspec*.json
if [[ -n "${TESTS_METADATA_S3_BUCKET}" ]]; then
if [[ "$CI_PIPELINE_SOURCE" == "schedule" ]]; then
scripts/sync-reports put "${TESTS_METADATA_S3_BUCKET}" "${KNAPSACK_RSPEC_SUITE_REPORT_PATH}"
else
echo "Not uplaoding report to S3 as the pipeline is not a scheduled one."
fi
fi
rm -f knapsack/rspec*.json rm -f knapsack/rspec*.json
export FLAKY_RSPEC_GENERATE_REPORT="true"
scripts/merge-reports "${FLAKY_RSPEC_SUITE_REPORT_PATH}" rspec_flaky/all_*.json scripts/merge-reports "${FLAKY_RSPEC_SUITE_REPORT_PATH}" rspec_flaky/all_*.json
export FLAKY_RSPEC_GENERATE_REPORT="true"
scripts/flaky_examples/prune-old-flaky-examples "${FLAKY_RSPEC_SUITE_REPORT_PATH}" scripts/flaky_examples/prune-old-flaky-examples "${FLAKY_RSPEC_SUITE_REPORT_PATH}"
if [[ -n ${TESTS_METADATA_S3_BUCKET} ]]; then
if [[ "$CI_PIPELINE_SOURCE" == "schedule" ]]; then
scripts/sync-reports put "${TESTS_METADATA_S3_BUCKET}" "${FLAKY_RSPEC_SUITE_REPORT_PATH}"
else
echo "Not uploading report to S3 as the pipeline is not a scheduled one."
fi
fi
rm -f rspec_flaky/all_*.json rspec_flaky/new_*.json rm -f rspec_flaky/all_*.json rspec_flaky/new_*.json
if [[ "$CI_PIPELINE_SOURCE" == "schedule" ]]; then if [[ "$CI_PIPELINE_SOURCE" == "schedule" ]]; then
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment