Skip to content

G
gitlab-ce
Commit e4a977c7 authored by Rémy Coutable's avatar Rémy Coutable
Browse files
Options

Use 'rules' for '.gitlab/ci/reports.gitlab-ci.yml' jobs 

Signed-off-by: default avatarRémy Coutable <remy@rymai.me>
parent b9940c46
Hide whitespace changes
Inline Side-by-side
Showing with 176 additions and 105 deletions
.gitlab/ci/reports.gitlab-ci.yml
View file @ e4a977c7
# Make sure to update all the similar conditions in other CI config files if you modify these conditions
.if-canonical-gitlab-merge-request: &if-canonical-gitlab-merge-request
if: '$CI_SERVER_HOST == "gitlab.com" && $CI_PROJECT_NAMESPACE == "gitlab-org" && $CI_MERGE_REQUEST_IID'
# Make sure to update all the similar conditions in other CI config files if you modify these conditions
.if-canonical-dot-com-gitlab-org-group-schedule: &if-canonical-dot-com-gitlab-org-group-schedule
if: '$CI_SERVER_HOST == "gitlab.com" && $CI_PROJECT_NAMESPACE == "gitlab-org" && $CI_PIPELINE_SOURCE == "schedule"'
# Make sure to update all the similar conditions in other CI config files if you modify these conditions
.if-master-refs: &if-master-refs
if: '$CI_COMMIT_REF_NAME == "master"'
# Make sure to update all the similar conditions in other CI config files if you modify these conditions
.if-default-refs: &if-default-refs
if: '$CI_COMMIT_REF_NAME == "master" || $CI_COMMIT_REF_NAME =~ /^[\d-]+-stable(-ee)?$/ || $CI_COMMIT_REF_NAME =~ /^\d+-\d+-auto-deploy-\d+$/ || $CI_COMMIT_REF_NAME =~ /^security\// || $CI_MERGE_REQUEST_IID || $CI_COMMIT_TAG'
# Make sure to update all the similar patterns in other CI config files if you modify these patterns
.code-backstage-patterns: &code-backstage-patterns
- ".gitlab/ci/**/*"
- ".{eslintignore,gitattributes,nvmrc,prettierrc,stylelintrc,yamllint}"
- ".{codeclimate,eslintrc,gitlab-ci,haml-lint,haml-lint_todo,rubocop,rubocop_todo,scss-lint}.yml"
- ".csscomb.json"
- "Dockerfile.assets"
- "*_VERSION"
- "Gemfile{,.lock}"
- "Rakefile"
- "{babel.config,jest.config}.js"
- "config.ru"
- "{package.json,yarn.lock}"
- "{,ee/}{app,bin,config,db,haml_lint,lib,locale,public,scripts,symbol,vendor}/**/*"
- "doc/api/graphql/reference/*" # Files in this folder are auto-generated
# Backstage changes
- "Dangerfile"
- "danger/**/*"
- "{,ee/}fixtures/**/*"
- "{,ee/}rubocop/**/*"
- "{,ee/}spec/**/*"
- "doc/README.md" # Some RSpec test rely on this file
# Make sure to update all the similar patterns in other CI config files if you modify these patterns
.code-qa-patterns: &code-qa-patterns
- ".gitlab/ci/**/*"
- ".{eslintignore,gitattributes,nvmrc,prettierrc,stylelintrc,yamllint}"
- ".{codeclimate,eslintrc,gitlab-ci,haml-lint,haml-lint_todo,rubocop,rubocop_todo,scss-lint}.yml"
- ".csscomb.json"
- "Dockerfile.assets"
- "*_VERSION"
- "Gemfile{,.lock}"
- "Rakefile"
- "{babel.config,jest.config}.js"
- "config.ru"
- "{package.json,yarn.lock}"
- "{,ee/}{app,bin,config,db,haml_lint,lib,locale,public,scripts,symbol,vendor}/**/*"
- "doc/api/graphql/reference/*" # Files in this folder are auto-generated
# QA changes
- ".dockerignore"
- "qa/**/*"
# Make sure to update all the similar patterns in other CI config files if you modify these patterns
.code-backstage-qa-patterns: &code-backstage-qa-patterns
- ".gitlab/ci/**/*"
- ".{eslintignore,gitattributes,nvmrc,prettierrc,stylelintrc,yamllint}"
- ".{codeclimate,eslintrc,gitlab-ci,haml-lint,haml-lint_todo,rubocop,rubocop_todo,scss-lint}.yml"
- ".csscomb.json"
- "Dockerfile.assets"
- "*_VERSION"
- "Gemfile{,.lock}"
- "Rakefile"
- "{babel.config,jest.config}.js"
- "config.ru"
- "{package.json,yarn.lock}"
- "{,ee/}{app,bin,config,db,haml_lint,lib,locale,public,scripts,symbol,vendor}/**/*"
- "doc/api/graphql/reference/*" # Files in this folder are auto-generated
# Backstage changes
- "Dangerfile"
- "danger/**/*"
- "{,ee/}fixtures/**/*"
- "{,ee/}rubocop/**/*"
- "{,ee/}spec/**/*"
- "doc/README.md" # Some RSpec test rely on this file
# QA changes
- ".dockerignore"
- "qa/**/*"
.reports:rules:code_quality:
rules:
- if: '$CODE_QUALITY_DISABLED'
when: never
# - <<: *if-master-refs # To be done in a later iteration: https://gitlab.com/gitlab-org/gitlab/issues/31160#note_278188255
- <<: *if-default-refs
changes: *code-backstage-patterns
.reports:rules:sast:
rules:
- if: '$SAST_DISABLED || $GITLAB_FEATURES !~ /\bsast\b/'
when: never
# - <<: *if-master-refs # To be done in a later iteration: https://gitlab.com/gitlab-org/gitlab/issues/31160#note_278188255
- <<: *if-default-refs
changes: *code-backstage-qa-patterns
.reports:rules:dependency_scanning:
rules:
- if: '$DEPENDENCY_SCANNING_DISABLED || $GITLAB_FEATURES !~ /\bdependency_scanning\b/'
when: never
# - <<: *if-master-refs # To be done in a later iteration: https://gitlab.com/gitlab-org/gitlab/issues/31160#note_278188255
- <<: *if-default-refs
changes: *code-backstage-qa-patterns
.reports:rules:dast:
rules:
- if: '$DAST_DISABLED || $GITLAB_FEATURES !~ /\bdast\b/'
when: never
- <<: *if-canonical-gitlab-merge-request
changes: *code-qa-patterns
# include: # include:
# - template: Code-Quality.gitlab-ci.yml # - template: Jobs/Code-Quality.gitlab-ci.yml
# - template: Security/SAST.gitlab-ci.yml # - template: Security/SAST.gitlab-ci.yml
# - template: Security/Dependency-Scanning.gitlab-ci.yml # - template: Security/Dependency-Scanning.gitlab-ci.yml
# - template: Security/DAST.gitlab-ci.yml # - template: Security/DAST.gitlab-ci.yml
...@@ -10,8 +125,7 @@ ...@@ -10,8 +125,7 @@
code_quality: code_quality:
extends: extends:
- .default-retry - .default-retry
- .default-only - .reports:rules:code_quality
- .only:changes-code-backstage
stage: test stage: test
image: docker:stable image: docker:stable
allow_failure: true allow_failure: true
...@@ -38,12 +152,9 @@ code_quality: ...@@ -38,12 +152,9 @@ code_quality:
reports: reports:
codequality: gl-code-quality-report.json codequality: gl-code-quality-report.json
paths: paths:
- gl-code-quality-report.json - gl-code-quality-report.json # GitLab-specific
expire_in: 1 week expire_in: 1 week # GitLab-specific
dependencies: [] dependencies: []
except:
variables:
- $CODE_QUALITY_DISABLED
# We need to duplicate this job's definition because it seems it's impossible to # We need to duplicate this job's definition because it seems it's impossible to
# override an included `only.refs`. # override an included `only.refs`.
...@@ -53,16 +164,22 @@ code_quality: ...@@ -53,16 +164,22 @@ code_quality:
sast: sast:
extends: extends:
- .default-retry - .default-retry
- .default-only - .reports:rules:sast
- .only:changes-code-backstage-qa
stage: test stage: test
allow_failure: true
dependencies: [] # GitLab-specific
artifacts:
paths:
- gl-sast-report.json # GitLab-specific
reports:
sast: gl-sast-report.json
expire_in: 1 week # GitLab-specific
image: docker:stable image: docker:stable
variables: variables:
DOCKER_DRIVER: overlay2 DOCKER_DRIVER: overlay2
DOCKER_TLS_CERTDIR: "" DOCKER_TLS_CERTDIR: ""
SAST_BRAKEMAN_LEVEL: 2 SAST_BRAKEMAN_LEVEL: 2 # GitLab-specific
SAST_EXCLUDED_PATHS: qa,spec,doc,ee/spec SAST_EXCLUDED_PATHS: qa,spec,doc,ee/spec # GitLab-specific
allow_failure: true
services: services:
- docker:stable-dind - docker:stable-dind
script: script:
...@@ -73,61 +190,12 @@ sast: ...@@ -73,61 +190,12 @@ sast:
export DOCKER_HOST='tcp://localhost:2375' export DOCKER_HOST='tcp://localhost:2375'
fi fi
fi fi
- | # this is required to avoid undesirable reset of Docker image ENV variables being set on build stage
function propagate_env_vars() {
CURRENT_ENV=$(printenv)
for VAR_NAME; do
echo $CURRENT_ENV | grep "${VAR_NAME}=" > /dev/null && echo "--env $VAR_NAME "
done
}
- | - |
docker run \ ENVS=`printenv | grep -vE '^(DOCKER_|CI|GITLAB_|FF_|HOME|PWD|OLDPWD|PATH|SHLVL|HOSTNAME)' | sed -n '/^[^\t]/s/=.*//p' | sed '/^$/d' | sed 's/^/-e /g' | tr '\n' ' '`
$(propagate_env_vars \ docker run "$ENVS" \
SAST_BANDIT_EXCLUDED_PATHS \
SAST_ANALYZER_IMAGES \
SAST_ANALYZER_IMAGE_PREFIX \
SAST_ANALYZER_IMAGE_TAG \
SAST_DEFAULT_ANALYZERS \
SAST_PULL_ANALYZER_IMAGES \
SAST_BRAKEMAN_LEVEL \
SAST_FLAWFINDER_LEVEL \
SAST_GITLEAKS_ENTROPY_LEVEL \
SAST_GOSEC_LEVEL \
SAST_EXCLUDED_PATHS \
SAST_DOCKER_CLIENT_NEGOTIATION_TIMEOUT \
SAST_PULL_ANALYZER_IMAGE_TIMEOUT \
SAST_RUN_ANALYZER_TIMEOUT \
SAST_JAVA_VERSION \
ANT_HOME \
ANT_PATH \
GRADLE_PATH \
JAVA_OPTS \
JAVA_PATH \
JAVA_8_VERSION \
JAVA_11_VERSION \
MAVEN_CLI_OPTS \
MAVEN_PATH \
MAVEN_REPO_PATH \
SBT_PATH \
FAIL_NEVER \
) \
--volume "$PWD:/code" \ --volume "$PWD:/code" \
--volume /var/run/docker.sock:/var/run/docker.sock \ --volume /var/run/docker.sock:/var/run/docker.sock \
"registry.gitlab.com/gitlab-org/security-products/sast:$SAST_VERSION" /app/bin/run /code "registry.gitlab.com/gitlab-org/security-products/sast:$SAST_VERSION" /app/bin/run /code
artifacts:
expire_in: 7 days
paths:
- gl-sast-report.json
reports:
sast: gl-sast-report.json
dependencies: []
only:
variables:
- $GITLAB_FEATURES =~ /\bsast\b/
except:
variables:
- $SAST_DISABLED
# We need to duplicate this job's definition because it seems it's impossible to # We need to duplicate this job's definition because it seems it's impossible to
# override an included `only.refs`. # override an included `only.refs`.
...@@ -135,14 +203,13 @@ sast: ...@@ -135,14 +203,13 @@ sast:
dependency_scanning: dependency_scanning:
extends: extends:
- .default-retry - .default-retry
- .default-only - .reports:rules:dependency_scanning
- .only:changes-code-backstage-qa
stage: test stage: test
image: docker:stable image: docker:stable
variables: variables:
DOCKER_DRIVER: overlay2 DOCKER_DRIVER: overlay2
DOCKER_TLS_CERTDIR: "" DOCKER_TLS_CERTDIR: ""
DS_EXCLUDED_PATHS: "qa/qa/ee/fixtures/secure_premade_reports,spec,ee/spec" DS_EXCLUDED_PATHS: "qa/qa/ee/fixtures/secure_premade_reports,spec,ee/spec" # GitLab-specific
allow_failure: true allow_failure: true
services: services:
- docker:stable-dind - docker:stable-dind
...@@ -174,23 +241,29 @@ dependency_scanning: ...@@ -174,23 +241,29 @@ dependency_scanning:
DS_PULL_ANALYZER_IMAGE_TIMEOUT \ DS_PULL_ANALYZER_IMAGE_TIMEOUT \
DS_RUN_ANALYZER_TIMEOUT \ DS_RUN_ANALYZER_TIMEOUT \
DS_PYTHON_VERSION \ DS_PYTHON_VERSION \
DS_PIP_VERSION \
DS_PIP_DEPENDENCY_PATH \ DS_PIP_DEPENDENCY_PATH \
GEMNASIUM_DB_LOCAL_PATH \
GEMNASIUM_DB_REMOTE_URL \
GEMNASIUM_DB_REF_NAME \
PIP_INDEX_URL \ PIP_INDEX_URL \
PIP_EXTRA_INDEX_URL \ PIP_EXTRA_INDEX_URL \
PIP_REQUIREMENTS_FILE \
MAVEN_CLI_OPTS \
BUNDLER_AUDIT_UPDATE_DISABLED \
BUNDLER_AUDIT_ADVISORY_DB_URL \
BUNDLER_AUDIT_ADVISORY_DB_REF_NAME \
) \ ) \
--volume "$PWD:/code" \ --volume "$PWD:/code" \
--volume /var/run/docker.sock:/var/run/docker.sock \ --volume /var/run/docker.sock:/var/run/docker.sock \
"registry.gitlab.com/gitlab-org/security-products/dependency-scanning:$DS_VERSION" /code "registry.gitlab.com/gitlab-org/security-products/dependency-scanning:$DS_VERSION" /code
artifacts: artifacts:
paths:
- gl-dependency-scanning-report.json # GitLab-specific
reports: reports:
dependency_scanning: gl-dependency-scanning-report.json dependency_scanning: gl-dependency-scanning-report.json
expire_in: 1 week # GitLab-specific
dependencies: [] dependencies: []
only:
variables:
- $GITLAB_FEATURES =~ /\bdependency_scanning\b/
except:
variables:
- $DEPENDENCY_SCANNING_DISABLED
# We need to duplicate this job's definition because it seems it's impossible to # We need to duplicate this job's definition because it seems it's impossible to
# override an included `only.refs`. # override an included `only.refs`.
...@@ -198,40 +271,38 @@ dependency_scanning: ...@@ -198,40 +271,38 @@ dependency_scanning:
dast: dast:
extends: extends:
- .default-retry - .default-retry
- .default-only - .reports:rules:dast
- .only:changes-code-qa needs:
- .only-review - job: review-deploy
stage: qa artifacts: true
needs: ["review-deploy"] stage: qa # GitLab-specific
dependencies: ["review-deploy"]
before_script:
- export DAST_WEBSITE="$(cat review_app_url.txt)"
image: image:
name: "registry.gitlab.com/gitlab-org/security-products/dast:$CI_SERVER_VERSION_MAJOR-$CI_SERVER_VERSION_MINOR-stable" name: "registry.gitlab.com/gitlab-org/security-products/dast:$DAST_VERSION"
variables: variables:
# URL to scan: # To be done in a later iteration
# DAST_WEBSITE: https://example.com/ # DAST_USERNAME: "root"
# # DAST_USERNAME_FIELD: "user[login]"
# Time limit for target availability (scan is attempted even when timeout): # DAST_PASSWORD_FIELD: "user[passowrd]"
# DAST_TARGET_AVAILABILITY_TIMEOUT: 60
#
# Set these variables to scan with an authenticated user:
# DAST_AUTH_URL: https://example.com/sign-in
# DAST_USERNAME: john.doe@example.com
# DAST_PASSWORD: john-doe-password
# DAST_USERNAME_FIELD: session[user] # the name of username field at the sign-in HTML form
# DAST_PASSWORD_FIELD: session[password] # the name of password field at the sign-in HTML form
# DAST_AUTH_EXCLUDE_URLS: http://example.com/sign-out,http://example.com/sign-out-2 # optional: URLs to skip during the authenticated scan; comma-separated, no spaces in between
#
# Perform ZAP Full Scan, which includes both passive and active scanning:
# DAST_FULL_SCAN_ENABLED: "true"
allow_failure: true allow_failure: true
script: script:
- export DAST_WEBSITE=${DAST_WEBSITE:-$(cat environment_url.txt)} - 'export DAST_WEBSITE="${DAST_WEBSITE:-$(cat environment_url.txt)}"'
# To be done in a later iteration
# - 'export DAST_AUTH_URL="${DAST_WEBSITE}/users/sign_in"'
# - 'export DAST_PASSWORD="${REVIEW_APPS_ROOT_PASSWORD}"'
- /analyze -t $DAST_WEBSITE - /analyze -t $DAST_WEBSITE
artifacts: artifacts:
expire_in: 7 days
paths: paths:
- gl-dast-report.json - gl-dast-report.json # GitLab-specific
reports: reports:
dast: gl-dast-report.json dast: gl-dast-report.json
expire_in: 1 week # GitLab-specific
# To be done in a later iteration: https://gitlab.com/gitlab-org/gitlab/issues/31160#note_278188255
# schedule:dast:
# extends: dast
# rules:
# - if: '$DAST_DISABLED || $GITLAB_FEATURES !~ /\bdast\b/'
# when: never
# - <<: *if-canonical-dot-com-gitlab-org-group-schedule
# variables:
# DAST_FULL_SCAN_ENABLED: "true"
.gitlab/ci/review.gitlab-ci.yml
View file @ e4a977c7
...@@ -148,7 +148,7 @@ review-deploy: ...@@ -148,7 +148,7 @@ review-deploy:
- export GITLAB_SHELL_VERSION=$(<GITLAB_SHELL_VERSION) - export GITLAB_SHELL_VERSION=$(<GITLAB_SHELL_VERSION)
- export GITALY_VERSION=$(<GITALY_SERVER_VERSION) - export GITALY_VERSION=$(<GITALY_SERVER_VERSION)
- export GITLAB_WORKHORSE_VERSION=$(<GITLAB_WORKHORSE_VERSION) - export GITLAB_WORKHORSE_VERSION=$(<GITLAB_WORKHORSE_VERSION)
- echo "${CI_ENVIRONMENT_URL}" > review_app_url.txt - echo "${CI_ENVIRONMENT_URL}" > environment_url.txt
- source ./scripts/utils.sh - source ./scripts/utils.sh
- install_api_client_dependencies_with_apk - install_api_client_dependencies_with_apk
- source scripts/review_apps/review-apps.sh - source scripts/review_apps/review-apps.sh
...@@ -161,7 +161,7 @@ review-deploy: ...@@ -161,7 +161,7 @@ review-deploy:
- date - date
- deploy || (display_deployment_debug && exit 1) - deploy || (display_deployment_debug && exit 1)
artifacts: artifacts:
paths: [review_app_url.txt] paths: [environment_url.txt]
expire_in: 2 days expire_in: 2 days
when: always when: always
...@@ -216,7 +216,7 @@ review-stop: ...@@ -216,7 +216,7 @@ review-stop:
before_script: before_script:
- '[[ ! -d "ee/" ]] || export GITLAB_EDITION="ee"' - '[[ ! -d "ee/" ]] || export GITLAB_EDITION="ee"'
- export QA_IMAGE="${CI_REGISTRY}/${CI_PROJECT_PATH}/gitlab/gitlab-${GITLAB_EDITION}-qa:${CI_COMMIT_REF_SLUG}" - export QA_IMAGE="${CI_REGISTRY}/${CI_PROJECT_PATH}/gitlab/gitlab-${GITLAB_EDITION}-qa:${CI_COMMIT_REF_SLUG}"
- export CI_ENVIRONMENT_URL="$(cat review_app_url.txt)" - export CI_ENVIRONMENT_URL="$(cat environment_url.txt)"
- echo "${CI_ENVIRONMENT_URL}" - echo "${CI_ENVIRONMENT_URL}"
- echo "${QA_IMAGE}" - echo "${QA_IMAGE}"
- source scripts/utils.sh - source scripts/utils.sh
...@@ -255,7 +255,7 @@ review-performance: ...@@ -255,7 +255,7 @@ review-performance:
artifacts: true artifacts: true
allow_failure: true allow_failure: true
before_script: before_script:
- export CI_ENVIRONMENT_URL="$(cat review_app_url.txt)" - export CI_ENVIRONMENT_URL="$(cat environment_url.txt)"
- echo "${CI_ENVIRONMENT_URL}" - echo "${CI_ENVIRONMENT_URL}"
- mkdir -p gitlab-exporter - mkdir -p gitlab-exporter
- wget -O ./gitlab-exporter/index.js https://gitlab.com/gitlab-org/gl-performance/raw/master/index.js - wget -O ./gitlab-exporter/index.js https://gitlab.com/gitlab-org/gl-performance/raw/master/index.js
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7