Commit 4592b03a authored by Nicolò Maria Mezzopera's avatar Nicolò Maria Mezzopera

Merge branch 'ps-add-generate-startup-css-script' into 'master'

Add generate:startup_css yarn script

See merge request gitlab-org/gitlab!62830
parents d78398ce c80cc1cd
......@@ -33,6 +33,8 @@
"lint:stylelint:fix": "yarn run lint:stylelint --fix",
"lint:stylelint:staged": "scripts/frontend/execute-on-staged-files.sh stylelint '(css|scss)' -q",
"lint:stylelint:staged:fix": "yarn run lint:stylelint:staged --fix",
"generate:startup_css": "scripts/frontend/startup_css/setup.sh && node scripts/frontend/startup_css/main.js",
"generate:startup_css:full": "scripts/frontend/startup_css/setup.sh force && node scripts/frontend/startup_css/main.js",
"markdownlint": "markdownlint --config .markdownlint.yml",
"markdownlint:no-trailing-spaces": "markdownlint --config doc/.markdownlint/markdownlint-no-trailing-spaces.yml",
"markdownlint:no-trailing-spaces:fix": "yarn run markdownlint:no-trailing-spaces --fix",
......@@ -200,6 +202,7 @@
"babel-plugin-dynamic-import-node": "^2.3.3",
"babel-plugin-istanbul": "^6.0.0",
"chalk": "^2.4.1",
"cheerio": "^1.0.0-rc.9",
"commander": "^2.18.0",
"custom-jquery-matchers": "^2.1.0",
"docdash": "^1.0.2",
......@@ -241,6 +244,7 @@
"nodemon": "^2.0.4",
"postcss": "^7.0.14",
"prettier": "2.2.1",
"purgecss": "^4.0.3",
"readdir-enhanced": "^2.2.4",
"sass": "^1.32.12",
"timezone-mock": "^1.0.8",
......
const { memoize, isString, isRegExp } = require('lodash');
const { parse } = require('postcss');
const { CSS_TO_REMOVE } = require('./constants');
const getSelectorRemoveTesters = memoize(() =>
CSS_TO_REMOVE.map((x) => {
if (isString(x)) {
return (selector) => x === selector;
}
if (isRegExp(x)) {
return (selector) => x.test(selector);
}
throw new Error(`Unexpected type in CSS_TO_REMOVE content "${x}". Expected String or RegExp.`);
}),
);
const getRemoveTesters = memoize(() => {
const selectorTesters = getSelectorRemoveTesters();
// These are mostly carried over from the previous project
// https://gitlab.com/gitlab-org/frontend/gitlab-css-statistics/-/blob/2aa00af25dba08fc71081c77206f45efe817ea4b/lib/gl_startup_extract.js
return [
(node) => node.type === 'comment',
(node) =>
node.type === 'atrule' &&
(node.params === 'print' ||
node.params === 'prefers-reduced-motion: reduce' ||
node.name === 'keyframe' ||
node.name === 'charset'),
(node) => node.selector && node.selectors && !node.selectors.length,
(node) => node.selector && selectorTesters.some((fn) => fn(node.selector)),
(node) =>
node.type === 'decl' &&
(node.prop === 'transition' ||
node.prop.indexOf('-webkit-') > -1 ||
node.prop.indexOf('-ms-') > -1),
];
});
const getNodesToRemove = (nodes) => {
const removeTesters = getRemoveTesters();
const remNodes = [];
nodes.forEach((node) => {
if (removeTesters.some((fn) => fn(node))) {
remNodes.push(node);
} else if (node.nodes?.length) {
remNodes.push(...getNodesToRemove(node.nodes));
}
});
return remNodes;
};
const getEmptyNodesToRemove = (nodes) =>
nodes
.filter((node) => node.nodes)
.reduce((acc, node) => {
if (node.nodes.length) {
acc.push(...getEmptyNodesToRemove(node.nodes));
} else {
acc.push(node);
}
return acc;
}, []);
const cleanCSS = (css) => {
const cssRoot = parse(css);
getNodesToRemove(cssRoot.nodes).forEach((node) => {
node.remove();
});
getEmptyNodesToRemove(cssRoot.nodes).forEach((node) => {
node.remove();
});
return cssRoot.toResult().css;
};
module.exports = { cleanCSS };
const path = require('path');
const IS_EE = require('../../../config/helpers/is_ee_env');
// controls --------------------------------------------------------------------
const HTML_TO_REMOVE = [
'style',
'script',
'link[rel="stylesheet"]',
'.content-wrapper',
'#js-peek',
'.modal',
'.feature-highlight',
// We don't want to capture all the children of a dropdown-menu
'.dropdown-menu',
];
const CSS_TO_REMOVE = [
'.tooltip',
'.tooltip.show',
'.fa',
'.gl-accessibility:focus',
'.toasted-container',
'body .toasted-container.bottom-left',
'.popover',
'.with-performance-bar .navbar-gitlab',
'.text-secondary',
/\.feature-highlight-popover-content/,
/\.commit/,
/\.md/,
/\.with-performance-bar/,
/\.identicon/,
];
const APPLICATION_CSS_PREFIX = 'application';
const APPLICATION_DARK_CSS_PREFIX = 'application_dark';
const UTILITIES_CSS_PREFIX = 'application_utilities';
const UTILITIES_DARK_CSS_PREFIX = 'application_utilities_dark';
// paths -----------------------------------------------------------------------
const ROOT = path.resolve(__dirname, '../../..');
const FIXTURES_FOLDER_NAME = IS_EE ? 'fixtures-ee' : 'fixtures';
const FIXTURES_ROOT = path.join(ROOT, 'tmp/tests/frontend', FIXTURES_FOLDER_NAME);
const PATH_SIGNIN_HTML = path.join(FIXTURES_ROOT, 'startup_css/sign-in.html');
const PATH_ASSETS = path.join(ROOT, 'tmp/startup_css_assets');
const PATH_STARTUP_SCSS = path.join(ROOT, 'app/assets/stylesheets/startup');
const OUTPUTS = [
{
outFile: 'startup-general',
htmlPaths: [
path.join(FIXTURES_ROOT, 'startup_css/project-general.html'),
path.join(FIXTURES_ROOT, 'startup_css/project-general-legacy-menu.html'),
path.join(FIXTURES_ROOT, 'startup_css/project-general-signed-out.html'),
],
cssKeys: [APPLICATION_CSS_PREFIX, UTILITIES_CSS_PREFIX],
// We want to include the root dropdown-menu style since it should be hidden by default
purgeOptions: {
safelist: {
standard: ['dropdown-menu'],
},
},
},
{
outFile: 'startup-dark',
htmlPaths: [
path.join(FIXTURES_ROOT, 'startup_css/project-dark.html'),
path.join(FIXTURES_ROOT, 'startup_css/project-dark-legacy-menu.html'),
path.join(FIXTURES_ROOT, 'startup_css/project-dark-signed-out.html'),
],
cssKeys: [APPLICATION_DARK_CSS_PREFIX, UTILITIES_DARK_CSS_PREFIX],
// We want to include the root dropdown-menu styles since it should be hidden by default
purgeOptions: {
safelist: {
standard: ['dropdown-menu'],
},
},
},
{
outFile: 'startup-signin',
htmlPaths: [PATH_SIGNIN_HTML],
cssKeys: [APPLICATION_CSS_PREFIX, UTILITIES_CSS_PREFIX],
purgeOptions: {
safelist: {
standard: ['fieldset'],
deep: [/login-page$/],
},
},
},
];
module.exports = {
HTML_TO_REMOVE,
CSS_TO_REMOVE,
APPLICATION_CSS_PREFIX,
APPLICATION_DARK_CSS_PREFIX,
UTILITIES_CSS_PREFIX,
UTILITIES_DARK_CSS_PREFIX,
ROOT,
PATH_ASSETS,
PATH_STARTUP_SCSS,
OUTPUTS,
};
const fs = require('fs');
const path = require('path');
const { memoize } = require('lodash');
const { PATH_ASSETS } = require('./constants');
const { die } = require('./utils');
const listAssetsDir = memoize(() => fs.readdirSync(PATH_ASSETS));
const getCSSPath = (prefix) => {
const matcher = new RegExp(`^${prefix}-[^-]+\\.css$`);
const cssPath = listAssetsDir().find((x) => matcher.test(x));
if (!cssPath) {
die(
`Could not find the CSS asset matching "${prefix}". Have you run "scripts/frontend/startup_css/setup.sh"?`,
);
}
return path.join(PATH_ASSETS, cssPath);
};
module.exports = { getCSSPath };
const fs = require('fs');
const cheerio = require('cheerio');
const { mergeWith, isArray } = require('lodash');
const { PurgeCSS } = require('purgecss');
const { cleanCSS } = require('./clean_css');
const { HTML_TO_REMOVE } = require('./constants');
const { die } = require('./utils');
const cleanHtml = (html) => {
const $ = cheerio.load(html);
HTML_TO_REMOVE.forEach((selector) => {
$(selector).remove();
});
return $.html();
};
const mergePurgeCSSOptions = (...options) =>
mergeWith(...options, (objValue, srcValue) => {
if (isArray(objValue)) {
return objValue.concat(srcValue);
}
return undefined;
});
const getStartupCSS = async ({ htmlPaths, cssPaths, purgeOptions }) => {
const content = htmlPaths.map((htmlPath) => {
if (!fs.existsSync(htmlPath)) {
die(`Could not find fixture "${htmlPath}". Have you run the fixtures?`);
}
const rawHtml = fs.readFileSync(htmlPath);
const html = cleanHtml(rawHtml);
return { raw: html, extension: 'html' };
});
const purgeCSSResult = await new PurgeCSS().purge({
content,
css: cssPaths,
...mergePurgeCSSOptions(
{
fontFace: true,
variables: true,
keyframes: true,
blocklist: [/:hover/, /:focus/, /-webkit-/, /-moz-focusring-/, /-ms-expand/],
safelist: {
standard: ['brand-header-logo'],
},
// By default, PurgeCSS ignores special characters, but our utilities use "!"
defaultExtractor: (x) => x.match(/[\w-!]+/g),
},
purgeOptions,
),
});
return purgeCSSResult.map(({ css }) => cleanCSS(css)).join('\n');
};
module.exports = { getStartupCSS };
const { memoize } = require('lodash');
const { OUTPUTS } = require('./constants');
const { getCSSPath } = require('./get_css_path');
const { getStartupCSS } = require('./get_startup_css');
const { log, die } = require('./utils');
const { writeStartupSCSS } = require('./write_startup_scss');
const memoizedCSSPath = memoize(getCSSPath);
const runTask = async ({ outFile, htmlPaths, cssKeys, purgeOptions = {} }) => {
try {
log(`Generating startup CSS for HTML files: ${htmlPaths}`);
const generalCSS = await getStartupCSS({
htmlPaths,
cssPaths: cssKeys.map(memoizedCSSPath),
purgeOptions,
});
log(`Writing to startup CSS...`);
const startupCSSPath = writeStartupSCSS(outFile, generalCSS);
log(`Finished writing to ${startupCSSPath}`);
return {
success: true,
outFile,
};
} catch (e) {
log(`ERROR! Unexpected error occurred while generating startup CSS for: ${outFile}`);
log(e);
return {
success: false,
outFile,
};
}
};
const main = async () => {
const result = await Promise.all(OUTPUTS.map(runTask));
const fullSuccess = result.every((x) => x.success);
log('RESULTS:');
log('--------');
result.forEach(({ success, outFile }) => {
const status = success ? '' : '';
log(`${status}: ${outFile}`);
});
log('--------');
if (fullSuccess) {
log('Done!');
} else {
die('Some tasks have failed');
}
};
main();
path_public_dir="public"
path_tmp="tmp"
path_dest="$path_tmp/startup_css_assets"
glob_css_dest="$path_dest/application*.css"
glob_css_src="$path_public_dir/assets/application*.css"
should_clean=false
should_force() {
$1=="force"
}
has_dest_already() {
find $glob_css_dest -quit
}
has_src_already() {
find $glob_css_src -quit
}
compile_assets() {
# We need to build the same test bundle that is built in CI
RAILS_ENV=test bundle exec rake rake:assets:precompile
}
clean_assets() {
bundle exec rake rake:assets:clobber
}
copy_assets() {
rm -rf $path_dest
mkdir $path_dest
cp $glob_css_src $path_dest
}
echo "-----------------------------------------------------------"
echo "If you are run into any issues with Startup CSS generation,"
echo "please check out the feedback issue:"
echo ""
echo "https://gitlab.com/gitlab-org/gitlab/-/issues/331812"
echo "-----------------------------------------------------------"
if [ ! -e $path_public_dir ]; then
echo "Could not find '$path_public_dir/'. This script must be run in the root directory of the gitlab project."
exit 1
fi
if [ ! -e $path_tmp ]; then
echo "Could not find '$path_tmp/'. This script must be run in the root directory of the gitlab project."
exit 1
fi
if [ "$1" != "force" ] && has_dest_already; then
echo "Already found assets for '$glob_css_dest'. Did you want to run this script with 'force' argument?"
exit 0
fi
# If we are in CI, don't recompile things...
if [ -n "$CI" ]; then
if ! has_src_already; then
echo "Could not find '$glob_css_src'. Expected these artifacts to be generated by CI pipeline."
exit 1
fi
elif has_src_already; then
echo "Found '$glob_css_src'. Skipping compile assets..."
else
echo "Starting compile assets process..."
compile_assets
should_clean=true
fi
copy_assets
if $should_clean; then
echo "Starting cleanup..."
clean_assets
fi
const die = (message) => {
console.log(message);
process.exit(1);
};
const log = (message) => console.error(`[gitlab.startup_css] ${message}`);
module.exports = { die, log };
const { writeFileSync } = require('fs');
const path = require('path');
const prettier = require('prettier');
const { PATH_STARTUP_SCSS } = require('./constants');
const buildFinalContent = (raw) => {
const content = `// DO NOT EDIT! This is auto-generated from "yarn run generate:startup_css"
// Please see the feedback issue for more details and help:
// https://gitlab.com/gitlab-org/gitlab/-/issues/331812
@charset "UTF-8";
${raw}
@import 'cloaking';
@include cloak-startup-scss(none);
`;
// We run prettier so that there is more determinism with the generated file.
return prettier.format(content, { parser: 'scss' });
};
const writeStartupSCSS = (name, raw) => {
const fullPath = path.join(PATH_STARTUP_SCSS, `${name}.scss`);
writeFileSync(fullPath, buildFinalContent(raw));
return fullPath;
};
module.exports = { writeStartupSCSS };
# frozen_string_literal: true
require 'spec_helper'
RSpec.describe 'Startup CSS fixtures', type: :controller do
include JavaScriptFixturesHelpers
let(:use_full_html) { true }
render_views
before(:all) do
stub_feature_flags(combined_menu: true)
clean_frontend_fixtures('startup_css/')
end
shared_examples 'startup css project fixtures' do |type|
let(:user) { create(:user, :admin) }
let(:project) { create(:project, :public, :repository, description: 'Code and stuff', avatar: fixture_file_upload('spec/fixtures/dk.png', 'image/png'), creator: user) }
before do
sign_in(user)
end
it "startup_css/project-#{type}-legacy-menu.html" do
stub_feature_flags(combined_menu: false)
get :show, params: {
namespace_id: project.namespace.to_param,
id: project
}
expect(response).to be_successful
end
it "startup_css/project-#{type}.html" do
get :show, params: {
namespace_id: project.namespace.to_param,
id: project
}
expect(response).to be_successful
end
it "startup_css/project-#{type}-signed-out.html" do
sign_out(user)
get :show, params: {
namespace_id: project.namespace.to_param,
id: project
}
expect(response).to be_successful
end
end
describe ProjectsController, '(Startup CSS fixtures)', type: :controller do
it_behaves_like 'startup css project fixtures', 'general'
end
describe ProjectsController, '(Startup CSS fixtures)', type: :controller do
before do
user.update!(theme_id: 11)
end
it_behaves_like 'startup css project fixtures', 'dark'
end
describe RegistrationsController, '(Startup CSS fixtures)', type: :controller do
it 'startup_css/sign-in.html' do
get :new
expect(response).to be_successful
end
end
end
......@@ -66,6 +66,14 @@ module JavaScriptFixturesHelpers
File.write(full_fixture_path, fixture)
end
def parse_html(fixture)
if respond_to?(:use_full_html) && public_send(:use_full_html)
Nokogiri::HTML::Document.parse(fixture)
else
Nokogiri::HTML::DocumentFragment.parse(fixture)
end
end
# Private: Prepare a response object for use as a frontend fixture
#
# response - response object to prepare
......@@ -76,7 +84,7 @@ module JavaScriptFixturesHelpers
response_mime_type = Mime::Type.lookup(response.media_type)
if response_mime_type.html?
doc = Nokogiri::HTML::DocumentFragment.parse(fixture)
doc = parse_html(fixture)
link_tags = doc.css('link')
link_tags.remove
......
This diff is collapsed.
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment