Merge branch 'ps-add-generate-startup-css-script' into 'master'

Add generate:startup_css yarn script

See merge request gitlab-org/gitlab!62830

package.json

@@ -33,6 +33,8 @@
     "lint:stylelint:fix": "yarn run lint:stylelint --fix",
     "lint:stylelint:staged": "scripts/frontend/execute-on-staged-files.sh stylelint '(css|scss)' -q",
     "lint:stylelint:staged:fix": "yarn run lint:stylelint:staged --fix",
+    "generate:startup_css": "scripts/frontend/startup_css/setup.sh && node scripts/frontend/startup_css/main.js",
+    "generate:startup_css:full": "scripts/frontend/startup_css/setup.sh force && node scripts/frontend/startup_css/main.js",
     "markdownlint": "markdownlint --config .markdownlint.yml",
     "markdownlint:no-trailing-spaces": "markdownlint --config doc/.markdownlint/markdownlint-no-trailing-spaces.yml",
     "markdownlint:no-trailing-spaces:fix": "yarn run markdownlint:no-trailing-spaces --fix",
@@ -200,6 +202,7 @@
     "babel-plugin-dynamic-import-node": "^2.3.3",
     "babel-plugin-istanbul": "^6.0.0",
     "chalk": "^2.4.1",
+    "cheerio": "^1.0.0-rc.9",
     "commander": "^2.18.0",
     "custom-jquery-matchers": "^2.1.0",
     "docdash": "^1.0.2",
@@ -241,6 +244,7 @@
     "nodemon": "^2.0.4",
     "postcss": "^7.0.14",
     "prettier": "2.2.1",
+    "purgecss": "^4.0.3",
     "readdir-enhanced": "^2.2.4",
     "sass": "^1.32.12",
     "timezone-mock": "^1.0.8",

scripts/frontend/startup_css/clean_css.js

+const { memoize, isString, isRegExp } = require('lodash');
+const { parse } = require('postcss');
+const { CSS_TO_REMOVE } = require('./constants');
+
+const getSelectorRemoveTesters = memoize(() =>
+  CSS_TO_REMOVE.map((x) => {
+    if (isString(x)) {
+      return (selector) => x === selector;
+    }
+    if (isRegExp(x)) {
+      return (selector) => x.test(selector);
+    }
+
+    throw new Error(`Unexpected type in CSS_TO_REMOVE content "${x}". Expected String or RegExp.`);
+  }),
+);
+
+const getRemoveTesters = memoize(() => {
+  const selectorTesters = getSelectorRemoveTesters();
+
+  // These are mostly carried over from the previous project
+  // https://gitlab.com/gitlab-org/frontend/gitlab-css-statistics/-/blob/2aa00af25dba08fc71081c77206f45efe817ea4b/lib/gl_startup_extract.js
+  return [
+    (node) => node.type === 'comment',
+    (node) =>
+      node.type === 'atrule' &&
+      (node.params === 'print' ||
+        node.params === 'prefers-reduced-motion: reduce' ||
+        node.name === 'keyframe' ||
+        node.name === 'charset'),
+    (node) => node.selector && node.selectors && !node.selectors.length,
+    (node) => node.selector && selectorTesters.some((fn) => fn(node.selector)),
+    (node) =>
+      node.type === 'decl' &&
+      (node.prop === 'transition' ||
+        node.prop.indexOf('-webkit-') > -1 ||
+        node.prop.indexOf('-ms-') > -1),
+  ];
+});
+
+const getNodesToRemove = (nodes) => {
+  const removeTesters = getRemoveTesters();
+  const remNodes = [];
+
+  nodes.forEach((node) => {
+    if (removeTesters.some((fn) => fn(node))) {
+      remNodes.push(node);
+    } else if (node.nodes?.length) {
+      remNodes.push(...getNodesToRemove(node.nodes));
+    }
+  });
+
+  return remNodes;
+};
+
+const getEmptyNodesToRemove = (nodes) =>
+  nodes
+    .filter((node) => node.nodes)
+    .reduce((acc, node) => {
+      if (node.nodes.length) {
+        acc.push(...getEmptyNodesToRemove(node.nodes));
+      } else {
+        acc.push(node);
+      }
+
+      return acc;
+    }, []);
+
+const cleanCSS = (css) => {
+  const cssRoot = parse(css);
+
+  getNodesToRemove(cssRoot.nodes).forEach((node) => {
+    node.remove();
+  });
+
+  getEmptyNodesToRemove(cssRoot.nodes).forEach((node) => {
+    node.remove();
+  });
+
+  return cssRoot.toResult().css;
+};
+
+module.exports = { cleanCSS };

scripts/frontend/startup_css/constants.js

+const path = require('path');
+const IS_EE = require('../../../config/helpers/is_ee_env');
+
+// controls --------------------------------------------------------------------
+const HTML_TO_REMOVE = [
+  'style',
+  'script',
+  'link[rel="stylesheet"]',
+  '.content-wrapper',
+  '#js-peek',
+  '.modal',
+  '.feature-highlight',
+  // We don't want to capture all the children of a dropdown-menu
+  '.dropdown-menu',
+];
+const CSS_TO_REMOVE = [
+  '.tooltip',
+  '.tooltip.show',
+  '.fa',
+  '.gl-accessibility:focus',
+  '.toasted-container',
+  'body .toasted-container.bottom-left',
+  '.popover',
+  '.with-performance-bar .navbar-gitlab',
+  '.text-secondary',
+  /\.feature-highlight-popover-content/,
+  /\.commit/,
+  /\.md/,
+  /\.with-performance-bar/,
+  /\.identicon/,
+];
+const APPLICATION_CSS_PREFIX = 'application';
+const APPLICATION_DARK_CSS_PREFIX = 'application_dark';
+const UTILITIES_CSS_PREFIX = 'application_utilities';
+const UTILITIES_DARK_CSS_PREFIX = 'application_utilities_dark';
+
+// paths -----------------------------------------------------------------------
+const ROOT = path.resolve(__dirname, '../../..');
+const FIXTURES_FOLDER_NAME = IS_EE ? 'fixtures-ee' : 'fixtures';
+const FIXTURES_ROOT = path.join(ROOT, 'tmp/tests/frontend', FIXTURES_FOLDER_NAME);
+const PATH_SIGNIN_HTML = path.join(FIXTURES_ROOT, 'startup_css/sign-in.html');
+const PATH_ASSETS = path.join(ROOT, 'tmp/startup_css_assets');
+const PATH_STARTUP_SCSS = path.join(ROOT, 'app/assets/stylesheets/startup');
+const OUTPUTS = [
+  {
+    outFile: 'startup-general',
+    htmlPaths: [
+      path.join(FIXTURES_ROOT, 'startup_css/project-general.html'),
+      path.join(FIXTURES_ROOT, 'startup_css/project-general-legacy-menu.html'),
+      path.join(FIXTURES_ROOT, 'startup_css/project-general-signed-out.html'),
+    ],
+    cssKeys: [APPLICATION_CSS_PREFIX, UTILITIES_CSS_PREFIX],
+    // We want to include the root dropdown-menu style since it should be hidden by default
+    purgeOptions: {
+      safelist: {
+        standard: ['dropdown-menu'],
+      },
+    },
+  },
+  {
+    outFile: 'startup-dark',
+    htmlPaths: [
+      path.join(FIXTURES_ROOT, 'startup_css/project-dark.html'),
+      path.join(FIXTURES_ROOT, 'startup_css/project-dark-legacy-menu.html'),
+      path.join(FIXTURES_ROOT, 'startup_css/project-dark-signed-out.html'),
+    ],
+    cssKeys: [APPLICATION_DARK_CSS_PREFIX, UTILITIES_DARK_CSS_PREFIX],
+    // We want to include the root dropdown-menu styles since it should be hidden by default
+    purgeOptions: {
+      safelist: {
+        standard: ['dropdown-menu'],
+      },
+    },
+  },
+  {
+    outFile: 'startup-signin',
+    htmlPaths: [PATH_SIGNIN_HTML],
+    cssKeys: [APPLICATION_CSS_PREFIX, UTILITIES_CSS_PREFIX],
+    purgeOptions: {
+      safelist: {
+        standard: ['fieldset'],
+        deep: [/login-page$/],
+      },
+    },
+  },
+];
+
+module.exports = {
+  HTML_TO_REMOVE,
+  CSS_TO_REMOVE,
+  APPLICATION_CSS_PREFIX,
+  APPLICATION_DARK_CSS_PREFIX,
+  UTILITIES_CSS_PREFIX,
+  UTILITIES_DARK_CSS_PREFIX,
+  ROOT,
+  PATH_ASSETS,
+  PATH_STARTUP_SCSS,
+  OUTPUTS,
+};

scripts/frontend/startup_css/get_css_path.js

+const fs = require('fs');
+const path = require('path');
+const { memoize } = require('lodash');
+const { PATH_ASSETS } = require('./constants');
+const { die } = require('./utils');
+
+const listAssetsDir = memoize(() => fs.readdirSync(PATH_ASSETS));
+
+const getCSSPath = (prefix) => {
+  const matcher = new RegExp(`^${prefix}-[^-]+\\.css$`);
+  const cssPath = listAssetsDir().find((x) => matcher.test(x));
+
+  if (!cssPath) {
+    die(
+      `Could not find the CSS asset matching "${prefix}". Have you run "scripts/frontend/startup_css/setup.sh"?`,
+    );
+  }
+
+  return path.join(PATH_ASSETS, cssPath);
+};
+
+module.exports = { getCSSPath };

scripts/frontend/startup_css/get_startup_css.js

+const fs = require('fs');
+const cheerio = require('cheerio');
+const { mergeWith, isArray } = require('lodash');
+const { PurgeCSS } = require('purgecss');
+const { cleanCSS } = require('./clean_css');
+const { HTML_TO_REMOVE } = require('./constants');
+const { die } = require('./utils');
+
+const cleanHtml = (html) => {
+  const $ = cheerio.load(html);
+
+  HTML_TO_REMOVE.forEach((selector) => {
+    $(selector).remove();
+  });
+
+  return $.html();
+};
+
+const mergePurgeCSSOptions = (...options) =>
+  mergeWith(...options, (objValue, srcValue) => {
+    if (isArray(objValue)) {
+      return objValue.concat(srcValue);
+    }
+
+    return undefined;
+  });
+
+const getStartupCSS = async ({ htmlPaths, cssPaths, purgeOptions }) => {
+  const content = htmlPaths.map((htmlPath) => {
+    if (!fs.existsSync(htmlPath)) {
+      die(`Could not find fixture "${htmlPath}". Have you run the fixtures?`);
+    }
+
+    const rawHtml = fs.readFileSync(htmlPath);
+    const html = cleanHtml(rawHtml);
+
+    return { raw: html, extension: 'html' };
+  });
+
+  const purgeCSSResult = await new PurgeCSS().purge({
+    content,
+    css: cssPaths,
+    ...mergePurgeCSSOptions(
+      {
+        fontFace: true,
+        variables: true,
+        keyframes: true,
+        blocklist: [/:hover/, /:focus/, /-webkit-/, /-moz-focusring-/, /-ms-expand/],
+        safelist: {
+          standard: ['brand-header-logo'],
+        },
+        // By default, PurgeCSS ignores special characters, but our utilities use "!"
+        defaultExtractor: (x) => x.match(/[\w-!]+/g),
+      },
+      purgeOptions,
+    ),
+  });
+
+  return purgeCSSResult.map(({ css }) => cleanCSS(css)).join('\n');
+};
+
+module.exports = { getStartupCSS };

scripts/frontend/startup_css/main.js

+const { memoize } = require('lodash');
+const { OUTPUTS } = require('./constants');
+const { getCSSPath } = require('./get_css_path');
+const { getStartupCSS } = require('./get_startup_css');
+const { log, die } = require('./utils');
+const { writeStartupSCSS } = require('./write_startup_scss');
+
+const memoizedCSSPath = memoize(getCSSPath);
+
+const runTask = async ({ outFile, htmlPaths, cssKeys, purgeOptions = {} }) => {
+  try {
+    log(`Generating startup CSS for HTML files: ${htmlPaths}`);
+    const generalCSS = await getStartupCSS({
+      htmlPaths,
+      cssPaths: cssKeys.map(memoizedCSSPath),
+      purgeOptions,
+    });
+
+    log(`Writing to startup CSS...`);
+    const startupCSSPath = writeStartupSCSS(outFile, generalCSS);
+    log(`Finished writing to ${startupCSSPath}`);
+
+    return {
+      success: true,
+      outFile,
+    };
+  } catch (e) {
+    log(`ERROR! Unexpected error occurred while generating startup CSS for: ${outFile}`);
+    log(e);
+
+    return {
+      success: false,
+      outFile,
+    };
+  }
+};
+
+const main = async () => {
+  const result = await Promise.all(OUTPUTS.map(runTask));
+  const fullSuccess = result.every((x) => x.success);
+
+  log('RESULTS:');
+  log('--------');
+
+  result.forEach(({ success, outFile }) => {
+    const status = success ? '✓' : 'ⅹ';
+
+    log(`${status}: ${outFile}`);
+  });
+
+  log('--------');
+
+  if (fullSuccess) {
+    log('Done!');
+  } else {
+    die('Some tasks have failed');
+  }
+};
+
+main();

scripts/frontend/startup_css/setup.sh

+path_public_dir="public"
+path_tmp="tmp"
+path_dest="$path_tmp/startup_css_assets"
+glob_css_dest="$path_dest/application*.css"
+glob_css_src="$path_public_dir/assets/application*.css"
+should_clean=false
+
+should_force() {
+  $1=="force"
+}
+
+has_dest_already() {
+ find $glob_css_dest -quit
+}
+
+has_src_already() {
+ find $glob_css_src -quit
+}
+
+compile_assets() {
+  # We need to build the same test bundle that is built in CI
+  RAILS_ENV=test bundle exec rake rake:assets:precompile
+}
+
+clean_assets() {
+  bundle exec rake rake:assets:clobber
+}
+
+copy_assets() {
+  rm -rf $path_dest
+  mkdir $path_dest
+  cp $glob_css_src $path_dest
+}
+
+echo "-----------------------------------------------------------"
+echo "If you are run into any issues with Startup CSS generation,"
+echo "please check out the feedback issue:"
+echo ""
+echo "https://gitlab.com/gitlab-org/gitlab/-/issues/331812"
+echo "-----------------------------------------------------------"
+
+if [ ! -e $path_public_dir ]; then
+  echo "Could not find '$path_public_dir/'. This script must be run in the root directory of the gitlab project."
+  exit 1
+fi
+
+if [ ! -e $path_tmp ]; then
+  echo "Could not find '$path_tmp/'. This script must be run in the root directory of the gitlab project."
+  exit 1
+fi
+
+if [ "$1" != "force" ] && has_dest_already; then
+  echo "Already found assets for '$glob_css_dest'. Did you want to run this script with 'force' argument?"
+  exit 0
+fi
+
+# If we are in CI, don't recompile things...
+if [ -n "$CI" ]; then
+  if ! has_src_already; then
+    echo "Could not find '$glob_css_src'. Expected these artifacts to be generated by CI pipeline."
+    exit 1
+  fi
+elif has_src_already; then
+  echo "Found '$glob_css_src'. Skipping compile assets..."
+else
+  echo "Starting compile assets process..."
+  compile_assets
+  should_clean=true
+fi
+
+copy_assets
+
+if $should_clean; then
+  echo "Starting cleanup..."
+  clean_assets
+fi

scripts/frontend/startup_css/utils.js

+const die = (message) => {
+  console.log(message);
+  process.exit(1);
+};
+
+const log = (message) => console.error(`[gitlab.startup_css] ${message}`);
+
+module.exports = { die, log };

scripts/frontend/startup_css/write_startup_scss.js

+const { writeFileSync } = require('fs');
+const path = require('path');
+const prettier = require('prettier');
+const { PATH_STARTUP_SCSS } = require('./constants');
+
+const buildFinalContent = (raw) => {
+  const content = `// DO NOT EDIT! This is auto-generated from "yarn run generate:startup_css"
+// Please see the feedback issue for more details and help:
+// https://gitlab.com/gitlab-org/gitlab/-/issues/331812
+@charset "UTF-8";
+${raw}
+@import 'cloaking';
+@include cloak-startup-scss(none);
+`;
+
+  // We run prettier so that there is more determinism with the generated file.
+  return prettier.format(content, { parser: 'scss' });
+};
+
+const writeStartupSCSS = (name, raw) => {
+  const fullPath = path.join(PATH_STARTUP_SCSS, `${name}.scss`);
+
+  writeFileSync(fullPath, buildFinalContent(raw));
+
+  return fullPath;
+};
+
+module.exports = { writeStartupSCSS };

spec/frontend/fixtures/startup_css.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+
+RSpec.describe 'Startup CSS fixtures', type: :controller do
+  include JavaScriptFixturesHelpers
+
+  let(:use_full_html) { true }
+
+  render_views
+
+  before(:all) do
+    stub_feature_flags(combined_menu: true)
+    clean_frontend_fixtures('startup_css/')
+  end
+
+  shared_examples 'startup css project fixtures' do |type|
+    let(:user) { create(:user, :admin) }
+    let(:project) { create(:project, :public, :repository, description: 'Code and stuff', avatar: fixture_file_upload('spec/fixtures/dk.png', 'image/png'), creator: user) }
+
+    before do
+      sign_in(user)
+    end
+
+    it "startup_css/project-#{type}-legacy-menu.html" do
+      stub_feature_flags(combined_menu: false)
+
+      get :show, params: {
+        namespace_id: project.namespace.to_param,
+        id: project
+      }
+
+      expect(response).to be_successful
+    end
+
+    it "startup_css/project-#{type}.html" do
+      get :show, params: {
+        namespace_id: project.namespace.to_param,
+        id: project
+      }
+
+      expect(response).to be_successful
+    end
+
+    it "startup_css/project-#{type}-signed-out.html" do
+      sign_out(user)
+
+      get :show, params: {
+        namespace_id: project.namespace.to_param,
+        id: project
+      }
+
+      expect(response).to be_successful
+    end
+  end
+
+  describe ProjectsController, '(Startup CSS fixtures)', type: :controller do
+    it_behaves_like 'startup css project fixtures', 'general'
+  end
+
+  describe ProjectsController, '(Startup CSS fixtures)', type: :controller do
+    before do
+      user.update!(theme_id: 11)
+    end
+
+    it_behaves_like 'startup css project fixtures', 'dark'
+  end
+
+  describe RegistrationsController, '(Startup CSS fixtures)', type: :controller do
+    it 'startup_css/sign-in.html' do
+      get :new
+
+      expect(response).to be_successful
+    end
+  end
+end

spec/support/helpers/javascript_fixtures_helpers.rb

@@ -66,6 +66,14 @@ module JavaScriptFixturesHelpers
     File.write(full_fixture_path, fixture)
   end
 
+  def parse_html(fixture)
+    if respond_to?(:use_full_html) && public_send(:use_full_html)
+      Nokogiri::HTML::Document.parse(fixture)
+    else
+      Nokogiri::HTML::DocumentFragment.parse(fixture)
+    end
+  end
+
   # Private: Prepare a response object for use as a frontend fixture
   #
   # response - response object to prepare
@@ -76,7 +84,7 @@ module JavaScriptFixturesHelpers
 
     response_mime_type = Mime::Type.lookup(response.media_type)
     if response_mime_type.html?
-      doc = Nokogiri::HTML::DocumentFragment.parse(fixture)
+      doc = parse_html(fixture)
 
       link_tags = doc.css('link')
       link_tags.remove