Merge branch 'mg-automatically-compile-dlls' into 'master'

Automatically compile vendor DLLs on-demand and enable in CI

See merge request gitlab-org/gitlab!25131

.gitlab/ci/frontend.gitlab-ci.yml

@@ -6,6 +6,7 @@
       - tmp/cache/assets/sprockets
       - tmp/cache/babel-loader
       - tmp/cache/vue-loader
+      - tmp/cache/webpack-dlls
 
 .gitlab:assets:compile-metadata:
   extends:
@@ -78,6 +79,7 @@ gitlab:assets:compile pull-cache:
     SETUP_DB: "false"
     # we override the max_old_space_size to prevent OOM errors
     NODE_OPTIONS: --max_old_space_size=3584
+    WEBPACK_VENDOR_DLL: "true"
   cache:
     key: "assets-compile:v9"
   artifacts:

config/webpack.config.js

@@ -117,23 +117,18 @@ if (IS_EE) {
   });
 }
 
-// if there is a compiled DLL with a matching hash string, use it
 let dll;
 
 if (VENDOR_DLL && !IS_PRODUCTION) {
   const dllHash = vendorDllHash();
   const dllCachePath = path.join(ROOT_PATH, `tmp/cache/webpack-dlls/${dllHash}`);
-  if (fs.existsSync(dllCachePath)) {
-    console.log(`Using vendor DLL found at: ${dllCachePath}`);
-    dll = {
-      manifestPath: path.join(dllCachePath, 'vendor.dll.manifest.json'),
-      cacheFrom: dllCachePath,
-      cacheTo: path.join(ROOT_PATH, `public/assets/webpack/dll.${dllHash}/`),
-      publicPath: `dll.${dllHash}/vendor.dll.bundle.js`,
-    };
-  } else {
-    console.log(`Warning: No vendor DLL found at: ${dllCachePath}. DllPlugin disabled.`);
-  }
+  dll = {
+    manifestPath: path.join(dllCachePath, 'vendor.dll.manifest.json'),
+    cacheFrom: dllCachePath,
+    cacheTo: path.join(ROOT_PATH, `public/assets/webpack/dll.${dllHash}/`),
+    publicPath: `dll.${dllHash}/vendor.dll.bundle.js`,
+    exists: null,
+  };
 }
 
 module.exports = {
@@ -314,6 +309,51 @@ module.exports = {
       jQuery: 'jquery',
     }),
 
+    // if DLLs are enabled, detect whether the DLL exists and create it automatically if necessary
+    dll && {
+      apply(compiler) {
+        compiler.hooks.beforeCompile.tapAsync('DllAutoCompilePlugin', (params, callback) => {
+          if (dll.exists) {
+            callback();
+          } else if (fs.existsSync(dll.manifestPath)) {
+            console.log(`Using vendor DLL found at: ${dll.cacheFrom}`);
+            dll.exists = true;
+            callback();
+          } else {
+            console.log(
+              `Warning: No vendor DLL found at: ${dll.cacheFrom}. Compiling DLL automatically.`,
+            );
+
+            const dllConfig = require('./webpack.vendor.config.js');
+            const dllCompiler = webpack(dllConfig);
+
+            dllCompiler.run((err, stats) => {
+              if (err) {
+                return callback(err);
+              }
+
+              const info = stats.toJson();
+
+              if (stats.hasErrors()) {
+                console.error(info.errors.join('\n\n'));
+                return callback('DLL not compiled successfully.');
+              }
+
+              if (stats.hasWarnings()) {
+                console.warn(info.warnings.join('\n\n'));
+                console.warn('DLL compiled with warnings.');
+              } else {
+                console.log('DLL compiled successfully.');
+              }
+
+              dll.exists = true;
+              callback();
+            });
+          }
+        });
+      },
+    },
+
     // reference our compiled DLL modules
     dll &&
       new webpack.DllReferencePlugin({

config/webpack.vendor.config.js

@@ -15,6 +15,9 @@ module.exports = {
     extensions: ['.js'],
   },
 
+  // ensure output is not generated when errors are encountered
+  bail: true,
+
   context: ROOT_PATH,
 
   entry: {

scripts/clean-old-cached-assets

@@ -2,5 +2,7 @@
 
 # Clean up cached files that are older than 4 days
 find tmp/cache/assets/sprockets/ -type f -mtime +4 -execdir rm -- "{}" \;
+find tmp/cache/webpack-dlls/ -maxdepth 1 -type d -mtime +4 -exec rm -rf -- "{}" \;
 
 du -d 0 -h tmp/cache/assets/sprockets | cut -f1 | xargs -I % echo "tmp/cache/assets/sprockets/ is currently %"
+du -d 0 -h tmp/cache/webpack-dlls | cut -f1 | xargs -I % echo "tmp/cache/webpack-dlls is currently %"