Merge branch 'incremental-webpack-loading' into 'master'

Add incremental compiler to webpack dev server

See merge request gitlab-org/gitlab!52350

app/assets/javascripts/webpack_non_compiled_placeholder.js

+const div = document.createElement('div');
+
+Object.assign(div.style, {
+  width: '100vw',
+  height: '100vh',
+  position: 'fixed',
+  top: 0,
+  left: 0,
+  'z-index': 100000,
+  background: 'rgba(0,0,0,0.9)',
+  'font-size': '25px',
+  'font-family': 'monospace',
+  color: 'white',
+  padding: '2.5em',
+  'text-align': 'center',
+});
+
+div.innerHTML = `
+<h1 style="color:white">🧙 Webpack is doing its magic 🧙</h1>
+<p>If you use Hot Module reloading, the page will reload in a few seconds.</p>
+<p>If you do not use Hot Module reloading, please <a href="">reload the page manually in a few seconds</a></p>
+`;
+
+document.body.append(div);

config/helpers/incremental_webpack_compiler.js

+const fs = require('fs');
+const path = require('path');
+
+const log = (msg, ...rest) => console.log(`IncrementalWebpackCompiler: ${msg}`, ...rest);
+
+// If we force a recompile immediately, the page reload doesn't seem to work.
+// Five seconds seem to work fine and the user can read the message
+const TIMEOUT = 5000;
+
+class NoopCompiler {
+  enabled = false;
+
+  filterEntryPoints(entryPoints) {
+    return entryPoints;
+  }
+
+  logStatus() {}
+
+  setupMiddleware() {}
+}
+
+class IncrementalWebpackCompiler extends NoopCompiler {
+  enabled = true;
+
+  constructor(historyFilePath) {
+    super();
+    this.history = {};
+    this.compiledEntryPoints = new Set([
+      // Login page
+      'pages.sessions.new',
+      // Explore page
+      'pages.root',
+    ]);
+    this.historyFilePath = historyFilePath;
+    this.loadFromHistory();
+  }
+
+  filterEntryPoints(entrypoints) {
+    return Object.fromEntries(
+      Object.entries(entrypoints).map(([key, val]) => {
+        if (this.compiledEntryPoints.has(key)) {
+          return [key, val];
+        }
+        return [key, ['./webpack_non_compiled_placeholder.js']];
+      }),
+    );
+  }
+
+  logStatus(totalCount) {
+    const current = this.compiledEntryPoints.size;
+    log(`Currently compiling route entrypoints: ${current} of ${totalCount}`);
+  }
+
+  setupMiddleware(app, server) {
+    app.use((req, res, next) => {
+      const fileName = path.basename(req.url);
+
+      /**
+       * We are only interested in files that have a name like `pages.foo.bar.chunk.js`
+       * because those are the ones corresponding to our entry points.
+       *
+       * This filters out hot update files that are for example named "pages.foo.bar.[hash].hot-update.js"
+       */
+      if (fileName.startsWith('pages.') && fileName.endsWith('.chunk.js')) {
+        const chunk = fileName.replace(/\.chunk\.js$/, '');
+
+        this.addToHistory(chunk);
+
+        if (!this.compiledEntryPoints.has(chunk)) {
+          log(`First time we are seeing ${chunk}. Adding to compilation.`);
+
+          this.compiledEntryPoints.add(chunk);
+
+          setTimeout(() => {
+            server.middleware.invalidate(() => {
+              if (server.sockets) {
+                server.sockWrite(server.sockets, 'content-changed');
+              }
+            });
+          }, TIMEOUT);
+        }
+      }
+
+      next();
+    });
+  }
+
+  // private methods
+
+  addToHistory(chunk) {
+    if (!this.history[chunk]) {
+      this.history[chunk] = { lastVisit: null, count: 0 };
+    }
+    this.history[chunk].lastVisit = Date.now();
+    this.history[chunk].count += 1;
+
+    try {
+      fs.writeFileSync(this.historyFilePath, JSON.stringify(this.history), 'utf8');
+    } catch (e) {
+      log('Warning – Could not write to history', e.message);
+    }
+  }
+
+  loadFromHistory() {
+    try {
+      this.history = JSON.parse(fs.readFileSync(this.historyFilePath, 'utf8'));
+      const entryPoints = Object.keys(this.history);
+      log(`Successfully loaded history containing ${entryPoints.length} entry points`);
+      /*
+      TODO: Let's ask a few folks to give us their history file after a milestone of usage
+            Then we can make smarter decisions on when to throw out rather than rendering everything
+            Something like top 20/30/40 entries visited in the last 7/10/15 days might be sufficient
+       */
+      this.compiledEntryPoints = new Set([...this.compiledEntryPoints, ...entryPoints]);
+    } catch (e) {
+      log(`No history found...`);
+    }
+  }
+}
+
+module.exports = (enabled, historyFilePath) => {
+  log(`Status – ${enabled ? 'enabled' : 'disabled'}`);
+
+  if (enabled) {
+    return new IncrementalWebpackCompiler(historyFilePath);
+  }
+  return new NoopCompiler();
+};

config/webpack.config.js

@@ -9,6 +9,7 @@ const MonacoWebpackPlugin = require('./plugins/monaco_webpack');
 const BundleAnalyzerPlugin = require('webpack-bundle-analyzer').BundleAnalyzerPlugin;
 const CopyWebpackPlugin = require('copy-webpack-plugin');
 const vendorDllHash = require('./helpers/vendor_dll_hash');
+const createIncrementalWebpackCompiler = require('./helpers/incremental_webpack_compiler');
 
 const ROOT_PATH = path.resolve(__dirname, '..');
 const VENDOR_DLL = process.env.WEBPACK_VENDOR_DLL && process.env.WEBPACK_VENDOR_DLL !== 'false';
@@ -23,6 +24,10 @@ const DEV_SERVER_ALLOWED_HOSTS =
   process.env.DEV_SERVER_ALLOWED_HOSTS && process.env.DEV_SERVER_ALLOWED_HOSTS.split(',');
 const DEV_SERVER_HTTPS = process.env.DEV_SERVER_HTTPS && process.env.DEV_SERVER_HTTPS !== 'false';
 const DEV_SERVER_LIVERELOAD = IS_DEV_SERVER && process.env.DEV_SERVER_LIVERELOAD !== 'false';
+const INCREMENTAL_COMPILER_ENABLED =
+  IS_DEV_SERVER &&
+  process.env.DEV_SERVER_INCREMENTAL &&
+  process.env.DEV_SERVER_INCREMENTAL !== 'false';
 const WEBPACK_REPORT = process.env.WEBPACK_REPORT && process.env.WEBPACK_REPORT !== 'false';
 const WEBPACK_MEMORY_TEST =
   process.env.WEBPACK_MEMORY_TEST && process.env.WEBPACK_MEMORY_TEST !== 'false';
@@ -48,6 +53,11 @@ let autoEntriesCount = 0;
 let watchAutoEntries = [];
 const defaultEntries = ['./main'];
 
+const incrementalCompiler = createIncrementalWebpackCompiler(
+  INCREMENTAL_COMPILER_ENABLED,
+  path.join(CACHE_PATH, 'incremental-webpack-compiler-history.json'),
+);
+
 function generateEntries() {
   // generate automatic entry points
   const autoEntries = {};
@@ -97,7 +107,7 @@ function generateEntries() {
     jira_connect_app: './jira_connect/index.js',
   };
 
-  return Object.assign(manualEntries, autoEntries);
+  return Object.assign(manualEntries, incrementalCompiler.filterEntryPoints(autoEntries));
 }
 
 const alias = {
@@ -495,9 +505,13 @@ module.exports = {
           watchAutoEntries.forEach((watchPath) => compilation.contextDependencies.add(watchPath));
 
           // report our auto-generated bundle count
+          if (incrementalCompiler.enabled) {
+            incrementalCompiler.logStatus(autoEntriesCount);
+          } else {
             console.log(
               `${autoEntriesCount} entries from '/pages' automatically added to webpack output.`,
             );
+          }
 
           callback();
         });
@@ -576,8 +590,10 @@ module.exports = {
     */
     new webpack.IgnorePlugin(/moment/, /pikaday/),
   ].filter(Boolean),
-
   devServer: {
+    before(app, server) {
+      incrementalCompiler.setupMiddleware(app, server);
+    },
     host: DEV_SERVER_HOST,
     port: DEV_SERVER_PORT,
     public: DEV_SERVER_PUBLIC_ADDR,