Implement built_assets directory (#27468) (#28442)

* [sass] build sass into built_assets directory

* [optimize/dlls] move dll optimization into built_assets directory

* [dlls] update path to dll manifest

* [built_assets] ignore when searching for notice files

* [sass] build into variable directory

* revert changes to ui_export_types/style_sheet_paths.js

* revert changes to ui_export_types/style_sheet_paths.test.js

* [uiExports/styleSheetPaths] switch .scss extension for stylesheets that are already compiled

* update comment for clarity

* [testBundle] include css from built_assets

* [dlls] move url to built_assets/dlls

* fix merge conflict issues

* [server/sass] pass log in test

.eslintignore

@@ -5,7 +5,7 @@ bower_components
 /.es
 /plugins
 /optimize
-/dlls
+/built_assets
 /src/fixtures/vislib/mock_data
 /src/ui/public/angular-bootstrap
 /src/ui/public/flot-charts

.gitignore

@@ -9,7 +9,7 @@ node_modules
 !/src/dev/notice/__fixtures__/node_modules
 trash
 /optimize
-/dlls
+/built_assets
 target
 /build
 .jruby
@@ -43,4 +43,3 @@ package-lock.json
 .vscode
 npm-debug.log*
 .tern-project
-index.css

package.json

@@ -46,7 +46,7 @@
       "extraPatterns": [
         "build",
         "optimize",
-        "dlls",
+        "built_assets",
         ".eslintcache"
       ]
     }

src/dev/build/tasks/nodejs_modules/clean_client_modules_on_dll_task.js

@@ -64,7 +64,7 @@ export const CleanClientModulesOnDLLTask = {
     ];
 
     // Resolve the client vendors dll manifest path
-    const dllManifestPath = `${baseDir}/dlls/vendors.manifest.dll.json`;
+    const dllManifestPath = `${baseDir}/built_assets/dlls/vendors.manifest.dll.json`;
 
     // Get dll entries filtering out the ones
     // from any whitelisted module

src/dev/build/tasks/transpile_scss_task.js

@@ -34,7 +34,7 @@ export const TranspileScssTask = {
     const uiExports = collectUiExports(enabledPlugins);
 
     try {
-      const bundles = await buildAll(uiExports.styleSheetPaths, log);
+      const bundles = await buildAll(uiExports.styleSheetPaths, log, build.resolvePath('built_assets/css'));
       bundles.forEach(bundle => log.info(`Compiled SCSS: ${bundle.source}`));
     } catch (error) {
       const { message, line, file } = error;

src/dev/notice/generate_notice_from_source.js

@@ -42,7 +42,7 @@ export async function generateNoticeFromSource({ productName, directory, log })
     cwd: directory,
     nodir: true,
     ignore: [
-      '{node_modules,build,target,dist,optimize,dlls}/**',
+      '{node_modules,build,target,dist,optimize,built_assets}/**',
       'packages/*/{node_modules,build,target,dist}/**',
       'x-pack/{node_modules,build,target,dist,optimize}/**',
       'x-pack/packages/*/{node_modules,build,target,dist}/**',

src/legacy/core_plugins/tests_bundle/index.js

@@ -70,7 +70,7 @@ export default (kibana) => {
             }
 
             testGlobs.push(`${plugin.publicDir}/**/__tests__/**/*.js`);
-            testGlobs.push(`${plugin.publicDir}/**/*.css`);
+            testGlobs.push(`built_assets/css/plugins/${plugin.id}/**/*.css`);
           });
         } else {
           // add the modules from all of the apps
@@ -80,7 +80,7 @@ export default (kibana) => {
 
           for (const plugin of plugins) {
             testGlobs.push(`${plugin.publicDir}/**/__tests__/**/*.js`);
-            testGlobs.push(`${plugin.publicDir}/**/*.css`);
+            testGlobs.push(`built_assets/css/plugins/${plugin.id}/**/*.css`);
           }
         }
 

src/optimize/bundles_route/bundles_route.js

@@ -59,7 +59,7 @@ export function createBundlesRoute({ regularBundlesPath, dllBundlesPath, basePub
 
   return [
     buildRouteForBundles(basePublicPath, '/bundles/', regularBundlesPath, fileHashCache),
-    buildRouteForBundles(basePublicPath, '/dlls/', dllBundlesPath, fileHashCache),
+    buildRouteForBundles(basePublicPath, '/built_assets/dlls/', dllBundlesPath, fileHashCache),
   ];
 }
 

src/optimize/bundles_route/proxy_bundles_route.js

@@ -20,7 +20,7 @@
 export function createProxyBundlesRoute({ host, port }) {
   return [
     buildProxyRouteForBundles('/bundles/', host, port),
-    buildProxyRouteForBundles('/dlls/', host, port)
+    buildProxyRouteForBundles('/built_assets/dlls/', host, port)
   ];
 }
 

src/optimize/dynamic_dll_plugin/dll_compiler.js

@@ -46,7 +46,7 @@ export class DllCompiler {
       dllExt: '.bundle.dll.js',
       manifestExt: '.manifest.dll.json',
       styleExt: '.style.dll.css',
-      outputPath: fromRoot('./dlls'),
+      outputPath: fromRoot('built_assets/dlls'),
       publicPath: PUBLIC_PATH_PLACEHOLDER
     };
   }

src/optimize/index.js

@@ -29,7 +29,7 @@ export default async (kbnServer, server, config) => {
   // bundles in a "middleware" style.
   //
   // the server listening on 5601 may be restarted a number of times, depending
-  // on the watch setup managed by the cli. It proxies all bundles/* and dlls/*
+  // on the watch setup managed by the cli. It proxies all bundles/* and built_assets/dlls/*
   // requests to the other server. The server on 5602 is long running, in order
   // to prevent complete rebuilds of the optimize content.
   const watch = config.get('optimize.watch');

src/optimize/watch/watch.js

@@ -33,7 +33,7 @@ export default async kbnServer => {
    *   while the optimizer is running
    *
    * server: this process runs the entire kibana server and proxies
-   *   all requests for /bundles/* or /dlls/* to the optmzr process
+   *   all requests for /bundles/* or /built_assets/dlls/* to the optmzr process
    *
    * @param  {string} process.env.kbnWorkerType
    */

src/server/sass/__fixtures__/index.scss

@@ -0,0 +1,5 @@
+foo {
+  bar {
+    display: flex;
+  }
+}

src/server/sass/build.js

@@ -23,30 +23,24 @@ import fs from 'fs';
 import sass from 'node-sass';
 import autoprefixer from 'autoprefixer';
 import postcss from 'postcss';
+import mkdirp from 'mkdirp';
 
 const renderSass = promisify(sass.render);
 const writeFile = promisify(fs.writeFile);
+const mkdirpAsync = promisify(mkdirp);
 
 export class Build {
-  constructor(source, log) {
+  constructor(source, log, targetPath) {
     this.source = source;
     this.log = log;
+    this.targetPath = targetPath;
     this.includedFiles = [source];
   }
 
-  outputPath() {
-    const fileName = path.basename(this.source, path.extname(this.source)) + '.css';
-    return path.join(path.dirname(this.source), fileName);
-  }
-
   /**
    * Glob based on source path
    */
 
-  getGlob() {
-    return path.join(path.dirname(this.source), '**', '*.s{a,c}ss');
-  }
-
   async buildIfIncluded(path) {
     if (this.includedFiles && this.includedFiles.includes(path)) {
       await this.build();
@@ -61,11 +55,9 @@ export class Build {
    */
 
   async build() {
-    const outFile = this.outputPath();
-
     const rendered = await renderSass({
       file: this.source,
-      outFile,
+      outFile: this.targetPath,
       sourceMap: true,
       sourceMapEmbed: true,
       includePaths: [
@@ -78,7 +70,8 @@ export class Build {
 
     this.includedFiles = rendered.stats.includedFiles;
 
-    await writeFile(outFile, prefixed.css);
+    await mkdirpAsync(path.dirname(this.targetPath));
+    await writeFile(this.targetPath, prefixed.css);
 
     return this;
   }

src/server/sass/build.test.js

@@ -17,34 +17,35 @@
  * under the License.
  */
 
-import path from 'path';
-import sass from 'node-sass';
+import { resolve } from 'path';
+import { readFileSync } from 'fs';
+
+import del from 'del';
+
 import { Build } from './build';
 
-jest.mock('node-sass');
+const TMP = resolve(__dirname, '__tmp__');
+const FIXTURE = resolve(__dirname, '__fixtures__/index.scss');
 
-describe('SASS builder', () => {
-  jest.mock('fs');
+afterEach(async () => {
+  await del(TMP);
+});
 
-  it('generates a glob', () => {
-    const builder = new Build('/foo/style.sass');
-    expect(builder.getGlob()).toEqual(path.join('/foo', '**', '*.s{a,c}ss'));
-  });
+it('builds SASS', async () => {
+  const cssPath = resolve(TMP, 'style.css');
+  await (new Build(FIXTURE, {
+    info: () => {},
+    warn: () => {},
+    error: () => {},
+  }, cssPath)).build();
 
-  it('builds SASS', () => {
-    sass.render.mockImplementation(() => Promise.resolve(null, { css: 'test' }));
-    const builder = new Build('/foo/style.sass');
-    builder.build();
-
-    const sassCall = sass.render.mock.calls[0][0];
-    expect(sassCall.file).toEqual('/foo/style.sass');
-    expect(sassCall.outFile).toEqual(path.join('/foo', 'style.css'));
-    expect(sassCall.sourceMap).toBe(true);
-    expect(sassCall.sourceMapEmbed).toBe(true);
-  });
-
-  it('has an output file with a different extension', () => {
-    const builder = new Build('/foo/style.sass');
-    expect(builder.outputPath()).toEqual(path.join('/foo', 'style.css'));
-  });
-});
+  expect(readFileSync(cssPath, 'utf8').replace(/(\/\*# sourceMappingURL=).*( \*\/)/, '$1...$2'))
+    .toMatchInlineSnapshot(`
+"foo bar {
+  display: -webkit-box;
+  display: -webkit-flex;
+  display: -ms-flexbox;
+  display: flex; }
+/*# sourceMappingURL=... */"
+`);
+});

src/server/sass/build_all.js

@@ -17,16 +17,18 @@
  * under the License.
  */
 
+import { resolve } from 'path';
+
 import { Build } from './build';
 
-export async function buildAll(styleSheets = [], log) {
+export async function buildAll(styleSheets, log, buildDir) {
   const bundles = await Promise.all(styleSheets.map(async styleSheet => {
 
     if (!styleSheet.localPath.endsWith('.scss')) {
       return;
     }
 
-    const bundle = new Build(styleSheet.localPath, log);
+    const bundle = new Build(styleSheet.localPath, log, resolve(buildDir, styleSheet.publicPath));
     await bundle.build();
 
     return bundle;

src/server/sass/index.js

@@ -17,7 +17,7 @@
  * under the License.
  */
 
-import { IS_KIBANA_DISTRIBUTABLE } from '../../utils';
+import { IS_KIBANA_DISTRIBUTABLE, fromRoot } from '../../utils';
 
 export async function sassMixin(kbnServer, server, config) {
   if (process.env.kbnWorkerType === 'optmzr') {
@@ -45,7 +45,7 @@ export async function sassMixin(kbnServer, server, config) {
   };
 
   try {
-    scssBundles = await buildAll(kbnServer.uiExports.styleSheetPaths, log);
+    scssBundles = await buildAll(kbnServer.uiExports.styleSheetPaths, log, fromRoot('built_assets/css'));
 
     scssBundles.forEach(bundle => {
       bundle.includedFiles.forEach(file => trackedFiles.add(file));

src/ui/ui_exports/ui_export_types/style_sheet_paths.js

@@ -18,6 +18,7 @@
  */
 
 import path from 'path';
+import { existsSync } from 'fs';
 import { flatConcatAtType } from './reduce';
 import { mapSpec, wrap } from './modify_reduce';
 
@@ -46,16 +47,22 @@ function normalize(localPath, type, pluginSpec) {
     );
   }
 
+  // replace the extension of localPath to be .css
+  // publicPath will always point to the css file
+  const localCssPath = localPath.slice(0, -extname.length) + '.css';
+
+  // update localPath to point to the .css file if it exists and
+  // the .scss path does not, which is the case for built plugins
+  if (extname === '.scss' && !existsSync(localPath) && existsSync(localCssPath)) {
+    localPath = localCssPath;
+  }
+
   // get the path of the stylesheet relative to the public dir for the plugin
-  let relativePath = path.relative(publicDir, localPath);
+  let relativePath = path.relative(publicDir, localCssPath);
 
   // replace back slashes on windows
   relativePath = relativePath.split('\\').join('/');
 
-  // replace the extension of relativePath to be .css
-  // publicPath will always point to the css file
-  relativePath = relativePath.slice(0, -extname.length) + '.css';
-
   const publicPath = `plugins/${pluginSpec.getId()}/${relativePath}`;
 
   return {
@@ -64,4 +71,4 @@ function normalize(localPath, type, pluginSpec) {
   };
 }
 
-export const styleSheetPaths = wrap(mapSpec(normalize), flatConcatAtType);
+export const styleSheetPaths = wrap(mapSpec(normalize), flatConcatAtType);

src/ui/ui_render/ui_render_mixin.js

@@ -23,6 +23,7 @@ import { resolve } from 'path';
 import { i18n } from '@kbn/i18n';
 import { AppBootstrap } from './bootstrap';
 import { mergeVariables } from './lib';
+import { fromRoot } from '../../utils';
 
 export function uiRenderMixin(kbnServer, server, config) {
   function replaceInjectedVars(request, injectedVars) {
@@ -50,6 +51,9 @@ export function uiRenderMixin(kbnServer, server, config) {
   // render all views from ./views
   server.setupViews(resolve(__dirname, 'views'));
 
+  // expose built css
+  server.exposeStaticDir('/built_assets/css/{path*}', fromRoot('built_assets/css'));
+
   server.route({
     path: '/bundles/app/{id}/bootstrap.js',
     method: 'GET',
@@ -63,12 +67,19 @@ export function uiRenderMixin(kbnServer, server, config) {
 
       const basePath = config.get('server.basePath');
       const regularBundlePath = `${basePath}/bundles`;
-      const dllBundlePath = `${basePath}/dlls`;
+      const dllBundlePath = `${basePath}/built_assets/dlls`;
       const styleSheetPaths = [
         `${dllBundlePath}/vendors.style.dll.css`,
         `${regularBundlePath}/commons.style.css`,
         `${regularBundlePath}/${app.getId()}.style.css`,
-      ].concat(kbnServer.uiExports.styleSheetPaths.map(path => `${basePath}/${path.publicPath}`).reverse());
+        ...kbnServer.uiExports.styleSheetPaths
+          .map(path => (
+            path.localPath.endsWith('.scss')
+              ? `${basePath}/built_assets/css/${path.publicPath}`
+              : `${basePath}/${path.publicPath}`
+          ))
+          .reverse()
+      ];
 
       const bootstrap = new AppBootstrap({
         templateData: {

tasks/config/karma.js

@@ -85,17 +85,17 @@ module.exports = function (grunt) {
 
       // list of files / patterns to load in the browser
       files: [
-        'http://localhost:5610/dlls/vendors.bundle.dll.js',
+        'http://localhost:5610/built_assets/dlls/vendors.bundle.dll.js',
         'http://localhost:5610/bundles/tests.bundle.js',
 
-        'http://localhost:5610/dlls/vendors.style.dll.css',
+        'http://localhost:5610/built_assets/dlls/vendors.style.dll.css',
         'http://localhost:5610/bundles/tests.style.css'
       ],
 
       proxies: {
         '/tests/': 'http://localhost:5610/tests/',
         '/bundles/': 'http://localhost:5610/bundles/',
-        '/dlls/': 'http://localhost:5610/dlls/'
+        '/built_assets/dlls/': 'http://localhost:5610/built_assets/dlls/'
       },
 
       client: {
@@ -176,10 +176,10 @@ module.exports = function (grunt) {
       singleRun: true,
       options: {
         files: [
-          'http://localhost:5610/dlls/vendors.bundle.dll.js',
+          'http://localhost:5610/built_assets/dlls/vendors.bundle.dll.js',
           `http://localhost:5610/bundles/tests.bundle.js?shards=${TOTAL_CI_SHARDS}&shard_num=${n}`,
 
-          'http://localhost:5610/dlls/vendors.style.dll.css',
+          'http://localhost:5610/built_assets/dlls/vendors.style.dll.css',
           'http://localhost:5610/bundles/tests.style.css'
         ]
       }