Move audit logs to a dedicated logs directory (#116562) (#118134)

Co-authored-by: Aleh Zasypkin <aleh.zasypkin@gmail.com>

Co-authored-by: Thomas Watson <w@tson.dk>
Co-authored-by: Aleh Zasypkin <aleh.zasypkin@gmail.com>

docs/settings/security-settings.asciidoc

@@ -332,7 +332,7 @@ For more details and a reference of audit events, refer to <<xpack-security-audi
 xpack.security.audit.enabled: true
 xpack.security.audit.appender: <1>
   type: rolling-file
-  fileName: ./data/audit.log
+  fileName: ./logs/audit.log
   policy:
     type: time-interval
     interval: 24h <2>

packages/kbn-utils/src/path/index.test.ts

@@ -7,21 +7,35 @@
  */
 
 import { accessSync, constants } from 'fs';
-import { getConfigPath, getDataPath, getConfigDirectory } from './';
+import { createAbsolutePathSerializer } from '@kbn/dev-utils';
+import { getConfigPath, getDataPath, getLogsPath, getConfigDirectory } from './';
+
+expect.addSnapshotSerializer(createAbsolutePathSerializer());
 
 describe('Default path finder', () => {
-  it('should find a kibana.yml', () => {
-    const configPath = getConfigPath();
-    expect(() => accessSync(configPath, constants.R_OK)).not.toThrow();
+  it('should expose a path to the config directory', () => {
+    expect(getConfigDirectory()).toMatchInlineSnapshot('<absolute path>/config');
   });
 
-  it('should find a data directory', () => {
-    const dataPath = getDataPath();
-    expect(() => accessSync(dataPath, constants.R_OK)).not.toThrow();
+  it('should expose a path to the kibana.yml', () => {
+    expect(getConfigPath()).toMatchInlineSnapshot('<absolute path>/config/kibana.yml');
+  });
+
+  it('should expose a path to the data directory', () => {
+    expect(getDataPath()).toMatchInlineSnapshot('<absolute path>/data');
+  });
+
+  it('should expose a path to the logs directory', () => {
+    expect(getLogsPath()).toMatchInlineSnapshot('<absolute path>/logs');
   });
 
   it('should find a config directory', () => {
     const configDirectory = getConfigDirectory();
     expect(() => accessSync(configDirectory, constants.R_OK)).not.toThrow();
   });
+
+  it('should find a kibana.yml', () => {
+    const configPath = getConfigPath();
+    expect(() => accessSync(configPath, constants.R_OK)).not.toThrow();
+  });
 });

packages/kbn-utils/src/path/index.ts

@@ -27,6 +27,8 @@ const CONFIG_DIRECTORIES = [
 
 const DATA_PATHS = [join(REPO_ROOT, 'data'), '/var/lib/kibana'].filter(isString);
 
+const LOGS_PATHS = [join(REPO_ROOT, 'logs'), '/var/log/kibana'].filter(isString);
+
 function findFile(paths: string[]) {
   const availablePath = paths.find((configPath) => {
     try {
@@ -57,6 +59,12 @@ export const getConfigDirectory = () => findFile(CONFIG_DIRECTORIES);
  */
 export const getDataPath = () => findFile(DATA_PATHS);
 
+/**
+ * Get the directory containing logs
+ * @internal
+ */
+export const getLogsPath = () => findFile(LOGS_PATHS);
+
 export type PathConfigType = TypeOf<typeof config.schema>;
 
 export const config = {

src/dev/build/tasks/clean_tasks.ts

@@ -196,6 +196,7 @@ export const CleanEmptyFolders: Task = {
     await deleteEmptyFolders(log, build.resolvePath('.'), [
       build.resolvePath('plugins'),
       build.resolvePath('data'),
+      build.resolvePath('logs'),
     ]);
   },
 };

src/dev/build/tasks/create_empty_dirs_and_files_task.ts

@@ -12,6 +12,10 @@ export const CreateEmptyDirsAndFiles: Task = {
   description: 'Creating some empty directories and files to prevent file-permission issues',
 
   async run(config, log, build) {
-    await Promise.all([mkdirp(build.resolvePath('plugins')), mkdirp(build.resolvePath('data'))]);
+    await Promise.all([
+      mkdirp(build.resolvePath('plugins')),
+      mkdirp(build.resolvePath('data')),
+      mkdirp(build.resolvePath('logs')),
+    ]);
   },
 };

src/dev/build/tasks/os_packages/run_fpm.ts

@@ -113,6 +113,8 @@ export async function runFpm(
     '--exclude',
     `usr/share/kibana/data`,
     '--exclude',
+    `usr/share/kibana/logs`,
+    '--exclude',
     'run/kibana/.gitempty',
 
     // flags specific to the package we are building, supplied by tasks below
@@ -129,6 +131,9 @@ export async function runFpm(
     // copy the data directory at /var/lib/kibana
     `${resolveWithTrailingSlash(fromBuild('data'))}=/var/lib/kibana/`,
 
+    // copy the logs directory at /var/log/kibana
+    `${resolveWithTrailingSlash(fromBuild('logs'))}=/var/log/kibana/`,
+
     // copy package configurations
     `${resolveWithTrailingSlash(__dirname, 'service_templates/systemd/')}=/`,
 

x-pack/plugins/security/server/config.test.ts

@@ -11,7 +11,7 @@ jest.mock('crypto', () => ({
 }));
 
 jest.mock('@kbn/utils', () => ({
-  getDataPath: () => '/mock/kibana/data/path',
+  getLogsPath: () => '/mock/kibana/logs/path',
 }));
 
 import { loggingSystemMock } from 'src/core/server/mocks';
@@ -1720,7 +1720,7 @@ describe('createConfig()', () => {
       ).audit.appender
     ).toMatchInlineSnapshot(`
       Object {
-        "fileName": "/mock/kibana/data/path/audit.log",
+        "fileName": "/mock/kibana/logs/path/audit.log",
         "layout": Object {
           "type": "json",
         },

x-pack/plugins/security/server/config.ts

@@ -12,7 +12,7 @@ import path from 'path';
 import type { Type, TypeOf } from '@kbn/config-schema';
 import { schema } from '@kbn/config-schema';
 import { i18n } from '@kbn/i18n';
-import { getDataPath } from '@kbn/utils';
+import { getLogsPath } from '@kbn/utils';
 import type { AppenderConfigType, Logger } from 'src/core/server';
 
 import { config as coreConfig } from '../../../../src/core/server';
@@ -378,7 +378,7 @@ export function createConfig(
     config.audit.appender ??
     ({
       type: 'rolling-file',
-      fileName: path.join(getDataPath(), 'audit.log'),
+      fileName: path.join(getLogsPath(), 'audit.log'),
       layout: {
         type: 'json',
       },