[DataForge] Change service.logs to mimic Nginx logs (#189567)

## Summary

This PR modifies the `service.log` dataset to change the message from a
random set of strings to something that mimics `Nginx` logs. It also
includes `log.level`, `url.*`, and `http.*` metadata structures.

Before

<img width="2048" alt="image"
src="https://github.com/user-attachments/assets/6c5e7277-b0de-46f8-825d-f13c8b77d7fd">


After

<img width="2045" alt="image"
src="https://github.com/user-attachments/assets/b74608a9-b1ca-4db8-8228-35bbbac82c0d">
This commit is contained in:
Chris Cowan 2024-07-31 12:32:59 -06:00 committed by GitHub
parent 4b701de036
commit 6de843b37f
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
2 changed files with 63 additions and 2 deletions

View file

@ -5,11 +5,11 @@
* 2.0.
*/
import { faker } from '@faker-js/faker';
import { omit, sample } from 'lodash';
import { SERVICE_LOGS } from '../../constants';
import { GeneratorFunction } from '../../types';
import { generateService } from './lib/generate_service';
import { generateLogMessage } from './lib/generate_log_message';
export const generateEvent: GeneratorFunction = (_config, _schedule, index, timestamp) => {
const service = generateService(index + 1);
@ -19,10 +19,10 @@ export const generateEvent: GeneratorFunction = (_config, _schedule, index, time
{
namespace: SERVICE_LOGS,
'@timestamp': timestamp.toISOString(),
message: faker.git.commitMessage(),
data_stream: { type: 'logs', dataset: SERVICE_LOGS, namespace: 'default' },
service: omit(service, 'hostsWithCloud'),
...hostWithCloud,
...generateLogMessage(timestamp),
},
];
};

View file

@ -0,0 +1,61 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import { faker } from '@faker-js/faker';
import { Moment } from 'moment';
import { URL } from 'node:url';
// $remote_addr - $remote_user [$time_local] "$request" $status $body_bytes_sent "$http_referer" "$http_user_agent"
export function generateLogMessage(timestamp: Moment) {
const statusCode = faker.internet.httpStatusCode();
const method = faker.internet.httpMethod();
const logLevel = statusCode >= 500 ? 'error' : 'info';
const userAgent = faker.internet.userAgent();
const remoteAddress = faker.internet.ipv4();
const path = `/api/${faker.word.noun()}/${faker.word.verb()}`;
const rawUrl = `${faker.internet.url()}/${path}`;
const parsedUrl = new URL(rawUrl);
const bytesSent = parseInt(
faker.string.numeric({ length: { min: 3, max: 10 }, allowLeadingZeros: false }),
10
);
const message = `${remoteAddress} - - [${timestamp.toISOString()}] "${method} ${path} HTTP/1.1" ${statusCode} ${bytesSent} "${
parsedUrl.origin
}" "${userAgent}"`;
return {
message,
log: { level: logLevel },
url: {
domain: parsedUrl.hostname,
port: parsedUrl.port || 80,
full: rawUrl,
original: rawUrl,
path,
scheme: parsedUrl.protocol,
},
http: {
request: {
method,
referrer: parsedUrl.origin,
mime_type: 'application/json',
body: {
bytes: parseInt(faker.string.numeric(3), 10),
},
},
response: {
bytes: bytesSent,
mime_type: 'application/json',
status_code: statusCode,
},
version: '1.1',
},
user_agent: {
original: userAgent,
},
};
}