github-mirrors/kibana
1
0
Fork 0
mirror of https://github.com/elastic/kibana.git synced 2025-04-23 09:19:04 -04:00
Code Issues Projects Releases Packages Wiki Activity

Use DNS caching (#184760)

Browse source 
Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com>
Co-authored-by: Jean-Louis Leysens <jeanlouis.leysens@elastic.co>
This commit is contained in:
Alejandro Fernández Haro 2024-06-10 14:33:38 +02:00 • committed by GitHub
parent cf7196fa1f
commit b1ff240cc4
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
17 changed files with 75 additions and 20 deletions
Download patch file Download diff file Expand all files Collapse all files

1
package.json
View file

@ -980,6 +980,7 @@
"brace": "0.11.1",
"brok": "^5.0.2",
"byte-size": "^8.1.0",
"cacheable-lookup": "6",
"camelcase-keys": "7.0.2",
"canvg": "^3.0.9",
"cbor-x": "^1.3.3",

18
packages/core/elasticsearch/core-elasticsearch-client-server-internal/src/agent_manager.test.ts
View file

@ -33,7 +33,7 @@ describe('AgentManager', () => {
describe('#getAgentFactory()', () => {
it('provides factories which are different at each call', () => {
const agentManager = new AgentManager(logger);
const agentManager = new AgentManager(logger, { dnsCacheTtlInSeconds: 0 });
const agentFactory1 = agentManager.getAgentFactory();
const agentFactory2 = agentManager.getAgentFactory();
expect(agentFactory1).not.toEqual(agentFactory2);
@ -45,7 +45,7 @@ describe('AgentManager', () => {
HttpAgentMock.mockImplementationOnce(() => mockedHttpAgent);
const mockedHttpsAgent = new HttpsAgent();
HttpsAgentMock.mockImplementationOnce(() => mockedHttpsAgent);
const agentManager = new AgentManager(logger);
const agentManager = new AgentManager(logger, { dnsCacheTtlInSeconds: 0 });
const agentFactory = agentManager.getAgentFactory();
const httpAgent = agentFactory({ url: new URL('http://elastic-node-1:9200') });
const httpsAgent = agentFactory({ url: new URL('https://elastic-node-1:9200') });
@ -54,7 +54,7 @@ describe('AgentManager', () => {
});
it('takes into account the provided configurations', () => {
const agentManager = new AgentManager(logger);
const agentManager = new AgentManager(logger, { dnsCacheTtlInSeconds: 0 });
const agentFactory = agentManager.getAgentFactory({
maxTotalSockets: 1024,
scheduling: 'fifo',
@ -77,7 +77,7 @@ describe('AgentManager', () => {
});
it('provides Agents that match the URLs protocol', () => {
const agentManager = new AgentManager(logger);
const agentManager = new AgentManager(logger, { dnsCacheTtlInSeconds: 0 });
const agentFactory = agentManager.getAgentFactory();
agentFactory({ url: new URL('http://elastic-node-1:9200') });
expect(HttpAgent).toHaveBeenCalledTimes(1);
@ -88,7 +88,7 @@ describe('AgentManager', () => {
});
it('provides the same Agent if URLs use the same protocol', () => {
const agentManager = new AgentManager(logger);
const agentManager = new AgentManager(logger, { dnsCacheTtlInSeconds: 0 });
const agentFactory = agentManager.getAgentFactory();
const agent1 = agentFactory({ url: new URL('http://elastic-node-1:9200') });
const agent2 = agentFactory({ url: new URL('http://elastic-node-2:9200') });
@ -101,7 +101,7 @@ describe('AgentManager', () => {
});
it('dereferences an agent instance when the agent is closed', () => {
const agentManager = new AgentManager(logger);
const agentManager = new AgentManager(logger, { dnsCacheTtlInSeconds: 0 });
const agentFactory = agentManager.getAgentFactory();
const agent = agentFactory({ url: new URL('http://elastic-node-1:9200') });
// eslint-disable-next-line dot-notation
@ -114,7 +114,7 @@ describe('AgentManager', () => {
describe('two agent factories', () => {
it('never provide the same Agent instance even if they use the same type', () => {
const agentManager = new AgentManager(logger);
const agentManager = new AgentManager(logger, { dnsCacheTtlInSeconds: 0 });
const agentFactory1 = agentManager.getAgentFactory();
const agentFactory2 = agentManager.getAgentFactory();
const agent1 = agentFactory1({ url: new URL('http://elastic-node-1:9200') });
@ -126,7 +126,7 @@ describe('AgentManager', () => {
describe('#getAgentsStats()', () => {
it('returns the stats of the agents', () => {
const agentManager = new AgentManager(logger);
const agentManager = new AgentManager(logger, { dnsCacheTtlInSeconds: 0 });
const metrics: ElasticsearchClientsMetrics = {
totalQueuedRequests: 0,
totalIdleSockets: 100,
@ -138,7 +138,7 @@ describe('AgentManager', () => {
});
it('warns when there are queued requests (requests unassigned to any socket)', () => {
const agentManager = new AgentManager(logger);
const agentManager = new AgentManager(logger, { dnsCacheTtlInSeconds: 0 });
const metrics: ElasticsearchClientsMetrics = {
totalQueuedRequests: 2,
totalIdleSockets: 100, // There may be idle sockets when many clients are initialized. It should not be taken as an indicator of health.

20
packages/core/elasticsearch/core-elasticsearch-client-server-internal/src/agent_manager.ts
View file

@ -8,6 +8,7 @@
import { Agent as HttpAgent, type AgentOptions } from 'http';
import { Agent as HttpsAgent } from 'https';
import CacheableLookup from 'cacheable-lookup';
import type { ConnectionOptions, HttpAgentOptions } from '@elastic/elasticsearch';
import type { Logger } from '@kbn/logging';
import type { ElasticsearchClientsMetrics } from '@kbn/core-metrics-server';
@ -22,6 +23,14 @@ export interface AgentFactoryProvider {
getAgentFactory(agentOptions?: HttpAgentOptions): AgentFactory;
}
export interface AgentManagerOptions {
/**
* The maximum number of seconds to retain the DNS lookup resolutions.
* Set to 0 to disable the cache (default Node.js behavior)
*/
dnsCacheTtlInSeconds: number;
}
/**
* Exposes the APIs to fetch stats of the existing agents.
*/
@ -45,9 +54,16 @@ export interface AgentStatsProvider {
**/
export class AgentManager implements AgentFactoryProvider, AgentStatsProvider {
private readonly agents: Set<HttpAgent>;
private readonly cacheableLookup?: CacheableLookup;
constructor(private readonly logger: Logger) {
constructor(private readonly logger: Logger, options: AgentManagerOptions) {
this.agents = new Set();
// Use DNS caching to avoid too many repetitive (and CPU-blocking) dns.lookup calls
if (options.dnsCacheTtlInSeconds > 0) {
this.cacheableLookup = new CacheableLookup({
maxTtl: options.dnsCacheTtlInSeconds,
});
}
}
public getAgentFactory(agentOptions?: AgentOptions): AgentFactory {
@ -63,6 +79,7 @@ export class AgentManager implements AgentFactoryProvider, AgentStatsProvider {
httpsAgent = new HttpsAgent(config);
this.agents.add(httpsAgent);
dereferenceOnDestroy(this.agents, httpsAgent);
this.cacheableLookup?.install(httpsAgent);
}
return httpsAgent;
@ -72,6 +89,7 @@ export class AgentManager implements AgentFactoryProvider, AgentStatsProvider {
httpAgent = new HttpAgent(agentOptions);
this.agents.add(httpAgent);
dereferenceOnDestroy(this.agents, httpAgent);
this.cacheableLookup?.install(httpAgent);
}
return httpAgent;

1
packages/core/elasticsearch/core-elasticsearch-client-server-internal/src/client_config.test.ts
View file

@ -25,6 +25,7 @@ const createConfig = (
sniffInterval: false,
requestHeadersWhitelist: ['authorization'],
hosts: ['http://localhost:80'],
dnsCacheTtlInSeconds: 0,
...parts,
};
};

3
packages/core/elasticsearch/core-elasticsearch-client-server-internal/src/cluster_client.test.ts
View file

@ -34,6 +34,7 @@ const createConfig = (
requestHeadersWhitelist: ['authorization'],
customHeaders: {},
hosts: ['http://localhost'],
dnsCacheTtlInSeconds: 0,
...parts,
};
};
@ -57,7 +58,7 @@ describe('ClusterClient', () => {
logger = loggingSystemMock.createLogger();
internalClient = createClient();
scopedClient = createClient();
agentFactoryProvider = new AgentManager(logger);
agentFactoryProvider = new AgentManager(logger, { dnsCacheTtlInSeconds: 0 });
authHeaders = httpServiceMock.createAuthHeaderStorage();
authHeaders.get.mockImplementation(() => ({

2
packages/core/elasticsearch/core-elasticsearch-client-server-internal/src/configure_client.test.ts
View file

@ -54,7 +54,7 @@ describe('configureClient', () => {
config = createFakeConfig();
parseClientOptionsMock.mockReturnValue({});
ClientMock.mockImplementation(() => createFakeClient());
agentFactoryProvider = new AgentManager(logger);
agentFactoryProvider = new AgentManager(logger, { dnsCacheTtlInSeconds: 0 });
});
afterEach(() => {

1
packages/core/elasticsearch/core-elasticsearch-server-internal/src/elasticsearch_config.test.ts
View file

@ -33,6 +33,7 @@ test('set correct defaults', () => {
"apisToRedactInLogs": Array [],
"compression": false,
"customHeaders": Object {},
"dnsCacheTtlInSeconds": 0,
"healthCheckDelay": "PT2.5S",
"healthCheckStartupDelay": "PT0.5S",
"hosts": Array [

8
packages/core/elasticsearch/core-elasticsearch-server-internal/src/elasticsearch_config.ts
View file

@ -186,6 +186,7 @@ export const configSchema = schema.object({
}),
{ defaultValue: [] }
),
dnsCacheTtlInSeconds: schema.number({ defaultValue: 0, min: 0, max: Infinity }),
});
const deprecations: ConfigDeprecationProvider = () => [
@ -427,6 +428,12 @@ export class ElasticsearchConfig implements IElasticsearchConfig {
*/
public readonly apisToRedactInLogs: ElasticsearchApiToRedactInLogs[];
/**
* The maximum number of seconds to retain the DNS lookup resolutions.
* Set to 0 to disable the cache (default Node.js behavior)
*/
public readonly dnsCacheTtlInSeconds: number;
constructor(rawConfig: ElasticsearchConfigType) {
this.ignoreVersionMismatch = rawConfig.ignoreVersionMismatch;
this.apiVersion = rawConfig.apiVersion;
@ -452,6 +459,7 @@ export class ElasticsearchConfig implements IElasticsearchConfig {
this.compression = rawConfig.compression;
this.skipStartupConnectionCheck = rawConfig.skipStartupConnectionCheck;
this.apisToRedactInLogs = rawConfig.apisToRedactInLogs;
this.dnsCacheTtlInSeconds = rawConfig.dnsCacheTtlInSeconds;
const { alwaysPresentCertificate, verificationMode } = rawConfig.ssl;
const { key, keyPassphrase, certificate, certificateAuthorities } = readKeyAndCerts(rawConfig);

16
packages/core/elasticsearch/core-elasticsearch-server-internal/src/elasticsearch_service.ts
View file

@ -61,7 +61,7 @@ export class ElasticsearchService
private client?: ClusterClient;
private clusterInfo$?: Observable<ClusterInfo>;
private unauthorizedErrorHandler?: UnauthorizedErrorHandler;
private agentManager: AgentManager;
private agentManager?: AgentManager;
constructor(private readonly coreContext: CoreContext) {
this.kibanaVersion = coreContext.env.packageInfo.version;
@ -69,7 +69,6 @@ export class ElasticsearchService
this.config$ = coreContext.configService
.atPath<ElasticsearchConfigType>('elasticsearch')
.pipe(map((rawConfig) => new ElasticsearchConfig(rawConfig)));
this.agentManager = new AgentManager(this.log.get('agent-manager'));
}
public async preboot(): Promise<InternalElasticsearchServicePreboot> {
@ -93,6 +92,8 @@ export class ElasticsearchService
const config = await firstValueFrom(this.config$);
const agentManager = this.getAgentManager(config);
this.authHeaders = deps.http.authRequestHeaders;
this.executionContextClient = deps.executionContext;
this.client = this.createClusterClient('data', config);
@ -125,7 +126,7 @@ export class ElasticsearchService
this.unauthorizedErrorHandler = handler;
},
agentStatsProvider: {
getAgentsStats: this.agentManager.getAgentsStats.bind(this.agentManager),
getAgentsStats: agentManager.getAgentsStats.bind(agentManager),
},
};
}
@ -218,8 +219,15 @@ export class ElasticsearchService
authHeaders: this.authHeaders,
getExecutionContext: () => this.executionContextClient?.getAsHeader(),
getUnauthorizedErrorHandler: () => this.unauthorizedErrorHandler,
agentFactoryProvider: this.agentManager,
agentFactoryProvider: this.getAgentManager(baseConfig),
kibanaVersion: this.kibanaVersion,
});
}
private getAgentManager({ dnsCacheTtlInSeconds }: ElasticsearchClientConfig): AgentManager {
if (!this.agentManager) {
this.agentManager = new AgentManager(this.log.get('agent-manager'), { dnsCacheTtlInSeconds });
}
return this.agentManager;
}
}

1
packages/core/elasticsearch/core-elasticsearch-server/src/client/client_config.ts
View file

@ -50,6 +50,7 @@ export interface ElasticsearchClientConfig {
caFingerprint?: string;
ssl?: ElasticsearchClientSslConfig;
apisToRedactInLogs?: ElasticsearchApiToRedactInLogs[];
dnsCacheTtlInSeconds: number;
}
/**

6
packages/core/elasticsearch/core-elasticsearch-server/src/elasticsearch_config.ts
View file

@ -149,6 +149,12 @@ export interface IElasticsearchConfig {
* Extends the list of APIs that should be redacted in logs.
*/
readonly apisToRedactInLogs: ElasticsearchApiToRedactInLogs[];
/**
* The maximum number of seconds to retain the DNS lookup resolutions.
* Set to 0 to disable the cache (default Node.js behavior)
*/
readonly dnsCacheTtlInSeconds: number;
}
/**

2
packages/core/metrics/core-metrics-server-internal/src/ops_metrics_collector.test.ts
View file

@ -29,7 +29,7 @@ describe('OpsMetricsCollector', () => {
beforeEach(() => {
const hapiServer = httpServiceMock.createInternalSetupContract().server;
const agentManager = new AgentManager(loggerMock.create());
const agentManager = new AgentManager(loggerMock.create(), { dnsCacheTtlInSeconds: 0 });
collector = new OpsMetricsCollector(hapiServer, agentManager, { logger: loggerMock.create() });
mockOsCollector.collect.mockResolvedValue('osMetrics');

3
packages/core/test-helpers/core-test-helpers-model-versions/src/test_bed/test_kit.ts
View file

@ -202,7 +202,8 @@ const getElasticsearchClient = async (
logger: loggerFactory.get('elasticsearch'),
type: 'data',
agentFactoryProvider: new AgentManager(
loggerFactory.get('elasticsearch-service', 'agent-manager')
loggerFactory.get('elasticsearch-service', 'agent-manager'),
{ dnsCacheTtlInSeconds: 0 }
),
kibanaVersion,
});

4
src/cli_setup/utils.ts
View file

@ -49,7 +49,9 @@ export const elasticsearch = new ElasticsearchService(logger, kibanaPackageJson.
logger,
type,
// we use an independent AgentManager for cli_setup, no need to track performance of this one
agentFactoryProvider: new AgentManager(logger.get('agent-manager')),
agentFactoryProvider: new AgentManager(logger.get('agent-manager'), {
dnsCacheTtlInSeconds: 0,
}),
kibanaVersion: kibanaPackageJson.version,
});
},

3
src/core/server/integration_tests/saved_objects/migrations/kibana_migrator_test_kit.ts
View file

@ -266,7 +266,8 @@ const getElasticsearchClient = async (
logger: loggerFactory.get('elasticsearch'),
type: 'data',
agentFactoryProvider: new AgentManager(
loggerFactory.get('elasticsearch-service', 'agent-manager')
loggerFactory.get('elasticsearch-service', 'agent-manager'),
{ dnsCacheTtlInSeconds: 0 }
),
kibanaVersion,
});

1
x-pack/plugins/monitoring/server/config.test.ts
View file

@ -59,6 +59,7 @@ describe('config schema', () => {
"apisToRedactInLogs": Array [],
"compression": false,
"customHeaders": Object {},
"dnsCacheTtlInSeconds": 0,
"healthCheck": Object {
"delay": "PT2.5S",
"startupDelay": "PT0.5S",

5
yarn.lock
View file

@ -13461,6 +13461,11 @@ cache-base@^1.0.1:
union-value "^1.0.0"
unset-value "^1.0.0"
cacheable-lookup@6:
version "6.1.0"
resolved "https://registry.yarnpkg.com/cacheable-lookup/-/cacheable-lookup-6.1.0.tgz#0330a543471c61faa4e9035db583aad753b36385"
integrity sha512-KJ/Dmo1lDDhmW2XDPMo+9oiy/CeqosPguPCrgcVzKyZrL6pM1gU2GmPY/xo6OQPTUaA/c0kwHuywB4E6nmT9ww==
cacheable-lookup@^5.0.3:
version "5.0.3"
resolved "https://registry.yarnpkg.com/cacheable-lookup/-/cacheable-lookup-5.0.3.tgz#049fdc59dffdd4fc285e8f4f82936591bd59fec3"