github-mirrors/kibana
1
0
Fork 0
mirror of https://github.com/elastic/kibana.git synced 2025-04-24 09:48:58 -04:00
Code Issues Projects Releases Packages Wiki Activity

[8.6] [APM][AWS] fix compute usage calc (#146328) (#146510)

Browse source 
# Backport

This will backport the following commits from `main` to `8.6`:
- [[APM][AWS] fix compute usage calc
(#146328)](https://github.com/elastic/kibana/pull/146328)

<!--- Backport version: 8.9.7 -->

### Questions ?
Please refer to the [Backport tool
documentation](https://github.com/sqren/backport)

<!--BACKPORT [{"author":{"name":"Cauê
Marcondes","email":"55978943+cauemarcondes@users.noreply.github.com"},"sourceCommit":{"committedDate":"2022-11-29T09:42:59Z","message":"[APM][AWS]
fix compute usage calc (#146328)\n\ncloses
https://github.com/elastic/kibana/issues/146206\r\n\r\n**Before** we
were averaging the memory and billed duration and then we\r\ncalculated
the compute usage.\r\n**Now** We first calculate the compute usage then
get the average and\r\nthen convert to GB-Sec.\r\n\r\nCo-authored-by:
Kibana Machine
<42973632+kibanamachine@users.noreply.github.com>","sha":"9cadd361ded281514626db5d0d49154a62d7484a","branchLabelMapping":{"^v8.7.0$":"main","^v(\\d+).(\\d+).\\d+$":"$1.$2"}},"sourcePullRequest":{"labels":["Team:APM","release_note:skip","v8.6.0","v8.7.0"],"number":146328,"url":"https://github.com/elastic/kibana/pull/146328","mergeCommit":{"message":"[APM][AWS]
fix compute usage calc (#146328)\n\ncloses
https://github.com/elastic/kibana/issues/146206\r\n\r\n**Before** we
were averaging the memory and billed duration and then we\r\ncalculated
the compute usage.\r\n**Now** We first calculate the compute usage then
get the average and\r\nthen convert to GB-Sec.\r\n\r\nCo-authored-by:
Kibana Machine
<42973632+kibanamachine@users.noreply.github.com>","sha":"9cadd361ded281514626db5d0d49154a62d7484a"}},"sourceBranch":"main","suggestedTargetBranches":["8.6"],"targetPullRequestStates":[{"branch":"8.6","label":"v8.6.0","labelRegex":"^v(\\d+).(\\d+).\\d+$","isSourceBranch":false,"state":"NOT_CREATED"},{"branch":"main","label":"v8.7.0","labelRegex":"^v8.7.0$","isSourceBranch":true,"state":"MERGED","url":"https://github.com/elastic/kibana/pull/146328","number":146328,"mergeCommit":{"message":"[APM][AWS]
fix compute usage calc (#146328)\n\ncloses
https://github.com/elastic/kibana/issues/146206\r\n\r\n**Before** we
were averaging the memory and billed duration and then we\r\ncalculated
the compute usage.\r\n**Now** We first calculate the compute usage then
get the average and\r\nthen convert to GB-Sec.\r\n\r\nCo-authored-by:
Kibana Machine
<42973632+kibanamachine@users.noreply.github.com>","sha":"9cadd361ded281514626db5d0d49154a62d7484a"}}]}]
BACKPORT-->

Co-authored-by: Cauê Marcondes <55978943+cauemarcondes@users.noreply.github.com>
This commit is contained in:
Kibana Machine 2022-11-29 05:47:50 -05:00 • committed by GitHub
parent 1b2ce1a5a8
commit 2458f6f283
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
4 changed files with 79 additions and 49 deletions
Download patch file Download diff file Expand all files Collapse all files

22
x-pack/plugins/apm/server/routes/metrics/serverless/get_compute_usage_chart.ts
View file

@ -25,7 +25,13 @@ import { environmentQuery } from '../../../../common/utils/environment_query';
import { getMetricsDateHistogramParams } from '../../../lib/helpers/metrics';
import { GenericMetricsChart } from '../fetch_and_transform_metrics';
import { APMEventClient } from '../../../lib/helpers/create_es_client/create_apm_event_client';
import { calcComputeUsageGBSeconds } from './helper';
import { convertComputeUsageToGbSec } from './helper';
export const computeUsageAvgScript = {
avg: {
script: `return doc['${METRIC_SYSTEM_TOTAL_MEMORY}'].value * doc['${FAAS_BILLED_DURATION}'].value`,
},
};
export async function getComputeUsageChart({
environment,
@ -47,9 +53,8 @@ export async function getComputeUsageChart({
serverlessId?: string;
}): Promise<GenericMetricsChart> {
const aggs = {
avgFaasBilledDuration: { avg: { field: FAAS_BILLED_DURATION } },
avgTotalMemory: { avg: { field: METRIC_SYSTEM_TOTAL_MEMORY } },
countInvocations: { value_count: { field: FAAS_BILLED_DURATION } },
avgComputeUsageBytesMs: computeUsageAvgScript,
};
const params = {
@ -117,17 +122,16 @@ export async function getComputeUsageChart({
key: 'compute_usage',
type: 'bar',
overallValue:
calcComputeUsageGBSeconds({
billedDuration: aggregations?.avgFaasBilledDuration.value,
totalMemory: aggregations?.avgTotalMemory.value,
convertComputeUsageToGbSec({
computeUsageBytesMs:
aggregations?.avgComputeUsageBytesMs.value,
countInvocations: aggregations?.countInvocations.value,
}) ?? 0,
color: theme.euiColorVis0,
data: timeseriesData.buckets.map((bucket) => {
const computeUsage =
calcComputeUsageGBSeconds({
billedDuration: bucket.avgFaasBilledDuration.value,
totalMemory: bucket.avgTotalMemory.value,
convertComputeUsageToGbSec({
computeUsageBytesMs: bucket.avgComputeUsageBytesMs.value,
countInvocations: bucket.countInvocations.value,
}) ?? 0;
return {

16
x-pack/plugins/apm/server/routes/metrics/serverless/get_serverless_summary.ts
View file

@ -22,7 +22,12 @@ import {
} from '../../../../common/elasticsearch_fieldnames';
import { environmentQuery } from '../../../../common/utils/environment_query';
import { APMEventClient } from '../../../lib/helpers/create_es_client/create_apm_event_client';
import { calcEstimatedCost, calcMemoryUsedRate } from './helper';
import { computeUsageAvgScript } from './get_compute_usage_chart';
import {
calcEstimatedCost,
calcMemoryUsedRate,
convertComputeUsageToGbSec,
} from './helper';
export type AwsLambdaArchitecture = 'arm' | 'x86_64';
@ -121,6 +126,7 @@ export async function getServerlessSummary({
avgTotalMemory: { avg: { field: METRIC_SYSTEM_TOTAL_MEMORY } },
avgFreeMemory: { avg: { field: METRIC_SYSTEM_FREE_MEMORY } },
countInvocations: { value_count: { field: FAAS_BILLED_DURATION } },
avgComputeUsageBytesMs: computeUsageAvgScript,
sample: {
top_metrics: {
metrics: [{ field: HOST_ARCHITECTURE }],
@ -159,9 +165,11 @@ export async function getServerlessSummary({
HOST_ARCHITECTURE
] as AwsLambdaArchitecture | undefined,
transactionThroughput,
billedDuration: response.aggregations?.faasBilledDurationAvg.value,
totalMemory: response.aggregations?.avgTotalMemory.value,
countInvocations: response.aggregations?.countInvocations.value,
computeUsageGbSec: convertComputeUsageToGbSec({
computeUsageBytesMs:
response.aggregations?.avgComputeUsageBytesMs.value,
countInvocations: response.aggregations?.countInvocations.value,
}),
}),
};
}

56
x-pack/plugins/apm/server/routes/metrics/serverless/helper.test.ts
View file

@ -8,7 +8,32 @@ import {
calcMemoryUsed,
calcMemoryUsedRate,
calcEstimatedCost,
convertComputeUsageToGbSec,
} from './helper';
describe('convertComputeUsageToGbSec', () => {
it('returns undefined', () => {
[
{ computeUsageBytesMs: undefined, countInvocations: 1 },
{ computeUsageBytesMs: null, countInvocations: 1 },
{ computeUsageBytesMs: 1, countInvocations: undefined },
{ computeUsageBytesMs: 1, countInvocations: null },
].forEach(({ computeUsageBytesMs, countInvocations }) => {
expect(
convertComputeUsageToGbSec({ computeUsageBytesMs, countInvocations })
).toBeUndefined();
});
});
it('converts to gb sec', () => {
const totalMemory = 536870912; // 0.5gb
const billedDuration = 4000;
const computeUsageBytesMs = totalMemory * billedDuration;
expect(
convertComputeUsageToGbSec({ computeUsageBytesMs, countInvocations: 1 })
).toBe(computeUsageBytesMs / 1024 ** 3 / 1000);
});
});
describe('calcMemoryUsed', () => {
it('returns undefined when memory values are no a number', () => {
[
@ -49,13 +74,19 @@ const AWS_LAMBDA_PRICE_FACTOR = {
};
describe('calcEstimatedCost', () => {
const totalMemory = 536870912; // 0.5gb
const billedDuration = 4000;
const computeUsageBytesMs = totalMemory * billedDuration;
const computeUsageGbSec = convertComputeUsageToGbSec({
computeUsageBytesMs,
countInvocations: 1,
});
it('returns undefined when price factor is not defined', () => {
expect(
calcEstimatedCost({
totalMemory: 1,
billedDuration: 1,
transactionThroughput: 1,
architecture: 'arm',
computeUsageGbSec,
})
).toBeUndefined();
});
@ -63,10 +94,9 @@ describe('calcEstimatedCost', () => {
it('returns undefined when architecture is not defined', () => {
expect(
calcEstimatedCost({
totalMemory: 1,
billedDuration: 1,
transactionThroughput: 1,
awsLambdaPriceFactor: AWS_LAMBDA_PRICE_FACTOR,
computeUsageGbSec,
})
).toBeUndefined();
});
@ -84,11 +114,10 @@ describe('calcEstimatedCost', () => {
it('returns undefined when request cost per million is not defined', () => {
expect(
calcEstimatedCost({
totalMemory: 1,
billedDuration: 1,
transactionThroughput: 1,
awsLambdaPriceFactor: AWS_LAMBDA_PRICE_FACTOR,
architecture: 'arm',
computeUsageGbSec,
})
).toBeUndefined();
});
@ -100,11 +129,9 @@ describe('calcEstimatedCost', () => {
calcEstimatedCost({
awsLambdaPriceFactor: AWS_LAMBDA_PRICE_FACTOR,
architecture,
billedDuration: 4000,
totalMemory: 536870912, // 0.5gb
transactionThroughput: 100000,
awsLambdaRequestCostPerMillion: 0.2,
countInvocations: 1,
computeUsageGbSec,
})
).toEqual(0.03);
});
@ -112,15 +139,20 @@ describe('calcEstimatedCost', () => {
describe('arm architecture', () => {
const architecture = 'arm';
it('returns correct cost', () => {
const _totalMemory = 536870912; // 0.5gb
const _billedDuration = 8000;
const _computeUsageBytesMs = _totalMemory * _billedDuration;
const _computeUsageGbSec = convertComputeUsageToGbSec({
computeUsageBytesMs: _computeUsageBytesMs,
countInvocations: 1,
});
expect(
calcEstimatedCost({
awsLambdaPriceFactor: AWS_LAMBDA_PRICE_FACTOR,
architecture,
billedDuration: 8000,
totalMemory: 536870912, // 0.5gb
transactionThroughput: 200000,
awsLambdaRequestCostPerMillion: 0.2,
countInvocations: 1,
computeUsageGbSec: _computeUsageGbSec,
})
).toEqual(0.05);
});

34
x-pack/plugins/apm/server/routes/metrics/serverless/helper.ts
View file

@ -44,57 +44,43 @@ const GB = 1024 ** 3;
* But the result of this calculation is in Bytes-milliseconds, as the "system.memory.total" is stored in bytes and the "faas.billed_duration" is stored in milliseconds.
* But to calculate the overall cost AWS uses GB-second, so we need to convert the result to this unit.
*/
export function calcComputeUsageGBSeconds({
billedDuration,
totalMemory,
export function convertComputeUsageToGbSec({
computeUsageBytesMs,
countInvocations,
}: {
billedDuration?: number | null;
totalMemory?: number | null;
computeUsageBytesMs?: number | null;
countInvocations?: number | null;
}) {
if (
!isFiniteNumber(billedDuration) ||
!isFiniteNumber(totalMemory) ||
!isFiniteNumber(computeUsageBytesMs) ||
!isFiniteNumber(countInvocations)
) {
return undefined;
}
const totalMemoryGB = totalMemory / GB;
const faasBilledDurationSec = billedDuration / 1000;
return totalMemoryGB * faasBilledDurationSec * countInvocations;
const computeUsageGbSec = computeUsageBytesMs / GB / 1000;
return computeUsageGbSec * countInvocations;
}
export function calcEstimatedCost({
awsLambdaPriceFactor,
architecture,
transactionThroughput,
billedDuration,
totalMemory,
awsLambdaRequestCostPerMillion,
countInvocations,
computeUsageGbSec,
}: {
awsLambdaPriceFactor?: AWSLambdaPriceFactor;
architecture?: AwsLambdaArchitecture;
transactionThroughput: number;
billedDuration?: number | null;
totalMemory?: number | null;
awsLambdaRequestCostPerMillion?: number;
countInvocations?: number | null;
computeUsageGbSec?: number;
}) {
try {
const computeUsage = calcComputeUsageGBSeconds({
billedDuration,
totalMemory,
countInvocations,
});
if (
!awsLambdaPriceFactor ||
!architecture ||
!isFiniteNumber(awsLambdaRequestCostPerMillion) ||
!isFiniteNumber(awsLambdaPriceFactor?.[architecture]) ||
!isFiniteNumber(computeUsage)
!isFiniteNumber(computeUsageGbSec)
) {
return undefined;
}
@ -102,7 +88,7 @@ export function calcEstimatedCost({
const priceFactor = awsLambdaPriceFactor?.[architecture];
const estimatedCost =
computeUsage * priceFactor +
computeUsageGbSec * priceFactor +
transactionThroughput * (awsLambdaRequestCostPerMillion / 1000000);
// Rounds up the decimals