[Maps] Geo containment latency and concurrent containment fix (#86980)

This commit is contained in:
Aaron Caldwell 2021-01-28 09:18:59 -07:00 committed by GitHub
parent 7593cf7ea5
commit 80b720da11
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
5 changed files with 289 additions and 145 deletions

View file

@ -18,7 +18,6 @@ export interface GeoContainmentAlertParams extends AlertTypeParams {
boundaryIndexId: string;
boundaryGeoField: string;
boundaryNameField?: string;
delayOffsetWithUnits?: string;
indexQuery?: Query;
boundaryIndexQuery?: Query;
}

View file

@ -98,7 +98,6 @@ export const ParamsSchema = schema.object({
boundaryIndexId: schema.string({ minLength: 1 }),
boundaryGeoField: schema.string({ minLength: 1 }),
boundaryNameField: schema.maybe(schema.string({ minLength: 1 })),
delayOffsetWithUnits: schema.maybe(schema.string({ minLength: 1 })),
indexQuery: schema.maybe(schema.any({})),
boundaryIndexQuery: schema.maybe(schema.any({})),
});
@ -114,7 +113,6 @@ export interface GeoContainmentParams extends AlertTypeParams {
boundaryIndexId: string;
boundaryGeoField: string;
boundaryNameField?: string;
delayOffsetWithUnits?: string;
indexQuery?: Query;
boundaryIndexQuery?: Query;
}

View file

@ -24,7 +24,7 @@ export function transformResults(
results: SearchResponse<unknown> | undefined,
dateField: string,
geoField: string
): Map<string, LatestEntityLocation> {
): Map<string, LatestEntityLocation[]> {
if (!results) {
return new Map();
}
@ -64,12 +64,15 @@ export function transformResults(
// Get unique
.reduce(
(
accu: Map<string, LatestEntityLocation>,
accu: Map<string, LatestEntityLocation[]>,
el: LatestEntityLocation & { entityName: string }
) => {
const { entityName, ...locationData } = el;
if (!accu.has(entityName)) {
accu.set(entityName, locationData);
if (entityName) {
if (!accu.has(entityName)) {
accu.set(entityName, []);
}
accu.get(entityName)!.push(locationData);
}
return accu;
},
@ -78,26 +81,9 @@ export function transformResults(
return orderedResults;
}
function getOffsetTime(delayOffsetWithUnits: string, oldTime: Date): Date {
const timeUnit = delayOffsetWithUnits.slice(-1);
const time: number = +delayOffsetWithUnits.slice(0, -1);
const adjustedDate = new Date(oldTime.getTime());
if (timeUnit === 's') {
adjustedDate.setSeconds(adjustedDate.getSeconds() - time);
} else if (timeUnit === 'm') {
adjustedDate.setMinutes(adjustedDate.getMinutes() - time);
} else if (timeUnit === 'h') {
adjustedDate.setHours(adjustedDate.getHours() - time);
} else if (timeUnit === 'd') {
adjustedDate.setDate(adjustedDate.getDate() - time);
}
return adjustedDate;
}
export function getActiveEntriesAndGenerateAlerts(
prevLocationMap: Record<string, LatestEntityLocation>,
currLocationMap: Map<string, LatestEntityLocation>,
prevLocationMap: Map<string, LatestEntityLocation[]>,
currLocationMap: Map<string, LatestEntityLocation[]>,
alertInstanceFactory: AlertServices<
GeoContainmentInstanceState,
GeoContainmentInstanceContext,
@ -106,32 +92,55 @@ export function getActiveEntriesAndGenerateAlerts(
shapesIdsNamesMap: Record<string, unknown>,
currIntervalEndTime: Date
) {
const allActiveEntriesMap: Map<string, LatestEntityLocation> = new Map([
...Object.entries(prevLocationMap || {}),
const allActiveEntriesMap: Map<string, LatestEntityLocation[]> = new Map([
...prevLocationMap,
...currLocationMap,
]);
allActiveEntriesMap.forEach(({ location, shapeLocationId, dateInShape, docId }, entityName) => {
const containingBoundaryName = shapesIdsNamesMap[shapeLocationId] || shapeLocationId;
const context = {
entityId: entityName,
entityDateTime: dateInShape ? new Date(dateInShape).toISOString() : null,
entityDocumentId: docId,
detectionDateTime: new Date(currIntervalEndTime).toISOString(),
entityLocation: `POINT (${location[0]} ${location[1]})`,
containingBoundaryId: shapeLocationId,
containingBoundaryName,
};
const alertInstanceId = `${entityName}-${containingBoundaryName}`;
if (shapeLocationId === OTHER_CATEGORY) {
allActiveEntriesMap.forEach((locationsArr, entityName) => {
// Generate alerts
locationsArr.forEach(({ location, shapeLocationId, dateInShape, docId }) => {
const context = {
entityId: entityName,
entityDateTime: dateInShape ? new Date(dateInShape).toISOString() : null,
entityDocumentId: docId,
detectionDateTime: new Date(currIntervalEndTime).toISOString(),
entityLocation: `POINT (${location[0]} ${location[1]})`,
containingBoundaryId: shapeLocationId,
containingBoundaryName: shapesIdsNamesMap[shapeLocationId] || shapeLocationId,
};
const alertInstanceId = `${entityName}-${context.containingBoundaryName}`;
if (shapeLocationId !== OTHER_CATEGORY) {
alertInstanceFactory(alertInstanceId).scheduleActions(ActionGroupId, context);
}
});
if (locationsArr[0].shapeLocationId === OTHER_CATEGORY) {
allActiveEntriesMap.delete(entityName);
return;
}
const otherCatIndex = locationsArr.findIndex(
({ shapeLocationId }) => shapeLocationId === OTHER_CATEGORY
);
if (otherCatIndex >= 0) {
const afterOtherLocationsArr = locationsArr.slice(0, otherCatIndex);
allActiveEntriesMap.set(entityName, afterOtherLocationsArr);
} else {
alertInstanceFactory(alertInstanceId).scheduleActions(ActionGroupId, context);
allActiveEntriesMap.set(entityName, locationsArr);
}
});
return allActiveEntriesMap;
}
export const getGeoContainmentExecutor = (log: Logger): GeoContainmentAlertType['executor'] =>
async function ({ previousStartedAt, startedAt, services, params, alertId, state }) {
async function ({
previousStartedAt: currIntervalStartTime,
startedAt: currIntervalEndTime,
services,
params,
alertId,
state,
}) {
const { shapesFilters, shapesIdsNamesMap } = state.shapesFilters
? state
: await getShapesFilters(
@ -147,15 +156,6 @@ export const getGeoContainmentExecutor = (log: Logger): GeoContainmentAlertType[
const executeEsQuery = await executeEsQueryFactory(params, services, log, shapesFilters);
let currIntervalStartTime = previousStartedAt;
let currIntervalEndTime = startedAt;
if (params.delayOffsetWithUnits) {
if (currIntervalStartTime) {
currIntervalStartTime = getOffsetTime(params.delayOffsetWithUnits, currIntervalStartTime);
}
currIntervalEndTime = getOffsetTime(params.delayOffsetWithUnits, currIntervalEndTime);
}
// Start collecting data only on the first cycle
let currentIntervalResults: SearchResponse<unknown> | undefined;
if (!currIntervalStartTime) {
@ -169,14 +169,17 @@ export const getGeoContainmentExecutor = (log: Logger): GeoContainmentAlertType[
currentIntervalResults = await executeEsQuery(currIntervalStartTime, currIntervalEndTime);
}
const currLocationMap: Map<string, LatestEntityLocation> = transformResults(
const currLocationMap: Map<string, LatestEntityLocation[]> = transformResults(
currentIntervalResults,
params.dateField,
params.geoField
);
const prevLocationMap: Map<string, LatestEntityLocation[]> = new Map([
...Object.entries((state.prevLocationMap as Record<string, LatestEntityLocation[]>) || {}),
]);
const allActiveEntriesMap = getActiveEntriesAndGenerateAlerts(
state.prevLocationMap as Record<string, LatestEntityLocation>,
prevLocationMap,
currLocationMap,
services.alertInstanceFactory,
shapesIdsNamesMap,

View file

@ -38,7 +38,6 @@ describe('alertType', () => {
boundaryIndexId: 'testIndex',
boundaryGeoField: 'testField',
boundaryNameField: 'testField',
delayOffsetWithUnits: 'testOffset',
};
expect(alertType.validate?.params?.validate(params)).toBeTruthy();

View file

@ -27,39 +27,47 @@ describe('geo_containment', () => {
new Map([
[
'936',
{
dateInShape: '2020-09-28T18:01:41.190Z',
docId: 'N-ng1XQB6yyY-xQxnGSM',
location: [-82.8814151789993, 40.62806099653244],
shapeLocationId: '0DrJu3QB6yyY-xQxv6Ip',
},
[
{
dateInShape: '2020-09-28T18:01:41.190Z',
docId: 'N-ng1XQB6yyY-xQxnGSM',
location: [-82.8814151789993, 40.62806099653244],
shapeLocationId: '0DrJu3QB6yyY-xQxv6Ip',
},
],
],
[
'AAL2019',
{
dateInShape: '2020-09-28T18:01:41.191Z',
docId: 'iOng1XQB6yyY-xQxnGSM',
location: [-82.22068064846098, 39.006176185794175],
shapeLocationId: '0DrJu3QB6yyY-xQxv6Ip',
},
[
{
dateInShape: '2020-09-28T18:01:41.191Z',
docId: 'iOng1XQB6yyY-xQxnGSM',
location: [-82.22068064846098, 39.006176185794175],
shapeLocationId: '0DrJu3QB6yyY-xQxv6Ip',
},
],
],
[
'AAL2323',
{
dateInShape: '2020-09-28T18:01:41.191Z',
docId: 'n-ng1XQB6yyY-xQxnGSM',
location: [-84.71324851736426, 41.6677269525826],
shapeLocationId: '0DrJu3QB6yyY-xQxv6Ip',
},
[
{
dateInShape: '2020-09-28T18:01:41.191Z',
docId: 'n-ng1XQB6yyY-xQxnGSM',
location: [-84.71324851736426, 41.6677269525826],
shapeLocationId: '0DrJu3QB6yyY-xQxv6Ip',
},
],
],
[
'ABD5250',
{
dateInShape: '2020-09-28T18:01:41.192Z',
docId: 'GOng1XQB6yyY-xQxnGWM',
location: [6.073727197945118, 39.07997465226799],
shapeLocationId: '0DrJu3QB6yyY-xQxv6Ip',
},
[
{
dateInShape: '2020-09-28T18:01:41.192Z',
docId: 'GOng1XQB6yyY-xQxnGWM',
location: [6.073727197945118, 39.07997465226799],
shapeLocationId: '0DrJu3QB6yyY-xQxv6Ip',
},
],
],
])
);
@ -77,39 +85,47 @@ describe('geo_containment', () => {
new Map([
[
'936',
{
dateInShape: '2020-09-28T18:01:41.190Z',
docId: 'N-ng1XQB6yyY-xQxnGSM',
location: [-82.8814151789993, 40.62806099653244],
shapeLocationId: '0DrJu3QB6yyY-xQxv6Ip',
},
[
{
dateInShape: '2020-09-28T18:01:41.190Z',
docId: 'N-ng1XQB6yyY-xQxnGSM',
location: [-82.8814151789993, 40.62806099653244],
shapeLocationId: '0DrJu3QB6yyY-xQxv6Ip',
},
],
],
[
'AAL2019',
{
dateInShape: '2020-09-28T18:01:41.191Z',
docId: 'iOng1XQB6yyY-xQxnGSM',
location: [-82.22068064846098, 39.006176185794175],
shapeLocationId: '0DrJu3QB6yyY-xQxv6Ip',
},
[
{
dateInShape: '2020-09-28T18:01:41.191Z',
docId: 'iOng1XQB6yyY-xQxnGSM',
location: [-82.22068064846098, 39.006176185794175],
shapeLocationId: '0DrJu3QB6yyY-xQxv6Ip',
},
],
],
[
'AAL2323',
{
dateInShape: '2020-09-28T18:01:41.191Z',
docId: 'n-ng1XQB6yyY-xQxnGSM',
location: [-84.71324851736426, 41.6677269525826],
shapeLocationId: '0DrJu3QB6yyY-xQxv6Ip',
},
[
{
dateInShape: '2020-09-28T18:01:41.191Z',
docId: 'n-ng1XQB6yyY-xQxnGSM',
location: [-84.71324851736426, 41.6677269525826],
shapeLocationId: '0DrJu3QB6yyY-xQxv6Ip',
},
],
],
[
'ABD5250',
{
dateInShape: '2020-09-28T18:01:41.192Z',
docId: 'GOng1XQB6yyY-xQxnGWM',
location: [6.073727197945118, 39.07997465226799],
shapeLocationId: '0DrJu3QB6yyY-xQxv6Ip',
},
[
{
dateInShape: '2020-09-28T18:01:41.192Z',
docId: 'GOng1XQB6yyY-xQxnGWM',
location: [6.073727197945118, 39.07997465226799],
shapeLocationId: '0DrJu3QB6yyY-xQxv6Ip',
},
],
],
])
);
@ -131,30 +147,36 @@ describe('geo_containment', () => {
const currLocationMap = new Map([
[
'a',
{
location: [0, 0],
shapeLocationId: '123',
dateInShape: 'Wed Dec 09 2020 14:31:31 GMT-0700 (Mountain Standard Time)',
docId: 'docId1',
},
[
{
location: [0, 0],
shapeLocationId: '123',
dateInShape: 'Wed Dec 09 2020 14:31:31 GMT-0700 (Mountain Standard Time)',
docId: 'docId1',
},
],
],
[
'b',
{
location: [0, 0],
shapeLocationId: '456',
dateInShape: 'Wed Dec 09 2020 15:31:31 GMT-0700 (Mountain Standard Time)',
docId: 'docId2',
},
[
{
location: [0, 0],
shapeLocationId: '456',
dateInShape: 'Wed Dec 16 2020 15:31:31 GMT-0700 (Mountain Standard Time)',
docId: 'docId2',
},
],
],
[
'c',
{
location: [0, 0],
shapeLocationId: '789',
dateInShape: 'Wed Dec 09 2020 16:31:31 GMT-0700 (Mountain Standard Time)',
docId: 'docId3',
},
[
{
location: [0, 0],
shapeLocationId: '789',
dateInShape: 'Wed Dec 23 2020 16:31:31 GMT-0700 (Mountain Standard Time)',
docId: 'docId3',
},
],
],
]);
@ -215,7 +237,7 @@ describe('geo_containment', () => {
const currentDateTime = new Date();
it('should use currently active entities if no older entity entries', () => {
const emptyPrevLocationMap = {};
const emptyPrevLocationMap = new Map();
const allActiveEntriesMap = getActiveEntriesAndGenerateAlerts(
emptyPrevLocationMap,
currLocationMap,
@ -227,14 +249,19 @@ describe('geo_containment', () => {
expect(testAlertActionArr).toMatchObject(expectedContext);
});
it('should overwrite older identical entity entries', () => {
const prevLocationMapWithIdenticalEntityEntry = {
a: {
location: [0, 0],
shapeLocationId: '999',
dateInShape: 'Wed Dec 09 2020 12:31:31 GMT-0700 (Mountain Standard Time)',
docId: 'docId7',
},
};
const prevLocationMapWithIdenticalEntityEntry = new Map([
[
'a',
[
{
location: [0, 0],
shapeLocationId: '999',
dateInShape: 'Wed Dec 09 2020 12:31:31 GMT-0700 (Mountain Standard Time)',
docId: 'docId7',
},
],
],
]);
const allActiveEntriesMap = getActiveEntriesAndGenerateAlerts(
prevLocationMapWithIdenticalEntityEntry,
currLocationMap,
@ -246,14 +273,19 @@ describe('geo_containment', () => {
expect(testAlertActionArr).toMatchObject(expectedContext);
});
it('should preserve older non-identical entity entries', () => {
const prevLocationMapWithNonIdenticalEntityEntry = {
d: {
location: [0, 0],
shapeLocationId: '999',
dateInShape: 'Wed Dec 09 2020 12:31:31 GMT-0700 (Mountain Standard Time)',
docId: 'docId7',
},
};
const prevLocationMapWithNonIdenticalEntityEntry = new Map([
[
'd',
[
{
location: [0, 0],
shapeLocationId: '999',
dateInShape: 'Wed Dec 09 2020 12:31:31 GMT-0700 (Mountain Standard Time)',
docId: 'docId7',
},
],
],
]);
const expectedContextPlusD = [
{
actionGroupId: 'Tracked entity contained',
@ -279,14 +311,17 @@ describe('geo_containment', () => {
expect(allActiveEntriesMap.has('d')).toBeTruthy();
expect(testAlertActionArr).toMatchObject(expectedContextPlusD);
});
it('should remove "other" entries and schedule the expected number of actions', () => {
const emptyPrevLocationMap = {};
const currLocationMapWithOther = new Map(currLocationMap).set('d', {
location: [0, 0],
shapeLocationId: OTHER_CATEGORY,
dateInShape: 'Wed Dec 09 2020 14:31:31 GMT-0700 (Mountain Standard Time)',
docId: 'docId1',
});
const emptyPrevLocationMap = new Map();
const currLocationMapWithOther = new Map([...currLocationMap]).set('d', [
{
location: [0, 0],
shapeLocationId: OTHER_CATEGORY,
dateInShape: 'Wed Dec 09 2020 14:31:31 GMT-0700 (Mountain Standard Time)',
docId: 'docId1',
},
]);
expect(currLocationMapWithOther).not.toEqual(currLocationMap);
const allActiveEntriesMap = getActiveEntriesAndGenerateAlerts(
emptyPrevLocationMap,
@ -298,5 +333,115 @@ describe('geo_containment', () => {
expect(allActiveEntriesMap).toEqual(currLocationMap);
expect(testAlertActionArr).toMatchObject(expectedContext);
});
it('should generate multiple alerts per entity if found in multiple shapes in interval', () => {
const emptyPrevLocationMap = new Map();
const currLocationMapWithThreeMore = new Map([...currLocationMap]).set('d', [
{
location: [0, 0],
shapeLocationId: '789',
dateInShape: 'Wed Dec 10 2020 14:31:31 GMT-0700 (Mountain Standard Time)',
docId: 'docId1',
},
{
location: [0, 0],
shapeLocationId: '123',
dateInShape: 'Wed Dec 08 2020 12:31:31 GMT-0700 (Mountain Standard Time)',
docId: 'docId2',
},
{
location: [0, 0],
shapeLocationId: '456',
dateInShape: 'Wed Dec 07 2020 10:31:31 GMT-0700 (Mountain Standard Time)',
docId: 'docId3',
},
]);
getActiveEntriesAndGenerateAlerts(
emptyPrevLocationMap,
currLocationMapWithThreeMore,
alertInstanceFactory,
emptyShapesIdsNamesMap,
currentDateTime
);
let numEntitiesInShapes = 0;
currLocationMapWithThreeMore.forEach((v) => {
numEntitiesInShapes += v.length;
});
expect(testAlertActionArr.length).toEqual(numEntitiesInShapes);
});
it('should not return entity as active entry if most recent location is "other"', () => {
const emptyPrevLocationMap = new Map();
const currLocationMapWithOther = new Map([...currLocationMap]).set('d', [
{
location: [0, 0],
shapeLocationId: OTHER_CATEGORY,
dateInShape: 'Wed Dec 10 2020 14:31:31 GMT-0700 (Mountain Standard Time)',
docId: 'docId1',
},
{
location: [0, 0],
shapeLocationId: '123',
dateInShape: 'Wed Dec 08 2020 12:31:31 GMT-0700 (Mountain Standard Time)',
docId: 'docId1',
},
{
location: [0, 0],
shapeLocationId: '456',
dateInShape: 'Wed Dec 07 2020 10:31:31 GMT-0700 (Mountain Standard Time)',
docId: 'docId1',
},
]);
expect(currLocationMapWithOther).not.toEqual(currLocationMap);
const allActiveEntriesMap = getActiveEntriesAndGenerateAlerts(
emptyPrevLocationMap,
currLocationMapWithOther,
alertInstanceFactory,
emptyShapesIdsNamesMap,
currentDateTime
);
expect(allActiveEntriesMap).toEqual(currLocationMap);
});
it('should return entity as active entry if "other" not the latest location but remove "other" and earlier entries', () => {
const emptyPrevLocationMap = new Map();
const currLocationMapWithOther = new Map([...currLocationMap]).set('d', [
{
location: [0, 0],
shapeLocationId: '123',
dateInShape: 'Wed Dec 10 2020 14:31:31 GMT-0700 (Mountain Standard Time)',
docId: 'docId1',
},
{
location: [0, 0],
shapeLocationId: OTHER_CATEGORY,
dateInShape: 'Wed Dec 08 2020 12:31:31 GMT-0700 (Mountain Standard Time)',
docId: 'docId1',
},
{
location: [0, 0],
shapeLocationId: '456',
dateInShape: 'Wed Dec 07 2020 10:31:31 GMT-0700 (Mountain Standard Time)',
docId: 'docId1',
},
]);
const allActiveEntriesMap = getActiveEntriesAndGenerateAlerts(
emptyPrevLocationMap,
currLocationMapWithOther,
alertInstanceFactory,
emptyShapesIdsNamesMap,
currentDateTime
);
expect(allActiveEntriesMap).toEqual(
new Map([...currLocationMap]).set('d', [
{
location: [0, 0],
shapeLocationId: '123',
dateInShape: 'Wed Dec 10 2020 14:31:31 GMT-0700 (Mountain Standard Time)',
docId: 'docId1',
},
])
);
});
});
});