github-mirrors/kibana
1
0
Fork 0
mirror of https://github.com/elastic/kibana.git synced 2025-04-24 09:48:58 -04:00
Code Issues Projects Releases Packages Wiki Activity

[Security solution] Security Assistant Cypress (#191000)

Browse source
This commit is contained in:
Steph Milovic 2024-08-29 14:21:33 -06:00 • committed by GitHub
parent dd221a7be1
commit a2315ab94c
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
33 changed files with 844 additions and 47 deletions
Download patch file Download diff file Expand all files Collapse all files

2
x-pack/packages/kbn-elastic-assistant/impl/assistant/assistant_body/empty_convo.tsx
View file

@ -38,7 +38,7 @@ export const EmptyConvo: React.FC<Props> = ({
setIsSettingsModalVisible,
}) => {
return (
<EuiFlexGroup alignItems="center" justifyContent="center">
<EuiFlexGroup alignItems="center" justifyContent="center" data-test-subj="emptyConvo">
<EuiFlexItem grow={false}>
<EuiPanel
hasShadow={false}

7
x-pack/packages/kbn-elastic-assistant/impl/assistant/assistant_body/welcome_setup.tsx
View file

@ -31,7 +31,12 @@ export const WelcomeSetup: React.FC<Props> = ({
text-align: center;
`}
>
<EuiFlexGroup alignItems="center" justifyContent="center" direction="column">
<EuiFlexGroup
alignItems="center"
justifyContent="center"
direction="column"
data-test-subj="welcome-setup"
>
<EuiFlexItem grow={false}>
<AssistantAnimatedIcon />
</EuiFlexItem>

1
x-pack/packages/kbn-elastic-assistant/impl/assistant/assistant_overlay/flyout_navigation.tsx
View file

@ -106,6 +106,7 @@ export const FlyoutNavigation = memo<FlyoutNavigationProps>(
size="xs"
color="primary"
iconType="newChat"
data-test-subj="newChatFromOverlay"
onClick={onConversationCreate}
disabled={isLoading || !isAssistantEnabled}
>

1
x-pack/packages/kbn-elastic-assistant/impl/assistant/assistant_title/index.tsx
View file

@ -59,6 +59,7 @@ export const AssistantTitle: React.FC<{
`}
>
<EuiInlineEditTitle
data-test-subj="conversationTitle"
heading="h2"
inputAriaLabel="Edit text inline"
value={newTitle ?? NEW_CHAT}

1
x-pack/packages/kbn-elastic-assistant/impl/assistant/conversations/conversation_sidepanel/index.tsx
View file

@ -197,6 +197,7 @@ export const ConversationSidePanel = React.memo<Props>(
onConversationSelected({ cId: conversation.id, cTitle: conversation.title })
}
label={conversation.title}
data-test-subj={`conversation-select-${conversation.title}`}
isActive={
!isEmpty(conversation.id)
? conversation.id === currentConversation?.id

2
x-pack/packages/kbn-elastic-assistant/impl/assistant/index.tsx
View file

@ -456,7 +456,7 @@ const AssistantComponent: React.FC<Props> = ({
overflow: hidden;
`}
>
<CommentContainer>
<CommentContainer data-test-subj="assistantChat">
<EuiFlexGroup
css={css`
overflow: hidden;

5
x-pack/packages/kbn-elastic-assistant/impl/assistant/prompt_editor/selected_prompt_contexts/index.tsx
View file

@ -45,10 +45,13 @@ const SelectedPromptContextsComponent: React.FC<Props> = ({
<EuiFlexGroup data-test-subj="selectedPromptContexts" direction="column" gutterSize={'s'}>
{Object.keys(selectedPromptContexts)
.sort()
.map((id) => (
.map((id, i) => (
<EuiFlexItem data-test-subj={`selectedPromptContext-${id}`} grow={false} key={id}>
<EuiAccordion
buttonContent={promptContexts[id]?.description}
buttonProps={{
'data-test-subj': `selectedPromptContext-${i}-button`,
}}
extraAction={
<EuiToolTip content={i18n.REMOVE_CONTEXT}>
<EuiButtonIcon

4
x-pack/packages/kbn-elastic-assistant/impl/assistant/prompt_editor/system_prompt/helpers.test.tsx
View file

@ -34,7 +34,9 @@ describe('helpers', () => {
render(<TestProviders>{option.dropdownDisplay}</TestProviders>);
expect(screen.getByTestId('name')).toHaveTextContent(mockSystemPrompt.name);
expect(screen.getByTestId(`systemPrompt-${mockSystemPrompt.name}`)).toHaveTextContent(
mockSystemPrompt.name
);
});
it('shows the expected prompt content in the dropdownDisplay', () => {

2
x-pack/packages/kbn-elastic-assistant/impl/assistant/prompt_editor/system_prompt/helpers.tsx
View file

@ -46,7 +46,7 @@ export const getOptionFromPrompt = ({
),
dropdownDisplay: (
<>
<Strong data-test-subj="name">{name}</Strong>
<Strong data-test-subj={`systemPrompt-${name}`}>{name}</Strong>
{/* Empty content tooltip gets around :hover styles from SuperSelectOptionButton */}
<EuiToolTip content={undefined}>

1
x-pack/packages/kbn-elastic-assistant/impl/assistant/prompt_editor/system_prompt/index.tsx
View file

@ -67,7 +67,6 @@ const SystemPromptComponent: React.FC<Props> = ({
isCleared={isCleared}
refetchConversations={refetchConversations}
isSettingsModalVisible={isSettingsModalVisible}
onSystemPromptSelectionChange={onSystemPromptSelectionChange}
selectedPrompt={selectedPrompt}
setIsSettingsModalVisible={setIsSettingsModalVisible}
/>

2
x-pack/packages/kbn-elastic-assistant/impl/assistant/prompt_editor/system_prompt/system_prompt_modal/system_prompt_settings.tsx
View file

@ -33,7 +33,7 @@ export const SystemPromptSettings: React.FC<SystemPromptSettingsProps> = React.m
return (
<>
<EuiTitle size={'s'}>
<h2>{i18n.SETTINGS_TITLE}</h2>
<h2 data-test-subj={`systemPromptSettingsTitle`}>{i18n.SETTINGS_TITLE}</h2>
</EuiTitle>
<EuiSpacer size="xs" />
<EuiText size={'s'}>{i18n.SETTINGS_DESCRIPTION}</EuiText>

1
x-pack/packages/kbn-elastic-assistant/impl/assistant/quick_prompts/prompt_context_selector/prompt_context_selector.tsx
View file

@ -88,6 +88,7 @@ export const PromptContextSelector: React.FC<Props> = React.memo(
<EuiComboBox
aria-label={i18n.PROMPT_CONTEXT_SELECTOR}
compressed
data-test-subj={'promptContextSelector'}
fullWidth
isDisabled={isDisabled}
placeholder={i18n.PROMPT_CONTEXT_SELECTOR_PLACEHOLDER}

1
x-pack/packages/kbn-elastic-assistant/impl/assistant/quick_prompts/quick_prompts.tsx
View file

@ -130,6 +130,7 @@ export const QuickPrompts: React.FC<QuickPromptsProps> = React.memo(
>
<EuiBadge
color={badge.color}
data-test-subj={`quickPrompt-${badge.name}`}
onClick={() => onClickAddQuickPrompt(badge)}
onClickAriaLabel={badge.name}
>

1
x-pack/packages/kbn-elastic-assistant/impl/assistant/upgrade_license_cta/index.tsx
View file

@ -25,6 +25,7 @@ export const UpgradeLicenseCallToAction: React.FC<Props> = ({ http }) => {
const basePath = http.basePath.get();
return (
<EuiFlexGroup
data-test-subj="upgradeLicenseCallToAction"
justifyContent="center"
direction="column"
alignItems="center"

2
x-pack/packages/kbn-elastic-assistant/impl/connectorland/connector_selector/index.tsx
View file

@ -115,7 +115,7 @@ export const ConnectorSelector: React.FC<Props> = React.memo(
dropdownDisplay: (
<React.Fragment key={connector.id}>
<EuiFlexGroup justifyContent="spaceBetween" gutterSize="none" alignItems="center">
<EuiFlexItem grow={false}>
<EuiFlexItem grow={false} data-test-subj={`connector-${connector.name}`}>
<strong>{connector.name}</strong>
{connectorDetails && (
<EuiText size="xs" color="subdued">

1
x-pack/plugins/security_solution/public/assistant/get_comments/index.tsx
View file

@ -184,6 +184,7 @@ export const getComments = ({
content={transformedMessage.content}
index={index}
isControlsEnabled={isControlsEnabled}
isError={message.isError}
// reader is used to determine if streaming controls are shown
reader={transformedMessage.reader}
regenerateMessage={regenerateMessageOfConversation}

9
x-pack/plugins/security_solution/public/assistant/get_comments/stream/index.tsx
View file

@ -102,7 +102,14 @@ export const StreamComment = ({
return (
<MessagePanel
body={<MessageText content={message} index={index} loading={isAnythingLoading} />}
body={
<MessageText
data-test-subj={isError ? 'errorComment' : undefined}
content={message}
index={index}
loading={isAnythingLoading}
/>
}
error={error ? new Error(error) : undefined}
controls={controls}
/>

5
x-pack/plugins/security_solution/public/assistant/get_comments/stream/message_text.tsx
View file

@ -30,6 +30,7 @@ interface Props {
content: string;
index: number;
loading: boolean;
['data-test-subj']?: string;
}
const ANIMATION_TIME = 1;
@ -143,7 +144,7 @@ const getPluginDependencies = () => {
};
};
export function MessageText({ loading, content, index }: Props) {
export function MessageText({ loading, content, index, 'data-test-subj': dataTestSubj }: Props) {
const containerClassName = css`
overflow-wrap: anywhere;
`;
@ -151,7 +152,7 @@ export function MessageText({ loading, content, index }: Props) {
const { parsingPluginList, processingPluginList } = getPluginDependencies();
return (
<EuiText className={containerClassName}>
<EuiText className={containerClassName} data-test-subj={dataTestSubj}>
<EuiMarkdownFormat
// used by augmentMessageCodeBlocks
className={`message-${index}`}

2
x-pack/plugins/security_solution/public/assistant/send_to_timeline/index.tsx
View file

@ -259,6 +259,7 @@ export const SendToTimelineButton: FC<PropsWithChildren<SendToTimelineButtonProp
isDisabled={isDisabled}
color="text"
flush="both"
data-test-subj="sendToTimelineEmptyButton"
size="xs"
>
<EuiToolTip position="right" content={toolTipText}>
@ -270,6 +271,7 @@ export const SendToTimelineButton: FC<PropsWithChildren<SendToTimelineButtonProp
aria-label={toolTipText}
isDisabled={isDisabled}
onClick={configureAndOpenTimeline}
data-test-subj="sendToTimelineButton"
{...rest}
>
<EuiToolTip position="right" content={toolTipText}>

35
x-pack/test/security_solution_cypress/cypress/e2e/ai_assistant/ai_assistant_feature_complete.cy.ts
View file

@ -1,35 +0,0 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import { AI_ASSISTANT_BUTTON } from '../../screens/ai_assistant';
import { login } from '../../tasks/login';
import { visitGetStartedPage } from '../../tasks/navigation';
describe(
'App Features for Security Complete',
{
tags: ['@serverless'],
env: {
ftrConfig: {
productTypes: [
{ product_line: 'security', product_tier: 'complete' },
{ product_line: 'endpoint', product_tier: 'complete' },
],
},
},
},
() => {
beforeEach(() => {
login();
});
it('should have have AI Assistant available', () => {
visitGetStartedPage();
cy.get(AI_ASSISTANT_BUTTON).should('exist');
});
}
);

173
x-pack/test/security_solution_cypress/cypress/e2e/ai_assistant/conversations.cy.ts Normal file
View file

@ -0,0 +1,173 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import {
EXPLAIN_THEN_SUMMARIZE_RULE_DETAILS,
RULE_MANAGEMENT_CONTEXT_DESCRIPTION,
} from '@kbn/security-solution-plugin/public/detections/pages/detection_engine/rules/translations';
import { EXPLAIN_THEN_SUMMARIZE_SUGGEST_INVESTIGATION_GUIDE_NON_I18N } from '@kbn/security-solution-plugin/public/assistant/content/prompts/user/translations';
import {
assertConnectorSelected,
assertNewConversation,
closeAssistant,
openAssistant,
selectConnector,
createNewChat,
selectConversation,
assertMessageSent,
typeAndSendMessage,
assertErrorResponse,
selectRule,
assertErrorToastShown,
updateConversationTitle,
assertSystemPrompt,
} from '../../tasks/assistant';
import { deleteConversations } from '../../tasks/api_calls/assistant';
import {
azureConnectorAPIPayload,
bedrockConnectorAPIPayload,
createAzureConnector,
createBedrockConnector,
} from '../../tasks/api_calls/connectors';
import { expandFirstAlert } from '../../tasks/alerts';
import { ALERTS_URL } from '../../urls/navigation';
import { waitForAlertsToPopulate } from '../../tasks/create_new_rule';
import { visitRulesManagementTable } from '../../tasks/rules_management';
import { deleteAlertsAndRules, deleteConnectors } from '../../tasks/api_calls/common';
import { createRule } from '../../tasks/api_calls/rules';
import { getExistingRule, getNewRule } from '../../objects/rule';
import { login } from '../../tasks/login';
import {
CONNECTOR_MISSING_CALLOUT,
PROMPT_CONTEXT_BUTTON,
USER_PROMPT,
} from '../../screens/ai_assistant';
import { visit, visitGetStartedPage } from '../../tasks/navigation';
describe('AI Assistant Conversations', { tags: ['@ess', '@serverless'] }, () => {
beforeEach(() => {
deleteConnectors();
deleteConversations();
deleteAlertsAndRules();
login();
});
describe('No connectors or conversations exist', () => {
it('Shows welcome setup when no connectors or conversations exist', () => {
visitGetStartedPage();
openAssistant();
assertNewConversation(true, 'Welcome');
});
});
describe('When no conversations exist but connectors do exist, show empty convo', () => {
beforeEach(() => {
createAzureConnector();
});
it('When invoked on AI Assistant click', () => {
visitGetStartedPage();
openAssistant();
assertNewConversation(false, 'Welcome');
assertConnectorSelected(azureConnectorAPIPayload.name);
assertSystemPrompt('Default system prompt');
cy.get(USER_PROMPT).should('not.have.text');
});
it('When invoked from rules page', () => {
createRule(getExistingRule({ rule_id: 'rule1', enabled: true })).then((createdRule) => {
visitRulesManagementTable();
selectRule(createdRule?.body?.id);
openAssistant('rule');
assertNewConversation(false, 'Detection Rules');
assertConnectorSelected(azureConnectorAPIPayload.name);
assertSystemPrompt('Default system prompt');
cy.get(USER_PROMPT).should('have.text', EXPLAIN_THEN_SUMMARIZE_RULE_DETAILS);
cy.get(PROMPT_CONTEXT_BUTTON(0)).should('have.text', RULE_MANAGEMENT_CONTEXT_DESCRIPTION);
});
});
it('When invoked from alert details', () => {
createRule(getNewRule());
visit(ALERTS_URL);
waitForAlertsToPopulate();
expandFirstAlert();
openAssistant('alert');
assertNewConversation(false, 'Alert summary');
assertConnectorSelected(azureConnectorAPIPayload.name);
assertSystemPrompt('Default system prompt');
cy.get(USER_PROMPT).should(
'have.text',
EXPLAIN_THEN_SUMMARIZE_SUGGEST_INVESTIGATION_GUIDE_NON_I18N
);
cy.get(PROMPT_CONTEXT_BUTTON(0)).should('have.text', 'Alert (from summary)');
});
it('Shows empty connector callout when a conversation that had a connector no longer does', () => {
visitGetStartedPage();
openAssistant();
assertConnectorSelected(azureConnectorAPIPayload.name);
closeAssistant();
deleteConnectors();
openAssistant();
cy.get(CONNECTOR_MISSING_CALLOUT).should('be.visible');
});
});
describe('Changing conversations', () => {
beforeEach(() => {
createAzureConnector();
createBedrockConnector();
});
it('Last conversation persists in memory from page to page', () => {
createRule(getNewRule());
visit(ALERTS_URL);
waitForAlertsToPopulate();
expandFirstAlert();
openAssistant('alert');
assertNewConversation(false, 'Alert summary');
closeAssistant();
visitGetStartedPage();
openAssistant();
assertNewConversation(false, 'Alert summary');
});
it('Properly switches back and forth between conversations', () => {
visitGetStartedPage();
openAssistant();
assertNewConversation(false, 'Welcome');
assertConnectorSelected(azureConnectorAPIPayload.name);
typeAndSendMessage('hello');
assertMessageSent('hello', true);
assertErrorResponse();
selectConversation('Alert summary');
selectConnector(bedrockConnectorAPIPayload.name);
typeAndSendMessage('goodbye');
assertMessageSent('goodbye', true);
assertErrorResponse();
selectConversation('Welcome');
assertConnectorSelected(azureConnectorAPIPayload.name);
assertMessageSent('hello', true);
selectConversation('Alert summary');
assertConnectorSelected(bedrockConnectorAPIPayload.name);
assertMessageSent('goodbye', true);
});
// This test is flakey due to the issue linked below and will be skipped until it is fixed
it.skip('Only allows one conversation called "New chat" at a time', () => {
visitGetStartedPage();
openAssistant();
createNewChat();
assertNewConversation(false, 'New chat');
assertConnectorSelected(azureConnectorAPIPayload.name);
typeAndSendMessage('hello');
// TODO fix bug with new chat and error message
// https://github.com/elastic/kibana/issues/191025
// assertMessageSent('hello', true);
assertErrorResponse();
selectConversation('Welcome');
createNewChat();
assertErrorToastShown('Error creating conversation with title New chat');
selectConversation('New chat');
updateConversationTitle('My other chat');
createNewChat();
assertNewConversation(false, 'New chat');
});
});
});

26
x-pack/test/security_solution_cypress/cypress/e2e/ai_assistant/ess_basic.cy.ts Normal file
View file

@ -0,0 +1,26 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import { startBasicLicense } from '../../tasks/api_calls/licensing';
import { UPGRADE_CTA } from '../../screens/ai_assistant';
import { login } from '../../tasks/login';
import { assertConversationReadOnly, openAssistant } from '../../tasks/assistant';
import { visitGetStartedPage } from '../../tasks/navigation';
describe('AI Assistant - Basic License', { tags: ['@ess'] }, () => {
beforeEach(() => {
login();
startBasicLicense();
visitGetStartedPage();
});
it('user with Basic license should not be able to use assistant', () => {
openAssistant();
cy.get(UPGRADE_CTA).should('be.visible');
assertConversationReadOnly();
});
});

21
x-pack/test/security_solution_cypress/cypress/e2e/ai_assistant/feature_complete.cy.ts Normal file
View file

@ -0,0 +1,21 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import { AI_ASSISTANT_BUTTON } from '../../screens/ai_assistant';
import { login } from '../../tasks/login';
import { visitGetStartedPage } from '../../tasks/navigation';
describe('App Features for Security Complete', { tags: ['@serverless'] }, () => {
beforeEach(() => {
login();
});
it('should have have AI Assistant available', () => {
visitGetStartedPage();
cy.get(AI_ASSISTANT_BUTTON).should('be.visible');
});
});

0
x-pack/test/security_solution_cypress/cypress/e2e/ai_assistant/ai_assistant_feature_essentials.cy.ts → x-pack/test/security_solution_cypress/cypress/e2e/ai_assistant/feature_essentials.cy.ts
View file

72
x-pack/test/security_solution_cypress/cypress/e2e/ai_assistant/messages.cy.ts Normal file
View file

@ -0,0 +1,72 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import { MessageRole } from '@kbn/elastic-assistant-common';
import { TIMELINE_QUERY } from '../../screens/timeline';
import { CASES_URL } from '../../urls/navigation';
import { SEND_TO_TIMELINE_BUTTON } from '../../screens/ai_assistant';
import { openAssistant, selectConversation, sendQueryToTimeline } from '../../tasks/assistant';
import {
deleteConversations,
deletePrompts,
waitForConversation,
} from '../../tasks/api_calls/assistant';
import { createAzureConnector } from '../../tasks/api_calls/connectors';
import { deleteConnectors } from '../../tasks/api_calls/common';
import { login } from '../../tasks/login';
import { visit, visitGetStartedPage } from '../../tasks/navigation';
describe(
'AI Assistant Messages',
// TODO - Fix this test to work in serverless - https://github.com/elastic/kibana/pull/190152
{ tags: ['@ess', '@serverless', '@skipInServerless'] },
() => {
const mockTimelineQuery = 'host.risk.keyword: "high"';
const mockConvo = {
id: 'spooky',
title: 'Spooky convo',
messages: [
{
timestamp: '2024-08-15T18:30:37.873Z',
content:
'You are a helpful, expert assistant who answers questions about Elastic Security. Do not answer questions unrelated to Elastic Security.\nIf you answer a question related to KQL, EQL, or ES|QL, it should be immediately usable within an Elastic Security timeline; please always format the output correctly with back ticks. Any answer provided for Query DSL should also be usable in a security timeline. This means you should only ever include the "filter" portion of the query.\n\nGive a query I can run in the timeline',
role: 'user' as MessageRole,
},
{
timestamp: '2024-08-15T18:31:24.008Z',
content:
'To query events from a high-risk host in the Elastic Security timeline, you can use the following KQL query:\n\n```kql\n' +
mockTimelineQuery +
'\n```',
role: 'assistant' as MessageRole,
traceData: {
traceId: '74d2fac29753adebd5c479e3d9e45da3',
transactionId: 'e13d97d138b8a13c',
},
},
],
};
beforeEach(() => {
deleteConnectors();
deleteConversations();
deletePrompts();
login();
createAzureConnector();
waitForConversation(mockConvo);
});
it('A message with a kql query can be used in the timeline only from pages with timeline', () => {
visitGetStartedPage();
openAssistant();
selectConversation(mockConvo.title);
cy.get(SEND_TO_TIMELINE_BUTTON).should('be.disabled');
visit(CASES_URL);
openAssistant();
sendQueryToTimeline();
cy.get(TIMELINE_QUERY).should('have.text', `${mockTimelineQuery}`);
});
}
);

132
x-pack/test/security_solution_cypress/cypress/e2e/ai_assistant/prompts.cy.ts Normal file
View file

@ -0,0 +1,132 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import { SUPERHERO_SYSTEM_PROMPT_NON_I18N } from '@kbn/security-solution-plugin/public/assistant/content/prompts/system/translations';
import { EXPLAIN_THEN_SUMMARIZE_SUGGEST_INVESTIGATION_GUIDE_NON_I18N } from '@kbn/security-solution-plugin/public/assistant/content/prompts/user/translations';
import { QUICK_PROMPT_BADGE, USER_PROMPT } from '../../screens/ai_assistant';
import { createRule } from '../../tasks/api_calls/rules';
import {
assertErrorResponse,
assertMessageSent,
assertSystemPrompt,
clearSystemPrompt,
createQuickPrompt,
createSystemPrompt,
openAssistant,
resetConversation,
selectConversation,
selectSystemPrompt,
sendQuickPrompt,
typeAndSendMessage,
} from '../../tasks/assistant';
import { deleteConversations, deletePrompts } from '../../tasks/api_calls/assistant';
import { createAzureConnector } from '../../tasks/api_calls/connectors';
import { deleteConnectors } from '../../tasks/api_calls/common';
import { login } from '../../tasks/login';
import { visit, visitGetStartedPage } from '../../tasks/navigation';
import { getNewRule } from '../../objects/rule';
import { ALERTS_URL } from '../../urls/navigation';
import { waitForAlertsToPopulate } from '../../tasks/create_new_rule';
import { expandFirstAlert } from '../../tasks/alerts';
const testPrompt = {
title: 'Cool prompt',
prompt: 'This is a super cool prompt.',
};
describe('AI Assistant Prompts', { tags: ['@ess', '@serverless'] }, () => {
beforeEach(() => {
deleteConnectors();
deleteConversations();
deletePrompts();
login();
createAzureConnector();
});
describe('System Prompts', () => {
it('Deselecting default system prompt prevents prompt from being sent. When conversation is then cleared, the prompt is reset.', () => {
visitGetStartedPage();
openAssistant();
clearSystemPrompt();
typeAndSendMessage('hello');
assertMessageSent('hello');
// ensure response before clearing convo
assertErrorResponse();
resetConversation();
typeAndSendMessage('hello');
assertMessageSent('hello', true);
});
it('Last selected system prompt persists in conversation', () => {
visitGetStartedPage();
openAssistant();
selectSystemPrompt('Enhanced system prompt');
typeAndSendMessage('hello');
assertMessageSent('hello', true, SUPERHERO_SYSTEM_PROMPT_NON_I18N);
resetConversation();
assertSystemPrompt('Enhanced system prompt');
selectConversation('Alert summary');
assertSystemPrompt('Default system prompt');
selectConversation('Welcome');
assertSystemPrompt('Enhanced system prompt');
});
it('Add prompt from system prompt selector without setting a default conversation', () => {
visitGetStartedPage();
openAssistant();
createSystemPrompt(testPrompt.title, testPrompt.prompt);
// we did not set a default conversation, so the prompt should not be set
assertSystemPrompt('Default system prompt');
selectSystemPrompt(testPrompt.title);
typeAndSendMessage('hello');
assertMessageSent('hello', true, testPrompt.prompt);
});
it('Add prompt from system prompt selector and set multiple conversations (including current) as default conversation', () => {
visitGetStartedPage();
openAssistant();
createSystemPrompt(testPrompt.title, testPrompt.prompt, ['Welcome', 'Alert summary']);
assertSystemPrompt(testPrompt.title);
typeAndSendMessage('hello');
assertMessageSent('hello', true, testPrompt.prompt);
// ensure response before changing convo
assertErrorResponse();
selectConversation('Alert summary');
assertSystemPrompt(testPrompt.title);
typeAndSendMessage('hello');
assertMessageSent('hello', true, testPrompt.prompt);
});
});
describe('User Prompts', () => {
it('Add a quick prompt and send it in the conversation', () => {
visitGetStartedPage();
openAssistant();
createQuickPrompt(testPrompt.title, testPrompt.prompt);
sendQuickPrompt(testPrompt.title);
assertMessageSent(testPrompt.prompt, true);
});
it('Add a quick prompt with context and it is only available in the selected context', () => {
visitGetStartedPage();
openAssistant();
createQuickPrompt(testPrompt.title, testPrompt.prompt, ['Alert (from view)']);
cy.get(QUICK_PROMPT_BADGE(testPrompt.title)).should('not.exist');
createRule(getNewRule());
visit(ALERTS_URL);
waitForAlertsToPopulate();
expandFirstAlert();
openAssistant('alert');
cy.get(QUICK_PROMPT_BADGE(testPrompt.title)).should('be.visible');
cy.get(USER_PROMPT).should(
'have.text',
EXPLAIN_THEN_SUMMARIZE_SUGGEST_INVESTIGATION_GUIDE_NON_I18N
);
cy.get(QUICK_PROMPT_BADGE(testPrompt.title)).click();
cy.get(USER_PROMPT).should('have.text', testPrompt.prompt);
});
// TODO delete quick prompt
// I struggled to do this since the element is hidden with css and I cannot get it to show
});
});

40
x-pack/test/security_solution_cypress/cypress/objects/assistant.ts Normal file
View file

@ -0,0 +1,40 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import {
ConversationCategory,
ConversationCreateProps,
ConversationResponse,
Provider,
} from '@kbn/elastic-assistant-common';
export const getMockConversation = (body?: Partial<ConversationCreateProps>) => ({
title: 'Test Conversation',
apiConfig: {
actionTypeId: '.gen-ai',
connectorId: '',
defaultSystemPromptId: 'default-system-prompt',
model: 'test-model',
provider: 'OpenAI' as Provider,
},
excludeFromLastConversationStorage: false,
isDefault: false,
messages: [],
replacements: {},
category: 'assistant' as ConversationCategory,
...body,
});
export const getMockConversationResponse = (
body?: Partial<ConversationCreateProps>
): ConversationResponse => ({
id: 'test-conversation-id',
createdAt: '2023-10-31T00:00:00.000Z',
users: [{ id: 'elastic', name: 'elastic@elastic.co' }],
namespace: 'default',
...getMockConversation(body),
});

45
x-pack/test/security_solution_cypress/cypress/screens/ai_assistant.ts
View file

@ -5,4 +5,49 @@
* 2.0.
*/
export const ADD_NEW_CONNECTOR = '[data-test-subj="addNewConnectorButton"]';
export const ADD_QUICK_PROMPT = '[data-test-subj="addQuickPrompt"]';
export const ASSISTANT_SETTINGS_BUTTON = 'button[data-test-subj="settings"]';
export const AI_ASSISTANT_BUTTON = '[data-test-subj="assistantHeaderLink"]';
export const ASSISTANT_CHAT_BODY = '[data-test-subj="assistantChat"]';
export const CHAT_CONTEXT_MENU = '[data-test-subj="chat-context-menu"]';
export const CHAT_ICON = '[data-test-subj="newChat"]';
export const CHAT_ICON_SM = '[data-test-subj="newChatByTitle"]';
export const CLEAR_CHAT = '[data-test-subj="clear-chat"]';
export const CLEAR_SYSTEM_PROMPT = '[data-test-subj="clearSystemPrompt"]';
export const CONFIRM_CLEAR_CHAT = '[data-test-subj="confirmModalConfirmButton"]';
export const CONNECTOR_MISSING_CALLOUT = '[data-test-subj="connectorMissingCallout"]';
export const CONNECTOR_SELECT = (c: string) => `[data-test-subj="connector-${c}"]`;
export const CONNECTOR_SELECTOR = '[data-test-subj="connector-selector"]';
export const CONVERSATION_MESSAGE = '[data-test-subj="messageText"]';
export const CONVERSATION_MESSAGE_ERROR =
'[data-test-subj="errorComment"] [data-test-subj="messageText"]';
export const CONVERSATION_MULTI_SELECTOR =
'[data-test-subj="conversationMultiSelector"] [data-test-subj="comboBoxSearchInput"]';
export const CONVERSATION_SELECT = (c: string) => `[data-test-subj="conversation-select-${c}"]`;
export const CONVERSATION_TITLE = '[data-test-subj="conversationTitle"]';
export const CONVERSATION_TITLE_SAVE_BUTTON = '[data-test-subj="euiInlineEditModeSaveButton"]';
export const CREATE_SYSTEM_PROMPT = '[data-test-subj="addSystemPrompt"]';
export const EMPTY_CONVO = '[data-test-subj="emptyConvo"]';
export const FLYOUT_NAV_TOGGLE = '[data-test-subj="aiAssistantFlyoutNavigationToggle"]';
export const MODAL_SAVE_BUTTON = '[data-test-subj="save-button"]';
export const NEW_CHAT = '[data-test-subj="newChatFromOverlay"]';
export const PROMPT_CONTEXT_SELECTOR =
'[data-test-subj="promptContextSelector"] [data-test-subj="comboBoxSearchInput"]';
export const PROMPT_CONTEXT_BUTTON = (i: string | number) =>
`[data-test-subj="selectedPromptContext-${i}-button"]`;
export const QUICK_PROMPT_TITLE_INPUT =
'[data-test-subj="quickPromptSelector"] [data-test-subj="comboBoxSearchInput"]';
export const QUICK_PROMPT_BADGE = (b: string) => `[data-test-subj="quickPrompt-${b}"]`;
export const QUICK_PROMPT_BODY_INPUT = '[data-test-subj="quick-prompt-prompt"]';
export const SEND_TO_TIMELINE_BUTTON = '[data-test-subj="sendToTimelineEmptyButton"]';
export const SHOW_ANONYMIZED_BUTTON = '[data-test-subj="showAnonymizedValues"]';
export const SUBMIT_CHAT = '[data-test-subj="submit-chat"]';
export const SYSTEM_PROMPT = '[data-test-subj="systemPromptText"]';
export const SYSTEM_PROMPT_BODY_INPUT = '[data-test-subj="systemPromptModalPromptText"]';
export const SYSTEM_PROMPT_TITLE_INPUT =
'[data-test-subj="systemPromptSelector"] [data-test-subj="comboBoxSearchInput"]';
export const SYSTEM_PROMPT_SELECT = (c: string) => `[data-test-subj="systemPrompt-${c}"]`;
export const UPGRADE_CTA = '[data-test-subj="upgradeLicenseCallToAction"]';
export const USER_PROMPT = '[data-test-subj="prompt-textarea"]';
export const WELCOME_SETUP = '[data-test-subj="welcome-setup"]';

38
x-pack/test/security_solution_cypress/cypress/tasks/api_calls/assistant.ts Normal file
View file

@ -0,0 +1,38 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import { ConversationCreateProps, ConversationResponse } from '@kbn/elastic-assistant-common';
import { deleteAllDocuments } from './elasticsearch';
import { getMockConversation } from '../../objects/assistant';
import { getSpaceUrl } from '../space';
import { rootRequest, waitForRootRequest } from './common';
const createConversation = (
body?: Partial<ConversationCreateProps>
): Cypress.Chainable<Cypress.Response<ConversationResponse>> =>
cy.currentSpace().then((spaceId) =>
rootRequest<ConversationResponse>({
method: 'POST',
url: spaceId
? getSpaceUrl(spaceId, `api/security_ai_assistant/current_user/conversations`)
: `api/security_ai_assistant/current_user/conversations`,
body: getMockConversation(body),
})
);
export const waitForConversation = (body?: Partial<ConversationCreateProps>) =>
waitForRootRequest<ConversationResponse>(createConversation(body));
export const deleteConversations = () => {
cy.log('Delete all conversations');
deleteAllDocuments(`.kibana-elastic-ai-assistant-conversations-*`);
};
export const deletePrompts = () => {
cy.log('Delete all prompts');
deleteAllDocuments(`.kibana-elastic-ai-assistant-prompts-*`);
};

13
x-pack/test/security_solution_cypress/cypress/tasks/api_calls/common.ts
View file

@ -40,6 +40,19 @@ export const rootRequest = <T = unknown>({
...restOptions,
});
// a helper function to wait for the root request to be successful
// defaults to 5 second intervals for 3 attempts
// can be helpful when waiting for a resource to be created before proceeding
export const waitForRootRequest = <T = unknown>(
fn: Cypress.Chainable<Cypress.Response<T>>,
interval = 5000,
timeout = 15000
) =>
cy.waitUntil(() => fn.then((response) => cy.wrap(response.status === 200)), {
interval,
timeout,
});
export const deleteAlertsAndRules = () => {
cy.log('Delete all alerts and rules');

27
x-pack/test/security_solution_cypress/cypress/tasks/api_calls/connectors.ts
View file

@ -21,4 +21,31 @@ const slackConnectorAPIPayload = {
name: 'Slack cypress test e2e connector',
};
export const azureConnectorAPIPayload = {
actionTypeId: '.gen-ai',
secrets: {
apiKey: '123',
},
config: {
apiUrl:
'https://goodurl.com/openai/deployments/good-gpt4o/chat/completions?api-version=2024-02-15-preview',
apiProvider: 'Azure OpenAI',
},
name: 'Azure OpenAI cypress test e2e connector',
};
export const bedrockConnectorAPIPayload = {
actionTypeId: '.bedrock',
secrets: {
accessKey: '123',
secret: '123',
},
config: {
apiUrl: 'https://bedrock.com',
},
name: 'Bedrock cypress test e2e connector',
};
export const createSlackConnector = () => createConnector(slackConnectorAPIPayload);
export const createAzureConnector = () => createConnector(azureConnectorAPIPayload);
export const createBedrockConnector = () => createConnector(bedrockConnectorAPIPayload);

218
x-pack/test/security_solution_cypress/cypress/tasks/assistant.ts Normal file
View file

@ -0,0 +1,218 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import { DEFAULT_SYSTEM_PROMPT_NON_I18N } from '@kbn/security-solution-plugin/public/assistant/content/prompts/system/translations';
import { TIMELINE_CHECKBOX } from '../screens/timelines';
import { CLOSE_FLYOUT } from '../screens/alerts';
import {
AI_ASSISTANT_BUTTON,
ASSISTANT_CHAT_BODY,
CHAT_ICON,
CHAT_ICON_SM,
CONNECTOR_SELECT,
CONNECTOR_SELECTOR,
CONVERSATION_TITLE,
EMPTY_CONVO,
WELCOME_SETUP,
NEW_CHAT,
CONVERSATION_SELECT,
FLYOUT_NAV_TOGGLE,
CONVERSATION_MESSAGE,
USER_PROMPT,
SUBMIT_CHAT,
CONVERSATION_MESSAGE_ERROR,
CLEAR_SYSTEM_PROMPT,
CHAT_CONTEXT_MENU,
CLEAR_CHAT,
CONFIRM_CLEAR_CHAT,
SYSTEM_PROMPT_SELECT,
SYSTEM_PROMPT,
CREATE_SYSTEM_PROMPT,
SYSTEM_PROMPT_TITLE_INPUT,
SYSTEM_PROMPT_BODY_INPUT,
CONVERSATION_MULTI_SELECTOR,
MODAL_SAVE_BUTTON,
ADD_QUICK_PROMPT,
QUICK_PROMPT_TITLE_INPUT,
QUICK_PROMPT_BODY_INPUT,
PROMPT_CONTEXT_SELECTOR,
QUICK_PROMPT_BADGE,
ADD_NEW_CONNECTOR,
SHOW_ANONYMIZED_BUTTON,
ASSISTANT_SETTINGS_BUTTON,
SEND_TO_TIMELINE_BUTTON,
} from '../screens/ai_assistant';
import { TOASTER } from '../screens/alerts_detection_rules';
export const openAssistant = (context?: 'rule' | 'alert') => {
if (!context) {
cy.get(AI_ASSISTANT_BUTTON).click();
return;
}
if (context === 'rule') {
cy.get(CHAT_ICON).should('be.visible');
cy.get(CHAT_ICON).click();
return;
}
if (context === 'alert') {
cy.get(CHAT_ICON_SM).should('be.visible');
cy.get(CHAT_ICON_SM).click();
return;
}
};
export const closeAssistant = () => {
cy.get(`${ASSISTANT_CHAT_BODY} ${CLOSE_FLYOUT}`).click();
};
export const createNewChat = () => {
cy.get(`${NEW_CHAT}`).click();
};
export const selectConnector = (connectorName: string) => {
cy.get(CONNECTOR_SELECTOR).click();
cy.get(CONNECTOR_SELECT(connectorName)).click();
assertConnectorSelected(connectorName);
};
export const resetConversation = () => {
cy.get(CHAT_CONTEXT_MENU).click();
cy.get(CLEAR_CHAT).click();
cy.get(CONFIRM_CLEAR_CHAT).click();
cy.get(EMPTY_CONVO).should('be.visible');
};
export const selectConversation = (conversationName: string) => {
cy.get(FLYOUT_NAV_TOGGLE).click();
cy.get(CONVERSATION_SELECT(conversationName)).click();
cy.get(CONVERSATION_TITLE + ' h2').should('have.text', conversationName);
cy.get(FLYOUT_NAV_TOGGLE).click();
};
export const updateConversationTitle = (newTitle: string) => {
cy.get(CONVERSATION_TITLE + ' h2').click();
cy.get(CONVERSATION_TITLE + ' input').clear();
cy.get(CONVERSATION_TITLE + ' input').type(newTitle);
cy.get(CONVERSATION_TITLE + ' input').type('{enter}');
cy.get(CONVERSATION_TITLE + ' h2').should('have.text', newTitle);
};
export const typeAndSendMessage = (message: string) => {
cy.get(USER_PROMPT).type(message);
cy.get(SUBMIT_CHAT).click();
};
export const sendQueryToTimeline = () => {
cy.get(SEND_TO_TIMELINE_BUTTON).click();
};
export const clearSystemPrompt = () => {
cy.get(CLEAR_SYSTEM_PROMPT).click();
};
export const sendQuickPrompt = (prompt: string) => {
cy.get(QUICK_PROMPT_BADGE(prompt)).click();
cy.get(SUBMIT_CHAT).click();
};
export const selectSystemPrompt = (systemPrompt: string) => {
cy.get(SYSTEM_PROMPT).click();
cy.get(SYSTEM_PROMPT_SELECT(systemPrompt)).click();
assertSystemPrompt(systemPrompt);
};
export const createSystemPrompt = (
title: string,
prompt: string,
defaultConversations?: string[]
) => {
cy.get(SYSTEM_PROMPT).click();
cy.get(CREATE_SYSTEM_PROMPT).click();
cy.get(SYSTEM_PROMPT_TITLE_INPUT).type(`${title}{enter}`);
cy.get(SYSTEM_PROMPT_BODY_INPUT).type(prompt);
if (defaultConversations && defaultConversations.length) {
defaultConversations.forEach((conversation) => {
cy.get(CONVERSATION_MULTI_SELECTOR).type(`${conversation}{enter}`);
});
}
cy.get(MODAL_SAVE_BUTTON).click();
};
export const createQuickPrompt = (
title: string,
prompt: string,
defaultConversations?: string[]
) => {
cy.get(ADD_QUICK_PROMPT).click();
cy.get(QUICK_PROMPT_TITLE_INPUT).type(`${title}{enter}`);
cy.get(QUICK_PROMPT_BODY_INPUT).type(prompt);
if (defaultConversations && defaultConversations.length) {
defaultConversations.forEach((conversation) => {
cy.get(PROMPT_CONTEXT_SELECTOR).type(`${conversation}{enter}`);
});
}
cy.get(MODAL_SAVE_BUTTON).click();
};
export const selectRule = (ruleId: string) => {
// not be.visible because of eui css
cy.get(TIMELINE_CHECKBOX(ruleId)).should('exist');
cy.get(TIMELINE_CHECKBOX(ruleId)).click();
};
/**
* Assertions
*/
export const assertNewConversation = (isWelcome: boolean, title: string) => {
if (isWelcome) {
cy.get(WELCOME_SETUP).should('be.visible');
} else {
cy.get(EMPTY_CONVO).should('be.visible');
}
cy.get(CONVERSATION_TITLE + ' h2').should('have.text', title);
};
export const assertMessageSent = (message: string, hasDefaultPrompt = false, prompt?: string) => {
cy.get(CONVERSATION_MESSAGE)
.first()
.should(
'contain',
hasDefaultPrompt ? `${prompt ?? DEFAULT_SYSTEM_PROMPT_NON_I18N}\n${message}` : message
);
};
export const assertErrorResponse = () => {
cy.get(CONVERSATION_MESSAGE_ERROR).should('be.visible');
};
export const assertSystemPrompt = (systemPrompt: string) => {
cy.get(SYSTEM_PROMPT).should('have.text', systemPrompt);
};
export const assertConnectorSelected = (connectorName: string) => {
cy.get(CONNECTOR_SELECTOR).should('have.text', connectorName);
};
export const assertErrorToastShown = (message?: string) => {
cy.get(TOASTER).should('be.visible');
if (message?.length) {
cy.get(TOASTER).should('contain', message);
}
};
const assertConversationTitleReadOnly = () => {
cy.get(CONVERSATION_TITLE + ' h2').click();
cy.get(CONVERSATION_TITLE + ' input').should('not.exist');
};
export const assertConversationReadOnly = () => {
assertConversationTitleReadOnly();
cy.get(ADD_NEW_CONNECTOR).should('be.disabled');
cy.get(SHOW_ANONYMIZED_BUTTON).should('be.disabled');
cy.get(CHAT_CONTEXT_MENU).should('be.disabled');
cy.get(FLYOUT_NAV_TOGGLE).should('be.disabled');
cy.get(NEW_CHAT).should('be.disabled');
cy.get(ASSISTANT_SETTINGS_BUTTON).should('be.disabled');
};

1
x-pack/test/security_solution_cypress/cypress/tsconfig.json
View file

@ -43,5 +43,6 @@
"@kbn/alerts-ui-shared",
"@kbn/securitysolution-endpoint-exceptions-common",
"@kbn/repo-info",
"@kbn/elastic-assistant-common",
]
}