[Osquery] Replace js-sql-parser (#128714)

package.json

@@ -97,6 +97,7 @@
     "puppeteer/node-fetch": "^2.6.7"
   },
   "dependencies": {
+    "@appland/sql-parser": "^1.5.1",
     "@babel/runtime": "^7.17.9",
     "@dnd-kit/core": "^3.1.1",
     "@dnd-kit/sortable": "^4.0.0",
@@ -297,7 +298,6 @@
     "js-levenshtein": "^1.1.6",
     "js-search": "^1.4.3",
     "js-sha256": "^0.9.0",
-    "js-sql-parser": "^1.4.1",
     "js-yaml": "^3.14.1",
     "json-stable-stringify": "^1.0.1",
     "json-stringify-pretty-compact": "1.2.0",

typings/@appland/sql_parser/index.d.ts

@@ -6,4 +6,4 @@
  * Side Public License, v 1.
  */
 
-declare module 'js-sql-parser';
+declare module '@appland/sql-parser';

x-pack/plugins/osquery/public/packs/queries/ecs_mapping_editor_field.tsx

@@ -34,7 +34,7 @@ import {
   EuiIcon,
   EuiSuperSelect,
 } from '@elastic/eui';
-import sqlParser from 'js-sql-parser';
+import sqliteParser from '@appland/sql-parser';
 import { FormattedMessage } from '@kbn/i18n-react';
 import { i18n } from '@kbn/i18n';
 import styled from 'styled-components';
@@ -777,7 +777,7 @@ export const ECSMappingEditorField = React.memo(
       let ast: Record<string, any> | undefined;
 
       try {
-        ast = sqlParser.parse(query)?.value;
+        ast = sqliteParser(query)?.statement?.[0];
       } catch (e) {
         return;
       }
@@ -789,44 +789,88 @@ export const ECSMappingEditorField = React.memo(
           order: number;
         }
       > =
-        ast?.from?.value?.reduce(
+        reduce(
-          (
+          ast,
-            acc: {
+          (acc, data) => {
-              [x: string]: {
+            // select * from uptime
-                columns: OsqueryColumn[];
+            if (data?.type === 'identifier' && data?.variant === 'table') {
-                order: number;
+              const osqueryTable = find(osquerySchema, ['name', data.name]);
-              };
+
-            },
+              if (osqueryTable) {
-            table: {
+                acc[data.alias || data.name] = {
-              value: {
+                  columns: osqueryTable.columns,
-                left?: { value: { value: string }; alias?: { value: string } };
+                  order: Object.keys(acc).length,
-                right?: { value: { value: string }; alias?: { value: string } };
+                };
-                value?: { value: string };
+              }
-                alias?: { value: string };
-              };
             }
-          ) => {
+
-            each(['value.left', 'value.right', 'value'], (valueKey) => {
+            // select * from uptime, routes
-              if (valueKey) {
+            if (data?.type === 'map' && data?.variant === 'join') {
-                const osqueryTable = find(osquerySchema, [
+              if (data?.source?.type === 'identifier' && data?.source?.variant === 'table') {
-                  'name',
+                const osqueryTable = find(osquerySchema, ['name', data?.source?.name]);
-                  get(table, `${valueKey}.value.value`),
-                ]);
 
                 if (osqueryTable) {
-                  acc[
+                  acc[data?.source?.alias || data?.source?.name] = {
-                    get(table, `${valueKey}.alias.value`) ?? get(table, `${valueKey}.value.value`)
-                  ] = {
                     columns: osqueryTable.columns,
                     order: Object.keys(acc).length,
                   };
                 }
               }
-            });
+
+              if (data?.source?.type === 'statement' && data?.source?.variant === 'compound') {
+                if (
+                  data?.source?.statement.from.type === 'identifier' &&
+                  data?.source?.statement.from.variant === 'table'
+                ) {
+                  const osqueryTable = find(osquerySchema, [
+                    'name',
+                    data?.source?.statement.from.name,
+                  ]);
+
+                  if (osqueryTable) {
+                    acc[data?.source?.statement.from.alias || data?.source?.statement.from.name] = {
+                      columns: osqueryTable.columns,
+                      order: Object.keys(acc).length,
+                    };
+                  }
+                }
+              }
+
+              each(
+                data?.map,
+                (mapValue: {
+                  type: string;
+                  // eslint-disable-next-line @typescript-eslint/no-explicit-any
+                  source: { type: string; variant: string; name: any | string; alias: any };
+                }) => {
+                  if (mapValue?.type === 'join') {
+                    if (
+                      mapValue?.source?.type === 'identifier' &&
+                      mapValue?.source?.variant === 'table'
+                    ) {
+                      const osqueryTable = find(osquerySchema, ['name', mapValue?.source?.name]);
+
+                      if (osqueryTable) {
+                        acc[mapValue?.source?.alias || mapValue?.source?.name] = {
+                          columns: osqueryTable.columns,
+                          order: Object.keys(acc).length,
+                        };
+                      }
+                    }
+                  }
+                }
+              );
+            }
 
             return acc;
           },
-          {}
+          {} as Record<
+            string,
+            {
+              columns: OsqueryColumn[];
+              order: number;
+            }
+          >
         ) ?? {};
 
       // Table doesn't exist in osquery schema
@@ -834,114 +878,114 @@ export const ECSMappingEditorField = React.memo(
         return;
       }
 
-      const suggestions =
+      const suggestions = isArray(ast?.result)
-        isArray(ast?.selectItems?.value) &&
+        ? ast?.result
-        ast?.selectItems?.value
+            ?.map((selectItem: { type: string; name: string; alias?: string }) => {
-          ?.map((selectItem: { type: string; value: string; hasAs: boolean; alias?: string }) => {
+              if (selectItem.type === 'identifier') {
-            if (selectItem.type === 'Identifier') {
+                /*
-              /*
+                  select * from routes, uptime;
-                select * from routes, uptime;
+                */
-              */
+                if (ast?.result.length === 1 && selectItem.name === '*') {
-              if (ast?.selectItems?.value.length === 1 && selectItem.value === '*') {
+                  return reduce(
-                return reduce(
+                    astOsqueryTables,
-                  astOsqueryTables,
+                    (acc, { columns: osqueryColumns, order: tableOrder }, table) => {
-                  (acc, { columns: osqueryColumns, order: tableOrder }, table) => {
+                      acc.push(
-                    acc.push(
+                        ...osqueryColumns.map((osqueryColumn) => ({
-                      ...osqueryColumns.map((osqueryColumn) => ({
+                          label: osqueryColumn.name,
-                        label: osqueryColumn.name,
+                          value: {
+                            name: osqueryColumn.name,
+                            description: osqueryColumn.description,
+                            table,
+                            tableOrder,
+                            suggestion_label: osqueryColumn.name,
+                          },
+                        }))
+                      );
+
+                      return acc;
+                    },
+                    [] as OsquerySchemaOption[]
+                  );
+                }
+
+                /*
+                  select i.*, p.resident_size, p.user_time, p.system_time, time.minutes as counter from osquery_info i, processes p, time where p.pid = i.pid;
+                */
+
+                const [table, column] = selectItem.name.includes('.')
+                  ? selectItem.name?.split('.')
+                  : [Object.keys(astOsqueryTables)[0], selectItem.name];
+
+                if (column === '*' && astOsqueryTables[table]) {
+                  const { columns: osqueryColumns, order: tableOrder } = astOsqueryTables[table];
+
+                  return osqueryColumns.map((osqueryColumn) => ({
+                    label: osqueryColumn.name,
+                    value: {
+                      name: osqueryColumn.name,
+                      description: osqueryColumn.description,
+                      table,
+                      tableOrder,
+                      suggestion_label: `${osqueryColumn.name}`,
+                    },
+                  }));
+                }
+
+                if (astOsqueryTables[table]) {
+                  const osqueryColumn = find(astOsqueryTables[table].columns, ['name', column]);
+
+                  if (osqueryColumn) {
+                    const label = selectItem.alias ?? column;
+
+                    return [
+                      {
+                        label,
                         value: {
                           name: osqueryColumn.name,
                           description: osqueryColumn.description,
                           table,
-                          tableOrder,
+                          tableOrder: astOsqueryTables[table].order,
-                          suggestion_label: osqueryColumn.name,
+                          suggestion_label: `${label}`,
                         },
-                      }))
+                      },
-                    );
+                    ];
-
+                  }
-                    return acc;
+                }
-                  },
-                  [] as OsquerySchemaOption[]
-                );
               }
 
               /*
-                select i.*, p.resident_size, p.user_time, p.system_time, time.minutes as counter from osquery_info i, processes p, time where p.pid = i.pid;
+                SELECT pid, uid, name, ROUND((
+                  (user_time + system_time) / (cpu_time.tsb - cpu_time.itsb)
+                ) * 100, 2) AS percentage
+                FROM processes, (
+                SELECT (
+                  SUM(user) + SUM(nice) + SUM(system) + SUM(idle) * 1.0) AS tsb,
+                  SUM(COALESCE(idle, 0)) + SUM(COALESCE(iowait, 0)) AS itsb
+                  FROM cpu_time
+                ) AS cpu_time
+                ORDER BY user_time+system_time DESC
+                LIMIT 5;
               */
 
-              const [table, column] = selectItem.value.includes('.')
+              if (selectItem.type === 'function' && selectItem.alias) {
-                ? selectItem.value?.split('.')
+                return [
-                : [Object.keys(astOsqueryTables)[0], selectItem.value];
+                  {
-
+                    label: selectItem.alias,
-              if (column === '*' && astOsqueryTables[table]) {
+                    value: {
-                const { columns: osqueryColumns, order: tableOrder } = astOsqueryTables[table];
+                      name: selectItem.alias,
-
+                      description: '',
-                return osqueryColumns.map((osqueryColumn) => ({
+                      table: '',
-                  label: osqueryColumn.name,
+                      tableOrder: -1,
-                  value: {
+                      suggestion_label: selectItem.alias,
-                    name: osqueryColumn.name,
-                    description: osqueryColumn.description,
-                    table,
-                    tableOrder,
-                    suggestion_label: `${osqueryColumn.name}`,
-                  },
-                }));
-              }
-
-              if (astOsqueryTables[table]) {
-                const osqueryColumn = find(astOsqueryTables[table].columns, ['name', column]);
-
-                if (osqueryColumn) {
-                  const label = selectItem.hasAs ? selectItem.alias : column;
-
-                  return [
-                    {
-                      label,
-                      value: {
-                        name: osqueryColumn.name,
-                        description: osqueryColumn.description,
-                        table,
-                        tableOrder: astOsqueryTables[table].order,
-                        suggestion_label: `${label}`,
-                      },
                     },
-                  ];
-                }
-              }
-            }
-
-            /*
-              SELECT pid, uid, name, ROUND((
-                (user_time + system_time) / (cpu_time.tsb - cpu_time.itsb)
-              ) * 100, 2) AS percentage
-              FROM processes, (
-              SELECT (
-                SUM(user) + SUM(nice) + SUM(system) + SUM(idle) * 1.0) AS tsb,
-                SUM(COALESCE(idle, 0)) + SUM(COALESCE(iowait, 0)) AS itsb
-                FROM cpu_time
-              ) AS cpu_time
-              ORDER BY user_time+system_time DESC
-              LIMIT 5;
-            */
-
-            if (selectItem.hasAs && selectItem.alias) {
-              return [
-                {
-                  label: selectItem.alias,
-                  value: {
-                    name: selectItem.alias,
-                    description: '',
-                    table: '',
-                    tableOrder: -1,
-                    suggestion_label: selectItem.alias,
                   },
-                },
+                ];
-              ];
+              }
-            }
 
-            return [];
+              return [];
-          })
+            })
-          .flat();
+            .flat()
+        : [];
 
       // Remove column duplicates by keeping the column from the table that appears last in the query
       const newOptions = sortedUniqBy(

yarn.lock

@@ -41,6 +41,11 @@
     call-me-maybe "^1.0.1"
     z-schema "^5.0.1"
 
+"@appland/sql-parser@^1.5.1":
+  version "1.5.1"
+  resolved "https://registry.yarnpkg.com/@appland/sql-parser/-/sql-parser-1.5.1.tgz#331d644364899858ba7aa6e884e2492596990626"
+  integrity sha512-R2FBHUOdzdBPUCCiL6WvXT9Fu+Xaj89exa1g+wMlatIe5z6vqitzLkY5a9zGDL3IByTiwbR0jiYuvFMfhp1Q+Q==
+
 "@babel/cli@^7.17.6":
   version "7.17.6"
   resolved "https://registry.yarnpkg.com/@babel/cli/-/cli-7.17.6.tgz#169e5935f1795f0b62ded5a2accafeedfe5c5363"
@@ -18331,11 +18336,6 @@ js-sha3@0.8.0:
   resolved "https://registry.yarnpkg.com/js-sha3/-/js-sha3-0.8.0.tgz#b9b7a5da73afad7dedd0f8c463954cbde6818840"
   integrity sha512-gF1cRrHhIzNfToc802P800N8PpXS+evLLXfsVpowqmAFR9uwbi89WvXg2QspOmXL8QL86J4T1EpFu+yUkwJY3Q==
 
-js-sql-parser@^1.4.1:
-  version "1.4.1"
-  resolved "https://registry.yarnpkg.com/js-sql-parser/-/js-sql-parser-1.4.1.tgz#775516b3187dd5872ecec04bef8ed4a430242fda"
-  integrity sha512-J8zi3+/yK4FWSnVvLOjS2HIGfJhR6v7ApwIF8gZ/SpaO/tFIDlsgugD6ZMn6flXiuMsCjJxvhE0+xBgbdzvDDw==
-
 js-string-escape@^1.0.1:
   version "1.0.1"
   resolved "https://registry.yarnpkg.com/js-string-escape/-/js-string-escape-1.0.1.tgz#e2625badbc0d67c7533e9edc1068c587ae4137ef"