Resolve url dynamically when requesting EMS data (#25685)

Url's from EMS ((files and tiles) should be resolved ad-hoc. The one that is stored in the saved object should not be used. This change removes the URL from the saved object. It also provides fallback functionality so old saved objects continue to be loaded correctly.

src/core_plugins/region_map/public/choropleth_layer.js

@@ -24,6 +24,7 @@ import d3 from 'd3';
 import { i18n }  from '@kbn/i18n';
 import { KibanaMapLayer } from 'ui/vis/map/kibana_map_layer';
 import { truncatedColorMaps } from 'ui/vislib/components/color/truncated_colormaps';
+import { uiModules } from 'ui/modules';
 import * as topojson from 'topojson-client';
 import { toastNotifications } from 'ui/notify';
 import * as colorUtil from 'ui/vis/map/color_util';
@@ -36,6 +37,14 @@ const EMPTY_STYLE = {
 };
 
 
+const emsServiceSettings = new Promise((resolve) => {
+  uiModules.get('kibana').run(($injector) => {
+    const serviceSttings = $injector.get('serviceSettings');
+    resolve(serviceSttings);
+  });
+});
+
+
 export default class ChoroplethLayer extends KibanaMapLayer {
 
   static _doInnerJoin(sortedMetrics, sortedGeojsonFeatures, joinField) {
@@ -69,7 +78,7 @@ export default class ChoroplethLayer extends KibanaMapLayer {
   }
 
 
-  constructor(geojsonUrl, attribution, format, showAllShapes, meta) {
+  constructor(name, attribution, format, showAllShapes, meta, layerConfig) {
     super();
 
     this._metrics = null;
@@ -79,9 +88,9 @@ export default class ChoroplethLayer extends KibanaMapLayer {
     this._tooltipFormatter = () => '';
     this._attribution = attribution;
     this._boundsOfData = null;
-
     this._showAllShapes = showAllShapes;
-    this._geojsonUrl = geojsonUrl;
+    this._layerName = name;
+    this._layerConfig = layerConfig;
 
     this._leafletLayer = L.geoJson(null, {
       onEachFeature: (feature, layer) => {
@@ -114,7 +123,7 @@ export default class ChoroplethLayer extends KibanaMapLayer {
     this._isJoinValid = false;
     this._whenDataLoaded = new Promise(async (resolve) => {
       try {
-        const data = await this._makeJsonAjaxCall(geojsonUrl);
+        const data = await this._makeJsonAjaxCall();
         let featureCollection;
         const formatType = typeof format === 'string' ? format : format.type;
         if (formatType === 'geojson') {
@@ -148,15 +157,15 @@ export default class ChoroplethLayer extends KibanaMapLayer {
         let errorMessage;
         if (e.status === 404) {
           errorMessage = i18n.translate('regionMap.choroplethLayer.downloadingVectorData404ErrorMessage', {
-            defaultMessage: 'Server responding with \'404\' when attempting to fetch {geojsonUrl}. \
+            defaultMessage: 'Server responding with \'404\' when attempting to fetch {name}. \
 Make sure the file exists at that location.',
-            values: { geojsonUrl },
+            values: { name: name },
           });
         } else {
           errorMessage = i18n.translate('regionMap.choroplethLayer.downloadingVectorDataErrorMessage', {
-            defaultMessage: 'Cannot download {geojsonUrl} file. Please ensure the \
+            defaultMessage: 'Cannot download {name} file. Please ensure the \
 CORS configuration of the server permits requests from the Kibana application on this host.',
-            values: { geojsonUrl },
+            values: { name: name },
           });
         }
 
@@ -174,11 +183,9 @@ CORS configuration of the server permits requests from the Kibana application on
   }
 
   //This method is stubbed in the tests to avoid network request during unit tests.
-  async _makeJsonAjaxCall(url) {
-    return await $.ajax({
-      dataType: 'json',
-      url: url
-    });
+  async _makeJsonAjaxCall() {
+    const serviceSettings = await emsServiceSettings;
+    return serviceSettings.getGeoJsonForRegionLayer(this._layerConfig);
   }
 
   _invalidateJoin() {
@@ -222,7 +229,7 @@ CORS configuration of the server permits requests from the Kibana application on
   }
 
   getUrl() {
-    return this._geojsonUrl;
+    return this._layerName;
   }
 
   setTooltipFormatter(tooltipFormatter, metricsAgg, fieldName) {
@@ -246,8 +253,8 @@ CORS configuration of the server permits requests from the Kibana application on
     this._setStyle();
   }
 
-  cloneChoroplethLayerForNewData(url, attribution, format, showAllData, meta) {
-    const clonedLayer = new ChoroplethLayer(url, attribution, format, showAllData, meta);
+  cloneChoroplethLayerForNewData(name, attribution, format, showAllData, meta, layerConfig) {
+    const clonedLayer = new ChoroplethLayer(name, attribution, format, showAllData, meta, layerConfig);
     clonedLayer.setJoinField(this._joinField);
     clonedLayer.setColorRamp(this._colorRamp);
     clonedLayer.setLineWeight(this._lineWeight);
@@ -301,12 +308,12 @@ CORS configuration of the server permits requests from the Kibana application on
     this._setStyle();
   }
 
-  canReuseInstance(geojsonUrl, showAllShapes) {
-    return this._geojsonUrl === geojsonUrl && this._showAllShapes === showAllShapes;
+  canReuseInstance(name, showAllShapes) {
+    return this._layerName === name && this._showAllShapes === showAllShapes;
   }
 
-  canReuseInstanceForNewMetrics(geojsonUrl, showAllShapes, newMetrics) {
-    if (this._geojsonUrl !== geojsonUrl) {
+  canReuseInstanceForNewMetrics(name, showAllShapes, newMetrics) {
+    if (this._layerName !== name) {
       return false;
     }
 

src/core_plugins/region_map/public/region_map_vis.js

@@ -25,12 +25,15 @@ import { truncatedColorMaps } from 'ui/vislib/components/color/truncated_colorma
 import { mapToLayerWithId } from './util';
 import { RegionMapsVisualizationProvider } from './region_map_visualization';
 import { Status } from 'ui/vis/update_status';
+import { ORIGIN } from 'ui/vis/map/origin';
 
 VisTypesRegistryProvider.register(function RegionMapProvider(Private, regionmapsConfig, config, i18n) {
+
   const VisFactory = Private(VisFactoryProvider);
   const RegionMapsVisualization = Private(RegionMapsVisualizationProvider);
 
-  const vectorLayers = regionmapsConfig.layers.map(mapToLayerWithId.bind(null, 'self_hosted', false));
+  const vectorLayers = regionmapsConfig.layers.map(mapToLayerWithId.bind(null, ORIGIN.KIBANA_YML));
+
   const selectedLayer = vectorLayers[0];
   const selectedJoinField = selectedLayer ? vectorLayers[0].fields[0] : null;
 

src/core_plugins/region_map/public/region_map_vis_params.js

@@ -22,6 +22,7 @@ import { toastNotifications } from 'ui/notify';
 import regionMapVisParamsTemplate from './region_map_vis_params.html';
 import { mapToLayerWithId } from './util';
 import '../../tile_map/public/editors/wms_options';
+import { ORIGIN } from 'ui/vis/map/origin';
 
 uiModules.get('kibana/region_map')
   .directive('regionMapVisParams', function (serviceSettings, regionmapsConfig) {
@@ -38,7 +39,7 @@ uiModules.get('kibana/region_map')
           serviceSettings.getFileLayers()
             .then(function (layersFromService) {
 
-              layersFromService = layersFromService.map(mapToLayerWithId.bind(null, 'elastic_maps_service', true));
+              layersFromService = layersFromService.map(mapToLayerWithId.bind(null, ORIGIN.EMS));
               const newVectorLayers = $scope.collections.vectorLayers.slice();
               for (let i = 0; i < layersFromService.length; i += 1) {
                 const layerFromService = layersFromService[i];

src/core_plugins/region_map/public/region_map_visualization.js

@@ -70,7 +70,7 @@ export function RegionMapsVisualizationProvider(Private, config, i18n) {
       }
 
       this._updateChoroplethLayerForNewMetrics(
-        this._vis.params.selectedLayer.url,
+        this._vis.params.selectedLayer.name,
         this._vis.params.selectedLayer.attribution,
         this._vis.params.showAllShapes,
         results
@@ -96,7 +96,7 @@ export function RegionMapsVisualizationProvider(Private, config, i18n) {
       }
 
       this._updateChoroplethLayerForNewProperties(
-        visParams.selectedLayer.url,
+        visParams.selectedLayer.name,
         visParams.selectedLayer.attribution,
         this._vis.params.showAllShapes
       );
@@ -107,40 +107,42 @@ export function RegionMapsVisualizationProvider(Private, config, i18n) {
 
     }
 
-    _updateChoroplethLayerForNewMetrics(url, attribution, showAllData, newMetrics) {
-      if (this._choroplethLayer && this._choroplethLayer.canReuseInstanceForNewMetrics(url, showAllData, newMetrics)) {
+    _updateChoroplethLayerForNewMetrics(name, attribution, showAllData, newMetrics) {
+      if (this._choroplethLayer && this._choroplethLayer.canReuseInstanceForNewMetrics(name, showAllData, newMetrics)) {
         return;
       }
-      return this._recreateChoroplethLayer(url, attribution, showAllData);
+      return this._recreateChoroplethLayer(name, attribution, showAllData);
     }
 
-    _updateChoroplethLayerForNewProperties(url, attribution, showAllData) {
-      if (this._choroplethLayer && this._choroplethLayer.canReuseInstance(url, showAllData)) {
+    _updateChoroplethLayerForNewProperties(name, attribution, showAllData) {
+      if (this._choroplethLayer && this._choroplethLayer.canReuseInstance(name, showAllData)) {
         return;
       }
-      return this._recreateChoroplethLayer(url, attribution, showAllData);
+      return this._recreateChoroplethLayer(name, attribution, showAllData);
     }
 
-    _recreateChoroplethLayer(url, attribution, showAllData) {
+    _recreateChoroplethLayer(name, attribution, showAllData) {
 
       this._kibanaMap.removeLayer(this._choroplethLayer);
 
 
       if (this._choroplethLayer) {
         this._choroplethLayer = this._choroplethLayer.cloneChoroplethLayerForNewData(
-          url,
+          name,
           attribution,
           this.vis.params.selectedLayer.format,
           showAllData,
-          this.vis.params.selectedLayer.meta
+          this.vis.params.selectedLayer.meta,
+          this.vis.params.selectedLayer
         );
       } else {
         this._choroplethLayer = new ChoroplethLayer(
-          url,
+          name,
           attribution,
           this.vis.params.selectedLayer.format,
           showAllData,
-          this.vis.params.selectedLayer.meta
+          this.vis.params.selectedLayer.meta,
+          this.vis.params.selectedLayer
         );
       }
 

src/core_plugins/region_map/public/util.js

@@ -18,10 +18,11 @@
  */
 
 import _ from 'lodash';
+import { ORIGIN } from 'ui/vis/map/origin';
 
-export function mapToLayerWithId(prefix, isEMS, layer) {
+export function mapToLayerWithId(prefix, layer) {
   const clonedLayer = _.cloneDeep(layer);
   clonedLayer.layerId = prefix + '.' + layer.name;
-  clonedLayer.isEMS = isEMS;
+  clonedLayer.isEMS = ORIGIN.EMS === prefix ?  true : false;
   return clonedLayer;
 }

src/core_plugins/tile_map/public/base_maps_visualization.js

@@ -23,10 +23,18 @@ import * as Rx from 'rxjs';
 import { filter, first } from 'rxjs/operators';
 import 'ui/vis/map/service_settings';
 import { toastNotifications } from 'ui/notify';
+import { uiModules } from 'ui/modules';
 
 const MINZOOM = 0;
 const MAXZOOM = 22;//increase this to 22. Better for WMS
 
+const emsServiceSettings = new Promise((resolve) => {
+  uiModules.get('kibana').run(($injector) => {
+    const serviceSttings = $injector.get('serviceSettings');
+    resolve(serviceSttings);
+  });
+});
+
 export function BaseMapsVisualizationProvider(serviceSettings, i18n) {
 
   /**
@@ -184,7 +192,8 @@ export function BaseMapsVisualizationProvider(serviceSettings, i18n) {
       if (this._kibanaMap.getZoomLevel() > tmsLayer.maxZoom) {
         this._kibanaMap.setZoomLevel(tmsLayer.maxZoom);
       }
-      const url = tmsLayer.url;
+      // const url = tmsLayer.url;
+      const url = await (await emsServiceSettings).getUrlTemplateForTMSLayer(tmsLayer);
       const options = _.cloneDeep(tmsLayer);
       delete options.id;
       delete options.url;
@@ -209,7 +218,7 @@ export function BaseMapsVisualizationProvider(serviceSettings, i18n) {
      */
     async _updateParams() {
       const mapParams = this._getMapsParams();
-      await this._updateBaseLayer(mapParams);
+      await this._updateBaseLayer();
       this._kibanaMap.setLegendPosition(mapParams.legendPosition);
       this._kibanaMap.setShowTooltip(mapParams.addTooltip);
       this._kibanaMap.useUiStateFromVisualization(this.vis);

src/core_plugins/tile_map/public/coordinate_maps_visualization.js

@@ -243,7 +243,6 @@ export function CoordinateMapsVisualizationProvider(Notifier, Private) {
       });
     }
 
-
     _isFilteredByCollar() {
       const DEFAULT = false;
       const agg = this._getGeoHashAgg();

src/core_plugins/vega/public/data_model/__tests__/vega_parser.js

@@ -79,7 +79,8 @@ describe('VegaParser._resolveEsQueries', () => {
   function test(spec, expected, warnCount) {
     return async () => {
       const vp = new VegaParser(spec, { search: async () => [[42]] }, 0, 0, {
-        getFileLayers: async () => [{ name: 'file1', url: 'url1' }]
+        getFileLayers: async () => [{ name: 'file1', url: 'url1' }],
+        getUrlForRegionLayer: async (layer) => { return layer.url;}
       });
       await vp._resolveDataUrls();
 

src/core_plugins/vega/public/data_model/ems_file_parser.js

@@ -72,7 +72,8 @@ export class EmsFileParser {
       }
 
       // This URL can bypass loader sanitization at the later stage
-      obj.url = bypassExternalUrlCheck(foundLayer.url);
+      const url = await this._serviceSettings.getUrlForRegionLayer(foundLayer);
+      obj.url = bypassExternalUrlCheck(url);
     }
   }
 

src/core_plugins/vega/public/vega_view/vega_map_view.js

@@ -37,6 +37,10 @@ export class VegaMapView extends VegaBaseView {
       if (!this._$container) return;
       const mapStyle = mapConfig.mapStyle === 'default' ? 'road_map' : mapConfig.mapStyle;
       baseMapOpts = tmsServices.find((s) => s.id === mapStyle);
+      baseMapOpts = {
+        url: await this._serviceSettings.getUrlTemplateForTMSLayer(baseMapOpts),
+        ...baseMapOpts
+      };
       if (!baseMapOpts) {
         this.onWarn(i18n.translate('vega.mapView.mapStyleNotFoundWarningMessage', {
           defaultMessage: '{mapStyleParam} was not found',

src/ui/public/vis/__tests__/map/service_settings.js

@@ -134,10 +134,19 @@ describe('service_settings (FKA tilemaptest)', function () {
 
   describe('TMS', function () {
 
-    it('should get url', async function () {
+    it('should NOT get url from the config', async function () {
       const tmsServices = await serviceSettings.getTMSServices();
       const tmsService = tmsServices[0];
-      const mapUrl = tmsService.url;
+      expect(typeof tmsService.url === 'undefined').to.equal(true);
+    });
+
+    it('should get url by resolving dynamically', async function () {
+
+      const tmsServices = await serviceSettings.getTMSServices();
+      const tmsService = tmsServices[0];
+      expect(typeof tmsService.url === 'undefined').to.equal(true);
+
+      const mapUrl = await serviceSettings.getUrlTemplateForTMSLayer(tmsService);
       expect(mapUrl).to.contain('{x}');
       expect(mapUrl).to.contain('{y}');
       expect(mapUrl).to.contain('{z}');
@@ -161,9 +170,8 @@ describe('service_settings (FKA tilemaptest)', function () {
 
       let tilemapServices;
 
-      function assertQuery(expected) {
-
-        const mapUrl = tilemapServices[0].url;
+      async function assertQuery(expected) {
+        const mapUrl = await serviceSettings.getUrlTemplateForTMSLayer(tilemapServices[0]);
         const urlObject = url.parse(mapUrl, true);
         Object.keys(expected).forEach(key => {
           expect(urlObject.query).to.have.property(key, expected[key]);
@@ -173,7 +181,7 @@ describe('service_settings (FKA tilemaptest)', function () {
       it('accepts an object', async () => {
         serviceSettings.addQueryParams({ foo: 'bar' });
         tilemapServices = await serviceSettings.getTMSServices();
-        assertQuery({ foo: 'bar' });
+        await assertQuery({ foo: 'bar' });
       });
 
       it('merged additions with previous values', async () => {
@@ -181,7 +189,7 @@ describe('service_settings (FKA tilemaptest)', function () {
         serviceSettings.addQueryParams({ foo: 'bar' });
         serviceSettings.addQueryParams({ bar: 'stool' });
         tilemapServices = await serviceSettings.getTMSServices();
-        assertQuery({ foo: 'bar', bar: 'stool' });
+        await assertQuery({ foo: 'bar', bar: 'stool' });
       });
 
       it('overwrites conflicting previous values', async () => {
@@ -190,14 +198,14 @@ describe('service_settings (FKA tilemaptest)', function () {
         serviceSettings.addQueryParams({ bar: 'stool' });
         serviceSettings.addQueryParams({ foo: 'tstool' });
         tilemapServices = await serviceSettings.getTMSServices();
-        assertQuery({ foo: 'tstool', bar: 'stool' });
+        await assertQuery({ foo: 'tstool', bar: 'stool' });
       });
 
       it('when overridden, should continue to work', async () => {
         mapConfig.manifestServiceUrl = manifestUrl2;
         serviceSettings.addQueryParams({ foo: 'bar' });
         tilemapServices = await serviceSettings.getTMSServices();
-        assertQuery({ foo: 'bar' });
+        await assertQuery({ foo: 'bar' });
       });
 
 
@@ -227,7 +235,17 @@ describe('service_settings (FKA tilemaptest)', function () {
             'subdomains': []
           }
         ];
-        expect(tilemapServices).to.eql(expected);
+
+
+        const assertions = tilemapServices.map(async (actualService, index) => {
+          const expectedService = expected[index];
+          expect(actualService.id).to.equal(expectedService.id);
+          expect(actualService.attribution).to.equal(expectedService.attribution);
+          const url = await serviceSettings.getUrlTemplateForTMSLayer(actualService);
+          expect(url).to.equal(expectedService.url);
+        });
+
+        return Promise.all(assertions);
 
       });
 
@@ -251,7 +269,11 @@ describe('service_settings (FKA tilemaptest)', function () {
             'id': 'TMS in config/kibana.yml'
           }
         ];
-        expect(tilemapServices).to.eql(expected);
+        expect(tilemapServices.length).to.eql(1);
+        expect(tilemapServices[0].attribution).to.eql(expected[0].attribution);
+        expect(tilemapServices[0].id).to.eql(expected[0].id);
+        const url = await serviceSettings.getUrlTemplateForTMSLayer(tilemapServices[0]);
+        expect(url).to.equal(expected[0].url);
 
       });
 
@@ -273,7 +295,7 @@ describe('service_settings (FKA tilemaptest)', function () {
     it('should load manifest', async function () {
       serviceSettings.addQueryParams({ foo: 'bar' });
       const fileLayers = await serviceSettings.getFileLayers();
-      fileLayers.forEach(function (fileLayer, index) {
+      const asserttions = fileLayers.map(async function (fileLayer, index) {
         const expected = vectorManifest.layers[index];
         expect(expected.attribution).to.eql(fileLayer.attribution);
         expect(expected.format).to.eql(fileLayer.format);
@@ -281,12 +303,15 @@ describe('service_settings (FKA tilemaptest)', function () {
         expect(expected.name).to.eql(fileLayer.name);
         expect(expected.created_at).to.eql(fileLayer.created_at);
 
-        const urlObject = url.parse(fileLayer.url, true);
+        const fileUrl = await serviceSettings.getUrlForRegionLayer(fileLayer);
+        const urlObject = url.parse(fileUrl, true);
         Object.keys({ foo: 'bar', elastic_tile_service_tos: 'agree' }).forEach(key => {
           expect(urlObject.query).to.have.property(key, expected[key]);
         });
 
       });
+
+      return Promise.all(asserttions);
     });
 
     it('should exclude all when not configured', async () => {

src/ui/public/vis/map/origin.js

@@ -0,0 +1,23 @@
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+export const ORIGIN = {
+  EMS: 'elastic_maps_service',
+  KIBANA_YML: 'self_hosted'
+};

src/ui/public/vis/map/service_settings.js

@@ -21,14 +21,17 @@ import { uiModules } from '../../modules';
 import _ from 'lodash';
 import MarkdownIt from 'markdown-it';
 import { modifyUrl } from '../../url';
+import { ORIGIN } from './origin';
 
 const markdownIt = new MarkdownIt({
   html: false,
   linkify: true
 });
 
+const TMS_IN_YML_ID = 'TMS in config/kibana.yml';
+
 uiModules.get('kibana')
-  .service('serviceSettings', function ($http, $sanitize, mapConfig, tilemapsConfig, kbnVersion) {
+  .service('serviceSettings', function ($http, $sanitize, mapConfig, regionmapsConfig, tilemapsConfig, kbnVersion) {
 
     const attributionFromConfig = $sanitize(markdownIt.render(tilemapsConfig.deprecated.config.options.attribution || ''));
     const tmsOptionsFromConfig = _.assign({}, tilemapsConfig.deprecated.config.options, { attribution: attributionFromConfig });
@@ -49,7 +52,6 @@ uiModules.get('kibana')
     };
 
 
-
     class ServiceSettings {
 
       constructor() {
@@ -63,6 +65,7 @@ uiModules.get('kibana')
 
         this._invalidateSettings();
       }
+
       _invalidateSettings() {
 
         this._loadCatalogue = _.once(async () => {
@@ -97,7 +100,6 @@ uiModules.get('kibana')
           const manifest = await this._getManifest(fileService.manifest, this._queryParams);
           const layers = manifest.data.layers.filter(layer => layer.format === 'geojson' || layer.format === 'topojson');
           layers.forEach((layer) => {
-            layer.url = this._extendUrlWithParams(layer.url);
             layer.attribution = $sanitize(markdownIt.render(layer.attribution));
           });
           return layers;
@@ -115,7 +117,7 @@ uiModules.get('kibana')
             const preppedService = _.cloneDeep(tmsService);
             preppedService.attribution = $sanitize(markdownIt.render(preppedService.attribution));
             preppedService.subdomains = preppedService.subdomains || [];
-            preppedService.url = this._extendUrlWithParams(preppedService.url);
+            preppedService.origin = ORIGIN.EMS;
             return preppedService;
           });
 
@@ -143,7 +145,13 @@ uiModules.get('kibana')
 
 
       async getFileLayers() {
-        return await this._loadFileLayers();
+        const fileLayers = await this._loadFileLayers();
+        return fileLayers.map(fileLayer => {
+          const massagedFileLayer = { ...fileLayer };
+          delete massagedFileLayer.url;
+          massagedFileLayer.origin = ORIGIN.EMS;
+          return massagedFileLayer;
+        });
       }
 
 
@@ -156,14 +164,22 @@ uiModules.get('kibana')
         const allServices = [];
         if (tilemapsConfig.deprecated.isOverridden) {//use tilemap.* settings from yml
           const tmsService = _.cloneDeep(tmsOptionsFromConfig);
-          tmsService.url = tilemapsConfig.deprecated.config.url;
-          tmsService.id = 'TMS in config/kibana.yml';
+          tmsService.id = TMS_IN_YML_ID;
+          tmsService.origin = ORIGIN.KIBANA_YML;
           allServices.push(tmsService);
         }
 
         const servicesFromManifest = await this._loadTMSServices();
-        return allServices.concat(servicesFromManifest);
 
+        const strippedServiceFromManifest = servicesFromManifest.map((service) => {
+          const strippedService = { ...service };
+          //do not expose url. needs to be resolved dynamically
+          delete strippedService.url;
+          strippedService.origin = ORIGIN.EMS;
+          return strippedService;
+        });
+
+        return allServices.concat(strippedServiceFromManifest);
       }
 
       /**
@@ -184,10 +200,78 @@ uiModules.get('kibana')
         }
       }
 
-      getEMSHotLink(fileLayer) {
+      async getEMSHotLink(fileLayer) {
         const id = `file/${fileLayer.name}`;
         return `${mapConfig.emsLandingPageUrl}#${id}`;
       }
+
+
+      async _getUrlTemplateForEMSTMSLayer(tmsServiceConfig) {
+        const tmsServices = await this._loadTMSServices();
+        const serviceConfig = tmsServices.find(service => {
+          return service.id === tmsServiceConfig.id;
+        });
+        return this._extendUrlWithParams(serviceConfig.url);
+      }
+
+      async getUrlTemplateForTMSLayer(tmsServiceConfig) {
+
+        if (tmsServiceConfig.origin === ORIGIN.EMS) {
+          return this._getUrlTemplateForEMSTMSLayer(tmsServiceConfig);
+        } else if (tmsServiceConfig.origin === ORIGIN.KIBANA_YML) {
+          return tilemapsConfig.deprecated.config.url;
+        } else {
+          //this is an older config. need to resolve this dynamically.
+          if (tmsServiceConfig.id === TMS_IN_YML_ID) {
+            return tilemapsConfig.deprecated.config.url;
+          } else {
+            //assume ems
+            return this._getUrlTemplateForEMSTMSLayer(tmsServiceConfig);
+          }
+        }
+
+      }
+
+      async _getUrlFromEms(fileLayerConfig) {
+        const fileLayers = await this._loadFileLayers();
+        const layerConfig = fileLayers.find(fileLayer => {
+          return fileLayer.name === fileLayerConfig.name;//the id is the filename
+        });
+
+        if (layerConfig) {
+          return this._extendUrlWithParams(layerConfig.url);
+        } else {
+          throw new Error(`File  ${fileLayerConfig.name} not recognized`);
+        }
+      }
+
+      async getUrlForRegionLayer(fileLayerConfig) {
+        let url;
+        if (fileLayerConfig.origin === ORIGIN.EMS) {
+          url = this._getUrlFromEms(fileLayerConfig);
+        } else if (fileLayerConfig.layerId && fileLayerConfig.layerId.startsWith(`${ORIGIN.EMS}.`)) {
+          //fallback for older saved objects
+          url = this._getUrlFromEms(fileLayerConfig);
+        } else if (fileLayerConfig.layerId && fileLayerConfig.layerId.startsWith(`${ORIGIN.KIBANA_YML}.`)) {
+          //fallback for older saved objects
+          url = fileLayerConfig.url;
+        } else {
+          //generic fallback
+          url = fileLayerConfig.url;
+        }
+        return url;
+      }
+
+      async getGeoJsonForRegionLayer(fileLayerConfig) {
+        const url = await this.getUrlForRegionLayer(fileLayerConfig);
+        const geojson = await $http({
+          url: url,
+          method: 'GET'
+        });
+        return geojson.data;
+      }
+
+
     }
 
     return new ServiceSettings();

x-pack/plugins/canvas/init.js

@@ -12,7 +12,7 @@ import { registerCanvasUsageCollector } from './server/usage';
 import { loadSampleData } from './server/sample_data';
 
 export default async function(server /*options*/) {
-  server.injectUiAppVars('canvas', () => {
+  server.injectUiAppVars('canvas', async () => {
     const config = server.config();
     const basePath = config.get('server.basePath');
     const reportingBrowserType = (() => {
@@ -20,8 +20,10 @@ export default async function(server /*options*/) {
       if (!config.has(configKey)) return null;
       return config.get(configKey);
     })();
+    const kibanaVars = await server.getInjectedUiAppVars('kibana');
 
     return {
+      ...kibanaVars,
       kbnIndex: config.get('kibana.index'),
       esShardTimeout: config.get('elasticsearch.shardTimeout'),
       esApiVersion: config.get('elasticsearch.apiVersion'),