chrome-devtools-frontend/front_end/core/sdk/PageResourceLoader.ts

Chrome DevTools UI

// Copyright 2020 The Chromium Authors
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

import type * as Protocol from '../../generated/protocol.js';
import * as Common from '../common/common.js';
import * as Host from '../host/host.js';
import * as i18n from '../i18n/i18n.js';
import type * as Platform from '../platform/platform.js';
import * as Root from '../root/root.js';

import {IOModel} from './IOModel.js';
import {MultitargetNetworkManager, NetworkManager} from './NetworkManager.js';
import {
  Events as ResourceTreeModelEvents,
  PrimaryPageChangeType,
  type ResourceTreeFrame,
  ResourceTreeModel,
} from './ResourceTreeModel.js';
import type {Target} from './Target.js';
import {TargetManager} from './TargetManager.js';

const UIStrings = {
  /**
   * @description Error message for canceled source map loads
   */
  loadCanceledDueToReloadOf: 'Load canceled due to reload of inspected page',
} as const;
const str_ = i18n.i18n.registerUIStrings('core/sdk/PageResourceLoader.ts', UIStrings);
const i18nString = i18n.i18n.getLocalizedString.bind(undefined, str_);

export interface ExtensionInitiator {
  target: null;
  frameId: null;
  initiatorUrl: Platform.DevToolsPath.UrlString;
  extensionId: string;
}

export type PageResourceLoadInitiator = {
  target: null,
  frameId: Protocol.Page.FrameId,
  initiatorUrl: Platform.DevToolsPath.UrlString|null,
}|{
  target: Target,
  frameId: Protocol.Page.FrameId | null,
  initiatorUrl: Platform.DevToolsPath.UrlString | null,
}|ExtensionInitiator;

function isExtensionInitiator(initiator: PageResourceLoadInitiator): initiator is ExtensionInitiator {
  return 'extensionId' in initiator;
}

export interface PageResource {
  success: boolean|null;
  errorMessage?: string;
  initiator: PageResourceLoadInitiator;
  url: Platform.DevToolsPath.UrlString;
  size: number|null;
  duration: number|null;
}

/** Used for revealing a resource. **/
export class ResourceKey {
  readonly key: string;

  constructor(key: string) {
    this.key = key;
  }
}

export type UserAgentProvider = Pick<MultitargetNetworkManager, 'currentUserAgent'>;

/**
 * The PageResourceLoader has two responsibilities: loading resources and tracking statistics scoped to targets
 * for the DeveloperResourcesPanel. Many places only require the former, so we expose that functionality via small
 * sub-interface. This makes it easier to test classes that require resource loading.
 */
export type ResourceLoader = Pick<PageResourceLoader, 'loadResource'>;

interface LoadQueueEntry {
  resolve: () => void;
  reject: (arg0: Error) => void;
}

/**
 * The page resource loader is a bottleneck for all DevTools-initiated resource loads. For each such load, it keeps a
 * `PageResource` object around that holds meta information. This can be as the basis for reporting to the user which
 * resources were loaded, and whether there was a load error.
 */
export class PageResourceLoader extends Common.ObjectWrapper.ObjectWrapper<EventTypes> implements ResourceLoader {
  readonly #targetManager: TargetManager;
  readonly #settings: Common.Settings.Settings;
  readonly #userAgentProvider: UserAgentProvider;
  #currentlyLoading = 0;
  #currentlyLoadingPerTarget = new Map<Protocol.Target.TargetID|'main', number>();
  readonly #maxConcurrentLoads: number;
  #pageResources = new Map<string, PageResource>();
  #queuedLoads: LoadQueueEntry[] = [];
  readonly #loadOverride: ((arg0: string) => Promise<{
                             success: boolean,
                             content: string|Uint8Array<ArrayBuffer>,
                             errorDescription: Host.ResourceLoader.LoadErrorDescription,
                           }>)|null;
  constructor(
      targetManager: TargetManager, settings: Common.Settings.Settings, userAgentProvider: UserAgentProvider,
      loadOverride: ((arg0: string) => Promise<{
                       success: boolean,
                       content: string|Uint8Array<ArrayBuffer>,
                       errorDescription: Host.ResourceLoader.LoadErrorDescription,
                     }>)|null,
      maxConcurrentLoads = 500) {
    super();
    this.#targetManager = targetManager;
    this.#settings = settings;
    this.#userAgentProvider = userAgentProvider;
    this.#maxConcurrentLoads = maxConcurrentLoads;
    this.#targetManager.addModelListener(
        ResourceTreeModel, ResourceTreeModelEvents.PrimaryPageChanged, this.onPrimaryPageChanged, this);
    this.#loadOverride = loadOverride;
  }

  static instance({forceNew, targetManager, settings, userAgentProvider, loadOverride, maxConcurrentLoads}: {
    forceNew: boolean,
    loadOverride: (null|((arg0: string) => Promise<{
                           success: boolean,
                           content: string|Uint8Array<ArrayBuffer>,
                           errorDescription: Host.ResourceLoader.LoadErrorDescription,
                         }>)),
    targetManager?: TargetManager,
    settings?: Common.Settings.Settings,
    userAgentProvider?: UserAgentProvider,
    maxConcurrentLoads?: number,
  } = {
    forceNew: false,
    loadOverride: null,
  }): PageResourceLoader {
    if (forceNew) {
      Root.DevToolsContext.globalInstance().set(
          PageResourceLoader,
          new PageResourceLoader(
              targetManager ?? TargetManager.instance(), settings ?? Common.Settings.Settings.instance(),
              userAgentProvider ?? MultitargetNetworkManager.instance(), loadOverride, maxConcurrentLoads));
    }

    return Root.DevToolsContext.globalInstance().get(PageResourceLoader);
  }

  static removeInstance(): void {
    Root.DevToolsContext.globalInstance().delete(PageResourceLoader);
  }

  onPrimaryPageChanged(
      event: Common.EventTarget.EventTargetEvent<{frame: ResourceTreeFrame, type: PrimaryPageChangeType}>): void {
    const {frame: mainFrame, type} = event.data;
    if (!mainFrame.isOutermostFrame()) {
      return;
    }
    for (const {reject} of this.#queuedLoads) {
      reject(new Error(i18nString(UIStrings.loadCanceledDueToReloadOf)));
    }
    this.#queuedLoads = [];
    const mainFrameTarget = mainFrame.resourceTreeModel().target();
    const keptResources = new Map<string, PageResource>();
    // If the navigation is a prerender-activation, the pageResources for the destination page have
    // already been preloaded. In such cases, we therefore don't just discard all pageResources, but
    // instead make sure to keep the pageResources for the prerendered target.
    for (const [key, pageResource] of this.#pageResources.entries()) {
      if ((type === PrimaryPageChangeType.ACTIVATION) && mainFrameTarget === pageResource.initiator.target) {
        keptResources.set(key, pageResource);
      }
    }
    this.#pageResources = keptResources;
    this.dispatchEventToListeners(Events.UPDATE);
  }

  getResourcesLoaded(): Map<string, PageResource> {
    return this.#pageResources;
  }

  getScopedResourcesLoaded(): Map<string, PageResource> {
    return new Map([...this.#pageResources].filter(
        ([_, pageResource]) => this.#targetManager.isInScope(pageResource.initiator.target) ||
            isExtensionInitiator(pageResource.initiator)));
  }

  /**
   * Loading is the number of currently loading and queued items. Resources is the total number of resources,
   * including loading and queued resources, but not including resources that are still loading but scheduled
   * for cancelation.;
   */
  getNumberOfResources(): {
    loading: number,
    queued: number,
    resources: number,
  } {
    return {loading: this.#currentlyLoading, queued: this.#queuedLoads.length, resources: this.#pageResources.size};
  }

  getScopedNumberOfResources(): {
    loading: number,
    resources: number,
  } {
    let loadingCount = 0;
    for (const [targetId, count] of this.#currentlyLoadingPerTarget) {
      const target = this.#targetManager.targetById(targetId);
      if (this.#targetManager.isInScope(target)) {
        loadingCount += count;
      }
    }
    return {loading: loadingCount, resources: this.getScopedResourcesLoaded().size};
  }

  private async acquireLoadSlot(target: Target|null): Promise<void> {
    this.#currentlyLoading++;
    if (target) {
      const currentCount = this.#currentlyLoadingPerTarget.get(target.id()) || 0;
      this.#currentlyLoadingPerTarget.set(target.id(), currentCount + 1);
    }
    if (this.#currentlyLoading > this.#maxConcurrentLoads) {
      const {
        promise: waitForCapacity,
        resolve,
        reject,
      } = Promise.withResolvers<void>();
      this.#queuedLoads.push({resolve, reject});
      await waitForCapacity;
    }
  }

  private releaseLoadSlot(target: Target|null): void {
    this.#currentlyLoading--;
    if (target) {
      const currentCount = this.#currentlyLoadingPerTarget.get(target.id());
      if (currentCount) {
        this.#currentlyLoadingPerTarget.set(target.id(), currentCount - 1);
      }
    }
    const entry = this.#queuedLoads.shift();
    if (entry) {
      entry.resolve();
    }
  }

  static makeExtensionKey(url: Platform.DevToolsPath.UrlString, initiator: PageResourceLoadInitiator): string {
    if (isExtensionInitiator(initiator) && initiator.extensionId) {
      return `${url}-${initiator.extensionId}`;
    }
    throw new Error('Invalid initiator');
  }

  static makeKey(url: Platform.DevToolsPath.UrlString, initiator: PageResourceLoadInitiator): string {
    if (initiator.frameId) {
      return `${url}-${initiator.frameId}`;
    }
    if (initiator.target) {
      return `${url}-${initiator.target.id()}`;
    }
    throw new Error('Invalid initiator');
  }

  resourceLoadedThroughExtension(pageResource: PageResource): void {
    const key = PageResourceLoader.makeExtensionKey(pageResource.url, pageResource.initiator);
    this.#pageResources.set(key, pageResource);
    this.dispatchEventToListeners(Events.UPDATE);
  }

  loadResource(url: Platform.DevToolsPath.UrlString, initiator: PageResourceLoadInitiator, isBinary: true): Promise<{
    content: Uint8Array<ArrayBuffer>,
  }>;
  loadResource(url: Platform.DevToolsPath.UrlString, initiator: PageResourceLoadInitiator, isBinary?: false): Promise<{
    content: string,
  }>;
  async loadResource(url: Platform.DevToolsPath.UrlString, initiator: PageResourceLoadInitiator, isBinary = false):
      Promise<{
        content: string | Uint8Array<ArrayBuffer>,
      }> {
    if (isExtensionInitiator(initiator)) {
      throw new Error('Invalid initiator');
    }
    const key = PageResourceLoader.makeKey(url, initiator);
    const pageResource:
        PageResource = {success: null, size: null, duration: null, errorMessage: undefined, url, initiator};
    this.#pageResources.set(key, pageResource);
    this.dispatchEventToListeners(Events.UPDATE);
    const startTime = performance.now();
    try {
      await this.acquireLoadSlot(initiator.target);
      const resultPromise = this.dispatchLoad(url, initiator, isBinary);
      const result = await resultPromise;
      pageResource.errorMessage = result.errorDescription.message;
      pageResource.success = result.success;
      if (result.success) {
        pageResource.size = result.content.length;
        return {content: result.content};
      }
      throw new Error(result.errorDescription.message);
    } catch (e) {
      if (pageResource.errorMessage === undefined) {
        pageResource.errorMessage = e.message;
      }
      if (pageResource.success === null) {
        pageResource.success = false;
      }
      throw e;
    } finally {
      pageResource.duration = performance.now() - startTime;
      this.releaseLoadSlot(initiator.target);
      this.dispatchEventToListeners(Events.UPDATE);
    }
  }

  private async dispatchLoad(
      url: Platform.DevToolsPath.UrlString, initiator: PageResourceLoadInitiator, isBinary: boolean): Promise<{
    success: boolean,
    content: string|Uint8Array<ArrayBuffer>,
    errorDescription: Host.ResourceLoader.LoadErrorDescription,
  }> {
    if (isExtensionInitiator(initiator)) {
      throw new Error('Invalid initiator');
    }

    const failureReason: string|null = null;
    if (this.#loadOverride) {
      return await this.#loadOverride(url);
    }
    const parsedURL = new Common.ParsedURL.ParsedURL(url);
    const eligibleForLoadFromTarget = this.getLoadThroughTargetSetting().get() && parsedURL &&
        parsedURL.scheme !== 'file' && parsedURL.scheme !== 'data' && parsedURL.scheme !== 'devtools' &&
        initiator.target;
    Host.userMetrics.developerResourceScheme(this.getDeveloperResourceScheme(parsedURL));
    if (eligibleForLoadFromTarget) {
      try {
        Host.userMetrics.developerResourceLoaded(Host.UserMetrics.DeveloperResourceLoaded.LOAD_THROUGH_PAGE_VIA_TARGET);
        const result = await this.loadFromTarget(initiator.target, initiator.frameId, url, isBinary);
        return result;
      } catch (e) {
        if (e instanceof Error) {
          Host.userMetrics.developerResourceLoaded(Host.UserMetrics.DeveloperResourceLoaded.LOAD_THROUGH_PAGE_FAILURE);
          if (e.message.includes('CSP violation')) {
            return {
              success: false,
              content: '',
              errorDescription:
                  {statusCode: 0, netError: undefined, netErrorName: undefined, message: e.message, urlValid: undefined}
            };
          }
        }
      }
      Host.userMetrics.developerResourceLoaded(Host.UserMetrics.DeveloperResourceLoaded.LOAD_THROUGH_PAGE_FALLBACK);
    } else {
      const code = this.getLoadThroughTargetSetting().get() ?
          Host.UserMetrics.DeveloperResourceLoaded.FALLBACK_PER_PROTOCOL :
          Host.UserMetrics.DeveloperResourceLoaded.FALLBACK_PER_OVERRIDE;
      Host.userMetrics.developerResourceLoaded(code);
    }

    const result = await this.loadFromHostBindings(url);
    if (eligibleForLoadFromTarget && !result.success) {
      Host.userMetrics.developerResourceLoaded(Host.UserMetrics.DeveloperResourceLoaded.FALLBACK_FAILURE);
    }
    if (failureReason) {
      // In case we have a success, add a note about why the load through the target failed.
      result.errorDescription.message =
          `Fetch through target failed: ${failureReason}; Fallback: ${result.errorDescription.message}`;
    }
    return result;
  }

  private getDeveloperResourceScheme(parsedURL: Common.ParsedURL.ParsedURL|null):
      Host.UserMetrics.DeveloperResourceScheme {
    if (!parsedURL || parsedURL.scheme === '') {
      return Host.UserMetrics.DeveloperResourceScheme.UKNOWN;
    }
    const isLocalhost = parsedURL.host === 'localhost' || parsedURL.host.endsWith('.localhost');
    switch (parsedURL.scheme) {
      case 'file':
        return Host.UserMetrics.DeveloperResourceScheme.FILE;
      case 'data':
        return Host.UserMetrics.DeveloperResourceScheme.DATA;
      case 'blob':
        return Host.UserMetrics.DeveloperResourceScheme.BLOB;
      case 'http':
        return isLocalhost ? Host.UserMetrics.DeveloperResourceScheme.HTTP_LOCALHOST :
                             Host.UserMetrics.DeveloperResourceScheme.HTTP;
      case 'https':
        return isLocalhost ? Host.UserMetrics.DeveloperResourceScheme.HTTPS_LOCALHOST :
                             Host.UserMetrics.DeveloperResourceScheme.HTTPS;
    }
    return Host.UserMetrics.DeveloperResourceScheme.OTHER;
  }

  private async loadFromTarget(
      target: Target, frameId: Protocol.Page.FrameId|null, url: Platform.DevToolsPath.UrlString,
      isBinary: boolean): Promise<{
    success: boolean,
    content: string|Uint8Array<ArrayBuffer>,
    errorDescription: {
      statusCode: number,
      netError: number|undefined,
      netErrorName: string|undefined,
      message: string,
      urlValid: undefined,
    },
  }> {
    const networkManager = (target.model(NetworkManager) as NetworkManager);
    const ioModel = (target.model(IOModel) as IOModel);
    const disableCache = this.#settings.moduleSetting('cache-disabled').get();
    const resource = await networkManager.loadNetworkResource(frameId, url, {disableCache, includeCredentials: true});
    try {
      const content = resource.stream ?
          (isBinary ? await ioModel.readToBuffer(resource.stream) : await ioModel.readToString(resource.stream)) :
          '';
      return {
        success: resource.success,
        content,
        errorDescription: {
          statusCode: resource.httpStatusCode || 0,
          netError: resource.netError,
          netErrorName: resource.netErrorName,
          message: Host.ResourceLoader.netErrorToMessage(
                       resource.netError, resource.httpStatusCode, resource.netErrorName) ||
              '',
          urlValid: undefined,
        },
      };
    } finally {
      if (resource.stream) {
        void ioModel.close(resource.stream);
      }
    }
  }

  private async loadFromHostBindings(url: Platform.DevToolsPath.UrlString): Promise<{
    success: boolean,
    content: string,
    errorDescription: Host.ResourceLoader.LoadErrorDescription,
  }> {
    const headers: Record<string, string> = {};

    const currentUserAgent = this.#userAgentProvider.currentUserAgent();
    if (currentUserAgent) {
      headers['User-Agent'] = currentUserAgent;
    }

    if (this.#settings.moduleSetting('cache-disabled').get()) {
      headers['Cache-Control'] = 'no-cache';
    }

    const allowRemoteFilePaths = this.#settings.moduleSetting('network.enable-remote-file-loading').get();

    return await new Promise(
        resolve => Host.ResourceLoader.load(url, headers, (success, _responseHeaders, content, errorDescription) => {
          resolve({success, content, errorDescription});
        }, allowRemoteFilePaths));
  }

  getLoadThroughTargetSetting(): Common.Settings.Setting<boolean> {
    return this.#settings.createSetting('load-through-target', true);
  }
}

export const enum Events {
  UPDATE = 'Update',
}

export interface EventTypes {
  [Events.UPDATE]: void;
}