@jupyter-lsp/jupyterlab-lsp/src/settings.ts

Language Server Protocol integration for JupyterLab

import { showDialog, Dialog } from '@jupyterlab/apputils';
import { ILanguageServerManager, LanguageServerManager } from '@jupyterlab/lsp';
import {
  ISettingRegistry,
  ISchemaValidator
} from '@jupyterlab/settingregistry';
import { TranslationBundle, nullTranslator } from '@jupyterlab/translation';
import {
  JSONExt,
  ReadonlyPartialJSONObject,
  ReadonlyJSONObject,
  PartialJSONObject,
  PromiseDelegate
} from '@lumino/coreutils';
import { Signal, ISignal } from '@lumino/signaling';
import { FieldProps } from '@rjsf/utils';

import { LanguageServers } from './_plugin';
import {
  renderLanguageServerSettings,
  renderCollapseConflicts
} from './components/serverSettings';
import { ILSPLogConsole } from './tokens';
import { collapseToDotted, expandDottedPaths } from './utils';

type ValueOf<T> = T[keyof T];
type ServerSchemaWrapper = ValueOf<
  Required<LanguageServers>['language_servers']
>;

/**
 * Only used for TypeScript type coercion, not meant to represent a property fully.
 */
interface IJSONProperty {
  type?: string | string[];
  description?: string;
  $ref?: string;
}

function isJSONProperty(obj: unknown): obj is IJSONProperty {
  return (
    typeof obj === 'object' && obj !== null && ('type' in obj || '$ref' in obj)
  );
}

/**
 * Settings keyed by language server name with values including
 * multiple properties, such as priority or workspace configuration
 */
type LanguageServerSettings = Record<string, ServerSchemaWrapper>;

/**
 * Default server priority; this should match the value defined in `plugin.json` schema.
 */
const DEAULT_SERVER_PRIORITY = 50;

/**
 * Get default values from JSON Schema properties field.
 */
function getDefaults(
  properties: ReadonlyPartialJSONObject | undefined
): Record<string, any> {
  if (properties == null) {
    return {};
  }
  // TODO: also get defaults from ref?
  const defaults: Record<string, any> = {};
  const entries = Object.entries(properties)
    .map(([key, value]) => [key, (value as any)?.default])
    .filter(([key, value]) => typeof value !== 'undefined');
  // TODO: use Object.fromEntries once we switch target
  for (let [key, value] of entries) {
    defaults[key] = value;
  }
  return defaults;
}

/**
 * Get a mutable property matching a dotted key and a properly nested value.
 *
 * Most LSP server schema properties are flattened using dotted convention,
 * e.g. a key for {pylsp: {plugins: {flake8: {enabled: true}}}}` is stored
 * as `pylsp.plugins.flake8.enabled`. However, some servers (e.g. pyright)
 * define specific properties as only partially doted, for example
 * `python.analysis.diagnosticSeverityOverrides` is an object with
 * properties like `reportGeneralTypeIssues` or `reportPropertyTypeMismatch`.
 * Only one level of nesting (on the finale level) is supported.
 */
function nestInSchema(
  properties: PartialJSONObject,
  key: string,
  value: PartialJSONObject
): { property: PartialJSONObject; value: PartialJSONObject } | null {
  if (properties.hasOwnProperty(key)) {
    return { property: properties[key] as PartialJSONObject, value };
  }
  const parts = key.split('.');
  const prefix = parts.slice(0, -1).join('.');
  const suffix = parts[parts.length - 1];
  if (properties.hasOwnProperty(prefix)) {
    const parent = properties[prefix] as PartialJSONObject;
    if (parent.type !== 'object') {
      return null;
    }
    const parentProperties = parent.properties as PartialJSONObject;
    if (parentProperties.hasOwnProperty(suffix)) {
      return {
        property: parent,
        value: { [suffix]: value }
      };
    }
  }
  return null;
}

function mergePropertyDefault(
  property: PartialJSONObject,
  value: PartialJSONObject
) {
  if (
    property.type === 'object' &&
    typeof property.default === 'object' &&
    typeof value === 'object'
  ) {
    property.default = {
      ...property.default,
      ...value
    };
  } else {
    property.default = value;
  }
}

/**
 * Schema and user data that for validation
 */
interface IValidationData {
  rawUserSettings: string;
  schema: ISettingRegistry.ISchema;
}

/**
 * Conflicts encountered when dot-collapsing settings
 * organised by server ID, and then as a mapping between
 * (dotted) setting ID and list of encountered values.
 * The last encountered values is preferred for use.
 */
type SettingsMergeConflicts = Record<string, Record<string, any[]>>;

interface ISettingsCollapseResult {
  settings: LanguageServerSettings;
  conflicts: SettingsMergeConflicts;
}

export class SettingsUIManager {
  constructor(
    protected options: {
      settingRegistry: ISettingRegistry;
      languageServerManager: ILanguageServerManager;
      console: ILSPLogConsole;
      trans: TranslationBundle;
      schemaValidated: ISignal<
        SettingsSchemaManager,
        ISchemaValidator.IError[]
      >;
    }
  ) {
    options.schemaValidated.connect((_, errors) => {
      this._validationErrors = errors;
    });
    this._validationErrors = [];
  }

  renderForm(props: FieldProps) {
    return renderLanguageServerSettings({
      settingRegistry: this.options.settingRegistry,
      languageServerManager: this.options.languageServerManager,
      trans: this.options.trans,
      validationErrors: this._validationErrors,
      ...props
    });
  }

  private _validationErrors: ISchemaValidator.IError[];
}

/**
 * Harmonize settings from schema, defaults from specification, and values set by user.
 */
export class SettingsSchemaManager {
  constructor(
    protected options: {
      settingRegistry: ISettingRegistry;
      languageServerManager: ILanguageServerManager;
      console: ILSPLogConsole;
      trans: TranslationBundle;
      /**
       * Promise resolved when JupyterLab splash screen disappears.
       */
      restored: Promise<void>;
    }
  ) {
    this._schemaValidated = new Signal(this);
    this._defaults = {};
    this._canonical = null;
    this._original = null;
    this._validationAttempt = 0;
    this._lastValidation = null;
    this._lastUserServerSettings = null;
    this._lastUserServerSettingsDoted = null;
    this._validationErrors = [];
  }

  get schemaValidated(): ISignal<
    SettingsSchemaManager,
    ISchemaValidator.IError[]
  > {
    return this._schemaValidated;
  }

  protected get console(): ILSPLogConsole {
    return this.options.console;
  }

  /**
   * Add schema for individual language servers into JSON schema.
   * This method has to be called before any other action
   * is performed on settingRegistry with regard to pluginId.
   */
  async setupSchemaTransform(pluginId: string): Promise<void> {
    const languageServerManager = this.options.languageServerManager;
    // To populate defaults we require specs to be available, so we need to
    // wait for until after the `languageServerManager` is ready.
    await languageServerManager.ready;

    // Transform the plugin object to return different schema than the default.
    this.options.settingRegistry.transform(pluginId, {
      fetch: plugin => {
        // Profiling data (earlier version):
        // Initial fetch: 61-64 ms
        // Subsequent without change: <1ms
        // Session change: 642 ms.
        // 91% spent on `validateData()` of which 10% in addSchema().
        // 1.8% spent on `deepCopy()`
        // 1.79% spend on other tasks in `populate()`
        // There is a limit on the transformation time, and failing to transform
        // in the default 1 second means that no settings whatsoever are available.
        // Therefore validation in `populate()` was moved into an async function;
        // this means that we need to trigger re-load of settings
        // if there validation errors.

        // Only store the original schema the first time.
        if (!this._original) {
          this._original = JSONExt.deepCopy(plugin.schema);
        }
        // Only override the canonical schema the first time (or after reset).
        if (!this._canonical) {
          this._canonical = JSONExt.deepCopy(plugin.schema);
          this._populate(plugin, this._canonical);
          this._defaultsPopulated.resolve(void 0);
        }

        return {
          data: plugin.data,
          id: plugin.id,
          raw: plugin.raw,
          schema: this._validationErrors.length
            ? this._original
            : this._canonical,
          version: plugin.version
        };
      }
    });

    // note: has to be after transform is called for the first time to avoid
    // race condition, see https://github.com/jupyterlab/jupyterlab/issues/12978
    languageServerManager.sessionsChanged.connect(async () => {
      this._canonical = null;
      this._defaultsPopulated = new PromiseDelegate();
      await this.options.settingRegistry.reload(pluginId);
    });
  }

  /**
   * Populate the plugin's schema defaults, transform descriptions.
   */
  private _populate(
    plugin: ISettingRegistry.IPlugin,
    schema: ISettingRegistry.ISchema
  ) {
    const languageServerManager = this.options.languageServerManager;

    const { properties, defaults } = SettingsSchemaManager.transformSchemas({
      schema,
      // TODO: expose `specs` upstream and use `ILanguageServerManager` instead
      specs: (languageServerManager as LanguageServerManager).specs,
      sessions: languageServerManager.sessions,
      console: this.console,
      trans: this.options.trans
    });

    schema.properties!.language_servers.properties = properties;
    schema.properties!.language_servers.default = defaults;

    this._validateSchemaLater(plugin, schema).catch(this.console.warn);
    this._defaults = defaults;
  }

  /**
   * Transform the plugin schema defaults, properties and descriptions
   */
  static transformSchemas(options: {
    schema: ISettingRegistry.ISchema;
    specs: LanguageServerManager['specs'];
    sessions: ILanguageServerManager['sessions'];
    console?: ILSPLogConsole;
    trans?: TranslationBundle;
  }) {
    const { schema, sessions, specs } = options;
    const trans = options.trans ?? nullTranslator.load('jupyterlab-lsp');
    const console = options.console ?? window.console;
    const baseServerSchema = (schema.definitions as any)['language-server'] as {
      description: string;
      title: string;
      definitions: Record<string, any>;
      properties: ServerSchemaWrapper;
    };

    const defaults: Record<string, any> = {};
    const knownServersConfig: Record<string, any> = {};
    // `sharedDefaults` may be empty as we do not define/receive custom
    // per-property defaults in schema as of the day of writing.
    const sharedDefaults = getDefaults(
      schema.properties!.language_servers.properties
    );
    const defaultsOverrides = schema.properties!.language_servers.default as
      | Record<string, any>
      | undefined;

    for (let [serverKey, serverSpec] of specs.entries()) {
      if ((serverKey as string) === '') {
        console.warn(
          `Empty server key - skipping transformation for ${serverSpec}`
        );
        continue;
      }

      const configSchema = serverSpec.config_schema;
      if (!configSchema) {
        console.warn(
          `No config schema - skipping transformation for ${serverKey}`
        );
        continue;
      }
      if (!configSchema.properties) {
        console.warn(
          `No properties in config schema - skipping transformation for ${serverKey}`
        );
        continue;
      }

      // let user know if server not available (installed, etc)
      if (!sessions.has(serverKey)) {
        configSchema.description = trans.__(
          'Settings that would be passed to `%1` server (this server was not detected as installed during startup) in `workspace/didChangeConfiguration` notification.',
          serverSpec.display_name
        );
      } else {
        configSchema.description = trans.__(
          'Settings to be passed to %1 in `workspace/didChangeConfiguration` notification.',
          serverSpec.display_name
        );
      }
      configSchema.title = trans.__('Workspace Configuration');

      // resolve refs
      for (let [key, value] of Object.entries(configSchema.properties)) {
        if (!isJSONProperty(value)) {
          continue;
        }
        if (typeof value.$ref === 'undefined') {
          continue;
        }
        if (value.$ref.startsWith('#/definitions/')) {
          const definitionID = value['$ref'].substring(14);
          const definition = configSchema.definitions[definitionID];
          if (definition == null) {
            console.warn('Definition not found');
          }
          for (let [defKey, defValue] of Object.entries(definition)) {
            configSchema.properties[key][defKey] = defValue;
          }
          delete value.$ref;
        } else {
          console.warn('Unsupported $ref', value['$ref']);
        }
      }

      // add default overrides from server-side spec (such as defined in `jupyter_server_config.py`)
      const workspaceConfigurationDefaults =
        serverSpec.workspace_configuration as Record<string, any> | undefined;
      if (workspaceConfigurationDefaults) {
        for (const [key, value] of Object.entries(
          workspaceConfigurationDefaults
        )) {
          const nested = nestInSchema(configSchema.properties, key, value);
          if (!nested) {
            console.warn(
              `"workspace_configuration" includes an override for "${key}" key which was not found in ${serverKey} schema'`
            );
            continue;
          }
          mergePropertyDefault(nested.property, nested.value);
        }
      }
      // add server-specific default overrides from `overrides.json` (and pre-defined in schema)
      const serverDefaultsOverrides =
        defaultsOverrides && defaultsOverrides.hasOwnProperty(serverKey)
          ? defaultsOverrides[serverKey]
          : {};
      if (serverDefaultsOverrides.serverSettings) {
        for (const [key, value] of Object.entries(
          serverDefaultsOverrides.serverSettings
        )) {
          const nested = nestInSchema(
            configSchema.properties,
            key,
            value as any
          );
          if (!nested) {
            console.warn(
              `"overrides.json" includes an override for "${key}" key which was not found in ${serverKey} schema`
            );
            continue;
          }
          mergePropertyDefault(nested.property, nested.value);
        }
      }

      const defaultMap = getDefaults(configSchema.properties);

      const baseSchemaCopy = JSONExt.deepCopy(baseServerSchema);
      baseSchemaCopy.properties.serverSettings = configSchema;
      knownServersConfig[serverKey] = baseSchemaCopy;
      defaults[serverKey] = {
        ...sharedDefaults,
        ...serverDefaultsOverrides,
        serverSettings: defaultMap
      };
    }

    return {
      properties: knownServersConfig,
      defaults
    };
  }

  /**
   * Expands dotted values into nested properties when the server config schema
   * indicates that this is needed. The schema is passed within the specs.
   *
   * This is needed because some settings, specifically pright's
   * `python.analysis.diagnosticSeverityOverrides` are defined as nested.
   */
  static expandDottedAsNeeded(options: {
    dottedSettings: LanguageServerSettings;
    specs: LanguageServerManager['specs'];
  }): LanguageServerSettings {
    const specs = options.specs;
    const partiallyUncollapsed = JSONExt.deepCopy(options.dottedSettings);

    for (let [serverKey, serverSpec] of specs.entries()) {
      const configSchema = serverSpec.config_schema;
      if (!partiallyUncollapsed.hasOwnProperty(serverKey)) {
        continue;
      }
      const settings = partiallyUncollapsed[serverKey].serverSettings;
      if (!configSchema || !settings) {
        continue;
      }
      const expanded = expandDottedPaths(settings);

      for (const [path, property] of Object.entries<PartialJSONObject>(
        configSchema.properties
      )) {
        if (property.type === 'object') {
          let value = expanded;
          for (const part of path.split('.')) {
            value = value[part] as ReadonlyJSONObject;
            if (typeof value === 'undefined') {
              break;
            }
          }
          if (typeof value === 'undefined') {
            continue;
          }
          // Add the uncollapsed value
          settings[path] = value;
          // Remove the collapsed values
          for (const k of Object.keys(value)) {
            const key = path + '.' + k;
            if (!settings.hasOwnProperty(key)) {
              throw Error(
                'Internal inconsistency: collapsed settings state does not match expanded object'
              );
            }
            delete settings[key];
          }
        }
      }
    }
    return partiallyUncollapsed;
  }

  /**
   * Normalize settings by dotted and nested specs, and merging with defaults.
   */
  async normalizeSettings(
    composite: Required<LanguageServers>
  ): Promise<Required<LanguageServers>> {
    await this._defaultsPopulated.promise;
    // Cache collapsed settings for speed and to only show dialog once.
    // Note that JupyterLab attempts to transform in "preload" step (before splash screen end)
    // and then again for deferred extensions if the initial transform in preload timed out.
    // We are hitting the timeout in preload step.
    if (
      this._lastUserServerSettings === null ||
      this._lastUserServerSettingsDoted === null ||
      !JSONExt.deepEqual(
        this._lastUserServerSettings,
        composite.language_servers
      )
    ) {
      this._lastUserServerSettings = composite.language_servers;

      const collapsedDefaults = this._collapseServerSettingsDotted(
        this._defaults
      );
      const collapsedUser = this._collapseServerSettingsDotted(
        composite.language_servers
      );

      const merged = SettingsSchemaManager.mergeByServer(
        collapsedDefaults.settings,
        collapsedUser.settings
      );

      // Uncollapse settings which need to be in the expanded form
      const languageServerManager = this.options.languageServerManager;
      const uncollapsed = SettingsSchemaManager.expandDottedAsNeeded({
        dottedSettings: merged,
        specs: (languageServerManager as LanguageServerManager).specs
      });

      composite.language_servers = uncollapsed;
      this._lastUserServerSettingsDoted = uncollapsed;

      if (Object.keys(collapsedUser.conflicts).length > 0) {
        this._warnConflicts(
          collapsedUser.conflicts,
          'Conflicts in user settings'
        ).catch(this.console.warn);
      }
      if (Object.keys(collapsedDefaults.conflicts).length > 0) {
        this._warnConflicts(
          collapsedDefaults.conflicts,
          'Conflicts in defaults'
        ).catch(this.console.warn);
      }
    } else {
      composite.language_servers = this._lastUserServerSettingsDoted;
    }

    // We do not filter out defaults at this level,
    // as it does not provide an obvious benefit:
    // - we would need to explicitly save the updated settings
    //   to get a clean version in JSON Setting Editor.
    // - if default changed on the LSP server side but schema did not get
    //   updated, LSP server would be using a different value than communicated
    //   to the user. It would be optimal to filter out defaults from
    //   user data and always keep them in composite,
    // - making Jupyter server-side `workspace_configuration` work would
    //   be more difficult

    // TODO: trigger update of settings to ensure that UI uses the same settings as collapsed?
    return composite;
  }

  private _wasPreviouslyValidated(
    plugin: ISettingRegistry.IPlugin,
    schema: ISettingRegistry.ISchema
  ) {
    return (
      this._lastValidation !== null &&
      this._lastValidation.rawUserSettings === plugin.raw &&
      JSONExt.deepEqual(this._lastValidation.schema, schema)
    );
  }

  /**
   * Validate user settings from plugin against provided schema,
   * asynchronously to avoid blocking the main thread.
   * Stores validation result in `this._validationErrors`.
   */
  private async _validateSchemaLater(
    plugin: ISettingRegistry.IPlugin,
    schema: ISettingRegistry.ISchema
  ) {
    // Ensure the subsequent code runs asynchronously; also reduce the CPU load on startup.
    await this.options.restored;

    // Do not re-validate if neither schema, nor user settings changed
    if (this._wasPreviouslyValidated(plugin, schema)) {
      return;
    }
    // Test if we can apply the schema without causing validation error
    // (is the configuration held by the user compatible with the schema?)
    this._validationAttempt += 1;
    // the validator will parse raw plugin data into this object;
    // we do not do anything with those right now.
    const parsedData = { composite: {}, user: {} };
    const validationErrors =
      this.options.settingRegistry.validator.validateData(
        {
          // The plugin schema is cached so we have to provide a dummy ID;
          // can be simplified once https://github.com/jupyterlab/jupyterlab/issues/12978 is fixed.
          id: `lsp-validation-attempt-${this._validationAttempt}`,
          raw: plugin.raw,
          data: parsedData,
          version: plugin.version,
          schema: schema
        },
        true
      );

    this._lastValidation = {
      rawUserSettings: plugin.raw,
      schema: schema
    };

    if (validationErrors) {
      console.error(
        'LSP server settings validation failed; graphical interface for settings will run in schema-free mode; errors:',
        validationErrors
      );
      this._validationErrors = validationErrors;
      this._schemaValidated.emit(validationErrors);
      if (!this._original) {
        console.error(
          'Original language servers schema not available to restore non-transformed values.'
        );
      } else {
        if (!this._original.properties!.language_servers.properties) {
          delete schema.properties!.language_servers.properties;
        }
        if (!this._original.properties!.language_servers.default) {
          delete schema.properties!.language_servers.default;
        }
      }

      // Reload settings to use non-restrictive schema; this requires fixing
      // https://github.com/jupyterlab/jupyterlab/issues/12978 upstream to work.
      await this.options.settingRegistry.reload(plugin.id);
    }
  }

  private async _warnConflicts(
    conflicts: SettingsMergeConflicts,
    title: string
  ) {
    // Ensure the subsequent code runs asynchronously, and delay
    // showing the dialog until the splash screen disappeared.
    await this.options.restored;

    showDialog({
      body: renderCollapseConflicts({
        conflicts: conflicts,
        trans: this.options.trans
      }),
      title: title,
      buttons: [Dialog.okButton()]
    }).catch(console.warn);
  }

  private _collapseServerSettingsDotted(
    settings: LanguageServerSettings
  ): ISettingsCollapseResult {
    const conflicts: Record<string, Record<string, any[]>> = {};
    const result = JSONExt.deepCopy(settings) as LanguageServerSettings;
    for (let [serverKey, serverSettingsGroup] of Object.entries(settings)) {
      if (!serverSettingsGroup || !serverSettingsGroup.serverSettings) {
        continue;
      }
      const collapsed = collapseToDotted(
        serverSettingsGroup.serverSettings as ReadonlyJSONObject
      );
      if (Object.keys(collapsed.conflicts).length) {
        conflicts[serverKey] = collapsed.conflicts;
      }
      result[serverKey]!.serverSettings = collapsed.result;
    }
    return {
      settings: result,
      conflicts: conflicts
    };
  }

  static mergeByServer(
    defaults: LanguageServerSettings,
    userSettings: LanguageServerSettings
  ): LanguageServerSettings {
    const result = JSONExt.deepCopy(defaults) as LanguageServerSettings;
    for (let [serverKey, serverSettingsGroup] of Object.entries(userSettings)) {
      if (!serverSettingsGroup || !serverSettingsGroup.serverSettings) {
        continue;
      }
      if (typeof result[serverKey] === 'undefined') {
        // nothing to merge with
        result[serverKey] = JSONExt.deepCopy(serverSettingsGroup);
      } else {
        // priority should come from (a) user (b) overrides (c) fallback default;
        // unfortunately the user and default values get merged in the form so we
        // cannot distinguish (a) from (c); as a workaround we can compare its value
        // with the default value.
        const userOrDefaultPriority = serverSettingsGroup.priority;
        const isPriorityUserSet =
          typeof userOrDefaultPriority !== 'undefined' &&
          userOrDefaultPriority !== DEAULT_SERVER_PRIORITY;
        const priority = isPriorityUserSet
          ? userOrDefaultPriority
          : result[serverKey].priority ?? DEAULT_SERVER_PRIORITY;
        const merged: Required<ServerSchemaWrapper> = {
          priority,
          // `serverSettings` entries are expected to be flattened to dot notation here.
          serverSettings: {
            ...(result[serverKey].serverSettings || {}),
            ...(serverSettingsGroup.serverSettings || {})
          }
        };
        result[serverKey] = merged;
      }
    }
    return result;
  }

  private _defaults: LanguageServerSettings;
  private _defaultsPopulated = new PromiseDelegate();
  private _validationErrors: ISchemaValidator.IError[];
  private _schemaValidated: Signal<
    SettingsSchemaManager,
    ISchemaValidator.IError[]
  >;
  private _validationAttempt: number;
  private _lastValidation: IValidationData | null;
  private _lastUserServerSettings: LanguageServerSettings | null;
  private _lastUserServerSettingsDoted: LanguageServerSettings | null;
  private _canonical: ISettingRegistry.ISchema | null;
  private _original: ISettingRegistry.ISchema | null;
}