webtorrent/lib/torrent.js

Streaming torrent client

import EventEmitter from 'events'
import fs from 'fs'
import net from 'net' // browser exclude
import os from 'os' // browser exclude
import path from 'path'
import addrToIPPort from 'addr-to-ip-port'
import BitField from 'bitfield'
import CacheChunkStore from 'cache-chunk-store'
import { chunkStoreWrite } from 'chunk-store-iterator'
import cpus from 'cpus'
import debugFactory from 'debug'
import Discovery from 'torrent-discovery'
import FSChunkStore from 'fs-chunk-store' // browser: `fsa-chunk-store`
import fetch from 'cross-fetch-ponyfill'
import ImmediateChunkStore from 'immediate-chunk-store'
import ltDontHave from 'lt_donthave'
import MemoryChunkStore from 'memory-chunk-store'
import joinIterator from 'join-async-iterator'
import parallel from 'run-parallel'
import parallelLimit from 'run-parallel-limit'
import parseTorrent, { toMagnetURI, toTorrentFile, remote } from 'parse-torrent'
import Piece from 'torrent-piece'
import queueMicrotask from 'queue-microtask'
import randomIterate from 'random-iterate'
import { hash, arr2hex } from 'uint8-util'
import throughput from 'throughput'
import utMetadata from 'ut_metadata'
import utPex from 'ut_pex' // browser exclude

import File from './file.js'
import Peer from './peer.js'
import RarityMap from './rarity-map.js'
import utp from './utp.cjs' // browser exclude
import WebConn from './webconn.js'
import { Selections } from './selections.js'

import VERSION from '../version.cjs'

// The following JSDoc comments are global types that can be used across the project

/**
 * This callback is called with an optional error, if the error is a falsy value the operation was executed successfully
 * @callback callbackWithError
 * @param {Error=} error
 */

/**
 * @typedef TorrentOpts
 * @type {object}
 * @property {Array<string>=} announce - Torrent trackers to use (added to list in .torrent or magnet uri)
 * @property {Array<string>=} urlList - Array of web seeds
 * @property {string=} path - Folder to download files to (default=`/tmp/webtorrent/`)
 * @property {boolean=} addUID - (Node.js only) If true, the torrent will be stored in it's infoHash folder to prevent file name collisions (default=false)
 * @property {FileSystemDirectoryHandle=} rootDir - *(browser only)* if supported by the browser, allows the user to specify a custom directory to stores the files in, retaining the torrent's folder and file structure
 * @property {boolean=} skipVerify - If true, client will skip verification of pieces for existing store and assume it's correct
 * @property {Uint8Array|ArrayLike<number>=} bitfield -  Preloaded numerical array/buffer to use to know what pieces are already downloaded (any type accepted by UInt8Array constructor is valid)
 * @property {FSChunkStore|MemoryChunkStore|Function=} store - Custom chunk store
 * @property {FSChunkStore|MemoryChunkStore|Function=} preloadedStore -  Custom, pre-loaded chunk store
 * @property {number=} storeCacheSlots - Number of chunk store entries (torrent pieces) to cache in memory [default=20]; 0 to disable caching
 * @property {boolean=} destroyStoreOnDestroy - If truthy, client will delete the torrent's chunk store (e.g. files on disk) when the torrent is destroyed
 * @property {object=} storeOpts - Custom options passed to the store
 * @property {boolean=} alwaysChokeSeeders - If true, client will automatically choke seeders if it's seeding. (default=true)
 * @property {function(): object=} getAnnounceOpts - Custom callback to allow sending extra parameters to the tracker
 * @property {boolean=} private - If true, client will not share the hash with the DHT nor with PEX (default is the privacy of the parsed torrent)
 * @property {'rarest'|'sequential'=} strategy - Piece selection strategy, `rarest` or `sequential`(defaut=`sequential`)
 * @property {number=} maxWebConns - Max number of simultaneous connections per web seed [default=4]
 * @property {number|false=} uploads - [default=10]
 * @property {number=} noPeersIntervalTime - The amount of time (in seconds) to wait between each check of the `noPeers` event (default=30)
 * @property {boolean=} deselect - If true, create the torrent with no pieces selected (default=false)
 * @property {boolean=} paused - If true, create the torrent in a paused state (default=false)
 * @property {Array<number>=} fileModtimes - An array containing a UNIX timestamp indicating the last change for each file of the torrent
 */

// End of JSDoc global declarations

const debug = debugFactory('webtorrent:torrent')
const MAX_BLOCK_LENGTH = 128 * 1024
const PIECE_TIMEOUT = 30_000
const CHOKE_TIMEOUT = 5_000
const SPEED_THRESHOLD = 3 * Piece.BLOCK_LENGTH

const PIPELINE_MIN_DURATION = 0.5
const PIPELINE_MAX_DURATION = 1

const RECHOKE_INTERVAL = 10_000 // 10 seconds
const RECHOKE_OPTIMISTIC_DURATION = 2 // 30 seconds

const DEFAULT_NO_PEERS_INTERVAL = 30_000 // 30 seconds

// IndexedDB chunk stores used in the browser benefit from high concurrency
const FILESYSTEM_CONCURRENCY = process.browser ? cpus().length : 2

const RECONNECT_WAIT = [1_000, 5_000, 15_000]

const USER_AGENT = `WebTorrent/${VERSION} (https://webtorrent.io)`

// if nodejs or browser that supports FSA
const SUPPORTS_FSA = globalThis.navigator?.storage?.getDirectory && globalThis.FileSystemFileHandle?.prototype?.createWritable
const FALLBACK_STORE = !process.browser || SUPPORTS_FSA
  ? FSChunkStore // Node or browser with FSA
  : MemoryChunkStore

let TMP
try {
  TMP = path.join(fs.statSync('/tmp') && '/tmp', 'webtorrent')
} catch (err) {
  TMP = path.join(typeof os.tmpdir === 'function' ? os.tmpdir() : '/', 'webtorrent')
}

const IDLE_CALLBACK = typeof window !== 'undefined' && typeof window.requestIdleCallback === 'function' && window.requestIdleCallback

export default class Torrent extends EventEmitter {
  /**
   * Start downloading a new torrent.
   * @param {string|ArrayBufferView|Object} torrentId
   * @param {import('../index.js').default} client
   * @param {TorrentOpts} opts
   */
  constructor (torrentId, client, opts) {
    super()

    this._debugId = 'unknown infohash'
    this.client = client

    this.announce = opts.announce
    this.urlList = opts.urlList

    this.path = opts.path || TMP
    this.addUID = opts.addUID || false
    this.rootDir = opts.rootDir || null
    this.skipVerify = !!opts.skipVerify
    this._startupBitfield = opts.bitfield
    this._store = opts.store || FALLBACK_STORE
    this._preloadedStore = opts.preloadedStore || null
    this._storeCacheSlots = opts.storeCacheSlots !== undefined ? opts.storeCacheSlots : 20
    this._destroyStoreOnDestroy = opts.destroyStoreOnDestroy || false
    this.store = null
    this.storeOpts = opts.storeOpts
    this.alwaysChokeSeeders = opts.alwaysChokeSeeders ?? true

    this._getAnnounceOpts = opts.getAnnounceOpts

    // if defined, `opts.private` overrides default privacy of torrent
    if (typeof opts.private === 'boolean') this.private = opts.private

    this.strategy = opts.strategy || 'sequential'

    this.maxWebConns = opts.maxWebConns || 4

    this._rechokeNumSlots = (opts.uploads === false || opts.uploads === 0)
      ? 0
      : (+opts.uploads || 10)
    this._rechokeOptimisticWire = null
    this._rechokeOptimisticTime = 0
    this._rechokeIntervalId = null
    this._noPeersIntervalId = null
    this._noPeersIntervalTime = opts.noPeersIntervalTime ? opts.noPeersIntervalTime * 1000 : DEFAULT_NO_PEERS_INTERVAL
    this._startAsDeselected = opts.deselect || false

    this.ready = false
    this.destroyed = false
    this.paused = opts.paused || false
    this.done = false

    this.metadata = null
    /**
     * Files of the torrent
     * @type {File[]}
     */
    this.files = []

    /**
     * Pieces that need to be downloaded, indexed by piece index
     * @type {Array<Piece|null>}
     */
    this.pieces = []

    this._amInterested = false
    this._selections = new Selections()
    this._critical = []
    /**
     * open wires (added *after* handshake)
     * @type {import('bittorrent-protocol').default[]}
     */
    this.wires = []

    this._queue = [] // queue of outgoing tcp peers to connect to
    /**
     * connected peers (addr/peerId -> Peer)
     * @type {Map<string, Peer>}
     */
    this._peers = new Map()
    this._peersLength = 0 // number of elements in `this._peers` (cache, for perf)

    // stats
    this.received = 0
    this.uploaded = 0
    this._downloadSpeed = throughput()
    this._uploadSpeed = throughput()

    // for cleanup
    this._servers = []
    this._xsRequests = []

    // TODO: remove this and expose a hook instead
    // optimization: don't recheck every file if it hasn't changed
    this._fileModtimes = opts.fileModtimes

    if (torrentId !== null) this._onTorrentId(torrentId)

    this._debug('new torrent')
  }

  get timeRemaining () {
    if (this.done) return 0
    if (this.downloadSpeed === 0) return Infinity
    return ((this.length - this.downloaded) / this.downloadSpeed) * 1000
  }

  get downloaded () {
    if (!this.bitfield) return 0
    let downloaded = 0
    for (let index = 0, len = this.pieces.length; index < len; ++index) {
      if (this.bitfield.get(index)) { // verified data
        downloaded += (index === len - 1) ? this.lastPieceLength : this.pieceLength
      } else { // "in progress" data
        const piece = this.pieces[index]
        downloaded += (piece.length - piece.missing)
      }
    }
    return downloaded
  }

  // TODO: re-enable this. The number of missing pieces. Used to implement 'end game' mode.
  // Object.defineProperty(Storage.prototype, 'numMissing', {
  //   get: function () {
  //     var self = this
  //     var numMissing = self.pieces.length
  //     for (var index = 0, len = self.pieces.length; index < len; index++) {
  //       numMissing -= self.bitfield.get(index)
  //     }
  //     return numMissing
  //   }
  // })

  get downloadSpeed () { return this._downloadSpeed() }

  get uploadSpeed () { return this._uploadSpeed() }

  get progress () { return this.length ? this.downloaded / this.length : 0 }

  get ratio () { return this.uploaded / (this.received || this.length) }

  get numPeers () { return this.wires.length }

  get torrentFileBlob () {
    if (!this.torrentFile) return null
    return new Blob([this.torrentFile], { type: 'application/x-bittorrent' })
  }

  get _numQueued () {
    return this._queue.length + (this._peersLength - this._numConns)
  }

  _numConns = 0

  /**
   * Parse a torrent from its magnet/torrent file/remote url and kickstart downloading it
   * @param {string|ArrayBufferView|Object} torrentId
   * @returns {Promise<void>}
   * @private
   */
  async _onTorrentId (torrentId) {
    if (this.destroyed) return

    let parsedTorrent
    try { parsedTorrent = await parseTorrent(torrentId) } catch (err) {}
    if (parsedTorrent) {
      // Attempt to set infoHash property synchronously
      this.infoHash = parsedTorrent.infoHash
      this._debugId = arr2hex(parsedTorrent.infoHash).substring(0, 7)
      queueMicrotask(() => {
        if (this.destroyed) return
        this._onParsedTorrent(parsedTorrent)
      })
    } else {
      // If torrentId failed to parse, it could be in a form that requires an async
      // operation, i.e. http/https link, filesystem path, or Blob.
      remote(torrentId, (err, parsedTorrent) => {
        if (this.destroyed) return
        if (err) return this._destroy(err)
        this._onParsedTorrent(parsedTorrent)
      })
    }
  }

  _onParsedTorrent (parsedTorrent) {
    if (this.destroyed) return

    this._processParsedTorrent(parsedTorrent)

    if (!this.infoHash) {
      return this._destroy(new Error('Malformed torrent data: No info hash'))
    }

    this._rechokeIntervalId = setInterval(() => {
      this._rechoke()
    }, RECHOKE_INTERVAL)
    if (this._rechokeIntervalId.unref) this._rechokeIntervalId.unref()

    // Private 'infoHash' event allows client.add to check for duplicate torrents and
    // destroy them before the normal 'infoHash' event is emitted. Prevents user
    // applications from needing to deal with duplicate 'infoHash' events.
    this.emit('_infoHash', this.infoHash)
    if (this.destroyed) return

    this.emit('infoHash', this.infoHash)
    if (this.destroyed) return // user might destroy torrent in event handler

    if (this.client.listening) {
      this._onListening()
    } else {
      this.client.once('listening', () => {
        this._onListening()
      })
    }
  }

  _processParsedTorrent (parsedTorrent) {
    this._debugId = arr2hex(parsedTorrent.infoHash).substring(0, 7)

    if (typeof this.private !== 'undefined') {
      // `private` option overrides default, only if it's defined
      parsedTorrent.private = this.private
    }

    if (Array.isArray(this.announce)) {
      // Allow specifying trackers via `opts` parameter
      parsedTorrent.announce = parsedTorrent.announce.concat(this.announce)
    }

    if (this.client.tracker && Array.isArray(this.client.tracker.announce) && !parsedTorrent.private) {
      // If the client has a default tracker, add it to the announce list if torrent is not private
      parsedTorrent.announce = parsedTorrent.announce.concat(this.client.tracker.announce)
    }

    if (this.client.tracker && global.WEBTORRENT_ANNOUNCE && !parsedTorrent.private) {
      // So `webtorrent-hybrid` can force specific trackers to be used
      parsedTorrent.announce = parsedTorrent.announce.concat(global.WEBTORRENT_ANNOUNCE)
    }

    if (this.urlList) {
      // Allow specifying web seeds via `opts` parameter
      parsedTorrent.urlList = parsedTorrent.urlList.concat(this.urlList)
    }

    // remove duplicates by converting to Set and back
    parsedTorrent.announce = Array.from(new Set(parsedTorrent.announce))
    parsedTorrent.urlList = Array.from(new Set(parsedTorrent.urlList))

    Object.assign(this, parsedTorrent)

    this.magnetURI = toMagnetURI(parsedTorrent)
    this.torrentFile = toTorrentFile(parsedTorrent)
  }

  _onListening () {
    if (this.destroyed) return

    if (this.info) {
      // if full metadata was included in initial torrent id, use it immediately. Otherwise,
      // wait for torrent-discovery to find peers and ut_metadata to get the metadata.
      this._onMetadata(this)
    } else {
      if (this.xs) this._getMetadataFromServer()
      this._startDiscovery()
    }
  }

  _startDiscovery () {
    if (this.discovery || this.destroyed) return

    let trackerOpts = this.client.tracker
    if (trackerOpts) {
      trackerOpts = Object.assign({}, this.client.tracker, {
        getAnnounceOpts: () => {
          if (this.destroyed) return

          const opts = {
            uploaded: this.uploaded,
            downloaded: this.downloaded,
            left: Math.max(this.length - this.downloaded, 0)
          }
          if (this.client.tracker.getAnnounceOpts) {
            Object.assign(opts, this.client.tracker.getAnnounceOpts())
          }
          if (this._getAnnounceOpts) {
            // TODO: consider deprecating this, as it's redundant with the former case
            Object.assign(opts, this._getAnnounceOpts())
          }
          return opts
        }
      })
    }

    // add BEP09 peer-address
    if (this.peerAddresses) {
      this.peerAddresses.forEach(peer => this.addPeer(peer, Peer.SOURCE_MANUAL))
    }

    // begin discovering peers via DHT and trackers
    this.discovery = new Discovery({
      infoHash: this.infoHash,
      announce: this.announce,
      peerId: this.client.peerId,
      dht: !this.private && this.client.dht,
      tracker: trackerOpts,
      port: this.client.torrentPort,
      userAgent: USER_AGENT,
      lsd: this.client.lsd
    })

    this.discovery.on('error', (err) => {
      this._destroy(err)
    })

    this.discovery.on('peer', (peer, source) => {
      this._debug('peer %s discovered via %s', peer, source)
      // Don't create new outgoing connections when torrent is done and seedOutgoingConnections is false.
      if (!this.client.seedOutgoingConnections && this.done) {
        this._debug('discovery ignoring peer %s: torrent is done and seedOutgoingConnections is false', peer)
        return
      }
      this.addPeer(peer, source)
    })

    this.discovery.on('trackerAnnounce', () => {
      this.emit('trackerAnnounce')
    })

    this.discovery.on('dhtAnnounce', () => {
      this.emit('dhtAnnounce')
    })

    this.discovery.on('warning', (err) => {
      this.emit('warning', err)
    })

    this._noPeersIntervalId = setInterval(() => {
      if (this.destroyed) return

      const counters = {
        [Peer.SOURCE_TRACKER]: {
          enabled: !!this.client.tracker,
          numPeers: 0
        },
        [Peer.SOURCE_DHT]: {
          enabled: !!this.client.dht,
          numPeers: 0
        },
        [Peer.SOURCE_LSD]: {
          enabled: !!this.client.lsd,
          numPeers: 0
        },
        [Peer.SOURCE_UT_PEX]: {
          enabled: (this.client.utPex && typeof utPex === 'function'),
          numPeers: 0
        }
      }
      for (const peer of this._peers.values()) {
        const counter = counters[peer.source]
        if (typeof counter !== 'undefined') counter.numPeers++
      }
      for (const source of Object.keys(counters)) {
        const counter = counters[source]
        if (counter.enabled && counter.numPeers === 0) this.emit('noPeers', source)
      }
    }, this._noPeersIntervalTime)
    if (this._noPeersIntervalId.unref) this._noPeersIntervalId.unref()
  }

  _getMetadataFromServer () {
    // to allow function hoisting
    const self = this

    const urls = Array.isArray(this.xs) ? this.xs : [this.xs]

    self._xsRequestsController = new AbortController()

    const signal = self._xsRequestsController.signal

    const tasks = urls.map(url => cb => {
      getMetadataFromURL(url, cb)
    })
    parallel(tasks)

    async function getMetadataFromURL (url, cb) {
      if (url.indexOf('http://') !== 0 && url.indexOf('https://') !== 0) {
        self.emit('warning', new Error(`skipping non-http xs param: ${url}`))
        return cb(null)
      }

      const opts = {
        method: 'GET',
        headers: {
          'user-agent': USER_AGENT
        },
        signal
      }
      let res
      try {
        res = await fetch(url, opts)
      } catch (err) {
        self.emit('warning', new Error(`http error from xs param: ${url}`))
        return cb(null)
      }

      if (self.destroyed) return cb(null)
      if (self.metadata) return cb(null)

      if (res.status !== 200) {
        self.emit('warning', new Error(`non-200 status code ${res.status} from xs param: ${url}`))
        return cb(null)
      }
      let torrent
      try {
        torrent = new Uint8Array(await res.arrayBuffer())
      } catch (e) {
        self.emit('warning', e)
        return cb(null)
      }

      let parsedTorrent
      try {
        parsedTorrent = await parseTorrent(torrent)
      } catch (err) {}

      if (!parsedTorrent) {
        self.emit('warning', new Error(`got invalid torrent file from xs param: ${url}`))
        return cb(null)
      }

      if (parsedTorrent.infoHash !== self.infoHash) {
        self.emit('warning', new Error(`got torrent file with incorrect info hash from xs param: ${url}`))
        return cb(null)
      }
      self._onMetadata(parsedTorrent)
      cb(null)
    }
  }

  /**
   * Called when the full torrent metadata is received.
   */
  async _onMetadata (metadata) {
    if (this.metadata || this.destroyed) return
    this._debug('got metadata')

    this._xsRequestsController?.abort()
    this._xsRequestsController = null

    let parsedTorrent
    if (metadata && metadata.infoHash) {
      // `metadata` is a parsed torrent (from parse-torrent module)
      parsedTorrent = metadata
    } else {
      try {
        parsedTorrent = await parseTorrent(metadata)
      } catch (err) {
        return this._destroy(err)
      }
    }

    this._processParsedTorrent(parsedTorrent)
    this.metadata = this.torrentFile

    // add web seed urls (BEP19)
    if (this.client.enableWebSeeds) {
      this.urlList.forEach(url => {
        this.addWebSeed(url)
      })
    }

    this._rarityMap = new RarityMap(this)

    this.files = this.files.map(file => new File(this, file))

    let rawStore = this._preloadedStore
    if (!rawStore) {
      rawStore = new this._store(this.pieceLength, {
        ...this.storeOpts,
        torrent: this,
        path: this.path,
        files: this.files,
        length: this.length,
        name: this.name + ' - ' + this.infoHash.slice(0, 8),
        addUID: this.addUID,
        rootDir: this.rootDir,
        max: this._storeCacheSlots
      })
    }

    // don't use the cache if the store is already in memory
    if (this._storeCacheSlots > 0 && !(rawStore instanceof MemoryChunkStore)) {
      rawStore = new CacheChunkStore(rawStore, {
        max: this._storeCacheSlots
      })
    }

    this.store = new ImmediateChunkStore(
      rawStore
    )

    // Select only specified files (BEP53) http://www.bittorrent.org/beps/bep_0053.html
    if (this.so && !this._startAsDeselected) {
      this.files.forEach((v, i) => {
        if (this.so.includes(i)) {
          this.files[i].select()
        }
      })
    } else {
      // start off selecting the entire torrent with low priority
      if (this.pieces.length !== 0 && !this._startAsDeselected) {
        this.select(0, this.pieces.length - 1)
      }
    }

    this._hashes = this.pieces
    // A startup bitfield can be used only when all the conditions are right:
    // - It exists
    // - It's the correct size ( rounded to the first byte )
    // - It will not be rewritten by _markAllVerified
    this._hasStartupBitfield = this._startupBitfield && this._startupBitfield.length === Math.ceil(this.pieces.length / 8) && !this.skipVerify

    this.bitfield = new BitField(this._hasStartupBitfield ? new Uint8Array(this._startupBitfield) : this.pieces.length)
    this._reservations = this._hasStartupBitfield
      ? this.pieces.map((_, index) => this.bitfield.get(index) ? null : [])
      : this.pieces.map(() => [])

    this.pieces = this.pieces.map((hash, i) => {
      if (this._hasStartupBitfield && this.bitfield.get(i)) {
        return null
      }
      const pieceLength = (i === this.pieces.length - 1)
        ? this.lastPieceLength
        : this.pieceLength
      return new Piece(pieceLength)
    })

    // Emit 'metadata' before 'ready' and 'done'
    this.emit('metadata')

    // User might destroy torrent in response to 'metadata' event
    if (this.destroyed) return

    if (this.skipVerify) {
      // Skip verifying exisitng data and just assume it's correct
      this._markAllVerified()
      this._onStore()
    } else {
      const onPiecesVerified = (err) => {
        if (err) return this._destroy(err)
        this._debug('done verifying')
        this._onStore()
      }

      this._debug('verifying existing torrent data')
      if (this._fileModtimes && this._store === FSChunkStore) {
        // don't verify if the files haven't been modified since we last checked
        this.getFileModtimes((err, fileModtimes) => {
          if (err) return this._destroy(err)

          const unchanged = this.files.map((_, index) => fileModtimes[index] === this._fileModtimes[index]).every(x => x)

          if (unchanged) {
            this._markAllVerified()
            this._onStore()
          } else {
            this._verifyPieces(onPiecesVerified)
          }
        })
      } else {
        this._verifyPieces(onPiecesVerified)
      }
    }
  }

  /*
   * TODO: remove this
   * Gets the last modified time of every file on disk for this torrent.
   * Only valid in Node, not in the browser.
   */
  getFileModtimes (cb) {
    const ret = []
    parallelLimit(this.files.map((file, index) => cb => {
      const filePath = this.addUID ? path.join(this.name + ' - ' + this.infoHash.slice(0, 8)) : path.join(this.path, file.path)
      fs.stat(filePath, (err, stat) => {
        if (err && err.code !== 'ENOENT') return cb(err)
        ret[index] = stat && stat.mtime.getTime()
        cb(null)
      })
    }), FILESYSTEM_CONCURRENCY, err => {
      this._debug('done getting file modtimes')
      cb(err, ret)
    })
  }

  /**
   * Callback called after a piece is verified
   * @callback verifyPieceCallback
   * @param {Error=} error
   * @param {boolean=} isValid
   */
  /**
   * Verify a single piece using hashing
   * @param index
   * @param {verifyPieceCallback} cb
   * @private
   */
  _verifyPiece (index, cb) {
    if (this.destroyed) return cb(new Error('torrent is destroyed'))

    const getOpts = {}
    // Specify length for the last piece in case it is zero-padded
    if (index === this.pieces.length - 1) {
      getOpts.length = this.lastPieceLength
    }
    this.store.get(index, getOpts, async (err, buf) => {
      if (this.destroyed) return cb(new Error('torrent is destroyed'))

      if (err) return queueMicrotask(() => cb(null, false)) // ignore error

      const hex = await hash(buf, 'hex')
      if (this.destroyed) return cb(new Error('torrent is destroyed'))

      cb(null, hex === this._hashes[index])
    })
  }

  /**
   * Verify pieces using bitfield, in case of in-congruences it will re-verify the file using hashing
   * @param {callbackWithError} cb
   * @private
   */
  _verifyPiecesUsingBitfield (cb) {
    const piecesToCheck = new Set()
    const piecesToFilesMap = new Map()
    // First step, optimistically mark what is verified and what is not by blindly trusting the bitfield
    // and construct a list of pieces to verify ( max 1 piece for each file, in some edge cases that piece could overlap )
    for (const file of this.files) {
      let checkFile = 2
      let pieceToCheckForThisFile = null
      for (let i = file._startPiece; i <= file._endPiece; ++i) {
        if (this.bitfield.get(i)) {
          if (checkFile) {
            pieceToCheckForThisFile = i
            checkFile--
          }
          if (!piecesToFilesMap.has(i)) {
            piecesToFilesMap.set(i, [])
          }
          piecesToFilesMap.get(i).push(file)
        }
      }
      if (pieceToCheckForThisFile !== null) {
        piecesToCheck.add(pieceToCheckForThisFile)
      }
    }
    // Second step, for each piece that needs to be verified we verify it using hashing
    this._verifyPiecesUsingHash([...piecesToCheck], (err) => {
      if (err) {
        return cb(err)
      }
      const filesToCheck = new Set()
      for (const piece of piecesToCheck) {
        if (!this.bitfield.get(piece)) {
          const filesOnPiece = piecesToFilesMap.get(piece)
          for (const file of filesOnPiece) {
            filesToCheck.add(file)
          }
        }
      }
      // Third step, if we need to recheck files we are going to fully recheck them
      if (filesToCheck.size) {
        const piecesToRecheck = []
        for (const file of filesToCheck) {
          for (let i = file._startPiece; i <= file._endPiece; ++i) {
            if (piecesToRecheck.indexOf(i) === -1) {
              piecesToRecheck.push(i)
            }
          }
        }
        return this._verifyPiecesUsingHash(piecesToRecheck, cb)
      }
      cb(null)
    })
  }

  /**
   * Verifies the pieces of the torrent using hashing ( it can be very slow )
   * @param {Array<number>} pieces
   * @param {callbackWithError} cb
   * @private
   */
  _verifyPiecesUsingHash (pieces, cb) {
    parallelLimit(pieces.map((piece, index) => cb => {
      const target = Number.isInteger(piece) ? piece : index
      this._verifyPiece(target, (err, isVerified) => {
        if (err) return cb(err)
        if (isVerified) {
          this._debug('piece verified %s', target)
          this._markVerified(target)
        } else {
          this._markUnverified(target)
          this._debug('piece invalid %s', target)
        }
        cb(null)
      })
    }), FILESYSTEM_CONCURRENCY, cb)
  }

  /**
   * Verifies the pieces of the torrent, if a startup bitfield is provided it will be used for verification
   * @param {callbackWithError} cb
   * @private
   */
  _verifyPieces (cb) {
    if (this._hasStartupBitfield) {
      return this._verifyPiecesUsingBitfield(cb)
    }
    this._verifyPiecesUsingHash(this.pieces, cb)
  }

  /**
   * Verify the hashes of all pieces in the store and update the bitfield for any new valid
   * pieces. Useful if data has been added to the store outside WebTorrent, e.g. if another
   * process puts a valid file in the right place. Once the scan is complete,
   * `callback(null)` will be called (if provided), unless the torrent was destroyed during
   * the scan, in which case `callback` will be called with an error.
   * @param  {callbackWithError=} cb
   */
  rescanFiles (cb) {
    if (this.destroyed) throw new Error('torrent is destroyed')
    if (!cb) cb = noop

    this._verifyPiecesUsingHash(this.pieces, (err) => {
      if (err) {
        this._destroy(err)
        return cb(err)
      }

      this._checkDone()
      cb(null)
    })
  }

  /**
   * Mark the entire torrent as verified ( i.e. fully downloaded )
   * @private
   */
  _markAllVerified () {
    for (let index = 0; index < this.pieces.length; index++) {
      this._markVerified(index)
    }
  }

  /**
   * Mark one piece as verified
   * @param {number} index
   * @private
   */
  _markVerified (index) {
    this.pieces[index] = null
    this._reservations[index] = null
    this.bitfield.set(index, true)
    this.emit('verified', index)
  }

  /**
   * Mark one piece as unverified
   * @param {number} index
   * @private
   */
  _markUnverified (index) {
    const len = (index === this.pieces.length - 1)
      ? this.lastPieceLength
      : this.pieceLength
    this.pieces[index] = new Piece(len)
    this.bitfield.set(index, false)
    if (!this._startAsDeselected) this.select(index, index)
    for (const file of this.files) {
      if (file.done && file.includes(index)) file.done = false
    }
  }

  _hasAllPieces () {
    for (let index = 0; index < this.pieces.length; index++) {
      if (!this.bitfield.get(index)) return false
    }
    return true
  }

  _hasNoPieces () {
    return !this._hasMorePieces(0)
  }

  _hasMorePieces (threshold) {
    let count = 0
    for (let index = 0; index < this.pieces.length; index++) {
      if (this.bitfield.get(index)) {
        count += 1
        if (count > threshold) return true
      }
    }
    return false
  }

  /**
   * Called when the metadata, listening server, and underlying chunk store is initialized.
   */
  _onStore () {
    if (this.destroyed) return
    this._debug('on store')

    // Start discovery before emitting 'ready'
    this._startDiscovery()

    this.ready = true
    this.emit('ready')

    // Files may start out done if the file was already in the store
    this._checkDone()

    // In case any selections were made before torrent was ready
    this._updateSelections()

    // Start requesting pieces after we have initially verified them
    this.wires.forEach(wire => {
      // If we didn't have the metadata at the time ut_metadata was initialized for this
      // wire, we still want to make it available to the peer in case they request it.
      if (wire.ut_metadata) wire.ut_metadata.setMetadata(this.metadata)

      this._onWireWithMetadata(wire)
    })
  }

  destroy (opts, cb) {
    if (typeof opts === 'function') return this.destroy(null, opts)

    this._destroy(null, opts, cb)
  }

  _destroy (err, opts, cb) {
    if (typeof opts === 'function') return this._destroy(err, null, opts)
    if (this.destroyed) return
    this.destroyed = true
    this._debug('destroy')

    this.client._remove(this)

    this._selections.clear()

    clearInterval(this._rechokeIntervalId)

    clearInterval(this._noPeersIntervalId)

    this._xsRequestsController?.abort()

    if (this._rarityMap) {
      this._rarityMap.destroy()
    }

    for (const id of this._peers.keys()) {
      this.removePeer(id)
    }

    for (const file of this.files) {
      if (file instanceof File) file._destroy()
    }

    const tasks = this._servers.map(server => cb => {
      server.destroy(cb)
    })

    if (this.discovery) {
      tasks.push(cb => {
        this.discovery.destroy(cb)
      })
    }

    if (this.store) {
      let destroyStore = this._destroyStoreOnDestroy
      if (opts && opts.destroyStore !== undefined) {
        destroyStore = opts.destroyStore
      }
      tasks.push(cb => {
        if (destroyStore) {
          this.store.destroy(cb)
        } else {
          this.store.close(cb)
        }
      })
    }

    parallel(tasks, cb)

    if (err) {
      // Torrent errors are emitted at `torrent.on('error')`. If there are no 'error'
      // event handlers on the torrent instance, then the error will be emitted at
      // `client.on('error')`. This prevents throwing an uncaught exception
      // (unhandled 'error' event), but it makes it impossible to distinguish client
      // errors versus torrent errors. Torrent errors are not fatal, and the client
      // is still usable afterwards. Therefore, always listen for errors in both
      // places (`client.on('error')` and `torrent.on('error')`).
      if (this.listenerCount('error') === 0) {
        this.client.emit('error', err)
      } else {
        this.emit('error', err)
      }
    }

    this.emit('close')

    this.client = null
    this.files = []
    this.discovery = null
    this.store = null
    this._rarityMap = null
    this._peers.clear()
    this._peers = null
    this._servers = null
    this._xsRequests = null
    this._queue = null
  }

  addPeer (peer, source) {
    if (this.destroyed) throw new Error('torrent is destroyed')
    if (!this.infoHash) throw new Error('addPeer() must not be called before the `infoHash` event')

    let host

    if (typeof peer === 'string') {
      let parts
      try {
        parts = addrToIPPort(peer)
      } catch (e) {
        this._debug('ignoring peer: invalid %s', peer)
        this.emit('invalidPeer', peer)
        return false
      }
      host = parts[0]
    } else if (typeof peer.remoteAddress === 'string') {
      host = peer.remoteAddress
    }

    if (this.client.blocked && host && this.client.blocked.contains(host)) {
      this._debug('ignoring peer: blocked %s', peer)
      if (typeof peer !== 'string') peer.destroy()
      this.emit('blockedPeer', peer)
      return false
    }

    // if the utp connection fails to connect, then it is replaced with a tcp connection to the same ip:port

    const type = (this.client.utp && this._isIPv4(host)) ? 'utp' : 'tcp'
    const wasAdded = !!this._addPeer(peer, type, source)

    if (wasAdded) {
      this.emit('peer', peer)
    } else {
      this.emit('invalidPeer', peer)
    }
    return wasAdded
  }

  _addPeer (peer, type, source) {
    if (this.destroyed) {
      if (typeof peer !== 'string') peer.destroy()
      return null
    }
    if (typeof peer === 'string' && !this._validAddr(peer)) {
      this._debug('ignoring peer: invalid %s', peer)
      return null
    }

    const id = (peer && peer.id) || peer
    if (this._peers.has(id)) {
      this._debug('ignoring peer: duplicate (%s)', id)
      if (typeof peer !== 'string') peer.destroy()
      return null
    }

    if (this.paused) {
      this._debug('ignoring peer: torrent is paused')
      if (typeof peer !== 'string') peer.destroy()
      return null
    }

    this._debug('add peer %s', id)

    let newPeer
    if (typeof peer === 'string') {
      // `peer` is an addr ("ip:port" string)
      newPeer = type === 'utp'
        ? Peer.createUTPOutgoingPeer(peer, this, this.client.throttleGroups, source)
        : Peer.createTCPOutgoingPeer(peer, this, this.client.throttleGroups, source)
    } else {
      // `peer` is a WebRTC connection (simple-peer)
      newPeer = Peer.createWebRTCPeer(peer, this, this.client.throttleGroups, source)
    }

    this._registerPeer(newPeer)

    if (typeof peer === 'string') {
      // `peer` is an addr ("ip:port" string)
      this._queue.push(newPeer)
      this._drain()
    }

    return newPeer
  }

  addWebSeed (urlOrConn) {
    if (this.destroyed) throw new Error('torrent is destroyed')

    let id
    let conn
    if (typeof urlOrConn === 'string') {
      id = urlOrConn

      if (!/^https?:\/\/.+/.test(id)) {
        this.emit('warning', new Error(`ignoring invalid web seed: ${id}`))
        this.emit('invalidPeer', id)
        return
      }

      if (this._peers.has(id)) {
        this.emit('warning', new Error(`ignoring duplicate web seed: ${id}`))
        this.emit('invalidPeer', id)
        return
      }

      conn = new WebConn(id, this)
    } else if (urlOrConn && typeof urlOrConn.connId === 'string') {
      conn = urlOrConn
      id = conn.connId

      if (this._peers.has(id)) {
        this.emit('warning', new Error(`ignoring duplicate web seed: ${id}`))
        this.emit('invalidPeer', id)
        return
      }
    } else {
      this.emit('warning', new Error('addWebSeed must be passed a string or connection object with id property'))
      return
    }

    this._debug('add web seed %s', id)

    const newPeer = Peer.createWebSeedPeer(conn, id, this, this.client.throttleGroups)

    this._registerPeer(newPeer)

    this.emit('peer', id)
  }

  /**
   * Called whenever a new incoming TCP peer connects to this torrent swarm. Called with a
   * peer that has already sent a handshake.
   */
  _addIncomingPeer (peer) {
    if (this.destroyed) return peer.destroy(new Error('torrent is destroyed'))
    if (this.paused) return peer.destroy(new Error('torrent is paused'))

    this._debug('add incoming peer %s', peer.id)

    this._registerPeer(peer)
  }

  /**
   * @param {Peer} newPeer
   */
  _registerPeer (newPeer) {
    newPeer.on('download', downloaded => {
      if (this.destroyed) return
      this.received += downloaded
      this._downloadSpeed(downloaded)
      this.client._downloadSpeed(downloaded)
      this.emit('download', downloaded)
      if (this.destroyed) return
      this.client.emit('download', downloaded)
    })

    newPeer.on('upload', uploaded => {
      if (this.destroyed) return
      this.uploaded += uploaded
      this._uploadSpeed(uploaded)
      this.client._uploadSpeed(uploaded)
      this.emit('upload', uploaded)
      if (this.destroyed) return
      this.client.emit('upload', uploaded)
    })

    if (newPeer.connected) {
      this._numConns += 1
    } else {
      newPeer.once('connect', () => {
        if (this.destroyed) return
        this._numConns += 1
      })
    }
    newPeer.once('disconnect', () => {
      this._numConns -= 1
    })

    this._peers.set(newPeer.id, newPeer)
    this._peersLength += 1
  }

  removePeer (peer) {
    const id = peer?.id || peer
    if (peer && !peer.id) peer = this._peers?.get(id)

    if (!peer) return
    peer.destroy()

    if (this.destroyed) return

    this._debug('removePeer %s', id)

    this._peers.delete(id)
    this._peersLength -= 1

    // If torrent swarm was at capacity before, try to open a new connection now
    this._drain()
  }

  _select (start = 0, end = this.pieces.length - 1, priority, notify, isStreamSelection = false) {
    if (this.destroyed) throw new Error('torrent is destroyed')

    if (start < 0 || end < start || this.pieces.length <= end) {
      throw new Error(`invalid selection ${start} : ${end}`)
    }
    priority = Number(priority) || 0

    this._debug('select %s-%s (priority %s)', start, end, priority)

    this._selections.insert({
      from: start,
      to: end,
      offset: 0,
      priority,
      notify,
      isStreamSelection
    })

    this._selections.sort((a, b) => b.priority - a.priority)

    this._updateSelections()
  }

  select (start, end, priority, notify) {
    this._select(start, end, priority, notify, false)
  }

  _deselect (from, to, isStreamSelection = false) {
    if (this.destroyed) throw new Error('torrent is destroyed')

    this._debug('deselect %s-%s', from, to)

    this._selections.remove({ from, to, isStreamSelection })

    this._updateSelections()
  }

  deselect (start, end) {
    this._deselect(start, end, false)
  }

  critical (start, end) {
    if (this.destroyed) throw new Error('torrent is destroyed')

    this._debug('critical %s-%s', start, end)

    for (let i = start; i <= end; ++i) {
      this._critical[i] = true
    }

    this._updateSelections()
  }

  _onWire (wire, addr) {
    this._debug('got wire %s (%s)', wire._debugId, addr || 'Unknown')

    this.wires.push(wire)

    if (addr) {
      // Sometimes RTCPeerConnection.getStats() doesn't return an ip:port for peers
      const parts = addrToIPPort(addr)
      wire.remoteAddress = parts[0]
      wire.remotePort = parts[1]
    }

    // When peer sends PORT message, add that DHT node to routing table
    if (this.client.dht && this.client.dht.listening) {
      wire.on('port', port => {
        if (this.destroyed || this.client.dht.destroyed) {
          return
        }
        if (!wire.remoteAddress) {
          return this._debug('ignoring PORT from peer with no address')
        }
        if (port === 0 || port > 65536) {
          return this._debug('ignoring invalid PORT from peer')
        }

        this._debug('port: %s (from %s)', port, addr)
        this.client.dht.addNode({ host: wire.remoteAddress, port })
      })
    }

    wire.on('timeout', () => {
      this._debug('wire timeout (%s)', addr)
      // TODO: this might be destroying wires too eagerly
      wire.destroy()
    })

    // Timeout for piece requests to this peer
    if (wire.type !== 'webSeed') { // webseeds always send 'unhave' on http timeout
      wire.setTimeout(PIECE_TIMEOUT, true)
    }

    // Send KEEP-ALIVE (every 60s) so peers will not disconnect the wire
    wire.setKeepAlive(true)

    // use ut_metadata extension
    wire.use(utMetadata(this.metadata))

    wire.ut_metadata.on('warning', err => {
      this._debug('ut_metadata warning: %s', err.message)
    })

    if (!this.metadata) {
      wire.ut_metadata.on('metadata', metadata => {
        this._debug('got metadata via ut_metadata')
        this._onMetadata(metadata)
      })
      wire.ut_metadata.fetch()
    }

    // use ut_pex extension if the torrent is not flagged as private
    if (this.client.utPex && typeof utPex === 'function' && !this.private) {
      wire.use(utPex())

      wire.ut_pex.on('peer', peer => {
        // Only add potential new peers when torrent is done and seedOutgoingConnections is false.
        if (!this.client.seedOutgoingConnections && this.done) {
          this._debug('ut_pex ignoring peer %s: torrent is done and seedOutgoingConnections is false', peer)
          return
        }
        this._debug('ut_pex: got peer: %s (from %s)', peer, addr)
        this.addPeer(peer, Peer.SOURCE_UT_PEX)
      })

      wire.ut_pex.on('dropped', peer => {
        // the remote peer believes a given peer has been dropped from the torrent swarm.
        // if we're not currently connected to it, then remove it from the queue.
        const peerObj = this._peers.get(peer)
        if (peerObj && !peerObj.connected) {
          this._debug('ut_pex: dropped peer: %s (from %s)', peer, addr)
          this.removePeer(peer)
        }
      })

      wire.once('close', () => {
        // Stop sending updates to remote peer
        wire.ut_pex.reset()
      })
    }

    wire.use(ltDontHave())

    // Hook to allow user-defined `bittorrent-protocol` extensions
    // More info: https://github.com/webtorrent/bittorrent-protocol#extension-api
    this.emit('wire', wire, addr)

    if (this.ready) {
      queueMicrotask(() => {
        // This allows wire.handshake() to be called (by Peer.onHandshake) before any
        // messages get sent on the wire
        this._onWireWithMetadata(wire)
      })
    }
  }

  /**
   * @param {import('bittorrent-protocol').default} wire
   */
  _onWireWithMetadata (wire) {
    let timeoutId = null

    const onChokeTimeout = () => {
      if (this.destroyed || wire.destroyed) return

      if (this._numQueued > 2 * (this._numConns - this.numPeers) &&
        wire.amInterested) {
        wire.destroy()
      } else {
        timeoutId = setTimeout(onChokeTimeout, CHOKE_TIMEOUT)
        if (timeoutId.unref) timeoutId.unref()
      }
    }

    let i
    const updateSeedStatus = () => {
      // bittorrent-protocol will set use fake bitfield if it gets a have-all message, which means its a seeder
      if (wire.peerPieces.buffer.length && !!wire.peerPieces.setAll) {
        if (wire.peerPieces.buffer.length !== this.bitfield.buffer.length) return
        for (i = 0; i < this.pieces.length; ++i) {
          if (!wire.peerPieces.get(i)) return
        }
      }
      wire.isSeeder = true
      if (this.alwaysChokeSeeders) wire.choke() // always choke seeders
    }

    wire.on('bitfield', () => {
      updateSeedStatus()
      this._update()
      this._updateWireInterest(wire)
    })

    wire.on('have', () => {
      updateSeedStatus()
      this._update()
      this._updateWireInterest(wire)
    })

    wire.lt_donthave.on('donthave', () => {
      updateSeedStatus()
      this._update()
      this._updateWireInterest(wire)
    })

    // fast extension (BEP6)
    wire.on('have-all', () => {
      wire.isSeeder = true
      if (this.alwaysChokeSeeders) wire.choke() // always choke seeders
      this._update()
      this._updateWireInterest(wire)
    })

    // fast extension (BEP6)
    wire.on('have-none', () => {
      wire.isSeeder = false
      this._update()
      this._updateWireInterest(wire)
    })

    // fast extension (BEP6)
    wire.on('allowed-fast', (index) => {
      this._update()
    })

    wire.once('interested', () => {
      wire.unchoke()
    })

    wire.once('close', () => {
      clearTimeout(timeoutId)
    })

    wire.on('choke', () => {
      clearTimeout(timeoutId)
      timeoutId = setTimeout(onChokeTimeout, CHOKE_TIMEOUT)
      if (timeoutId.unref) timeoutId.unref()
    })

    wire.on('unchoke', () => {
      clearTimeout(timeoutId)
      this._update()
    })

    wire.on('request', (index, offset, length, cb) => {
      if (length > MAX_BLOCK_LENGTH) {
        // Per spec, disconnect from peers that request >128KB
        return wire.destroy()
      }
      if (this.pieces[index]) return
      this.store.get(index, { offset, length }, cb)
    })

    // always send bitfield or equivalent fast extension message (required)
    if (wire.hasFast && this._hasAllPieces()) wire.haveAll()
    else if (wire.hasFast && this._hasNoPieces()) wire.haveNone()
    else wire.bitfield(this.bitfield)

    // initialize interest in case bitfield message was already received before above handler was registered
    this._updateWireInterest(wire)

    // Send PORT message to peers that support DHT
    if (wire.peerExtensions.dht && this.client.dht && this.client.dht.listening) {
      wire.port(this.client.dht.address().port)
    }

    if (wire.type !== 'webSeed') { // do not choke on webseeds
      timeoutId = setTimeout(onChokeTimeout, CHOKE_TIMEOUT)
      if (timeoutId.unref) timeoutId.unref()
    }

    wire.isSeeder = false
    updateSeedStatus()
  }

  /**
   * Called on selection changes.
   */
  _updateSelections () {
    if (!this.ready || this.destroyed) return

    queueMicrotask(() => {
      this._gcSelections()
    })
    this._updateInterest()
    this._update()
  }

  /**
   * Garbage collect selections with respect to the store's current state.
   */
  _gcSelections () {
    for (const s of this._selections) {
      const oldOffset = s.offset

      // check for newly downloaded pieces in selection
      while (this.bitfield.get(s.from + s.offset) && s.from + s.offset < s.to) {
        s.offset += 1
      }

      if (oldOffset !== s.offset) s.notify?.()
      if (s.to !== s.from + s.offset) continue
      if (!this.bitfield.get(s.from + s.offset)) continue

      s.remove() // remove fully downloaded selection
      s.notify?.()
      this._updateInterest()
    }

    if (!this._selections.length) this.emit('idle')
  }

  /**
   * Update interested status for all peers.
   */
  _updateInterest () {
    const prev = this._amInterested
    this._amInterested = !!this._selections.length

    this.wires.forEach(wire => this._updateWireInterest(wire))

    if (prev === this._amInterested) return
    if (this._amInterested) this.emit('interested')
    else this.emit('uninterested')
  }

  _updateWireInterest (wire) {
    let interested = false
    for (let index = 0; index < this.pieces.length; ++index) {
      if (this.pieces[index] && wire.peerPieces.get(index)) {
        interested = true
        break
      }
    }

    if (interested) wire.interested()
    else wire.uninterested()
  }

  /**
   * Heartbeat to update all peers and their requests.
   */
  _update () {
    if (IDLE_CALLBACK) {
      IDLE_CALLBACK(() => this._updateWireWrapper(), { timeout: 250 })
    } else {
      this._updateWireWrapper()
    }
  }

  _updateWireWrapper () {
    if (this.destroyed) return
    // update wires in random order for better request distribution
    const ite = randomIterate(this.wires)
    let wire
    while ((wire = ite())) {
      this._updateWire(wire)
    }
  }

  /**
   * Attempts to update a peer's requests
   * @param {import('bittorrent-protocol').default} wire
   */
  _updateWire (wire) {
    if (wire.destroyed) return false
    // to allow function hoisting
    const self = this

    const minOutstandingRequests = getBlockPipelineLength(wire, PIPELINE_MIN_DURATION)
    if (wire.requests.length >= minOutstandingRequests) return
    const maxOutstandingRequests = getBlockPipelineLength(wire, PIPELINE_MAX_DURATION)

    if (wire.peerChoking) {
      if (wire.hasFast && wire.peerAllowedFastSet.length > 0 &&
        !this._hasMorePieces(wire.peerAllowedFastSet.length - 1)) {
        requestAllowedFastS