diff --git a/.github/workflows/nodejs.yml b/.github/workflows/nodejs.yml index 4c3a77ec1e3..f75295fea02 100644 --- a/.github/workflows/nodejs.yml +++ b/.github/workflows/nodejs.yml @@ -23,14 +23,6 @@ jobs: post-test-steps: | - name: Coverage Report uses: codecov/codecov-action@v3 - include: | - - runs-on: ubuntu-latest - node-version: 16.8 - exclude: | - - runs-on: windows-latest - node-version: 14 - - runs-on: windows-latest - node-version: 16 automerge: if: > github.event_name == 'pull_request' && github.event.pull_request.user.login == 'dependabot[bot]' diff --git a/README.md b/README.md index 3ba89890df6..cec6f032e99 100644 --- a/README.md +++ b/README.md @@ -180,8 +180,6 @@ Implements [fetch](https://fetch.spec.whatwg.org/#fetch-method). * https://developer.mozilla.org/en-US/docs/Web/API/WindowOrWorkerGlobalScope/fetch * https://fetch.spec.whatwg.org/#fetch-method -Only supported on Node 16.8+. - Basic usage example: ```js diff --git a/benchmarks/benchmark-http2.js b/benchmarks/benchmark-http2.js index d8555de22b5..af7b3cdbd86 100644 --- a/benchmarks/benchmark-http2.js +++ b/benchmarks/benchmark-http2.js @@ -7,7 +7,6 @@ const path = require('path') const { readFileSync } = require('fs') const { table } = require('table') const { Writable } = require('stream') -const { WritableStream } = require('stream/web') const { isMainThread } = require('worker_threads') const { Pool, Client, fetch, Agent, setGlobalDispatcher } = require('..') diff --git a/benchmarks/benchmark-https.js b/benchmarks/benchmark-https.js index a364f0a0c43..b2ec951ddbc 100644 --- a/benchmarks/benchmark-https.js +++ b/benchmarks/benchmark-https.js @@ -6,7 +6,6 @@ const path = require('path') const { readFileSync } = require('fs') const { table } = require('table') const { Writable } = require('stream') -const { WritableStream } = require('stream/web') const { isMainThread } = require('worker_threads') const { Pool, Client, fetch, Agent, setGlobalDispatcher } = require('..') @@ -15,7 +14,7 @@ const ca = readFileSync(path.join(__dirname, '..', 'test', 'fixtures', 'ca.pem') const servername = 'agent1' const iterations = (parseInt(process.env.SAMPLES, 10) || 10) + 1 -const errorThreshold = parseInt(process.env.ERROR_TRESHOLD, 10) || 3 +const errorThreshold = parseInt(process.env.ERROR_THRESHOLD, 10) || 3 const connections = parseInt(process.env.CONNECTIONS, 10) || 50 const pipelining = parseInt(process.env.PIPELINING, 10) || 10 const parallelRequests = parseInt(process.env.PARALLEL, 10) || 100 diff --git a/benchmarks/benchmark.js b/benchmarks/benchmark.js index 5bf3d2ede4f..94ac1b45a81 100644 --- a/benchmarks/benchmark.js +++ b/benchmarks/benchmark.js @@ -5,13 +5,12 @@ const os = require('os') const path = require('path') const { table } = require('table') const { Writable } = require('stream') -const { WritableStream } = require('stream/web') const { isMainThread } = require('worker_threads') const { Pool, Client, fetch, Agent, setGlobalDispatcher } = require('..') const iterations = (parseInt(process.env.SAMPLES, 10) || 10) + 1 -const errorThreshold = parseInt(process.env.ERROR_TRESHOLD, 10) || 3 +const errorThreshold = parseInt(process.env.ERROR_THRESHOLD, 10) || 3 const connections = parseInt(process.env.CONNECTIONS, 10) || 50 const pipelining = parseInt(process.env.PIPELINING, 10) || 10 const parallelRequests = parseInt(process.env.PARALLEL, 10) || 100 diff --git a/docs/api/BalancedPool.md b/docs/api/BalancedPool.md index 290c734c022..183ef523185 100644 --- a/docs/api/BalancedPool.md +++ b/docs/api/BalancedPool.md @@ -50,7 +50,7 @@ Arguments: ### `BalancedPool.removeUpstream(upstream)` -Removes an upstream that was previously addded. +Removes an upstream that was previously added. ### `BalancedPool.close([callback])` diff --git a/index.js b/index.js index 26302cc8efa..66d60785613 100644 --- a/index.js +++ b/index.js @@ -98,58 +98,54 @@ function makeDispatcher (fn) { module.exports.setGlobalDispatcher = setGlobalDispatcher module.exports.getGlobalDispatcher = getGlobalDispatcher -if (util.nodeMajor > 16 || (util.nodeMajor === 16 && util.nodeMinor >= 8)) { - let fetchImpl = null - module.exports.fetch = async function fetch (resource) { - if (!fetchImpl) { - fetchImpl = require('./lib/fetch').fetch - } - - try { - return await fetchImpl(...arguments) - } catch (err) { - if (typeof err === 'object') { - Error.captureStackTrace(err, this) - } +let fetchImpl = null +module.exports.fetch = async function fetch (resource) { + if (!fetchImpl) { + fetchImpl = require('./lib/fetch').fetch + } - throw err + try { + return await fetchImpl(...arguments) + } catch (err) { + if (typeof err === 'object') { + Error.captureStackTrace(err, this) } + + throw err } - module.exports.Headers = require('./lib/fetch/headers').Headers - module.exports.Response = require('./lib/fetch/response').Response - module.exports.Request = require('./lib/fetch/request').Request - module.exports.FormData = require('./lib/fetch/formdata').FormData - module.exports.File = require('./lib/fetch/file').File - module.exports.FileReader = require('./lib/fileapi/filereader').FileReader +} +module.exports.Headers = require('./lib/fetch/headers').Headers +module.exports.Response = require('./lib/fetch/response').Response +module.exports.Request = require('./lib/fetch/request').Request +module.exports.FormData = require('./lib/fetch/formdata').FormData +module.exports.File = require('./lib/fetch/file').File +module.exports.FileReader = require('./lib/fileapi/filereader').FileReader - const { setGlobalOrigin, getGlobalOrigin } = require('./lib/fetch/global') +const { setGlobalOrigin, getGlobalOrigin } = require('./lib/fetch/global') - module.exports.setGlobalOrigin = setGlobalOrigin - module.exports.getGlobalOrigin = getGlobalOrigin +module.exports.setGlobalOrigin = setGlobalOrigin +module.exports.getGlobalOrigin = getGlobalOrigin - const { CacheStorage } = require('./lib/cache/cachestorage') - const { kConstruct } = require('./lib/cache/symbols') +const { CacheStorage } = require('./lib/cache/cachestorage') +const { kConstruct } = require('./lib/cache/symbols') - // Cache & CacheStorage are tightly coupled with fetch. Even if it may run - // in an older version of Node, it doesn't have any use without fetch. - module.exports.caches = new CacheStorage(kConstruct) -} +// Cache & CacheStorage are tightly coupled with fetch. Even if it may run +// in an older version of Node, it doesn't have any use without fetch. +module.exports.caches = new CacheStorage(kConstruct) -if (util.nodeMajor >= 16) { - const { deleteCookie, getCookies, getSetCookies, setCookie } = require('./lib/cookies') +const { deleteCookie, getCookies, getSetCookies, setCookie } = require('./lib/cookies') - module.exports.deleteCookie = deleteCookie - module.exports.getCookies = getCookies - module.exports.getSetCookies = getSetCookies - module.exports.setCookie = setCookie +module.exports.deleteCookie = deleteCookie +module.exports.getCookies = getCookies +module.exports.getSetCookies = getSetCookies +module.exports.setCookie = setCookie - const { parseMIMEType, serializeAMimeType } = require('./lib/fetch/dataURL') +const { parseMIMEType, serializeAMimeType } = require('./lib/fetch/dataURL') - module.exports.parseMIMEType = parseMIMEType - module.exports.serializeAMimeType = serializeAMimeType -} +module.exports.parseMIMEType = parseMIMEType +module.exports.serializeAMimeType = serializeAMimeType -if (util.nodeMajor >= 18 && hasCrypto) { +if (hasCrypto) { const { WebSocket } = require('./lib/websocket/websocket') module.exports.WebSocket = WebSocket diff --git a/lib/client.js b/lib/client.js index 22cb39039da..af16fd16d3e 100644 --- a/lib/client.js +++ b/lib/client.js @@ -249,7 +249,7 @@ class Client extends DispatcherBase { } if (maxConcurrentStreams != null && (typeof maxConcurrentStreams !== 'number' || maxConcurrentStreams < 1)) { - throw new InvalidArgumentError('maxConcurrentStreams must be a possitive integer, greater than 0') + throw new InvalidArgumentError('maxConcurrentStreams must be a positive integer, greater than 0') } if (typeof connect !== 'function') { @@ -296,7 +296,7 @@ class Client extends DispatcherBase { ? null : { // streams: null, // Fixed queue of streams - For future support of `push` - openStreams: 0, // Keep track of them to decide wether or not unref the session + openStreams: 0, // Keep track of them to decide whether or not unref the session maxConcurrentStreams: maxConcurrentStreams != null ? maxConcurrentStreams : 100 // Max peerConcurrentStreams for a Node h2 server } this[kHost] = `${this[kUrl].hostname}${this[kUrl].port ? `:${this[kUrl].port}` : ''}` @@ -1706,7 +1706,7 @@ function writeH2 (client, session, request) { } // https://tools.ietf.org/html/rfc7540#section-8.3 - // :path and :scheme headers must be omited when sending CONNECT + // :path and :scheme headers must be omitted when sending CONNECT headers[HTTP2_HEADER_PATH] = path headers[HTTP2_HEADER_SCHEME] = 'https' @@ -1833,7 +1833,7 @@ function writeH2 (client, session, request) { // }) // stream.on('push', headers => { - // // TODO(HTTP/2): Suppor push + // // TODO(HTTP/2): Support push // }) // stream.on('trailers', headers => { diff --git a/lib/compat/dispatcher-weakref.js b/lib/compat/dispatcher-weakref.js index 8cb99e21501..a2fd0020416 100644 --- a/lib/compat/dispatcher-weakref.js +++ b/lib/compat/dispatcher-weakref.js @@ -1,7 +1,5 @@ 'use strict' -/* istanbul ignore file: only for Node 12 */ - const { kConnected, kSize } = require('../core/symbols') class CompatWeakRef { @@ -41,8 +39,5 @@ module.exports = function () { FinalizationRegistry: CompatFinalizer } } - return { - WeakRef: global.WeakRef || CompatWeakRef, - FinalizationRegistry: global.FinalizationRegistry || CompatFinalizer - } + return { WeakRef, FinalizationRegistry } } diff --git a/lib/core/connect.js b/lib/core/connect.js index 33091173fa8..877956f7bee 100644 --- a/lib/core/connect.js +++ b/lib/core/connect.js @@ -183,7 +183,11 @@ function setupTimeout (onConnectTimeout, timeout) { } function onConnectTimeout (socket) { - util.destroy(socket, new ConnectTimeoutError()) + let message = 'Connect Timeout Error' + if (Array.isArray(socket.autoSelectFamilyAttemptedAddresses)) { + message = +` (attempted addresses: ${socket.autoSelectFamilyAttemptedAddresses.join(', ')})` + } + util.destroy(socket, new ConnectTimeoutError(message)) } module.exports = buildConnector diff --git a/lib/core/constants.js b/lib/core/constants.js new file mode 100644 index 00000000000..0f827cc4ae0 --- /dev/null +++ b/lib/core/constants.js @@ -0,0 +1,116 @@ +/** @type {Record} */ +const headerNameLowerCasedRecord = {} + +// https://developer.mozilla.org/docs/Web/HTTP/Headers +const wellknownHeaderNames = [ + 'Accept', + 'Accept-Encoding', + 'Accept-Language', + 'Accept-Ranges', + 'Access-Control-Allow-Credentials', + 'Access-Control-Allow-Headers', + 'Access-Control-Allow-Methods', + 'Access-Control-Allow-Origin', + 'Access-Control-Expose-Headers', + 'Access-Control-Max-Age', + 'Access-Control-Request-Headers', + 'Access-Control-Request-Method', + 'Age', + 'Allow', + 'Alt-Svc', + 'Alt-Used', + 'Authorization', + 'Cache-Control', + 'Clear-Site-Data', + 'Connection', + 'Content-Disposition', + 'Content-Encoding', + 'Content-Language', + 'Content-Length', + 'Content-Location', + 'Content-Range', + 'Content-Security-Policy', + 'Content-Security-Policy-Report-Only', + 'Content-Type', + 'Cookie', + 'Cross-Origin-Embedder-Policy', + 'Cross-Origin-Opener-Policy', + 'Cross-Origin-Resource-Policy', + 'Date', + 'Device-Memory', + 'Downlink', + 'ECT', + 'ETag', + 'Expect', + 'Expect-CT', + 'Expires', + 'Forwarded', + 'From', + 'Host', + 'If-Match', + 'If-Modified-Since', + 'If-None-Match', + 'If-Range', + 'If-Unmodified-Since', + 'Keep-Alive', + 'Last-Modified', + 'Link', + 'Location', + 'Max-Forwards', + 'Origin', + 'Permissions-Policy', + 'Pragma', + 'Proxy-Authenticate', + 'Proxy-Authorization', + 'RTT', + 'Range', + 'Referer', + 'Referrer-Policy', + 'Refresh', + 'Retry-After', + 'Sec-WebSocket-Accept', + 'Sec-WebSocket-Extensions', + 'Sec-WebSocket-Key', + 'Sec-WebSocket-Protocol', + 'Sec-WebSocket-Version', + 'Server', + 'Server-Timing', + 'Service-Worker-Allowed', + 'Service-Worker-Navigation-Preload', + 'Set-Cookie', + 'SourceMap', + 'Strict-Transport-Security', + 'Supports-Loading-Mode', + 'TE', + 'Timing-Allow-Origin', + 'Trailer', + 'Transfer-Encoding', + 'Upgrade', + 'Upgrade-Insecure-Requests', + 'User-Agent', + 'Vary', + 'Via', + 'WWW-Authenticate', + 'X-Content-Type-Options', + 'X-DNS-Prefetch-Control', + 'X-Frame-Options', + 'X-Permitted-Cross-Domain-Policies', + 'X-Powered-By', + 'X-Requested-With', + 'X-XSS-Protection' +] + +for (let i = 0; i < wellknownHeaderNames.length; ++i) { + const key = wellknownHeaderNames[i] + const lowerCasedKey = key.toLowerCase() + headerNameLowerCasedRecord[key] = headerNameLowerCasedRecord[lowerCasedKey] = + lowerCasedKey +} + +// Note: object prototypes should not be able to be referenced. e.g. `Object#hasOwnProperty`. +Object.setPrototypeOf(headerNameLowerCasedRecord, null) + +module.exports = { + wellknownHeaderNames, + headerNameLowerCasedRecord +} diff --git a/lib/core/request.js b/lib/core/request.js index 3697e6a3acc..b0bd870e33a 100644 --- a/lib/core/request.js +++ b/lib/core/request.js @@ -196,10 +196,6 @@ class Request { } if (util.isFormDataLike(this.body)) { - if (util.nodeMajor < 16 || (util.nodeMajor === 16 && util.nodeMinor < 8)) { - throw new InvalidArgumentError('Form-Data bodies are only supported in node v16.8 and newer.') - } - if (!extractBody) { extractBody = require('../fetch/body.js').extractBody } diff --git a/lib/core/util.js b/lib/core/util.js index 8d5450ba0c0..5a28529b663 100644 --- a/lib/core/util.js +++ b/lib/core/util.js @@ -9,6 +9,7 @@ const { InvalidArgumentError } = require('./errors') const { Blob } = require('buffer') const nodeUtil = require('util') const { stringify } = require('querystring') +const { headerNameLowerCasedRecord } = require('./constants') const [nodeMajor, nodeMinor] = process.versions.node.split('.').map(v => Number(v)) @@ -223,19 +224,21 @@ function parseHeaders (headers, obj = {}) { if (!Array.isArray(headers)) return headers for (let i = 0; i < headers.length; i += 2) { - const key = headers[i].toString().toLowerCase() - let val = obj[key] + const key = headers[i].toString() + const lowerCasedKey = headerNameLowerCasedRecord[key] ?? key.toLowerCase() + let val = obj[lowerCasedKey] if (!val) { - if (Array.isArray(headers[i + 1])) { - obj[key] = headers[i + 1].map(x => x.toString('utf8')) + const headersValue = headers[i + 1] + if (typeof headersValue === 'string') { + obj[lowerCasedKey] = headersValue } else { - obj[key] = headers[i + 1].toString('utf8') + obj[lowerCasedKey] = Array.isArray(headersValue) ? headersValue.map(x => x.toString('utf8')) : headersValue.toString('utf8') } } else { if (!Array.isArray(val)) { val = [val] - obj[key] = val + obj[lowerCasedKey] = val } val.push(headers[i + 1].toString('utf8')) } @@ -359,21 +362,9 @@ function getSocketInfo (socket) { } } -async function * convertIterableToBuffer (iterable) { - for await (const chunk of iterable) { - yield Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk) - } -} - -let ReadableStream +/** @type {globalThis['ReadableStream']} */ function ReadableStreamFrom (iterable) { - if (!ReadableStream) { - ReadableStream = require('stream/web').ReadableStream - } - - if (ReadableStream.from) { - return ReadableStream.from(convertIterableToBuffer(iterable)) - } + // We cannot use ReadableStream.from here because it does not return a byte stream. let iterator return new ReadableStream( @@ -386,18 +377,21 @@ function ReadableStreamFrom (iterable) { if (done) { queueMicrotask(() => { controller.close() + controller.byobRequest?.respond(0) }) } else { const buf = Buffer.isBuffer(value) ? value : Buffer.from(value) - controller.enqueue(new Uint8Array(buf)) + if (buf.byteLength) { + controller.enqueue(new Uint8Array(buf)) + } } return controller.desiredSize > 0 }, async cancel (reason) { await iterator.return() - } - }, - 0 + }, + type: 'bytes' + } ) } diff --git a/lib/fetch/body.js b/lib/fetch/body.js index fd8481b796d..2a8b38f7415 100644 --- a/lib/fetch/body.js +++ b/lib/fetch/body.js @@ -13,7 +13,6 @@ const { const { FormData } = require('./formdata') const { kState } = require('./symbols') const { webidl } = require('./webidl') -const { DOMException, structuredClone } = require('./constants') const { Blob, File: NativeFile } = require('buffer') const { kBodyUsed } = require('../core/symbols') const assert = require('assert') @@ -22,8 +21,6 @@ const { isUint8Array, isArrayBuffer } = require('util/types') const { File: UndiciFile } = require('./file') const { parseMIMEType, serializeAMimeType } = require('./dataURL') -let ReadableStream = globalThis.ReadableStream - /** @type {globalThis['File']} */ const File = NativeFile ?? UndiciFile const textEncoder = new TextEncoder() @@ -31,10 +28,6 @@ const textDecoder = new TextDecoder() // https://fetch.spec.whatwg.org/#concept-bodyinit-extract function extractBody (object, keepalive = false) { - if (!ReadableStream) { - ReadableStream = require('stream/web').ReadableStream - } - // 1. Let stream be null. let stream = null @@ -47,16 +40,19 @@ function extractBody (object, keepalive = false) { stream = object.stream() } else { // 4. Otherwise, set stream to a new ReadableStream object, and set - // up stream. + // up stream with byte reading support. stream = new ReadableStream({ async pull (controller) { - controller.enqueue( - typeof source === 'string' ? textEncoder.encode(source) : source - ) + const buffer = typeof source === 'string' ? textEncoder.encode(source) : source + + if (buffer.byteLength) { + controller.enqueue(buffer) + } + queueMicrotask(() => readableStreamClose(controller)) }, start () {}, - type: undefined + type: 'bytes' }) } @@ -223,13 +219,17 @@ function extractBody (object, keepalive = false) { // When running action is done, close stream. queueMicrotask(() => { controller.close() + controller.byobRequest?.respond(0) }) } else { // Whenever one or more bytes are available and stream is not errored, // enqueue a Uint8Array wrapping an ArrayBuffer containing the available // bytes into stream. if (!isErrored(stream)) { - controller.enqueue(new Uint8Array(value)) + const buffer = new Uint8Array(value) + if (buffer.byteLength) { + controller.enqueue(buffer) + } } } return controller.desiredSize > 0 @@ -237,7 +237,7 @@ function extractBody (object, keepalive = false) { async cancel (reason) { await iterator.return() }, - type: undefined + type: 'bytes' }) } @@ -251,11 +251,6 @@ function extractBody (object, keepalive = false) { // https://fetch.spec.whatwg.org/#bodyinit-safely-extract function safelyExtractBody (object, keepalive = false) { - if (!ReadableStream) { - // istanbul ignore next - ReadableStream = require('stream/web').ReadableStream - } - // To safely extract a body and a `Content-Type` value from // a byte sequence or BodyInit object object, run these steps: diff --git a/lib/fetch/constants.js b/lib/fetch/constants.js index 218fcbee4da..ada622feed5 100644 --- a/lib/fetch/constants.js +++ b/lib/fetch/constants.js @@ -1,7 +1,5 @@ 'use strict' -const { MessageChannel, receiveMessageOnPort } = require('worker_threads') - const corsSafeListedMethods = ['GET', 'HEAD', 'POST'] const corsSafeListedMethodsSet = new Set(corsSafeListedMethods) @@ -92,41 +90,7 @@ const subresource = [ ] const subresourceSet = new Set(subresource) -/** @type {globalThis['DOMException']} */ -const DOMException = globalThis.DOMException ?? (() => { - // DOMException was only made a global in Node v17.0.0, - // but fetch supports >= v16.8. - try { - atob('~') - } catch (err) { - return Object.getPrototypeOf(err).constructor - } -})() - -let channel - -/** @type {globalThis['structuredClone']} */ -const structuredClone = - globalThis.structuredClone ?? - // https://github.com/nodejs/node/blob/b27ae24dcc4251bad726d9d84baf678d1f707fed/lib/internal/structured_clone.js - // structuredClone was added in v17.0.0, but fetch supports v16.8 - function structuredClone (value, options = undefined) { - if (arguments.length === 0) { - throw new TypeError('missing argument') - } - - if (!channel) { - channel = new MessageChannel() - } - channel.port1.unref() - channel.port2.unref() - channel.port1.postMessage(value, options?.transfer) - return receiveMessageOnPort(channel.port2).message - } - module.exports = { - DOMException, - structuredClone, subresource, forbiddenMethods, requestBodyHeader, diff --git a/lib/fetch/dataURL.js b/lib/fetch/dataURL.js index 7b6a606106d..5d6b3940526 100644 --- a/lib/fetch/dataURL.js +++ b/lib/fetch/dataURL.js @@ -126,7 +126,13 @@ function URLSerializer (url, excludeFragment = false) { const href = url.href const hashLength = url.hash.length - return hashLength === 0 ? href : href.substring(0, href.length - hashLength) + const serialized = hashLength === 0 ? href : href.substring(0, href.length - hashLength) + + if (!hashLength && href.endsWith('#')) { + return serialized.slice(0, -1) + } + + return serialized } // https://infra.spec.whatwg.org/#collect-a-sequence-of-code-points @@ -543,7 +549,7 @@ function serializeAMimeType (mimeType) { // 4. If value does not solely contain HTTP token code // points or value is the empty string, then: if (!HTTP_TOKEN_CODEPOINTS.test(value)) { - // 1. Precede each occurence of U+0022 (") or + // 1. Precede each occurrence of U+0022 (") or // U+005C (\) in value with U+005C (\). value = value.replace(/(\\|")/g, '\\$1') diff --git a/lib/fetch/index.js b/lib/fetch/index.js index 17c3d87ea62..4471ee5d57c 100644 --- a/lib/fetch/index.js +++ b/lib/fetch/index.js @@ -40,25 +40,25 @@ const { isomorphicEncode, urlIsLocal, urlIsHttpHttpsScheme, - urlHasHttpsScheme + urlHasHttpsScheme, + simpleRangeHeaderValue, + buildContentRange } = require('./util') const { kState, kHeaders, kGuard, kRealm } = require('./symbols') const assert = require('assert') -const { safelyExtractBody } = require('./body') +const { safelyExtractBody, extractBody } = require('./body') const { redirectStatusSet, nullBodyStatus, safeMethodsSet, requestBodyHeader, - subresourceSet, - DOMException + subresourceSet } = require('./constants') -const { kHeadersList } = require('../core/symbols') +const { kHeadersList, kConstruct } = require('../core/symbols') const EE = require('events') const { Readable, pipeline } = require('stream') const { addAbortListener, isErrored, isReadable, nodeMajor, nodeMinor } = require('../core/util') -const { dataURLProcessor, serializeAMimeType } = require('./dataURL') -const { TransformStream } = require('stream/web') +const { dataURLProcessor, serializeAMimeType, parseMIMEType } = require('./dataURL') const { getGlobalDispatcher } = require('../global') const { webidl } = require('./webidl') const { STATUS_CODES } = require('http') @@ -66,7 +66,6 @@ const GET_OR_HEAD = ['GET', 'HEAD'] /** @type {import('buffer').resolveObjectURL} */ let resolveObjectURL -let ReadableStream = globalThis.ReadableStream class Fetch extends EE { constructor (dispatcher) { @@ -232,9 +231,10 @@ function fetch (input, init = {}) { // 4. Set responseObject to the result of creating a Response object, // given response, "immutable", and relevantRealm. - responseObject = new Response() + responseObject = new Response(kConstruct) responseObject[kState] = response responseObject[kRealm] = relevantRealm + responseObject[kHeaders] = new Headers(kConstruct) responseObject[kHeaders][kHeadersList] = response.headersList responseObject[kHeaders][kGuard] = 'immutable' responseObject[kHeaders][kRealm] = relevantRealm @@ -404,9 +404,9 @@ function fetching ({ // 5. Let timingInfo be a new fetch timing info whose start time and // post-redirect start time are the coarsened shared current time given // crossOriginIsolatedCapability. - const currenTime = coarsenedSharedCurrentTime(crossOriginIsolatedCapability) + const currentTime = coarsenedSharedCurrentTime(crossOriginIsolatedCapability) const timingInfo = createOpaqueTimingInfo({ - startTime: currenTime + startTime: currentTime }) // 6. Let fetchParams be a new fetch params whose @@ -815,38 +815,118 @@ function schemeFetch (fetchParams) { return Promise.resolve(makeNetworkError('NetworkError when attempting to fetch resource.')) } - const blobURLEntryObject = resolveObjectURL(blobURLEntry.toString()) + const blob = resolveObjectURL(blobURLEntry.toString()) // 2. If request’s method is not `GET`, blobURLEntry is null, or blobURLEntry’s // object is not a Blob object, then return a network error. - if (request.method !== 'GET' || !isBlobLike(blobURLEntryObject)) { + if (request.method !== 'GET' || !isBlobLike(blob)) { return Promise.resolve(makeNetworkError('invalid method')) } - // 3. Let bodyWithType be the result of safely extracting blobURLEntry’s object. - const bodyWithType = safelyExtractBody(blobURLEntryObject) + // 3. Let blob be blobURLEntry’s object. + // Note: done above - // 4. Let body be bodyWithType’s body. - const body = bodyWithType[0] + // 4. Let response be a new response. + const response = makeResponse() - // 5. Let length be body’s length, serialized and isomorphic encoded. - const length = isomorphicEncode(`${body.length}`) + // 5. Let fullLength be blob’s size. + const fullLength = blob.size - // 6. Let type be bodyWithType’s type if it is non-null; otherwise the empty byte sequence. - const type = bodyWithType[1] ?? '' + // 6. Let serializedFullLength be fullLength, serialized and isomorphic encoded. + const serializedFullLength = isomorphicEncode(`${fullLength}`) - // 7. Return a new response whose status message is `OK`, header list is - // « (`Content-Length`, length), (`Content-Type`, type) », and body is body. - const response = makeResponse({ - statusText: 'OK', - headersList: [ - ['content-length', { name: 'Content-Length', value: length }], - ['content-type', { name: 'Content-Type', value: type }] - ] - }) + // 7. Let type be blob’s type. + const type = blob.type + + // 8. If request’s header list does not contain `Range`: + // 9. Otherwise: + if (!request.headersList.contains('range')) { + // 1. Let bodyWithType be the result of safely extracting blob. + // Note: in the FileAPI a blob "object" is a Blob *or* a MediaSource. + // In node, this can only ever be a Blob. Therefore we can safely + // use extractBody directly. + const bodyWithType = extractBody(blob) + + // 2. Set response’s status message to `OK`. + response.statusText = 'OK' + + // 3. Set response’s body to bodyWithType’s body. + response.body = bodyWithType[0] + + // 4. Set response’s header list to « (`Content-Length`, serializedFullLength), (`Content-Type`, type) ». + response.headersList.set('content-length', serializedFullLength) + response.headersList.set('content-type', type) + } else { + // 1. Set response’s range-requested flag. + response.rangeRequested = true + + // 2. Let rangeHeader be the result of getting `Range` from request’s header list. + const rangeHeader = request.headersList.get('range') + + // 3. Let rangeValue be the result of parsing a single range header value given rangeHeader and true. + const rangeValue = simpleRangeHeaderValue(rangeHeader, true) + + // 4. If rangeValue is failure, then return a network error. + if (rangeValue === 'failure') { + return Promise.resolve(makeNetworkError('failed to fetch the data URL')) + } + + // 5. Let (rangeStart, rangeEnd) be rangeValue. + let { rangeStartValue: rangeStart, rangeEndValue: rangeEnd } = rangeValue + + // 6. If rangeStart is null: + // 7. Otherwise: + if (rangeStart === null) { + // 1. Set rangeStart to fullLength − rangeEnd. + rangeStart = fullLength - rangeEnd + + // 2. Set rangeEnd to rangeStart + rangeEnd − 1. + rangeEnd = rangeStart + rangeEnd - 1 + } else { + // 1. If rangeStart is greater than or equal to fullLength, then return a network error. + if (rangeStart >= fullLength) { + return Promise.resolve(makeNetworkError('Range start is greater than the blob\'s size.')) + } + + // 2. If rangeEnd is null or rangeEnd is greater than or equal to fullLength, then set + // rangeEnd to fullLength − 1. + if (rangeEnd === null || rangeEnd >= fullLength) { + rangeEnd = fullLength - 1 + } + } + + // 8. Let slicedBlob be the result of invoking slice blob given blob, rangeStart, + // rangeEnd + 1, and type. + const slicedBlob = blob.slice(rangeStart, rangeEnd, type) + + // 9. Let slicedBodyWithType be the result of safely extracting slicedBlob. + // Note: same reason as mentioned above as to why we use extractBody + const slicedBodyWithType = extractBody(slicedBlob) - response.body = body + // 10. Set response’s body to slicedBodyWithType’s body. + response.body = slicedBodyWithType[0] + // 11. Let serializedSlicedLength be slicedBlob’s size, serialized and isomorphic encoded. + const serializedSlicedLength = isomorphicEncode(`${slicedBlob.size}`) + + // 12. Let contentRange be the result of invoking build a content range given rangeStart, + // rangeEnd, and fullLength. + const contentRange = buildContentRange(rangeStart, rangeEnd, fullLength) + + // 13. Set response’s status to 206. + response.status = 206 + + // 14. Set response’s status message to `Partial Content`. + response.statusText = 'Partial Content' + + // 15. Set response’s header list to « (`Content-Length`, serializedSlicedLength), + // (`Content-Type`, type), (`Content-Range`, contentRange) ». + response.headersList.set('content-length', serializedSlicedLength) + response.headersList.set('content-type', type) + response.headersList.set('content-range', contentRange) + } + + // 10. Return response. return Promise.resolve(response) } case 'data:': { @@ -908,92 +988,147 @@ function finalizeResponse (fetchParams, response) { // https://fetch.spec.whatwg.org/#fetch-finale function fetchFinale (fetchParams, response) { - // 1. If response is a network error, then: - if (response.type === 'error') { - // 1. Set response’s URL list to « fetchParams’s request’s URL list[0] ». - response.urlList = [fetchParams.request.urlList[0]] - - // 2. Set response’s timing info to the result of creating an opaque timing - // info for fetchParams’s timing info. - response.timingInfo = createOpaqueTimingInfo({ - startTime: fetchParams.timingInfo.startTime - }) - } + // 1. Let timingInfo be fetchParams’s timing info. + let timingInfo = fetchParams.timingInfo + + // 2. If response is not a network error and fetchParams’s request’s client is a secure context, + // then set timingInfo’s server-timing headers to the result of getting, decoding, and splitting + // `Server-Timing` from response’s internal response’s header list. + // TODO - // 2. Let processResponseEndOfBody be the following steps: + // 3. Let processResponseEndOfBody be the following steps: const processResponseEndOfBody = () => { - // 1. Set fetchParams’s request’s done flag. - fetchParams.request.done = true - - // If fetchParams’s process response end-of-body is not null, - // then queue a fetch task to run fetchParams’s process response - // end-of-body given response with fetchParams’s task destination. - if (fetchParams.processResponseEndOfBody != null) { - queueMicrotask(() => fetchParams.processResponseEndOfBody(response)) + // 1. Let unsafeEndTime be the unsafe shared current time. + const unsafeEndTime = Date.now() // ? + + // 2. If fetchParams’s request’s destination is "document", then set fetchParams’s controller’s + // full timing info to fetchParams’s timing info. + if (fetchParams.request.destination === 'document') { + fetchParams.controller.fullTimingInfo = timingInfo + } + + // 3. Set fetchParams’s controller’s report timing steps to the following steps given a global object global: + fetchParams.controller.reportTimingSteps = () => { + // 1. If fetchParams’s request’s URL’s scheme is not an HTTP(S) scheme, then return. + if (fetchParams.request.url.protocol !== 'https:') { + return + } + + // 2. Set timingInfo’s end time to the relative high resolution time given unsafeEndTime and global. + timingInfo.endTime = unsafeEndTime + + // 3. Let cacheState be response’s cache state. + let cacheState = response.cacheState + + // 4. Let bodyInfo be response’s body info. + const bodyInfo = response.bodyInfo + + // 5. If response’s timing allow passed flag is not set, then set timingInfo to the result of creating an + // opaque timing info for timingInfo and set cacheState to the empty string. + if (!response.timingAllowPassed) { + timingInfo = createOpaqueTimingInfo(timingInfo) + + cacheState = '' + } + + // 6. Let responseStatus be 0. + let responseStatus = 0 + + // 7. If fetchParams’s request’s mode is not "navigate" or response’s has-cross-origin-redirects is false: + if (fetchParams.request.mode !== 'navigator' || !response.hasCrossOriginRedirects) { + // 1. Set responseStatus to response’s status. + responseStatus = response.status + + // 2. Let mimeType be the result of extracting a MIME type from response’s header list. + const mimeType = parseMIMEType(response.headersList.get('content-type')) // TODO: fix + + // 3. If mimeType is not failure, then set bodyInfo’s content type to the result of minimizing a supported MIME type given mimeType. + if (mimeType !== 'failure') { + // TODO + } + } + + // 8. If fetchParams’s request’s initiator type is non-null, then mark resource timing given timingInfo, + // fetchParams’s request’s URL, fetchParams’s request’s initiator type, global, cacheState, bodyInfo, + // and responseStatus. + if (fetchParams.request.initiatorType != null) { + // TODO: update markresourcetiming + markResourceTiming(timingInfo, fetchParams.request.url, fetchParams.request.initiatorType, globalThis, cacheState, bodyInfo, responseStatus) + } } + + // 4. Let processResponseEndOfBodyTask be the following steps: + const processResponseEndOfBodyTask = () => { + // 1. Set fetchParams’s request’s done flag. + fetchParams.request.done = true + + // 2. If fetchParams’s process response end-of-body is non-null, then run fetchParams’s process + // response end-of-body given response. + if (fetchParams.processResponseEndOfBody != null) { + queueMicrotask(() => fetchParams.processResponseEndOfBody(response)) + } + + // 3. If fetchParams’s request’s initiator type is non-null and fetchParams’s request’s client’s + // global object is fetchParams’s task destination, then run fetchParams’s controller’s report + // timing steps given fetchParams’s request’s client’s global object. + if (fetchParams.request.initiatorType != null) { + fetchParams.controller.reportTimingSteps() + } + } + + // 5. Queue a fetch task to run processResponseEndOfBodyTask with fetchParams’s task destination + queueMicrotask(() => processResponseEndOfBodyTask()) } - // 3. If fetchParams’s process response is non-null, then queue a fetch task - // to run fetchParams’s process response given response, with fetchParams’s - // task destination. + // 4. If fetchParams’s process response is non-null, then queue a fetch task to run fetchParams’s + // process response given response, with fetchParams’s task destination. if (fetchParams.processResponse != null) { queueMicrotask(() => fetchParams.processResponse(response)) } - // 4. If response’s body is null, then run processResponseEndOfBody. - if (response.body == null) { + // 5. Let internalResponse be response, if response is a network error; otherwise response’s internal response. + const internalResponse = response.type === 'error' ? response : (response.internalResponse ?? response) + + // 6. If internalResponse’s body is null, then run processResponseEndOfBody. + // 7. Otherwise: + if (internalResponse.body == null) { processResponseEndOfBody() } else { - // 5. Otherwise: - - // 1. Let transformStream be a new a TransformStream. - - // 2. Let identityTransformAlgorithm be an algorithm which, given chunk, - // enqueues chunk in transformStream. - const identityTransformAlgorithm = (chunk, controller) => { - controller.enqueue(chunk) - } - - // 3. Set up transformStream with transformAlgorithm set to identityTransformAlgorithm - // and flushAlgorithm set to processResponseEndOfBody. + // 1. Let transformStream be a new TransformStream. + // 2. Let identityTransformAlgorithm be an algorithm which, given chunk, enqueues chunk in transformStream. + // 3. Set up transformStream with transformAlgorithm set to identityTransformAlgorithm and flushAlgorithm + // set to processResponseEndOfBody. const transformStream = new TransformStream({ start () {}, - transform: identityTransformAlgorithm, + transform (chunk, controller) { + controller.enqueue(chunk) + }, flush: processResponseEndOfBody - }, { - size () { - return 1 - } - }, { - size () { - return 1 - } }) - // 4. Set response’s body to the result of piping response’s body through transformStream. - response.body = { stream: response.body.stream.pipeThrough(transformStream) } - } + // 4. Set internalResponse’s body’s stream to the result of internalResponse’s body’s stream piped through transformStream. + internalResponse.body.stream.pipeThrough(transformStream) + + const byteStream = new ReadableStream({ + readableStream: transformStream.readable, + async start (controller) { + const reader = this.readableStream.getReader() - // 6. If fetchParams’s process response consume body is non-null, then: - if (fetchParams.processResponseConsumeBody != null) { - // 1. Let processBody given nullOrBytes be this step: run fetchParams’s - // process response consume body given response and nullOrBytes. - const processBody = (nullOrBytes) => fetchParams.processResponseConsumeBody(response, nullOrBytes) + while (true) { + const { done, value } = await reader.read() - // 2. Let processBodyError be this step: run fetchParams’s process - // response consume body given response and failure. - const processBodyError = (failure) => fetchParams.processResponseConsumeBody(response, failure) + if (done) { + queueMicrotask(() => readableStreamClose(controller)) + break + } - // 3. If response’s body is null, then queue a fetch task to run processBody - // given null, with fetchParams’s task destination. - if (response.body == null) { - queueMicrotask(() => processBody(null)) - } else { - // 4. Otherwise, fully read response’s body given processBody, processBodyError, - // and fetchParams’s task destination. - return fullyReadBody(response.body, processBody, processBodyError) - } - return Promise.resolve() + controller.enqueue(value) + } + }, + type: 'bytes' + }) + + internalResponse.body.stream = byteStream } } @@ -1795,13 +1930,8 @@ async function httpNetworkFetch ( // TODO // 15. Let stream be a new ReadableStream. - // 16. Set up stream with pullAlgorithm set to pullAlgorithm, - // cancelAlgorithm set to cancelAlgorithm, highWaterMark set to - // highWaterMark, and sizeAlgorithm set to sizeAlgorithm. - if (!ReadableStream) { - ReadableStream = require('stream/web').ReadableStream - } - + // 16. Set up stream with byte reading support with pullAlgorithm set to pullAlgorithm, + // cancelAlgorithm set to cancelAlgorithm. const stream = new ReadableStream( { async start (controller) { @@ -1812,13 +1942,8 @@ async function httpNetworkFetch ( }, async cancel (reason) { await cancelAlgorithm(reason) - } - }, - { - highWaterMark: 0, - size () { - return 1 - } + }, + type: 'bytes' } ) @@ -1898,7 +2023,10 @@ async function httpNetworkFetch ( // 7. Enqueue a Uint8Array wrapping an ArrayBuffer containing bytes // into stream. - fetchParams.controller.controller.enqueue(new Uint8Array(bytes)) + const buffer = new Uint8Array(bytes) + if (buffer.byteLength) { + fetchParams.controller.controller.enqueue(buffer) + } // 8. If stream is errored, then terminate the ongoing fetch. if (isErrored(stream)) { diff --git a/lib/fetch/request.js b/lib/fetch/request.js index 6fe4dff64c4..85b8bb0df2f 100644 --- a/lib/fetch/request.js +++ b/lib/fetch/request.js @@ -32,8 +32,6 @@ const { kHeadersList, kConstruct } = require('../core/symbols') const assert = require('assert') const { getMaxListeners, setMaxListeners, getEventListeners, defaultMaxListeners } = require('events') -let TransformStream = globalThis.TransformStream - const kAbortController = Symbol('abortController') const requestFinalizer = new FinalizationRegistry(({ signal, abort }) => { @@ -516,10 +514,6 @@ class Request { } // 2. Set finalBody to the result of creating a proxy for inputBody. - if (!TransformStream) { - TransformStream = require('stream/web').TransformStream - } - // https://streams.spec.whatwg.org/#readablestream-create-a-proxy const identityTransform = new TransformStream() inputBody.stream.pipeThrough(identityTransform) diff --git a/lib/fetch/response.js b/lib/fetch/response.js index 73386123e33..b0b0a2ce543 100644 --- a/lib/fetch/response.js +++ b/lib/fetch/response.js @@ -15,8 +15,7 @@ const { } = require('./util') const { redirectStatusSet, - nullBodyStatus, - DOMException + nullBodyStatus } = require('./constants') const { kState, kHeaders, kGuard, kRealm } = require('./symbols') const { webidl } = require('./webidl') @@ -27,7 +26,6 @@ const { kHeadersList, kConstruct } = require('../core/symbols') const assert = require('assert') const { types } = require('util') -const ReadableStream = globalThis.ReadableStream || require('stream/web').ReadableStream const textEncoder = new TextEncoder('utf-8') // https://fetch.spec.whatwg.org/#response-class @@ -40,9 +38,10 @@ class Response { // The static error() method steps are to return the result of creating a // Response object, given a new network error, "immutable", and this’s // relevant Realm. - const responseObject = new Response() + const responseObject = new Response(kConstruct) responseObject[kState] = makeNetworkError() responseObject[kRealm] = relevantRealm + responseObject[kHeaders] = new Headers(kConstruct) responseObject[kHeaders][kHeadersList] = responseObject[kState].headersList responseObject[kHeaders][kGuard] = 'immutable' responseObject[kHeaders][kRealm] = relevantRealm @@ -68,8 +67,11 @@ class Response { // 3. Let responseObject be the result of creating a Response object, given a new response, // "response", and this’s relevant Realm. const relevantRealm = { settingsObject: {} } - const responseObject = new Response() + const responseObject = new Response(kConstruct) + responseObject[kState] = makeResponse({}) responseObject[kRealm] = relevantRealm + responseObject[kHeaders] = new Headers(kConstruct) + responseObject[kHeaders][kHeadersList] = responseObject[kState].headersList responseObject[kHeaders][kGuard] = 'response' responseObject[kHeaders][kRealm] = relevantRealm @@ -109,8 +111,11 @@ class Response { // 4. Let responseObject be the result of creating a Response object, // given a new response, "immutable", and this’s relevant Realm. - const responseObject = new Response() + const responseObject = new Response(kConstruct) + responseObject[kState] = makeResponse({}) responseObject[kRealm] = relevantRealm + responseObject[kHeaders] = new Headers(kConstruct) + responseObject[kHeaders][kHeadersList] = responseObject[kState].headersList responseObject[kHeaders][kGuard] = 'immutable' responseObject[kHeaders][kRealm] = relevantRealm @@ -129,6 +134,10 @@ class Response { // https://fetch.spec.whatwg.org/#dom-response constructor (body = null, init = {}) { + if (body === kConstruct) { + return + } + if (body !== null) { body = webidl.converters.BodyInit(body) } @@ -260,9 +269,10 @@ class Response { // 3. Return the result of creating a Response object, given // clonedResponse, this’s headers’s guard, and this’s relevant Realm. - const clonedResponseObject = new Response() + const clonedResponseObject = new Response(kConstruct) clonedResponseObject[kState] = clonedResponse clonedResponseObject[kRealm] = this[kRealm] + clonedResponseObject[kHeaders] = new Headers(kConstruct) clonedResponseObject[kHeaders][kHeadersList] = clonedResponse.headersList clonedResponseObject[kHeaders][kGuard] = this[kHeaders][kGuard] clonedResponseObject[kHeaders][kRealm] = this[kHeaders][kRealm] @@ -335,10 +345,10 @@ function makeResponse (init) { cacheState: '', statusText: '', ...init, - headersList: init.headersList - ? new HeadersList(init.headersList) + headersList: init?.headersList + ? new HeadersList(init?.headersList) : new HeadersList(), - urlList: init.urlList ? [...init.urlList] : [] + urlList: init?.urlList ? [...init.urlList] : [] } } diff --git a/lib/fetch/util.js b/lib/fetch/util.js index b12142c7f42..96711267681 100644 --- a/lib/fetch/util.js +++ b/lib/fetch/util.js @@ -890,14 +890,7 @@ async function fullyReadBody (body, processBody, processBodyError) { } } -/** @type {ReadableStream} */ -let ReadableStream = globalThis.ReadableStream - function isReadableStreamLike (stream) { - if (!ReadableStream) { - ReadableStream = require('stream/web').ReadableStream - } - return stream instanceof ReadableStream || ( stream[Symbol.toStringTag] === 'ReadableStream' && typeof stream.tee === 'function' @@ -928,9 +921,10 @@ function isomorphicDecode (input) { function readableStreamClose (controller) { try { controller.close() + controller.byobRequest?.respond(0) } catch (err) { // TODO: add comment explaining why this error occurs. - if (!err.message.includes('Controller is already closed')) { + if (!err.message.includes('Controller is already closed') && !err.message.includes('ReadableStream is already closed')) { throw err } } @@ -1018,10 +1012,172 @@ function urlIsHttpHttpsScheme (url) { return protocol === 'http:' || protocol === 'https:' } +/** @type {import('./dataURL')['collectASequenceOfCodePoints']} */ +let collectASequenceOfCodePoints + +/** + * @see https://fetch.spec.whatwg.org/#simple-range-header-value + * @param {string} value + * @param {boolean} allowWhitespace + */ +function simpleRangeHeaderValue (value, allowWhitespace) { + // Note: avoid circular require + collectASequenceOfCodePoints ??= require('./dataURL').collectASequenceOfCodePoints + + // 1. Let data be the isomorphic decoding of value. + // Note: isomorphic decoding takes a sequence of bytes (ie. a Uint8Array) and turns it into a string, + // nothing more. We obviously don't need to do that if value is a string already. + const data = value + + // 2. If data does not start with "bytes", then return failure. + if (!data.startsWith('bytes')) { + return 'failure' + } + + // 3. Let position be a position variable for data, initially pointing at the 5th code point of data. + const position = { position: 5 } + + // 4. If allowWhitespace is true, collect a sequence of code points that are HTTP tab or space, + // from data given position. + if (allowWhitespace) { + collectASequenceOfCodePoints( + (char) => char === '\t' || char === ' ', + data, + position + ) + } + + // 5. If the code point at position within data is not U+003D (=), then return failure. + if (data.charCodeAt(position.position) !== 0x3D) { + return 'failure' + } + + // 6. Advance position by 1. + position.position++ + + // 7. If allowWhitespace is true, collect a sequence of code points that are HTTP tab or space, from + // data given position. + if (allowWhitespace) { + collectASequenceOfCodePoints( + (char) => char === '\t' || char === ' ', + data, + position + ) + } + + // 8. Let rangeStart be the result of collecting a sequence of code points that are ASCII digits, + // from data given position. + const rangeStart = collectASequenceOfCodePoints( + (char) => { + const code = char.charCodeAt(0) + + return code >= 0x30 && code <= 0x39 + }, + data, + position + ) + + // 9. Let rangeStartValue be rangeStart, interpreted as decimal number, if rangeStart is not the + // empty string; otherwise null. + const rangeStartValue = rangeStart.length ? Number(rangeStart) : null + + // 10. If allowWhitespace is true, collect a sequence of code points that are HTTP tab or space, + // from data given position. + if (allowWhitespace) { + collectASequenceOfCodePoints( + (char) => char === '\t' || char === ' ', + data, + position + ) + } + + // 11. If the code point at position within data is not U+002D (-), then return failure. + if (data.charCodeAt(position.position) !== 0x2D) { + return 'failure' + } + + // 12. Advance position by 1. + position.position++ + + // 13. If allowWhitespace is true, collect a sequence of code points that are HTTP tab + // or space, from data given position. + // Note from Khafra: its the same fucking step again lol + if (allowWhitespace) { + collectASequenceOfCodePoints( + (char) => char === '\t' || char === ' ', + data, + position + ) + } + + // 14. Let rangeEnd be the result of collecting a sequence of code points that are + // ASCII digits, from data given position. + // Note from Khafra: you wouldn't guess it, but this is also the same step as #8 + const rangeEnd = collectASequenceOfCodePoints( + (char) => { + const code = char.charCodeAt(0) + + return code >= 0x30 && code <= 0x39 + }, + data, + position + ) + + // 15. Let rangeEndValue be rangeEnd, interpreted as decimal number, if rangeEnd + // is not the empty string; otherwise null. + // Note from Khafra: THE SAME STEP, AGAIN!!! + // Note: why interpret as a decimal if we only collect ascii digits? + const rangeEndValue = rangeEnd.length ? Number(rangeEnd) : null + + // 16. If position is not past the end of data, then return failure. + if (position.position < data.length) { + return 'failure' + } + + // 17. If rangeEndValue and rangeStartValue are null, then return failure. + if (rangeEndValue === null && rangeStartValue === null) { + return 'failure' + } + + // 18. If rangeStartValue and rangeEndValue are numbers, and rangeStartValue is + // greater than rangeEndValue, then return failure. + // Note: ... when can they not be numbers? + if (rangeStartValue > rangeEndValue) { + return 'failure' + } + + // 19. Return (rangeStartValue, rangeEndValue). + return { rangeStartValue, rangeEndValue } +} + /** - * Fetch supports node >= 16.8.0, but Object.hasOwn was added in v16.9.0. + * @see https://fetch.spec.whatwg.org/#build-a-content-range + * @param {number} rangeStart + * @param {number} rangeEnd + * @param {number} fullLength */ -const hasOwn = Object.hasOwn || ((dict, key) => Object.prototype.hasOwnProperty.call(dict, key)) +function buildContentRange (rangeStart, rangeEnd, fullLength) { + // 1. Let contentRange be `bytes `. + let contentRange = 'bytes ' + + // 2. Append rangeStart, serialized and isomorphic encoded, to contentRange. + contentRange += isomorphicEncode(`${rangeStart}`) + + // 3. Append 0x2D (-) to contentRange. + contentRange += '-' + + // 4. Append rangeEnd, serialized and isomorphic encoded to contentRange. + contentRange += isomorphicEncode(`${rangeEnd}`) + + // 5. Append 0x2F (/) to contentRange. + contentRange += '/' + + // 6. Append fullLength, serialized and isomorphic encoded to contentRange. + contentRange += isomorphicEncode(`${fullLength}`) + + // 7. Return contentRange. + return contentRange +} module.exports = { isAborted, @@ -1055,7 +1211,6 @@ module.exports = { makeIterator, isValidHeaderName, isValidHeaderValue, - hasOwn, isErrorLike, fullyReadBody, bytesMatch, @@ -1067,5 +1222,7 @@ module.exports = { urlHasHttpsScheme, urlIsHttpHttpsScheme, readAllBytes, - normalizeMethodRecord + normalizeMethodRecord, + simpleRangeHeaderValue, + buildContentRange } diff --git a/lib/fetch/webidl.js b/lib/fetch/webidl.js index 6fcf2ab6ad4..ca1019221d9 100644 --- a/lib/fetch/webidl.js +++ b/lib/fetch/webidl.js @@ -1,7 +1,7 @@ 'use strict' const { types } = require('util') -const { hasOwn, toUSVString } = require('./util') +const { toUSVString } = require('./util') /** @type {import('../../types/webidl').Webidl} */ const webidl = {} @@ -346,7 +346,7 @@ webidl.dictionaryConverter = function (converters) { const { key, defaultValue, required, converter } = options if (required === true) { - if (!hasOwn(dictionary, key)) { + if (!Object.hasOwn(dictionary, key)) { throw webidl.errors.exception({ header: 'Dictionary', message: `Missing required key "${key}".` @@ -355,7 +355,7 @@ webidl.dictionaryConverter = function (converters) { } let value = dictionary[key] - const hasDefault = hasOwn(options, 'defaultValue') + const hasDefault = Object.hasOwn(options, 'defaultValue') // Only use defaultValue if value is undefined and // a defaultValue options was provided. diff --git a/lib/fileapi/util.js b/lib/fileapi/util.js index 1d10899cee8..a29423d9d14 100644 --- a/lib/fileapi/util.js +++ b/lib/fileapi/util.js @@ -9,7 +9,6 @@ const { } = require('./symbols') const { ProgressEvent } = require('./progressevent') const { getEncoding } = require('./encoding') -const { DOMException } = require('../fetch/constants') const { serializeAMimeType, parseMIMEType } = require('../fetch/dataURL') const { types } = require('util') const { StringDecoder } = require('string_decoder') diff --git a/lib/handler/RedirectHandler.js b/lib/handler/RedirectHandler.js index baca27ed147..34bba185d96 100644 --- a/lib/handler/RedirectHandler.js +++ b/lib/handler/RedirectHandler.js @@ -135,7 +135,7 @@ class RedirectHandler { For status 300, which is "Multiple Choices", the spec mentions both generating a Location response header AND a response body with the other possible location to follow. - Since the spec explicitily chooses not to specify a format for such body and leave it to + Since the spec explicitly chooses not to specify a format for such body and leave it to servers and browsers implementors, we ignore the body as there is no specified way to eventually parse it. */ } else { @@ -151,7 +151,7 @@ class RedirectHandler { TLDR: undici always ignores 3xx response trailers as they are not expected in case of redirections and neither are useful if present. - See comment on onData method above for more detailed informations. + See comment on onData method above for more detailed information. */ this.location = null diff --git a/lib/handler/RetryHandler.js b/lib/handler/RetryHandler.js index 371044719fd..da35d256baa 100644 --- a/lib/handler/RetryHandler.js +++ b/lib/handler/RetryHandler.js @@ -172,13 +172,22 @@ class RetryHandler { this.retryCount += 1 if (statusCode >= 300) { - this.abort( - new RequestRetryError('Request failed', statusCode, { - headers, - count: this.retryCount - }) - ) - return false + if (this.retryOpts.statusCodes.includes(statusCode) === false) { + return this.handler.onHeaders( + statusCode, + rawHeaders, + resume, + statusMessage + ) + } else { + this.abort( + new RequestRetryError('Request failed', statusCode, { + headers, + count: this.retryCount + }) + ) + return false + } } // Checkpoint for resume from where we left it diff --git a/lib/websocket/connection.js b/lib/websocket/connection.js index e0fa69726b4..4bc60b3b90b 100644 --- a/lib/websocket/connection.js +++ b/lib/websocket/connection.js @@ -261,6 +261,7 @@ function onSocketClose () { // attribute initialized to the result of applying UTF-8 // decode without BOM to the WebSocket connection close // reason. + // TODO: process.nextTick fireEvent('close', ws, CloseEvent, { wasClean, code, reason }) diff --git a/lib/websocket/util.js b/lib/websocket/util.js index 6c59b2c2380..d15a63cde92 100644 --- a/lib/websocket/util.js +++ b/lib/websocket/util.js @@ -182,6 +182,7 @@ function failWebsocketConnection (ws, reason) { } if (reason) { + // TODO: process.nextTick fireEvent('error', ws, ErrorEvent, { error: new Error(reason) }) diff --git a/lib/websocket/websocket.js b/lib/websocket/websocket.js index e4aa58f52fc..1eac67b7828 100644 --- a/lib/websocket/websocket.js +++ b/lib/websocket/websocket.js @@ -1,7 +1,6 @@ 'use strict' const { webidl } = require('../fetch/webidl') -const { DOMException } = require('../fetch/constants') const { URLSerializer } = require('../fetch/dataURL') const { getGlobalOrigin } = require('../fetch/global') const { staticPropertyDescriptors, states, opcodes, emptyBuffer } = require('./constants') diff --git a/package.json b/package.json index 2b64daf41e2..3d0cc9a5da2 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "undici", - "version": "5.28.2", + "version": "6.0.0", "description": "An HTTP/1.1 client, written from scratch for Node.js", "homepage": "https://undici.nodejs.org", "bugs": { @@ -135,7 +135,7 @@ "ws": "^8.11.0" }, "engines": { - "node": ">=14.0" + "node": ">=18.0" }, "standard": { "env": [ diff --git a/test/balanced-pool.js b/test/balanced-pool.js index d20f926e931..7806652d95b 100644 --- a/test/balanced-pool.js +++ b/test/balanced-pool.js @@ -482,7 +482,7 @@ const cases = [ for (const [index, { config, expected, expectedRatios, iterations = 9, expectedConnectionRefusedErrors = 0, expectedSocketErrors = 0, maxWeightPerServer, errorPenalty = 10, only = false, skip = false }] of cases.entries()) { test(`weighted round robin - case ${index}`, { only, skip }, async (t) => { - // cerate an array to store succesfull reqeusts + // create an array to store successful requests const requestLog = [] // create instances of the test servers according to the config diff --git a/test/client-pipeline.js b/test/client-pipeline.js index 9b677a02774..75b59e83422 100644 --- a/test/client-pipeline.js +++ b/test/client-pipeline.js @@ -11,7 +11,6 @@ const { Writable, PassThrough } = require('stream') -const { nodeMajor } = require('../lib/core/util') test('pipeline get', (t) => { t.plan(17) @@ -535,12 +534,7 @@ test('pipeline abort piped res', (t) => { return pipeline(body, pt, () => {}) }) .on('error', (err) => { - // Node < 13 doesn't always detect premature close. - if (nodeMajor < 13) { - t.ok(err) - } else { - t.equal(err.code, 'UND_ERR_ABORTED') - } + t.equal(err.code, 'UND_ERR_ABORTED') }) .end() }) diff --git a/test/client-request.js b/test/client-request.js index 3e6670523b8..3f74ce8cc70 100644 --- a/test/client-request.js +++ b/test/client-request.js @@ -11,7 +11,6 @@ const { Readable } = require('stream') const net = require('net') const { promisify } = require('util') const { NotSupportedError } = require('../lib/core/errors') -const { nodeMajor } = require('../lib/core/util') const { parseFormDataString } = require('./utils/formdata') test('request dump', (t) => { @@ -387,7 +386,7 @@ test('request long multibyte text', (t) => { }) }) -test('request blob', { skip: nodeMajor < 16 }, (t) => { +test('request blob', (t) => { t.plan(2) const obj = { asd: true } @@ -436,7 +435,7 @@ test('request arrayBuffer', (t) => { }) }) -test('request body', { skip: nodeMajor < 16 }, (t) => { +test('request body', (t) => { t.plan(1) const obj = { asd: true } @@ -462,7 +461,7 @@ test('request body', { skip: nodeMajor < 16 }, (t) => { }) }) -test('request post body no missing data', { skip: nodeMajor < 16 }, (t) => { +test('request post body no missing data', (t) => { t.plan(2) const server = createServer(async (req, res) => { @@ -495,7 +494,7 @@ test('request post body no missing data', { skip: nodeMajor < 16 }, (t) => { }) }) -test('request post body no extra data handler', { skip: nodeMajor < 16 }, (t) => { +test('request post body no extra data handler', (t) => { t.plan(3) const server = createServer(async (req, res) => { @@ -663,7 +662,7 @@ test('request raw responseHeaders', async (t) => { t.pass() }) -test('request formData', { skip: nodeMajor < 16 }, (t) => { +test('request formData', (t) => { t.plan(1) const obj = { asd: true } @@ -718,7 +717,7 @@ test('request text2', (t) => { }) }) -test('request with FormData body', { skip: nodeMajor < 16 }, (t) => { +test('request with FormData body', (t) => { const { FormData } = require('../') const { Blob } = require('buffer') @@ -769,24 +768,6 @@ test('request with FormData body', { skip: nodeMajor < 16 }, (t) => { }) }) -test('request with FormData body on node < 16', { skip: nodeMajor >= 16 }, async (t) => { - t.plan(1) - - // a FormData polyfill, for example - class FormData {} - - const fd = new FormData() - - const client = new Client('http://localhost:3000') - t.teardown(client.destroy.bind(client)) - - await t.rejects(client.request({ - path: '/', - method: 'POST', - body: fd - }), errors.InvalidArgumentError) -}) - test('request post body Buffer from string', (t) => { t.plan(2) const requestBody = Buffer.from('abcdefghijklmnopqrstuvwxyz') diff --git a/test/esm-wrapper.js b/test/esm-wrapper.js index a593fbdb531..8adb3278719 100644 --- a/test/esm-wrapper.js +++ b/test/esm-wrapper.js @@ -1,19 +1,14 @@ 'use strict' -const { nodeMajor, nodeMinor } = require('../lib/core/util') -if (!((nodeMajor > 14 || (nodeMajor === 14 && nodeMajor > 13)) || (nodeMajor === 12 && nodeMinor > 20))) { - require('tap') // shows skipped -} else { - ;(async () => { - try { - await import('./utils/esm-wrapper.mjs') - } catch (e) { - if (e.message === 'Not supported') { - require('tap') // shows skipped - return - } - console.error(e.stack) - process.exitCode = 1 +;(async () => { + try { + await import('./utils/esm-wrapper.mjs') + } catch (e) { + if (e.message === 'Not supported') { + require('tap') // shows skipped + return } - })() -} + console.error(e.stack) + process.exitCode = 1 + } +})() diff --git a/test/fetch/abort.js b/test/fetch/abort.js index e1ca1ebf9ea..367201f3a16 100644 --- a/test/fetch/abort.js +++ b/test/fetch/abort.js @@ -4,8 +4,6 @@ const { test } = require('tap') const { fetch } = require('../..') const { createServer } = require('http') const { once } = require('events') -const { DOMException } = require('../../lib/fetch/constants') -const { nodeMajor } = require('../../lib/core/util') const { AbortController: NPMAbortController } = require('abort-controller') @@ -37,7 +35,7 @@ test('Allow the usage of custom implementation of AbortController', async (t) => } }) -test('allows aborting with custom errors', { skip: nodeMajor === 16 }, async (t) => { +test('allows aborting with custom errors', async (t) => { const server = createServer().listen(0) t.teardown(server.close.bind(server)) diff --git a/test/fetch/abort2.js b/test/fetch/abort2.js index 5f3853bcb7a..b07482d9706 100644 --- a/test/fetch/abort2.js +++ b/test/fetch/abort2.js @@ -4,7 +4,6 @@ const { test } = require('tap') const { fetch } = require('../..') const { createServer } = require('http') const { once } = require('events') -const { DOMException } = require('../../lib/fetch/constants') /* global AbortController */ diff --git a/test/fetch/bundle.js b/test/fetch/bundle.js index aa1257a49b4..93e605bf357 100644 --- a/test/fetch/bundle.js +++ b/test/fetch/bundle.js @@ -1,12 +1,6 @@ 'use strict' -const { test, skip } = require('tap') -const { nodeMajor } = require('../../lib/core/util') - -if (nodeMajor === 16) { - skip('esbuild uses static blocks with --keep-names which node 16.8 does not have') - process.exit() -} +const { test } = require('tap') const { Response, Request, FormData, Headers } = require('../../undici-fetch') diff --git a/test/fetch/client-fetch.js b/test/fetch/client-fetch.js index 9009d547ec6..4046785f795 100644 --- a/test/fetch/client-fetch.js +++ b/test/fetch/client-fetch.js @@ -4,7 +4,6 @@ const { test, teardown } = require('tap') const { createServer } = require('http') -const { ReadableStream } = require('stream/web') const { Blob } = require('buffer') const { fetch, Response, Request, FormData, File } = require('../..') const { Client, setGlobalDispatcher, Agent } = require('../..') diff --git a/test/fetch/client-node-max-header-size.js b/test/fetch/client-node-max-header-size.js index 737bae8ed6e..432a576b97e 100644 --- a/test/fetch/client-node-max-header-size.js +++ b/test/fetch/client-node-max-header-size.js @@ -1,13 +1,7 @@ 'use strict' const { execSync } = require('node:child_process') -const { test, skip } = require('tap') -const { nodeMajor } = require('../../lib/core/util') - -if (nodeMajor === 16) { - skip('esbuild uses static blocks with --keep-names which node 16.8 does not have') - process.exit() -} +const { test } = require('tap') const command = 'node -e "require(\'./undici-fetch.js\').fetch(\'https://httpbin.org/get\')"' diff --git a/test/fetch/issue-1447.js b/test/fetch/issue-1447.js index 503b34406d2..cfa5e94c0ed 100644 --- a/test/fetch/issue-1447.js +++ b/test/fetch/issue-1447.js @@ -1,12 +1,6 @@ 'use strict' -const { test, skip } = require('tap') -const { nodeMajor } = require('../../lib/core/util') - -if (nodeMajor === 16) { - skip('esbuild uses static blocks with --keep-names which node 16.8 does not have') - process.exit() -} +const { test } = require('tap') const undici = require('../..') const { fetch: theoreticalGlobalFetch } = require('../../undici-fetch') diff --git a/test/fetch/issue-2171.js b/test/fetch/issue-2171.js index b04ae0e6c38..72770f3713f 100644 --- a/test/fetch/issue-2171.js +++ b/test/fetch/issue-2171.js @@ -1,7 +1,6 @@ 'use strict' const { fetch } = require('../..') -const { DOMException } = require('../../lib/fetch/constants') const { once } = require('events') const { createServer } = require('http') const { test } = require('tap') diff --git a/test/fetch/resource-timing.js b/test/fetch/resource-timing.js index d266f28bdc8..32753f57a86 100644 --- a/test/fetch/resource-timing.js +++ b/test/fetch/resource-timing.js @@ -10,7 +10,7 @@ const { performance } = require('perf_hooks') -const skip = nodeMajor < 18 || (nodeMajor === 18 && nodeMinor < 2) +const skip = nodeMajor === 18 && nodeMinor < 2 test('should create a PerformanceResourceTiming after each fetch request', { skip }, (t) => { t.plan(8) diff --git a/test/fetch/response.js b/test/fetch/response.js index 422c7ef2e02..066adb1e7b3 100644 --- a/test/fetch/response.js +++ b/test/fetch/response.js @@ -4,7 +4,6 @@ const { test } = require('tap') const { Response } = require('../../') -const { ReadableStream } = require('stream/web') const { Blob: ThirdPartyBlob, FormData: ThirdPartyFormData diff --git a/test/fetch/user-agent.js b/test/fetch/user-agent.js index 2e37ea5883d..604305959ff 100644 --- a/test/fetch/user-agent.js +++ b/test/fetch/user-agent.js @@ -1,15 +1,9 @@ 'use strict' -const { test, skip } = require('tap') +const { test } = require('tap') const events = require('events') const http = require('http') const undici = require('../../') -const { nodeMajor } = require('../../lib/core/util') - -if (nodeMajor === 16) { - skip('esbuild uses static blocks with --keep-names which node 16.8 does not have') - process.exit() -} const nodeBuild = require('../../undici-fetch.js') diff --git a/test/http2.js b/test/http2.js index 71b77493255..06c79cda8e9 100644 --- a/test/http2.js +++ b/test/http2.js @@ -282,7 +282,7 @@ test('Should support H2 GOAWAY (server-side)', async t => { t.equal(err.message, 'HTTP/2: "GOAWAY" frame received with code 204') }) -test('Should throw if bad allowH2 has been pased', async t => { +test('Should throw if bad allowH2 has been passed', async t => { try { // eslint-disable-next-line new Client('https://localhost:1000', { @@ -294,7 +294,7 @@ test('Should throw if bad allowH2 has been pased', async t => { } }) -test('Should throw if bad maxConcurrentStreams has been pased', async t => { +test('Should throw if bad maxConcurrentStreams has been passed', async t => { try { // eslint-disable-next-line new Client('https://localhost:1000', { @@ -305,7 +305,7 @@ test('Should throw if bad maxConcurrentStreams has been pased', async t => { } catch (error) { t.equal( error.message, - 'maxConcurrentStreams must be a possitive integer, greater than 0' + 'maxConcurrentStreams must be a positive integer, greater than 0' ) } @@ -319,7 +319,7 @@ test('Should throw if bad maxConcurrentStreams has been pased', async t => { } catch (error) { t.equal( error.message, - 'maxConcurrentStreams must be a possitive integer, greater than 0' + 'maxConcurrentStreams must be a positive integer, greater than 0' ) } }) diff --git a/test/issue-1903.js b/test/issue-1903.js deleted file mode 100644 index 76ac81ef217..00000000000 --- a/test/issue-1903.js +++ /dev/null @@ -1,78 +0,0 @@ -'use strict' - -const { createServer } = require('http') -const { test } = require('tap') -const { request } = require('..') -const { nodeMajor } = require('../lib/core/util') - -function createPromise () { - const result = {} - result.promise = new Promise((resolve) => { - result.resolve = resolve - }) - return result -} - -test('should parse content-disposition consistently', { skip: nodeMajor >= 18 }, async (t) => { - t.plan(5) - - // create promise to allow server spinup in parallel - const spinup1 = createPromise() - const spinup2 = createPromise() - const spinup3 = createPromise() - - // variables to store content-disposition header - const header = [] - - const server = createServer((req, res) => { - res.writeHead(200, { - 'content-length': 2, - 'content-disposition': "attachment; filename='år.pdf'" - }) - header.push("attachment; filename='år.pdf'") - res.end('OK', spinup1.resolve) - }) - t.teardown(server.close.bind(server)) - server.listen(0, spinup1.resolve) - - const proxy1 = createServer(async (req, res) => { - const { statusCode, headers, body } = await request(`http://localhost:${server.address().port}`, { - method: 'GET' - }) - header.push(headers['content-disposition']) - delete headers['transfer-encoding'] - res.writeHead(statusCode, headers) - body.pipe(res) - }) - t.teardown(proxy1.close.bind(proxy1)) - proxy1.listen(0, spinup2.resolve) - - const proxy2 = createServer(async (req, res) => { - const { statusCode, headers, body } = await request(`http://localhost:${proxy1.address().port}`, { - method: 'GET' - }) - header.push(headers['content-disposition']) - delete headers['transfer-encoding'] - res.writeHead(statusCode, headers) - body.pipe(res) - }) - t.teardown(proxy2.close.bind(proxy2)) - proxy2.listen(0, spinup3.resolve) - - // wait until all server spinup - await Promise.all([spinup1.promise, spinup2.promise, spinup3.promise]) - - const { statusCode, headers, body } = await request(`http://localhost:${proxy2.address().port}`, { - method: 'GET' - }) - header.push(headers['content-disposition']) - t.equal(statusCode, 200) - t.equal(await body.text(), 'OK') - - // we check header - // must not be the same in first proxy - t.notSame(header[0], header[1]) - // chaining always the same value - t.equal(header[1], header[2]) - t.equal(header[2], header[3]) -}) diff --git a/test/issue-2065.js b/test/issue-2065.js index cc288c48a74..9cc015b9824 100644 --- a/test/issue-2065.js +++ b/test/issue-2065.js @@ -1,17 +1,11 @@ 'use strict' -const { test, skip } = require('tap') -const { nodeMajor, nodeMinor } = require('../lib/core/util') +const { test } = require('tap') const { createServer } = require('http') const { once } = require('events') const { createReadStream } = require('fs') const { File, FormData, request } = require('..') -if (nodeMajor < 16 || (nodeMajor === 16 && nodeMinor < 8)) { - skip('FormData is not available in node < v16.8.0') - process.exit() -} - test('undici.request with a FormData body should set content-length header', async (t) => { const server = createServer((req, res) => { t.ok(req.headers['content-length']) diff --git a/test/issue-2078.js b/test/issue-2078.js index d3aa868ef43..335d7ab3820 100644 --- a/test/issue-2078.js +++ b/test/issue-2078.js @@ -1,14 +1,8 @@ 'use strict' -const { test, skip } = require('tap') -const { nodeMajor, nodeMinor } = require('../lib/core/util') +const { test } = require('tap') const { MockAgent, getGlobalDispatcher, setGlobalDispatcher, fetch } = require('..') -if (nodeMajor < 16 || (nodeMajor === 16 && nodeMinor < 8)) { - skip('fetch is not supported in node < v16.8.0') - process.exit() -} - test('MockPool.reply headers are an object, not an array - issue #2078', async (t) => { const global = getGlobalDispatcher() const mockAgent = new MockAgent() diff --git a/test/issue-2349.js b/test/issue-2349.js index a82bb74a261..ae44543bb7a 100644 --- a/test/issue-2349.js +++ b/test/issue-2349.js @@ -1,15 +1,9 @@ 'use strict' -const { test, skip } = require('tap') -const { nodeMajor } = require('../lib/core/util') +const { test } = require('tap') const { Writable } = require('stream') const { MockAgent, errors, stream } = require('..') -if (nodeMajor < 16) { - skip('only for node 16') - process.exit(0) -} - test('stream() does not fail after request has been aborted', async (t) => { t.plan(1) diff --git a/test/jest/instanceof-error.test.js b/test/jest/instanceof-error.test.js index 8bb36d23c79..2fa71565172 100644 --- a/test/jest/instanceof-error.test.js +++ b/test/jest/instanceof-error.test.js @@ -8,12 +8,8 @@ const { once } = require('events') // https://github.com/facebook/jest/issues/11607#issuecomment-899068995 jest.useRealTimers() -const runIf = (condition) => condition ? it : it.skip -const nodeMajor = Number(process.versions.node.split('.', 1)[0]) - -runIf(nodeMajor >= 16)('isErrorLike sanity check', () => { +it('isErrorLike sanity check', () => { const { isErrorLike } = require('../../lib/fetch/util') - const { DOMException } = require('../../lib/fetch/constants') const error = new DOMException('') // https://github.com/facebook/jest/issues/2549 @@ -21,7 +17,7 @@ runIf(nodeMajor >= 16)('isErrorLike sanity check', () => { expect(isErrorLike(error)).toBeTruthy() }) -runIf(nodeMajor >= 16)('Real use-case', async () => { +it('Real use-case', async () => { const { fetch } = require('../..') const ac = new AbortController() diff --git a/test/jest/issue-1757.test.js b/test/jest/issue-1757.test.js index b6519d9da14..e419e7c6c6b 100644 --- a/test/jest/issue-1757.test.js +++ b/test/jest/issue-1757.test.js @@ -23,10 +23,7 @@ class MiniflareDispatcher extends Dispatcher { } } -const runIf = (condition) => condition ? it : it.skip -const nodeMajor = Number(process.versions.node.split('.', 1)[0]) - -runIf(nodeMajor >= 16)('https://github.com/nodejs/undici/issues/1757', async () => { +it('https://github.com/nodejs/undici/issues/1757', async () => { // fetch isn't exported in <16.8 const { fetch } = require('../..') diff --git a/test/jest/mock-scope.test.js b/test/jest/mock-scope.test.js index cab77f6b112..f0b2378cbea 100644 --- a/test/jest/mock-scope.test.js +++ b/test/jest/mock-scope.test.js @@ -2,16 +2,13 @@ const { MockAgent, setGlobalDispatcher, request } = require('../../index') /* global afterAll, expect, it, AbortController */ -const runIf = (condition) => condition ? it : it.skip - -const nodeMajor = Number(process.versions.node.split('.', 1)[0]) const mockAgent = new MockAgent() afterAll(async () => { await mockAgent.close() }) -runIf(nodeMajor >= 16)('Jest works with MockScope.delay - issue #1327', async () => { +it('Jest works with MockScope.delay - issue #1327', async () => { mockAgent.disableNetConnect() setGlobalDispatcher(mockAgent) diff --git a/test/mock-agent.js b/test/mock-agent.js index c9ffda443aa..7c1ff660622 100644 --- a/test/mock-agent.js +++ b/test/mock-agent.js @@ -7,12 +7,12 @@ const { request, setGlobalDispatcher, MockAgent, Agent } = require('..') const { getResponse } = require('../lib/mock/mock-utils') const { kClients, kConnected } = require('../lib/core/symbols') const { InvalidArgumentError, ClientDestroyedError } = require('../lib/core/errors') -const { nodeMajor } = require('../lib/core/util') const MockClient = require('../lib/mock/mock-client') const MockPool = require('../lib/mock/mock-pool') const { kAgent } = require('../lib/mock/mock-symbols') const Dispatcher = require('../lib/dispatcher') const { MockNotMatchedError } = require('../lib/mock/mock-errors') +const { fetch } = require('..') test('MockAgent - constructor', t => { t.plan(5) @@ -2399,9 +2399,7 @@ test('MockAgent - clients are not garbage collected', async (t) => { }) // https://github.com/nodejs/undici/issues/1321 -test('MockAgent - using fetch yields correct statusText', { skip: nodeMajor < 16 }, async (t) => { - const { fetch } = require('..') - +test('MockAgent - using fetch yields correct statusText', async (t) => { const mockAgent = new MockAgent() mockAgent.disableNetConnect() setGlobalDispatcher(mockAgent) @@ -2432,9 +2430,7 @@ test('MockAgent - using fetch yields correct statusText', { skip: nodeMajor < 16 }) // https://github.com/nodejs/undici/issues/1556 -test('MockAgent - using fetch yields a headers object in the reply callback', { skip: nodeMajor < 16 }, async (t) => { - const { fetch } = require('..') - +test('MockAgent - using fetch yields a headers object in the reply callback', async (t) => { const mockAgent = new MockAgent() mockAgent.disableNetConnect() t.teardown(mockAgent.close.bind(mockAgent)) @@ -2464,9 +2460,7 @@ test('MockAgent - using fetch yields a headers object in the reply callback', { }) // https://github.com/nodejs/undici/issues/1579 -test('MockAgent - headers in mock dispatcher intercept should be case-insensitive', { skip: nodeMajor < 16 }, async (t) => { - const { fetch } = require('..') - +test('MockAgent - headers in mock dispatcher intercept should be case-insensitive', async (t) => { const mockAgent = new MockAgent() mockAgent.disableNetConnect() setGlobalDispatcher(mockAgent) @@ -2495,10 +2489,7 @@ test('MockAgent - headers in mock dispatcher intercept should be case-insensitiv }) // https://github.com/nodejs/undici/issues/1757 -test('MockAgent - reply callback can be asynchronous', { skip: nodeMajor < 16 }, async (t) => { - const { fetch } = require('..') - const ReadableStream = globalThis.ReadableStream || require('stream/web').ReadableStream - +test('MockAgent - reply callback can be asynchronous', async (t) => { class MiniflareDispatcher extends Dispatcher { constructor (inner, options) { super(options) @@ -2602,10 +2593,8 @@ test('MockAgent - headers should be array of strings', async (t) => { }) // https://github.com/nodejs/undici/issues/2418 -test('MockAgent - Sending ReadableStream body', { skip: nodeMajor < 16 }, async (t) => { +test('MockAgent - Sending ReadableStream body', async (t) => { t.plan(1) - const { fetch } = require('..') - const ReadableStream = globalThis.ReadableStream || require('stream/web').ReadableStream const mockAgent = new MockAgent() setGlobalDispatcher(mockAgent) diff --git a/test/mock-pool.js b/test/mock-pool.js index 0ac1aac49ce..682d02aab68 100644 --- a/test/mock-pool.js +++ b/test/mock-pool.js @@ -5,12 +5,12 @@ const { createServer } = require('http') const { promisify } = require('util') const { MockAgent, MockPool, getGlobalDispatcher, setGlobalDispatcher, request } = require('..') const { kUrl } = require('../lib/core/symbols') -const { nodeMajor } = require('../lib/core/util') const { kDispatches } = require('../lib/mock/mock-symbols') const { InvalidArgumentError } = require('../lib/core/errors') const { MockInterceptor } = require('../lib/mock/mock-interceptor') const { getResponse } = require('../lib/mock/mock-utils') const Dispatcher = require('../lib/dispatcher') +const { fetch } = require('..') test('MockPool - constructor', t => { t.plan(3) @@ -323,9 +323,7 @@ test('MockPool - correct errors when consuming invalid JSON body', async (t) => t.end() }) -test('MockPool - allows matching headers in fetch', { skip: nodeMajor < 16 }, async (t) => { - const { fetch } = require('../index') - +test('MockPool - allows matching headers in fetch', async (t) => { const oldDispatcher = getGlobalDispatcher() const baseUrl = 'http://localhost:9999' diff --git a/test/mock-utils.js b/test/mock-utils.js index 7799803ad2f..4acc8558390 100644 --- a/test/mock-utils.js +++ b/test/mock-utils.js @@ -1,7 +1,6 @@ 'use strict' const { test } = require('tap') -const { nodeMajor } = require('../lib/core/util') const { MockNotMatchedError } = require('../lib/mock/mock-errors') const { deleteMockDispatch, @@ -145,16 +144,14 @@ test('getHeaderByName', (t) => { t.equal(getHeaderByName(headersArray, 'key'), 'value') t.equal(getHeaderByName(headersArray, 'anotherKey'), undefined) - if (nodeMajor >= 16) { - const { Headers } = require('../index') + const { Headers } = require('../index') - const headers = new Headers([ - ['key', 'value'] - ]) + const headers = new Headers([ + ['key', 'value'] + ]) - t.equal(getHeaderByName(headers, 'key'), 'value') - t.equal(getHeaderByName(headers, 'anotherKey'), null) - } + t.equal(getHeaderByName(headers, 'key'), 'value') + t.equal(getHeaderByName(headers, 'anotherKey'), null) t.end() }) diff --git a/test/node-fetch/main.js b/test/node-fetch/main.js index 358a969b574..d702a2743fc 100644 --- a/test/node-fetch/main.js +++ b/test/node-fetch/main.js @@ -27,7 +27,6 @@ const RequestOrig = require('../../lib/fetch/request.js').Request const ResponseOrig = require('../../lib/fetch/response.js').Response const TestServer = require('./utils/server.js') const chaiTimeout = require('./utils/chai-timeout.js') -const { ReadableStream } = require('stream/web') function isNodeLowerThan (version) { return !~process.version.localeCompare(version, undefined, { numeric: true }) diff --git a/test/pool.js b/test/pool.js index 8cf7195bb21..816bc5e2d31 100644 --- a/test/pool.js +++ b/test/pool.js @@ -24,7 +24,7 @@ const { errors } = require('..') -test('throws when connection is inifinite', (t) => { +test('throws when connection is infinite', (t) => { t.plan(2) try { diff --git a/test/proxy-agent.js b/test/proxy-agent.js index 0a9212639f8..7177ef8735a 100644 --- a/test/proxy-agent.js +++ b/test/proxy-agent.js @@ -3,7 +3,6 @@ const { test, teardown } = require('tap') const { request, fetch, setGlobalDispatcher, getGlobalDispatcher } = require('..') const { InvalidArgumentError } = require('../lib/core/errors') -const { nodeMajor } = require('../lib/core/util') const { readFileSync } = require('fs') const { join } = require('path') const ProxyAgent = require('../lib/proxy-agent') @@ -367,7 +366,7 @@ test('use proxy-agent with setGlobalDispatcher', async (t) => { proxyAgent.close() }) -test('ProxyAgent correctly sends headers when using fetch - #1355, #1623', { skip: nodeMajor < 16 }, async (t) => { +test('ProxyAgent correctly sends headers when using fetch - #1355, #1623', async (t) => { t.plan(2) const defaultDispatcher = getGlobalDispatcher() @@ -443,7 +442,7 @@ test('should throw when proxy does not return 200', async (t) => { t.end() }) -test('pass ProxyAgent proxy status code error when using fetch - #2161', { skip: nodeMajor < 16 }, async (t) => { +test('pass ProxyAgent proxy status code error when using fetch - #2161', async (t) => { const server = await buildServer() const proxy = await buildProxy() diff --git a/test/redirect-request.js b/test/redirect-request.js index 5a1ae6de5b4..a51166adb76 100644 --- a/test/redirect-request.js +++ b/test/redirect-request.js @@ -2,7 +2,6 @@ const t = require('tap') const undici = require('..') -const { nodeMajor } = require('../lib/core/util') const { startRedirectingServer, startRedirectingWithBodyServer, @@ -312,7 +311,7 @@ for (const factory of [ } }) - t.test('should not follow redirects when using ReadableStream request bodies', { skip: nodeMajor < 16 }, async t => { + t.test('should not follow redirects when using ReadableStream request bodies', async t => { const server = await startRedirectingServer(t) const { statusCode, headers, body: bodyStream } = await request(t, server, undefined, `http://${server}/301`, { diff --git a/test/request-timeout.js b/test/request-timeout.js index 3ec5c107e79..23115ea4b9e 100644 --- a/test/request-timeout.js +++ b/test/request-timeout.js @@ -4,7 +4,6 @@ const { test } = require('tap') const { createReadStream, writeFileSync, unlinkSync } = require('fs') const { Client, errors } = require('..') const { kConnect } = require('../lib/core/symbols') -const { nodeMajor } = require('../lib/core/util') const timers = require('../lib/timers') const { createServer } = require('http') const EventEmitter = require('events') @@ -57,7 +56,7 @@ test('request timeout with readable body', (t) => { t.type(err, errors.HeadersTimeoutError) }) }) -}, { skip: nodeMajor < 14 }) +}) test('body timeout', (t) => { t.plan(2) diff --git a/test/retry-handler.js b/test/retry-handler.js index a4577a6a3e1..d90ff5ebc45 100644 --- a/test/retry-handler.js +++ b/test/retry-handler.js @@ -620,3 +620,72 @@ tap.test('retrying a request with a body', t => { ) }) }) + +tap.test('should not error if request is not meant to be retried', t => { + const server = createServer() + server.on('request', (req, res) => { + res.writeHead(400) + res.end('Bad request') + }) + + t.plan(3) + + const dispatchOptions = { + retryOptions: { + method: 'GET', + path: '/', + headers: { + 'content-type': 'application/json' + } + } + } + + server.listen(0, () => { + const client = new Client(`http://localhost:${server.address().port}`) + const chunks = [] + const handler = new RetryHandler(dispatchOptions, { + dispatch: client.dispatch.bind(client), + handler: { + onConnect () { + t.pass() + }, + onBodySent () { + t.pass() + }, + onHeaders (status, _rawHeaders, resume, _statusMessage) { + t.equal(status, 400) + return true + }, + onData (chunk) { + chunks.push(chunk) + return true + }, + onComplete () { + t.equal(Buffer.concat(chunks).toString('utf-8'), 'Bad request') + }, + onError (err) { + console.log({ err }) + t.fail() + } + } + }) + + t.teardown(async () => { + await client.close() + server.close() + + await once(server, 'close') + }) + + client.dispatch( + { + method: 'GET', + path: '/', + headers: { + 'content-type': 'application/json' + } + }, + handler + ) + }) +}) diff --git a/test/tls-client-cert.js b/test/tls-client-cert.js deleted file mode 100644 index 8ae301d8fca..00000000000 --- a/test/tls-client-cert.js +++ /dev/null @@ -1,70 +0,0 @@ -'use strict' - -const { readFileSync } = require('fs') -const { join } = require('path') -const https = require('https') -const { test } = require('tap') -const { Client } = require('..') -const { kSocket } = require('../lib/core/symbols') -const { nodeMajor } = require('../lib/core/util') - -const serverOptions = { - ca: [ - readFileSync(join(__dirname, 'fixtures', 'client-ca-crt.pem'), 'utf8') - ], - key: readFileSync(join(__dirname, 'fixtures', 'key.pem'), 'utf8'), - cert: readFileSync(join(__dirname, 'fixtures', 'cert.pem'), 'utf8'), - requestCert: true, - rejectUnauthorized: false -} - -test('Client using valid client certificate', { skip: nodeMajor > 16 }, t => { - t.plan(5) - - const server = https.createServer(serverOptions, (req, res) => { - const authorized = req.client.authorized - t.ok(authorized) - - res.setHeader('Content-Type', 'text/plain') - res.end('hello') - }) - - server.listen(0, function () { - const tls = { - ca: [ - readFileSync(join(__dirname, 'fixtures', 'ca.pem'), 'utf8') - ], - key: readFileSync(join(__dirname, 'fixtures', 'client-key.pem'), 'utf8'), - cert: readFileSync(join(__dirname, 'fixtures', 'client-crt.pem'), 'utf8'), - rejectUnauthorized: false, - servername: 'agent1' - } - const client = new Client(`https://localhost:${server.address().port}`, { - connect: tls - }) - - t.teardown(() => { - client.close() - server.close() - }) - - client.request({ - path: '/', - method: 'GET' - }, (err, { statusCode, body }) => { - t.error(err) - t.equal(statusCode, 200) - - const authorized = client[kSocket].authorized - t.ok(authorized) - - const bufs = [] - body.on('data', (buf) => { - bufs.push(buf) - }) - body.on('end', () => { - t.equal('hello', Buffer.concat(bufs).toString('utf8')) - }) - }) - }) -}) diff --git a/test/tls-session-reuse.js b/test/tls-session-reuse.js index ab012f16756..7a7657b4e4c 100644 --- a/test/tls-session-reuse.js +++ b/test/tls-session-reuse.js @@ -7,7 +7,6 @@ const crypto = require('crypto') const { test, teardown } = require('tap') const { Client, Pool } = require('..') const { kSocket } = require('../lib/core/symbols') -const { nodeMajor } = require('../lib/core/util') const options = { key: readFileSync(join(__dirname, 'fixtures', 'key.pem'), 'utf8'), @@ -15,9 +14,7 @@ const options = { } const ca = readFileSync(join(__dirname, 'fixtures', 'ca.pem'), 'utf8') -test('A client should disable session caching', { - skip: nodeMajor < 11 // tls socket session event has been added in Node 11. Cf. https://nodejs.org/api/tls.html#tls_event_session -}, t => { +test('A client should disable session caching', t => { const clientSessions = {} let serverRequests = 0 @@ -93,9 +90,7 @@ test('A client should disable session caching', { t.end() }) -test('A pool should be able to reuse TLS sessions between clients', { - skip: nodeMajor < 11 // tls socket session event has been added in Node 11. Cf. https://nodejs.org/api/tls.html#tls_event_session -}, t => { +test('A pool should be able to reuse TLS sessions between clients', t => { let serverRequests = 0 const REQ_COUNT = 10 diff --git a/test/util.js b/test/util.js index 794c68e3f77..75a4d8c1617 100644 --- a/test/util.js +++ b/test/util.js @@ -6,6 +6,7 @@ const { Stream } = require('stream') const { EventEmitter } = require('events') const util = require('../lib/core/util') +const { headerNameLowerCasedRecord } = require('../lib/core/constants') const { InvalidArgumentError } = require('../lib/core/errors') test('isStream', (t) => { @@ -97,7 +98,7 @@ test('parseRawHeaders', (t) => { t.same(util.parseRawHeaders(['key', 'value', Buffer.from('key'), Buffer.from('value')]), ['key', 'value', 'key', 'value']) }) -test('buildURL', { skip: util.nodeMajor >= 12 }, (t) => { +test('buildURL', (t) => { const tests = [ [{ id: BigInt(123456) }, 'id=123456'], [{ date: new Date() }, 'date='], @@ -121,3 +122,8 @@ test('buildURL', { skip: util.nodeMajor >= 12 }, (t) => { t.end() }) + +test('headerNameLowerCasedRecord', (t) => { + t.plan(1) + t.ok(typeof headerNameLowerCasedRecord.hasOwnProperty === 'undefined') +}) diff --git a/test/utils/stream.js b/test/utils/stream.js index b78ff5c5538..e35ca29fe44 100644 --- a/test/utils/stream.js +++ b/test/utils/stream.js @@ -2,8 +2,6 @@ const { Readable, Writable } = require('stream') -let ReadableStream - function createReadable (data) { return new Readable({ read () { @@ -41,7 +39,6 @@ class Source { } function createReadableStream (data) { - ReadableStream = require('stream/web').ReadableStream return new ReadableStream(new Source(data)) } diff --git a/test/wpt/server/server.mjs b/test/wpt/server/server.mjs index 82b9080f407..d12b4936978 100644 --- a/test/wpt/server/server.mjs +++ b/test/wpt/server/server.mjs @@ -36,6 +36,7 @@ const server = createServer(async (req, res) => { res.setHeader('content-type', 'text/html') // fall through } + case '/fetch/content-encoding/resources/big.text.gz': case '/service-workers/cache-storage/resources/simple.txt': case '/fetch/content-encoding/resources/foo.octetstream.gz': case '/fetch/content-encoding/resources/foo.text.gz': diff --git a/test/wpt/status/fetch.status.json b/test/wpt/status/fetch.status.json index 5910bf37f6f..34f91ed8375 100644 --- a/test/wpt/status/fetch.status.json +++ b/test/wpt/status/fetch.status.json @@ -71,6 +71,10 @@ "Fetch with TacO and mode \"cors\" needs an Origin header" ] }, + "request-private-network-headers.tentative.any.js": { + "note": "undici doesn't filter headers", + "skip": true + }, "request-referrer.any.js": { "note": "TODO(@KhafraDev): url referrer test could probably be fixed", "fail": [ @@ -228,6 +232,10 @@ "note": "document is not defined", "skip": true }, + "redirect-keepalive.https.any.js": { + "note": "document is not defined", + "skip": true + }, "redirect-location-escape.tentative.any.js": { "note": "TODO(@KhafraDev): crashes runner", "skip": true diff --git a/test/wpt/status/websockets.status.json b/test/wpt/status/websockets.status.json index 68bc6e2ebe7..9f5a6308a1d 100644 --- a/test/wpt/status/websockets.status.json +++ b/test/wpt/status/websockets.status.json @@ -4,6 +4,18 @@ "skip": true } }, + "interfaces": { + "WebSocket": { + "close": { + "close-connecting-async.any.js": { + "note": "TODO - need to add route for handshake delay", + "fail": [ + "close event should be fired asynchronously when WebSocket is connecting" + ] + } + } + } + }, "Create-blocked-port.any.js": { "note": "TODO(@KhafraDev): investigate failure", "fail": [ diff --git a/test/wpt/tests/.azure-pipelines.yml b/test/wpt/tests/.azure-pipelines.yml index 75a87df90f0..1a21d2f7a00 100644 --- a/test/wpt/tests/.azure-pipelines.yml +++ b/test/wpt/tests/.azure-pipelines.yml @@ -100,9 +100,6 @@ jobs: inputs: versionSpec: '3.11' - template: tools/ci/azure/checkout.yml - - template: tools/ci/azure/pip_install.yml - parameters: - packages: virtualenv - template: tools/ci/azure/install_fonts.yml - template: tools/ci/azure/install_certs.yml - template: tools/ci/azure/color_profile.yml @@ -144,8 +141,7 @@ jobs: steps: - task: UsePythonVersion@0 inputs: - # TODO(#40525): Revert back to 3.7 once the Mac agent's Python v3.7 contains bz2 again. - versionSpec: '3.7.16' + versionSpec: '3.7' - template: tools/ci/azure/checkout.yml - template: tools/ci/azure/tox_pytest.yml parameters: @@ -177,8 +173,7 @@ jobs: steps: - task: UsePythonVersion@0 inputs: - # TODO(#40525): Revert back to 3.7 once the Mac agent's Python v3.7 contains bz2 again. - versionSpec: '3.7.16' + versionSpec: '3.7' - template: tools/ci/azure/checkout.yml - template: tools/ci/azure/tox_pytest.yml parameters: @@ -211,8 +206,7 @@ jobs: # full checkout required - task: UsePythonVersion@0 inputs: - # TODO(#40525): Revert back to 3.7 once the Mac agent's Python v3.7 contains bz2 again. - versionSpec: '3.7.16' + versionSpec: '3.7' - template: tools/ci/azure/install_chrome.yml - template: tools/ci/azure/install_firefox.yml - template: tools/ci/azure/update_hosts.yml @@ -373,9 +367,6 @@ jobs: versionSpec: '3.11' - template: tools/ci/azure/system_info.yml - template: tools/ci/azure/checkout.yml - - template: tools/ci/azure/pip_install.yml - parameters: - packages: virtualenv - template: tools/ci/azure/install_certs.yml - template: tools/ci/azure/install_edge.yml parameters: @@ -412,9 +403,6 @@ jobs: versionSpec: '3.11' - template: tools/ci/azure/system_info.yml - template: tools/ci/azure/checkout.yml - - template: tools/ci/azure/pip_install.yml - parameters: - packages: virtualenv - template: tools/ci/azure/install_certs.yml - template: tools/ci/azure/install_edge.yml parameters: @@ -450,9 +438,6 @@ jobs: inputs: versionSpec: '3.11' - template: tools/ci/azure/checkout.yml - - template: tools/ci/azure/pip_install.yml - parameters: - packages: virtualenv - template: tools/ci/azure/install_certs.yml - template: tools/ci/azure/install_edge.yml parameters: @@ -488,9 +473,6 @@ jobs: inputs: versionSpec: '3.11' - template: tools/ci/azure/checkout.yml - - template: tools/ci/azure/pip_install.yml - parameters: - packages: virtualenv - template: tools/ci/azure/install_certs.yml - template: tools/ci/azure/color_profile.yml - template: tools/ci/azure/install_safari.yml @@ -531,9 +513,6 @@ jobs: inputs: versionSpec: '3.11' - template: tools/ci/azure/checkout.yml - - template: tools/ci/azure/pip_install.yml - parameters: - packages: virtualenv - template: tools/ci/azure/install_certs.yml - template: tools/ci/azure/color_profile.yml - template: tools/ci/azure/install_safari.yml @@ -571,9 +550,6 @@ jobs: inputs: versionSpec: '3.11' - template: tools/ci/azure/checkout.yml - - template: tools/ci/azure/pip_install.yml - parameters: - packages: virtualenv - template: tools/ci/azure/install_certs.yml - template: tools/ci/azure/color_profile.yml - template: tools/ci/azure/update_hosts.yml diff --git a/test/wpt/tests/.taskcluster.yml b/test/wpt/tests/.taskcluster.yml index c817999b8e6..2a005df772f 100644 --- a/test/wpt/tests/.taskcluster.yml +++ b/test/wpt/tests/.taskcluster.yml @@ -7,7 +7,7 @@ tasks: run_task: $if: 'tasks_for == "github-push"' then: - $if: 'event.ref in ["refs/heads/master", "refs/heads/epochs/daily", "refs/heads/epochs/weekly", "refs/heads/triggers/chrome_stable", "refs/heads/triggers/chrome_beta", "refs/heads/triggers/chrome_dev", "refs/heads/triggers/chrome_nightly", "refs/heads/triggers/firefox_stable", "refs/heads/triggers/firefox_beta", "refs/heads/triggers/firefox_nightly", "refs/heads/triggers/webkitgtk_minibrowser_stable", "refs/heads/triggers/webkitgtk_minibrowser_beta", "refs/heads/triggers/webkitgtk_minibrowser_nightly", "refs/heads/triggers/servo_nightly"]' + $if: 'event.ref in ["refs/heads/master", "refs/heads/epochs/daily", "refs/heads/epochs/weekly", "refs/heads/triggers/chrome_stable", "refs/heads/triggers/chrome_beta", "refs/heads/triggers/chrome_dev", "refs/heads/triggers/chrome_nightly", "refs/heads/triggers/firefox_stable", "refs/heads/triggers/firefox_beta", "refs/heads/triggers/firefox_nightly", "refs/heads/triggers/firefox_android_nightly", "refs/heads/triggers/webkitgtk_minibrowser_stable", "refs/heads/triggers/webkitgtk_minibrowser_beta", "refs/heads/triggers/webkitgtk_minibrowser_nightly", "refs/heads/triggers/servo_nightly"]' then: true else: false else: @@ -57,7 +57,7 @@ tasks: owner: ${owner} source: ${event.repository.clone_url} payload: - image: webplatformtests/wpt:0.54 + image: webplatformtests/wpt:0.55 maxRunTime: 7200 artifacts: public/results: diff --git a/test/wpt/tests/FileAPI/META.yml b/test/wpt/tests/FileAPI/META.yml index 506a59fec1e..66227c8224b 100644 --- a/test/wpt/tests/FileAPI/META.yml +++ b/test/wpt/tests/FileAPI/META.yml @@ -1,6 +1,6 @@ spec: https://w3c.github.io/FileAPI/ suggested_reviewers: - inexorabletash - - zqzhang - jdm - mkruisselbrink + - annevk diff --git a/test/wpt/tests/FileAPI/blob/Blob-stream.any.js b/test/wpt/tests/FileAPI/blob/Blob-stream.any.js index 87710a171a9..453144cac96 100644 --- a/test/wpt/tests/FileAPI/blob/Blob-stream.any.js +++ b/test/wpt/tests/FileAPI/blob/Blob-stream.any.js @@ -70,7 +70,18 @@ promise_test(async() => { await garbageCollect(); const chunks = await read_all_chunks(stream, { perform_gc: true }); assert_array_equals(chunks, input_arr); -}, "Blob.stream() garbage collection of blob shouldn't break stream" + +}, "Blob.stream() garbage collection of blob shouldn't break stream " + + "consumption") + +promise_test(async() => { + const input_arr = [8, 241, 48, 123, 151]; + const typed_arr = new Uint8Array(input_arr); + let blob = new Blob([typed_arr]); + const chunksPromise = read_all_chunks(blob.stream()); + // It somehow matters to do GC here instead of doing `perform_gc: true` + await garbageCollect(); + assert_array_equals(await chunksPromise, input_arr); +}, "Blob.stream() garbage collection of stream shouldn't break stream " + "consumption") promise_test(async () => { diff --git a/test/wpt/tests/common/META.yml b/test/wpt/tests/common/META.yml index ca4d2e523c0..963dff9d1f1 100644 --- a/test/wpt/tests/common/META.yml +++ b/test/wpt/tests/common/META.yml @@ -1,3 +1,2 @@ suggested_reviewers: - - zqzhang - deniak diff --git a/test/wpt/tests/common/dispatcher/dispatcher.js b/test/wpt/tests/common/dispatcher/dispatcher.js index a0f9f43e622..ce17a7c9145 100644 --- a/test/wpt/tests/common/dispatcher/dispatcher.js +++ b/test/wpt/tests/common/dispatcher/dispatcher.js @@ -1,7 +1,26 @@ // Define a universal message passing API. It works cross-origin and across // browsing context groups. const dispatcher_path = "/common/dispatcher/dispatcher.py"; -const dispatcher_url = new URL(dispatcher_path, location.href).href; + +// Finds the nearest ancestor window that has a non srcdoc location. This should +// give us a usable location for constructing further URLs. +function findLocationFromAncestors(w) { + if (w.location.href == 'about:srcdoc') { + return findLocationFromAncestors(w.parent); + } + return w.location; +} + +// Handles differences between workers vs frames (src vs srcdoc). +function findLocation() { + if (location.href == 'about:srcdoc') { + return findLocationFromAncestors(window.parent); + } + return location; +} + +const dispatcherLocation = findLocation(); +const dispatcher_url = new URL(dispatcher_path, dispatcherLocation).href; // Return a promise, limiting the number of concurrent accesses to a shared // resources to |max_concurrent_access|. @@ -138,7 +157,7 @@ const cacheableShowRequestHeaders = function(origin, uuid) { // protocol: (optional) Sets the returned URL's `protocol` property. // } function remoteExecutorUrl(uuid, options) { - const url = new URL("/common/dispatcher/remote-executor.html", location); + const url = new URL("/common/dispatcher/remote-executor.html", dispatcherLocation); url.searchParams.set("uuid", uuid); if (options?.host) { diff --git a/test/wpt/tests/common/dummy.json b/test/wpt/tests/common/dummy.json new file mode 100644 index 00000000000..0967ef424bc --- /dev/null +++ b/test/wpt/tests/common/dummy.json @@ -0,0 +1 @@ +{} diff --git a/test/wpt/tests/common/security-features/subresource/video.py b/test/wpt/tests/common/security-features/subresource/video.py index 7cfbbfa68c8..9db8e9fbb5a 100644 --- a/test/wpt/tests/common/security-features/subresource/video.py +++ b/test/wpt/tests/common/security-features/subresource/video.py @@ -4,7 +4,7 @@ subresource = importlib.import_module("common.security-features.subresource.subresource") def generate_payload(request, server_data): - file = os.path.join(request.doc_root, u"media", u"movie_5.ogv") + file = os.path.join(request.doc_root, u"media", u"movie_5.webm") return open(file, "rb").read() @@ -14,4 +14,4 @@ def main(request, response): response, payload_generator = handler, access_control_allow_origin = b"*", - content_type = b"video/ogg") + content_type = b"video/webm") diff --git a/test/wpt/tests/common/top-layer.js b/test/wpt/tests/common/top-layer.js new file mode 100644 index 00000000000..2dc8ce3893a --- /dev/null +++ b/test/wpt/tests/common/top-layer.js @@ -0,0 +1,29 @@ +// This function is a version of test_driver.bless which works while there are +// elements in the top layer: +// https://github.com/web-platform-tests/wpt/issues/41218. +// Pass it the element at the top of the top layer stack. +window.blessTopLayer = async (topLayerElement) => { + const button = document.createElement('button'); + topLayerElement.append(button); + let wait_click = new Promise(resolve => button.addEventListener("click", resolve, {once: true})); + await test_driver.click(button); + await wait_click; + button.remove(); +}; + +window.isTopLayer = (el) => { + // A bit of a hack. Just test a few properties of the ::backdrop pseudo + // element that change when in the top layer. + const properties = ['right','background']; + const testEl = document.createElement('div'); + document.body.appendChild(testEl); + const computedStyle = getComputedStyle(testEl, '::backdrop'); + const nonTopLayerValues = properties.map(p => computedStyle[p]); + testEl.remove(); + for(let i=0;i + + + + + + + +
+ diff --git a/test/wpt/tests/fetch/api/cors/cors-keepalive.any.js b/test/wpt/tests/fetch/api/cors/cors-keepalive.any.js index f68d90ef9aa..08229f9cfc7 100644 --- a/test/wpt/tests/fetch/api/cors/cors-keepalive.any.js +++ b/test/wpt/tests/fetch/api/cors/cors-keepalive.any.js @@ -75,10 +75,10 @@ keepaliveCorsBasicTest( function keepaliveCorsInUnloadTest(description, origin, method) { const evt = 'unload'; for (const mode of ['no-cors', 'cors']) { - for (const disallowOrigin of [false, true]) { + for (const disallowCrossOrigin of [false, true]) { const desc = `${description} ${method} request in ${evt} [${mode} mode` + - (disallowOrigin ? `, server forbid CORS]` : `]`); - const shouldPass = !disallowOrigin || mode === 'no-cors'; + (disallowCrossOrigin ? ']' : ', server forbid CORS]'); + const expectTokenExist = !disallowCrossOrigin || mode === 'no-cors'; promise_test(async (test) => { const token1 = token(); const iframe = document.createElement('iframe'); @@ -87,14 +87,14 @@ function keepaliveCorsInUnloadTest(description, origin, method) { requestOrigin: origin, sendOn: evt, mode: mode, - disallowOrigin + disallowCrossOrigin }); document.body.appendChild(iframe); await iframeLoaded(iframe); iframe.remove(); assert_equals(await getTokenFromMessage(), token1); - assertStashedTokenAsync(desc, token1, {shouldPass}); + assertStashedTokenAsync(desc, token1, {expectTokenExist}); }, `${desc}; setting up`); } } diff --git a/test/wpt/tests/fetch/api/redirect/redirect-keepalive.any.js b/test/wpt/tests/fetch/api/redirect/redirect-keepalive.any.js index bcfc444f5a6..beda8bb8e78 100644 --- a/test/wpt/tests/fetch/api/redirect/redirect-keepalive.any.js +++ b/test/wpt/tests/fetch/api/redirect/redirect-keepalive.any.js @@ -14,66 +14,6 @@ const { HTTP_REMOTE_ORIGIN_WITH_DIFFERENT_PORT } = get_host_info(); -/** - * In an iframe, test to fetch a keepalive URL that involves in redirect to - * another URL. - */ -function keepaliveRedirectTest( - desc, {origin1 = '', origin2 = '', withPreflight = false} = {}) { - desc = `[keepalive] ${desc}`; - promise_test(async (test) => { - const tokenToStash = token(); - const iframe = document.createElement('iframe'); - iframe.src = getKeepAliveAndRedirectIframeUrl( - tokenToStash, origin1, origin2, withPreflight); - document.body.appendChild(iframe); - await iframeLoaded(iframe); - assert_equals(await getTokenFromMessage(), tokenToStash); - iframe.remove(); - - assertStashedTokenAsync(desc, tokenToStash); - }, `${desc}; setting up`); -} - -/** - * Opens a different site window, and in `unload` event handler, test to fetch - * a keepalive URL that involves in redirect to another URL. - */ -function keepaliveRedirectInUnloadTest(desc, { - origin1 = '', - origin2 = '', - url2 = '', - withPreflight = false, - shouldPass = true -} = {}) { - desc = `[keepalive][new window][unload] ${desc}`; - - promise_test(async (test) => { - const targetUrl = - `${HTTP_NOTSAMESITE_ORIGIN}/fetch/api/resources/keepalive-redirect-window.html?` + - `origin1=${origin1}&` + - `origin2=${origin2}&` + - `url2=${url2}&` + (withPreflight ? `with-headers` : ``); - const w = window.open(targetUrl); - const token = await getTokenFromMessage(); - w.close(); - - assertStashedTokenAsync(desc, token, {shouldPass}); - }, `${desc}; setting up`); -} - -keepaliveRedirectTest(`same-origin redirect`); -keepaliveRedirectTest( - `same-origin redirect + preflight`, {withPreflight: true}); -keepaliveRedirectTest(`cross-origin redirect`, { - origin1: HTTP_REMOTE_ORIGIN, - origin2: HTTP_REMOTE_ORIGIN_WITH_DIFFERENT_PORT -}); -keepaliveRedirectTest(`cross-origin redirect + preflight`, { - origin1: HTTP_REMOTE_ORIGIN, - origin2: HTTP_REMOTE_ORIGIN_WITH_DIFFERENT_PORT, - withPreflight: true -}); keepaliveRedirectInUnloadTest('same-origin redirect'); keepaliveRedirectInUnloadTest( @@ -88,7 +28,9 @@ keepaliveRedirectInUnloadTest('cross-origin redirect + preflight', { withPreflight: true }); keepaliveRedirectInUnloadTest( - 'redirect to file URL', {url2: 'file://tmp/bar.txt', shouldPass: false}); -keepaliveRedirectInUnloadTest( - 'redirect to data URL', - {url2: 'data:text/plain;base64,cmVzcG9uc2UncyBib2R5', shouldPass: false}); + 'redirect to file URL', + {url2: 'file://tmp/bar.txt', expectFetchSucceed: false}); +keepaliveRedirectInUnloadTest('redirect to data URL', { + url2: 'data:text/plain;base64,cmVzcG9uc2UncyBib2R5', + expectFetchSucceed: false +}); diff --git a/test/wpt/tests/fetch/api/redirect/redirect-keepalive.https.any.js b/test/wpt/tests/fetch/api/redirect/redirect-keepalive.https.any.js new file mode 100644 index 00000000000..6765ecac6d7 --- /dev/null +++ b/test/wpt/tests/fetch/api/redirect/redirect-keepalive.https.any.js @@ -0,0 +1,20 @@ +// META: global=window +// META: title=Fetch API: keepalive handling +// META: script=/resources/testharness.js +// META: script=/resources/testharnessreport.js +// META: script=/common/utils.js +// META: script=/common/get-host-info.sub.js +// META: script=../resources/keepalive-helper.js + +'use strict'; + +const { + HTTP_NOTSAMESITE_ORIGIN, + HTTPS_NOTSAMESITE_ORIGIN, +} = get_host_info(); + +keepaliveRedirectTest(`mixed content redirect`, { + origin1: HTTPS_NOTSAMESITE_ORIGIN, + origin2: HTTP_NOTSAMESITE_ORIGIN, + expectFetchSucceed: false +}); diff --git a/test/wpt/tests/fetch/api/request/destination/fetch-destination.https.html b/test/wpt/tests/fetch/api/request/destination/fetch-destination.https.html index 0094b0b6fe8..1b6cf169141 100644 --- a/test/wpt/tests/fetch/api/request/destination/fetch-destination.https.html +++ b/test/wpt/tests/fetch/api/request/destination/fetch-destination.https.html @@ -194,6 +194,40 @@ }); }, 'HTMLLinkElement with rel=stylesheet fetches with a "style" Request.destination'); +// Import declaration with `type: "css"` - style destination +promise_test(t => { + return new Promise((resolve, reject) => { + frame.contentWindow.onerror = reject; + + let node = frame.contentWindow.document.createElement("script"); + node.onload = resolve; + node.onerror = reject; + node.src = "import-declaration-type-css.js"; + node.type = "module"; + frame.contentWindow.document.body.appendChild(node); + }).then(() => { + frame.contentWindow.onerror = null; + }); +}, 'Import declaration with `type: "css"` fetches with a "style" Request.destination'); + +// JSON destination +/////////////////// + +// Import declaration with `type: "json"` - json destination +promise_test(t => { + return new Promise((resolve, reject) => { + frame.contentWindow.onerror = reject; + let node = frame.contentWindow.document.createElement("script"); + node.onload = resolve; + node.onerror = reject; + node.src = "import-declaration-type-json.js"; + node.type = "module"; + frame.contentWindow.document.body.appendChild(node); + }).then(() => { + frame.contentWindow.onerror = null; + }); +}, 'Import declaration with `type: "json"` fetches with a "json" Request.destination'); + // Preload tests //////////////// // HTMLLinkElement with rel=preload and as=fetch - empty string destination @@ -232,6 +266,22 @@ }); }, 'HTMLLinkElement with rel=preload and as=style fetches with a "style" Request.destination'); +// HTMLLinkElement with rel=preload and as=json - json destination +promise_test(t => { + return new Promise((resolve, reject) => { + let node = frame.contentWindow.document.createElement("link"); + node.rel = "preload"; + node.as = "json"; + if (node.as != "json") { + resolve(); + } + node.onload = resolve; + node.onerror = reject; + node.href = "dummy.json?t=2&dest=json"; + frame.contentWindow.document.body.appendChild(node); + }); +}, 'HTMLLinkElement with rel=preload and as=json fetches with a "json" Request.destination'); + // HTMLLinkElement with rel=preload and as=script - script destination promise_test(async t => { await new Promise((resolve, reject) => { diff --git a/test/wpt/tests/fetch/api/request/destination/resources/dummy.css b/test/wpt/tests/fetch/api/request/destination/resources/dummy.css new file mode 100644 index 00000000000..e69de29bb2d diff --git a/test/wpt/tests/fetch/api/request/destination/resources/dummy.json b/test/wpt/tests/fetch/api/request/destination/resources/dummy.json new file mode 100644 index 00000000000..0967ef424bc --- /dev/null +++ b/test/wpt/tests/fetch/api/request/destination/resources/dummy.json @@ -0,0 +1 @@ +{} diff --git a/test/wpt/tests/fetch/api/request/destination/resources/import-declaration-type-css.js b/test/wpt/tests/fetch/api/request/destination/resources/import-declaration-type-css.js new file mode 100644 index 00000000000..3c8cf1f44b7 --- /dev/null +++ b/test/wpt/tests/fetch/api/request/destination/resources/import-declaration-type-css.js @@ -0,0 +1 @@ +import "./dummy.css?dest=style" with { type: "css" }; diff --git a/test/wpt/tests/fetch/api/request/destination/resources/import-declaration-type-json.js b/test/wpt/tests/fetch/api/request/destination/resources/import-declaration-type-json.js new file mode 100644 index 00000000000..b2d964dd824 --- /dev/null +++ b/test/wpt/tests/fetch/api/request/destination/resources/import-declaration-type-json.js @@ -0,0 +1 @@ +import "./dummy.json?dest=json" with { type: "json" }; diff --git a/test/wpt/tests/fetch/api/request/request-headers.any.js b/test/wpt/tests/fetch/api/request/request-headers.any.js index 22925e01b69..a766bcb5fff 100644 --- a/test/wpt/tests/fetch/api/request/request-headers.any.js +++ b/test/wpt/tests/fetch/api/request/request-headers.any.js @@ -18,7 +18,6 @@ var invalidRequestHeaders = [ ["Accept-Encoding", "KO"], ["Access-Control-Request-Headers", "KO"], ["Access-Control-Request-Method", "KO"], - ["Access-Control-Request-Private-Network", "KO"], ["Connection", "KO"], ["Content-Length", "KO"], ["Cookie", "KO"], diff --git a/test/wpt/tests/fetch/api/resources/keepalive-helper.js b/test/wpt/tests/fetch/api/resources/keepalive-helper.js index ad1d4b2c7c3..f6f511631e5 100644 --- a/test/wpt/tests/fetch/api/resources/keepalive-helper.js +++ b/test/wpt/tests/fetch/api/resources/keepalive-helper.js @@ -11,14 +11,14 @@ // `sendOn` to specify the name of the event when the keepalive request should // be sent instead of the default 'load'. // `mode` to specify the fetch request's CORS mode. -// `disallowOrigin` to ask the iframe to set up a server that forbids CORS -// requests. +// `disallowCrossOrigin` to ask the iframe to set up a server that disallows +// cross origin requests. function getKeepAliveIframeUrl(token, method, { frameOrigin = 'DEFAULT', requestOrigin = '', sendOn = 'load', mode = 'cors', - disallowOrigin = false + disallowCrossOrigin = false } = {}) { const https = location.protocol.startsWith('https'); frameOrigin = frameOrigin === 'DEFAULT' ? @@ -28,7 +28,7 @@ function getKeepAliveIframeUrl(token, method, { `token=${token}&` + `method=${method}&` + `sendOn=${sendOn}&` + - `mode=${mode}&` + (disallowOrigin ? `disallowOrigin=1&` : ``) + + `mode=${mode}&` + (disallowCrossOrigin ? `disallowCrossOrigin=1&` : ``) + `origin=${requestOrigin}`; } @@ -72,28 +72,105 @@ async function queryToken(token) { return json; } +// A helper to assert the existence of `token` that should have been stored in +// the server by fetching ../resources/stash-put.py. +// +// This function simply wait for a custom amount of time before trying to +// retrieve `token` from the server. +// `expectTokenExist` tells if `token` should be present or not. +// +// NOTE: // In order to parallelize the work, we are going to have an async_test // for the rest of the work. Note that we want the serialized behavior // for the steps so far, so we don't want to make the entire test case // an async_test. -function assertStashedTokenAsync(testName, token, {shouldPass = true} = {}) { - async_test((test) => { - new Promise((resolve) => test.step_timeout(resolve, 3000)) - .then(() => { +function assertStashedTokenAsync( + testName, token, {expectTokenExist = true} = {}) { + async_test(test => { + new Promise(resolve => test.step_timeout(resolve, 3000 /*ms*/)) + .then(test.step_func(() => { return queryToken(token); - }) - .then((result) => { - assert_equals(result, 'on'); - }) - .then(() => { - test.done(); - }) - .catch(test.step_func((e) => { - if (shouldPass) { - assert_unreached(e); + })) + .then(test.step_func(result => { + if (expectTokenExist) { + assert_equals(result, 'on', `token should be on (stashed).`); + test.done(); + } else { + assert_not_equals( + result, 'on', `token should not be on (stashed).`); + return Promise.reject(`Failed to retrieve token from server`); + } + })) + .catch(test.step_func(e => { + if (expectTokenExist) { + test.unreached_func(e); } else { test.done(); } })); }, testName); } + +/** + * In an iframe, and in `load` event handler, test to fetch a keepalive URL that + * involves in redirect to another URL. + * + * `unloadIframe` to unload the iframe before verifying stashed token to + * simulate the situation that unloads after fetching. Note that this test is + * different from `keepaliveRedirectInUnloadTest()` in that the the latter + * performs fetch() call directly in `unload` event handler, while this test + * does it in `load`. + */ +function keepaliveRedirectTest(desc, { + origin1 = '', + origin2 = '', + withPreflight = false, + unloadIframe = false, + expectFetchSucceed = true, +} = {}) { + desc = `[keepalive][iframe][load] ${desc}` + + (unloadIframe ? ' [unload at end]' : ''); + promise_test(async (test) => { + const tokenToStash = token(); + const iframe = document.createElement('iframe'); + iframe.src = getKeepAliveAndRedirectIframeUrl( + tokenToStash, origin1, origin2, withPreflight); + document.body.appendChild(iframe); + await iframeLoaded(iframe); + assert_equals(await getTokenFromMessage(), tokenToStash); + if (unloadIframe) { + iframe.remove(); + } + + assertStashedTokenAsync( + desc, tokenToStash, {expectTokenExist: expectFetchSucceed}); + }, `${desc}; setting up`); +} + +/** + * Opens a different site window, and in `unload` event handler, test to fetch + * a keepalive URL that involves in redirect to another URL. + */ +function keepaliveRedirectInUnloadTest(desc, { + origin1 = '', + origin2 = '', + url2 = '', + withPreflight = false, + expectFetchSucceed = true +} = {}) { + desc = `[keepalive][new window][unload] ${desc}`; + + promise_test(async (test) => { + const targetUrl = + `${HTTP_NOTSAMESITE_ORIGIN}/fetch/api/resources/keepalive-redirect-window.html?` + + `origin1=${origin1}&` + + `origin2=${origin2}&` + + `url2=${url2}&` + (withPreflight ? `with-headers` : ``); + const w = window.open(targetUrl); + const token = await getTokenFromMessage(); + w.close(); + + assertStashedTokenAsync( + desc, token, {expectTokenExist: expectFetchSucceed}); + }, `${desc}; setting up`); +} diff --git a/test/wpt/tests/fetch/api/resources/keepalive-iframe.html b/test/wpt/tests/fetch/api/resources/keepalive-iframe.html index 335a1f8e318..f9dae5a34ec 100644 --- a/test/wpt/tests/fetch/api/resources/keepalive-iframe.html +++ b/test/wpt/tests/fetch/api/resources/keepalive-iframe.html @@ -4,14 +4,15 @@ + + + + + + + + + diff --git a/test/wpt/tests/fetch/fetch-later/headers/header-referrer-no-referrer.tentative.https.html b/test/wpt/tests/fetch/fetch-later/headers/header-referrer-no-referrer.tentative.https.html new file mode 100644 index 00000000000..75e9ece7ba5 --- /dev/null +++ b/test/wpt/tests/fetch/fetch-later/headers/header-referrer-no-referrer.tentative.https.html @@ -0,0 +1,19 @@ + + + +FetchLater Referrer Header: No Referrer Policy + + + + + + + + + + + diff --git a/test/wpt/tests/fetch/fetch-later/headers/header-referrer-origin-when-cross-origin.tentative.https.html b/test/wpt/tests/fetch/fetch-later/headers/header-referrer-origin-when-cross-origin.tentative.https.html new file mode 100644 index 00000000000..b9f14171ba3 --- /dev/null +++ b/test/wpt/tests/fetch/fetch-later/headers/header-referrer-origin-when-cross-origin.tentative.https.html @@ -0,0 +1,25 @@ + + + +FetchLater Referrer Header: Origin When Cross Origin Policy + + + + + + + + + + + diff --git a/test/wpt/tests/fetch/fetch-later/headers/header-referrer-origin.tentative.https.html b/test/wpt/tests/fetch/fetch-later/headers/header-referrer-origin.tentative.https.html new file mode 100644 index 00000000000..ce7abf92039 --- /dev/null +++ b/test/wpt/tests/fetch/fetch-later/headers/header-referrer-origin.tentative.https.html @@ -0,0 +1,23 @@ + + + +FetchLater Referrer Header: Origin Policy + + + + + + + + + + + diff --git a/test/wpt/tests/fetch/fetch-later/headers/header-referrer-same-origin.tentative.https.html b/test/wpt/tests/fetch/fetch-later/headers/header-referrer-same-origin.tentative.https.html new file mode 100644 index 00000000000..264beddc03a --- /dev/null +++ b/test/wpt/tests/fetch/fetch-later/headers/header-referrer-same-origin.tentative.https.html @@ -0,0 +1,24 @@ + + + +FetchLater Referrer Header: Same Origin Policy + + + + + + + + + + + diff --git a/test/wpt/tests/fetch/fetch-later/headers/header-referrer-strict-origin-when-cross-origin.tentative.https.html b/test/wpt/tests/fetch/fetch-later/headers/header-referrer-strict-origin-when-cross-origin.tentative.https.html new file mode 100644 index 00000000000..9133f2496fe --- /dev/null +++ b/test/wpt/tests/fetch/fetch-later/headers/header-referrer-strict-origin-when-cross-origin.tentative.https.html @@ -0,0 +1,24 @@ + + + +FetchLater Referrer Header: Strict Origin Policy + + + + + + + + + + + diff --git a/test/wpt/tests/fetch/fetch-later/headers/header-referrer-strict-origin.tentative.https.html b/test/wpt/tests/fetch/fetch-later/headers/header-referrer-strict-origin.tentative.https.html new file mode 100644 index 00000000000..943d70bbc58 --- /dev/null +++ b/test/wpt/tests/fetch/fetch-later/headers/header-referrer-strict-origin.tentative.https.html @@ -0,0 +1,24 @@ + + + +FetchLater Referrer Header: Strict Origin Policy + + + + + + + + + + + diff --git a/test/wpt/tests/fetch/fetch-later/headers/header-referrer-unsafe-url.tentative.https.html b/test/wpt/tests/fetch/fetch-later/headers/header-referrer-unsafe-url.tentative.https.html new file mode 100644 index 00000000000..a602e0003a4 --- /dev/null +++ b/test/wpt/tests/fetch/fetch-later/headers/header-referrer-unsafe-url.tentative.https.html @@ -0,0 +1,24 @@ + + + +FetchLater Referrer Header: Unsafe Url Policy + + + + + + + + + + + diff --git a/test/wpt/tests/fetch/fetch-later/iframe.tentative.https.window.js b/test/wpt/tests/fetch/fetch-later/iframe.tentative.https.window.js new file mode 100644 index 00000000000..62505bc81d9 --- /dev/null +++ b/test/wpt/tests/fetch/fetch-later/iframe.tentative.https.window.js @@ -0,0 +1,65 @@ +// META: script=/resources/testharness.js +// META: script=/resources/testharnessreport.js +// META: script=/common/utils.js +// META: script=/common/get-host-info.sub.js +// META: script=/pending-beacon/resources/pending_beacon-helper.js + +'use strict'; + +const { + HTTPS_ORIGIN, + HTTPS_NOTSAMESITE_ORIGIN, +} = get_host_info(); + +async function loadElement(el) { + const loaded = new Promise(resolve => el.onload = resolve); + document.body.appendChild(el); + await loaded; +} + +// `host` may be cross-origin +async function loadFetchLaterIframe(host, targetUrl) { + const url = `${host}/fetch/fetch-later/resources/fetch-later.html?url=${ + encodeURIComponent(targetUrl)}`; + const iframe = document.createElement('iframe'); + iframe.src = url; + await loadElement(iframe); + return iframe; +} + +parallelPromiseTest(async t => { + const uuid = token(); + const url = generateSetBeaconURL(uuid); + + // Loads a blank iframe that fires a fetchLater request. + const iframe = document.createElement('iframe'); + iframe.addEventListener('load', () => { + fetchLater(url, {activateAfter: 0}); + }); + await loadElement(iframe); + + // The iframe should have sent the request. + await expectBeacon(uuid, {count: 1}); +}, 'A blank iframe can trigger fetchLater.'); + +parallelPromiseTest(async t => { + const uuid = token(); + const url = generateSetBeaconURL(uuid); + + // Loads a same-origin iframe that fires a fetchLater request. + await loadFetchLaterIframe(HTTPS_ORIGIN, url); + + // The iframe should have sent the request. + await expectBeacon(uuid, {count: 1}); +}, 'A same-origin iframe can trigger fetchLater.'); + +parallelPromiseTest(async t => { + const uuid = token(); + const url = generateSetBeaconURL(uuid); + + // Loads a same-origin iframe that fires a fetchLater request. + await loadFetchLaterIframe(HTTPS_NOTSAMESITE_ORIGIN, url); + + // The iframe should have sent the request. + await expectBeacon(uuid, {count: 1}); +}, 'A cross-origin iframe can trigger fetchLater.'); diff --git a/test/wpt/tests/fetch/fetch-later/new-window.tentative.https.window.js b/test/wpt/tests/fetch/fetch-later/new-window.tentative.https.window.js new file mode 100644 index 00000000000..37b38d7f1dc --- /dev/null +++ b/test/wpt/tests/fetch/fetch-later/new-window.tentative.https.window.js @@ -0,0 +1,77 @@ +// META: script=/resources/testharness.js +// META: script=/resources/testharnessreport.js +// META: script=/common/utils.js +// META: script=/common/get-host-info.sub.js +// META: script=/pending-beacon/resources/pending_beacon-helper.js + +'use strict'; + +const { + HTTPS_ORIGIN, + HTTPS_NOTSAMESITE_ORIGIN, +} = get_host_info(); + +function fetchLaterPopupUrl(host, targetUrl) { + return `${host}/fetch/fetch-later/resources/fetch-later.html?url=${ + encodeURIComponent(targetUrl)}`; +} + +for (const target of ['', '_blank']) { + for (const features in ['', 'popup', 'popup,noopener']) { + parallelPromiseTest( + async t => { + const uuid = token(); + const url = + generateSetBeaconURL(uuid, {host: HTTPS_NOTSAMESITE_ORIGIN}); + + // Opens a blank popup window that fires a fetchLater request. + const w = window.open( + `javascript: fetchLater("${url}", {activateAfter: 0})`, target, + features); + await new Promise(resolve => w.addEventListener('load', resolve)); + + // The popup should have sent the request. + await expectBeacon(uuid, {count: 1}); + w.close(); + }, + `A blank window[target='${target}'][features='${ + features}'] can trigger fetchLater.`); + + parallelPromiseTest( + async t => { + const uuid = token(); + const popupUrl = + fetchLaterPopupUrl(HTTPS_ORIGIN, generateSetBeaconURL(uuid)); + + // Opens a same-origin popup that fires a fetchLater request. + const w = window.open(popupUrl, target, features); + await new Promise(resolve => w.addEventListener('load', resolve)); + + // The popup should have sent the request. + await expectBeacon(uuid, {count: 1}); + w.close(); + }, + `A same-origin window[target='${target}'][features='${ + features}'] can trigger fetchLater.`); + + parallelPromiseTest( + async t => { + const uuid = token(); + const popupUrl = fetchLaterPopupUrl( + HTTPS_NOTSAMESITE_ORIGIN, generateSetBeaconURL(uuid)); + + // Opens a cross-origin popup that fires a fetchLater request. + const w = window.open(popupUrl, target, features); + // As events from cross-origin window is not accessible, waiting for + // its message instead. + await new Promise( + resolve => window.addEventListener('message', resolve)); + + // The popup should have sent the request. + await expectBeacon(uuid, {count: 1}); + w.close(); + }, + `A cross-origin window[target='${target}'][features='${ + features}'] can trigger fetchLater.`); + } +} diff --git a/test/wpt/tests/fetch/fetch-later/policies/csp-allowed.tentative.https.window.js b/test/wpt/tests/fetch/fetch-later/policies/csp-allowed.tentative.https.window.js new file mode 100644 index 00000000000..5aa759c2346 --- /dev/null +++ b/test/wpt/tests/fetch/fetch-later/policies/csp-allowed.tentative.https.window.js @@ -0,0 +1,28 @@ +// META: title=FetchLater: allowed by CSP +// META: script=/resources/testharness.js +// META: script=/resources/testharnessreport.js +// META: script=/common/utils.js +// META: script=/common/get-host-info.sub.js +// META: script=/pending-beacon/resources/pending_beacon-helper.js +'use strict'; + +const { + HTTPS_NOTSAMESITE_ORIGIN, +} = get_host_info(); + +// FetchLater requests allowed by Content Security Policy. +// https://w3c.github.io/webappsec-csp/#should-block-request + +const meta = document.createElement('meta'); +meta.setAttribute('http-equiv', 'Content-Security-Policy'); +meta.setAttribute('content', `connect-src 'self' ${HTTPS_NOTSAMESITE_ORIGIN}`); +document.head.appendChild(meta); + +promise_test(async t => { + const uuid = token(); + const url = generateSetBeaconURL(uuid, {host: HTTPS_NOTSAMESITE_ORIGIN}); + fetchLater(url, {activateAfter: 0}); + + await expectBeacon(uuid, {count: 1}); + t.done(); +}, 'FetchLater allowed by CSP should succeed'); diff --git a/test/wpt/tests/fetch/fetch-later/policies/csp-blocked.tentative.https.window.js b/test/wpt/tests/fetch/fetch-later/policies/csp-blocked.tentative.https.window.js new file mode 100644 index 00000000000..88490950d3a --- /dev/null +++ b/test/wpt/tests/fetch/fetch-later/policies/csp-blocked.tentative.https.window.js @@ -0,0 +1,33 @@ +// META: title=FetchLater: blocked by CSP +// META: script=/resources/testharness.js +// META: script=/resources/testharnessreport.js +// META: script=/common/utils.js +// META: script=/common/get-host-info.sub.js +// META: script=/pending-beacon/resources/pending_beacon-helper.js +'use strict'; + +const { + HTTPS_NOTSAMESITE_ORIGIN, +} = get_host_info(); + +// FetchLater requests blocked by Content Security Policy are rejected. +// https://w3c.github.io/webappsec-csp/#should-block-request + +const meta = document.createElement('meta'); +meta.setAttribute('http-equiv', 'Content-Security-Policy'); +meta.setAttribute('content', 'connect-src \'self\''); +document.head.appendChild(meta); + +promise_test(async t => { + const uuid = token(); + const cspViolationUrl = + generateSetBeaconURL(uuid, {host: HTTPS_NOTSAMESITE_ORIGIN}); + fetchLater(cspViolationUrl, {activateAfter: 0}); + + await new Promise( + resolve => window.addEventListener('securitypolicyviolation', e => { + assert_equals(e.violatedDirective, 'connect-src'); + resolve(); + })); + t.done(); +}, 'FetchLater blocked by CSP should reject'); diff --git a/test/wpt/tests/fetch/fetch-later/policies/csp-redirect-to-blocked.tentative.https.window.js b/test/wpt/tests/fetch/fetch-later/policies/csp-redirect-to-blocked.tentative.https.window.js new file mode 100644 index 00000000000..db6b4234b97 --- /dev/null +++ b/test/wpt/tests/fetch/fetch-later/policies/csp-redirect-to-blocked.tentative.https.window.js @@ -0,0 +1,35 @@ +// META: title=FetchLater: redirect blocked by CSP +// META: script=/resources/testharness.js +// META: script=/resources/testharnessreport.js +// META: script=/common/utils.js +// META: script=/common/get-host-info.sub.js +// META: script=/pending-beacon/resources/pending_beacon-helper.js +// META: timeout=long + +'use strict'; + +const { + HTTPS_NOTSAMESITE_ORIGIN, +} = get_host_info(); + +// FetchLater requests redirect to URL blocked by Content Security Policy. +// https://w3c.github.io/webappsec-csp/#should-block-request + +const meta = document.createElement('meta'); +meta.setAttribute('http-equiv', 'Content-Security-Policy'); +meta.setAttribute('content', 'connect-src \'self\''); +document.head.appendChild(meta); + +promise_test(async t => { + const uuid = token(); + const cspViolationUrl = + generateSetBeaconURL(uuid, {host: HTTPS_NOTSAMESITE_ORIGIN}); + const url = + `/common/redirect.py?location=${encodeURIComponent(cspViolationUrl)}`; + fetchLater(url, {activateAfter: 0}); + + // TODO(crbug.com/1465781): redirect csp check is handled in browser, of which + // result cannot be populated to renderer at this moment. + await expectBeacon(uuid, {count: 0}); + t.done(); +}, 'FetchLater redirect blocked by CSP should reject'); diff --git a/test/wpt/tests/fetch/fetch-later/quota.tentative.https.window.js b/test/wpt/tests/fetch/fetch-later/quota.tentative.https.window.js new file mode 100644 index 00000000000..4fc5979374c --- /dev/null +++ b/test/wpt/tests/fetch/fetch-later/quota.tentative.https.window.js @@ -0,0 +1,130 @@ +// META: script=/resources/testharness.js +// META: script=/resources/testharnessreport.js +// META: script=/common/get-host-info.sub.js +// META: script=/common/utils.js +// META: script=/pending-beacon/resources/pending_beacon-helper.js + +'use strict'; + +const kQuotaPerOrigin = 64 * 1024; // 64 kilobytes per spec. +const {ORIGIN, HTTPS_NOTSAMESITE_ORIGIN} = get_host_info(); + +// Runs a test case that cover a single fetchLater() call with `body` in its +// request payload. The call is not expected to throw any errors. +function fetchLaterPostTest(body, description) { + test(() => { + const controller = new AbortController(); + const result = fetchLater( + '/fetch-later', + {method: 'POST', signal: controller.signal, body: body}); + assert_false(result.activated); + // Release quota taken by the pending request for subsequent tests. + controller.abort(); + }, description); +} + +// Test small payload for each supported data types. +for (const [dataType, skipCharset] of Object.entries( + BeaconDataTypeToSkipCharset)) { + fetchLaterPostTest( + makeBeaconData(generateSequentialData(0, 1024, skipCharset), dataType), + `A fetchLater() call accept small data in POST request of ${dataType}.`); +} + +// Test various size of payloads for the same origin. +for (const dataType in BeaconDataType) { + if (dataType !== BeaconDataType.FormData && + dataType !== BeaconDataType.URLSearchParams) { + // Skips FormData & URLSearchParams, as browser adds extra bytes to them + // in addition to the user-provided content. It is difficult to test a + // request right at the quota limit. + fetchLaterPostTest( + // Generates data that is exactly 64 kilobytes. + makeBeaconData(generatePayload(kQuotaPerOrigin), dataType), + `A single fetchLater() call takes up the per-origin quota for its ` + + `body of ${dataType}.`); + } +} + +// Test empty payload. +for (const dataType in BeaconDataType) { + test( + () => { + assert_throws_js( + TypeError, () => fetchLater('/', {method: 'POST', body: ''})); + }, + `A single fetchLater() call does not accept empty data in POST request ` + + `of ${dataType}.`); +} + +// Test oversized payload. +for (const dataType in BeaconDataType) { + test( + () => { + assert_throws_dom( + 'QuotaExceededError', + () => fetchLater('/fetch-later', { + method: 'POST', + // Generates data that exceeds 64 kilobytes. + body: + makeBeaconData(generatePayload(kQuotaPerOrigin + 1), dataType) + })); + }, + `A single fetchLater() call is not allowed to exceed per-origin quota ` + + `for its body of ${dataType}.`); +} + +// Test accumulated oversized request. +for (const dataType in BeaconDataType) { + test( + () => { + const controller = new AbortController(); + // Makes the 1st call that sends only half of allowed quota. + fetchLater('/fetch-later', { + method: 'POST', + signal: controller.signal, + body: makeBeaconData(generatePayload(kQuotaPerOrigin / 2), dataType) + }); + + // Makes the 2nd call that sends half+1 of allowed quota. + assert_throws_dom('QuotaExceededError', () => { + fetchLater('/fetch-later', { + method: 'POST', + signal: controller.signal, + body: makeBeaconData( + generatePayload(kQuotaPerOrigin / 2 + 1), dataType) + }); + }); + // Release quota taken by the pending requests for subsequent tests. + controller.abort(); + }, + `The 2nd fetchLater() call is not allowed to exceed per-origin quota ` + + `for its body of ${dataType}.`); +} + +// Test various size of payloads across different origins. +for (const dataType in BeaconDataType) { + test( + () => { + const controller = new AbortController(); + // Makes the 1st call that sends only half of allowed quota. + fetchLater('/fetch-later', { + method: 'POST', + signal: controller.signal, + body: makeBeaconData(generatePayload(kQuotaPerOrigin / 2), dataType) + }); + + // Makes the 2nd call that sends half+1 of allowed quota, but to a + // different origin. + fetchLater(`${HTTPS_NOTSAMESITE_ORIGIN}/fetch-later`, { + method: 'POST', + signal: controller.signal, + body: + makeBeaconData(generatePayload(kQuotaPerOrigin / 2 + 1), dataType) + }); + // Release quota taken by the pending requests for subsequent tests. + controller.abort(); + }, + `The 2nd fetchLater() call to another origin does not exceed per-origin` + + ` quota for its body of ${dataType}.`); +} diff --git a/test/wpt/tests/fetch/fetch-later/resources/fetch-later.html b/test/wpt/tests/fetch/fetch-later/resources/fetch-later.html new file mode 100644 index 00000000000..b569e1a076a --- /dev/null +++ b/test/wpt/tests/fetch/fetch-later/resources/fetch-later.html @@ -0,0 +1,14 @@ + + + + + + diff --git a/test/wpt/tests/fetch/fetch-later/resources/header-referrer-helper.js b/test/wpt/tests/fetch/fetch-later/resources/header-referrer-helper.js new file mode 100644 index 00000000000..374097614ae --- /dev/null +++ b/test/wpt/tests/fetch/fetch-later/resources/header-referrer-helper.js @@ -0,0 +1,39 @@ +'use strict'; + +// https://w3c.github.io/webappsec-referrer-policy/#determine-requests-referrer +const REFERRER_ORIGIN = self.location.origin + '/'; +const REFERRER_URL = self.location.href; + +function testReferrerHeader(id, host, expectedReferrer) { + const url = `${ + host}/beacon/resources/inspect-header.py?header=referer&cmd=put&id=${id}`; + + promise_test(t => { + fetchLater(url, {activateAfter: 0}); + return pollResult(expectedReferrer, id).then(result => { + assert_equals(result, expectedReferrer, 'Correct referrer header result'); + }); + }, `Test referer header ${host}`); +} + +function pollResult(expectedReferrer, id) { + const checkUrl = + `/beacon/resources/inspect-header.py?header=referer&cmd=get&id=${id}`; + + return new Promise(resolve => { + function checkResult() { + fetch(checkUrl).then(response => { + assert_equals( + response.status, 200, 'Inspect header response\'s status is 200'); + let result = response.headers.get('x-request-referer'); + + if (result != undefined) { + resolve(result); + } else { + step_timeout(checkResult.bind(this), 100); + } + }); + } + checkResult(); + }); +} diff --git a/test/wpt/tests/fetch/fetch-later/send-on-deactivate.tentative.https.window.js b/test/wpt/tests/fetch/fetch-later/send-on-deactivate.tentative.https.window.js new file mode 100644 index 00000000000..94877e8321a --- /dev/null +++ b/test/wpt/tests/fetch/fetch-later/send-on-deactivate.tentative.https.window.js @@ -0,0 +1,185 @@ +// META: script=/resources/testharness.js +// META: script=/resources/testharnessreport.js +// META: script=/common/dispatcher/dispatcher.js +// META: script=/common/get-host-info.sub.js +// META: script=/common/utils.js +// META: script=/html/browsers/browsing-the-web/remote-context-helper/resources/remote-context-helper.js +// META: script=/html/browsers/browsing-the-web/back-forward-cache/resources/rc-helper.js +// META: script=/pending-beacon/resources/pending_beacon-helper.js + +'use strict'; + +// NOTE: Due to the restriction of WPT runner, the following tests are all run +// with BackgroundSync off, which is different from some browsers, +// e.g. Chrome, default behavior, as the testing infra does not support enabling +// it. + +parallelPromiseTest(async t => { + const uuid = token(); + const url = generateSetBeaconURL(uuid); + // Sets no option to test the default behavior when a document enters BFCache. + const helper = new RemoteContextHelper(); + // Opens a window with noopener so that BFCache will work. + const rc1 = await helper.addWindow( + /*config=*/ null, /*options=*/ {features: 'noopener'}); + + // Creates a fetchLater request with default config in remote, which should + // only be sent on page discarded (not on entering BFCache). + await rc1.executeScript(url => { + fetchLater(url); + // Add a pageshow listener to stash the BFCache event. + window.addEventListener('pageshow', e => { + window.pageshowEvent = e; + }); + }, [url]); + // Navigates away to let page enter BFCache. + const rc2 = await rc1.navigateToNew(); + // Navigates back. + await rc2.historyBack(); + // Verifies the page was BFCached. + assert_true(await rc1.executeScript(() => { + return window.pageshowEvent.persisted; + })); + + // Theoretically, the request should still be pending thus 0 request received. + // However, 1 request is sent, as by default the WPT test runner, e.g. + // content_shell in Chromium, does not enable BackgroundSync permission, + // resulting in forcing request sending on every navigation. + await expectBeacon(uuid, {count: 1}); +}, `fetchLater() sends on page entering BFCache if BackgroundSync is off.`); + +parallelPromiseTest(async t => { + const uuid = token(); + const url = generateSetBeaconURL(uuid); + const helper = new RemoteContextHelper(); + // Opens a window with noopener so that BFCache will work. + const rc1 = await helper.addWindow( + /*config=*/ null, /*options=*/ {features: 'noopener'}); + + // When the remote is put into BFCached, creates a fetchLater request w/ + // activateAfter = 0s. It should be sent out immediately. + await rc1.executeScript(url => { + window.addEventListener('pagehide', e => { + if (e.persisted) { + fetchLater(url, {activateAfter: 0}); + } + }); + // Add a pageshow listener to stash the BFCache event. + window.addEventListener('pageshow', e => { + window.pageshowEvent = e; + }); + }, [url]); + // Navigates away to trigger request sending. + const rc2 = await rc1.navigateToNew(); + // Navigates back. + await rc2.historyBack(); + // Verifies the page was BFCached. + assert_true(await rc1.executeScript(() => { + return window.pageshowEvent.persisted; + })); + + // NOTE: In this case, it does not matter if BackgroundSync is on or off. + await expectBeacon(uuid, {count: 1}); +}, `Call fetchLater() when BFCached with activateAfter=0 sends immediately.`); + +parallelPromiseTest(async t => { + const uuid = token(); + const url = generateSetBeaconURL(uuid); + // Sets no option to test the default behavior when a document gets discarded + // on navigated away. + const helper = new RemoteContextHelper(); + // Opens a window without BFCache. + const rc1 = await helper.addWindow(); + + // Creates a fetchLater request in remote which should only be sent on + // navigating away. + await rc1.executeScript(url => { + fetchLater(url); + // Add a pageshow listener to stash the BFCache event. + window.addEventListener('pageshow', e => { + window.pageshowEvent = e; + }); + }, [url]); + // Navigates away to trigger request sending. + const rc2 = await rc1.navigateToNew(); + // Navigates back. + await rc2.historyBack(); + // Verifies the page was NOT BFCached. + assert_equals(undefined, await rc1.executeScript(() => { + return window.pageshowEvent; + })); + + // NOTE: In this case, it does not matter if BackgroundSync is on or off. + await expectBeacon(uuid, {count: 1}); +}, `fetchLater() sends on navigating away a page w/o BFCache.`); + +parallelPromiseTest(async t => { + const uuid = token(); + const url = generateSetBeaconURL(uuid); + // Sets no option to test the default behavior when a document gets discarded + // on navigated away. + const helper = new RemoteContextHelper(); + // Opens a window without BFCache. + const rc1 = await helper.addWindow(); + + // Creates 2 fetchLater requests in remote, and one of them is aborted + // immediately. The other one should only be sent right on navigating away. + await rc1.executeScript(url => { + const controller = new AbortController(); + fetchLater(url, {signal: controller.signal}); + fetchLater(url); + controller.abort(); + // Add a pageshow listener to stash the BFCache event. + window.addEventListener('pageshow', e => { + window.pageshowEvent = e; + }); + }, [url]); + // Navigates away to trigger request sending. + const rc2 = await rc1.navigateToNew(); + // Navigates back. + await rc2.historyBack(); + // Verifies the page was NOT BFCached. + assert_equals(undefined, await rc1.executeScript(() => { + return window.pageshowEvent; + })); + + // NOTE: In this case, it does not matter if BackgroundSync is on or off. + await expectBeacon(uuid, {count: 1}); +}, `fetchLater() does not send aborted request on navigating away a page w/o BFCache.`); + +parallelPromiseTest(async t => { + const uuid = token(); + const url = generateSetBeaconURL(uuid); + const options = {activateAfter: 60000}; + const helper = new RemoteContextHelper(); + // Opens a window with noopener so that BFCache will work. + const rc1 = await helper.addWindow( + /*config=*/ null, /*options=*/ {features: 'noopener'}); + + // Creates a fetchLater request in remote which should only be sent on + // navigating away. + await rc1.executeScript((url) => { + // Sets activateAfter = 1m to indicate the request should NOT be sent out + // immediately. + fetchLater(url, {activateAfter: 60000}); + // Adds a pageshow listener to stash the BFCache event. + window.addEventListener('pageshow', e => { + window.pageshowEvent = e; + }); + }, [url]); + // Navigates away to trigger request sending. + const rc2 = await rc1.navigateToNew(); + // Navigates back. + await rc2.historyBack(); + // Verifies the page was BFCached. + assert_true(await rc1.executeScript(() => { + return window.pageshowEvent.persisted; + })); + + // Theoretically, the request should still be pending thus 0 request received. + // However, 1 request is sent, as by default the WPT test runner, e.g. + // content_shell in Chromium, does not enable BackgroundSync permission, + // resulting in forcing request sending on every navigation, even if page is + // put into BFCache. + await expectBeacon(uuid, {count: 1}); +}, `fetchLater() with activateAfter=1m sends on page entering BFCache if BackgroundSync is off.`); diff --git a/test/wpt/tests/fetch/fetch-later/send-on-discard/not-send-after-abort.tentative.https.window.js b/test/wpt/tests/fetch/fetch-later/send-on-discard/not-send-after-abort.tentative.https.window.js new file mode 100644 index 00000000000..c49e0bde87b --- /dev/null +++ b/test/wpt/tests/fetch/fetch-later/send-on-discard/not-send-after-abort.tentative.https.window.js @@ -0,0 +1,25 @@ +// META: script=/resources/testharness.js +// META: script=/resources/testharnessreport.js +// META: script=/common/utils.js +// META: script=/pending-beacon/resources/pending_beacon-helper.js + +'use strict'; + +parallelPromiseTest(async t => { + const uuid = token(); + const url = generateSetBeaconURL(uuid); + + // Loads an iframe that creates 2 fetchLater requests. One of them is aborted. + const iframe = await loadScriptAsIframe(` + const url = '${url}'; + const controller = new AbortController(); + fetchLater(url, {signal: controller.signal}); + fetchLater(url, {method: 'POST'}); + controller.abort(); + `); + // Delete the iframe to trigger deferred request sending. + document.body.removeChild(iframe); + + // The iframe should not send the aborted request. + await expectBeacon(uuid, {count: 1}); +}, 'A discarded document does not send an already aborted fetchLater request.'); diff --git a/test/wpt/tests/fetch/fetch-later/send-on-discard/send-multiple-with-activate-after.tentative.https.window.js b/test/wpt/tests/fetch/fetch-later/send-on-discard/send-multiple-with-activate-after.tentative.https.window.js new file mode 100644 index 00000000000..03078b2b516 --- /dev/null +++ b/test/wpt/tests/fetch/fetch-later/send-on-discard/send-multiple-with-activate-after.tentative.https.window.js @@ -0,0 +1,32 @@ +// META: script=/resources/testharness.js +// META: script=/resources/testharnessreport.js +// META: script=/common/utils.js +// META: script=/pending-beacon/resources/pending_beacon-helper.js +// META: timeout=long + +'use strict'; + +parallelPromiseTest(async t => { + const uuid = token(); + const url = generateSetBeaconURL(uuid); + const numPerMethod = 20; + const total = numPerMethod * 2; + + // Loads an iframe that creates `numPerMethod` GET & POST fetchLater requests. + const iframe = await loadScriptAsIframe(` + const url = '${url}'; + for (let i = 0; i < ${numPerMethod}; i++) { + // Changing the URL of each request to avoid HTTP Cache issue. + // See crbug.com/1498203#c17. + fetchLater(url + "&method=GET&i=" + i, + {method: 'GET', activateAfter: 10000}); // 10s + fetchLater(url + "&method=POST&i=" + i, + {method: 'POST', activateAfter: 8000}); // 8s + } + `); + // Delete the iframe to trigger deferred request sending. + document.body.removeChild(iframe); + + // The iframe should have sent all requests. + await expectBeacon(uuid, {count: total}); +}, 'A discarded document sends all its fetchLater requests, no matter how much their activateAfter timeout remain.'); diff --git a/test/wpt/tests/fetch/fetch-later/sendondiscard.tentative.https.window.js b/test/wpt/tests/fetch/fetch-later/send-on-discard/send-multiple.tentative.https.window.js similarity index 69% rename from test/wpt/tests/fetch/fetch-later/sendondiscard.tentative.https.window.js rename to test/wpt/tests/fetch/fetch-later/send-on-discard/send-multiple.tentative.https.window.js index 0613d18dffb..25ce98d446e 100644 --- a/test/wpt/tests/fetch/fetch-later/sendondiscard.tentative.https.window.js +++ b/test/wpt/tests/fetch/fetch-later/send-on-discard/send-multiple.tentative.https.window.js @@ -2,6 +2,7 @@ // META: script=/resources/testharnessreport.js // META: script=/common/utils.js // META: script=/pending-beacon/resources/pending_beacon-helper.js +// META: timeout=long 'use strict'; @@ -13,16 +14,17 @@ parallelPromiseTest(async t => { // Loads an iframe that creates `numPerMethod` GET & POST fetchLater requests. const iframe = await loadScriptAsIframe(` - const url = "${url}"; + const url = '${url}'; for (let i = 0; i < ${numPerMethod}; i++) { - let get = fetchLater(url); - let post = fetchLater(url, {method: 'POST'}); + // Changing the URL of each request to avoid HTTP Cache issue. + // See crbug.com/1498203#c17. + fetchLater(url + "&method=GET&i=" + i); + fetchLater(url + "&method=POST&i=" + i, {method: 'POST'}); } `); - // Delete the iframe to trigger deferred request sending. document.body.removeChild(iframe); // The iframe should have sent all requests. await expectBeacon(uuid, {count: total}); -}, 'A discarded document sends all its fetchLater requests with default config.'); +}, 'A discarded document sends all its fetchLater requests.'); diff --git a/test/wpt/tests/fetch/metadata/portal.https.sub.html b/test/wpt/tests/fetch/metadata/portal.https.sub.html deleted file mode 100644 index 55b555a1b8e..00000000000 --- a/test/wpt/tests/fetch/metadata/portal.https.sub.html +++ /dev/null @@ -1,50 +0,0 @@ - - - - - - - - - - diff --git a/test/wpt/tests/fetch/orb/resources/utils.js b/test/wpt/tests/fetch/orb/resources/utils.js index 94a2177f079..45fbc4cb38e 100644 --- a/test/wpt/tests/fetch/orb/resources/utils.js +++ b/test/wpt/tests/fetch/orb/resources/utils.js @@ -10,9 +10,92 @@ function contentTypeOptions(type) { return header("X-Content-Type-Options", type); } -function fetchORB(file, options, ...pipe) { - return fetch(`${file}${pipe.length ? `?pipe=${pipe.join("|")}` : ""}`, { - ...(options || {}), +function testFetchNoCors(_t, path, { headers }) { + return fetch(path, { + ...(headers ? { headers } : {}), mode: "no-cors", }); } + +function testElementInitiator(t, path, name) { + let element = document.createElement(name); + element.src = path; + t.add_cleanup(() => element.remove()); + return new Promise((resolve, reject) => { + element.onerror = e => reject(new TypeError()); + element.onload = resolve; + + document.body.appendChild(element); + }); +} + +function testImageInitiator(t, path) { + return testElementInitiator(t, path, "img"); +} + +function testAudioInitiator(t, path) { + return testElementInitiator(t, path, "audio"); +} + +function testVideoInitiator(t, path) { + return testElementInitiator(t, path, "video"); +} + +function testScriptInitiator(t, path) { + return testElementInitiator(t, path, "script"); +} + +function runTest(t, test, file, options, ...pipe) { + const path = `${file}${pipe.length ? `?pipe=${pipe.join("|")}` : ""}`; + return test(t, path, options) +} + +function testRunAll(file, testCallback, adapter, options) { + let testcase = function (test, message, skip) { + return {test, message, skip}; + }; + + const name = "..."; + [ testcase(testFetchNoCors, `fetch(${name}, {mode: "no-cors"})`, false || options.skip.includes("fetch")), + testcase(testImageInitiator, ``, options.onlyFetch || options.skip.includes("image")), + testcase(testAudioInitiator, `