From 6763bc82cea5e8d05fe8b4d96c4852c9440e071d Mon Sep 17 00:00:00 2001 From: Antonio Barcelos Date: Mon, 14 Mar 2022 15:27:27 +0100 Subject: [PATCH 01/35] Add Testkit support to DenoJS Enabling the support for the acceptance tests to the generated DenoJS driver is a step towards to release it as library. --- .../src/channel/browser/browser-channel.js | 48 +++--- packages/testkit-backend/package.json | 6 +- .../src/cypher-native-binders.js | 2 +- .../testkit-backend/src/feature/common.js | 5 +- packages/testkit-backend/src/feature/index.js | 6 +- packages/testkit-backend/src/index.deno.ts | 138 ++++++++++++++++++ packages/testkit-backend/src/neo4j.deno.js | 3 + .../testkit-backend/src/request-handlers.js | 4 +- .../src/skipped-tests/browser.js | 2 +- .../src/skipped-tests/common.js | 17 +-- .../src/skipped-tests/index.js | 6 +- .../testkit-backend/src/skipped-tests/rx.js | 2 +- 12 files changed, 191 insertions(+), 48 deletions(-) create mode 100644 packages/testkit-backend/src/index.deno.ts create mode 100644 packages/testkit-backend/src/neo4j.deno.js diff --git a/packages/bolt-connection/src/channel/browser/browser-channel.js b/packages/bolt-connection/src/channel/browser/browser-channel.js index 8959e6e44..532fcbefa 100644 --- a/packages/bolt-connection/src/channel/browser/browser-channel.js +++ b/packages/bolt-connection/src/channel/browser/browser-channel.js @@ -144,17 +144,26 @@ export default class WebSocketChannel { if (this._pending !== null) { this._pending.push(buffer) } else if (buffer instanceof ChannelBuffer) { - try { - this._ws.send(buffer._buffer) - } catch (error) { - if (this._ws.readyState !== WS_OPEN) { - // Websocket has been closed - this._handleConnectionError() - } else { - // Some other error occured - throw error + // We should wait for the connection state change before sending + setTimeout(() => { + try { + if (this._ws.readyState !== WS_OPEN){ + console.log('WebSocket not open') + return; + } + this._ws.send(buffer._buffer) + } catch (error) { + if (this._ws.readyState !== WS_OPEN) { + // Websocket has been closed + this._handleConnectionError() + } else { + console.log('errror', error) + // Some other error occured + throw error + } } - } + }, 500) + console.log('after timeout') } else { throw newError("Don't know how to send buffer: " + buffer) } @@ -166,14 +175,17 @@ export default class WebSocketChannel { */ close () { return new Promise((resolve, reject) => { - if (this._ws && this._ws.readyState !== WS_CLOSED) { - this._open = false - this._clearConnectionTimeout() - this._ws.onclose = () => resolve() - this._ws.close() - } else { - resolve() - } + setTimeout(() => { + if (this._ws && this._ws.readyState !== WS_CLOSED && this._ws.readyState !== WS_CLOSING) { + this._open = false + this._clearConnectionTimeout() + this._ws.onclose = () => resolve() + this._ws.close() + } else { + resolve() + } + }, 500) + }) } diff --git a/packages/testkit-backend/package.json b/packages/testkit-backend/package.json index c00e95f9d..c5e4bd015 100644 --- a/packages/testkit-backend/package.json +++ b/packages/testkit-backend/package.json @@ -10,8 +10,10 @@ }, "type": "module", "scripts": { - "build": "rollup src/index.js --config rollup.config.js", - "start": "node --version | grep -q v10. && node -r esm src/index.js || node --experimental-specifier-resolution=node src/index.js", + "build": "echo 'Error: no build script specified'", + "build::": "rollup src/index.js --config rollup.config.js", + "start::": "node --version | grep -q v10. && node -r esm src/index.js || node --experimental-specifier-resolution=node src/index.js", + "start": "deno run --allow-read --allow-write --allow-net --allow-env --allow-run src/index.deno.ts", "clean": "rm -fr node_modules public/index.js", "prepare": "npm run build", "node": "node" diff --git a/packages/testkit-backend/src/cypher-native-binders.js b/packages/testkit-backend/src/cypher-native-binders.js index f7de792d9..2bd03a0e6 100644 --- a/packages/testkit-backend/src/cypher-native-binders.js +++ b/packages/testkit-backend/src/cypher-native-binders.js @@ -1,4 +1,4 @@ -import neo4j from './neo4j' +import neo4j from './neo4j.deno.js' export function valueResponse (name, value) { return { name: name, data: { value: value } } diff --git a/packages/testkit-backend/src/feature/common.js b/packages/testkit-backend/src/feature/common.js index d941af6e2..18bf3d2d1 100644 --- a/packages/testkit-backend/src/feature/common.js +++ b/packages/testkit-backend/src/feature/common.js @@ -1,6 +1,9 @@ -import tls from 'tls' + const SUPPORTED_TLS = (() => { + const tls = { + DEFAULT_MAX_VERSION: false + } if (tls.DEFAULT_MAX_VERSION) { const min = Number(tls.DEFAULT_MIN_VERSION.split('TLSv')[1]) const max = Number(tls.DEFAULT_MAX_VERSION.split('TLSv')[1]) diff --git a/packages/testkit-backend/src/feature/index.js b/packages/testkit-backend/src/feature/index.js index 1e95fa4ec..49e486a0d 100644 --- a/packages/testkit-backend/src/feature/index.js +++ b/packages/testkit-backend/src/feature/index.js @@ -1,6 +1,6 @@ -import commonFeatures from './common' -import rxFeatures from './rx' -import asyncFeatures from './async' +import commonFeatures from './common.js' +import rxFeatures from './rx.js' +import asyncFeatures from './async.js' const featuresByContext = new Map([ ['async', asyncFeatures], diff --git a/packages/testkit-backend/src/index.deno.ts b/packages/testkit-backend/src/index.deno.ts new file mode 100644 index 000000000..ab7c05b9f --- /dev/null +++ b/packages/testkit-backend/src/index.deno.ts @@ -0,0 +1,138 @@ +import Context from './context.js'; +import { getShouldRunTest } from './skipped-tests/index.js'; +import neo4j from "../../neo4j-driver-deno/lib/mod.ts"; +import { createGetFeatures } from './feature/index.js'; +import * as handlers from './request-handlers.js'; + +const listener = Deno.listen({ port: 9876 }); +let index = 0; +const contexts = new Map(); + +interface RequestHandler { + (c: Context, data: any, wire: any): void +} + +interface RequestHandlerMap { + [key: string]: RequestHandler +} + +addEventListener('uncaughtException', (event) => { + console.log('unhandled rejection', event); +}) + +// @ts-ignore +const requestHandlers: RequestHandlerMap = handlers as RequestHandlerMap + +addEventListener('events.errorMonitor', (event) => { + console.log('something here ========================') +}) + +interface Backend { + openContext(contextId: number): void + closeContext(contextId: number): void + handle(contextId: number, conn: Deno.Conn, request: { name: string, data: object }): void +} + +function write(conn: Deno.Conn, response: object) { + const responseStr = JSON.stringify(response, (_, value) => + typeof value === 'bigint' ? `${value}n` : value + ) + const responseArr = ['#response begin', responseStr, '#response end'].join('\n') + '\n' + console.log('response', responseArr); + conn.write(new TextEncoder().encode(responseArr)) + .catch(e => { + console.log('error writing to connection', e); + }); +} + +const descriptor = [] as string[] +const shouldRunTest = getShouldRunTest(descriptor); +const getFeatures = createGetFeatures(descriptor); + +const backend: Backend = { + openContext: (contextId) => { + console.log("Open context:", contextId); + contexts.set(contextId, new Context(shouldRunTest, getFeatures)); + }, + closeContext: (contextId) => { + console.log('Close context', contextId); + contexts.delete(contextId); + }, + handle: (contextId, conn, req) => { + const { data, name } = req; + if (!contexts.has(contextId)) { + throw new Error(`Context ${contextId} does not exist`) + } else if (!(name in requestHandlers)) { + console.log('Unknown request: ' + name) + throw new Error(`Unknown request: ${name}`) + } + + console.log('Handle', req.name, req.data); + + const handler: (c: Context, data: any, wire: any) => void = requestHandlers[name as string]; + + handler(contexts.get(contextId)!!, data, { + writeResponse: (response: object) => write(conn, response), + writeError: (e: Error) => { + console.log('writeError', e); + if (e.name && e instanceof neo4j.Neo4jError) { + if (contexts.has(contextId)) { + const id = contexts.get(contextId)!!.addError(e) + write(conn, { name: 'DriverError', data: { id, msg: e.message + ' (' + e.code + ')', code: e.code } }) + return + } else { + console.log('Context does not exist', contextId, e) + } + return + } + write(conn, { name: 'BackendError', data: { msg: e.message } }) + + }, + writeBackendError: (msg: string) => write(conn, { name: 'BackendError', data: { msg } }) + }); + } +} + +for await (const conn of listener) { + const contextId = index++; + handleConnection(conn, contextId) +} + +async function handleConnection( conn: Deno.Conn, contextId: number): Promise { + backend.openContext(contextId); + let inRequest = false + let requestString = '' + for await (const message of Deno.iter(conn)) { + const rawTxtMessage = new TextDecoder().decode(message); + const lines = rawTxtMessage.split('\n'); + for (const line of lines) { + switch (line) { + case '#request begin': + if(inRequest) { + throw new Error('Already in request'); + } + inRequest = true; + break; + case '#request end': + if(!inRequest) { + throw new Error('Not in request'); + } + const request = JSON.parse(requestString); + backend.handle(contextId, conn, request); + inRequest = false; + requestString = ''; + break + case '': + // ignore empty lines + break; + default: + if(!inRequest) { + throw new Error('Not in request'); + } + requestString += line; + break + } + } + } + backend.closeContext(contextId); +} diff --git a/packages/testkit-backend/src/neo4j.deno.js b/packages/testkit-backend/src/neo4j.deno.js new file mode 100644 index 000000000..76bb25e5a --- /dev/null +++ b/packages/testkit-backend/src/neo4j.deno.js @@ -0,0 +1,3 @@ +import neo4j from '../../neo4j-driver-deno/lib/mod.ts' + +export default neo4j diff --git a/packages/testkit-backend/src/request-handlers.js b/packages/testkit-backend/src/request-handlers.js index b845c42b1..3af193249 100644 --- a/packages/testkit-backend/src/request-handlers.js +++ b/packages/testkit-backend/src/request-handlers.js @@ -1,4 +1,4 @@ -import neo4j from './neo4j' +import neo4j from './neo4j.deno.js' import { cypherToNative } from './cypher-native-binders.js' import * as responses from './responses.js' @@ -52,7 +52,7 @@ export function NewDriver (context, data, wire) { userAgent, resolver, useBigInt: true, - logging: neo4j.logging.console(process.env.LOG_LEVEL || context.logLevel) + logging: neo4j.logging.console('debug') } if ('encrypted' in data) { config.encrypted = data.encrypted ? 'ENCRYPTION_ON' : 'ENCRYPTION_OFF' diff --git a/packages/testkit-backend/src/skipped-tests/browser.js b/packages/testkit-backend/src/skipped-tests/browser.js index 788e1657a..b8313e21d 100644 --- a/packages/testkit-backend/src/skipped-tests/browser.js +++ b/packages/testkit-backend/src/skipped-tests/browser.js @@ -1,4 +1,4 @@ -import skip, { ifEndsWith, ifEquals, ifStartsWith } from './skip' +import skip, { ifEndsWith, ifEquals, ifStartsWith } from './skip.js' const skippedTests = [ skip( "Browser doesn't support socket timeouts", diff --git a/packages/testkit-backend/src/skipped-tests/common.js b/packages/testkit-backend/src/skipped-tests/common.js index bbe75f9be..385218ba5 100644 --- a/packages/testkit-backend/src/skipped-tests/common.js +++ b/packages/testkit-backend/src/skipped-tests/common.js @@ -1,21 +1,6 @@ -import skip, { ifEquals, ifEndsWith, endsWith, ifStartsWith, startsWith, not } from './skip' +import skip, { ifEquals, ifEndsWith } from './skip' const skippedTests = [ - skip( - 'Driver does not return offset for old DateTime implementations', - ifStartsWith('stub.types.test_temporal_types.TestTemporalTypes') - .and(not(startsWith('stub.types.test_temporal_types.TestTemporalTypesV5'))) - .and(endsWith('test_zoned_date_time')), - ifEquals('neo4j.datatypes.test_temporal_types.TestDataTypes.test_nested_datetime'), - ifEquals('neo4j.datatypes.test_temporal_types.TestDataTypes.test_should_echo_all_timezone_ids'), - ifEquals('neo4j.datatypes.test_temporal_types.TestDataTypes.test_cypher_created_datetime') - ), - skip( - 'Using numbers out of bound', - ifEquals('neo4j.datatypes.test_temporal_types.TestDataTypes.test_should_echo_temporal_type'), - ifEquals('neo4j.datatypes.test_temporal_types.TestDataTypes.test_nested_duration'), - ifEquals('neo4j.datatypes.test_temporal_types.TestDataTypes.test_duration_components') - ), skip( 'Testkit implemenation is deprecated', ifEquals('stub.basic_query.test_basic_query.TestBasicQuery.test_5x0_populates_node_only_element_id'), diff --git a/packages/testkit-backend/src/skipped-tests/index.js b/packages/testkit-backend/src/skipped-tests/index.js index e1ee901be..2ef02a884 100644 --- a/packages/testkit-backend/src/skipped-tests/index.js +++ b/packages/testkit-backend/src/skipped-tests/index.js @@ -1,6 +1,6 @@ -import commonSkippedTests from './common' -import browserSkippedTests from './browser' -import rxSessionSkippedTests from './rx' +import commonSkippedTests from './common.js' +import browserSkippedTests from './browser.js' +import rxSessionSkippedTests from './rx.js' const skippedTestsByContext = new Map([ ['browser', browserSkippedTests], diff --git a/packages/testkit-backend/src/skipped-tests/rx.js b/packages/testkit-backend/src/skipped-tests/rx.js index 0d0be9514..558e17f4e 100644 --- a/packages/testkit-backend/src/skipped-tests/rx.js +++ b/packages/testkit-backend/src/skipped-tests/rx.js @@ -1,4 +1,4 @@ -import { skip, ifEquals } from './skip' +import { skip, ifEquals } from './skip.js' const skippedTests = [ skip( From 0dff616696d4c65e997c765e4ec23d8f19a50ad0 Mon Sep 17 00:00:00 2001 From: Antonio Barcelos Date: Fri, 23 Sep 2022 17:46:50 +0200 Subject: [PATCH 02/35] Add `deno-channel` for non-encrypted connection and fix deno-backend : --- packages/core/src/transaction.ts | 1 + packages/neo4j-driver-deno/generate.ts | 6 + .../neo4j-driver-deno/src/deno-channel.js | 286 ++++++++++++++++++ packages/testkit-backend/src/index.deno.ts | 104 ++++--- .../src/skipped-tests/common.js | 17 +- 5 files changed, 371 insertions(+), 43 deletions(-) create mode 100644 packages/neo4j-driver-deno/src/deno-channel.js diff --git a/packages/core/src/transaction.ts b/packages/core/src/transaction.ts index 45d3c9ba9..57b775332 100644 --- a/packages/core/src/transaction.ts +++ b/packages/core/src/transaction.ts @@ -109,6 +109,7 @@ class Transaction { this._lowRecordWatermak = lowRecordWatermark this._highRecordWatermark = highRecordWatermark this._bookmarks = Bookmarks.empty() + this._acceptActive = () => { } // satisfy DenoJS this._activePromise = new Promise((resolve, reject) => { this._acceptActive = resolve }) diff --git a/packages/neo4j-driver-deno/generate.ts b/packages/neo4j-driver-deno/generate.ts index a323a62a1..1f7a5d983 100644 --- a/packages/neo4j-driver-deno/generate.ts +++ b/packages/neo4j-driver-deno/generate.ts @@ -164,6 +164,12 @@ await Deno.writeTextFile( `export default "${version}" // Specified using --version when running generate.ts\n`, ); +// Copy custom files +await Deno.copyFile( + "src/deno-channel.js", + join(rootOutDir, "/bolt-connection/channel/browser/browser-channel.js")); + + //////////////////////////////////////////////////////////////////////////////// // Warnings show up at the end if (!doTransform) { diff --git a/packages/neo4j-driver-deno/src/deno-channel.js b/packages/neo4j-driver-deno/src/deno-channel.js new file mode 100644 index 000000000..35b5f9110 --- /dev/null +++ b/packages/neo4j-driver-deno/src/deno-channel.js @@ -0,0 +1,286 @@ +/** + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +/* eslint-env browser */ +import ChannelBuffer from '../channel-buf.js' +import { newError, internal } from '../../../core/index.ts' + +const { + util: { ENCRYPTION_OFF, ENCRYPTION_ON } +} = internal + +let _CONNECTION_IDGEN = 0 +/** + * Create a new DenoChannel to be used in web browsers. + * @access private + */ +export default class DenoChannel { + /** + * Create new instance + * @param {ChannelConfig} config - configuration for this channel. + * @param {function(): string} protocolSupplier - function that detects protocol of the web page. Should only be used in tests. + */ + constructor ( + config + ) { + this.id = _CONNECTION_IDGEN++ + this._conn = null + this._pending = [] + this._error = null + this._open = true + this._config = config + + this._receiveTimeout = null + this._receiveTimeoutStarted = false + this._receiveTimeoutId = null + + this._connectionErrorCode = config.connectionErrorCode + this._handleConnectionError = this._handleConnectionError.bind(this) + this._handleConnectionTerminated = this._handleConnectionTerminated.bind( + this + ) + + this._socketPromise = Deno.connect({ + hostname: config.address.host(), + port: config.address.port() + }) + .then(conn => { + this._clearConnectionTimeout() + if (!this._open) { + return conn.close() + } + this._conn = conn + + setupReader(this) + .catch(this._handleConnectionError) + + const pending = this._pending + this._pending = null + for (let i = 0; i < pending.length; i++) { + this.write(pending[i]) + } + }) + .catch(this._handleConnectionError) + + this._connectionTimeoutFired = false + this._connectionTimeoutId = this._setupConnectionTimeout() + } + + _setupConnectionTimeout () { + const timeout = this._config.connectionTimeout + if (timeout) { + return setTimeout(() => { + this._connectionTimeoutFired = true + this.close() + .catch(this._handleConnectionError) + }, timeout) + } + return null + } + + /** + * Remove active connection timeout, if any. + * @private + */ + _clearConnectionTimeout () { + const timeoutId = this._connectionTimeoutId + if (timeoutId !== null) { + this._connectionTimeoutFired = false + this._connectionTimeoutId = null + clearTimeout(timeoutId) + } + } + + _handleConnectionError (err) { + let msg = + 'Failed to connect to server. ' + + 'Please ensure that your database is listening on the correct host and port ' + + 'and that you have compatible encryption settings both on Neo4j server and driver. ' + + 'Note that the default encryption setting has changed in Neo4j 4.0.' + if (err.message) msg += ' Caused by: ' + err.message + this._error = newError(msg, this._connectionErrorCode) + if (this.onerror) { + this.onerror(this._error) + } + } + + _handleConnectionTerminated () { + this._open = false + this._error = newError( + 'Connection was closed by server', + this._connectionErrorCode + ) + if (this.onerror) { + this.onerror(this._error) + } + } + + + /** + * Write the passed in buffer to connection + * @param {ChannelBuffer} buffer - Buffer to write + */ + write (buffer) { + if (this._pending !== null) { + this._pending.push(buffer) + } else if (buffer instanceof ChannelBuffer) { + this._conn.write(buffer._buffer) + } else { + throw newError("Don't know how to send buffer: " + buffer) + } + } + + /** + * Close the connection + * @returns {Promise} A promise that will be resolved after channel is closed + */ + async close () { + if (this._open) { + this._open = false + if (this._conn != null) { + await this._conn.close() + } + } + } + + /** + * Setup the receive timeout for the channel. + * + * Not supported for the browser channel. + * + * @param {number} receiveTimeout The amount of time the channel will keep without receive any data before timeout (ms) + * @returns {void} + */ + setupReceiveTimeout (receiveTimeout) { + this._receiveTimeout = receiveTimeout + } + + /** + * Stops the receive timeout for the channel. + */ + stopReceiveTimeout () { + if (this._receiveTimeout !== null && this._receiveTimeoutStarted) { + this._receiveTimeoutStarted = false + if (this._receiveTimeoutId != null) { + clearTimeout(this._receiveTimeoutId) + } + this._receiveTimeoutId = null + } + } + + /** + * Start the receive timeout for the channel. + */ + startReceiveTimeout () { + if (this._receiveTimeout !== null && !this._receiveTimeoutStarted) { + this._receiveTimeoutStarted = true + this._resetTimeout() + } + } + + _resetTimeout () { + if (!this._receiveTimeoutStarted) { + return + } + + if (this._receiveTimeoutId !== null) { + clearTimeout(this._receiveTimeoutId) + } + + this._receiveTimeoutId = setTimeout(() => { + this._receiveTimeoutId = null + this._timedout = true + this.stopReceiveTimeout() + this._error = newError( + `Connection lost. Server didn't respond in ${this._receiveTimeout}ms`, + this._config.connectionErrorCode + ) + + this.close() + if (this.onerror) { + this.onerror(this._error) + } + }, this._receiveTimeout) + } +} + +async function setupReader (channel) { + try { + for await (const message of Deno.iter(channel._conn)) { + channel._resetTimeout() + + if (!channel._open) { + return + } + if (channel.onmessage) { + channel.onmessage(new ChannelBuffer(message)) + } + } + channel._handleConnectionTerminated() + } catch (error) { + if (channel._open) { + channel._handleConnectionError(error) + } + } +} + +/** + * @param {ChannelConfig} config - configuration for the channel. + * @return {boolean} `true` if encryption enabled in the config, `false` otherwise. + */ +function isEncryptionExplicitlyTurnedOn (config) { + return config.encrypted === true || config.encrypted === ENCRYPTION_ON +} + +/** + * @param {ChannelConfig} config - configuration for the channel. + * @return {boolean} `true` if encryption disabled in the config, `false` otherwise. + */ +function isEncryptionExplicitlyTurnedOff (config) { + return config.encrypted === false || config.encrypted === ENCRYPTION_OFF +} + +/** + * @param {function(): string} protocolSupplier - function that detects protocol of the web page. + * @return {boolean} `true` if protocol returned by the given function is secure, `false` otherwise. + */ +function isProtocolSecure (protocolSupplier) { + const protocol = + typeof protocolSupplier === 'function' ? protocolSupplier() : '' + return protocol && protocol.toLowerCase().indexOf('https') >= 0 +} + +function verifyEncryptionSettings (encryptionOn, encryptionOff, secureProtocol) { + if (secureProtocol === null) { + // do nothing sice the protocol could not be identified + } else if (encryptionOn && !secureProtocol) { + // encryption explicitly turned on for a driver used on a HTTP web page + console.warn( + 'Neo4j driver is configured to use secure WebSocket on a HTTP web page. ' + + 'WebSockets might not work in a mixed content environment. ' + + 'Please consider configuring driver to not use encryption.' + ) + } else if (encryptionOff && secureProtocol) { + // encryption explicitly turned off for a driver used on a HTTPS web page + console.warn( + 'Neo4j driver is configured to use insecure WebSocket on a HTTPS web page. ' + + 'WebSockets might not work in a mixed content environment. ' + + 'Please consider configuring driver to use encryption.' + ) + } +} diff --git a/packages/testkit-backend/src/index.deno.ts b/packages/testkit-backend/src/index.deno.ts index ab7c05b9f..e701a59ae 100644 --- a/packages/testkit-backend/src/index.deno.ts +++ b/packages/testkit-backend/src/index.deno.ts @@ -39,10 +39,19 @@ function write(conn: Deno.Conn, response: object) { ) const responseArr = ['#response begin', responseStr, '#response end'].join('\n') + '\n' console.log('response', responseArr); - conn.write(new TextEncoder().encode(responseArr)) - .catch(e => { - console.log('error writing to connection', e); - }); + const buffer = new Uint8Array(responseArr.length); + new TextEncoder().encodeInto(responseArr, buffer); + + async function writeBuffer(buff: Uint8Array, size: number) { + let bytesWritten = 0; + while(bytesWritten < size) { + const writtenInSep = await conn.write(buff.slice(bytesWritten)); + bytesWritten += writtenInSep; + } + } + + writeBuffer(buffer, buffer.byteLength); + } const descriptor = [] as string[] @@ -74,19 +83,25 @@ const backend: Backend = { handler(contexts.get(contextId)!!, data, { writeResponse: (response: object) => write(conn, response), writeError: (e: Error) => { - console.log('writeError', e); - if (e.name && e instanceof neo4j.Neo4jError) { - if (contexts.has(contextId)) { - const id = contexts.get(contextId)!!.addError(e) - write(conn, { name: 'DriverError', data: { id, msg: e.message + ' (' + e.code + ')', code: e.code } }) - return + console.error(e) + if (e.name) { + if (e.message === 'TestKit FrontendError') { + write(conn, { name: 'FrontendError', data: { + msg: 'Simulating the client code throwing some error.' + }}) } else { - console.log('Context does not exist', contextId, e) + const id = contexts.get(contextId)?.addError(e) + write(conn, { name: 'DriverError', data: { + id, + msg: e.message, + // @ts-ignore + code: e.code + }}) } return } - write(conn, { name: 'BackendError', data: { msg: e.message } }) - + const msg = e.message + write(conn, { name: 'BackendError', data: { msg } }) }, writeBackendError: (msg: string) => write(conn, { name: 'BackendError', data: { msg } }) }); @@ -102,37 +117,42 @@ async function handleConnection( conn: Deno.Conn, contextId: number): Promise Date: Mon, 26 Sep 2022 16:18:31 +0200 Subject: [PATCH 03/35] Add support to TLS --- .../neo4j-driver-deno/src/deno-channel.js | 146 ++++++++++++------ packages/testkit-backend/src/feature/deno.js | 7 + packages/testkit-backend/src/feature/index.js | 4 +- packages/testkit-backend/src/index.deno.ts | 2 +- .../testkit-backend/src/skipped-tests/deno.js | 17 ++ .../src/skipped-tests/index.js | 4 +- testkit/Dockerfile | 2 + 7 files changed, 128 insertions(+), 54 deletions(-) create mode 100644 packages/testkit-backend/src/feature/deno.js create mode 100644 packages/testkit-backend/src/skipped-tests/deno.js diff --git a/packages/neo4j-driver-deno/src/deno-channel.js b/packages/neo4j-driver-deno/src/deno-channel.js index 35b5f9110..ddd263c68 100644 --- a/packages/neo4j-driver-deno/src/deno-channel.js +++ b/packages/neo4j-driver-deno/src/deno-channel.js @@ -33,10 +33,10 @@ export default class DenoChannel { /** * Create new instance * @param {ChannelConfig} config - configuration for this channel. - * @param {function(): string} protocolSupplier - function that detects protocol of the web page. Should only be used in tests. */ constructor ( - config + config, + connect = _connect ) { this.id = _CONNECTION_IDGEN++ this._conn = null @@ -55,10 +55,7 @@ export default class DenoChannel { this ) - this._socketPromise = Deno.connect({ - hostname: config.address.host(), - port: config.address.port() - }) + this._socketPromise = connect(config) .then(conn => { this._clearConnectionTimeout() if (!this._open) { @@ -219,6 +216,98 @@ export default class DenoChannel { } } +const TrustStrategy = { + TRUST_CUSTOM_CA_SIGNED_CERTIFICATES: async function (config) { + if ( + !config.trustedCertificates || + config.trustedCertificates.length === 0 + ) { + throw newError( + 'You are using TRUST_CUSTOM_CA_SIGNED_CERTIFICATES as the method ' + + 'to verify trust for encrypted connections, but have not configured any ' + + 'trustedCertificates. You must specify the path to at least one trusted ' + + 'X.509 certificate for this to work. Two other alternatives is to use ' + + 'TRUST_ALL_CERTIFICATES or to disable encryption by setting encrypted="' + + ENCRYPTION_OFF + + '"' + + 'in your driver configuration.' + ); + } + + const caCerts = await Promise.all( + config.trustedCertificates.map(f => Deno.readTextFile(f)) + ) + + return Deno.connectTls({ + hostname: config.address.resolvedHost(), + port: config.address.port(), + caCerts + }) + }, + TRUST_SYSTEM_CA_SIGNED_CERTIFICATES: async function (config) { + return Deno.connectTls({ + hostname: config.address.resolvedHost(), + port: config.address.port() + }) + }, + TRUST_ALL_CERTIFICATES: async function (config) { + throw newError( + `"${config.trust}" is not available in DenoJS. ` + + 'For trust in any certificates, you should use the DenoJS flag ' + + '"--unsafely-ignore-certificate-errors". '+ + 'See, https://deno.com/blog/v1.13#disable-tls-verification' + ) + } +} + +async function _connect (config) { + if (!isEncrypted(config)) { + return Deno.connect({ + hostname: config.address.resolvedHost(), + port: config.address.port() + }) + } + const trustStrategyName = getTrustStrategyName(config) + const trustStrategy = TrustStrategy[trustStrategyName] + + if (trustStrategy != null) { + return await trustStrategy(config) + } + + throw newError( + 'Unknown trust strategy: ' + + config.trust + + '. Please use either ' + + "trust:'TRUST_CUSTOM_CA_SIGNED_CERTIFICATES' configuration " + + 'or the System CA. ' + + 'Alternatively, you can disable encryption by setting ' + + '`encrypted:"' + + ENCRYPTION_OFF + + '"`. There is no mechanism to use encryption without trust verification, ' + + 'because this incurs the overhead of encryption without improving security. If ' + + 'the driver does not verify that the peer it is connected to is really Neo4j, it ' + + 'is very easy for an attacker to bypass the encryption by pretending to be Neo4j.' + + ) +} + +function isEncrypted (config) { + const encryptionNotConfigured = + config.encrypted == null || config.encrypted === undefined + if (encryptionNotConfigured) { + // default to using encryption if trust-all-certificates is available + return false + } + return config.encrypted === true || config.encrypted === ENCRYPTION_ON +} + +function getTrustStrategyName (config) { + if (config.trust) { + return config.trust + } + return 'TRUST_SYSTEM_CA_SIGNED_CERTIFICATES' +} + async function setupReader (channel) { try { for await (const message of Deno.iter(channel._conn)) { @@ -239,48 +328,3 @@ async function setupReader (channel) { } } -/** - * @param {ChannelConfig} config - configuration for the channel. - * @return {boolean} `true` if encryption enabled in the config, `false` otherwise. - */ -function isEncryptionExplicitlyTurnedOn (config) { - return config.encrypted === true || config.encrypted === ENCRYPTION_ON -} - -/** - * @param {ChannelConfig} config - configuration for the channel. - * @return {boolean} `true` if encryption disabled in the config, `false` otherwise. - */ -function isEncryptionExplicitlyTurnedOff (config) { - return config.encrypted === false || config.encrypted === ENCRYPTION_OFF -} - -/** - * @param {function(): string} protocolSupplier - function that detects protocol of the web page. - * @return {boolean} `true` if protocol returned by the given function is secure, `false` otherwise. - */ -function isProtocolSecure (protocolSupplier) { - const protocol = - typeof protocolSupplier === 'function' ? protocolSupplier() : '' - return protocol && protocol.toLowerCase().indexOf('https') >= 0 -} - -function verifyEncryptionSettings (encryptionOn, encryptionOff, secureProtocol) { - if (secureProtocol === null) { - // do nothing sice the protocol could not be identified - } else if (encryptionOn && !secureProtocol) { - // encryption explicitly turned on for a driver used on a HTTP web page - console.warn( - 'Neo4j driver is configured to use secure WebSocket on a HTTP web page. ' + - 'WebSockets might not work in a mixed content environment. ' + - 'Please consider configuring driver to not use encryption.' - ) - } else if (encryptionOff && secureProtocol) { - // encryption explicitly turned off for a driver used on a HTTPS web page - console.warn( - 'Neo4j driver is configured to use insecure WebSocket on a HTTPS web page. ' + - 'WebSockets might not work in a mixed content environment. ' + - 'Please consider configuring driver to use encryption.' - ) - } -} diff --git a/packages/testkit-backend/src/feature/deno.js b/packages/testkit-backend/src/feature/deno.js new file mode 100644 index 000000000..5b662da9a --- /dev/null +++ b/packages/testkit-backend/src/feature/deno.js @@ -0,0 +1,7 @@ + +const features = [ + 'Feature:TLS:1.2', + 'Feature:TLS:1.3' +] + +export default features diff --git a/packages/testkit-backend/src/feature/index.js b/packages/testkit-backend/src/feature/index.js index 49e486a0d..81f67f198 100644 --- a/packages/testkit-backend/src/feature/index.js +++ b/packages/testkit-backend/src/feature/index.js @@ -1,10 +1,12 @@ import commonFeatures from './common.js' import rxFeatures from './rx.js' import asyncFeatures from './async.js' +import denoFeatures from './deno.js' const featuresByContext = new Map([ ['async', asyncFeatures], - ['rx', rxFeatures] + ['rx', rxFeatures], + ['deno', denoFeatures] ]) export function createGetFeatures (contexts) { diff --git a/packages/testkit-backend/src/index.deno.ts b/packages/testkit-backend/src/index.deno.ts index e701a59ae..34f51baf7 100644 --- a/packages/testkit-backend/src/index.deno.ts +++ b/packages/testkit-backend/src/index.deno.ts @@ -54,7 +54,7 @@ function write(conn: Deno.Conn, response: object) { } -const descriptor = [] as string[] +const descriptor = ['async', 'deno'] as string[] const shouldRunTest = getShouldRunTest(descriptor); const getFeatures = createGetFeatures(descriptor); diff --git a/packages/testkit-backend/src/skipped-tests/deno.js b/packages/testkit-backend/src/skipped-tests/deno.js new file mode 100644 index 000000000..cf56b5a8e --- /dev/null +++ b/packages/testkit-backend/src/skipped-tests/deno.js @@ -0,0 +1,17 @@ +import skip, { ifEndsWith, ifStartsWith } from './skip.js' + +const skippedTests = [ + skip('DenoJS fail hard on certificate error', + ifEndsWith('test_trusted_ca_expired_server_correct_hostname'), + ifEndsWith('test_trusted_ca_wrong_hostname'), + ifEndsWith('test_unencrypted'), + ifEndsWith('test_untrusted_ca_correct_hostname'), + ifEndsWith('test_1_1'), + ), + skip('Trust All is no available as configuration', + ifStartsWith('tls.test_self_signed_scheme.TestTrustAllCertsConfig.'), + ifStartsWith('tls.test_self_signed_scheme.TestSelfSignedScheme.') + ) +] + +export default skippedTests diff --git a/packages/testkit-backend/src/skipped-tests/index.js b/packages/testkit-backend/src/skipped-tests/index.js index 2ef02a884..324d07706 100644 --- a/packages/testkit-backend/src/skipped-tests/index.js +++ b/packages/testkit-backend/src/skipped-tests/index.js @@ -1,10 +1,12 @@ import commonSkippedTests from './common.js' import browserSkippedTests from './browser.js' import rxSessionSkippedTests from './rx.js' +import denoSkippedTests from './deno.js' const skippedTestsByContext = new Map([ ['browser', browserSkippedTests], - ['rx', rxSessionSkippedTests] + ['rx', rxSessionSkippedTests], + ['deno', denoSkippedTests] ]) export function getShouldRunTest (contexts) { diff --git a/testkit/Dockerfile b/testkit/Dockerfile index a65f7ad1c..e64130231 100644 --- a/testkit/Dockerfile +++ b/testkit/Dockerfile @@ -49,6 +49,8 @@ RUN update-ca-certificates --verbose # Add Deno RUN curl -fsSL https://deno.land/x/install/install.sh | sh RUN mv /root/.deno/bin/deno /usr/bin/ +# Using System CA in Deno +ENV DENO_TLS_CA_STORE=system # Creating an user for building the driver and running the tests RUN useradd -m driver && echo "driver:driver" | chpasswd && adduser driver sudo From 83a77a0d764fedb7638772c46a66e8c24d95d703 Mon Sep 17 00:00:00 2001 From: Antonio Barcelos Date: Mon, 26 Sep 2022 16:38:55 +0200 Subject: [PATCH 04/35] Ajust testkit scripts --- package.json | 1 + packages/testkit-backend/package.json | 7 +++---- testkit/backend.py | 8 +++++++- testkit/build.py | 2 +- testkit/common.py | 4 ++++ testkit/integration.py | 5 ++++- testkit/stress.py | 4 +++- testkit/unittests.py | 5 ++--- 8 files changed, 25 insertions(+), 11 deletions(-) diff --git a/package.json b/package.json index 5e1a6db33..f6ae20460 100644 --- a/package.json +++ b/package.json @@ -46,6 +46,7 @@ "start-neo4j": "lerna run start-neo4j --scope neo4j-driver", "stop-neo4j": "lerna run stop-neo4j --scope neo4j-driver", "start-testkit-backend": "lerna run start --scope testkit-backend --stream", + "start-testkit-backend::deno": "lerna run start::deno --scope testkit-backend --stream", "lerna": "lerna", "prepare": "husky install", "lint-staged": "lint-staged", diff --git a/packages/testkit-backend/package.json b/packages/testkit-backend/package.json index c5e4bd015..7674a54d3 100644 --- a/packages/testkit-backend/package.json +++ b/packages/testkit-backend/package.json @@ -10,10 +10,9 @@ }, "type": "module", "scripts": { - "build": "echo 'Error: no build script specified'", - "build::": "rollup src/index.js --config rollup.config.js", - "start::": "node --version | grep -q v10. && node -r esm src/index.js || node --experimental-specifier-resolution=node src/index.js", - "start": "deno run --allow-read --allow-write --allow-net --allow-env --allow-run src/index.deno.ts", + "build": "rollup src/index.js --config rollup.config.js", + "start": "node --version | grep -q v10. && node -r esm src/index.js || node --experimental-specifier-resolution=node src/index.js", + "start::deno": "deno run --allow-read --allow-write --allow-net --allow-env --allow-run src/index.deno.ts", "clean": "rm -fr node_modules public/index.js", "prepare": "npm run build", "node": "node" diff --git a/testkit/backend.py b/testkit/backend.py index 4acdce471..7e1bb6784 100644 --- a/testkit/backend.py +++ b/testkit/backend.py @@ -6,12 +6,18 @@ from common import ( open_proccess_in_driver_repo, is_browser, + is_deno, + run_in_driver_repo ) import os import time if __name__ == "__main__": print("starting backend") + backend_script = "start-testkit-backend" + if is_deno(): + backend_script = "start-testkit-backend::deno" + if is_browser(): print("Testkit should test browser") os.environ["TEST_ENVIRONMENT"] = "REMOTE" @@ -22,7 +28,7 @@ print("npm run start-testkit-backend") with open_proccess_in_driver_repo([ - "npm", "run", "start-testkit-backend" + "npm", "run", backend_script ], env=os.environ) as backend: if (is_browser()): time.sleep(5) diff --git a/testkit/build.py b/testkit/build.py index ae9292b17..ad428a0b5 100644 --- a/testkit/build.py +++ b/testkit/build.py @@ -2,7 +2,7 @@ Executed in Javascript driver container. Responsible for building driver and test backend. """ -from common import run, run_in_driver_repo, DRIVER_REPO +from common import is_deno, run, run_in_driver_repo, DRIVER_REPO import os diff --git a/testkit/common.py b/testkit/common.py index a6e58adff..93717b9d9 100644 --- a/testkit/common.py +++ b/testkit/common.py @@ -42,3 +42,7 @@ def is_lite(): def is_browser(): return is_enabled(os.environ.get("TEST_DRIVER_BROWSER", "false")) + + +def is_deno(): + return is_enabled(os.environ.get("TEST_DRIVER_DENO", "false")) diff --git a/testkit/integration.py b/testkit/integration.py index 1af24054e..165d84fb5 100644 --- a/testkit/integration.py +++ b/testkit/integration.py @@ -2,6 +2,7 @@ import os from common import ( is_browser, + is_deno, is_lite, run_in_driver_repo, ) @@ -14,7 +15,9 @@ else: ignore = "--ignore=neo4j-driver-lite" - if is_browser(): + if is_deno(): + pass + elif is_browser(): run_in_driver_repo(["npm", "run", "test::browser", "--", ignore]) else: run_in_driver_repo(["npm", "run", "test::integration", "--", ignore]) diff --git a/testkit/stress.py b/testkit/stress.py index 7481e6bae..43332f83e 100644 --- a/testkit/stress.py +++ b/testkit/stress.py @@ -1,6 +1,7 @@ import os from common import ( is_browser, + is_deno, is_lite, run_in_driver_repo, ) @@ -17,4 +18,5 @@ else: ignore = "--ignore=neo4j-driver-lite" - run_in_driver_repo(["npm", "run", "test::stress", "--", ignore]) + if not is_deno(): + run_in_driver_repo(["npm", "run", "test::stress", "--", ignore]) diff --git a/testkit/unittests.py b/testkit/unittests.py index 24e5a9c99..791590a7c 100644 --- a/testkit/unittests.py +++ b/testkit/unittests.py @@ -3,12 +3,11 @@ Responsible for running unit tests. Assumes driver has been setup by build script prior to this. """ -import os -from common import run_in_driver_repo, is_lite +from common import is_deno, run_in_driver_repo, is_lite if __name__ == "__main__": - if is_lite(): + if is_lite() or is_deno(): ignore = "--ignore=neo4j-driver" else: ignore = "--ignore=neo4j-driver-lite" From 4d89c83c0dadbb06a724f6ba3a1be90a4a8d82ea Mon Sep 17 00:00:00 2001 From: Antonio Barcelos Date: Mon, 26 Sep 2022 17:47:08 +0200 Subject: [PATCH 05/35] Adjust neo4j import and tls import --- .../testkit-backend/src/controller/local.js | 5 +- .../src/cypher-native-binders.js | 455 +++++++++--------- .../testkit-backend/src/feature/common.js | 18 +- packages/testkit-backend/src/feature/index.js | 4 +- packages/testkit-backend/src/index.deno.ts | 6 +- packages/testkit-backend/src/index.js | 19 +- packages/testkit-backend/src/neo4j.deno.js | 3 - .../src/request-handlers-rx.js | 34 +- .../testkit-backend/src/request-handlers.js | 81 ++-- packages/testkit-backend/src/responses.js | 16 +- .../testkit-backend/src/summary-binder.js | 16 +- 11 files changed, 331 insertions(+), 326 deletions(-) delete mode 100644 packages/testkit-backend/src/neo4j.deno.js diff --git a/packages/testkit-backend/src/controller/local.js b/packages/testkit-backend/src/controller/local.js index e8a838563..6ef154aa8 100644 --- a/packages/testkit-backend/src/controller/local.js +++ b/packages/testkit-backend/src/controller/local.js @@ -9,12 +9,13 @@ import { isFrontendError } from '../request-handlers' * This controller is used when testing browser and locally. */ export default class LocalController extends Controller { - constructor (requestHandlers = {}, shouldRunTest = () => {}, getFeatures = () => []) { + constructor (requestHandlers = {}, shouldRunTest = () => {}, getFeatures = () => [], neo4j) { super() this._requestHandlers = requestHandlers this._shouldRunTest = shouldRunTest this._getFeatures = getFeatures this._contexts = new Map() + this._neo4j = neo4j } openContext (contextId) { @@ -34,7 +35,7 @@ export default class LocalController extends Controller { throw new Error(`Unknown request: ${name}`) } - return await this._requestHandlers[name](this._contexts.get(contextId), data, { + return await this._requestHandlers[name](this._neo4j, this._contexts.get(contextId), data, { writeResponse: (response) => this._writeResponse(contextId, response), writeError: (e) => this._writeError(contextId, e), writeBackendError: (msg) => this._writeBackendError(contextId, msg) diff --git a/packages/testkit-backend/src/cypher-native-binders.js b/packages/testkit-backend/src/cypher-native-binders.js index 2bd03a0e6..8890c8720 100644 --- a/packages/testkit-backend/src/cypher-native-binders.js +++ b/packages/testkit-backend/src/cypher-native-binders.js @@ -1,252 +1,259 @@ -import neo4j from './neo4j.deno.js' -export function valueResponse (name, value) { - return { name: name, data: { value: value } } -} - -export function objectToCypher (obj) { - return objectMapper(obj, nativeToCypher) -} - -export function objectMemberBitIntToNumber (obj, recursive = false) { - return objectMapper(obj, val => { - if (typeof val === 'bigint') { - return Number(val) - } else if (recursive && typeof val === 'object') { - return objectMemberBitIntToNumber(val) - } else if (recursive && Array.isArray(val)) { - return val.map(item => objectMemberBitIntToNumber(item, true)) - } - return val - }) -} - -function objectMapper (obj, mapper) { - if (obj === null || obj === undefined) { - return obj +export default function CypherNativeBinders (neo4j) { + function valueResponse (name, value) { + return { name: name, data: { value: value } } } - return Object.keys(obj).reduce((acc, key) => { - return { ...acc, [key]: mapper(obj[key]) } - }, {}) -} - -export function nativeToCypher (x) { - if (x == null) { - return valueResponse('CypherNull', null) + + function objectToCypher (obj) { + return objectMapper(obj, nativeToCypher) } - switch (typeof x) { - case 'number': - if (Number.isInteger(x)) { - return valueResponse('CypherInt', x) + + function objectMemberBitIntToNumber (obj, recursive = false) { + return objectMapper(obj, val => { + if (typeof val === 'bigint') { + return Number(val) + } else if (recursive && typeof val === 'object') { + return objectMemberBitIntToNumber(val) + } else if (recursive && Array.isArray(val)) { + return val.map(item => objectMemberBitIntToNumber(item, true)) } - return valueResponse('CypherFloat', x) - case 'bigint': - return valueResponse('CypherInt', neo4j.int(x).toNumber()) - case 'string': - return valueResponse('CypherString', x) - case 'boolean': - return valueResponse('CypherBool', x) - case 'object': - return valueResponseOfObject(x) - } - console.log(`type of ${x} is ${typeof x}`) - const err = 'Unable to convert ' + x + ' to cypher type' - console.log(err) - throw Error(err) -} - -function valueResponseOfObject (x) { - if (neo4j.isInt(x)) { - // TODO: Broken!!! - return valueResponse('CypherInt', x.toInt()) - } - if (Array.isArray(x)) { - const values = x.map(nativeToCypher) - return valueResponse('CypherList', values) + return val + }) } - if (x instanceof neo4j.types.Node) { - const node = { - id: nativeToCypher(x.identity), - labels: nativeToCypher(x.labels), - props: nativeToCypher(x.properties), - elementId: nativeToCypher(x.elementId) + + function objectMapper (obj, mapper) { + if (obj === null || obj === undefined) { + return obj } - return { name: 'CypherNode', data: node } + return Object.keys(obj).reduce((acc, key) => { + return { ...acc, [key]: mapper(obj[key]) } + }, {}) } - if (x instanceof neo4j.types.Relationship) { - const relationship = { - id: nativeToCypher(x.identity), - startNodeId: nativeToCypher(x.start), - endNodeId: nativeToCypher(x.end), - type: nativeToCypher(x.type), - props: nativeToCypher(x.properties), - elementId: nativeToCypher(x.elementId), - startNodeElementId: nativeToCypher(x.startNodeElementId), - endNodeElementId: nativeToCypher(x.endNodeElementId) + + function nativeToCypher (x) { + if (x == null) { + return valueResponse('CypherNull', null) } - return { name: 'CypherRelationship', data: relationship } - } - if (x instanceof neo4j.types.Path) { - const path = x.segments - .map(segment => { - return { - nodes: [segment.end], - relationships: [segment.relationship] + switch (typeof x) { + case 'number': + if (Number.isInteger(x)) { + return valueResponse('CypherInt', x) } - }) - .reduce( - (previous, current) => { + return valueResponse('CypherFloat', x) + case 'bigint': + return valueResponse('CypherInt', neo4j.int(x).toNumber()) + case 'string': + return valueResponse('CypherString', x) + case 'boolean': + return valueResponse('CypherBool', x) + case 'object': + return valueResponseOfObject(x) + } + console.log(`type of ${x} is ${typeof x}`) + const err = 'Unable to convert ' + x + ' to cypher type' + console.log(err) + throw Error(err) + } + + function valueResponseOfObject (x) { + if (neo4j.isInt(x)) { + // TODO: Broken!!! + return valueResponse('CypherInt', x.toInt()) + } + if (Array.isArray(x)) { + const values = x.map(nativeToCypher) + return valueResponse('CypherList', values) + } + if (x instanceof neo4j.types.Node) { + const node = { + id: nativeToCypher(x.identity), + labels: nativeToCypher(x.labels), + props: nativeToCypher(x.properties), + elementId: nativeToCypher(x.elementId) + } + return { name: 'CypherNode', data: node } + } + if (x instanceof neo4j.types.Relationship) { + const relationship = { + id: nativeToCypher(x.identity), + startNodeId: nativeToCypher(x.start), + endNodeId: nativeToCypher(x.end), + type: nativeToCypher(x.type), + props: nativeToCypher(x.properties), + elementId: nativeToCypher(x.elementId), + startNodeElementId: nativeToCypher(x.startNodeElementId), + endNodeElementId: nativeToCypher(x.endNodeElementId) + } + return { name: 'CypherRelationship', data: relationship } + } + if (x instanceof neo4j.types.Path) { + const path = x.segments + .map(segment => { return { - nodes: [...previous.nodes, ...current.nodes], - relationships: [ - ...previous.relationships, - ...current.relationships - ] + nodes: [segment.end], + relationships: [segment.relationship] } - }, - { nodes: [x.start], relationships: [] } - ) - - return { - name: 'CypherPath', - data: { - nodes: nativeToCypher(path.nodes), - relationships: nativeToCypher(path.relationships) + }) + .reduce( + (previous, current) => { + return { + nodes: [...previous.nodes, ...current.nodes], + relationships: [ + ...previous.relationships, + ...current.relationships + ] + } + }, + { nodes: [x.start], relationships: [] } + ) + + return { + name: 'CypherPath', + data: { + nodes: nativeToCypher(path.nodes), + relationships: nativeToCypher(path.relationships) + } } } + + if (neo4j.isDate(x)) { + return structResponse('CypherDate', { + year: x.year, + month: x.month, + day: x.day + }) + } else if (neo4j.isDateTime(x) || neo4j.isLocalDateTime(x)) { + return structResponse('CypherDateTime', { + year: x.year, + month: x.month, + day: x.day, + hour: x.hour, + minute: x.minute, + second: x.second, + nanosecond: x.nanosecond, + utc_offset_s: x.timeZoneOffsetSeconds || (x.timeZoneId == null ? undefined : 0), + timezone_id: x.timeZoneId + }) + } else if (neo4j.isTime(x) || neo4j.isLocalTime(x)) { + return structResponse('CypherTime', { + hour: x.hour, + minute: x.minute, + second: x.second, + nanosecond: x.nanosecond, + utc_offset_s: x.timeZoneOffsetSeconds + }) + } else if (neo4j.isDuration(x)) { + return structResponse('CypherDuration', { + months: x.months, + days: x.days, + seconds: x.seconds, + nanoseconds: x.nanoseconds + }) + } + + // If all failed, interpret as a map + const map = {} + for (const [key, value] of Object.entries(x)) { + map[key] = nativeToCypher(value) + } + return valueResponse('CypherMap', map) } - - if (neo4j.isDate(x)) { - return structResponse('CypherDate', { - year: x.year, - month: x.month, - day: x.day - }) - } else if (neo4j.isDateTime(x) || neo4j.isLocalDateTime(x)) { - return structResponse('CypherDateTime', { - year: x.year, - month: x.month, - day: x.day, - hour: x.hour, - minute: x.minute, - second: x.second, - nanosecond: x.nanosecond, - utc_offset_s: x.timeZoneOffsetSeconds || (x.timeZoneId == null ? undefined : 0), - timezone_id: x.timeZoneId - }) - } else if (neo4j.isTime(x) || neo4j.isLocalTime(x)) { - return structResponse('CypherTime', { - hour: x.hour, - minute: x.minute, - second: x.second, - nanosecond: x.nanosecond, - utc_offset_s: x.timeZoneOffsetSeconds - }) - } else if (neo4j.isDuration(x)) { - return structResponse('CypherDuration', { - months: x.months, - days: x.days, - seconds: x.seconds, - nanoseconds: x.nanoseconds - }) - } - - // If all failed, interpret as a map - const map = {} - for (const [key, value] of Object.entries(x)) { - map[key] = nativeToCypher(value) - } - return valueResponse('CypherMap', map) -} - -function structResponse (name, data) { - const map = {} - for (const [key, value] of Object.entries(data)) { - map[key] = typeof value === 'bigint' || neo4j.isInt(value) - ? neo4j.int(value).toNumber() - : value + + function structResponse (name, data) { + const map = {} + for (const [key, value] of Object.entries(data)) { + map[key] = typeof value === 'bigint' || neo4j.isInt(value) + ? neo4j.int(value).toNumber() + : value + } + return { name, data: map } } - return { name, data: map } -} - -export function cypherToNative (c) { - const { - name, - data - } = c - switch (name) { - case 'CypherString': - return data.value - case 'CypherInt': - return BigInt(data.value) - case 'CypherFloat': - return data.value - case 'CypherNull': - return data.value - case 'CypherBool': - return data.value - case 'CypherList': - return data.value.map(cypherToNative) - case 'CypherDateTime': - if (data.utc_offset_s == null && data.timezone_id == null) { - return new neo4j.LocalDateTime( + + function cypherToNative (c) { + const { + name, + data + } = c + switch (name) { + case 'CypherString': + return data.value + case 'CypherInt': + return BigInt(data.value) + case 'CypherFloat': + return data.value + case 'CypherNull': + return data.value + case 'CypherBool': + return data.value + case 'CypherList': + return data.value.map(cypherToNative) + case 'CypherDateTime': + if (data.utc_offset_s == null && data.timezone_id == null) { + return new neo4j.LocalDateTime( + data.year, + data.month, + data.day, + data.hour, + data.minute, + data.second, + data.nanosecond + ) + } + return new neo4j.DateTime( data.year, data.month, data.day, data.hour, data.minute, data.second, - data.nanosecond + data.nanosecond, + data.utc_offset_s, + data.timezone_id ) - } - return new neo4j.DateTime( - data.year, - data.month, - data.day, - data.hour, - data.minute, - data.second, - data.nanosecond, - data.utc_offset_s, - data.timezone_id - ) - case 'CypherTime': - if (data.utc_offset_s == null) { - return new neo4j.LocalTime( + case 'CypherTime': + if (data.utc_offset_s == null) { + return new neo4j.LocalTime( + data.hour, + data.minute, + data.second, + data.nanosecond + ) + } + return new neo4j.Time( data.hour, data.minute, data.second, - data.nanosecond + data.nanosecond, + data.utc_offset_s ) - } - return new neo4j.Time( - data.hour, - data.minute, - data.second, - data.nanosecond, - data.utc_offset_s - ) - case 'CypherDate': - return new neo4j.Date( - data.year, - data.month, - data.day - ) - case 'CypherDuration': - return new neo4j.Duration( - data.months, - data.days, - data.seconds, - data.nanoseconds - ) - case 'CypherMap': - return Object.entries(data.value).reduce((acc, [key, val]) => { - acc[key] = cypherToNative(val) - return acc - }, {}) + case 'CypherDate': + return new neo4j.Date( + data.year, + data.month, + data.day + ) + case 'CypherDuration': + return new neo4j.Duration( + data.months, + data.days, + data.seconds, + data.nanoseconds + ) + case 'CypherMap': + return Object.entries(data.value).reduce((acc, [key, val]) => { + acc[key] = cypherToNative(val) + return acc + }, {}) + } + console.log(`Type ${name} is not handle by cypherToNative`, c) + const err = 'Unable to convert ' + c + ' to native type' + console.log(err) + throw Error(err) } - console.log(`Type ${name} is not handle by cypherToNative`, c) - const err = 'Unable to convert ' + c + ' to native type' - console.log(err) - throw Error(err) + + this.valueResponse = valueResponse + this.objectToCypher = objectToCypher + this.objectMemberBitIntToNumber = objectMemberBitIntToNumber + this.nativeToCypher = nativeToCypher + this.cypherToNative = cypherToNative } diff --git a/packages/testkit-backend/src/feature/common.js b/packages/testkit-backend/src/feature/common.js index 18bf3d2d1..e7c8c5a2a 100644 --- a/packages/testkit-backend/src/feature/common.js +++ b/packages/testkit-backend/src/feature/common.js @@ -1,20 +1,5 @@ -const SUPPORTED_TLS = (() => { - const tls = { - DEFAULT_MAX_VERSION: false - } - if (tls.DEFAULT_MAX_VERSION) { - const min = Number(tls.DEFAULT_MIN_VERSION.split('TLSv')[1]) - const max = Number(tls.DEFAULT_MAX_VERSION.split('TLSv')[1]) - const result = [] - for (let version = min > 1 ? min : 1.1; version <= max; version = Number((version + 0.1).toFixed(1))) { - result.push(`Feature:TLS:${version.toFixed(1)}`) - } - return result - } - return [] -})() const features = [ 'Feature:Auth:Custom', @@ -40,8 +25,7 @@ const features = [ 'Optimization:EagerTransactionBegin', 'Optimization:ImplicitDefaultArguments', 'Optimization:MinimalBookmarksSet', - 'Optimization:MinimalResets', - ...SUPPORTED_TLS + 'Optimization:MinimalResets' ] export default features diff --git a/packages/testkit-backend/src/feature/index.js b/packages/testkit-backend/src/feature/index.js index 81f67f198..4d9639af1 100644 --- a/packages/testkit-backend/src/feature/index.js +++ b/packages/testkit-backend/src/feature/index.js @@ -9,11 +9,11 @@ const featuresByContext = new Map([ ['deno', denoFeatures] ]) -export function createGetFeatures (contexts) { +export function createGetFeatures (contexts, extraFeatures = []) { const features = contexts .filter(context => featuresByContext.has(context)) .map(context => featuresByContext.get(context)) - .reduce((previous, current) => [...previous, ...current], commonFeatures) + .reduce((previous, current) => [...previous, ...current], [...commonFeatures, ...extraFeatures]) return () => features } diff --git a/packages/testkit-backend/src/index.deno.ts b/packages/testkit-backend/src/index.deno.ts index 34f51baf7..781e60bab 100644 --- a/packages/testkit-backend/src/index.deno.ts +++ b/packages/testkit-backend/src/index.deno.ts @@ -1,6 +1,6 @@ import Context from './context.js'; import { getShouldRunTest } from './skipped-tests/index.js'; -import neo4j from "../../neo4j-driver-deno/lib/mod.ts"; +import neo4j from '../../neo4j-driver-deno/lib/mod.ts' import { createGetFeatures } from './feature/index.js'; import * as handlers from './request-handlers.js'; @@ -78,9 +78,9 @@ const backend: Backend = { console.log('Handle', req.name, req.data); - const handler: (c: Context, data: any, wire: any) => void = requestHandlers[name as string]; + const handler: (neo4j: any, c: Context, data: any, wire: any) => void = requestHandlers[name as string]; - handler(contexts.get(contextId)!!, data, { + handler(neo4j, contexts.get(contextId)!!, data, { writeResponse: (response: object) => write(conn, response), writeError: (e: Error) => { console.error(e) diff --git a/packages/testkit-backend/src/index.js b/packages/testkit-backend/src/index.js index c9a4b16eb..feaae6274 100644 --- a/packages/testkit-backend/src/index.js +++ b/packages/testkit-backend/src/index.js @@ -1,4 +1,6 @@ import Backend from './backend' +import tls from 'tls' +import neo4j from './neo4j.js' import { SocketChannel, WebSocketChannel } from './channel' import { LocalController, RemoteController } from './controller' import { getShouldRunTest } from './skipped-tests' @@ -6,6 +8,19 @@ import { createGetFeatures } from './feature' import * as REQUEST_HANDLERS from './request-handlers.js' import * as RX_REQUEST_HANDLERS from './request-handlers-rx.js' +const SUPPORTED_TLS = (() => { + if (tls.DEFAULT_MAX_VERSION) { + const min = Number(tls.DEFAULT_MIN_VERSION.split('TLSv')[1]) + const max = Number(tls.DEFAULT_MAX_VERSION.split('TLSv')[1]) + const result = [] + for (let version = min > 1 ? min : 1.1; version <= max; version = Number((version + 0.1).toFixed(1))) { + result.push(`Feature:TLS:${version.toFixed(1)}`) + } + return result + } + return [] +})() + /** * Responsible for configure and run the backend server. */ @@ -21,7 +36,7 @@ function main () { .split(',').map(s => s.trim().toLowerCase()) const shouldRunTest = getShouldRunTest([...driverDescriptorList, sessionTypeDescriptor]) - const getFeatures = createGetFeatures([sessionTypeDescriptor]) + const getFeatures = createGetFeatures([sessionTypeDescriptor], SUPPORTED_TLS) const newChannel = () => { if (channelType.toUpperCase() === 'WEBSOCKET') { @@ -34,7 +49,7 @@ function main () { if (testEnviroment.toUpperCase() === 'REMOTE') { return new RemoteController(webserverPort) } - return new LocalController(getRequestHandlers(sessionType), shouldRunTest, getFeatures) + return new LocalController(getRequestHandlers(sessionType), shouldRunTest, getFeatures, neo4j) } const backend = new Backend(newController, newChannel) diff --git a/packages/testkit-backend/src/neo4j.deno.js b/packages/testkit-backend/src/neo4j.deno.js deleted file mode 100644 index 76bb25e5a..000000000 --- a/packages/testkit-backend/src/neo4j.deno.js +++ /dev/null @@ -1,3 +0,0 @@ -import neo4j from '../../neo4j-driver-deno/lib/mod.ts' - -export default neo4j diff --git a/packages/testkit-backend/src/request-handlers-rx.js b/packages/testkit-backend/src/request-handlers-rx.js index 088b89aff..f638593c9 100644 --- a/packages/testkit-backend/src/request-handlers-rx.js +++ b/packages/testkit-backend/src/request-handlers-rx.js @@ -1,8 +1,5 @@ import * as responses from './responses.js' -import neo4j from './neo4j.js' -import { - cypherToNative -} from './cypher-native-binders.js' +import CypherNativeBinders from './cypher-native-binders.js' import { from } from 'rxjs' // Handlers which didn't change depending @@ -28,7 +25,7 @@ export { StartSubTest } from './request-handlers.js' -export function NewSession (context, data, wire) { +export function NewSession (neo4j, context, data, wire) { let { driverId, accessMode, bookmarks, database, fetchSize, impersonatedUser, bookmarkManagerId } = data switch (accessMode) { case 'r': @@ -62,7 +59,7 @@ export function NewSession (context, data, wire) { wire.writeResponse(responses.Session({ id })) } -export function SessionClose (context, data, wire) { +export function SessionClose (_, context, data, wire) { const { sessionId } = data const session = context.getSession(sessionId) return session @@ -72,12 +69,12 @@ export function SessionClose (context, data, wire) { .catch(err => wire.writeError(err)) } -export function SessionRun (context, data, wire) { +export function SessionRun (_, context, data, wire) { const { sessionId, cypher, params, txMeta: metadata, timeout } = data const session = context.getSession(sessionId) if (params) { for (const [key, value] of Object.entries(params)) { - params[key] = cypherToNative(value) + params[key] = binder.cypherToNative(value) } } @@ -104,18 +101,19 @@ export function SessionRun (context, data, wire) { }) } -export function ResultConsume (context, data, wire) { +export function ResultConsume (neo4j, context, data, wire) { const { resultId } = data const result = context.getResult(resultId) + const binder = new CypherNativeBinders(neo4j) return result.consume() .toPromise() .then(summary => { - wire.writeResponse(responses.Summary({ summary })) + wire.writeResponse(responses.Summary({ summary }, { binder })) }).catch(e => wire.writeError(e)) } -export function SessionBeginTransaction (context, data, wire) { +export function SessionBeginTransaction (_, context, data, wire) { const { sessionId, txMeta: metadata, timeout } = data const session = context.getSession(sessionId) @@ -135,12 +133,12 @@ export function SessionBeginTransaction (context, data, wire) { } } -export function TransactionRun (context, data, wire) { +export function TransactionRun (_, context, data, wire) { const { txId, cypher, params } = data const tx = context.getTx(txId) if (params) { for (const [key, value] of Object.entries(params)) { - params[key] = cypherToNative(value) + params[key] = binder.cypherToNative(value) } } @@ -158,7 +156,7 @@ export function TransactionRun (context, data, wire) { }) } -export function TransactionRollback (context, data, wire) { +export function TransactionRollback (_, context, data, wire) { const { txId: id } = data const { tx } = context.getTx(id) return tx.rollback() @@ -170,7 +168,7 @@ export function TransactionRollback (context, data, wire) { }) } -export function TransactionCommit (context, data, wire) { +export function TransactionCommit (_, context, data, wire) { const { txId: id } = data const { tx } = context.getTx(id) return tx.commit() @@ -182,7 +180,7 @@ export function TransactionCommit (context, data, wire) { }) } -export function TransactionClose (context, data, wire) { +export function TransactionClose (_, context, data, wire) { const { txId: id } = data const { tx } = context.getTx(id) return tx.close() @@ -191,7 +189,7 @@ export function TransactionClose (context, data, wire) { .catch(e => wire.writeError(e)) } -export function SessionReadTransaction (context, data, wire) { +export function SessionReadTransaction (_, context, data, wire) { const { sessionId, txMeta: metadata } = data const session = context.getSession(sessionId) @@ -210,7 +208,7 @@ export function SessionReadTransaction (context, data, wire) { } } -export function SessionWriteTransaction (context, data, wire) { +export function SessionWriteTransaction (_, context, data, wire) { const { sessionId, txMeta: metadata } = data const session = context.getSession(sessionId) diff --git a/packages/testkit-backend/src/request-handlers.js b/packages/testkit-backend/src/request-handlers.js index 3af193249..ef36c4417 100644 --- a/packages/testkit-backend/src/request-handlers.js +++ b/packages/testkit-backend/src/request-handlers.js @@ -1,5 +1,4 @@ -import neo4j from './neo4j.deno.js' -import { cypherToNative } from './cypher-native-binders.js' +import CypherNativeBinders from './cypher-native-binders.js' import * as responses from './responses.js' export function throwFrontendError () { @@ -10,7 +9,7 @@ export function isFrontendError (error) { return error.message === 'TestKit FrontendError' } -export function NewDriver (context, data, wire) { +export function NewDriver (neo4j, context, data, wire) { const { uri, authorizationToken: { data: authToken }, @@ -95,7 +94,7 @@ export function NewDriver (context, data, wire) { wire.writeResponse(responses.Driver({ id })) } -export function DriverClose (context, data, wire) { +export function DriverClose (_, context, data, wire) { const { driverId } = data const driver = context.getDriver(driverId) return driver @@ -106,7 +105,7 @@ export function DriverClose (context, data, wire) { .catch(err => wire.writeError(err)) } -export function NewSession (context, data, wire) { +export function NewSession (neo4j, context, data, wire) { let { driverId, accessMode, bookmarks, database, fetchSize, impersonatedUser, bookmarkManagerId } = data switch (accessMode) { case 'r': @@ -140,7 +139,7 @@ export function NewSession (context, data, wire) { wire.writeResponse(responses.Session({ id })) } -export function SessionClose (context, data, wire) { +export function SessionClose (_, context, data, wire) { const { sessionId } = data const session = context.getSession(sessionId) return session @@ -151,12 +150,13 @@ export function SessionClose (context, data, wire) { .catch(err => wire.writeError(err)) } -export function SessionRun (context, data, wire) { +export function SessionRun (neo4j, context, data, wire) { const { sessionId, cypher, params, txMeta: metadata, timeout } = data const session = context.getSession(sessionId) + const binder = new CypherNativeBinders(neo4j) if (params) { for (const [key, value] of Object.entries(params)) { - params[key] = cypherToNative(value) + params[key] = binder.cypherToNative(value) } } @@ -174,9 +174,10 @@ export function SessionRun (context, data, wire) { wire.writeResponse(responses.Result({ id })) } -export function ResultNext (context, data, wire) { +export function ResultNext (neo4j, context, data, wire) { const { resultId } = data const result = context.getResult(resultId) + const binder = new CypherNativeBinders(neo4j) if (!('recordIt' in result)) { result.recordIt = result[Symbol.asyncIterator]() } @@ -184,7 +185,7 @@ export function ResultNext (context, data, wire) { if (done) { wire.writeResponse(responses.NullRecord()) } else { - wire.writeResponse(responses.Record({ record: value })) + wire.writeResponse(responses.Record({ record: value }, { binder })) } }).catch(e => { console.log('got some err: ' + JSON.stringify(e)) @@ -192,9 +193,10 @@ export function ResultNext (context, data, wire) { }) } -export function ResultPeek (context, data, wire) { +export function ResultPeek (neo4j, context, data, wire) { const { resultId } = data const result = context.getResult(resultId) + const binder = new CypherNativeBinders(neo4j) if (!('recordIt' in result)) { result.recordIt = result[Symbol.asyncIterator]() } @@ -202,7 +204,7 @@ export function ResultPeek (context, data, wire) { if (done) { wire.writeResponse(responses.NullRecord()) } else { - wire.writeResponse(responses.Record({ record: value })) + wire.writeResponse(responses.Record({ record: value }, { binder })) } }).catch(e => { console.log('got some err: ' + JSON.stringify(e)) @@ -210,28 +212,29 @@ export function ResultPeek (context, data, wire) { }) } -export function ResultConsume (context, data, wire) { +export function ResultConsume (neo4j, context, data, wire) { const { resultId } = data const result = context.getResult(resultId) + const binder = new CypherNativeBinders(neo4j) return result.summary().then(summary => { - wire.writeResponse(responses.Summary({ summary })) + wire.writeResponse(responses.Summary({ summary }, { binder })) }).catch(e => wire.writeError(e)) } -export function ResultList (context, data, wire) { +export function ResultList (neo4j, context, data, wire) { const { resultId } = data - + const binder = new CypherNativeBinders(neo4j) const result = context.getResult(resultId) return result .then(({ records }) => { - wire.writeResponse(responses.RecordList({ records })) + wire.writeResponse(responses.RecordList({ records }, { binder })) }) .catch(error => wire.writeError(error)) } -export function SessionReadTransaction (context, data, wire) { +export function SessionReadTransaction (_, context, data, wire) { const { sessionId, txMeta: metadata } = data const session = context.getSession(sessionId) return session @@ -246,12 +249,13 @@ export function SessionReadTransaction (context, data, wire) { .catch(error => wire.writeError(error)) } -export function TransactionRun (context, data, wire) { +export function TransactionRun (_, context, data, wire) { const { txId, cypher, params } = data const tx = context.getTx(txId) + const binder = new CypherNativeBinders(neo4j) if (params) { for (const [key, value] of Object.entries(params)) { - params[key] = cypherToNative(value) + params[key] = binder.cypherToNative(value) } } const result = tx.tx.run(cypher, params) @@ -260,14 +264,14 @@ export function TransactionRun (context, data, wire) { wire.writeResponse(responses.Result({ id })) } -export function RetryablePositive (context, data, wire) { +export function RetryablePositive (_, context, data, wire) { const { sessionId } = data context.getTxsBySessionId(sessionId).forEach(tx => { tx.resolve() }) } -export function RetryableNegative (context, data, wire) { +export function RetryableNegative (_, context, data, wire) { const { sessionId, errorId } = data const error = context.getError(errorId) || new Error('TestKit FrontendError') context.getTxsBySessionId(sessionId).forEach(tx => { @@ -275,7 +279,7 @@ export function RetryableNegative (context, data, wire) { }) } -export function SessionBeginTransaction (context, data, wire) { +export function SessionBeginTransaction (_, context, data, wire) { const { sessionId, txMeta: metadata, timeout } = data const session = context.getSession(sessionId) @@ -294,7 +298,7 @@ export function SessionBeginTransaction (context, data, wire) { } } -export function TransactionCommit (context, data, wire) { +export function TransactionCommit (_, context, data, wire) { const { txId: id } = data const { tx } = context.getTx(id) return tx.commit() @@ -305,7 +309,7 @@ export function TransactionCommit (context, data, wire) { }) } -export function TransactionRollback (context, data, wire) { +export function TransactionRollback (_, context, data, wire) { const { txId: id } = data const { tx } = context.getTx(id) return tx.rollback() @@ -313,7 +317,7 @@ export function TransactionRollback (context, data, wire) { .catch(e => wire.writeError(e)) } -export function TransactionClose (context, data, wire) { +export function TransactionClose (_, context, data, wire) { const { txId: id } = data const { tx } = context.getTx(id) return tx.close() @@ -321,14 +325,14 @@ export function TransactionClose (context, data, wire) { .catch(e => wire.writeError(e)) } -export function SessionLastBookmarks (context, data, wire) { +export function SessionLastBookmarks (_, context, data, wire) { const { sessionId } = data const session = context.getSession(sessionId) const bookmarks = session.lastBookmarks() wire.writeResponse(responses.Bookmarks({ bookmarks })) } -export function SessionWriteTransaction (context, data, wire) { +export function SessionWriteTransaction (_, context, data, wire) { const { sessionId, txMeta: metadata } = data const session = context.getSession(sessionId) return session @@ -343,7 +347,7 @@ export function SessionWriteTransaction (context, data, wire) { .catch(error => wire.writeError(error)) } -export function StartTest (context, { testName }, wire) { +export function StartTest (_, context, { testName }, wire) { if (testName.endsWith('.test_disconnect_session_on_tx_pull_after_record') || testName.endsWith('test_no_reset_on_clean_connection')) { context.logLevel = 'debug' } else { @@ -361,7 +365,7 @@ export function StartTest (context, { testName }, wire) { }) } -export function StartSubTest (context, { testName, subtestArguments }, wire) { +export function StartSubTest (_, context, { testName, subtestArguments }, wire) { if (testName === 'neo4j.datatypes.test_temporal_types.TestDataTypes.test_date_time_cypher_created_tz_id') { try { Intl.DateTimeFormat(undefined, { timeZone: subtestArguments.tz_id }) @@ -374,13 +378,13 @@ export function StartSubTest (context, { testName, subtestArguments }, wire) { } } -export function GetFeatures (context, _params, wire) { +export function GetFeatures (_, context, _params, wire) { wire.writeResponse(responses.FeatureList({ features: context.getFeatures() })) } -export function VerifyConnectivity (context, { driverId }, wire) { +export function VerifyConnectivity (_, context, { driverId }, wire) { const driver = context.getDriver(driverId) return driver .verifyConnectivity() @@ -388,7 +392,7 @@ export function VerifyConnectivity (context, { driverId }, wire) { .catch(error => wire.writeError(error)) } -export function GetServerInfo (context, { driverId }, wire) { +export function GetServerInfo (_, context, { driverId }, wire) { const driver = context.getDriver(driverId) return driver .getServerInfo() @@ -396,7 +400,7 @@ export function GetServerInfo (context, { driverId }, wire) { .catch(error => wire.writeError(error)) } -export function CheckMultiDBSupport (context, { driverId }, wire) { +export function CheckMultiDBSupport (_, context, { driverId }, wire) { const driver = context.getDriver(driverId) return driver .supportsMultiDb() @@ -407,6 +411,7 @@ export function CheckMultiDBSupport (context, { driverId }, wire) { } export function ResolverResolutionCompleted ( + _, context, { requestId, addresses }, wire @@ -416,6 +421,7 @@ export function ResolverResolutionCompleted ( } export function NewBookmarkManager ( + _, context, { initialBookmarks, @@ -458,6 +464,7 @@ export function NewBookmarkManager ( } export function BookmarkManagerClose ( + _, context, { id @@ -469,6 +476,7 @@ export function BookmarkManagerClose ( } export function BookmarksSupplierCompleted ( + _, context, { requestId, @@ -480,6 +488,7 @@ export function BookmarksSupplierCompleted ( } export function BookmarksConsumerCompleted ( + _, context, { requestId @@ -489,7 +498,7 @@ export function BookmarksConsumerCompleted ( notifyBookmarksRequest.resolve() } -export function GetRoutingTable (context, { driverId, database }, wire) { +export function GetRoutingTable (_, context, { driverId, database }, wire) { const driver = context.getDriver(driverId) const routingTable = driver && @@ -512,7 +521,7 @@ export function GetRoutingTable (context, { driverId, database }, wire) { } } -export function ForcedRoutingTableUpdate (context, { driverId, database, bookmarks }, wire) { +export function ForcedRoutingTableUpdate (_, context, { driverId, database, bookmarks }, wire) { const driver = context.getDriver(driverId) const provider = driver._getOrCreateConnectionProvider() diff --git a/packages/testkit-backend/src/responses.js b/packages/testkit-backend/src/responses.js index 000d906e4..044f02e34 100644 --- a/packages/testkit-backend/src/responses.js +++ b/packages/testkit-backend/src/responses.js @@ -1,7 +1,3 @@ -import { - nativeToCypher -} from './cypher-native-binders.js' - import { nativeToTestkitSummary } from './summary-binder.js' @@ -50,20 +46,20 @@ export function NullRecord () { return response('NullRecord', null) } -export function Record ({ record }) { - const values = Array.from(record.values()).map(nativeToCypher) +export function Record ({ record }, { binder }) { + const values = Array.from(record.values()).map(binder.nativeToCypher) return response('Record', { values }) } -export function RecordList ({ records }) { +export function RecordList ({ records }, { binder }) { const cypherRecords = records.map(rec => { - return { values: Array.from(rec.values()).map(nativeToCypher) } + return { values: Array.from(rec.values()).map(binder.nativeToCypher) } }) return response('RecordList', { records: cypherRecords }) } -export function Summary ({ summary }) { - return response('Summary', nativeToTestkitSummary(summary)) +export function Summary ({ summary }, { binder }) { + return response('Summary', nativeToTestkitSummary(summary, binder)) } export function Bookmarks ({ bookmarks }) { diff --git a/packages/testkit-backend/src/summary-binder.js b/packages/testkit-backend/src/summary-binder.js index 24e5a46ef..d7221a3ea 100644 --- a/packages/testkit-backend/src/summary-binder.js +++ b/packages/testkit-backend/src/summary-binder.js @@ -1,5 +1,3 @@ -import { objectToCypher, objectMemberBitIntToNumber } from './cypher-native-binders.js' - function mapPlan (plan) { return { operatorType: plan.operatorType, @@ -18,10 +16,10 @@ function mapCounters (stats) { } } -function mapProfile (profile, child = false) { - const mapChild = (child) => mapProfile(child, true) +function mapProfile (profile, child = false, binder) { + const mapChild = (child) => mapProfile(child, true, binder) const obj = { - args: objectMemberBitIntToNumber(profile.arguments), + args: binder.objectMemberBitIntToNumber(profile.arguments), dbHits: Number(profile.dbHits), identifiers: profile.identifiers, operatorType: profile.operatorType, @@ -48,13 +46,13 @@ function mapNotification (notification) { } } -export function nativeToTestkitSummary (summary) { +export function nativeToTestkitSummary (summary, binder) { return { - ...objectMemberBitIntToNumber(summary), + ...binder.objectMemberBitIntToNumber(summary), database: summary.database.name, query: { text: summary.query.text, - parameters: objectToCypher(summary.query.parameters) + parameters: binder.objectToCypher(summary.query.parameters) }, serverInfo: { agent: summary.server.agent, @@ -62,7 +60,7 @@ export function nativeToTestkitSummary (summary) { }, counters: mapCounters(summary.counters), plan: mapPlan(summary.plan), - profile: mapProfile(summary.profile), + profile: mapProfile(summary.profile, false, binder), notifications: summary.notifications.map(mapNotification) } } From 286d20833d0190f6998249219537879edae1201c Mon Sep 17 00:00:00 2001 From: Antonio Barcelos Date: Mon, 26 Sep 2022 17:49:05 +0200 Subject: [PATCH 06/35] Revert browser channel --- .../src/channel/browser/browser-channel.js | 48 +++++++------------ 1 file changed, 18 insertions(+), 30 deletions(-) diff --git a/packages/bolt-connection/src/channel/browser/browser-channel.js b/packages/bolt-connection/src/channel/browser/browser-channel.js index 532fcbefa..8959e6e44 100644 --- a/packages/bolt-connection/src/channel/browser/browser-channel.js +++ b/packages/bolt-connection/src/channel/browser/browser-channel.js @@ -144,26 +144,17 @@ export default class WebSocketChannel { if (this._pending !== null) { this._pending.push(buffer) } else if (buffer instanceof ChannelBuffer) { - // We should wait for the connection state change before sending - setTimeout(() => { - try { - if (this._ws.readyState !== WS_OPEN){ - console.log('WebSocket not open') - return; - } - this._ws.send(buffer._buffer) - } catch (error) { - if (this._ws.readyState !== WS_OPEN) { - // Websocket has been closed - this._handleConnectionError() - } else { - console.log('errror', error) - // Some other error occured - throw error - } + try { + this._ws.send(buffer._buffer) + } catch (error) { + if (this._ws.readyState !== WS_OPEN) { + // Websocket has been closed + this._handleConnectionError() + } else { + // Some other error occured + throw error } - }, 500) - console.log('after timeout') + } } else { throw newError("Don't know how to send buffer: " + buffer) } @@ -175,17 +166,14 @@ export default class WebSocketChannel { */ close () { return new Promise((resolve, reject) => { - setTimeout(() => { - if (this._ws && this._ws.readyState !== WS_CLOSED && this._ws.readyState !== WS_CLOSING) { - this._open = false - this._clearConnectionTimeout() - this._ws.onclose = () => resolve() - this._ws.close() - } else { - resolve() - } - }, 500) - + if (this._ws && this._ws.readyState !== WS_CLOSED) { + this._open = false + this._clearConnectionTimeout() + this._ws.onclose = () => resolve() + this._ws.close() + } else { + resolve() + } }) } From 65d269144ca84ae87c35e36b28d70b060e306c19 Mon Sep 17 00:00:00 2001 From: Antonio Barcelos Date: Tue, 27 Sep 2022 12:26:44 +0200 Subject: [PATCH 07/35] Fix missing neo4j definition --- packages/testkit-backend/src/request-handlers-rx.js | 6 ++++-- packages/testkit-backend/src/request-handlers.js | 12 ++++++------ 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/packages/testkit-backend/src/request-handlers-rx.js b/packages/testkit-backend/src/request-handlers-rx.js index f638593c9..ee8a0dc1e 100644 --- a/packages/testkit-backend/src/request-handlers-rx.js +++ b/packages/testkit-backend/src/request-handlers-rx.js @@ -69,10 +69,11 @@ export function SessionClose (_, context, data, wire) { .catch(err => wire.writeError(err)) } -export function SessionRun (_, context, data, wire) { +export function SessionRun (neo4j, context, data, wire) { const { sessionId, cypher, params, txMeta: metadata, timeout } = data const session = context.getSession(sessionId) if (params) { + const binder = new CypherNativeBinders(neo4j) for (const [key, value] of Object.entries(params)) { params[key] = binder.cypherToNative(value) } @@ -133,9 +134,10 @@ export function SessionBeginTransaction (_, context, data, wire) { } } -export function TransactionRun (_, context, data, wire) { +export function TransactionRun (neo4j, context, data, wire) { const { txId, cypher, params } = data const tx = context.getTx(txId) + const binder = new CypherNativeBinders(neo4j) if (params) { for (const [key, value] of Object.entries(params)) { params[key] = binder.cypherToNative(value) diff --git a/packages/testkit-backend/src/request-handlers.js b/packages/testkit-backend/src/request-handlers.js index ef36c4417..87602177b 100644 --- a/packages/testkit-backend/src/request-handlers.js +++ b/packages/testkit-backend/src/request-handlers.js @@ -42,10 +42,10 @@ export function NewDriver (neo4j, context, data, wire) { } const resolver = resolverRegistered ? address => - new Promise((resolve, reject) => { - const id = context.addResolverRequest(resolve, reject) - wire.writeResponse(responses.ResolverResolutionRequired({ id, address })) - }) + new Promise((resolve, reject) => { + const id = context.addResolverRequest(resolve, reject) + wire.writeResponse(responses.ResolverResolutionRequired({ id, address })) + }) : undefined const config = { userAgent, @@ -249,7 +249,7 @@ export function SessionReadTransaction (_, context, data, wire) { .catch(error => wire.writeError(error)) } -export function TransactionRun (_, context, data, wire) { +export function TransactionRun (neo4j, context, data, wire) { const { txId, cypher, params } = data const tx = context.getTx(txId) const binder = new CypherNativeBinders(neo4j) @@ -421,7 +421,7 @@ export function ResolverResolutionCompleted ( } export function NewBookmarkManager ( - _, + neo4j, context, { initialBookmarks, From 6ea7a14f487f7958f0fe7345d1fa7331e4f84293 Mon Sep 17 00:00:00 2001 From: Antonio Barcelos Date: Tue, 27 Sep 2022 12:35:53 +0200 Subject: [PATCH 08/35] Move Deno implemenation for its own folder --- .../testkit-backend/{src/index.deno.ts => deno/index.ts} | 8 ++++---- packages/testkit-backend/package.json | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) rename packages/testkit-backend/{src/index.deno.ts => deno/index.ts} (95%) diff --git a/packages/testkit-backend/src/index.deno.ts b/packages/testkit-backend/deno/index.ts similarity index 95% rename from packages/testkit-backend/src/index.deno.ts rename to packages/testkit-backend/deno/index.ts index 781e60bab..b81856bac 100644 --- a/packages/testkit-backend/src/index.deno.ts +++ b/packages/testkit-backend/deno/index.ts @@ -1,8 +1,8 @@ -import Context from './context.js'; -import { getShouldRunTest } from './skipped-tests/index.js'; +import Context from '../src/context.js'; +import { getShouldRunTest } from '../src/skipped-tests/index.js'; import neo4j from '../../neo4j-driver-deno/lib/mod.ts' -import { createGetFeatures } from './feature/index.js'; -import * as handlers from './request-handlers.js'; +import { createGetFeatures } from '../src/feature/index.js'; +import * as handlers from '../src/request-handlers.js'; const listener = Deno.listen({ port: 9876 }); let index = 0; diff --git a/packages/testkit-backend/package.json b/packages/testkit-backend/package.json index 7674a54d3..4feed4551 100644 --- a/packages/testkit-backend/package.json +++ b/packages/testkit-backend/package.json @@ -12,7 +12,7 @@ "scripts": { "build": "rollup src/index.js --config rollup.config.js", "start": "node --version | grep -q v10. && node -r esm src/index.js || node --experimental-specifier-resolution=node src/index.js", - "start::deno": "deno run --allow-read --allow-write --allow-net --allow-env --allow-run src/index.deno.ts", + "start::deno": "deno run --allow-read --allow-write --allow-net --allow-env --allow-run deno/index.ts", "clean": "rm -fr node_modules public/index.js", "prepare": "npm run build", "node": "node" From f2e9436fd98d5ff6e0ef7fe7832316d12cc28501 Mon Sep 17 00:00:00 2001 From: Antonio Barcelos Date: Tue, 27 Sep 2022 14:22:29 +0200 Subject: [PATCH 09/35] Refactory DenoJS implementation --- packages/testkit-backend/deno/README.md | 25 ++++ packages/testkit-backend/deno/channel.ts | 87 ++++++++++++ packages/testkit-backend/deno/controller.ts | 56 ++++++++ packages/testkit-backend/deno/domain.ts | 19 +++ packages/testkit-backend/deno/index.ts | 150 ++------------------ packages/testkit-backend/deno/tsconfig.json | 5 + 6 files changed, 201 insertions(+), 141 deletions(-) create mode 100644 packages/testkit-backend/deno/README.md create mode 100644 packages/testkit-backend/deno/channel.ts create mode 100644 packages/testkit-backend/deno/controller.ts create mode 100644 packages/testkit-backend/deno/domain.ts create mode 100644 packages/testkit-backend/deno/tsconfig.json diff --git a/packages/testkit-backend/deno/README.md b/packages/testkit-backend/deno/README.md new file mode 100644 index 000000000..d9d595ee8 --- /dev/null +++ b/packages/testkit-backend/deno/README.md @@ -0,0 +1,25 @@ +# Deno Testkit Backend Specific implementations + +This directory contains Deno specific implementations which depends on having `Deno` global variable available or being able to load `Deno` specific libraries such as the `Neo4j Deno Module`. +Files like `../feature/deno.js` and `../skipped-tests/deno.js` are outside this directory since they are pure javascript configuration files and they don't depends on the environment. + +## Starting Backend + +### Pre-requisites + +First, you need to build the `Neo4j Deno Module` by running `npm run build::deno` in the repository root folder. + +### The start command +For starting this backend, you should run the following command in the current directory: + +``` +deno run --allow-read --allow-write --allow-net --allow-env --allow-run index.ts +``` + +Alternatively, you could run `'npm run start::deno'` in the root package of the `testkit-backend` or `npm run start-testkit-backend::deno` in the repository root folder. + +## Project Structure + +* `index.ts` is responsible for configuring the backend to run. +* `channel.ts` is responsible for the communication with testkit. +* `controller.ts` is responsible for routing the request to the service. diff --git a/packages/testkit-backend/deno/channel.ts b/packages/testkit-backend/deno/channel.ts new file mode 100644 index 000000000..26812b17e --- /dev/null +++ b/packages/testkit-backend/deno/channel.ts @@ -0,0 +1,87 @@ +import { TestkitRequest, TestkitResponse } from "./domain.ts"; + +export interface TestkitClient { + id: number, + requests: () => AsyncIterable, + reply: (response: TestkitResponse) => Promise +} + + +export async function* listen (port: number): AsyncIterable { + let clientId = 0; + const listener = Deno.listen({ port }); + + for await (const conn of listener) { + const id = clientId++; + const requests = () => readRequests(conn); + const reply = createReply(conn); + yield { id, requests, reply }; + } +} + +async function* readRequests (conn: Deno.Conn): AsyncIterable { + let inRequest = false + let requestString = '' + for await (const message of Deno.iter(conn)) { + const rawTxtMessage = new TextDecoder().decode(message); + const lines = rawTxtMessage.split('\n'); + for (const line of lines) { + switch (line) { + case '#request begin': + if(inRequest) { + throw new Error('Already in request'); + } + inRequest = true; + break; + case '#request end': + if(!inRequest) { + throw new Error('Not in request'); + } + const request = JSON.parse(requestString); + yield request + inRequest = false; + requestString = ''; + break + case '': + // ignore empty lines + break; + default: + if(!inRequest) { + throw new Error('Not in request'); + } + requestString += line; + break + } + } + } +} + +function createReply (conn: Deno.Conn) { + return async function (response: TestkitResponse): Promise { + const responseStr = JSON.stringify(response, (_, value) => + typeof value === 'bigint' ? `${value}n` : value + ) + + const responseArr = ['#response begin', responseStr, '#response end'].join('\n') + '\n' + console.log('response', responseArr); + const buffer = new Uint8Array(responseArr.length); + new TextEncoder().encodeInto(responseArr, buffer); + + async function writeBuffer(buff: Uint8Array, size: number) { + try { + let bytesWritten = 0; + while(bytesWritten < size) { + const writtenInSep = await conn.write(buff.slice(bytesWritten)); + bytesWritten += writtenInSep; + } + } catch (error) { + console.error(error) + } + } + await writeBuffer(buffer, buffer.byteLength); + } +} + +export default { + listen +} diff --git a/packages/testkit-backend/deno/controller.ts b/packages/testkit-backend/deno/controller.ts new file mode 100644 index 000000000..9c7af28c5 --- /dev/null +++ b/packages/testkit-backend/deno/controller.ts @@ -0,0 +1,56 @@ +import Context from '../src/context.js'; +import { TestkitRequest, TestkitResponse, RequestHandlerMap } from "./domain.ts"; + +interface Reply { + (response: TestkitResponse): Promise +} + +function newWire (context: Context, reply: Reply ): any { + return { + writeResponse: (response: TestkitResponse) => reply(response), + writeError: (e: Error) => { + console.error(e) + if (e.name) { + if (e.message === 'TestKit FrontendError') { + reply({ name: 'FrontendError', data: { + msg: 'Simulating the client code throwing some error.' + }}) + } else { + const id = context.addError(e) + reply({ name: 'DriverError', data: { + id, + msg: e.message, + // @ts-ignore + code: e.code + }}) + } + return + } + const msg = e.message + reply({ name: 'BackendError', data: { msg } }) + }, + writeBackendError: (msg: string) => reply({ name: 'BackendError', data: { msg } }) + } +} + +export function createHandler (neo4j: any, newContext: () => Context, requestHandlers: RequestHandlerMap) { + return async function (reply: Reply, requests: () => AsyncIterable) { + const context = newContext() + const wire = newWire(context, reply) + for await (const request of requests()) { + const { data, name } = request; + if (!(name in requestHandlers)) { + console.log('Unknown request: ' + name) + wire.writeBackendError('Unknown request: ' + name) + } + + const handleRequest = requestHandlers[name] + + handleRequest(neo4j, context, data, wire) + } + } +} + +export default { + createHandler +} diff --git a/packages/testkit-backend/deno/domain.ts b/packages/testkit-backend/deno/domain.ts new file mode 100644 index 000000000..562d69658 --- /dev/null +++ b/packages/testkit-backend/deno/domain.ts @@ -0,0 +1,19 @@ +import Context from '../src/context.js'; + +export interface TestkitRequest { + name: string, + data?: any +} + +export interface TestkitResponse { + name: string, + data?: any +} + +export interface RequestHandler { + (neo4j: any, c: Context, data: any, wire: any): void +} + +export interface RequestHandlerMap { + [key: string]: RequestHandler +} diff --git a/packages/testkit-backend/deno/index.ts b/packages/testkit-backend/deno/index.ts index b81856bac..c921098ec 100644 --- a/packages/testkit-backend/deno/index.ts +++ b/packages/testkit-backend/deno/index.ts @@ -3,156 +3,24 @@ import { getShouldRunTest } from '../src/skipped-tests/index.js'; import neo4j from '../../neo4j-driver-deno/lib/mod.ts' import { createGetFeatures } from '../src/feature/index.js'; import * as handlers from '../src/request-handlers.js'; - -const listener = Deno.listen({ port: 9876 }); -let index = 0; -const contexts = new Map(); - -interface RequestHandler { - (c: Context, data: any, wire: any): void -} - -interface RequestHandlerMap { - [key: string]: RequestHandler -} - -addEventListener('uncaughtException', (event) => { - console.log('unhandled rejection', event); -}) +import channel from './channel.ts'; +import controller from './controller.ts'; // @ts-ignore const requestHandlers: RequestHandlerMap = handlers as RequestHandlerMap addEventListener('events.errorMonitor', (event) => { - console.log('something here ========================') + console.log('something here ========================', event) }) -interface Backend { - openContext(contextId: number): void - closeContext(contextId: number): void - handle(contextId: number, conn: Deno.Conn, request: { name: string, data: object }): void -} - -function write(conn: Deno.Conn, response: object) { - const responseStr = JSON.stringify(response, (_, value) => - typeof value === 'bigint' ? `${value}n` : value - ) - const responseArr = ['#response begin', responseStr, '#response end'].join('\n') + '\n' - console.log('response', responseArr); - const buffer = new Uint8Array(responseArr.length); - new TextEncoder().encodeInto(responseArr, buffer); - - async function writeBuffer(buff: Uint8Array, size: number) { - let bytesWritten = 0; - while(bytesWritten < size) { - const writtenInSep = await conn.write(buff.slice(bytesWritten)); - bytesWritten += writtenInSep; - } - } - - writeBuffer(buffer, buffer.byteLength); - -} - -const descriptor = ['async', 'deno'] as string[] +const descriptor = ['async', 'deno'] const shouldRunTest = getShouldRunTest(descriptor); const getFeatures = createGetFeatures(descriptor); +const createContext = () => new Context(shouldRunTest, getFeatures) -const backend: Backend = { - openContext: (contextId) => { - console.log("Open context:", contextId); - contexts.set(contextId, new Context(shouldRunTest, getFeatures)); - }, - closeContext: (contextId) => { - console.log('Close context', contextId); - contexts.delete(contextId); - }, - handle: (contextId, conn, req) => { - const { data, name } = req; - if (!contexts.has(contextId)) { - throw new Error(`Context ${contextId} does not exist`) - } else if (!(name in requestHandlers)) { - console.log('Unknown request: ' + name) - throw new Error(`Unknown request: ${name}`) - } - - console.log('Handle', req.name, req.data); - - const handler: (neo4j: any, c: Context, data: any, wire: any) => void = requestHandlers[name as string]; - - handler(neo4j, contexts.get(contextId)!!, data, { - writeResponse: (response: object) => write(conn, response), - writeError: (e: Error) => { - console.error(e) - if (e.name) { - if (e.message === 'TestKit FrontendError') { - write(conn, { name: 'FrontendError', data: { - msg: 'Simulating the client code throwing some error.' - }}) - } else { - const id = contexts.get(contextId)?.addError(e) - write(conn, { name: 'DriverError', data: { - id, - msg: e.message, - // @ts-ignore - code: e.code - }}) - } - return - } - const msg = e.message - write(conn, { name: 'BackendError', data: { msg } }) - }, - writeBackendError: (msg: string) => write(conn, { name: 'BackendError', data: { msg } }) - }); - } -} - -for await (const conn of listener) { - const contextId = index++; - handleConnection(conn, contextId) -} +const listener = channel.listen(9876) +const handle = controller.createHandler(neo4j, createContext, requestHandlers) -async function handleConnection( conn: Deno.Conn, contextId: number): Promise { - backend.openContext(contextId); - let inRequest = false - let requestString = '' - try { - for await (const message of Deno.iter(conn)) { - const rawTxtMessage = new TextDecoder().decode(message); - const lines = rawTxtMessage.split('\n'); - for (const line of lines) { - switch (line) { - case '#request begin': - if(inRequest) { - throw new Error('Already in request'); - } - inRequest = true; - break; - case '#request end': - if(!inRequest) { - throw new Error('Not in request'); - } - const request = JSON.parse(requestString); - backend.handle(contextId, conn, request); - inRequest = false; - requestString = ''; - break - case '': - // ignore empty lines - break; - default: - if(!inRequest) { - throw new Error('Not in request'); - } - requestString += line; - break - } - } - } - } catch (err) { - console.error(err) - } - - backend.closeContext(contextId); +for await (const client of listener) { + handle(client.reply, client.requests) } diff --git a/packages/testkit-backend/deno/tsconfig.json b/packages/testkit-backend/deno/tsconfig.json new file mode 100644 index 000000000..b18e5bc35 --- /dev/null +++ b/packages/testkit-backend/deno/tsconfig.json @@ -0,0 +1,5 @@ +{ + "compilerOptions": { + "allowJs": true + } +} From 2de0f8db5934c0fb9dafa01df763f4bdcdf9bc4e Mon Sep 17 00:00:00 2001 From: Antonio Barcelos Date: Tue, 27 Sep 2022 14:23:08 +0200 Subject: [PATCH 10/35] deno fmt --- packages/testkit-backend/deno/README.md | 26 +++--- packages/testkit-backend/deno/channel.ts | 71 ++++++++-------- packages/testkit-backend/deno/controller.ts | 90 ++++++++++++--------- packages/testkit-backend/deno/domain.ts | 16 ++-- packages/testkit-backend/deno/index.ts | 32 ++++---- 5 files changed, 131 insertions(+), 104 deletions(-) diff --git a/packages/testkit-backend/deno/README.md b/packages/testkit-backend/deno/README.md index d9d595ee8..7218b28cc 100644 --- a/packages/testkit-backend/deno/README.md +++ b/packages/testkit-backend/deno/README.md @@ -1,25 +1,33 @@ # Deno Testkit Backend Specific implementations -This directory contains Deno specific implementations which depends on having `Deno` global variable available or being able to load `Deno` specific libraries such as the `Neo4j Deno Module`. -Files like `../feature/deno.js` and `../skipped-tests/deno.js` are outside this directory since they are pure javascript configuration files and they don't depends on the environment. +This directory contains Deno specific implementations which depends on having +`Deno` global variable available or being able to load `Deno` specific libraries +such as the `Neo4j Deno Module`. Files like `../feature/deno.js` and +`../skipped-tests/deno.js` are outside this directory since they are pure +javascript configuration files and they don't depends on the environment. ## Starting Backend -### Pre-requisites +### Pre-requisites -First, you need to build the `Neo4j Deno Module` by running `npm run build::deno` in the repository root folder. +First, you need to build the `Neo4j Deno Module` by running +`npm run build::deno` in the repository root folder. ### The start command -For starting this backend, you should run the following command in the current directory: + +For starting this backend, you should run the following command in the current +directory: ``` deno run --allow-read --allow-write --allow-net --allow-env --allow-run index.ts ``` -Alternatively, you could run `'npm run start::deno'` in the root package of the `testkit-backend` or `npm run start-testkit-backend::deno` in the repository root folder. +Alternatively, you could run `'npm run start::deno'` in the root package of the +`testkit-backend` or `npm run start-testkit-backend::deno` in the repository +root folder. ## Project Structure -* `index.ts` is responsible for configuring the backend to run. -* `channel.ts` is responsible for the communication with testkit. -* `controller.ts` is responsible for routing the request to the service. +- `index.ts` is responsible for configuring the backend to run. +- `channel.ts` is responsible for the communication with testkit. +- `controller.ts` is responsible for routing the request to the service. diff --git a/packages/testkit-backend/deno/channel.ts b/packages/testkit-backend/deno/channel.ts index 26812b17e..ae97360b0 100644 --- a/packages/testkit-backend/deno/channel.ts +++ b/packages/testkit-backend/deno/channel.ts @@ -1,13 +1,12 @@ import { TestkitRequest, TestkitResponse } from "./domain.ts"; export interface TestkitClient { - id: number, - requests: () => AsyncIterable, - reply: (response: TestkitResponse) => Promise + id: number; + requests: () => AsyncIterable; + reply: (response: TestkitResponse) => Promise; } - -export async function* listen (port: number): AsyncIterable { +export async function* listen(port: number): AsyncIterable { let clientId = 0; const listener = Deno.listen({ port }); @@ -19,69 +18,71 @@ export async function* listen (port: number): AsyncIterable { } } -async function* readRequests (conn: Deno.Conn): AsyncIterable { - let inRequest = false - let requestString = '' +async function* readRequests(conn: Deno.Conn): AsyncIterable { + let inRequest = false; + let requestString = ""; for await (const message of Deno.iter(conn)) { const rawTxtMessage = new TextDecoder().decode(message); - const lines = rawTxtMessage.split('\n'); + const lines = rawTxtMessage.split("\n"); for (const line of lines) { switch (line) { - case '#request begin': - if(inRequest) { - throw new Error('Already in request'); + case "#request begin": + if (inRequest) { + throw new Error("Already in request"); } inRequest = true; break; - case '#request end': - if(!inRequest) { - throw new Error('Not in request'); + case "#request end": + if (!inRequest) { + throw new Error("Not in request"); } const request = JSON.parse(requestString); - yield request + yield request; inRequest = false; - requestString = ''; - break - case '': + requestString = ""; + break; + case "": // ignore empty lines break; default: - if(!inRequest) { - throw new Error('Not in request'); + if (!inRequest) { + throw new Error("Not in request"); } requestString += line; - break + break; } } } } -function createReply (conn: Deno.Conn) { - return async function (response: TestkitResponse): Promise { - const responseStr = JSON.stringify(response, (_, value) => - typeof value === 'bigint' ? `${value}n` : value - ) +function createReply(conn: Deno.Conn) { + return async function (response: TestkitResponse): Promise { + const responseStr = JSON.stringify( + response, + (_, value) => typeof value === "bigint" ? `${value}n` : value, + ); - const responseArr = ['#response begin', responseStr, '#response end'].join('\n') + '\n' - console.log('response', responseArr); + const responseArr = + ["#response begin", responseStr, "#response end"].join("\n") + "\n"; + console.log("response", responseArr); const buffer = new Uint8Array(responseArr.length); new TextEncoder().encodeInto(responseArr, buffer); - + async function writeBuffer(buff: Uint8Array, size: number) { try { let bytesWritten = 0; - while(bytesWritten < size) { + while (bytesWritten < size) { const writtenInSep = await conn.write(buff.slice(bytesWritten)); bytesWritten += writtenInSep; } } catch (error) { - console.error(error) + console.error(error); } } await writeBuffer(buffer, buffer.byteLength); - } + }; } export default { - listen -} + listen, +}; diff --git a/packages/testkit-backend/deno/controller.ts b/packages/testkit-backend/deno/controller.ts index 9c7af28c5..beb2e617f 100644 --- a/packages/testkit-backend/deno/controller.ts +++ b/packages/testkit-backend/deno/controller.ts @@ -1,56 +1,74 @@ -import Context from '../src/context.js'; -import { TestkitRequest, TestkitResponse, RequestHandlerMap } from "./domain.ts"; +import Context from "../src/context.js"; +import { + RequestHandlerMap, + TestkitRequest, + TestkitResponse, +} from "./domain.ts"; interface Reply { - (response: TestkitResponse): Promise + (response: TestkitResponse): Promise; } -function newWire (context: Context, reply: Reply ): any { +function newWire(context: Context, reply: Reply): any { return { - writeResponse: (response: TestkitResponse) => reply(response), - writeError: (e: Error) => { - console.error(e) - if (e.name) { - if (e.message === 'TestKit FrontendError') { - reply({ name: 'FrontendError', data: { - msg: 'Simulating the client code throwing some error.' - }}) - } else { - const id = context.addError(e) - reply({ name: 'DriverError', data: { + writeResponse: (response: TestkitResponse) => reply(response), + writeError: (e: Error) => { + console.error(e); + if (e.name) { + if (e.message === "TestKit FrontendError") { + reply({ + name: "FrontendError", + data: { + msg: "Simulating the client code throwing some error.", + }, + }); + } else { + const id = context.addError(e); + reply({ + name: "DriverError", + data: { id, msg: e.message, // @ts-ignore - code: e.code - }}) - } - return + code: e.code, + }, + }); } - const msg = e.message - reply({ name: 'BackendError', data: { msg } }) - }, - writeBackendError: (msg: string) => reply({ name: 'BackendError', data: { msg } }) - } + return; + } + const msg = e.message; + reply({ name: "BackendError", data: { msg } }); + }, + writeBackendError: (msg: string) => + reply({ name: "BackendError", data: { msg } }), + }; } -export function createHandler (neo4j: any, newContext: () => Context, requestHandlers: RequestHandlerMap) { - return async function (reply: Reply, requests: () => AsyncIterable) { - const context = newContext() - const wire = newWire(context, reply) +export function createHandler( + neo4j: any, + newContext: () => Context, + requestHandlers: RequestHandlerMap, +) { + return async function ( + reply: Reply, + requests: () => AsyncIterable, + ) { + const context = newContext(); + const wire = newWire(context, reply); for await (const request of requests()) { const { data, name } = request; if (!(name in requestHandlers)) { - console.log('Unknown request: ' + name) - wire.writeBackendError('Unknown request: ' + name) + console.log("Unknown request: " + name); + wire.writeBackendError("Unknown request: " + name); } - - const handleRequest = requestHandlers[name] - handleRequest(neo4j, context, data, wire) + const handleRequest = requestHandlers[name]; + + handleRequest(neo4j, context, data, wire); } - } + }; } export default { - createHandler -} + createHandler, +}; diff --git a/packages/testkit-backend/deno/domain.ts b/packages/testkit-backend/deno/domain.ts index 562d69658..49004baf8 100644 --- a/packages/testkit-backend/deno/domain.ts +++ b/packages/testkit-backend/deno/domain.ts @@ -1,19 +1,19 @@ -import Context from '../src/context.js'; +import Context from "../src/context.js"; -export interface TestkitRequest { - name: string, - data?: any +export interface TestkitRequest { + name: string; + data?: any; } export interface TestkitResponse { - name: string, - data?: any + name: string; + data?: any; } export interface RequestHandler { - (neo4j: any, c: Context, data: any, wire: any): void + (neo4j: any, c: Context, data: any, wire: any): void; } export interface RequestHandlerMap { - [key: string]: RequestHandler + [key: string]: RequestHandler; } diff --git a/packages/testkit-backend/deno/index.ts b/packages/testkit-backend/deno/index.ts index c921098ec..651b9c811 100644 --- a/packages/testkit-backend/deno/index.ts +++ b/packages/testkit-backend/deno/index.ts @@ -1,26 +1,26 @@ -import Context from '../src/context.js'; -import { getShouldRunTest } from '../src/skipped-tests/index.js'; -import neo4j from '../../neo4j-driver-deno/lib/mod.ts' -import { createGetFeatures } from '../src/feature/index.js'; -import * as handlers from '../src/request-handlers.js'; -import channel from './channel.ts'; -import controller from './controller.ts'; +import Context from "../src/context.js"; +import { getShouldRunTest } from "../src/skipped-tests/index.js"; +import neo4j from "../../neo4j-driver-deno/lib/mod.ts"; +import { createGetFeatures } from "../src/feature/index.js"; +import * as handlers from "../src/request-handlers.js"; +import channel from "./channel.ts"; +import controller from "./controller.ts"; // @ts-ignore -const requestHandlers: RequestHandlerMap = handlers as RequestHandlerMap +const requestHandlers: RequestHandlerMap = handlers as RequestHandlerMap; -addEventListener('events.errorMonitor', (event) => { - console.log('something here ========================', event) -}) +addEventListener("events.errorMonitor", (event) => { + console.log("something here ========================", event); +}); -const descriptor = ['async', 'deno'] +const descriptor = ["async", "deno"]; const shouldRunTest = getShouldRunTest(descriptor); const getFeatures = createGetFeatures(descriptor); -const createContext = () => new Context(shouldRunTest, getFeatures) +const createContext = () => new Context(shouldRunTest, getFeatures); -const listener = channel.listen(9876) -const handle = controller.createHandler(neo4j, createContext, requestHandlers) +const listener = channel.listen(9876); +const handle = controller.createHandler(neo4j, createContext, requestHandlers); for await (const client of listener) { - handle(client.reply, client.requests) + handle(client.reply, client.requests); } From 8385ef6f5d0afb2c7d8c5f5bb1c68284b3d886aa Mon Sep 17 00:00:00 2001 From: Antonio Barcelos Date: Tue, 27 Sep 2022 15:09:39 +0200 Subject: [PATCH 11/35] Revert changes in the log config --- packages/testkit-backend/src/request-handlers.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/testkit-backend/src/request-handlers.js b/packages/testkit-backend/src/request-handlers.js index 87602177b..2fba333c1 100644 --- a/packages/testkit-backend/src/request-handlers.js +++ b/packages/testkit-backend/src/request-handlers.js @@ -51,7 +51,7 @@ export function NewDriver (neo4j, context, data, wire) { userAgent, resolver, useBigInt: true, - logging: neo4j.logging.console('debug') + logging: neo4j.logging.console(process.env.LOG_LEVEL || context.logLevel) } if ('encrypted' in data) { config.encrypted = data.encrypted ? 'ENCRYPTION_ON' : 'ENCRYPTION_OFF' From 04894e61aef445124cf1e9779af59d12f76574da Mon Sep 17 00:00:00 2001 From: Antonio Barcelos Date: Tue, 27 Sep 2022 18:12:24 +0200 Subject: [PATCH 12/35] Adjust linter issues --- packages/neo4j-driver-deno/src/deno-channel.js | 9 ++++++--- packages/testkit-backend/deno/channel.ts | 7 +++---- packages/testkit-backend/deno/controller.ts | 4 ++-- packages/testkit-backend/deno/index.ts | 2 +- packages/testkit-backend/src/request-handlers.js | 4 +++- 5 files changed, 15 insertions(+), 11 deletions(-) diff --git a/packages/neo4j-driver-deno/src/deno-channel.js b/packages/neo4j-driver-deno/src/deno-channel.js index ddd263c68..058d99e23 100644 --- a/packages/neo4j-driver-deno/src/deno-channel.js +++ b/packages/neo4j-driver-deno/src/deno-channel.js @@ -19,6 +19,9 @@ /* eslint-env browser */ import ChannelBuffer from '../channel-buf.js' import { newError, internal } from '../../../core/index.ts' +import { iterateReader } from 'https://deno.land/std@0.157.0/streams/conversion.ts'; + + const { util: { ENCRYPTION_OFF, ENCRYPTION_ON } @@ -244,13 +247,13 @@ const TrustStrategy = { caCerts }) }, - TRUST_SYSTEM_CA_SIGNED_CERTIFICATES: async function (config) { + TRUST_SYSTEM_CA_SIGNED_CERTIFICATES: function (config) { return Deno.connectTls({ hostname: config.address.resolvedHost(), port: config.address.port() }) }, - TRUST_ALL_CERTIFICATES: async function (config) { + TRUST_ALL_CERTIFICATES: function (config) { throw newError( `"${config.trust}" is not available in DenoJS. ` + 'For trust in any certificates, you should use the DenoJS flag ' + @@ -310,7 +313,7 @@ function getTrustStrategyName (config) { async function setupReader (channel) { try { - for await (const message of Deno.iter(channel._conn)) { + for await (const message of iterateReader(channel._conn)) { channel._resetTimeout() if (!channel._open) { diff --git a/packages/testkit-backend/deno/channel.ts b/packages/testkit-backend/deno/channel.ts index ae97360b0..b70a03cfb 100644 --- a/packages/testkit-backend/deno/channel.ts +++ b/packages/testkit-backend/deno/channel.ts @@ -1,5 +1,5 @@ import { TestkitRequest, TestkitResponse } from "./domain.ts"; - +import { iterateReader } from "./deps.ts"; export interface TestkitClient { id: number; requests: () => AsyncIterable; @@ -21,7 +21,7 @@ export async function* listen(port: number): AsyncIterable { async function* readRequests(conn: Deno.Conn): AsyncIterable { let inRequest = false; let requestString = ""; - for await (const message of Deno.iter(conn)) { + for await (const message of iterateReader(conn)) { const rawTxtMessage = new TextDecoder().decode(message); const lines = rawTxtMessage.split("\n"); for (const line of lines) { @@ -36,8 +36,7 @@ async function* readRequests(conn: Deno.Conn): AsyncIterable { if (!inRequest) { throw new Error("Not in request"); } - const request = JSON.parse(requestString); - yield request; + yield JSON.parse(requestString); inRequest = false; requestString = ""; break; diff --git a/packages/testkit-backend/deno/controller.ts b/packages/testkit-backend/deno/controller.ts index beb2e617f..58ce6338a 100644 --- a/packages/testkit-backend/deno/controller.ts +++ b/packages/testkit-backend/deno/controller.ts @@ -13,7 +13,6 @@ function newWire(context: Context, reply: Reply): any { return { writeResponse: (response: TestkitResponse) => reply(response), writeError: (e: Error) => { - console.error(e); if (e.name) { if (e.message === "TestKit FrontendError") { reply({ @@ -29,7 +28,7 @@ function newWire(context: Context, reply: Reply): any { data: { id, msg: e.message, - // @ts-ignore + // @ts-ignore Code Neo4jError does have code code: e.code, }, }); @@ -45,6 +44,7 @@ function newWire(context: Context, reply: Reply): any { } export function createHandler( + // deno-lint-ignore no-explicit-any neo4j: any, newContext: () => Context, requestHandlers: RequestHandlerMap, diff --git a/packages/testkit-backend/deno/index.ts b/packages/testkit-backend/deno/index.ts index 651b9c811..22476e40b 100644 --- a/packages/testkit-backend/deno/index.ts +++ b/packages/testkit-backend/deno/index.ts @@ -5,8 +5,8 @@ import { createGetFeatures } from "../src/feature/index.js"; import * as handlers from "../src/request-handlers.js"; import channel from "./channel.ts"; import controller from "./controller.ts"; +import { RequestHandlerMap } from "./domain.ts"; -// @ts-ignore const requestHandlers: RequestHandlerMap = handlers as RequestHandlerMap; addEventListener("events.errorMonitor", (event) => { diff --git a/packages/testkit-backend/src/request-handlers.js b/packages/testkit-backend/src/request-handlers.js index 2fba333c1..1748e77ee 100644 --- a/packages/testkit-backend/src/request-handlers.js +++ b/packages/testkit-backend/src/request-handlers.js @@ -47,11 +47,13 @@ export function NewDriver (neo4j, context, data, wire) { wire.writeResponse(responses.ResolverResolutionRequired({ id, address })) }) : undefined + + const envLogLevel = typeof process === 'undefined' ? undefined : process.env.LOG_LEVEL const config = { userAgent, resolver, useBigInt: true, - logging: neo4j.logging.console(process.env.LOG_LEVEL || context.logLevel) + logging: neo4j.logging.console(envLogLevel || context.logLevel) } if ('encrypted' in data) { config.encrypted = data.encrypted ? 'ENCRYPTION_ON' : 'ENCRYPTION_OFF' From 2af4009c3454e6f28484fbff6d03b4d2460b8baf Mon Sep 17 00:00:00 2001 From: Antonio Barcelos Date: Wed, 28 Sep 2022 11:44:16 +0200 Subject: [PATCH 13/35] Improve logs in the backend --- packages/testkit-backend/deno/channel.ts | 1 - packages/testkit-backend/deno/controller.ts | 7 ++++++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/packages/testkit-backend/deno/channel.ts b/packages/testkit-backend/deno/channel.ts index b70a03cfb..b6ef1c770 100644 --- a/packages/testkit-backend/deno/channel.ts +++ b/packages/testkit-backend/deno/channel.ts @@ -63,7 +63,6 @@ function createReply(conn: Deno.Conn) { const responseArr = ["#response begin", responseStr, "#response end"].join("\n") + "\n"; - console.log("response", responseArr); const buffer = new Uint8Array(responseArr.length); new TextEncoder().encodeInto(responseArr, buffer); diff --git a/packages/testkit-backend/deno/controller.ts b/packages/testkit-backend/deno/controller.ts index 58ce6338a..eee83c20b 100644 --- a/packages/testkit-backend/deno/controller.ts +++ b/packages/testkit-backend/deno/controller.ts @@ -54,8 +54,13 @@ export function createHandler( requests: () => AsyncIterable, ) { const context = newContext(); - const wire = newWire(context, reply); + const wire = newWire(context, response => { + console.log('response:', response) + return reply(response) + }); + for await (const request of requests()) { + console.log('request:', request) const { data, name } = request; if (!(name in requestHandlers)) { console.log("Unknown request: " + name); From 00fbae25ae2d2f25ec62bdea4faf6fd95a9210b9 Mon Sep 17 00:00:00 2001 From: Antonio Barcelos Date: Wed, 28 Sep 2022 12:39:41 +0200 Subject: [PATCH 14/35] Add dependencies --- packages/testkit-backend/deno/deps.ts | 1 + 1 file changed, 1 insertion(+) create mode 100644 packages/testkit-backend/deno/deps.ts diff --git a/packages/testkit-backend/deno/deps.ts b/packages/testkit-backend/deno/deps.ts new file mode 100644 index 000000000..9a1782abc --- /dev/null +++ b/packages/testkit-backend/deno/deps.ts @@ -0,0 +1 @@ +export { iterateReader } from "https://deno.land/std@0.157.0/streams/conversion.ts"; From 8f5022e055d8ad438ca7572b81400f1b2a49a840 Mon Sep 17 00:00:00 2001 From: Antonio Barcelos Date: Wed, 28 Sep 2022 14:07:44 +0200 Subject: [PATCH 15/35] Prevent write in a broken socket --- packages/testkit-backend/deno/channel.ts | 85 +++++++++++++++--------- 1 file changed, 53 insertions(+), 32 deletions(-) diff --git a/packages/testkit-backend/deno/channel.ts b/packages/testkit-backend/deno/channel.ts index b6ef1c770..148f1f9f9 100644 --- a/packages/testkit-backend/deno/channel.ts +++ b/packages/testkit-backend/deno/channel.ts @@ -12,50 +12,71 @@ export async function* listen(port: number): AsyncIterable { for await (const conn of listener) { const id = clientId++; - const requests = () => readRequests(conn); - const reply = createReply(conn); + const { requests, reply } = setupRequestsAndReply(conn) yield { id, requests, reply }; } } -async function* readRequests(conn: Deno.Conn): AsyncIterable { +interface State { + finishedReading: boolean +} + +function setupRequestsAndReply (conn: Deno.Conn) { + const state = { finishedReading: false } + const requests = () => readRequests(conn, state); + const reply = createReply(conn, state); + + return { requests, reply } +} + +async function* readRequests(conn: Deno.Conn, state: State ): AsyncIterable { let inRequest = false; let requestString = ""; - for await (const message of iterateReader(conn)) { - const rawTxtMessage = new TextDecoder().decode(message); - const lines = rawTxtMessage.split("\n"); - for (const line of lines) { - switch (line) { - case "#request begin": - if (inRequest) { - throw new Error("Already in request"); - } - inRequest = true; - break; - case "#request end": - if (!inRequest) { - throw new Error("Not in request"); - } - yield JSON.parse(requestString); - inRequest = false; - requestString = ""; - break; - case "": - // ignore empty lines - break; - default: - if (!inRequest) { - throw new Error("Not in request"); - } - requestString += line; - break; + try { + for await (const message of iterateReader(conn)) { + const rawTxtMessage = new TextDecoder().decode(message); + const lines = rawTxtMessage.split("\n"); + for (const line of lines) { + switch (line) { + case "#request begin": + if (inRequest) { + throw new Error("Already in request"); + } + inRequest = true; + break; + case "#request end": + if (!inRequest) { + throw new Error("Not in request"); + } + yield JSON.parse(requestString); + inRequest = false; + requestString = ""; + break; + case "": + // ignore empty lines + break; + default: + if (!inRequest) { + throw new Error("Not in request"); + } + requestString += line; + break; + } } } + } finally { + state.finishedReading = true } + + } -function createReply(conn: Deno.Conn) { +function createReply(conn: Deno.Conn, state: State) { return async function (response: TestkitResponse): Promise { + if (state.finishedReading) { + console.warn('Discarded response:', response) + return + } const responseStr = JSON.stringify( response, (_, value) => typeof value === "bigint" ? `${value}n` : value, From f541c0b778f0492067e7b2e7fa0f243a9fc1c40f Mon Sep 17 00:00:00 2001 From: Antonio Barcelos Date: Wed, 28 Sep 2022 14:47:16 +0200 Subject: [PATCH 16/35] Capture and log unhandledrejection --- packages/testkit-backend/deno/channel.ts | 2 -- packages/testkit-backend/deno/index.ts | 5 +++++ 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/packages/testkit-backend/deno/channel.ts b/packages/testkit-backend/deno/channel.ts index 148f1f9f9..78da277be 100644 --- a/packages/testkit-backend/deno/channel.ts +++ b/packages/testkit-backend/deno/channel.ts @@ -67,8 +67,6 @@ async function* readRequests(conn: Deno.Conn, state: State ): AsyncIterable { console.log("something here ========================", event); }); +addEventListener("unhandledrejection", (event) => { + console.log("unhandledrejection", event) + console.error("unhandledrejection", event) +}) + const descriptor = ["async", "deno"]; const shouldRunTest = getShouldRunTest(descriptor); const getFeatures = createGetFeatures(descriptor); From 577a37536f4067501b79c0bb57d5d86f6acf351a Mon Sep 17 00:00:00 2001 From: Antonio Barcelos Date: Wed, 28 Sep 2022 16:07:53 +0200 Subject: [PATCH 17/35] Move dependencies to deps and fix text encode. --- packages/testkit-backend/deno/channel.ts | 4 ++-- packages/testkit-backend/deno/deps.ts | 5 +++++ packages/testkit-backend/deno/index.ts | 12 +++++++----- 3 files changed, 14 insertions(+), 7 deletions(-) diff --git a/packages/testkit-backend/deno/channel.ts b/packages/testkit-backend/deno/channel.ts index 78da277be..8fa08a4d1 100644 --- a/packages/testkit-backend/deno/channel.ts +++ b/packages/testkit-backend/deno/channel.ts @@ -70,6 +70,7 @@ async function* readRequests(conn: Deno.Conn, state: State ): AsyncIterable { if (state.finishedReading) { console.warn('Discarded response:', response) @@ -82,8 +83,7 @@ function createReply(conn: Deno.Conn, state: State) { const responseArr = ["#response begin", responseStr, "#response end"].join("\n") + "\n"; - const buffer = new Uint8Array(responseArr.length); - new TextEncoder().encodeInto(responseArr, buffer); + const buffer = textEncoder.encode(responseArr) async function writeBuffer(buff: Uint8Array, size: number) { try { diff --git a/packages/testkit-backend/deno/deps.ts b/packages/testkit-backend/deno/deps.ts index 9a1782abc..6046a70e4 100644 --- a/packages/testkit-backend/deno/deps.ts +++ b/packages/testkit-backend/deno/deps.ts @@ -1 +1,6 @@ export { iterateReader } from "https://deno.land/std@0.157.0/streams/conversion.ts"; +export { default as Context } from "../src/context.js"; +export { getShouldRunTest } from "../src/skipped-tests/index.js"; +export { default as neo4j } from "../../neo4j-driver-deno/lib/mod.ts"; +export { createGetFeatures } from "../src/feature/index.js"; +export * as handlers from "../src/request-handlers.js"; diff --git a/packages/testkit-backend/deno/index.ts b/packages/testkit-backend/deno/index.ts index fbaa05e7f..826610f31 100644 --- a/packages/testkit-backend/deno/index.ts +++ b/packages/testkit-backend/deno/index.ts @@ -1,8 +1,10 @@ -import Context from "../src/context.js"; -import { getShouldRunTest } from "../src/skipped-tests/index.js"; -import neo4j from "../../neo4j-driver-deno/lib/mod.ts"; -import { createGetFeatures } from "../src/feature/index.js"; -import * as handlers from "../src/request-handlers.js"; +import { + handlers, + getShouldRunTest, + createGetFeatures, + Context, + neo4j +} from "./deps.ts" import channel from "./channel.ts"; import controller from "./controller.ts"; import { RequestHandlerMap } from "./domain.ts"; From 9edf9ec8a0f604007682e4f37de9be4cf4d6cdd9 Mon Sep 17 00:00:00 2001 From: Antonio Barcelos Date: Wed, 28 Sep 2022 16:34:26 +0200 Subject: [PATCH 18/35] Move deno-channel implemantation to bolt-connection --- .../src/channel/deno}/deno-channel.js | 8 +++----- packages/neo4j-driver-deno/generate.ts | 10 ++-------- 2 files changed, 5 insertions(+), 13 deletions(-) rename packages/{neo4j-driver-deno/src => bolt-connection/src/channel/deno}/deno-channel.js (98%) diff --git a/packages/neo4j-driver-deno/src/deno-channel.js b/packages/bolt-connection/src/channel/deno/deno-channel.js similarity index 98% rename from packages/neo4j-driver-deno/src/deno-channel.js rename to packages/bolt-connection/src/channel/deno/deno-channel.js index 058d99e23..c498502fa 100644 --- a/packages/neo4j-driver-deno/src/deno-channel.js +++ b/packages/bolt-connection/src/channel/deno/deno-channel.js @@ -16,13 +16,11 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -/* eslint-env browser */ -import ChannelBuffer from '../channel-buf.js' -import { newError, internal } from '../../../core/index.ts' +/* eslint-env deno */ +import ChannelBuffer from '../channel-buf' +import { newError, internal } from 'neo4j-driver-core' import { iterateReader } from 'https://deno.land/std@0.157.0/streams/conversion.ts'; - - const { util: { ENCRYPTION_OFF, ENCRYPTION_ON } } = internal diff --git a/packages/neo4j-driver-deno/generate.ts b/packages/neo4j-driver-deno/generate.ts index 1f7a5d983..6888891ea 100644 --- a/packages/neo4j-driver-deno/generate.ts +++ b/packages/neo4j-driver-deno/generate.ts @@ -130,11 +130,11 @@ async function copyAndTransform(inDir: string, outDir: string) { } // Special fix for bolt-connection/channel/index.js - // Replace the "node channel" with the "browser channel", since Deno supports browser APIs + // Replace the "node channel" with the "deno channel", since Deno supports browser APIs if (inPath.endsWith("channel/index.js")) { contents = contents.replace( `export * from './node/index.js'`, - `export * from './browser/index.js'`, + `export * from './deno/index.js'`, ); } @@ -164,12 +164,6 @@ await Deno.writeTextFile( `export default "${version}" // Specified using --version when running generate.ts\n`, ); -// Copy custom files -await Deno.copyFile( - "src/deno-channel.js", - join(rootOutDir, "/bolt-connection/channel/browser/browser-channel.js")); - - //////////////////////////////////////////////////////////////////////////////// // Warnings show up at the end if (!doTransform) { From a7beadc5f71ceb28634fc1750d74902dff090d64 Mon Sep 17 00:00:00 2001 From: Antonio Barcelos Date: Wed, 28 Sep 2022 16:53:18 +0200 Subject: [PATCH 19/35] Fix deps --- .../channel/deno/deno-host-name-resolver.js | 29 ++++++++++++++++ .../bolt-connection/src/channel/deno/index.js | 34 +++++++++++++++++++ packages/testkit-backend/deno/deps.ts | 2 +- 3 files changed, 64 insertions(+), 1 deletion(-) create mode 100644 packages/bolt-connection/src/channel/deno/deno-host-name-resolver.js create mode 100644 packages/bolt-connection/src/channel/deno/index.js diff --git a/packages/bolt-connection/src/channel/deno/deno-host-name-resolver.js b/packages/bolt-connection/src/channel/deno/deno-host-name-resolver.js new file mode 100644 index 000000000..99fb8174a --- /dev/null +++ b/packages/bolt-connection/src/channel/deno/deno-host-name-resolver.js @@ -0,0 +1,29 @@ +/** + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { internal } from 'neo4j-driver-core' + +const { + resolver: { BaseHostNameResolver } +} = internal + +export default class DenoHostNameResolver extends BaseHostNameResolver { + resolve (address) { + return this._resolveToItself(address) + } +} diff --git a/packages/bolt-connection/src/channel/deno/index.js b/packages/bolt-connection/src/channel/deno/index.js new file mode 100644 index 000000000..d84e98195 --- /dev/null +++ b/packages/bolt-connection/src/channel/deno/index.js @@ -0,0 +1,34 @@ +/** + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import DenoChannel from './deno-channel' +import DenoHostNameResolver from './deno-host-name-resolver' + +/* + + This module exports a set of components to be used in deno environment. + They are not compatible with NodeJS environment. + All files import/require APIs from `node/index.js` by default. + Such imports are replaced at build time with `browser/index.js` when building a browser bundle. + + NOTE: exports in this module should have exactly the same names/structure as exports in `node/index.js`. + + */ +export const Channel = DenoChannel +export const HostNameResolver = DenoHostNameResolver diff --git a/packages/testkit-backend/deno/deps.ts b/packages/testkit-backend/deno/deps.ts index 6046a70e4..5fb388e3f 100644 --- a/packages/testkit-backend/deno/deps.ts +++ b/packages/testkit-backend/deno/deps.ts @@ -1,4 +1,4 @@ -export { iterateReader } from "https://deno.land/std@0.157.0/streams/conversion.ts"; +export { iterateReader } from "https://deno.land/std@0.119.0/streams/conversion.ts"; export { default as Context } from "../src/context.js"; export { getShouldRunTest } from "../src/skipped-tests/index.js"; export { default as neo4j } from "../../neo4j-driver-deno/lib/mod.ts"; From 1cbdecae5d063098c3f9df5986aae93be3a8fe77 Mon Sep 17 00:00:00 2001 From: Antonio Barcelos Date: Wed, 28 Sep 2022 17:42:28 +0200 Subject: [PATCH 20/35] Disable eslint for Deno files --- packages/bolt-connection/src/channel/deno/deno-channel.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/bolt-connection/src/channel/deno/deno-channel.js b/packages/bolt-connection/src/channel/deno/deno-channel.js index c498502fa..3ea05f424 100644 --- a/packages/bolt-connection/src/channel/deno/deno-channel.js +++ b/packages/bolt-connection/src/channel/deno/deno-channel.js @@ -16,7 +16,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -/* eslint-env deno */ +/* eslint-disable */ import ChannelBuffer from '../channel-buf' import { newError, internal } from 'neo4j-driver-core' import { iterateReader } from 'https://deno.land/std@0.157.0/streams/conversion.ts'; From ccdc4afae4c012577402959fca791fd411dd23a5 Mon Sep 17 00:00:00 2001 From: Antonio Barcelos Date: Wed, 28 Sep 2022 18:31:03 +0200 Subject: [PATCH 21/35] Add copyright notice to version.ts --- packages/neo4j-driver-deno/copyright.txt | 18 ++++++++++++++++++ packages/neo4j-driver-deno/generate.ts | 3 ++- 2 files changed, 20 insertions(+), 1 deletion(-) create mode 100644 packages/neo4j-driver-deno/copyright.txt diff --git a/packages/neo4j-driver-deno/copyright.txt b/packages/neo4j-driver-deno/copyright.txt new file mode 100644 index 000000000..9ee3abe74 --- /dev/null +++ b/packages/neo4j-driver-deno/copyright.txt @@ -0,0 +1,18 @@ +/** + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ diff --git a/packages/neo4j-driver-deno/generate.ts b/packages/neo4j-driver-deno/generate.ts index 6888891ea..a7792c5e9 100644 --- a/packages/neo4j-driver-deno/generate.ts +++ b/packages/neo4j-driver-deno/generate.ts @@ -159,9 +159,10 @@ await copyAndTransform( await copyAndTransform("../neo4j-driver-lite/src", rootOutDir); // Deno convention is to use "mod.ts" not "index.ts", so let's do that at least for the main/root import: await Deno.rename(join(rootOutDir, "index.ts"), join(rootOutDir, "mod.ts")) +const copyright = await Deno.readTextFile("./copyright.txt"); await Deno.writeTextFile( join(rootOutDir, "version.ts"), - `export default "${version}" // Specified using --version when running generate.ts\n`, + [copyright, `export default "${version}" // Specified using --version when running generate.ts\n`].join('\n'), ); //////////////////////////////////////////////////////////////////////////////// From befa3c815d52249cc864a47393c08988ea74fb87 Mon Sep 17 00:00:00 2001 From: Antonio Barcelos Date: Thu, 29 Sep 2022 12:28:41 +0200 Subject: [PATCH 22/35] Configuring linter and formatter fr the deno backend files --- package.json | 5 +++++ packages/testkit-backend/deno/controller.ts | 16 +++++++++++----- packages/testkit-backend/deno/domain.ts | 1 + 3 files changed, 17 insertions(+), 5 deletions(-) diff --git a/package.json b/package.json index f6ae20460..a0cb4ac09 100644 --- a/package.json +++ b/package.json @@ -30,6 +30,11 @@ "packages/neo4j-driver/**/*.ts": [ "npm run ts-standard::neo4j-driver", "git add" + ], + "packages/testkit-backend/deno/**/*.{ts,js}": [ + "deno fmt", + "deno lint", + "git add" ] }, "scripts": { diff --git a/packages/testkit-backend/deno/controller.ts b/packages/testkit-backend/deno/controller.ts index eee83c20b..fdde73fba 100644 --- a/packages/testkit-backend/deno/controller.ts +++ b/packages/testkit-backend/deno/controller.ts @@ -9,7 +9,13 @@ interface Reply { (response: TestkitResponse): Promise; } -function newWire(context: Context, reply: Reply): any { +interface Wire { + writeResponse(response: TestkitRequest): Promise; + writeError(e: Error): Promise; + writeBackendError(msg: string): Promise; +} + +function newWire(context: Context, reply: Reply): Wire { return { writeResponse: (response: TestkitResponse) => reply(response), writeError: (e: Error) => { @@ -54,13 +60,13 @@ export function createHandler( requests: () => AsyncIterable, ) { const context = newContext(); - const wire = newWire(context, response => { - console.log('response:', response) - return reply(response) + const wire = newWire(context, (response) => { + console.log("response:", response); + return reply(response); }); for await (const request of requests()) { - console.log('request:', request) + console.log("request:", request); const { data, name } = request; if (!(name in requestHandlers)) { console.log("Unknown request: " + name); diff --git a/packages/testkit-backend/deno/domain.ts b/packages/testkit-backend/deno/domain.ts index 49004baf8..74d5e817b 100644 --- a/packages/testkit-backend/deno/domain.ts +++ b/packages/testkit-backend/deno/domain.ts @@ -1,3 +1,4 @@ +// deno-lint-ignore-file no-explicit-any import Context from "../src/context.js"; export interface TestkitRequest { From 6d56f087629fa402c706b19660b0428d17cb10ff Mon Sep 17 00:00:00 2001 From: Antonio Barcelos Date: Thu, 29 Sep 2022 12:31:32 +0200 Subject: [PATCH 23/35] Fix `writeError` function --- packages/testkit-backend/deno/controller.ts | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/packages/testkit-backend/deno/controller.ts b/packages/testkit-backend/deno/controller.ts index fdde73fba..eafd454c5 100644 --- a/packages/testkit-backend/deno/controller.ts +++ b/packages/testkit-backend/deno/controller.ts @@ -21,7 +21,7 @@ function newWire(context: Context, reply: Reply): Wire { writeError: (e: Error) => { if (e.name) { if (e.message === "TestKit FrontendError") { - reply({ + return reply({ name: "FrontendError", data: { msg: "Simulating the client code throwing some error.", @@ -29,7 +29,7 @@ function newWire(context: Context, reply: Reply): Wire { }); } else { const id = context.addError(e); - reply({ + return reply({ name: "DriverError", data: { id, @@ -39,10 +39,9 @@ function newWire(context: Context, reply: Reply): Wire { }, }); } - return; } const msg = e.message; - reply({ name: "BackendError", data: { msg } }); + return reply({ name: "BackendError", data: { msg } }); }, writeBackendError: (msg: string) => reply({ name: "BackendError", data: { msg } }), From ed8f1c4eaca811807a38022510f0a1d0e8c0ffbf Mon Sep 17 00:00:00 2001 From: Antonio Barcelos Date: Thu, 29 Sep 2022 13:15:55 +0200 Subject: [PATCH 24/35] Skip test which timesout --- packages/testkit-backend/src/skipped-tests/deno.js | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/packages/testkit-backend/src/skipped-tests/deno.js b/packages/testkit-backend/src/skipped-tests/deno.js index cf56b5a8e..0eec78cf3 100644 --- a/packages/testkit-backend/src/skipped-tests/deno.js +++ b/packages/testkit-backend/src/skipped-tests/deno.js @@ -1,4 +1,4 @@ -import skip, { ifEndsWith, ifStartsWith } from './skip.js' +import skip, { ifEndsWith, ifStartsWith, ifEquals } from './skip.js' const skippedTests = [ skip('DenoJS fail hard on certificate error', @@ -6,11 +6,14 @@ const skippedTests = [ ifEndsWith('test_trusted_ca_wrong_hostname'), ifEndsWith('test_unencrypted'), ifEndsWith('test_untrusted_ca_correct_hostname'), - ifEndsWith('test_1_1'), + ifEndsWith('test_1_1') ), skip('Trust All is no available as configuration', ifStartsWith('tls.test_self_signed_scheme.TestTrustAllCertsConfig.'), ifStartsWith('tls.test_self_signed_scheme.TestSelfSignedScheme.') + ), + skip('Takes a bit longer to complete in TeamCity', + ifEquals('neo4j.test_session_run.TestSessionRun.test_long_string') ) ] From a79e61f60f3fdb97f5abbb41584825a940881e99 Mon Sep 17 00:00:00 2001 From: Antonio Barcelos Date: Thu, 29 Sep 2022 13:20:56 +0200 Subject: [PATCH 25/35] Fix listener --- packages/testkit-backend/deno/index.ts | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/packages/testkit-backend/deno/index.ts b/packages/testkit-backend/deno/index.ts index 826610f31..c2c3108a4 100644 --- a/packages/testkit-backend/deno/index.ts +++ b/packages/testkit-backend/deno/index.ts @@ -1,10 +1,10 @@ import { - handlers, - getShouldRunTest, - createGetFeatures, Context, - neo4j -} from "./deps.ts" + createGetFeatures, + getShouldRunTest, + handlers, + neo4j, +} from "./deps.ts"; import channel from "./channel.ts"; import controller from "./controller.ts"; import { RequestHandlerMap } from "./domain.ts"; @@ -15,10 +15,10 @@ addEventListener("events.errorMonitor", (event) => { console.log("something here ========================", event); }); -addEventListener("unhandledrejection", (event) => { - console.log("unhandledrejection", event) - console.error("unhandledrejection", event) -}) +addEventListener("unhandledrejection", (event: Event) => { + // @ts-ignore PromiseRejectionEvent has reason property + console.error("unhandledrejection", event.reason); +}); const descriptor = ["async", "deno"]; const shouldRunTest = getShouldRunTest(descriptor); From b6e659ac92da24d78cf4b4d26f3ab051de7e85d5 Mon Sep 17 00:00:00 2001 From: Antonio Barcelos Date: Thu, 29 Sep 2022 13:25:08 +0200 Subject: [PATCH 26/35] Handling connection write errors --- packages/bolt-connection/src/channel/deno/deno-channel.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/bolt-connection/src/channel/deno/deno-channel.js b/packages/bolt-connection/src/channel/deno/deno-channel.js index 3ea05f424..4e1bf4780 100644 --- a/packages/bolt-connection/src/channel/deno/deno-channel.js +++ b/packages/bolt-connection/src/channel/deno/deno-channel.js @@ -137,7 +137,7 @@ export default class DenoChannel { if (this._pending !== null) { this._pending.push(buffer) } else if (buffer instanceof ChannelBuffer) { - this._conn.write(buffer._buffer) + this._conn.write(buffer._buffer).catch(this._handleConnectionError) } else { throw newError("Don't know how to send buffer: " + buffer) } From 468dc5b916ed77e3a55ef5f52a2ab16f1b11948d Mon Sep 17 00:00:00 2001 From: Antonio Barcelos Date: Thu, 29 Sep 2022 13:51:15 +0200 Subject: [PATCH 27/35] Skip timedout test --- packages/testkit-backend/src/skipped-tests/deno.js | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/packages/testkit-backend/src/skipped-tests/deno.js b/packages/testkit-backend/src/skipped-tests/deno.js index 0eec78cf3..50879400f 100644 --- a/packages/testkit-backend/src/skipped-tests/deno.js +++ b/packages/testkit-backend/src/skipped-tests/deno.js @@ -13,7 +13,8 @@ const skippedTests = [ ifStartsWith('tls.test_self_signed_scheme.TestSelfSignedScheme.') ), skip('Takes a bit longer to complete in TeamCity', - ifEquals('neo4j.test_session_run.TestSessionRun.test_long_string') + ifEquals('neo4j.test_session_run.TestSessionRun.test_long_string'), + ifEquals('stub.driver_parameters.test_connection_acquisition_timeout_ms.TestConnectionAcquisitionTimeoutMs.test_should_fail_when_connection_timeout_is_reached_first)') ) ] From 46f95b46f86c2f398a2be445e0c3071bec825e6b Mon Sep 17 00:00:00 2001 From: Antonio Barcelos Date: Thu, 29 Sep 2022 14:01:20 +0200 Subject: [PATCH 28/35] fix connection timeout --- packages/bolt-connection/src/channel/deno/deno-channel.js | 2 +- packages/testkit-backend/src/skipped-tests/deno.js | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/packages/bolt-connection/src/channel/deno/deno-channel.js b/packages/bolt-connection/src/channel/deno/deno-channel.js index 4e1bf4780..fa069186c 100644 --- a/packages/bolt-connection/src/channel/deno/deno-channel.js +++ b/packages/bolt-connection/src/channel/deno/deno-channel.js @@ -85,6 +85,7 @@ export default class DenoChannel { return setTimeout(() => { this._connectionTimeoutFired = true this.close() + .then(e => this._handleConnectionError(newError(`Connection timeout after ${timeout} ms`))) .catch(this._handleConnectionError) }, timeout) } @@ -202,7 +203,6 @@ export default class DenoChannel { this._receiveTimeoutId = setTimeout(() => { this._receiveTimeoutId = null - this._timedout = true this.stopReceiveTimeout() this._error = newError( `Connection lost. Server didn't respond in ${this._receiveTimeout}ms`, diff --git a/packages/testkit-backend/src/skipped-tests/deno.js b/packages/testkit-backend/src/skipped-tests/deno.js index 50879400f..0eec78cf3 100644 --- a/packages/testkit-backend/src/skipped-tests/deno.js +++ b/packages/testkit-backend/src/skipped-tests/deno.js @@ -13,8 +13,7 @@ const skippedTests = [ ifStartsWith('tls.test_self_signed_scheme.TestSelfSignedScheme.') ), skip('Takes a bit longer to complete in TeamCity', - ifEquals('neo4j.test_session_run.TestSessionRun.test_long_string'), - ifEquals('stub.driver_parameters.test_connection_acquisition_timeout_ms.TestConnectionAcquisitionTimeoutMs.test_should_fail_when_connection_timeout_is_reached_first)') + ifEquals('neo4j.test_session_run.TestSessionRun.test_long_string') ) ] From d95095715cffbe9d3b5454687d2da7f82253b65f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Antonio=20Barc=C3=A9los?= Date: Mon, 3 Oct 2022 15:58:19 +0200 Subject: [PATCH 29/35] Apply suggestions from code review Co-authored-by: Robsdedude --- .../src/channel/deno/deno-channel.js | 21 +++++++++---------- .../bolt-connection/src/channel/deno/index.js | 2 +- packages/neo4j-driver-deno/generate.ts | 2 +- packages/testkit-backend/deno/README.md | 12 +++++------ .../testkit-backend/src/skipped-tests/deno.js | 4 ++-- 5 files changed, 20 insertions(+), 21 deletions(-) diff --git a/packages/bolt-connection/src/channel/deno/deno-channel.js b/packages/bolt-connection/src/channel/deno/deno-channel.js index fa069186c..8383d6554 100644 --- a/packages/bolt-connection/src/channel/deno/deno-channel.js +++ b/packages/bolt-connection/src/channel/deno/deno-channel.js @@ -27,7 +27,7 @@ const { let _CONNECTION_IDGEN = 0 /** - * Create a new DenoChannel to be used in web browsers. + * Create a new DenoChannel to be used in Deno runtime. * @access private */ export default class DenoChannel { @@ -42,21 +42,20 @@ export default class DenoChannel { this.id = _CONNECTION_IDGEN++ this._conn = null this._pending = [] - this._error = null this._open = true - this._config = config - - this._receiveTimeout = null - this._receiveTimeoutStarted = false - this._receiveTimeoutId = null - - this._connectionErrorCode = config.connectionErrorCode + this._error = null this._handleConnectionError = this._handleConnectionError.bind(this) this._handleConnectionTerminated = this._handleConnectionTerminated.bind( this ) + this._connectionErrorCode = config.connectionErrorCode + this._receiveTimeout = null + this._receiveTimeoutStarted = false + this._receiveTimeoutId = null + + this._config = config - this._socketPromise = connect(config) + connect(config) .then(conn => { this._clearConnectionTimeout() if (!this._open) { @@ -68,7 +67,7 @@ export default class DenoChannel { .catch(this._handleConnectionError) const pending = this._pending - this._pending = null + this._pending = null for (let i = 0; i < pending.length; i++) { this.write(pending[i]) } diff --git a/packages/bolt-connection/src/channel/deno/index.js b/packages/bolt-connection/src/channel/deno/index.js index d84e98195..7ed3a270b 100644 --- a/packages/bolt-connection/src/channel/deno/index.js +++ b/packages/bolt-connection/src/channel/deno/index.js @@ -25,7 +25,7 @@ import DenoHostNameResolver from './deno-host-name-resolver' This module exports a set of components to be used in deno environment. They are not compatible with NodeJS environment. All files import/require APIs from `node/index.js` by default. - Such imports are replaced at build time with `browser/index.js` when building a browser bundle. + Such imports are replaced at build time with `deno/index.js` when building a deno bundle. NOTE: exports in this module should have exactly the same names/structure as exports in `node/index.js`. diff --git a/packages/neo4j-driver-deno/generate.ts b/packages/neo4j-driver-deno/generate.ts index a7792c5e9..ed5741c62 100644 --- a/packages/neo4j-driver-deno/generate.ts +++ b/packages/neo4j-driver-deno/generate.ts @@ -130,7 +130,7 @@ async function copyAndTransform(inDir: string, outDir: string) { } // Special fix for bolt-connection/channel/index.js - // Replace the "node channel" with the "deno channel", since Deno supports browser APIs + // Replace the "node channel" with the "deno channel", since Deno supports different APIs if (inPath.endsWith("channel/index.js")) { contents = contents.replace( `export * from './node/index.js'`, diff --git a/packages/testkit-backend/deno/README.md b/packages/testkit-backend/deno/README.md index 7218b28cc..783ab655a 100644 --- a/packages/testkit-backend/deno/README.md +++ b/packages/testkit-backend/deno/README.md @@ -1,21 +1,21 @@ -# Deno Testkit Backend Specific implementations +# Deno-Specific Testkit Backend Implementations -This directory contains Deno specific implementations which depends on having +This directory contains Deno specific implementations which depend on having `Deno` global variable available or being able to load `Deno` specific libraries such as the `Neo4j Deno Module`. Files like `../feature/deno.js` and `../skipped-tests/deno.js` are outside this directory since they are pure -javascript configuration files and they don't depends on the environment. +javascript configuration files, and they don't depend on the environment. ## Starting Backend -### Pre-requisites +### Pre-Requisites First, you need to build the `Neo4j Deno Module` by running `npm run build::deno` in the repository root folder. -### The start command +### The Start Command -For starting this backend, you should run the following command in the current +For starting this backend, you should run the following command in this directory: ``` diff --git a/packages/testkit-backend/src/skipped-tests/deno.js b/packages/testkit-backend/src/skipped-tests/deno.js index 0eec78cf3..e2dd2eec2 100644 --- a/packages/testkit-backend/src/skipped-tests/deno.js +++ b/packages/testkit-backend/src/skipped-tests/deno.js @@ -1,14 +1,14 @@ import skip, { ifEndsWith, ifStartsWith, ifEquals } from './skip.js' const skippedTests = [ - skip('DenoJS fail hard on certificate error', + skip('DenoJS fails hard on certificate error', ifEndsWith('test_trusted_ca_expired_server_correct_hostname'), ifEndsWith('test_trusted_ca_wrong_hostname'), ifEndsWith('test_unencrypted'), ifEndsWith('test_untrusted_ca_correct_hostname'), ifEndsWith('test_1_1') ), - skip('Trust All is no available as configuration', + skip('Trust All is not available as configuration', ifStartsWith('tls.test_self_signed_scheme.TestTrustAllCertsConfig.'), ifStartsWith('tls.test_self_signed_scheme.TestSelfSignedScheme.') ), From bfb81bcd2f69f4e989203ad75a7a5587b487a837 Mon Sep 17 00:00:00 2001 From: Antonio Barcelos Date: Mon, 3 Oct 2022 16:47:37 +0200 Subject: [PATCH 30/35] Move binder creation and log env-variable read to the controller. --- packages/testkit-backend/deno/deps.ts | 1 + packages/testkit-backend/deno/index.ts | 6 +++- packages/testkit-backend/src/context.js | 12 ++++++- .../testkit-backend/src/controller/local.js | 4 ++- .../src/request-handlers-rx.js | 16 ++++----- .../testkit-backend/src/request-handlers.js | 34 +++++++------------ 6 files changed, 39 insertions(+), 34 deletions(-) diff --git a/packages/testkit-backend/deno/deps.ts b/packages/testkit-backend/deno/deps.ts index 5fb388e3f..898990acd 100644 --- a/packages/testkit-backend/deno/deps.ts +++ b/packages/testkit-backend/deno/deps.ts @@ -4,3 +4,4 @@ export { getShouldRunTest } from "../src/skipped-tests/index.js"; export { default as neo4j } from "../../neo4j-driver-deno/lib/mod.ts"; export { createGetFeatures } from "../src/feature/index.js"; export * as handlers from "../src/request-handlers.js"; +export { default as CypherNativeBinders } from "../src/cypher-native-binders.js"; diff --git a/packages/testkit-backend/deno/index.ts b/packages/testkit-backend/deno/index.ts index c2c3108a4..60efbff5d 100644 --- a/packages/testkit-backend/deno/index.ts +++ b/packages/testkit-backend/deno/index.ts @@ -1,6 +1,7 @@ import { Context, createGetFeatures, + CypherNativeBinders, getShouldRunTest, handlers, neo4j, @@ -20,10 +21,13 @@ addEventListener("unhandledrejection", (event: Event) => { console.error("unhandledrejection", event.reason); }); +const binder = new CypherNativeBinders(neo4j); const descriptor = ["async", "deno"]; const shouldRunTest = getShouldRunTest(descriptor); const getFeatures = createGetFeatures(descriptor); -const createContext = () => new Context(shouldRunTest, getFeatures); +const logLevel = Deno.env.get("LOG_LEVEL"); +const createContext = () => + new Context(shouldRunTest, getFeatures, binder, logLevel); const listener = channel.listen(9876); const handle = controller.createHandler(neo4j, createContext, requestHandlers); diff --git a/packages/testkit-backend/src/context.js b/packages/testkit-backend/src/context.js index fad3a5329..ab4733c0b 100644 --- a/packages/testkit-backend/src/context.js +++ b/packages/testkit-backend/src/context.js @@ -1,5 +1,5 @@ export default class Context { - constructor (shouldRunTest, getFeatures) { + constructor (shouldRunTest, getFeatures, binder, environmentLogLevel) { this._id = 0 this._drivers = {} this._sessions = {} @@ -12,6 +12,16 @@ export default class Context { this._bookmarkSupplierRequests = {} this._notifyBookmarksRequests = {} this._bookmarksManagers = {} + this._binder = binder + this._environmentLogLevel = environmentLogLevel + } + + get binder () { + return this._binder + } + + get environmentLogLevel () { + return this._environmentLogLevel } addDriver (driver) { diff --git a/packages/testkit-backend/src/controller/local.js b/packages/testkit-backend/src/controller/local.js index 6ef154aa8..a670cd654 100644 --- a/packages/testkit-backend/src/controller/local.js +++ b/packages/testkit-backend/src/controller/local.js @@ -2,6 +2,7 @@ import Context from '../context' import Controller from './interface' import stringify from '../stringify' import { isFrontendError } from '../request-handlers' +import CypherNativeBinders from './cypher-native-binders' /** * Local controller handles the requests locally by redirecting them to the correct request handler/service. @@ -16,10 +17,11 @@ export default class LocalController extends Controller { this._getFeatures = getFeatures this._contexts = new Map() this._neo4j = neo4j + this._binder = new CypherNativeBinders(neo4j) } openContext (contextId) { - this._contexts.set(contextId, new Context(this._shouldRunTest, this._getFeatures)) + this._contexts.set(contextId, new Context(this._shouldRunTest, this._getFeatures, this._binder, process.env.LOG_LEVEL)) } closeContext (contextId) { diff --git a/packages/testkit-backend/src/request-handlers-rx.js b/packages/testkit-backend/src/request-handlers-rx.js index ee8a0dc1e..8a882dfa4 100644 --- a/packages/testkit-backend/src/request-handlers-rx.js +++ b/packages/testkit-backend/src/request-handlers-rx.js @@ -1,5 +1,4 @@ import * as responses from './responses.js' -import CypherNativeBinders from './cypher-native-binders.js' import { from } from 'rxjs' // Handlers which didn't change depending @@ -69,13 +68,12 @@ export function SessionClose (_, context, data, wire) { .catch(err => wire.writeError(err)) } -export function SessionRun (neo4j, context, data, wire) { +export function SessionRun (_, context, data, wire) { const { sessionId, cypher, params, txMeta: metadata, timeout } = data const session = context.getSession(sessionId) if (params) { - const binder = new CypherNativeBinders(neo4j) for (const [key, value] of Object.entries(params)) { - params[key] = binder.cypherToNative(value) + params[key] = context.binder.cypherToNative(value) } } @@ -102,15 +100,14 @@ export function SessionRun (neo4j, context, data, wire) { }) } -export function ResultConsume (neo4j, context, data, wire) { +export function ResultConsume (_, context, data, wire) { const { resultId } = data const result = context.getResult(resultId) - const binder = new CypherNativeBinders(neo4j) return result.consume() .toPromise() .then(summary => { - wire.writeResponse(responses.Summary({ summary }, { binder })) + wire.writeResponse(responses.Summary({ summary }, { binder: context.binder })) }).catch(e => wire.writeError(e)) } @@ -134,13 +131,12 @@ export function SessionBeginTransaction (_, context, data, wire) { } } -export function TransactionRun (neo4j, context, data, wire) { +export function TransactionRun (_, context, data, wire) { const { txId, cypher, params } = data const tx = context.getTx(txId) - const binder = new CypherNativeBinders(neo4j) if (params) { for (const [key, value] of Object.entries(params)) { - params[key] = binder.cypherToNative(value) + params[key] = context.binder.cypherToNative(value) } } diff --git a/packages/testkit-backend/src/request-handlers.js b/packages/testkit-backend/src/request-handlers.js index 1748e77ee..49b8fb0a2 100644 --- a/packages/testkit-backend/src/request-handlers.js +++ b/packages/testkit-backend/src/request-handlers.js @@ -1,4 +1,3 @@ -import CypherNativeBinders from './cypher-native-binders.js' import * as responses from './responses.js' export function throwFrontendError () { @@ -48,12 +47,11 @@ export function NewDriver (neo4j, context, data, wire) { }) : undefined - const envLogLevel = typeof process === 'undefined' ? undefined : process.env.LOG_LEVEL const config = { userAgent, resolver, useBigInt: true, - logging: neo4j.logging.console(envLogLevel || context.logLevel) + logging: neo4j.logging.console(context.logLevel || context.environmentLogLevel) } if ('encrypted' in data) { config.encrypted = data.encrypted ? 'ENCRYPTION_ON' : 'ENCRYPTION_OFF' @@ -152,13 +150,12 @@ export function SessionClose (_, context, data, wire) { .catch(err => wire.writeError(err)) } -export function SessionRun (neo4j, context, data, wire) { +export function SessionRun (_, context, data, wire) { const { sessionId, cypher, params, txMeta: metadata, timeout } = data const session = context.getSession(sessionId) - const binder = new CypherNativeBinders(neo4j) if (params) { for (const [key, value] of Object.entries(params)) { - params[key] = binder.cypherToNative(value) + params[key] = context.binder.cypherToNative(value) } } @@ -176,10 +173,9 @@ export function SessionRun (neo4j, context, data, wire) { wire.writeResponse(responses.Result({ id })) } -export function ResultNext (neo4j, context, data, wire) { +export function ResultNext (_, context, data, wire) { const { resultId } = data const result = context.getResult(resultId) - const binder = new CypherNativeBinders(neo4j) if (!('recordIt' in result)) { result.recordIt = result[Symbol.asyncIterator]() } @@ -187,7 +183,7 @@ export function ResultNext (neo4j, context, data, wire) { if (done) { wire.writeResponse(responses.NullRecord()) } else { - wire.writeResponse(responses.Record({ record: value }, { binder })) + wire.writeResponse(responses.Record({ record: value }, { binder: context.binder })) } }).catch(e => { console.log('got some err: ' + JSON.stringify(e)) @@ -195,10 +191,9 @@ export function ResultNext (neo4j, context, data, wire) { }) } -export function ResultPeek (neo4j, context, data, wire) { +export function ResultPeek (_, context, data, wire) { const { resultId } = data const result = context.getResult(resultId) - const binder = new CypherNativeBinders(neo4j) if (!('recordIt' in result)) { result.recordIt = result[Symbol.asyncIterator]() } @@ -206,7 +201,7 @@ export function ResultPeek (neo4j, context, data, wire) { if (done) { wire.writeResponse(responses.NullRecord()) } else { - wire.writeResponse(responses.Record({ record: value }, { binder })) + wire.writeResponse(responses.Record({ record: value }, { binder: context.binder })) } }).catch(e => { console.log('got some err: ' + JSON.stringify(e)) @@ -214,24 +209,22 @@ export function ResultPeek (neo4j, context, data, wire) { }) } -export function ResultConsume (neo4j, context, data, wire) { +export function ResultConsume (_, context, data, wire) { const { resultId } = data const result = context.getResult(resultId) - const binder = new CypherNativeBinders(neo4j) return result.summary().then(summary => { - wire.writeResponse(responses.Summary({ summary }, { binder })) + wire.writeResponse(responses.Summary({ summary }, { binder: context.binder })) }).catch(e => wire.writeError(e)) } -export function ResultList (neo4j, context, data, wire) { +export function ResultList (_, context, data, wire) { const { resultId } = data - const binder = new CypherNativeBinders(neo4j) const result = context.getResult(resultId) return result .then(({ records }) => { - wire.writeResponse(responses.RecordList({ records }, { binder })) + wire.writeResponse(responses.RecordList({ records }, { binder: context.binder })) }) .catch(error => wire.writeError(error)) } @@ -251,13 +244,12 @@ export function SessionReadTransaction (_, context, data, wire) { .catch(error => wire.writeError(error)) } -export function TransactionRun (neo4j, context, data, wire) { +export function TransactionRun (_, context, data, wire) { const { txId, cypher, params } = data const tx = context.getTx(txId) - const binder = new CypherNativeBinders(neo4j) if (params) { for (const [key, value] of Object.entries(params)) { - params[key] = binder.cypherToNative(value) + params[key] = context.binder.cypherToNative(value) } } const result = tx.tx.run(cypher, params) From e31cdddbf68659b724c364904ac2bd0ae9ca1f73 Mon Sep 17 00:00:00 2001 From: Antonio Barcelos Date: Mon, 3 Oct 2022 17:11:33 +0200 Subject: [PATCH 31/35] Waiting for connection be closed during timeout --- .../bolt-connection/src/channel/deno/deno-channel.js | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/packages/bolt-connection/src/channel/deno/deno-channel.js b/packages/bolt-connection/src/channel/deno/deno-channel.js index 8383d6554..acb5365bb 100644 --- a/packages/bolt-connection/src/channel/deno/deno-channel.js +++ b/packages/bolt-connection/src/channel/deno/deno-channel.js @@ -209,9 +209,15 @@ export default class DenoChannel { ) this.close() - if (this.onerror) { - this.onerror(this._error) - } + .catch(() => { + // ignoring error during the close timeout connections since they + // not valid + }) + .finally(() => { + if (this.onerror) { + this.onerror(this._error) + } + }) }, this._receiveTimeout) } } From 3d68810ce38bfc2d482a71485893f813be1cbccf Mon Sep 17 00:00:00 2001 From: Antonio Barcelos Date: Mon, 3 Oct 2022 17:19:41 +0200 Subject: [PATCH 32/35] docs improvement --- packages/neo4j-driver-deno/README.md | 41 ++++++++++++++++++++++++++-- 1 file changed, 38 insertions(+), 3 deletions(-) diff --git a/packages/neo4j-driver-deno/README.md b/packages/neo4j-driver-deno/README.md index f7a97c58e..26eaa3cec 100644 --- a/packages/neo4j-driver-deno/README.md +++ b/packages/neo4j-driver-deno/README.md @@ -21,7 +21,7 @@ The script will: 1. Copy `neo4j-driver-lite` and the Neo4j packages it uses into a subfolder here called `lib`. 1. Rewrite all imports to Deno-compatible versions -1. Replace the "node channel" with the "browser channel" +1. Replace the "node channel" with the "deno channel" 1. Test that the resulting driver can be imported by Deno and passes type checks It is not necessary to do any other setup first; in particular, you don't need @@ -53,7 +53,42 @@ you don't have a running Neo4j instance, you can use `docker run --rm -p 7687:7687 -e NEO4J_AUTH=neo4j/driverdemo neo4j:4.4` to quickly spin one up. +## TLS + +For using system certificates, the `DENO_TLS_CA_STORE` should be set to `"system"`. +`TRUST_ALL_CERTIFICATES` should be handle by `--unsafely-ignore-certificate-errors` and not by driver configuration. See, https://deno.com/blog/v1.13#disable-tls-verification; + ## Tests -It is not yet possible to run the test suite with this driver. Contributions to -make that possible would be welcome. +Tests **require** latest [Testkit 5.0](https://github.com/neo4j-drivers/testkit/tree/5.0), Python3 and Docker. + +Testkit is needed to be cloned and configured to run against the Javascript Lite Driver. Use the following steps to configure Testkit. + +1. Clone the Testkit repository + +``` +git clone https://github.com/neo4j-drivers/testkit.git +``` + +2. Under the Testkit folder, install the requirements. + +``` +pip3 install -r requirements.txt +``` + +3. Define some enviroment variables to configure Testkit + +``` +export TEST_DRIVER_NAME=javascript +export TEST_DRIVER_REPO= +export TEST_DRIVER_DENO=1 +``` + +To run test against against some Neo4j version: + +``` +python3 main.py +``` + +More details about how to use Teskit could be found on [its repository](https://github.com/neo4j-drivers/testkit/tree/5.0) + From 6852df6bc1259e85ed10409e9c7099e4b0f93704 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Antonio=20Barc=C3=A9los?= Date: Mon, 3 Oct 2022 17:21:49 +0200 Subject: [PATCH 33/35] Apply suggestions from code review Co-authored-by: Robsdedude --- packages/testkit-backend/src/cypher-native-binders.js | 1 - 1 file changed, 1 deletion(-) diff --git a/packages/testkit-backend/src/cypher-native-binders.js b/packages/testkit-backend/src/cypher-native-binders.js index 8890c8720..712685b5b 100644 --- a/packages/testkit-backend/src/cypher-native-binders.js +++ b/packages/testkit-backend/src/cypher-native-binders.js @@ -3,7 +3,6 @@ export default function CypherNativeBinders (neo4j) { function valueResponse (name, value) { return { name: name, data: { value: value } } } - function objectToCypher (obj) { return objectMapper(obj, nativeToCypher) } From b47d05e456b6700abc3209f6e5d6adc70220e9d4 Mon Sep 17 00:00:00 2001 From: Antonio Barcelos Date: Mon, 3 Oct 2022 17:39:26 +0200 Subject: [PATCH 34/35] Fix CypherNativeBinder import --- packages/testkit-backend/src/controller/local.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/testkit-backend/src/controller/local.js b/packages/testkit-backend/src/controller/local.js index a670cd654..2d5c80af0 100644 --- a/packages/testkit-backend/src/controller/local.js +++ b/packages/testkit-backend/src/controller/local.js @@ -2,7 +2,7 @@ import Context from '../context' import Controller from './interface' import stringify from '../stringify' import { isFrontendError } from '../request-handlers' -import CypherNativeBinders from './cypher-native-binders' +import CypherNativeBinders from '../cypher-native-binders' /** * Local controller handles the requests locally by redirecting them to the correct request handler/service. From d9c32e1d53c5682c28836b3824a96ebb65080d99 Mon Sep 17 00:00:00 2001 From: Antonio Barcelos Date: Tue, 4 Oct 2022 16:01:52 +0200 Subject: [PATCH 35/35] Install specific Deno version --- testkit/Dockerfile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/testkit/Dockerfile b/testkit/Dockerfile index e64130231..15e30041a 100644 --- a/testkit/Dockerfile +++ b/testkit/Dockerfile @@ -1,6 +1,7 @@ FROM ubuntu:20.04 ARG NODE_VERSION=10 +ARG DENO_VERSION=1.19.3 ENV DEBIAN_FRONTEND noninteractive ENV NODE_OPTIONS --max_old_space_size=4096 --use-openssl-ca @@ -47,7 +48,7 @@ COPY CustomCAs/* /usr/local/share/custom-ca-certificates/ RUN update-ca-certificates --verbose # Add Deno -RUN curl -fsSL https://deno.land/x/install/install.sh | sh +RUN curl -fsSL https://deno.land/x/install/install.sh | sh -s v$DENO_VERSION RUN mv /root/.deno/bin/deno /usr/bin/ # Using System CA in Deno ENV DENO_TLS_CA_STORE=system