Added ability to enforce max connection lifetime

Driver keeps idle network connections in a connection pool. These
connections can get invalidated while resting idle in the pool.
They can be killed by network equipment like load balancer, proxy or
firewall. It is also safer to refresh connections once in a while so
that database can renew corresponding resources.

This commit adds `maxConnectionLifetime` setting and makes driver
close too old connections. Checking and closing happens during
connection acquisition.

Also removed manual `Date.now()` mocking in favour of
existing library - Lolex.

Author: lutovich

src/v1/driver.js

@@ -47,6 +47,8 @@ class Driver {
    * @protected
    */
   constructor(url, userAgent, token = {}, config = {}) {
+    sanitizeConfig(config);
+
     this._url = url;
     this._userAgent = userAgent;
     this._openSessions = {};
@@ -56,7 +58,7 @@ class Driver {
     this._pool = new Pool(
       this._createConnection.bind(this),
       this._destroyConnection.bind(this),
-      Driver._validateConnection.bind(this),
+      this._validateConnection.bind(this),
       config.connectionPoolSize
     );
 
@@ -90,8 +92,20 @@ class Driver {
    * @return {boolean} true if the connection is open
    * @access private
    **/
-  static _validateConnection(conn) {
-    return conn.isOpen();
+  _validateConnection(conn) {
+    if (!conn.isOpen()) {
+      return false;
+    }
+
+    const maxConnectionLifetime = this._config.maxConnectionLifetime;
+    if (maxConnectionLifetime) {
+      const lifetime = Date.now() - conn.creationTimestamp;
+      if (lifetime > maxConnectionLifetime) {
+        return false;
+      }
+    }
+
+    return true;
   }
 
   /**
@@ -213,7 +227,22 @@ class _ConnectionStreamObserver extends StreamObserver {
   }
 }
 
-
+/**
+ * @private
+ */
+function sanitizeConfig(config) {
+  const maxConnectionLifetime = config.maxConnectionLifetime;
+  if (maxConnectionLifetime) {
+    const sanitizedMaxConnectionLifetime = parseInt(maxConnectionLifetime, 10);
+    if (sanitizedMaxConnectionLifetime && sanitizedMaxConnectionLifetime > 0) {
+      config.maxConnectionLifetime = sanitizedMaxConnectionLifetime;
+    } else {
+      config.maxConnectionLifetime = null;
+    }
+  } else {
+    config.maxConnectionLifetime = null;
+  }
+}
 
 export {Driver, READ, WRITE}
 

src/v1/index.js

@@ -93,7 +93,7 @@ const USER_AGENT = "neo4j-javascript/" + VERSION;
  *       // TRUST_SYSTEM_CA_SIGNED_CERTIFICATES meand that you trust whatever certificates
  *       // are in the default certificate chain of th
  *       trust: "TRUST_ALL_CERTIFICATES" | "TRUST_ON_FIRST_USE" | "TRUST_SIGNED_CERTIFICATES" |
-  *       "TRUST_CUSTOM_CA_SIGNED_CERTIFICATES" | "TRUST_SYSTEM_CA_SIGNED_CERTIFICATES",
+ *       "TRUST_CUSTOM_CA_SIGNED_CERTIFICATES" | "TRUST_SYSTEM_CA_SIGNED_CERTIFICATES",
  *
  *       // List of one or more paths to trusted encryption certificates. This only
  *       // works in the NodeJS bundle, and only matters if you use "TRUST_CUSTOM_CA_SIGNED_CERTIFICATES".
@@ -112,11 +112,20 @@ const USER_AGENT = "neo4j-javascript/" + VERSION;
  *       // Connection will be destroyed if this threshold is exceeded.
  *       connectionPoolSize: 50,
  *
+ *       // The maximum allowed lifetime for a pooled connection in milliseconds. Pooled connections older than this
+ *       // threshold will be closed and removed from the pool. Such discarding happens during connection acquisition
+ *       // so that new session is never backed by an old connection. Setting this option to a low value will cause
+ *       // a high connection churn and might result in a performance hit. It is recommended to set maximum lifetime
+ *       // to a slightly smaller value than the one configured in network equipment (load balancer, proxy, firewall,
+ *       // etc. can also limit maximum connection lifetime). No maximum lifetime limit is imposed by default. Zero
+ *       // and negative values result in lifetime not being checked.
+ *       maxConnectionLifetime: 30 * 60 * 1000, // 30 minutes
+ *
  *       // Specify the maximum time in milliseconds transactions are allowed to retry via
  *       // {@link Session#readTransaction()} and {@link Session#writeTransaction()} functions. These functions
  *       // will retry the given unit of work on `ServiceUnavailable`, `SessionExpired` and transient errors with
  *       // exponential backoff using initial delay of 1 second. Default value is 30000 which is 30 seconds.
- *       maxTransactionRetryTime: 30000,
+ *       maxTransactionRetryTime: 30000, // 30 seconds
  *
  *       // Provide an alternative load balancing strategy for the routing driver to use.
  *       // Driver uses "least_connected" by default.

src/v1/internal/connector.js

@@ -174,6 +174,7 @@ class Connection {
      */
     this.url = url;
     this.server = {address: url};
+    this.creationTimestamp = Date.now();
     this._pendingObservers = [];
     this._currentObserver = undefined;
     this._ch = channel;

src/v1/internal/transaction-executor.js

@@ -54,7 +54,8 @@ export default class TransactionExecutor {
   }
 
   _retryTransactionPromise(transactionCreator, transactionWork, error, retryStartTime, retryDelayMs) {
-    const elapsedTimeMs = Date.now() - retryStartTime;
+    const now = Date.now();
+    const elapsedTimeMs = now - retryStartTime;
 
     if (elapsedTimeMs > this._maxRetryTimeMs || !TransactionExecutor._canRetryOn(error)) {
       return Promise.reject(error);

test/internal/connector.test.js

@@ -25,12 +25,19 @@ import {alloc} from '../../src/v1/internal/buf';
 import {Neo4jError} from '../../src/v1/error';
 import sharedNeo4j from '../internal/shared-neo4j';
 import {ServerVersion} from '../../src/v1/internal/server-version';
+import lolex from 'lolex';
 
 describe('connector', () => {
 
+  let clock;
   let connection;
 
   afterEach(done => {
+    if (clock) {
+      clock.uninstall();
+      clock = null;
+    }
+
     const usedConnection = connection;
     connection = null;
     if (usedConnection) {
@@ -39,6 +46,15 @@ describe('connector', () => {
     done();
   });
 
+  it('should have correct creation timestamp', () => {
+    clock = lolex.install();
+    clock.setSystemTime(424242);
+
+    connection = connect('bolt://localhost');
+
+    expect(connection.creationTimestamp).toEqual(424242);
+  });
+
   it('should read/write basic messages', done => {
     // Given
     connection = connect("bolt://localhost");

test/internal/fake-connection.js

@@ -27,6 +27,9 @@
 export default class FakeConnection {
 
   constructor() {
+    this._open = true;
+    this.creationTimestamp = Date.now();
+
     this.resetInvoked = 0;
     this.resetAsyncInvoked = 0;
     this.syncInvoked = 0;
@@ -68,6 +71,10 @@ export default class FakeConnection {
     return this._initializationPromise;
   }
 
+  isOpen() {
+    return this._open;
+  }
+
   isReleasedOnceOnSessionClose() {
     return this.isReleasedOnSessionCloseTimes(1);
   }
@@ -103,4 +110,14 @@ export default class FakeConnection {
     this._initializationPromise = Promise.reject(error);
     return this;
   }
+
+  withCreationTimestamp(value) {
+    this.creationTimestamp = value;
+    return this;
+  }
+
+  closed() {
+    this._open = false;
+    return this;
+  }
 };

test/internal/routing-util.test.js

@@ -30,12 +30,11 @@ describe('RoutingUtil', () => {
 
   let clock;
 
-  beforeAll(() => {
-    clock = lolex.install();
-  });
-
-  afterAll(() => {
-    clock.uninstall();
+  afterEach(() => {
+    if (clock) {
+      clock.uninstall();
+      clock = null;
+    }
   });
 
   it('should return retrieved records when query succeeds', done => {
@@ -141,6 +140,8 @@ describe('RoutingUtil', () => {
   });
 
   it('should parse valid ttl', () => {
+    clock = lolex.install();
+
     testValidTtlParsing(100, 5);
     testValidTtlParsing(Date.now(), 3600); // 1 hour
     testValidTtlParsing(Date.now(), 86400); // 24 hours
@@ -152,6 +153,7 @@ describe('RoutingUtil', () => {
 
   it('should not overflow parsing huge ttl', () => {
     const record = newRecord({ttl: Integer.MAX_VALUE});
+    clock = lolex.install();
     clock.setSystemTime(42);
 
     const expirationTime = parseTtl(record);
@@ -161,6 +163,7 @@ describe('RoutingUtil', () => {
 
   it('should return valid value parsing negative ttl', () => {
     const record = newRecord({ttl: int(-42)});
+    clock = lolex.install();
     clock.setSystemTime(42);
 
     const expirationTime = parseTtl(record);

test/internal/timers-util.js

@@ -66,21 +66,3 @@ class SetTimeoutMock {
 }
 
 export const setTimeoutMock = new SetTimeoutMock();
-
-export function hijackNextDateNowCall(newValue) {
-  const originalDate = global.Date;
-  global.Date = new FakeDate(originalDate, newValue);
-}
-
-class FakeDate {
-
-  constructor(originalDate, nextNowValue) {
-    this._originalDate = originalDate;
-    this._nextNowValue = nextNowValue;
-  }
-
-  now() {
-    global.Date = this._originalDate;
-    return this._nextNowValue;
-  }
-}

test/internal/transaction-executor.test.js

@@ -19,7 +19,8 @@
 
 import TransactionExecutor from '../../src/v1/internal/transaction-executor';
 import {newError, SERVICE_UNAVAILABLE, SESSION_EXPIRED} from '../../src/v1/error';
-import {hijackNextDateNowCall, setTimeoutMock} from './timers-util';
+import {setTimeoutMock} from './timers-util';
+import lolex from 'lolex';
 
 const TRANSIENT_ERROR_1 = 'Neo.TransientError.Transaction.DeadlockDetected';
 const TRANSIENT_ERROR_2 = 'Neo.TransientError.Network.CommunicationError';
@@ -30,13 +31,18 @@ const OOM_ERROR = 'Neo.DatabaseError.General.OutOfMemoryError';
 
 describe('TransactionExecutor', () => {
 
+  let clock;
   let fakeSetTimeout;
 
   beforeEach(() => {
     fakeSetTimeout = setTimeoutMock.install();
   });
 
   afterEach(() => {
+    if (clock) {
+      clock.uninstall();
+      clock = null;
+    }
     fakeSetTimeout.uninstall();
   });
 
@@ -81,7 +87,9 @@ describe('TransactionExecutor', () => {
       expect(tx).toBeDefined();
       workInvocationCounter++;
       if (workInvocationCounter === 3) {
-        hijackNextDateNowCall(Date.now() + 30001); // move next `Date.now()` call forward by 30 seconds
+        const currentTime = Date.now();
+        clock = lolex.install();
+        clock.setSystemTime(currentTime + 30001); // move `Date.now()` call forward by 30 seconds
       }
       return realWork();
     });