diff --git a/src/VirtualList.html b/src/VirtualList.html
index 0d8c3c3..bf89bd4 100644
--- a/src/VirtualList.html
+++ b/src/VirtualList.html
@@ -163,6 +163,19 @@
 					start: newStart,
 					end: newEnd
 				});
+
+				if (newStart < start) {
+					let d = 0;
+
+					for (let i = newStart; i < start; i += 1) {
+						const expectedHeight = this.heightMap[i];
+						const actualHeight = this.rows[i - newStart].offsetHeight;
+
+						d += actualHeight - expectedHeight;
+					}
+
+					this.refs.viewport.scrollTo(0, this.refs.viewport.scrollTop + d);
+				}
 			}
 		}
 	};
diff --git a/test/src/index.js b/test/src/index.js
index a2a2330..cceac71 100644
--- a/test/src/index.js
+++ b/test/src/index.js
@@ -260,5 +260,43 @@ test('updates when items change from an empty list', t => {
 	list.destroy();
 });
 
+test('handles unexpected height changes when scrolling up', async t => {
+	const Row = svelte.create(`
+		<div style="height: {rowHeight}px;">test</div>
+	`);
+
+	const list = new VirtualList({
+		target,
+		data: {
+			items: Array(20).fill().map(() => ({})),
+			component: Row,
+			height: '500px',
+			rowHeight: 50
+		}
+	});
+
+	const { viewport } = list.refs;
+
+	await scroll(viewport, 500);
+	assert.equal(viewport.scrollTop, 500);
+
+	list.set({ rowHeight: 100 });
+	await scroll(viewport, 475);
+	assert.equal(viewport.scrollTop, 525);
+
+	list.destroy();
+});
+
+function scroll(element, y) {
+	return new Promise(fulfil => {
+		element.addEventListener('scroll', function handler() {
+			element.removeEventListener('scroll', handler);
+			fulfil();
+		});
+
+		element.scrollTo(0, y);
+	});
+}
+
 // this allows us to close puppeteer once tests have completed
 window.done = done;
\ No newline at end of file