Merge libdispatch-749

Signed-off-by: Daniel A. Steffen <dsteffen@apple.com>

Author: das

os/firehose_buffer_private.h

@@ -99,27 +99,29 @@ static inline firehose_tracepoint_t
 _firehose_tracepoint_reader_next(const uint8_t **ptr, const uint8_t *end)
 {
 	const uint16_t ft_size = offsetof(struct firehose_tracepoint_s, ft_data);
-	firehose_tracepoint_t ft;
+	struct ft_unaligned_s {
+		struct firehose_tracepoint_s ft;
+	} __attribute__((packed, aligned(1))) *uft;
 
 	do {
-		ft = (firehose_tracepoint_t)*ptr;
-		if (ft->ft_data >= end) {
+		uft = (struct ft_unaligned_s *)*ptr;
+		if (uft->ft.ft_data >= end) {
 			// reached the end
 			return NULL;
 		}
-		if (!ft->ft_length) {
+		if (!uft->ft.ft_length) {
 			// tracepoint write didn't even start
 			return NULL;
 		}
-		if (ft->ft_length > end - ft->ft_data) {
+		if (uft->ft.ft_length > end - uft->ft.ft_data) {
 			// invalid length
 			return NULL;
 		}
-		*ptr += roundup(ft_size + ft->ft_length, 8);
+		*ptr += roundup(ft_size + uft->ft.ft_length, 8);
 		// test whether write of the tracepoint was finished
-	} while (os_unlikely(ft->ft_id.ftid_value == 0));
+	} while (os_unlikely(uft->ft.ft_id.ftid_value == 0));
 
-	return ft;
+	return (firehose_tracepoint_t)uft;
 }
 
 #define firehose_tracepoint_foreach(ft, fbc) \

os/voucher_activity_private.h

@@ -42,12 +42,6 @@
 #define OS_VOUCHER_EXPORT OS_EXPORT
 #endif
 
-#define __VOUCHER_ACTIVITY_IGNORE_DEPRECATION_PUSH \
-	_Pragma("clang diagnostic push") \
-	_Pragma("clang diagnostic ignored \"-Wdeprecated-declarations\"")
-#define __VOUCHER_ACTIVITY_IGNORE_DEPRECATION_POP \
-	_Pragma("clang diagnostic pop")
-
 __BEGIN_DECLS
 
 /*!
@@ -273,11 +267,12 @@ voucher_activity_trace_with_private_strings(firehose_stream_t stream,
 		const void *privdata, size_t privlen);
 
 typedef const struct voucher_activity_hooks_s {
-#define VOUCHER_ACTIVITY_HOOKS_VERSION     3
+#define VOUCHER_ACTIVITY_HOOKS_VERSION     4
 	long vah_version;
 	mach_port_t (*vah_get_logd_port)(void);
 	dispatch_mach_handler_function_t vah_debug_channel_handler;
 	kern_return_t (*vah_get_reconnect_info)(mach_vm_address_t *, mach_vm_size_t *);
+	void (*vah_metadata_init)(void *metadata_buffer, size_t size);
 } *voucher_activity_hooks_t;
 
 /*!

src/allocator.c

@@ -274,22 +274,16 @@ mark_bitmap_as_full_if_still_full(volatile bitmap_t *supermap,
 	dispatch_assert(bitmap_index < BITMAPS_PER_SUPERMAP);
 #endif
 	const bitmap_t mask = BITMAP_C(1) << bitmap_index;
-	bitmap_t s, s_new, s_masked;
+	bitmap_t s, s_new;
 
-	if (!bitmap_is_full(*bitmap)) {
-		return;
-	}
-	s_new = *supermap;
-	for (;;) {
-		// No barriers because supermaps are only advisory, they
-		// don't protect access to other memory.
-		s = s_new;
-		s_masked = s | mask;
-		if (os_atomic_cmpxchgvw(supermap, s, s_masked, &s_new, relaxed) ||
-				!bitmap_is_full(*bitmap)) {
-			return;
+	// No barriers because supermaps are only advisory, they
+	// don't protect access to other memory.
+	os_atomic_rmw_loop(supermap, s, s_new, relaxed, {
+		if (!bitmap_is_full(*bitmap)) {
+			os_atomic_rmw_loop_give_up(return);
 		}
-	}
+		s_new = s | mask;
+	});
 }
 
 #pragma mark -

src/event/event_config.h

@@ -88,19 +88,19 @@
 #	endif
 
 #	if !defined(NOTE_MEMORYSTATUS_PROC_LIMIT_WARN) || \
-		!DISPATCH_HOST_SUPPORTS_OSX(101200)
+		!DISPATCH_MIN_REQUIRED_OSX_AT_LEAST(101200)
 #	undef NOTE_MEMORYSTATUS_PROC_LIMIT_WARN
 #	define NOTE_MEMORYSTATUS_PROC_LIMIT_WARN 0
 #	endif // NOTE_MEMORYSTATUS_PROC_LIMIT_WARN
 
 #	if !defined(NOTE_MEMORYSTATUS_PROC_LIMIT_CRITICAL) || \
-		!DISPATCH_HOST_SUPPORTS_OSX(101200)
+		!DISPATCH_MIN_REQUIRED_OSX_AT_LEAST(101200)
 #	undef NOTE_MEMORYSTATUS_PROC_LIMIT_CRITICAL
 #	define NOTE_MEMORYSTATUS_PROC_LIMIT_CRITICAL 0
 #	endif // NOTE_MEMORYSTATUS_PROC_LIMIT_CRITICAL
 
 #	ifndef DISPATCH_KEVENT_TREAT_ENOENT_AS_EINPROGRESS
-#	if TARGET_OS_MAC && !DISPATCH_HOST_SUPPORTS_OSX(101200)
+#	if TARGET_OS_MAC && !DISPATCH_MIN_REQUIRED_OSX_AT_LEAST(101200)
 	// deferred delete can return bogus ENOENTs on older kernels
 #	define DISPATCH_KEVENT_TREAT_ENOENT_AS_EINPROGRESS 1
 #	else

src/firehose/firehose.defs

@@ -35,7 +35,8 @@ register(
 			comm_recvp		: mach_port_move_receive_t;
 			comm_sendp		: mach_port_make_send_t;
 			extra_info_port : mach_port_move_send_t;
-			extra_info_size : mach_vm_size_t
+			extra_info_size : mach_vm_size_t;
+			ServerAuditToken atoken : audit_token_t
 );
 
 routine

src/firehose/firehose_server.c

@@ -74,7 +74,7 @@ firehose_client_notify(firehose_client_t fc, mach_port_t reply_port)
 	firehose_atomic_max2o(fc, fc_io_sent_flushed_pos,
 			push_reply.fpr_io_flushed_pos, relaxed);
 
-	if (fc->fc_is_kernel) {
+	if (!fc->fc_pid) {
 		if (ioctl(server_config.fs_kernel_fd, LOGFLUSHED, &push_reply) < 0) {
 			dispatch_assume_zero(errno);
 		}
@@ -209,18 +209,25 @@ firehose_client_drain(firehose_client_t fc, mach_port_t port, uint32_t flags)
 			ref = (flushed + count) & FIREHOSE_RING_POS_IDX_MASK;
 			ref = os_atomic_load(&fbh_ring[ref], relaxed);
 			ref &= FIREHOSE_RING_POS_IDX_MASK;
-		} while (fc->fc_is_kernel && !ref);
+		} while (!fc->fc_pid && !ref);
 		count++;
 		if (!ref) {
 			_dispatch_debug("Ignoring invalid page reference in ring: %d", ref);
 			continue;
 		}
 
 		fbc = firehose_buffer_ref_to_chunk(fb, ref);
+		if (fbc->fc_pos.fcp_stream == firehose_stream_metadata) {
+			// serialize with firehose_client_metadata_stream_peek
+			os_unfair_lock_lock(&fc->fc_lock);
+		}
 		server_config.fs_handler(fc, evt, fbc);
 		if (slowpath(snapshot)) {
 			snapshot->handler(fc, evt, fbc);
 		}
+		if (fbc->fc_pos.fcp_stream == firehose_stream_metadata) {
+			os_unfair_lock_unlock(&fc->fc_lock);
+		}
 		// clients not using notifications (single threaded) always drain fully
 		// because they use all their limit, always
 	} while (!fc->fc_use_notifs || count < DRAIN_BATCH_SIZE || snapshot);
@@ -238,7 +245,7 @@ firehose_client_drain(firehose_client_t fc, mach_port_t port, uint32_t flags)
 			client_flushed = os_atomic_load2o(&fb->fb_header,
 				fbh_ring_tail.frp_mem_flushed, relaxed);
 		}
-		if (fc->fc_is_kernel) {
+		if (!fc->fc_pid) {
 			// will fire firehose_client_notify() because port is MACH_PORT_DEAD
 			port = fc->fc_sendp;
 		} else if (!port && client_flushed == sent_flushed && fc->fc_use_notifs) {
@@ -253,7 +260,7 @@ firehose_client_drain(firehose_client_t fc, mach_port_t port, uint32_t flags)
 	if (port) {
 		firehose_client_notify(fc, port);
 	}
-	if (fc->fc_is_kernel) {
+	if (!fc->fc_pid) {
 		if (!(flags & FIREHOSE_DRAIN_POLL)) {
 			// see firehose_client_kernel_source_handle_event
 			dispatch_resume(fc->fc_kernel_source);
@@ -283,7 +290,7 @@ firehose_client_drain(firehose_client_t fc, mach_port_t port, uint32_t flags)
 	// from now on all IO/mem drains depending on `for_io` will be no-op
 	// (needs_<for_io>_snapshot: false, memory_corrupted: true). we can safely
 	// silence the corresponding source of drain wake-ups.
-	if (!fc->fc_is_kernel) {
+	if (fc->fc_pid) {
 		dispatch_source_cancel(for_io ? fc->fc_io_source : fc->fc_mem_source);
 	}
 }
@@ -502,7 +509,7 @@ firehose_client_resume(firehose_client_t fc,
 	dispatch_assert_queue(server_config.fs_io_drain_queue);
 	TAILQ_INSERT_TAIL(&server_config.fs_clients, fc, fc_entry);
 	server_config.fs_handler(fc, FIREHOSE_EVENT_CLIENT_CONNECTED, (void *)fcci);
-	if (fc->fc_is_kernel) {
+	if (!fc->fc_pid) {
 		dispatch_activate(fc->fc_kernel_source);
 	} else {
 		dispatch_mach_connect(fc->fc_mach_channel,
@@ -553,14 +560,15 @@ _firehose_client_create(firehose_buffer_t fb)
 }
 
 static firehose_client_t
-firehose_client_create(firehose_buffer_t fb,
+firehose_client_create(firehose_buffer_t fb, pid_t pid,
 		mach_port_t comm_recvp, mach_port_t comm_sendp)
 {
 	uint64_t unique_pid = fb->fb_header.fbh_uniquepid;
 	firehose_client_t fc = _firehose_client_create(fb);
 	dispatch_mach_t dm;
 	dispatch_source_t ds;
 
+	fc->fc_pid = pid;
 	ds = dispatch_source_create(DISPATCH_SOURCE_TYPE_DATA_OR, 0, 0,
 			server_config.fs_mem_drain_queue);
 	_os_object_retain_internal_inline(&fc->fc_as_os_object);
@@ -622,7 +630,6 @@ firehose_kernel_client_create(void)
 	}
 
 	fc = _firehose_client_create((firehose_buffer_t)(uintptr_t)fb_map.fbmi_addr);
-	fc->fc_is_kernel = true;
 	ds = dispatch_source_create(DISPATCH_SOURCE_TYPE_READ, (uintptr_t)fd, 0,
 			fs->fs_ipc_queue);
 	dispatch_set_context(ds, fc);
@@ -663,12 +670,9 @@ uint64_t
 firehose_client_get_unique_pid(firehose_client_t fc, pid_t *pid_out)
 {
 	firehose_buffer_header_t fbh = &fc->fc_buffer->fb_header;
-	if (fc->fc_is_kernel) {
-		if (pid_out) *pid_out = 0;
-		return 0;
-	}
-	if (pid_out) *pid_out = fbh->fbh_pid ?: ~(pid_t)0;
-	return fbh->fbh_uniquepid ?: ~0ull;
+	if (pid_out) *pid_out = fc->fc_pid;
+	if (!fc->fc_pid) return 0;
+	return fbh->fbh_uniquepid ? fbh->fbh_uniquepid : ~0ull;
 }
 
 void *
@@ -800,47 +804,42 @@ firehose_server_copy_queue(firehose_server_queue_t which)
 
 void
 firehose_client_metadata_stream_peek(firehose_client_t fc,
-		firehose_event_t context, bool (^peek_should_start)(void),
+		OS_UNUSED firehose_event_t context, bool (^peek_should_start)(void),
 		bool (^peek)(firehose_chunk_t fbc))
 {
-	if (context != FIREHOSE_EVENT_MEM_BUFFER_RECEIVED) {
-		return dispatch_sync(server_config.fs_mem_drain_queue, ^{
-			firehose_client_metadata_stream_peek(fc,
-					FIREHOSE_EVENT_MEM_BUFFER_RECEIVED, peek_should_start, peek);
-		});
-	}
+	os_unfair_lock_lock(&fc->fc_lock);
 
-	if (peek_should_start && !peek_should_start()) {
-		return;
-	}
-
-	firehose_buffer_t fb = fc->fc_buffer;
-	firehose_buffer_header_t fbh = &fb->fb_header;
-	uint64_t bitmap = fbh->fbh_bank.fbb_metadata_bitmap;
+	if (peek_should_start && peek_should_start()) {
+		firehose_buffer_t fb = fc->fc_buffer;
+		firehose_buffer_header_t fbh = &fb->fb_header;
+		uint64_t bitmap = fbh->fbh_bank.fbb_metadata_bitmap;
 
-	while (bitmap) {
-		uint16_t ref = firehose_bitmap_first_set(bitmap);
-		firehose_chunk_t fbc = firehose_buffer_ref_to_chunk(fb, ref);
-		uint16_t fbc_length = fbc->fc_pos.fcp_next_entry_offs;
+		while (bitmap) {
+			uint16_t ref = firehose_bitmap_first_set(bitmap);
+			firehose_chunk_t fbc = firehose_buffer_ref_to_chunk(fb, ref);
+			uint16_t fbc_length = fbc->fc_pos.fcp_next_entry_offs;
 
-		bitmap &= ~(1ULL << ref);
-		if (fbc->fc_start + fbc_length <= fbc->fc_data) {
-			// this page has its "recycle-requeue" done, but hasn't gone
-			// through "recycle-reuse", or it has no data, ditch it
-			continue;
-		}
-		if (!((firehose_tracepoint_t)fbc->fc_data)->ft_length) {
-			// this thing has data, but the first tracepoint is unreadable
-			// so also just ditch it
-			continue;
-		}
-		if (fbc->fc_pos.fcp_stream != firehose_stream_metadata) {
-			continue;
-		}
-		if (!peek(fbc)) {
-			break;
+			bitmap &= ~(1ULL << ref);
+			if (fbc->fc_start + fbc_length <= fbc->fc_data) {
+				// this page has its "recycle-requeue" done, but hasn't gone
+				// through "recycle-reuse", or it has no data, ditch it
+				continue;
+			}
+			if (!((firehose_tracepoint_t)fbc->fc_data)->ft_length) {
+				// this thing has data, but the first tracepoint is unreadable
+				// so also just ditch it
+				continue;
+			}
+			if (fbc->fc_pos.fcp_stream != firehose_stream_metadata) {
+				continue;
+			}
+			if (!peek(fbc)) {
+				break;
+			}
 		}
 	}
+
+	os_unfair_lock_unlock(&fc->fc_lock);
 }
 
 OS_NOINLINE OS_COLD
@@ -925,7 +924,7 @@ firehose_snapshot_start(void *ctxt)
 	// 1. mark all the clients participating in the current snapshot
 	//    and enter the group for each bit set
 	TAILQ_FOREACH(fci, &server_config.fs_clients, fc_entry) {
-		if (fci->fc_is_kernel) {
+		if (!fci->fc_pid) {
 #if TARGET_OS_SIMULATOR
 			continue;
 #endif
@@ -965,7 +964,7 @@ firehose_snapshot_start(void *ctxt)
 			//    were removed from the list have already left the group
 			//    (see firehose_client_finalize())
 			TAILQ_FOREACH(fcj, &server_config.fs_clients, fc_entry) {
-				if (fcj->fc_is_kernel) {
+				if (!fcj->fc_pid) {
 #if !TARGET_OS_SIMULATOR
 					firehose_client_kernel_source_handle_event(fcj);
 #endif
@@ -1028,7 +1027,8 @@ kern_return_t
 firehose_server_register(mach_port_t server_port OS_UNUSED,
 		mach_port_t mem_port, mach_vm_size_t mem_size,
 		mach_port_t comm_recvp, mach_port_t comm_sendp,
-		mach_port_t extra_info_port, mach_vm_size_t extra_info_size)
+		mach_port_t extra_info_port, mach_vm_size_t extra_info_size,
+		audit_token_t atoken)
 {
 	mach_vm_address_t base_addr = 0;
 	firehose_client_t fc = NULL;
@@ -1077,7 +1077,9 @@ firehose_server_register(mach_port_t server_port OS_UNUSED,
 		fcci.fcci_size = (size_t)extra_info_size;
 	}
 
-	fc = firehose_client_create((firehose_buffer_t)base_addr,
+	pid_t pid = (pid_t)atoken.val[5]; // same as audit_token_to_pid()
+	if (pid == 0) pid = ~0;
+	fc = firehose_client_create((firehose_buffer_t)base_addr, pid,
 			comm_recvp, comm_sendp);
 	dispatch_async(server_config.fs_io_drain_queue, ^{
 		firehose_client_resume(fc, &fcci);

src/firehose/firehose_server_internal.h

@@ -53,11 +53,12 @@ struct firehose_client_s {
 	dispatch_source_t	fc_mem_source;
 	mach_port_t			fc_recvp;
 	mach_port_t			fc_sendp;
+	os_unfair_lock      fc_lock;
+	pid_t               fc_pid;
 	bool				fc_use_notifs;
 	bool				fc_memory_corrupted;
 	bool				fc_needs_io_snapshot;
 	bool				fc_needs_mem_snapshot;
-	bool				fc_is_kernel;
 };
 
 void