|
4 | 4 | import logging
|
5 | 5 | import argparse
|
6 | 6 | import os
|
| 7 | +import re |
7 | 8 | import sys
|
8 | 9 | from pathlib import Path
|
9 | 10 | from typing import Any
|
@@ -244,26 +245,58 @@ def dump_markdown_metadata(reader: GGUFReader, args: argparse.Namespace) -> None
|
244 | 245 | else:
|
245 | 246 | pretty_type = str(field.types[-1].name)
|
246 | 247 |
|
| 248 | + def escape_markdown_inline_code(value_string): |
| 249 | + # Find the longest contiguous sequence of backticks in the string then |
| 250 | + # wrap string with appropriate number of backticks required to escape it |
| 251 | + max_backticks = max((len(match.group(0)) for match in re.finditer(r'`+', value_string)), default=0) |
| 252 | + inline_code_marker = '`' * (max_backticks + 1) |
| 253 | + |
| 254 | + # If the string starts or ends with a backtick, add a space at the beginning and end |
| 255 | + if value_string.startswith('`') or value_string.endswith('`'): |
| 256 | + value_string = f" {value_string} " |
| 257 | + |
| 258 | + return f"{inline_code_marker}{value_string}{inline_code_marker}" |
| 259 | + |
247 | 260 | total_elements = len(field.data)
|
248 | 261 | value = ""
|
249 | 262 | if len(field.types) == 1:
|
250 | 263 | curr_type = field.types[0]
|
251 | 264 | if curr_type == GGUFValueType.STRING:
|
252 |
| - value = repr(str(bytes(field.parts[-1]), encoding='utf-8')[:60]) |
| 265 | + truncate_length = 60 |
| 266 | + value_string = str(bytes(field.parts[-1]), encoding='utf-8') |
| 267 | + if len(value_string) > truncate_length: |
| 268 | + head = escape_markdown_inline_code(value_string[:truncate_length // 2]) |
| 269 | + tail = escape_markdown_inline_code(value_string[-truncate_length // 2:]) |
| 270 | + value = "{head}...{tail}".format(head=head, tail=tail) |
| 271 | + else: |
| 272 | + value = escape_markdown_inline_code(value_string) |
253 | 273 | elif curr_type in reader.gguf_scalar_to_np:
|
254 | 274 | value = str(field.parts[-1][0])
|
255 | 275 | else:
|
256 | 276 | if field.types[0] == GGUFValueType.ARRAY:
|
257 | 277 | curr_type = field.types[1]
|
| 278 | + array_elements = [] |
| 279 | + |
258 | 280 | if curr_type == GGUFValueType.STRING:
|
259 | 281 | render_element = min(5, total_elements)
|
260 | 282 | for element_pos in range(render_element):
|
261 |
| - value += repr(str(bytes(field.parts[-1 - element_pos]), encoding='utf-8')[:5]) + (", " if total_elements > 1 else "") |
| 283 | + truncate_length = 30 |
| 284 | + value_string = str(bytes(field.parts[-1 - (total_elements - element_pos - 1) * 2]), encoding='utf-8') |
| 285 | + if len(value_string) > truncate_length: |
| 286 | + head = escape_markdown_inline_code(value_string[:truncate_length // 2]) |
| 287 | + tail = escape_markdown_inline_code(value_string[-truncate_length // 2:]) |
| 288 | + value = "{head}...{tail}".format(head=head, tail=tail) |
| 289 | + else: |
| 290 | + value = escape_markdown_inline_code(value_string) |
| 291 | + array_elements.append(value) |
| 292 | + |
262 | 293 | elif curr_type in reader.gguf_scalar_to_np:
|
263 | 294 | render_element = min(7, total_elements)
|
264 | 295 | for element_pos in range(render_element):
|
265 |
| - value += str(field.parts[-1 - element_pos][0]) + (", " if total_elements > 1 else "") |
266 |
| - value = f'[ {value}{" ..." if total_elements > 1 else ""} ]' |
| 296 | + array_elements.append(str(field.parts[-1 - (total_elements - element_pos - 1)][0])) |
| 297 | + |
| 298 | + value = f'[ {", ".join(array_elements).strip()}{", ..." if total_elements > len(array_elements) else ""} ]' |
| 299 | + |
267 | 300 | kv_dump_table.append({"n":n, "pretty_type":pretty_type, "total_elements":total_elements, "field_name":field.name, "value":value})
|
268 | 301 |
|
269 | 302 | kv_dump_table_header_map = [
|
@@ -382,7 +415,7 @@ def dump_markdown_metadata(reader: GGUFReader, args: argparse.Namespace) -> None
|
382 | 415 | markdown_content += f"- Percentage of total elements: {group_percentage:.2f}%\n"
|
383 | 416 | markdown_content += "\n\n"
|
384 | 417 |
|
385 |
| - print(markdown_content) # noqa: NP100 |
| 418 | + print(markdown_content) # noqa: NP100 |
386 | 419 |
|
387 | 420 |
|
388 | 421 | def main() -> None:
|
|
0 commit comments