Skip to content

main: log file #2748

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 33 commits into from
Aug 30, 2023
Merged

main: log file #2748

Show file tree
Hide file tree
Changes from 2 commits
Commits
Show all changes
33 commits
Select commit Hold shift + click to select a range
62de8a6
initial, base LOG macro
staviq Aug 23, 2023
727af3e
add *.log to .gitignore
staviq Aug 23, 2023
d5156d3
added basic log file handler
staviq Aug 23, 2023
356a166
reverted log auto endline to better mimic printf
staviq Aug 23, 2023
71d05b9
remove atomics and add dynamic log target
staviq Aug 23, 2023
47b9f2d
log_enable/disable, LOG_TEE, basic usage doc
staviq Aug 24, 2023
f5080da
update .gitignore
staviq Aug 24, 2023
8054c32
Merge branch 'master' into betterlogs
staviq Aug 24, 2023
45f1a43
mv include to common, params, help msg
staviq Aug 24, 2023
360b36c
log tostring helpers, token vectors pretty prints
staviq Aug 25, 2023
181e8a9
main: replaced fprintf/LOG_TEE, some trace logging
staviq Aug 25, 2023
54e81ba
Merge branch 'ggerganov:master' into betterlogs
staviq Aug 25, 2023
4fdcede
LOG_DISABLE_LOGS compile flag, wrapped f in macros
staviq Aug 25, 2023
c8a1118
fix LOG_TEELN and configchecker
staviq Aug 25, 2023
9bace22
stub LOG_DUMP_CMDLINE for WIN32 for now
staviq Aug 25, 2023
0b36725
fix msvc
staviq Aug 26, 2023
e99f039
cleanup main.cpp:273
staviq Aug 26, 2023
5031c50
Merge branch 'master' into betterlogs
staviq Aug 26, 2023
5cea869
fix stray whitespace after master sync
staviq Aug 26, 2023
2c1930d
Merge branch 'master' into HEAD
ggerganov Aug 29, 2023
b97958a
log : fix compile warnings
ggerganov Aug 29, 2023
6b4c65b
log : do not append to existing log + disable file line func by default
ggerganov Aug 29, 2023
891ac40
Merge branch 'master' into HEAD
ggerganov Aug 29, 2023
5c978f4
log : try to fix Windows build
ggerganov Aug 29, 2023
c72d344
main : wip logs
ggerganov Aug 29, 2023
ecdf113
main : add trace log
ggerganov Aug 29, 2023
f7ac431
Merge branch 'ggerganov:master' into betterlogs
staviq Aug 29, 2023
3a10f5a
review: macro f lowercase, str append to sstream
staviq Aug 29, 2023
ba5590f
review: simplify ifs and str comparisons
staviq Aug 29, 2023
3edee3f
fix MSVC, formatting, FMT/VAL placeholders
staviq Aug 29, 2023
6fa208e
review: if/else cleanup
staviq Aug 29, 2023
f60f7d3
review: if/else cleanup (2)
staviq Aug 29, 2023
89dc100
replace _ prefix with _impl suffix
staviq Aug 29, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
144 changes: 126 additions & 18 deletions common/log.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,8 @@
#include <sstream>
#include <iostream>
#include <thread>
#include <vector>
#include <algorithm>

// --------------------------------
//
Expand All @@ -15,6 +17,9 @@
// The LOG() and LOG_TEE() macros are ready to go by default
// they do not require any initialization.
//
// LOGLN() and LOG_TEELN() are variants which automatically
// include \n character at the end of the log string.
//
// LOG() behaves exactly like printf, by default writing to a logfile.
// LOG_TEE() additionally, prints to the screen too ( mimics Unix tee command ).
//
Expand Down Expand Up @@ -147,6 +152,14 @@ inline std::string _log_filename_generator(std::string log_file_basename, std::s
#define LOG_TIMESTAMP_VAL
#endif

#ifdef LOG_TEE_TIMESTAMPS
#define LOG_TEE_TIMESTAMP_FMT "[%lu]"
#define LOG_TEE_TIMESTAMP_VAL , (std::chrono::duration_cast<std::chrono::duration<std::uint64_t>>(std::chrono::system_clock::now().time_since_epoch())).count()
#else
#define LOG_TEE_TIMESTAMP_FMT
#define LOG_TEE_TIMESTAMP_VAL
#endif

// Allows disabling file/line/function prefix
// in order to disable, define LOG_NO_FILE_LINE_FUNCTION
// like so:
Expand All @@ -162,6 +175,14 @@ inline std::string _log_filename_generator(std::string log_file_basename, std::s
#define LOG_FLF_VAL
#endif

#ifdef LOG_TEE_FILE_LINE_FUNCTION
#define LOG_TEE_FLF_FMT "[%24s:%5d][%24s] "
#define LOG_TEE_FLF_VAL , __FILE__, __LINE__, __FUNCTION__
#else
#define LOG_TEE_FLF_FMT
#define LOG_TEE_FLF_VAL
#endif

// Utility for synchronizing log configuration state
// since std::optional was introduced only in c++17
enum LogTriState
Expand All @@ -179,27 +200,27 @@ enum LogTriState
/*fprintf(stderr, "DBG:" str, ##__VA_ARGS__);*/ \
if (LOG_TARGET != nullptr) \
{ \
fprintf(LOG_TARGET, LOG_TIMESTAMP_FMT LOG_FLF_FMT str "%c" LOG_TIMESTAMP_VAL LOG_FLF_VAL, ##__VA_ARGS__); \
fprintf(LOG_TARGET, LOG_TIMESTAMP_FMT LOG_FLF_FMT str "%s" LOG_TIMESTAMP_VAL LOG_FLF_VAL, ##__VA_ARGS__); \
fflush(LOG_TARGET); /*fprintf(stderr, "DBGEND\n");*/ \
} \
}

// INTERNAL, DO NOT USE
// USE LOG_TEE() INSTEAD
//
#define _LOG_TEE(str, ...) \
{ \
/*fprintf(stderr, "DBG:" str, ##__VA_ARGS__);*/ \
if (LOG_TARGET != nullptr) \
{ \
fprintf(LOG_TARGET, LOG_TIMESTAMP_FMT LOG_FLF_FMT str "%c" LOG_TIMESTAMP_VAL LOG_FLF_VAL, ##__VA_ARGS__); \
fflush(LOG_TARGET); /*fprintf(stderr, "DBGEND\n");*/ \
} \
if (LOG_TARGET != nullptr && LOG_TARGET != stdout && LOG_TARGET != stderr && LOG_TEE_TARGET != nullptr) \
{ \
fprintf(LOG_TEE_TARGET, LOG_TIMESTAMP_FMT LOG_FLF_FMT str "%c" LOG_TIMESTAMP_VAL LOG_FLF_VAL, ##__VA_ARGS__); \
fflush(LOG_TEE_TARGET); /*fprintf(stderr, "DBGEND\n");*/ \
} \
#define _LOG_TEE(str, ...) \
{ \
/*fprintf(stderr, "DBG:" str, ##__VA_ARGS__);*/ \
if (LOG_TARGET != nullptr) \
{ \
fprintf(LOG_TARGET, LOG_TIMESTAMP_FMT LOG_FLF_FMT str "%s" LOG_TIMESTAMP_VAL LOG_FLF_VAL, ##__VA_ARGS__); \
fflush(LOG_TARGET); /*fprintf(stderr, "DBGEND\n");*/ \
} \
if (LOG_TARGET != nullptr && LOG_TARGET != stdout && LOG_TARGET != stderr && LOG_TEE_TARGET != nullptr) \
{ \
fprintf(LOG_TEE_TARGET, LOG_TEE_TIMESTAMP_FMT LOG_TEE_FLF_FMT str "%s" LOG_TEE_TIMESTAMP_VAL LOG_TEE_FLF_VAL, ##__VA_ARGS__); \
fflush(LOG_TEE_TARGET); /*fprintf(stderr, "DBGEND\n");*/ \
} \
}

// The '\0' as a last argument, is a trick to bypass the silly
Expand All @@ -209,7 +230,7 @@ enum LogTriState
// Main LOG macro.
// behaves like printf, and supports arguments the exact same way.
//
#define LOG(...) _LOG(__VA_ARGS__, '\0')
#define LOG(...) _LOG(__VA_ARGS__, "")

// Main TEE macro.
// does the same as LOG
Expand All @@ -219,7 +240,11 @@ enum LogTriState
// Secondary target can be changed just like LOG_TARGET
// by defining LOG_TEE_TARGET
//
#define LOG_TEE(...) _LOG_TEE(__VA_ARGS__, '\0')
#define LOG_TEE(...) _LOG_TEE(__VA_ARGS__, "")

// LOG macro variants with auto endline.
#define LOGLN(...) _LOG(__VA_ARGS__, "\n")
#define LOG_TEELN(...) _LOG(__VA_ARGS__, "\n")

// INTERNAL, DO NOT USE
inline FILE *_log_handler1(bool change = false, LogTriState disable = LogTriStateSame, std::string filename = LOG_DEFAULT_FILE_NAME, FILE *target = nullptr)
Expand Down Expand Up @@ -396,7 +421,7 @@ inline void log_test()
log_set_target(LOG_FILENAME_GENERATOR("llama_autonamed", "log"));
LOG("14 Hello World in log with generated filename!\n")

//exit(0);
// exit(0);
}

inline bool log_param_single_parse(std::string param)
Expand Down Expand Up @@ -430,7 +455,7 @@ inline bool log_param_pair_parse(bool check_but_dont_parse, std::string param, s

if (std::string("--log-file").compare(param) == 0)
{
if( check_but_dont_parse )
if (check_but_dont_parse)
{
return true;
}
Expand All @@ -456,3 +481,86 @@ inline void log_print_usage()
fprintf(stdout, " --log-file Specify a log filename (without extension)\n");
fprintf(stdout, " Log file will be tagged with unique ID and written as \"<name>.<ID>.log\"\n"); /* */
}

inline void log_dump_cmdline(int argc, char **argv)
{
std::string buf;
for (int i = 0; i < argc; ++i)
{
if (std::string(argv[i]).find(' ') != std::string::npos)
{
buf.append(" \"").append(argv[i]).append("\"");
}
else
{
buf.append(" ").append(argv[i]);
}
}
LOGLN("Cmd:%s", buf.c_str())
}

#define LOG_TOSTR(var) _log_var_to_string(var).c_str()

inline std::string _log_var_to_string(bool var)
{
return var ? "true" : "false";
}

inline std::string _log_var_to_string(std::string var)
{
return var;
}

inline std::string _log_var_to_string(std::vector<int> var)
{
std::string buf;
buf.append("[ ");
bool first = true;
for (auto e : var)
{
if (first)
{
first = false;
}
else
{
buf.append(", ");
}
buf.append(std::to_string(e));
}
buf.append(" ]");

return buf;
}

#define LOG_TOKENS_TOSTR_PRETTY(tokens, ctx) \
[&tokens, &ctx]() \
{ \
std::string buf("[ "); \
bool first = true; \
for (const auto &token : tokens) \
{ \
if (!first) \
buf.append(", "); \
else \
first = false; \
\
auto detokenized = llama_token_to_str(ctx, token); \
\
detokenized.erase( \
std::remove_if( \
detokenized.begin(), \
detokenized.end(), \
[](const char c) { return !std::isprint(c); }), \
detokenized.end()); \
\
buf \
.append("'") \
.append(detokenized) \
.append("'") \
.append(":") \
.append(std::to_string(token)); \
} \
return buf.append(" ]"); \
}() \
.c_str()
Loading