Skip to content

optimise template load time #241

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 1 commit into from
Jun 16, 2023
Merged

optimise template load time #241

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
13 changes: 12 additions & 1 deletion CMakeLists.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,9 @@ endif()
set(JINJA2CPP_SANITIZERS address+undefined memory)
set(JINJA2CPP_WITH_SANITIZERS none CACHE STRING "Build with sanitizer")
set_property(CACHE JINJA2CPP_WITH_SANITIZERS PROPERTY STRINGS ${JINJA2CPP_SANITIZERS})
set(JINJA2CPP_SUPPORTED_REGEX std boost)
set(JINJA2CPP_USE_REGEX boost CACHE STRING "Use regex parser in lexer, boost works faster on most platforms")
set_property(CACHE JINJA2CPP_USE_REGEX PROPERTY STRINGS ${JINJA2CPP_SUPPORTED_REGEX})
set(JINJA2CPP_WITH_JSON_BINDINGS boost nlohmann rapid all none)
set(JINJA2CPP_WITH_JSON_BINDINGS all CACHE STRING "Build with json support")
set_property(CACHE JINJA2CPP_WITH_JSON_BINDINGS PROPERTY STRINGS ${JINJA2CPP_WITH_JSON_BINDINGS})
Expand Down Expand Up @@ -210,7 +213,15 @@ elseif (CMAKE_CXX_COMPILER_ID MATCHES "MSVC")
target_compile_options(${LIB_TARGET_NAME} PRIVATE ${MSVC_CXX_FLAGS})
endif ()

target_compile_definitions(${LIB_TARGET_NAME} PUBLIC -DBOOST_SYSTEM_NO_DEPRECATED -DBOOST_ERROR_CODE_HEADER_ONLY)
if ("${JINJA2CPP_USE_REGEX}" STREQUAL "boost")
set(_regex_define "-DJINJA2CPP_USE_REGEX_BOOST")
endif()
target_compile_definitions(${LIB_TARGET_NAME}
PUBLIC
-DBOOST_SYSTEM_NO_DEPRECATED
-DBOOST_ERROR_CODE_HEADER_ONLY
${_regex_define}
)

if (JINJA2CPP_BUILD_SHARED)
target_compile_definitions(${LIB_TARGET_NAME} PRIVATE -DJINJA2CPP_BUILD_AS_SHARED PUBLIC -DJINJA2CPP_LINK_AS_SHARED)
Expand Down
43 changes: 30 additions & 13 deletions src/template_parser.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,11 +17,28 @@
#include <nonstd/expected.hpp>

#include <list>
#include <regex>
#include <sstream>
#include <string>
#include <vector>

#ifdef JINJA2CPP_USE_REGEX_BOOST
#include <boost/regex.hpp>
template <typename CharType>
using BasicRegex = boost::basic_regex<CharType>;
using Regex = boost::regex;
using WideRegex = boost::wregex;
template <typename CharIterator>
using RegexIterator = boost::regex_iterator<CharIterator>;
#else
#include <regex>
template <typename CharType>
using BasicRegex = std::basic_regex<CharType>;
using Regex = std::regex;
using WideRegex = std::wregex;
template <typename CharIterator>
using RegexIterator = std::regex_iterator<CharIterator>;
#endif

namespace jinja2
{
template<typename CharT>
Expand Down Expand Up @@ -58,9 +75,9 @@ MultiStringLiteral ParserTraitsBase<T>::s_regexp = UNIVERSAL_STR(
template<>
struct ParserTraits<char> : public ParserTraitsBase<>
{
static std::regex GetRoughTokenizer()
{ return std::regex(s_regexp.GetValueStr<char>()); }
static std::regex GetKeywords()
static Regex GetRoughTokenizer()
{ return Regex(s_regexp.GetValueStr<char>()); }
static Regex GetKeywords()
{
std::string pattern;
std::string prefix("(^");
Expand All @@ -76,7 +93,7 @@ struct ParserTraits<char> : public ParserTraitsBase<>

pattern += prefix + info.name.charValue + postfix;
}
return std::regex(pattern);
return Regex(pattern);
}
static std::string GetAsString(const std::string& str, CharRange range) { return str.substr(range.startOffset, range.size()); }
static InternalValue RangeToNum(const std::string& str, CharRange range, Token::Type hint)
Expand Down Expand Up @@ -109,9 +126,9 @@ struct ParserTraits<char> : public ParserTraitsBase<>
template<>
struct ParserTraits<wchar_t> : public ParserTraitsBase<>
{
static std::wregex GetRoughTokenizer()
{ return std::wregex(s_regexp.GetValueStr<wchar_t>()); }
static std::wregex GetKeywords()
static WideRegex GetRoughTokenizer()
{ return WideRegex(s_regexp.GetValueStr<wchar_t>()); }
static WideRegex GetKeywords()
{
std::wstring pattern;
std::wstring prefix(L"(^");
Expand All @@ -127,7 +144,7 @@ struct ParserTraits<wchar_t> : public ParserTraitsBase<>

pattern += prefix + info.name.wcharValue + postfix;
}
return std::wregex(pattern);
return WideRegex(pattern);
}
static std::string GetAsString(const std::wstring& str, CharRange range)
{
Expand Down Expand Up @@ -248,7 +265,7 @@ class TemplateParser : public LexerHelper
public:
using string_t = std::basic_string<CharT>;
using traits_t = ParserTraits<CharT>;
using sregex_iterator = std::regex_iterator<typename string_t::const_iterator>;
using sregex_iterator = RegexIterator<typename string_t::const_iterator>;
using ErrorInfo = ErrorInfoTpl<CharT>;
using ParseResult = nonstd::expected<RendererPtr, std::vector<ErrorInfo>>;

Expand Down Expand Up @@ -437,7 +454,7 @@ class TemplateParser : public LexerHelper
FinishCurrentLine(match.position() + 2);
return MakeParseError(ErrorCode::UnexpectedCommentEnd, MakeToken(Token::CommentEnd, { matchStart, matchStart + 2 }));
}

m_currentBlockInfo.range.startOffset = FinishCurrentBlock(matchStart, TextBlockType::RawText);
break;
case RM_ExprBegin:
Expand Down Expand Up @@ -925,8 +942,8 @@ class TemplateParser : public LexerHelper
std::string m_templateName;
const Settings& m_settings;
TemplateEnv* m_env = nullptr;
std::basic_regex<CharT> m_roughTokenizer;
std::basic_regex<CharT> m_keywords;
BasicRegex<CharT> m_roughTokenizer;
BasicRegex<CharT> m_keywords;
std::vector<LineInfo> m_lines;
std::vector<TextBlockInfo> m_textBlocks;
LineInfo m_currentLineInfo = {};
Expand Down
20 changes: 20 additions & 0 deletions test/perf_test.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -181,6 +181,26 @@ TEST(PerfTests, ForLoopIfText)
std::cout << result << std::endl;
}

TEST(PerfTests, LoadTemplate)
{
std::string source = "{{ title }}";
jinja2::Template tpl;

const int N = 100000;

for (int i = 0; i < N; i++)
{
tpl.Load(source);
}

ValuesMap data;
data["title"] = "My title";
data["t"] = "My List";
std::string result = tpl.RenderAsString(data).value();;

std::cout << result << std::endl;
}

TEST(PerfTests, DISABLED_TestMatsuhiko)
{
std::string source = R"(
Expand Down
2 changes: 1 addition & 1 deletion thirdparty/CMakeLists.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -68,7 +68,7 @@ endif()

if (NOT DEFINED JINJA2_PRIVATE_LIBS_INT)
set(JINJA2CPP_PRIVATE_LIBS ${JINJA2CPP_PRIVATE_LIBS} Boost::variant
Boost::filesystem Boost::algorithm Boost::lexical_cast Boost::json fmt RapidJson)
Boost::filesystem Boost::algorithm Boost::lexical_cast Boost::json Boost::regex fmt RapidJson)
else ()
set (JINJA2CPP_PRIVATE_LIBS ${JINJA2_PRIVATE_LIBS_INT})
endif ()
Expand Down
13 changes: 10 additions & 3 deletions thirdparty/external_boost_deps.cmake
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,30 +24,37 @@ find_package(boost_filesystem ${FIND_BOOST_PACKAGE_QUIET})
find_package(boost_json ${FIND_BOOST_PACKAGE_QUIET})
find_package(boost_optional ${FIND_BOOST_PACKAGE_QUIET})
find_package(boost_variant ${FIND_BOOST_PACKAGE_QUIET})
find_package(boost_regex ${FIND_BOOST_PACKAGE_QUIET})

if (boost_algorithm_FOUND AND
boost_filesystem_FOUND AND
boost_json_FOUND AND
boost_optional_FOUND AND
boost_variant_FOUND)
boost_variant_FOUND AND boost_regex_FOUND)
imported_target_alias(boost_algorithm ALIAS boost_algorithm::boost_algorithm)
imported_target_alias(boost_filesystem ALIAS boost_filesystem::boost_filesystem)
imported_target_alias(boost_json ALIAS boost_json::boost_json)
imported_target_alias(boost_optional ALIAS boost_optional::boost_optional)
imported_target_alias(boost_variant ALIAS boost_variant::boost_variant)
imported_target_alias(boost_regex ALIAS boost_regex::boost_regex)
else ()
find_package(Boost COMPONENTS system filesystem json ${FIND_BOOST_PACKAGE_QUIET} REQUIRED)
find_package(Boost COMPONENTS system filesystem json regex ${FIND_BOOST_PACKAGE_QUIET} REQUIRED)

if (Boost_FOUND)
imported_target_alias(boost_algorithm ALIAS Boost::boost)
imported_target_alias(boost_filesystem ALIAS Boost::filesystem)
imported_target_alias(boost_json ALIAS Boost::json)
imported_target_alias(boost_optional ALIAS Boost::boost)
imported_target_alias(boost_variant ALIAS Boost::boost)
imported_target_alias(boost_regex ALIAS Boost::regex)
endif ()
endif ()

install(TARGETS boost_algorithm boost_filesystem boost_json boost_optional boost_variant
set(_additional_boost_install_targets)
if ("${JINJA2CPP_USE_REGEX}" STREQUAL "boost")
set(_additional_boost_install_targets "boost_regex")
endif()
install(TARGETS boost_algorithm boost_filesystem boost_json boost_optional boost_variant ${_additional_boost_install_targets}
EXPORT InstallTargets
RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR}
LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR}
Expand Down