1 // Copyright 2015 The Shaderc Authors. All rights reserved. 2 // 3 // Licensed under the Apache License, Version 2.0 (the "License"); 4 // you may not use this file except in compliance with the License. 5 // You may obtain a copy of the License at 6 // 7 // http://www.apache.org/licenses/LICENSE-2.0 8 // 9 // Unless required by applicable law or agreed to in writing, software 10 // distributed under the License is distributed on an "AS IS" BASIS, 11 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 // See the License for the specific language governing permissions and 13 // limitations under the License. 14 15 #include "libshaderc_util/compiler.h" 16 17 #include <cstdint> 18 #include <tuple> 19 20 #include "libshaderc_util/format.h" 21 #include "libshaderc_util/io.h" 22 #include "libshaderc_util/message.h" 23 #include "libshaderc_util/resources.h" 24 #include "libshaderc_util/shader_stage.h" 25 #include "libshaderc_util/spirv_tools_wrapper.h" 26 #include "libshaderc_util/string_piece.h" 27 #include "libshaderc_util/version_profile.h" 28 29 #include "SPIRV/GlslangToSpv.h" 30 31 namespace { 32 using shaderc_util::string_piece; 33 34 // For use with glslang parsing calls. 35 const bool kNotForwardCompatible = false; 36 37 // Returns true if #line directive sets the line number for the next line in the 38 // given version and profile. 39 inline bool LineDirectiveIsForNextLine(int version, EProfile profile) { 40 return profile == EEsProfile || version >= 330; 41 } 42 43 // Returns a #line directive whose arguments are line and filename. 44 inline std::string GetLineDirective(int line, const string_piece& filename) { 45 return "#line " + std::to_string(line) + " \"" + filename.str() + "\"\n"; 46 } 47 48 // Given a canonicalized #line directive (starting exactly with "#line", using 49 // single spaces to separate different components, and having an optional 50 // newline at the end), returns the line number and string name/number. If no 51 // string name/number is provided, the second element in the returned pair is an 52 // empty string_piece. Behavior is undefined if the directive parameter is not a 53 // canonicalized #line directive. 54 std::pair<int, string_piece> DecodeLineDirective(string_piece directive) { 55 const string_piece kLineDirective = "#line "; 56 assert(directive.starts_with(kLineDirective)); 57 directive = directive.substr(kLineDirective.size()); 58 59 const int line = std::atoi(directive.data()); 60 const size_t space_loc = directive.find_first_of(' '); 61 if (space_loc == string_piece::npos) return std::make_pair(line, ""); 62 63 directive = directive.substr(space_loc); 64 directive = directive.strip("\" \n"); 65 return std::make_pair(line, directive); 66 } 67 68 // Returns the Glslang message rules for the given target environment, 69 // source language, and whether we want HLSL offset rules. We assume 70 // only valid combinations are used. 71 EShMessages GetMessageRules(shaderc_util::Compiler::TargetEnv env, 72 shaderc_util::Compiler::SourceLanguage lang, 73 bool hlsl_offsets) { 74 using shaderc_util::Compiler; 75 EShMessages result = EShMsgCascadingErrors; 76 if (lang == Compiler::SourceLanguage::HLSL) { 77 result = static_cast<EShMessages>(result | EShMsgReadHlsl); 78 } 79 switch (env) { 80 case Compiler::TargetEnv::OpenGLCompat: 81 break; 82 case Compiler::TargetEnv::OpenGL: 83 result = static_cast<EShMessages>(result | EShMsgSpvRules); 84 break; 85 case Compiler::TargetEnv::Vulkan: 86 result = 87 static_cast<EShMessages>(result | EShMsgSpvRules | EShMsgVulkanRules); 88 break; 89 } 90 if (hlsl_offsets) { 91 result = static_cast<EShMessages>(result | EShMsgHlslOffsets); 92 } 93 return result; 94 } 95 96 } // anonymous namespace 97 98 namespace shaderc_util { 99 100 void Compiler::SetLimit(Compiler::Limit limit, int value) { 101 switch (limit) { 102 #define RESOURCE(NAME, FIELD, CNAME) \ 103 case Limit::NAME: \ 104 limits_.FIELD = value; \ 105 break; 106 #include "libshaderc_util/resources.inc" 107 #undef RESOURCE 108 } 109 } 110 111 int Compiler::GetLimit(Compiler::Limit limit) const { 112 switch (limit) { 113 #define RESOURCE(NAME, FIELD, CNAME) \ 114 case Limit::NAME: \ 115 return limits_.FIELD; 116 #include "libshaderc_util/resources.inc" 117 #undef RESOURCE 118 } 119 return 0; // Unreachable 120 } 121 122 std::tuple<bool, std::vector<uint32_t>, size_t> Compiler::Compile( 123 const string_piece& input_source_string, EShLanguage forced_shader_stage, 124 const std::string& error_tag, const char* entry_point_name, 125 const std::function<EShLanguage(std::ostream* error_stream, 126 const string_piece& error_tag)>& 127 stage_callback, 128 CountingIncluder& includer, OutputType output_type, 129 std::ostream* error_stream, size_t* total_warnings, size_t* total_errors, 130 GlslangInitializer* initializer) const { 131 // Compilation results to be returned: 132 // Initialize the result tuple as a failed compilation. In error cases, we 133 // should return result_tuple directly without setting its members. 134 auto result_tuple = 135 std::make_tuple(false, std::vector<uint32_t>(), (size_t)0u); 136 // Get the reference of the members of the result tuple. We should set their 137 // values for succeeded compilation before returning the result tuple. 138 bool& succeeded = std::get<0>(result_tuple); 139 std::vector<uint32_t>& compilation_output_data = std::get<1>(result_tuple); 140 size_t& compilation_output_data_size_in_bytes = std::get<2>(result_tuple); 141 142 auto token = initializer->Acquire(); 143 EShLanguage used_shader_stage = forced_shader_stage; 144 const std::string macro_definitions = 145 shaderc_util::format(predefined_macros_, "#define ", " ", "\n"); 146 const std::string pound_extension = 147 "#extension GL_GOOGLE_include_directive : enable\n"; 148 const std::string preamble = macro_definitions + pound_extension; 149 150 std::string preprocessed_shader; 151 152 // If only preprocessing, we definitely need to preprocess. Otherwise, if 153 // we don't know the stage until now, we need the preprocessed shader to 154 // deduce the shader stage. 155 if (output_type == OutputType::PreprocessedText || 156 used_shader_stage == EShLangCount) { 157 bool success; 158 std::string glslang_errors; 159 std::tie(success, preprocessed_shader, glslang_errors) = 160 PreprocessShader(error_tag, input_source_string, preamble, includer); 161 162 success &= PrintFilteredErrors(error_tag, error_stream, warnings_as_errors_, 163 /* suppress_warnings = */ true, 164 glslang_errors.c_str(), total_warnings, 165 total_errors); 166 if (!success) return result_tuple; 167 // Because of the behavior change of the #line directive, the #line 168 // directive introducing each file's content must use the syntax for the 169 // specified version. So we need to probe this shader's version and 170 // profile. 171 int version; 172 EProfile profile; 173 std::tie(version, profile) = DeduceVersionProfile(preprocessed_shader); 174 const bool is_for_next_line = LineDirectiveIsForNextLine(version, profile); 175 176 preprocessed_shader = 177 CleanupPreamble(preprocessed_shader, error_tag, pound_extension, 178 includer.num_include_directives(), is_for_next_line); 179 180 if (output_type == OutputType::PreprocessedText) { 181 // Set the values of the result tuple. 182 succeeded = true; 183 compilation_output_data = ConvertStringToVector(preprocessed_shader); 184 compilation_output_data_size_in_bytes = preprocessed_shader.size(); 185 return result_tuple; 186 } else if (used_shader_stage == EShLangCount) { 187 std::string errors; 188 std::tie(used_shader_stage, errors) = 189 GetShaderStageFromSourceCode(error_tag, preprocessed_shader); 190 if (!errors.empty()) { 191 *error_stream << errors; 192 return result_tuple; 193 } 194 if (used_shader_stage == EShLangCount) { 195 if ((used_shader_stage = stage_callback(error_stream, error_tag)) == 196 EShLangCount) { 197 return result_tuple; 198 } 199 } 200 } 201 } 202 203 // Parsing requires its own Glslang symbol tables. 204 glslang::TShader shader(used_shader_stage); 205 const char* shader_strings = input_source_string.data(); 206 const int shader_lengths = static_cast<int>(input_source_string.size()); 207 const char* string_names = error_tag.c_str(); 208 shader.setStringsWithLengthsAndNames(&shader_strings, &shader_lengths, 209 &string_names, 1); 210 shader.setPreamble(preamble.c_str()); 211 shader.setEntryPoint(entry_point_name); 212 shader.setAutoMapBindings(auto_bind_uniforms_); 213 const auto& bases = auto_binding_base_[static_cast<int>(used_shader_stage)]; 214 shader.setShiftImageBinding(bases[static_cast<int>(UniformKind::Image)]); 215 shader.setShiftSamplerBinding(bases[static_cast<int>(UniformKind::Sampler)]); 216 shader.setShiftTextureBinding(bases[static_cast<int>(UniformKind::Texture)]); 217 shader.setShiftUboBinding(bases[static_cast<int>(UniformKind::Buffer)]); 218 shader.setShiftSsboBinding(bases[static_cast<int>(UniformKind::StorageBuffer)]); 219 shader.setShiftUavBinding(bases[static_cast<int>(UniformKind::UnorderedAccessView)]); 220 shader.setHlslIoMapping(hlsl_iomap_); 221 shader.setResourceSetBinding( 222 hlsl_explicit_bindings_[static_cast<int>(used_shader_stage)]); 223 224 // TODO(dneto): Generate source-level debug info if requested. 225 bool success = shader.parse( 226 &limits_, default_version_, default_profile_, force_version_profile_, 227 kNotForwardCompatible, 228 GetMessageRules(target_env_, source_language_, hlsl_offsets_), includer); 229 230 success &= PrintFilteredErrors(error_tag, error_stream, warnings_as_errors_, 231 suppress_warnings_, shader.getInfoLog(), 232 total_warnings, total_errors); 233 if (!success) return result_tuple; 234 235 glslang::TProgram program; 236 program.addShader(&shader); 237 success = program.link(EShMsgDefault) && program.mapIO(); 238 success &= PrintFilteredErrors(error_tag, error_stream, warnings_as_errors_, 239 suppress_warnings_, program.getInfoLog(), 240 total_warnings, total_errors); 241 if (!success) return result_tuple; 242 243 // 'spirv' is an alias for the compilation_output_data. This alias is added 244 // to serve as an input for the call to DissassemblyBinary. 245 std::vector<uint32_t>& spirv = compilation_output_data; 246 // Note the call to GlslangToSpv also populates compilation_output_data. 247 glslang::GlslangToSpv(*program.getIntermediate(used_shader_stage), spirv); 248 249 // Set the tool field (the top 16-bits) in the generator word to 250 // 'Shaderc over Glslang'. 251 const uint32_t shaderc_generator_word = 13; // From SPIR-V XML Registry 252 const uint32_t generator_word_index = 2; // SPIR-V 2.3: Physical layout 253 assert(spirv.size() > generator_word_index); 254 spirv[generator_word_index] = 255 (spirv[generator_word_index] & 0xffff) | (shaderc_generator_word << 16); 256 257 if (!enabled_opt_passes_.empty()) { 258 std::string opt_errors; 259 if (!SpirvToolsOptimize(target_env_, enabled_opt_passes_, &spirv, 260 &opt_errors)) { 261 *error_stream << "shaderc: internal error: compilation succeeded but " 262 "failed to optimize: " 263 << opt_errors << "\n"; 264 return result_tuple; 265 } 266 } 267 268 if (output_type == OutputType::SpirvAssemblyText) { 269 std::string text_or_error; 270 if (!SpirvToolsDisassemble(target_env_, spirv, &text_or_error)) { 271 *error_stream << "shaderc: internal error: compilation succeeded but " 272 "failed to disassemble: " 273 << text_or_error << "\n"; 274 return result_tuple; 275 } 276 succeeded = true; 277 compilation_output_data = ConvertStringToVector(text_or_error); 278 compilation_output_data_size_in_bytes = text_or_error.size(); 279 return result_tuple; 280 } else { 281 succeeded = true; 282 // Note compilation_output_data is already populated in GlslangToSpv(). 283 compilation_output_data_size_in_bytes = spirv.size() * sizeof(spirv[0]); 284 return result_tuple; 285 } 286 } 287 288 void Compiler::AddMacroDefinition(const char* macro, size_t macro_length, 289 const char* definition, 290 size_t definition_length) { 291 predefined_macros_[std::string(macro, macro_length)] = 292 definition ? std::string(definition, definition_length) : ""; 293 } 294 295 void Compiler::SetTargetEnv(Compiler::TargetEnv env) { target_env_ = env; } 296 297 void Compiler::SetSourceLanguage(Compiler::SourceLanguage lang) { 298 source_language_ = lang; 299 } 300 301 void Compiler::SetForcedVersionProfile(int version, EProfile profile) { 302 default_version_ = version; 303 default_profile_ = profile; 304 force_version_profile_ = true; 305 } 306 307 void Compiler::SetWarningsAsErrors() { warnings_as_errors_ = true; } 308 309 void Compiler::SetGenerateDebugInfo() { 310 generate_debug_info_ = true; 311 for (size_t i = 0; i < enabled_opt_passes_.size(); ++i) { 312 if (enabled_opt_passes_[i] == PassId::kStripDebugInfo) { 313 enabled_opt_passes_[i] = PassId::kNullPass; 314 } 315 } 316 } 317 318 void Compiler::SetOptimizationLevel(Compiler::OptimizationLevel level) { 319 // Clear previous settings first. 320 enabled_opt_passes_.clear(); 321 322 switch (level) { 323 case OptimizationLevel::Size: 324 if (!generate_debug_info_) { 325 enabled_opt_passes_.push_back(PassId::kStripDebugInfo); 326 } 327 enabled_opt_passes_.push_back(PassId::kUnifyConstant); 328 break; 329 default: 330 break; 331 } 332 } 333 334 void Compiler::SetSuppressWarnings() { suppress_warnings_ = true; } 335 336 std::tuple<bool, std::string, std::string> Compiler::PreprocessShader( 337 const std::string& error_tag, const string_piece& shader_source, 338 const string_piece& shader_preamble, CountingIncluder& includer) const { 339 // The stage does not matter for preprocessing. 340 glslang::TShader shader(EShLangVertex); 341 const char* shader_strings = shader_source.data(); 342 const int shader_lengths = static_cast<int>(shader_source.size()); 343 const char* string_names = error_tag.c_str(); 344 shader.setStringsWithLengthsAndNames(&shader_strings, &shader_lengths, 345 &string_names, 1); 346 shader.setPreamble(shader_preamble.data()); 347 348 // The preprocessor might be sensitive to the target environment. 349 // So combine the existing rules with the just-give-me-preprocessor-output 350 // flag. 351 const auto rules = static_cast<EShMessages>( 352 EShMsgOnlyPreprocessor | 353 GetMessageRules(target_env_, source_language_, hlsl_offsets_)); 354 355 std::string preprocessed_shader; 356 const bool success = shader.preprocess( 357 &limits_, default_version_, default_profile_, force_version_profile_, 358 kNotForwardCompatible, rules, &preprocessed_shader, includer); 359 360 if (success) { 361 return std::make_tuple(true, preprocessed_shader, shader.getInfoLog()); 362 } 363 return std::make_tuple(false, "", shader.getInfoLog()); 364 } 365 366 std::string Compiler::CleanupPreamble(const string_piece& preprocessed_shader, 367 const string_piece& error_tag, 368 const string_piece& pound_extension, 369 int num_include_directives, 370 bool is_for_next_line) const { 371 // Those #define directives in preamble will become empty lines after 372 // preprocessing. We also injected an #extension directive to turn on #include 373 // directive support. In the original preprocessing output from glslang, it 374 // appears before the user source string. We need to do proper adjustment: 375 // * Remove empty lines generated from #define directives in preamble. 376 // * If there is no #include directive in the source code, we do not need to 377 // output the injected #extension directive. Otherwise, 378 // * If there exists a #version directive in the source code, it should be 379 // placed at the first line. Its original line will be filled with an empty 380 // line as placeholder to maintain the code structure. 381 382 const std::vector<string_piece> lines = 383 preprocessed_shader.get_fields('\n', /* keep_delimiter = */ true); 384 385 std::ostringstream output_stream; 386 387 size_t pound_extension_index = lines.size(); 388 size_t pound_version_index = lines.size(); 389 for (size_t i = 0; i < lines.size(); ++i) { 390 if (lines[i] == pound_extension) { 391 pound_extension_index = std::min(i, pound_extension_index); 392 } else if (lines[i].starts_with("#version")) { 393 // In a preprocessed shader, directives are in a canonical format, so we 394 // can confidently compare to '#version' verbatim, without worrying about 395 // whitespace. 396 pound_version_index = i; 397 if (num_include_directives > 0) output_stream << lines[i]; 398 break; 399 } 400 } 401 // We know that #extension directive exists and appears before #version 402 // directive (if any). 403 assert(pound_extension_index < lines.size()); 404 405 for (size_t i = 0; i < pound_extension_index; ++i) { 406 // All empty lines before the #line directive we injected are generated by 407 // preprocessing preamble. Do not output them. 408 if (lines[i].strip_whitespace().empty()) continue; 409 output_stream << lines[i]; 410 } 411 412 if (num_include_directives > 0) { 413 output_stream << pound_extension; 414 // Also output a #line directive for the main file. 415 output_stream << GetLineDirective(is_for_next_line, error_tag); 416 } 417 418 for (size_t i = pound_extension_index + 1; i < lines.size(); ++i) { 419 if (i == pound_version_index) { 420 if (num_include_directives > 0) { 421 output_stream << "\n"; 422 } else { 423 output_stream << lines[i]; 424 } 425 } else { 426 output_stream << lines[i]; 427 } 428 } 429 430 return output_stream.str(); 431 } 432 433 std::pair<EShLanguage, std::string> Compiler::GetShaderStageFromSourceCode( 434 string_piece filename, const std::string& preprocessed_shader) const { 435 const string_piece kPragmaShaderStageDirective = "#pragma shader_stage"; 436 const string_piece kLineDirective = "#line"; 437 438 int version; 439 EProfile profile; 440 std::tie(version, profile) = DeduceVersionProfile(preprocessed_shader); 441 const bool is_for_next_line = LineDirectiveIsForNextLine(version, profile); 442 443 std::vector<string_piece> lines = 444 string_piece(preprocessed_shader).get_fields('\n'); 445 // The filename, logical line number (which starts from 1 and is sensitive to 446 // #line directives), and stage value for #pragma shader_stage() directives. 447 std::vector<std::tuple<string_piece, size_t, string_piece>> stages; 448 // The physical line numbers of the first #pragma shader_stage() line and 449 // first non-preprocessing line in the preprocessed shader text. 450 size_t first_pragma_physical_line = lines.size() + 1; 451 size_t first_non_pp_line = lines.size() + 1; 452 453 for (size_t i = 0, logical_line_no = 1; i < lines.size(); ++i) { 454 const string_piece current_line = lines[i].strip_whitespace(); 455 if (current_line.starts_with(kPragmaShaderStageDirective)) { 456 const string_piece stage_value = 457 current_line.substr(kPragmaShaderStageDirective.size()).strip("()"); 458 stages.emplace_back(filename, logical_line_no, stage_value); 459 first_pragma_physical_line = std::min(first_pragma_physical_line, i + 1); 460 } else if (!current_line.empty() && !current_line.starts_with("#")) { 461 first_non_pp_line = std::min(first_non_pp_line, i + 1); 462 } 463 464 // Update logical line number for the next line. 465 if (current_line.starts_with(kLineDirective)) { 466 string_piece name; 467 std::tie(logical_line_no, name) = DecodeLineDirective(current_line); 468 if (!name.empty()) filename = name; 469 // Note that for core profile, the meaning of #line changed since version 470 // 330. The line number given by #line used to mean the logical line 471 // number of the #line line. Now it means the logical line number of the 472 // next line after the #line line. 473 if (!is_for_next_line) ++logical_line_no; 474 } else { 475 ++logical_line_no; 476 } 477 } 478 if (stages.empty()) return std::make_pair(EShLangCount, ""); 479 480 std::string error_message; 481 482 const string_piece& first_pragma_filename = std::get<0>(stages[0]); 483 const std::string first_pragma_line = std::to_string(std::get<1>(stages[0])); 484 const string_piece& first_pragma_stage = std::get<2>(stages[0]); 485 486 if (first_pragma_physical_line > first_non_pp_line) { 487 error_message += first_pragma_filename.str() + ":" + first_pragma_line + 488 ": error: '#pragma': the first 'shader_stage' #pragma " 489 "must appear before any non-preprocessing code\n"; 490 } 491 492 EShLanguage stage = MapStageNameToLanguage(first_pragma_stage); 493 if (stage == EShLangCount) { 494 error_message += 495 first_pragma_filename.str() + ":" + first_pragma_line + 496 ": error: '#pragma': invalid stage for 'shader_stage' #pragma: '" + 497 first_pragma_stage.str() + "'\n"; 498 } 499 500 for (size_t i = 1; i < stages.size(); ++i) { 501 const string_piece& current_stage = std::get<2>(stages[i]); 502 if (current_stage != first_pragma_stage) { 503 const string_piece& current_filename = std::get<0>(stages[i]); 504 const std::string current_line = std::to_string(std::get<1>(stages[i])); 505 error_message += current_filename.str() + ":" + current_line + 506 ": error: '#pragma': conflicting stages for " 507 "'shader_stage' #pragma: '" + 508 current_stage.str() + "' (was '" + 509 first_pragma_stage.str() + "' at " + 510 first_pragma_filename.str() + ":" + first_pragma_line + 511 ")\n"; 512 } 513 } 514 515 return std::make_pair(error_message.empty() ? stage : EShLangCount, 516 error_message); 517 } 518 519 std::pair<int, EProfile> Compiler::DeduceVersionProfile( 520 const std::string& preprocessed_shader) const { 521 int version = default_version_; 522 EProfile profile = default_profile_; 523 if (!force_version_profile_) { 524 std::tie(version, profile) = 525 GetVersionProfileFromSourceCode(preprocessed_shader); 526 if (version == 0 && profile == ENoProfile) { 527 version = default_version_; 528 profile = default_profile_; 529 } 530 } 531 return std::make_pair(version, profile); 532 } 533 534 std::pair<int, EProfile> Compiler::GetVersionProfileFromSourceCode( 535 const std::string& preprocessed_shader) const { 536 string_piece pound_version = preprocessed_shader; 537 const size_t pound_version_loc = pound_version.find("#version"); 538 if (pound_version_loc == string_piece::npos) { 539 return std::make_pair(0, ENoProfile); 540 } 541 pound_version = 542 pound_version.substr(pound_version_loc + std::strlen("#version")); 543 pound_version = pound_version.substr(0, pound_version.find_first_of("\n")); 544 545 std::string version_profile; 546 for (const auto character : pound_version) { 547 if (character != ' ') version_profile += character; 548 } 549 550 int version; 551 EProfile profile; 552 if (!ParseVersionProfile(version_profile, &version, &profile)) { 553 return std::make_pair(0, ENoProfile); 554 } 555 return std::make_pair(version, profile); 556 } 557 558 // Converts a string to a vector of uint32_t by copying the content of a given 559 // string to a vector<uint32_t> and returns it. Appends '\0' at the end if extra 560 // bytes are required to complete the last element. 561 std::vector<uint32_t> ConvertStringToVector(const std::string& str) { 562 size_t num_bytes_str = str.size() + 1u; 563 size_t vector_length = 564 (num_bytes_str + sizeof(uint32_t) - 1) / sizeof(uint32_t); 565 std::vector<uint32_t> result_vec(vector_length, 0); 566 std::strncpy(reinterpret_cast<char*>(result_vec.data()), str.c_str(), 567 str.size()); 568 return result_vec; 569 } 570 571 } // namesapce shaderc_util 572