From ef72003049a56ceafdd45912f9a007360507d3b2 Mon Sep 17 00:00:00 2001 From: James Martin Date: Sun, 17 Oct 2021 23:09:46 -0700 Subject: [PATCH] Initial commit --- .editorconfig | 9 + LICENSE | 24 ++ README.md | 98 +++++++++ meson.build | 15 ++ subprojects/blob-object-coff/meson.build | 19 ++ subprojects/blob-object-coff/src/coff.h | 51 +++++ subprojects/blob-object-coff/src/main.c | 174 +++++++++++++++ subprojects/blob-object-elf/meson.build | 5 + subprojects/blob-object-elf/src/main.c | 266 +++++++++++++++++++++++ 9 files changed, 661 insertions(+) create mode 100644 .editorconfig create mode 100644 LICENSE create mode 100644 README.md create mode 100644 meson.build create mode 100644 subprojects/blob-object-coff/meson.build create mode 100644 subprojects/blob-object-coff/src/coff.h create mode 100644 subprojects/blob-object-coff/src/main.c create mode 100644 subprojects/blob-object-elf/meson.build create mode 100644 subprojects/blob-object-elf/src/main.c diff --git a/.editorconfig b/.editorconfig new file mode 100644 index 0000000..82af483 --- /dev/null +++ b/.editorconfig @@ -0,0 +1,9 @@ +# https://EditorConfig.org/ +root = true + +[*] +indent_size = 4 +charset = "utf-8" +indent_style = "space" +trim_trailing_whitespace = "true" +insert_final_newline = "true" \ No newline at end of file diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..00d2e13 --- /dev/null +++ b/LICENSE @@ -0,0 +1,24 @@ +This is free and unencumbered software released into the public domain. + +Anyone is free to copy, modify, publish, use, compile, sell, or +distribute this software, either in source code form or as a compiled +binary, for any purpose, commercial or non-commercial, and by any +means. + +In jurisdictions that recognize copyright laws, the author or authors +of this software dedicate any and all copyright interest in the +software to the public domain. We make this dedication for the benefit +of the public at large and to the detriment of our heirs and +successors. We intend this dedication to be an overt act of +relinquishment in perpetuity of all present and future rights to this +software under copyright law. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR +OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +OTHER DEALINGS IN THE SOFTWARE. + +For more information, please refer to \ No newline at end of file diff --git a/README.md b/README.md new file mode 100644 index 0000000..f33cd7e --- /dev/null +++ b/README.md @@ -0,0 +1,98 @@ +# Blob Object +A simple utility for embedding an arbitrary binary blob into an ELF or COFF object file. +You can then link your program against that object file to be able to access +the binary blob as a C array. + +This project will only be updated rarely, if ever. +This isn't because the project has been abandoned; +it's just trivial enough that (other than the caveats described below) +there are few reasons that an update would be necessary. + +This project is public domain (see `LICENSE`), so feel free to use it however you'd like. +Credit is optional. + +## Installation +This project has no dependencies and can be built with Meson: +`meson setup build && cd build && meson compile && sudo meson install` + +You can also compile it manually with your C compiler; meson is just for convenience: +```bash +cc -o blob-object-elf subprojects/blob-object-elf/src/main.c +cc -o blob-object-coff subprojects/blob-object-coff/src/main.c +``` + +## Usage +`blob-object-(elf|coff)
` + +You can include the generated blob object in C using these declarations: +```c +// the size in bytes of the blob. can be uint64_t with ELF, but COFF is restricted to 32-bit. +extern uint32_t const _size; +// the address of this constant is the beginning of the blob +extern uint8_t const ; + +// get a reference to your data +uint8_t* my_data = &; +``` + +### Example for compiling GLSL shaders to SPIR-V and into your Vulkan program +Here's an example of how you might use this to include compiled SPIR-V shaders in your executable: +* `glslangValidator -V100 -o "frag_shader.spv" "frag.glsl"` +* `blob-object-elf frag_shader.o shaders frag_shader frag_shader.spv` +* `gcc my-vulkan-program.c frag_shader.o` + +I wrote a hack to make shader compilation part of the build processess with meson: +```meson +compile_shader = find_program(meson.source_root() / 'compile-shader.sh') +shaders_gen = generator( + compile_shader, + output: '@PLAINNAME@.o', + arguments: [ + '@BUILD_ROOT@', + '@INPUT@', + '@OUTPUT@', + '@PLAINNAME@', + ] +) + +sources += [ + shaders_gen.process('shaders/shader.frag'), + shaders_gen.process('shaders/shader.vert') +] +``` + +`compile-shader.sh`: +```bash +#!/bin/bash +build_root="$1" +src="$2" +obj="$3" +name="$4" + +if [[ "$OSTYPE" == "msys" ]]; then + make_blob="$build_root/subprojects/make-coff-blob/blob-object-coff.exe" +else + make_blob="$build_root/subprojects/make-elf-blob/blob-object-elf" +fi + +base_name="${name%.*}" +ext="${name#*.}" + +spv="$obj.spv" + +glslangValidator -V100 -o "$spv" "$src" && $make_blob "$obj" shaders "cg_${base_name}_${ext}" "$spv" +``` + +## Caveats +* `blob-object-coff` is restricted to <4GB blobs because COFF is a 32-bit format +* `blob-object-elf` cannot be used on native Windows because it depends on POSIX + (fix: this program should use standard library headers instead of POSIX and bundle its own elf.h) +* both generators are hardcoded to generate object files flagged for x86_64 for their respective platforms + (fix: add an argument to select which platform to build for) + +Fixing these things would be pretty trivial; feel free to send a PR. +I'd also be willing to do it myself if anyone ends up actually using this, +so you could open an issue instead too. + +It'd also be nice to support Mach-O (MacOS) and combine both blob-object commands +into a single executable. \ No newline at end of file diff --git a/meson.build b/meson.build new file mode 100644 index 0000000..21b4f81 --- /dev/null +++ b/meson.build @@ -0,0 +1,15 @@ +project('blob-object', 'c', default_options: ['c_std=c17', 'warning_level=3']) + +compiler = meson.get_compiler('c') +if compiler.get_id() == 'clang' or compiler.get_id() == 'gcc' + # MSVC does not support VLAs for us to disable them. + add_global_arguments('-Werror=vla', language: 'c') +endif + +if target_machine.system() == 'windows' + add_global_arguments('-DUNICODE', '-D_CRT_SECURE_NO_WARNINGS', language: 'c') +else + # the ELF blob maker is not portable currently + subproject('blob-object-elf') +endif +subproject('blob-object-coff') \ No newline at end of file diff --git a/subprojects/blob-object-coff/meson.build b/subprojects/blob-object-coff/meson.build new file mode 100644 index 0000000..83df455 --- /dev/null +++ b/subprojects/blob-object-coff/meson.build @@ -0,0 +1,19 @@ +project('blob-object-coff', 'c') + +if meson.get_compiler('c').get_id() == 'msvc' + add_project_arguments( + # we must downcast a few things because COFF is 32-bit + '/IGNORE:4267', + language: 'c' + ) +else + add_project_arguments( + '-Wno-pointer-arith', + '-Wno-sign-compare', + '-Wno-missing-braces', + '-fno-strict-aliasing', + language: 'c' + ) +endif + +make_coff_blob = executable('blob-object-coff', 'src/main.c', install: true) \ No newline at end of file diff --git a/subprojects/blob-object-coff/src/coff.h b/subprojects/blob-object-coff/src/coff.h new file mode 100644 index 0000000..c6b9738 --- /dev/null +++ b/subprojects/blob-object-coff/src/coff.h @@ -0,0 +1,51 @@ +#ifndef _COFF_H +#define _COFF_H + +#include + +struct coff_header { + uint16_t magic; + uint16_t nscns; /* number of sections */ + uint32_t timdat; /* time and date stamp */ + uint32_t symptr; /* file offset of symbol table */ + uint32_t nsyms; /* number of symbols */ + uint16_t opthdr; /* size of optional header */ + uint16_t flags; /* flags */ +}; + +#define MAGIC_AMD64 0x8664 + +struct coff_section { + char name[8]; /* section name */ + uint32_t paddr; /* physical address */ + uint32_t vaddr; /* virtual address */ + uint32_t size; /* section size in bytes */ + uint32_t scnptr; /* file offset to section data */ + uint32_t relptr; /* file offset to relocation table */ + uint32_t lnnoptr; /* file offset to line number table */ + uint16_t nreloc; /* number of relocation table entries */ + uint16_t nlnno; /* number of line number table entries */ + uint32_t flags; /* section flags */ +}; + +/* section types */ +#define STYP_TEXT 0x0020 +#define STYP_DATA 0x0040 +#define STYP_BSS 0x0080 + +struct coff_symtab { + uint32_t zeroes; /* must be zero; used as part of a feature we don't care about */ + uint32_t name; /* offset into the string table */ + uint32_t value; + uint16_t scnum; /* section number */ + uint16_t type; + uint8_t sclass; /* storage class */ + uint8_t numaux; /* auxiliary count */ +}; +#define COFF_SYMTAB_SIZE 18 + +/* symbol storage classes */ +#define C_EXT 2 /* global symbol storage class */ +#define C_STAT 3 /* static symbol storage class */ + +#endif /* _COFF_H */ diff --git a/subprojects/blob-object-coff/src/main.c b/subprojects/blob-object-coff/src/main.c new file mode 100644 index 0000000..59ddf9c --- /dev/null +++ b/subprojects/blob-object-coff/src/main.c @@ -0,0 +1,174 @@ +#ifndef _WIN32 +#include +#endif +#include +#include +#include +#include + +#include "coff.h" + +static const char* const usage = + "usage: blob-object-coff
\n"; + +int main(int argc, char** argv) { + if (argc < 5) { + fprintf(stderr, "not enough arguments\n%s", usage); + return 1; + } + if (argc > 5) { + fprintf(stderr, "too many arguments\n%s", usage); + return 1; + } + + char* output_file_name = argv[1]; + char* section_name = argv[2]; + uint32_t section_name_size = strlen(section_name); + char* symbol_name = argv[3]; + uint32_t symbol_name_size = strlen(symbol_name) + 1; + char* blob_file_name = argv[4]; + + if (section_name_size > 8) { + fprintf(stderr, "section name is too long for COFF\n"); + return 1; + } + + FILE* blob_file = fopen(blob_file_name, "rb"); + if (blob_file == NULL) { + goto blob_fail; + } + if (fseek(blob_file, 0, SEEK_END) != 0) { + goto blob_fail; + } + long blob_size = ftell(blob_file); + if (blob_size == -1L) { + goto blob_fail; + } + rewind(blob_file); + void* blob = malloc(blob_size); + if (fread(blob, 1, blob_size, blob_file) < blob_size) { + goto blob_fail; + } + fclose(blob_file); + + FILE* output_file = fopen(output_file_name, "wb"); + if (output_file == NULL) { + goto output_fail; + } + + uint32_t size_name_size = symbol_name_size + 5; + uint32_t blob_name_size = symbol_name_size; + + uint32_t strtab_size_name_off = 4; + uint32_t strtab_blob_name_off = strtab_size_name_off + size_name_size; + uint32_t strtab_size = 4 + strtab_blob_name_off + blob_name_size; + + uint32_t coff_header_off = 0; + uint32_t section_header_off = sizeof(struct coff_header); + uint32_t symbol_header_off = section_header_off + sizeof(struct coff_section); + uint32_t size_symbol_off = symbol_header_off; + uint32_t blob_symbol_off = size_symbol_off + COFF_SYMTAB_SIZE; + uint32_t string_table_off = blob_symbol_off + COFF_SYMTAB_SIZE; + uint32_t blob_size_off = string_table_off + strtab_size; + uint32_t blob_data_off = blob_size_off + sizeof(uint64_t); + uint32_t file_end_off = blob_data_off + blob_size; + + uint8_t* data = calloc(file_end_off, 1); + + time_t now = time(0); + *(struct coff_header*) (data + coff_header_off) = (struct coff_header) { + .magic = MAGIC_AMD64, + .nscns = 1, + .timdat = (uint32_t) now, + .symptr = symbol_header_off, + .nsyms = 2, + .opthdr = 0, + .flags = 0, + }; + + struct coff_section data_section = { + .name = 0, + .paddr = 0, + .vaddr = 0, + .size = sizeof(uint64_t) + blob_size, + .scnptr = blob_size_off, + .relptr = 0, + .lnnoptr = 0, + .nreloc = 0, + .nlnno = 0, + .flags = STYP_DATA, + }; + strncpy(data_section.name, section_name, section_name_size); + memcpy(data + section_header_off, &data_section, sizeof(struct coff_section)); + + struct coff_symtab size_symbol = { + .zeroes = 0, + .name = strtab_size_name_off, + .value = 0, + .scnum = 1, + .type = 0, + .sclass = C_EXT, + .numaux = 0, + }; + memcpy(data + size_symbol_off, &size_symbol, COFF_SYMTAB_SIZE); + + struct coff_symtab blob_symbol = { + .zeroes = 0, + .name = strtab_blob_name_off, + .value = sizeof(uint64_t), + .scnum = 1, + .type = 0, + .sclass = C_EXT, + .numaux = 0, + }; + memcpy(data + blob_symbol_off, &blob_symbol, COFF_SYMTAB_SIZE); + + memcpy(data + string_table_off, &strtab_size, 4); + memcpy( + data + string_table_off + strtab_size_name_off, + symbol_name, + size_name_size - 1 + ); + memcpy( + data + string_table_off + strtab_size_name_off + symbol_name_size - 1, + "_size\0", + 6 + ); + memcpy( + data + string_table_off + strtab_blob_name_off, + symbol_name, + blob_name_size + ); + + *(uint64_t*) (data + blob_size_off) = blob_size; + memcpy(data + blob_data_off, blob, blob_size); + + if (fwrite(data, 1, file_end_off, output_file) < file_end_off) { + goto output_fail; + } + if (fflush(output_file) != 0) { + goto output_fail; + } + + return 0; + + blob_fail: + fprintf( + stderr, + "failed to read blob file %s: %s\n%s", + blob_file_name, + strerror(errno), + usage + ); + return 1; + + output_fail: + fprintf( + stderr, + "failed to write output file %s: %s\n %s", + output_file_name, + strerror(errno), + usage + ); + return 1; +} diff --git a/subprojects/blob-object-elf/meson.build b/subprojects/blob-object-elf/meson.build new file mode 100644 index 0000000..5918402 --- /dev/null +++ b/subprojects/blob-object-elf/meson.build @@ -0,0 +1,5 @@ +project('blob-object-elf', 'c') + +add_project_arguments('-Wno-pointer-arith', '-fno-strict-aliasing', language: 'c') + +executable('blob-object-elf', 'src/main.c', install: true) \ No newline at end of file diff --git a/subprojects/blob-object-elf/src/main.c b/subprojects/blob-object-elf/src/main.c new file mode 100644 index 0000000..9f65226 --- /dev/null +++ b/subprojects/blob-object-elf/src/main.c @@ -0,0 +1,266 @@ +#define _POSIX_C_SOURCE 200112L + +#include +#include +#include +#include +#include +#include +#include +#include + +#ifndef align_up +#define align_up(num, align) (((num) + ((align)-1)) & ~((align)-1)) +#endif + +static const char* const usage = + "usage: blob-object-elf
\n"; + +int main(int argc, char** argv) { + if (argc < 5) { + fprintf(stderr, "not enough arguments\n%s", usage); + return 1; + } + if (argc > 5) { + fprintf(stderr, "too many arguments\n%s", usage); + return 1; + } + + char* output_file_name = argv[1]; + char* section_name = argv[2]; + size_t section_name_size = strlen(section_name) + 1; + char* symbol_name = argv[3]; + size_t symbol_name_size = strlen(symbol_name) + 1; + char* blob_file_name = argv[4]; + + int blob_fd = open(blob_file_name, O_RDONLY); + if (blob_fd < 0) { + fprintf( + stderr, + "failed to open blob file %s: %s\n%s", + output_file_name, + strerror(errno), + usage + ); + return 1; + } + + off_t blob_size = lseek(blob_fd, 0, SEEK_END); + if (blob_size == (off_t) -1) { + fprintf( + stderr, + "failed to get size of blob file: %s\n", + strerror(errno) + ); + return 1; + } + + void* blob = + mmap((void*) -1, blob_size, PROT_READ, MAP_PRIVATE, blob_fd, 0); + if (blob == (void*) -1) { + fprintf(stderr, "failed to mmap blob file: %s\n", strerror(errno)); + return 1; + } + + close(blob_fd); + + // only write is needed, but mmap requires O_RDWR + int output_fd = open(output_file_name, O_RDWR | O_CREAT | O_TRUNC); + if (output_fd < 0) { + fprintf( + stderr, + "failed to open output file %s: %s\n%s", + output_file_name, + strerror(errno), + usage + ); + return 1; + } + + // + // Pre-calculate file layout. It looks like this: + // * ELF header + // * Section headers: + // * .shstrtab + // * .strtab + // * .symtab + // * + // * .shstrtab data + // * .strtab data + // * .symtab data + // * _size + // * + // + + char* shstrtab_fixed_data = "\0.shstrtab\0.strtab\0.symtab\0"; + // offsets into .shstrtab + size_t shstrtab_shstrtab_off = 1; + size_t shstrtab_strtab_off = 11; + size_t shstrtab_symtab_off = 19; + size_t shstrtab_blob_off = 28; + size_t shstrtab_fixed_data_size = 28; + size_t shstrtab_data_size = shstrtab_fixed_data_size + section_name_size; + + // offsets into .strtab + size_t strtab_name_off = 1; + size_t strtab_size_off = strtab_name_off + symbol_name_size; + size_t strtab_end_off = strtab_size_off + symbol_name_size + 5; + + // three symbols: the zero symbol, the blob, and the blob size + size_t symtab_data_size = 3 * sizeof(Elf64_Sym); + + // the absolute offsets into the file of each section as described above + Elf64_Off ehdr_off = 0; + Elf64_Off shtab_off = ehdr_off + sizeof(Elf64_Ehdr); + Elf64_Off shstrtab_hdr_off = shtab_off + sizeof(Elf64_Shdr); + Elf64_Off strtab_hdr_off = shstrtab_hdr_off + sizeof(Elf64_Shdr); + Elf64_Off symtab_hdr_off = strtab_hdr_off + sizeof(Elf64_Shdr); + Elf64_Off blob_hdr_off = symtab_hdr_off + sizeof(Elf64_Shdr); + Elf64_Off shstrtab_data_off = blob_hdr_off + sizeof(Elf64_Shdr); + Elf64_Off strtab_data_off = shstrtab_data_off + shstrtab_data_size; + Elf64_Off strtab_data_name_off = strtab_data_off + strtab_name_off; + Elf64_Off strtab_data_size_off = strtab_data_off + strtab_size_off; + Elf64_Off symtab_data_off = strtab_data_off + strtab_end_off; + Elf64_Off blob_size_off = symtab_data_off + symtab_data_size; + Elf64_Off blob_data_off = blob_size_off + sizeof(uint64_t); + Elf64_Off file_end_off = blob_data_off + blob_size; + + if (ftruncate(output_fd, file_end_off) != 0) { + fprintf( + stderr, + "failed to truncate output file to size: %s\n", + strerror(errno) + ); + return 1; + } + + void* data = + mmap((void*) -1, file_end_off, PROT_WRITE, MAP_SHARED, output_fd, 0); + if (data == (void*) -1) { + fprintf( + stderr, + "failed to mmap output file: %s\n", + strerror(errno) + ); + return 1; + } + + close(output_fd); + + *(Elf64_Ehdr*) (data + ehdr_off) = (Elf64_Ehdr) { + .e_ident = { 0x7F, 'E', 'L', 'F', ELFCLASS64, ELFDATA2LSB, EV_CURRENT }, + .e_type = ET_REL, + .e_machine = EM_X86_64, + .e_version = EV_CURRENT, + .e_entry = 0, + .e_phoff = 0, + .e_shoff = shtab_off, + .e_flags = 0, + .e_ehsize = sizeof(Elf64_Ehdr), + .e_phentsize = 0, + .e_phnum = 0, + .e_shentsize = sizeof(Elf64_Shdr), + .e_shnum = 5, + .e_shstrndx = 1, + }; + + *(Elf64_Shdr*) (data + shstrtab_hdr_off) = (Elf64_Shdr) { + .sh_name = shstrtab_shstrtab_off, + .sh_type = SHT_STRTAB, + .sh_flags = 0, + .sh_addr = 0, + .sh_offset = shstrtab_data_off, + .sh_size = shstrtab_data_size, + .sh_link = 0, + .sh_info = 0, + .sh_addralign = 0, + .sh_entsize = 0, + }; + + *(Elf64_Shdr*) (data + strtab_hdr_off) = (Elf64_Shdr) { + .sh_name = shstrtab_strtab_off, + .sh_type = SHT_STRTAB, + .sh_flags = 0, + .sh_addr = 0, + .sh_offset = strtab_data_off, + .sh_size = strtab_end_off, + .sh_link = 0, + .sh_info = 0, + .sh_addralign = 0, + .sh_entsize = 0, + }; + + *(Elf64_Shdr*) (data + symtab_hdr_off) = (Elf64_Shdr) { + .sh_name = shstrtab_symtab_off, + .sh_type = SHT_SYMTAB, + .sh_flags = 0, + .sh_addr = 0, + .sh_offset = symtab_data_off, + .sh_size = symtab_data_size, + .sh_link = 2, + .sh_info = 1, + .sh_addralign = 0, + .sh_entsize = sizeof(Elf64_Sym), + }; + + *(Elf64_Shdr*) (data + blob_hdr_off) = (Elf64_Shdr) { + .sh_name = shstrtab_blob_off, + .sh_type = SHT_PROGBITS, + .sh_flags = SHF_ALLOC, + .sh_addr = 0, + .sh_offset = blob_size_off, + .sh_size = sizeof(uint64_t) + blob_size, + .sh_link = 0, + .sh_info = 0, + .sh_addralign = 0, + .sh_entsize = 0, + }; + + memcpy( + data + shstrtab_data_off, + shstrtab_fixed_data, + shstrtab_fixed_data_size + ); + memcpy( + data + shstrtab_data_off + shstrtab_fixed_data_size, + section_name, + section_name_size + ); + + memcpy(data + strtab_data_name_off, symbol_name, symbol_name_size); + memcpy( + data + strtab_data_size_off, + symbol_name, + symbol_name_size - 1 + ); + memcpy( + data + strtab_data_size_off + symbol_name_size - 1, + "_size", + 5 + ); + + *(Elf64_Sym*) (data + symtab_data_off + sizeof(Elf64_Sym)) = (Elf64_Sym) { + .st_name = strtab_name_off, + .st_info = ELF64_ST_INFO(STB_GLOBAL, STT_OBJECT), + .st_other = ELF64_ST_VISIBILITY(STV_DEFAULT), + .st_shndx = 4, + .st_value = sizeof(uint64_t), + .st_size = blob_size, + }; + + // I wanted to make this an absolute symbol, + // but it segfaulted for whatever reason, probably relocation-related. + *(Elf64_Sym*) (data + symtab_data_off + 2 * sizeof(Elf64_Sym)) = (Elf64_Sym) { + .st_name = strtab_size_off, + .st_info = ELF64_ST_INFO(STB_GLOBAL, STT_OBJECT), + .st_other = ELF64_ST_VISIBILITY(STV_DEFAULT), + .st_shndx = 4, + .st_value = 0, + .st_size = sizeof(uint64_t), + }; + + *(uint64_t*) (data + blob_size_off) = blob_size; + memcpy(data + blob_data_off, blob, blob_size); + + return 0; +}