diff --git a/src/coders/binary_json_spec.md b/src/coders/binary_json_spec.md new file mode 100644 index 00000000..4e694247 --- /dev/null +++ b/src/coders/binary_json_spec.md @@ -0,0 +1,56 @@ +# Binary JSON Format Specification + +Format version: 1.0 + +This binary data format is developed for use as binary version of JSON in [VoxelEngine-Cpp](https://github.com/MihailRis/VoxelEngine-Cpp) and not compatible with [BSON](https://bsonspec.org/spec.html) due to elements/entries syntax and type codes differences + +## Basic types + +byteorder: little-endian +| Name | Size | Definition | +| ------- | ------- | ----------------------- | +| byte | 1 byte | 8 bit unsigned integer | +| int16 | 2 bytes | 16 bit signed integer | +| int32 | 4 bytes | 32 bit signed integer | +| int64 | 8 bytes | 64 bit unsigned integer | +| uint32 | 4 bytes | 32 bit unsigned integer | +| float64 | 8 bytes | 64 bit floating point | + +## Syntax (RFC 5234) + +```bnf +file = %x01 document / cdocument file content +document = uint32 (*entry) %x00 uint32 stores bytes number + of the encoded document + including the uint32 size +entry = cstring value +value = %x01 document + / %x02 (*value) %x00 list of values + / %x03 byte 8 bit integer + / %x04 int16 16 bit integer + / %x05 int32 32 bit integer + / %x06 int64 64 bit integer + / %x07 float64 number + / %x08 string utf-8 encoded string + / %x09 uint32 (*byte) bytes array + / %x0A boolean 'false' + / %x0B boolean 'true' + / %x0C null value +cdocument = %x1F %x8B (16*byte) gzip-compressed document +cstring = (*%x01-FF) %x00 +string = uint32 (*byte) uint32 stores number of the + encoded string bytes + +float64 = 8byte +int64 = 8byte +uint32 = 4byte +int32 = 4byte +int16 = 2byte +byte = %x00-FF +``` + +## VoxelEngine format support + +Current implementation does not support types: bytes array, null, compressed document. + +All unsupported types will be implemented in future. diff --git a/src/coders/byte_utils.cpp b/src/coders/byte_utils.cpp new file mode 100644 index 00000000..5a404ae5 --- /dev/null +++ b/src/coders/byte_utils.cpp @@ -0,0 +1,212 @@ +#include "byte_utils.h" + +#include +#include +#include + +void ByteBuilder::put(ubyte b) { + buffer.push_back(b); +} + +void ByteBuilder::putCStr(const char* str) { + size_t size = strlen(str)+1; + buffer.reserve(buffer.size() + size); + for (size_t i = 0; i < size; i++) { + buffer.push_back(str[i]); + } +} + +void ByteBuilder::put(const std::string& s) { + size_t len = s.length(); + putInt32(len); + put((const ubyte*)s.data(), len); +} + +void ByteBuilder::put(const ubyte* arr, size_t size) { + buffer.reserve(buffer.size() + size); + for (size_t i = 0; i < size; i++) { + buffer.push_back(arr[i]); + } +} + +void ByteBuilder::putInt16(int16_t val) { + buffer.push_back((char) (val >> 0 & 255)); + buffer.push_back((char) (val >> 8 & 255)); +} + +void ByteBuilder::putInt32(int32_t val) { + buffer.reserve(buffer.size() + 4); + buffer.push_back((char) (val >> 0 & 255)); + buffer.push_back((char) (val >> 8 & 255)); + buffer.push_back((char) (val >> 16 & 255)); + buffer.push_back((char) (val >> 24 & 255)); +} + +void ByteBuilder::putInt64(int64_t val) { + buffer.reserve(buffer.size() + 8); + buffer.push_back((char) (val >> 0 & 255)); + buffer.push_back((char) (val >> 8 & 255)); + buffer.push_back((char) (val >> 16 & 255)); + buffer.push_back((char) (val >> 24 & 255)); + buffer.push_back((char) (val >> 32 & 255)); + buffer.push_back((char) (val >> 40 & 255)); + buffer.push_back((char) (val >> 48 & 255)); + buffer.push_back((char) (val >> 56 & 255)); +} + +void ByteBuilder::putFloat32(float val) { + union { + int32_t vali32; + float valfloat; + } value; + value.valfloat = val; + putInt32(value.vali32); +} + +void ByteBuilder::putFloat64(double val) { + union { + int64_t vali64; + double valfloat; + } value; + value.valfloat = val; + putInt64(value.vali64); +} + +void ByteBuilder::set(size_t position, ubyte val) { + buffer[position] = val; +} + +void ByteBuilder::setInt16(size_t position, int16_t val) { + buffer[position++] = val >> 0 & 255; + buffer[position] = val >> 8 & 255; +} + +void ByteBuilder::setInt32(size_t position, int32_t val) { + buffer[position++] = val >> 0 & 255; + buffer[position++] = val >> 8 & 255; + buffer[position++] = val >> 16 & 255; + buffer[position] = val >> 24 & 255; +} + +void ByteBuilder::setInt64(size_t position, int64_t val) { + buffer[position++] = val >> 0 & 255; + buffer[position++] = val >> 8 & 255; + buffer[position++] = val >> 16 & 255; + buffer[position++] = val >> 24 & 255; + + buffer[position++] = val >> 32 & 255; + buffer[position++] = val >> 40 & 255; + buffer[position++] = val >> 48 & 255; + buffer[position] = val >> 56 & 255; +} + +std::vector ByteBuilder::build() { + return buffer; +} + +ByteReader::ByteReader(const ubyte* data, size_t size) + : data(data), size(size), pos(0) { +} + +ByteReader::ByteReader(const ubyte* data) + : data(data), size(4), pos(0) { + size = getInt32(); +} + +void ByteReader::checkMagic(const char* data, size_t size) { + if (pos + size >= this->size) { + throw std::runtime_error("invalid magic number"); + } + for (size_t i = 0; i < size; i++) { + if (this->data[pos + i] != (ubyte)data[i]){ + throw std::runtime_error("invalid magic number"); + } + } + pos += size; +} + +ubyte ByteReader::get() { + if (pos == size) { + throw std::underflow_error("buffer underflow"); + } + return data[pos++]; +} + +ubyte ByteReader::peek() { + if (pos == size) { + throw std::underflow_error("buffer underflow"); + } + return data[pos]; +} + +int16_t ByteReader::getInt16() { + if (pos+2 > size) { + throw std::underflow_error("unexpected end"); + } + pos += 2; + return (data[pos - 1] << 8) | + (data[pos - 2]); +} + +int32_t ByteReader::getInt32() { + if (pos+4 > size) { + throw std::underflow_error("unexpected end"); + } + pos += 4; + return (data[pos - 1] << 24) | + (data[pos - 2] << 16) | + (data[pos - 3] << 8) | + (data[pos - 4]); +} + +int64_t ByteReader::getInt64() { + if (pos+8 > size) { + throw std::underflow_error("unexpected end"); + } + pos += 8; + return ((int64_t)data[pos - 1] << 56) | + ((int64_t)data[pos - 2] << 48) | + ((int64_t)data[pos - 3] << 40) | + ((int64_t)data[pos - 4] << 32) | + ((int64_t)data[pos - 5] << 24) | + ((int64_t)data[pos - 6] << 16) | + ((int64_t)data[pos - 7] << 8) | + ((int64_t)data[pos - 8]); +} + +float ByteReader::getFloat32() { + union { + int32_t vali32; + float valfloat; + } value; + value.vali32 = getInt32(); + return value.valfloat; +} + +double ByteReader::getFloat64() { + union { + int64_t vali64; + double valfloat; + } value; + value.vali64 = getInt64(); + return value.valfloat; +} + +const char* ByteReader::getCString() { + const char* cstr = (const char*)(data+pos); + pos += strlen(cstr) + 1; + return cstr; +} + +std::string ByteReader::getString() { + uint32_t length = (uint32_t)getInt32(); + if (pos+length > size) { + throw std::underflow_error("unexpected end"); + } + pos += length; + return std::string((const char*)(data+pos-length), length); +} + +bool ByteReader::hasNext() const { + return pos < size; +} diff --git a/src/coders/byte_utils.h b/src/coders/byte_utils.h new file mode 100644 index 00000000..7bd2846d --- /dev/null +++ b/src/coders/byte_utils.h @@ -0,0 +1,76 @@ +#ifndef CODERS_BYTE_UTILS_H_ +#define CODERS_BYTE_UTILS_H_ + +#include +#include +#include "../typedefs.h" + +/* byteorder: little-endian */ +class ByteBuilder { + std::vector buffer; +public: + /* Write one byte (8 bit unsigned integer) */ + void put(ubyte b); + /* Write c-string (bytes array terminated with '\00') */ + void putCStr(const char* str); + /* Write signed 16 bit integer */ + void putInt16(int16_t val); + /* Write signed 32 bit integer */ + void putInt32(int32_t val); + /* Write signed 64 bit integer */ + void putInt64(int64_t val); + /* Write 32 bit floating-point number */ + void putFloat32(float val); + /* Write 64 bit floating-point number */ + void putFloat64(double val); + + /* Write string (uint32 length + bytes) */ + void put(const std::string& s); + /* Write sequence of bytes without any header */ + void put(const ubyte* arr, size_t size); + + void set(size_t position, ubyte val); + void setInt16(size_t position, int16_t val); + void setInt32(size_t position, int32_t val); + void setInt64(size_t position, int64_t val); + + inline size_t size() const { + return buffer.size(); + } + inline const ubyte* data() const { + return buffer.data(); + } + + std::vector build(); +}; + +/* byteorder: little-endian */ +class ByteReader { + const ubyte* data; + size_t size; + size_t pos; +public: + ByteReader(const ubyte* data, size_t size); + ByteReader(const ubyte* data); + + void checkMagic(const char* data, size_t size); + /* Read one byte (unsigned 8 bit integer) */ + ubyte get(); + /* Read one byte (unsigned 8 bit integer) without pointer move */ + ubyte peek(); + /* Read signed 16 bit integer */ + int16_t getInt16(); + /* Read signed 32 bit integer */ + int32_t getInt32(); + /* Read signed 64 bit integer */ + int64_t getInt64(); + /* Read 32 bit floating-point number */ + float getFloat32(); + /* Read 64 bit floating-point number */ + double getFloat64(); + const char* getCString(); + std::string getString(); + bool hasNext() const; +}; + +#endif // CODERS_BYTE_UTILS_H_ diff --git a/src/coders/commons.cpp b/src/coders/commons.cpp index ee7a6841..3a504f4b 100644 --- a/src/coders/commons.cpp +++ b/src/coders/commons.cpp @@ -1,10 +1,9 @@ #include "commons.h" #include +#include #include -using std::string; - inline int char2int(int c) { if (c >= '0' && c <= '9') { return c - '0'; @@ -26,23 +25,24 @@ inline double power(double base, int64_t power) { return result; } -parsing_error::parsing_error(string message, - string filename, - string source, - uint pos, - uint line, - uint linestart) +parsing_error::parsing_error( + std::string message, + std::string filename, + std::string source, + uint pos, + uint line, + uint linestart) : std::runtime_error(message), filename(filename), source(source), pos(pos), line(line), linestart(linestart) { } -string parsing_error::errorLog() const { +std::string parsing_error::errorLog() const { std::stringstream ss; uint linepos = pos - linestart; ss << "parsing error in file '" << filename; ss << "' at " << (line+1) << ":" << linepos << ": " << this->what() << "\n"; size_t end = source.find("\n", linestart); - if (end == string::npos) { + if (end == std::string::npos) { end = source.length(); } ss << source.substr(linestart, end-linestart) << "\n"; @@ -54,7 +54,7 @@ string parsing_error::errorLog() const { } -string escape_string(string s) { +std::string escape_string(std::string s) { std::stringstream ss; ss << '"'; for (char c : s) { @@ -124,7 +124,7 @@ char BasicParser::nextChar() { void BasicParser::expect(char expected) { char c = peek(); if (c != expected) { - throw error("'"+string({expected})+"' expected"); + throw error("'"+std::string({expected})+"' expected"); } pos++; } @@ -153,7 +153,7 @@ char BasicParser::peek() { return source[pos]; } -string BasicParser::parseName() { +std::string BasicParser::parseName() { char c = peek(); if (!is_identifier_start(c)) { if (c == '"') { @@ -262,7 +262,7 @@ bool BasicParser::parseNumber(int sign, number_u& out) { return true; } -string BasicParser::parseString(char quote, bool closeRequired) { +std::string BasicParser::parseString(char quote, bool closeRequired) { std::stringstream ss; while (hasNext()) { char c = source[pos]; @@ -290,7 +290,8 @@ string BasicParser::parseString(char quote, bool closeRequired) { case '/': ss << '/'; break; case '\n': pos++; continue; default: - throw error("'\\" + string({c}) + "' is an illegal escape"); + throw error("'\\" + std::string({c}) + + "' is an illegal escape"); } continue; } @@ -308,4 +309,4 @@ string BasicParser::parseString(char quote, bool closeRequired) { parsing_error BasicParser::error(std::string message) { return parsing_error(message, filename, source, pos, line, linestart); -} \ No newline at end of file +} diff --git a/src/coders/json.cpp b/src/coders/json.cpp index a4559771..64e38383 100644 --- a/src/coders/json.cpp +++ b/src/coders/json.cpp @@ -6,17 +6,28 @@ #include #include "commons.h" +#include "byte_utils.h" using namespace json; -using std::string; -using std::vector; -using std::unique_ptr; -using std::unordered_map; -using std::stringstream; -using std::make_pair; +const int BJSON_END = 0x0; +const int BJSON_TYPE_DOCUMENT = 0x1; +const int BJSON_TYPE_LIST = 0x2; +const int BJSON_TYPE_BYTE = 0x3; +const int BJSON_TYPE_INT16 = 0x4; +const int BJSON_TYPE_INT32 = 0x5; +const int BJSON_TYPE_INT64 = 0x6; +const int BJSON_TYPE_NUMBER = 0x7; +const int BJSON_TYPE_STRING = 0x8; +const int BJSON_TYPE_BYTES = 0x9; +const int BJSON_TYPE_FALSE = 0xA; +const int BJSON_TYPE_TRUE = 0xB; +const int BJSON_TYPE_NULL = 0xC; +const int BJSON_TYPE_CDOCUMENT = 0x1F; -inline void newline(stringstream& ss, bool nice, uint indent, const string indentstr) { +inline void newline(std::stringstream& ss, + bool nice, uint indent, + const std::string& indentstr) { if (nice) { ss << "\n"; for (uint i = 0; i < indent; i++) { @@ -27,29 +38,28 @@ inline void newline(stringstream& ss, bool nice, uint indent, const string inden } } -void stringify(Value* value, - stringstream& ss, +void stringify(const Value* value, + std::stringstream& ss, int indent, - string indentstr, + const std::string& indentstr, bool nice); -void stringifyObj(JObject* obj, - stringstream& ss, +void stringifyObj(const JObject* obj, + std::stringstream& ss, int indent, - string indentstr, + const std::string& indentstr, bool nice); -#include -void stringify(Value* value, - stringstream& ss, +void stringify(const Value* value, + std::stringstream& ss, int indent, - string indentstr, + const std::string& indentstr, bool nice) { if (value->type == valtype::object) { stringifyObj(value->value.obj, ss, indent, indentstr, nice); } else if (value->type == valtype::array) { - vector& list = value->value.arr->values; + std::vector& list = value->value.arr->values; if (list.empty()) { ss << "[]"; return; @@ -81,7 +91,11 @@ void stringify(Value* value, } } -void stringifyObj(JObject* obj, stringstream& ss, int indent, string indentstr, bool nice) { +void stringifyObj(const JObject* obj, + std::stringstream& ss, + int indent, + const std::string& indentstr, + bool nice) { if (obj->map.empty()) { ss << "{}"; return; @@ -107,12 +121,165 @@ void stringifyObj(JObject* obj, stringstream& ss, int indent, string indentstr, ss << '}'; } -string json::stringify(JObject* obj, bool nice, string indent) { - stringstream ss; +std::string json::stringify( + const JObject* obj, + bool nice, + const std::string& indent) { + std::stringstream ss; stringifyObj(obj, ss, 1, indent, nice); return ss.str(); } +static void to_binary(ByteBuilder& builder, const Value* value) { + switch (value->type) { + case valtype::object: { + std::vector bytes = to_binary(value->value.obj); + builder.put(bytes.data(), bytes.size()); + break; + } + case valtype::array: + builder.put(BJSON_TYPE_LIST); + for (Value* element : value->value.arr->values) { + to_binary(builder, element); + } + builder.put(BJSON_END); + break; + case valtype::integer: { + int64_t val = value->value.integer; + if (val >= 0 && val <= 255) { + builder.put(BJSON_TYPE_BYTE); + builder.put(val); + } else if (val >= INT16_MIN && val <= INT16_MAX){ + builder.put(BJSON_TYPE_INT16); + builder.putInt16(val); + } else if (val >= INT32_MIN && val <= INT32_MAX) { + builder.put(BJSON_TYPE_INT32); + builder.putInt32(val); + } else { + builder.put(BJSON_TYPE_INT64); + builder.putInt64(val); + } + break; + } + case valtype::number: + builder.put(BJSON_TYPE_NUMBER); + builder.putFloat64(value->value.decimal); + break; + case valtype::boolean: + builder.put(BJSON_TYPE_FALSE + value->value.boolean); + break; + case valtype::string: + builder.put(BJSON_TYPE_STRING); + builder.put(*value->value.str); + break; + } +} + +static JArray* array_from_binary(ByteReader& reader); +static JObject* object_from_binary(ByteReader& reader); + +std::vector json::to_binary(const JObject* obj) { + ByteBuilder builder; + // type byte + builder.put(BJSON_TYPE_DOCUMENT); + // document size + builder.putInt32(0); + + // writing entries + for (auto& entry : obj->map) { + builder.putCStr(entry.first.c_str()); + to_binary(builder, entry.second); + } + // terminating byte + builder.put(BJSON_END); + + // updating document size + builder.setInt32(1, builder.size()); + return builder.build(); +} + +static Value* value_from_binary(ByteReader& reader) { + ubyte typecode = reader.get(); + valtype type; + valvalue val; + switch (typecode) { + case BJSON_TYPE_DOCUMENT: + type = valtype::object; + reader.getInt32(); + val.obj = object_from_binary(reader); + break; + case BJSON_TYPE_LIST: + type = valtype::array; + val.arr = array_from_binary(reader); + break; + case BJSON_TYPE_BYTE: + type = valtype::integer; + val.integer = reader.get(); + break; + case BJSON_TYPE_INT16: + type = valtype::integer; + val.integer = reader.getInt16(); + break; + case BJSON_TYPE_INT32: + type = valtype::integer; + val.integer = reader.getInt32(); + break; + case BJSON_TYPE_INT64: + type = valtype::integer; + val.integer = reader.getInt64(); + break; + case BJSON_TYPE_NUMBER: + type = valtype::number; + val.decimal = reader.getFloat64(); + break; + case BJSON_TYPE_FALSE: + case BJSON_TYPE_TRUE: + type = valtype::boolean; + val.boolean = typecode - BJSON_TYPE_FALSE; + break; + case BJSON_TYPE_STRING: + type = valtype::string; + val.str = new std::string(reader.getString()); + break; + default: + throw std::runtime_error( + "type "+std::to_string(typecode)+" is not supported"); + } + return new Value(type, val); +} + +static JArray* array_from_binary(ByteReader& reader) { + auto array = std::make_unique(); + auto& items = array->values; + while (reader.peek() != BJSON_END) { + items.push_back(value_from_binary(reader)); + } + reader.get(); + return array.release(); +} + +static JObject* object_from_binary(ByteReader& reader) { + auto obj = std::make_unique(); + auto& map = obj->map; + while (reader.peek() != BJSON_END) { + const char* key = reader.getCString(); + Value* value = value_from_binary(reader); + map.insert(std::make_pair(key, value)); + } + reader.get(); + return obj.release(); +} + +JObject* json::from_binary(const ubyte* src, size_t size) { + ByteReader reader(src, size); + std::unique_ptr value (value_from_binary(reader)); + if (value->type != valtype::object) { + throw std::runtime_error("root value is not an object"); + } + JObject* obj = value->value.obj; + value->value.obj = nullptr; + return obj; +} JArray::~JArray() { for (auto value : values) { @@ -168,9 +335,9 @@ bool JArray::flag(size_t index) const { return values[index]->value.boolean; } -JArray& JArray::put(string value) { +JArray& JArray::put(std::string value) { valvalue val; - val.str = new string(value); + val.str = new std::string(value); values.push_back(new Value(valtype::string, val)); return *this; } @@ -248,11 +415,11 @@ JObject::~JObject() { } } -void JObject::str(string key, string& dst) const { +void JObject::str(std::string key, std::string& dst) const { dst = getStr(key, dst); } -string JObject::getStr(string key, const string& def) const { +std::string JObject::getStr(std::string key, const std::string& def) const { auto found = map.find(key); if (found == map.end()) return def; @@ -266,7 +433,7 @@ string JObject::getStr(string key, const string& def) const { } } -double JObject::getNum(string key, double def) const { +double JObject::getNum(std::string key, double def) const { auto found = map.find(key); if (found == map.end()) return def; @@ -280,7 +447,7 @@ double JObject::getNum(string key, double def) const { } } -int64_t JObject::getInteger(string key, int64_t def) const { +int64_t JObject::getInteger(std::string key, int64_t def) const { auto found = map.find(key); if (found == map.end()) return def; @@ -294,7 +461,7 @@ int64_t JObject::getInteger(string key, int64_t def) const { } } -void JObject::num(string key, double& dst) const { +void JObject::num(std::string key, double& dst) const { dst = getNum(key, dst); } @@ -346,93 +513,93 @@ void JObject::flag(std::string key, bool& dst) const { dst = found->second->value.boolean; } -JObject& JObject::put(string key, uint value) { +JObject& JObject::put(std::string key, uint value) { return put(key, (int64_t)value); } -JObject& JObject::put(string key, int value) { +JObject& JObject::put(std::string key, int value) { return put(key, (int64_t)value); } -JObject& JObject::put(string key, int64_t value) { +JObject& JObject::put(std::string key, int64_t value) { auto found = map.find(key); if (found != map.end()) found->second; valvalue val; val.integer = value; - map.insert(make_pair(key, new Value(valtype::integer, val))); + map.insert(std::make_pair(key, new Value(valtype::integer, val))); return *this; } -JObject& JObject::put(string key, uint64_t value) { +JObject& JObject::put(std::string key, uint64_t value) { return put(key, (int64_t)value); } -JObject& JObject::put(string key, float value) { +JObject& JObject::put(std::string key, float value) { return put(key, (double)value); } -JObject& JObject::put(string key, double value) { +JObject& JObject::put(std::string key, double value) { auto found = map.find(key); if (found != map.end()) delete found->second; valvalue val; val.decimal = value; - map.insert(make_pair(key, new Value(valtype::number, val))); + map.insert(std::make_pair(key, new Value(valtype::number, val))); return *this; } -JObject& JObject::put(string key, string value){ +JObject& JObject::put(std::string key, std::string value){ auto found = map.find(key); if (found != map.end()) delete found->second; valvalue val; - val.str = new string(value); - map.insert(make_pair(key, new Value(valtype::string, val))); + val.str = new std::string(value); + map.insert(std::make_pair(key, new Value(valtype::string, val))); return *this; } JObject& JObject::put(std::string key, const char* value) { - return put(key, string(value)); + return put(key, std::string(value)); } -JObject& JObject::put(string key, JObject* value){ +JObject& JObject::put(std::string key, JObject* value){ auto found = map.find(key); if (found != map.end()) delete found->second; valvalue val; val.obj = value; - map.insert(make_pair(key, new Value(valtype::object, val))); + map.insert(std::make_pair(key, new Value(valtype::object, val))); return *this; } -JObject& JObject::put(string key, JArray* value){ +JObject& JObject::put(std::string key, JArray* value){ auto found = map.find(key); if (found != map.end()) delete found->second; valvalue val; val.arr = value; - map.insert(make_pair(key, new Value(valtype::array, val))); + map.insert(std::make_pair(key, new Value(valtype::array, val))); return *this; } -JObject& JObject::put(string key, bool value){ +JObject& JObject::put(std::string key, bool value){ auto found = map.find(key); if (found != map.end()) delete found->second; valvalue val; val.boolean = value; - map.insert(make_pair(key, new Value(valtype::boolean, val))); + map.insert(std::make_pair(key, new Value(valtype::boolean, val))); return *this; } -JArray& JObject::putArray(string key) { +JArray& JObject::putArray(std::string key) { JArray* arr = new JArray(); put(key, arr); return *arr; } -JObject& JObject::putObj(string key) { +JObject& JObject::putObj(std::string key) { JObject* obj = new JObject(); put(key, obj); return *obj; } -bool JObject::has(string key) { +bool JObject::has(std::string key) { return map.find(key) != map.end(); } @@ -449,7 +616,8 @@ Value::~Value() { } } -Parser::Parser(string filename, string source) : BasicParser(filename, source) { +Parser::Parser(std::string filename, std::string source) + : BasicParser(filename, source) { } JObject* Parser::parse() { @@ -462,20 +630,20 @@ JObject* Parser::parse() { JObject* Parser::parseObject() { expect('{'); - unique_ptr obj(new JObject()); - unordered_map& map = obj->map; + auto obj = std::make_unique(); + auto& map = obj->map; while (peek() != '}') { if (peek() == '#') { skipLine(); continue; } - string key = parseName(); + std::string key = parseName(); char next = peek(); if (next != ':') { throw error("':' expected"); } pos++; - map.insert(make_pair(key, parseValue())); + map.insert(std::make_pair(key, parseValue())); next = peek(); if (next == ',') { pos++; @@ -491,8 +659,8 @@ JObject* Parser::parseObject() { JArray* Parser::parseArray() { expect('['); - unique_ptr arr(new JArray()); - vector& values = arr->values; + auto arr = std::make_unique(); + auto& values = arr->values; while (peek() != ']') { if (peek() == '#') { skipLine(); @@ -530,7 +698,7 @@ Value* Parser::parseValue() { return new Value(type, val); } if (is_identifier_start(next)) { - string literal = parseName(); + std::string literal = parseName(); if (literal == "true") { val.boolean = true; return new Value(valtype::boolean, val); @@ -568,17 +736,17 @@ Value* Parser::parseValue() { } if (next == '"' || next == '\'') { pos++; - val.str = new string(parseString(next)); + val.str = new std::string(parseString(next)); return new Value(valtype::string, val); } - throw error("unexpected character '"+string({next})+"'"); + throw error("unexpected character '"+std::string({next})+"'"); } -JObject* json::parse(string filename, string source) { +JObject* json::parse(std::string filename, std::string source) { Parser parser(filename, source); return parser.parse(); } -JObject* json::parse(string source) { +JObject* json::parse(std::string source) { return parse("", source); } diff --git a/src/coders/json.h b/src/coders/json.h index dd170864..c90e3314 100644 --- a/src/coders/json.h +++ b/src/coders/json.h @@ -16,7 +16,9 @@ namespace json { class JArray; class Value; - extern std::string stringify(JObject* obj, bool nice, std::string indent); + extern std::string stringify(const JObject* obj, bool nice, const std::string& indent); + extern std::vector to_binary(const JObject* obj); + extern JObject* from_binary(const ubyte* src, size_t size); enum class valtype { object, array, number, integer, string, boolean diff --git a/src/content/ContentLoader.cpp b/src/content/ContentLoader.cpp index 2f7d7b22..f0c803c6 100644 --- a/src/content/ContentLoader.cpp +++ b/src/content/ContentLoader.cpp @@ -87,7 +87,7 @@ void ContentLoader::fixPackIndices() { if (modified){ // rewrite modified json std::cout << indexFile << std::endl; - files::write_string(indexFile, json::stringify(root.get(), true, " ")); + files::write_json(indexFile, root.get()); } } diff --git a/src/files/WorldFiles.cpp b/src/files/WorldFiles.cpp index 3896a67e..84e857c0 100644 --- a/src/files/WorldFiles.cpp +++ b/src/files/WorldFiles.cpp @@ -1,7 +1,6 @@ #include "WorldFiles.h" #include "rle.h" -#include "binary_io.h" #include "../window/Camera.h" #include "../content/Content.h" #include "../objects/Player.h" @@ -477,7 +476,7 @@ void WorldFiles::writeIndices(const ContentIndices* indices) { items.put(def->name); } - files::write_string(getIndicesFile(), json::stringify(&root, true, " ")); + files::write_json(getIndicesFile(), &root); } void WorldFiles::writeWorldInfo(const World* world) { @@ -494,7 +493,7 @@ void WorldFiles::writeWorldInfo(const World* world) { timeobj.put("day-time", world->daytime); timeobj.put("day-time-speed", world->daytimeSpeed); - files::write_string(getWorldFile(), json::stringify(&root, true, " ")); + files::write_json(getWorldFile(), &root); } bool WorldFiles::readWorldInfo(World* world) { @@ -540,7 +539,7 @@ void WorldFiles::writePlayer(Player* player){ root.put("flight", player->flight); root.put("noclip", player->noclip); - files::write_string(getPlayerFile(), json::stringify(&root, true, " ")); + files::write_json(getPlayerFile(), &root); } bool WorldFiles::readPlayer(Player* player) { diff --git a/src/files/binary_io.cpp b/src/files/binary_io.cpp deleted file mode 100644 index c16083a3..00000000 --- a/src/files/binary_io.cpp +++ /dev/null @@ -1,165 +0,0 @@ -#include "binary_io.h" - -#include -#include -#include - -void BinaryWriter::put(ubyte b) { - buffer.push_back(b); -} - -void BinaryWriter::putCStr(const char* str) { - size_t size = strlen(str)+1; - buffer.reserve(buffer.size() + size); - for (size_t i = 0; i < size; i++) { - buffer.push_back(str[i]); - } -} - -void BinaryWriter::put(const std::string& s) { - size_t len = s.length(); - if (len > INT16_MAX) { - throw std::domain_error("length > INT16_MAX"); - } - putInt16(len); - put((const ubyte*)s.data(), len); -} - -void BinaryWriter::putShortStr(const std::string& s) { - size_t len = s.length(); - if (len > 255) { - throw std::domain_error("length > 255"); - } - put(len); - put((const ubyte*)s.data(), len); -} - -void BinaryWriter::put(const ubyte* arr, size_t size) { - buffer.reserve(buffer.size() + size); - for (size_t i = 0; i < size; i++) { - buffer.push_back(arr[i]); - } -} - -void BinaryWriter::putInt16(int16_t val) { - buffer.push_back((char) (val >> 8 & 255)); - buffer.push_back((char) (val >> 0 & 255)); -} - -void BinaryWriter::putInt32(int32_t val) { - buffer.reserve(buffer.size() + 4); - buffer.push_back((char) (val >> 24 & 255)); - buffer.push_back((char) (val >> 16 & 255)); - buffer.push_back((char) (val >> 8 & 255)); - buffer.push_back((char) (val >> 0 & 255)); -} - -void BinaryWriter::putInt64(int64_t val) { - buffer.reserve(buffer.size() + 8); - buffer.push_back((char) (val >> 56 & 255)); - buffer.push_back((char) (val >> 48 & 255)); - buffer.push_back((char) (val >> 40 & 255)); - buffer.push_back((char) (val >> 32 & 255)); - buffer.push_back((char) (val >> 24 & 255)); - buffer.push_back((char) (val >> 16 & 255)); - buffer.push_back((char) (val >> 8 & 255)); - buffer.push_back((char) (val >> 0 & 255)); -} - -void BinaryWriter::putFloat32(float val) { - union { - int32_t vali32; - float valfloat; - } value; - value.valfloat = val; - putInt32(value.vali32); -} - -BinaryReader::BinaryReader(const ubyte* data, size_t size) - : data(data), size(size), pos(0) { -} - -void BinaryReader::checkMagic(const char* data, size_t size) { - if (pos + size >= this->size) { - throw std::runtime_error("invalid magic number"); - } - for (size_t i = 0; i < size; i++) { - if (this->data[pos + i] != (ubyte)data[i]){ - throw std::runtime_error("invalid magic number"); - } - } - pos += size; -} - -ubyte BinaryReader::get() { - if (pos == size) { - throw std::underflow_error("buffer underflow"); - } - return data[pos++]; -} - -int16_t BinaryReader::getInt16() { - if (pos+2 > size) { - throw std::underflow_error("unexpected end"); - } - pos += 2; - return (data[pos - 2] << 8) | - (data[pos - 1]); -} - -int32_t BinaryReader::getInt32() { - if (pos+4 > size) { - throw std::underflow_error("unexpected end"); - } - pos += 4; - return (data[pos - 4] << 24) | - (data[pos - 3] << 16) | - (data[pos - 2] << 8) | - (data[pos - 1]); -} - -int64_t BinaryReader::getInt64() { - if (pos+8 > size) { - throw std::underflow_error("unexpected end"); - } - pos += 8; - return ((int64_t)data[pos - 8] << 56) | - ((int64_t)data[pos - 7] << 48) | - ((int64_t)data[pos - 6] << 40) | - ((int64_t)data[pos - 5] << 32) | - ((int64_t)data[pos - 4] << 24) | - ((int64_t)data[pos - 3] << 16) | - ((int64_t)data[pos - 2] << 8) | - ((int64_t)data[pos - 1]); -} - -float BinaryReader::getFloat32() { - union { - int32_t vali32; - float valfloat; - } value; - value.vali32 = getInt32(); - return value.valfloat; -} - -std::string BinaryReader::getString() { - uint16_t length = (uint16_t)getInt16(); - if (pos+length > size) { - throw std::underflow_error("unexpected end"); - } - pos += length; - return std::string((const char*)(data+pos-length), length); -} - -std::string BinaryReader::getShortString() { - ubyte length = get(); - if (pos+length > size) { - throw std::underflow_error("unexpected end"); - } - pos += length; - return std::string((const char*)(data+pos-length), length); -} - -bool BinaryReader::hasNext() const { - return pos < size; -} diff --git a/src/files/binary_io.h b/src/files/binary_io.h deleted file mode 100644 index ac7a7299..00000000 --- a/src/files/binary_io.h +++ /dev/null @@ -1,47 +0,0 @@ -#ifndef FILES_BINARY_WRITER_H_ -#define FILES_BINARY_WRITER_H_ - -#include -#include -#include "../typedefs.h" - -class BinaryWriter { - std::vector buffer; -public: - void put(ubyte b); - void putCStr(const char* str); - void putInt16(int16_t val); - void putInt32(int32_t val); - void putInt64(int64_t val); - void putFloat32(float val); - void put(const std::string& s); - void put(const ubyte* arr, size_t size); - void putShortStr(const std::string& s); - - inline size_t size() const { - return buffer.size(); - } - inline const ubyte* data() const { - return buffer.data(); - } -}; - -class BinaryReader { - const ubyte* data; - size_t size; - size_t pos; -public: - BinaryReader(const ubyte* data, size_t size); - - void checkMagic(const char* data, size_t size); - ubyte get(); - int16_t getInt16(); - int32_t getInt32(); - int64_t getInt64(); - float getFloat32(); - std::string getString(); - std::string getShortString(); - bool hasNext() const; -}; - -#endif // FILES_BINARY_WRITER_H_ \ No newline at end of file diff --git a/src/files/files.cpp b/src/files/files.cpp index 52eb925d..d2161b6c 100644 --- a/src/files/files.cpp +++ b/src/files/files.cpp @@ -92,16 +92,41 @@ bool files::write_string(fs::path filename, const std::string content) { return true; } -json::JObject* files::read_json(fs::path file) { - std::string text = files::read_string(file); +bool files::write_json(fs::path filename, const json::JObject* obj, bool nice) { + // -- binary json tests + //return write_binary_json(fs::path(filename.u8string()+".bin"), obj); + return files::write_string(filename, json::stringify(obj, nice, " ")); +} + +bool files::write_binary_json(fs::path filename, const json::JObject* obj) { + std::vector bytes = json::to_binary(obj); + return files::write_bytes(filename, (const char*)bytes.data(), bytes.size()); +} + +json::JObject* files::read_json(fs::path filename) { + // binary json tests + // fs::path binfile = fs::path(filename.u8string()+".bin"); + // if (fs::is_regular_file(binfile)){ + // return read_binary_json(binfile); + // } + + std::string text = files::read_string(filename); try { - return json::parse(file.string(), text); + auto obj = json::parse(filename.string(), text); + //write_binary_json(binfile, obj); + return obj; } catch (const parsing_error& error) { std::cerr << error.errorLog() << std::endl; - throw std::runtime_error("could not to parse "+file.string()); + throw std::runtime_error("could not to parse "+filename.string()); } } +json::JObject* files::read_binary_json(fs::path file) { + size_t size; + std::unique_ptr bytes (files::read_bytes(file, size)); + return json::from_binary((const ubyte*)bytes.get(), size); +} + std::vector files::read_list(fs::path filename) { std::ifstream file(filename); if (!file) { diff --git a/src/files/files.h b/src/files/files.h index ddffc814..2ad4a879 100644 --- a/src/files/files.h +++ b/src/files/files.h @@ -7,6 +7,8 @@ #include #include "../typedefs.h" +namespace fs = std::filesystem; + namespace json { class JObject; } @@ -25,14 +27,18 @@ namespace files { }; - extern bool write_bytes(std::filesystem::path, const char* data, size_t size); - extern uint append_bytes(std::filesystem::path, const char* data, size_t size); - extern bool read(std::filesystem::path, char* data, size_t size); - extern char* read_bytes(std::filesystem::path, size_t& length); - extern std::string read_string(std::filesystem::path filename); - extern bool write_string(std::filesystem::path filename, const std::string content); - extern json::JObject* read_json(std::filesystem::path file); - extern std::vector read_list(std::filesystem::path file); + extern bool write_bytes(fs::path, const char* data, size_t size); + extern uint append_bytes(fs::path, const char* data, size_t size); + extern bool write_string(fs::path filename, const std::string content); + extern bool write_json(fs::path filename, const json::JObject* obj, bool nice=true); + extern bool write_binary_json(fs::path filename, const json::JObject* obj); + + extern bool read(fs::path, char* data, size_t size); + extern char* read_bytes(fs::path, size_t& length); + extern std::string read_string(fs::path filename); + extern json::JObject* read_json(fs::path file); + extern json::JObject* read_binary_json(fs::path file); + extern std::vector read_list(fs::path file); } #endif /* FILES_FILES_H_ */ \ No newline at end of file diff --git a/src/logic/scripting/api_lua.cpp b/src/logic/scripting/api_lua.cpp index a73c966e..26ade8a6 100644 --- a/src/logic/scripting/api_lua.cpp +++ b/src/logic/scripting/api_lua.cpp @@ -14,6 +14,13 @@ #include "../../lighting/Lighting.h" #include "../../logic/BlocksController.h" +inline int lua_pushivec3(lua_State* L, int x, int y, int z) { + lua_pushinteger(L, x); + lua_pushinteger(L, y); + lua_pushinteger(L, z); + return 3; +} + inline void luaL_openlib(lua_State* L, const char* name, const luaL_Reg* libfuncs, int nup) { lua_newtable(L); luaL_setfuncs(L, libfuncs, nup); @@ -44,11 +51,6 @@ static const luaL_Reg worldlib [] = { {NULL, NULL} }; -int luaopen_world(lua_State* L) { - luaL_openlib(L, "world", worldlib, 0); - return 1; -} - /* == player library ==*/ static int l_player_get_pos(lua_State* L) { int playerid = lua_tointeger(L, 1); @@ -102,11 +104,6 @@ static const luaL_Reg playerlib [] = { {NULL, NULL} }; -int luaopen_player(lua_State* L) { - luaL_openlib(L, "player", playerlib, 0); - return 1; -} - /* == blocks-related functions == */ static int l_block_name(lua_State* L) { int id = lua_tointeger(L, 1); @@ -158,13 +155,6 @@ static int l_get_block(lua_State* L) { return 1; } -inline int lua_pushivec3(lua_State* L, int x, int y, int z) { - lua_pushinteger(L, x); - lua_pushinteger(L, y); - lua_pushinteger(L, z); - return 3; -} - static int l_get_block_x(lua_State* L) { int x = lua_tointeger(L, 1); int y = lua_tointeger(L, 2); @@ -275,8 +265,8 @@ static int l_is_replaceable_at(lua_State* L) { lua_setglobal(L, NAME)) void apilua::create_funcs(lua_State* L) { - luaopen_world(L); - luaopen_player(L); + luaL_openlib(L, "world", worldlib, 0); + luaL_openlib(L, "player", playerlib, 0); lua_addfunc(L, l_block_index, "block_index"); lua_addfunc(L, l_block_name, "block_name");