mysql-server/sql-common/json_diff.cc

/* Copyright (c) 2017, 2024, Oracle and/or its affiliates.

   This program is free software; you can redistribute it and/or modify
   it under the terms of the GNU General Public License, version 2.0,
   as published by the Free Software Foundation.

   This program is designed to work with certain software (including
   but not limited to OpenSSL) that is licensed under separate terms,
   as designated in a particular file or component or in included license
   documentation.  The authors of MySQL hereby grant you an additional
   permission to link the program and your derivative works with the
   separately licensed software that they have either included with
   the program or referenced in the documentation.

   This program is distributed in the hope that it will be useful,
   but WITHOUT ANY WARRANTY; without even the implied warranty of
   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
   GNU General Public License, version 2.0, for more details.

   You should have received a copy of the GNU General Public License
   along with this program; if not, write to the Free Software
   Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301  USA */

#include "sql-common/json_diff.h"

#include <cassert>
#include <optional>
#include <utility>

#include "my_alloc.h"
#include "my_byteorder.h"
#include "my_dbug.h"  // assert
#include "my_inttypes.h"
#include "my_sys.h"
#include "mysql/components/services/bits/psi_bits.h"
#include "mysql_com.h"
#include "mysqld_error.h"
#include "sql-common/json_binary.h"
#include "sql-common/json_dom.h"  // Json_dom, Json_wrapper
#include "sql-common/json_error_handler.h"
#include "sql-common/json_path.h"  // Json_path
#include "sql/current_thd.h"       // current_thd
#include "sql/log_event.h"         // net_field_length_checked
#include "sql/psi_memory_key.h"
#include "sql/sql_class.h"
#include "sql/sql_const.h"
#include "sql/system_variables.h"
#include "sql/table.h"
#include "sql_string.h"  // StringBuffer
#include "template_utils.h"

using std::nullopt;
using std::optional;

// Define constructor and destructor here instead of in the header file, to
// avoid dependencies on psi_memory_key.h and json_dom.h from the header file.

Json_diff::Json_diff(const Json_seekable_path &path,
                     enum_json_diff_operation operation,
                     std::unique_ptr<Json_dom> value)
    : m_path(key_memory_JSON),
      m_operation(operation),
      m_value(std::move(value)) {
  for (const Json_path_leg *leg : path) m_path.append(*leg);
}

Json_diff::~Json_diff() = default;

Json_wrapper Json_diff::value() const {
  Json_wrapper result(m_value.get());
  result.set_alias();
  return result;
}

/**
  Return the total size of a data field, plus the size of the
  preceding integer that describes the length, when the integer is
  stored in net_field_length() format

  @param length The length of the data
  @return The length of the data plus the length of the length field.
*/
static size_t length_of_length_and_string(size_t length) {
  return length + net_length_size(length);
}

/**
  Encode a String as (length, data) pair, with length being stored in
  net_field_length() format.

  @param to Buffer where length and data will be stored.
  @param from Source string containing the data.
  @return true on out of memory, false on success.
*/
static bool write_length_and_string(String *to, const String &from) {
  // Serialize length.
  size_t length = from.length();
  DBUG_EXECUTE_IF("binlog_corrupt_write_length_and_string_bad_length", {
    DBUG_SET("-d,binlog_corrupt_write_length_and_string_bad_length");
    length = 1 << 30;
  });
  char length_buf[9];
  const size_t length_length =
      net_store_length((uchar *)length_buf, length) - (uchar *)length_buf;
  DBUG_PRINT("info", ("write_length_and_string: length=%lu length_length=%lu",
                      (unsigned long)length, (unsigned long)length_length));
  DBUG_EXECUTE_IF(
      "binlog_corrupt_write_length_and_string_truncate_before_string", {
        DBUG_SET(
            "-d,binlog_corrupt_write_length_and_string_truncate_before_string");
        return false;
      });
  DBUG_EXECUTE_IF("binlog_corrupt_write_length_and_string_bad_char", {
    DBUG_SET("-d,binlog_corrupt_write_length_and_string_bad_char");
    // Instead of "some text", write "\xffsome tex"
    // This is sure to corrupt both JSON paths and
    // binary JSON.
    return to->append(length_buf, length_length) ||
           to->append(static_cast<char>(0xff)) ||
           to->append(from.ptr(), from.length() - 1);
  });
  // Allocate memory and append
  return to->append(length_buf, length_length) || to->append(from);
}

size_t Json_diff::binary_length() const {
  DBUG_TRACE;

  // operation
  size_t ret = ENCODED_OPERATION_BYTES;

  /*
    It would be better to compute length without serializing the path
    and json.  And given that we serialize the path and json, it would
    be better if we dealt with out-of-memory errors in a better way.

    In the future, we should remove the need to pre-compute the size.
    Currently this is only needed by the binlog writer.  And it would
    be better to rewrite the binlog writer so that it streams rows
    directly to the thread caches instead of storing them in memory.

    And currently, this function is used from
    Row_data_memory::max_row_length, and the return value of that
    function is used in Row_data_memory::allocate_memory, which
    doesn't check out-of-memory conditions at all and might just
    dereference nullptr in case of out of memory.  So these asserts do
    not make the situation worse.
  */
  StringBuffer<STRING_BUFFER_USUAL_SIZE> buf;

  // path
  if (m_path.to_string(&buf)) assert(0); /* purecov: deadcode */
  ret += length_of_length_and_string(buf.length());

  if (operation() != enum_json_diff_operation::REMOVE) {
    // value
    buf.length(0);
    const THD *const thd = current_thd;
    if (value().to_binary(JsonSerializationDefaultErrorHandler(thd), &buf))
      assert(0); /* purecov: deadcode */
    if (buf.length() > thd->variables.max_allowed_packet) {
      my_error(ER_WARN_ALLOWED_PACKET_OVERFLOWED, MYF(0),
               "json_binary::serialize", thd->variables.max_allowed_packet);
      assert(0);
    }
    ret += length_of_length_and_string(buf.length());
  }

  return ret;
}

bool Json_diff::write_binary(String *to) const {
  DBUG_TRACE;

  // Serialize operation
  char operation = (char)m_operation;
  DBUG_EXECUTE_IF("binlog_corrupt_json_diff_bad_op", {
    DBUG_SET("-d,binlog_corrupt_json_diff_bad_op");
    operation = 127;
  });
  if (to->append(&operation, ENCODED_OPERATION_BYTES))
    return true; /* purecov: inspected */  // OOM, error is reported
  DBUG_PRINT("info", ("wrote JSON operation=%d", (int)operation));

  /**
    @todo This first serializes in one buffer and then copies to
    another buffer.  It would be better if we could write directly to
    the output and save a round of memory allocation + copy. /Sven
  */

  // Serialize JSON path
  StringBuffer<STRING_BUFFER_USUAL_SIZE> buf;
#ifndef NDEBUG
  bool return_early = false;
  DBUG_EXECUTE_IF("binlog_corrupt_json_diff_truncate_before_path_length", {
    DBUG_SET("-d,binlog_corrupt_json_diff_truncate_before_path_length");
    return false;
  });
  DBUG_EXECUTE_IF("binlog_corrupt_json_diff_bad_path_length", {
    DBUG_SET("-d,binlog_corrupt_json_diff_bad_path_length");
    DBUG_SET("+d,binlog_corrupt_write_length_and_string_bad_length");
  });
  DBUG_EXECUTE_IF("binlog_corrupt_json_diff_truncate_before_path", {
    DBUG_SET("-d,binlog_corrupt_json_diff_truncate_before_path");
    DBUG_SET(
        "+d,binlog_corrupt_write_length_and_string_truncate_before_string");
    return_early = true;
  });
  DBUG_EXECUTE_IF("binlog_corrupt_json_diff_bad_path_char", {
    DBUG_SET("-d,binlog_corrupt_json_diff_bad_path_char");
    DBUG_SET("+d,binlog_corrupt_write_length_and_string_bad_char");
  });
#endif  // ifndef NDEBUG
  if (m_path.to_string(&buf) || write_length_and_string(to, buf))
    return true; /* purecov: inspected */  // OOM, error is reported
#ifndef NDEBUG
  if (return_early) return false;
#endif
  DBUG_PRINT("info", ("wrote JSON path '%s' of length %lu", buf.ptr(),
                      (unsigned long)buf.length()));

  if (m_operation != enum_json_diff_operation::REMOVE) {
    // Serialize JSON value
    buf.length(0);
#ifndef NDEBUG
    DBUG_EXECUTE_IF("binlog_corrupt_json_diff_truncate_before_doc_length", {
      DBUG_SET("-d,binlog_corrupt_json_diff_truncate_before_doc_length");
      return false;
    });
    DBUG_EXECUTE_IF("binlog_corrupt_json_diff_bad_doc_length", {
      DBUG_SET("-d,binlog_corrupt_json_diff_bad_doc_length");
      DBUG_SET("+d,binlog_corrupt_write_length_and_string_bad_length");
    });
    DBUG_EXECUTE_IF("binlog_corrupt_json_diff_truncate_before_doc", {
      DBUG_SET("-d,binlog_corrupt_json_diff_truncate_before_doc");
      DBUG_SET(
          "+d,binlog_corrupt_write_length_and_string_truncate_before_string");
    });
    DBUG_EXECUTE_IF("binlog_corrupt_json_diff_bad_doc_char", {
      DBUG_SET("-d,binlog_corrupt_json_diff_bad_doc_char");
      DBUG_SET("+d,binlog_corrupt_write_length_and_string_bad_char");
    });
#endif  // ifndef NDEBUG
    const THD *const thd = current_thd;
    if (value().to_binary(JsonSerializationDefaultErrorHandler(thd), &buf) ||
        write_length_and_string(to, buf)) {
      return true; /* purecov: inspected */  // OOM, error is reported
    }
    if (buf.length() > thd->variables.max_allowed_packet) {
      my_error(ER_WARN_ALLOWED_PACKET_OVERFLOWED, MYF(0),
               "json_binary::serialize", thd->variables.max_allowed_packet);
      return true;
    }
    DBUG_PRINT("info",
               ("wrote JSON value of length %lu", (unsigned long)buf.length()));
  }

  return false;
}

Json_diff_vector::Json_diff_vector(allocator_type arg)
    : m_vector(std::vector<Json_diff, allocator_type>(arg)),
      m_binary_length(0) {}

static MEM_ROOT empty_json_diff_vector_mem_root(PSI_NOT_INSTRUMENTED, 256);
const Json_diff_vector Json_diff_vector::EMPTY_JSON_DIFF_VECTOR{
    Json_diff_vector::allocator_type{&empty_json_diff_vector_mem_root}};

void Json_diff_vector::add_diff(Json_diff diff) {
  m_vector.push_back(std::move(diff));
  m_binary_length += m_vector.back().binary_length();
}

void Json_diff_vector::add_diff(const Json_seekable_path &path,
                                enum_json_diff_operation operation,
                                std::unique_ptr<Json_dom> dom) {
  add_diff({path, operation, std::move(dom)});
}

void Json_diff_vector::add_diff(const Json_seekable_path &path,
                                enum_json_diff_operation operation) {
  add_diff({path, operation, /*value=*/nullptr});
}

void Json_diff_vector::clear() {
  m_vector.clear();
  m_binary_length = 0;
}

size_t Json_diff_vector::binary_length(bool include_metadata) const {
  return m_binary_length + (include_metadata ? ENCODED_LENGTH_BYTES : 0);
}

bool Json_diff_vector::write_binary(String *to) const {
  // Insert placeholder where we will store the length, once that is known.
  char length_buf[ENCODED_LENGTH_BYTES] = {0, 0, 0, 0};
  if (to->append(length_buf, ENCODED_LENGTH_BYTES))
    return true; /* purecov: inspected */  // OOM, error is reported

  // Store all the diffs.
  for (const Json_diff &diff : *this)
    if (diff.write_binary(to))
      return true; /* purecov: inspected */  // OOM, error is reported

  // Store the length.
  const size_t length = to->length() - ENCODED_LENGTH_BYTES;
  int4store(to->ptr(), (uint32)length);

  DBUG_PRINT("info", ("Wrote JSON diff vector length %lu=%02x %02x %02x %02x",
                      (unsigned long)length, length_buf[0], length_buf[1],
                      length_buf[2], length_buf[3]));

  return false;
}

bool Json_diff_vector::read_binary(const char **from, const TABLE *table,
                                   const char *field_name) {
  DBUG_TRACE;
  const uchar *p = pointer_cast<const uchar *>(*from);

  // Caller should have validated that the buffer is least 4 + length
  // bytes long.
  size_t length = uint4korr(p);
  p += 4;

  DBUG_PRINT("info", ("length=%zu p=%p", length, p));

  while (length > 0) {
    DBUG_PRINT(
        "info",
        ("length=%zu bytes remaining to decode into Json_diff_vector", length));

    optional<ReadJsonDiffResult> result = read_json_diff(p, length);
    if (!result.has_value()) {
      if (current_thd->is_error()) return true;
      my_error(ER_CORRUPTED_JSON_DIFF, MYF(0),
               static_cast<int>(table->s->table_name.length),
               table->s->table_name.str, field_name);
      return true;
    }

    auto &[diff, bytes_read] = result.value();
#ifndef NDEBUG
    if (Json_wrapper wrapper = diff.value(); !wrapper.empty()) {
      wrapper.dbug_print("", JsonDepthErrorHandler);
    }
#endif

    p += bytes_read;
    length -= bytes_read;
    add_diff(std::move(diff));
  }

  *from = pointer_cast<const char *>(p);
  return false;
}

optional<ReadJsonDiffResult> read_json_diff(const unsigned char *pos,
                                            size_t length) {
  const unsigned char *const start = pos;
  const unsigned char *const end = pos + length;

  // Read operation.
  if (pos >= end) return nullopt;
  const int operation_number = *pos;
  DBUG_PRINT("info", ("operation_number=%d", operation_number));
  if (operation_number >= JSON_DIFF_OPERATION_COUNT) return nullopt;
  const enum_json_diff_operation operation =
      static_cast<enum_json_diff_operation>(operation_number);
  ++pos;

  // Read path length.
  size_t path_length;
  if (size_t max_length = end - pos;
      net_field_length_checked<size_t>(&pos, &max_length, &path_length) ||
      path_length > max_length) {
    return nullopt;
  }

  // Read path.
  Json_path path{key_memory_JSON};
  DBUG_PRINT("info", ("path='%.*s'", static_cast<int>(path_length), pos));
  if (size_t bad_index; parse_path(path_length, pointer_cast<const char *>(pos),
                                   &path, &bad_index)) {
    return nullopt;
  }
  pos += path_length;

  if (operation == enum_json_diff_operation::REMOVE) {
    return ReadJsonDiffResult{{path, operation, /*value=*/nullptr},
                              static_cast<size_t>(pos - start)};
  }

  // Read value length.
  size_t value_length;
  if (size_t max_length = end - pos;
      net_field_length_checked<size_t>(&pos, &max_length, &value_length) ||
      value_length > max_length) {
    return std::nullopt;
  }

  // Read value.
  json_binary::Value value =
      json_binary::parse_binary(pointer_cast<const char *>(pos), value_length);
  if (value.type() == json_binary::Value::ERROR) return nullopt;
  std::unique_ptr<Json_dom> dom = Json_dom::parse(value);
  if (dom == nullptr) return nullopt;
  pos += value_length;

  return ReadJsonDiffResult{{path, operation, std::move(dom)},
                            static_cast<size_t>(pos - start)};
}

/**
  Find the value at the specified path in a JSON DOM. The path should
  not contain any wildcard or ellipsis, only simple array cells or
  member names. Auto-wrapping is not performed.

  @param dom        the root of the DOM
  @param first_leg  the first path leg
  @param last_leg   the last path leg (exclusive)
  @return the JSON DOM at the given path, or `nullptr` if the path is not found
 */
static Json_dom *seek_exact_path(Json_dom *dom,
                                 const Json_path_iterator &first_leg,
                                 const Json_path_iterator &last_leg) {
  for (auto it = first_leg; it != last_leg; ++it) {
    const Json_path_leg *leg = *it;
    const auto leg_type = leg->get_type();
    assert(leg_type == jpl_member || leg_type == jpl_array_cell);
    switch (dom->json_type()) {
      case enum_json_type::J_ARRAY: {
        const auto array = down_cast<Json_array *>(dom);
        if (leg_type != jpl_array_cell) return nullptr;
        Json_array_index idx = leg->first_array_index(array->size());
        if (!idx.within_bounds()) return nullptr;
        dom = (*array)[idx.position()];
        continue;
      }
      case enum_json_type::J_OBJECT: {
        const auto object = down_cast<Json_object *>(dom);
        if (leg_type != jpl_member) return nullptr;
        dom = object->get(leg->get_member_name());
        if (dom == nullptr) return nullptr;
        continue;
      }
      default:
        return nullptr;
    }
  }

  return dom;
}

enum_json_diff_status apply_json_diff(const Json_diff &diff, Json_dom *dom) {
  Json_wrapper val_to_apply = diff.value();
  const Json_path &path = diff.path();

  switch (diff.operation()) {
    case enum_json_diff_operation::REPLACE: {
      assert(path.leg_count() > 0);
      Json_dom *old = seek_exact_path(dom, path.begin(), path.end());
      if (old == nullptr) return enum_json_diff_status::REJECTED;
      assert(old->parent() != nullptr);
      old->parent()->replace_dom_in_container(old, val_to_apply.clone_dom());
      return enum_json_diff_status::SUCCESS;
    }
    case enum_json_diff_operation::INSERT: {
      assert(path.leg_count() > 0);
      Json_dom *parent = seek_exact_path(dom, path.begin(), path.end() - 1);
      if (parent == nullptr) return enum_json_diff_status::REJECTED;
      const Json_path_leg *last_leg = path.last_leg();
      if (parent->json_type() == enum_json_type::J_OBJECT &&
          last_leg->get_type() == jpl_member) {
        auto obj = down_cast<Json_object *>(parent);
        if (obj->get(last_leg->get_member_name()) != nullptr)
          return enum_json_diff_status::REJECTED;
        if (obj->add_alias(last_leg->get_member_name(),
                           val_to_apply.clone_dom()))
          return enum_json_diff_status::ERROR; /* purecov: inspected */
        return enum_json_diff_status::SUCCESS;
      }
      if (parent->json_type() == enum_json_type::J_ARRAY &&
          last_leg->get_type() == jpl_array_cell) {
        auto array = down_cast<Json_array *>(parent);
        Json_array_index idx = last_leg->first_array_index(array->size());
        if (array->insert_alias(idx.position(), val_to_apply.clone_dom()))
          return enum_json_diff_status::ERROR; /* purecov: inspected */
        return enum_json_diff_status::SUCCESS;
      }
      return enum_json_diff_status::REJECTED;
    }
    case enum_json_diff_operation::REMOVE: {
      assert(path.leg_count() > 0);
      Json_dom *parent = seek_exact_path(dom, path.begin(), path.end() - 1);
      if (parent == nullptr) return enum_json_diff_status::REJECTED;
      const Json_path_leg *last_leg = path.last_leg();
      if (parent->json_type() == enum_json_type::J_OBJECT) {
        auto object = down_cast<Json_object *>(parent);
        if (last_leg->get_type() != jpl_member ||
            !object->remove(last_leg->get_member_name()))
          return enum_json_diff_status::REJECTED;
      } else if (parent->json_type() == enum_json_type::J_ARRAY) {
        if (last_leg->get_type() != jpl_array_cell)
          return enum_json_diff_status::REJECTED;
        auto array = down_cast<Json_array *>(parent);
        Json_array_index idx = last_leg->first_array_index(array->size());
        if (!idx.within_bounds() || !array->remove(idx.position()))
          return enum_json_diff_status::REJECTED;
      } else {
        return enum_json_diff_status::REJECTED;
      }
      return enum_json_diff_status::SUCCESS;
    }
  }

  /* purecov: begin deadcode */
  assert(false);
  return enum_json_diff_status::ERROR;
  /* purecov: end */
}