diff --git a/be/src/io/file_factory.cpp b/be/src/io/file_factory.cpp index c17742d9e2eb27..bb0eade505758b 100644 --- a/be/src/io/file_factory.cpp +++ b/be/src/io/file_factory.cpp @@ -45,6 +45,7 @@ #include "io/fs/s3_file_writer.h" #include "io/fs/stream_load_pipe.h" #include "io/hdfs_builder.h" +#include "io/hdfs_tvf_policy.h" #include "io/hdfs_util.h" #include "load/stream_load/new_load_stream_mgr.h" #include "load/stream_load/stream_load_context.h" @@ -204,7 +205,8 @@ Result FileFactory::create_file_writer( THdfsParams hdfs_params = parse_properties(properties); std::shared_ptr handler; RETURN_IF_ERROR_RESULT(ExecEnv::GetInstance()->hdfs_mgr()->get_or_create_fs( - hdfs_params, hdfs_params.fs_name, &handler)); + hdfs_params, hdfs_params.fs_name, &handler, + io::convert_path(path, hdfs_params.fs_name).native())); return io::HdfsFileWriter::create(path, handler, hdfs_params.fs_name, &options); } default: @@ -261,9 +263,11 @@ Result FileFactory::_create_file_reader_internal( fs_name = &system_properties.hdfs_params.fs_name; } RETURN_IF_ERROR_RESULT(ExecEnv::GetInstance()->hdfs_mgr()->get_or_create_fs( - system_properties.hdfs_params, *fs_name, &handler)); + system_properties.hdfs_params, *fs_name, &handler, + io::convert_path(file_description.path, *fs_name).native())); // The reader's path drops the name node, so the cache key takes it from here. - auto options = with_storage_in_cache_key(reader_options, *fs_name); + auto options = with_storage_in_cache_key( + reader_options, io::HdfsTvfPolicy::storage_cache_identity(*handler, *fs_name)); return io::HdfsFileReader::create(file_description.path, handler->hdfs_fs, *fs_name, options) .and_then([&options](auto&& reader) { diff --git a/be/src/io/fs/hdfs/hdfs_mgr.cpp b/be/src/io/fs/hdfs/hdfs_mgr.cpp index f4c97fe5b535d6..e274c98ac96a83 100644 --- a/be/src/io/fs/hdfs/hdfs_mgr.cpp +++ b/be/src/io/fs/hdfs/hdfs_mgr.cpp @@ -18,8 +18,9 @@ #include "io/fs/hdfs/hdfs_mgr.h" #include -#include +#include +#include #include #include @@ -28,9 +29,11 @@ #include "core/string_ref.h" #include "io/fs/err_utils.h" #include "io/hdfs_builder.h" +#include "io/hdfs_tvf_policy.h" #include "io/hdfs_util.h" #include "runtime/exec_env.h" #include "runtime/thread_context.h" +#include "util/jni-util.h" #include "util/jvm_launcher.h" #include "util/thread.h" @@ -73,10 +76,10 @@ void HdfsMgr::_cleanup_loop() { std::vector> handlers_to_cleanup; { std::lock_guard lock(_mutex); - std::vector to_remove; // Find expired handlers - for (const auto& entry : _fs_handlers) { + for (auto it = _fs_handlers.begin(); it != _fs_handlers.end();) { + const auto& entry = *it; bool is_expired = current_time - entry.second->last_access_time >= _instance_timeout_seconds; // bool is_krb_expired = @@ -91,15 +94,12 @@ void HdfsMgr::_cleanup_loop() { << ", fs_name=" << entry.second->fs_name << ", is_expired=" << is_expired; // << ", is_krb_expire=" << is_krb_expired; - to_remove.push_back(entry.first); handlers_to_cleanup.push_back(entry.second); + it = _fs_handlers.erase(it); + } else { + ++it; } } - - // Remove expired handlers from map under lock - for (uint64_t hash_code : to_remove) { - _fs_handlers.erase(hash_code); - } } // Cleanup handlers outside lock @@ -128,56 +128,87 @@ void HdfsMgr::_cleanup_loop() { } Status HdfsMgr::get_or_create_fs(const THdfsParams& hdfs_params, const std::string& fs_name, - std::shared_ptr* fs_handler) { + std::shared_ptr* fs_handler, + const std::string& native_path) { + bool guarded = HdfsTvfPolicy::requested(hdfs_params); + if (guarded && native_path.empty()) { + return Status::InvalidArgument("TVF Hadoop IO requires its actual native path"); + } uint64_t hash_code = _hdfs_hash_code(hdfs_params, fs_name); - - // First check without lock - { - std::lock_guard lock(_mutex); - auto it = _fs_handlers.find(hash_code); - if (it != _fs_handlers.end()) { - LOG(INFO) << "Reuse existing HDFS handler, hash_code=" << hash_code - << ", is_kerberos=" << it->second->is_kerberos_auth - << ", principal=" << it->second->principal << ", fs_name=" << fs_name; - it->second->update_access_time(); - *fs_handler = it->second; + auto matches_policy = [&](const std::shared_ptr& handler) { + return guarded ? handler->tvf_params && *handler->tvf_params == hdfs_params && + handler->fs_name == fs_name + : !handler->tvf_params; + }; + auto validate = [&](const std::shared_ptr& handler) { + if (!guarded) { return Status::OK(); } - } - - // Create new hdfsFS handler outside the lock - LOG(INFO) << "Start to create new HDFS handler, hash_code=" << hash_code - << ", fs_name=" << fs_name; - - std::shared_ptr new_fs_handler; - RETURN_IF_ERROR(_create_hdfs_fs(hdfs_params, fs_name, &new_fs_handler)); - - // Double check with lock before inserting + auto check = [&] { return _check_tvf_path_impl(*handler, fs_name, native_path); }; + // A direct remote provider never dispatches by path; avoid a pthread per cached file. + if (handler->tvf_context && !handler->tvf_context->requires_jni()) { + return check(); + } + return HdfsTvfPolicy::on_pthread(check); + }; + std::shared_ptr selected; { - std::lock_guard lock(_mutex); - auto it = _fs_handlers.find(hash_code); - if (it != _fs_handlers.end()) { - // Another thread has created the handler, use it instead - LOG(INFO) << "Another thread created HDFS handler, reuse it, hash_code=" << hash_code - << ", is_kerberos=" << it->second->is_kerberos_auth - << ", principal=" << it->second->principal << ", fs_name=" << fs_name; - it->second->update_access_time(); - *fs_handler = it->second; - return Status::OK(); + std::scoped_lock lock(_mutex); + auto [first, last] = _fs_handlers.equal_range(hash_code); + auto it = std::find_if(first, last, + [&](const auto& entry) { return matches_policy(entry.second); }); + if (it != last) { + selected = it->second; } - - // Store the new handler - *fs_handler = new_fs_handler; - _fs_handlers[hash_code] = new_fs_handler; - - LOG(INFO) << "Finished create new HDFS handler, hash_code=" << hash_code - << ", is_kerberos=" << new_fs_handler->is_kerberos_auth - << ", principal=" << new_fs_handler->principal << ", fs_name=" << fs_name; } - + if (!selected) { + THdfsParams creation = hdfs_params; + if (guarded) { + // Internal metadata is attached to the private connection request, never the cache key. + for (const auto& entry : {std::make_pair(HdfsTvfPolicy::ROOT_KEY, fs_name), + std::make_pair(HdfsTvfPolicy::PATH_KEY, native_path)}) { + THdfsConf conf; + conf.__set_key(entry.first); + conf.__set_value(entry.second); + creation.hdfs_conf.push_back(std::move(conf)); + } + creation.__isset.hdfs_conf = true; + } + std::shared_ptr created; + RETURN_IF_ERROR(_create_hdfs_fs(creation, fs_name, &created)); + if (guarded) { + created->tvf_params = std::make_shared(hdfs_params); + } + RETURN_IF_ERROR(validate(created)); + { + std::scoped_lock lock(_mutex); + auto [first, last] = _fs_handlers.equal_range(hash_code); + auto it = std::find_if(first, last, + [&](const auto& entry) { return matches_policy(entry.second); }); + if (it != last) { + selected = it->second; + } else { + selected = created; + // Keep exact identities in the same hash bucket, including CRC collisions. + // Each remains reusable and cleanup erases only the expired entry. + _fs_handlers.emplace(hash_code, created); + } + } + } + RETURN_IF_ERROR(validate(selected)); + selected->update_access_time(); + *fs_handler = std::move(selected); return Status::OK(); } +Status HdfsMgr::_check_tvf_path_impl(const HdfsHandler& handler, const std::string& fs_name, + const std::string& native_path) { + if (!handler.tvf_context) { + return Status::NotSupported("TVF Hadoop connection has no runtime configuration context"); + } + return handler.tvf_context->check(fs_name, native_path); +} + Status HdfsMgr::_create_hdfs_fs_impl(const THdfsParams& hdfs_params, const std::string& fs_name, std::shared_ptr* fs_handler) { // Right next to the connect, because the connect is the thing it guards: hdfsBuilderConnect() @@ -192,19 +223,35 @@ Status HdfsMgr::_create_hdfs_fs_impl(const THdfsParams& hdfs_params, const std:: // // No thread switch needed: _create_hdfs_fs() has put us on a pthread already. RETURN_IF_ERROR(Jni::JvmLauncher::ensure_jvm()); + bool guarded = HdfsTvfPolicy::requested(hdfs_params); + if (guarded) { + RETURN_IF_ERROR(HdfsTvfPolicy::prepare_runtime()); + } HDFSCommonBuilder builder; RETURN_IF_ERROR(create_hdfs_builder(hdfs_params, fs_name, &builder)); // release(), not get(): hdfsBuilderConnect() frees the builder it is handed, so this is // where ownership leaves HDFSCommonBuilder - whose destructor frees whatever it still owns. hdfsFS hdfs_fs = hdfsBuilderConnect(builder.release()); if (hdfs_fs == nullptr) { + if (guarded) { + RETURN_IF_ERROR(HdfsTvfPolicy::clear_runtime()); + } return Status::InternalError("failed to connect to hdfs {}: {}", fs_name, hdfs_error()); } + std::shared_ptr context; + if (guarded) { + Status captured = HdfsTvfPolicy::capture(fs_name, &context); + if (!captured.ok()) { + hdfsDisconnect(hdfs_fs); + return captured; + } + } bool is_kerberos = builder.is_kerberos(); *fs_handler = std::make_shared( hdfs_fs, is_kerberos, is_kerberos ? hdfs_params.hdfs_kerberos_principal : "", is_kerberos ? hdfs_params.hdfs_kerberos_keytab : "", fs_name); + (*fs_handler)->tvf_context = std::move(context); // builder.get_ticket_cache()); return Status::OK(); } @@ -214,42 +261,8 @@ Status HdfsMgr::_create_hdfs_fs_impl(const THdfsParams& hdfs_params, const std:: // bthreads so create a pthread for creating hdfs connection if necessary. Status HdfsMgr::_create_hdfs_fs(const THdfsParams& hdfs_params, const std::string& fs_name, std::shared_ptr* fs_handler) { - bool is_pthread = bthread_self() == 0; - LOG(INFO) << "create hdfs fs, is_pthread=" << is_pthread << " fs_name=" << fs_name; - if (is_pthread) { // running in pthread - return _create_hdfs_fs_impl(hdfs_params, fs_name, fs_handler); - } - - // running in bthread, switch to a pthread and wait - Status st; - auto btx = bthread::butex_create(); - *(int*)btx = 0; - std::thread t([&] { - // Named and given a thread context like every other pthread the BE starts. This one is - // where the JVM is most often created for the first time: _create_hdfs_fs_impl() calls - // ensure_jvm(), and ensure_jvm() only switches to a pthread of its own when it is called - // FROM a bthread - which it is not here, because this switch already happened. Without - // these two lines the whole bootstrap - seconds of hadoop ServiceLoader scanning, and any - // hang inside it - runs on an anonymous thread with no memory tracking, and an operator - // looking at the stacks has nothing to grep for. - SCOPED_INIT_THREAD_CONTEXT(); - Thread::set_self_name("hdfs_fs_connect"); - st = _create_hdfs_fs_impl(hdfs_params, fs_name, fs_handler); - *(int*)btx = 1; - bthread::butex_wake_all(btx); - }); - std::unique_ptr> defer((int*)0x01, [&t, &btx](...) { - if (t.joinable()) t.join(); - bthread::butex_destroy(btx); - }); - timespec tmout {.tv_sec = std::chrono::system_clock::now().time_since_epoch().count() + 60, - .tv_nsec = 0}; - if (int ret = bthread::butex_wait(btx, 1, &tmout); ret != 0) { - std::string msg = "failed to wait create_hdfs_fs finish. fs_name=" + fs_name; - LOG(WARNING) << msg << " error=" << std::strerror(errno); - st = Status::Error(msg); - } - return st; + return HdfsTvfPolicy::on_pthread( + [&] { return _create_hdfs_fs_impl(hdfs_params, fs_name, fs_handler); }); } uint64_t HdfsMgr::_hdfs_hash_code(const THdfsParams& hdfs_params, const std::string& fs_name) { diff --git a/be/src/io/fs/hdfs/hdfs_mgr.h b/be/src/io/fs/hdfs/hdfs_mgr.h index 579f6efcc32484..eb06d9e3211814 100644 --- a/be/src/io/fs/hdfs/hdfs_mgr.h +++ b/be/src/io/fs/hdfs/hdfs_mgr.h @@ -39,7 +39,8 @@ class HdfsMgr { // Get or create a hdfsFS instance based on the given parameters Status get_or_create_fs(const THdfsParams& hdfs_params, const std::string& fs_name, - std::shared_ptr* fs_handler); + std::shared_ptr* fs_handler, + const std::string& native_path = ""); virtual ~HdfsMgr(); @@ -58,6 +59,9 @@ class HdfsMgr { } uint64_t _hdfs_hash_code(const THdfsParams& hdfs_params, const std::string& fs_name); + virtual Status _check_tvf_path_impl(const HdfsHandler& handler, const std::string& fs_name, + const std::string& native_path); + virtual Status _create_hdfs_fs_impl(const THdfsParams& hdfs_params, const std::string& fs_name, std::shared_ptr* fs_handler); @@ -79,7 +83,7 @@ class HdfsMgr { private: std::mutex _mutex; - std::unordered_map> _fs_handlers; + std::unordered_multimap> _fs_handlers; std::atomic _should_stop_cleanup_thread; std::unique_ptr _cleanup_thread; diff --git a/be/src/io/fs/local_file_system.cpp b/be/src/io/fs/local_file_system.cpp index 421fb71bbbdd51..6fad144d195b11 100644 --- a/be/src/io/fs/local_file_system.cpp +++ b/be/src/io/fs/local_file_system.cpp @@ -471,14 +471,27 @@ const std::shared_ptr& global_local_filesystem() { Status LocalFileSystem::canonicalize_local_file(const std::string& dir, const std::string& file_path, std::string* full_path) { - const std::string absolute_path = dir + "/" + file_path; + // An empty root would turn a relative request into an unrestricted absolute path. + if (dir.empty()) { + return Status::InvalidArgument("user_files_secure_path must not be empty"); + } + // Reject embedded NULs before passing paths to filesystem APIs that use C strings. + if (file_path.empty() || file_path.find('\0') != std::string::npos) { + return Status::InvalidArgument("local file path must not be empty or contain NUL"); + } + Path local_path; + // Convert file: URIs to filesystem paths and reject remote schemes before any reader opens them. + RETURN_IF_ERROR(convert_to_abs_path(file_path, local_path)); + std::string secure_path; + RETURN_IF_ERROR(canonicalize(dir, &secure_path)); std::string canonical_path; - RETURN_IF_ERROR(canonicalize(absolute_path, &canonical_path)); - if (!contain_path(dir, canonical_path)) { - return Status::InvalidArgument("file path is not allowed: {}", canonical_path); + const auto status = canonicalize(Path(secure_path) / local_path, &canonical_path); + // A glob result can reveal external names even when resolution fails on a dangling symlink. + // Reject both failed resolution and out-of-root targets without exposing expanded names. + if (!status.ok() || !contain_path(secure_path, canonical_path)) { + return Status::InvalidArgument("file path is not allowed"); } - - *full_path = canonical_path; + *full_path = std::move(canonical_path); return Status::OK(); } @@ -486,13 +499,29 @@ Status LocalFileSystem::safe_glob(const std::string& path, std::vector if (path.find("..") != std::string::npos) { return Status::InvalidArgument("can not contain '..' in path"); } - std::string full_path = config::user_files_secure_path + "/" + path; + // An empty root would turn the prefix below into "/" and allow any local file. + if (config::user_files_secure_path.empty()) { + return Status::InvalidArgument("user_files_secure_path must not be empty"); + } + std::string secure_path; + RETURN_IF_ERROR(canonicalize(config::user_files_secure_path, &secure_path)); + // A symlink root may canonicalize to a directory such as "user_files[prod]". + // The root is literal; only the user-supplied suffix may contain glob operators. + std::string full_path; + for (char c : secure_path) { + if (c == '\\' || c == '*' || c == '?' || c == '[' || c == ']') { + full_path += '\\'; + } + full_path += c; + } + full_path += "/" + path; std::vector files; RETURN_IF_ERROR(_glob(full_path, &files)); for (auto& file : files) { FileInfo fi; fi.is_file = true; - RETURN_IF_ERROR(canonicalize_local_file("", file, &(fi.file_name))); + // Glob classes and symlinks can escape without a literal ".." in the pattern. + RETURN_IF_ERROR(canonicalize_local_file(secure_path, file, &(fi.file_name))); RETURN_IF_ERROR(file_size_impl(fi.file_name, &(fi.file_size))); res->push_back(std::move(fi)); } diff --git a/be/src/io/fs/local_file_system.h b/be/src/io/fs/local_file_system.h index b381e960a7bdba..1c23612f8e282c 100644 --- a/be/src/io/fs/local_file_system.h +++ b/be/src/io/fs/local_file_system.h @@ -75,6 +75,8 @@ class LocalFileSystem final : public FileSystem { static std::filesystem::perms PERMS_OWNER_RW; + // Resolve a local path relative to a nonempty secure root, then check canonical containment. + // Absolute paths and file: URIs must resolve inside the same root; remote schemes are rejected. Status canonicalize_local_file(const std::string& dir, const std::string& file_path, std::string* full_path); diff --git a/be/src/io/hdfs_tvf_policy.cpp b/be/src/io/hdfs_tvf_policy.cpp new file mode 100644 index 00000000000000..5e4fe3c7f07a31 --- /dev/null +++ b/be/src/io/hdfs_tvf_policy.cpp @@ -0,0 +1,217 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "io/hdfs_tvf_policy.h" + +#include +#include +#include + +#include +#include + +#include "io/hdfs_util.h" +#include "runtime/thread_context.h" +#include "util/thread.h" +#ifndef USE_LIBHDFS3 +#include "util/jni-util.h" +#endif + +namespace doris::io { + +// Hadoop JNI must execute on an OS thread, including checks on cached connections. +Status HdfsTvfPolicy::on_pthread(const std::function& operation) { + if (bthread_self() == 0) { + return operation(); + } + Status status; + bthread::CountdownEvent complete(1); + std::thread worker([&] { + SCOPED_INIT_THREAD_CONTEXT(); + Thread::set_self_name("hdfs_tvf_io"); + status = operation(); + complete.signal(); + }); + complete.wait(); + worker.join(); + return status; +} + +bool HdfsTvfPolicy::requested(const THdfsParams& params) { + return std::ranges::any_of(params.hdfs_conf, [](const auto& conf) { + return conf.key == POLICY_KEY && (conf.value == "native" || conf.value == "native-write"); + }); +} + +#ifndef USE_LIBHDFS3 +struct HdfsTvfPolicy::Impl { + Jni::GlobalObject context; + Jni::MethodId check_path; + bool direct_remote = false; + std::string profile; + std::string fs_name; +}; + +Status HdfsTvfPolicy::clear_runtime() { + JNIEnv* env = nullptr; + RETURN_IF_ERROR(Jni::Env::Get(&env)); + Jni::LocalClass guard; + RETURN_IF_ERROR( + Jni::LocalClass::find_class(env, "org/apache/hadoop/fs/FileSystem$TvfGuard", &guard)); + Jni::MethodId clear; + RETURN_IF_ERROR(guard.get_static_method(env, "clearNativeContext", "()V", &clear)); + return guard.call_static_void_method(env, clear).call(); +} + +Status HdfsTvfPolicy::prepare_runtime() { + JNIEnv* env = nullptr; + RETURN_IF_ERROR(Jni::Env::Get(&env)); + jclass clazz = env->FindClass("org/apache/hadoop/fs/FileSystem"); + if (env->ExceptionCheck()) { + env->ExceptionClear(); + } + if (clazz == nullptr) { + return Status::NotSupported("TVF requires the patched Hadoop FileSystem runtime"); + } + jmethodID method = env->GetStaticMethodID(clazz, "getDorisTvfGuardVersion", "()I"); + if (env->ExceptionCheck()) { + env->ExceptionClear(); + } + if (method == nullptr) { + env->DeleteLocalRef(clazz); + return Status::NotSupported("TVF requires the patched Hadoop FileSystem runtime"); + } + jint version = env->CallStaticIntMethod(clazz, method); + bool failed = env->ExceptionCheck(); + if (failed) { + env->ExceptionClear(); + } + env->DeleteLocalRef(clazz); + if (failed || version != 1) { + return Status::NotSupported("TVF Hadoop runtime policy version mismatch"); + } + return clear_runtime(); +} + +Status HdfsTvfPolicy::capture(const std::string& fs_name, std::shared_ptr* result) { + JNIEnv* env = nullptr; + RETURN_IF_ERROR(Jni::Env::Get(&env)); + Jni::LocalClass guard; + RETURN_IF_ERROR( + Jni::LocalClass::find_class(env, "org/apache/hadoop/fs/FileSystem$TvfGuard", &guard)); + Jni::MethodId take; + RETURN_IF_ERROR(guard.get_static_method(env, "takeNativeContext", + "()Lorg/apache/hadoop/fs/FileSystem$TvfGuard$Context;", + &take)); + Jni::LocalObject context; + RETURN_IF_ERROR(guard.call_static_object_method(env, take).call(&context)); + if (context.uninitialized()) { + return Status::NotSupported("Hadoop runtime did not capture the TVF configuration"); + } + auto impl = std::make_unique(); + RETURN_IF_ERROR(Jni::GlobalObject::create(env, context, &impl->context)); + Jni::LocalClass context_class; + RETURN_IF_ERROR(Jni::LocalClass::find_class( + env, "org/apache/hadoop/fs/FileSystem$TvfGuard$Context", &context_class)); + RETURN_IF_ERROR(context_class.get_method( + env, "checkNative", "(Ljava/lang/String;Ljava/lang/String;)V", &impl->check_path)); + Jni::MethodId direct; + RETURN_IF_ERROR(context_class.get_method(env, "isDirectRemote", "()Z", &direct)); + jboolean direct_remote; + RETURN_IF_ERROR(context.call_boolean_method(env, direct).call(&direct_remote)); + impl->direct_remote = direct_remote; + impl->fs_name = fs_name; + Jni::MethodId profile; + RETURN_IF_ERROR(context_class.get_method(env, "profile", "()Ljava/lang/String;", &profile)); + Jni::LocalString text; + RETURN_IF_ERROR(context.call_object_method(env, profile).call(&text)); + const char* chars = env->GetStringUTFChars(static_cast(text.get()), nullptr); + RETURN_ERROR_IF_EXC(env); + impl->profile = chars; + env->ReleaseStringUTFChars(static_cast(text.get()), chars); + *result = std::shared_ptr(new HdfsTvfPolicy(std::move(impl))); + return Status::OK(); +} + +bool HdfsTvfPolicy::requires_jni() const { + return !_impl->direct_remote; +} + +Status HdfsTvfPolicy::check(const std::string& fs_name, const std::string& native_path) { + if (fs_name != _impl->fs_name) { + return Status::InvalidArgument("TVF Hadoop connection identity mismatch"); + } + if (_impl->direct_remote) { + return Status::OK(); + } + // libhdfs consumes C strings, so bytes after the first NUL are not part of its path. + const auto path = native_path.substr(0, native_path.find('\0')); + JNIEnv* env = nullptr; + RETURN_IF_ERROR(Jni::Env::Get(&env)); + Jni::LocalString root; + Jni::LocalString file; + RETURN_IF_ERROR(Jni::LocalString::new_string(env, fs_name.c_str(), &root)); + RETURN_IF_ERROR(Jni::LocalString::new_string(env, path.c_str(), &file)); + RETURN_IF_ERROR(_impl->context.call_void_method(env, _impl->check_path) + .with_arg(root) + .with_arg(file) + .call()); + return Status::OK(); +} +#else +struct HdfsTvfPolicy::Impl { + std::string profile; +}; +Status HdfsTvfPolicy::prepare_runtime() { + return Status::NotSupported("Hadoop JVM is not enabled"); +} +Status HdfsTvfPolicy::clear_runtime() { + return Status::OK(); +} +Status HdfsTvfPolicy::capture(const std::string&, std::shared_ptr*) { + return Status::NotSupported("Hadoop JVM is not enabled"); +} +bool HdfsTvfPolicy::requires_jni() const { + return false; +} +Status HdfsTvfPolicy::check(const std::string&, const std::string&) { + return Status::OK(); +} +#endif + +std::string HdfsTvfPolicy::storage_cache_identity(const HdfsHandler& handler, + const std::string& fs_name) { + if (!handler.tvf_params) { + return fs_name; + } + return "tvf:" + (handler.tvf_context ? handler.tvf_context->_impl->profile : "native-hdfs3") + + ":" + fs_name; +} + +HdfsTvfPolicy::HdfsTvfPolicy(std::unique_ptr impl) : _impl(std::move(impl)) {} +HdfsTvfPolicy::~HdfsTvfPolicy() { + // A collision-only connection can lose its last reader on a bthread. Release its Java + // global reference on an OS thread, just like creation and route checks. + if (_impl) { + static_cast(on_pthread([&] { + _impl.reset(); + return Status::OK(); + })); + } +} + +} // namespace doris::io diff --git a/be/src/io/hdfs_tvf_policy.h b/be/src/io/hdfs_tvf_policy.h new file mode 100644 index 00000000000000..e9523905b7d4e4 --- /dev/null +++ b/be/src/io/hdfs_tvf_policy.h @@ -0,0 +1,55 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include +#include + +#include "common/status.h" + +namespace doris { +class THdfsParams; +namespace io { +class HdfsHandler; + +// Owns the private Java configuration used to initialize one native Hadoop connection. +class HdfsTvfPolicy { +public: + static constexpr const char* POLICY_KEY = "fs.doris.tvf.remote.paths"; + static constexpr const char* ROOT_KEY = "fs.doris.tvf.root"; + static constexpr const char* PATH_KEY = "fs.doris.tvf.path"; + static Status on_pthread(const std::function& operation); + static bool requested(const THdfsParams& params); + static Status prepare_runtime(); + static Status clear_runtime(); + static Status capture(const std::string& fs_name, std::shared_ptr* result); + static std::string storage_cache_identity(const HdfsHandler& handler, + const std::string& fs_name); + + ~HdfsTvfPolicy(); + bool requires_jni() const; + Status check(const std::string& fs_name, const std::string& native_path); + +private: + struct Impl; + explicit HdfsTvfPolicy(std::unique_ptr impl); + std::unique_ptr _impl; +}; +} // namespace io +} // namespace doris diff --git a/be/src/io/hdfs_util.h b/be/src/io/hdfs_util.h index d7a74134db64ec..dbd9867c3cc357 100644 --- a/be/src/io/hdfs_util.h +++ b/be/src/io/hdfs_util.h @@ -40,6 +40,8 @@ class THdfsParams; namespace io { +class HdfsTvfPolicy; + class HdfsHandler { public: hdfsFS hdfs_fs; @@ -47,6 +49,8 @@ class HdfsHandler { std::string principal; std::string keytab_path; std::string fs_name; + std::shared_ptr tvf_params; + std::shared_ptr tvf_context; uint64_t create_time; std::atomic last_access_time; // std::shared_ptr ticket_cache; diff --git a/be/test/io/fs/hdfs/hdfs_mgr_test.cpp b/be/test/io/fs/hdfs/hdfs_mgr_test.cpp index ab4f3ac576a6c7..bd7666119ba548 100644 --- a/be/test/io/fs/hdfs/hdfs_mgr_test.cpp +++ b/be/test/io/fs/hdfs/hdfs_mgr_test.cpp @@ -20,6 +20,7 @@ #include #include +#include #include #include @@ -65,6 +66,9 @@ int MockKerberosTicketCache::instance_count = 0; // Mock HdfsMgr class to override the actual HDFS operations class MockHdfsMgr : public HdfsMgr { public: + MOCK_METHOD(Status, _check_tvf_path_impl, + (const HdfsHandler&, const std::string&, const std::string&), (override)); + MockHdfsMgr() : HdfsMgr() {} MOCK_METHOD(Status, _create_hdfs_fs_impl, @@ -97,6 +101,8 @@ class HdfsMgrTest : public testing::Test { // Reset MockKerberosTicketCache instance count MockKerberosTicketCache::reset_instance_count(); + ON_CALL(*_hdfs_mgr, _check_tvf_path_impl(_, _, _)).WillByDefault(Return(Status::OK())); + // Setup default mock behavior ON_CALL(*_hdfs_mgr, _create_hdfs_fs_impl(_, _, _)) .WillByDefault([](const THdfsParams& params, const std::string& fs_name, @@ -108,6 +114,35 @@ class HdfsMgrTest : public testing::Test { void TearDown() override { _hdfs_mgr.reset(); } + void seed_collision(const THdfsParams& params, const std::string& fs_name, + const std::shared_ptr& handler) { + std::scoped_lock lock(_hdfs_mgr->_mutex); + _hdfs_mgr->_fs_handlers.clear(); + _hdfs_mgr->_fs_handlers.emplace(_hdfs_mgr->_hdfs_hash_code(params, fs_name), handler); + } + + bool contains_handler(const THdfsParams& params, const std::string& fs_name, + const std::shared_ptr& handler) { + std::scoped_lock lock(_hdfs_mgr->_mutex); + auto [first, last] = + _hdfs_mgr->_fs_handlers.equal_range(_hdfs_mgr->_hdfs_hash_code(params, fs_name)); + return std::any_of(first, last, [&](const auto& entry) { return entry.second == handler; }); + } + + void set_timeout_locked(int64_t seconds) { + std::scoped_lock lock(_hdfs_mgr->_mutex); + _hdfs_mgr->set_instance_timeout_seconds(seconds); + } + + THdfsParams guarded_params(const std::string& policy) { + THdfsParams params; + THdfsConf conf; + conf.__set_key("fs.doris.tvf.remote.paths"); + conf.__set_value(policy); + params.__set_hdfs_conf({conf}); + return params; + } + THdfsParams create_test_params(const std::string& user = "", const std::string& principal = "", const std::string& keytab = "") { THdfsParams params; @@ -379,4 +414,134 @@ TEST_F(HdfsMgrTest, ConcurrentAccess) { // ASSERT_EQ(MockKerberosTicketCache::get_instance_count(), 0); // } +TEST_F(HdfsMgrTest, TvfPolicyAndAuthorityMustMatchAtFirstLookup) { + for (int scenario = 0; scenario < 3; ++scenario) { + THdfsParams params = guarded_params("native"); + auto wrong = std::make_shared(nullptr, false, "", "", "viewfs://cluster"); + if (scenario != 0) { + auto stored = params; + if (scenario == 1) { + stored.__set_user("different-user"); + } + wrong->tvf_params = std::make_shared(stored); + } + if (scenario == 2) { + wrong->fs_name = "viewfs://other"; + } + seed_collision(params, "viewfs://cluster", wrong); + EXPECT_CALL(*_hdfs_mgr, _create_hdfs_fs_impl(_, _, _)).Times(1); + std::shared_ptr actual; + ASSERT_TRUE(_hdfs_mgr->get_or_create_fs(params, "viewfs://cluster", &actual, "/data/a.csv") + .ok()); + EXPECT_NE(actual, wrong); + ASSERT_TRUE(actual->tvf_params); + EXPECT_EQ(*actual->tvf_params, params); + EXPECT_EQ(actual->fs_name, "viewfs://cluster"); + testing::Mock::VerifyAndClearExpectations(_hdfs_mgr.get()); + } +} + +TEST_F(HdfsMgrTest, TvfCollisionAtSecondLookupCannotReplaceOrReuseAnotherPolicy) { + THdfsParams params = guarded_params("native"); + auto wrong = std::make_shared(nullptr, false, "", "", "viewfs://other"); + wrong->tvf_params = std::make_shared(params); + auto fresh = std::make_shared(nullptr, false, "", "", "viewfs://cluster"); + EXPECT_CALL(*_hdfs_mgr, _create_hdfs_fs_impl(_, _, _)) + .WillOnce( + [&](const THdfsParams&, const std::string&, std::shared_ptr* out) { + seed_collision(params, "viewfs://cluster", wrong); + *out = fresh; + return Status::OK(); + }); + std::shared_ptr actual; + ASSERT_TRUE( + _hdfs_mgr->get_or_create_fs(params, "viewfs://cluster", &actual, "/data/a.csv").ok()); + EXPECT_EQ(actual, fresh); + ASSERT_TRUE(actual->tvf_params); + EXPECT_EQ(*actual->tvf_params, params); + EXPECT_TRUE(contains_handler(params, "viewfs://cluster", wrong)); + EXPECT_TRUE(contains_handler(params, "viewfs://cluster", actual)); + testing::Mock::VerifyAndClearExpectations(_hdfs_mgr.get()); + EXPECT_CALL(*_hdfs_mgr, _create_hdfs_fs_impl(_, _, _)).Times(0); + std::shared_ptr reused; + ASSERT_TRUE( + _hdfs_mgr->get_or_create_fs(params, "viewfs://cluster", &reused, "/data/b.csv").ok()); + EXPECT_EQ(actual, reused); + EXPECT_EQ(wrong->fs_name, "viewfs://other"); +} + +TEST_F(HdfsMgrTest, IdenticalTvfPolicyAndAuthorityKeepCacheReuse) { + THdfsParams params = guarded_params("native"); + EXPECT_CALL(*_hdfs_mgr, _create_hdfs_fs_impl(_, _, _)).Times(1); + std::shared_ptr first; + std::shared_ptr second; + ASSERT_TRUE( + _hdfs_mgr->get_or_create_fs(params, "viewfs://cluster", &first, "/data/a.csv").ok()); + ASSERT_TRUE( + _hdfs_mgr->get_or_create_fs(params, "viewfs://cluster", &second, "/data/b.csv").ok()); + EXPECT_EQ(first, second); +} + +TEST_F(HdfsMgrTest, CachedTvfConnectionStillRejectsLocalActualPath) { + auto params = guarded_params("native"); + std::shared_ptr remote; + ASSERT_TRUE( + _hdfs_mgr->get_or_create_fs(params, "viewfs://cluster", &remote, "/data/a.csv").ok()); + EXPECT_CALL(*_hdfs_mgr, _create_hdfs_fs_impl(_, _, _)).Times(0); + EXPECT_CALL(*_hdfs_mgr, _check_tvf_path_impl(_, "viewfs://cluster", "/local/b.csv")) + .WillOnce(Return(Status::InvalidArgument("local route"))); + std::shared_ptr local; + EXPECT_FALSE( + _hdfs_mgr->get_or_create_fs(params, "viewfs://cluster", &local, "/local/b.csv").ok()); + EXPECT_FALSE(local); + EXPECT_FALSE(_hdfs_mgr->get_or_create_fs(params, "viewfs://cluster", &local).ok()); +} + +TEST_F(HdfsMgrTest, ExpiredCollisionDoesNotEvictLiveTvfConnection) { + auto params = guarded_params("native"); + set_timeout_locked(100); + auto expired = std::make_shared(nullptr, false, "", "", "viewfs://other"); + expired->tvf_params = std::make_shared(params); + expired->last_access_time = 0; + seed_collision(params, "viewfs://cluster", expired); + std::shared_ptr live; + ASSERT_TRUE(_hdfs_mgr->get_or_create_fs(params, "viewfs://cluster", &live, "/data/a.csv").ok()); + std::this_thread::sleep_for(std::chrono::seconds(2)); + EXPECT_FALSE(contains_handler(params, "viewfs://cluster", expired)); + EXPECT_TRUE(contains_handler(params, "viewfs://cluster", live)); + EXPECT_CALL(*_hdfs_mgr, _create_hdfs_fs_impl(_, _, _)).Times(0); + std::shared_ptr reused; + ASSERT_TRUE( + _hdfs_mgr->get_or_create_fs(params, "viewfs://cluster", &reused, "/data/b.csv").ok()); + EXPECT_EQ(live, reused); +} + +TEST_F(HdfsMgrTest, NativeWritePolicyIsGuardedAndCannotReuseReadContext) { + auto write_params = guarded_params("native-write"); + auto read = std::make_shared(nullptr, false, "", "", "viewfs://cluster"); + read->tvf_params = std::make_shared(guarded_params("native")); + seed_collision(write_params, "viewfs://cluster", read); + EXPECT_CALL(*_hdfs_mgr, _create_hdfs_fs_impl(_, _, _)).Times(1); + std::shared_ptr write; + ASSERT_TRUE( + _hdfs_mgr->get_or_create_fs(write_params, "viewfs://cluster", &write, "/data/prefix_") + .ok()); + EXPECT_NE(read, write); + ASSERT_TRUE(write->tvf_params); + EXPECT_EQ(write_params, *write->tvf_params); + std::shared_ptr reused; + ASSERT_TRUE( + _hdfs_mgr->get_or_create_fs(write_params, "viewfs://cluster", &reused, "/data/next_") + .ok()); + EXPECT_EQ(write, reused); + EXPECT_CALL(*_hdfs_mgr, _check_tvf_path_impl(_, "viewfs://cluster", "/local/leaf")) + .WillOnce(Return(Status::InvalidArgument("local create parent"))); + std::shared_ptr denied; + EXPECT_FALSE( + _hdfs_mgr->get_or_create_fs(write_params, "viewfs://cluster", &denied, "/local/leaf") + .ok()); + EXPECT_FALSE(denied); + EXPECT_FALSE(_hdfs_mgr->get_or_create_fs(write_params, "viewfs://cluster", &denied).ok()); +} + } // namespace doris::io diff --git a/be/test/io/fs/local_file_system_test.cpp b/be/test/io/fs/local_file_system_test.cpp index a3b5549f98e893..985f9626c0b888 100644 --- a/be/test/io/fs/local_file_system_test.cpp +++ b/be/test/io/fs/local_file_system_test.cpp @@ -31,12 +31,14 @@ #include #include +#include "common/config.h" #include "common/status.h" #include "cpp/sync_point.h" #include "gtest/gtest_pred_impl.h" #include "io/fs/file_reader.h" #include "io/fs/file_writer.h" #include "io/fs/local_file_writer.h" +#include "util/defer_op.h" #include "util/slice.h" namespace doris { @@ -568,6 +570,10 @@ TEST_F(LocalFileSystemTest, AbnormalWriteRead) { } TEST_F(LocalFileSystemTest, TestGlob) { + const std::string original_secure_path = config::user_files_secure_path; + DEFER(config::user_files_secure_path = original_secure_path); + // Bind this fixture to its own files, independent of the runner's build-profile directory. + config::user_files_secure_path = std::filesystem::absolute("./be/ut_build_ASAN/test").string(); std::string path = "./be/ut_build_ASAN/test/file_path/"; EXPECT_TRUE(io::global_local_filesystem()->delete_directory(path).ok()); EXPECT_TRUE(io::global_local_filesystem() @@ -599,6 +605,112 @@ TEST_F(LocalFileSystemTest, TestGlob) { EXPECT_TRUE(io::global_local_filesystem()->delete_directory(path).ok()); } +TEST_F(LocalFileSystemTest, TestSafeGlobPathContainment) { + auto fs = io::global_local_filesystem(); + const std::string original_secure_path = config::user_files_secure_path; + DEFER(config::user_files_secure_path = original_secure_path); + const auto root = std::filesystem::canonical(test_dir); + const auto secure = root / "secure"; + const auto outside = root / "secure-other"; + std::filesystem::create_directories(secure / "nested"); + std::filesystem::create_directory(outside); + ASSERT_TRUE(save_string_file((secure / "nested/inside.csv").string(), "inside").ok()); + ASSERT_TRUE(save_string_file((outside / "outside.csv").string(), "outside").ok()); + std::filesystem::create_directory_symlink(outside, secure / "escape"); + std::filesystem::create_symlink(outside / "outside.csv", secure / "outside.csv"); + std::filesystem::create_symlink(outside / "missing.csv", outside / "broken.csv"); + std::filesystem::create_directory_symlink(secure / "nested", secure / "inside"); + config::user_files_secure_path = secure.string(); + + for (const auto* pattern : {".[.]/secure-other/outside.csv", ".?/secure-other/outside.csv", + ".[.-.]/secure-other/outside.csv", "escape/*.csv", "outside.csv", + ".[.]/secure-other/*.csv", "escape/broken*.csv"}) { + std::vector files; + const auto status = fs->safe_glob(pattern, &files); + EXPECT_FALSE(status.ok()) << pattern; + EXPECT_NE(status.to_string().find("file path is not allowed"), std::string::npos) << status; + // The pattern need not name the matched file, so errors must not reveal that name. + const auto message = status.to_string(); + EXPECT_EQ(std::string::npos, message.find(root.string())) << message; + for (const auto* name : {"outside.csv", "broken.csv", "missing.csv"}) { + EXPECT_EQ(std::string::npos, message.find(name)) << message; + } + EXPECT_TRUE(files.empty()); + } + + for (const auto* pattern : {"nested/*.csv", "inside/*.csv"}) { + std::vector files; + ASSERT_TRUE(fs->safe_glob(pattern, &files).ok()) << pattern; + ASSERT_EQ(1, files.size()); + EXPECT_EQ((secure / "nested/inside.csv").string(), files[0].file_name); + EXPECT_EQ(6, files[0].file_size); + } +} + +TEST_F(LocalFileSystemTest, TestSafeGlobSecureRoot) { + auto fs = io::global_local_filesystem(); + const std::string original_secure_path = config::user_files_secure_path; + DEFER(config::user_files_secure_path = original_secure_path); + const auto root = std::filesystem::canonical(test_dir); + std::filesystem::create_directory(root / "secure"); + std::filesystem::create_directory_symlink(root / "secure", root / "secure-link"); + ASSERT_TRUE(save_string_file((root / "secure/inside.csv").string(), "inside").ok()); + + for (const auto& secure : {std::string(test_dir) + "/secure", (root / "secure/").string(), + (root / "secure-link").string()}) { + config::user_files_secure_path = secure; + std::vector files; + ASSERT_TRUE(fs->safe_glob("*.csv", &files).ok()) << secure; + ASSERT_EQ(1, files.size()); + EXPECT_EQ((root / "secure/inside.csv").string(), files[0].file_name); + } + const auto special_root = root / "secure[prod]*?\\root"; + std::filesystem::create_directory(special_root); + std::filesystem::create_directory_symlink(special_root, root / "special-root-link"); + ASSERT_TRUE(save_string_file((special_root / "inside.csv").string(), "inside").ok()); + for (const auto& secure : {special_root, root / "special-root-link"}) { + config::user_files_secure_path = secure.string(); + std::vector matches; + ASSERT_TRUE(fs->safe_glob("*.csv", &matches).ok()) << secure; + ASSERT_EQ(1, matches.size()); + EXPECT_EQ((special_root / "inside.csv").string(), matches[0].file_name); + } + config::user_files_secure_path = ""; + std::vector files; + EXPECT_FALSE(fs->safe_glob("*.csv", &files).ok()); + config::user_files_secure_path = (root / "missing").string(); + EXPECT_FALSE(fs->safe_glob("*.csv", &files).ok()); +} + +TEST_F(LocalFileSystemTest, TestCanonicalizeSecureLocalPath) { + auto fs = io::global_local_filesystem(); + const auto root = std::filesystem::canonical(test_dir); + const auto secure = root / "secure"; + std::filesystem::create_directories(secure / "nested"); + std::filesystem::create_directory(root / "secure-other"); + std::filesystem::create_directory_symlink(root / "secure-other", secure / "escape"); + std::filesystem::create_directory_symlink(secure / "nested", secure / "inside"); + for (const auto& path : + {std::string("nested"), std::string("nested/../inside"), (secure / "nested").string(), + "file://" + (secure / "nested").string()}) { + std::string resolved; + ASSERT_TRUE(fs->canonicalize_local_file(secure.string(), path, &resolved).ok()) << path; + EXPECT_EQ((secure / "nested").string(), resolved); + } + for (const auto& path : + {std::string("../secure-other"), std::string("escape"), (root / "secure-other").string(), + std::string("https://invalid/dataset"), std::string("s3://bucket/dataset"), + std::string("http://invalid/dataset"), std::string(""), + std::string("nested\0ignored", 14)}) { + std::string resolved; + const auto status = fs->canonicalize_local_file(secure.string(), path, &resolved); + EXPECT_FALSE(status.ok()) << path; + EXPECT_EQ(std::string::npos, status.to_string().find(root.string())) << status; + } + std::string resolved; + EXPECT_FALSE(fs->canonicalize_local_file("", secure.string(), &resolved).ok()); +} + TEST_F(LocalFileSystemTest, TestConvertToAbsPath) { io::Path abs_path; Status st; @@ -658,4 +770,30 @@ TEST_F(LocalFileSystemTest, TestEqualOrSubPath) { EXPECT_FALSE(io::LocalFileSystem::equal_or_sub_path("/data/store1", "/data/store11/snapshot")); EXPECT_FALSE(io::LocalFileSystem::equal_or_sub_path("/data/store/snapshot", "/data/store")); } + +TEST_F(LocalFileSystemTest, TestSafeGlobPreservesWhitespaceIdentity) { + const auto original_secure_path = config::user_files_secure_path; + DEFER(config::user_files_secure_path = original_secure_path); + const auto root = std::filesystem::canonical(test_dir); + const auto secure = root / "whitespace-secure"; + const auto outside = root / "whitespace-outside.csv"; + ASSERT_TRUE(std::filesystem::create_directory(secure)); + ASSERT_TRUE(save_string_file(outside.string(), "outside").ok()); + ASSERT_TRUE(save_string_file((secure / "name.csv ").string(), "safe").ok()); + std::filesystem::create_symlink(outside, secure / "name.csv"); + config::user_files_secure_path = secure.string(); + std::vector files; + auto fs = io::global_local_filesystem(); + ASSERT_TRUE(fs->safe_glob("name.csv?", &files).ok()); + ASSERT_EQ(1, files.size()); + EXPECT_EQ((secure / "name.csv ").string(), files[0].file_name); + io::FileReaderSPtr reader; + ASSERT_TRUE(fs->open_file(files[0].file_name, &reader).ok()); + char buffer[4]; + size_t bytes_read = 0; + ASSERT_TRUE(reader->read_at(0, {buffer, sizeof(buffer)}, &bytes_read).ok()); + EXPECT_EQ("safe", std::string(buffer, bytes_read)); + EXPECT_FALSE(fs->safe_glob("name.csv", &files).ok()); +} + } // namespace doris diff --git a/fe/be-java-extensions/hadoop-deps/src/main/java/org/apache/hadoop/fs/FileSystem.java b/fe/be-java-extensions/hadoop-deps/src/main/java/org/apache/hadoop/fs/FileSystem.java index 6046bb5b723e5e..a404f69f958a41 100644 --- a/fe/be-java-extensions/hadoop-deps/src/main/java/org/apache/hadoop/fs/FileSystem.java +++ b/fe/be-java-extensions/hadoop-deps/src/main/java/org/apache/hadoop/fs/FileSystem.java @@ -31,13 +31,15 @@ // lands in fe/lib/, and bin/start_fe.sh prepends it explicitly; plugin // classloaders resolve org.apache.hadoop.* parent-first, so they see it. // -// The only functional change: Cache.Key carries one extra dimension read from +// Credential cache isolation: Cache.Key carries an extra dimension read from // the configuration property "doris.fs.cache.key". FE injects a per-catalog / // per-TVF credential fingerprint under that property, so FileSystem instances // created with different credentials for the same scheme://authority no longer // collide in the global FileSystem.CACHE. When the property is absent the key // is the empty string and behavior is byte-for-byte identical to vanilla -// Hadoop. Grep for "DORIS-PATCH" to find every modified line. +// Hadoop. TVF-only route policies also check the effective Configuration before +// provider construction and isolate policy/authority identities in that cache. +// Grep for "DORIS-PATCH" to find every modified line. // // When upgrading hadoop.version, re-copy this file from the matching sources // jar and re-apply the patch; hadoop-deps/pom.xml pins the expected version. @@ -54,6 +56,7 @@ import java.net.URISyntaxException; import java.security.PrivilegedExceptionAction; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collection; import java.util.Collections; import java.util.EnumSet; @@ -91,6 +94,7 @@ import org.apache.hadoop.fs.permission.FsAction; import org.apache.hadoop.fs.permission.FsCreateModes; import org.apache.hadoop.fs.permission.FsPermission; +import org.apache.hadoop.fs.viewfs.FsGetter; import org.apache.hadoop.io.IOUtils; import org.apache.hadoop.io.MultipleIOException; import org.apache.hadoop.net.NetUtils; @@ -313,6 +317,8 @@ public FileSystem run() throws IOException { * @throws IOException If an I/O error occurred. */ public static FileSystem get(Configuration conf) throws IOException { + // DORIS-PATCH: resolve defaults and authority from the same TVF configuration snapshot. + conf = TvfGuard.prepare(conf); return get(getDefaultUri(conf), conf); } @@ -561,6 +567,8 @@ public static LocalFileSystem getLocal(Configuration conf) * @throws IOException if the FileSystem cannot be instantiated. */ public static FileSystem get(URI uri, Configuration conf) throws IOException { + // DORIS-PATCH: resolve defaults and authority from the same TVF configuration snapshot. + conf = TvfGuard.prepare(conf); String scheme = uri.getScheme(); String authority = uri.getAuthority(); @@ -576,7 +584,9 @@ public static FileSystem get(URI uri, Configuration conf) throws IOException { } } String disableCacheName = String.format("fs.%s.impl.disable.cache", scheme); - if (conf.getBoolean(disableCacheName, false)) { + if (conf.getBoolean(disableCacheName, false) + || (TvfGuard.isJavaRequest(conf) + && getFileSystemClass(scheme, conf) == org.apache.hadoop.fs.viewfs.ViewFileSystem.class)) { LOGGER.debug("Bypassing cache to create filesystem {}", uri); return createFileSystem(uri, conf); } @@ -620,6 +630,8 @@ public FileSystem run() throws IOException { */ public static FileSystem newInstance(URI uri, Configuration config) throws IOException { + // DORIS-PATCH: resolve defaults and authority from the same TVF configuration snapshot. + config = TvfGuard.prepare(config); String scheme = uri.getScheme(); String authority = uri.getAuthority(); @@ -646,6 +658,8 @@ public static FileSystem newInstance(URI uri, Configuration config) * @throws IOException FS creation or initialization failure. */ public static FileSystem newInstance(Configuration conf) throws IOException { + // DORIS-PATCH: resolve defaults and authority from the same TVF configuration snapshot. + conf = TvfGuard.prepare(conf); return newInstance(getDefaultUri(conf), conf); } @@ -2727,6 +2741,10 @@ public void completeLocalOutput(Path fsOutputFile, Path tmpLocalFile) */ @Override public void close() throws IOException { + closeBase(); + } + + private void closeBase() throws IOException { debugLogFileSystemClose("close", "Key: " + key + "; URI: " + getUri() + "; Object Identity Hash: " + Integer.toHexString(System.identityHashCode(this))); @@ -3642,6 +3660,754 @@ public static Class getFileSystemClass(String scheme, * @return the initialized filesystem. * @throws IOException problems loading or initializing the FileSystem */ + // DORIS-PATCH: TVF policy is checked against the frozen configuration used by the consumer. + public static int getDorisTvfGuardVersion() { + return 1; + } + + public static final class TvfGuard { + public static final String POLICY_KEY = "fs.doris.tvf.remote.paths"; + public static final String NATIVE_POLICY = "native"; + public static final String NATIVE_WRITE_POLICY = "native-write"; + public static final String JAVA_READ_POLICY = "java-read"; + public static final String ROOT_KEY = "fs.doris.tvf.root"; + public static final String PATH_KEY = "fs.doris.tvf.path"; + public static final String CONTEXT_KEY = "fs.doris.tvf.context"; + private static final ThreadLocal LAST_NATIVE_CONTEXT = new ThreadLocal<>(); + + private TvfGuard() { + } + + public static final class Rule { + private final String filesystem; + private final String path; + private final boolean descendants; + + public Rule(String filesystem, String path, boolean descendants) { + this.filesystem = filesystem; + this.path = path; + this.descendants = descendants; + } + } + + public static String encode(List rules) { + if (FileSystem.getDorisTvfGuardVersion() != 1) { + throw new IllegalStateException("TVF Hadoop runtime policy version mismatch"); + } + try { + java.io.ByteArrayOutputStream bytes = new java.io.ByteArrayOutputStream(); + java.io.DataOutputStream out = new java.io.DataOutputStream(bytes); + out.writeInt(1); + out.writeInt(rules.size()); + for (Rule rule : rules) { + out.writeUTF(rule.filesystem); + out.writeUTF(rule.path); + out.writeBoolean(rule.descendants); + } + out.flush(); + return java.util.Base64.getEncoder().encodeToString(bytes.toByteArray()); + } catch (IOException e) { + throw new IllegalArgumentException("Cannot encode TVF Hadoop route policy", e); + } + } + + /** Literal probes and subtree prefixes touched by Globber, without filesystem IO. */ + public static List globRules(URI filesystem, String pattern) throws IOException { + List rules = new ArrayList<>(); + for (String expanded : GlobExpander.expand(pattern)) { + Path flattened = new Path(expanded.isEmpty() ? Path.CUR_DIR : expanded); + String path = flattened.toUri().getPath(); + Path root = new Path(flattened.isAbsolute() ? "/" : Path.CUR_DIR); + Path candidate = root; + List components = new ArrayList<>(); + List patterned = new ArrayList<>(); + for (String component : path.split("/")) { + if (component.isEmpty()) { + continue; + } + GlobFilter filter = new GlobFilter(component); + if (filter.hasPattern()) { + // Keep every real listing parent, even if later escaped '..' removes it. + rules.add(new Rule(filesystem.toString(), + globLiteralPrefix(root, components, patterned).toUri().getPath(), true)); + } + component = component.replaceAll("\\\\(.)", "$1"); + candidate = new Path(candidate, component); + if ("..".equals(component) && !components.isEmpty() + && !"..".equals(components.get(components.size() - 1))) { + components.remove(components.size() - 1); + patterned.remove(patterned.size() - 1); + } else if (!".".equals(component)) { + components.add(component); + patterned.add(filter.hasPattern()); + } + } + boolean subtree = patterned.contains(Boolean.TRUE); + Path probe = subtree ? globLiteralPrefix(root, components, patterned) : candidate; + rules.add(new Rule(filesystem.toString(), probe.toUri().getPath(), subtree)); + } + return rules; + } + + private static Path globLiteralPrefix(Path root, List components, List patterned) { + Path prefix = root; + for (int i = 0; i < components.size() && !patterned.get(i); i++) { + prefix = new Path(prefix, components.get(i)); + } + return prefix; + } + + /** Java requests own their mount clients; native connections keep their existing cache. */ + private static final class RequestViewFileSystem extends org.apache.hadoop.fs.viewfs.ViewFileSystem { + private final Map, FileSystem> children = new HashMap<>(); + private final Set owned = Collections.newSetFromMap(new java.util.IdentityHashMap<>()); + private boolean closed; + + private RequestViewFileSystem() throws IOException { + } + + private final FsGetter getter = new FsGetter() { + @Override + public FileSystem get(URI uri, Configuration conf) throws IOException { + synchronized (RequestViewFileSystem.this) { + List key = Arrays.asList(uri.getScheme(), uri.getRawAuthority()); + FileSystem child = children.get(key); + if (child == null) { + child = getNewInstance(uri, conf); + children.put(key, child); + } + return child; + } + } + + @Override + public FileSystem getNewInstance(URI uri, Configuration conf) throws IOException { + synchronized (RequestViewFileSystem.this) { + if (closed) { + throw new IOException("TVF ViewFS request is closed"); + } + FileSystem child = FileSystem.newInstance(uri, conf); + owned.add(child); + return child; + } + } + }; + + @Override + protected FsGetter fsGetter() { + return getter; + } + + @Override + public synchronized void close() throws IOException { + if (closed) { + return; + } + List failures = new ArrayList<>(); + try { + // delete-on-exit may resolve a previously unused mount, so do this before sealing. + ((FileSystem) this).closeBase(); + } catch (IOException e) { + failures.add(e); + } finally { + closed = true; + // Enumerating ViewFS target.getChildFileSystems() misses Nfly's actual peers. + // The stable getter owns those peers directly, including partial initialization. + for (FileSystem child : owned) { + try { + child.close(); + } catch (IOException e) { + failures.add(e); + } catch (RuntimeException e) { + failures.add(new IOException("Cannot close a TVF mount client", e)); + } + } + owned.clear(); + children.clear(); + } + if (!failures.isEmpty()) { + throw MultipleIOException.createIOException(failures); + } + } + } + + private static boolean isNative(String policy) { + return NATIVE_POLICY.equals(policy) || NATIVE_WRITE_POLICY.equals(policy); + } + + private static boolean isJavaRequest(Configuration conf) { + String policy = conf.getRaw(POLICY_KEY); + return policy != null && !policy.isEmpty() && !isNative(policy); + } + + private static boolean isRequestViewFs(Class implementation, Configuration conf) { + return isJavaRequest(conf) && implementation == org.apache.hadoop.fs.viewfs.ViewFileSystem.class; + } + + private static IOException rejected() { + return new IOException("Hadoop TVF requires a proven remote filesystem route. " + + "Use local() with LOCAL TVF privileges and secure-path restrictions."); + } + + private static Configuration freeze(Configuration conf) throws IOException { + // Service providers (notably HDFS) register their default/site resources on first load. + // Load them before copying raw values, as an ordinary FileSystem lookup does. + if (!FILE_SYSTEMS_LOADED) { + loadFileSystems(); + } + if (conf instanceof FrozenConfiguration) { + return conf; + } + return new FrozenConfiguration(conf); + } + + private static Configuration prepare(Configuration conf) throws IOException { + String policy = conf.getRaw(POLICY_KEY); + if (policy == null || policy.isEmpty()) { + return conf; + } + Configuration frozen = freeze(conf); + Context context = ((FrozenConfiguration) frozen).context; + if (isNative(policy)) { + context.checkNative(frozen.getRaw(ROOT_KEY), frozen.getRaw(PATH_KEY)); + } else if (JAVA_READ_POLICY.equals(policy)) { + context.checkJava(new Path(frozen.getRaw(PATH_KEY))); + } else { + context.checkRules(policy); + } + return frozen; + } + + public static void check(URI ignored, Configuration conf) throws IOException { + prepare(conf); + } + + public static void clearNativeContext() { + LAST_NATIVE_CONTEXT.remove(); + } + + public static Context takeNativeContext() { + Context result = LAST_NATIVE_CONTEXT.get(); + LAST_NATIVE_CONTEXT.remove(); + return result; + } + + public static Configuration prepareJavaRead(Configuration conf, Path path, String marker) throws IOException { + if (marker == null || marker.isEmpty()) { + return conf; + } + if (!NATIVE_POLICY.equals(marker)) { + throw rejected(); + } + Configuration request = new Configuration(conf); + request.set(POLICY_KEY, JAVA_READ_POLICY); + request.set(ROOT_KEY, path.toUri().toString()); + request.set(PATH_KEY, path.toString()); + return prepare(request); + } + + public static void checkJavaRead(FileSystem fs, Path path, String marker) throws IOException { + if (marker == null || marker.isEmpty()) { + return; + } + if (!NATIVE_POLICY.equals(marker)) { + throw rejected(); + } + // The marker is request-owned, never inferred from an unmarked cached filesystem. + Configuration conf = fs.getConf(); + if (JAVA_READ_POLICY.equals(conf.getRaw(POLICY_KEY))) { + // S3A derives a Configuration for bucket options. Its copy preserves our internal + // marker and raw snapshot values, but not the private Configuration subclass. + ((FrozenConfiguration) freeze(conf)).context.checkJava(path); + } else { + throw rejected(); + } + } + + private static final class FrozenConfiguration extends Configuration { + private final Context context; + + private FrozenConfiguration(Configuration source) throws IOException { + super(false); + setClassLoader(source.getClassLoader()); + java.util.SortedMap values = new java.util.TreeMap<>(); + for (Map.Entry entry : source) { + String key = entry.getKey(); + String value = source.getRaw(key); + set(key, value); + if (!PATH_KEY.equals(key) && !ROOT_KEY.equals(key) && !CONTEXT_KEY.equals(key)) { + values.put(key, value); + } + } + // ViewFS's private InnerCache folds authority case (A/a). Guarded children must + // instead use FileSystem's exact-authority/profile cache below. + set("fs.viewfs.enable.inner.cache", "false"); + values.put("fs.viewfs.enable.inner.cache", "false"); + try { + java.security.MessageDigest digest = java.security.MessageDigest.getInstance("SHA-256"); + java.io.DataOutputStream output = new java.io.DataOutputStream( + new java.security.DigestOutputStream(new java.io.OutputStream() { + @Override + public void write(int value) { + } + + @Override + public void write(byte[] values, int offset, int length) { + } + }, digest)); + for (Map.Entry entry : values.entrySet()) { + for (String value : new String[] {entry.getKey(), entry.getValue()}) { + // Preserve raw Java code units, including unpaired surrogates. UTF-8 replacement + // would give a literal '?' and an unpaired surrogate the same cache profile. + output.writeInt(value.length()); + output.writeChars(value); + } + } + output.flush(); + StringBuilder identity = new StringBuilder(); + for (byte value : digest.digest()) { + identity.append(Character.forDigit((value & 255) >>> 4, 16)); + identity.append(Character.forDigit(value & 15, 16)); + } + set(CONTEXT_KEY, identity.toString()); + } catch (java.security.NoSuchAlgorithmException e) { + throw new IOException("SHA-256 is required for TVF Hadoop cache isolation"); + } + context = new Context(this); + } + } + + public static final class Context { + private final Configuration conf; + private final Map routingKeys = new HashMap<>(); + + private Context(Configuration conf) { + this.conf = conf; + for (Map.Entry entry : conf) { + String key = entry.getKey(); + if ("fs.defaultFS".equals(key) || "fs.default.name".equals(key) + || key.startsWith("fs.viewfs.") || (key.startsWith("fs.") && key.endsWith(".impl"))) { + routingKeys.put(key, entry.getValue()); + } + } + } + + private EffectiveRoute route(boolean descendants) { + return new EffectiveRoute(routingKeys, descendants, conf); + } + + public String profile() { + return conf.getRaw(CONTEXT_KEY); + } + + public boolean isDirectRemote() throws IOException { + String root = conf.getRaw(ROOT_KEY); + if (root == null) { + throw rejected(); + } + try { + URI uri = nativeUri(root); + return route(false).provider(uri.getScheme()) == Route.Provider.REMOTE; + } catch (RuntimeException | URISyntaxException e) { + throw rejected(); + } + } + + private static URI nativeUri(String root) throws URISyntaxException { + root = root.trim(); + if (root.isEmpty() || "default".equals(root)) { + throw new URISyntaxException("", "Unproven native filesystem"); + } + return new URI(root.contains("://") ? root : "hdfs://" + root); + } + + public void checkNative(String root, String path) throws IOException { + if (root == null || path == null || path.isEmpty()) { + throw rejected(); + } + try { + URI uri = nativeUri(root); + Path actualPath = new Path(path); + if (NATIVE_WRITE_POLICY.equals(conf.getRaw(POLICY_KEY))) { + // ViewFS.create resolves the parent, without following the final component. + Path parent = actualPath.getParent(); + actualPath = parent == null ? new Path("/") : parent; + } + route(false).checkRoute(uri, actualPath.toUri().getPath(), new HashSet<>()); + } catch (RuntimeException | URISyntaxException e) { + throw rejected(); + } + } + + private void checkJava(Path path) throws IOException { + try { + URI uri = path.toUri(); + route(false).checkRoute(uri, uri.getPath(), new HashSet<>()); + } catch (RuntimeException | URISyntaxException e) { + throw rejected(); + } + } + + private void checkRules(String policy) throws IOException { + try { + byte[] bytes = java.util.Base64.getDecoder().decode(policy); + java.io.DataInputStream in = new java.io.DataInputStream(new java.io.ByteArrayInputStream(bytes)); + int version = in.readInt(); + int count = in.readInt(); + if (version != 1 || count <= 0 || count > in.available() / 5) { + throw rejected(); + } + for (int i = 0; i < count; i++) { + URI filesystem = new URI(in.readUTF()); + String path = in.readUTF(); + boolean descendants = in.readBoolean(); + route(descendants).checkRoute(filesystem, path, new HashSet<>()); + } + if (in.available() != 0) { + throw rejected(); + } + } catch (RuntimeException | URISyntaxException e) { + throw rejected(); + } + } + } + + private static final class EffectiveRoute extends Route { + private final Configuration conf; + + private EffectiveRoute(Map properties, boolean descendants, Configuration conf) { + super(properties, descendants); + this.conf = conf; + } + + private EffectiveRoute(EffectiveRoute original, boolean descendants) { + super(original, descendants); + conf = original.conf; + } + + @Override + protected Route withDescendants(boolean value) { + return new EffectiveRoute(this, value); + } + + @Override + protected String property(String key) { + return conf.get(key); + } + + @Override + protected Provider provider(String scheme) { + if ("file".equalsIgnoreCase(scheme)) { + return Provider.LOCAL; + } + try { + String implementation = FileSystem.getFileSystemClass(scheme, conf).getName(); + return implementationProvider(implementation); + } catch (IOException | RuntimeException e) { + return Provider.UNKNOWN; + } + } + } + + public static class Route { + protected enum Provider { + REMOTE, + LOCAL, + MOUNT_TABLE, + DYNAMIC, + UNKNOWN + } + + protected static final Set REMOTE_SCHEMES = new HashSet<>(Arrays.asList( + "hdfs", "s3a", "s3n", "oss", "ofs", "jfs", "abfs", "abfss", "wasb", "wasbs", "gs", "cosn")); + protected static final Set REMOTE_IMPLEMENTATIONS = new HashSet<>(Arrays.asList( + "org.apache.hadoop.hdfs.DistributedFileSystem", + "org.apache.hadoop.fs.s3a.S3AFileSystem", + "org.apache.hadoop.fs.azurebfs.AzureBlobFileSystem", + "org.apache.hadoop.fs.azurebfs.SecureAzureBlobFileSystem", + "org.apache.hadoop.fs.azure.NativeAzureFileSystem", + "org.apache.hadoop.fs.azure.NativeAzureFileSystem$Secure", + "io.juicefs.JuiceFileSystem", + "org.apache.hadoop.fs.ozone.OzoneFileSystem", + "org.apache.hadoop.fs.ozone.RootedOzoneFileSystem", + "org.apache.hadoop.fs.cosn.CosNFileSystem", + "com.qcloud.chdfs.fs.CHDFSHadoopFileSystemAdapter", + "org.apache.hadoop.fs.aliyun.oss.AliyunOSSFileSystem", + "com.aliyun.jindodata.oss.JindoOssFileSystem", + "org.apache.hadoop.fs.obs.OBSFileSystem", + "com.google.cloud.hadoop.fs.gcs.GoogleHadoopFileSystem")); + + private final boolean descendants; + private final Map properties; + + protected Route(Route original, boolean descendants) { + this(original.properties, descendants); + } + + protected Route withDescendants(boolean value) { + return new Route(this, value); + } + + public Route(Map properties, boolean descendants) { + // Both callers supply a statement-local snapshot that is never mutated afterward. + this.properties = properties; + this.descendants = descendants; + } + + private List defaults() throws URISyntaxException { + List defaults = new ArrayList<>(); + for (String key : Arrays.asList("fs.defaultFS", "fs.default.name")) { + String value = property(key); + if (value != null) { + value = selector(value).trim(); + // FileSystem.fixName treats slash-free host[:port] strings as HDFS defaults. + defaults.add(new URI("local".equals(value) ? "file:///" + : (value.indexOf('/') < 0 ? "hdfs://" + value : value))); + } + } + if (defaults.isEmpty()) { + defaults.add(new URI("file:///")); + } + return defaults; + } + + public void checkRoute(URI filesystem, String path, Set activeTables) throws URISyntaxException { + if (filesystem.getScheme() == null) { + for (URI defaultFs : defaults()) { + checkViewFsEnabled(defaultFs, path, activeTables); + } + return; + } + if (filesystem.getAuthority() == null) { + boolean completed = false; + for (URI defaultFs : defaults()) { + if (filesystem.getScheme().equals(defaultFs.getScheme()) && defaultFs.getAuthority() != null) { + checkViewFsEnabled(defaultFs, path, activeTables); + completed = true; + } + } + if (completed) { + return; + } + } + checkViewFsEnabled(filesystem, path, activeTables); + } + + public void checkGlobRoute(URI filesystem, String pattern, Set activeTables) + throws IOException, URISyntaxException { + for (Rule rule : globRules(filesystem, pattern)) { + withDescendants(rule.descendants).checkRoute(filesystem, rule.path, activeTables); + } + } + + protected Provider provider(String scheme) { + if ("file".equalsIgnoreCase(scheme)) { + return Provider.LOCAL; + } + String implementation = property("fs." + scheme + ".impl"); + if (implementation != null) { + implementation = selector(implementation).trim(); + return implementationProvider(implementation); + } + if ("viewfs".equals(scheme)) { + return Provider.MOUNT_TABLE; + } + return REMOTE_SCHEMES.contains(scheme) ? Provider.REMOTE : Provider.UNKNOWN; + } + + protected Provider implementationProvider(String implementation) { + if ("org.apache.hadoop.fs.LocalFileSystem".equals(implementation) + || "org.apache.hadoop.fs.RawLocalFileSystem".equals(implementation)) { + return Provider.LOCAL; + } + if ("org.apache.hadoop.fs.viewfs.ViewFileSystem".equals(implementation)) { + return Provider.MOUNT_TABLE; + } + if ("org.apache.hadoop.fs.viewfs.ViewFileSystemOverloadScheme".equals(implementation) + || "org.apache.hadoop.hdfs.ViewDistributedFileSystem".equals(implementation)) { + return Provider.DYNAMIC; + } + return REMOTE_IMPLEMENTATIONS.contains(implementation) ? Provider.REMOTE : Provider.UNKNOWN; + } + + private String[] targets(String key, String prefix) { + String value = selector(property(key)); + // InodeTree treats SINGLE, SINGLE_FALLBACK and MERGE_SLASH as one URI. + // Commas delimit targets only for MERGE and NFLY links. + String entry = key.substring(prefix.length()); + boolean multiple = entry.startsWith("linkMerge.") || entry.startsWith("linkNfly."); + return multiple ? value.split(",", -1) : new String[] {value}; + } + + private void checkGlobTargets(String key, String prefix, Set activeTables) throws URISyntaxException { + for (String target : targets(key, prefix)) { + if (target.isEmpty()) { + reject(); + } + URI uri = new URI(target); + // Listing can touch any descendant of a selected mount. Reuse the same route + // interpreter for nested tables instead of assuming their root is representative. + String path = uri.getPath() == null ? "/" : uri.getPath(); + checkRoute(uri, path, activeTables); + } + } + + private void checkViewFsEnabled(URI filesystem, String path, Set activeTables) { + String scheme = filesystem.getScheme(); + Provider provider = provider(scheme); + if (provider == Provider.LOCAL || provider == Provider.UNKNOWN || provider == Provider.DYNAMIC) { + reject(); + } + if (provider != Provider.MOUNT_TABLE) { + return; + } + if (!path.startsWith("/")) { + // Java resolves relative paths against the selected filesystem's working directory. + // Its user/home configuration is not a proven absolute mount-table path here. + reject(); + } + String table = filesystem.getAuthority(); + String ignorePort = property("fs.viewfs.ignore.port.in.mount.table.name"); + if (ignorePort != null && Boolean.parseBoolean(selector(ignorePort).trim()) + && table != null && filesystem.getPort() != -1) { + table = filesystem.getHost(); + if (table == null) { + reject(); + } + } + if (table == null || table.isEmpty()) { + table = property("fs.viewfs.mounttable.default.name.key"); + table = table == null ? "default" : selector(table); + } + // Nested ViewFS cycles or arbitrarily deep delegation cannot establish remote storage. + if (activeTables.size() >= 32 || !activeTables.add(table)) { + reject(); + } + try { + String prefix = "fs.viewfs.mounttable." + table + "."; + String source = ""; + List selectedKeys = new ArrayList<>(); + List mountSources = new ArrayList<>(); + String fallback = null; + String rootMerge = null; + for (String key : properties.keySet()) { + if (!key.startsWith(prefix)) { + continue; + } + String entry = key.substring(prefix.length()); + if (entry.equals("linkFallback")) { + fallback = key; + continue; + } + if (entry.equals("linkMergeSlash")) { + rootMerge = key; + continue; + } + if (entry.startsWith("linkRegex")) { + // Regex interception rewrites runtime paths before ordinary mount resolution. + reject(); + } + int slash = entry.indexOf('/'); + if (slash < 0 || !(entry.startsWith("link.") || entry.startsWith("linkMerge.") + || entry.startsWith("linkNfly."))) { + continue; + } + String candidate = entry.substring(slash); + // InodeTree splits source components and discards trailing empty components. + while (candidate.length() > 1 && candidate.endsWith("/")) { + candidate = candidate.substring(0, candidate.length() - 1); + } + mountSources.add(candidate); + if (!(path.equals(candidate) || path.startsWith(candidate + "/"))) { + continue; + } + if (candidate.length() > source.length()) { + source = candidate; + selectedKeys.clear(); + } + if (candidate.equals(source)) { + selectedKeys.add(key); + } + } + if (descendants) { + String subtree = path.endsWith("/") ? path : path + "/"; + for (String key : properties.keySet()) { + if (key.startsWith(prefix)) { + String entry = key.substring(prefix.length()); + int sourceStart = entry.indexOf('/'); + if (sourceStart >= 0 && entry.substring(sourceStart).startsWith(subtree)) { + checkGlobTargets(key, prefix, activeTables); + } + } + } + } + if (rootMerge != null) { + selectedKeys.add(rootMerge); + source = ""; + } else if (selectedKeys.isEmpty() && fallback != null) { + selectedKeys.add(fallback); + } + if (selectedKeys.isEmpty()) { + // Missing inline fallback does not prove absence of one in default resources. + // Keep rejecting unproven routes, including globs that cross mount names. + reject(); + } + for (String key : selectedKeys) { + for (String target : targets(key, prefix)) { + if (target.isEmpty()) { + reject(); + } + URI uri = new URI(target); + if (uri.getScheme() == null) { + reject(); + } + if (provider(uri.getScheme()) == Provider.MOUNT_TABLE) { + for (String nested : mountSources) { + if (!nested.startsWith(source + "/")) { + continue; + } + int branchEnd = nested.indexOf('/', source.length() + 1); + String branch = branchEnd < 0 ? nested : nested.substring(0, branchEnd); + if (path.equals(branch) || path.startsWith(branch + "/")) { + // A missing child below an internal nested-mount directory uses + // a different suffix in InodeTree. Do not guess its delegation. + reject(); + } + } + } + String base = uri.getPath() == null ? "/" : uri.getPath(); + String remaining = path.substring(source.length()); + String nextPath = new URI(null, null, + (base.endsWith("/") ? base : base + "/") + + (remaining.startsWith("/") ? remaining.substring(1) : remaining), null) + .normalize().getPath(); + checkRoute(uri, nextPath, activeTables); + } + } + } catch (URISyntaxException e) { + reject(); + } finally { + activeTables.remove(table); + } + } + + protected String property(String key) { + return properties.get(key); + } + + private String selector(String value) { + if (value.contains("${")) { + reject(); + } + return value; + } + + protected void reject() { + throw new IllegalArgumentException("Hadoop TVF requires a proven remote filesystem route"); + } + } + } + private static FileSystem createFileSystem(URI uri, Configuration conf) throws IOException { Tracer tracer = FsTracer.get(conf); @@ -3649,9 +4415,13 @@ private static FileSystem createFileSystem(URI uri, Configuration conf) DurationInfo ignored = new DurationInfo(LOGGER, false, "Creating FS %s", uri)) { scope.addKVAnnotation("scheme", uri.getScheme()); + // DORIS-PATCH: initialize from the exact private snapshot retained for subsequent TVF IO. + conf = TvfGuard.prepare(conf); Class clazz = getFileSystemClass(uri.getScheme(), conf); - FileSystem fs = ReflectionUtils.newInstance(clazz, conf); + FileSystem fs = TvfGuard.isRequestViewFs(clazz, conf) + ? ReflectionUtils.newInstance(TvfGuard.RequestViewFileSystem.class, conf) + : ReflectionUtils.newInstance(clazz, conf); try { fs.initialize(uri, conf); } catch (IOException | RuntimeException e) { @@ -3662,9 +4432,13 @@ private static FileSystem createFileSystem(URI uri, Configuration conf) LOGGER.debug("Failed to initialize filesystem", e); // then (robustly) close the FS, so as to invoke any // cleanup code. + TvfGuard.clearNativeContext(); IOUtils.cleanupWithLogger(LOGGER, fs); throw e; } + if (TvfGuard.isNative(conf.getRaw(TvfGuard.POLICY_KEY))) { + TvfGuard.LAST_NATIVE_CONTEXT.set(((TvfGuard.FrozenConfiguration) conf).context); + } return fs; } } @@ -3704,12 +4478,14 @@ static final class Cache { } FileSystem get(URI uri, Configuration conf) throws IOException{ + conf = TvfGuard.prepare(conf); Key key = new Key(uri, conf); return getInternal(uri, conf, key); } /** The objects inserted into the cache using this method are all unique. */ FileSystem getUnique(URI uri, Configuration conf) throws IOException{ + conf = TvfGuard.prepare(conf); Key key = new Key(uri, conf, unique.getAndIncrement()); return getInternal(uri, conf, key); } @@ -3926,6 +4702,9 @@ static class Key { final UserGroupInformation ugi; final long unique; // an artificial way to make a key unique final String dorisCacheKey; // DORIS-PATCH + final String tvfPolicy; // DORIS-PATCH: separate from user cache fingerprints. + final String tvfContext; + final String tvfFileSystem; // DORIS-PATCH: ViewFS table authority is case-sensitive. Key(URI uri, Configuration conf) throws IOException { this(uri, conf, 0); @@ -3937,6 +4716,11 @@ static class Key { authority = uri.getAuthority()==null ? "" : StringUtils.toLowerCase(uri.getAuthority()); this.unique = unique; + this.tvfPolicy = conf.getRaw(TvfGuard.POLICY_KEY) == null + ? "" : conf.getRaw(TvfGuard.POLICY_KEY); + this.tvfContext = tvfPolicy.isEmpty() ? "" : conf.get(TvfGuard.CONTEXT_KEY, ""); + this.tvfFileSystem = tvfPolicy.isEmpty() ? "" : uri.getScheme() + "://" + + (uri.getRawAuthority() == null ? "" : uri.getRawAuthority()); // DORIS-PATCH: getTrimmed, so a hand-written value with stray whitespace still keys the // same entry as the FE-generated one. this.dorisCacheKey = conf.getTrimmed(DORIS_FS_CACHE_KEY_PROPERTY + "." + scheme, @@ -3948,7 +4732,7 @@ static class Key { @Override public int hashCode() { return (scheme + authority).hashCode() + ugi.hashCode() + (int)unique - + dorisCacheKey.hashCode(); // DORIS-PATCH + + dorisCacheKey.hashCode() + tvfPolicy.hashCode() + tvfContext.hashCode() + tvfFileSystem.hashCode(); // DORIS-PATCH } static boolean isEqual(Object a, Object b) { @@ -3966,6 +4750,9 @@ public boolean equals(Object obj) { && isEqual(this.authority, that.authority) && isEqual(this.ugi, that.ugi) && isEqual(this.dorisCacheKey, that.dorisCacheKey) // DORIS-PATCH + && isEqual(this.tvfPolicy, that.tvfPolicy) + && isEqual(this.tvfContext, that.tvfContext) + && isEqual(this.tvfFileSystem, that.tvfFileSystem) && (this.unique == that.unique); } return false; diff --git a/fe/be-java-extensions/hadoop-deps/src/test/java/org/apache/hadoop/fs/DorisFileSystemCacheKeyTest.java b/fe/be-java-extensions/hadoop-deps/src/test/java/org/apache/hadoop/fs/DorisFileSystemCacheKeyTest.java index 8b96ee499b9149..b6f6d640da64ba 100644 --- a/fe/be-java-extensions/hadoop-deps/src/test/java/org/apache/hadoop/fs/DorisFileSystemCacheKeyTest.java +++ b/fe/be-java-extensions/hadoop-deps/src/test/java/org/apache/hadoop/fs/DorisFileSystemCacheKeyTest.java @@ -169,4 +169,70 @@ public void testServiceScanIgnoresTheThreadContextClassLoader() throws Exception loadedFlag.set(null, wasLoaded); } } + @Test + public void testTvfPolicyIsIndependentFromUserCacheFingerprints() throws IOException { + Configuration plain = new Configuration(false); + Configuration first = new Configuration(false); + first.set(FileSystem.TvfGuard.POLICY_KEY, "first-policy"); + Configuration second = new Configuration(false); + second.set(FileSystem.TvfGuard.POLICY_KEY, "second-policy"); + first.set("doris.fs.cache.key", "same-user-value"); + second.set("doris.fs.cache.key", "same-user-value"); + FileSystem.Cache.Key a = new FileSystem.Cache.Key(LOCAL, first); + FileSystem.Cache.Key b = new FileSystem.Cache.Key(LOCAL, second); + Assertions.assertNotEquals(a, b); + Assertions.assertNotEquals(a, new FileSystem.Cache.Key(LOCAL, plain)); + } + + @Test + public void testGuardedViewFsAuthorityCaseIsAnExactCacheDimension() throws IOException { + Configuration guarded = new Configuration(false); + guarded.set(FileSystem.TvfGuard.POLICY_KEY, "one-policy-with-both-tables"); + Assertions.assertNotEquals(new FileSystem.Cache.Key(URI.create("viewfs://Cluster"), guarded), + new FileSystem.Cache.Key(URI.create("viewfs://cluster"), guarded)); + Configuration plain = new Configuration(false); + Assertions.assertEquals(new FileSystem.Cache.Key(URI.create("viewfs://Cluster"), plain), + new FileSystem.Cache.Key(URI.create("viewfs://cluster"), plain)); + } + + + @Test + public void testActualSnapshotProfilePartitionsCacheBeforeLookup() throws IOException { + // A credential helper may initialize local FS while its guarded data route is remote. + Configuration first = nativeRequest(); + first.set("remote.cluster.setting", "first"); + Configuration second = nativeRequest(); + second.set("remote.cluster.setting", "second"); + FileSystem a = FileSystem.get(LOCAL, first); + FileSystem b = FileSystem.get(LOCAL, second); + Assertions.assertNotSame(a, b); + Assertions.assertNotEquals(a.getConf().getRaw(FileSystem.TvfGuard.CONTEXT_KEY), + b.getConf().getRaw(FileSystem.TvfGuard.CONTEXT_KEY)); + Configuration nextPath = new Configuration(first); + nextPath.set(FileSystem.TvfGuard.PATH_KEY, "/data/next.csv"); + Assertions.assertSame(a, FileSystem.get(LOCAL, nextPath)); + FileSystem.TvfGuard.clearNativeContext(); + } + + @Test + public void testRawConfigurationCodeUnitsRemainDistinctCacheProfiles() throws IOException { + java.util.Set instances = new java.util.HashSet<>(); + for (String value : new String[] {"?", String.valueOf((char) 0xD800), String.valueOf((char) 0xDC00)}) { + Configuration conf = nativeRequest(); + conf.set("unrelated.raw.setting", value); + FileSystem fs = FileSystem.get(LOCAL, conf); + Assertions.assertEquals(value, fs.getConf().getRaw("unrelated.raw.setting")); + Assertions.assertTrue(instances.add(fs), "different raw settings must not reuse a guarded filesystem"); + } + FileSystem.TvfGuard.clearNativeContext(); + } + + private static Configuration nativeRequest() { + Configuration conf = new Configuration(false); + conf.set(FileSystem.TvfGuard.POLICY_KEY, FileSystem.TvfGuard.NATIVE_POLICY); + conf.set(FileSystem.TvfGuard.ROOT_KEY, "hdfs://runtime"); + conf.set(FileSystem.TvfGuard.PATH_KEY, "/data/file.csv"); + return conf; + } + } diff --git a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java index d032e271112078..a92117c3b39385 100644 --- a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java +++ b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java @@ -2815,6 +2815,10 @@ public static void validateMtmvCacheConfig() throws ConfigException { + "configuration to prevent command injection attacks.") public static boolean enable_java_udf = true; + @ConfField(description = "Whether to enable LOCAL TVF and FILE TVF with local storage. Disabled by default. " + + "Set this in fe.conf and restart every FE to enable it. Existing access checks still apply.") + public static boolean enable_local_tvf = false; + @ConfField(mutable = true, masterOnly = true, description = "When enabled, data can be processed using the " + "globally created Java UDF function during import. " + "The default setting is false.") public static boolean enable_udf_in_load = false; diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindSink.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindSink.java index 4f4ece3d15e23b..78d4b9e6f4fe84 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindSink.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindSink.java @@ -46,7 +46,9 @@ import org.apache.doris.datasource.plugin.PluginDrivenExternalCatalog; import org.apache.doris.datasource.plugin.PluginDrivenExternalTable; import org.apache.doris.datasource.plugin.PluginDrivenMetadata; +import org.apache.doris.datasource.storage.StorageAdapter; import org.apache.doris.dictionary.Dictionary; +import org.apache.doris.filesystem.FileSystemUtil; import org.apache.doris.mtmv.ivm.IvmUtil; import org.apache.doris.nereids.CascadesContext; import org.apache.doris.nereids.StatementContext; @@ -104,6 +106,8 @@ import org.apache.doris.qe.AutoCloseSessionVariable; import org.apache.doris.qe.ConnectContext; import org.apache.doris.qe.SessionVariable; +import org.apache.doris.tablefunction.HadoopTvfAdmission; +import org.apache.doris.tablefunction.LocalTableValuedFunction; import org.apache.doris.thrift.TPartialUpdateNewRowPolicy; import com.google.common.annotations.VisibleForTesting; @@ -701,6 +705,9 @@ private Plan bindTVFTableSink(MatchingContext> ctx) { "INSERT INTO TVF only supports local/s3/hdfs, but got: " + tvfName); } + // INSERT/EXPLAIN INSERT bind a sink without constructing the LOCAL read TVF. + LocalTableValuedFunction.checkSinkEnabled(tvfName, properties); + // Validate required properties if (!properties.containsKey("file_path")) { throw new AnalysisException("TVF sink requires 'file_path' property"); @@ -727,6 +734,19 @@ private Plan bindTVFTableSink(MatchingContext> ctx) { "delete_existing_files=true is not supported for local TVF"); } + if ("hdfs".equals(tvfName)) { + // Storage normalization loads operator-controlled XML before route classification, + // while leaving filesystem construction to execution. + Map normalized = StorageAdapter.of(properties).getBackendConfigProperties(); + if (!properties.containsKey("writer_class")) { + HadoopTvfAdmission.check(HadoopTvfAdmission.Access.NATIVE_WRITE, filePath, normalized); + } + if (deleteExisting) { + HadoopTvfAdmission.check(HadoopTvfAdmission.Access.JAVA_DELETE, + FileSystemUtil.extractParentDirectory(filePath), normalized); + } + } + LogicalPlan child = ((LogicalPlan) sink.child()); // Always derive schema from child query output diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/InsertIntoTVFCommand.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/InsertIntoTVFCommand.java index 30cd5da46e6218..716ec2e9421ef6 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/InsertIntoTVFCommand.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/InsertIntoTVFCommand.java @@ -24,6 +24,7 @@ import org.apache.doris.common.Status; import org.apache.doris.common.UserException; import org.apache.doris.datasource.storage.StorageAdapter; +import org.apache.doris.datasource.storage.StorageTypeId; import org.apache.doris.filesystem.FileSystemUtil; import org.apache.doris.filesystem.Location; import org.apache.doris.fs.FileSystemFactory; @@ -44,6 +45,8 @@ import org.apache.doris.qe.QeProcessorImpl; import org.apache.doris.qe.QeProcessorImpl.QueryInfo; import org.apache.doris.qe.StmtExecutor; +import org.apache.doris.tablefunction.HadoopTvfAdmission; +import org.apache.doris.tablefunction.LocalTableValuedFunction; import org.apache.doris.task.LoadEtlTask; import org.apache.doris.thrift.TStatusCode; import org.apache.doris.thrift.TUniqueId; @@ -173,8 +176,10 @@ public Plan getExplainPlan(ConnectContext ctx) { private void deleteExistingFilesInFE(String tvfName, Map props) throws Exception { + LocalTableValuedFunction.checkSinkEnabled(tvfName, props); // Copy props for building the storage adapter (exclude write-specific params) Map fsCopyProps = new HashMap<>(props); + HadoopTvfAdmission.removeRuntimePolicy(fsCopyProps); fsCopyProps.remove("file_path"); fsCopyProps.remove("format"); fsCopyProps.remove("delete_existing_files"); @@ -185,11 +190,27 @@ private void deleteExistingFilesInFE(String tvfName, Map props) fsCopyProps.remove("compress_type"); StorageAdapter storageAdapter = StorageAdapter.of(fsCopyProps); + if (storageAdapter.getType() == StorageTypeId.LOCAL) { + LocalTableValuedFunction.checkEnabled(); + } + if (storageAdapter.getType() == StorageTypeId.HDFS) { + HadoopTvfAdmission.check(HadoopTvfAdmission.Access.JAVA_DELETE, + FileSystemUtil.extractParentDirectory(props.get("file_path")), + storageAdapter.getBackendConfigProperties()); + } // Concrete filesystems only accept their native schemes; normalize legacy compatibility // schemes (e.g. cos:// with s3.* properties) before crossing the plugin boundary. String filePath = storageAdapter.validateAndNormalizeUri(props.get("file_path")); // Extract parent directory from prefix path: s3://bucket/path/to/prefix_ -> s3://bucket/path/to/ String parentDir = FileSystemUtil.extractParentDirectory(filePath); + if (storageAdapter.getType() == StorageTypeId.HDFS) { + HadoopTvfAdmission.check(HadoopTvfAdmission.Access.JAVA_DELETE, + parentDir, storageAdapter.getBackendConfigProperties()); + } + if (storageAdapter.getType() == StorageTypeId.HDFS) { + storageAdapter = StorageAdapter.of(HadoopTvfAdmission.protectJava( + HadoopTvfAdmission.Access.JAVA_DELETE, parentDir, fsCopyProps)); + } LOG.info("TVF sink: deleting existing files in directory: {}", parentDir); try (org.apache.doris.filesystem.FileSystem fs = FileSystemFactory.getFileSystem(storageAdapter)) { fs.delete(Location.of(parentDir), true); diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/TVFTableSink.java b/fe/fe-core/src/main/java/org/apache/doris/planner/TVFTableSink.java index 507e1c6595cb02..e8a052775b0f62 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/planner/TVFTableSink.java +++ b/fe/fe-core/src/main/java/org/apache/doris/planner/TVFTableSink.java @@ -25,6 +25,8 @@ import org.apache.doris.datasource.property.fileformat.FileFormatProperties; import org.apache.doris.datasource.property.fileformat.OrcFileFormatProperties; import org.apache.doris.datasource.storage.StorageAdapter; +import org.apache.doris.tablefunction.HadoopTvfAdmission; +import org.apache.doris.tablefunction.LocalTableValuedFunction; import org.apache.doris.thrift.TColumn; import org.apache.doris.thrift.TDataSink; import org.apache.doris.thrift.TDataSinkType; @@ -83,6 +85,7 @@ public long getBackendId() { } public void bindDataSink() throws AnalysisException { + LocalTableValuedFunction.checkSinkEnabled(tvfName, properties); TTVFTableSink tSink = new TTVFTableSink(); tSink.setTvfName(tvfName); @@ -114,6 +117,7 @@ public void bindDataSink() throws AnalysisException { tSink.setFileType(fileType); // --- 2. Parse storage/connection properties (reuse read-side storage adapter) --- + HadoopTvfAdmission.removeRuntimePolicy(propsCopy); Map backendConnectProps; if (tvfName.equals("local")) { // Local TVF: pass properties as-is (same as the local backend config map) @@ -129,6 +133,13 @@ public void bindDataSink() throws AnalysisException { } } + if (fileType == TFileType.FILE_HDFS && !properties.containsKey("writer_class")) { + HadoopTvfAdmission.check(HadoopTvfAdmission.Access.NATIVE_WRITE, + properties.get("file_path"), backendConnectProps); + backendConnectProps = HadoopTvfAdmission.protectNativeWrite( + properties.get("file_path"), backendConnectProps); + } + String filePath = properties.get("file_path"); tSink.setFilePath(filePath); diff --git a/fe/fe-core/src/main/java/org/apache/doris/tablefunction/ExternalFileTableValuedFunction.java b/fe/fe-core/src/main/java/org/apache/doris/tablefunction/ExternalFileTableValuedFunction.java index 81d7f834c78fd9..312ace0b2a3778 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/tablefunction/ExternalFileTableValuedFunction.java +++ b/fe/fe-core/src/main/java/org/apache/doris/tablefunction/ExternalFileTableValuedFunction.java @@ -152,6 +152,9 @@ public TFileCompressType getTFileCompressType() { } public Map getBackendConnectProperties() { + if (getTFileType() == TFileType.FILE_HDFS || getTFileType() == TFileType.FILE_S3) { + return HadoopTvfAdmission.protectNativeRead(backendConnectProperties); + } return backendConnectProperties; } @@ -229,6 +232,7 @@ protected Map parseCommonProperties(Map properti // Copy the properties, because we will remove the key from properties. Map copiedProps = Maps.newTreeMap(String.CASE_INSENSITIVE_ORDER); copiedProps.putAll(mergedProperties); + HadoopTvfAdmission.removeRuntimePolicy(copiedProps); tableId = Long.valueOf(getOrDefaultAndRemove(copiedProps, PROP_TABLE_ID, "-1")); @@ -295,6 +299,10 @@ public ScanNode getScanNode(PlanNodeId id, TupleDescriptor desc, SessionVariable ScanContext.builder().clusterName(sv.resolveCloudClusterName()).build()); } + protected boolean needsSchemaDiscovery() { + return csvSchema.isEmpty(); + } + @Override public List getTableColumns() throws AnalysisException { if (!csvSchema.isEmpty()) { @@ -524,7 +532,7 @@ private PFetchTableSchemaRequest getFetchTableStructureRequest(Iterator beProperties = new HashMap<>(); - beProperties.putAll(backendConnectProperties); + beProperties.putAll(getBackendConnectProperties()); fileScanRangeParams.setProperties(beProperties); fileScanRangeParams.setFileAttributes(getFileAttributes()); ConnectContext ctx = ConnectContext.get(); @@ -545,8 +553,8 @@ private PFetchTableSchemaRequest getFetchTableStructureRequest(Iterator properties = new HashMap<>(); + + private HadoopTvfAdmission(Access access, Map normalized, boolean nativeConsumer) { + this(access, normalized, nativeConsumer, false); + } + + private HadoopTvfAdmission(Access access, Map normalized, boolean nativeConsumer, + boolean discovery) { + this.access = access; + Set nativeRoutingKeys = new HashSet<>(); + for (Map.Entry entry : normalized.entrySet()) { + // parse_properties / generateHdfsParam consumes the canonical NameNode before JNI. + if (nativeConsumer && "fs.defaultFS".equalsIgnoreCase(entry.getKey())) { + continue; + } + if (nativeConsumer && isRoutingKey(asciiCaseFold(nativeStringPrefix(entry.getKey()).trim()))) { + checkNativeCharacters(access, entry.getKey()); + checkNativeCharacters(access, entry.getValue()); + } + String key = (nativeConsumer ? nativeStringPrefix(entry.getKey()) : entry.getKey()).trim(); + // Native writes first pass through StringCaseMap, before JNI truncates strings. + // Its comparator uses strncasecmp followed by the full UTF-8 byte length. + if (nativeConsumer && access == Access.NATIVE_WRITE && isRoutingKey(asciiCaseFold(key))) { + String identity = asciiCaseFold(nativeStringPrefix(entry.getKey())) + "\0" + + entry.getKey().getBytes(StandardCharsets.UTF_8).length; + if (!nativeRoutingKeys.add(identity)) { + reject(); + } + } + String value = nativeConsumer ? nativeStringPrefix(entry.getValue()) : entry.getValue(); + String previous = properties.put(key, value); + if (previous != null && !previous.equals(value) && isRoutingKey(key)) { + reject(); + } + } + routes = new TvfGuard.Route(properties, discovery); + } + + /** Check literal read, native write and cleanup paths before their filesystem IO. */ + public static void check(Access access, String path, Map normalized) { + check(access, path, normalized, true); + } + + /** A declared CSV schema skips the schema RPC; listing and data admission still apply. */ + public static void check(Access access, String path, Map normalized, + boolean needsSchemaDiscovery) { + if (access != Access.HADOOP_READ && Config.enable_local_tvf) { + return; + } + try { + if (access == Access.NATIVE_WRITE) { + String nameNode = nativeNameNode(access, normalized); + URI output = nativeFileUri(access, path + "doris-tvf-file", nameNode); + String parent = output.getPath(); + parent = parent.substring(0, parent.lastIndexOf('/') + 1); + if (parent.length() > 1) { + parent = parent.substring(0, parent.length() - 1); + } + new HadoopTvfAdmission(access, normalized, true).checkNative(nameNode, parent); + } else { + HadoopTvfAdmission javaConsumer = new HadoopTvfAdmission(access, normalized, false); + // Project the Java consumer independently from native convert_path/JNI. + URI javaPath = hadoopPathUri(path); + javaConsumer.checkRoute(javaPath, javaPath.getPath(), new HashSet<>()); + if (access == Access.HADOOP_READ) { + // FileQueryScanNode sends this filesystem identifier in range.fs_name. BE + // falls back to hdfs_params.fs_name only when that range field is empty. + String nameNode = nativeReadNameNode(path, normalized); + HadoopTvfAdmission nativeConsumer = new HadoopTvfAdmission(access, normalized, true); + nativeConsumer.checkNativeRead(nameNode, path); + // Schema RPCs have no range.fs_name; they use THdfsParams.fs_name instead. + if (needsSchemaDiscovery) { + nativeConsumer.checkNativeRead(nativeNameNode(access, normalized), path); + } + } + } + } catch (URISyntaxException | IllegalArgumentException e) { + new HadoopTvfAdmission(access, normalized, false).reject(); + } + } + + /** Prove the Java listing probes; native readers consume the returned literal paths. */ + public static void checkReadDiscovery(String path, Map normalized) { + try { + HadoopTvfAdmission javaConsumer = new HadoopTvfAdmission(Access.HADOOP_READ, normalized, false, true); + if (hasListingGlob(path)) { + URI listingPath = hadoopPathUri(GlobPattern.expandNumericRanges(path)); + javaConsumer.routes.checkGlobRoute(listingPath, listingPath.getPath(), new HashSet<>()); + } else { + URI listingPath = hadoopPathUri(path); + javaConsumer.checkRoute(listingPath, listingPath.getPath(), new HashSet<>()); + } + } catch (IOException | URISyntaxException | IllegalArgumentException e) { + reject(Access.HADOOP_READ); + } + } + + private static String nativeReadNameNode(String path, Map normalized) { + String nameNode = LocationPath.of(path).getFsIdentifier(); + if (nameNode.isEmpty()) { + nameNode = nativeNameNode(Access.HADOOP_READ, normalized); + } + return nameNode; + } + + /** Reserved internal policy must not be selected by SQL/resource property aliases. */ + public static void removeRuntimePolicy(Map properties) { + properties.keySet().removeIf(key -> asciiCaseFold(nativeStringPrefix(key)).trim() + .startsWith("fs.doris.tvf.")); + } + + private static boolean hasListingGlob(String path) { + return path.indexOf('*') >= 0 || path.indexOf('?') >= 0 + || path.indexOf('[') >= 0 || path.indexOf('{') >= 0; + } + + public static Map protectJava(Access access, String path, Map properties) { + Map guarded = new HashMap<>(properties); + removeRuntimePolicy(guarded); + if (access == Access.HADOOP_READ || !Config.enable_local_tvf) { + try { + URI uri = hadoopPathUri(path); + java.util.List rules; + if (access == Access.HADOOP_READ && hasListingGlob(path)) { + URI listingPath = hadoopPathUri(GlobPattern.expandNumericRanges(path)); + rules = TvfGuard.globRules(listingPath, listingPath.getPath()); + } else { + rules = java.util.Collections.singletonList( + new TvfGuard.Rule(uri.toString(), uri.getPath(), access == Access.HADOOP_READ)); + } + guarded.put(TvfGuard.POLICY_KEY, TvfGuard.encode(rules)); + } catch (IOException | URISyntaxException | IllegalArgumentException e) { + reject(access); + } + } + return guarded; + } + + public static Map protectNativeWrite(String path, Map normalized) { + Map guarded = new HashMap<>(normalized); + removeRuntimePolicy(guarded); + if (!Config.enable_local_tvf) { + guarded.put(TvfGuard.POLICY_KEY, TvfGuard.NATIVE_WRITE_POLICY); + } + return guarded; + } + + public static Map protectNativeRead(Map normalized) { + Map guarded = new HashMap<>(normalized); + removeRuntimePolicy(guarded); + guarded.put(TvfGuard.POLICY_KEY, TvfGuard.NATIVE_POLICY); + return guarded; + } + + private static String nativeNameNode(Access access, Map normalized) { + String result = null; + for (Map.Entry entry : normalized.entrySet()) { + if ("fs.defaultFS".equalsIgnoreCase(entry.getKey())) { + if (result != null && !result.equals(entry.getValue())) { + reject(access); + } + result = entry.getValue(); + } + } + return result == null ? "" : result; + } + + private void checkNative(String nameNode, String path) throws URISyntaxException { + checkNativeCharacters(access, nameNode); + nameNode = selector(nativeStringPrefix(nameNode)).trim(); + if (nameNode.isEmpty() && access == Access.NATIVE_WRITE) { + // BE passes fs_name.c_str(), never NULL. libhdfs prefixes this empty string + // with hdfs://, whose URI construction fails before any filesystem is selected. + // Preserve existing analysis; execution reports the invalid NameNode as before. + return; + } + if ("default".equals(nameNode) || nameNode.isEmpty()) { + reject(); + } + checkRoute(new URI(nameNode.contains("://") ? nameNode : "hdfs://" + nameNode), + path, new HashSet<>()); + } + + private void checkNativeRead(String nameNode, String path) throws URISyntaxException { + checkNative(nameNode, nativeFileUri(access, path, nameNode).getPath()); + } + + private void checkRoute(URI filesystem, String path, Set activeTables) throws URISyntaxException { + routes.checkRoute(filesystem, path, activeTables); + } + + private static URI nativeFileUri(Access access, String path, String nameNode) + throws URISyntaxException { + checkNativeCharacters(access, path); + // Match BE convert_path before the JNI C-string conversion. A different URI prefix + // becomes a literal rooted Hadoop path; it must not be parsed as that URI's path. + if (!nameNode.isEmpty() && path.contains(nameNode)) { + if (!path.startsWith(nameNode)) { + // convert_path truncates bytes at NameNode.length even for a non-prefix match. + // Java character offsets cannot establish that malformed path's mount identity. + reject(access); + } + path = path.substring(nameNode.length()); + } + if (!path.startsWith("/")) { + path = "/" + path; + } + return hadoopPathUri(nativeStringPrefix(path)); + } + + private static URI hadoopPathUri(String path) throws URISyntaxException { + String scheme = uriScheme(path); + String authority = null; + String relative = scheme.isEmpty() ? path : path.substring(scheme.length() + 1); + if (relative.startsWith("//")) { + int slash = relative.indexOf('/', 2); + authority = slash < 0 ? relative.substring(2) : relative.substring(2, slash); + relative = slash < 0 ? "/" : relative.substring(slash); + } + if (!scheme.isEmpty() && !relative.startsWith("/")) { + relative = "/" + relative; + } + // Hadoop Path(String) quotes path bytes before URI normalization; percent text is literal. + return new URI(scheme.isEmpty() ? null : scheme, authority, relative, null, null).normalize(); + } + + private static String uriScheme(String path) { + int colon = path.indexOf(':'); + int slash = path.indexOf('/'); + return colon > 0 && (slash < 0 || colon < slash) ? path.substring(0, colon) : ""; + } + + private String selector(String value) { + if (value.contains("${")) { + // Runtime environment and Hadoop substitutions are not available to this FE projection. + reject(); + } + return value; + } + + private static boolean isRoutingKey(String key) { + return "fs.defaultFS".equalsIgnoreCase(key) || "fs.default.name".equals(key) + || key.startsWith("fs.viewfs.") || (key.startsWith("fs.") && key.endsWith(".impl")); + } + + private static String asciiCaseFold(String value) { + StringBuilder folded = new StringBuilder(value.length()); + for (int i = 0; i < value.length(); i++) { + char c = value.charAt(i); + folded.append(c >= 'A' && c <= 'Z' ? (char) (c + ('a' - 'A')) : c); + } + return folded.toString(); + } + + private static void checkNativeCharacters(Access access, String value) { + // libhdfs uses NewStringUTF on standard UTF-8 bytes. Surrogates encode differently + // in modified UTF-8, so supplementary/unpaired input cannot prove the same route. + for (int i = 0; i < value.length() && value.charAt(i) != '\0'; i++) { + if (Character.isSurrogate(value.charAt(i))) { + reject(access); + } + } + } + + private static String nativeStringPrefix(String value) { + int nul = value.indexOf('\0'); + return nul < 0 ? value : value.substring(0, nul); + } + + private void reject() { + reject(access); + } + + private static void reject(Access access) { + if (access == Access.HADOOP_READ) { + throw new AnalysisException("Hadoop TVFs require a proven remote filesystem route. " + + "Use local() for local files with LOCAL TVF privileges and secure-path restrictions."); + } + LocalTableValuedFunction.checkEnabled(); + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/tablefunction/HdfsTableValuedFunction.java b/fe/fe-core/src/main/java/org/apache/doris/tablefunction/HdfsTableValuedFunction.java index fd4189dfc53fbc..c967012ad82afe 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/tablefunction/HdfsTableValuedFunction.java +++ b/fe/fe-core/src/main/java/org/apache/doris/tablefunction/HdfsTableValuedFunction.java @@ -22,8 +22,10 @@ import org.apache.doris.common.AnalysisException; import org.apache.doris.common.FeConstants; import org.apache.doris.datasource.storage.StorageAdapter; +import org.apache.doris.thrift.TBrokerFileStatus; import org.apache.doris.thrift.TFileType; +import org.apache.hadoop.fs.FileSystem.TvfGuard; import org.apache.log4j.LogManager; import org.apache.log4j.Logger; @@ -56,9 +58,16 @@ private void init(Map properties) throws AnalysisException { throw new AnalysisException("Failed check storage props, " + e.getMessage(), e); } + HadoopTvfAdmission.checkReadDiscovery(filePath, backendConnectProperties); + processedParams.put(TvfGuard.POLICY_KEY, HadoopTvfAdmission.protectJava( + HadoopTvfAdmission.Access.HADOOP_READ, filePath, processedParams).get(TvfGuard.POLICY_KEY)); if (!FeConstants.runningUnitTest) { // 4. parse file parseFile(); + for (TBrokerFileStatus file : fileStatuses) { + HadoopTvfAdmission.check(HadoopTvfAdmission.Access.HADOOP_READ, + file.getPath(), backendConnectProperties, needsSchemaDiscovery()); + } } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/tablefunction/LocalTableValuedFunction.java b/fe/fe-core/src/main/java/org/apache/doris/tablefunction/LocalTableValuedFunction.java index b462c395b1c3b1..a2791325e1683c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/tablefunction/LocalTableValuedFunction.java +++ b/fe/fe-core/src/main/java/org/apache/doris/tablefunction/LocalTableValuedFunction.java @@ -21,6 +21,7 @@ import org.apache.doris.analysis.StorageBackend.StorageType; import org.apache.doris.catalog.Env; import org.apache.doris.common.AnalysisException; +import org.apache.doris.common.Config; import org.apache.doris.common.ErrorCode; import org.apache.doris.datasource.storage.StorageAdapter; import org.apache.doris.mysql.privilege.PrivPredicate; @@ -63,6 +64,8 @@ public class LocalTableValuedFunction extends ExternalFileTableValuedFunction { private boolean sharedStorage = false; public LocalTableValuedFunction(Map properties) throws AnalysisException { + checkEnabled(); + // Check privileges before listing files or discovering their schema. checkAuth(ConnectContext.get()); @@ -129,7 +132,7 @@ private void getFileListFromBackend() throws AnalysisException { + ", error msg: " + globResponse.getStatus().getErrorMsgsList()); } for (PFileInfo file : globResponse.getFilesList()) { - fileStatuses.add(new TBrokerFileStatus(file.getFile().trim(), false, file.getSize(), true)); + fileStatuses.add(new TBrokerFileStatus(file.getFile(), false, file.getSize(), true)); LOG.info("get file from backend success. file: {}, size: {}", file.getFile(), file.getSize()); } } catch (Exception e) { @@ -137,6 +140,33 @@ private void getFileListFromBackend() throws AnalysisException { } } + /** Apply LOCAL admission to the effective sink before planning or filesystem side effects. */ + public static void checkSinkEnabled(String tvfName, Map properties) { + if ("local".equalsIgnoreCase(tvfName) || isLocalWriter(properties.getOrDefault("writer_class", ""))) { + checkEnabled(); + } + } + + private static boolean isLocalWriter(String writer) { + // BE splits the complete identifier first, then passes each half through JNI C strings. + int separator = writer.indexOf(':'); + return separator > 0 && "java-writer".equals(nativeStringPrefix(writer.substring(0, separator))) + && "local-file".equals(nativeStringPrefix(writer.substring(separator + 1))); + } + + private static String nativeStringPrefix(String value) { + int nul = value.indexOf('\0'); + return nul < 0 ? value : value.substring(0, nul); + } + + /** Check read and write admission before properties or backend access. */ + public static void checkEnabled() { + if (!Config.enable_local_tvf) { + throw new org.apache.doris.nereids.exceptions.AnalysisException( + "LOCAL TVF is disabled. Set enable_local_tvf=true in fe.conf and restart every FE to enable it."); + } + } + @Override public void checkAuth(ConnectContext ctx) { if (!Env.getCurrentEnv().getAccessManager().checkGlobalPriv(ctx, PrivPredicate.ADMIN) diff --git a/fe/fe-core/src/main/java/org/apache/doris/tablefunction/S3TableValuedFunction.java b/fe/fe-core/src/main/java/org/apache/doris/tablefunction/S3TableValuedFunction.java index 11d44962e15bbe..6a2a76601019fa 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/tablefunction/S3TableValuedFunction.java +++ b/fe/fe-core/src/main/java/org/apache/doris/tablefunction/S3TableValuedFunction.java @@ -20,10 +20,14 @@ import org.apache.doris.analysis.BrokerDesc; import org.apache.doris.common.AnalysisException; import org.apache.doris.common.FeConstants; +import org.apache.doris.common.util.LocationPath; import org.apache.doris.datasource.storage.StorageAdapter; +import org.apache.doris.datasource.storage.StorageTypeId; import org.apache.doris.thrift.TBrokerFileStatus; import org.apache.doris.thrift.TFileType; +import org.apache.hadoop.fs.FileSystem.TvfGuard; + import java.util.List; import java.util.Map; @@ -54,11 +58,30 @@ public S3TableValuedFunction(Map properties, List analyzeInsert(prefix + "INSERT INTO local(" + properties + ") SELECT 1")); + Assertions.assertTrue(exception.getMessage().contains("LOCAL TVF is disabled")); + } + } + } + + @Test + void enabledLocalSinkKeepsPropertyValidation() { + Config.enable_local_tvf = true; + AnalysisException exception = Assertions.assertThrows(AnalysisException.class, + () -> analyzeInsert("INSERT INTO local('format'='csv') SELECT 1")); + Assertions.assertTrue(exception.getMessage().contains("requires 'file_path'")); + } + + @Test + void disabledLocalDoesNotBlockRemoteSinks() { + for (String name : Arrays.asList("s3", "hdfs")) { + Assertions.assertInstanceOf(LogicalTVFTableSink.class, + analyzeInsert("INSERT INTO " + name + "('file_path'='remote/data', 'format'='csv') SELECT 1")); + } + } + + @Test + void disabledLocalWriterRejectsRemoteInsertAndExplain() { + for (String prefix : Arrays.asList("", "EXPLAIN ")) { + for (String name : Arrays.asList("s3", "hdfs")) { + for (String writer : Arrays.asList("java-writer:local-file")) { + for (String props : Arrays.asList("", "'file_path'='/tmp/local_gate_', 'format'='csv',")) { + AnalysisException exception = Assertions.assertThrows(AnalysisException.class, + () -> analyzeInsert(prefix + "INSERT INTO " + name + "(" + props + + "'writer_class'='" + writer + "') SELECT 1")); + Assertions.assertTrue(exception.getMessage().contains("LOCAL TVF is disabled")); + } + } + } + } + } + + @Test + void directSinkBindingChecksEffectiveLocalWriter() throws Exception { + Map properties = new HashMap<>(); + properties.put("file_path", "s3://bucket/data"); + properties.put("format", "csv"); + properties.put("s3.endpoint", "http://127.0.0.1:9000"); + properties.put("s3.region", "us-east-1"); + properties.put("s3.access_key", "test-key"); + properties.put("s3.secret_key", "test-secret"); + for (String name : Arrays.asList("s3", "hdfs", "local")) { + for (String writer : Arrays.asList("java-writer:local-file", + "java-writer\0ignored:local-file", "java-writer:local-file\0ignored")) { + properties.put("writer_class", writer); + TVFTableSink sink = new TVFTableSink(new PlanNodeId(0), name, properties, + Collections.emptyList()); + AnalysisException exception = Assertions.assertThrows(AnalysisException.class, sink::bindDataSink); + Assertions.assertTrue(exception.getMessage().contains("LOCAL TVF is disabled")); + } + } + Config.enable_local_tvf = true; + properties.put("writer_class", "java-writer:local-file"); + new TVFTableSink(new PlanNodeId(0), "s3", properties, Collections.emptyList()).bindDataSink(); + Config.enable_local_tvf = false; + properties.remove("writer_class"); + new TVFTableSink(new PlanNodeId(0), "s3", properties, Collections.emptyList()).bindDataSink(); + properties.put("writer_class", "java-writer:other-writer"); + Assertions.assertDoesNotThrow(() -> LocalTableValuedFunction.checkSinkEnabled("s3", properties)); + properties.put("writer_class", "java-writer:local-file:other"); + Assertions.assertDoesNotThrow(() -> LocalTableValuedFunction.checkSinkEnabled("s3", properties)); + } + + @Test + void enabledLocalWriterKeepsRemotePropertyValidation() { + Config.enable_local_tvf = true; + for (String name : Arrays.asList("s3", "hdfs")) { + AnalysisException exception = Assertions.assertThrows(AnalysisException.class, + () -> analyzeInsert("INSERT INTO " + name + + "('writer_class'='java-writer:local-file') SELECT 1")); + Assertions.assertTrue(exception.getMessage().contains("requires 'file_path'")); + } + } + + @Test + void loadUserCannotExecuteLocalWriterWhileDisabled() { + ConnectContext context = MemoTestUtils.createConnectContext(); + String sql = "INSERT INTO s3('file_path'='/tmp/local_gate_', 'format'='csv'," + + "'writer_class'='java-writer:local-file') SELECT 1"; + MemoTestUtils.createStatementContext(context, sql); + InsertIntoTVFCommand command = (InsertIntoTVFCommand) new NereidsParser().parseSingle(sql); + Env env = Mockito.spy(Env.getCurrentEnv()); + AccessControllerManager accessManager = Mockito.mock(AccessControllerManager.class); + Mockito.doReturn(accessManager).when(env).getAccessManager(); + Mockito.when(accessManager.checkGlobalPriv(context, PrivPredicate.LOAD)).thenReturn(true); + StmtExecutor executor = Mockito.mock(StmtExecutor.class); + try (MockedStatic mockedEnv = Mockito.mockStatic(Env.class, Mockito.CALLS_REAL_METHODS)) { + mockedEnv.when(Env::getCurrentEnv).thenReturn(env); + AnalysisException exception = Assertions.assertThrows(AnalysisException.class, + () -> command.run(context, executor)); + Assertions.assertTrue(exception.getMessage().contains("LOCAL TVF is disabled")); + Mockito.verify(accessManager).checkGlobalPriv(context, PrivPredicate.ADMIN); + Mockito.verify(accessManager).checkGlobalPriv(context, PrivPredicate.LOAD); + Mockito.verifyNoInteractions(executor); + } + } + + @Test + void disabledNativeHadoopLocalSinksRejectInsertAndExplain() { + for (String prefix : Arrays.asList("", "EXPLAIN ")) { + for (String storage : Arrays.asList("'fs.defaultFS'='file:///'", + "'fs.defaultFS'='hdfs://namenode:8020'," + + "'fs.hdfs.impl'='org.apache.hadoop.fs.LocalFileSystem'", + "'fs.defaultFS'='hdfs://namenode:8020'," + + "'fs.hdfs.impl'='org.apache.hadoop.fs.RawLocalFileSystem'", + "'fs.defaultFS'='hdfs://namenode:8020','fs.hdfs.impl'='${fs.file.impl}'," + + "'fs.file.impl'='org.apache.hadoop.fs.LocalFileSystem'", + "'fs.defaultFS'='hdfs://namenode:8020'," + + "'fs.hdfs.impl '='org.apache.hadoop.fs.LocalFileSystem'")) { + for (String cleanup : Arrays.asList("false", "true")) { + AnalysisException exception = Assertions.assertThrows(AnalysisException.class, + () -> analyzeInsert(prefix + "INSERT INTO hdfs('file_path'='/tmp/local_gate_'," + + "'format'='csv','delete_existing_files'='" + cleanup + "'," + + storage + ") SELECT 1")); + Assertions.assertTrue(exception.getMessage().contains("LOCAL TVF is disabled")); + } + } + } + AnalysisException exception = Assertions.assertThrows(AnalysisException.class, + () -> analyzeInsert("INSERT INTO hdfs('fs.defaultFS'='file:///') SELECT 1")); + Assertions.assertTrue(exception.getMessage().contains("requires 'file_path'")); + } + + @Test + void directNativeHadoopBindingHonorsLocalOptIn() throws Exception { + Map properties = new HashMap<>(); + properties.put("file_path", "/tmp/local_gate_"); + properties.put("format", "csv"); + for (String defaultFs : Arrays.asList("file:///", "default")) { + properties.put("fs.defaultFS", defaultFs); + TVFTableSink sink = new TVFTableSink(new PlanNodeId(0), "hdfs", properties, + Collections.emptyList()); + AnalysisException exception = Assertions.assertThrows(AnalysisException.class, sink::bindDataSink); + Assertions.assertTrue(exception.getMessage().contains("LOCAL TVF is disabled")); + } + properties.put("fs.defaultFS", "hdfs://namenode:8020"); + properties.put("fs.hdfs.impl", "org.apache.hadoop.fs.RawLocalFileSystem"); + TVFTableSink sink = new TVFTableSink(new PlanNodeId(0), "hdfs", properties, Collections.emptyList()); + Assertions.assertThrows(AnalysisException.class, sink::bindDataSink); + Config.enable_local_tvf = true; + sink.bindDataSink(); + properties.put("fs.defaultFS", "file:///"); + sink.bindDataSink(); + Config.enable_local_tvf = false; + properties.put("fs.defaultFS", "hdfs://namenode:8020"); + properties.remove("fs.hdfs.impl"); + // Hadoop normally provides a local implementation for file; it is not selected by hdfs. + properties.put("fs.file.impl", "org.apache.hadoop.fs.LocalFileSystem"); + sink.bindDataSink(); + Assertions.assertInstanceOf(LogicalTVFTableSink.class, + analyzeInsert("INSERT INTO hdfs('file_path'='/tmp/remote_', 'format'='csv'," + + "'fs.defaultFS'='hdfs://namenode:8020'," + + "'fs.file.impl'='org.apache.hadoop.fs.LocalFileSystem') SELECT 1")); + } + + @Test + void normalizedHadoopConfigIsCheckedBeforeNativeBinding() throws Exception { + String originalConfigDir = Config.hadoop_config_dir; + String originalConfigProperty = System.getProperty("doris.hadoop.config.dir"); + Path directory = Files.createTempDirectory("tvf-hadoop-admission-"); + Path resource = directory.resolve("local-filesystem.xml"); + try { + Files.write(resource, ("fs.defaultFS" + + "file:///").getBytes(StandardCharsets.UTF_8)); + Config.hadoop_config_dir = directory.toString() + "/"; + Map properties = new HashMap<>(); + properties.put("file_path", "/tmp/local_gate_"); + properties.put("format", "csv"); + properties.put("fs.hdfs.support", "true"); + properties.put("hadoop.config.resources", resource.getFileName().toString()); + TVFTableSink sink = new TVFTableSink(new PlanNodeId(0), "hdfs", properties, + Collections.emptyList()); + AnalysisException exception = Assertions.assertThrows(AnalysisException.class, sink::bindDataSink); + Assertions.assertTrue(exception.getMessage().contains("LOCAL TVF is disabled")); + Config.enable_local_tvf = true; + sink.bindDataSink(); + } finally { + Config.hadoop_config_dir = originalConfigDir; + if (originalConfigProperty == null) { + System.clearProperty("doris.hadoop.config.dir"); + } else { + System.setProperty("doris.hadoop.config.dir", originalConfigProperty); + } + Files.deleteIfExists(resource); + Files.deleteIfExists(directory); + } + } + + @Test + void hadoopCleanupChecksSelectedFilesystemWithoutOpeningIt() { + Map properties = new HashMap<>(); + properties.put("fs.defaultFS", "hdfs://namenode:8020"); + properties.put("fs.file.impl", "org.apache.hadoop.fs.LocalFileSystem"); + Assertions.assertDoesNotThrow(() -> HadoopTvfAdmission.check(HadoopTvfAdmission.Access.JAVA_DELETE, + "/remote/path", properties)); + Assertions.assertThrows(AnalysisException.class, + () -> HadoopTvfAdmission.check(HadoopTvfAdmission.Access.JAVA_DELETE, + "file:///tmp/prefix_", properties)); + properties.remove("fs.defaultFS"); + for (String defaultFs : Arrays.asList("local", "file:///")) { + properties.put("fs.default.name", defaultFs); + Assertions.assertThrows(AnalysisException.class, + () -> HadoopTvfAdmission.check(HadoopTvfAdmission.Access.JAVA_DELETE, "/tmp/prefix_", properties)); + } + properties.clear(); + Assertions.assertThrows(AnalysisException.class, + () -> HadoopTvfAdmission.check(HadoopTvfAdmission.Access.JAVA_DELETE, "/tmp/prefix_", properties)); + properties.put("fs.defaultFS", "hdfs://namenode:8020"); + properties.put("fs.default.name", "file:///"); + Assertions.assertThrows(AnalysisException.class, + () -> HadoopTvfAdmission.check(HadoopTvfAdmission.Access.JAVA_DELETE, "/tmp/prefix_", properties)); + properties.put("fs.default.name", "hdfs://other-namenode:8020"); + Assertions.assertDoesNotThrow(() -> HadoopTvfAdmission.check(HadoopTvfAdmission.Access.JAVA_DELETE, + "/remote/path", properties)); + properties.remove("fs.defaultFS"); + Assertions.assertDoesNotThrow(() -> HadoopTvfAdmission.check(HadoopTvfAdmission.Access.JAVA_DELETE, + "/remote/path", properties)); + properties.clear(); + properties.put("fs.defaultFS", "viewfs://cluster"); + properties.put("fs.viewfs.impl", "org.apache.hadoop.fs.RawLocalFileSystem"); + Assertions.assertThrows(AnalysisException.class, + () -> HadoopTvfAdmission.check(HadoopTvfAdmission.Access.JAVA_DELETE, + "viewfs://cluster/tmp", properties)); + properties.clear(); + properties.put("fs.defaultFS", "${fs.selected.uri}"); + properties.put("fs.selected.uri", "file:///"); + Assertions.assertThrows(AnalysisException.class, + () -> HadoopTvfAdmission.check(HadoopTvfAdmission.Access.JAVA_DELETE, "/tmp/prefix_", properties)); + properties.put("fs.selected.uri", "hdfs://namenode:8020"); + Assertions.assertThrows(AnalysisException.class, + () -> HadoopTvfAdmission.check(HadoopTvfAdmission.Access.JAVA_DELETE, "/remote/path", properties)); + Config.enable_local_tvf = true; + Assertions.assertDoesNotThrow(() -> HadoopTvfAdmission.check(HadoopTvfAdmission.Access.JAVA_DELETE, + "/remote/path", properties)); + Config.enable_local_tvf = false; + properties.put("fs.defaultFS", "hdfs://namenode:8020"); + properties.put("fs.hdfs.impl", "${fs.selected.impl}"); + properties.put("fs.selected.impl", "org.apache.hadoop.fs.RawLocalFileSystem"); + Assertions.assertThrows(AnalysisException.class, + () -> HadoopTvfAdmission.check(HadoopTvfAdmission.Access.NATIVE_WRITE, "", properties)); + properties.put("fs.selected.impl", "org.apache.hadoop.hdfs.DistributedFileSystem"); + Assertions.assertThrows(AnalysisException.class, + () -> HadoopTvfAdmission.check(HadoopTvfAdmission.Access.NATIVE_WRITE, "", properties)); + Config.enable_local_tvf = true; + Assertions.assertDoesNotThrow(() -> HadoopTvfAdmission.check(HadoopTvfAdmission.Access.NATIVE_WRITE, + "", properties)); + Config.enable_local_tvf = false; + properties.put("fs.hdfs.impl", "${fs.selected.impl}"); + properties.remove("fs.selected.impl"); + properties.put("fs.selected.impl ", "org.apache.hadoop.fs.LocalFileSystem"); + Assertions.assertThrows(AnalysisException.class, + () -> HadoopTvfAdmission.check(HadoopTvfAdmission.Access.NATIVE_WRITE, "", properties)); + properties.put("fs.selected.impl ", "org.apache.hadoop.hdfs.DistributedFileSystem"); + Assertions.assertThrows(AnalysisException.class, + () -> HadoopTvfAdmission.check(HadoopTvfAdmission.Access.NATIVE_WRITE, "", properties)); + Config.enable_local_tvf = true; + Assertions.assertDoesNotThrow(() -> HadoopTvfAdmission.check(HadoopTvfAdmission.Access.NATIVE_WRITE, + "", properties)); + Config.enable_local_tvf = false; + // BE consumes canonical defaultFS as its NameNode; the old alias can redefine + // that key inside Hadoop. Do not infer this indirect provider from the FE's raw map. + properties.put("fs.default.name", "org.apache.hadoop.fs.LocalFileSystem"); + properties.put("fs.hdfs.impl", "${fs.defaultFS}"); + Assertions.assertThrows(AnalysisException.class, + () -> HadoopTvfAdmission.check(HadoopTvfAdmission.Access.NATIVE_WRITE, "", properties)); + properties.put("fs.hdfs.impl", "${env:TVF_TEST_UNRESOLVED_IMPL}"); + Assertions.assertThrows(AnalysisException.class, + () -> HadoopTvfAdmission.check(HadoopTvfAdmission.Access.NATIVE_WRITE, "", properties)); + Config.enable_local_tvf = true; + Assertions.assertDoesNotThrow(() -> HadoopTvfAdmission.check(HadoopTvfAdmission.Access.NATIVE_WRITE, + "", properties)); + Config.enable_local_tvf = false; + properties.clear(); + properties.put("fs.defaultFS", "hdfs://namenode:8020"); + properties.put("fs.hdfs.impl\0ignored", "org.apache.hadoop.fs.LocalFileSystem\0ignored"); + Assertions.assertThrows(AnalysisException.class, + () -> HadoopTvfAdmission.check(HadoopTvfAdmission.Access.NATIVE_WRITE, "", properties)); + } + + @Test + void cleanupEntryRejectsHadoopLocalStorageBeforeFilesystemCreation() throws Exception { + InsertIntoTVFCommand command = (InsertIntoTVFCommand) new NereidsParser().parseSingle( + "INSERT INTO s3('file_path'='/tmp/local_gate_', 'format'='csv') SELECT 1"); + Method cleanup = InsertIntoTVFCommand.class.getDeclaredMethod( + "deleteExistingFilesInFE", String.class, Map.class); + cleanup.setAccessible(true); + Map properties = new HashMap<>(); + properties.put("file_path", "/tmp/local_gate_"); + properties.put("format", "csv"); + properties.put("delete_existing_files", "true"); + properties.put("fs.defaultFS", "file:///"); + try (MockedStatic factory = Mockito.mockStatic(FileSystemFactory.class)) { + // The nominal S3 name must not hide Hadoop storage selected for FE-side cleanup. + InvocationTargetException exception = Assertions.assertThrows(InvocationTargetException.class, + () -> cleanup.invoke(command, "s3", properties)); + Assertions.assertInstanceOf(AnalysisException.class, exception.getCause()); + Assertions.assertTrue(exception.getCause().getMessage().contains("LOCAL TVF is disabled")); + factory.verifyNoInteractions(); + } + } + + @Test + void loadUserCannotReachHadoopLocalCleanupOrExecution() { + ConnectContext context = MemoTestUtils.createConnectContext(); + String sql = "INSERT INTO hdfs('file_path'='/tmp/local_gate_', 'format'='csv'," + + "'delete_existing_files'='true', 'fs.defaultFS'='file:///') SELECT 1"; + MemoTestUtils.createStatementContext(context, sql); + InsertIntoTVFCommand command = (InsertIntoTVFCommand) new NereidsParser().parseSingle(sql); + Env env = Mockito.spy(Env.getCurrentEnv()); + AccessControllerManager accessManager = Mockito.mock(AccessControllerManager.class); + Mockito.doReturn(accessManager).when(env).getAccessManager(); + Mockito.when(accessManager.checkGlobalPriv(context, PrivPredicate.LOAD)).thenReturn(true); + StmtExecutor executor = Mockito.mock(StmtExecutor.class); + try (MockedStatic mockedEnv = Mockito.mockStatic(Env.class, Mockito.CALLS_REAL_METHODS); + MockedStatic factory = Mockito.mockStatic(FileSystemFactory.class)) { + mockedEnv.when(Env::getCurrentEnv).thenReturn(env); + AnalysisException exception = Assertions.assertThrows(AnalysisException.class, + () -> command.run(context, executor)); + Assertions.assertTrue(exception.getMessage().contains("LOCAL TVF is disabled")); + Mockito.verify(accessManager).checkGlobalPriv(context, PrivPredicate.ADMIN); + Mockito.verify(accessManager).checkGlobalPriv(context, PrivPredicate.LOAD); + Mockito.verifyNoInteractions(executor); + factory.verifyNoInteractions(); + } + } + + @Test + void disabledViewFsLocalMountsRejectInsertAndExplain() { + for (String prefix : Arrays.asList("", "EXPLAIN ")) { + for (String cleanup : Arrays.asList("false", "true")) { + AnalysisException exception = Assertions.assertThrows(AnalysisException.class, + () -> analyzeInsert(prefix + "INSERT INTO hdfs('file_path'='/local/sub/prefix_'," + + "'format'='csv','delete_existing_files'='" + cleanup + "'," + + "'fs.defaultFS'='viewfs://cluster'," + + "'fs.viewfs.mounttable.cluster.link./local'='file:///tmp') SELECT 1")); + Assertions.assertTrue(exception.getMessage().contains("LOCAL TVF is disabled")); + } + } + } + + @Test + void viewFsChecksSelectedTargetsAndPreservesRemoteMounts() throws Exception { + Map properties = new HashMap<>(); + properties.put("file_path", "/remote/prefix_"); + properties.put("format", "csv"); + properties.put("fs.defaultFS", "viewfs://cluster"); + properties.put("fs.viewfs.mounttable.cluster.link./remote", "hdfs://namenode:8020/base"); + properties.put("fs.viewfs.mounttable.cluster.link./local", "file:///tmp"); + properties.put("fs.viewfs.mounttable.cluster.linkFallback", "hdfs://namenode:8020/fallback"); + new TVFTableSink(new PlanNodeId(0), "hdfs", properties, Collections.emptyList()).bindDataSink(); + Assertions.assertDoesNotThrow(() -> HadoopTvfAdmission.check(HadoopTvfAdmission.Access.NATIVE_WRITE, + "/locality/prefix_", properties)); + // A prefix naming a mount point generates a sibling filename; a trailing slash writes inside it. + Assertions.assertDoesNotThrow(() -> HadoopTvfAdmission.check(HadoopTvfAdmission.Access.NATIVE_WRITE, + "/local", properties)); + Assertions.assertInstanceOf(LogicalTVFTableSink.class, + analyzeInsert("INSERT INTO hdfs('file_path'='/local', 'format'='csv'," + + "'delete_existing_files'='true', 'fs.defaultFS'='viewfs://cluster'," + + "'fs.viewfs.mounttable.cluster.link./local'='file:///tmp'," + + "'fs.viewfs.mounttable.cluster.linkFallback'='hdfs://namenode:8020/base') SELECT 1")); + Assertions.assertThrows(AnalysisException.class, + () -> HadoopTvfAdmission.check(HadoopTvfAdmission.Access.NATIVE_WRITE, "/local/", properties)); + properties.put("file_path", "/local/sub/prefix_"); + TVFTableSink sink = new TVFTableSink(new PlanNodeId(0), "hdfs", properties, Collections.emptyList()); + Assertions.assertThrows(AnalysisException.class, sink::bindDataSink); + Config.enable_local_tvf = true; + sink.bindDataSink(); + Config.enable_local_tvf = false; + properties.remove("fs.viewfs.mounttable.cluster.link./local"); + properties.put("fs.viewfs.mounttable.cluster.link./local/", "file:///tmp"); + Assertions.assertThrows(AnalysisException.class, + () -> HadoopTvfAdmission.check(HadoopTvfAdmission.Access.NATIVE_WRITE, + "/local/sub/prefix_", properties)); + properties.remove("fs.viewfs.mounttable.cluster.link./local/"); + properties.put("fs.viewfs.mounttable.cluster.link./local", "file:///tmp"); + properties.put("fs.viewfs.mounttable.cluster.link./local/remote", "hdfs://namenode:8020/base"); + Assertions.assertDoesNotThrow(() -> HadoopTvfAdmission.check(HadoopTvfAdmission.Access.NATIVE_WRITE, + "/local/remote/prefix_", properties)); + properties.put("fs.defaultFS", "viewfs://cl%75ster"); + Assertions.assertThrows(AnalysisException.class, + () -> HadoopTvfAdmission.check(HadoopTvfAdmission.Access.NATIVE_WRITE, + "/local/sub/prefix_", properties)); + properties.put("fs.defaultFS", "viewfs://cluster_name"); + properties.put("fs.viewfs.ignore.port.in.mount.table.name", "true"); + properties.put("fs.viewfs.mounttable.cluster_name.link./remote", "hdfs://namenode:8020/base"); + Assertions.assertDoesNotThrow(() -> HadoopTvfAdmission.check(HadoopTvfAdmission.Access.NATIVE_WRITE, + "/remote/prefix_", properties)); + properties.put("fs.defaultFS", "viewfs://cluster:8020"); + properties.put("fs.viewfs.ignore.port.in.mount.table.name", "true"); + Assertions.assertThrows(AnalysisException.class, + () -> HadoopTvfAdmission.check(HadoopTvfAdmission.Access.NATIVE_WRITE, + "/local/sub/prefix_", properties)); + properties.clear(); + properties.put("fs.defaultFS", "viewfs:///"); + properties.put("fs.viewfs.mounttable.default.name.key", "named"); + properties.put("fs.viewfs.mounttable.named.link./local", "file:///tmp"); + Assertions.assertThrows(AnalysisException.class, + () -> HadoopTvfAdmission.check(HadoopTvfAdmission.Access.NATIVE_WRITE, + "/local/sub/prefix_", properties)); + properties.clear(); + properties.put("fs.defaultFS", "viewfs://cluster"); + properties.put("fs.viewfs.mounttable.cluster.linkFallback", "file:///tmp"); + Assertions.assertThrows(AnalysisException.class, + () -> HadoopTvfAdmission.check(HadoopTvfAdmission.Access.NATIVE_WRITE, + "/unmapped/prefix_", properties)); + properties.remove("fs.viewfs.mounttable.cluster.linkFallback"); + for (String key : Arrays.asList("linkMerge./local", + "linkNfly.minReplication=2,repairOnRead=true./local")) { + String mountKey = "fs.viewfs.mounttable.cluster." + key; + properties.put(mountKey, "hdfs://namenode:8020/base,file:///tmp"); + Assertions.assertThrows(AnalysisException.class, + () -> HadoopTvfAdmission.check(HadoopTvfAdmission.Access.NATIVE_WRITE, + "/local/sub/prefix_", properties)); + properties.put(mountKey, "hdfs://namenode:8020/base,hdfs://other:8020/base"); + Assertions.assertDoesNotThrow(() -> HadoopTvfAdmission.check(HadoopTvfAdmission.Access.NATIVE_WRITE, + "/local/sub/prefix_", properties)); + properties.remove(mountKey); + } + properties.put("fs.viewfs.mounttable.cluster.linkMergeSlash", "file:///tmp"); + Assertions.assertThrows(AnalysisException.class, + () -> HadoopTvfAdmission.check(HadoopTvfAdmission.Access.NATIVE_WRITE, + "/local/sub/prefix_", properties)); + properties.put("fs.viewfs.mounttable.cluster.linkMergeSlash", "hdfs://namenode:8020/base,archive"); + Assertions.assertDoesNotThrow(() -> HadoopTvfAdmission.check(HadoopTvfAdmission.Access.NATIVE_WRITE, + "/local/sub/prefix_", properties)); + properties.remove("fs.viewfs.mounttable.cluster.linkMergeSlash"); + properties.put("fs.viewfs.mounttable.cluster.link./local", "viewfs://inner/root"); + properties.put("fs.viewfs.mounttable.inner.link./root", "file:///tmp"); + Assertions.assertThrows(AnalysisException.class, + () -> HadoopTvfAdmission.check(HadoopTvfAdmission.Access.NATIVE_WRITE, + "/local/sub/prefix_", properties)); + properties.put("fs.viewfs.mounttable.inner.link./root", "hdfs://namenode:8020/base"); + Assertions.assertDoesNotThrow(() -> HadoopTvfAdmission.check(HadoopTvfAdmission.Access.NATIVE_WRITE, + "/local/sub/prefix_", properties)); + properties.put("fs.viewfs.mounttable.inner.link./root", "viewfs://cluster/local"); + Assertions.assertThrows(AnalysisException.class, + () -> HadoopTvfAdmission.check(HadoopTvfAdmission.Access.NATIVE_WRITE, + "/local/sub/prefix_", properties)); + properties.clear(); + properties.put("fs.defaultFS", "viewfs://A"); + properties.put("fs.viewfs.mounttable.A.link./a", "viewfs://B/base"); + properties.put("fs.viewfs.mounttable.A.link./a/b/c", "hdfs://namenode:8020/remote"); + properties.put("fs.viewfs.mounttable.B.link./base/d", "file:///tmp"); + properties.put("fs.viewfs.mounttable.B.link./base/b/d", "hdfs://namenode:8020/remote"); + properties.put("fs.viewfs.mounttable.B.link./base/z", "hdfs://namenode:8020/remote"); + Assertions.assertDoesNotThrow(() -> HadoopTvfAdmission.check(HadoopTvfAdmission.Access.NATIVE_WRITE, + "/a/z/sub/prefix_", properties)); + Assertions.assertThrows(AnalysisException.class, + () -> HadoopTvfAdmission.check(HadoopTvfAdmission.Access.NATIVE_WRITE, + "/a/b/d/sub/prefix_", properties)); + properties.clear(); + properties.put("fs.defaultFS", "hdfs://cluster:8020"); + properties.put("fs.hdfs.impl", "org.apache.hadoop.fs.viewfs.ViewFileSystemOverloadScheme"); + properties.put("fs.viewfs.mounttable.cluster:8020.link./local", "hdfs://namenode:8020/base"); + properties.put("fs.viewfs.mounttable.cluster.link./local", "file:///tmp"); + Assertions.assertThrows(AnalysisException.class, + () -> HadoopTvfAdmission.check(HadoopTvfAdmission.Access.NATIVE_WRITE, + "/local/sub/prefix_", properties)); + properties.clear(); + properties.put("fs.defaultFS", "viewfs://inherited"); + Assertions.assertThrows(AnalysisException.class, + () -> HadoopTvfAdmission.check(HadoopTvfAdmission.Access.NATIVE_WRITE, "/unknown/prefix_", properties)); + } + + @Test + void explicitRemoteViewFsResourceIsNormalizedBeforeAdmission() throws Exception { + String originalConfigDir = Config.hadoop_config_dir; + String originalConfigProperty = System.getProperty("doris.hadoop.config.dir"); + Path directory = Files.createTempDirectory("tvf-remote-viewfs-"); + Path resource = directory.resolve("remote.xml"); + try { + Files.write(resource, ("" + + "fs.viewfs.mounttable.cluster.link./data" + + "hdfs://remote/base") + .getBytes(StandardCharsets.UTF_8)); + Config.hadoop_config_dir = directory.toString() + "/"; + String statement = "INSERT INTO hdfs('file_path'='/data/sub/prefix_', 'format'='csv'," + + "'fs.defaultFS'='viewfs://cluster', 'hadoop.config.resources'='remote.xml') SELECT 1"; + try (MockedStatic factory = Mockito.mockStatic(FileSystemFactory.class)) { + for (String prefix : Arrays.asList("", "EXPLAIN ")) { + Assertions.assertInstanceOf(LogicalTVFTableSink.class, analyzeInsert(prefix + statement)); + } + Map properties = new HashMap<>(); + properties.put("file_path", "/data/sub/prefix_"); + properties.put("format", "csv"); + properties.put("fs.defaultFS", "viewfs://cluster"); + properties.put("hadoop.config.resources", "remote.xml"); + TVFTableSink sink = new TVFTableSink(new PlanNodeId(0), "hdfs", properties, + Collections.emptyList()); + Assertions.assertDoesNotThrow(sink::bindDataSink); + factory.verifyNoInteractions(); + } + } finally { + Config.hadoop_config_dir = originalConfigDir; + if (originalConfigProperty == null) { + System.clearProperty("doris.hadoop.config.dir"); + } else { + System.setProperty("doris.hadoop.config.dir", originalConfigProperty); + } + Files.deleteIfExists(resource); + Files.deleteIfExists(directory); + } + } + + @Test + void viewFsResourcesAndCleanupRejectBeforeFilesystemCreation() throws Exception { + String originalConfigDir = Config.hadoop_config_dir; + String originalConfigProperty = System.getProperty("doris.hadoop.config.dir"); + Path directory = Files.createTempDirectory("tvf-viewfs-admission-"); + Path resource = directory.resolve("viewfs.xml"); + try { + Files.write(resource, ("fs.defaultFS" + + "viewfs://cluster" + + "fs.viewfs.mounttable.cluster.link./local" + + "file:///tmp").getBytes(StandardCharsets.UTF_8)); + Config.hadoop_config_dir = directory.toString() + "/"; + Map properties = new HashMap<>(); + properties.put("file_path", "/local/sub/prefix_"); + properties.put("format", "csv"); + properties.put("fs.hdfs.support", "true"); + properties.put("hadoop.config.resources", resource.getFileName().toString()); + TVFTableSink sink = new TVFTableSink(new PlanNodeId(0), "hdfs", properties, + Collections.emptyList()); + Assertions.assertThrows(AnalysisException.class, sink::bindDataSink); + InsertIntoTVFCommand command = (InsertIntoTVFCommand) new NereidsParser().parseSingle( + "INSERT INTO s3('file_path'='/local/sub/prefix_', 'format'='csv') SELECT 1"); + Method cleanup = InsertIntoTVFCommand.class.getDeclaredMethod( + "deleteExistingFilesInFE", String.class, Map.class); + cleanup.setAccessible(true); + properties.put("delete_existing_files", "true"); + try (MockedStatic factory = Mockito.mockStatic(FileSystemFactory.class)) { + InvocationTargetException exception = Assertions.assertThrows(InvocationTargetException.class, + () -> cleanup.invoke(command, "s3", properties)); + Assertions.assertInstanceOf(AnalysisException.class, exception.getCause()); + factory.verifyNoInteractions(); + } + } finally { + Config.hadoop_config_dir = originalConfigDir; + if (originalConfigProperty == null) { + System.clearProperty("doris.hadoop.config.dir"); + } else { + System.setProperty("doris.hadoop.config.dir", originalConfigProperty); + } + Files.deleteIfExists(resource); + Files.deleteIfExists(directory); + } + } + + @Test + void loadUserCannotExecuteViewFsLocalWritesOrCleanup() { + ConnectContext context = MemoTestUtils.createConnectContext(); + String sql = "INSERT INTO hdfs('file_path'='/local/sub/prefix_', 'format'='csv'," + + "'delete_existing_files'='true','fs.defaultFS'='viewfs://cluster'," + + "'fs.viewfs.mounttable.cluster.link./local'='file:///tmp') SELECT 1"; + MemoTestUtils.createStatementContext(context, sql); + InsertIntoTVFCommand command = (InsertIntoTVFCommand) new NereidsParser().parseSingle(sql); + Env env = Mockito.spy(Env.getCurrentEnv()); + AccessControllerManager accessManager = Mockito.mock(AccessControllerManager.class); + Mockito.doReturn(accessManager).when(env).getAccessManager(); + Mockito.when(accessManager.checkGlobalPriv(context, PrivPredicate.LOAD)).thenReturn(true); + StmtExecutor executor = Mockito.mock(StmtExecutor.class); + try (MockedStatic mockedEnv = Mockito.mockStatic(Env.class, Mockito.CALLS_REAL_METHODS); + MockedStatic factory = Mockito.mockStatic(FileSystemFactory.class)) { + mockedEnv.when(Env::getCurrentEnv).thenReturn(env); + AnalysisException exception = Assertions.assertThrows(AnalysisException.class, + () -> command.run(context, executor)); + Assertions.assertTrue(exception.getMessage().contains("LOCAL TVF is disabled")); + Mockito.verify(accessManager).checkGlobalPriv(context, PrivPredicate.ADMIN); + Mockito.verify(accessManager).checkGlobalPriv(context, PrivPredicate.LOAD); + Mockito.verifyNoInteractions(executor); + factory.verifyNoInteractions(); + } + } + + @Test + void cleanupEntryRejectsSelectedLocalSpiBeforeFilesystemCreation() throws Exception { + InsertIntoTVFCommand command = (InsertIntoTVFCommand) new NereidsParser().parseSingle( + "INSERT INTO s3('file_path'='/tmp/local_gate_/prefix_', 'format'='csv') SELECT 1"); + Method cleanup = InsertIntoTVFCommand.class.getDeclaredMethod( + "deleteExistingFilesInFE", String.class, Map.class); + cleanup.setAccessible(true); + Map properties = new HashMap<>(); + properties.put("file_path", "/tmp/local_gate_/prefix_"); + properties.put("format", "csv"); + properties.put("delete_existing_files", "true"); + properties.put("fs.local.support", "true"); + try (MockedStatic factory = Mockito.mockStatic(FileSystemFactory.class)) { + InvocationTargetException exception = Assertions.assertThrows(InvocationTargetException.class, + () -> cleanup.invoke(command, "s3", properties)); + Assertions.assertInstanceOf(AnalysisException.class, exception.getCause()); + Assertions.assertTrue(exception.getCause().getMessage().contains("LOCAL TVF is disabled")); + factory.verifyNoInteractions(); + } + } + + @Test + void nativeUriPrefixesCannotSelectAnUncheckedLocalMount() throws Exception { + String properties = "'file_path'='hdfs://unrelated/remote/prefix_', 'format'='csv'," + + "'fs.defaultFS'='viewfs://cluster'," + + "'fs.viewfs.mounttable.cluster.link./remote'='hdfs://remote/base'," + + "'fs.viewfs.mounttable.cluster.link./hdfs:'='file:///tmp'"; + for (String prefix : Arrays.asList("", "EXPLAIN ")) { + Assertions.assertThrows(AnalysisException.class, + () -> analyzeInsert(prefix + "INSERT INTO hdfs(" + properties + ") SELECT 1")); + } + Map props = new HashMap<>(); + props.put("file_path", "hdfs://unrelated/remote/prefix_"); + props.put("format", "csv"); + props.put("fs.defaultFS", "viewfs://cluster"); + props.put("fs.viewfs.mounttable.cluster.link./remote", "hdfs://remote/base"); + props.put("fs.viewfs.mounttable.cluster.link./hdfs:", "file:///tmp"); + TVFTableSink sink = new TVFTableSink(new PlanNodeId(0), "hdfs", props, Collections.emptyList()); + Assertions.assertThrows(AnalysisException.class, sink::bindDataSink); + } + + @Test + void emptyViewFsTargetRejectsBeforeCleanupFilesystemCreation() throws Exception { + Map properties = new HashMap<>(); + properties.put("file_path", "/data/sub/prefix_"); + properties.put("format", "csv"); + properties.put("fs.defaultFS", "viewfs://cluster"); + properties.put("fs.viewfs.mounttable.cluster.link./data", ""); + properties.put("fs.viewfs.mounttable.cluster.linkFallback", "file:///tmp"); + InsertIntoTVFCommand command = (InsertIntoTVFCommand) new NereidsParser().parseSingle( + "INSERT INTO hdfs('file_path'='/data/sub/prefix_', 'format'='csv') SELECT 1"); + Method cleanup = InsertIntoTVFCommand.class.getDeclaredMethod( + "deleteExistingFilesInFE", String.class, Map.class); + cleanup.setAccessible(true); + try (MockedStatic factory = Mockito.mockStatic(FileSystemFactory.class)) { + InvocationTargetException exception = Assertions.assertThrows(InvocationTargetException.class, + () -> cleanup.invoke(command, "hdfs", properties)); + Assertions.assertInstanceOf(AnalysisException.class, exception.getCause()); + factory.verifyNoInteractions(); + } + } + + private Plan analyzeInsert(String sql) { + ConnectContext context = MemoTestUtils.createConnectContext(); + LogicalPlan parsed = new NereidsParser().parseSingle(sql); + if (parsed instanceof ExplainCommand) { + parsed = ((ExplainCommand) parsed).getLogicalPlan(); + } + InsertIntoTVFCommand command = (InsertIntoTVFCommand) parsed; + return PlanChecker.from(context).analyze(command.getExplainPlan(context)).getPlan(); + } @Test void duplicateOutputNamesKeepDistinctPositions() { + Config.enable_local_tvf = true; ConnectContext context = MemoTestUtils.createConnectContext(); InsertIntoTVFCommand command = (InsertIntoTVFCommand) new NereidsParser().parseSingle( "INSERT INTO local(" diff --git a/fe/fe-core/src/test/java/org/apache/doris/tablefunction/HadoopTvfAdmissionTest.java b/fe/fe-core/src/test/java/org/apache/doris/tablefunction/HadoopTvfAdmissionTest.java new file mode 100644 index 00000000000000..845bfbefd5c041 --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/tablefunction/HadoopTvfAdmissionTest.java @@ -0,0 +1,438 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.tablefunction; + +import org.apache.doris.common.Config; +import org.apache.doris.nereids.exceptions.AnalysisException; +import org.apache.doris.tablefunction.HadoopTvfAdmission.Access; + +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +import java.util.HashMap; +import java.util.Map; + +class HadoopTvfAdmissionTest { + private boolean originalEnable; + + @BeforeEach + void setUp() { + originalEnable = Config.enable_local_tvf; + Config.enable_local_tvf = false; + } + + @AfterEach + void tearDown() { + Config.enable_local_tvf = originalEnable; + } + + @Test + void javaMountKeysPreserveNulWhileNativeKeysAreCStringPrefixes() { + Map properties = viewFs(); + properties.put("fs.viewfs.mounttable.cluster.link./data\0ignored", "hdfs://remote/base"); + properties.put("fs.viewfs.mounttable.cluster.linkFallback", "file:///tmp"); + Assertions.assertDoesNotThrow(() -> HadoopTvfAdmission.check( + Access.NATIVE_WRITE, "/data/sub/prefix_", properties)); + Assertions.assertThrows(AnalysisException.class, () -> HadoopTvfAdmission.check( + Access.JAVA_DELETE, "/data/sub/", properties)); + Assertions.assertThrows(AnalysisException.class, () -> HadoopTvfAdmission.check( + Access.HADOOP_READ, "viewfs://cluster/data/sub/file.csv", properties)); + } + + @Test + void javaImplementationKeysAreNotTruncatedAtNul() { + Map properties = new HashMap<>(); + properties.put("fs.defaultFS", "hdfs://remote"); + properties.put("fs.hdfs.impl\0ignored", "org.apache.hadoop.fs.LocalFileSystem\0ignored"); + Assertions.assertDoesNotThrow(() -> HadoopTvfAdmission.check( + Access.JAVA_DELETE, "/data/sub/", properties)); + Assertions.assertThrows(AnalysisException.class, () -> HadoopTvfAdmission.check( + Access.NATIVE_WRITE, "/data/sub/prefix_", properties)); + } + + @Test + void nativeAuthorityCompletionUsesTheRemainingLegacyConfiguration() { + Map properties = new HashMap<>(); + properties.put("fs.defaultFS", "viewfs:///"); + properties.put("fs.default.name", "viewfs://local"); + properties.put("fs.viewfs.mounttable.default.link./data", "hdfs://remote/base"); + properties.put("fs.viewfs.mounttable.local.link./data", "file:///tmp"); + Assertions.assertThrows(AnalysisException.class, () -> HadoopTvfAdmission.check( + Access.NATIVE_WRITE, "/data/sub/prefix_", properties)); + properties.put("fs.viewfs.mounttable.local.link./data", "hdfs://remote/base"); + Assertions.assertDoesNotThrow(() -> HadoopTvfAdmission.check( + Access.NATIVE_WRITE, "/data/sub/prefix_", properties)); + } + + @Test + void distributedViewWrappersCannotBeMistakenForDirectRemoteProviders() { + Map properties = new HashMap<>(); + properties.put("fs.defaultFS", "hdfs://cluster"); + properties.put("fs.hdfs.impl", "org.apache.hadoop.hdfs.ViewDistributedFileSystem"); + properties.put("fs.viewfs.mounttable.cluster.link./data", "file:///tmp"); + for (Access access : Access.values()) { + Assertions.assertThrows(AnalysisException.class, () -> HadoopTvfAdmission.check( + access, "hdfs://cluster/data/sub/file.csv", properties)); + } + Config.enable_local_tvf = true; + Assertions.assertDoesNotThrow(() -> HadoopTvfAdmission.check( + Access.NATIVE_WRITE, "/data/sub/prefix_", properties)); + Assertions.assertThrows(AnalysisException.class, () -> HadoopTvfAdmission.check( + Access.HADOOP_READ, "hdfs://cluster/data/sub/file.csv", properties)); + } + + @Test + void hadoopReadsRejectLocalDelegationEvenWithLocalOptIn() { + Map properties = viewFs(); + properties.put("fs.viewfs.mounttable.cluster.link./data", "file:///tmp"); + for (boolean enabled : new boolean[] {false, true}) { + Config.enable_local_tvf = enabled; + AnalysisException error = Assertions.assertThrows(AnalysisException.class, + () -> HadoopTvfAdmission.check(Access.HADOOP_READ, + "viewfs://cluster/data/sub/file.csv", properties)); + Assertions.assertTrue(error.getMessage().contains("Use local()")); + } + } + + @Test + void remoteReadsDoNotRejectAnUnrelatedLocalSiblingMount() { + Map properties = viewFs(); + properties.put("fs.viewfs.mounttable.cluster.link./data", "hdfs://remote/base"); + properties.put("fs.viewfs.mounttable.cluster.link./local", "file:///tmp"); + Assertions.assertDoesNotThrow(() -> HadoopTvfAdmission.check( + Access.HADOOP_READ, "viewfs://cluster/data/file.csv", properties)); + } + + @Test + void readPatternsCheckEveryIntersectingMountBeforeListing() { + Map properties = viewFs(); + properties.put("fs.viewfs.mounttable.cluster.link./data", "hdfs://remote/base"); + properties.put("fs.viewfs.mounttable.cluster.link./data/local", "file:///tmp"); + for (String pattern : new String[] {"/data/*", "/data/?/file.csv", "/data/{a,b}/file.csv", "/data"}) { + Assertions.assertThrows(AnalysisException.class, () -> HadoopTvfAdmission.checkReadDiscovery("viewfs://cluster" + pattern, properties)); + } + Assertions.assertDoesNotThrow(() -> HadoopTvfAdmission.checkReadDiscovery("viewfs://cluster/data/remote/file.csv", properties)); + properties.put("fs.viewfs.mounttable.cluster.link./data/local", "hdfs://remote/other"); + Assertions.assertDoesNotThrow(() -> HadoopTvfAdmission.checkReadDiscovery("viewfs://cluster/data/*", properties)); + } + + @Test + void schemaAndDataReadNameNodesAreCheckedIndependently() { + Map properties = new HashMap<>(); + properties.put("fs.defaultFS", "viewfs://schema"); + properties.put("fs.viewfs.mounttable.schema.link./viewfs:", "file:///tmp"); + properties.put("fs.viewfs.mounttable.data.link./data", "hdfs://remote/base"); + Assertions.assertThrows(AnalysisException.class, () -> HadoopTvfAdmission.check( + Access.HADOOP_READ, "viewfs://data/data/file.csv", properties)); + properties.put("fs.viewfs.mounttable.schema.link./viewfs:", "hdfs://remote/schema"); + Assertions.assertDoesNotThrow(() -> HadoopTvfAdmission.check( + Access.HADOOP_READ, "viewfs://data/data/file.csv", properties)); + } + + @Test + void unknownImplementationsAndRuntimeMountsNeedExplicitWriteOptIn() { + Map properties = new HashMap<>(); + properties.put("fs.defaultFS", "hdfs://remote"); + properties.put("fs.hdfs.impl", "example.DynamicFileSystem"); + Assertions.assertThrows(AnalysisException.class, () -> HadoopTvfAdmission.check( + Access.NATIVE_WRITE, "/data/prefix_", properties)); + Config.enable_local_tvf = true; + Assertions.assertDoesNotThrow(() -> HadoopTvfAdmission.check( + Access.NATIVE_WRITE, "/data/prefix_", properties)); + Assertions.assertThrows(AnalysisException.class, () -> HadoopTvfAdmission.check( + Access.HADOOP_READ, "hdfs://remote/data/file.csv", properties)); + } + + @Test + void nativeProjectionRejectsConflictingCStringKeys() { + Map properties = viewFs(); + properties.put("fs.viewfs.mounttable.cluster.link./data", "hdfs://remote/base"); + properties.put("fs.viewfs.mounttable.cluster.link./data\0ignored", "file:///tmp"); + Assertions.assertThrows(AnalysisException.class, () -> HadoopTvfAdmission.check( + Access.NATIVE_WRITE, "/data/sub/prefix_", properties)); + } + + @Test + void nativeNameNodeCaseCollisionsCannotChooseAnUncheckedProvider() { + Map properties = new HashMap<>(); + properties.put("fs.defaultFS", "hdfs://remote"); + properties.put("fs.DEFAULTFS", "file:///"); + Assertions.assertThrows(AnalysisException.class, () -> HadoopTvfAdmission.check( + Access.NATIVE_WRITE, "/data/prefix_", properties)); + } + + @Test + void nestedRemoteReadPatternsReuseTheSameMountInterpreter() { + Map properties = viewFs(); + properties.put("fs.viewfs.mounttable.cluster.link./data", "viewfs://inner/base"); + properties.put("fs.viewfs.mounttable.inner.link./base", "hdfs://remote/base"); + Assertions.assertDoesNotThrow(() -> HadoopTvfAdmission.checkReadDiscovery("viewfs://cluster/data/*", properties)); + properties.put("fs.viewfs.mounttable.inner.link./base/private", "file:///tmp"); + Assertions.assertThrows(AnalysisException.class, () -> HadoopTvfAdmission.checkReadDiscovery("viewfs://cluster/data/*", properties)); + } + + @Test + void emptyNativeWriteNameNodeKeepsExistingAnalysisWithoutSelectingLocal() { + Map properties = new HashMap<>(); + Assertions.assertDoesNotThrow(() -> HadoopTvfAdmission.check( + Access.NATIVE_WRITE, "remote/data", properties)); + Assertions.assertThrows(AnalysisException.class, () -> HadoopTvfAdmission.check( + Access.HADOOP_READ, "hdfs://remote/data", properties)); + properties.put("fs.defaultFS", "default"); + Assertions.assertThrows(AnalysisException.class, () -> HadoopTvfAdmission.check( + Access.NATIVE_WRITE, "remote/data", properties)); + properties.put("fs.defaultFS", "\0ignored"); + Assertions.assertDoesNotThrow(() -> HadoopTvfAdmission.check( + Access.NATIVE_WRITE, "remote/data", properties)); + } + + @Test + void builtInRemoteProviderImplementationsRemainAvailable() { + for (String[] provider : new String[][] { + {"oss", "com.aliyun.jindodata.oss.JindoOssFileSystem"}, + {"obs", "org.apache.hadoop.fs.obs.OBSFileSystem"}}) { + Map properties = new HashMap<>(); + properties.put("fs.defaultFS", provider[0] + "://bucket"); + properties.put("fs." + provider[0] + ".impl", provider[1]); + for (Access access : Access.values()) { + Assertions.assertDoesNotThrow(() -> HadoopTvfAdmission.check( + access, provider[0] + "://bucket/data/file.csv", properties)); + } + } + } + + @Test + void nativePathsAreRootedBeforeHadoopUriParsing() { + Map properties = viewFs(); + properties.put("fs.viewfs.mounttable.cluster.link./remote", "hdfs://remote/base"); + properties.put("fs.viewfs.mounttable.cluster.link./hdfs:", "file:///tmp"); + Assertions.assertThrows(AnalysisException.class, () -> HadoopTvfAdmission.check( + Access.NATIVE_WRITE, "hdfs://unrelated/remote/prefix_", properties)); + Assertions.assertDoesNotThrow(() -> HadoopTvfAdmission.check( + Access.NATIVE_WRITE, "remote/prefix_", properties)); + + properties.put("fs.defaultFS", "viewfs://schema"); + properties.put("fs.viewfs.mounttable.data.link./remote", "hdfs://remote/base"); + properties.put("fs.viewfs.mounttable.schema.link./remote", "hdfs://remote/base"); + properties.put("fs.viewfs.mounttable.schema.link./viewfs:", "file:///tmp"); + Assertions.assertThrows(AnalysisException.class, () -> HadoopTvfAdmission.check( + Access.HADOOP_READ, "viewfs://data/remote/file.csv", properties)); + properties.put("fs.viewfs.mounttable.schema.link./viewfs:", "hdfs://remote/schema"); + Assertions.assertDoesNotThrow(() -> HadoopTvfAdmission.check( + Access.HADOOP_READ, "viewfs://data/remote/file.csv", properties)); + } + + @Test + void nonPrefixNameNodeMatchesRequireAnEstablishedNativePath() { + String nameNode = "viewfs://clüster"; + Map properties = new HashMap<>(); + properties.put("fs.defaultFS", nameNode); + properties.put("fs.viewfs.mounttable.clüster.link./x", "hdfs://remote/base"); + properties.put("fs.viewfs.mounttable.clüster.link./local", "file:///tmp"); + String path = "abcdefghijklmnopx/local/prefix_" + nameNode + "/a"; + Assertions.assertThrows(AnalysisException.class, () -> HadoopTvfAdmission.check( + Access.NATIVE_WRITE, path, properties)); + Assertions.assertDoesNotThrow(() -> HadoopTvfAdmission.check( + Access.NATIVE_WRITE, nameNode + "/x/prefix_", properties)); + Config.enable_local_tvf = true; + Assertions.assertDoesNotThrow(() -> HadoopTvfAdmission.check( + Access.NATIVE_WRITE, path, properties)); + } + + @Test + void relativeJavaViewFsPathsCannotAssumeTheFilesystemRoot() { + Map properties = viewFs(); + properties.put("fs.viewfs.mounttable.cluster.link./data", "hdfs://remote/base"); + properties.put("fs.viewfs.mounttable.cluster.link./user/tvf/data", "file:///tmp"); + Assertions.assertDoesNotThrow(() -> HadoopTvfAdmission.check( + Access.NATIVE_WRITE, "data/sub/prefix_", properties)); + Assertions.assertThrows(AnalysisException.class, () -> HadoopTvfAdmission.check( + Access.JAVA_DELETE, "data/sub/", properties)); + properties.put("fs.defaultFS", "hdfs://remote"); + Assertions.assertDoesNotThrow(() -> HadoopTvfAdmission.check( + Access.JAVA_DELETE, "data/sub/", properties)); + properties.put("fs.defaultFS", "viewfs://cluster"); + Config.enable_local_tvf = true; + Assertions.assertDoesNotThrow(() -> HadoopTvfAdmission.check( + Access.JAVA_DELETE, "data/sub/", properties)); + } + + @Test + void selectedMountTargetsMustNotBeEmpty() { + Map properties = viewFs(); + properties.put("fs.viewfs.mounttable.cluster.linkFallback", "file:///tmp"); + for (String target : new String[] {"", ","}) { + properties.put("fs.viewfs.mounttable.cluster.link./data", target); + for (Access access : new Access[] {Access.NATIVE_WRITE, Access.JAVA_DELETE}) { + Assertions.assertThrows(AnalysisException.class, () -> HadoopTvfAdmission.check( + access, "/data/sub/prefix_", properties)); + } + } + properties.put("fs.viewfs.mounttable.cluster.link./data", "hdfs://remote/base,"); + Assertions.assertDoesNotThrow(() -> HadoopTvfAdmission.check( + Access.NATIVE_WRITE, "/data/sub/prefix_", properties)); + properties.remove("fs.viewfs.mounttable.cluster.link./data"); + properties.put("fs.viewfs.mounttable.cluster.linkMerge./data", "hdfs://remote/base,"); + Assertions.assertThrows(AnalysisException.class, () -> HadoopTvfAdmission.check( + Access.NATIVE_WRITE, "/data/sub/prefix_", properties)); + properties.remove("fs.viewfs.mounttable.cluster.linkMerge./data"); + properties.put("fs.viewfs.mounttable.cluster.link./data", "hdfs://remote/base"); + properties.put("fs.viewfs.mounttable.cluster.link./unused", ""); + Assertions.assertDoesNotThrow(() -> HadoopTvfAdmission.check( + Access.NATIVE_WRITE, "/data/sub/prefix_", properties)); + } + + @Test + void nativeWriterRejectsCaseAndNulMapCollisions() { + Map properties = viewFs(); + String upper = "fs.viewfs.mounttable.CLUSTER.link./data\0A"; + String lower = "fs.viewfs.mounttable.cluster.link./data\0b"; + properties.put(upper, "hdfs://remote/base"); + properties.put(lower, "hdfs://remote/base"); + properties.put("fs.viewfs.mounttable.cluster.linkFallback", "file:///tmp"); + Assertions.assertThrows(AnalysisException.class, () -> HadoopTvfAdmission.check( + Access.NATIVE_WRITE, "/data/sub/prefix_", properties)); + properties.put("fs.viewfs.mounttable.cluster.link./public", "hdfs://remote/public"); + Assertions.assertDoesNotThrow(() -> HadoopTvfAdmission.check( + Access.HADOOP_READ, "viewfs://cluster/public/file.csv", properties)); + properties.remove(upper); + properties.put(upper + "long", "hdfs://remote/base"); + Assertions.assertDoesNotThrow(() -> HadoopTvfAdmission.check( + Access.NATIVE_WRITE, "/data/sub/prefix_", properties)); + properties.remove(upper + "long"); + properties.put(upper, "hdfs://remote/base"); + Config.enable_local_tvf = true; + Assertions.assertDoesNotThrow(() -> HadoopTvfAdmission.check( + Access.NATIVE_WRITE, "/data/sub/prefix_", properties)); + Config.enable_local_tvf = false; + properties.remove(upper); + properties.remove(lower); + properties.put("fs.viewfs.mounttable.cluster.link./data", "hdfs://remote/base"); + properties.put("dfs.test.VALUE\0a", "one"); + properties.put("dfs.test.value\0b", "two"); + Assertions.assertDoesNotThrow(() -> HadoopTvfAdmission.check( + Access.NATIVE_WRITE, "/data/sub/prefix_", properties)); + } + + @Test + void crossMountGlobsRequireAnExplicitRemoteRoute() { + Map properties = viewFs(); + properties.put("fs.viewfs.mounttable.cluster.link./data", "hdfs://remote/base"); + properties.put("fs.viewfs.mount.links.as.symlinks", "false"); + Assertions.assertThrows(AnalysisException.class, () -> HadoopTvfAdmission.checkReadDiscovery("viewfs://cluster/*/*.csv", properties)); + properties.put("fs.viewfs.mounttable.cluster.linkFallback", "hdfs://remote/fallback"); + for (String pattern : new String[] {"/*/*.csv", "/d*/*.csv", "/{data,other}/*.csv"}) { + Assertions.assertDoesNotThrow(() -> HadoopTvfAdmission.checkReadDiscovery("viewfs://cluster" + pattern, properties)); + } + properties.put("fs.viewfs.mounttable.cluster.link./private", "file:///tmp"); + Assertions.assertThrows(AnalysisException.class, () -> HadoopTvfAdmission.checkReadDiscovery("viewfs://cluster/*/*.csv", properties)); + properties.remove("fs.viewfs.mounttable.cluster.link./private"); + properties.put("fs.viewfs.mounttable.cluster.linkFallback", "file:///tmp"); + Assertions.assertThrows(AnalysisException.class, () -> HadoopTvfAdmission.checkReadDiscovery("viewfs://cluster/*/*.csv", properties)); + properties.remove("fs.viewfs.mounttable.cluster.linkFallback"); + properties.put("fs.viewfs.mounttable.cluster.link./data", ""); + Assertions.assertThrows(AnalysisException.class, () -> HadoopTvfAdmission.checkReadDiscovery("viewfs://cluster/*/*.csv", properties)); + } + + @Test + void defaultFilesystemHostPortShorthandRemainsRemote() { + for (String key : new String[] {"fs.defaultFS", "fs.default.name"}) { + Map properties = new HashMap<>(); + properties.put(key, "namenode:8020"); + Assertions.assertDoesNotThrow(() -> HadoopTvfAdmission.check( + Access.JAVA_DELETE, "/exports/", properties)); + properties.put(key, "local"); + Assertions.assertThrows(AnalysisException.class, () -> HadoopTvfAdmission.check( + Access.JAVA_DELETE, "/exports/", properties)); + } + } + + @Test + void singleViewFsTargetsKeepCommasWhileMultiTargetsStillRejectLocal() { + for (String key : new String[] {"link./data", "linkFallback", "linkMergeSlash"}) { + Map properties = viewFs(); + properties.put("fs.viewfs.mounttable.cluster." + key, "hdfs://remote/base,archive"); + for (Access access : Access.values()) { + String path = access == Access.HADOOP_READ + ? "viewfs://cluster/data/file.csv" : "/data/prefix_"; + Assertions.assertDoesNotThrow(() -> HadoopTvfAdmission.check(access, path, properties)); + } + Assertions.assertDoesNotThrow(() -> HadoopTvfAdmission.checkReadDiscovery("viewfs://cluster/data/*.csv", properties)); + } + for (String key : new String[] {"linkMerge./data", "linkNfly.minReplication=2./data"}) { + Map properties = viewFs(); + properties.put("fs.viewfs.mounttable.cluster." + key, "hdfs://remote/base,file:///tmp"); + for (Access access : Access.values()) { + Assertions.assertThrows(AnalysisException.class, () -> HadoopTvfAdmission.check( + access, "viewfs://cluster/data/file.csv", properties)); + } + } + } + + @Test + void nativeRoutesRejectSupplementaryCharactersBeforeJniDecoding() { + String supplementary = new String(Character.toChars(0x1F600)) + "AAA"; + Map properties = viewFs(); + properties.put("fs.viewfs.mounttable.cluster.link./data" + supplementary, "hdfs://remote/base"); + properties.put("fs.viewfs.mounttable.cluster.linkFallback", "file:///tmp"); + Assertions.assertThrows(AnalysisException.class, () -> HadoopTvfAdmission.check( + Access.NATIVE_WRITE, "/data" + supplementary + "/prefix_", properties)); + // Java deletion does not cross JNI and preserves this otherwise valid URI identity. + Assertions.assertDoesNotThrow(() -> HadoopTvfAdmission.check( + Access.JAVA_DELETE, "/data" + supplementary + "/", properties)); + Map bmp = viewFs(); + bmp.put("fs.viewfs.mounttable.cluster.link./数据", "hdfs://remote/目录"); + Assertions.assertDoesNotThrow(() -> HadoopTvfAdmission.check( + Access.NATIVE_WRITE, "/数据/prefix_", bmp)); + Config.enable_local_tvf = true; + Assertions.assertDoesNotThrow(() -> HadoopTvfAdmission.check( + Access.NATIVE_WRITE, "/data" + supplementary + "/prefix_", properties)); + } + + @Test + void nativeCharacterChecksCoverRoutingKeysValuesAndReadPaths() { + String supplementary = new String(Character.toChars(0x1F600)) + "AAA"; + Map properties = viewFs(); + properties.put("fs.viewfs.mounttable.cluster.link./data", "hdfs://remote/base"); + for (String value : new String[] {"hdfs://remote/base" + supplementary, "hdfs://remote/base" + (char) 0xD800}) { + properties.put("fs.viewfs.mounttable.cluster.link./data", value); + Assertions.assertThrows(AnalysisException.class, () -> HadoopTvfAdmission.check( + Access.NATIVE_WRITE, "/data/prefix_", properties)); + Assertions.assertThrows(AnalysisException.class, () -> HadoopTvfAdmission.check( + Access.HADOOP_READ, "viewfs://cluster/data/file.csv", properties)); + } + properties.put("fs.viewfs.mounttable.cluster.link./data", "hdfs://remote/base"); + Assertions.assertThrows(AnalysisException.class, () -> HadoopTvfAdmission.check( + Access.HADOOP_READ, "viewfs://cluster/data/" + supplementary + ".csv", properties)); + properties.put("fs.viewfs.mounttable.cluster.link./unused" + supplementary, "hdfs://remote/unused"); + Assertions.assertThrows(AnalysisException.class, () -> HadoopTvfAdmission.check( + Access.NATIVE_WRITE, "/data/prefix_", properties)); + properties.remove("fs.viewfs.mounttable.cluster.link./unused" + supplementary); + properties.put("fs.viewfs.mounttable.cluster.link./data", "hdfs://remote/base\0" + supplementary); + Assertions.assertDoesNotThrow(() -> HadoopTvfAdmission.check( + Access.NATIVE_WRITE, "/data/prefix_", properties)); + } + + private static Map viewFs() { + Map properties = new HashMap<>(); + properties.put("fs.defaultFS", "viewfs://cluster"); + return properties; + } +} diff --git a/fe/fe-core/src/test/java/org/apache/doris/tablefunction/HadoopTvfReadAdmissionTest.java b/fe/fe-core/src/test/java/org/apache/doris/tablefunction/HadoopTvfReadAdmissionTest.java new file mode 100644 index 00000000000000..7daedd6431d04e --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/tablefunction/HadoopTvfReadAdmissionTest.java @@ -0,0 +1,350 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.tablefunction; + +import org.apache.doris.common.Config; +import org.apache.doris.common.FeConstants; +import org.apache.doris.fs.FileSystemFactory; +import org.apache.doris.nereids.exceptions.AnalysisException; +import org.apache.doris.thrift.TBrokerFileStatus; + +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.mockito.MockedStatic; +import org.mockito.Mockito; + +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +class HadoopTvfReadAdmissionTest { + private boolean originalEnable; + private boolean originalUnitTest; + + @BeforeEach + void setUp() { + originalEnable = Config.enable_local_tvf; + originalUnitTest = FeConstants.runningUnitTest; + FeConstants.runningUnitTest = false; + } + + @AfterEach + void tearDown() { + Config.enable_local_tvf = originalEnable; + FeConstants.runningUnitTest = originalUnitTest; + } + + @Test + void hdfsAndFileRejectLocalRoutesBeforeFilesystemCreation() { + Map properties = properties("local"); + for (boolean enabled : new boolean[] {false, true}) { + Config.enable_local_tvf = enabled; + try (MockedStatic factory = Mockito.mockStatic(FileSystemFactory.class)) { + Assertions.assertThrows(AnalysisException.class, + () -> new HdfsTableValuedFunction(properties)); + Assertions.assertThrows(AnalysisException.class, + () -> new FileTableValuedFunction(properties)); + factory.verifyNoInteractions(); + } + } + } + + @Test + void remoteReadAdmissionAllowsTheExistingListingPath() throws Exception { + Config.enable_local_tvf = false; + HdfsTableValuedFunction tvf = new ListedHdfsTvf(properties("remote"), false); + Assertions.assertEquals(1, tvf.getFileStatuses().size()); + Assertions.assertEquals("viewfs://review/remote/listed.csv", tvf.getFileStatuses().get(0).getPath()); + } + + @Test + void returnedFilePathsAreCheckedBeforeSchemaOrDataAccess() { + Config.enable_local_tvf = true; + Assertions.assertThrows(AnalysisException.class, + () -> new ListedHdfsTvf(properties("remote"), true)); + } + + @Test + void fileKeepsTheBuiltInOssHdfsStorageSelection() throws Exception { + FeConstants.runningUnitTest = true; + Config.enable_local_tvf = false; + Map properties = new HashMap<>(); + properties.put("uri", "oss://bucket/file.csv"); + properties.put("fs.defaultFS", "oss://bucket"); + properties.put("fs.oss-hdfs.support", "true"); + properties.put("oss.endpoint", "cn-beijing.oss-dls.aliyuncs.com"); + properties.put("oss.access_key", "test-access-key"); + properties.put("oss.secret_key", "test-secret-key"); + properties.put("format", "csv"); + properties.put("csv_schema", "id:int"); + FileTableValuedFunction file = new FileTableValuedFunction(properties); + Assertions.assertEquals(org.apache.doris.thrift.TFileType.FILE_HDFS, file.getTFileType()); + Assertions.assertEquals("com.aliyun.jindodata.oss.JindoOssFileSystem", + file.getBackendConnectProperties().get("fs.oss.impl")); + S3TableValuedFunction s3 = new S3TableValuedFunction(properties); + Assertions.assertEquals("com.aliyun.jindodata.oss.JindoOssFileSystem", + s3.getBackendConnectProperties().get("fs.oss.impl")); + } + + @Test + void s3CsvWithHadoopStorageRejectsLocalRoutesBeforeListing() { + Map properties = properties("local"); + properties.put("fs.hdfs.support", "true"); + for (boolean enabled : new boolean[] {false, true}) { + Config.enable_local_tvf = enabled; + try (MockedStatic factory = Mockito.mockStatic(FileSystemFactory.class)) { + Assertions.assertThrows(AnalysisException.class, + () -> new ListedS3Tvf(properties, "local")); + factory.verifyNoInteractions(); + } + } + } + + @Test + void s3HadoopProvidedFilesAreCheckedInBothConstructorBranches() { + Config.enable_local_tvf = true; + Map properties = properties("remote"); + properties.put("fs.hdfs.support", "true"); + TBrokerFileStatus local = new TBrokerFileStatus("viewfs://review/local/file.csv", false, 10, true); + for (boolean unitTest : new boolean[] {false, true}) { + FeConstants.runningUnitTest = unitTest; + try (MockedStatic factory = Mockito.mockStatic(FileSystemFactory.class)) { + Assertions.assertThrows(AnalysisException.class, + () -> new S3TableValuedFunction(properties, Collections.singletonList(local))); + factory.verifyNoInteractions(); + } + } + } + + @Test + void s3HadoopRemoteCsvKeepsDeclaredSchemaAndHdfsRanges() throws Exception { + Config.enable_local_tvf = false; + Map properties = properties("remote"); + properties.put("fs.hdfs.support", "true"); + S3TableValuedFunction tvf = new ListedS3Tvf(properties, "remote"); + Assertions.assertEquals("id", tvf.getTableColumns().get(0).getName()); + Assertions.assertEquals(1, tvf.getFileStatuses().size()); + String path = tvf.getFileStatuses().get(0).getPath(); + Assertions.assertEquals("viewfs://review/remote/listed.csv", path); + Assertions.assertEquals(org.apache.doris.thrift.TFileType.FILE_HDFS, + org.apache.doris.common.util.LocationPath.getTFileTypeForBE(path)); + } + + @Test + void explicitRemoteFallbackKeepsCrossMountGlobAdmission() throws Exception { + Config.enable_local_tvf = false; + Map properties = properties("remote"); + properties.put("uri", "viewfs://review/*/*.csv"); + properties.remove("fs.viewfs.mounttable.review.link./local"); + properties.put("fs.viewfs.mount.links.as.symlinks", "false"); + properties.put("fs.viewfs.mounttable.review.linkFallback", "hdfs://remote/fallback"); + Assertions.assertEquals(1, new ListedHdfsTvf(properties, false).getFileStatuses().size()); + FeConstants.runningUnitTest = true; + Assertions.assertDoesNotThrow(() -> new FileTableValuedFunction(properties)); + } + + @Test + void s3LocalAdapterRejectsBeforeListingEvenWithDeclaredSchema() { + Map properties = new HashMap<>(); + properties.put("file_path", "/tmp/local_gate_dir"); + properties.put("fs.local.support", "true"); + properties.put("format", "csv"); + properties.put("csv_schema", "id:int"); + for (boolean enabled : new boolean[] {false, true}) { + Config.enable_local_tvf = enabled; + try (MockedStatic factory = Mockito.mockStatic(FileSystemFactory.class)) { + Exception error = Assertions.assertThrows(Exception.class, + () -> new S3TableValuedFunction(properties)); + Assertions.assertTrue(error.getMessage().contains("Use local()"), error.getMessage()); + factory.verifyNoInteractions(); + } + } + } + + @Test + void s3BrokerHadoopRangesCannotUseLocalImplementation() { + Config.enable_local_tvf = false; + Map properties = new HashMap<>(); + properties.put("uri", "hdfs://remote/srv/private.csv"); + properties.put("fs.defaultFS", "hdfs://remote"); + properties.put("fs.broker.support", "true"); + properties.put("broker.name", "review-broker"); + properties.put("fs.hdfs.impl", "org.apache.hadoop.fs.RawLocalFileSystem"); + properties.put("format", "csv"); + properties.put("csv_schema", "id:int"); + try (MockedStatic factory = Mockito.mockStatic(FileSystemFactory.class)) { + Assertions.assertThrows(AnalysisException.class, + () -> new S3TableValuedFunction(properties)); + factory.verifyNoInteractions(); + } + } + + @Test + void declaredSchemaDoesNotOpenTheUnusedLocalDefault() throws Exception { + Config.enable_local_tvf = false; + FeConstants.runningUnitTest = true; + Map properties = new HashMap<>(); + properties.put("uri", "hdfs://remote/data.csv"); + properties.put("fs.defaultFS", "file:///"); + properties.put("format", "csv"); + properties.put("csv_schema", "id:int"); + HdfsTableValuedFunction hdfs = new HdfsTableValuedFunction(properties); + Assertions.assertEquals("id", hdfs.getTableColumns().get(0).getName()); + FileTableValuedFunction file = new FileTableValuedFunction(properties); + Assertions.assertEquals("id", file.getTableColumns().get(0).getName()); + properties.put("fs.hdfs.support", "true"); + Assertions.assertEquals("id", new S3TableValuedFunction(properties).getTableColumns().get(0).getName()); + properties.remove("csv_schema"); + FeConstants.runningUnitTest = false; + Assertions.assertThrows(AnalysisException.class, () -> new SchemaListedHdfsTvf(properties)); + } + + @Test + void s3ChecksSuppliedRangeConsumersIndependentlyFromBrokerListing() throws Exception { + Map properties = new HashMap<>(); + properties.put("uri", "s3://bucket/listing.csv"); + properties.put("fs.broker.support", "true"); + properties.put("broker.name", "review-broker"); + properties.put("fs.defaultFS", "hdfs://remote"); + properties.put("fs.hdfs.impl", "org.apache.hadoop.fs.RawLocalFileSystem"); + properties.put("format", "csv"); + properties.put("csv_schema", "id:int"); + for (boolean unitTest : new boolean[] {false, true}) { + FeConstants.runningUnitTest = unitTest; + for (String path : new String[] {"hdfs://remote/private.csv", "file:///tmp/private.csv"}) { + List files = Collections.singletonList( + new TBrokerFileStatus(path, false, 10, true)); + Assertions.assertThrows(AnalysisException.class, + () -> new SuppliedS3Tvf(properties, files)); + } + } + FeConstants.runningUnitTest = true; + properties.put("fs.hdfs.impl", "org.apache.hadoop.hdfs.DistributedFileSystem"); + S3TableValuedFunction remote = new S3TableValuedFunction(properties, Collections.singletonList( + new TBrokerFileStatus("hdfs://remote/public.csv", false, 10, true))); + Assertions.assertEquals(1, remote.getFileStatuses().size()); + Assertions.assertEquals("id", remote.getTableColumns().get(0).getName()); + } + + @Test + void suppliedHadoopFilesSkipDiscoveryAndKeepTheirLiteralPaths() throws Exception { + Map properties = properties("local"); + properties.put("fs.hdfs.support", "true"); + String remoteLiteral = "viewfs://review/remote/{../local}/file.csv"; + for (boolean enabled : new boolean[] {false, true}) { + Config.enable_local_tvf = enabled; + List files = Collections.singletonList( + new TBrokerFileStatus(remoteLiteral, false, 10, true)); + S3TableValuedFunction tvf = new SuppliedS3Tvf(properties, files); + Assertions.assertEquals(remoteLiteral, tvf.getFileStatuses().get(0).getPath()); + List localFiles = Collections.singletonList( + new TBrokerFileStatus("viewfs://review/local/{../remote}/file.csv", false, 10, true)); + Assertions.assertThrows(AnalysisException.class, () -> new SuppliedS3Tvf(properties, localFiles)); + } + } + + private static class SuppliedS3Tvf extends S3TableValuedFunction { + SuppliedS3Tvf(Map properties, List files) + throws org.apache.doris.common.AnalysisException { + super(properties, files); + } + + @Override + protected void parseFile(List files) { + fileStatuses.addAll(files); + } + } + + @Test + void schemaDiscoveryAlsoChecksTheReturnedFilesDefaultRoute() throws Exception { + Config.enable_local_tvf = false; + Map properties = new HashMap<>(); + properties.put("uri", "hdfs://initial/data.csv"); + properties.put("fs.defaultFS", "viewfs://schema"); + properties.put("fs.viewfs.mounttable.schema.link./hdfs:/initial", "hdfs://remote/base"); + properties.put("fs.viewfs.mounttable.schema.linkFallback", "file:///tmp"); + properties.put("format", "csv"); + // The fake listing separates initial/data/schema identities without filesystem IO. + Assertions.assertThrows(AnalysisException.class, () -> new SchemaListedHdfsTvf(properties)); + properties.put("csv_schema", "id:int"); + Assertions.assertEquals("id", new SchemaListedHdfsTvf(properties).getTableColumns().get(0).getName()); + } + + private static class SchemaListedHdfsTvf extends HdfsTableValuedFunction { + SchemaListedHdfsTvf(Map properties) throws org.apache.doris.common.AnalysisException { + super(properties); + } + + @Override + protected void parseFile() { + fileStatuses.add(new TBrokerFileStatus("hdfs://returned/data.csv", false, 10, true)); + } + } + + private static Map properties(String mount) { + Map properties = new HashMap<>(); + properties.put("uri", "viewfs://review/" + mount + "/file.csv"); + properties.put("fs.defaultFS", "viewfs://review"); + properties.put("fs.viewfs.mounttable.review.link./remote", "hdfs://remote/base"); + properties.put("fs.viewfs.mounttable.review.link./local", "file:///tmp"); + properties.put("format", "csv"); + properties.put("csv_schema", "id:int"); + return properties; + } + + private static class ListedS3Tvf extends S3TableValuedFunction { + ListedS3Tvf(Map properties, String mount) + throws org.apache.doris.common.AnalysisException { + super(withListing(properties, mount)); + } + + private static Map withListing(Map properties, String mount) { + properties.put("fs.test.listing", mount); + return properties; + } + + @Override + protected void parseFile(List files) { + String mount = backendConnectProperties.get("fs.test.listing"); + fileStatuses.add(new TBrokerFileStatus("viewfs://review/" + mount + "/listed.csv", false, 10, true)); + } + } + + private static class ListedHdfsTvf extends HdfsTableValuedFunction { + // Constructor-time listing is the existing TVF lifecycle. Select the fake listing through + // statement properties, which are available before super() calls parseFile(). + ListedHdfsTvf(Map properties, boolean localListing) + throws org.apache.doris.common.AnalysisException { + super(withListing(properties, localListing)); + } + + private static Map withListing(Map properties, boolean localListing) { + properties.put("fs.test.listing", localListing ? "local" : "remote"); + return properties; + } + + @Override + protected void parseFile() { + String mount = backendConnectProperties.get("fs.test.listing"); + fileStatuses.add(new TBrokerFileStatus("viewfs://review/" + mount + "/listed.csv", false, 10, true)); + } + } +} diff --git a/fe/fe-core/src/test/java/org/apache/doris/tablefunction/HadoopTvfRuntimeAdmissionTest.java b/fe/fe-core/src/test/java/org/apache/doris/tablefunction/HadoopTvfRuntimeAdmissionTest.java new file mode 100644 index 00000000000000..f4bdad2434918e --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/tablefunction/HadoopTvfRuntimeAdmissionTest.java @@ -0,0 +1,954 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.tablefunction; + +import org.apache.doris.common.Config; +import org.apache.doris.common.FeConstants; +import org.apache.doris.planner.PlanNodeId; +import org.apache.doris.planner.TVFTableSink; +import org.apache.doris.proto.InternalService.PFetchTableSchemaRequest; +import org.apache.doris.qe.ConnectContext; +import org.apache.doris.thrift.TBrokerFileStatus; +import org.apache.doris.thrift.TDataSink; +import org.apache.doris.thrift.TFileScanRange; +import org.apache.doris.thrift.TUniqueId; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.FileSystem.TvfGuard; +import org.apache.hadoop.fs.LocalFileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.viewfs.ViewFileSystem; +import org.apache.hadoop.fs.viewfs.ViewFileSystemOverloadScheme; +import org.apache.hadoop.hdfs.DistributedFileSystem; +import org.apache.hadoop.util.ReflectionUtils; +import org.apache.thrift.TDeserializer; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; +import org.mockito.MockedStatic; +import org.mockito.Mockito; + +import java.io.IOException; +import java.lang.reflect.Field; +import java.lang.reflect.Method; +import java.net.URI; +import java.net.URL; +import java.net.URLClassLoader; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicReference; + +class HadoopTvfRuntimeAdmissionTest { + @TempDir + java.nio.file.Path temporary; + private boolean originalEnable; + private boolean originalUnitTest; + private ConnectContext originalContext; + private final List ownedLoaders = new ArrayList<>(); + + @BeforeEach + void setUp() { + originalEnable = Config.enable_local_tvf; + originalUnitTest = FeConstants.runningUnitTest; + originalContext = ConnectContext.get(); + Config.enable_local_tvf = false; + FeConstants.runningUnitTest = true; + } + + @AfterEach + void tearDown() throws IOException { + Config.enable_local_tvf = originalEnable; + FeConstants.runningUnitTest = originalUnitTest; + ConnectContext.remove(); + if (originalContext != null) { + originalContext.setThreadLocalInfo(); + } + FileSystem.closeAll(); + for (URLClassLoader loader : ownedLoaders) { + loader.close(); + } + } + + @Test + void implicitLocalWriteProviderRejectsBeforeInitialization() throws Exception { + Map properties = hdfs("hdfs://runtime"); + Map guarded = HadoopTvfAdmission.protectNativeWrite("/data/prefix_", properties); + Configuration consumer = defaults(Collections.singletonMap("fs.hdfs.impl", LocalFileSystem.class.getName())); + overlay(consumer, guarded); + nativeRequest(consumer, "hdfs://runtime", "/data/prefix_"); + try (MockedStatic factory = Mockito.mockStatic( + ReflectionUtils.class, Mockito.CALLS_REAL_METHODS)) { + Assertions.assertThrows(IOException.class, + () -> FileSystem.newInstance(URI.create("hdfs://runtime"), consumer)); + factory.verify(() -> ReflectionUtils.newInstance(LocalFileSystem.class, consumer), Mockito.never()); + } + } + + @Test + void implicitOverloadedReadProviderRejectsWithLocalOptInToo() throws Exception { + Config.enable_local_tvf = true; + HdfsTableValuedFunction tvf = new HdfsTableValuedFunction(readProperties("hdfs://runtime/local/data.csv")); + Configuration consumer = defaults(Collections.singletonMap( + "fs.hdfs.impl", ViewFileSystemOverloadScheme.class.getName())); + consumer.set("fs.viewfs.mounttable.runtime.link./local", "file:///tmp"); + overlay(consumer, tvf.getBackendConnectProperties()); + nativeRequest(consumer, "hdfs://runtime", "/local/data.csv"); + try (MockedStatic factory = Mockito.mockStatic( + ReflectionUtils.class, Mockito.CALLS_REAL_METHODS)) { + Assertions.assertThrows(IOException.class, + () -> FileSystem.newInstance(URI.create("hdfs://runtime"), consumer)); + factory.verify(() -> ReflectionUtils.newInstance( + ViewFileSystemOverloadScheme.class, consumer), Mockito.never()); + } + } + + @Test + void implicitChildMountRejectsBeforeViewFsInitialization() throws Exception { + Map properties = hdfs("viewfs://cluster"); + properties.put("fs.viewfs.mounttable.cluster.link./data", "hdfs://remote/base"); + HadoopTvfAdmission.check(HadoopTvfAdmission.Access.NATIVE_WRITE, "/data/local/sub/prefix_", properties); + Configuration consumer = defaults(Collections.singletonMap( + "fs.viewfs.mounttable.cluster.link./data/local", "file:///tmp")); + overlay(consumer, HadoopTvfAdmission.protectNativeWrite("/data/local/sub/prefix_", properties)); + nativeRequest(consumer, "viewfs://cluster", "/data/local/sub/prefix_"); + try (MockedStatic factory = Mockito.mockStatic( + ReflectionUtils.class, Mockito.CALLS_REAL_METHODS)) { + Assertions.assertThrows(IOException.class, + () -> FileSystem.newInstance(URI.create("viewfs://cluster"), consumer)); + factory.verify(() -> ReflectionUtils.newInstance(ViewFileSystem.class, consumer), Mockito.never()); + } + } + + @Test + void remoteRoutesKeepUnrelatedLocalSiblingsAndCommaDirectories() throws Exception { + Map properties = hdfs("viewfs://cluster"); + properties.put("fs.viewfs.mounttable.cluster.link./data", "hdfs://remote/base,archive"); + Configuration consumer = defaults(Collections.singletonMap( + "fs.viewfs.mounttable.cluster.link./local", "file:///tmp")); + overlay(consumer, HadoopTvfAdmission.protectNativeWrite("/data/sub/prefix_", properties)); + nativeRequest(consumer, "viewfs://cluster", "/data/sub/prefix_"); + Assertions.assertDoesNotThrow(() -> TvfGuard.check(URI.create("viewfs://cluster"), consumer)); + overlay(consumer, HadoopTvfAdmission.protectNativeRead(properties)); + nativeRequest(consumer, "viewfs://cluster", "/data/file.csv"); + Assertions.assertDoesNotThrow(() -> TvfGuard.check(URI.create("viewfs://cluster"), consumer)); + } + + @Test + void actualProviderClassAndExpandedDefaultValuesAreChecked() throws Exception { + Map values = new HashMap<>(); + values.put("runtime.provider", LocalFileSystem.class.getName()); + values.put("fs.hdfs.impl", "${runtime.provider}"); + Configuration consumer = defaults(values); + overlay(consumer, HadoopTvfAdmission.protectNativeWrite("/data/prefix_", hdfs("hdfs://runtime"))); + nativeRequest(consumer, "hdfs://runtime", "/data/prefix_"); + Assertions.assertThrows(IOException.class, () -> TvfGuard.check(URI.create("hdfs://runtime"), consumer)); + } + + @Test + void unusedConfigurationValuesAreNotEagerlyExpanded() throws Exception { + Configuration consumer = defaults(Collections.emptyMap()); + consumer.set("cycle.a", "${cycle.b}x"); + consumer.set("cycle.b", "${cycle.a}y"); + consumer.set("fs.viewfs.mounttable.unused.link./data", "${cycle.a}"); + consumer.set("unrelated.setting", "${cycle.a}"); + overlay(consumer, HadoopTvfAdmission.protectNativeWrite("/data/prefix_", hdfs("hdfs://runtime"))); + nativeRequest(consumer, "hdfs://runtime", "/data/prefix_"); + Assertions.assertDoesNotThrow(() -> TvfGuard.check(URI.create("hdfs://runtime"), consumer)); + Assertions.assertThrows(IllegalStateException.class, () -> consumer.get("unrelated.setting")); + } + + @Test + void ordinaryUnmarkedFilesystemRetainsItsBehavior() throws Exception { + Configuration consumer = defaults(Collections.singletonMap("fs.hdfs.impl", LocalFileSystem.class.getName())); + try (FileSystem fs = FileSystem.newInstance(URI.create("hdfs://runtime"), consumer)) { + Assertions.assertInstanceOf(LocalFileSystem.class, fs); + } + Config.enable_local_tvf = true; + Assertions.assertFalse(HadoopTvfAdmission.protectNativeWrite("/data/prefix_", hdfs("hdfs://runtime")) + .containsKey(TvfGuard.POLICY_KEY)); + } + + @Test + void listingUsesJavaPolicyAndReadSenderUsesNativePolicy() throws Exception { + HdfsTableValuedFunction tvf = new HdfsTableValuedFunction(readProperties("hdfs://runtime/data.csv")); + Map javaProperties = tvf.getBrokerDesc().getStorageAdapter().getBackendConfigProperties(); + Assertions.assertNotNull(javaProperties.get(TvfGuard.POLICY_KEY)); + Configuration consumer = defaults(Collections.singletonMap("fs.hdfs.impl", LocalFileSystem.class.getName())); + overlay(consumer, javaProperties); + Assertions.assertThrows(IOException.class, () -> TvfGuard.check(URI.create("hdfs://runtime"), consumer)); + Assertions.assertNotNull(tvf.getBackendConnectProperties().get(TvfGuard.POLICY_KEY)); + } + + @Test + void schemaRequestCarriesPolicyInBothPropertiesAndHdfsParams() throws Exception { + ConnectContext context = new ConnectContext(); + context.setQueryId(new TUniqueId(1, 2)); + context.setThreadLocalInfo(); + Map properties = readProperties("hdfs://runtime/data.csv"); + properties.remove("csv_schema"); + HdfsTableValuedFunction tvf = new HdfsTableValuedFunction(properties); + tvf.getFileStatuses().add(new TBrokerFileStatus("hdfs://runtime/returned.csv", false, 100, true)); + Method method = ExternalFileTableValuedFunction.class.getDeclaredMethod( + "getFetchTableStructureRequest", java.util.Iterator.class); + method.setAccessible(true); + PFetchTableSchemaRequest request = (PFetchTableSchemaRequest) method.invoke( + tvf, tvf.getFileStatuses().iterator()); + TFileScanRange range = new TFileScanRange(); + new TDeserializer().deserialize(range, request.getFileScanRange().toByteArray()); + String policy = range.getParams().getProperties().get(TvfGuard.POLICY_KEY); + Assertions.assertNotNull(policy); + Assertions.assertTrue(range.getParams().getHdfsParams().getHdfsConf().stream() + .anyMatch(conf -> TvfGuard.POLICY_KEY.equals(conf.getKey()) && policy.equals(conf.getValue()))); + } + + @Test + void unusedInitialSchemaPathDoesNotRejectTheActualReturnedFile() throws Exception { + Map properties = hdfs("viewfs://schema"); + properties.put("uri", "hdfs://initial/input.csv"); + properties.put("format", "csv"); + properties.put("fs.viewfs.mounttable.schema.linkFallback", "hdfs://remote/base"); + HdfsTableValuedFunction tvf = new HdfsTableValuedFunction(properties); + tvf.getFileStatuses().add(new TBrokerFileStatus("hdfs://returned/data.csv", false, 100, true)); + Configuration consumer = defaults(Collections.singletonMap( + "fs.viewfs.mounttable.schema.link./hdfs:/initial", "file:///tmp")); + overlay(consumer, tvf.getBackendConnectProperties()); + nativeRequest(consumer, "viewfs://schema", "hdfs://returned/data.csv"); + Assertions.assertDoesNotThrow(() -> TvfGuard.check(URI.create("viewfs://schema"), consumer)); + } + + @Test + void brokerToHadoopDataSenderCannotDropRuntimePolicy() throws Exception { + Map properties = readProperties("hdfs://runtime/data.csv"); + properties.put("fs.broker.support", "true"); + properties.put("broker.name", "runtime-review"); + S3TableValuedFunction tvf = new S3TableValuedFunction(properties, Collections.singletonList( + new TBrokerFileStatus("hdfs://runtime/returned.csv", false, 100, true))); + String policy = tvf.getBackendConnectProperties().get(TvfGuard.POLICY_KEY); + Assertions.assertNotNull(policy); + Configuration consumer = defaults(Collections.singletonMap("fs.hdfs.impl", LocalFileSystem.class.getName())); + overlay(consumer, tvf.getBackendConnectProperties()); + Assertions.assertThrows(IOException.class, () -> TvfGuard.check(URI.create("hdfs://runtime"), consumer)); + } + + @Test + void finalSinkOverridesUserPolicyAliasesAndEnabledWritesClearThem() throws Exception { + Map properties = hdfs("hdfs://runtime"); + properties.put("file_path", "/data/prefix_"); + properties.put("format", "csv"); + properties.put(TvfGuard.POLICY_KEY.toUpperCase(java.util.Locale.ROOT) + "\0suffix", "attacker-value"); + properties.put(TvfGuard.POLICY_KEY, "attacker-value"); + TVFTableSink sink = new TVFTableSink(new PlanNodeId(0), "hdfs", properties, Collections.emptyList()); + sink.bindDataSink(); + Method method = TVFTableSink.class.getDeclaredMethod("toThrift"); + method.setAccessible(true); + TDataSink thrift = (TDataSink) method.invoke(sink); + Map sent = thrift.getTvfTableSink().getProperties(); + Assertions.assertNotNull(sent.get(TvfGuard.POLICY_KEY)); + Assertions.assertNotEquals("attacker-value", sent.get(TvfGuard.POLICY_KEY)); + Assertions.assertFalse(sent.containsKey(TvfGuard.POLICY_KEY.toUpperCase(java.util.Locale.ROOT) + "\0suffix")); + Config.enable_local_tvf = true; + sink.bindDataSink(); + sent = ((TDataSink) method.invoke(sink)).getTvfTableSink().getProperties(); + Assertions.assertFalse(sent.containsKey(TvfGuard.POLICY_KEY)); + } + + @Test + void nativeContextChecksEachPathAgainstTheConnectionSnapshot() throws Exception { + Configuration consumer = defaults(Collections.emptyMap()); + consumer.set("fs.viewfs.mounttable.cluster.link./data", "hdfs://remote/base"); + consumer.set("fs.viewfs.mounttable.cluster.link./local", "file:///tmp"); + consumer.set(TvfGuard.POLICY_KEY, TvfGuard.NATIVE_POLICY); + nativeRequest(consumer, "viewfs://cluster", "/data/a.csv"); + TvfGuard.clearNativeContext(); + try (MockedStatic factory = Mockito.mockStatic( + ReflectionUtils.class, Mockito.CALLS_REAL_METHODS)) { + factory.when(() -> ReflectionUtils.newInstance(Mockito.eq(ViewFileSystem.class), + Mockito.any(Configuration.class))).thenReturn(Mockito.mock(ViewFileSystem.class)); + FileSystem.newInstance(URI.create("viewfs://cluster"), consumer); + TvfGuard.Context context = TvfGuard.takeNativeContext(); + Assertions.assertNotNull(context); + Assertions.assertNull(TvfGuard.takeNativeContext()); + Assertions.assertDoesNotThrow(() -> context.checkNative("viewfs://cluster", "/data/b.csv")); + Assertions.assertThrows(IOException.class, + () -> context.checkNative("viewfs://cluster", "/local/b.csv")); + consumer.set("fs.viewfs.mounttable.cluster.link./data", "file:///tmp"); + Assertions.assertDoesNotThrow(() -> context.checkNative("viewfs://cluster", "/data/c.csv")); + } + } + + @Test + void javaReadBindsActualPathAndRejectsUnmarkedCachedInstances() throws Exception { + Configuration consumer = defaults(Collections.emptyMap()); + consumer.set("fs.viewfs.mounttable.cluster.link./data", "hdfs://remote/base"); + consumer.set("fs.viewfs.mounttable.cluster.link./local", "file:///tmp"); + Path remote = new Path("viewfs://cluster/data/a%2fb c.csv"); + Configuration frozen = TvfGuard.prepareJavaRead(consumer, remote, TvfGuard.NATIVE_POLICY); + FileSystem actual = Mockito.mock(FileSystem.class); + Mockito.when(actual.getConf()).thenReturn(frozen); + Assertions.assertDoesNotThrow(() -> TvfGuard.checkJavaRead(actual, remote, TvfGuard.NATIVE_POLICY)); + Assertions.assertThrows(IOException.class, () -> TvfGuard.checkJavaRead(actual, + new Path("viewfs://cluster/local/data.csv"), TvfGuard.NATIVE_POLICY)); + Mockito.when(actual.getConf()).thenReturn(new Configuration(frozen)); + Assertions.assertDoesNotThrow(() -> TvfGuard.checkJavaRead(actual, remote, TvfGuard.NATIVE_POLICY)); + Assertions.assertThrows(IOException.class, () -> TvfGuard.checkJavaRead(actual, + new Path("viewfs://cluster/local/data.csv"), TvfGuard.NATIVE_POLICY)); + Mockito.when(actual.getConf()).thenReturn(consumer); + Assertions.assertThrows(IOException.class, () -> TvfGuard.checkJavaRead(actual, remote, TvfGuard.NATIVE_POLICY)); + Assertions.assertDoesNotThrow(() -> TvfGuard.checkJavaRead(actual, remote, null)); + } + + @Test + void actualS3BucketConfigurationRetainsTheReadGuard() throws Exception { + Configuration consumer = defaults(Collections.emptyMap()); + Path path = new Path("s3a://bucket/data.avro"); + Configuration frozen = TvfGuard.prepareJavaRead(consumer, path, TvfGuard.NATIVE_POLICY); + Configuration derived = org.apache.hadoop.fs.s3a.S3AUtils.propagateBucketOptions(frozen, "bucket"); + Assertions.assertEquals(Configuration.class, derived.getClass()); + FileSystem actual = Mockito.mock(FileSystem.class); + Mockito.when(actual.getConf()).thenReturn(derived); + Assertions.assertDoesNotThrow(() -> TvfGuard.checkJavaRead(actual, path, TvfGuard.NATIVE_POLICY)); + Mockito.when(actual.getConf()).thenReturn(new org.apache.hadoop.mapred.JobConf(derived)); + Assertions.assertDoesNotThrow(() -> TvfGuard.checkJavaRead(actual, path, TvfGuard.NATIVE_POLICY)); + } + + @Test + void nativeRequestRequiresActualPathAndConstantMarkerDoesNotGrowWithFileCount() throws Exception { + Map properties = HadoopTvfAdmission.protectNativeRead(hdfs("hdfs://runtime")); + Assertions.assertEquals(TvfGuard.NATIVE_POLICY, properties.get(TvfGuard.POLICY_KEY)); + Configuration consumer = defaults(Collections.emptyMap()); + overlay(consumer, properties); + Assertions.assertThrows(IOException.class, () -> TvfGuard.check(URI.create("hdfs://runtime"), consumer)); + } + + private static void nativeRequest(Configuration configuration, String root, String path) { + configuration.set(TvfGuard.ROOT_KEY, root); + configuration.set(TvfGuard.PATH_KEY, path); + } + + @Test + void escapedListingComponentsAreCheckedBeforeDiscovery() throws Exception { + Map properties = readProperties("viewfs://cluster/da\\ta/*.csv"); + properties.put("fs.defaultFS", "viewfs://cluster"); + properties.put("fs.viewfs.mounttable.cluster.link./data", "file:///tmp"); + properties.put("fs.viewfs.mounttable.cluster.linkFallback", "hdfs://remote/base"); + Assertions.assertThrows(org.apache.doris.nereids.exceptions.AnalysisException.class, + () -> new HdfsTableValuedFunction(properties)); + properties.put("fs.viewfs.mounttable.cluster.link./data", "hdfs://remote/data"); + Assertions.assertDoesNotThrow(() -> new HdfsTableValuedFunction(properties)); + Configuration actual = defaults(Collections.singletonMap( + "fs.viewfs.mounttable.cluster.link./data", "file:///tmp")); + properties.remove("fs.viewfs.mounttable.cluster.link./data"); + overlay(actual, HadoopTvfAdmission.protectJava( + HadoopTvfAdmission.Access.HADOOP_READ, properties.get("uri"), properties)); + Assertions.assertThrows(IOException.class, + () -> TvfGuard.check(URI.create("viewfs://cluster"), actual)); + } + + @Test + void globTraversalChecksIntermediateParentsAndEscapedDotSegments() { + Map properties = hdfs("viewfs://cluster"); + properties.put("fs.viewfs.mounttable.cluster.link./data", "file:///tmp"); + properties.put("fs.viewfs.mounttable.cluster.linkFallback", "hdfs://remote/base"); + for (String path : new String[] {"viewfs://cluster/da\\ta/*/\\.\\./other/*.csv", + "viewfs://cluster/{da\\ta/x,remote/y}/*.csv"}) { + Assertions.assertThrows(org.apache.doris.nereids.exceptions.AnalysisException.class, + () -> HadoopTvfAdmission.checkReadDiscovery(path, properties)); + } + properties.put("fs.viewfs.mounttable.cluster.link./data", "hdfs://remote/data"); + Assertions.assertDoesNotThrow(() -> HadoopTvfAdmission.checkReadDiscovery("viewfs://cluster/da\\ta/\\*/file.csv", properties)); + properties.put("fs.viewfs.mounttable.cluster.link./local", "file:///tmp"); + properties.put("fs.viewfs.mounttable.cluster.link./literal\\name", "hdfs://remote/base"); + Assertions.assertDoesNotThrow(() -> HadoopTvfAdmission.check(HadoopTvfAdmission.Access.JAVA_DELETE, + "viewfs://cluster/literal\\name/file.csv", properties)); + } + + @Test + void explicitJuiceFsUsesTheSameKnownRemoteProviderAsRuntime() throws Exception { + for (boolean enabled : new boolean[] {false, true}) { + Config.enable_local_tvf = enabled; + Map properties = readProperties("jfs://cluster/data.csv"); + properties.put("fs.defaultFS", "jfs://cluster"); + properties.put("fs.jfs.impl", "io.juicefs.JuiceFileSystem"); + properties.put("juicefs.cluster.meta", "redis://meta:6379/1"); + Assertions.assertDoesNotThrow(() -> new HdfsTableValuedFunction(properties)); + properties.put("fs.jfs.impl", LocalFileSystem.class.getName()); + Assertions.assertThrows(org.apache.doris.nereids.exceptions.AnalysisException.class, + () -> new HdfsTableValuedFunction(properties)); + } + } + + @Test + void secureWasbDefaultAndDelegatedRoutesKeepRemoteAdmission() throws Exception { + Configuration direct = defaults(Collections.emptyMap()); + Assertions.assertEquals("org.apache.hadoop.fs.azure.NativeAzureFileSystem$Secure", + FileSystem.getFileSystemClass("wasbs", direct).getName()); + direct.set(TvfGuard.POLICY_KEY, TvfGuard.NATIVE_POLICY); + nativeRequest(direct, "wasbs://container@account.blob.core.windows.net", "/exports/file"); + Assertions.assertDoesNotThrow(() -> TvfGuard.check(URI.create("wasbs://container@account.blob.core.windows.net"), direct)); + Configuration delegated = new Configuration(direct); + nativeRequest(delegated, "viewfs://cluster", "/exports/file"); + delegated.set("fs.viewfs.mounttable.cluster.link./exports", + "wasbs://container@account.blob.core.windows.net/base"); + Assertions.assertDoesNotThrow(() -> TvfGuard.check(URI.create("viewfs://cluster"), delegated)); + delegated.set("fs.wasbs.impl", LocalFileSystem.class.getName()); + Assertions.assertThrows(IOException.class, () -> TvfGuard.check(URI.create("viewfs://cluster"), delegated)); + } + + @Test + void guardedSnapshotDisablesOnlyViewFsInnerCache() throws Exception { + Configuration source = defaults(Collections.emptyMap()); + source.setBoolean("fs.viewfs.enable.inner.cache", true); + Configuration frozen = TvfGuard.prepareJavaRead(source, new Path("hdfs://runtime/data.csv"), TvfGuard.NATIVE_POLICY); + Assertions.assertFalse(frozen.getBoolean("fs.viewfs.enable.inner.cache", true)); + Assertions.assertTrue(source.getBoolean("fs.viewfs.enable.inner.cache", false)); + Assertions.assertSame(source, TvfGuard.prepareJavaRead(source, new Path("file:///unused"), null)); + } + + @Test + void noGlobBackslashMatchesTheBranchListingConsumer() { + Map properties = hdfs("viewfs://cluster"); + properties.put("fs.viewfs.mounttable.cluster.link./data", "file:///tmp"); + properties.put("fs.viewfs.mounttable.cluster.linkFallback", "hdfs://remote/base"); + Assertions.assertDoesNotThrow(() -> HadoopTvfAdmission.checkReadDiscovery("viewfs://cluster/da\\ta/file.csv", properties)); + } + + @Test + void nestedViewFsCaseDistinctTargetsKeepTheirActualIdentityAfterWarmup() throws Exception { + Configuration source = defaults(Collections.emptyMap()); + source.set(TvfGuard.POLICY_KEY, TvfGuard.NATIVE_POLICY); + nativeRequest(source, "viewfs://outer", "/warm/file.csv"); + source.setBoolean("fs.viewfs.enable.inner.cache", true); + source.set("fs.viewfs.mounttable.outer.link./warm", "viewfs://A/remote"); + source.set("fs.viewfs.mounttable.outer.link./evil", "viewfs://a/secret"); + source.set("fs.viewfs.mounttable.A.link./remote", "hdfs://nn/base"); + source.set("fs.viewfs.mounttable.A.link./secret", "file:///unused-local"); + source.set("fs.viewfs.mounttable.a.linkFallback", "hdfs://nn/lower"); + java.util.List accesses = new java.util.ArrayList<>(); + LocalFileSystem local = Mockito.mock(LocalFileSystem.class); + Mockito.when(local.getUri()).thenReturn(URI.create("file:///")); + Mockito.when(local.getWorkingDirectory()).thenReturn(new Path("/")); + try (MockedStatic factory = Mockito.mockStatic( + ReflectionUtils.class, Mockito.CALLS_REAL_METHODS)) { + factory.when(() -> ReflectionUtils.newInstance(Mockito.eq(DistributedFileSystem.class), + Mockito.any(Configuration.class))).thenAnswer(invocation -> { + DistributedFileSystem remote = Mockito.mock(DistributedFileSystem.class); + Mockito.when(remote.getConf()).thenReturn(invocation.getArgument(1)); + Mockito.when(remote.getUri()).thenReturn(URI.create("hdfs://nn")); + Mockito.when(remote.getWorkingDirectory()).thenReturn(new Path("/")); + Mockito.when(remote.getFileStatus(Mockito.any(Path.class))).thenAnswer(call -> { + Path path = call.getArgument(0); + accesses.add(path.toUri().getPath()); + return new org.apache.hadoop.fs.FileStatus(1, false, 1, 1, 0, path); + }); + return remote; + }); + factory.when(() -> ReflectionUtils.newInstance(Mockito.eq(LocalFileSystem.class), + Mockito.any(Configuration.class))).thenReturn(local); + FileSystem outer = FileSystem.newInstance(URI.create("viewfs://outer"), source); + TvfGuard.Context context = TvfGuard.takeNativeContext(); + Assertions.assertNotNull(context); + Assertions.assertFalse(outer.getConf().getBoolean("fs.viewfs.enable.inner.cache", true)); + outer.getFileStatus(new Path("/warm/file.csv")); + context.checkNative("viewfs://outer", "/evil/file.csv"); + outer.getFileStatus(new Path("/evil/file.csv")); + outer.getFileStatus(new Path("/evil/next.csv")); + Assertions.assertTrue(accesses.contains("/base/file.csv"), accesses.toString()); + Assertions.assertTrue(accesses.contains("/lower/secret/file.csv"), accesses.toString()); + Assertions.assertTrue(accesses.contains("/lower/secret/next.csv"), accesses.toString()); + Mockito.verify(local, Mockito.never()).getFileStatus(Mockito.any(Path.class)); + } + } + + @Test + void slashBracesCheckFlattenedRoutesBeforeStaticOrRuntimeDiscovery() throws Exception { + Map properties = hdfs("viewfs://braces"); + properties.put("fs.viewfs.mounttable.braces.link./remote", "hdfs://nn/base"); + properties.put("fs.viewfs.mounttable.braces.linkFallback", "hdfs://nn/fallback"); + for (String pattern : new String[] {"/remote/{../local}/*.csv", + "/remote/{ok,../local}/*.csv", "/remote/{{../local}}/*.csv"}) { + String uri = "viewfs://braces" + pattern; + Map statement = readProperties(uri); + statement.putAll(properties); + statement.put("fs.viewfs.mounttable.braces.link./local", "file:///unused-local"); + Assertions.assertThrows(org.apache.doris.nereids.exceptions.AnalysisException.class, + () -> HadoopTvfAdmission.checkReadDiscovery(uri, statement), uri); + if ("/remote/{../local}/*.csv".equals(pattern)) { + // The reported singleton URI reaches admission through the actual TVF constructor. + // Other brace alternatives may be rejected earlier by the existing URI parameter parser. + Assertions.assertThrows(org.apache.doris.nereids.exceptions.AnalysisException.class, + () -> new HdfsTableValuedFunction(statement), uri); + } + Configuration runtime = defaults(Collections.singletonMap( + "fs.viewfs.mounttable.braces.link./local", "file:///unused-local")); + overlay(runtime, HadoopTvfAdmission.protectJava(HadoopTvfAdmission.Access.HADOOP_READ, uri, properties)); + Assertions.assertThrows(IOException.class, () -> TvfGuard.check(URI.create("viewfs://braces"), runtime), uri); + } + properties.put("fs.viewfs.mounttable.braces.link./local", "file:///unused-local"); + String remote = "viewfs://braces/remote/{../other}/*.csv"; + Assertions.assertDoesNotThrow(() -> HadoopTvfAdmission.checkReadDiscovery(remote, properties)); + Assertions.assertDoesNotThrow(() -> TvfGuard.check(URI.create("viewfs://braces"), + javaRequest(properties, HadoopTvfAdmission.Access.HADOOP_READ, remote))); + } + + @Test + void exactCleanupKeepsLiteralBracesButRejectsSelectedAndNestedLocalTargets() throws Exception { + Map properties = hdfs("viewfs://cleanup"); + properties.put("fs.viewfs.mounttable.cleanup.link./data", "hdfs://nn/base"); + properties.put("fs.viewfs.mounttable.cleanup.link./data/local", "file:///unused-local"); + properties.put("fs.viewfs.mounttable.cleanup.link./nested", "viewfs://inner/base"); + properties.put("fs.viewfs.mounttable.inner.link./base", "file:///unused-local"); + String literal = "viewfs://cleanup/data/literal{tag}"; + Assertions.assertDoesNotThrow(() -> HadoopTvfAdmission.check( + HadoopTvfAdmission.Access.JAVA_DELETE, literal, properties)); + Configuration runtime = javaRequest(properties, HadoopTvfAdmission.Access.JAVA_DELETE, literal); + try (RemoteClients clients = new RemoteClients(); + FileSystem outer = FileSystem.get(URI.create("viewfs://cleanup"), runtime)) { + Assertions.assertTrue(outer.delete(new Path(literal), true)); + Assertions.assertTrue(clients.events.contains("delete:0:/base/literal{tag}")); + Assertions.assertThrows(IOException.class, () -> outer.delete(new Path("/data"), true)); + Assertions.assertThrows(IOException.class, () -> outer.delete(new Path("/"), true)); + } + for (String selected : new String[] {"/data/local/file", "/nested/file"}) { + String uri = "viewfs://cleanup" + selected; + Assertions.assertThrows(org.apache.doris.nereids.exceptions.AnalysisException.class, + () -> HadoopTvfAdmission.check(HadoopTvfAdmission.Access.JAVA_DELETE, uri, properties)); + // The same selected local route can exist only in effective core-site.xml. + Configuration effective = defaults(properties); + overlay(effective, HadoopTvfAdmission.protectJava( + HadoopTvfAdmission.Access.JAVA_DELETE, uri, hdfs("viewfs://cleanup"))); + Assertions.assertThrows(IOException.class, () -> TvfGuard.check(URI.create("viewfs://cleanup"), effective)); + } + } + + @Test + void nativeWriteChecksCreateParentOnInitializationAndRetainedContextReuse() throws Exception { + Map write = HadoopTvfAdmission.protectNativeWrite("/safe/prefix_", hdfs("viewfs://create")); + Assertions.assertEquals(TvfGuard.NATIVE_WRITE_POLICY, write.get(TvfGuard.POLICY_KEY)); + Assertions.assertNotEquals(HadoopTvfAdmission.protectNativeRead(hdfs("viewfs://create")), write); + for (int i = 0; i < 16; i++) { + Assertions.assertEquals(write, HadoopTvfAdmission.protectNativeWrite( + "/safe/file" + i + "\0ignored-suffix", hdfs("viewfs://create"))); + } + Map mounts = new HashMap<>(); + mounts.put("fs.viewfs.mounttable.create.link./safe", "hdfs://nn/base"); + mounts.put("fs.viewfs.mounttable.create.link./parent", "file:///unused-local"); + mounts.put("fs.viewfs.mounttable.create.link./parent/final", "hdfs://nn/final"); + mounts.put("fs.viewfs.mounttable.create.link./remote", "hdfs://nn/base"); + mounts.put("fs.viewfs.mounttable.create.link./remote/final", "file:///unused-local"); + mounts.put("fs.viewfs.mounttable.create.link./nested", "viewfs://inner/parent"); + mounts.put("fs.viewfs.mounttable.inner.link./parent", "file:///unused-local"); + mounts.put("fs.viewfs.mounttable.inner.link./parent/final", "hdfs://nn/final"); + Configuration runtime = defaults(mounts); + overlay(runtime, write); + nativeRequest(runtime, "viewfs://create", "/parent/final"); + Assertions.assertThrows(IOException.class, () -> FileSystem.newInstance(URI.create("viewfs://create"), runtime)); + nativeRequest(runtime, "viewfs://create", "/safe/file"); + try (MockedStatic factory = Mockito.mockStatic( + ReflectionUtils.class, Mockito.CALLS_REAL_METHODS)) { + factory.when(() -> ReflectionUtils.newInstance(Mockito.eq(ViewFileSystem.class), + Mockito.any(Configuration.class))).thenAnswer(call -> Mockito.mock(ViewFileSystem.class)); + TvfGuard.clearNativeContext(); + FileSystem.newInstance(URI.create("viewfs://create"), runtime); + TvfGuard.Context writer = TvfGuard.takeNativeContext(); + Assertions.assertNotNull(writer); + Assertions.assertDoesNotThrow(() -> writer.checkNative("viewfs://create", "/safe/next")); + Assertions.assertDoesNotThrow(() -> writer.checkNative("viewfs://create", "/remote/final")); + for (String path : new String[] {"/parent/final", "/nested/final"}) { + Assertions.assertThrows(IOException.class, () -> writer.checkNative("viewfs://create", path)); + } + Configuration read = new Configuration(runtime); + read.set(TvfGuard.POLICY_KEY, TvfGuard.NATIVE_POLICY); + FileSystem.newInstance(URI.create("viewfs://create"), read); + TvfGuard.Context reader = TvfGuard.takeNativeContext(); + Assertions.assertNotNull(reader); + Assertions.assertNotEquals(writer.profile(), reader.profile()); + Assertions.assertThrows(IOException.class, () -> reader.checkNative("viewfs://create", "/remote/final")); + } + } + + @Test + void differentJavaPathsCloseOwnedClientsAndReturnGlobalCacheToBaseline() throws Exception { + Map properties = hdfs("viewfs://owned"); + properties.put("fs.viewfs.mounttable.owned.link./data", "hdfs://nn/base"); + properties.put("fs.viewfs.mounttable.owned.link./alias", "hdfs://nn/alternate"); + int baseline = filesystemCacheSize(); + try (RemoteClients clients = new RemoteClients()) { + for (int i = 0; i < 6; i++) { + String path = "viewfs://owned/data/file" + i; + try (FileSystem outer = FileSystem.get(URI.create("viewfs://owned"), + javaRequest(properties, HadoopTvfAdmission.Access.HADOOP_READ, path))) { + outer.getFileStatus(new Path(path)); + outer.getFileStatus(new Path("/alias/file" + i)); + Assertions.assertEquals(i + 1, clients.clients.size()); + } + Assertions.assertEquals(baseline, filesystemCacheSize(), path); + } + clients.verifyClosedOnce(); + } + } + + @Test + void equalJavaPoliciesOwnIndependentClientsAndDeleteBeforeClosingThem() throws Exception { + Map properties = hdfs("viewfs://independent"); + properties.put("fs.viewfs.mounttable.independent.link./data", "hdfs://nn/base"); + Configuration source = javaRequest(properties, HadoopTvfAdmission.Access.HADOOP_READ, + "viewfs://independent/data/file.csv"); + int baseline = filesystemCacheSize(); + try (RemoteClients clients = new RemoteClients()) { + try (FileSystem first = FileSystem.get(URI.create("viewfs://independent"), source); + FileSystem second = FileSystem.get(URI.create("viewfs://independent"), source)) { + Assertions.assertNotSame(first, second); + first.getFileStatus(new Path("/data/file.csv")); + second.getFileStatus(new Path("/data/file.csv")); + Assertions.assertEquals(2, clients.clients.size()); + Assertions.assertTrue(first.deleteOnExit(new Path("/data/file.csv"))); + first.close(); + Assertions.assertTrue(clients.events.contains("delete:0:/base/file.csv")); + Assertions.assertTrue(clients.events.indexOf("delete:0:/base/file.csv") + < clients.events.indexOf("close:0"), clients.events.toString()); + Mockito.verify(clients.clients.get(1), Mockito.never()).close(); + Assertions.assertDoesNotThrow(() -> second.getFileStatus(new Path("/data/file.csv"))); + first.close(); + Mockito.verify(clients.clients.get(0), Mockito.times(1)).close(); + } + clients.verifyClosedOnce(); + Assertions.assertEquals(baseline, filesystemCacheSize()); + } + } + + @Test + void nestedJavaViewFsKeepsCaseDistinctAuthoritiesAndClosesEachActualClient() throws Exception { + Map properties = hdfs("viewfs://outer-owned"); + properties.put("fs.viewfs.mounttable.outer-owned.link./warm", "viewfs://A/remote"); + properties.put("fs.viewfs.mounttable.outer-owned.link./other", "viewfs://a/secret"); + properties.put("fs.viewfs.mounttable.A.link./remote", "hdfs://upper/base"); + properties.put("fs.viewfs.mounttable.A.link./secret", "file:///unused-local"); + properties.put("fs.viewfs.mounttable.a.linkFallback", "hdfs://lower/lower"); + int baseline = filesystemCacheSize(); + try (RemoteClients clients = new RemoteClients()) { + try (FileSystem outer = FileSystem.get(URI.create("viewfs://outer-owned"), javaRequest( + properties, HadoopTvfAdmission.Access.HADOOP_READ, "viewfs://outer-owned/warm/file.csv"))) { + outer.getFileStatus(new Path("/warm/file.csv")); + outer.getFileStatus(new Path("/other/file.csv")); + outer.getFileStatus(new Path("/other/next.csv")); + Assertions.assertTrue(clients.events.contains("stat:0:/base/file.csv")); + Assertions.assertTrue(clients.events.contains("stat:1:/lower/secret/file.csv")); + Assertions.assertTrue(clients.events.contains("stat:1:/lower/secret/next.csv")); + Assertions.assertEquals(URI.create("hdfs://upper"), clients.clients.get(0).getUri()); + Assertions.assertEquals(URI.create("hdfs://lower"), clients.clients.get(1).getUri()); + } + Assertions.assertEquals(2, clients.clients.size()); + clients.verifyClosedOnce(); + Assertions.assertEquals(baseline, filesystemCacheSize()); + } + } + + @Test + void nflyOwnsSameAuthorityPeersAndClosesOthersAfterOneCloseFails() throws Exception { + Map properties = hdfs("viewfs://peers"); + properties.put("fs.viewfs.mounttable.peers.linkNfly.minReplication=2./data", + "hdfs://nn/first,hdfs://nn/second"); + int baseline = filesystemCacheSize(); + try (RemoteClients clients = new RemoteClients()) { + FileSystem outer = FileSystem.get(URI.create("viewfs://peers"), javaRequest( + properties, HadoopTvfAdmission.Access.HADOOP_READ, "viewfs://peers/data/file.csv")); + Assertions.assertEquals(2, clients.clients.size()); + Assertions.assertNotSame(clients.clients.get(0), clients.clients.get(1)); + clients.failNextClose = true; + Assertions.assertThrows(IOException.class, outer::close); + Assertions.assertDoesNotThrow(outer::close); + clients.verifyClosedOnce(); + Assertions.assertEquals(baseline, filesystemCacheSize()); + } + } + + @Test + void failedNflyInitializationClosesSuccessfulAndFailingPeers() throws Exception { + Map properties = hdfs("viewfs://partial"); + properties.put("fs.viewfs.mounttable.partial.linkNfly.minReplication=2./data", + "hdfs://good/first,hdfs://fail/second"); + int baseline = filesystemCacheSize(); + try (RemoteClients clients = new RemoteClients()) { + clients.failInitializeAuthority = "fail"; + Assertions.assertThrows(IOException.class, () -> FileSystem.get(URI.create("viewfs://partial"), + javaRequest(properties, HadoopTvfAdmission.Access.HADOOP_READ, "viewfs://partial/data/file.csv"))); + Assertions.assertEquals(2, clients.clients.size()); + clients.verifyClosedOnce(); + Assertions.assertEquals(baseline, filesystemCacheSize()); + } + } + + @Test + void discoveryOnlyChecksNormalizedProbesWhileReturnedPathsRemainLiteral() throws Exception { + Map properties = hdfs("viewfs://normalized"); + properties.put("fs.viewfs.mounttable.normalized.link./remote", "file:///unused-local"); + properties.put("fs.viewfs.mounttable.normalized.link./other", "hdfs://nn/base"); + properties.put("fs.viewfs.mounttable.normalized.linkFallback", "hdfs://nn/fallback"); + for (boolean enabled : new boolean[] {false, true}) { + Config.enable_local_tvf = enabled; + try (RemoteClients clients = new RemoteClients()) { + int baseline = filesystemCacheSize(); + for (String pattern : new String[] {"/remote/{../other}/*.csv", + "/remote/{{../other}}/*.csv", "/remote/\\.\\./other/*.csv"}) { + String uri = "viewfs://normalized" + pattern; + Assertions.assertDoesNotThrow(() -> HadoopTvfAdmission.checkReadDiscovery(uri, properties)); + // A supplied/returned file with this spelling is literal and still selects LOCAL. + Assertions.assertThrows(org.apache.doris.nereids.exceptions.AnalysisException.class, + () -> HadoopTvfAdmission.check(HadoopTvfAdmission.Access.HADOOP_READ, + uri, properties, false)); + if ("/remote/{../other}/*.csv".equals(pattern)) { + Map statement = readProperties(uri); + statement.putAll(properties); + Assertions.assertDoesNotThrow(() -> new HdfsTableValuedFunction(statement)); + } + Configuration request = javaRequest(properties, HadoopTvfAdmission.Access.HADOOP_READ, uri); + try (FileSystem filesystem = FileSystem.get(URI.create("viewfs://normalized"), request)) { + org.apache.hadoop.fs.FileStatus[] matches = filesystem.globStatus(new Path(uri)); + Assertions.assertEquals(1, matches.length, uri + clients.events); + Assertions.assertEquals("/other/probe.csv", matches[0].getPath().toUri().getPath(), uri); + Assertions.assertDoesNotThrow(() -> HadoopTvfAdmission.check( + HadoopTvfAdmission.Access.HADOOP_READ, + matches[0].getPath().toString(), properties, false)); + } + clients.verifyClosedOnce(); + Assertions.assertEquals(baseline, filesystemCacheSize()); + } + } + } + } + + @Test + void escapedMetacharactersKeepLiteralIdentityBeforeAndAfterRealWildcards() throws Exception { + Map properties = hdfs("viewfs://literal"); + properties.put("fs.viewfs.mounttable.literal.link./data", "hdfs://nn/base"); + properties.put("fs.viewfs.mounttable.literal.link./data/private", "file:///unused-local"); + properties.put("fs.viewfs.mounttable.literal.linkFallback", "hdfs://nn/fallback"); + String[][] paths = { + {"/data/\\*/file.csv", "/data/*/file.csv"}, + {"/data/\\*/p?obe.csv", "/data/*/probe.csv"}, + {"/data/\\?/p?obe.csv", "/data/?/probe.csv"}, + {"/data/\\[x\\]/p?obe.csv", "/data/[x]/probe.csv"}, + {"/data/\\{x\\}/p?obe.csv", "/data/{x}/probe.csv"}, + {"/data/literal\\\\name/p?obe.csv", "/data/literal\\name/probe.csv"} + }; + for (boolean enabled : new boolean[] {false, true}) { + Config.enable_local_tvf = enabled; + try (RemoteClients clients = new RemoteClients()) { + int baseline = filesystemCacheSize(); + for (String[] entry : paths) { + String uri = "viewfs://literal" + entry[0]; + Assertions.assertDoesNotThrow(() -> HadoopTvfAdmission.checkReadDiscovery(uri, properties), uri); + Configuration request = javaRequest(properties, HadoopTvfAdmission.Access.HADOOP_READ, uri); + try (FileSystem filesystem = FileSystem.get(URI.create("viewfs://literal"), request)) { + org.apache.hadoop.fs.FileStatus[] matches = filesystem.globStatus(new Path(uri)); + Assertions.assertEquals(1, matches.length, uri + clients.events); + Assertions.assertEquals(entry[1], matches[0].getPath().toUri().getPath(), uri); + Assertions.assertDoesNotThrow(() -> HadoopTvfAdmission.check( + HadoopTvfAdmission.Access.HADOOP_READ, + matches[0].getPath().toString(), properties, false)); + } + clients.verifyClosedOnce(); + Assertions.assertEquals(baseline, filesystemCacheSize()); + } + } + String actualWildcard = "viewfs://literal/data/*/file.csv"; + Assertions.assertThrows(org.apache.doris.nereids.exceptions.AnalysisException.class, + () -> HadoopTvfAdmission.checkReadDiscovery(actualWildcard, properties)); + Assertions.assertThrows(IOException.class, () -> TvfGuard.check(URI.create("viewfs://literal"), + javaRequest(properties, HadoopTvfAdmission.Access.HADOOP_READ, actualWildcard))); + properties.put("fs.viewfs.mounttable.literal.link./data/*", "file:///unused-local"); + String selectedLocal = "viewfs://literal/data/\\*/file.csv"; + Assertions.assertThrows(org.apache.doris.nereids.exceptions.AnalysisException.class, + () -> HadoopTvfAdmission.checkReadDiscovery(selectedLocal, properties)); + Assertions.assertThrows(IOException.class, () -> TvfGuard.check(URI.create("viewfs://literal"), + javaRequest(properties, HadoopTvfAdmission.Access.HADOOP_READ, selectedLocal))); + properties.remove("fs.viewfs.mounttable.literal.link./data/*"); + } + } + + @Test + void escapedParentNormalizationRetainsEarlierWildcardProbes() throws Exception { + Map properties = hdfs("viewfs://parents"); + properties.put("fs.viewfs.mounttable.parents.link./data", "hdfs://nn/base"); + properties.put("fs.viewfs.mounttable.parents.link./other", "hdfs://nn/other"); + properties.put("fs.viewfs.mounttable.parents.link./other/private", "file:///unused-local"); + properties.put("fs.viewfs.mounttable.parents.linkFallback", "hdfs://nn/fallback"); + String uri = "viewfs://parents/data/*/\\.\\./\\.\\./other/\\*/p?obe.csv"; + Assertions.assertDoesNotThrow(() -> HadoopTvfAdmission.checkReadDiscovery(uri, properties)); + Assertions.assertDoesNotThrow(() -> TvfGuard.check(URI.create("viewfs://parents"), + javaRequest(properties, HadoopTvfAdmission.Access.HADOOP_READ, uri))); + properties.put("fs.viewfs.mounttable.parents.link./data/private", "file:///unused-local"); + Assertions.assertThrows(org.apache.doris.nereids.exceptions.AnalysisException.class, + () -> HadoopTvfAdmission.checkReadDiscovery(uri, properties)); + Assertions.assertThrows(IOException.class, () -> TvfGuard.check(URI.create("viewfs://parents"), + javaRequest(properties, HadoopTvfAdmission.Access.HADOOP_READ, uri))); + } + + private Configuration javaRequest(Map properties, HadoopTvfAdmission.Access access, + String path) throws Exception { + Configuration configuration = defaults(Collections.emptyMap()); + overlay(configuration, HadoopTvfAdmission.protectJava(access, path, properties)); + // Exercise the default-cached remote-provider case that used to retain every path policy. + configuration.setBoolean("fs.hdfs.impl.disable.cache", false); + return configuration; + } + + private static int filesystemCacheSize() throws Exception { + Field cacheField = FileSystem.class.getDeclaredField("CACHE"); + cacheField.setAccessible(true); + Object cache = cacheField.get(null); + Field mapField = cache.getClass().getDeclaredField("map"); + mapField.setAccessible(true); + synchronized (cache) { + return ((Map) mapField.get(cache)).size(); + } + } + + /** Real ViewFS dispatch/cache, with all storage operations replaced by recognized remote clients. */ + private static final class RemoteClients implements AutoCloseable { + private final List clients = new ArrayList<>(); + private final List events = new ArrayList<>(); + private final MockedStatic factory; + private final Method baseClose; + private String failInitializeAuthority; + private boolean failNextClose; + + private RemoteClients() throws Exception { + // The mock must retain FileSystem.close's real cache removal without invoking DFS RPC cleanup. + baseClose = FileSystem.class.getDeclaredMethod("closeBase"); + baseClose.setAccessible(true); + factory = Mockito.mockStatic(ReflectionUtils.class, Mockito.CALLS_REAL_METHODS); + factory.when(() -> ReflectionUtils.newInstance(Mockito.eq(DistributedFileSystem.class), + Mockito.any(Configuration.class))).thenAnswer(call -> remote(call.getArgument(1))); + factory.when(() -> ReflectionUtils.newInstance(Mockito.eq(LocalFileSystem.class), + Mockito.any(Configuration.class))).thenThrow(new AssertionError("Unexpected local storage access")); + } + + private DistributedFileSystem remote(Configuration configuration) throws IOException { + int index = clients.size(); + DistributedFileSystem remote = Mockito.mock(DistributedFileSystem.class); + clients.add(remote); + AtomicReference uri = new AtomicReference<>(); + AtomicBoolean closed = new AtomicBoolean(); + Set directories = new HashSet<>(); + Mockito.when(remote.getConf()).thenReturn(configuration); + Mockito.when(remote.getUri()).thenAnswer(call -> uri.get()); + Mockito.when(remote.getWorkingDirectory()).thenReturn(new Path("/")); + Mockito.when(remote.getHomeDirectory()).thenReturn(new Path("/")); + Mockito.doAnswer(call -> { + URI target = call.getArgument(0); + uri.set(URI.create(target.getScheme() + "://" + target.getRawAuthority())); + if (target.getAuthority().equals(failInitializeAuthority)) { + throw new IOException("Injected remote initialization failure"); + } + return null; + }).when(remote).initialize(Mockito.any(URI.class), Mockito.any(Configuration.class)); + Mockito.when(remote.getFileStatus(Mockito.any(Path.class))).thenAnswer(call -> { + if (closed.get()) { + throw new IOException("Remote client is closed"); + } + Path path = call.getArgument(0); + events.add("stat:" + index + ":" + path.toUri().getPath()); + return new org.apache.hadoop.fs.FileStatus(1, directories.contains(path), 1, 1, 0, path); + }); + Mockito.when(remote.listStatus(Mockito.any(Path.class))).thenAnswer(call -> { + Path path = call.getArgument(0); + events.add("list:" + index + ":" + path.toUri().getPath()); + directories.add(path); + return new org.apache.hadoop.fs.FileStatus[] {new org.apache.hadoop.fs.FileStatus( + 1, false, 1, 1, 0, new Path(path, "probe.csv"))}; + }); + Mockito.when(remote.delete(Mockito.any(Path.class), Mockito.anyBoolean())).thenAnswer(call -> { + events.add("delete:" + index + ":" + ((Path) call.getArgument(0)).toUri().getPath()); + return true; + }); + Mockito.doAnswer(call -> { + events.add("close:" + index); + closed.set(true); + baseClose.invoke(remote); + if (failNextClose) { + failNextClose = false; + throw new IOException("Injected remote close failure"); + } + return null; + }).when(remote).close(); + return remote; + } + + private void verifyClosedOnce() throws IOException { + for (DistributedFileSystem client : clients) { + Mockito.verify(client, Mockito.times(1)).close(); + } + } + + @Override + public void close() { + factory.close(); + } + } + + private Configuration defaults(Map entries) throws Exception { + StringBuilder xml = new StringBuilder(""); + entries.forEach((key, value) -> xml.append("").append(key) + .append("").append(value).append("")); + Files.write(temporary.resolve("core-site.xml"), + xml.append("").toString().getBytes(StandardCharsets.UTF_8)); + URLClassLoader loader = new URLClassLoader(new URL[] {temporary.toUri().toURL()}, + FileSystem.class.getClassLoader()) { + @Override + public URL getResource(String name) { + return "core-site.xml".equals(name) ? findResource(name) : super.getResource(name); + } + }; + ownedLoaders.add(loader); + Configuration configuration = new Configuration(); + configuration.setClassLoader(loader); + entries.forEach((key, value) -> Assertions.assertEquals( + value.contains("${") ? LocalFileSystem.class.getName() : value, configuration.get(key))); + return configuration; + } + + private static void overlay(Configuration configuration, Map properties) { + properties.forEach(configuration::set); + } + + private static Map hdfs(String nameNode) { + Map properties = new HashMap<>(); + properties.put("fs.defaultFS", nameNode); + return properties; + } + + private static Map readProperties(String uri) { + Map properties = hdfs("hdfs://runtime"); + properties.put("uri", uri); + properties.put("format", "csv"); + properties.put("csv_schema", "id:int"); + return properties; + } +} diff --git a/fe/fe-core/src/test/java/org/apache/doris/tablefunction/LocalTableValuedFunctionAuthTest.java b/fe/fe-core/src/test/java/org/apache/doris/tablefunction/LocalTableValuedFunctionAuthTest.java index c82cf2eebd93f0..5821ab1a3776d2 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/tablefunction/LocalTableValuedFunctionAuthTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/tablefunction/LocalTableValuedFunctionAuthTest.java @@ -18,10 +18,12 @@ package org.apache.doris.tablefunction; import org.apache.doris.catalog.Env; +import org.apache.doris.common.Config; import org.apache.doris.mysql.privilege.AccessControllerManager; import org.apache.doris.mysql.privilege.PrivPredicate; import org.apache.doris.nereids.exceptions.AnalysisException; import org.apache.doris.proto.InternalService.PGlobResponse; +import org.apache.doris.proto.InternalService.PGlobResponse.PFileInfo; import org.apache.doris.proto.Types.PStatus; import org.apache.doris.qe.ConnectContext; import org.apache.doris.qe.SessionVariable; @@ -50,12 +52,15 @@ public class LocalTableValuedFunctionAuthTest { private final AccessControllerManager accessManager = Mockito.mock(AccessControllerManager.class); private final SystemInfoService systemInfo = Mockito.mock(SystemInfoService.class); private final BackendServiceProxy proxy = Mockito.mock(BackendServiceProxy.class); + private boolean originalEnableLocalTvf; private MockedStatic mockedEnv; private MockedStatic mockedContext; private MockedStatic mockedProxy; @BeforeEach public void setUp() throws Exception { + originalEnableLocalTvf = Config.enable_local_tvf; + Config.enable_local_tvf = true; mockedEnv = Mockito.mockStatic(Env.class); mockedEnv.when(Env::getCurrentEnv).thenReturn(env); mockedEnv.when(Env::getCurrentSystemInfo).thenReturn(systemInfo); @@ -76,6 +81,7 @@ public void setUp() throws Exception { @AfterEach public void tearDown() { + Config.enable_local_tvf = originalEnableLocalTvf; mockedProxy.close(); mockedContext.close(); mockedEnv.close(); @@ -118,6 +124,25 @@ public void testCheckAuthRejectsRevokedPrivileges() throws Exception { Assertions.assertThrows(AnalysisException.class, () -> file.checkAuth(context)); } + @Test + public void testCanonicalGlobPathsKeepTheirExactWhitespace() throws Exception { + Mockito.when(accessManager.checkGlobalPriv(context, PrivPredicate.ADMIN)).thenReturn(true); + String[] paths = {"/secure/name.csv ", "/secure/ name.csv", "/secure/\tname.csv\t"}; + PGlobResponse.Builder response = PGlobResponse.newBuilder() + .setStatus(PStatus.newBuilder().setStatusCode(0)); + for (String path : paths) { + response.addFiles(PFileInfo.newBuilder().setFile(path).setSize(4)); + } + Mockito.when(proxy.glob(Mockito.any(), Mockito.any())) + .thenReturn(CompletableFuture.completedFuture(response.build())); + LocalTableValuedFunction local = new LocalTableValuedFunction(PROPERTIES); + FileTableValuedFunction file = new FileTableValuedFunction(PROPERTIES); + for (int i = 0; i < paths.length; i++) { + Assertions.assertEquals(paths[i], local.getFileStatuses().get(i).getPath()); + Assertions.assertEquals(paths[i], file.getFileStatuses().get(i).getPath()); + } + } + private void assertPrivilegedAccess(PrivPredicate privilege) throws Exception { Mockito.when(accessManager.checkGlobalPriv(context, privilege)).thenReturn(true); LocalTableValuedFunction local = new LocalTableValuedFunction(PROPERTIES); diff --git a/fe/fe-core/src/test/java/org/apache/doris/tablefunction/LocalTableValuedFunctionEnableTest.java b/fe/fe-core/src/test/java/org/apache/doris/tablefunction/LocalTableValuedFunctionEnableTest.java new file mode 100644 index 00000000000000..5d07ed2ac4c635 --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/tablefunction/LocalTableValuedFunctionEnableTest.java @@ -0,0 +1,151 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.tablefunction; + +import org.apache.doris.catalog.Env; +import org.apache.doris.common.Config; +import org.apache.doris.mysql.privilege.AccessControllerManager; +import org.apache.doris.mysql.privilege.PrivPredicate; +import org.apache.doris.nereids.exceptions.AnalysisException; +import org.apache.doris.proto.InternalService.PGlobResponse; +import org.apache.doris.proto.Types.PStatus; +import org.apache.doris.qe.ConnectContext; +import org.apache.doris.qe.SessionVariable; +import org.apache.doris.rpc.BackendServiceProxy; +import org.apache.doris.system.Backend; +import org.apache.doris.system.SystemInfoService; +import org.apache.doris.thrift.TNetworkAddress; + +import com.google.common.collect.ImmutableMap; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.mockito.MockedConstruction; +import org.mockito.MockedStatic; +import org.mockito.Mockito; + +import java.util.Map; +import java.util.concurrent.CompletableFuture; + +public class LocalTableValuedFunctionEnableTest { + private static final Map PROPERTIES = ImmutableMap.of( + "file_path", "local_privilege_test.csv", "backend_id", "1", "format", "csv"); + + private final Env env = Mockito.mock(Env.class); + private final ConnectContext context = Mockito.mock(ConnectContext.class); + private final AccessControllerManager accessManager = Mockito.mock(AccessControllerManager.class); + private final SystemInfoService systemInfo = Mockito.mock(SystemInfoService.class); + private final BackendServiceProxy proxy = Mockito.mock(BackendServiceProxy.class); + private boolean originalEnableLocalTvf; + private MockedStatic mockedEnv; + private MockedStatic mockedContext; + private MockedStatic mockedProxy; + + @BeforeEach + public void setUp() throws Exception { + originalEnableLocalTvf = Config.enable_local_tvf; + Config.enable_local_tvf = false; + mockedEnv = Mockito.mockStatic(Env.class); + mockedEnv.when(Env::getCurrentEnv).thenReturn(env); + mockedEnv.when(Env::getCurrentSystemInfo).thenReturn(systemInfo); + mockedContext = Mockito.mockStatic(ConnectContext.class); + mockedContext.when(ConnectContext::get).thenReturn(context); + mockedProxy = Mockito.mockStatic(BackendServiceProxy.class); + mockedProxy.when(BackendServiceProxy::getInstance).thenReturn(proxy); + Mockito.when(env.getAccessManager()).thenReturn(accessManager); + Mockito.when(context.getSessionVariable()).thenReturn(new SessionVariable()); + Mockito.when(context.getQueryTimeoutS()).thenReturn(60); + + Backend backend = Mockito.mock(Backend.class); + Mockito.when(systemInfo.getBackend(1L)).thenReturn(backend); + Mockito.when(backend.getBrpcAddress()).thenReturn(new TNetworkAddress("127.0.0.1", 8060)); + Mockito.when(proxy.glob(Mockito.any(), Mockito.any())).thenReturn(CompletableFuture.completedFuture( + PGlobResponse.newBuilder().setStatus(PStatus.newBuilder().setStatusCode(0)).build())); + } + + @AfterEach + public void tearDown() { + Config.enable_local_tvf = originalEnableLocalTvf; + mockedProxy.close(); + mockedContext.close(); + mockedEnv.close(); + } + + @Test + public void testDisabledRejectsLocalBeforePropertyParsing() { + AnalysisException exception = Assertions.assertThrows(AnalysisException.class, + () -> new LocalTableValuedFunction(ImmutableMap.of())); + Assertions.assertTrue(exception.getMessage().contains("enable_local_tvf=true")); + Mockito.verifyNoInteractions(accessManager, systemInfo, proxy); + } + + @Test + public void testDisabledRejectsCsvAndLanceBeforeFileAccess() { + for (String format : new String[] {"csv", "lance"}) { + Map properties = ImmutableMap.of( + "file_path", "local_enable_test.csv", "backend_id", "1", "format", format); + AnalysisException exception = Assertions.assertThrows(AnalysisException.class, + () -> new LocalTableValuedFunction(properties)); + Assertions.assertTrue(exception.getMessage().contains("LOCAL TVF is disabled")); + } + Mockito.verifyNoInteractions(accessManager, systemInfo, proxy); + } + + @Test + public void testEnabledLocalReachesExistingFileAccess() throws Exception { + Config.enable_local_tvf = true; + Mockito.when(accessManager.checkGlobalPriv(context, PrivPredicate.ADMIN)).thenReturn(true); + new LocalTableValuedFunction(PROPERTIES); + Mockito.verify(proxy).glob(Mockito.any(), Mockito.any()); + } + + @Test + public void testDisabledRejectsFileLocalBeforeFileAccess() { + AnalysisException exception = Assertions.assertThrows(AnalysisException.class, + () -> new FileTableValuedFunction(PROPERTIES)); + Assertions.assertTrue(exception.getMessage().contains("LOCAL TVF is disabled")); + Mockito.verifyNoInteractions(accessManager, systemInfo, proxy); + } + + @Test + public void testEnabledFileLocalReachesExistingFileAccess() throws Exception { + Config.enable_local_tvf = true; + Mockito.when(accessManager.checkGlobalPriv(context, PrivPredicate.ADMIN)).thenReturn(true); + new FileTableValuedFunction(PROPERTIES); + Mockito.verify(proxy).glob(Mockito.any(), Mockito.any()); + } + + @Test + public void testDisabledLocalDoesNotBlockFileS3() { + Map properties = ImmutableMap.builder() + .put("uri", "s3://test-bucket/local_enable_test.csv") + .put("s3.endpoint", "s3.us-east-1.amazonaws.com") + .put("s3.region", "us-east-1") + .put("s3.access_key", "test-access-key") + .put("s3.secret_key", "test-secret-key") + .put("format", "csv") + .build(); + try (MockedConstruction mockedS3 = + Mockito.mockConstruction(S3TableValuedFunction.class)) { + Assertions.assertDoesNotThrow(() -> new FileTableValuedFunction(properties)); + Assertions.assertEquals(1, mockedS3.constructed().size()); + } + Mockito.verifyNoInteractions(accessManager, systemInfo, proxy); + } +} diff --git a/regression-test/pipeline/cloud_p0/conf/fe_custom.conf b/regression-test/pipeline/cloud_p0/conf/fe_custom.conf index 68012788a9eb87..4f34a255a0ca6b 100644 --- a/regression-test/pipeline/cloud_p0/conf/fe_custom.conf +++ b/regression-test/pipeline/cloud_p0/conf/fe_custom.conf @@ -1,3 +1,6 @@ +# Exercise LOCAL TVF explicitly; production defaults to disabled. +enable_local_tvf=true + stream_load_default_timeout_second = 72000 replication_num_forced_in_cloud_mode = true ignore_unsupported_properties_in_cloud_mode = true diff --git a/regression-test/pipeline/cloud_p1/conf/fe_custom.conf b/regression-test/pipeline/cloud_p1/conf/fe_custom.conf index 157f3a07a09b47..e671d0a7dd15b2 100644 --- a/regression-test/pipeline/cloud_p1/conf/fe_custom.conf +++ b/regression-test/pipeline/cloud_p1/conf/fe_custom.conf @@ -1,3 +1,6 @@ +# Exercise LOCAL TVF explicitly; production defaults to disabled. +enable_local_tvf=true + stream_load_default_timeout_second = 72000 replication_num_forced_in_cloud_mode = true ignore_unsupported_properties_in_cloud_mode = true diff --git a/regression-test/pipeline/external/conf/fe.conf b/regression-test/pipeline/external/conf/fe.conf index 34fa5a440a527e..88118ec677cdf6 100644 --- a/regression-test/pipeline/external/conf/fe.conf +++ b/regression-test/pipeline/external/conf/fe.conf @@ -1,3 +1,6 @@ +# Exercise LOCAL TVF explicitly; production defaults to disabled. +enable_local_tvf=true + # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information diff --git a/regression-test/pipeline/nonConcurrent/conf/fe.conf b/regression-test/pipeline/nonConcurrent/conf/fe.conf index d283b2a15cfa6a..de57276c9bcd79 100644 --- a/regression-test/pipeline/nonConcurrent/conf/fe.conf +++ b/regression-test/pipeline/nonConcurrent/conf/fe.conf @@ -1,3 +1,6 @@ +# Exercise LOCAL TVF explicitly; production defaults to disabled. +enable_local_tvf=true + # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information diff --git a/regression-test/pipeline/p0/conf/fe.conf b/regression-test/pipeline/p0/conf/fe.conf index 61c6a5a49621bb..4b73152ba7bf7c 100644 --- a/regression-test/pipeline/p0/conf/fe.conf +++ b/regression-test/pipeline/p0/conf/fe.conf @@ -1,3 +1,6 @@ +# Exercise LOCAL TVF explicitly; production defaults to disabled. +enable_local_tvf=true + # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information diff --git a/regression-test/pipeline/p1/conf/fe.conf b/regression-test/pipeline/p1/conf/fe.conf index be302d2908f7e6..f2932470f237e1 100644 --- a/regression-test/pipeline/p1/conf/fe.conf +++ b/regression-test/pipeline/p1/conf/fe.conf @@ -1,3 +1,6 @@ +# Exercise LOCAL TVF explicitly; production defaults to disabled. +enable_local_tvf=true + # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information diff --git a/regression-test/pipeline/vault_p0/conf/fe_custom.conf b/regression-test/pipeline/vault_p0/conf/fe_custom.conf index 9b622795168540..88cff4dd920ec5 100644 --- a/regression-test/pipeline/vault_p0/conf/fe_custom.conf +++ b/regression-test/pipeline/vault_p0/conf/fe_custom.conf @@ -1,3 +1,6 @@ +# Exercise LOCAL TVF explicitly; production defaults to disabled. +enable_local_tvf=true + stream_load_default_timeout_second = 72000 replication_num_forced_in_cloud_mode = true ignore_unsupported_properties_in_cloud_mode = true diff --git a/regression-test/suites/account_p0/test_local_tvf_enable.groovy b/regression-test/suites/account_p0/test_local_tvf_enable.groovy new file mode 100644 index 00000000000000..3cc8bf71ce3d92 --- /dev/null +++ b/regression-test/suites/account_p0/test_local_tvf_enable.groovy @@ -0,0 +1,46 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_local_tvf_enable", "p0") { + def enabled = (sql "ADMIN SHOW FRONTEND CONFIG LIKE 'enable_local_tvf'")[0][1].toBoolean() + for (def functionName : ["LOCAL", "FILE"]) { + for (def prefix : ["", "EXPLAIN "]) { + test { + sql """${prefix}SELECT * FROM ${functionName}( + "file_path" = "local_enable_test.csv", + "backend_id" = "-1", + "format" = "csv" + )""" + // With opt-in enabled, the existing backend validation must be reached. + exception (enabled ? "'backend_id' is required" : "LOCAL TVF is disabled") + } + } + } + if (!enabled) { + for (def prefix : ["", "EXPLAIN "]) { + test { + sql """${prefix}INSERT INTO LOCAL( + "file_path" = "/tmp/local_enable_test_", + "backend_id" = "-1", + "format" = "csv" + ) SELECT 1""" + exception "LOCAL TVF is disabled" + } + } + } + +} diff --git a/regression-test/suites/account_p0/test_local_tvf_privilege.groovy b/regression-test/suites/account_p0/test_local_tvf_privilege.groovy index 4dfd4da49c85e4..2f262b3aa17a14 100644 --- a/regression-test/suites/account_p0/test_local_tvf_privilege.groovy +++ b/regression-test/suites/account_p0/test_local_tvf_privilege.groovy @@ -16,6 +16,8 @@ // under the License. suite("test_local_tvf_privilege", "p0") { + def enabled = (sql "ADMIN SHOW FRONTEND CONFIG LIKE 'enable_local_tvf'")[0][1].toBoolean() + def expectedError = enabled ? "ADMIN/OPERATOR" : "LOCAL TVF is disabled" def user = "local_tvf_privilege_user" sql "DROP USER IF EXISTS '${user}'" sql "CREATE USER '${user}' IDENTIFIED BY 'LocalTvf123!'" @@ -36,7 +38,7 @@ suite("test_local_tvf_privilege", "p0") { "backend_id" = "0", "format" = "csv" )""" - exception "ADMIN/OPERATOR" + exception expectedError } } }