diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml index 9bbc284266..a31949b20a 100644 --- a/.github/workflows/e2e.yaml +++ b/.github/workflows/e2e.yaml @@ -154,144 +154,8 @@ jobs: run: | ./scripts/e2e.sh e2e - # E2E测试 - Docker Engine版本兼容性测试 - E2EWithDockerEngineVersions: - runs-on: ubuntu-22.04 - needs: - - GenerateE2ETestImage - strategy: - matrix: - docker_engine_version: ["29.1", "28.5", "27.5", "26.1", "25.0", "24.0", "23.0", "20.10"] - test_case: ["input_container_stdio", "input_docker_static_file"] - fail-fast: false - timeout-minutes: 60 - steps: - - name: Set up Go - uses: actions/setup-go@v4 - with: - go-version: 1.23.12 - - - name: Check out code - uses: actions/checkout@v4 - with: - submodules: false - - - name: Install Docker Engine ${{ matrix.docker_engine_version }} - run: | - # 移除现有的docker(包括docker-ce) - sudo apt-get remove -y docker docker-engine docker.io docker-ce docker-ce-cli containerd runc 2>/dev/null || true - sudo apt-get purge -y docker docker-engine docker.io docker-ce docker-ce-cli containerd runc 2>/dev/null || true - - # 安装依赖 - sudo apt-get update - sudo apt-get install -y \ - ca-certificates \ - curl \ - gnupg \ - lsb-release \ - apt-transport-https - - # 添加Docker官方GPG密钥 - sudo mkdir -p /etc/apt/keyrings - curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo gpg --dearmor -o /etc/apt/keyrings/docker.gpg - - # 设置Docker仓库 - echo \ - "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.gpg] https://download.docker.com/linux/ubuntu \ - $(lsb_release -cs) stable" | sudo tee /etc/apt/sources.list.d/docker.list > /dev/null - - # 更新包列表 - sudo apt-get update - - # 根据版本号确定要安装的版本 - VERSION="${{ matrix.docker_engine_version }}" - - # 查找可用的docker-ce版本 - AVAILABLE_VERSIONS=$(apt-cache madison docker-ce | awk '{print $3}' | sort -V) - echo "Available Docker CE versions:" - echo "$AVAILABLE_VERSIONS" - - # 根据主版本号匹配 - MAJOR=$(echo $VERSION | cut -d. -f1) - MINOR=$(echo $VERSION | cut -d. -f2) - - # 查找匹配的版本,选择最高的小版本 - MATCHED_VERSION=$(echo "$AVAILABLE_VERSIONS" | grep "^5:$MAJOR\.$MINOR\." | tail -1) - - if [ -z "$MATCHED_VERSION" ]; then - # 如果找不到精确匹配,尝试匹配主版本号,选择最高版本 - MATCHED_VERSION=$(echo "$AVAILABLE_VERSIONS" | grep "^5:$MAJOR\." | tail -1) - fi - - if [ -n "$MATCHED_VERSION" ]; then - echo "Installing Docker CE version: $MATCHED_VERSION" - sudo apt-get install -y --allow-downgrades docker-ce=$MATCHED_VERSION docker-ce-cli=$MATCHED_VERSION containerd.io - else - echo "Warning: Exact version $VERSION not found, trying to install latest version in major version $MAJOR" - LATEST_MAJOR=$(echo "$AVAILABLE_VERSIONS" | grep "^5:$MAJOR\." | tail -1) - if [ -n "$LATEST_MAJOR" ]; then - echo "Installing latest available version in major version $MAJOR: $LATEST_MAJOR" - sudo apt-get install -y --allow-downgrades docker-ce=$LATEST_MAJOR docker-ce-cli=$LATEST_MAJOR containerd.io - else - echo "::warning::Docker $VERSION is not available on this Ubuntu version" - echo "Available versions:" - echo "$AVAILABLE_VERSIONS" - echo "Skipping Docker $VERSION test" - exit 0 - fi - fi - - # 启动Docker服务 - sudo systemctl start docker || sudo service docker start - sudo systemctl enable docker || true - - # 等待Docker服务就绪 - sleep 5 - - # 输出Docker Engine版本信息 - echo "=========================================" - echo "Docker Engine Version Information:" - echo "=========================================" - docker --version - echo "" - echo "Docker Engine Server Version:" - docker version --format '{{.Server.Version}}' - echo "=========================================" - - - name: Update Docker-compose to v2 - run: | - sudo curl -SL https://github.com/docker/compose/releases/download/v2.7.0/docker-compose-linux-x86_64 -o /usr/local/bin/docker-compose - sudo chmod +x /usr/local/bin/docker-compose - - - name: Download Artifact - uses: actions/download-artifact@v4 - with: - name: loongcollector-image - - - name: Check if Docker is installed - id: check_docker - run: | - if command -v docker &> /dev/null; then - echo "docker_installed=true" >> $GITHUB_OUTPUT - echo "Docker is installed" - else - echo "docker_installed=false" >> $GITHUB_OUTPUT - echo "Docker is not installed, skipping test" - fi - - - name: Import Image from Tar - if: steps.check_docker.outputs.docker_installed == 'true' - run: docker load -i ./image.tar - - - name: E2E Test with Docker Engine ${{ matrix.docker_engine_version }} - ${{ matrix.test_case }} - if: steps.check_docker.outputs.docker_installed == 'true' - env: - TEST_CASE: ${{ matrix.test_case }} - run: | - ./scripts/e2e.sh e2e - actions-timeline: - needs: [E2E, E2EWithDockerEngineVersions] + needs: [E2E] runs-on: ubuntu-latest permissions: actions: read diff --git a/.github/workflows/release-edge.yaml b/.github/workflows/release-edge.yaml index 4a3c434f89..3b4909118d 100644 --- a/.github/workflows/release-edge.yaml +++ b/.github/workflows/release-edge.yaml @@ -28,6 +28,7 @@ on: - "test/**" - "tools/**" - "CHANGELOG.md" + workflow_dispatch: concurrency: group: ${{ github.workflow }}-${{ github.ref }} diff --git a/core/app_config/AppConfig.cpp b/core/app_config/AppConfig.cpp index cade277f06..03fad0ddc3 100644 --- a/core/app_config/AppConfig.cpp +++ b/core/app_config/AppConfig.cpp @@ -20,7 +20,6 @@ #include #include -#include "boost/filesystem.hpp" #include "json/value.h" #include "RuntimeUtil.h" @@ -29,7 +28,6 @@ #include "common/FileSystemUtil.h" #include "common/JsonUtil.h" #include "common/LogtailCommonFlags.h" -#include "common/version.h" #include "config/InstanceConfigManager.h" #include "config/watcher/InstanceConfigWatcher.h" #include "constants/Constants.h" @@ -37,7 +35,6 @@ #include "file_server/reader/LogFileReader.h" #include "logger/Logger.h" #include "monitor/AlarmManager.h" -#include "monitor/Monitor.h" #ifdef __ENTERPRISE__ #include "config/provider/EnterpriseConfigProvider.h" #endif @@ -51,7 +48,6 @@ DEFINE_FLAG_BOOL(logtail_mode, "logtail mode", true); #else DEFINE_FLAG_BOOL(logtail_mode, "logtail mode", false); #endif -DEFINE_FLAG_INT32(max_buffer_num, "max size", 40); DEFINE_FLAG_INT32(default_send_byte_per_sec, "the max send speed per sec, replay buffer thread", 2 * 1024 * 1024); DEFINE_FLAG_INT32(default_buffer_file_num, "how many buffer files in default", 50); DEFINE_FLAG_INT32(default_local_file_size, "default size of one buffer file", 20 * 1024 * 1024); @@ -121,16 +117,9 @@ DECLARE_FLAG_INT32(ignore_file_modify_timeout); DEFINE_FLAG_STRING(host_path_blacklist, "host path matches substring in blacklist will be ignored", ""); DEFINE_FLAG_STRING(ALIYUN_LOG_FILE_TAGS, "default env file key to load tags", ""); DEFINE_FLAG_STRING(LOONG_FILE_TAGS, "default env file key to load tags", ""); -DEFINE_FLAG_INT32(max_holded_data_size, - "for every id and metric name, the max data size can be holded in memory (default 512KB)", - 512 * 1024); -DEFINE_FLAG_STRING(metrics_report_method, - "method to report metrics (default none, means logtail will not report metrics)", - "sls"); DEFINE_FLAG_STRING(operator_service, "loong collector operator service", ""); DEFINE_FLAG_INT32(operator_service_port, "loong collector operator service port", 8888); -DEFINE_FLAG_INT32(k8s_meta_service_port, "loong collector operator service port", 9000); DEFINE_FLAG_STRING(k8s_metadata_server_name, "loong collector singleton service for k8s metadata server", @@ -142,8 +131,6 @@ DEFINE_FLAG_STRING(app_info_file, "", "app_info.json"); DEFINE_FLAG_STRING(crash_stack_file_name, "crash stack back trace file name", "backtrace.dat"); DEFINE_FLAG_STRING(local_event_data_file_name, "local event data file name", "local_event.json"); DEFINE_FLAG_STRING(inotify_watcher_dirs_dump_filename, "", "inotify_watcher_dirs"); -DEFINE_FLAG_STRING(logtail_snapshot_dir, "snapshot dir on local disk", "snapshot"); -DEFINE_FLAG_STRING(logtail_profile_snapshot, "reader profile on local disk", "logtail_profile_snapshot"); DEFINE_FLAG_STRING(ilogtail_config_env_name, "config file path", "ALIYUN_LOGTAIL_CONFIG"); #if defined(__linux__) @@ -152,10 +139,6 @@ DEFINE_FLAG_STRING(check_point_filename, "", "/tmp/logtail_check_point"); DEFINE_FLAG_STRING(check_point_filename, "", "C:\\LogtailData\\logtail_check_point"); #endif -DEFINE_FLAG_STRING(sls_observer_ebpf_host_path, - "the backup real host path for store libebpf.so", - "/etc/ilogtail/ebpf/"); - namespace logtail { const int32_t kDefaultMaxSendBytePerSec = 25 * 1024 * 1024; // the max send speed per sec, realtime thread @@ -166,9 +149,6 @@ const double GLOBAL_CONCURRENCY_FREE_PERCENTAGE_FOR_ONE_REGION = 0.5; const int32_t MIN_SEND_REQUEST_CONCURRENCY = 15; // 单地域并发度最大值 const int32_t MAX_SEND_REQUEST_CONCURRENCY = 80; -// 并发度统计数量&&时间间隔 -const uint32_t CONCURRENCY_STATISTIC_THRESHOLD = 10; -const uint32_t CONCURRENCY_STATISTIC_INTERVAL_THRESHOLD_SECONDS = 3; // 并发度不回退百分比阈值 const uint32_t NO_FALL_BACK_FAIL_PERCENTAGE = 10; // 并发度慢回退百分比阈值 @@ -496,14 +476,6 @@ string GetAgentLogName() { #endif } -string GetObserverEbpfHostPath() { - if (BOOL_FLAG(logtail_mode)) { - return STRING_FLAG(sls_observer_ebpf_host_path); - } else { - return GetAgentDataDir(); - } -} - string GetSendBufferFileNamePrefix() { if (BOOL_FLAG(logtail_mode)) { return "logtail_buffer_file_"; @@ -522,10 +494,10 @@ string GetLegacyUserLocalConfigFilePath() { string GetExactlyOnceCheckpoint() { if (BOOL_FLAG(logtail_mode)) { - auto fp = boost::filesystem::path(AppConfig::GetInstance()->GetLoongcollectorConfDir()); + auto fp = filesystem::path(AppConfig::GetInstance()->GetLoongcollectorConfDir()); return (fp / "checkpoint_v2").string(); } else { - auto fp = boost::filesystem::path(GetAgentDataDir()); + auto fp = filesystem::path(GetAgentDataDir()); return (fp / "exactly_once_checkpoint").string(); } } @@ -606,22 +578,16 @@ std::string GetPluginBaseName() { AppConfig::AppConfig() { LOG_INFO(sLogger, ("AppConfig AppConfig", "success")); SetIlogtailConfigJson(""); - // mStreamLogAddress = "0.0.0.0"; - // mIsOldPubRegion = false; - // mOpenStreamLog = false; mSendRequestConcurrency = INT32_FLAG(send_request_concurrency); mProcessThreadCount = INT32_FLAG(process_thread_count); - // mMappingConfigPath = STRING_FLAG(default_mapping_config_path); mMachineCpuUsageThreshold = DOUBLE_FLAG(default_machine_cpu_usage_threshold); mCpuUsageUpLimit = DOUBLE_FLAG(cpu_usage_up_limit); mScaledCpuUsageUpLimit = DOUBLE_FLAG(cpu_usage_up_limit); mMemUsageUpLimit = INT64_FLAG(memory_usage_up_limit); mResourceAutoScale = BOOL_FLAG(default_resource_auto_scale); mInputFlowControl = BOOL_FLAG(default_input_flow_control); - // mDefaultRegion = STRING_FLAG(default_region_name); mAcceptMultiConfigFlag = BOOL_FLAG(default_accept_multi_config); mMaxMultiConfigSize = INT32_FLAG(max_multi_config_size); - // mUserConfigPath = STRING_FLAG(user_log_config); mIgnoreDirInodeChanged = false; mLogParseAlarmFlag = true; mNoInotify = false; @@ -915,16 +881,6 @@ void AppConfig::LoadResourceConf(const Json::Value& confJson) { else mLocalFileSize = INT32_FLAG(default_local_file_size); - if (confJson.isMember("buffer_map_size") && confJson["buffer_map_size"].isInt()) - mMaxHoldedDataSize = confJson["buffer_map_size"].asInt(); - else - mMaxHoldedDataSize = INT32_FLAG(max_holded_data_size); - - if (confJson.isMember("buffer_map_num") && confJson["buffer_map_num"].isInt()) - mMaxBufferNum = confJson["buffer_map_num"].asInt(); - else - mMaxBufferNum = INT32_FLAG(max_buffer_num); - if (confJson.isMember("send_request_concurrency") && confJson["send_request_concurrency"].isInt()) mSendRequestConcurrency = confJson["send_request_concurrency"].asInt(); else @@ -1030,56 +986,13 @@ void AppConfig::LoadResourceConf(const Json::Value& confJson) { mIgnoreDirInodeChanged = confJson["ignore_dir_inode_changed"].asBool(); } - // LoadStringParameter(mUserConfigPath, confJson, "user_config_file_path", "ALIYUN_LOGTAIL_USER_CONIFG_PATH"); - - // LoadStringParameter( - // mUserLocalConfigPath, confJson, "user_local_config_filename", "ALIYUN_LOGTAIL_USER_LOCAL_CONFIG_FILENAME"); - LoadBooleanParameter( BOOL_FLAG(ilogtail_discard_old_data), confJson, "discard_old_data", "ALIYUN_LOGTAIL_DISCARD_OLD_DATA"); - // if (confJson.isMember("container_mount_path") && confJson["container_mount_path"].isString()) { - // mContainerMountConfigPath = confJson["container_mount_path"].asString(); - // } else { - // mContainerMountConfigPath = GetProcessExecutionDir() + STRING_FLAG(default_container_mount_path); - // } - LoadStringParameter(mConfigIP, confJson, "working_ip", "ALIYUN_LOGTAIL_WORKING_IP"); - // LoadStringParameter(mCustomizedConfigIP, confJson, "customized_config_ip", - // "ALIYUN_LOGTAIL_CUSTOMIZED_CONFIG_IP"); - LoadStringParameter(mConfigHostName, confJson, "working_hostname", "ALIYUN_LOGTAIL_WORKING_HOSTNAME"); - // // try to get zone env name from conf json - // if (confJson.isMember("alipay_zone_env_name") && confJson["alipay_zone_env_name"].isString()) { - // STRING_FLAG(alipay_zone_env_name) = confJson["alipay_zone_env_name"].asString(); - // } - // // get zone info from env, for ant - // do { - // if (!STRING_FLAG(alipay_zone_env_name).empty()) { - // const char* alipayZone = getenv(STRING_FLAG(alipay_zone_env_name).c_str()); - // if (alipayZone != NULL) { - // mAlipayZone = alipayZone; - // break; - // } - // } - // const char* alipayZone = getenv(STRING_FLAG(alipay_app_zone).c_str()); - // if (alipayZone != NULL) { - // mAlipayZone = alipayZone; - // break; - // } - // alipayZone = getenv(STRING_FLAG(alipay_zone).c_str()); - // if (alipayZone != NULL) { - // mAlipayZone = alipayZone; - // break; - // } - // } while (false); - - // if (confJson.isMember("alipay_zone") && confJson["alipay_zone"].isString()) { - // mAlipayZone = confJson["alipay_zone"].asString(); - // } - if (confJson.isMember("system_boot_time") && confJson["system_boot_time"].isInt()) { mSystemBootTime = confJson["system_boot_time"].asInt(); } @@ -1158,26 +1071,11 @@ void AppConfig::LoadResourceConf(const Json::Value& confJson) { "check_point_dump_interval", "ALIYUN_LOGTAIL_CHECKPOINT_DUMP_INTERVAL"); - // if (confJson.isMember("rapid_retry_update_config") && confJson["rapid_retry_update_config"].isBool()) { - // BOOL_FLAG(rapid_retry_update_config) = confJson["rapid_retry_update_config"].asBool(); - // LOG_INFO(sLogger, ("set rapid_retry_update_config", BOOL_FLAG(rapid_retry_update_config))); - // } - if (confJson.isMember("check_profile_region") && confJson["check_profile_region"].isBool()) { BOOL_FLAG(check_profile_region) = confJson["check_profile_region"].asBool(); LOG_INFO(sLogger, ("set check_profile_region", BOOL_FLAG(check_profile_region))); } - // if (confJson.isMember("enable_collection_mark") && confJson["enable_collection_mark"].isBool()) { - // BOOL_FLAG(enable_collection_mark) = confJson["enable_collection_mark"].asBool(); - // LOG_INFO(sLogger, ("set enable_collection_mark", BOOL_FLAG(enable_collection_mark))); - // } - - // LoadBooleanParameter(BOOL_FLAG(enable_collection_mark), - // confJson, - // "enable_env_ref_in_config", - // "ALIYUN_LOGTAIL_ENABLE_ENV_REF_IN_CONFIG"); - LoadBooleanParameter( mEnableHostIPReplace, confJson, "enable_host_ip_replace", "ALIYUN_LOGTAIL_ENABLE_HOST_IP_REPLACE"); @@ -1662,9 +1560,6 @@ bool AppConfig::IsInInotifyBlackList(const std::string& path) const { return rst; } -// TODO: Use Boost instead. -// boost::filesystem::directory_iterator end; -// try { boost::filesystem::directory_iterator(path); } catch (...) { // failed } // OK void AppConfig::SetLoongcollectorConfDir(const std::string& dirPath) { mLoongcollectorConfDir = dirPath; if (dirPath.back() != '/' && dirPath.back() != '\\') { @@ -1696,22 +1591,6 @@ void AppConfig::SetLoongcollectorConfDir(const std::string& dirPath) { mLoongcollectorConfDir = GetAgentConfDir(); } #endif - - // Update related configurations (local user config). - // if (STRING_FLAG(ilogtail_local_config).empty()) { - // LOG_WARNING(sLogger, ("flag error", "ilogtail_local_config must be non-empty")); - // STRING_FLAG(ilogtail_local_config) = DEFAULT_ILOGTAIL_LOCAL_CONFIG_FLAG_VALUE; - // } - // if (STRING_FLAG(ilogtail_local_config_dir).empty()) { - // LOG_WARNING(sLogger, ("flag error", "ilogtail_local_config_dir must be non-empty")); - // STRING_FLAG(ilogtail_local_config_dir) = DEFAULT_ILOGTAIL_LOCAL_CONFIG_DIR_FLAG_VALUE; - // } - // mUserLocalConfigPath = AbsolutePath(STRING_FLAG(ilogtail_local_config), mLogtailSysConfDir); - // mUserLocalConfigDirPath = AbsolutePath(STRING_FLAG(ilogtail_local_config_dir), mLogtailSysConfDir) + - // PATH_SEPARATOR; mUserLocalYamlConfigDirPath - // = AbsolutePath(STRING_FLAG(ilogtail_local_yaml_config_dir), mLogtailSysConfDir) + PATH_SEPARATOR; - // mUserRemoteYamlConfigDirPath - // = AbsolutePath(STRING_FLAG(ilogtail_remote_yaml_config_dir), mLogtailSysConfDir) + PATH_SEPARATOR; LOG_INFO(sLogger, ("set " + GetAgentName() + " conf dir", mLoongcollectorConfDir)); } diff --git a/core/app_config/AppConfig.h b/core/app_config/AppConfig.h index ea2b48141b..9e7a3cb8d9 100644 --- a/core/app_config/AppConfig.h +++ b/core/app_config/AppConfig.h @@ -35,8 +35,6 @@ extern const int32_t kDefaultMaxSendBytePerSec; extern const double GLOBAL_CONCURRENCY_FREE_PERCENTAGE_FOR_ONE_REGION; extern const int32_t MIN_SEND_REQUEST_CONCURRENCY; extern const int32_t MAX_SEND_REQUEST_CONCURRENCY; -extern const uint32_t CONCURRENCY_STATISTIC_THRESHOLD; -extern const uint32_t CONCURRENCY_STATISTIC_INTERVAL_THRESHOLD_SECONDS; extern const uint32_t NO_FALL_BACK_FAIL_PERCENTAGE; extern const uint32_t SLOW_FALL_BACK_FAIL_PERCENTAGE; extern const std::string LOONGCOLLECTOR_ENV_PREFIX; @@ -62,12 +60,10 @@ std::string GetLocalEventDataFileName(); std::string GetInotifyWatcherDirsDumpFileName(); std::string GetAgentLoggersPrefix(); std::string GetAgentLogName(); -std::string GetObserverEbpfHostPath(); std::string GetSendBufferFileNamePrefix(); std::string GetLegacyUserLocalConfigFilePath(); std::string GetExactlyOnceCheckpoint(); std::string GetContinuousPipelineConfigDir(); -std::string GetPipelineConfigDir(); std::string GetPluginLogName(); std::string GetVersionTag(); std::string GetGoPluginCheckpoint(); @@ -110,7 +106,6 @@ class AppConfig { DoubleBuffer> mFileTags; std::string mFileTagsDir; - DoubleBuffer> mAgentAttrs; Json::Value mFileTagsJson; @@ -119,13 +114,6 @@ class AppConfig { // loongcollector_config.json content for rebuild std::string mIlogtailConfigJson; - // syslog - // std::string mStreamLogAddress; - // uint32_t mStreamLogTcpPort; - // uint32_t mStreamLogPoolSizeInMb; - // uint32_t mStreamLogRcvLenPerCall; - // bool mOpenStreamLog; - // performance float mCpuUsageUpLimit; #if defined(__linux__) || defined(__APPLE__) @@ -140,8 +128,6 @@ class AppConfig { float mScaledCpuUsageUpLimit; // sender - int32_t mMaxHoldedDataSize; - int32_t mMaxBufferNum; int32_t mBytePerSec; int32_t mMaxBytePerSec; int32_t mNumOfBufferFile; @@ -153,32 +139,18 @@ class AppConfig { // checkpoint std::string mCheckPointFilePath; - // local config - // std::string mMappingConfigPath; - - int32_t mMaxMultiConfigSize; bool mAcceptMultiConfigFlag; bool mIgnoreDirInodeChanged; - // std::string mUserConfigPath; - // std::string mUserLocalConfigPath; - // std::string mUserLocalConfigDirPath; - // std::string mUserLocalYamlConfigDirPath; - // std::string mUserRemoteYamlConfigDirPath; bool mLogParseAlarmFlag; std::string mProcessExecutionDir; std::string mWorkingDir; - // std::string mContainerMountConfigPath; std::string mConfigIP; std::string mConfigHostName; - // std::string mAlipayZone; int32_t mSystemBootTime = -1; - // used to get log config instead of mConfigIp if set, eg: "127.0.0.1.fuse", - // std::string mCustomizedConfigIP; - // config file path to save docker file cmd info std::string mDockerFilePathConfig; @@ -225,13 +197,6 @@ class AppConfig { std::string mBindInterface; - // /** - // * @brief Load ConfigServer, DataServer and network interface - // * - // * @param confJson - // */ - // virtual void LoadAddrConfig(const Json::Value& confJson) = 0; - /** * @brief Auto scale buffer, file and network parameters according to mem limit. * @@ -259,7 +224,6 @@ class AppConfig { * @param confJson json value to append to */ void LoadIncludeConfig(Json::Value& confJson); - // void LoadSyslogConf(const Json::Value& confJson); void DumpAllFlagsToMap(std::unordered_map& flagMap); void ReadFlagsFromMap(const std::unordered_map& flagMap); @@ -287,7 +251,6 @@ class AppConfig { */ void LoadResourceConf(const Json::Value& confJson); void LoadOtherConf(const Json::Value& confJson); - // void LoadGlobalFuseConf(const Json::Value& confJson); void SetIlogtailConfigJson(const std::string& configJson) { std::lock_guard lock(mAppConfigLock); mIlogtailConfigJson = configJson; @@ -392,11 +355,6 @@ class AppConfig { std::string GetFileTagsDir() { return mFileTagsDir; } - // Agent属性相关,获取从文件中来的attrs - std::map& GetAgentAttrs() { return mAgentAttrs.getReadBuffer(); } - // 更新从文件中来的attrs - void UpdateAgentAttrs(); - // Legacy:获取各种参数 bool NoInotify() const { return mNoInotify; } @@ -404,20 +362,6 @@ class AppConfig { bool IsLogParseAlarmValid() const { return mLogParseAlarmFlag; } - // std::string GetDefaultRegion() const; - - // void SetDefaultRegion(const std::string& region); - - // uint32_t GetStreamLogTcpPort() const { return mStreamLogTcpPort; } - - // const std::string& GetStreamLogAddress() const { return mStreamLogAddress; } - - // uint32_t GetStreamLogPoolSizeInMb() const { return mStreamLogPoolSizeInMb; } - - // uint32_t GetStreamLogRcvLenPerCall() const { return mStreamLogRcvLenPerCall; } - - // bool GetOpenStreamLog() const { return mOpenStreamLog; } - std::string GetIlogtailConfigJson() { std::lock_guard lock(mAppConfigLock); return mIlogtailConfigJson; @@ -425,12 +369,8 @@ class AppConfig { bool IsAcceptMultiConfig() const { return mAcceptMultiConfigFlag; } - void SetAcceptMultiConfig(bool flag) { mAcceptMultiConfigFlag = flag; } - int32_t GetMaxMultiConfigSize() const { return mMaxMultiConfigSize; } - void SetMaxMultiConfigSize(int32_t maxSize) { mMaxMultiConfigSize = maxSize; } - const std::string& GetCheckPointFilePath() const { return mCheckPointFilePath; } bool IsInputFlowControl() const { return mInputFlowControl; } @@ -439,10 +379,6 @@ class AppConfig { int64_t GetMemUsageUpLimit() const { return mMemUsageUpLimit; } - int32_t GetMaxHoldedDataSize() const { return mMaxHoldedDataSize; } - - uint32_t GetMaxBufferNum() const { return mMaxBufferNum; } - int32_t GetMaxBytePerSec() const { return mMaxBytePerSec; } void SetMaxBytePerSec(int32_t maxBytePerSec) { mMaxBytePerSec = maxBytePerSec; } @@ -465,18 +401,6 @@ class AppConfig { int32_t GetProcessThreadCount() const { return mProcessThreadCount; } - // const std::string& GetMappingConfigPath() const { return mMappingConfigPath; } - - // const std::string& GetUserConfigPath() const { return mUserConfigPath; } - - // const std::string& GetLocalUserConfigPath() const { return mUserLocalConfigPath; } - - // const std::string& GetLocalUserConfigDirPath() const { return mUserLocalConfigDirPath; } - - // const std::string& GetLocalUserYamlConfigDirPath() const { return mUserLocalYamlConfigDirPath; } - - // const std::string& GetRemoteUserYamlConfigDirPath() const { return mUserRemoteYamlConfigDirPath; } - bool IgnoreDirInodeChanged() const { return mIgnoreDirInodeChanged; } void SetProcessExecutionDir(const std::string& dir) { mProcessExecutionDir = dir; } @@ -487,12 +411,8 @@ class AppConfig { const std::string& GetWorkingDir() const { return mWorkingDir; } - // const std::string& GetContainerMountConfigPath() const { return mContainerMountConfigPath; } - const std::string& GetConfigIP() const { return mConfigIP; } - // const std::string& GetCustomizedConfigIp() const { return mCustomizedConfigIP; } - const std::string& GetConfigHostName() const { return mConfigHostName; } int32_t GetSystemBootTime() const { return mSystemBootTime; } @@ -509,8 +429,6 @@ class AppConfig { int32_t GetForceQuitReadTimeout() const { return mForceQuitReadTimeout; } - // const std::string& GetAlipayZone() const { return mAlipayZone; } - // If @dirPath is not accessible, GetProcessExecutionDir will be set. void SetLoongcollectorConfDir(const std::string& dirPath); @@ -520,8 +438,6 @@ class AppConfig { inline bool IsResponseVerificationEnabled() const { return mEnableResponseVerification; } - // EndpointAddressType GetConfigServerAddressNetType() const { return mConfigServerAddressNetType; } - inline bool EnableCheckpointSyncWrite() const { return mEnableCheckpointSyncWrite; } inline bool EnableLogTimeAutoAdjust() const { return mEnableLogTimeAutoAdjust; } @@ -534,10 +450,6 @@ class AppConfig { const std::string& GetBindInterface() const { return mBindInterface; } #ifdef APSARA_UNIT_TEST_MAIN - friend class SenderUnittest; - friend class ConfigUpdatorUnittest; - friend class MultiServerConfigUpdatorUnitest; - friend class UtilUnittest; friend class AppConfigUnittest; friend class PipelineUnittest; friend class InputFileUnittest; diff --git a/core/application/Application.cpp b/core/application/Application.cpp index 458b447995..ba9d9e3f26 100644 --- a/core/application/Application.cpp +++ b/core/application/Application.cpp @@ -380,7 +380,7 @@ void Application::Start() { // GCOVR_EXCL_START EventDispatcher::GetInstance()->DumpCheckPointPeriod(curTime); if (curTime - lastContainerCheckTime >= 3) { - if (ContainerManager::GetInstance()->CheckContainerDiffForAllConfig()) { + if (ContainerManager::GetInstance()->CheckFileServerContainerDiffs()) { FileServer::GetInstance()->Pause(); FileServer::GetInstance()->Resume(false, true); } diff --git a/core/common/FileEncryption.h b/core/common/FileEncryption.h index 5f6a316bdf..5877e6997a 100644 --- a/core/common/FileEncryption.h +++ b/core/common/FileEncryption.h @@ -66,10 +66,6 @@ class FileEncryption { static const int32_t FIRST_KEY_VERSION; static const std::string FIRST_KEY_VALUE; - -#ifdef APSARA_UNIT_TEST_MAIN - friend class SenderUnittest; -#endif }; } // namespace logtail diff --git a/core/common/FileSystemUtil.cpp b/core/common/FileSystemUtil.cpp index 822e219e55..52264e4d45 100644 --- a/core/common/FileSystemUtil.cpp +++ b/core/common/FileSystemUtil.cpp @@ -27,8 +27,6 @@ #endif #include -#include "boost/filesystem.hpp" - #include "EncodingConverter.h" #include "RuntimeUtil.h" #include "StringTools.h" @@ -45,14 +43,13 @@ const std::string PATH_SEPARATOR = "\\"; #endif std::string ParentPath(const std::string& path) { - boost::filesystem::path p(path); + std::filesystem::path p(path); return p.parent_path().string(); } bool CheckExistance(const std::string& path) { - boost::system::error_code ec; - boost::filesystem::path p(path); - return boost::filesystem::exists(p, ec); + std::error_code ec; + return std::filesystem::exists(path, ec); } bool Mkdirs(const std::string& dirPath) { @@ -62,7 +59,7 @@ bool Mkdirs(const std::string& dirPath) { if (errno != ENOENT) { return false; } - boost::filesystem::path p(dirPath); + std::filesystem::path p(dirPath); if (!p.has_parent_path()) { return false; } @@ -89,20 +86,19 @@ bool Mkdir(const std::string& dirPath) { } bool IsRelativePath(const std::string& path) { - boost::filesystem::path checkPointFilePath(path); + std::filesystem::path checkPointFilePath(path); return checkPointFilePath.is_relative(); } std::string AbsolutePath(const std::string& path, const std::string& basepath) { - return boost::filesystem::absolute(path, basepath).string(); -} - -std::string NormalizePath(const std::string& path) { - boost::filesystem::path abs(path); - if (abs.filename_is_dot() || abs.filename_is_dot_dot()) { - abs.remove_filename(); + if (path.empty()) { + return basepath; + } + std::filesystem::path p(path); + if (p.is_absolute()) { + return p.string(); } - return abs.string(); + return (std::filesystem::path(basepath) / p).string(); } int FSeek(FILE* stream, int64_t offset, int origin) { @@ -272,9 +268,9 @@ bool WriteFile(const std::string& fileName, const std::string& content, std::str } bool IsAccessibleDirectory(const std::string& dirPath) { - boost::filesystem::directory_iterator end; + std::filesystem::directory_iterator end; try { - boost::filesystem::directory_iterator dirIter(dirPath); + std::filesystem::directory_iterator dirIter(dirPath); return (dirIter != end); } catch (...) { return false; diff --git a/core/common/FileSystemUtil.h b/core/common/FileSystemUtil.h index 8437f4c856..7faba0be90 100644 --- a/core/common/FileSystemUtil.h +++ b/core/common/FileSystemUtil.h @@ -71,9 +71,6 @@ bool IsRelativePath(const std::string& path); // . -> /usr/local/ilogtail/. // ./a.txt -> /usr/local/ilogtail/./a.txt std::string AbsolutePath(const std::string& path, const std::string& basepath); -// /usr/local/ilogtail/. -> /usr/local/ilogtail -// /usr/local/ilogtail/./a.txt -> /usr/local/ilogtail/a.txt -std::string NormalizePath(const std::string& path); // FSeek, FTell that can handle file large than 2GB on 32bits OS. int FSeek(FILE* stream, int64_t offset, int origin); diff --git a/core/common/StringTools.cpp b/core/common/StringTools.cpp index 2194f7bfe1..5c3b4313a6 100644 --- a/core/common/StringTools.cpp +++ b/core/common/StringTools.cpp @@ -18,7 +18,6 @@ #include "boost/algorithm/string.hpp" #include "boost/exception/all.hpp" -#include "boost/filesystem.hpp" #include "logger/Logger.h" #if defined(_MSC_VER) @@ -389,20 +388,6 @@ bool NormalizeTopicRegFormat(std::string& topicFormat) { return true; } -bool isRootDirectory(const std::string& pathStr) { - boost::filesystem::path path(pathStr); - return path.has_root_directory() && !path.has_parent_path(); -} - -void RemoveFilePathTrailingSlash(std::string& filePath) { - if (isRootDirectory(filePath)) { - return; - } - boost::filesystem::path path(filePath); - path.remove_trailing_separator(); - filePath = path.string(); -} - bool IsInt(const char* sz) { bool ok = (sz != nullptr && *sz != '\0'); for (const auto* it = sz; ok && *it; ++it) { diff --git a/core/common/StringTools.h b/core/common/StringTools.h index df31bf7cdd..bd9994aaf0 100644 --- a/core/common/StringTools.h +++ b/core/common/StringTools.h @@ -155,8 +155,6 @@ bool ExtractTopics(const std::string& val, bool NormalizeTopicRegFormat(std::string& regStr); -void RemoveFilePathTrailingSlash(std::string& path); - bool IsInt(const char* sz); inline bool IsInt(const std::string& str) { diff --git a/core/container_manager/ContainerDiscoveryOptions.h b/core/container_manager/ContainerDiscoveryOptions.h index 1c64e4d2bd..27627f260b 100644 --- a/core/container_manager/ContainerDiscoveryOptions.h +++ b/core/container_manager/ContainerDiscoveryOptions.h @@ -137,6 +137,7 @@ struct MatchedContainerInfo { std::string PathExistInputContainerIDs; std::string SourceAddress; std::string InputType; + std::string InputIndex; std::string InputIsContainerFile; std::string FlusherType; std::string FlusherTargetAddress; @@ -151,6 +152,7 @@ struct MatchedContainerInfo { ss << "PathExistInputContainerIDs: " << PathExistInputContainerIDs << std::endl; ss << "SourceAddress: " << SourceAddress << std::endl; ss << "InputType: " << InputType << std::endl; + ss << "InputIndex: " << InputIndex << std::endl; ss << "InputIsContainerFile: " << InputIsContainerFile << std::endl; ss << "FlusherType: " << FlusherType << std::endl; ss << "FlusherTargetAddress: " << FlusherTargetAddress << std::endl; diff --git a/core/container_manager/ContainerManager.cpp b/core/container_manager/ContainerManager.cpp index 112b7ccf68..813f72705a 100644 --- a/core/container_manager/ContainerManager.cpp +++ b/core/container_manager/ContainerManager.cpp @@ -5,6 +5,7 @@ #include #include #include +#include #include "json/json.h" @@ -19,6 +20,7 @@ #include "container_manager/ContainerDiff.h" #include "container_manager/ContainerDiscoveryOptions.h" #include "file_server/FileServer.h" +#include "file_server/StaticFileServer.h" #include "go_pipeline/LogtailPlugin.h" #include "monitor/Monitor.h" #include "monitor/SelfMonitorServer.h" @@ -99,110 +101,137 @@ void ContainerManager::pollingLoop() { } } -void ContainerManager::ApplyContainerDiffs() { +void ContainerManager::ApplyFileServerContainerDiffs() { std::vector> configResults; + std::vector> fileKeys; + + // Collect keys that belong to FileServer (pipeline name + input index 0 convention). + FileServer::GetInstance()->WithFileDiscoveryConfigs( + [&](const std::unordered_map& nameConfigMap) { + for (const auto& e : mConfigContainerDiffMap) { + if (e.first.second != 0) { + continue; + } + if (nameConfigMap.find(e.first.first) != nameConfigMap.end()) { + fileKeys.push_back(e.first); + } + } + }); - // Write lock: apply structural mutations to each config's container info. FileServer::GetInstance()->WithFileDiscoveryConfigsMutable( [&](std::unordered_map& nameConfigMap) { - for (auto& pair : mConfigContainerDiffMap) { - const auto& itr = nameConfigMap.find(pair.first); + for (const auto& key : fileKeys) { + const auto itDiff = mConfigContainerDiffMap.find(key); + if (itDiff == mConfigContainerDiffMap.end()) { + continue; + } + const auto itr = nameConfigMap.find(key.first); if (itr == nameConfigMap.end()) { continue; } const auto& options = itr->second.first; const auto& ctx = itr->second.second; - const auto& diff = pair.second; - - LOG_INFO(sLogger, ("ApplyContainerDiffs diff", diff->ToString())("configName", ctx->GetConfigName())); + const auto& diff = itDiff->second; + LOG_INFO(sLogger, + ("ApplyFileServerContainerDiffs diff", diff->ToString())("configName", ctx->GetConfigName())); + applyContainerDiffToOptions(options, ctx, diff); + } + }); - if (diff->mRefreshAllContainers) { - options->ClearContainerInfo(); + FileServer::GetInstance()->WithFileDiscoveryConfigs( + [&](const std::unordered_map& nameConfigMap) { + for (const auto& key : fileKeys) { + const auto itr = nameConfigMap.find(key.first); + if (itr == nameConfigMap.end()) { + continue; } + appendMatchedContainerInfoForConfig( + key.first, key.second, itr->second.first, itr->second.second, configResults); + } + }); - for (const auto& pair : diff->mLegacyCheckpointAdded) { - const std::string& basePathInCheckpoint = pair.first; - const std::shared_ptr& container = pair.second; - options->UpdateRawContainerInfo(container, ctx, basePathInCheckpoint); - } + for (const auto& key : fileKeys) { + mConfigContainerDiffMap.erase(key); + } + sendMatchedContainerInfo(configResults); +} - for (const auto& container : diff->mAdded) { - options->UpdateRawContainerInfo(container, ctx); - } - for (const auto& container : diff->mModified) { - options->UpdateRawContainerInfo(container, ctx); - } - for (const auto& container : diff->mRemoved) { - options->DeleteRawContainerInfo(container); +void ContainerManager::ApplyStaticFileServerContainerDiffs() { + std::vector> configResults; + std::vector> staticKeys; + + StaticFileServer::GetInstance()->WithFileDiscoveryConfigs( + [&](const std::map, FileDiscoveryConfig>& nameConfigMap) { + for (const auto& e : mConfigContainerDiffMap) { + if (nameConfigMap.find(e.first) != nameConfigMap.end()) { + staticKeys.push_back(e.first); } } }); - // Read lock: build MatchedContainerInfo from the now-stable container state. - // Filesystem existence checks (CheckExistance) can be slow, so they are kept - // outside the write lock to avoid blocking concurrent ReadLock holders. - // mMatchedContainerInfo is only written here and read in sendAllMatchedContainerInfo, - // both of which run on the single polling thread, so no write-under-read-lock hazard. - FileServer::GetInstance()->WithFileDiscoveryConfigs( - [&](const std::unordered_map& nameConfigMap) { - for (const auto& pair : mConfigContainerDiffMap) { - const auto& itr = nameConfigMap.find(pair.first); + StaticFileServer::GetInstance()->WithFileDiscoveryConfigsMutable( + [&](std::map, FileDiscoveryConfig>& nameConfigMap) { + for (const auto& key : staticKeys) { + const auto itDiff = mConfigContainerDiffMap.find(key); + if (itDiff == mConfigContainerDiffMap.end()) { + continue; + } + const auto itr = nameConfigMap.find(key); if (itr == nameConfigMap.end()) { continue; } const auto& options = itr->second.first; const auto& ctx = itr->second.second; + const auto& diff = itDiff->second; + LOG_INFO(sLogger, + ("ApplyStaticFileServerContainerDiffs diff", + diff->ToString())("configName", ctx->GetConfigName())("inputIndex", key.second)); + applyContainerDiffToOptions(options, ctx, diff); + } + }); - if (options->GetContainerDiscoveryOptions().mCollectingContainersMeta) { - std::vector pathExistContainerIDs; - std::vector pathNotExistContainerIDs; - const auto& containerInfos = options->GetContainerInfo(); - if (containerInfos) { - const auto& basePathInfos = options->GetBasePathInfos(); - for (const auto& info : *containerInfos) { - if (!info.mRawContainerInfo) { - continue; - } - - if (info.mRealBaseDirs.size() != basePathInfos.size()) { - LOG_WARNING(sLogger, - ("mRealBaseDirs size mismatch", - "container may have incomplete path mapping")("container id", - info.mRawContainerInfo->mID)( - "expected size", basePathInfos.size())("actual size", - info.mRealBaseDirs.size())); - } - - bool anyExists = false; - for (const auto& realBaseDir : info.mRealBaseDirs) { - if (!realBaseDir.empty() && CheckExistance(realBaseDir)) { - anyExists = true; - break; - } - } - - if (anyExists) { - pathExistContainerIDs.push_back(info.mRawContainerInfo->mID); - } else { - pathNotExistContainerIDs.push_back(info.mRawContainerInfo->mID); - } - } - } - auto configResult = std::make_shared( - CreateMatchedContainerInfo(options, ctx, pathExistContainerIDs, pathNotExistContainerIDs)); - options->GetContainerDiscoveryOptions().mMatchedContainerInfo = configResult; - configResults.push_back(configResult); - LOG_DEBUG(sLogger, ("configResult", configResult->ToString())("configName", ctx->GetConfigName())); + StaticFileServer::GetInstance()->WithFileDiscoveryConfigs( + [&](const std::map, FileDiscoveryConfig>& nameConfigMap) { + for (const auto& key : staticKeys) { + const auto itr = nameConfigMap.find(key); + if (itr == nameConfigMap.end()) { + continue; } + appendMatchedContainerInfoForConfig( + key.first, key.second, itr->second.first, itr->second.second, configResults); } }); + for (const auto& key : staticKeys) { + mConfigContainerDiffMap.erase(key); + } sendMatchedContainerInfo(configResults); - mConfigContainerDiffMap.clear(); } +void ContainerManager::applyContainerDiffToOptions(FileDiscoveryOptions* options, + const CollectionPipelineContext* ctx, + const std::shared_ptr& diff) { + if (!options || !ctx || !diff) { + return; + } + if (diff->mRefreshAllContainers) { + options->ClearContainerInfo(); + } + for (const auto& p : diff->mLegacyCheckpointAdded) { + options->UpdateRawContainerInfo(p.second, ctx, p.first); + } + for (const auto& container : diff->mAdded) { + options->UpdateRawContainerInfo(container, ctx); + } + for (const auto& container : diff->mModified) { + options->UpdateRawContainerInfo(container, ctx); + } + for (const auto& containerId : diff->mRemoved) { + options->DeleteRawContainerInfo(containerId); + } +} -bool ContainerManager::CheckContainerDiffForAllConfig() { +bool ContainerManager::CheckFileServerContainerDiffs() { if (!mIsRunning) { return false; } @@ -212,7 +241,25 @@ bool ContainerManager::CheckContainerDiffForAllConfig() { [&](std::unordered_map& nameConfigMap) { for (const auto& [name, cfg] : nameConfigMap) { if (cfg.first->IsContainerDiscoveryEnabled()) { - if (checkContainerDiffForOneConfig(cfg.first, cfg.second)) { + if (checkContainerDiffForOneConfig(name, 0, cfg.first, cfg.second)) { + isUpdate = true; + } + } + } + }); + return isUpdate; +} + +bool ContainerManager::CheckStaticFileServerContainerDiffs() { + if (!mIsRunning) { + return false; + } + bool isUpdate = false; + StaticFileServer::GetInstance()->WithFileDiscoveryConfigsMutable( + [&](std::map, FileDiscoveryConfig>& nameConfigMap) { + for (const auto& [key, cfg] : nameConfigMap) { + if (cfg.first->IsContainerDiscoveryEnabled()) { + if (checkContainerDiffForOneConfig(key.first, key.second, cfg.first, cfg.second)) { isUpdate = true; } } @@ -236,6 +283,17 @@ void ContainerManager::sendAllMatchedContainerInfo() { } } }); + StaticFileServer::GetInstance()->WithFileDiscoveryConfigs( + [&](const std::map, FileDiscoveryConfig>& nameConfigMap) { + for (const auto& [key, cfg] : nameConfigMap) { + FileDiscoveryOptions* options = cfg.first; + if (options->IsContainerDiscoveryEnabled() + && options->GetContainerDiscoveryOptions().mCollectingContainersMeta + && options->GetContainerDiscoveryOptions().mMatchedContainerInfo) { + configResults.push_back(options->GetContainerDiscoveryOptions().mMatchedContainerInfo); + } + } + }); sendMatchedContainerInfo(configResults); } @@ -294,6 +352,7 @@ void ContainerManager::sendMatchedContainerInfo(std::vectorSetContent("input.path_not_exist_container_ids", itr->PathNotExistInputContainerIDs); logEventPtr->SetContent("input.type", itr->InputType); logEventPtr->SetContent("input.container_file", itr->InputIsContainerFile); + logEventPtr->SetContent("input.index", itr->InputIndex); logEventPtr->SetContent("flusher.type", itr->FlusherType); logEventPtr->SetContent("flusher.target_addresses", itr->FlusherTargetAddress); } @@ -305,28 +364,27 @@ void ContainerManager::sendMatchedContainerInfo(std::vectorGetLastContainerUpdateTime(); - int64_t newContainerUpdateTime; - if (lastConfigContainerUpdateTime < mLastFullUpdateTime - || (mLastFullUpdateTime == 0 && mLastIncrementalUpdateTime == 0 && lastConfigContainerUpdateTime == 0)) { + int64_t newContainerUpdateTime = 0; + if (lastConfigContainerUpdateTime < mLastFullUpdateTime.load() + || (mLastFullUpdateTime.load() == 0 && mLastIncrementalUpdateTime.load() == 0 + && lastConfigContainerUpdateTime == 0)) { refreshAllContainers = true; - newContainerUpdateTime = (mLastFullUpdateTime == 0) ? static_cast(1) : mLastFullUpdateTime.load(); - } else if (lastConfigContainerUpdateTime < mLastIncrementalUpdateTime) { + newContainerUpdateTime + = (mLastFullUpdateTime.load() == 0) ? static_cast(1) : mLastFullUpdateTime.load(); + } else if (lastConfigContainerUpdateTime < mLastIncrementalUpdateTime.load()) { refreshAllContainers = false; - newContainerUpdateTime = mLastIncrementalUpdateTime; + newContainerUpdateTime = mLastIncrementalUpdateTime.load(); } else { return false; } std::unordered_map> containerInfoMap; - std::vector removedList; - std::vector matchAddedList; ContainerDiff diff; if (refreshAllContainers) { options->SetFullContainerList(std::make_shared>()); @@ -356,12 +414,13 @@ bool ContainerManager::checkContainerDiffForOneConfig(FileDiscoveryOptions* opti LOG_DEBUG( sLogger, - ("diff", diff.ToString())("configName", ctx->GetConfigName())( + ("diff", diff.ToString())("configName", configName)("inputIndex", inputIndex)( "containerFilters", options->GetContainerDiscoveryOptions().mContainerFilters.ToString())( "fullContainerList", options->GetFullContainerList()->size())("containerInfoMap", containerInfoMap.size())( - "lastConfigContainerUpdateTime", lastConfigContainerUpdateTime)("mLastFullUpdateTime", mLastFullUpdateTime)( - "mLastIncrementalUpdateTime", mLastIncrementalUpdateTime)("newContainerUpdateTime", - newContainerUpdateTime)); + "lastConfigContainerUpdateTime", lastConfigContainerUpdateTime)("mLastFullUpdateTime", + mLastFullUpdateTime.load())( + "mLastIncrementalUpdateTime", mLastIncrementalUpdateTime.load())("newContainerUpdateTime", + newContainerUpdateTime)); // Update the config's container update time when there are changes options->SetLastContainerUpdateTime(newContainerUpdateTime); @@ -370,7 +429,8 @@ bool ContainerManager::checkContainerDiffForOneConfig(FileDiscoveryOptions* opti return false; } - mConfigContainerDiffMap[ctx->GetConfigName()] = std::make_shared(diff); + mConfigContainerDiffMap[std::pair(configName, inputIndex)] + = std::make_shared(diff); return true; } @@ -425,7 +485,7 @@ void ContainerManager::incrementallyUpdateContainersSnapshot() { } if (hasChanges) { - mLastIncrementalUpdateTime = time(nullptr); + mLastIncrementalUpdateTime.store(time(nullptr)); } } @@ -459,21 +519,22 @@ void ContainerManager::refreshAllContainersSnapshot() { WriteLock lock(mContainerMapRWLock); mContainerMap.swap(tmpContainerMap); } - mLastFullUpdateTime = time(nullptr); + mLastFullUpdateTime.store(time(nullptr)); tmpContainerMap.clear(); } - MatchedContainerInfo -ContainerManager::CreateMatchedContainerInfo(const FileDiscoveryOptions* options, +ContainerManager::createMatchedContainerInfo(const std::string& configName, + size_t inputIndex, + const FileDiscoveryOptions* options, const CollectionPipelineContext* ctx, const std::vector& pathExistContainerIDs, const std::vector& pathNotExistContainerIDs) { MatchedContainerInfo result; if (!options || !ctx) { - LOG_WARNING(sLogger, ("CreateMatchedContainerInfo failed", "options or ctx is null")); + LOG_WARNING(sLogger, ("createMatchedContainerInfo failed", "options or ctx is null")); return result; } @@ -481,7 +542,8 @@ ContainerManager::CreateMatchedContainerInfo(const FileDiscoveryOptions* options result.DataType = "container_config_result"; result.Project = ctx->GetProjectName(); result.Logstore = ctx->GetLogstoreName(); - result.ConfigName = ctx->GetConfigName(); + result.ConfigName = configName; + result.InputIndex = std::to_string(inputIndex); // Container IDs - join with semicolon like in Go version result.PathExistInputContainerIDs = joinContainerIDs(pathExistContainerIDs); @@ -496,6 +558,54 @@ ContainerManager::CreateMatchedContainerInfo(const FileDiscoveryOptions* options return result; } +void ContainerManager::appendMatchedContainerInfoForConfig( + const std::string& configName, + size_t inputIndex, + FileDiscoveryOptions* options, + const CollectionPipelineContext* ctx, + std::vector>& configResults) { + if (!options || !ctx) { + return; + } + if (!options->GetContainerDiscoveryOptions().mCollectingContainersMeta) { + return; + } + std::vector pathExistContainerIDs; + std::vector pathNotExistContainerIDs; + const auto& containerInfos = options->GetContainerInfo(); + if (containerInfos) { + const auto& basePathInfos = options->GetBasePathInfos(); + for (const auto& info : *containerInfos) { + if (!info.mRawContainerInfo) { + continue; + } + if (info.mRealBaseDirs.size() != basePathInfos.size()) { + LOG_WARNING(sLogger, + ("mRealBaseDirs size mismatch", + "container may have incomplete path mapping")("container id", info.mRawContainerInfo->mID)( + "expected size", basePathInfos.size())("actual size", info.mRealBaseDirs.size())); + } + bool anyExists = false; + for (const auto& realBaseDir : info.mRealBaseDirs) { + if (!realBaseDir.empty() && CheckExistance(realBaseDir)) { + anyExists = true; + break; + } + } + if (anyExists) { + pathExistContainerIDs.push_back(info.mRawContainerInfo->mID); + } else { + pathNotExistContainerIDs.push_back(info.mRawContainerInfo->mID); + } + } + } + auto configResult = std::make_shared(createMatchedContainerInfo( + configName, inputIndex, options, ctx, pathExistContainerIDs, pathNotExistContainerIDs)); + options->GetContainerDiscoveryOptions().mMatchedContainerInfo = configResult; + configResults.push_back(configResult); + LOG_DEBUG(sLogger, ("configResult", configResult->ToString())("configName", configName)("inputIndex", inputIndex)); +} + std::string ContainerManager::joinContainerIDs(const std::vector& containerIDs) { if (containerIDs.empty()) { return ""; @@ -528,7 +638,7 @@ std::string ContainerManager::joinContainerIDs(const std::vector& c return result; } -void ContainerManager::GetContainerStoppedEvents(std::vector& eventVec) { +void ContainerManager::GetFileServerContainerStoppedEvents(std::vector& eventVec) { std::vector stoppedContainerIDs; { std::lock_guard lock(mStoppedContainerIDsMutex); @@ -543,40 +653,67 @@ void ContainerManager::GetContainerStoppedEvents(std::vector& eventVec) [&](const std::unordered_map& nameConfigMap) { for (const auto& containerId : stoppedContainerIDs) { for (const auto& [configName, cfg] : nameConfigMap) { - const FileDiscoveryOptions* options = cfg.first; - if (!options->IsContainerDiscoveryEnabled()) { - continue; - } - // GetContainerInfo() returns a shared_ptr; even if ApplyContainerDiffs - // replaces the pointed-to vector, the ref-count keeps this copy alive. - auto containerInfosSnapshot = options->GetContainerInfo(); - if (!containerInfosSnapshot) { - continue; - } - for (auto& info : *containerInfosSnapshot) { - if (!info.mRawContainerInfo) { - continue; - } - if (info.mRawContainerInfo->mID == containerId) { - for (const auto& realBaseDir : info.mRealBaseDirs) { - if (!realBaseDir.empty()) { - Event* pStoppedEvent - = new Event(realBaseDir, "", EVENT_ISDIR | EVENT_CONTAINER_STOPPED, -1, 0); - pStoppedEvent->SetConfigName(configName); - pStoppedEvent->SetContainerID(containerId); - eventVec.push_back(pStoppedEvent); - } - } - info.mRawContainerInfo->mStopped.store(true); - LOG_DEBUG(sLogger, - ("generate stop event, containerId", containerId)("configName", configName)); - } - } + getContainerStoppedEventForOneConfig(configName, 0, containerId, cfg, &eventVec); + } + } + }); +} + +void ContainerManager::GetStaticFileServerContainerStoppedEvents() { + std::vector stoppedContainerIDs; + { + std::lock_guard lock(mStoppedContainerIDsMutex); + stoppedContainerIDs.swap(mStoppedContainerIDs); + } + if (stoppedContainerIDs.empty()) { + return; + } + LOG_INFO(sLogger, ("stoppedContainerIDs", ToString(stoppedContainerIDs))); + StaticFileServer::GetInstance()->WithFileDiscoveryConfigs( + [&](const std::map, FileDiscoveryConfig>& nameConfigMap) { + for (const auto& containerId : stoppedContainerIDs) { + for (const auto& [key, cfg] : nameConfigMap) { + getContainerStoppedEventForOneConfig(key.first, key.second, containerId, cfg, nullptr); } } }); } +void ContainerManager::getContainerStoppedEventForOneConfig(const std::string& configName, + size_t inputIndex, + const std::string& containerID, + const FileDiscoveryConfig& config, + std::vector* eventVec) { + const FileDiscoveryOptions* options = config.first; + if (!options->IsContainerDiscoveryEnabled()) { + return; + } + const auto& containerInfos = options->GetContainerInfo(); + if (!containerInfos) { + return; + } + for (auto& info : *containerInfos) { + if (!info.mRawContainerInfo) { + continue; + } + if (info.mRawContainerInfo->mID == containerID) { + // 为每个真实路径生成停止事件 + for (const auto& realBaseDir : info.mRealBaseDirs) { + if (eventVec && !realBaseDir.empty()) { + Event* pStoppedEvent = new Event(realBaseDir, "", EVENT_ISDIR | EVENT_CONTAINER_STOPPED, -1, 0); + pStoppedEvent->SetConfigName(configName); + pStoppedEvent->SetInputIndex(std::to_string(inputIndex)); + pStoppedEvent->SetContainerID(containerID); + eventVec->push_back(pStoppedEvent); + } + } + info.mRawContainerInfo->mStopped.store(true); + LOG_DEBUG( + sLogger, + ("generate stop event, containerId", containerID)("configName", configName)("inputIndex", inputIndex)); + } + } +} bool IsMapLabelsMatch(const MatchCriteriaFilter& filter, const std::unordered_map& labels) { if (!filter.mIncludeFields.mFieldsMap.empty() || !filter.mIncludeFields.mFieldsRegMap.empty()) { @@ -623,7 +760,6 @@ bool IsMapLabelsMatch(const MatchCriteriaFilter& filter, const std::unordered_ma return true; } - bool IsK8sFilterMatch(const K8sFilter& filter, const K8sInfo& k8sInfo) { if (k8sInfo.mPausedContainer) { return false; @@ -644,7 +780,6 @@ bool IsK8sFilterMatch(const K8sFilter& filter, const K8sInfo& k8sInfo) { return IsMapLabelsMatch(filter.mK8sLabelFilter, k8sInfo.mLabels); } - void ContainerManager::computeMatchedContainersDiff( std::set& fullContainerIDList, const std::unordered_map>& matchList, @@ -918,7 +1053,7 @@ void ContainerManager::LoadContainerInfo() { loadContainerInfoFromContainersFormat(root, configPath); } // Apply container diffs immediately after loading - ApplyContainerDiffs(); + ApplyFileServerContainerDiffs(); } void ContainerManager::loadContainerInfoFromDetailFormat(const Json::Value& root, const std::string& configPath) { @@ -928,7 +1063,7 @@ void ContainerManager::loadContainerInfoFromDetailFormat(const Json::Value& root } std::unordered_map> tmpContainerMap; - std::unordered_map>>> + std::map, std::vector>>> configContainerMap; const auto& arr = root["detail"]; @@ -938,6 +1073,10 @@ void ContainerManager::loadContainerInfoFromDetailFormat(const Json::Value& root if (item.isMember("config_name") && item["config_name"].isString()) { configName = item["config_name"].asString(); } + size_t inputIndex = 0; + if (item.isMember("input_index") && item["input_index"].isInt()) { + inputIndex = item["input_index"].asInt(); + } if (item.isMember("params") && item["params"].isString()) { std::string paramsStr = item["params"].asString(); @@ -1034,7 +1173,8 @@ void ContainerManager::loadContainerInfoFromDetailFormat(const Json::Value& root tmpContainerMap[info->mID] = info; // Also associate with config if config name is available if (!configName.empty()) { - configContainerMap[configName].push_back(std::make_pair(basePathInCheckpoint, info)); + configContainerMap[std::pair(configName, inputIndex)].emplace_back( + std::make_pair(basePathInCheckpoint, info)); } } } else { @@ -1048,9 +1188,11 @@ void ContainerManager::loadContainerInfoFromDetailFormat(const Json::Value& root WriteLock lock(mContainerMapRWLock); mContainerMap.swap(tmpContainerMap); } + // Update config container diffs for each config for (const auto& configPair : configContainerMap) { - const std::string& configName = configPair.first; + const std::string& configName = configPair.first.first; + size_t inputIndex = configPair.first.second; const auto& containers = configPair.second; ContainerDiff diff; @@ -1059,18 +1201,27 @@ void ContainerManager::loadContainerInfoFromDetailFormat(const Json::Value& root } if (!diff.IsEmpty()) { - mConfigContainerDiffMap[configName] = std::make_shared(diff); + mConfigContainerDiffMap[std::pair(configName, inputIndex)] + = std::make_shared(diff); } } FileServer::GetInstance()->WithFileDiscoveryConfigsMutable( - [&](const std::unordered_map& nameConfigMap) { + [&](std::unordered_map& nameConfigMap) { for (const auto& [name, cfg] : nameConfigMap) { if (cfg.first->IsContainerDiscoveryEnabled()) { cfg.first->SetLastContainerUpdateTime(1); } } }); + StaticFileServer::GetInstance()->WithFileDiscoveryConfigsMutable( + [&](std::map, FileDiscoveryConfig>& nameConfigMap) { + for (const auto& [key, cfg] : nameConfigMap) { + if (cfg.first->IsContainerDiscoveryEnabled()) { + cfg.first->SetLastContainerUpdateTime(1); + } + } + }); LOG_INFO(sLogger, ("load container state from docker_path_config.json (v0.1.0)", configPath)); } @@ -1100,10 +1251,19 @@ void ContainerManager::loadContainerInfoFromContainersFormat(const Json::Value& // Apply containers to all existing configs FileServer::GetInstance()->WithFileDiscoveryConfigsMutable( [&](std::unordered_map& nameConfigMap) { - LOG_DEBUG(sLogger, ("recover containers to nameConfigMap", nameConfigMap.size())); + LOG_DEBUG(sLogger, ("recover containers to FileServer nameConfigMap", nameConfigMap.size())); for (const auto& [name, cfg] : nameConfigMap) { if (cfg.first->IsContainerDiscoveryEnabled()) { - checkContainerDiffForOneConfig(cfg.first, cfg.second); + checkContainerDiffForOneConfig(name, 0, cfg.first, cfg.second); + } + } + }); + StaticFileServer::GetInstance()->WithFileDiscoveryConfigsMutable( + [&](std::map, FileDiscoveryConfig>& nameConfigMap) { + LOG_DEBUG(sLogger, ("recover containers to StaticFileServer nameConfigMap", nameConfigMap.size())); + for (const auto& [key, cfg] : nameConfigMap) { + if (cfg.first->IsContainerDiscoveryEnabled()) { + checkContainerDiffForOneConfig(key.first, key.second, cfg.first, cfg.second); } } }); @@ -1111,4 +1271,84 @@ void ContainerManager::loadContainerInfoFromContainersFormat(const Json::Value& } } +void ContainerManager::updateContainerInfoPointersInAllConfigs() { + auto updateOptions = [&](FileDiscoveryOptions* options) { + if (!options->IsContainerDiscoveryEnabled()) { + return; + } + const auto& containerInfos = options->GetContainerInfo(); + if (!containerInfos) { + return; + } + for (auto& containerInfo : *containerInfos) { + if (!containerInfo.mRawContainerInfo) { + continue; + } + const std::string& containerId = containerInfo.mRawContainerInfo->mID; + { + WriteLock lock(mContainerMapRWLock); + auto it = mContainerMap.find(containerId); + if (it != mContainerMap.end()) { + containerInfo.mRawContainerInfo = it->second; + } else { + LOG_WARNING(sLogger, ("container not found in global map during pointer update", containerId)); + } + } + } + }; + + FileServer::GetInstance()->WithFileDiscoveryConfigsMutable( + [&](std::unordered_map& nameConfigMap) { + for (auto& [name, cfg] : nameConfigMap) { + updateOptions(cfg.first); + } + }); + StaticFileServer::GetInstance()->WithFileDiscoveryConfigsMutable( + [&](std::map, FileDiscoveryConfig>& nameConfigMap) { + for (auto& [key, cfg] : nameConfigMap) { + updateOptions(cfg.first); + } + }); +} + +void ContainerManager::updateContainerInfoPointersForContainers(const std::vector& containerIDs) { + auto updateOptions = [&](FileDiscoveryOptions* options) { + if (!options->IsContainerDiscoveryEnabled()) { + return; + } + const auto& containerInfos = options->GetContainerInfo(); + if (!containerInfos) { + return; + } + for (auto& containerInfo : *containerInfos) { + if (!containerInfo.mRawContainerInfo) { + continue; + } + const std::string& containerId = containerInfo.mRawContainerInfo->mID; + if (std::find(containerIDs.begin(), containerIDs.end(), containerId) != containerIDs.end()) { + WriteLock lock(mContainerMapRWLock); + auto it = mContainerMap.find(containerId); + if (it != mContainerMap.end()) { + containerInfo.mRawContainerInfo = it->second; + } else { + LOG_WARNING(sLogger, ("container not found in global map during pointer update", containerId)); + } + } + } + }; + + FileServer::GetInstance()->WithFileDiscoveryConfigsMutable( + [&](std::unordered_map& nameConfigMap) { + for (auto& [name, cfg] : nameConfigMap) { + updateOptions(cfg.first); + } + }); + StaticFileServer::GetInstance()->WithFileDiscoveryConfigsMutable( + [&](std::map, FileDiscoveryConfig>& nameConfigMap) { + for (auto& [key, cfg] : nameConfigMap) { + updateOptions(cfg.first); + } + }); +} + } // namespace logtail diff --git a/core/container_manager/ContainerManager.h b/core/container_manager/ContainerManager.h index bb38c2ae69..5bcc6ad8e4 100644 --- a/core/container_manager/ContainerManager.h +++ b/core/container_manager/ContainerManager.h @@ -47,19 +47,24 @@ class ContainerManager { void Init(); void Stop(); - void ApplyContainerDiffs(); - bool CheckContainerDiffForAllConfig(); + // 由于获取容器信息可能耗时较大,这里通过更新时间是否更新过判断是否成功拉取过一次容器信息 + bool IsReady() { + return mIsRunning.load() && (mLastFullUpdateTime.load() > 0 || mLastIncrementalUpdateTime.load() > 0); + } + + // file server + void ApplyFileServerContainerDiffs(); + bool CheckFileServerContainerDiffs(); + void GetFileServerContainerStoppedEvents(std::vector& eventVec); + // static file server + void ApplyStaticFileServerContainerDiffs(); + bool CheckStaticFileServerContainerDiffs(); + void GetStaticFileServerContainerStoppedEvents(); - void GetContainerStoppedEvents(std::vector& eventVec); // Persist/restore container runtime state void SaveContainerInfo(); void LoadContainerInfo(); - MatchedContainerInfo CreateMatchedContainerInfo(const FileDiscoveryOptions* options, - const CollectionPipelineContext* ctx, - const std::vector& pathExistContainerIDs, - const std::vector& pathNotExistContainerIDs); - void UpdateMatchedContainerInfoPipeline(CollectionPipelineContext* ctx, size_t inputIndex); void RemoveMatchedContainerInfoPipeline(); @@ -68,7 +73,17 @@ class ContainerManager { void refreshAllContainersSnapshot(); void incrementallyUpdateContainersSnapshot(); - bool checkContainerDiffForOneConfig(FileDiscoveryOptions* options, const CollectionPipelineContext* ctx); + bool checkContainerDiffForOneConfig(const std::string& configName, + size_t inputIndex, + FileDiscoveryOptions* options, + const CollectionPipelineContext* ctx); + void getContainerStoppedEventForOneConfig(const std::string& configName, + size_t inputIndex, + const std::string& containerID, + const FileDiscoveryConfig& config, + std::vector* eventVec = nullptr); + void updateContainerInfoPointersInAllConfigs(); + void updateContainerInfoPointersForContainers(const std::vector& containerIDs); void computeMatchedContainersDiff(std::set& fullContainerIDList, const std::unordered_map>& matchList, @@ -83,12 +98,28 @@ class ContainerManager { void sendMatchedContainerInfo(std::vector> configResults); void sendAllMatchedContainerInfo(); - // Helper method for joining container IDs - std::string joinContainerIDs(const std::vector& containerIDs); + static std::string joinContainerIDs(const std::vector& containerIDs); + + static MatchedContainerInfo createMatchedContainerInfo(const std::string& configName, + size_t inputIndex, + const FileDiscoveryOptions* options, + const CollectionPipelineContext* ctx, + const std::vector& pathExistContainerIDs, + const std::vector& pathNotExistContainerIDs); + + static void appendMatchedContainerInfoForConfig(const std::string& configName, + size_t inputIndex, + FileDiscoveryOptions* options, + const CollectionPipelineContext* ctx, + std::vector>& configResults); + + static void applyContainerDiffToOptions(FileDiscoveryOptions* options, + const CollectionPipelineContext* ctx, + const std::shared_ptr& diff); std::unordered_map> mContainerMap; - std::unordered_map> mConfigContainerDiffMap; - std::unordered_map> mConfigContainerResultMap; + std::map, std::shared_ptr> mConfigContainerDiffMap; + std::map, std::shared_ptr> mConfigContainerResultMap; mutable ReadWriteLock mContainerMapRWLock; std::vector mStoppedContainerIDs; std::mutex mStoppedContainerIDsMutex; diff --git a/core/file_server/ConfigManager.cpp b/core/file_server/ConfigManager.cpp index f4f427de56..8f3426b535 100644 --- a/core/file_server/ConfigManager.cpp +++ b/core/file_server/ConfigManager.cpp @@ -940,8 +940,8 @@ void ConfigManager::GetRelatedConfigs(const std::string& path, std::vector& eventVec) { - ContainerManager::GetInstance()->GetContainerStoppedEvents(eventVec); +void ConfigManager::GetFileServerContainerStoppedEvents(std::vector& eventVec) { + ContainerManager::GetInstance()->GetFileServerContainerStoppedEvents(eventVec); } diff --git a/core/file_server/ConfigManager.h b/core/file_server/ConfigManager.h index f201bfe3f2..eeb6dc71a0 100644 --- a/core/file_server/ConfigManager.h +++ b/core/file_server/ConfigManager.h @@ -439,7 +439,7 @@ class ConfigManager { // std::string GetAllProjectsSet(); - void GetContainerStoppedEvents(std::vector& eventVec); + void GetFileServerContainerStoppedEvents(std::vector& eventVec); /** @@ -565,15 +565,11 @@ class ConfigManager { void CleanEnviroments(); friend class EventDispatcherTest; - friend class SenderUnittest; - friend class UtilUnittest; - friend class ConfigUpdatorUnittest; friend class ConfigMatchUnittest; friend class FuxiSceneUnittest; friend class SymlinkInotifyTest; friend class LogFilterUnittest; friend class FuseFileUnittest; - friend class MultiServerConfigUpdatorUnitest; friend class CreateModifyHandlerUnittest; friend class ProcessorDesensitizeNativeUnittest; friend class ConfigContainerUnittest; diff --git a/core/file_server/EventDispatcher.h b/core/file_server/EventDispatcher.h index 89b552c8dd..d7f41aa201 100644 --- a/core/file_server/EventDispatcher.h +++ b/core/file_server/EventDispatcher.h @@ -278,14 +278,11 @@ class EventDispatcher { #ifdef APSARA_UNIT_TEST_MAIN friend class EventDispatcherTest; - friend class ConfigUpdatorUnittest; friend class ConfigMatchUnittest; friend class FuxiSceneUnittest; friend class LogtailWriterTest; friend class SymlinkInotifyTest; - friend class SenderUnittest; friend class FuseFileUnittest; - friend class MultiServerConfigUpdatorUnitest; friend class EventDispatcherDirUnittest; friend class ModifyHandlerUnittest; friend class PipelineUpdateUnittest; diff --git a/core/file_server/FileDiscoveryOptions.cpp b/core/file_server/FileDiscoveryOptions.cpp index 632a2cb2bd..01c6ab286d 100644 --- a/core/file_server/FileDiscoveryOptions.cpp +++ b/core/file_server/FileDiscoveryOptions.cpp @@ -26,6 +26,7 @@ #include "common/LogtailCommonFlags.h" #include "common/ParamExtractor.h" #include "common/StringTools.h" +#include "logger/Logger.h" using namespace std; @@ -795,6 +796,75 @@ size_t FileDiscoveryOptions::GetRealBaseDirIndex(const ContainerInfo* containerI return bestMatchIndex; } +// 为 ContainerInfo 设置多个真实路径,与 FileDiscoveryOptions 的多路径对应 +bool SetContainerBaseDirs(ContainerInfo& containerInfo, const FileDiscoveryOptions* fileDiscovery) { + if (!containerInfo.mRealBaseDirs.empty()) { + return true; // 已经设置过 + } + + if (!fileDiscovery || containerInfo.mRawContainerInfo == nullptr) { + return false; + } + + const auto& pathInfos = fileDiscovery->GetBasePathInfos(); + bool hasSuccess = false; + + // 为每个配置路径计算对应的容器真实路径 + for (const auto& pathInfo : pathInfos) { + string logPath = pathInfo.hasWildcard() ? pathInfo.wildcardPaths[0] : pathInfo.basePath; + + // 计算该路径在容器中的映射 + string realBaseDir; + if (SetContainerBaseDirForPath(containerInfo, logPath, realBaseDir)) { + containerInfo.mRealBaseDirs.push_back(realBaseDir); + hasSuccess = true; + } else { + // 映射失败,存储空字符串以保持索引对应 + LOG_WARNING(sLogger, + ("failed to map container path", "path mapping failed for this config path")( + "container id", containerInfo.mRawContainerInfo->mID)("log path", logPath)); + containerInfo.mRealBaseDirs.push_back(""); + } + } + + return hasSuccess; +} + +// 计算单个配置路径在容器中的映射路径(不修改 containerInfo) +bool SetContainerBaseDirForPath(const ContainerInfo& containerInfo, const string& logPath, string& outRealBaseDir) { + if (containerInfo.mRawContainerInfo == nullptr) { + return false; + } + + size_t pthSize = logPath.size(); + size_t size = containerInfo.mRawContainerInfo->mMounts.size(); + size_t bestMatchedMountsIndex = size; + + // ParseByJSONObj 确保 Destination、Source、mUpperDir 不会以\或者/结尾 + for (size_t i = 0; i < size; ++i) { + StringView dst = containerInfo.mRawContainerInfo->mMounts[i].mDestination; + size_t dstSize = dst.size(); + + if (StartWith(logPath, dst) + && (pthSize == dstSize || (pthSize > dstSize && (logPath[dstSize] == '/' || logPath[dstSize] == '\\'))) + && (bestMatchedMountsIndex == size + || containerInfo.mRawContainerInfo->mMounts[bestMatchedMountsIndex].mDestination.size() < dstSize)) { + bestMatchedMountsIndex = i; + } + } + if (bestMatchedMountsIndex < size) { + outRealBaseDir = STRING_FLAG(default_container_host_path) + + containerInfo.mRawContainerInfo->mMounts[bestMatchedMountsIndex].mSource + + logPath.substr(containerInfo.mRawContainerInfo->mMounts[bestMatchedMountsIndex].mDestination.size()); + } else { + outRealBaseDir + = STRING_FLAG(default_container_host_path) + containerInfo.mRawContainerInfo->mUpperDir + logPath; + } + LOG_INFO(sLogger, + ("set container base dir for path", + logPath)("real dir", outRealBaseDir)("container id", containerInfo.mRawContainerInfo->mID)); + return true; +} bool FileDiscoveryOptions::UpdateRawContainerInfo(const std::shared_ptr& rawContainerInfo, const CollectionPipelineContext* ctx, diff --git a/core/file_server/FileDiscoveryOptions.h b/core/file_server/FileDiscoveryOptions.h index 5a66e86dc3..4b5425ecef 100644 --- a/core/file_server/FileDiscoveryOptions.h +++ b/core/file_server/FileDiscoveryOptions.h @@ -204,4 +204,12 @@ void BuildAndSortPathItems(const std::unordered_map& outSortedPaths, std::vector& outWildcardPaths); +// 为 ContainerInfo 设置多路径对应的容器真实路径(与 FileDiscoveryOptions::GetBasePathInfos 一一对应),供 input_file / +// input_static_file_onetime 等复用 +bool SetContainerBaseDirs(ContainerInfo& containerInfo, const FileDiscoveryOptions* fileDiscovery); +// 计算单个配置路径在容器中的映射路径 +bool SetContainerBaseDirForPath(const ContainerInfo& containerInfo, + const std::string& logPath, + std::string& outRealBaseDir); + } // namespace logtail diff --git a/core/file_server/FileServer.cpp b/core/file_server/FileServer.cpp index 5e77bfdc86..82173bc20e 100644 --- a/core/file_server/FileServer.cpp +++ b/core/file_server/FileServer.cpp @@ -101,13 +101,13 @@ void FileServer::PauseInner() { // 恢复文件服务,重新注册事件处理程序和恢复日志输入 void FileServer::Resume(bool isConfigUpdate, bool isContainerUpdate) { if (isConfigUpdate) { - if (ContainerManager::GetInstance()->CheckContainerDiffForAllConfig()) { - ContainerManager::GetInstance()->ApplyContainerDiffs(); + if (ContainerManager::GetInstance()->CheckFileServerContainerDiffs()) { + ContainerManager::GetInstance()->ApplyFileServerContainerDiffs(); ContainerManager::GetInstance()->SaveContainerInfo(); } } else { if (isContainerUpdate) { - ContainerManager::GetInstance()->ApplyContainerDiffs(); + ContainerManager::GetInstance()->ApplyFileServerContainerDiffs(); ContainerManager::GetInstance()->SaveContainerInfo(); } } diff --git a/core/file_server/StaticFileServer.cpp b/core/file_server/StaticFileServer.cpp index 9ecbeb0d90..8349294e21 100644 --- a/core/file_server/StaticFileServer.cpp +++ b/core/file_server/StaticFileServer.cpp @@ -14,9 +14,20 @@ #include "file_server/StaticFileServer.h" +#include + +#if defined(__linux__) +#include +#else +#include "common/StringTools.h" +#endif + #include "collection_pipeline/queue/ProcessQueueManager.h" #include "common/FileSystemUtil.h" #include "common/LogtailCommonFlags.h" +#include "common/StringTools.h" +#include "container_manager/ContainerManager.h" +#include "file_server/ContainerInfo.h" #include "file_server/checkpoint/CheckPointManager.h" #include "file_server/checkpoint/InputStaticFileCheckpointManager.h" #include "runner/ProcessorRunner.h" @@ -41,6 +52,7 @@ StaticFileServer::StaticFileServer() { } void StaticFileServer::Init() { + ContainerManager::GetInstance()->LoadContainerInfo(); InputStaticFileCheckpointManager::GetInstance()->GetAllCheckpointFileNames(); mIsThreadRunning = true; mThreadRes = async(launch::async, &StaticFileServer::Run, this); @@ -66,7 +78,7 @@ void StaticFileServer::Stop() { } bool StaticFileServer::HasRegisteredPlugins() const { - lock_guard lock(mUpdateMux); + ReadLock lock(mConfigReadWriteLock); return !mInputFileReaderConfigsMap.empty(); } @@ -81,11 +93,14 @@ void StaticFileServer::ClearUnusedCheckpoints() { void StaticFileServer::RemoveInput(const string& configName, size_t idx, bool keepingCheckpoint) { { lock_guard lock(mUpdateMux); - mInputFileDiscoveryConfigsMap.erase(make_pair(configName, idx)); - mInputFileReaderConfigsMap.erase(make_pair(configName, idx)); - mInputMultilineConfigsMap.erase(make_pair(configName, idx)); - mInputFileTagConfigsMap.erase(make_pair(configName, idx)); - mDeletedInputs.emplace(configName, idx); + WriteLock clock(mConfigReadWriteLock); + std::pair configInfo = make_pair(configName, idx); + mInputFileDiscoveryConfigsMap.erase(configInfo); + mFileContainerMetaMap.erase(configInfo); + mInputFileReaderConfigsMap.erase(configInfo); + mInputMultilineConfigsMap.erase(configInfo); + mInputFileTagConfigsMap.erase(configInfo); + mDeletedInputs.emplace(configInfo); } if (!keepingCheckpoint) { InputStaticFileCheckpointManager::GetInstance()->DeleteCheckpoint(configName, idx); @@ -94,11 +109,11 @@ void StaticFileServer::RemoveInput(const string& configName, size_t idx, bool ke void StaticFileServer::AddInput(const string& configName, size_t idx, - const optional>& files, FileDiscoveryOptions* fileDiscoveryOpts, const FileReaderOptions* fileReaderOpts, const MultilineOptions* multilineOpts, const FileTagOptions* fileTagOpts, + unordered_map& fileContainerMetas, const CollectionPipelineContext* ctx) { // check if the server is started, if not, start it { @@ -108,24 +123,20 @@ void StaticFileServer::AddInput(const string& configName, } } - // 安全获取Pipeline的时间值,避免空指针解引用 - uint32_t startTime = 0; - uint32_t expireTime = 0; - - if (ctx != nullptr && ctx->HasValidPipeline()) { - const auto& pipeline = ctx->GetPipeline(); - startTime = pipeline.GetOnetimeStartTime().value_or(0); - expireTime = pipeline.GetOnetimeExpireTime().value_or(0); + if (fileDiscoveryOpts->IsContainerDiscoveryEnabled()) { + ContainerManager::GetInstance()->Init(); } - InputStaticFileCheckpointManager::GetInstance()->CreateCheckpoint(configName, idx, files, startTime, expireTime); { lock_guard lock(mUpdateMux); - mInputFileDiscoveryConfigsMap.try_emplace(make_pair(configName, idx), make_pair(fileDiscoveryOpts, ctx)); - mInputFileReaderConfigsMap.try_emplace(make_pair(configName, idx), make_pair(fileReaderOpts, ctx)); - mInputMultilineConfigsMap.try_emplace(make_pair(configName, idx), make_pair(multilineOpts, ctx)); - mInputFileTagConfigsMap.try_emplace(make_pair(configName, idx), make_pair(fileTagOpts, ctx)); - mAddedInputs.emplace(configName, idx); + WriteLock clock(mConfigReadWriteLock); + std::pair configInfo = make_pair(configName, idx); + mInputFileDiscoveryConfigsMap.try_emplace(configInfo, make_pair(fileDiscoveryOpts, ctx)); + mFileContainerMetaMap.try_emplace(configInfo, fileContainerMetas); + mInputFileReaderConfigsMap.try_emplace(configInfo, make_pair(fileReaderOpts, ctx)); + mInputMultilineConfigsMap.try_emplace(configInfo, make_pair(multilineOpts, ctx)); + mInputFileTagConfigsMap.try_emplace(configInfo, make_pair(fileTagOpts, ctx)); + mAddedInputs.emplace(configInfo, ctx); } } @@ -133,12 +144,22 @@ void StaticFileServer::Run() { LOG_INFO(sLogger, ("static file server", "started")); unique_lock lock(mThreadRunningMux); time_t lastDumpCheckpointTime = time(nullptr); + time_t lastContainerCheckTime = time(nullptr); while (mIsThreadRunning) { lock.unlock(); UpdateInputs(); ReadFiles(); auto cur = time(nullptr); + if (cur - lastContainerCheckTime >= 3) { + if (ContainerManager::GetInstance()->IsReady()) { + if (ContainerManager::GetInstance()->CheckStaticFileServerContainerDiffs()) { + ContainerManager::GetInstance()->ApplyStaticFileServerContainerDiffs(); + } + ContainerManager::GetInstance()->GetStaticFileServerContainerStoppedEvents(); + } + lastContainerCheckTime = cur; + } if (cur - lastDumpCheckpointTime >= INT32_FLAG(input_static_file_checkpoint_dump_interval_sec)) { InputStaticFileCheckpointManager::GetInstance()->DumpAllCheckpointFiles(); lastDumpCheckpointTime = cur; @@ -152,20 +173,19 @@ void StaticFileServer::Run() { } void StaticFileServer::ReadFiles() { - for (auto& item : mPipelineNameReadersMap) { + lock_guard lock(mUpdateMux); + for (auto& item : mInputReadersMap) { { - lock_guard lock(mUpdateMux); - const auto& configName = item.first; - auto inputIdx = item.second.first; - if (mDeletedInputs.find(make_pair(configName, inputIdx)) != mDeletedInputs.end()) { + const auto& configInfo = item.first; + if (mDeletedInputs.find(configInfo) != mDeletedInputs.end()) { continue; } - auto& reader = item.second.second; + auto& reader = item.second; auto cur = chrono::system_clock::now(); while (chrono::system_clock::now() - cur < chrono::milliseconds(50)) { if (!reader) { - reader = GetNextAvailableReader(configName, inputIdx); + reader = GetNextAvailableReader(configInfo.first, configInfo.second); if (!reader) { break; } @@ -173,6 +193,34 @@ void StaticFileServer::ReadFiles() { bool skip = false; while (chrono::system_clock::now() - cur < chrono::milliseconds(50)) { + if (reader) { + FileFingerprint fingerprint; + if (InputStaticFileCheckpointManager::GetInstance()->GetCurrentFileFingerprint( + configInfo.first, configInfo.second, &fingerprint) + && !fingerprint.mContainerID.empty()) { + string currentID; + bool currentStopped = false; + if (GetCurrentContainerInfoForPath(configInfo.first, + configInfo.second, + reader->GetHostLogPath(), + ¤tID, + ¤tStopped)) { + if (currentID != fingerprint.mContainerID || currentStopped) { + LOG_INFO(sLogger, + ("abort reading: container changed or stopped", + "")("config", configInfo.first)("input idx", configInfo.second)( + "filepath", reader->GetHostLogPath())("checkpoint container id", + fingerprint.mContainerID)( + "current container id", currentID)("current stopped", currentStopped)); + InputStaticFileCheckpointManager::GetInstance()->InvalidateCurrentFileCheckpoint( + configInfo.first, configInfo.second); + reader = nullptr; + skip = true; + break; + } + } + } + } if (!ProcessQueueManager::GetInstance()->IsValidToPush(reader->GetQueueKey())) { skip = true; break; @@ -181,14 +229,15 @@ void StaticFileServer::ReadFiles() { auto logBuffer = make_unique(); bool moreData = reader->ReadLog(*logBuffer, nullptr, true); auto group = LogFileReader::GenerateEventGroup(reader, logBuffer.get()); - if (!ProcessorRunner::GetInstance()->PushQueue(reader->GetQueueKey(), inputIdx, std::move(group))) { + if (!ProcessorRunner::GetInstance()->PushQueue( + reader->GetQueueKey(), configInfo.second, std::move(group))) { // should not happend, since only one thread is pushing to the queue LOG_ERROR(sLogger, - ("failed to push to process queue", "discard data")("config", configName)( - "input idx", inputIdx)("filepath", reader->GetHostLogPath())); + ("failed to push to process queue", "discard data")("config", configInfo.first)( + "input idx", configInfo.second)("filepath", reader->GetHostLogPath())); } InputStaticFileCheckpointManager::GetInstance()->UpdateCurrentFileCheckpoint( - configName, inputIdx, reader->GetLastFilePos()); + configInfo.first, configInfo.second, reader->GetLastFilePos()); if (!moreData) { reader = nullptr; skip = true; @@ -201,7 +250,7 @@ void StaticFileServer::ReadFiles() { } } { - lock_guard lock(mThreadRunningMux); + lock_guard tlock(mThreadRunningMux); if (!mIsThreadRunning) { return; } @@ -287,28 +336,108 @@ LogFileReaderPtr StaticFileServer::GetNextAvailableReader(const string& configNa void StaticFileServer::UpdateInputs() { unique_lock lock(mUpdateMux); for (const auto& item : mDeletedInputs) { - mPipelineNameReadersMap.erase(item.first); + mInputReadersMap.erase(item); } mDeletedInputs.clear(); - for (const auto& item : mAddedInputs) { - mPipelineNameReadersMap.emplace(item.first, make_pair(item.second, LogFileReaderPtr())); + for (auto it = mAddedInputs.begin(); it != mAddedInputs.end();) { + const auto& configInfo = it->first; + const auto& ctx = it->second; + if (!ctx) { + it = mAddedInputs.erase(it); + continue; + } + + // 获取Pipeline的时间值 + uint32_t startTime = 0; + uint32_t expireTime = 0; + if (ctx->HasValidPipeline()) { + const auto& pipeline = ctx->GetPipeline(); + startTime = pipeline.GetOnetimeStartTime().value_or(0); + expireTime = pipeline.GetOnetimeExpireTime().value_or(0); + } + + // 获取文件列表 + optional> files; + auto* fileDiscoveryOpts = GetFileDiscoveryConfig(configInfo.first, configInfo.second).first; + if (fileDiscoveryOpts->IsContainerDiscoveryEnabled()) { + if (!ContainerManager::GetInstance()->IsReady()) { + // 如果容器发现模块未准备好,跳过本次,等待下次重试 + ++it; + continue; + } + // 刷新容器信息 + if (ContainerManager::GetInstance()->CheckStaticFileServerContainerDiffs()) { + ContainerManager::GetInstance()->ApplyStaticFileServerContainerDiffs(); + } + ContainerManager::GetInstance()->GetStaticFileServerContainerStoppedEvents(); + } + { + ReadLock clk(mConfigReadWriteLock); + auto fileContainerMetas = mFileContainerMetaMap.find(configInfo)->second; + if (!ctx->IsOnetimePipelineRunningBeforeStart()) { + files = GetFiles(fileDiscoveryOpts, ctx, fileContainerMetas); + } + InputStaticFileCheckpointManager::GetInstance()->CreateCheckpoint( + configInfo.first, configInfo.second, files, startTime, expireTime, fileContainerMetas); + } + mInputReadersMap.emplace(configInfo, LogFileReaderPtr()); + it = mAddedInputs.erase(it); } - mAddedInputs.clear(); - SET_GAUGE(mActiveInputsTotalGauge, mPipelineNameReadersMap.size()); + SET_GAUGE(mActiveInputsTotalGauge, mInputReadersMap.size()); } -FileDiscoveryConfig StaticFileServer::GetFileDiscoveryConfig(const std::string& name, size_t idx) const { - auto it = mInputFileDiscoveryConfigsMap.find(make_pair(name, idx)); - if (it == mInputFileDiscoveryConfigsMap.end()) { - // should not happen - return make_pair(nullptr, nullptr); +bool StaticFileServer::GetCurrentContainerInfoForPath( + const string& configName, size_t idx, const string& hostPath, string* outContainerID, bool* outStopped) const { + if (!outContainerID || !outStopped) { + return false; } - return it->second; + *outContainerID = ""; + *outStopped = true; + FileDiscoveryConfig config = GetFileDiscoveryConfig(configName, idx); + if (!config.first) { + return false; + } + if (!config.first->IsContainerDiscoveryEnabled()) { + return false; + } + const auto& containerInfos = config.first->GetContainerInfo(); + if (!containerInfos || containerInfos->empty()) { + return true; + } + for (const auto& info : *containerInfos) { + if (!info.mRawContainerInfo) { + continue; + } + for (const auto& realBaseDir : info.mRealBaseDirs) { + if (realBaseDir.empty()) { + continue; + } + if (hostPath.size() >= realBaseDir.size() && hostPath.compare(0, realBaseDir.size(), realBaseDir) == 0 + && (realBaseDir.size() == hostPath.size() || hostPath[realBaseDir.size()] == '/')) { + *outContainerID = info.mRawContainerInfo->mID; + *outStopped = info.mRawContainerInfo->mStopped; + return true; + } + } + } + return true; +} + +FileDiscoveryConfig StaticFileServer::GetFileDiscoveryConfig(const std::string& name, size_t idx) const { + FileDiscoveryConfig result = make_pair(nullptr, nullptr); + WithFileDiscoveryConfigs([&](const auto& discMap) { + auto it = discMap.find(make_pair(name, idx)); + if (it != discMap.end()) { + result = it->second; + } + }); + return result; } FileReaderConfig StaticFileServer::GetFileReaderConfig(const std::string& name, size_t idx) const { + ReadLock lock(mConfigReadWriteLock); auto it = mInputFileReaderConfigsMap.find(make_pair(name, idx)); if (it == mInputFileReaderConfigsMap.end()) { // should not happen @@ -318,6 +447,7 @@ FileReaderConfig StaticFileServer::GetFileReaderConfig(const std::string& name, } MultilineConfig StaticFileServer::GetMultilineConfig(const std::string& name, size_t idx) const { + ReadLock lock(mConfigReadWriteLock); auto it = mInputMultilineConfigsMap.find(make_pair(name, idx)); if (it == mInputMultilineConfigsMap.end()) { // should not happen @@ -327,6 +457,7 @@ MultilineConfig StaticFileServer::GetMultilineConfig(const std::string& name, si } FileTagConfig StaticFileServer::GetFileTagConfig(const std::string& name, size_t idx) const { + ReadLock lock(mConfigReadWriteLock); auto it = mInputFileTagConfigsMap.find(make_pair(name, idx)); if (it == mInputFileTagConfigsMap.end()) { // should not happen @@ -335,15 +466,231 @@ FileTagConfig StaticFileServer::GetFileTagConfig(const std::string& name, size_t return it->second; } +vector +StaticFileServer::GetFiles(const FileDiscoveryOptions* fileDiscoveryOpts, + const CollectionPipelineContext* ctx, + std::unordered_map& fileContainerMetas) { + vector res; + const auto& pathInfos = fileDiscoveryOpts->GetBasePathInfos(); + fileContainerMetas.clear(); + + if (!fileDiscoveryOpts->IsContainerDiscoveryEnabled()) { + set visitedDirs; + // Process each path configuration + for (const auto& pathInfo : pathInfos) { + vector baseDirs; + if (pathInfo.hasWildcard()) { + GetValidBaseDirs(pathInfo, pathInfo.wildcardPaths[0], 0, baseDirs); + if (baseDirs.empty()) { + LOG_WARNING(sLogger, + ("no files found", "base dir path invalid")("base dir", pathInfo.basePath)( + "config", ctx->GetConfigName())); + continue; + } + } else { + baseDirs.emplace_back(pathInfo.basePath); + } + + for (const auto& dir : baseDirs) { + if (IsValidDir(dir)) { + GetFiles(fileDiscoveryOpts, + dir, + fileDiscoveryOpts->mMaxDirSearchDepth, + &pathInfo, + nullptr, + visitedDirs, + res); + } + } + } + LOG_INFO(sLogger, ("total files cnt", res.size())("files", ToString(res))("config", ctx->GetConfigName())); + } else { + // TODO: support symlink in container + set visitedDirs; + for (const auto& item : *fileDiscoveryOpts->GetContainerInfo()) { + const string& containerID = item.mRawContainerInfo->mID; + // 遍历配置路径和对应的容器真实路径 + for (size_t idx = 0; idx < pathInfos.size(); ++idx) { + if (idx >= item.mRealBaseDirs.size()) + break; + + const auto& pathInfo = pathInfos[idx]; + const string& realBaseDir = item.mRealBaseDirs[idx]; + + if (realBaseDir.empty()) + continue; + + vector baseDirs; + if (pathInfo.hasWildcard()) { + GetValidBaseDirs(pathInfo, realBaseDir, 0, baseDirs); + if (baseDirs.empty()) { + LOG_DEBUG(sLogger, + ("no files found", "base dir path invalid")("container id", containerID)( + "real base dir", realBaseDir)("config", ctx->GetConfigName())); + continue; + } + } else { + baseDirs.emplace_back(realBaseDir); + } + + auto prevCnt = res.size(); + for (const auto& dir : baseDirs) { + if (IsValidDir(dir)) { + GetFiles(fileDiscoveryOpts, + dir, + fileDiscoveryOpts->mMaxDirSearchDepth, + &pathInfo, + &realBaseDir, + visitedDirs, + res); + } + } + // 记录新发现的文件与容器元信息(FileCheckpoint::ContainerMeta,含容器内路径) + const auto* raw = item.mRawContainerInfo.get(); + for (size_t i = prevCnt; i < res.size(); ++i) { + const string& hostPath = res[i].string(); + FileCheckpoint::ContainerMeta meta; + meta.mContainerID = raw->mID; + meta.mContainerName = raw->mName; + meta.mPodName = raw->mK8sInfo.mPod; + meta.mNamespace = raw->mK8sInfo.mNamespace; + meta.mContainerPath = pathInfo.basePath + hostPath.substr(realBaseDir.size()); + fileContainerMetas[hostPath] = std::move(meta); + } + if (res.size() > prevCnt) { + LOG_INFO(sLogger, + ("container files cnt", res.size() - prevCnt)("container id", containerID)( + "real base dir", realBaseDir)("files", ToString(res))("config", ctx->GetConfigName())); + } else { + LOG_DEBUG(sLogger, + ("no files found, container id", + containerID)("real base dir", realBaseDir)("config", ctx->GetConfigName())); + } + } + } + LOG_INFO(sLogger, ("total files cnt", res.size())("config", ctx->GetConfigName())); + } + return res; +} + +void StaticFileServer::GetFiles(const FileDiscoveryOptions* fileDiscoveryOpts, + const filesystem::path& dir, + uint32_t depth, + const BasePathInfo* pathInfo, + const std::string* containerBaseDir, + std::set& visitedDir, + std::vector& files) { + error_code ec; + for (auto const& entry : filesystem::directory_iterator(dir, ec)) { + const auto& path = entry.path(); + auto pathStr = path.string(); + if (containerBaseDir && pathInfo) { + // Map container path to config path + pathStr = pathInfo->basePath + pathStr.substr(containerBaseDir->size()); + } + const auto& status = entry.status(); + if (filesystem::is_regular_file(status)) { + const auto& filename = path.filename().string(); + if (pathInfo) { + // Use the specific pathInfo's filePattern if provided, otherwise check all patterns + bool filenameMatched = fileDiscoveryOpts->IsFilenameMatched(filename, *pathInfo); + + if (filenameMatched && !fileDiscoveryOpts->IsFilenameInBlacklist(filename) + && !fileDiscoveryOpts->IsFilepathInBlacklist(pathStr)) { + files.emplace_back(path); + } + } + } else if (filesystem::is_directory(status)) { + auto devInode = GetFileDevInode(path.string()); + if (!devInode.IsValid() || visitedDir.find(devInode) != visitedDir.end()) { + // avoid loop + continue; + } + visitedDir.emplace(devInode); + if (depth > 0 && !AppConfig::GetInstance()->IsHostPathMatchBlacklist(path.string()) + && !fileDiscoveryOpts->IsDirectoryInBlacklist(pathStr)) { + GetFiles(fileDiscoveryOpts, path, depth - 1, pathInfo, containerBaseDir, visitedDir, files); + } + } + } +} + +void StaticFileServer::GetValidBaseDirs(const BasePathInfo& pathInfo, + const filesystem::path& dir, + uint32_t depth, + vector& filepaths) { + const auto& wildcardPaths = pathInfo.wildcardPaths; + bool finish = false; + if (depth + 2 == wildcardPaths.size()) { + finish = true; + } + + if (depth == 0 && !IsValidDir(wildcardPaths[depth])) { + return; + } + + const auto& subdir = pathInfo.constWildcardPaths[depth]; + if (!subdir.empty()) { + auto path = dir / subdir; + error_code ec; + filesystem::file_status s = filesystem::status(path, ec); + if (ec || !filesystem::exists(s) || !filesystem::is_directory(s)) { + return; + } + if (finish) { + filepaths.emplace_back(path); + } else { + GetValidBaseDirs(pathInfo, path, depth + 1, filepaths); + } + } else { + auto pattern = filesystem::path(wildcardPaths[depth + 1]).filename(); + error_code ec; + for (auto const& entry : filesystem::directory_iterator(dir, ec)) { + const auto& path = entry.path(); + const auto& status = entry.status(); + if (filesystem::is_directory(status) + && (fnmatch(pattern.string().c_str(), path.filename().string().c_str(), FNM_PATHNAME) == 0)) { + if (finish) { + filepaths.emplace_back(path); + } else { + GetValidBaseDirs(pathInfo, path, depth + 1, filepaths); + } + } + } + } +} + +bool StaticFileServer::IsValidDir(const filesystem::path& dir) { + error_code ec; + filesystem::file_status s = filesystem::status(dir, ec); + if (ec) { + LOG_WARNING(sLogger, + ("failed to get base dir path info", + "skip")("dir path", dir.string())("error code", ec.value())("error msg", ec.message())); + return false; + } + if (!filesystem::exists(s)) { + LOG_WARNING(sLogger, ("base dir path not existed", "skip")("dir path", dir.string())); + return false; + } + if (!filesystem::is_directory(s)) { + LOG_WARNING(sLogger, ("base dir path is not a directory", "skip")("dir path", dir.string())); + return false; + } + return true; +} + #ifdef APSARA_UNIT_TEST_MAIN void StaticFileServer::Clear() { Stop(); lock_guard lock(mUpdateMux); + WriteLock clock(mConfigReadWriteLock); mInputFileDiscoveryConfigsMap.clear(); + mFileContainerMetaMap.clear(); mInputFileReaderConfigsMap.clear(); mInputMultilineConfigsMap.clear(); mInputFileTagConfigsMap.clear(); - mPipelineNameReadersMap.clear(); + mInputReadersMap.clear(); mAddedInputs.clear(); mDeletedInputs.clear(); } diff --git a/core/file_server/StaticFileServer.h b/core/file_server/StaticFileServer.h index b61a96adfc..70b6bac6ff 100644 --- a/core/file_server/StaticFileServer.h +++ b/core/file_server/StaticFileServer.h @@ -16,20 +16,26 @@ #pragma once +#include + #include #include #include #include +#include #include #include #include #include +#include #include #include "collection_pipeline/CollectionPipelineContext.h" +#include "common/Lock.h" #include "file_server/FileDiscoveryOptions.h" #include "file_server/FileTagOptions.h" #include "file_server/MultilineOptions.h" +#include "file_server/checkpoint/FileCheckpoint.h" #include "file_server/reader/FileReaderOptions.h" #include "file_server/reader/LogFileReader.h" #include "monitor/MetricManager.h" @@ -56,14 +62,31 @@ class StaticFileServer : public InputRunner { void AddInput(const std::string& configName, size_t idx, - const std::optional>& files, FileDiscoveryOptions* fileDiscoveryOpts, const FileReaderOptions* fileReaderOpts, const MultilineOptions* multilineOpts, const FileTagOptions* fileTagOpts, + std::unordered_map& fileContainerMetas, const CollectionPipelineContext* ctx); void RemoveInput(const std::string& configName, size_t idx, bool keepingCheckpoint = false); + const std::map, FileDiscoveryConfig>& GetAllFileDiscoveryConfigs() const { + return mInputFileDiscoveryConfigsMap; + } + + // Config maps only (same idea as FileServer::WithFileDiscoveryConfigs*). + template + void WithFileDiscoveryConfigs(Func&& fn) const { + ReadLock lock(mConfigReadWriteLock); + fn(mInputFileDiscoveryConfigsMap); + } + + template + void WithFileDiscoveryConfigsMutable(Func&& fn) { + WriteLock lock(mConfigReadWriteLock); + fn(mInputFileDiscoveryConfigsMap); + } + #ifdef APSARA_UNIT_TEST_MAIN void Clear(); #endif @@ -82,6 +105,32 @@ class StaticFileServer : public InputRunner { MultilineConfig GetMultilineConfig(const std::string& name, size_t idx) const; FileTagConfig GetFileTagConfig(const std::string& name, size_t idx) const; + // If container discovery is enabled, finds the container that owns hostPath and fills outContainerID/outStopped. + // Returns true when container discovery is enabled (caller should validate); outContainerID empty and outStopped + // true means path is no longer under any container (e.g. container removed). + bool GetCurrentContainerInfoForPath(const std::string& configName, + size_t idx, + const std::string& hostPath, + std::string* outContainerID, + bool* outStopped) const; + + static std::vector + GetFiles(const FileDiscoveryOptions* fileDiscoveryOpts, + const CollectionPipelineContext* ctx, + std::unordered_map& fileContainerMetas); + static void GetFiles(const FileDiscoveryOptions* fileDiscoveryOpts, + const std::filesystem::path& dir, + uint32_t depth, + const BasePathInfo* pathInfo, + const std::string* containerBaseDir, + std::set& visitedDir, + std::vector& files); + static bool IsValidDir(const std::filesystem::path& dir); + static void GetValidBaseDirs(const BasePathInfo& pathInfo, + const std::filesystem::path& dir, + uint32_t depth, + std::vector& filepaths); + std::future mThreadRes; mutable std::mutex mThreadRunningMux; bool mIsThreadRunning = false; @@ -94,16 +143,21 @@ class StaticFileServer : public InputRunner { time_t mStartTime = 0; bool mIsUnusedCheckpointsCleared = false; - std::multimap> mPipelineNameReadersMap; - - // accessed by main thread and input runner thread + // Runner state: multimap / mAddedInputs / mDeletedInputs. + // Lock order when both needed: mUpdateMux first, then mConfigReadWriteLock. mutable std::mutex mUpdateMux; + std::multimap, LogFileReaderPtr> mInputReadersMap; + std::multimap, const CollectionPipelineContext*> mAddedInputs; + std::set> mDeletedInputs; + + // Plugin / config registration (same pattern as FileServer::mReadWriteLock). + mutable ReadWriteLock mConfigReadWriteLock; std::map, FileDiscoveryConfig> mInputFileDiscoveryConfigsMap; + std::map, std::unordered_map> + mFileContainerMetaMap; std::map, FileReaderConfig> mInputFileReaderConfigsMap; std::map, MultilineConfig> mInputMultilineConfigsMap; std::map, FileTagConfig> mInputFileTagConfigsMap; - std::multimap mAddedInputs; - std::set> mDeletedInputs; #ifdef APSARA_UNIT_TEST_MAIN friend class StaticFileServerUnittest; diff --git a/core/file_server/checkpoint/CheckPointManager.h b/core/file_server/checkpoint/CheckPointManager.h index 00a270cca9..07f9a316d6 100644 --- a/core/file_server/checkpoint/CheckPointManager.h +++ b/core/file_server/checkpoint/CheckPointManager.h @@ -157,7 +157,6 @@ class CheckPointManager { } #ifdef APSARA_UNIT_TEST_MAIN - friend class ConfigUpdatorUnittest; void RemoveLocalCheckPoint(); void PrintStatus(); #endif diff --git a/core/file_server/checkpoint/CheckpointManagerV2.h b/core/file_server/checkpoint/CheckpointManagerV2.h index fd93d1d78b..bc7821ec6c 100644 --- a/core/file_server/checkpoint/CheckpointManagerV2.h +++ b/core/file_server/checkpoint/CheckpointManagerV2.h @@ -182,7 +182,6 @@ class CheckpointManagerV2 { #ifdef APSARA_UNIT_TEST_MAIN friend class CheckpointManagerV2Unittest; friend class ExactlyOnceReaderUnittest; - friend class SenderUnittest; void rebuild(); #endif diff --git a/core/file_server/checkpoint/FileCheckpoint.h b/core/file_server/checkpoint/FileCheckpoint.h index 2ef3e182a5..89d2d67990 100644 --- a/core/file_server/checkpoint/FileCheckpoint.h +++ b/core/file_server/checkpoint/FileCheckpoint.h @@ -37,8 +37,16 @@ const std::string& FileStatusToString(FileStatus status); FileStatus GetFileStatusFromString(const std::string& status); struct FileCheckpoint { + // 容器场景下与文件绑定的元信息 + struct ContainerMeta { + std::string mContainerID; + std::string mContainerName; + std::string mPodName; + std::string mNamespace; + std::string mContainerPath; // 容器内路径 + }; + std::filesystem::path mFilePath; - // std::string mRealFileName; DevInode mDevInode; uint64_t mSignatureHash = 0; uint32_t mSignatureSize = 0; @@ -47,18 +55,21 @@ struct FileCheckpoint { FileStatus mStatus = FileStatus::WAITING; int32_t mStartTime = 0; int32_t mLastUpdateTime = 0; + ContainerMeta mContainerMeta; // 容器场景下使用,非容器时各字段为空 FileCheckpoint() = default; FileCheckpoint(const std::filesystem::path& filename, const DevInode& devInode, uint64_t signatureHash, uint32_t signatureSize, - uint64_t initialSize = 0) + uint64_t initialSize = 0, + const ContainerMeta& containerMeta = {}) : mFilePath(filename), mDevInode(devInode), mSignatureHash(signatureHash), mSignatureSize(signatureSize), - mSize(initialSize) {} + mSize(initialSize), + mContainerMeta(containerMeta) {} }; struct FileFingerprint { @@ -68,6 +79,7 @@ struct FileFingerprint { uint64_t mSignatureHash; uint64_t mSize = 0; // 初始文件大小,用于限制 StaticFileServer reader 的读取范围 uint64_t mOffset = 0; + std::string mContainerID; // 容器场景下当前文件所属容器 ID,非容器为空;用于读文件时校验容器是否变更/停止 }; } // namespace logtail diff --git a/core/file_server/checkpoint/InputStaticFileCheckpoint.cpp b/core/file_server/checkpoint/InputStaticFileCheckpoint.cpp index 9081cb4850..f4a77062a3 100644 --- a/core/file_server/checkpoint/InputStaticFileCheckpoint.cpp +++ b/core/file_server/checkpoint/InputStaticFileCheckpoint.cpp @@ -162,6 +162,7 @@ bool InputStaticFileCheckpoint::GetCurrentFileFingerprint(FileFingerprint* cpt) cpt->mSignatureSize = fileCpt.mSignatureSize; cpt->mSize = fileCpt.mSize; cpt->mOffset = fileCpt.mOffset; + cpt->mContainerID = fileCpt.mContainerMeta.mContainerID; return true; } @@ -201,6 +202,21 @@ bool InputStaticFileCheckpoint::Serialize(string* res) const { file["dev"] = cpt.mDevInode.dev; file["inode"] = cpt.mDevInode.inode; file["size"] = cpt.mSize; + if (!cpt.mContainerMeta.mContainerID.empty()) { + file["container_id"] = cpt.mContainerMeta.mContainerID; + } + if (!cpt.mContainerMeta.mContainerName.empty()) { + file["container_name"] = cpt.mContainerMeta.mContainerName; + } + if (!cpt.mContainerMeta.mPodName.empty()) { + file["pod_name"] = cpt.mContainerMeta.mPodName; + } + if (!cpt.mContainerMeta.mNamespace.empty()) { + file["namespace"] = cpt.mContainerMeta.mNamespace; + } + if (!cpt.mContainerMeta.mContainerPath.empty()) { + file["container_path"] = cpt.mContainerMeta.mContainerPath; + } switch (cpt.mStatus) { case FileStatus::WAITING: file["sig_hash"] = cpt.mSignatureHash; @@ -375,6 +391,17 @@ bool InputStaticFileCheckpoint::Deserialize(const string& str, string* errMsg) { *errMsg = "mandatory string param " + outerKey + ".status is not valid"; return false; } + // 读取可选的容器元信息 + if (GetOptionalStringParam(fileCpt, outerKey + ".container_id", cpt.mContainerMeta.mContainerID, *errMsg)) { + } + if (GetOptionalStringParam(fileCpt, outerKey + ".container_name", cpt.mContainerMeta.mContainerName, *errMsg)) { + } + if (GetOptionalStringParam(fileCpt, outerKey + ".pod_name", cpt.mContainerMeta.mPodName, *errMsg)) { + } + if (GetOptionalStringParam(fileCpt, outerKey + ".namespace", cpt.mContainerMeta.mNamespace, *errMsg)) { + } + if (GetOptionalStringParam(fileCpt, outerKey + ".container_path", cpt.mContainerMeta.mContainerPath, *errMsg)) { + } mFileCheckpoints.emplace_back(std::move(cpt)); } @@ -393,6 +420,21 @@ string buildFileInfoJson(const FileCheckpoint& cpt) { fileInfo["sig_hash"] = cpt.mSignatureHash; fileInfo["sig_size"] = cpt.mSignatureSize; fileInfo["size"] = cpt.mSize; + if (!cpt.mContainerMeta.mContainerID.empty()) { + fileInfo["container_id"] = cpt.mContainerMeta.mContainerID; + } + if (!cpt.mContainerMeta.mContainerName.empty()) { + fileInfo["container_name"] = cpt.mContainerMeta.mContainerName; + } + if (!cpt.mContainerMeta.mPodName.empty()) { + fileInfo["pod_name"] = cpt.mContainerMeta.mPodName; + } + if (!cpt.mContainerMeta.mNamespace.empty()) { + fileInfo["namespace"] = cpt.mContainerMeta.mNamespace; + } + if (!cpt.mContainerMeta.mContainerPath.empty()) { + fileInfo["container_path"] = cpt.mContainerMeta.mContainerPath; + } switch (cpt.mStatus) { case FileStatus::WAITING: @@ -483,6 +525,21 @@ bool InputStaticFileCheckpoint::SerializeToLogEvents() const { logEvent->SetContent("finish_time", ToString(mFinishTime)); } logEvent->SetContent("file_info", buildFileInfoJson(cpt)); + if (!cpt.mContainerMeta.mContainerID.empty()) { + logEvent->SetContent("container_id", cpt.mContainerMeta.mContainerID); + } + if (!cpt.mContainerMeta.mContainerName.empty()) { + logEvent->SetContent("container_name", cpt.mContainerMeta.mContainerName); + } + if (!cpt.mContainerMeta.mPodName.empty()) { + logEvent->SetContent("pod_name", cpt.mContainerMeta.mPodName); + } + if (!cpt.mContainerMeta.mNamespace.empty()) { + logEvent->SetContent("namespace", cpt.mContainerMeta.mNamespace); + } + if (!cpt.mContainerMeta.mContainerPath.empty()) { + logEvent->SetContent("container_path", cpt.mContainerMeta.mContainerPath); + } if (cpt.mStatus == FileStatus::FINISHED || cpt.mStatus == FileStatus::ABORT) { mLastSentIndex = i + 1; diff --git a/core/file_server/checkpoint/InputStaticFileCheckpointManager.cpp b/core/file_server/checkpoint/InputStaticFileCheckpointManager.cpp index 3a8623d899..b4a204a99b 100644 --- a/core/file_server/checkpoint/InputStaticFileCheckpointManager.cpp +++ b/core/file_server/checkpoint/InputStaticFileCheckpointManager.cpp @@ -39,11 +39,13 @@ InputStaticFileCheckpointManager::InputStaticFileCheckpointManager() } } -bool InputStaticFileCheckpointManager::CreateCheckpoint(const string& configName, - size_t idx, - const optional>& files, - uint32_t startTime, - uint32_t expireTime) { +bool InputStaticFileCheckpointManager::CreateCheckpoint( + const string& configName, + size_t idx, + const optional>& files, + uint32_t startTime, + uint32_t expireTime, + const std::unordered_map& fileContainerMetas) { if (!files.has_value()) { InputStaticFileCheckpoint cpt; if (RetrieveCheckpointFromFile(configName, idx, &cpt)) { @@ -122,11 +124,20 @@ bool InputStaticFileCheckpointManager::CreateCheckpoint(const string& configName signature.resize(is.gcount()); auto sigHash = static_cast(HashSignatureString(signature.c_str(), signature.size())); - fileCpts.emplace_back(file, devInode, sigHash, signature.size(), initialSize); + // 查找文件对应的容器元信息 + FileCheckpoint::ContainerMeta containerMeta; + if (!fileContainerMetas.empty()) { + auto cit = fileContainerMetas.find(file.string()); + if (cit != fileContainerMetas.end()) { + containerMeta = cit->second; + } + } + fileCpts.emplace_back(file, devInode, sigHash, signature.size(), initialSize, containerMeta); LOG_INFO(sLogger, - ("create file checkpoint succeeded, config", - configName)("input idx", idx)("filepath", file)("device", devInode.dev)("inode", devInode.inode)( - "signature hash", sigHash)("signature size", signature.size())("initial size", initialSize)); + ("create file checkpoint succeeded, config", configName)("input idx", idx)("filepath", file)( + "device", devInode.dev)("inode", devInode.inode)("signature hash", sigHash)( + "signature size", signature.size())("initial size", initialSize)("container id", + containerMeta.mContainerID)); } LOG_INFO(sLogger, ("create checkpoint succeeded, config", configName)("input idx", idx)("file count", fileCpts.size())); diff --git a/core/file_server/checkpoint/InputStaticFileCheckpointManager.h b/core/file_server/checkpoint/InputStaticFileCheckpointManager.h index f62a6e3054..a88cfe39f1 100644 --- a/core/file_server/checkpoint/InputStaticFileCheckpointManager.h +++ b/core/file_server/checkpoint/InputStaticFileCheckpointManager.h @@ -20,14 +20,17 @@ #include #include +#include #include #include #include #include +#include #include #include "json/json.h" +#include "file_server/checkpoint/FileCheckpoint.h" #include "file_server/checkpoint/InputStaticFileCheckpoint.h" namespace logtail { @@ -46,7 +49,9 @@ class InputStaticFileCheckpointManager { size_t idx, const std::optional>& files = std::nullopt, uint32_t startTime = 0, - uint32_t expireTime = 0); + uint32_t expireTime = 0, + const std::unordered_map& fileContainerMetas + = {}); bool DeleteCheckpoint(const std::string& configName, size_t idx); bool UpdateCurrentFileCheckpoint(const std::string& configName, size_t idx, uint64_t offset); bool InvalidateCurrentFileCheckpoint(const std::string& configName, size_t idx); diff --git a/core/file_server/event/Event.h b/core/file_server/event/Event.h index 3201acb392..92f7eb8f77 100644 --- a/core/file_server/event/Event.h +++ b/core/file_server/event/Event.h @@ -47,6 +47,7 @@ class Event { uint64_t mInode; int64_t mHashKey; std::string mConfigName; + std::string mInputIndex; std::string mContainerID; // for read timeout @@ -100,7 +101,7 @@ class Event { int GetWd() const { return mWd; } - const uint32_t GetCookie() const { return mCookie; } + uint32_t GetCookie() const { return mCookie; } int64_t GetHashKey() { return mHashKey; } @@ -122,6 +123,8 @@ class Event { void SetConfigName(const std::string& configName) { mConfigName = configName; } + void SetInputIndex(const std::string& inputIndex) { mInputIndex = inputIndex; } + void SetContainerID(const std::string& containerID) { mContainerID = containerID; } void SetLastReadPos(int64_t lastReadPos) { mLastReadPos = lastReadPos; } diff --git a/core/file_server/event_handler/EventHandler.h b/core/file_server/event_handler/EventHandler.h index 49e7904087..d1aee45059 100644 --- a/core/file_server/event_handler/EventHandler.h +++ b/core/file_server/event_handler/EventHandler.h @@ -123,9 +123,7 @@ class ModifyHandler : public EventHandler { const std::string& GetConfigName() const { return mConfigName; } #ifdef APSARA_UNIT_TEST_MAIN - friend class ConfigUpdatorUnittest; friend class EventDispatcherTest; - friend class SenderUnittest; friend class ModifyHandlerUnittest; friend class ForceReadUnittest; friend class CreateModifyHandlerUnittest; @@ -183,9 +181,7 @@ class CreateModifyHandler : public EventHandler { #ifdef APSARA_UNIT_TEST_MAIN friend class CreateModifyHandlerUnittest; - friend class ConfigUpdatorUnittest; friend class EventDispatcherTest; - friend class SenderUnittest; friend class EventDispatcherContainerUnittest; friend class LogInputReaderUnittest; #endif diff --git a/core/file_server/event_handler/LogInput.cpp b/core/file_server/event_handler/LogInput.cpp index 7a5bbf0976..1fa6a3be6c 100644 --- a/core/file_server/event_handler/LogInput.cpp +++ b/core/file_server/event_handler/LogInput.cpp @@ -145,7 +145,7 @@ void LogInput::TryReadEvents(bool forceRead) { } std::vector containerStoppedEvents; - ConfigManager::GetInstance()->GetContainerStoppedEvents(containerStoppedEvents); + ConfigManager::GetInstance()->GetFileServerContainerStoppedEvents(containerStoppedEvents); if (containerStoppedEvents.size() > 0) { PushEventQueue(containerStoppedEvents); } diff --git a/core/file_server/event_handler/LogInput.h b/core/file_server/event_handler/LogInput.h index 5c714a2014..6713846d6a 100644 --- a/core/file_server/event_handler/LogInput.h +++ b/core/file_server/event_handler/LogInput.h @@ -97,9 +97,7 @@ class LogInput : public LogRunnable { #ifdef APSARA_UNIT_TEST_MAIN friend class LogInputUnittest; friend class EventDispatcherTest; - friend class ConfigUpdatorUnittest; friend class SymlinkInotifyTest; - friend class SenderUnittest; friend class FuxiSceneUnittest; friend class ConfigMatchUnittest; friend class FuseFileUnittest; diff --git a/core/file_server/reader/LogFileReader.cpp b/core/file_server/reader/LogFileReader.cpp index 4a378b2549..ea0f0769e7 100644 --- a/core/file_server/reader/LogFileReader.cpp +++ b/core/file_server/reader/LogFileReader.cpp @@ -31,7 +31,6 @@ #include #include -#include "boost/filesystem.hpp" #include "boost/regex.hpp" #include "rapidjson/document.h" @@ -511,7 +510,7 @@ bool LogFileReader::validatePrimaryCheckpoint(const PrimaryCheckpointPB& cpt) { return false; } - auto dirPath = boost::filesystem::path(filePath).parent_path(); + auto dirPath = std::filesystem::path(filePath).parent_path(); const auto searchResult = SearchFilePathByDevInodeInDirectory(dirPath.string(), 0, mDevInode, nullptr); if (!searchResult) { LOG_WARNING(sLogger, METHOD_LOG_PATTERN("can not find file with dev inode", mDevInode.inode)); diff --git a/core/file_server/reader/LogFileReader.h b/core/file_server/reader/LogFileReader.h index bedded7656..efc7cc1d0c 100644 --- a/core/file_server/reader/LogFileReader.h +++ b/core/file_server/reader/LogFileReader.h @@ -712,7 +712,6 @@ class LogFileReader { friend class LogFileReaderUnittest; friend class LogMultiBytesUnittest; friend class ExactlyOnceReaderUnittest; - friend class SenderUnittest; friend class AppConfigUnittest; friend class ModifyHandlerUnittest; friend class LogSplitUnittest; diff --git a/core/host_monitor/collector/ProcessCollector.h b/core/host_monitor/collector/ProcessCollector.h index 761bc98d4f..c017e27577 100644 --- a/core/host_monitor/collector/ProcessCollector.h +++ b/core/host_monitor/collector/ProcessCollector.h @@ -13,7 +13,6 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -#include #include #include "common/ProcParser.h" diff --git a/core/host_monitor/collector/ProcessEntityCollector.h b/core/host_monitor/collector/ProcessEntityCollector.h index b15619b696..16ff59cc2f 100644 --- a/core/host_monitor/collector/ProcessEntityCollector.h +++ b/core/host_monitor/collector/ProcessEntityCollector.h @@ -21,7 +21,6 @@ #include #include -#include #include #include #include diff --git a/core/logger/Logger.cpp b/core/logger/Logger.cpp index 702c313bd6..d1d03dd48a 100644 --- a/core/logger/Logger.cpp +++ b/core/logger/Logger.cpp @@ -16,7 +16,6 @@ #include -#include "boost/filesystem.hpp" #include "json/json.h" #include "spdlog/async.h" #include "spdlog/sinks/rotating_file_sink.h" @@ -426,10 +425,10 @@ void Logger::EnsureSnapshotDirExist(std::map& sinkCfgs) SinkConfig sinkCfg = sink.second; if (sinkName == "AsyncFileSinkStatus" || sinkName == "AsyncFileSinkProfile") { - boost::filesystem::path logFilePath(sinkCfg.logFilePath); + std::filesystem::path logFilePath(sinkCfg.logFilePath); try { - boost::filesystem::create_directories(logFilePath.parent_path()); - } catch (const boost::filesystem::filesystem_error& e) { + std::filesystem::create_directories(logFilePath.parent_path()); + } catch (const std::filesystem::filesystem_error& e) { LogMsg(std::string("Create snapshot dir error ") + logFilePath.parent_path().string() + ", error" + std::string(e.what())); } diff --git a/core/logtail.cpp b/core/logtail.cpp index e2a1a5ec4d..d1bc927025 100644 --- a/core/logtail.cpp +++ b/core/logtail.cpp @@ -44,7 +44,6 @@ DECLARE_FLAG_STRING(logtail_sys_conf_dir); DECLARE_FLAG_STRING(check_point_filename); DECLARE_FLAG_STRING(default_buffer_file_path); DECLARE_FLAG_STRING(ilogtail_docker_file_path_config); -DECLARE_FLAG_STRING(metrics_report_method); DECLARE_FLAG_INT32(data_server_port); DECLARE_FLAG_BOOL(enable_env_ref_in_config); DECLARE_FLAG_BOOL(enable_sls_metrics_format); @@ -82,7 +81,6 @@ static void overwrite_community_edition_flags() { STRING_FLAG(default_buffer_file_path) = "checkpoint"; STRING_FLAG(ilogtail_docker_file_path_config) = "checkpoint/docker_path_config.json"; } - STRING_FLAG(metrics_report_method) = ""; INT32_FLAG(data_server_port) = 443; BOOL_FLAG(enable_env_ref_in_config) = true; BOOL_FLAG(enable_sls_metrics_format) = false; diff --git a/core/logtail_windows.cpp b/core/logtail_windows.cpp index 4eb7a2aec7..a14d531f2a 100644 --- a/core/logtail_windows.cpp +++ b/core/logtail_windows.cpp @@ -29,14 +29,12 @@ DECLARE_FLAG_STRING(logtail_sys_conf_dir); DECLARE_FLAG_STRING(check_point_filename); DECLARE_FLAG_STRING(default_buffer_file_path); DECLARE_FLAG_STRING(ilogtail_docker_file_path_config); -DECLARE_FLAG_STRING(metrics_report_method); DECLARE_FLAG_INT32(data_server_port); DECLARE_FLAG_BOOL(enable_env_ref_in_config); DECLARE_FLAG_BOOL(enable_sls_metrics_format); static void overwrite_community_edition_flags() { // support run in installation dir on default - STRING_FLAG(metrics_report_method) = ""; INT32_FLAG(data_server_port) = 443; BOOL_FLAG(enable_env_ref_in_config) = true; BOOL_FLAG(enable_sls_metrics_format) = false; diff --git a/core/monitor/Monitor.h b/core/monitor/Monitor.h index 3a09d325b8..ea020c22bb 100644 --- a/core/monitor/Monitor.h +++ b/core/monitor/Monitor.h @@ -180,10 +180,6 @@ class LogtailMonitor { int32_t mCpuArrayForScaleIdx; float mScaledCpuUsageStep; #endif - -#ifdef APSARA_UNIT_TEST_MAIN - friend class ConfigUpdatorUnittest; -#endif }; class LoongCollectorMonitor { diff --git a/core/plugin/input/InputFile.cpp b/core/plugin/input/InputFile.cpp index dd5c9d6931..0e97cb3fb9 100644 --- a/core/plugin/input/InputFile.cpp +++ b/core/plugin/input/InputFile.cpp @@ -249,85 +249,4 @@ bool InputFile::CreateInnerProcessors() { return true; } - -// 为 ContainerInfo 设置多个真实路径,与 FileDiscoveryOptions 的多路径对应 -bool InputFile::SetContainerBaseDirs(ContainerInfo& containerInfo, const FileDiscoveryOptions* fileDiscovery) { - if (!containerInfo.mRealBaseDirs.empty()) { - return true; // 已经设置过 - } - - if (!fileDiscovery || containerInfo.mRawContainerInfo == nullptr) { - return false; - } - - const auto& pathInfos = fileDiscovery->GetBasePathInfos(); - bool hasSuccess = false; - - // 为每个配置路径计算对应的容器真实路径 - for (const auto& pathInfo : pathInfos) { - string logPath; - if (pathInfo.hasWildcard()) { - logPath = pathInfo.wildcardPaths[0]; - } else { - logPath = pathInfo.basePath; - } - - // 计算该路径在容器中的映射 - string realBaseDir; - if (SetContainerBaseDirForPath(containerInfo, logPath, realBaseDir)) { - containerInfo.mRealBaseDirs.push_back(realBaseDir); - hasSuccess = true; - } else { - // 映射失败,存储空字符串以保持索引对应 - LOG_WARNING(sLogger, - ("failed to map container path", "path mapping failed for this config path")( - "container id", containerInfo.mRawContainerInfo->mID)("log path", logPath)); - containerInfo.mRealBaseDirs.push_back(""); - } - } - - return hasSuccess; -} - -// 计算单个配置路径在容器中的映射路径(不修改 containerInfo) -bool InputFile::SetContainerBaseDirForPath(const ContainerInfo& containerInfo, - const string& logPath, - string& outRealBaseDir) { - if (containerInfo.mRawContainerInfo == nullptr) { - return false; - } - - size_t pthSize = logPath.size(); - size_t size = containerInfo.mRawContainerInfo->mMounts.size(); - size_t bestMatchedMountsIndex = size; - - // ParseByJSONObj 确保 Destination、Source、mUpperDir 不会以\或者/结尾 - for (size_t i = 0; i < size; ++i) { - StringView dst = containerInfo.mRawContainerInfo->mMounts[i].mDestination; - size_t dstSize = dst.size(); - - if (StartWith(logPath, dst) - && (pthSize == dstSize || (pthSize > dstSize && (logPath[dstSize] == '/' || logPath[dstSize] == '\\'))) - && (bestMatchedMountsIndex == size - || containerInfo.mRawContainerInfo->mMounts[bestMatchedMountsIndex].mDestination.size() < dstSize)) { - bestMatchedMountsIndex = i; - } - } - - if (bestMatchedMountsIndex < size) { - outRealBaseDir = STRING_FLAG(default_container_host_path) - + containerInfo.mRawContainerInfo->mMounts[bestMatchedMountsIndex].mSource - + logPath.substr(containerInfo.mRawContainerInfo->mMounts[bestMatchedMountsIndex].mDestination.size()); - } else { - outRealBaseDir - = STRING_FLAG(default_container_host_path) + containerInfo.mRawContainerInfo->mUpperDir + logPath; - } - - LOG_INFO(sLogger, - ("set container base dir for path", - logPath)("real dir", outRealBaseDir)("container id", containerInfo.mRawContainerInfo->mID)); - - return true; -} - } // namespace logtail diff --git a/core/plugin/input/InputFile.h b/core/plugin/input/InputFile.h index 40cf94667d..8f4dfa1f81 100644 --- a/core/plugin/input/InputFile.h +++ b/core/plugin/input/InputFile.h @@ -58,14 +58,6 @@ class InputFile : public Input { private: bool CreateInnerProcessors(); - // 为 ContainerInfo 设置多个真实路径,与 FileDiscoveryOptions 的多路径对应 - static bool SetContainerBaseDirs(ContainerInfo& containerInfo, const FileDiscoveryOptions* fileDiscovery); - - // 计算单个配置路径在容器中的映射路径(不修改 containerInfo) - static bool SetContainerBaseDirForPath(const ContainerInfo& containerInfo, - const std::string& logPath, - std::string& outRealBaseDir); - #ifdef APSARA_UNIT_TEST_MAIN friend class InputFileUnittest; #endif diff --git a/core/plugin/input/InputStaticFile.cpp b/core/plugin/input/InputStaticFile.cpp index 6bdc805bab..52c165b5f8 100644 --- a/core/plugin/input/InputStaticFile.cpp +++ b/core/plugin/input/InputStaticFile.cpp @@ -14,12 +14,6 @@ #include "plugin/input/InputStaticFile.h" -#if defined(__linux__) -#include -#else -#include "common/StringTools.h" -#endif - #include "app_config/AppConfig.h" #include "collection_pipeline/CollectionPipeline.h" #include "collection_pipeline/plugin/PluginRegistry.h" @@ -34,26 +28,6 @@ using namespace std; namespace logtail { -static bool IsValidDir(const filesystem::path& dir) { - error_code ec; - filesystem::file_status s = filesystem::status(dir, ec); - if (ec) { - LOG_WARNING(sLogger, - ("failed to get base dir path info", - "skip")("dir path", dir.string())("error code", ec.value())("error msg", ec.message())); - return false; - } - if (!filesystem::exists(s)) { - LOG_WARNING(sLogger, ("base dir path not existed", "skip")("dir path", dir.string())); - return false; - } - if (!filesystem::is_directory(s)) { - LOG_WARNING(sLogger, ("base dir path is not a directory", "skip")("dir path", dir.string())); - return false; - } - return true; -} - const string InputStaticFile::sName = "input_static_file_onetime"; bool InputStaticFile::Init(const Json::Value& config, Json::Value& optionalGoPipeline) { @@ -89,10 +63,21 @@ bool InputStaticFile::Init(const Json::Value& config, Json::Value& optionalGoPip } if (mEnableContainerDiscovery) { if (!mContainerDiscovery.Init(config, *mContext, sName)) { - // should not happen - return false; + PARAM_ERROR_RETURN(mContext->GetLogger(), + mContext->GetAlarm(), + "container discovery config is invalid", + sName, + mContext->GetConfigName(), + mContext->GetProjectName(), + mContext->GetLogstoreName(), + mContext->GetRegion()); } + mContainerDiscovery.mIsStdio = false; mFileDiscovery.SetEnableContainerDiscoveryFlag(true); + mFileDiscovery.SetDeduceAndSetContainerBaseDirFunc(DeduceAndSetContainerBaseDir); + mContainerDiscovery.GenerateContainerMetaFetchingGoPipeline( + optionalGoPipeline, &mFileDiscovery, mContext->GetPipeline().GenNextPluginMeta(false)); + mFileDiscovery.SetContainerDiscoveryOptions(std::move(mContainerDiscovery)); } if (!mFileReader.Init(config, *mContext, sName)) { @@ -144,19 +129,21 @@ bool InputStaticFile::Init(const Json::Value& config, Json::Value& optionalGoPip bool InputStaticFile::Start() { if (mEnableContainerDiscovery) { - // TODO: get container info - // mFileDiscovery.SetContainerInfo(); + mFileDiscovery.SetContainerInfo(std::make_shared>()); + mFileDiscovery.SetFullContainerList(std::make_shared>()); } // Add plugin metric manager FileServer::GetInstance()->AddPluginMetricManager(mContext->GetConfigName(), mPluginMetricManager); - optional> files; - if (!mContext->IsOnetimePipelineRunningBeforeStart()) { - files = GetFiles(); - } - StaticFileServer::GetInstance()->AddInput( - mContext->GetConfigName(), mIndex, files, &mFileDiscovery, &mFileReader, &mMultiline, &mFileTag, mContext); + StaticFileServer::GetInstance()->AddInput(mContext->GetConfigName(), + mIndex, + &mFileDiscovery, + &mFileReader, + &mMultiline, + &mFileTag, + mFileContainerMetaMap, + mContext); return true; } @@ -170,171 +157,6 @@ bool InputStaticFile::Stop(bool isPipelineRemoving) { return true; } -vector InputStaticFile::GetFiles() const { - vector res; - const auto& pathInfos = mFileDiscovery.GetBasePathInfos(); - - if (!mEnableContainerDiscovery) { - set visitedDirs; - // Process each path configuration - for (const auto& pathInfo : pathInfos) { - vector baseDirs; - if (pathInfo.hasWildcard()) { - GetValidBaseDirs(pathInfo, pathInfo.wildcardPaths[0], 0, baseDirs); - if (baseDirs.empty()) { - LOG_WARNING(sLogger, - ("no files found", "base dir path invalid")("base dir", pathInfo.basePath)( - "config", mContext->GetConfigName())); - continue; - } - } else { - baseDirs.emplace_back(pathInfo.basePath); - } - - for (const auto& dir : baseDirs) { - if (IsValidDir(dir)) { - GetFiles(dir, mFileDiscovery.mMaxDirSearchDepth, &pathInfo, nullptr, visitedDirs, res); - } - } - } - LOG_INFO(sLogger, ("total files cnt", res.size())("files", ToString(res))("config", mContext->GetConfigName())); - } else { - // TODO: support symlink in container - set visitedDirs; - for (const auto& item : *mFileDiscovery.GetContainerInfo()) { - // 遍历配置路径和对应的容器真实路径 - for (size_t idx = 0; idx < pathInfos.size(); ++idx) { - if (idx >= item.mRealBaseDirs.size()) - break; - - const auto& pathInfo = pathInfos[idx]; - const string& realBaseDir = item.mRealBaseDirs[idx]; - - if (realBaseDir.empty()) - continue; - - vector baseDirs; - if (pathInfo.hasWildcard()) { - GetValidBaseDirs(pathInfo, realBaseDir, 0, baseDirs); - if (baseDirs.empty()) { - LOG_DEBUG( - sLogger, - ("no files found", "base dir path invalid")("container id", item.mRawContainerInfo->mID)( - "real base dir", realBaseDir)("config", mContext->GetConfigName())); - continue; - } - } else { - baseDirs.emplace_back(realBaseDir); - } - - auto prevCnt = res.size(); - for (const auto& dir : baseDirs) { - if (IsValidDir(dir)) { - GetFiles(dir, mFileDiscovery.mMaxDirSearchDepth, &pathInfo, &realBaseDir, visitedDirs, res); - } - } - if (res.size() > prevCnt) { - LOG_INFO( - sLogger, - ("container files cnt", res.size() - prevCnt)("container id", item.mRawContainerInfo->mID)( - "real base dir", realBaseDir)("files", ToString(res))("config", mContext->GetConfigName())); - } else { - LOG_DEBUG(sLogger, - ("no files found, container id", item.mRawContainerInfo->mID)( - "real base dir", realBaseDir)("config", mContext->GetConfigName())); - } - } - } - LOG_INFO(sLogger, ("total files cnt", res.size())("config", mContext->GetConfigName())); - } - return res; -} - -void InputStaticFile::GetValidBaseDirs(const BasePathInfo& pathInfo, - const filesystem::path& dir, - uint32_t depth, - vector& filepaths) const { - const auto& wildcardPaths = pathInfo.wildcardPaths; - bool finish = false; - if (depth + 2 == wildcardPaths.size()) { - finish = true; - } - - if (depth == 0 && !IsValidDir(wildcardPaths[depth])) { - return; - } - - const auto& subdir = pathInfo.constWildcardPaths[depth]; - if (!subdir.empty()) { - auto path = dir / subdir; - error_code ec; - filesystem::file_status s = filesystem::status(path, ec); - if (ec || !filesystem::exists(s) || !filesystem::is_directory(s)) { - return; - } - if (finish) { - filepaths.emplace_back(path); - } else { - GetValidBaseDirs(pathInfo, path, depth + 1, filepaths); - } - } else { - auto pattern = filesystem::path(wildcardPaths[depth + 1]).filename(); - error_code ec; - for (auto const& entry : filesystem::directory_iterator(dir, ec)) { - const auto& path = entry.path(); - const auto& status = entry.status(); - if (filesystem::is_directory(status) - && (fnmatch(pattern.string().c_str(), path.filename().string().c_str(), FNM_PATHNAME) == 0)) { - if (finish) { - filepaths.emplace_back(path); - } else { - GetValidBaseDirs(pathInfo, path, depth + 1, filepaths); - } - } - } - } -} - -void InputStaticFile::GetFiles(const filesystem::path& dir, - uint32_t depth, - const BasePathInfo* pathInfo, - const string* containerBaseDir, - set& visitedDir, - vector& files) const { - error_code ec; - for (auto const& entry : filesystem::directory_iterator(dir, ec)) { - const auto& path = entry.path(); - auto pathStr = path.string(); - if (containerBaseDir && pathInfo) { - // Map container path to config path - pathStr = pathInfo->basePath + pathStr.substr(containerBaseDir->size()); - } - const auto& status = entry.status(); - if (filesystem::is_regular_file(status)) { - const auto& filename = path.filename().string(); - if (pathInfo) { - // Use the specific pathInfo's filePattern if provided, otherwise check all patterns - bool filenameMatched = mFileDiscovery.IsFilenameMatched(filename, *pathInfo); - - if (filenameMatched && !mFileDiscovery.IsFilenameInBlacklist(filename) - && !mFileDiscovery.IsFilepathInBlacklist(pathStr)) { - files.emplace_back(path); - } - } - } else if (filesystem::is_directory(status)) { - auto devInode = GetFileDevInode(path.string()); - if (!devInode.IsValid() || visitedDir.find(devInode) != visitedDir.end()) { - // avoid loop - continue; - } - visitedDir.emplace(devInode); - if (depth > 0 && !AppConfig::GetInstance()->IsHostPathMatchBlacklist(path.string()) - && !mFileDiscovery.IsDirectoryInBlacklist(pathStr)) { - GetFiles(path, depth - 1, pathInfo, containerBaseDir, visitedDir, files); - } - } - } -} bool InputStaticFile::CreateInnerProcessors() { unique_ptr processor; @@ -375,4 +197,10 @@ bool InputStaticFile::CreateInnerProcessors() { return true; } +bool InputStaticFile::DeduceAndSetContainerBaseDir(ContainerInfo& containerInfo, + const CollectionPipelineContext*, + const FileDiscoveryOptions* fileDiscovery) { + return SetContainerBaseDirs(containerInfo, fileDiscovery); +} + } // namespace logtail diff --git a/core/plugin/input/InputStaticFile.h b/core/plugin/input/InputStaticFile.h index 4e258dc1bc..4124300412 100644 --- a/core/plugin/input/InputStaticFile.h +++ b/core/plugin/input/InputStaticFile.h @@ -16,18 +16,15 @@ #pragma once -#include - -#include -#include -#include +#include +#include #include "collection_pipeline/plugin/interface/Input.h" -#include "common/DevInode.h" #include "container_manager/ContainerDiscoveryOptions.h" #include "file_server/FileDiscoveryOptions.h" #include "file_server/FileTagOptions.h" #include "file_server/MultilineOptions.h" +#include "file_server/checkpoint/FileCheckpoint.h" #include "file_server/reader/FileReaderOptions.h" #include "monitor/metric_models/ReentrantMetricsRecord.h" @@ -37,6 +34,10 @@ class InputStaticFile : public Input { public: static const std::string sName; + static bool DeduceAndSetContainerBaseDir(ContainerInfo& containerInfo, + const CollectionPipelineContext*, + const FileDiscoveryOptions*); + const std::string& Name() const override { return sName; } bool Init(const Json::Value& config, Json::Value& optionalGoPipeline) override; bool Start() override; @@ -54,17 +55,9 @@ class InputStaticFile : public Input { PluginMetricManagerPtr mPluginMetricManager; IntGaugePtr mMonitorFileTotal; - void GetValidBaseDirs(const BasePathInfo& pathInfo, - const std::filesystem::path& dir, - uint32_t depth, - std::vector& filepaths) const; - std::vector GetFiles() const; - void GetFiles(const std::filesystem::path& dir, - uint32_t depth, - const BasePathInfo* pathInfo, - const std::string* containerBaseDir, - std::set& visitedDir, - std::vector& files) const; + // 文件路径到容器元信息的映射(容器场景下使用) + std::unordered_map mFileContainerMetaMap; + bool CreateInnerProcessors(); #ifdef APSARA_UNIT_TEST_MAIN diff --git a/core/unittest/Unittest.h b/core/unittest/Unittest.h index 59e028ee1c..afc46fc758 100644 --- a/core/unittest/Unittest.h +++ b/core/unittest/Unittest.h @@ -20,7 +20,8 @@ #endif #include -#include "boost/filesystem.hpp" +#include + #include "gtest/gtest.h" #include "common/HashUtil.h" @@ -30,7 +31,7 @@ #include "common/TimeUtil.h" #include "logger/Logger.h" -namespace bfs = boost::filesystem; +namespace fs = std::filesystem; #define APSARA_TEST_TRUE(condition) \ do { \ diff --git a/core/unittest/checkpoint/CheckpointManagerUnittest.cpp b/core/unittest/checkpoint/CheckpointManagerUnittest.cpp index 9f3c60b119..1ebf75b6e3 100644 --- a/core/unittest/checkpoint/CheckpointManagerUnittest.cpp +++ b/core/unittest/checkpoint/CheckpointManagerUnittest.cpp @@ -26,13 +26,13 @@ std::string kTestRootDir; class CheckpointManagerUnittest : public ::testing::Test { public: static void SetUpTestCase() { - kTestRootDir = (bfs::path(GetProcessExecutionDir()) / "CheckpointManagerUnittest").string(); - bfs::remove_all(kTestRootDir); - bfs::create_directories(kTestRootDir); + kTestRootDir = (fs::path(GetProcessExecutionDir()) / "CheckpointManagerUnittest").string(); + fs::remove_all(kTestRootDir); + fs::create_directories(kTestRootDir); AppConfig::GetInstance()->SetLoongcollectorConfDir(kTestRootDir); } - static void TearDownTestCase() { bfs::remove_all(kTestRootDir); } + static void TearDownTestCase() { fs::remove_all(kTestRootDir); } void TestSearchFilePathByDevInodeInDirectory(); }; @@ -42,9 +42,9 @@ UNIT_TEST_CASE(CheckpointManagerUnittest, TestSearchFilePathByDevInodeInDirector void CheckpointManagerUnittest::TestSearchFilePathByDevInodeInDirectory() { const std::string kRotateFileName = "test.log.5"; const std::string kFileName = "test.log"; - const std::string kFilePath = (bfs::path(kTestRootDir) / kFileName).string(); - const std::string kRotateFilePath = (bfs::path(kTestRootDir) / kRotateFileName).string(); - const std::string kTempPath = (bfs::path(kTestRootDir) / ".." / kFileName).string(); + const std::string kFilePath = (fs::path(kTestRootDir) / kFileName).string(); + const std::string kRotateFilePath = (fs::path(kTestRootDir) / kRotateFileName).string(); + const std::string kTempPath = (fs::path(kTestRootDir) / ".." / kFileName).string(); std::ofstream(kFilePath) << ""; fsutil::PathStat ps; @@ -52,7 +52,7 @@ void CheckpointManagerUnittest::TestSearchFilePathByDevInodeInDirectory() { auto devInode = ps.GetDevInode(); // Rotate file in current directory. - bfs::rename(kFilePath, kRotateFilePath); + fs::rename(kFilePath, kRotateFilePath); // Normal search. { @@ -63,7 +63,7 @@ void CheckpointManagerUnittest::TestSearchFilePathByDevInodeInDirectory() { // Exceed limit when search. { - bfs::rename(kRotateFilePath, kTempPath); + fs::rename(kRotateFilePath, kTempPath); auto bakLimit = INT32_FLAG(checkpoint_find_max_file_count); INT32_FLAG(checkpoint_find_max_file_count) = 2; @@ -76,14 +76,14 @@ void CheckpointManagerUnittest::TestSearchFilePathByDevInodeInDirectory() { EXPECT_EQ(cache.size(), INT32_FLAG(checkpoint_find_max_file_count) + 1); INT32_FLAG(checkpoint_find_max_file_count) = bakLimit; - bfs::rename(kTempPath, kRotateFilePath); + fs::rename(kTempPath, kRotateFilePath); } // File is moved to sub-directory. - const auto kSubDir = bfs::path(kTestRootDir) / "sub" / "sub"; - bfs::create_directories(kSubDir); + const auto kSubDir = fs::path(kTestRootDir) / "sub" / "sub"; + fs::create_directories(kSubDir); const auto kSubDirFilePath = (kSubDir / kRotateFileName).string(); - bfs::rename(kRotateFilePath, kSubDirFilePath); + fs::rename(kRotateFilePath, kSubDirFilePath); // Search with depth. { diff --git a/core/unittest/checkpoint/CheckpointManagerV2Unittest.cpp b/core/unittest/checkpoint/CheckpointManagerV2Unittest.cpp index dd921c51f8..af537b37d3 100644 --- a/core/unittest/checkpoint/CheckpointManagerV2Unittest.cpp +++ b/core/unittest/checkpoint/CheckpointManagerV2Unittest.cpp @@ -34,16 +34,16 @@ const auto kDevInode = DevInode(100, 1000); class CheckpointManagerV2Unittest : public ::testing::Test { public: static void SetUpTestCase() { - kTestRootDir = (bfs::path(GetProcessExecutionDir()) / "CheckpointManagerV2Unittest").string(); - if (bfs::exists(kTestRootDir)) { - bfs::remove_all(kTestRootDir); + kTestRootDir = (fs::path(GetProcessExecutionDir()) / "CheckpointManagerV2Unittest").string(); + if (fs::exists(kTestRootDir)) { + fs::remove_all(kTestRootDir); } - bfs::create_directories(kTestRootDir); + fs::create_directories(kTestRootDir); AppConfig::GetInstance()->SetLoongcollectorConfDir(kTestRootDir); INT32_FLAG(logtail_checkpoint_check_gc_interval_sec) = 1; } - static void TearDownTestCase() { bfs::remove_all(kTestRootDir); } + static void TearDownTestCase() { fs::remove_all(kTestRootDir); } void TestBaseMethod(); diff --git a/core/unittest/checkpoint/InputStaticFileCheckpointManagerUnittest.cpp b/core/unittest/checkpoint/InputStaticFileCheckpointManagerUnittest.cpp index c3ac56ca59..889e4e0d29 100644 --- a/core/unittest/checkpoint/InputStaticFileCheckpointManagerUnittest.cpp +++ b/core/unittest/checkpoint/InputStaticFileCheckpointManagerUnittest.cpp @@ -32,14 +32,14 @@ class InputStaticFileCheckpointManagerUnittest : public testing::Test { protected: void SetUp() override { sManager = InputStaticFileCheckpointManager::GetInstance(); - sManager->mCheckpointRootPath = filesystem::path("./input_static_file"); - filesystem::create_directories(sManager->mCheckpointRootPath); + sManager->mCheckpointRootPath = fs::path("./input_static_file"); + fs::create_directories(sManager->mCheckpointRootPath); } void TearDown() override { sManager->ClearUnusedCheckpoints(); sManager->mInputCheckpointMap.clear(); - filesystem::remove_all(sManager->mCheckpointRootPath); + fs::remove_all(sManager->mCheckpointRootPath); } private: @@ -48,8 +48,8 @@ class InputStaticFileCheckpointManagerUnittest : public testing::Test { void InputStaticFileCheckpointManagerUnittest::TestUpdateCheckpointMap() const { // prepare logs - filesystem::create_directories("test_logs"); - vector files{"./test_logs/test_file_1.log", "./test_logs/test_file_2.log"}; + fs::create_directories("test_logs"); + vector files{"./test_logs/test_file_1.log", "./test_logs/test_file_2.log"}; vector contents{string(2000, 'a') + "\n", string(200, 'b') + "\n"}; vector fingerprints; for (size_t i = 0; i < files.size(); ++i) { @@ -101,7 +101,7 @@ void InputStaticFileCheckpointManagerUnittest::TestUpdateCheckpointMap() const { // create config { // new config - optional> filesOpt({files[0]}); + optional> filesOpt({files[0]}); APSARA_TEST_TRUE(sManager->CreateCheckpoint("test_config_1", 0, filesOpt)); APSARA_TEST_EQUAL(1U, sManager->mInputCheckpointMap.size()); const auto& cpt = sManager->mInputCheckpointMap.at(make_pair("test_config_1", 0)); @@ -120,11 +120,11 @@ void InputStaticFileCheckpointManagerUnittest::TestUpdateCheckpointMap() const { APSARA_TEST_EQUAL(FileStatus::WAITING, cpt.mFileCheckpoints[0].mStatus); APSARA_TEST_EQUAL(0, cpt.mFileCheckpoints[0].mStartTime); APSARA_TEST_EQUAL(0, cpt.mFileCheckpoints[0].mLastUpdateTime); - APSARA_TEST_TRUE(filesystem::exists(sManager->mCheckpointRootPath / "test_config_1@0.json")); + APSARA_TEST_TRUE(fs::exists(sManager->mCheckpointRootPath / "test_config_1@0.json")); } { // old config, but changed on restart - optional> filesOpt({files[1]}); + optional> filesOpt({files[1]}); APSARA_TEST_TRUE(sManager->CreateCheckpoint("test_config_2", 0, filesOpt)); APSARA_TEST_EQUAL(2U, sManager->mInputCheckpointMap.size()); const auto& cpt = sManager->mInputCheckpointMap.at(make_pair("test_config_2", 0)); @@ -143,8 +143,8 @@ void InputStaticFileCheckpointManagerUnittest::TestUpdateCheckpointMap() const { APSARA_TEST_EQUAL(FileStatus::WAITING, cpt.mFileCheckpoints[0].mStatus); APSARA_TEST_EQUAL(0, cpt.mFileCheckpoints[0].mStartTime); APSARA_TEST_EQUAL(0, cpt.mFileCheckpoints[0].mLastUpdateTime); - APSARA_TEST_TRUE(filesystem::exists(sManager->mCheckpointRootPath / "test_config_2@0.json")); - APSARA_TEST_NOT_EQUAL(0U, std::filesystem::file_size(sManager->mCheckpointRootPath / "test_config_2@0.json")); + APSARA_TEST_TRUE(fs::exists(sManager->mCheckpointRootPath / "test_config_2@0.json")); + APSARA_TEST_NOT_EQUAL(0U, fs::file_size(sManager->mCheckpointRootPath / "test_config_2@0.json")); } { // old config, no change @@ -165,7 +165,7 @@ void InputStaticFileCheckpointManagerUnittest::TestUpdateCheckpointMap() const { APSARA_TEST_EQUAL(FileStatus::READING, cpt.mFileCheckpoints[0].mStatus); APSARA_TEST_EQUAL(1739349980, cpt.mFileCheckpoints[0].mStartTime); APSARA_TEST_EQUAL(1739349981, cpt.mFileCheckpoints[0].mLastUpdateTime); - APSARA_TEST_TRUE(filesystem::exists(sManager->mCheckpointRootPath / "test_config_3@0.json")); + APSARA_TEST_TRUE(fs::exists(sManager->mCheckpointRootPath / "test_config_3@0.json")); } { // old config, no change, but checkpoint file not existed @@ -177,7 +177,7 @@ void InputStaticFileCheckpointManagerUnittest::TestUpdateCheckpointMap() const { APSARA_TEST_EQUAL(StaticFileReadingStatus::ABORT, cpt.mStatus); APSARA_TEST_EQUAL(0U, cpt.mCurrentFileIndex); APSARA_TEST_TRUE(cpt.mFileCheckpoints.empty()); - APSARA_TEST_TRUE(filesystem::exists(sManager->mCheckpointRootPath / "test_config_4@0.json")); + APSARA_TEST_TRUE(fs::exists(sManager->mCheckpointRootPath / "test_config_4@0.json")); } // delete config @@ -187,24 +187,23 @@ void InputStaticFileCheckpointManagerUnittest::TestUpdateCheckpointMap() const { APSARA_TEST_EQUAL(3U, sManager->mInputCheckpointMap.size()); APSARA_TEST_EQUAL(sManager->mInputCheckpointMap.end(), sManager->mInputCheckpointMap.find(make_pair("test_config_1", 0))); - APSARA_TEST_FALSE(filesystem::exists(sManager->mCheckpointRootPath / "test_config_1@0.json")); + APSARA_TEST_FALSE(fs::exists(sManager->mCheckpointRootPath / "test_config_1@0.json")); } { // checkpoint file not existed - filesystem::remove(sManager->mCheckpointRootPath / "test_config_2@0.json"); + fs::remove(sManager->mCheckpointRootPath / "test_config_2@0.json"); APSARA_TEST_TRUE(sManager->DeleteCheckpoint("test_config_2", 0)); APSARA_TEST_EQUAL(2U, sManager->mInputCheckpointMap.size()); APSARA_TEST_EQUAL(sManager->mInputCheckpointMap.end(), sManager->mInputCheckpointMap.find(make_pair("test_config_2", 0))); } - filesystem::remove_all("test_logs"); + fs::remove_all("test_logs"); } void InputStaticFileCheckpointManagerUnittest::TestUpdateCheckpoint() const { // prepare logs - filesystem::create_directories("test_logs"); - vector files{ - "./test_logs/test_file_1.log", "./test_logs/test_file_2.log", "./test_logs/test_file_3.log"}; + fs::create_directories("test_logs"); + vector files{"./test_logs/test_file_1.log", "./test_logs/test_file_2.log", "./test_logs/test_file_3.log"}; vector contents{string(2000, 'a') + "\n", string(200, 'b') + "\n", string(500, 'c') + "\n"}; vector fingerprints; for (size_t i = 0; i < files.size(); ++i) { @@ -221,7 +220,7 @@ void InputStaticFileCheckpointManagerUnittest::TestUpdateCheckpoint() const { } sManager->CreateCheckpoint("test_config_1", 0, files); - sManager->CreateCheckpoint("test_config_2", 0, optional>({files[0]})); + sManager->CreateCheckpoint("test_config_2", 0, optional>({files[0]})); { // from waiting to reading uint64_t initialSize = static_cast(contents[0].size()); @@ -235,7 +234,7 @@ void InputStaticFileCheckpointManagerUnittest::TestUpdateCheckpoint() const { APSARA_TEST_EQUAL(FileStatus::READING, cpt.mFileCheckpoints[0].mStatus); APSARA_TEST_NOT_EQUAL(0, cpt.mFileCheckpoints[0].mStartTime); APSARA_TEST_NOT_EQUAL(0, cpt.mFileCheckpoints[0].mLastUpdateTime); - APSARA_TEST_TRUE(filesystem::exists(sManager->mCheckpointRootPath / "test_config_1@0.json")); + APSARA_TEST_TRUE(fs::exists(sManager->mCheckpointRootPath / "test_config_1@0.json")); InputStaticFileCheckpoint cptLoaded; sManager->LoadCheckpointFile(sManager->mCheckpointRootPath / "test_config_1@0.json", &cptLoaded); APSARA_TEST_EQUAL(cpt.mFileCheckpoints[0].mOffset, cptLoaded.mFileCheckpoints[0].mOffset); @@ -263,7 +262,7 @@ void InputStaticFileCheckpointManagerUnittest::TestUpdateCheckpoint() const { APSARA_TEST_EQUAL(FileStatus::READING, cpt.mFileCheckpoints[0].mStatus); APSARA_TEST_NOT_EQUAL(0, cpt.mFileCheckpoints[0].mStartTime); APSARA_TEST_NOT_EQUAL(0, cpt.mFileCheckpoints[0].mLastUpdateTime); - APSARA_TEST_TRUE(filesystem::exists(sManager->mCheckpointRootPath / "test_config_1@0.json")); + APSARA_TEST_TRUE(fs::exists(sManager->mCheckpointRootPath / "test_config_1@0.json")); InputStaticFileCheckpoint cptLoaded; sManager->LoadCheckpointFile(sManager->mCheckpointRootPath / "test_config_1@0.json", &cptLoaded); APSARA_TEST_EQUAL(1000U, cptLoaded.mFileCheckpoints[0].mOffset); @@ -287,7 +286,7 @@ void InputStaticFileCheckpointManagerUnittest::TestUpdateCheckpoint() const { APSARA_TEST_EQUAL(FileStatus::FINISHED, cpt.mFileCheckpoints[0].mStatus); APSARA_TEST_NOT_EQUAL(0, cpt.mFileCheckpoints[0].mStartTime); APSARA_TEST_NOT_EQUAL(0, cpt.mFileCheckpoints[0].mLastUpdateTime); - APSARA_TEST_TRUE(filesystem::exists(sManager->mCheckpointRootPath / "test_config_1@0.json")); + APSARA_TEST_TRUE(fs::exists(sManager->mCheckpointRootPath / "test_config_1@0.json")); InputStaticFileCheckpoint cptLoaded; sManager->LoadCheckpointFile(sManager->mCheckpointRootPath / "test_config_1@0.json", &cptLoaded); APSARA_TEST_EQUAL(cpt.mFileCheckpoints[0].mSize, cptLoaded.mFileCheckpoints[0].mSize); @@ -343,12 +342,12 @@ void InputStaticFileCheckpointManagerUnittest::TestUpdateCheckpoint() const { FileFingerprint fp; APSARA_TEST_FALSE(sManager->GetCurrentFileFingerprint("test_config_2", 0, &fp)); } - filesystem::remove_all("test_logs"); + fs::remove_all("test_logs"); } void InputStaticFileCheckpointManagerUnittest::TestCheckpointFileNames() const { // valid checkpoint root path - filesystem::create_directories(sManager->mCheckpointRootPath / "dir"); + fs::create_directories(sManager->mCheckpointRootPath / "dir"); { ofstream fout(sManager->mCheckpointRootPath / "unsupported_extenstion.yaml", std::ios_base::binary); } { ofstream fout(sManager->mCheckpointRootPath / "invalid_filename.json", std::ios_base::binary); } { ofstream fout(sManager->mCheckpointRootPath / "test_config@invalid_idx.json", std::ios_base::binary); } @@ -365,14 +364,14 @@ void InputStaticFileCheckpointManagerUnittest::TestCheckpointFileNames() const { APSARA_TEST_NOT_EQUAL(sManager->mCheckpointFileNamesOnInit.end(), sManager->mCheckpointFileNamesOnInit.find(make_pair("test_config", 1))); - filesystem::remove(sManager->mCheckpointRootPath / "test_config@0.json"); + fs::remove(sManager->mCheckpointRootPath / "test_config@0.json"); sManager->ClearUnusedCheckpoints(); APSARA_TEST_TRUE(sManager->mInputCheckpointMap.empty()); - APSARA_TEST_FALSE(filesystem::exists(sManager->mCheckpointRootPath / "test_config@0.json")); - APSARA_TEST_FALSE(filesystem::exists(sManager->mCheckpointRootPath / "test_config@1.json")); + APSARA_TEST_FALSE(fs::exists(sManager->mCheckpointRootPath / "test_config@0.json")); + APSARA_TEST_FALSE(fs::exists(sManager->mCheckpointRootPath / "test_config@1.json")); // no checkpoint root path - filesystem::remove_all(sManager->mCheckpointRootPath); + fs::remove_all(sManager->mCheckpointRootPath); EXPECT_NO_THROW(sManager->GetAllCheckpointFileNames()); // invalid checkpoint root path @@ -381,11 +380,11 @@ void InputStaticFileCheckpointManagerUnittest::TestCheckpointFileNames() const { } void InputStaticFileCheckpointManagerUnittest::TestDumpCheckpoints() const { - filesystem::create_directories("test_logs"); - vector files{"./test_logs/test_file_1.log", - "./test_logs/test_file_2.log", - "./test_logs/test_file_3.log", - "./test_logs/test_file_4.log"}; + fs::create_directories("test_logs"); + vector files{"./test_logs/test_file_1.log", + "./test_logs/test_file_2.log", + "./test_logs/test_file_3.log", + "./test_logs/test_file_4.log"}; { ofstream fout(files[0], std::ios_base::binary); fout << string(2000, 'a') << endl; @@ -409,7 +408,7 @@ void InputStaticFileCheckpointManagerUnittest::TestDumpCheckpoints() const { sManager->InvalidateCurrentFileCheckpoint("test_config_1", 0); sManager->UpdateCurrentFileCheckpoint("test_config_1", 0, 100); // job_2 finished - optional> filesOpt({files[0]}); + optional> filesOpt({files[0]}); sManager->CreateCheckpoint("test_config_2", 0, filesOpt); sManager->UpdateCurrentFileCheckpoint("test_config_2", 0, file1InitialSize); @@ -455,18 +454,18 @@ void InputStaticFileCheckpointManagerUnittest::TestDumpCheckpoints() const { APSARA_TEST_EQUAL(expectedCpt.mStatus, cpt.mStatus); APSARA_TEST_EQUAL(expectedCpt.mFileCheckpoints.size(), cpt.mFileCheckpoints.size()); } - filesystem::remove_all("test_logs"); + fs::remove_all("test_logs"); } void InputStaticFileCheckpointManagerUnittest::TestInvalidCheckpointFile() const { - filesystem::path cptPath = sManager->mCheckpointRootPath / "test_config@0.json"; + fs::path cptPath = sManager->mCheckpointRootPath / "test_config@0.json"; InputStaticFileCheckpoint cpt; // no file APSARA_TEST_FALSE(sManager->LoadCheckpointFile(cptPath, &cpt)); // not regular file - filesystem::create_directories(cptPath); + fs::create_directories(cptPath); APSARA_TEST_FALSE(sManager->LoadCheckpointFile(cptPath, &cpt)); - filesystem::remove_all(cptPath); + fs::remove_all(cptPath); // empty file { ofstream fout(cptPath, std::ios_base::binary); } APSARA_TEST_FALSE(sManager->LoadCheckpointFile(cptPath, &cpt)); @@ -646,15 +645,15 @@ void InputStaticFileCheckpointManagerUnittest::TestInvalidCheckpointFile() const } void InputStaticFileCheckpointManagerUnittest::TestSizeRemainsConstant() const { - filesystem::create_directories("test_logs"); - filesystem::path testFile = "./test_logs/test_file.log"; + fs::create_directories("test_logs"); + fs::path testFile = "./test_logs/test_file.log"; string content = string(2000, 'a') + "\n"; { ofstream fout(testFile, std::ios_base::binary); fout << content; } - optional> filesOpt({testFile}); + optional> filesOpt({testFile}); sManager->CreateCheckpoint("test_config", 0, filesOpt); const auto& cpt = sManager->mInputCheckpointMap.at(make_pair("test_config", 0)); @@ -675,7 +674,7 @@ void InputStaticFileCheckpointManagerUnittest::TestSizeRemainsConstant() const { sManager->UpdateCurrentFileCheckpoint("test_config", 0, initialSize); APSARA_TEST_EQUAL(initialSize, cpt.mFileCheckpoints[0].mSize); - filesystem::remove_all("test_logs"); + fs::remove_all("test_logs"); } UNIT_TEST_CASE(InputStaticFileCheckpointManagerUnittest, TestUpdateCheckpointMap) diff --git a/core/unittest/common/DevInodeUnittest.h b/core/unittest/common/DevInodeUnittest.h index 3d8400bbfb..9531d4de68 100644 --- a/core/unittest/common/DevInodeUnittest.h +++ b/core/unittest/common/DevInodeUnittest.h @@ -28,24 +28,24 @@ class DevInodeUnittest : public ::testing::Test { public: static void SetUpTestCase() { - bfs::path rootPath(GetProcessExecutionDir()); + fs::path rootPath(GetProcessExecutionDir()); rootPath /= "DevInodeUnittestDir"; mRootDir = rootPath.string(); - EXPECT_TRUE(bfs::create_directories(rootPath)); + EXPECT_TRUE(fs::create_directories(rootPath)); } - static void TearDownTestCase() { EXPECT_TRUE(bfs::remove_all(bfs::path(mRootDir))); } + static void TearDownTestCase() { EXPECT_TRUE(fs::remove_all(fs::path(mRootDir))); } }; std::string DevInodeUnittest::mRootDir; TEST_F(DevInodeUnittest, TestGetFileDevInode) { - bfs::path testRoot(mRootDir + "/TestGetFileDevInode"); - ASSERT_TRUE(bfs::create_directories(testRoot)); + fs::path testRoot(mRootDir + "/TestGetFileDevInode"); + ASSERT_TRUE(fs::create_directories(testRoot)); std::string subFileName = "file"; std::string subDirName = "dir"; std::ofstream((testRoot / subFileName).string()); - bfs::create_directory(testRoot / subDirName); + fs::create_directory(testRoot / subDirName); auto fileDevInode = GetFileDevInode((testRoot / subFileName).string()); EXPECT_TRUE(fileDevInode.IsValid()); diff --git a/core/unittest/common/FileSystemUtilUnittest.h b/core/unittest/common/FileSystemUtilUnittest.h index 8a7f772d05..8ce61a2813 100644 --- a/core/unittest/common/FileSystemUtilUnittest.h +++ b/core/unittest/common/FileSystemUtilUnittest.h @@ -33,17 +33,17 @@ class FileSystemUtilUnittest : public ::testing::Test { // Because file system operation can't lock any files or dirs, so it // is important to test if the test root directory can be removed. // Use mTestRoot to record the test root directory of each case. - bfs::path mTestRoot; + fs::path mTestRoot; void SetUp() { auto testCaseName = ::testing::UnitTest::GetInstance()->current_test_info()->test_case_name(); - mTestRoot = bfs::path(mRootDir) / testCaseName; - ASSERT_TRUE(bfs::create_directories(mTestRoot)); + mTestRoot = fs::path(mRootDir) / testCaseName; + ASSERT_TRUE(fs::create_directories(mTestRoot)); } void TearDown() { // Delete the whole folder to prove that Dir will not locking any files or dirs. try { - bfs::remove_all(mTestRoot); + fs::remove_all(mTestRoot); } catch (...) { ASSERT_TRUE(false); } @@ -51,13 +51,13 @@ class FileSystemUtilUnittest : public ::testing::Test { public: static void SetUpTestCase() { - bfs::path rootPath(GetProcessExecutionDir()); + fs::path rootPath(GetProcessExecutionDir()); rootPath /= "CommonUnittest"; - bfs::remove_all(rootPath); - EXPECT_TRUE(bfs::create_directories(rootPath)); + fs::remove_all(rootPath); + EXPECT_TRUE(fs::create_directories(rootPath)); mRootDir = rootPath.string(); } - static void TearDownTestCase() { EXPECT_TRUE(bfs::remove_all(mRootDir)); } + static void TearDownTestCase() { EXPECT_TRUE(fs::remove_all(mRootDir)); } }; std::string FileSystemUtilUnittest::mRootDir; @@ -98,9 +98,9 @@ TEST_F(FileSystemUtilUnittest, TestDirNormal) { for (auto& f : hiddenFiles) std::ofstream((mTestRoot / f).string()); for (auto& d : subDirs) - bfs::create_directory(mTestRoot / d); + fs::create_directory(mTestRoot / d); for (auto& d : hiddenDirs) - bfs::create_directory(mTestRoot / d); + fs::create_directory(mTestRoot / d); // Use Dir to iterate it. fsutil::Dir rootDir(mTestRoot.string()); @@ -118,7 +118,7 @@ TEST_F(FileSystemUtilUnittest, TestDirNormal) { // Delete the first file to prove that iteration will not lock the file. static bool isFirstFile = true; if (isFirstFile) { - EXPECT_TRUE(bfs::remove(mTestRoot / entry.Name())); + EXPECT_TRUE(fs::remove(mTestRoot / entry.Name())); isFirstFile = false; } } else { @@ -132,7 +132,7 @@ TEST_F(FileSystemUtilUnittest, TestDirNormal) { #ifndef _MSC_VER TEST_F(FileSystemUtilUnittest, TestDirSymbolic) { { std::ofstream((mTestRoot / "f1").string()); } - bfs::create_directory(mTestRoot / "d1"); + fs::create_directory(mTestRoot / "d1"); std::map symbolics = {{"s1", "f1"}, {"s2", "d1"}}; for (auto& s : symbolics) { system(boost::str(boost::format("ln -s %s %s") % (mTestRoot / s.second) % (mTestRoot / s.first)).c_str()); @@ -171,7 +171,7 @@ TEST_F(FileSystemUtilUnittest, TestCheckExistance) { std::string subFileName = "file"; std::string subDirName = "dir"; std::ofstream((mTestRoot / subFileName).string()); - bfs::create_directory(mTestRoot / subDirName); + fs::create_directory(mTestRoot / subDirName); EXPECT_TRUE(CheckExistance((mTestRoot / subFileName).string())); EXPECT_TRUE(CheckExistance((mTestRoot / subDirName).string())); @@ -200,7 +200,7 @@ TEST_F(FileSystemUtilUnittest, TestPathStat_stat) { { auto dirPath = ((mTestRoot / "dir")).string(); - EXPECT_TRUE(bfs::create_directory(dirPath)); + EXPECT_TRUE(fs::create_directory(dirPath)); fsutil::PathStat stat; EXPECT_TRUE(fsutil::PathStat::stat(dirPath, stat)); DevInode devInode = stat.GetDevInode(); @@ -299,13 +299,13 @@ TEST_F(FileSystemUtilUnittest, TestFileReadOnlyOpen) { // After closing, the file is no more existing. auto* file = FileReadOnlyOpen(filePath.c_str(), "r"); EXPECT_TRUE(file != nullptr); - EXPECT_TRUE(bfs::remove(filePath)); + EXPECT_TRUE(fs::remove(filePath)); std::vector buffer(fileContent.length(), '\0'); auto nbytes = fread(buffer.data(), sizeof(char), fileContent.length(), file); EXPECT_EQ(nbytes, fileContent.length()); EXPECT_EQ(std::string(buffer.data(), nbytes), fileContent); fclose(file); - EXPECT_FALSE(bfs::exists(filePath)); + EXPECT_FALSE(fs::exists(filePath)); } TEST_F(FileSystemUtilUnittest, TestFileWriteOnlyOpen) { @@ -352,9 +352,9 @@ TEST_F(FileSystemUtilUnittest, TestFileWriteOnlyOpen) { in.close(); fclose(cfile); // Delete the file. - EXPECT_TRUE(bfs::remove(filePath)); + EXPECT_TRUE(fs::remove(filePath)); fclose(file); - EXPECT_FALSE(bfs::exists(filePath)); + EXPECT_FALSE(fs::exists(filePath)); } // Case #2: File is existing, open will truncate it. @@ -404,9 +404,9 @@ TEST_F(FileSystemUtilUnittest, TestFileAppendOpen) { EXPECT_EQ(std::string(buffer.data(), buffer.size()), fileContent + fileContent); fclose(in); // Delete the file. - EXPECT_TRUE(bfs::remove(filePath)); + EXPECT_TRUE(fs::remove(filePath)); fclose(file); - EXPECT_FALSE(bfs::exists(filePath)); + EXPECT_FALSE(fs::exists(filePath)); } // Case #3: Open existing file, check its cursor position. @@ -480,6 +480,60 @@ TEST_F(FileSystemUtilUnittest, TestPathJoin) { #endif } +TEST_F(FileSystemUtilUnittest, TestAbsolutePath) { +#ifndef _MSC_VER + // Expected values are based on boost::filesystem::absolute results. + + // Case 1: relative path "." with basepath + { + std::string result = AbsolutePath(".", "/usr/local/ilogtail"); + EXPECT_EQ(result, "/usr/local/ilogtail/."); + } + + // Case 2: relative path "./a.txt" with basepath + { + std::string result = AbsolutePath("./a.txt", "/usr/local/ilogtail"); + EXPECT_EQ(result, "/usr/local/ilogtail/./a.txt"); + } + + // Case 3: relative path without dot prefix + { + std::string result = AbsolutePath("a/b.txt", "/home/user"); + EXPECT_EQ(result, "/home/user/a/b.txt"); + } + + // Case 4: absolute path should be returned as-is + { + std::string result = AbsolutePath("/already/absolute", "/some/base"); + EXPECT_EQ(result, "/already/absolute"); + } + + // Case 5: basepath with trailing slash + { + std::string result = AbsolutePath("file.txt", "/base/path/"); + EXPECT_EQ(result, "/base/path/file.txt"); + } + + // Case 6: empty path + { + std::string result = AbsolutePath("", "/base/path"); + EXPECT_EQ(result, "/base/path"); + } + + // Case 7: path with ".." component (no normalization expected) + { + std::string result = AbsolutePath("../sibling/file.txt", "/base/path"); + EXPECT_EQ(result, "/base/path/../sibling/file.txt"); + } + + // Case 8: root path "/" as basepath + { + std::string result = AbsolutePath("data", "/"); + EXPECT_EQ(result, "/data"); + } +#endif +} + TEST_F(FileSystemUtilUnittest, TestReadFileContent) { auto filePath = (mTestRoot / "file").string(); diff --git a/core/unittest/common/LogFileOperatorUnittest.cpp b/core/unittest/common/LogFileOperatorUnittest.cpp index 88546f06ca..1da2af91aa 100644 --- a/core/unittest/common/LogFileOperatorUnittest.cpp +++ b/core/unittest/common/LogFileOperatorUnittest.cpp @@ -86,12 +86,12 @@ void LogFileOperatorUnittest::SetUpTestCase() { srand(time(NULL)); gRootDir = GetProcessExecutionDir(); gRootDir += "LogFileOperatorUnittest"; - bfs::remove_all(gRootDir); - bfs::create_directories(gRootDir); + fs::remove_all(gRootDir); + fs::create_directories(gRootDir); } void LogFileOperatorUnittest::TearDownTestCase() { - bfs::remove_all(gRootDir); + fs::remove_all(gRootDir); } std::string LogFileOperatorUnittest::GenerateData(size_t lines, size_t line_size) { @@ -137,7 +137,7 @@ void LogFileOperatorUnittest::TestOpen() { // non fuse, file no existed & existed { std::string file = gRootDir + PATH_SEPARATOR + gTestFile; - bfs::remove(file); + fs::remove(file); LogFileOperator logFileOp; diff --git a/core/unittest/common/ProcParserUnittest.cpp b/core/unittest/common/ProcParserUnittest.cpp index e3cbe4e76c..1d5bc57922 100644 --- a/core/unittest/common/ProcParserUnittest.cpp +++ b/core/unittest/common/ProcParserUnittest.cpp @@ -15,7 +15,6 @@ #include #include -#include #include #include #include @@ -71,22 +70,22 @@ class ProcParserUnittest : public ::testing::Test { protected: void SetUp() override { - mTestRoot = std::filesystem::path(GetProcessExecutionDir()) / "ProcParserUnittestDir"; + mTestRoot = fs::path(GetProcessExecutionDir()) / "ProcParserUnittestDir"; mProcDir = mTestRoot / "proc"; - std::filesystem::create_directories(mProcDir); + fs::create_directories(mProcDir); mParser = std::make_unique(mTestRoot.string()); } - void TearDown() override { std::filesystem::remove_all(mTestRoot); } + void TearDown() override { fs::remove_all(mTestRoot); } - void WriteStringWithNulls(const std::filesystem::path& path, const char* data, size_t size) { + void WriteStringWithNulls(const fs::path& path, const char* data, size_t size) { std::ofstream ofs(path, std::ios::binary); ofs.write(data, size); } void CreateProcTestFiles(int pid) { auto pidDir = mProcDir / std::to_string(pid); - std::filesystem::create_directories(pidDir); + fs::create_directories(pidDir); // Create cmdline file with null separators const char cmdline[] = {'t', 'e', 's', 't', ' ', 'p', 'r', 'o', 'g', 'r', 'a', @@ -117,22 +116,22 @@ class ProcParserUnittest : public ::testing::Test { << "0::/docker/1234567890abcdef1234567890abcdef1234567890abcdef1234567890abcdef"; // Create exe symlink - std::filesystem::create_directories(mTestRoot / "usr" / "bin"); + fs::create_directories(mTestRoot / "usr" / "bin"); std::ofstream(mTestRoot / "usr" / "bin" / "test_program") << "test program binary"; - std::filesystem::create_symlink(mTestRoot / "usr" / "bin" / "test_program", pidDir / "exe"); + fs::create_symlink(mTestRoot / "usr" / "bin" / "test_program", pidDir / "exe"); // Create cwd symlink - std::filesystem::create_directories(mTestRoot / "home" / "user"); - std::filesystem::create_symlink(mTestRoot / "home" / "user", pidDir / "cwd"); + fs::create_directories(mTestRoot / "home" / "user"); + fs::create_symlink(mTestRoot / "home" / "user", pidDir / "cwd"); // Create ns directory and net symlink - std::filesystem::create_directories(pidDir / "ns"); - std::filesystem::create_symlink("net:[4026531992]", pidDir / "ns" / "net"); + fs::create_directories(pidDir / "ns"); + fs::create_symlink("net:[4026531992]", pidDir / "ns" / "net"); } void CreateProcTestFile(int pid, const std::string& filename, const std::string& cgroupContent) { auto pidDir = mProcDir / std::to_string(pid); - std::filesystem::create_directories(pidDir); + fs::create_directories(pidDir); std::ofstream(pidDir / filename) << cgroupContent; } @@ -149,8 +148,8 @@ class ProcParserUnittest : public ::testing::Test { } private: - std::filesystem::path mTestRoot; - std::filesystem::path mProcDir; + fs::path mTestRoot; + fs::path mProcDir; std::unique_ptr mParser; }; diff --git a/core/unittest/common/StringToolsUnittest.cpp b/core/unittest/common/StringToolsUnittest.cpp index feb318eab8..8c840c6504 100644 --- a/core/unittest/common/StringToolsUnittest.cpp +++ b/core/unittest/common/StringToolsUnittest.cpp @@ -116,15 +116,6 @@ TEST_F(StringToolsUnittest, TestGetTopicNames) { } } -TEST_F(StringToolsUnittest, TestRemoveFilePathTrailingSlash) { - std::string filePath = "/aaa/aa/"; - RemoveFilePathTrailingSlash(filePath); - APSARA_TEST_EQUAL("/aaa/aa", filePath); - filePath = "/"; - RemoveFilePathTrailingSlash(filePath); - APSARA_TEST_EQUAL("/", filePath); -} - TEST_F(StringToolsUnittest, TestBoostRegexSearch) { { // ^(\[\d+-\d+-\d+\].*)|(\[\d+\].*) diff --git a/core/unittest/config/CollectionConfigUnittest.cpp b/core/unittest/config/CollectionConfigUnittest.cpp index 9e858c8bc4..4b41057047 100644 --- a/core/unittest/config/CollectionConfigUnittest.cpp +++ b/core/unittest/config/CollectionConfigUnittest.cpp @@ -44,7 +44,7 @@ class CollectionConfigUnittest : public testing::Test { private: const string configName = "test"; - const filesystem::path filepath = "/path/to/test"; + const fs::path filepath = "/path/to/test"; }; void CollectionConfigUnittest::HandleValidConfig() const { diff --git a/core/unittest/config/CommonConfigProviderUnittest.cpp b/core/unittest/config/CommonConfigProviderUnittest.cpp index 642f4670ad..4e21790204 100644 --- a/core/unittest/config/CommonConfigProviderUnittest.cpp +++ b/core/unittest/config/CommonConfigProviderUnittest.cpp @@ -88,7 +88,7 @@ class CommonConfigProviderUnittest : public ::testing::Test { void SetUp() override { if (BOOL_FLAG(logtail_mode)) { mRootDir = GetProcessExecutionDir(); - bfs::create_directories(mRootDir); + fs::create_directories(mRootDir); ilogtailConfigPath = mRootDir + PS + STRING_FLAG(ilogtail_config); std::ofstream fout(ilogtailConfigPath.c_str()); fout << "" << std::endl; @@ -96,8 +96,8 @@ class CommonConfigProviderUnittest : public ::testing::Test { MockCommonConfigProvider provider; provider.Init("common_v2"); provider.Stop(); - bfs::remove_all(provider.mContinuousPipelineConfigDir.string()); - bfs::remove_all(provider.mInstanceSourceDir.string()); + fs::remove_all(provider.mContinuousPipelineConfigDir.string()); + fs::remove_all(provider.mInstanceSourceDir.string()); } else { CreateAgentDir(); ilogtailConfigPath = GetAgentConfDir() + "/instance_config/local/loongcollector_config.json"; @@ -108,8 +108,8 @@ class CommonConfigProviderUnittest : public ::testing::Test { MockCommonConfigProvider provider; provider.Init("common_v2"); provider.Stop(); - bfs::remove_all(provider.mContinuousPipelineConfigDir.string()); - bfs::remove_all(provider.mInstanceSourceDir.string()); + fs::remove_all(provider.mContinuousPipelineConfigDir.string()); + fs::remove_all(provider.mInstanceSourceDir.string()); } } @@ -118,8 +118,8 @@ class CommonConfigProviderUnittest : public ::testing::Test { MockCommonConfigProvider provider; provider.Init("common_v2"); provider.Stop(); - bfs::remove_all(provider.mContinuousPipelineConfigDir.string()); - bfs::remove_all(provider.mInstanceSourceDir.string()); + fs::remove_all(provider.mContinuousPipelineConfigDir.string()); + fs::remove_all(provider.mInstanceSourceDir.string()); } void TestInit(); diff --git a/core/unittest/config/ConfigManagerUnittest.cpp b/core/unittest/config/ConfigManagerUnittest.cpp index 8f476768b3..e3bb66d6f1 100644 --- a/core/unittest/config/ConfigManagerUnittest.cpp +++ b/core/unittest/config/ConfigManagerUnittest.cpp @@ -16,7 +16,7 @@ #if defined(__linux__) #include #endif -#include + #include #include @@ -54,14 +54,14 @@ class ConfigManagerUnittest : public ::testing::Test { mTestRootDir.resize(mTestRootDir.size() - 1); } mTestRootDir += PATH_SEPARATOR + "ConfigManagerTest"; - std::filesystem::remove_all(mTestRootDir); - std::filesystem::create_directories(mTestRootDir); + fs::remove_all(mTestRootDir); + fs::create_directories(mTestRootDir); } void TearDown() override { CheckPointManager::Instance()->RemoveAllCheckPoint(); ConfigManager::GetInstance()->ClearFilePipelineMatchCache(); - std::filesystem::remove_all(mTestRootDir); + fs::remove_all(mTestRootDir); } FileDiscoveryConfig CreateTestConfig(const string& basePath) { @@ -89,7 +89,7 @@ void ConfigManagerUnittest::TestRegisterHandlersWithinDepthDanglingRef() { for (int i = 0; i < numSubdirs; ++i) { subdirs.push_back(baseDir + PS + "subdir" + ToString(i)); for (int j = 0; j < 5; ++j) { - std::filesystem::create_directories(subdirs.back() + PS + "nested" + ToString(j)); + fs::create_directories(subdirs.back() + PS + "nested" + ToString(j)); } } @@ -123,10 +123,10 @@ void ConfigManagerUnittest::TestRegisterHandlersWithinDepthDanglingRef() { int startIdx = numSubdirs / 10; int endIdx = numSubdirs * 9 / 10; for (int i = startIdx; i < endIdx; ++i) { - std::filesystem::remove_all(subdirs[i]); - std::filesystem::create_directories(subdirs[i]); + fs::remove_all(subdirs[i]); + fs::create_directories(subdirs[i]); for (int j = 0; j < 5; ++j) { - std::filesystem::create_directories(subdirs[i] + PS + "nested" + ToString(j)); + fs::create_directories(subdirs[i] + PS + "nested" + ToString(j)); } } LOG_INFO(sLogger, ("Subdirs recreated with new inode", "")("count", endIdx - startIdx)); diff --git a/core/unittest/config/ConfigUpdateUnittest.cpp b/core/unittest/config/ConfigUpdateUnittest.cpp index 3a918e0ed3..455de3805c 100644 --- a/core/unittest/config/ConfigUpdateUnittest.cpp +++ b/core/unittest/config/ConfigUpdateUnittest.cpp @@ -12,7 +12,6 @@ // See the License for the specific language governing permissions and // limitations under the License. -#include #include #include #include @@ -64,7 +63,7 @@ class ConfigUpdateUnittest : public testing::Test { } void SetUp() override { - filesystem::create_directories(configDir); + fs::create_directories(configDir); PipelineConfigWatcher::GetInstance()->AddSource(configDir.string()); #ifdef __ENTERPRISE__ builtinPipelineCnt = EnterpriseConfigProvider::GetInstance()->GetAllBuiltInPipelineConfigs().size(); @@ -75,7 +74,7 @@ class ConfigUpdateUnittest : public testing::Test { PipelineManagerMock::GetInstance()->ClearEnvironment(); TaskPipelineManager::GetInstance()->ClearEnvironment(); PipelineConfigWatcher::GetInstance()->ClearEnvironment(); - filesystem::remove_all(configDir); + fs::remove_all(configDir); } private: @@ -83,15 +82,15 @@ class ConfigUpdateUnittest : public testing::Test { void GenerateInitialConfigs() const; size_t builtinPipelineCnt = 0; - filesystem::path configDir = "./continuous_pipeline_config"; - vector pipelineConfigPaths = {configDir / "pipeline_invalid_format.json", - configDir / "pipeline_invalid_detail.json", - configDir / "pipeline_enabled_valid.json", - configDir / "pipeline_disabled_valid.json"}; - vector taskConfigPaths = {configDir / "task_invalid_format.json", - configDir / "task_invalid_detail.json", - configDir / "task_enabled_valid.json", - configDir / "task_disabled_valid.json"}; + fs::path configDir = "./continuous_pipeline_config"; + vector pipelineConfigPaths = {configDir / "pipeline_invalid_format.json", + configDir / "pipeline_invalid_detail.json", + configDir / "pipeline_enabled_valid.json", + configDir / "pipeline_disabled_valid.json"}; + vector taskConfigPaths = {configDir / "task_invalid_format.json", + configDir / "task_invalid_detail.json", + configDir / "task_enabled_valid.json", + configDir / "task_disabled_valid.json"}; const string invalidPipelineConfigWithInvalidFormat = R"({"inputs":{}})"; const string invalidPipelineConfigWithInvalidDetail = R"( { @@ -274,7 +273,7 @@ void ConfigUpdateUnittest::OnConfigDelete() const { APSARA_TEST_EQUAL(1U + builtinPipelineCnt, PipelineManagerMock::GetInstance()->GetAllConfigNames().size()); APSARA_TEST_EQUAL(1U, TaskPipelineManager::GetInstance()->GetAllPipelineNames().size()); - filesystem::remove_all(configDir); + fs::remove_all(configDir); auto diff = PipelineConfigWatcher::GetInstance()->CheckConfigDiff(); APSARA_TEST_TRUE(diff.first.HasDiff()); APSARA_TEST_TRUE(diff.first.mAdded.empty()); @@ -433,12 +432,12 @@ void ConfigUpdateUnittest::OnConfigUnchanged() const { GenerateInitialConfigs(); // mandatorily overwrite modify time in case of no update when file content remains the same. for (const auto& path : pipelineConfigPaths) { - filesystem::file_time_type fTime = filesystem::last_write_time(path); - filesystem::last_write_time(path, fTime + 1s); + fs::file_time_type fTime = fs::last_write_time(path); + fs::last_write_time(path, fTime + 1s); } for (const auto& path : taskConfigPaths) { - filesystem::file_time_type fTime = filesystem::last_write_time(path); - filesystem::last_write_time(path, fTime + 1s); + fs::file_time_type fTime = fs::last_write_time(path); + fs::last_write_time(path, fTime + 1s); } diff = PipelineConfigWatcher::GetInstance()->CheckConfigDiff(); APSARA_TEST_TRUE(diff.first.HasDiff()); diff --git a/core/unittest/config/ConfigWatcherUnittest.cpp b/core/unittest/config/ConfigWatcherUnittest.cpp index 452751c164..0df45efa4c 100644 --- a/core/unittest/config/ConfigWatcherUnittest.cpp +++ b/core/unittest/config/ConfigWatcherUnittest.cpp @@ -13,7 +13,6 @@ // limitations under the License. #include -#include #include #include "collection_pipeline/CollectionPipelineManager.h" @@ -58,12 +57,12 @@ class ConfigWatcherUnittest : public testing::Test { } private: - static const filesystem::path configDir; - static const filesystem::path instanceConfigDir; + static const fs::path configDir; + static const fs::path instanceConfigDir; }; -const filesystem::path ConfigWatcherUnittest::configDir = "./continuous_pipeline_config"; -const filesystem::path ConfigWatcherUnittest::instanceConfigDir = "./instance_config"; +const fs::path ConfigWatcherUnittest::configDir = "./continuous_pipeline_config"; +const fs::path ConfigWatcherUnittest::instanceConfigDir = "./instance_config"; void ConfigWatcherUnittest::InvalidConfigDirFound() const { { @@ -79,7 +78,7 @@ void ConfigWatcherUnittest::InvalidConfigDirFound() const { diff = PipelineConfigWatcher::GetInstance()->CheckConfigDiff(); APSARA_TEST_FALSE(diff.first.HasDiff()); APSARA_TEST_FALSE(diff.second.HasDiff()); - filesystem::remove_all("continuous_pipeline_config"); + fs::remove_all("continuous_pipeline_config"); } { InstanceConfigDiff diff = InstanceConfigWatcher::GetInstance()->CheckConfigDiff(); @@ -88,15 +87,15 @@ void ConfigWatcherUnittest::InvalidConfigDirFound() const { { ofstream fout("instance_config"); } diff = InstanceConfigWatcher::GetInstance()->CheckConfigDiff(); APSARA_TEST_FALSE(diff.HasDiff()); - filesystem::remove_all("instance_config"); + fs::remove_all("instance_config"); } } void ConfigWatcherUnittest::InvalidConfigFileFound() const { { - filesystem::create_directories(configDir); + fs::create_directories(configDir); - filesystem::create_directories(configDir / "dir"); + fs::create_directories(configDir / "dir"); { ofstream fout(configDir / "unsupported_extenstion.zip"); } { ofstream fout(configDir / "empty_file.json"); } { @@ -110,12 +109,12 @@ void ConfigWatcherUnittest::InvalidConfigFileFound() const { #endif APSARA_TEST_EQUAL(0U + builtinPipelineCnt, diff.first.mAdded.size()); APSARA_TEST_FALSE(diff.second.HasDiff()); - filesystem::remove_all(configDir); + fs::remove_all(configDir); } { - filesystem::create_directories(instanceConfigDir); + fs::create_directories(instanceConfigDir); - filesystem::create_directories(instanceConfigDir / "dir"); + fs::create_directories(instanceConfigDir / "dir"); { ofstream fout(instanceConfigDir / "unsupported_extenstion.zip"); } { ofstream fout(instanceConfigDir / "empty_file.json"); } { @@ -124,7 +123,7 @@ void ConfigWatcherUnittest::InvalidConfigFileFound() const { } InstanceConfigDiff diff = InstanceConfigWatcher::GetInstance()->CheckConfigDiff(); APSARA_TEST_FALSE(diff.HasDiff()); - filesystem::remove_all(instanceConfigDir); + fs::remove_all(instanceConfigDir); } } @@ -133,9 +132,9 @@ void ConfigWatcherUnittest::DuplicateConfigs() const { PipelineConfigWatcher::GetInstance()->AddSource("dir1"); PipelineConfigWatcher::GetInstance()->AddSource("dir2"); - filesystem::create_directories("continuous_pipeline_config"); - filesystem::create_directories("dir1"); - filesystem::create_directories("dir2"); + fs::create_directories("continuous_pipeline_config"); + fs::create_directories("dir1"); + fs::create_directories("dir2"); { ofstream fout("dir1/config.json"); @@ -163,17 +162,17 @@ void ConfigWatcherUnittest::DuplicateConfigs() const { APSARA_TEST_TRUE(diff.first.HasDiff()); APSARA_TEST_EQUAL(1U + builtinPipelineCnt, diff.first.mAdded.size()); - filesystem::remove_all("dir1"); - filesystem::remove_all("dir2"); - filesystem::remove_all("continuous_pipeline_config"); + fs::remove_all("dir1"); + fs::remove_all("dir2"); + fs::remove_all("continuous_pipeline_config"); } { InstanceConfigWatcher::GetInstance()->AddSource("dir1"); InstanceConfigWatcher::GetInstance()->AddSource("dir2"); - filesystem::create_directories("instance_config"); - filesystem::create_directories("dir1"); - filesystem::create_directories("dir2"); + fs::create_directories("instance_config"); + fs::create_directories("dir1"); + fs::create_directories("dir2"); { ofstream fout("dir1/config.json"); @@ -191,14 +190,14 @@ void ConfigWatcherUnittest::DuplicateConfigs() const { APSARA_TEST_TRUE(diff.HasDiff()); APSARA_TEST_EQUAL(1U, diff.mAdded.size()); - filesystem::remove_all("dir1"); - filesystem::remove_all("dir2"); - filesystem::remove_all("instance_config"); + fs::remove_all("dir1"); + fs::remove_all("dir2"); + fs::remove_all("instance_config"); } } void ConfigWatcherUnittest::IgnoreNewLowerPrioritySingletonConfig() const { - filesystem::create_directories("continuous_pipeline_config"); + fs::create_directories("continuous_pipeline_config"); size_t builtinPipelineCnt = 0; #ifdef __ENTERPRISE__ builtinPipelineCnt += EnterpriseConfigProvider::GetInstance()->GetAllBuiltInPipelineConfigs().size(); @@ -258,11 +257,11 @@ void ConfigWatcherUnittest::IgnoreNewLowerPrioritySingletonConfig() const { APSARA_TEST_TRUE(std::find(diff2.first.mIgnored.begin(), diff2.first.mIgnored.end(), std::string("b")) != diff2.first.mIgnored.end()); - filesystem::remove_all("continuous_pipeline_config"); + fs::remove_all("continuous_pipeline_config"); } void ConfigWatcherUnittest::IgnoreModifiedLowerPrioritySingletonConfig() const { - filesystem::create_directories("continuous_pipeline_config"); + fs::create_directories("continuous_pipeline_config"); size_t builtinPipelineCnt = 0; #ifdef __ENTERPRISE__ builtinPipelineCnt += EnterpriseConfigProvider::GetInstance()->GetAllBuiltInPipelineConfigs().size(); @@ -338,11 +337,11 @@ void ConfigWatcherUnittest::IgnoreModifiedLowerPrioritySingletonConfig() const { APSARA_TEST_TRUE(std::find(diff2.first.mIgnored.begin(), diff2.first.mIgnored.end(), std::string("b")) != diff2.first.mIgnored.end()); - filesystem::remove_all("continuous_pipeline_config"); + fs::remove_all("continuous_pipeline_config"); } void ConfigWatcherUnittest::HigherPriorityOverrideLowerPrioritySingletonConfig() const { - filesystem::create_directories("continuous_pipeline_config"); + fs::create_directories("continuous_pipeline_config"); size_t builtinPipelineCnt = 0; #ifdef __ENTERPRISE__ builtinPipelineCnt += EnterpriseConfigProvider::GetInstance()->GetAllBuiltInPipelineConfigs().size(); @@ -425,7 +424,7 @@ void ConfigWatcherUnittest::HigherPriorityOverrideLowerPrioritySingletonConfig() [](const CollectionConfig& c) { return c.mName == "a"; }) != diff2.first.mAdded.end()); - filesystem::remove_all("continuous_pipeline_config"); + fs::remove_all("continuous_pipeline_config"); } UNIT_TEST_CASE(ConfigWatcherUnittest, InvalidConfigDirFound) diff --git a/core/unittest/config/OnetimeConfigInfoManagerUnittest.cpp b/core/unittest/config/OnetimeConfigInfoManagerUnittest.cpp index 9ef7e17a31..a82dec9692 100644 --- a/core/unittest/config/OnetimeConfigInfoManagerUnittest.cpp +++ b/core/unittest/config/OnetimeConfigInfoManagerUnittest.cpp @@ -29,12 +29,12 @@ class OnetimeConfigInfoManagerUnittest : public testing::Test { void TestDumpCheckpointFile() const; protected: - static void SetUpTestCase() { sManager->mCheckpointFilePath = filesystem::path(".") / "onetime_config_info.json"; } + static void SetUpTestCase() { sManager->mCheckpointFilePath = fs::path(".") / "onetime_config_info.json"; } void TearDown() override { sManager->Clear(); error_code ec; - filesystem::remove(sManager->mCheckpointFilePath, ec); + fs::remove(sManager->mCheckpointFilePath, ec); } private: @@ -50,9 +50,9 @@ void OnetimeConfigInfoManagerUnittest::TestLoadCheckpointFile() const { } { // invalid checkpoint file - filesystem::create_directories("onetime_config_info.json"); + fs::create_directories("onetime_config_info.json"); APSARA_TEST_FALSE(sManager->LoadCheckpointFile()); - filesystem::remove_all("onetime_config_info.json"); + fs::remove_all("onetime_config_info.json"); } { // empty content @@ -167,37 +167,22 @@ void OnetimeConfigInfoManagerUnittest::TestGetOnetimeConfigStatusFromCheckpoint( } void OnetimeConfigInfoManagerUnittest::TestUpdateConfig() const { - filesystem::create_directories("test_config"); + fs::create_directories("test_config"); { ofstream fout("test_config/test_config_1.json"); } { ofstream fout("test_config/test_config_2.json"); } // restart - APSARA_TEST_TRUE(sManager->UpdateConfig("test_config_1", - ConfigType::Collection, - filesystem::path("test_config/test_config_1.json"), - 1, - 1000000000, - 0, - 3600)); - APSARA_TEST_TRUE(sManager->UpdateConfig("test_config_2", - ConfigType::Collection, - filesystem::path("test_config/test_config_2.json"), - 2, - 1500000000, - 0, - 1800)); - APSARA_TEST_TRUE(sManager->UpdateConfig("test_config_3", - ConfigType::Collection, - filesystem::path("test_config/test_config_3.json"), - 3, - 4000000000, - 0, - 7200)); + APSARA_TEST_TRUE(sManager->UpdateConfig( + "test_config_1", ConfigType::Collection, fs::path("test_config/test_config_1.json"), 1, 1000000000, 0, 3600)); + APSARA_TEST_TRUE(sManager->UpdateConfig( + "test_config_2", ConfigType::Collection, fs::path("test_config/test_config_2.json"), 2, 1500000000, 0, 1800)); + APSARA_TEST_TRUE(sManager->UpdateConfig( + "test_config_3", ConfigType::Collection, fs::path("test_config/test_config_3.json"), 3, 4000000000, 0, 7200)); APSARA_TEST_EQUAL(3U, sManager->mConfigInfoMap.size()); { const auto& info = sManager->mConfigInfoMap.at("test_config_1"); APSARA_TEST_EQUAL(ConfigType::Collection, info.mType); - APSARA_TEST_EQUAL(filesystem::path("test_config/test_config_1.json"), info.mFilepath); + APSARA_TEST_EQUAL(fs::path("test_config/test_config_1.json"), info.mFilepath); APSARA_TEST_EQUAL(1U, info.mConfigHash); APSARA_TEST_EQUAL(1000000000U, info.mExpireTime); APSARA_TEST_EQUAL(0U, info.mInputsHash); @@ -206,7 +191,7 @@ void OnetimeConfigInfoManagerUnittest::TestUpdateConfig() const { { const auto& info = sManager->mConfigInfoMap.at("test_config_2"); APSARA_TEST_EQUAL(ConfigType::Collection, info.mType); - APSARA_TEST_EQUAL(filesystem::path("test_config/test_config_2.json"), info.mFilepath); + APSARA_TEST_EQUAL(fs::path("test_config/test_config_2.json"), info.mFilepath); APSARA_TEST_EQUAL(2U, info.mConfigHash); APSARA_TEST_EQUAL(1500000000U, info.mExpireTime); APSARA_TEST_EQUAL(0U, info.mInputsHash); @@ -215,7 +200,7 @@ void OnetimeConfigInfoManagerUnittest::TestUpdateConfig() const { { const auto& info = sManager->mConfigInfoMap.at("test_config_3"); APSARA_TEST_EQUAL(ConfigType::Collection, info.mType); - APSARA_TEST_EQUAL(filesystem::path("test_config/test_config_3.json"), info.mFilepath); + APSARA_TEST_EQUAL(fs::path("test_config/test_config_3.json"), info.mFilepath); APSARA_TEST_EQUAL(3U, info.mConfigHash); APSARA_TEST_EQUAL(4000000000U, info.mExpireTime); APSARA_TEST_EQUAL(0U, info.mInputsHash); @@ -223,19 +208,14 @@ void OnetimeConfigInfoManagerUnittest::TestUpdateConfig() const { } // update - APSARA_TEST_TRUE(sManager->UpdateConfig("test_config_1", - ConfigType::Collection, - filesystem::path("test_config/test_config_1.json"), - 1, - 1200000000, - 0, - 3600)); + APSARA_TEST_TRUE(sManager->UpdateConfig( + "test_config_1", ConfigType::Collection, fs::path("test_config/test_config_1.json"), 1, 1200000000, 0, 3600)); APSARA_TEST_FALSE(sManager->RemoveConfig("test_config_4")); APSARA_TEST_EQUAL(3U, sManager->mConfigInfoMap.size()); { const auto& info = sManager->mConfigInfoMap.at("test_config_1"); APSARA_TEST_EQUAL(ConfigType::Collection, info.mType); - APSARA_TEST_EQUAL(filesystem::path("test_config/test_config_1.json"), info.mFilepath); + APSARA_TEST_EQUAL(fs::path("test_config/test_config_1.json"), info.mFilepath); APSARA_TEST_EQUAL(1U, info.mConfigHash); APSARA_TEST_EQUAL(1200000000U, info.mExpireTime); APSARA_TEST_EQUAL(0U, info.mInputsHash); @@ -243,13 +223,13 @@ void OnetimeConfigInfoManagerUnittest::TestUpdateConfig() const { } // delete timeout config - filesystem::remove("test_config/test_config_1.json"); + fs::remove("test_config/test_config_1.json"); sManager->DeleteTimeoutConfigFiles(); APSARA_TEST_EQUAL(1U, sManager->mConfigInfoMap.size()); APSARA_TEST_NOT_EQUAL(sManager->mConfigInfoMap.end(), sManager->mConfigInfoMap.find("test_config_3")); - APSARA_TEST_FALSE(filesystem::exists("test_config/test_config_2.json")); + APSARA_TEST_FALSE(fs::exists("test_config/test_config_2.json")); - filesystem::remove_all("test_config"); + fs::remove_all("test_config"); } void OnetimeConfigInfoManagerUnittest::TestDumpCheckpointFile() const { @@ -288,29 +268,14 @@ void OnetimeConfigInfoManagerUnittest::TestDumpCheckpointFile() const { uint32_t expireTime = 0; sManager->GetOnetimeConfigStatus("test_config_1", 1U, false, 0U, 3600U, &expireTime); - sManager->UpdateConfig("test_config_1", - ConfigType::Collection, - filesystem::path("test_config/test_config_1.json"), - 1, - 2000000000, - 0, - 3600); + sManager->UpdateConfig( + "test_config_1", ConfigType::Collection, fs::path("test_config/test_config_1.json"), 1, 2000000000, 0, 3600); sManager->GetOnetimeConfigStatus("test_config_2", 2U, false, 0U, 1800U, &expireTime); sManager->GetOnetimeConfigStatus("test_config_3", 4U, false, 0U, 7200U, &expireTime); - sManager->UpdateConfig("test_config_3", - ConfigType::Collection, - filesystem::path("test_config/test_config_3.json"), - 4, - 2200000000, - 0, - 7200); - sManager->UpdateConfig("test_config_5", - ConfigType::Collection, - filesystem::path("test_config/test_config_5.json"), - 5, - 2100000000, - 0, - 600); + sManager->UpdateConfig( + "test_config_3", ConfigType::Collection, fs::path("test_config/test_config_3.json"), 4, 2200000000, 0, 7200); + sManager->UpdateConfig( + "test_config_5", ConfigType::Collection, fs::path("test_config/test_config_5.json"), 5, 2100000000, 0, 600); sManager->DumpCheckpointFile(); diff --git a/core/unittest/config/OnetimeConfigUpdateUnittest.cpp b/core/unittest/config/OnetimeConfigUpdateUnittest.cpp index 6b5eddb684..61eaadea04 100644 --- a/core/unittest/config/OnetimeConfigUpdateUnittest.cpp +++ b/core/unittest/config/OnetimeConfigUpdateUnittest.cpp @@ -37,7 +37,7 @@ class OnetimeConfigUpdateUnittest : public testing::Test { static void TearDownTestCase() { PluginRegistry::GetInstance()->UnloadPlugins(); } void SetUp() override { - filesystem::create_directories(mConfigDir); + fs::create_directories(mConfigDir); PipelineConfigWatcher::GetInstance()->AddSource(mConfigDir.string()); } @@ -46,15 +46,15 @@ class OnetimeConfigUpdateUnittest : public testing::Test { // PipelineManagerMock::GetInstance()->ClearEnvironment(); PipelineConfigWatcher::GetInstance()->ClearEnvironment(); sConfigManager->Clear(); - filesystem::remove_all(mConfigDir); + fs::remove_all(mConfigDir); error_code ec; - filesystem::remove(sConfigManager->mCheckpointFilePath, ec); + fs::remove(sConfigManager->mCheckpointFilePath, ec); } private: static OnetimeConfigInfoManager* sConfigManager; - filesystem::path mConfigDir = "continuous_pipeline_config"; + fs::path mConfigDir = "continuous_pipeline_config"; }; OnetimeConfigInfoManager* OnetimeConfigUpdateUnittest::sConfigManager = OnetimeConfigInfoManager::GetInstance(); @@ -238,20 +238,24 @@ void OnetimeConfigUpdateUnittest::OnCollectionConfigUpdate() const { auto diff = PipelineConfigWatcher::GetInstance()->CheckConfigDiff(); APSARA_TEST_TRUE(diff.first.HasDiff()); + time_t beforeUpdate = time(nullptr); CollectionPipelineManager::GetInstance()->UpdatePipelines(diff.first); sConfigManager->DumpCheckpointFile(); + time_t afterUpdate = time(nullptr); APSARA_TEST_EQUAL(3U, sConfigManager->mConfigInfoMap.size()); { const auto& item = sConfigManager->mConfigInfoMap.at("new_config"); - APSARA_TEST_EQUAL(time(nullptr) + 3600U, item.mExpireTime); + APSARA_TEST_TRUE(item.mExpireTime >= static_cast(beforeUpdate) + 3600U + && item.mExpireTime <= static_cast(afterUpdate) + 3600U + 1); APSARA_TEST_EQUAL(configHash["new_config.json"], item.mConfigHash); APSARA_TEST_EQUAL(ConfigType::Collection, item.mType); APSARA_TEST_EQUAL(mConfigDir / filenames[0], item.mFilepath); } { const auto& item = sConfigManager->mConfigInfoMap.at("changed_config"); - APSARA_TEST_EQUAL(time(nullptr) + 7200U, item.mExpireTime); + APSARA_TEST_TRUE(item.mExpireTime >= static_cast(beforeUpdate) + 7200U + && item.mExpireTime <= static_cast(afterUpdate) + 7200U + 1); APSARA_TEST_EQUAL(configHash["changed_config.json"], item.mConfigHash); APSARA_TEST_EQUAL(ConfigType::Collection, item.mType); APSARA_TEST_EQUAL(mConfigDir / filenames[1], item.mFilepath); @@ -304,13 +308,13 @@ void OnetimeConfigUpdateUnittest::OnCollectionConfigUpdate() const { })"}; vector filenames = {"new_config.json", "old_config.json"}; for (size_t i = 0; i < configDetails.size(); ++i) { - filesystem::path filePath = mConfigDir / filenames[i]; + fs::path filePath = mConfigDir / filenames[i]; ofstream fout(filePath, ios::binary); fout << configDetails[i]; fout.close(); // 强制更新文件修改时间 - filesystem::file_time_type newTime = filesystem::file_time_type::clock::now(); - filesystem::last_write_time(filePath, newTime); + fs::file_time_type newTime = fs::file_time_type::clock::now(); + fs::last_write_time(filePath, newTime); // 添加一个小延迟确保文件系统更新 this_thread::sleep_for(chrono::milliseconds(10)); } @@ -319,7 +323,7 @@ void OnetimeConfigUpdateUnittest::OnCollectionConfigUpdate() const { fout << unusedConfigDetail; fout.close(); } - filesystem::remove(mConfigDir / "changed_config.json"); + fs::remove(mConfigDir / "changed_config.json"); // compute config hash, inputs hash and excution timeout for (size_t i = 0; i < configDetails.size(); ++i) { @@ -333,20 +337,25 @@ void OnetimeConfigUpdateUnittest::OnCollectionConfigUpdate() const { auto diff = PipelineConfigWatcher::GetInstance()->CheckConfigDiff(); APSARA_TEST_TRUE(diff.first.HasDiff()); + time_t beforeUpdate = time(nullptr); CollectionPipelineManager::GetInstance()->UpdatePipelines(diff.first); sConfigManager->DumpCheckpointFile(); + time_t afterUpdate = time(nullptr); APSARA_TEST_EQUAL(3U, sConfigManager->mConfigInfoMap.size()); { const auto& item = sConfigManager->mConfigInfoMap.at("new_config"); - APSARA_TEST_EQUAL(time(nullptr) + 1000U, item.mExpireTime); + // mExpireTime 在 UpdatePipelines 内部用当时的 time(nullptr) 设置,允许 1 秒执行误差 + APSARA_TEST_TRUE(item.mExpireTime >= static_cast(beforeUpdate) + 1000U + && item.mExpireTime <= static_cast(afterUpdate) + 1000U + 1); APSARA_TEST_EQUAL(configHash["new_config.json"], item.mConfigHash); APSARA_TEST_EQUAL(ConfigType::Collection, item.mType); APSARA_TEST_EQUAL(mConfigDir / filenames[0], item.mFilepath); } { const auto& item = sConfigManager->mConfigInfoMap.at("old_config"); - APSARA_TEST_EQUAL(time(nullptr) + 1200U, item.mExpireTime); + APSARA_TEST_TRUE(item.mExpireTime >= static_cast(beforeUpdate) + 1200U + && item.mExpireTime <= static_cast(afterUpdate) + 1200U + 1); APSARA_TEST_EQUAL(configHash["old_config.json"], item.mConfigHash); APSARA_TEST_EQUAL(ConfigType::Collection, item.mType); APSARA_TEST_EQUAL(mConfigDir / filenames[1], item.mFilepath); diff --git a/core/unittest/config/PipelineConfigUnittest.cpp b/core/unittest/config/PipelineConfigUnittest.cpp index cc8320e68d..9f8bfaccbc 100644 --- a/core/unittest/config/PipelineConfigUnittest.cpp +++ b/core/unittest/config/PipelineConfigUnittest.cpp @@ -22,7 +22,7 @@ using namespace std; namespace logtail { struct ConfigMock : public PipelineConfig { - ConfigMock(const string& name, unique_ptr&& detail, const filesystem::path& filepath) + ConfigMock(const string& name, unique_ptr&& detail, const fs::path& filepath) : PipelineConfig(name, std::move(detail), filepath) {} bool Parse() override { return true; } @@ -36,7 +36,7 @@ class PipelineConfigUnittest : public testing::Test { void TearDown() override { sConfigManager->Clear(); error_code ec; - filesystem::remove(sConfigManager->mCheckpointFilePath, ec); + fs::remove(sConfigManager->mCheckpointFilePath, ec); } private: @@ -47,7 +47,7 @@ OnetimeConfigInfoManager* PipelineConfigUnittest::sConfigManager = OnetimeConfig void PipelineConfigUnittest::TestOnetimeConfig() const { - filesystem::path filepath("test_config.json"); + fs::path filepath("test_config.json"); { // enable is removed auto configJson = make_unique(); @@ -135,7 +135,7 @@ void PipelineConfigUnittest::TestOnetimeConfig() const { } sConfigManager->LoadCheckpointFile(); - filesystem::create_directories("config"); + fs::create_directories("config"); { // new config { ofstream fout("config/new_config.json", ios::binary); } @@ -172,7 +172,7 @@ void PipelineConfigUnittest::TestOnetimeConfig() const { APSARA_TEST_FALSE(config.GetExpireTimeIfOneTime((*config.mDetail)["global"])); APSARA_TEST_EQUAL(sConfigManager->mConfigCheckpointMap.end(), sConfigManager->mConfigCheckpointMap.find("obsolete_config_1")); - APSARA_TEST_FALSE(filesystem::exists("obsolete_config_1.json")); + APSARA_TEST_FALSE(fs::exists("obsolete_config_1.json")); } { // obsolete config, config file existed @@ -184,7 +184,7 @@ void PipelineConfigUnittest::TestOnetimeConfig() const { } // 使用错误处理来安全地删除目录 error_code ec; - filesystem::remove_all("config", ec); + fs::remove_all("config", ec); if (ec) { // 如果删除失败,记录警告但不抛出异常 LOG_WARNING(sLogger, ("failed to remove config directory", ec.message())); diff --git a/core/unittest/config/SingletonInputCollectionConfigUpdateUnittest.cpp b/core/unittest/config/SingletonInputCollectionConfigUpdateUnittest.cpp index 2c79193e05..6a9d207876 100644 --- a/core/unittest/config/SingletonInputCollectionConfigUpdateUnittest.cpp +++ b/core/unittest/config/SingletonInputCollectionConfigUpdateUnittest.cpp @@ -57,9 +57,9 @@ class SingletonInputCollectionConfigUpdateUnittest : public testing::Test { private: void PrepareConfig() { - filesystem::create_directories(configDir1); + fs::create_directories(configDir1); PipelineConfigWatcher::GetInstance()->AddSource(configDir1.string()); - filesystem::create_directories(configDir2); + fs::create_directories(configDir2); PipelineConfigWatcher::GetInstance()->AddSource(configDir2.string()); #ifdef __ENTERPRISE__ builtinPipelineCnt = EnterpriseConfigProvider::GetInstance()->GetAllBuiltInPipelineConfigs().size(); @@ -69,13 +69,13 @@ class SingletonInputCollectionConfigUpdateUnittest : public testing::Test { void ClearConfig() { PipelineManagerMock::GetInstance()->ClearEnvironment(); PipelineConfigWatcher::GetInstance()->ClearEnvironment(); - filesystem::remove_all(configDir1); - filesystem::remove_all(configDir2); + fs::remove_all(configDir1); + fs::remove_all(configDir2); } size_t builtinPipelineCnt = 0; - filesystem::path configDir1 = "./continuous_pipeline_config1"; - filesystem::path configDir2 = "./continuous_pipeline_config2"; + fs::path configDir1 = "./continuous_pipeline_config1"; + fs::path configDir2 = "./continuous_pipeline_config2"; const std::string greaterPriorityConfig = R"( { @@ -340,8 +340,8 @@ void SingletonInputCollectionConfigUpdateUnittest::TestLoadAddedSingletonConfig( fout.open(configDir1 / "test1.json", ios::trunc); fout << greaterPriorityConfig; fout.close(); - filesystem::remove(configDir2 / "test2.json"); - filesystem::remove(configDir2 / "test-other.json"); + fs::remove(configDir2 / "test2.json"); + fs::remove(configDir2 / "test-other.json"); diff = PipelineConfigWatcher::GetInstance()->CheckConfigDiff(); APSARA_TEST_EQUAL_FATAL(1U, diff.first.mAdded.size()); APSARA_TEST_EQUAL_FATAL(0U, diff.first.mModified.size()); @@ -370,8 +370,8 @@ void SingletonInputCollectionConfigUpdateUnittest::TestLoadAddedSingletonConfig( fout.open(configDir1 / "test1.json", ios::trunc); fout << lessPriorityConfig; fout.close(); - filesystem::remove(configDir2 / "test2.json"); - filesystem::remove(configDir2 / "test-other.json"); + fs::remove(configDir2 / "test2.json"); + fs::remove(configDir2 / "test-other.json"); diff = PipelineConfigWatcher::GetInstance()->CheckConfigDiff(); APSARA_TEST_EQUAL_FATAL(1U, diff.first.mAdded.size()); APSARA_TEST_EQUAL_FATAL(0U, diff.first.mModified.size()); @@ -710,8 +710,8 @@ void SingletonInputCollectionConfigUpdateUnittest::TestLoadModifiedSingletonConf fout.open(configDir1 / "test1.json", ios::trunc); fout << modifiedGreaterPriorityConfig; fout.close(); - filesystem::remove(configDir2 / "test2.json"); - filesystem::remove(configDir2 / "test-other.json"); + fs::remove(configDir2 / "test2.json"); + fs::remove(configDir2 / "test-other.json"); diff = PipelineConfigWatcher::GetInstance()->CheckConfigDiff(); APSARA_TEST_EQUAL_FATAL(0U, diff.first.mAdded.size()); APSARA_TEST_EQUAL_FATAL(1U, diff.first.mModified.size()); @@ -744,8 +744,8 @@ void SingletonInputCollectionConfigUpdateUnittest::TestLoadModifiedSingletonConf fout.open(configDir1 / "test1.json", ios::trunc); fout << modifiedLessPriorityConfig; fout.close(); - filesystem::remove(configDir2 / "test2.json"); - filesystem::remove(configDir2 / "test-other.json"); + fs::remove(configDir2 / "test2.json"); + fs::remove(configDir2 / "test-other.json"); diff = PipelineConfigWatcher::GetInstance()->CheckConfigDiff(); APSARA_TEST_EQUAL_FATAL(1U, diff.first.mAdded.size()); APSARA_TEST_EQUAL_FATAL(0U, diff.first.mModified.size()); @@ -819,7 +819,7 @@ void SingletonInputCollectionConfigUpdateUnittest::TestLoadModifiedSingletonConf fout.open(configDir1 / "test1.json", ios::trunc); fout << modifiedLessPriorityConfig; fout.close(); - filesystem::remove(configDir2 / "test2.json"); + fs::remove(configDir2 / "test2.json"); diff = PipelineConfigWatcher::GetInstance()->CheckConfigDiff(); APSARA_TEST_EQUAL_FATAL(1U, diff.first.mAdded.size()); APSARA_TEST_EQUAL_FATAL(0U, diff.first.mModified.size()); @@ -912,7 +912,7 @@ void SingletonInputCollectionConfigUpdateUnittest::TestLoadRemovedSingletonConfi APSARA_TEST_EQUAL_FATAL(1U + builtinPipelineCnt, PipelineManagerMock::GetInstance()->GetAllConfigNames().size()); - filesystem::remove(configDir1 / "test1.json"); + fs::remove(configDir1 / "test1.json"); fout.open(configDir2 / "test2.json", ios::trunc); fout << lessPriorityConfig; fout.close(); @@ -942,7 +942,7 @@ void SingletonInputCollectionConfigUpdateUnittest::TestLoadRemovedSingletonConfi APSARA_TEST_EQUAL_FATAL(1U + builtinPipelineCnt, PipelineManagerMock::GetInstance()->GetAllConfigNames().size()); - filesystem::remove(configDir1 / "test1.json"); + fs::remove(configDir1 / "test1.json"); fout.open(configDir2 / "test2.json", ios::trunc); fout << greaterPriorityConfig; fout.close(); @@ -987,7 +987,7 @@ void SingletonInputCollectionConfigUpdateUnittest::TestLoadRemovedSingletonConfi PipelineManagerMock::GetInstance()->GetAllConfigNames().size()); this_thread::sleep_for(chrono::milliseconds(1)); - filesystem::remove(configDir1 / "test1.json"); + fs::remove(configDir1 / "test1.json"); fout.open(configDir2 / "test2.json", ios::trunc); fout << modifiedLessPriorityConfig; fout.close(); @@ -1024,7 +1024,7 @@ void SingletonInputCollectionConfigUpdateUnittest::TestLoadRemovedSingletonConfi PipelineManagerMock::GetInstance()->GetAllConfigNames().size()); this_thread::sleep_for(chrono::milliseconds(1)); - filesystem::remove(configDir1 / "test1.json"); + fs::remove(configDir1 / "test1.json"); fout.open(configDir2 / "test2.json", ios::trunc); fout << modifiedGreaterPriorityConfig; fout.close(); @@ -1068,9 +1068,9 @@ void SingletonInputCollectionConfigUpdateUnittest::TestLoadRemovedSingletonConfi APSARA_TEST_EQUAL_FATAL(2U + builtinPipelineCnt, PipelineManagerMock::GetInstance()->GetAllConfigNames().size()); - filesystem::remove(configDir1 / "test1.json"); - filesystem::remove(configDir2 / "test2.json"); - filesystem::remove(configDir2 / "test-other.json"); + fs::remove(configDir1 / "test1.json"); + fs::remove(configDir2 / "test2.json"); + fs::remove(configDir2 / "test-other.json"); diff = PipelineConfigWatcher::GetInstance()->CheckConfigDiff(); APSARA_TEST_EQUAL_FATAL(0U, diff.first.mAdded.size()); APSARA_TEST_EQUAL_FATAL(0U, diff.first.mModified.size()); @@ -1097,9 +1097,9 @@ void SingletonInputCollectionConfigUpdateUnittest::TestLoadRemovedSingletonConfi APSARA_TEST_EQUAL_FATAL(2U + builtinPipelineCnt, PipelineManagerMock::GetInstance()->GetAllConfigNames().size()); - filesystem::remove(configDir1 / "test1.json"); - filesystem::remove(configDir2 / "test2.json"); - filesystem::remove(configDir2 / "test-other.json"); + fs::remove(configDir1 / "test1.json"); + fs::remove(configDir2 / "test2.json"); + fs::remove(configDir2 / "test-other.json"); diff = PipelineConfigWatcher::GetInstance()->CheckConfigDiff(); APSARA_TEST_EQUAL_FATAL(0U, diff.first.mAdded.size()); APSARA_TEST_EQUAL_FATAL(0U, diff.first.mModified.size()); @@ -1134,7 +1134,7 @@ void SingletonInputCollectionConfigUpdateUnittest::TestLoadRemovedSingletonConfi APSARA_TEST_EQUAL_FATAL(2U + builtinPipelineCnt, PipelineManagerMock::GetInstance()->GetAllConfigNames().size()); - filesystem::remove(configDir1 / "test1.json"); + fs::remove(configDir1 / "test1.json"); diff = PipelineConfigWatcher::GetInstance()->CheckConfigDiff(); APSARA_TEST_EQUAL_FATAL(1U, diff.first.mAdded.size()); APSARA_TEST_EQUAL_FATAL(0U, diff.first.mModified.size()); @@ -1164,7 +1164,7 @@ void SingletonInputCollectionConfigUpdateUnittest::TestLoadRemovedSingletonConfi APSARA_TEST_EQUAL_FATAL(2U + builtinPipelineCnt, PipelineManagerMock::GetInstance()->GetAllConfigNames().size()); - filesystem::remove(configDir1 / "test1.json"); + fs::remove(configDir1 / "test1.json"); diff = PipelineConfigWatcher::GetInstance()->CheckConfigDiff(); APSARA_TEST_EQUAL_FATAL(0U, diff.first.mAdded.size()); APSARA_TEST_EQUAL_FATAL(0U, diff.first.mModified.size()); @@ -1410,8 +1410,8 @@ void SingletonInputCollectionConfigUpdateUnittest::TestLoadUnchangedSingletonCon APSARA_TEST_EQUAL_FATAL(2U + builtinPipelineCnt, PipelineManagerMock::GetInstance()->GetAllConfigNames().size()); - filesystem::remove(configDir2 / "test2.json"); - filesystem::remove(configDir2 / "test-other.json"); + fs::remove(configDir2 / "test2.json"); + fs::remove(configDir2 / "test-other.json"); diff = PipelineConfigWatcher::GetInstance()->CheckConfigDiff(); APSARA_TEST_EQUAL_FATAL(0U, diff.first.mAdded.size()); APSARA_TEST_EQUAL_FATAL(0U, diff.first.mModified.size()); @@ -1440,8 +1440,8 @@ void SingletonInputCollectionConfigUpdateUnittest::TestLoadUnchangedSingletonCon APSARA_TEST_EQUAL_FATAL(2U + builtinPipelineCnt, PipelineManagerMock::GetInstance()->GetAllConfigNames().size()); - filesystem::remove(configDir2 / "test2.json"); - filesystem::remove(configDir2 / "test-other.json"); + fs::remove(configDir2 / "test2.json"); + fs::remove(configDir2 / "test-other.json"); diff = PipelineConfigWatcher::GetInstance()->CheckConfigDiff(); APSARA_TEST_EQUAL_FATAL(1U, diff.first.mAdded.size()); APSARA_TEST_EQUAL_FATAL(0U, diff.first.mModified.size()); diff --git a/core/unittest/config/TaskConfigUnittest.cpp b/core/unittest/config/TaskConfigUnittest.cpp index a112dc6836..0bfc5d3459 100644 --- a/core/unittest/config/TaskConfigUnittest.cpp +++ b/core/unittest/config/TaskConfigUnittest.cpp @@ -39,7 +39,7 @@ class TaskConfigUnittest : public testing::Test { private: const string configName = "test"; - const filesystem::path filepath = "/path/to/test"; + const fs::path filepath = "/path/to/test"; }; void TaskConfigUnittest::HandleValidConfig() const { diff --git a/core/unittest/container_manager/ContainerManagerUnittest.cpp b/core/unittest/container_manager/ContainerManagerUnittest.cpp index f67e6628f9..74b8239804 100644 --- a/core/unittest/container_manager/ContainerManagerUnittest.cpp +++ b/core/unittest/container_manager/ContainerManagerUnittest.cpp @@ -630,9 +630,9 @@ void ContainerManagerUnittest::TestLoadContainerInfoFromDetailFormat() const { EXPECT_FALSE(foundImageName); // Verify config diffs are created - EXPECT_TRUE(containerManager.mConfigContainerDiffMap.find("##1.0##config1") + EXPECT_TRUE(containerManager.mConfigContainerDiffMap.find(std::make_pair(std::string("##1.0##config1"), size_t(0))) != containerManager.mConfigContainerDiffMap.end()); - EXPECT_TRUE(containerManager.mConfigContainerDiffMap.find("##1.0##config2") + EXPECT_TRUE(containerManager.mConfigContainerDiffMap.find(std::make_pair(std::string("##1.0##config2"), size_t(0))) != containerManager.mConfigContainerDiffMap.end()); } @@ -713,10 +713,10 @@ void ContainerManagerUnittest::TestLoadContainerInfoFromDetailFormatWithTags() c EXPECT_TRUE(foundContainerIp); EXPECT_TRUE(foundPodUid); - // Verify config diff is created - EXPECT_TRUE( - containerManager.mConfigContainerDiffMap.find("##1.0##k8s-log-c84f2ea7e4b1641c882a67ea014247720$file-test") - != containerManager.mConfigContainerDiffMap.end()); + // Verify config diff is created (key is pair, inputIndex defaults to 0) + EXPECT_TRUE(containerManager.mConfigContainerDiffMap.find(std::make_pair( + std::string("##1.0##k8s-log-c84f2ea7e4b1641c882a67ea014247720$file-test"), size_t(0))) + != containerManager.mConfigContainerDiffMap.end()); } void ContainerManagerUnittest::TestLoadContainerInfoFromDetailFormatWithMounts() const { @@ -1591,9 +1591,9 @@ void ContainerManagerUnittest::runConcurrentContainerMapAccessTest(bool enableT2 // Test scenarios: // 1. Thread 1: Continuously calls refreshAllContainersSnapshot() and incrementallyUpdateContainersSnapshot() // (simulates real Polling thread behavior, updates mContainerMap with new RawContainerInfo) - // 2. Thread 2 (optional): Calls CheckContainerDiffForAllConfig() and ApplyContainerDiffs() + // 2. Thread 2 (optional): Calls CheckFileServerContainerDiffs() and ApplyFileServerContainerDiffs() // (simulates FileServer::Resume() behavior, can trigger GetCustomExternalTags crash) - // 3. Thread 3 (optional): Calls GetContainerStoppedEvents() + // 3. Thread 3 (optional): Calls GetFileServerContainerStoppedEvents() // (tests mRawContainerInfo access with proper locking) // // This test can expose race conditions including: @@ -1604,7 +1604,7 @@ void ContainerManagerUnittest::runConcurrentContainerMapAccessTest(bool enableT2 // - Null pointer dereference if mRawContainerInfo is not checked ContainerManager containerManager; - // Set running state so CheckContainerDiffForAllConfig can execute + // Set running state so CheckFileServerContainerDiffs can execute containerManager.mIsRunning = true; // Initialize with some containers @@ -1616,7 +1616,7 @@ void ContainerManagerUnittest::runConcurrentContainerMapAccessTest(bool enableT2 containerManager.mContainerMap[info.mID] = std::make_shared(info); } - // Setup FileServer with a test config to enable CheckContainerDiffForAllConfig and ApplyContainerDiffs + // Setup FileServer with a test config to enable CheckFileServerContainerDiffs and ApplyFileServerContainerDiffs ctx.SetConfigName(testName); // Create FileDiscoveryOptions with container discovery enabled @@ -1644,7 +1644,7 @@ void ContainerManagerUnittest::runConcurrentContainerMapAccessTest(bool enableT2 return true; }); - // Add config to FileServer so CheckContainerDiffForAllConfig can find it + // Add config to FileServer so CheckFileServerContainerDiffs can find it FileServer::GetInstance()->AddFileDiscoveryConfig(testName, &mDiscoveryOpts, &ctx); std::atomic testRunning{true}; @@ -1726,7 +1726,7 @@ void ContainerManagerUnittest::runConcurrentContainerMapAccessTest(bool enableT2 } }; - // Thread 2: Continuously call CheckContainerDiffForAllConfig and ApplyContainerDiffs + // Thread 2: Continuously call CheckFileServerContainerDiffs and ApplyFileServerContainerDiffs // This simulates the real FileServer::Resume() behavior auto readThread = [&]() { // Random initial delay (0-50ms) to vary thread interleaving @@ -1735,13 +1735,13 @@ void ContainerManagerUnittest::runConcurrentContainerMapAccessTest(bool enableT2 while (testRunning) { try { // Check container diffs for all configs (fills mConfigContainerDiffMap) - containerManager.CheckContainerDiffForAllConfig(); + containerManager.CheckFileServerContainerDiffs(); // Apply the diffs (reads and clears mConfigContainerDiffMap) // This is where GetCustomExternalTags gets called and may crash // When thread 1 replaces RawContainerInfo in mContainerMap, the old info's // mEnv and mK8sInfo.mLabels may be accessed here causing use-after-free - containerManager.ApplyContainerDiffs(); + containerManager.ApplyFileServerContainerDiffs(); iterationCount++; @@ -1753,7 +1753,7 @@ void ContainerManagerUnittest::runConcurrentContainerMapAccessTest(bool enableT2 } }; - // Thread 3: Call GetContainerStoppedEvents to test concurrent access to mRawContainerInfo + // Thread 3: Call GetFileServerContainerStoppedEvents to test concurrent access to mRawContainerInfo // This tests the fix for accessing mRawContainerInfo->mID without proper locking @@ -1771,10 +1771,10 @@ void ContainerManagerUnittest::runConcurrentContainerMapAccessTest(bool enableT2 containerManager.mStoppedContainerIDs.push_back("dynamic_" + std::to_string(counter + 100)); } - // Call GetContainerStoppedEvents which accesses mRawContainerInfo + // Call GetFileServerContainerStoppedEvents which accesses mRawContainerInfo // This method was also fixed in commit e0b33e88 to add proper locking std::vector events; - containerManager.GetContainerStoppedEvents(events); + containerManager.GetFileServerContainerStoppedEvents(events); // Clean up events for (auto* event : events) { @@ -1850,13 +1850,13 @@ void ContainerManagerUnittest::TestConcurrentContainerMapAccess_T1T2T3() { void ContainerManagerUnittest::TestSequentialContainerDiffAndApply() { // This test verifies that after multiple refreshAllContainersSnapshot and incrementallyUpdateContainersSnapshot - // operations, CheckContainerDiffForAllConfig and ApplyContainerDiffs produce correct results. + // operations, CheckFileServerContainerDiffs and ApplyFileServerContainerDiffs produce correct results. // // Test flow: // 1. Setup FileServer with test config // 2. Execute refreshAllContainersSnapshot multiple times to simulate container updates // 3. Execute incrementallyUpdateContainersSnapshot to simulate incremental updates - // 4. Call CheckContainerDiffForAllConfig and ApplyContainerDiffs multiple times + // 4. Call CheckFileServerContainerDiffs and ApplyFileServerContainerDiffs multiple times // 5. Verify the FileDiscoveryOptions container info is correctly updated after each round const std::string testName = "test_sequential_diff_apply"; @@ -1913,7 +1913,7 @@ void ContainerManagerUnittest::TestSequentialContainerDiffAndApply() { LogtailPluginMock::GetInstance()->SetUpContainersMeta(metaBuilder1.str()); containerManager.refreshAllContainersSnapshot(); - containerManager.mLastFullUpdateTime = 100; // Manually set timestamp for predictable testing + containerManager.mLastFullUpdateTime.store(100); // Manually set timestamp for predictable testing // Verify mContainerMap has 5 containers { @@ -1923,14 +1923,14 @@ void ContainerManagerUnittest::TestSequentialContainerDiffAndApply() { // Check diffs and apply (Round 1) mDiscoveryOpts.SetLastContainerUpdateTime(50); // Older than mLastFullUpdateTime - bool hasUpdate = containerManager.CheckContainerDiffForAllConfig(); - EXPECT_TRUE(hasUpdate) << "First CheckContainerDiffForAllConfig should detect new containers"; + bool hasUpdate = containerManager.CheckFileServerContainerDiffs(); + EXPECT_TRUE(hasUpdate) << "First CheckFileServerContainerDiffs should detect new containers"; - auto diff = containerManager.mConfigContainerDiffMap[testName]; + auto diff = containerManager.mConfigContainerDiffMap[std::make_pair(testName, 0)]; EXPECT_TRUE(diff != nullptr); EXPECT_EQ(diff->mAdded.size(), 5) << "Should have 5 added containers in first round"; - containerManager.ApplyContainerDiffs(); + containerManager.ApplyFileServerContainerDiffs(); // Verify FileDiscoveryOptions container info was populated auto containerInfo = mDiscoveryOpts.GetContainerInfo(); @@ -1956,7 +1956,7 @@ void ContainerManagerUnittest::TestSequentialContainerDiffAndApply() { LogtailPluginMock::GetInstance()->SetUpDiffContainersMeta(diffBuilder1.str()); containerManager.incrementallyUpdateContainersSnapshot(); - containerManager.mLastIncrementalUpdateTime = 200; // Manually set timestamp + containerManager.mLastIncrementalUpdateTime.store(200); // Manually set timestamp // Verify incremental update in mContainerMap { @@ -1969,10 +1969,10 @@ void ContainerManagerUnittest::TestSequentialContainerDiffAndApply() { // Check diffs and apply (Round 2) mDiscoveryOpts.SetLastContainerUpdateTime(150); // Between mLastFullUpdateTime and mLastIncrementalUpdateTime - hasUpdate = containerManager.CheckContainerDiffForAllConfig(); - EXPECT_TRUE(hasUpdate) << "Second CheckContainerDiffForAllConfig should detect updates"; + hasUpdate = containerManager.CheckFileServerContainerDiffs(); + EXPECT_TRUE(hasUpdate) << "Second CheckFileServerContainerDiffs should detect updates"; - diff = containerManager.mConfigContainerDiffMap[testName]; + diff = containerManager.mConfigContainerDiffMap[std::make_pair(testName, 0)]; EXPECT_TRUE(diff != nullptr); EXPECT_EQ(diff->mAdded.size(), 1) << "Should have 1 added container (container_5)"; EXPECT_EQ(diff->mModified.size(), 1) << "Should have 1 modified container (container_0)"; @@ -1985,7 +1985,7 @@ void ContainerManagerUnittest::TestSequentialContainerDiffAndApply() { EXPECT_EQ(diff->mModified[0]->mLogPath, "/var/log/container_0_updated"); } - containerManager.ApplyContainerDiffs(); + containerManager.ApplyFileServerContainerDiffs(); // Verify state after second round containerInfo = mDiscoveryOpts.GetContainerInfo(); @@ -2010,7 +2010,7 @@ void ContainerManagerUnittest::TestSequentialContainerDiffAndApply() { LogtailPluginMock::GetInstance()->SetUpDiffContainersMeta(diffBuilder2.str()); containerManager.incrementallyUpdateContainersSnapshot(); - containerManager.mLastIncrementalUpdateTime = 300; // Manually set timestamp + containerManager.mLastIncrementalUpdateTime.store(300); // Manually set timestamp // Verify deletions in mContainerMap { @@ -2022,14 +2022,14 @@ void ContainerManagerUnittest::TestSequentialContainerDiffAndApply() { // Check diffs and apply (Round 3) mDiscoveryOpts.SetLastContainerUpdateTime(250); // Between 200 and 300 - hasUpdate = containerManager.CheckContainerDiffForAllConfig(); - EXPECT_TRUE(hasUpdate) << "Third CheckContainerDiffForAllConfig should detect deletions"; + hasUpdate = containerManager.CheckFileServerContainerDiffs(); + EXPECT_TRUE(hasUpdate) << "Third CheckFileServerContainerDiffs should detect deletions"; - diff = containerManager.mConfigContainerDiffMap[testName]; + diff = containerManager.mConfigContainerDiffMap[std::make_pair(testName, 0)]; EXPECT_TRUE(diff != nullptr); EXPECT_EQ(diff->mRemoved.size(), 2) << "Should have 2 removed containers (container_1, container_2)"; - containerManager.ApplyContainerDiffs(); + containerManager.ApplyFileServerContainerDiffs(); // Verify final state containerInfo = mDiscoveryOpts.GetContainerInfo(); @@ -2081,7 +2081,7 @@ void ContainerManagerUnittest::TestSequentialContainerDiffAndApply() { LogtailPluginMock::GetInstance()->SetUpContainersMeta(metaBuilder3.str()); containerManager.refreshAllContainersSnapshot(); - containerManager.mLastFullUpdateTime = 400; // Manually set timestamp + containerManager.mLastFullUpdateTime.store(400); // Manually set timestamp // Verify mContainerMap has 9 containers { @@ -2097,15 +2097,15 @@ void ContainerManagerUnittest::TestSequentialContainerDiffAndApply() { // Check diffs and apply (Round 4) mDiscoveryOpts.SetLastContainerUpdateTime(350); // Older than mLastFullUpdateTime - hasUpdate = containerManager.CheckContainerDiffForAllConfig(); - EXPECT_TRUE(hasUpdate) << "Fourth CheckContainerDiffForAllConfig should detect full refresh changes"; + hasUpdate = containerManager.CheckFileServerContainerDiffs(); + EXPECT_TRUE(hasUpdate) << "Fourth CheckFileServerContainerDiffs should detect full refresh changes"; - diff = containerManager.mConfigContainerDiffMap[testName]; + diff = containerManager.mConfigContainerDiffMap[std::make_pair(testName, 0)]; EXPECT_TRUE(diff != nullptr); // Full refresh resets baseline, so all 9 containers are treated as "added" EXPECT_EQ(diff->mAdded.size(), 9) << "Should have 9 added containers (full refresh resets baseline)"; - containerManager.ApplyContainerDiffs(); + containerManager.ApplyFileServerContainerDiffs(); // Verify FileDiscoveryOptions has 9 containers containerInfo = mDiscoveryOpts.GetContainerInfo(); @@ -2157,8 +2157,8 @@ void ContainerManagerUnittest::TestSequentialContainerDiffAndApply() { // ===== Round 5: Verify no changes when config is up-to-date ===== mDiscoveryOpts.SetLastContainerUpdateTime(450); // Newer than mLastFullUpdateTime - hasUpdate = containerManager.CheckContainerDiffForAllConfig(); - EXPECT_FALSE(hasUpdate) << "CheckContainerDiffForAllConfig should not detect updates when config is up-to-date"; + hasUpdate = containerManager.CheckFileServerContainerDiffs(); + EXPECT_FALSE(hasUpdate) << "CheckFileServerContainerDiffs should not detect updates when config is up-to-date"; // Cleanup FileServer::GetInstance()->RemoveFileDiscoveryConfig(testName); @@ -2283,7 +2283,7 @@ void ContainerManagerUnittest::TestClearContainerInfo() const { void ContainerManagerUnittest::TestApplyContainerDiffsRefreshAllClearsStaleContainers() { // This test validates the core fix introduced in commit 6d1c1b5: - // When a full refresh occurs (mRefreshAllContainers=true), ApplyContainerDiffs must call + // When a full refresh occurs (mRefreshAllContainers=true), ApplyFileServerContainerDiffs must call // ClearContainerInfo() before applying the diff so that containers present in the previous // snapshot but absent from the new runtime snapshot are not retained as stale entries. // @@ -2377,15 +2377,15 @@ void ContainerManagerUnittest::TestApplyContainerDiffsRefreshAllClearsStaleConta } mDiscoveryOpts.SetLastContainerUpdateTime(0); // 0 < 100 -> full refresh path - EXPECT_TRUE(containerManager.CheckContainerDiffForAllConfig()); + EXPECT_TRUE(containerManager.CheckFileServerContainerDiffs()); - auto diff1 = containerManager.mConfigContainerDiffMap[testName]; + auto diff1 = containerManager.mConfigContainerDiffMap[std::make_pair(testName, 0)]; ASSERT_NE(diff1, nullptr); EXPECT_TRUE(diff1->mRefreshAllContainers) << "Full refresh must set mRefreshAllContainers=true"; EXPECT_EQ(diff1->mAdded.size(), 3u) << "All three containers must appear as added"; EXPECT_EQ(diff1->mRemoved.size(), 0u) << "Empty matchList produces no removals in full refresh path"; - containerManager.ApplyContainerDiffs(); + containerManager.ApplyFileServerContainerDiffs(); ASSERT_NE(mDiscoveryOpts.GetContainerInfo(), nullptr); EXPECT_EQ(mDiscoveryOpts.GetContainerInfo()->size(), 3u); @@ -2405,18 +2405,18 @@ void ContainerManagerUnittest::TestApplyContainerDiffsRefreshAllClearsStaleConta // lastConfigContainerUpdateTime(100) < mLastFullUpdateTime(200) -> full refresh path mDiscoveryOpts.SetLastContainerUpdateTime(100); - EXPECT_TRUE(containerManager.CheckContainerDiffForAllConfig()); + EXPECT_TRUE(containerManager.CheckFileServerContainerDiffs()); - auto diff2 = containerManager.mConfigContainerDiffMap[testName]; + auto diff2 = containerManager.mConfigContainerDiffMap[std::make_pair(testName, 0)]; ASSERT_NE(diff2, nullptr); EXPECT_TRUE(diff2->mRefreshAllContainers) << "Second full refresh must also set mRefreshAllContainers=true"; // The full refresh path seeds an empty matchList, so no mRemoved entries are produced for // container_A / container_C even though they disappeared from the runtime. - // ClearContainerInfo() inside ApplyContainerDiffs compensates for this gap. + // ClearContainerInfo() inside ApplyFileServerContainerDiffs compensates for this gap. EXPECT_EQ(diff2->mAdded.size(), 2u) << "container_B and container_D must be added"; EXPECT_EQ(diff2->mRemoved.size(), 0u) << "Full refresh path produces no explicit removals"; - containerManager.ApplyContainerDiffs(); + containerManager.ApplyFileServerContainerDiffs(); // Critical assertion: only container_B and container_D must survive. // Without the ClearContainerInfo() fix the stale container_A and container_C would remain, @@ -2444,15 +2444,15 @@ void ContainerManagerUnittest::TestApplyContainerDiffsRefreshAllClearsStaleConta // lastConfigContainerUpdateTime(200) == mLastFullUpdateTime(200) < mLastIncrementalUpdateTime(300) // -> incremental path (refreshAllContainers=false) mDiscoveryOpts.SetLastContainerUpdateTime(200); - EXPECT_TRUE(containerManager.CheckContainerDiffForAllConfig()); + EXPECT_TRUE(containerManager.CheckFileServerContainerDiffs()); - auto diff3 = containerManager.mConfigContainerDiffMap[testName]; + auto diff3 = containerManager.mConfigContainerDiffMap[std::make_pair(testName, 0)]; ASSERT_NE(diff3, nullptr); EXPECT_FALSE(diff3->mRefreshAllContainers) << "Incremental update must set mRefreshAllContainers=false"; EXPECT_EQ(diff3->mAdded.size(), 1u) << "Only container_E should be added incrementally"; EXPECT_EQ(diff3->mAdded[0]->mID, "container_E"); - containerManager.ApplyContainerDiffs(); + containerManager.ApplyFileServerContainerDiffs(); EXPECT_EQ(mDiscoveryOpts.GetContainerInfo()->size(), 3u) << "container_B, container_D, container_E must be present after incremental add"; @@ -2464,7 +2464,7 @@ void ContainerManagerUnittest::TestLoadContainerInfoAfterConfigInit() { // Validates the real startup sequence: // 1. Pipeline config is registered with FileServer (ContainerManager Init). // 2. ContainerManager::LoadContainerInfo() is called to restore the checkpoint. - // 3. LoadContainerInfo() internally calls ApplyContainerDiffs(); containers + // 3. LoadContainerInfo() internally calls ApplyFileServerContainerDiffs(); containers // must appear in FileDiscoveryOptions without any extra polling-loop step. // // Two checkpoint formats are exercised: @@ -2472,11 +2472,11 @@ void ContainerManagerUnittest::TestLoadContainerInfoAfterConfigInit() { // A. v1.0.0 format (Containers array, no per-config name): // loadContainerInfoFromContainersFormat calls checkContainerDiffForOneConfig // for every already-registered config, queuing a full-refresh diff. - // The following ApplyContainerDiffs() applies it immediately. + // The following ApplyFileServerContainerDiffs() applies it immediately. // // B. v0.1.0 format (detail array, config name embedded): // loadContainerInfoFromDetailFormat creates mLegacyCheckpointAdded entries - // keyed by config name. ApplyContainerDiffs() finds the registered config + // keyed by config name. ApplyFileServerContainerDiffs() finds the registered config // and applies them via UpdateRawContainerInfo(..., basePathInCheckpoint). // In unit tests GetAgentDataDir() == GetProcessExecutionDir() (see AppConfig.cpp). @@ -2535,7 +2535,7 @@ void ContainerManagerUnittest::TestLoadContainerInfoAfterConfigInit() { // Step 3: load the checkpoint. loadContainerInfoFromContainersFormat iterates // already-registered configs, calls checkContainerDiffForOneConfig for each - // (full-refresh path since both timestamps are 0), then ApplyContainerDiffs() + // (full-refresh path since both timestamps are 0), then ApplyFileServerContainerDiffs() // applies the queued diff — all within the single LoadContainerInfo() call. ContainerManager containerManager; containerManager.LoadContainerInfo(); @@ -2565,31 +2565,31 @@ void ContainerManagerUnittest::TestLoadContainerInfoAfterConfigInit() { containerManager.mIsRunning = true; // ── Phase 4a: before any real snapshot (both timestamps still 0) ────── - // ApplyContainerDiffs() set lastConfigContainerUpdateTime = 1 (sentinel). + // ApplyFileServerContainerDiffs() set lastConfigContainerUpdateTime = 1 (sentinel). // checkContainerDiffForOneConfig: 1 < 0 → false; (0==0 && 0==0 && 1==0) → false // → returns false immediately. No spurious re-trigger, no ClearContainerInfo. - EXPECT_FALSE(containerManager.CheckContainerDiffForAllConfig()) - << "[v1.0.0] CheckContainerDiffForAllConfig must NOT re-trigger before first snapshot"; + EXPECT_FALSE(containerManager.CheckFileServerContainerDiffs()) + << "[v1.0.0] CheckFileServerContainerDiffs must NOT re-trigger before first snapshot"; EXPECT_TRUE(containerManager.mConfigContainerDiffMap.empty()) << "[v1.0.0] No new diff must be queued when sentinel prevents re-trigger"; EXPECT_EQ(mDiscoveryOpts.GetContainerInfo()->size(), 2u) - << "[v1.0.0] Containers must survive the non-triggering CheckContainerDiffForAllConfig"; + << "[v1.0.0] Containers must survive the non-triggering CheckFileServerContainerDiffs"; // ── Phase 4b: first real snapshot arrives ───────────────────────────── // Simulate refreshAllContainersSnapshot() setting a real timestamp. containerManager.mLastFullUpdateTime = 100; // lastConfigContainerUpdateTime(1) < mLastFullUpdateTime(100) → full refresh. - EXPECT_TRUE(containerManager.CheckContainerDiffForAllConfig()) - << "[v1.0.0] CheckContainerDiffForAllConfig must trigger on first real snapshot"; + EXPECT_TRUE(containerManager.CheckFileServerContainerDiffs()) + << "[v1.0.0] CheckFileServerContainerDiffs must trigger on first real snapshot"; { - auto diff = containerManager.mConfigContainerDiffMap[configName]; + auto diff = containerManager.mConfigContainerDiffMap[std::make_pair(configName, 0)]; ASSERT_NE(diff, nullptr); EXPECT_TRUE(diff->mRefreshAllContainers) << "[v1.0.0] First real snapshot must be a full refresh"; EXPECT_EQ(diff->mAdded.size(), 2u) << "[v1.0.0] Checkpoint containers must be re-confirmed by real snapshot"; } - containerManager.ApplyContainerDiffs(); + containerManager.ApplyFileServerContainerDiffs(); EXPECT_EQ(mDiscoveryOpts.GetContainerInfo()->size(), 2u) << "[v1.0.0] Containers must still be present after first real snapshot apply"; // lastConfigContainerUpdateTime must now be 100 (real timestamp). @@ -2598,8 +2598,8 @@ void ContainerManagerUnittest::TestLoadContainerInfoAfterConfigInit() { // ── Phase 4c: no new snapshot — must not re-trigger ─────────────────── // lastConfigContainerUpdateTime(100) == mLastFullUpdateTime(100) → false. - EXPECT_FALSE(containerManager.CheckContainerDiffForAllConfig()) - << "[v1.0.0] CheckContainerDiffForAllConfig must not fire again without a new snapshot"; + EXPECT_FALSE(containerManager.CheckFileServerContainerDiffs()) + << "[v1.0.0] CheckFileServerContainerDiffs must not fire again without a new snapshot"; FileServer::GetInstance()->RemoveFileDiscoveryConfig(configName); } @@ -2646,7 +2646,7 @@ void ContainerManagerUnittest::TestLoadContainerInfoAfterConfigInit() { // Step 3: load the checkpoint. loadContainerInfoFromDetailFormat creates // mLegacyCheckpointAdded entries for the registered config name. - // ApplyContainerDiffs() calls UpdateRawContainerInfo(..., basePathInCheckpoint) + // ApplyFileServerContainerDiffs() calls UpdateRawContainerInfo(..., basePathInCheckpoint) // for each entry — all within the single LoadContainerInfo() call. ContainerManager containerManager; containerManager.LoadContainerInfo(); @@ -2676,40 +2676,40 @@ void ContainerManagerUnittest::TestLoadContainerInfoAfterConfigInit() { containerManager.mIsRunning = true; // ── Phase 4a: before any real snapshot (both timestamps still 0) ────── - // ApplyContainerDiffs() set lastConfigContainerUpdateTime = 1 (sentinel) via the + // ApplyFileServerContainerDiffs() set lastConfigContainerUpdateTime = 1 (sentinel) via the // unified sentinel check, even though loadContainerInfoFromDetailFormat never // calls checkContainerDiffForOneConfig. // checkContainerDiffForOneConfig: 1 < 0 → false; (0==0 && 0==0 && 1==0) → false // → returns false. No spurious re-trigger, containers unchanged. - EXPECT_FALSE(containerManager.CheckContainerDiffForAllConfig()) - << "[v0.1.0] CheckContainerDiffForAllConfig must NOT re-trigger before first snapshot"; + EXPECT_FALSE(containerManager.CheckFileServerContainerDiffs()) + << "[v0.1.0] CheckFileServerContainerDiffs must NOT re-trigger before first snapshot"; EXPECT_TRUE(containerManager.mConfigContainerDiffMap.empty()) << "[v0.1.0] No new diff must be queued when sentinel prevents re-trigger"; EXPECT_EQ(mDiscoveryOpts.GetContainerInfo()->size(), 2u) - << "[v0.1.0] Containers must survive the non-triggering CheckContainerDiffForAllConfig"; + << "[v0.1.0] Containers must survive the non-triggering CheckFileServerContainerDiffs"; // ── Phase 4b: first real snapshot arrives ───────────────────────────── containerManager.mLastFullUpdateTime = 100; // lastConfigContainerUpdateTime(1) < mLastFullUpdateTime(100) → full refresh. - EXPECT_TRUE(containerManager.CheckContainerDiffForAllConfig()) - << "[v0.1.0] CheckContainerDiffForAllConfig must trigger on first real snapshot"; + EXPECT_TRUE(containerManager.CheckFileServerContainerDiffs()) + << "[v0.1.0] CheckFileServerContainerDiffs must trigger on first real snapshot"; { - auto diff = containerManager.mConfigContainerDiffMap[configName]; + auto diff = containerManager.mConfigContainerDiffMap[std::make_pair(configName, 0)]; ASSERT_NE(diff, nullptr); EXPECT_TRUE(diff->mRefreshAllContainers) << "[v0.1.0] First real snapshot must be a full refresh"; EXPECT_EQ(diff->mAdded.size(), 2u) << "[v0.1.0] Checkpoint containers must be re-confirmed by real snapshot"; } - containerManager.ApplyContainerDiffs(); + containerManager.ApplyFileServerContainerDiffs(); EXPECT_EQ(mDiscoveryOpts.GetContainerInfo()->size(), 2u) << "[v0.1.0] Containers must still be present after first real snapshot apply"; EXPECT_EQ(mDiscoveryOpts.GetLastContainerUpdateTime(), 100) << "[v0.1.0] lastConfigContainerUpdateTime must advance to mLastFullUpdateTime(100)"; // ── Phase 4c: no new snapshot — must not re-trigger ─────────────────── - EXPECT_FALSE(containerManager.CheckContainerDiffForAllConfig()) - << "[v0.1.0] CheckContainerDiffForAllConfig must not fire again without a new snapshot"; + EXPECT_FALSE(containerManager.CheckFileServerContainerDiffs()) + << "[v0.1.0] CheckFileServerContainerDiffs must not fire again without a new snapshot"; FileServer::GetInstance()->RemoveFileDiscoveryConfig(configName); } diff --git a/core/unittest/event_handler/CreateModifyHandlerUnittest.cpp b/core/unittest/event_handler/CreateModifyHandlerUnittest.cpp index 69651acc31..ecd56c5b32 100644 --- a/core/unittest/event_handler/CreateModifyHandlerUnittest.cpp +++ b/core/unittest/event_handler/CreateModifyHandlerUnittest.cpp @@ -16,7 +16,6 @@ #include #include -#include #include #include @@ -64,13 +63,13 @@ class CreateModifyHandlerUnittest : public ::testing::Test { if (PATH_SEPARATOR[0] == gRootDir.at(gRootDir.size() - 1)) gRootDir.resize(gRootDir.size() - 1); gRootDir += PATH_SEPARATOR + "ModifyHandlerUnittest"; - filesystem::remove_all(gRootDir); + fs::remove_all(gRootDir); } static void TearDownTestCase() {} void SetUp() override { - bfs::create_directories(gRootDir); + fs::create_directories(gRootDir); // create a file for reader std::string logPath = UnitTestHelper::JsonEscapeDirPath(gRootDir + PATH_SEPARATOR + gLogName); writeLog(logPath, "a sample log\n"); @@ -108,7 +107,7 @@ class CreateModifyHandlerUnittest : public ::testing::Test { APSARA_TEST_TRUE(ParseJsonTable(configStr, *configJson, errorMsg)); Json::Value inputConfigJson = (*configJson)["inputs"][0]; - config.reset(new CollectionConfig(mConfigName, std::move(configJson), filesystem::path("."))); + config.reset(new CollectionConfig(mConfigName, std::move(configJson), fs::path("."))); APSARA_TEST_TRUE(config->Parse()); pipeline.reset(new CollectionPipeline()); APSARA_TEST_TRUE(pipeline->Init(std::move(*config))); @@ -154,7 +153,7 @@ class CreateModifyHandlerUnittest : public ::testing::Test { discoveryOpts.SetContainerInfo(containerInfo); } - void TearDown() override { filesystem::remove_all(gRootDir); } + void TearDown() override { fs::remove_all(gRootDir); } static std::string gRootDir; static std::string gLogName; diff --git a/core/unittest/event_handler/LogInputReaderUnittest.cpp b/core/unittest/event_handler/LogInputReaderUnittest.cpp index 4069d233f7..ad65dcc6ee 100644 --- a/core/unittest/event_handler/LogInputReaderUnittest.cpp +++ b/core/unittest/event_handler/LogInputReaderUnittest.cpp @@ -19,8 +19,6 @@ #include #endif -#include -#include #include #include @@ -126,7 +124,7 @@ class LogInputReaderUnittest : public ::testing::Test { gRootDir.resize(gRootDir.size() - 1); } gRootDir += PATH_SEPARATOR + "LogInputReaderUnittest"; - filesystem::remove_all(gRootDir); + fs::remove_all(gRootDir); } static void TearDownTestCase() {} @@ -163,7 +161,7 @@ class LogInputReaderUnittest : public ::testing::Test { // Double-check: ensure path is not registered APSARA_TEST_TRUE_FATAL(!EventDispatcher::GetInstance()->IsRegistered(gRootDir)); - bfs::create_directories(gRootDir); + fs::create_directories(gRootDir); // create a file for reader std::string logPath = gRootDir + PATH_SEPARATOR + "**" + PATH_SEPARATOR + "*.log"; writeLog(logPath, ""); @@ -253,7 +251,7 @@ class LogInputReaderUnittest : public ::testing::Test { // Pre-create and register handler for link directory (for symlink tests) std::string linkDir = gRootDir + PATH_SEPARATOR + "link"; - bfs::create_directories(linkDir); + fs::create_directories(linkDir); EventHandler* linkHandler = new CreateModifyHandler(new CreateHandler()); EventHandler* registeredLinkHandler = linkHandler; bool linkRegistered @@ -325,7 +323,7 @@ class LogInputReaderUnittest : public ::testing::Test { // Clear ConfigManager cache to avoid stale pointers ConfigManager::GetInstance()->ClearFilePipelineMatchCache(); - filesystem::remove_all(gRootDir); + fs::remove_all(gRootDir); ProcessQueueManager::GetInstance()->Clear(); LogInput::GetInstance()->CleanEnviroments(); EventDispatcher::GetInstance()->UnregisterEventHandler(gRootDir); @@ -362,8 +360,8 @@ class LogInputReaderUnittest : public ::testing::Test { } void createSymlink(const std::string& target, const std::string& linkPath) { - boost::filesystem::remove(linkPath); - boost::filesystem::create_symlink(target, linkPath); + fs::remove(linkPath); + fs::create_symlink(target, linkPath); } // Helper function to setup symlink test environment @@ -374,14 +372,14 @@ class LogInputReaderUnittest : public ::testing::Test { std::string linkDir = gRootDir + PATH_SEPARATOR + "link"; // Remove existing link directory if it exists - bfs::remove(linkDir); + fs::remove(linkDir); // Create real directory structure - bfs::create_directories(realDir); + fs::create_directories(realDir); // Create real file std::string realPath = realDir + PATH_SEPARATOR + "test.log"; - bfs::remove(realPath); + fs::remove(realPath); writeLog(realPath, content); // Create directory symlink: link -> real @@ -523,7 +521,7 @@ std::string LogInputReaderUnittest::gLogName; void LogInputReaderUnittest::CreateEmpty_Modify_Modify() { LOG_INFO(sLogger, ("TestNormal_CreateModifyModify_EmptyUnchanged() begin", time(nullptr))); std::string logPath = gRootDir + PATH_SEPARATOR + gLogName; - bfs::remove(logPath); + fs::remove(logPath); writeLog(logPath, ""); DevInode devInode = GetFileDevInode(logPath); @@ -550,7 +548,7 @@ void LogInputReaderUnittest::CreateEmpty_Modify_Modify() { void LogInputReaderUnittest::CreateNonEmpty_Modify_Modify() { LOG_INFO(sLogger, ("TestNormal_CreateModifyModify_NonEmptyUnchanged() begin", time(nullptr))); std::string logPath = gRootDir + PATH_SEPARATOR + gLogName; - bfs::remove(logPath); + fs::remove(logPath); writeLog(logPath, "test content\n"); DevInode devInode = GetFileDevInode(logPath); @@ -577,7 +575,7 @@ void LogInputReaderUnittest::CreateNonEmpty_Modify_Modify() { void LogInputReaderUnittest::CreateEmpty_Modify_Change_Close_Modify() { LOG_INFO(sLogger, ("TestNormal_CreateModifyCloseModify_EmptyChanged_BeforeClose() begin", time(nullptr))); std::string logPath = gRootDir + PATH_SEPARATOR + gLogName; - bfs::remove(logPath); + fs::remove(logPath); writeLog(logPath, ""); DevInode devInode = GetFileDevInode(logPath); @@ -594,7 +592,7 @@ void LogInputReaderUnittest::CreateEmpty_Modify_Change_Close_Modify() { ProcessSingleEvent(modifyEv1); std::string newLogPath = gRootDir + PATH_SEPARATOR + "new.log"; - bfs::rename(logPath, newLogPath); + fs::rename(logPath, newLogPath); // Process close event Event* closeEv = new Event(gRootDir, gLogName, EVENT_DELETE, 0, 0, devInode.dev, devInode.inode); @@ -611,7 +609,7 @@ void LogInputReaderUnittest::CreateEmpty_Modify_Change_Close_Modify() { void LogInputReaderUnittest::CreateEmpty_Modify_Close_Change_Modify() { LOG_INFO(sLogger, ("TestNormal_CreateModifyCloseModify_EmptyChanged_AfterClose() begin", time(nullptr))); std::string logPath = gRootDir + PATH_SEPARATOR + gLogName; - bfs::remove(logPath); + fs::remove(logPath); writeLog(logPath, ""); DevInode devInode = GetFileDevInode(logPath); @@ -632,7 +630,7 @@ void LogInputReaderUnittest::CreateEmpty_Modify_Close_Change_Modify() { ProcessSingleEvent(closeEv); std::string newLogPath = gRootDir + PATH_SEPARATOR + "new.log"; - bfs::rename(logPath, newLogPath); + fs::rename(logPath, newLogPath); std::string newLogName = "new.log"; Event* modifyEv2 = new Event(gRootDir, newLogName, EVENT_MODIFY, 0, 0, devInode.dev, devInode.inode); @@ -645,7 +643,7 @@ void LogInputReaderUnittest::CreateEmpty_Modify_Close_Change_Modify() { void LogInputReaderUnittest::CreateEmpty_Modify_Close_Modify() { LOG_INFO(sLogger, ("TestNormal_CreateModifyCloseModify_EmptyUnchanged() begin", time(nullptr))); std::string logPath = gRootDir + PATH_SEPARATOR + gLogName; - bfs::remove(logPath); + fs::remove(logPath); writeLog(logPath, ""); DevInode devInode = GetFileDevInode(logPath); @@ -676,7 +674,7 @@ void LogInputReaderUnittest::CreateNonEmpty_Modify_Change_Close_Modify_SameSigna LOG_INFO(sLogger, ("TestNormal_CreateModifyCloseModify_NonEmptyChanged_SameSignature_BeforeClose() begin", time(nullptr))); std::string logPath = gRootDir + PATH_SEPARATOR + gLogName; - bfs::remove(logPath); + fs::remove(logPath); std::string signature = "test content\n"; writeLog(logPath, signature); @@ -694,7 +692,7 @@ void LogInputReaderUnittest::CreateNonEmpty_Modify_Change_Close_Modify_SameSigna ProcessSingleEvent(modifyEv1); std::string newLogPath = gRootDir + PATH_SEPARATOR + "new.log"; - bfs::rename(logPath, newLogPath); + fs::rename(logPath, newLogPath); // Process close event Event* closeEv = new Event(gRootDir, gLogName, EVENT_DELETE, 0, 0, devInode.dev, devInode.inode); @@ -712,7 +710,7 @@ void LogInputReaderUnittest::CreateNonEmpty_Modify_Close_Change_Modify_SameSigna LOG_INFO(sLogger, ("TestNormal_CreateModifyCloseModify_NonEmptyChanged_SameSignature_AfterClose() begin", time(nullptr))); std::string logPath = gRootDir + PATH_SEPARATOR + gLogName; - bfs::remove(logPath); + fs::remove(logPath); std::string signature = "test content\n"; writeLog(logPath, signature); @@ -734,7 +732,7 @@ void LogInputReaderUnittest::CreateNonEmpty_Modify_Close_Change_Modify_SameSigna ProcessSingleEvent(closeEv); std::string newLogPath = gRootDir + PATH_SEPARATOR + "new.log"; - bfs::rename(logPath, newLogPath); + fs::rename(logPath, newLogPath); std::string newLogName = "new.log"; Event* modifyEv2 = new Event(gRootDir, newLogName, EVENT_MODIFY, 0, 0, devInode.dev, devInode.inode); @@ -748,7 +746,7 @@ void LogInputReaderUnittest::CreateNonEmpty_Modify_Change_Close_Modify_DiffSigna LOG_INFO(sLogger, ("TestNormal_CreateModifyCloseModify_NonEmptyChanged_DiffSignature_BeforeClose() begin", time(nullptr))); std::string logPath = gRootDir + PATH_SEPARATOR + gLogName; - bfs::remove(logPath); + fs::remove(logPath); writeLog(logPath, "old content\n"); DevInode devInode = GetFileDevInode(logPath); @@ -765,7 +763,7 @@ void LogInputReaderUnittest::CreateNonEmpty_Modify_Change_Close_Modify_DiffSigna ProcessSingleEvent(modifyEv1); std::string newLogPath = gRootDir + PATH_SEPARATOR + "new.log"; - bfs::rename(logPath, newLogPath); + fs::rename(logPath, newLogPath); overwriteLog(newLogPath, "new content\n"); // Process close event @@ -784,7 +782,7 @@ void LogInputReaderUnittest::CreateNonEmpty_Modify_Close_Change_Modify_DiffSigna LOG_INFO(sLogger, ("TestNormal_CreateModifyCloseModify_NonEmptyChanged_DiffSignature_AfterClose() begin", time(nullptr))); std::string logPath = gRootDir + PATH_SEPARATOR + gLogName; - bfs::remove(logPath); + fs::remove(logPath); writeLog(logPath, "old content\n"); DevInode devInode = GetFileDevInode(logPath); @@ -805,7 +803,7 @@ void LogInputReaderUnittest::CreateNonEmpty_Modify_Close_Change_Modify_DiffSigna ProcessSingleEvent(closeEv); std::string newLogPath = gRootDir + PATH_SEPARATOR + "new.log"; - bfs::rename(logPath, newLogPath); + fs::rename(logPath, newLogPath); overwriteLog(newLogPath, "new content\n"); std::string newLogName = "new.log"; @@ -819,7 +817,7 @@ void LogInputReaderUnittest::CreateNonEmpty_Modify_Close_Change_Modify_DiffSigna void LogInputReaderUnittest::CreateNonEmpty_Modify_Close_Modify() { LOG_INFO(sLogger, ("TestNormal_CreateModifyCloseModify_NonEmptyUnchanged() begin", time(nullptr))); std::string logPath = gRootDir + PATH_SEPARATOR + gLogName; - bfs::remove(logPath); + fs::remove(logPath); writeLog(logPath, "test content\n"); DevInode devInode = GetFileDevInode(logPath); @@ -849,7 +847,7 @@ void LogInputReaderUnittest::CreateNonEmpty_Modify_Close_Modify() { void LogInputReaderUnittest::CreateEmpty_Change_Close_Modify() { LOG_INFO(sLogger, ("TestNormal_CreateCloseModify_EmptyChanged_BeforeClose() begin", time(nullptr))); std::string logPath = gRootDir + PATH_SEPARATOR + gLogName; - bfs::remove(logPath); + fs::remove(logPath); writeLog(logPath, ""); DevInode devInode = GetFileDevInode(logPath); @@ -862,7 +860,7 @@ void LogInputReaderUnittest::CreateEmpty_Change_Close_Modify() { LogFileReaderPtr originalReader = GetReader(devInode); std::string newLogPath = gRootDir + PATH_SEPARATOR + "new.log"; - bfs::rename(logPath, newLogPath); + fs::rename(logPath, newLogPath); // Process close event Event* closeEv = new Event(gRootDir, gLogName, EVENT_DELETE, 0, 0, devInode.dev, devInode.inode); @@ -879,7 +877,7 @@ void LogInputReaderUnittest::CreateEmpty_Change_Close_Modify() { void LogInputReaderUnittest::CreateEmpty_Close_Change_Modify() { LOG_INFO(sLogger, ("TestNormal_CreateCloseModify_EmptyChanged_AfterClose() begin", time(nullptr))); std::string logPath = gRootDir + PATH_SEPARATOR + gLogName; - bfs::remove(logPath); + fs::remove(logPath); writeLog(logPath, ""); DevInode devInode = GetFileDevInode(logPath); @@ -896,7 +894,7 @@ void LogInputReaderUnittest::CreateEmpty_Close_Change_Modify() { ProcessSingleEvent(closeEv); std::string newLogPath = gRootDir + PATH_SEPARATOR + "new.log"; - bfs::rename(logPath, newLogPath); + fs::rename(logPath, newLogPath); std::string newLogName = "new.log"; Event* modifyEv = new Event(gRootDir, newLogName, EVENT_MODIFY, 0, 0, devInode.dev, devInode.inode); @@ -909,7 +907,7 @@ void LogInputReaderUnittest::CreateEmpty_Close_Change_Modify() { void LogInputReaderUnittest::CreateEmpty_Close_Modify() { LOG_INFO(sLogger, ("TestNormal_CreateCloseModify_EmptyUnchanged() begin", time(nullptr))); std::string logPath = gRootDir + PATH_SEPARATOR + gLogName; - bfs::remove(logPath); + fs::remove(logPath); writeLog(logPath, ""); DevInode devInode = GetFileDevInode(logPath); @@ -936,7 +934,7 @@ void LogInputReaderUnittest::CreateNonEmpty_Change_Close_Modify_SameSignature() LOG_INFO(sLogger, ("TestNormal_CreateCloseModify_NonEmptyChanged_SameSignature_BeforeClose() begin", time(nullptr))); std::string logPath = gRootDir + PATH_SEPARATOR + gLogName; - bfs::remove(logPath); + fs::remove(logPath); std::string signature = "test content\n"; writeLog(logPath, signature); @@ -950,7 +948,7 @@ void LogInputReaderUnittest::CreateNonEmpty_Change_Close_Modify_SameSignature() LogFileReaderPtr originalReader = GetReader(devInode); std::string newLogPath = gRootDir + PATH_SEPARATOR + "new.log"; - bfs::rename(logPath, newLogPath); + fs::rename(logPath, newLogPath); // Process close event Event* closeEv = new Event(gRootDir, gLogName, EVENT_DELETE, 0, 0, devInode.dev, devInode.inode); @@ -967,7 +965,7 @@ void LogInputReaderUnittest::CreateNonEmpty_Change_Close_Modify_SameSignature() void LogInputReaderUnittest::CreateNonEmpty_Close_Change_Modify_SameSignature() { LOG_INFO(sLogger, ("TestNormal_CreateCloseModify_NonEmptyChanged_SameSignature_AfterClose() begin", time(nullptr))); std::string logPath = gRootDir + PATH_SEPARATOR + gLogName; - bfs::remove(logPath); + fs::remove(logPath); std::string signature = "test content\n"; writeLog(logPath, signature); @@ -985,7 +983,7 @@ void LogInputReaderUnittest::CreateNonEmpty_Close_Change_Modify_SameSignature() ProcessSingleEvent(closeEv); std::string newLogPath = gRootDir + PATH_SEPARATOR + "new.log"; - bfs::rename(logPath, newLogPath); + fs::rename(logPath, newLogPath); std::string newLogName = "new.log"; Event* modifyEv = new Event(gRootDir, newLogName, EVENT_MODIFY, 0, 0, devInode.dev, devInode.inode); @@ -999,7 +997,7 @@ void LogInputReaderUnittest::CreateNonEmpty_Change_Close_Modify_DiffSignature() LOG_INFO(sLogger, ("TestNormal_CreateCloseModify_NonEmptyChanged_DiffSignature_BeforeClose() begin", time(nullptr))); std::string logPath = gRootDir + PATH_SEPARATOR + gLogName; - bfs::remove(logPath); + fs::remove(logPath); writeLog(logPath, "old content\n"); DevInode devInode = GetFileDevInode(logPath); @@ -1012,7 +1010,7 @@ void LogInputReaderUnittest::CreateNonEmpty_Change_Close_Modify_DiffSignature() LogFileReaderPtr originalReader = GetReader(devInode); std::string newLogPath = gRootDir + PATH_SEPARATOR + "new.log"; - bfs::rename(logPath, newLogPath); + fs::rename(logPath, newLogPath); overwriteLog(newLogPath, "new content\n"); // Process close event @@ -1030,7 +1028,7 @@ void LogInputReaderUnittest::CreateNonEmpty_Change_Close_Modify_DiffSignature() void LogInputReaderUnittest::CreateNonEmpty_Close_Change_Modify_DiffSignature() { LOG_INFO(sLogger, ("TestNormal_CreateCloseModify_NonEmptyChanged_DiffSignature_AfterClose() begin", time(nullptr))); std::string logPath = gRootDir + PATH_SEPARATOR + gLogName; - bfs::remove(logPath); + fs::remove(logPath); writeLog(logPath, "old content\n"); DevInode devInode = GetFileDevInode(logPath); @@ -1047,7 +1045,7 @@ void LogInputReaderUnittest::CreateNonEmpty_Close_Change_Modify_DiffSignature() ProcessSingleEvent(closeEv); std::string newLogPath = gRootDir + PATH_SEPARATOR + "new.log"; - bfs::rename(logPath, newLogPath); + fs::rename(logPath, newLogPath); overwriteLog(newLogPath, "new content\n"); std::string newLogName = "new.log"; @@ -1061,7 +1059,7 @@ void LogInputReaderUnittest::CreateNonEmpty_Close_Change_Modify_DiffSignature() void LogInputReaderUnittest::CreateNonEmpty_Close_Modify() { LOG_INFO(sLogger, ("TestNormal_CreateCloseModify_NonEmptyUnchanged() begin", time(nullptr))); std::string logPath = gRootDir + PATH_SEPARATOR + gLogName; - bfs::remove(logPath); + fs::remove(logPath); writeLog(logPath, "test content\n"); DevInode devInode = GetFileDevInode(logPath); @@ -1170,7 +1168,7 @@ void LogInputReaderUnittest::SymlinkCreateEmpty_Modify_Change_Close_Modify() { // Real file is renamed to a new file - BEFORE close std::string newRealPath = realDir + PATH_SEPARATOR + "new_test.log"; - bfs::rename(realPath, newRealPath); + fs::rename(realPath, newRealPath); // Process close event Event* closeEv = new Event(linkDir, "test.log", EVENT_DELETE, 0, 0, devInode.dev, devInode.inode); @@ -1212,7 +1210,7 @@ void LogInputReaderUnittest::SymlinkCreateEmpty_Modify_Close_Change_Modify() { // Real file is renamed to a new file - this happens AFTER close std::string newRealPath = realDir + PATH_SEPARATOR + "new_test.log"; - bfs::rename(realPath, newRealPath); + fs::rename(realPath, newRealPath); Event* modifyEv2 = new Event(linkDir, "new_test.log", EVENT_MODIFY, 0, 0, devInode.dev, devInode.inode); ProcessSingleEvent(modifyEv2); @@ -1281,7 +1279,7 @@ void LogInputReaderUnittest::SymlinkCreateNonEmpty_Modify_Change_Close_Modify_Sa // Real file is renamed to a new file - this happens BEFORE close std::string newRealPath = realDir + PATH_SEPARATOR + "new_test.log"; - bfs::rename(realPath, newRealPath); + fs::rename(realPath, newRealPath); // Process close event Event* closeEv = new Event(linkDir, "test.log", EVENT_DELETE, 0, 0, devInode.dev, devInode.inode); @@ -1325,7 +1323,7 @@ void LogInputReaderUnittest::SymlinkCreateNonEmpty_Modify_Close_Change_Modify_Sa // Real file is renamed to a new file - this happens AFTER close std::string newRealPath = realDir + PATH_SEPARATOR + "new_test.log"; - bfs::rename(realPath, newRealPath); + fs::rename(realPath, newRealPath); Event* modifyEv2 = new Event(linkDir, "new_test.log", EVENT_MODIFY, 0, 0, devInode.dev, devInode.inode); ProcessSingleEvent(modifyEv2); @@ -1361,7 +1359,7 @@ void LogInputReaderUnittest::SymlinkCreateNonEmpty_Modify_Change_Close_Modify_Di // Real file is renamed to a new file - this happens BEFORE close // Create new symlink pointing to the new real file std::string newRealPath = realDir + PATH_SEPARATOR + "new_test.log"; - bfs::rename(realPath, newRealPath); + fs::rename(realPath, newRealPath); overwriteLog(newRealPath, "new content\n"); // Process close event @@ -1406,7 +1404,7 @@ void LogInputReaderUnittest::SymlinkCreateNonEmpty_Modify_Close_Change_Modify_Di // Real file is renamed to a new file - this happens AFTER close // Since linkDir is a directory symlink, no need to recreate file symlinks std::string newRealPath = realDir + PATH_SEPARATOR + "new_test.log"; - bfs::rename(realPath, newRealPath); + fs::rename(realPath, newRealPath); overwriteLog(newRealPath, "new content\n"); Event* modifyEv2 = new Event(linkDir, "new_test.log", EVENT_MODIFY, 0, 0, devInode.dev, devInode.inode); @@ -1470,7 +1468,7 @@ void LogInputReaderUnittest::SymlinkCreateEmpty_Change_Close_Modify() { // Real file is renamed to a new file - BEFORE close // Since linkDir is a directory symlink, no need to recreate file symlinks std::string newRealPath = realDir + PATH_SEPARATOR + "new_test.log"; - bfs::rename(realPath, newRealPath); + fs::rename(realPath, newRealPath); // Process close event Event* closeEv = new Event(linkDir, "test.log", EVENT_DELETE, 0, 0, devInode.dev, devInode.inode); @@ -1507,7 +1505,7 @@ void LogInputReaderUnittest::SymlinkCreateEmpty_Close_Change_Modify() { // Real file is renamed to a new file - this happens AFTER close // Since linkDir is a directory symlink, no need to recreate file symlinks std::string newRealPath = realDir + PATH_SEPARATOR + "new_test.log"; - bfs::rename(realPath, newRealPath); + fs::rename(realPath, newRealPath); Event* modifyEv = new Event(linkDir, "new_test.log", EVENT_MODIFY, 0, 0, devInode.dev, devInode.inode); ProcessSingleEvent(modifyEv); @@ -1567,7 +1565,7 @@ void LogInputReaderUnittest::SymlinkCreateNonEmpty_Change_Close_Modify_SameSigna // Real file is renamed to a new file - this happens BEFORE close std::string newRealPath = realDir + PATH_SEPARATOR + "new_test.log"; - bfs::rename(realPath, newRealPath); + fs::rename(realPath, newRealPath); // Process close event Event* closeEv = new Event(linkDir, "test.log", EVENT_DELETE, 0, 0, devInode.dev, devInode.inode); @@ -1607,7 +1605,7 @@ void LogInputReaderUnittest::SymlinkCreateNonEmpty_Close_Change_Modify_SameSigna // Real file is renamed to a new file - this happens AFTER close // Since linkDir is a directory symlink, no need to recreate file symlinks std::string newRealPath = realDir + PATH_SEPARATOR + "new_test.log"; - bfs::rename(realPath, newRealPath); + fs::rename(realPath, newRealPath); Event* modifyEv = new Event(linkDir, "new_test.log", EVENT_MODIFY, 0, 0, devInode.dev, devInode.inode); ProcessSingleEvent(modifyEv); @@ -1636,7 +1634,7 @@ void LogInputReaderUnittest::SymlinkCreateNonEmpty_Change_Close_Modify_DiffSigna // Real file is renamed to a new file - this happens BEFORE close std::string newRealPath = realDir + PATH_SEPARATOR + "new_test.log"; - bfs::rename(realPath, newRealPath); + fs::rename(realPath, newRealPath); overwriteLog(newRealPath, "new content\n"); // Process close event @@ -1676,7 +1674,7 @@ void LogInputReaderUnittest::SymlinkCreateNonEmpty_Close_Change_Modify_DiffSigna // Real file is renamed to a new file - this happens AFTER close // Since linkDir is a directory symlink, no need to recreate file symlinks std::string newRealPath = realDir + PATH_SEPARATOR + "new_test.log"; - bfs::rename(realPath, newRealPath); + fs::rename(realPath, newRealPath); overwriteLog(newRealPath, "new content\n"); Event* modifyEv = new Event(linkDir, "new_test.log", EVENT_MODIFY, 0, 0, devInode.dev, devInode.inode); @@ -1743,7 +1741,7 @@ void LogInputReaderUnittest::SymlinkCreateEmpty_Close_ModifyWrite_Change_Close_M // Real file is renamed to a new file - this happens after first modify std::string newRealPath = realDir + PATH_SEPARATOR + "new_test.log"; - bfs::rename(realPath, newRealPath); + fs::rename(realPath, newRealPath); // Process close event again with new file name Event* closeEv2 = new Event(linkDir, "new_test.log", EVENT_DELETE, 0, 0, devInode.dev, devInode.inode); @@ -1763,7 +1761,7 @@ void LogInputReaderUnittest::SymlinkCreateEmpty_Close_ModifyWrite_Change_Close_M void LogInputReaderUnittest::ModifyEmpty_Modify_Modify() { LOG_INFO(sLogger, ("TestNormal_ModifyModifyModify_EmptyUnchanged() begin", time(nullptr))); std::string logPath = gRootDir + PATH_SEPARATOR + gLogName; - bfs::remove(logPath); + fs::remove(logPath); writeLog(logPath, ""); DevInode devInode = GetFileDevInode(logPath); @@ -1790,7 +1788,7 @@ void LogInputReaderUnittest::ModifyEmpty_Modify_Modify() { void LogInputReaderUnittest::ModifyNonEmpty_Modify_Modify() { LOG_INFO(sLogger, ("TestNormal_ModifyModifyModify_NonEmptyUnchanged() begin", time(nullptr))); std::string logPath = gRootDir + PATH_SEPARATOR + gLogName; - bfs::remove(logPath); + fs::remove(logPath); writeLog(logPath, "test content\n"); DevInode devInode = GetFileDevInode(logPath); @@ -1817,7 +1815,7 @@ void LogInputReaderUnittest::ModifyNonEmpty_Modify_Modify() { void LogInputReaderUnittest::ModifyEmpty_Change_Close_Modify() { LOG_INFO(sLogger, ("TestNormal_ModifyCloseModify_EmptyChanged_BeforeClose() begin", time(nullptr))); std::string logPath = gRootDir + PATH_SEPARATOR + gLogName; - bfs::remove(logPath); + fs::remove(logPath); writeLog(logPath, ""); DevInode devInode = GetFileDevInode(logPath); @@ -1830,7 +1828,7 @@ void LogInputReaderUnittest::ModifyEmpty_Change_Close_Modify() { LogFileReaderPtr originalReader = GetReader(devInode); std::string newLogPath = gRootDir + PATH_SEPARATOR + "new.log"; - bfs::rename(logPath, newLogPath); + fs::rename(logPath, newLogPath); // Process close event Event* closeEv = new Event(gRootDir, gLogName, EVENT_DELETE, 0, 0, devInode.dev, devInode.inode); @@ -1847,7 +1845,7 @@ void LogInputReaderUnittest::ModifyEmpty_Change_Close_Modify() { void LogInputReaderUnittest::ModifyEmpty_Close_Change_Modify() { LOG_INFO(sLogger, ("TestNormal_ModifyCloseModify_EmptyChanged_AfterClose() begin", time(nullptr))); std::string logPath = gRootDir + PATH_SEPARATOR + gLogName; - bfs::remove(logPath); + fs::remove(logPath); writeLog(logPath, ""); DevInode devInode = GetFileDevInode(logPath); @@ -1864,7 +1862,7 @@ void LogInputReaderUnittest::ModifyEmpty_Close_Change_Modify() { ProcessSingleEvent(closeEv); std::string newLogPath = gRootDir + PATH_SEPARATOR + "new.log"; - bfs::rename(logPath, newLogPath); + fs::rename(logPath, newLogPath); std::string newLogName = "new.log"; Event* modifyEv2 = new Event(gRootDir, newLogName, EVENT_MODIFY, 0, 0, devInode.dev, devInode.inode); @@ -1877,7 +1875,7 @@ void LogInputReaderUnittest::ModifyEmpty_Close_Change_Modify() { void LogInputReaderUnittest::ModifyEmpty_Close_Modify() { LOG_INFO(sLogger, ("TestNormal_ModifyCloseModify_EmptyUnchanged() begin", time(nullptr))); std::string logPath = gRootDir + PATH_SEPARATOR + gLogName; - bfs::remove(logPath); + fs::remove(logPath); writeLog(logPath, ""); DevInode devInode = GetFileDevInode(logPath); @@ -1904,7 +1902,7 @@ void LogInputReaderUnittest::ModifyNonEmpty_Change_Close_Modify_SameSignature() LOG_INFO(sLogger, ("TestNormal_ModifyCloseModify_NonEmptyChanged_SameSignature_BeforeClose() begin", time(nullptr))); std::string logPath = gRootDir + PATH_SEPARATOR + gLogName; - bfs::remove(logPath); + fs::remove(logPath); std::string signature = "test content\n"; writeLog(logPath, signature); @@ -1918,7 +1916,7 @@ void LogInputReaderUnittest::ModifyNonEmpty_Change_Close_Modify_SameSignature() LogFileReaderPtr originalReader = GetReader(devInode); std::string newLogPath = gRootDir + PATH_SEPARATOR + "new.log"; - bfs::rename(logPath, newLogPath); + fs::rename(logPath, newLogPath); // Process close event Event* closeEv = new Event(gRootDir, gLogName, EVENT_DELETE, 0, 0, devInode.dev, devInode.inode); @@ -1935,7 +1933,7 @@ void LogInputReaderUnittest::ModifyNonEmpty_Change_Close_Modify_SameSignature() void LogInputReaderUnittest::ModifyNonEmpty_Close_Change_Modify_SameSignature() { LOG_INFO(sLogger, ("TestNormal_ModifyCloseModify_NonEmptyChanged_SameSignature_AfterClose() begin", time(nullptr))); std::string logPath = gRootDir + PATH_SEPARATOR + gLogName; - bfs::remove(logPath); + fs::remove(logPath); std::string signature = "test content\n"; writeLog(logPath, signature); @@ -1953,7 +1951,7 @@ void LogInputReaderUnittest::ModifyNonEmpty_Close_Change_Modify_SameSignature() ProcessSingleEvent(closeEv); std::string newLogPath = gRootDir + PATH_SEPARATOR + "new.log"; - bfs::rename(logPath, newLogPath); + fs::rename(logPath, newLogPath); std::string newLogName = "new.log"; Event* modifyEv2 = new Event(gRootDir, newLogName, EVENT_MODIFY, 0, 0, devInode.dev, devInode.inode); @@ -1967,7 +1965,7 @@ void LogInputReaderUnittest::ModifyNonEmpty_Change_Close_Modify_DiffSignature() LOG_INFO(sLogger, ("TestNormal_ModifyCloseModify_NonEmptyChanged_DiffSignature_BeforeClose() begin", time(nullptr))); std::string logPath = gRootDir + PATH_SEPARATOR + gLogName; - bfs::remove(logPath); + fs::remove(logPath); writeLog(logPath, "old content\n"); DevInode devInode = GetFileDevInode(logPath); @@ -1980,7 +1978,7 @@ void LogInputReaderUnittest::ModifyNonEmpty_Change_Close_Modify_DiffSignature() LogFileReaderPtr originalReader = GetReader(devInode); std::string newLogPath = gRootDir + PATH_SEPARATOR + "new.log"; - bfs::rename(logPath, newLogPath); + fs::rename(logPath, newLogPath); overwriteLog(newLogPath, "new content\n"); // Process close event @@ -1998,7 +1996,7 @@ void LogInputReaderUnittest::ModifyNonEmpty_Change_Close_Modify_DiffSignature() void LogInputReaderUnittest::ModifyNonEmpty_Close_Change_Modify_DiffSignature() { LOG_INFO(sLogger, ("TestNormal_ModifyCloseModify_NonEmptyChanged_DiffSignature_AfterClose() begin", time(nullptr))); std::string logPath = gRootDir + PATH_SEPARATOR + gLogName; - bfs::remove(logPath); + fs::remove(logPath); writeLog(logPath, "old content\n"); DevInode devInode = GetFileDevInode(logPath); @@ -2015,7 +2013,7 @@ void LogInputReaderUnittest::ModifyNonEmpty_Close_Change_Modify_DiffSignature() ProcessSingleEvent(closeEv); std::string newLogPath = gRootDir + PATH_SEPARATOR + "new.log"; - bfs::rename(logPath, newLogPath); + fs::rename(logPath, newLogPath); overwriteLog(newLogPath, "new content\n"); std::string newLogName = "new.log"; @@ -2029,7 +2027,7 @@ void LogInputReaderUnittest::ModifyNonEmpty_Close_Change_Modify_DiffSignature() void LogInputReaderUnittest::ModifyNonEmpty_Close_Modify() { LOG_INFO(sLogger, ("TestNormal_ModifyCloseModify_NonEmptyUnchanged() begin", time(nullptr))); std::string logPath = gRootDir + PATH_SEPARATOR + gLogName; - bfs::remove(logPath); + fs::remove(logPath); writeLog(logPath, "test content\n"); DevInode devInode = GetFileDevInode(logPath); @@ -2132,7 +2130,7 @@ void LogInputReaderUnittest::SymlinkModifyEmpty_Change_Close_Modify() { // Real file is renamed to a new file - BEFORE close // Since linkDir is a directory symlink, no need to recreate file symlinks std::string newRealPath = realDir + PATH_SEPARATOR + "new_test.log"; - bfs::rename(realPath, newRealPath); + fs::rename(realPath, newRealPath); // Process close event Event* closeEv = new Event(linkDir, "test.log", EVENT_DELETE, 0, 0, devInode.dev, devInode.inode); @@ -2171,7 +2169,7 @@ void LogInputReaderUnittest::SymlinkModifyEmpty_Close_Change_Modify() { // Since linkDir is a directory symlink, no need to recreate file symlinks std::string newRealPath = realDir + PATH_SEPARATOR + "new_test.log"; std::string newLinkPath = linkDir + PATH_SEPARATOR + "new_test.log"; - bfs::rename(realPath, newRealPath); + fs::rename(realPath, newRealPath); Event* modifyEv2 = new Event(linkDir, "new_test.log", EVENT_MODIFY, 0, 0, devInode.dev, devInode.inode); ProcessSingleEvent(modifyEv2); @@ -2230,7 +2228,7 @@ void LogInputReaderUnittest::SymlinkModifyNonEmpty_Change_Close_Modify_SameSigna // Real file is renamed to a new file - this happens BEFORE close std::string newRealPath = realDir + PATH_SEPARATOR + "new_test.log"; - bfs::rename(realPath, newRealPath); + fs::rename(realPath, newRealPath); // Process close event Event* closeEv = new Event(linkDir, "test.log", EVENT_DELETE, 0, 0, devInode.dev, devInode.inode); @@ -2270,7 +2268,7 @@ void LogInputReaderUnittest::SymlinkModifyNonEmpty_Close_Change_Modify_SameSigna // Real file is renamed to a new file - this happens AFTER close // Since linkDir is a directory symlink, no need to recreate file symlinks std::string newRealPath = realDir + PATH_SEPARATOR + "new_test.log"; - bfs::rename(realPath, newRealPath); + fs::rename(realPath, newRealPath); Event* modifyEv2 = new Event(linkDir, "new_test.log", EVENT_MODIFY, 0, 0, devInode.dev, devInode.inode); ProcessSingleEvent(modifyEv2); @@ -2301,7 +2299,7 @@ void LogInputReaderUnittest::SymlinkModifyNonEmpty_Change_Close_Modify_DiffSigna // Real file is renamed to a new file - this happens BEFORE close // Create new symlink pointing to the new real file std::string newRealPath = realDir + PATH_SEPARATOR + "new_test.log"; - bfs::rename(realPath, newRealPath); + fs::rename(realPath, newRealPath); overwriteLog(newRealPath, "new content\n"); // Process close event @@ -2340,7 +2338,7 @@ void LogInputReaderUnittest::SymlinkModifyNonEmpty_Close_Change_Modify_DiffSigna // Real file is renamed to a new file - this happens AFTER close std::string newRealPath = realDir + PATH_SEPARATOR + "new_test.log"; - bfs::rename(realPath, newRealPath); + fs::rename(realPath, newRealPath); overwriteLog(newRealPath, "new content\n"); Event* modifyEv2 = new Event(linkDir, "new_test.log", EVENT_MODIFY, 0, 0, devInode.dev, devInode.inode); diff --git a/core/unittest/event_handler/ModifyHandlerUnittest.cpp b/core/unittest/event_handler/ModifyHandlerUnittest.cpp index 61d3b2819c..57346fafcc 100644 --- a/core/unittest/event_handler/ModifyHandlerUnittest.cpp +++ b/core/unittest/event_handler/ModifyHandlerUnittest.cpp @@ -16,7 +16,6 @@ #include #include -#include #include #include #include @@ -79,13 +78,13 @@ class ModifyHandlerUnittest : public ::testing::Test { if (PATH_SEPARATOR[0] == gRootDir.at(gRootDir.size() - 1)) gRootDir.resize(gRootDir.size() - 1); gRootDir += PATH_SEPARATOR + "ModifyHandlerUnittest"; - filesystem::remove_all(gRootDir); + fs::remove_all(gRootDir); } static void TearDownTestCase() {} void SetUp() override { - bfs::create_directories(gRootDir); + fs::create_directories(gRootDir); // create a file for reader std::string logPath = gRootDir + PATH_SEPARATOR + gLogName; writeLog(logPath, "a sample log\n"); @@ -171,7 +170,7 @@ class ModifyHandlerUnittest : public ::testing::Test { addContainerInfo("1"); } void TearDown() override { - filesystem::remove_all(gRootDir); + fs::remove_all(gRootDir); ProcessQueueManager::GetInstance()->Clear(); } @@ -349,7 +348,7 @@ void ModifyHandlerUnittest::TestHandleBasicDeleteEvent() { // Actually delete the file std::string logPath = gRootDir + PATH_SEPARATOR + gLogName; - bfs::remove(logPath); + fs::remove(logPath); // Send DELETE event Event deleteEvent(gRootDir, gLogName, EVENT_DELETE, 0); @@ -538,10 +537,10 @@ void ModifyHandlerUnittest::TestDoublePopFrontBugWhenFileDeletedWithMultipleRead // Delete the physical file // In production, Linux marks the fd path with " (deleted)" suffix // In test environment, this may not be reliably detected - bfs::remove(logPath1); + fs::remove(logPath1); // Check if file still exists on filesystem - bool fileExists = bfs::exists(logPath1); + bool fileExists = fs::exists(logPath1); LOG_INFO(sLogger, ("File status", "after deletion")("file exists", fileExists)("fd is open", reader1->IsFileOpened())); @@ -603,7 +602,7 @@ void ModifyHandlerUnittest::TestDoublePopFrontBugWhenFileDeletedWithMultipleRead APSARA_TEST_EQUAL_FATAL(handlerPtr->mNameReaderMap[gLogName][0]->GetDevInode().inode, devInode2.inode); APSARA_TEST_EQUAL_FATAL(handlerPtr->mNameReaderMap[gLogName][1]->GetDevInode().inode, devInode3.inode); - // In this test scenario, the file was physically deleted via bfs::remove() + // In this test scenario, the file was physically deleted via fs::remove() // while the file descriptor was still open. With the refactored code, the // size > 1 condition is checked first, preventing the double pop_front bug. // CloseFilePtr should detect the deletion and return isFileReallyDeleted = true. @@ -718,9 +717,9 @@ void ModifyHandlerUnittest::TestFileDeletedWithTwoReaders_FileReallyDeleted() { // Physically delete the file while fd is still open // This simulates the real scenario where file is deleted from filesystem - bfs::remove(logPath1); + fs::remove(logPath1); - bool fileExists = bfs::exists(logPath1); + bool fileExists = fs::exists(logPath1); LOG_INFO( sLogger, ("File status", "file physically deleted")("file exists", fileExists)("fd is open", reader1->IsFileOpened())); @@ -745,7 +744,7 @@ void ModifyHandlerUnittest::TestFileDeletedWithTwoReaders_FileReallyDeleted() { // 3. Enter the first branch (not IsFileDeleted branch) // 4. ForceReadLogAndPush called // 5. RemoveReaderFromArrayAndMap called - reader1 removed from array and map - // 6. CloseFilePtr called - detects file was deleted (bfs::remove was called) + // 6. CloseFilePtr called - detects file was deleted (fs::remove was called) // 7. isFileReallyDeleted = true (file physically deleted from filesystem) // 8. reader1 NOT added to mRotatorReaderMap (because file really deleted) // Result: reader1 removed completely, only reader2 remains in array @@ -768,7 +767,7 @@ void ModifyHandlerUnittest::TestFileDeletedWithTwoReaders_FileReallyDeleted() { APSARA_TEST_EQUAL_FATAL(handlerPtr->mDevInodeReaderMap.count(devInode2), 1); APSARA_TEST_EQUAL_FATAL(handlerPtr->mNameReaderMap[gLogName][0]->GetDevInode().inode, devInode2.inode); - // In this test scenario, the file was physically deleted via bfs::remove() + // In this test scenario, the file was physically deleted via fs::remove() // while the file descriptor was still open. This means CloseFilePtr should // detect the deletion and return isFileReallyDeleted = true. // Therefore, reader1 should NOT be added to mRotatorReaderMap. @@ -880,7 +879,7 @@ void ModifyHandlerUnittest::TestFileDeletedWithTwoReaders_FileNotDeleted() { // DON'T delete the file - it still exists on filesystem // This simulates the case where DELETE event was received but file still exists - bool fileExists = bfs::exists(logPath1); + bool fileExists = fs::exists(logPath1); LOG_INFO(sLogger, ("File status", "file NOT physically deleted")("file exists", fileExists)( "fd is open", reader1->IsFileOpened())("deleted flag", reader1->IsFileDeleted())); @@ -1013,9 +1012,9 @@ void ModifyHandlerUnittest::TestFileDeletedWithSingleReader_FileReallyDeleted() APSARA_TEST_TRUE_FATAL(reader->IsFileOpened()); // Physically delete the file while fd is still open - bfs::remove(logPath); + fs::remove(logPath); - bool fileExists = bfs::exists(logPath); + bool fileExists = fs::exists(logPath); LOG_INFO( sLogger, ("File status", "file physically deleted")("file exists", fileExists)("fd is open", reader->IsFileOpened())); @@ -1038,7 +1037,7 @@ void ModifyHandlerUnittest::TestFileDeletedWithSingleReader_FileReallyDeleted() // 1. hasMoreData = false (already at end) // 2. readerArrayPtr->size() > 1 is false (size is 1) // 3. IsFileDeleted() is true - enter the second branch - // 4. CloseFilePtr called - detects file was deleted (bfs::remove was called) + // 4. CloseFilePtr called - detects file was deleted (fs::remove was called) // 5. isFileReallyDeleted = true (file physically deleted from filesystem) // 6. RemoveReaderFromArrayAndMap called - reader removed from array and map // Result: reader completely removed from both array and map @@ -1138,7 +1137,7 @@ void ModifyHandlerUnittest::TestFileDeletedWithSingleReader_FileNotDeleted() { // DON'T delete the file - it still exists on filesystem // This simulates the case where DELETE event was received but file still exists - bool fileExists = bfs::exists(logPath); + bool fileExists = fs::exists(logPath); LOG_INFO(sLogger, ("File status", "file NOT physically deleted")("file exists", fileExists)( "fd is open", reader->IsFileOpened())("deleted flag", reader->IsFileDeleted())); @@ -1270,9 +1269,9 @@ void ModifyHandlerUnittest::TestContainerStoppedWithSingleReader_FileReallyDelet APSARA_TEST_TRUE_FATAL(reader->IsFileOpened()); // Physically delete the file while fd is still open - bfs::remove(logPath); + fs::remove(logPath); - bool fileExists = bfs::exists(logPath); + bool fileExists = fs::exists(logPath); LOG_INFO(sLogger, ("File status", "file physically deleted")("file exists", fileExists)( "fd is open", reader->IsFileOpened())("container stopped", reader->IsContainerStopped())); @@ -1298,7 +1297,7 @@ void ModifyHandlerUnittest::TestContainerStoppedWithSingleReader_FileReallyDelet // 4. IsContainerStopped() is true - enter the third branch // 5. UpdateContainerInfo() - still stopped // 6. ForceReadLogAndPush called - // 7. CloseFilePtr called - detects file was deleted (bfs::remove was called) + // 7. CloseFilePtr called - detects file was deleted (fs::remove was called) // 8. isFileReallyDeleted = true (file physically deleted from filesystem) // 9. RemoveReaderFromArrayAndMap called - reader removed from array and map // Result: reader completely removed from both array and map @@ -1408,7 +1407,7 @@ void ModifyHandlerUnittest::TestContainerStoppedWithSingleReader_FileNotDeleted( // DON'T delete the file - it still exists on filesystem // This simulates the case where container is stopped but file still exists - bool fileExists = bfs::exists(logPath); + bool fileExists = fs::exists(logPath); LOG_INFO(sLogger, ("File status", "file NOT physically deleted")("file exists", fileExists)( "fd is open", reader->IsFileOpened())("container stopped", reader->IsContainerStopped())); @@ -2043,7 +2042,7 @@ void ModifyHandlerUnittest::TestClearReaderWhenFileDeleted() { // Actually delete the file while file descriptor is still open // This simulates the real scenario where file is deleted but fd is held std::string logPath = gRootDir + PATH_SEPARATOR + gLogName; - bfs::remove(logPath); + fs::remove(logPath); // Verify reader exists before handling event APSARA_TEST_EQUAL_FATAL(mHandlerPtr->mDevInodeReaderMap.size(), 1); diff --git a/core/unittest/file_source/FileDiscoveryOptionsUnittest.cpp b/core/unittest/file_source/FileDiscoveryOptionsUnittest.cpp index 4f06a2e915..585c0f7172 100644 --- a/core/unittest/file_source/FileDiscoveryOptionsUnittest.cpp +++ b/core/unittest/file_source/FileDiscoveryOptionsUnittest.cpp @@ -12,7 +12,6 @@ // See the License for the specific language governing permissions and // limitations under the License. -#include #include #include @@ -48,9 +47,9 @@ void FileDiscoveryOptionsUnittest::OnSuccessfulInit() const { unique_ptr config; Json::Value configJson; string configStr, errorMsg; - filesystem::path filePath = filesystem::absolute("*.log"); + fs::path filePath = fs::absolute("*.log"); filePath = NormalizeNativePath(filePath.string()); - filesystem::path ex1, ex2, ex3, ex4, ex5; + fs::path ex1, ex2, ex3, ex4, ex5; // only mandatory param configStr = R"( @@ -124,9 +123,9 @@ void FileDiscoveryOptionsUnittest::OnSuccessfulInit() const { APSARA_TEST_FALSE(config->mAllowingIncludedByMultiConfigs); // ExcludeFilePaths - ex1 = filesystem::path(".") / "test" / "a.log"; // not absolute - ex2 = filesystem::current_path() / "**" / "b.log"; // ML - ex3 = filesystem::absolute(ex1); + ex1 = fs::path(".") / "test" / "a.log"; // not absolute + ex2 = fs::current_path() / "**" / "b.log"; // ML + ex3 = fs::absolute(ex1); ex2 = NormalizeNativePath(ex2.string()); ex3 = NormalizeNativePath(ex3.string()); configStr = R"( @@ -149,7 +148,7 @@ void FileDiscoveryOptionsUnittest::OnSuccessfulInit() const { // ExcludeFiles ex1 = "a.log"; - ex2 = filesystem::current_path() / "b.log"; // has path separator + ex2 = fs::current_path() / "b.log"; // has path separator ex2 = NormalizeNativePath(ex2.string()); configStr = R"( { @@ -168,11 +167,11 @@ void FileDiscoveryOptionsUnittest::OnSuccessfulInit() const { APSARA_TEST_TRUE(config->mHasBlacklist); // ExcludeDirs - ex1 = filesystem::path(".") / "test"; // not absolute - ex2 = filesystem::current_path() / "**" / "test"; // ML - ex3 = filesystem::current_path() / "a*"; // * - ex4 = filesystem::current_path() / "a?"; // ? - ex5 = filesystem::absolute(ex1); + ex1 = fs::path(".") / "test"; // not absolute + ex2 = fs::current_path() / "**" / "test"; // ML + ex3 = fs::current_path() / "a*"; // * + ex4 = fs::current_path() / "a?"; // ? + ex5 = fs::absolute(ex1); ex2 = NormalizeNativePath(ex2.string()); ex3 = NormalizeNativePath(ex3.string()); ex4 = NormalizeNativePath(ex4.string()); @@ -217,7 +216,7 @@ void FileDiscoveryOptionsUnittest::OnFailedInit() const { unique_ptr config; Json::Value configJson; string configStr, errorMsg; - filesystem::path filePath = filesystem::absolute("*.log"); + fs::path filePath = fs::absolute("*.log"); filePath = NormalizeNativePath(filePath.string()); // no FilePaths @@ -260,7 +259,7 @@ void FileDiscoveryOptionsUnittest::OnFailedInit() const { APSARA_TEST_TRUE(config->Init(configJson, ctx, pluginType)); // invalid filepath - filePath = filesystem::current_path(); + filePath = fs::current_path(); filePath = NormalizeNativePath(filePath.string()); configStr = R"( { @@ -277,17 +276,17 @@ void FileDiscoveryOptionsUnittest::TestFilePaths() const { unique_ptr config; Json::Value configJson; CollectionPipelineContext ctx; - filesystem::path filePath; + fs::path filePath; // no wildcard - filePath = filesystem::path(".") / "test" / "*.log"; + filePath = fs::path(".") / "test" / "*.log"; configJson["FilePaths"].append(Json::Value(filePath.string())); configJson["MaxDirSearchDepth"] = Json::Value(1); config.reset(new FileDiscoveryOptions()); APSARA_TEST_TRUE(config->Init(configJson, ctx, pluginType)); APSARA_TEST_EQUAL(0, config->mMaxDirSearchDepth); - filesystem::path expectedBasePath = filesystem::current_path() / "test"; + fs::path expectedBasePath = fs::current_path() / "test"; expectedBasePath = NormalizeNativePath(expectedBasePath.string()); APSARA_TEST_EQUAL(1U, config->GetBasePathInfos().size()); APSARA_TEST_EQUAL(expectedBasePath.string(), config->GetBasePathInfos()[0].basePath); @@ -295,17 +294,17 @@ void FileDiscoveryOptionsUnittest::TestFilePaths() const { configJson.clear(); // with wildcard */? - filePath = filesystem::path(".") / "*" / "test" / "?" / "*.log"; + filePath = fs::path(".") / "*" / "test" / "?" / "*.log"; configJson["FilePaths"].append(Json::Value(filePath.string())); configJson["MaxDirSearchDepth"] = Json::Value(1); config.reset(new FileDiscoveryOptions()); APSARA_TEST_TRUE(config->Init(configJson, ctx, pluginType)); APSARA_TEST_EQUAL(0, config->mMaxDirSearchDepth); - filesystem::path expectedBasePath1 = filesystem::current_path() / "*" / "test" / "?"; - filesystem::path expectedWildcard0 = filesystem::current_path(); - filesystem::path expectedWildcard1 = filesystem::current_path() / "*"; - filesystem::path expectedWildcard2 = filesystem::current_path() / "*" / "test"; - filesystem::path expectedWildcard3 = filesystem::current_path() / "*" / "test" / "?"; + fs::path expectedBasePath1 = fs::current_path() / "*" / "test" / "?"; + fs::path expectedWildcard0 = fs::current_path(); + fs::path expectedWildcard1 = fs::current_path() / "*"; + fs::path expectedWildcard2 = fs::current_path() / "*" / "test"; + fs::path expectedWildcard3 = fs::current_path() / "*" / "test" / "?"; expectedBasePath1 = NormalizeNativePath(expectedBasePath1.string()); expectedWildcard0 = NormalizeNativePath(expectedWildcard0.string()); expectedWildcard1 = NormalizeNativePath(expectedWildcard1.string()); @@ -327,14 +326,14 @@ void FileDiscoveryOptionsUnittest::TestFilePaths() const { configJson.clear(); // with wildcard ** - filePath = filesystem::path(".") / "*" / "test" / "**" / "*.log"; + filePath = fs::path(".") / "*" / "test" / "**" / "*.log"; configJson["FilePaths"].append(Json::Value(filePath.string())); configJson["MaxDirSearchDepth"] = Json::Value(1); config.reset(new FileDiscoveryOptions()); APSARA_TEST_TRUE(config->Init(configJson, ctx, pluginType)); APSARA_TEST_EQUAL(1, config->mMaxDirSearchDepth); - filesystem::path expectedBasePath2 = filesystem::current_path() / "*" / "test"; + fs::path expectedBasePath2 = fs::current_path() / "*" / "test"; expectedBasePath2 = NormalizeNativePath(expectedBasePath2.string()); APSARA_TEST_EQUAL(1U, config->GetBasePathInfos().size()); APSARA_TEST_EQUAL(expectedBasePath2.string(), config->GetBasePathInfos()[0].basePath); @@ -349,7 +348,7 @@ void FileDiscoveryOptionsUnittest::TestWindowsRootPathCollection() const { // Test 1: Direct root path collection without AllowingCollectingFilesInRootDir flag // Expected: Init should succeed but mAllowingCollectingFilesInRootDir should be false { - filesystem::path filePath = "C:\\*.log"; + fs::path filePath = "C:\\*.log"; filePath = NormalizeNativePath(filePath.string()); configStr = R"( { @@ -367,7 +366,7 @@ void FileDiscoveryOptionsUnittest::TestWindowsRootPathCollection() const { // Expected: Flag should be enabled only when enable_root_path_collection is true { BOOL_FLAG(enable_root_path_collection) = true; - filesystem::path filePath = "C:\\*.log"; + fs::path filePath = "C:\\*.log"; filePath = NormalizeNativePath(filePath.string()); configStr = R"( { @@ -389,7 +388,7 @@ void FileDiscoveryOptionsUnittest::TestWindowsRootPathCollection() const { // Test 3: Multi-level path with wildcard at root (C:\*\logs\*.log) // Expected: Should parse correctly and set base path to C:\*\logs { - filesystem::path filePath = "C:\\*\\logs\\*.log"; + fs::path filePath = "C:\\*\\logs\\*.log"; filePath = NormalizeNativePath(filePath.string()); configStr = R"( { @@ -411,7 +410,7 @@ void FileDiscoveryOptionsUnittest::TestWindowsRootPathCollection() const { // Expected: Should work when both config and global flag are enabled { BOOL_FLAG(enable_root_path_collection) = true; - filesystem::path filePath = "C:\\*\\logs\\*.log"; + fs::path filePath = "C:\\*\\logs\\*.log"; filePath = NormalizeNativePath(filePath.string()); configStr = R"( { @@ -432,7 +431,7 @@ void FileDiscoveryOptionsUnittest::TestWindowsRootPathCollection() const { // Test 5: Recursive search from root (C:\**\*.log) // Expected: Should parse correctly with ** at root { - filesystem::path filePath = "C:\\**\\*.log"; + fs::path filePath = "C:\\**\\*.log"; filePath = NormalizeNativePath(filePath.string()); configStr = R"( { @@ -454,7 +453,7 @@ void FileDiscoveryOptionsUnittest::TestWindowsRootPathCollection() const { // Expected: Should work with flag enabled { BOOL_FLAG(enable_root_path_collection) = true; - filesystem::path filePath = "D:\\**\\*.log"; + fs::path filePath = "D:\\**\\*.log"; filePath = NormalizeNativePath(filePath.string()); configStr = R"( { @@ -483,7 +482,7 @@ void FileDiscoveryOptionsUnittest::TestChinesePathMatching() const { // Linux Test 1: Chinese path in FilePaths // Expected: Should successfully parse Chinese path { - filesystem::path filePath = filesystem::absolute("测试目录/**/*.log"); + fs::path filePath = fs::absolute("测试目录/**/*.log"); configStr = R"( { "FilePaths": [] @@ -501,8 +500,8 @@ void FileDiscoveryOptionsUnittest::TestChinesePathMatching() const { // Linux Test 2: Chinese directory in ExcludeDirs // Expected: Should successfully add Chinese directory to blacklist { - filesystem::path filePath = filesystem::absolute("日志/**/*.log"); - filesystem::path excludeDir = filesystem::absolute("日志/黑名单"); + fs::path filePath = fs::absolute("日志/**/*.log"); + fs::path excludeDir = fs::absolute("日志/黑名单"); configStr = R"( { "FilePaths": [], @@ -521,7 +520,7 @@ void FileDiscoveryOptionsUnittest::TestChinesePathMatching() const { // Linux Test 3: Chinese filename in ExcludeFiles // Expected: Should successfully add Chinese filename to blacklist { - filesystem::path filePath = filesystem::absolute("logs/*.log"); + fs::path filePath = fs::absolute("logs/*.log"); configStr = R"( { "FilePaths": [], @@ -546,7 +545,7 @@ void FileDiscoveryOptionsUnittest::TestChinesePathMatching() const { try { // UTF-8: "测试目录" string chineseDir = "\xE6\xB5\x8B\xE8\xAF\x95\xE7\x9B\xAE\xE5\xBD\x95"; - filesystem::path basePath = filesystem::current_path(); + fs::path basePath = fs::current_path(); string fullPath = basePath.string() + "\\" + chineseDir + "\\**\\*.log"; configStr = R"( { @@ -574,7 +573,7 @@ void FileDiscoveryOptionsUnittest::TestChinesePathMatching() const { ("Chinese path test", "SKIPPED - encoding conversion not supported on this system")); } } catch (const std::exception& e) { - // Encoding conversion or filesystem operation failed + // Encoding conversion or fs operation failed LOG_WARNING(sLogger, ("Chinese path test", "SKIPPED - exception")("error", e.what())); } } @@ -584,7 +583,7 @@ void FileDiscoveryOptionsUnittest::TestChinesePathMatching() const { // UTF-8: "日志" and "黑名单" string chineseLog = "\xE6\x97\xA5\xE5\xBF\x97"; string chineseBlacklist = "\xE9\xBB\x91\xE5\x90\x8D\xE5\x8D\x95"; - filesystem::path basePath = filesystem::current_path(); + fs::path basePath = fs::current_path(); string fullPath = basePath.string() + "\\" + chineseLog + "\\**\\*.log"; string excludePath = basePath.string() + "\\" + chineseLog + "\\" + chineseBlacklist; configStr = R"( @@ -618,7 +617,7 @@ void FileDiscoveryOptionsUnittest::TestChinesePathMatching() const { try { // UTF-8: "混合" string chineseMixed = "\xE6\xB7\xB7\xE5\x90\x88"; - filesystem::path basePath = filesystem::current_path(); + fs::path basePath = fs::current_path(); string fullPath = basePath.string() + "\\" + chineseMixed + "\\*.log"; // UTF-8: "排除.log" and "测试.log" string excludeFile1 = "\xE6\x8E\x92\xE9\x99\xA4.log"; @@ -660,7 +659,7 @@ void FileDiscoveryOptionsUnittest::TestChinesePathMatching() const { try { // UTF-8: "文档" string chineseDoc = "\xE6\x96\x87\xE6\xA1\xA3"; - filesystem::path basePath = filesystem::current_path(); + fs::path basePath = fs::current_path(); string fullPath = basePath.string() + "\\" + chineseDoc + "\\*.log"; // UTF-8: "排除文件.log" string excludeFileName = "\xE6\x8E\x92\xE9\x99\xA4\xE6\x96\x87\xE4\xBB\xB6.log"; @@ -696,8 +695,8 @@ void FileDiscoveryOptionsUnittest::TestChinesePathMatching() const { } // Windows Test 5: ASCII paths (baseline test to ensure basic functionality) { - filesystem::path filePath = filesystem::absolute("test_logs\\**\\*.log"); - filesystem::path excludeDir = filesystem::absolute("test_logs\\exclude"); + fs::path filePath = fs::absolute("test_logs\\**\\*.log"); + fs::path excludeDir = fs::absolute("test_logs\\exclude"); filePath = NormalizeNativePath(filePath.string()); excludeDir = NormalizeNativePath(excludeDir.string()); configStr = R"( @@ -711,7 +710,7 @@ void FileDiscoveryOptionsUnittest::TestChinesePathMatching() const { APSARA_TEST_TRUE(ParseJsonTable(configStr, configJson, errorMsg)); configJson["FilePaths"].append(Json::Value(filePath.string())); configJson["ExcludeDirs"].append(Json::Value(excludeDir.string())); - filesystem::path excludeFile = filesystem::absolute("test_logs\\exclude.log"); + fs::path excludeFile = fs::absolute("test_logs\\exclude.log"); excludeFile = NormalizeNativePath(excludeFile.string()); configJson["ExcludeFilePaths"].append(Json::Value(excludeFile.string())); config.reset(new FileDiscoveryOptions()); @@ -751,7 +750,7 @@ void FileDiscoveryOptionsUnittest::TestWindowsDriveLetterCaseInsensitive() const // Test 2: ExcludeDirs with lowercase drive letter // Expected: Should correctly add to blacklist regardless of case { - filesystem::path filePath = filesystem::absolute("test_logs\\*.log"); + fs::path filePath = fs::absolute("test_logs\\*.log"); string upperFilePath = filePath.string(); if (upperFilePath.size() >= 2 && upperFilePath[1] == ':') { upperFilePath[0] = toupper(upperFilePath[0]); @@ -779,8 +778,8 @@ void FileDiscoveryOptionsUnittest::TestWindowsDriveLetterCaseInsensitive() const // Test 3: ExcludeFilePaths with different drive letter case // Expected: Should correctly handle regardless of case { - filesystem::path filePath = filesystem::absolute("test_files\\*.log"); - filesystem::path excludeFile = filesystem::absolute("test_files\\exclude.log"); + fs::path filePath = fs::absolute("test_files\\*.log"); + fs::path excludeFile = fs::absolute("test_files\\exclude.log"); string filePathStr = filePath.string(); string excludeFileStr = excludeFile.string(); // Make drive letters different case @@ -826,9 +825,9 @@ void FileDiscoveryOptionsUnittest::TestWindowsDriveLetterCaseInsensitive() const // Test 5: Mixed case in base path and multiple blacklists // Expected: All path operations should be case-insensitive for drive letters { - filesystem::path filePath = filesystem::absolute("multi_test\\**\\*.log"); - filesystem::path excludeDir = filesystem::absolute("multi_test\\exclude_dir"); - filesystem::path excludeFile = filesystem::absolute("multi_test\\exclude.log"); + fs::path filePath = fs::absolute("multi_test\\**\\*.log"); + fs::path excludeDir = fs::absolute("multi_test\\exclude_dir"); + fs::path excludeFile = fs::absolute("multi_test\\exclude.log"); string filePathStr = filePath.string(); string excludeDirStr = excludeDir.string(); string excludeFileStr = excludeFile.string(); diff --git a/core/unittest/file_source/StaticFileServerUnittest.cpp b/core/unittest/file_source/StaticFileServerUnittest.cpp index 05518d1270..ee66047521 100644 --- a/core/unittest/file_source/StaticFileServerUnittest.cpp +++ b/core/unittest/file_source/StaticFileServerUnittest.cpp @@ -12,9 +12,14 @@ // See the License for the specific language governing permissions and // limitations under the License. +#include + #include "collection_pipeline/CollectionPipeline.h" +#include "collection_pipeline/CollectionPipelineContext.h" #include "collection_pipeline/plugin/PluginRegistry.h" +#include "common/FileSystemUtil.h" #include "common/JsonUtil.h" +#include "file_server/FileDiscoveryOptions.h" #include "file_server/StaticFileServer.h" #include "file_server/checkpoint/InputStaticFileCheckpointManager.h" #include "plugin/input/InputStaticFile.h" @@ -24,6 +29,15 @@ using namespace std; namespace logtail { +namespace { + +size_t CountInputReadersForPipeline(const std::multimap, LogFileReaderPtr>& m, + const string& pipelineName) { + return count_if(m.begin(), m.end(), [&](const auto& e) { return e.first.first == pipelineName; }); +} + +} // namespace + class StaticFileServerUnittest : public testing::Test { public: void TestGetNextAvailableReader() const; @@ -31,6 +45,7 @@ class StaticFileServerUnittest : public testing::Test { void TestClearUnusedCheckpoints() const; void TestSetExpectedFileSize() const; void TestFileRotationDetection() const; + void TestGetFiles() const; protected: static void SetUpTestCase() { PluginRegistry::GetInstance()->LoadPlugins(); } @@ -38,44 +53,65 @@ class StaticFileServerUnittest : public testing::Test { void SetUp() override { sManager = InputStaticFileCheckpointManager::GetInstance(); sServer = StaticFileServer::GetInstance(); - sManager->mCheckpointRootPath = filesystem::path("./input_static_file"); - filesystem::create_directories(sManager->mCheckpointRootPath); + sManager->mCheckpointRootPath = fs::path("./input_static_file"); + fs::create_directories(sManager->mCheckpointRootPath); } void TearDown() override { sServer->Clear(); sManager->ClearUnusedCheckpoints(); sManager->mInputCheckpointMap.clear(); - filesystem::remove_all(sManager->mCheckpointRootPath); + fs::remove_all(sManager->mCheckpointRootPath); } + // Inserts config into server maps and mAddedInputs without calling Init() (no Run() thread). + static void AddInputWithoutStartingThread(StaticFileServer* srv, + const string& configName, + size_t idx, + FileDiscoveryOptions* discoveryOpts, + const FileReaderOptions* fileReaderOpts, + const MultilineOptions* multilineOpts, + const FileTagOptions* fileTagOpts, + unordered_map& fileContainerMetas, + const CollectionPipelineContext* ctx); + private: InputStaticFileCheckpointManager* sManager; StaticFileServer* sServer; }; +void StaticFileServerUnittest::AddInputWithoutStartingThread( + StaticFileServer* srv, + const string& configName, + size_t idx, + FileDiscoveryOptions* discoveryOpts, + const FileReaderOptions* fileReaderOpts, + const MultilineOptions* multilineOpts, + const FileTagOptions* fileTagOpts, + unordered_map& fileContainerMetas, + const CollectionPipelineContext* ctx) { + // Lock order: mUpdateMux -> mConfigReadWriteLock (StaticFileServer.h); keep one writer section for all maps. + lock_guard rlk(srv->mUpdateMux); + WriteLock clk(srv->mConfigReadWriteLock); + const auto configInfo = make_pair(configName, idx); + srv->mInputFileDiscoveryConfigsMap.try_emplace(configInfo, make_pair(discoveryOpts, ctx)); + srv->mFileContainerMetaMap.try_emplace(configInfo, fileContainerMetas); + srv->mInputFileReaderConfigsMap.try_emplace(configInfo, make_pair(fileReaderOpts, ctx)); + srv->mInputMultilineConfigsMap.try_emplace(configInfo, make_pair(multilineOpts, ctx)); + srv->mInputFileTagConfigsMap.try_emplace(configInfo, make_pair(fileTagOpts, ctx)); + srv->mAddedInputs.emplace(configInfo, ctx); +} + void StaticFileServerUnittest::TestGetNextAvailableReader() const { - // prepare test log - filesystem::create_directories("test_logs"); - vector files{ - "./test_logs/test_file_1.log", "./test_logs/test_file_2.log", "./test_logs/test_file_3.log"}; + sServer->Stop(); // no Run() thread so test is deterministic + fs::create_directories("test_logs"); + vector files{"./test_logs/test_file_1.log", "./test_logs/test_file_2.log", "./test_logs/test_file_3.log"}; vector contents{string(500, 'a') + "\n", string(500, 'b') + "\n", string(500, 'c') + "\n"}; - vector fingerprints; for (size_t i = 0; i < files.size(); ++i) { - { - ofstream fout(files[i], ios::binary); - fout << contents[i]; - } - auto& item = fingerprints.emplace_back(); - item.mFilePath = files[i]; - item.mDevInode = GetFileDevInode(files[i].string()); - item.mSignatureSize = contents[i].size() > 1024 ? 1024 : contents[i].size(); - item.mSignatureHash - = HashSignatureString(contents[i].substr(0, item.mSignatureSize).c_str(), item.mSignatureSize); - item.mSize = contents[i].size(); + ofstream fout(files[i], ios::binary); + fout << contents[i]; } - // build input CollectionPipeline p; p.mName = "test_config"; p.mPluginID.store(0); @@ -83,7 +119,7 @@ void StaticFileServerUnittest::TestGetNextAvailableReader() const { ctx.SetConfigName("test_config"); ctx.SetPipeline(p); - filesystem::path filePath = filesystem::absolute("./test_logs/*.log"); + fs::path filePath = fs::absolute("./test_logs/*.log"); string configStr = R"( { "Type": "input_static_file_onetime", @@ -99,26 +135,32 @@ void StaticFileServerUnittest::TestGetNextAvailableReader() const { input.CreateMetricsRecordRef(InputFile::sName, "1"); input.Init(configJson, optionalGoPipeline); input.CommitMetricsRecordRef(); - input.Start(); + std::unordered_map emptyFileContainerMetas; + AddInputWithoutStartingThread(sServer, + "test_config", + 0, + &input.mFileDiscovery, + &input.mFileReader, + &input.mMultiline, + &input.mFileTag, + emptyFileContainerMetas, + &ctx); + sServer->UpdateInputs(); - vector cptFiles; + vector cptFiles; for (const auto& item : sManager->mInputCheckpointMap.at(make_pair("test_config", 0)).mFileCheckpoints) { cptFiles.push_back(item.mFilePath); } - sServer->UpdateInputs(); + auto reader = sServer->GetNextAvailableReader("test_config", 0); + auto const& cpt = sManager->mInputCheckpointMap.at(make_pair("test_config", 0)); + APSARA_TEST_NOT_EQUAL(nullptr, reader); + APSARA_TEST_EQUAL(StaticFileReadingStatus::RUNNING, cpt.mStatus); + APSARA_TEST_EQUAL(FileStatus::WAITING, cpt.mFileCheckpoints[0].mStatus); - { - // file 1 existed - APSARA_TEST_NOT_EQUAL(nullptr, sServer->GetNextAvailableReader("test_config", 0)); - auto const& cpt = sManager->mInputCheckpointMap.at(make_pair("test_config", 0)); - APSARA_TEST_EQUAL(StaticFileReadingStatus::RUNNING, cpt.mStatus); - APSARA_TEST_EQUAL(FileStatus::WAITING, cpt.mFileCheckpoints[0].mStatus); - } sManager->UpdateCurrentFileCheckpoint("test_config", 0, 501); { - // file 2 not existed && file 3 signature changed - filesystem::remove(cptFiles[1]); + fs::remove(cptFiles[1]); { ofstream fout(cptFiles[2], ios::binary); fout << string(10, 'd') << "\n"; @@ -126,41 +168,52 @@ void StaticFileServerUnittest::TestGetNextAvailableReader() const { APSARA_TEST_EQUAL(nullptr, sServer->GetNextAvailableReader("test_config", 0)); APSARA_TEST_EQUAL(1U, sServer->mDeletedInputs.size()); APSARA_TEST_NOT_EQUAL(sServer->mDeletedInputs.end(), sServer->mDeletedInputs.find(make_pair("test_config", 0))); - auto const& cpt = sManager->mInputCheckpointMap.at(make_pair("test_config", 0)); - APSARA_TEST_EQUAL(StaticFileReadingStatus::FINISHED, cpt.mStatus); - APSARA_TEST_EQUAL(FileStatus::FINISHED, cpt.mFileCheckpoints[0].mStatus); - APSARA_TEST_EQUAL(FileStatus::ABORT, cpt.mFileCheckpoints[1].mStatus); - APSARA_TEST_EQUAL(FileStatus::ABORT, cpt.mFileCheckpoints[2].mStatus); + auto const& cpt2 = sManager->mInputCheckpointMap.at(make_pair("test_config", 0)); + APSARA_TEST_EQUAL(StaticFileReadingStatus::FINISHED, cpt2.mStatus); + APSARA_TEST_EQUAL(FileStatus::FINISHED, cpt2.mFileCheckpoints[0].mStatus); + APSARA_TEST_EQUAL(FileStatus::ABORT, cpt2.mFileCheckpoints[1].mStatus); + APSARA_TEST_EQUAL(FileStatus::ABORT, cpt2.mFileCheckpoints[2].mStatus); } sServer->UpdateInputs(); - APSARA_TEST_EQUAL(0U, sServer->mPipelineNameReadersMap.size()); + APSARA_TEST_EQUAL(0U, sServer->mInputReadersMap.size()); APSARA_TEST_EQUAL(0U, sServer->mDeletedInputs.size()); - input.Stop(true); - filesystem::remove_all("test_logs"); + fs::remove_all("test_logs"); } void StaticFileServerUnittest::TestUpdateInputs() const { - // new config - sServer->AddInput("test_config_1", 0, nullopt, nullptr, nullptr, nullptr, nullptr, nullptr); - sServer->AddInput("test_config_2", 0, nullopt, nullptr, nullptr, nullptr, nullptr, nullptr); - sServer->AddInput("test_config_2", 1, nullopt, nullptr, nullptr, nullptr, nullptr, nullptr); + sServer->Stop(); // ensure Run() is not running so this test is deterministic + FileDiscoveryOptions emptyDiscoveryOpts; + CollectionPipeline emptyPipeline; + CollectionPipelineContext ctx; + ctx.SetConfigName("test_config_1"); + ctx.SetPipeline(emptyPipeline); + std::unordered_map emptyFileContainerMetas; + + AddInputWithoutStartingThread( + sServer, "test_config_1", 0, &emptyDiscoveryOpts, nullptr, nullptr, nullptr, emptyFileContainerMetas, &ctx); + ctx.SetConfigName("test_config_2"); + AddInputWithoutStartingThread( + sServer, "test_config_2", 0, &emptyDiscoveryOpts, nullptr, nullptr, nullptr, emptyFileContainerMetas, &ctx); + AddInputWithoutStartingThread( + sServer, "test_config_2", 1, &emptyDiscoveryOpts, nullptr, nullptr, nullptr, emptyFileContainerMetas, &ctx); sServer->UpdateInputs(); - APSARA_TEST_EQUAL(3U, sServer->mPipelineNameReadersMap.size()); - APSARA_TEST_EQUAL(1U, sServer->mPipelineNameReadersMap.count("test_config_1")); - APSARA_TEST_EQUAL(2U, sServer->mPipelineNameReadersMap.count("test_config_2")); + APSARA_TEST_EQUAL(3U, sServer->mInputReadersMap.size()); + APSARA_TEST_EQUAL(2U, CountInputReadersForPipeline(sServer->mInputReadersMap, "test_config_2")); + APSARA_TEST_EQUAL(1U, CountInputReadersForPipeline(sServer->mInputReadersMap, "test_config_1")); APSARA_TEST_TRUE(sServer->mAddedInputs.empty()); APSARA_TEST_TRUE(sServer->HasRegisteredPlugins()); // update config sServer->RemoveInput("test_config_2", 0); sServer->RemoveInput("test_config_2", 1); - sServer->AddInput("test_config_2", 0, nullopt, nullptr, nullptr, nullptr, nullptr, nullptr); + AddInputWithoutStartingThread( + sServer, "test_config_2", 0, &emptyDiscoveryOpts, nullptr, nullptr, nullptr, emptyFileContainerMetas, &ctx); sServer->UpdateInputs(); - APSARA_TEST_EQUAL(2U, sServer->mPipelineNameReadersMap.size()); - APSARA_TEST_EQUAL(1U, sServer->mPipelineNameReadersMap.count("test_config_1")); - APSARA_TEST_EQUAL(1U, sServer->mPipelineNameReadersMap.count("test_config_2")); + APSARA_TEST_EQUAL(2U, sServer->mInputReadersMap.size()); + APSARA_TEST_EQUAL(1U, CountInputReadersForPipeline(sServer->mInputReadersMap, "test_config_1")); + APSARA_TEST_EQUAL(1U, CountInputReadersForPipeline(sServer->mInputReadersMap, "test_config_2")); APSARA_TEST_TRUE(sServer->mDeletedInputs.empty()); APSARA_TEST_TRUE(sServer->mAddedInputs.empty()); APSARA_TEST_TRUE(sServer->HasRegisteredPlugins()); @@ -169,7 +222,7 @@ void StaticFileServerUnittest::TestUpdateInputs() const { sServer->RemoveInput("test_config_1", 0); sServer->RemoveInput("test_config_2", 0); sServer->UpdateInputs(); - APSARA_TEST_EQUAL(0U, sServer->mPipelineNameReadersMap.size()); + APSARA_TEST_EQUAL(0U, sServer->mInputReadersMap.size()); APSARA_TEST_TRUE(sServer->mDeletedInputs.empty()); APSARA_TEST_FALSE(sServer->HasRegisteredPlugins()); } @@ -190,16 +243,15 @@ void StaticFileServerUnittest::TestClearUnusedCheckpoints() const { } void StaticFileServerUnittest::TestSetExpectedFileSize() const { - // prepare test log - filesystem::create_directories("test_logs"); - filesystem::path testFile = filesystem::absolute("./test_logs/test_file.log"); + sServer->Stop(); + fs::create_directories("test_logs"); + fs::path testFile = fs::absolute("./test_logs/test_file.log"); string content = string(5000, 'a') + "\n"; { ofstream fout(testFile, ios::binary); fout << content; } - // build input CollectionPipeline p; p.mName = "test_config"; p.mPluginID.store(0); @@ -207,40 +259,48 @@ void StaticFileServerUnittest::TestSetExpectedFileSize() const { ctx.SetConfigName("test_config"); ctx.SetPipeline(p); - optional> filesOpt({testFile}); - FileDiscoveryOptions discoveryOpts; - FileReaderOptions readerOpts; - readerOpts.mInputType = FileReaderOptions::InputType::InputFile; - MultilineOptions multilineOpts; - FileTagOptions fileTagOpts; + string configStr = R"({"Type": "input_static_file_onetime", "FilePaths": []})"; + string errorMsg; + Json::Value configJson, optionalGoPipeline; + APSARA_TEST_TRUE(ParseJsonTable(configStr, configJson, errorMsg)); + configJson["FilePaths"].append(Json::Value(testFile.string())); + InputStaticFile input; + input.SetContext(ctx); + input.CreateMetricsRecordRef(InputStaticFile::sName, "1"); + APSARA_TEST_TRUE(input.Init(configJson, optionalGoPipeline)); + input.CommitMetricsRecordRef(); - sServer->AddInput("test_config", 0, filesOpt, &discoveryOpts, &readerOpts, &multilineOpts, &fileTagOpts, &ctx); + std::unordered_map emptyFileContainerMetas; + AddInputWithoutStartingThread(sServer, + "test_config", + 0, + &input.mFileDiscovery, + &input.mFileReader, + &input.mMultiline, + &input.mFileTag, + emptyFileContainerMetas, + &ctx); sServer->UpdateInputs(); - // Get reader and check if mExpectedFileSize is set correctly auto reader = sServer->GetNextAvailableReader("test_config", 0); APSARA_TEST_NOT_EQUAL(nullptr, reader); - // Check that mExpectedFileSize is set to initial file size FileFingerprint fingerprint; sManager->GetCurrentFileFingerprint("test_config", 0, &fingerprint); - // Note: mExpectedFileSize is protected, but we can verify through GetRawData behavior - // The actual size should match the initial file size APSARA_TEST_EQUAL(static_cast(content.size()), fingerprint.mSize); sServer->RemoveInput("test_config", 0); sServer->UpdateInputs(); - filesystem::remove_all("test_logs"); + fs::remove_all("test_logs"); } void StaticFileServerUnittest::TestFileRotationDetection() const { - // prepare test log - filesystem::create_directories("test_logs"); - filesystem::path originalFile = filesystem::absolute("test_logs/test_file.log"); - filesystem::path rotatedFile = filesystem::absolute("test_logs/test_file.log.1"); + sServer->Stop(); + fs::create_directories("test_logs"); + fs::path originalFile = fs::absolute("test_logs/test_file.log"); + fs::path rotatedFile = fs::absolute("test_logs/test_file.log.1"); string content = string(2000, 'a') + "\n"; - // Create original file and get its devinode DevInode originalDevInode; { ofstream fout(originalFile, ios::binary); @@ -249,7 +309,6 @@ void StaticFileServerUnittest::TestFileRotationDetection() const { originalDevInode = GetFileDevInode(originalFile.string()); APSARA_TEST_TRUE(originalDevInode.IsValid()); - // build input CollectionPipeline p; p.mName = "test_config"; p.mPluginID.store(0); @@ -257,23 +316,33 @@ void StaticFileServerUnittest::TestFileRotationDetection() const { ctx.SetConfigName("test_config"); ctx.SetPipeline(p); - optional> filesOpt({originalFile}); - FileDiscoveryOptions discoveryOpts; - FileReaderOptions readerOpts; - readerOpts.mInputType = FileReaderOptions::InputType::InputFile; - MultilineOptions multilineOpts; - FileTagOptions fileTagOpts; + string configStr = R"({"Type": "input_static_file_onetime", "FilePaths": []})"; + string errorMsg; + Json::Value configJson, optionalGoPipeline; + APSARA_TEST_TRUE(ParseJsonTable(configStr, configJson, errorMsg)); + configJson["FilePaths"].append(Json::Value(originalFile.string())); + InputStaticFile input; + input.SetContext(ctx); + input.CreateMetricsRecordRef(InputStaticFile::sName, "1"); + APSARA_TEST_TRUE(input.Init(configJson, optionalGoPipeline)); + input.CommitMetricsRecordRef(); - sServer->AddInput("test_config", 0, filesOpt, &discoveryOpts, &readerOpts, &multilineOpts, &fileTagOpts, &ctx); + std::unordered_map emptyFileContainerMetas; + AddInputWithoutStartingThread(sServer, + "test_config", + 0, + &input.mFileDiscovery, + &input.mFileReader, + &input.mMultiline, + &input.mFileTag, + emptyFileContainerMetas, + &ctx); sServer->UpdateInputs(); - // Rotate file: move original to rotated (this preserves devinode), create new file - filesystem::rename(originalFile, rotatedFile); - // Verify rotated file has the same devinode + fs::rename(originalFile, rotatedFile); auto rotatedDevInode = GetFileDevInode(rotatedFile.string()); APSARA_TEST_EQUAL(originalDevInode, rotatedDevInode); - // Create new file at original path (will have different devinode) { ofstream fout(originalFile, ios::binary); fout << string(100, 'b') + "\n"; @@ -281,14 +350,243 @@ void StaticFileServerUnittest::TestFileRotationDetection() const { auto newFileDevInode = GetFileDevInode(originalFile.string()); APSARA_TEST_NOT_EQUAL(originalDevInode, newFileDevInode); - // Get reader - should find rotated file by devinode auto reader = sServer->GetNextAvailableReader("test_config", 0); APSARA_TEST_NOT_EQUAL(nullptr, reader); APSARA_TEST_EQUAL(rotatedFile, reader->GetHostLogPath()); sServer->RemoveInput("test_config", 0); sServer->UpdateInputs(); - filesystem::remove_all("test_logs"); + fs::remove_all("test_logs"); +} + +void StaticFileServerUnittest::TestGetFiles() const { + CollectionPipeline p; + p.mName = "test_config"; + p.mPluginID.store(0); + CollectionPipelineContext ctx; + ctx.SetConfigName("test_config"); + ctx.SetPipeline(p); + Json::Value optionalGoPipeline; + std::unordered_map emptyFileContainerMetas; + + // Call GetFiles() directly; no AddInput/UpdateInputs/Run() - avoids thread races + auto getFileCount = [&ctx, &emptyFileContainerMetas](InputStaticFile& input) -> size_t { + return StaticFileServer::GetFiles(&input.mFileDiscovery, &ctx, emptyFileContainerMetas).size(); + }; + + // Directory name with dot (filename() not stem() for matching) + { + fs::create_directories("test_logs/app.v1/subdir"); + fs::create_directories("test_logs/app.v2/subdir"); + fs::create_directories("test_logs/notstem"); + { ofstream fout("test_logs/app.v1/subdir/test1.log"); } + { ofstream fout("test_logs/app.v2/subdir/test2.log"); } + { ofstream fout("test_logs/notstem/test3.log"); } + fs::path filePath = fs::absolute("test_logs/app.*/subdir/*.log"); + filePath = NormalizeNativePath(filePath.string()); + Json::Value configJson; + configJson["FilePaths"].append(Json::Value(filePath.string())); + InputStaticFile input; + input.SetContext(ctx); + input.CreateMetricsRecordRef(InputStaticFile::sName, "1"); + APSARA_TEST_TRUE(input.Init(configJson, optionalGoPipeline)); + input.CommitMetricsRecordRef(); + APSARA_TEST_EQUAL(2U, getFileCount(input)); + fs::remove_all("test_logs"); + } + { + fs::create_directories("test_logs/release.2024.01.15/logs"); + fs::create_directories("test_logs/release.2024.01.16/logs"); + { ofstream fout("test_logs/release.2024.01.15/logs/app.log"); } + { ofstream fout("test_logs/release.2024.01.16/logs/app.log"); } + fs::path filePath = fs::absolute("test_logs/release.*/logs/**/*.log"); + filePath = NormalizeNativePath(filePath.string()); + Json::Value configJson; + configJson["FilePaths"].append(Json::Value(filePath.string())); + configJson["MaxDirSearchDepth"] = 10; + InputStaticFile input; + input.SetContext(ctx); + input.CreateMetricsRecordRef(InputStaticFile::sName, "1"); + APSARA_TEST_TRUE(input.Init(configJson, optionalGoPipeline)); + input.CommitMetricsRecordRef(); + APSARA_TEST_EQUAL(2U, getFileCount(input)); + fs::remove_all("test_logs"); + } + { + fs::create_directories("test_logs/node_modules.backup/lib"); + { ofstream fout("test_logs/node_modules.backup/lib/test.log"); } + fs::path filePath = fs::absolute("test_logs/node_modules.backup/lib/*.log"); + filePath = NormalizeNativePath(filePath.string()); + Json::Value configJson; + configJson["FilePaths"].append(Json::Value(filePath.string())); + InputStaticFile input; + input.SetContext(ctx); + input.CreateMetricsRecordRef(InputStaticFile::sName, "1"); + APSARA_TEST_TRUE(input.Init(configJson, optionalGoPipeline)); + input.CommitMetricsRecordRef(); + APSARA_TEST_EQUAL(1U, getFileCount(input)); + fs::remove_all("test_logs"); + } + + // wildcard dir + { + fs::create_directories("invalid_dir"); + fs::path filePath = fs::absolute("test_logs/*/**/*.log"); + filePath = NormalizeNativePath(filePath.string()); + Json::Value configJson; + configJson["FilePaths"].append(Json::Value(filePath.string())); + InputStaticFile input; + input.SetContext(ctx); + input.CreateMetricsRecordRef(InputStaticFile::sName, "1"); + APSARA_TEST_TRUE(input.Init(configJson, optionalGoPipeline)); + input.CommitMetricsRecordRef(); + APSARA_TEST_EQUAL(0U, getFileCount(input)); + fs::remove_all("invalid_dir"); + } + { + fs::create_directories("test_logs/dir"); + fs::path filePath = fs::absolute("test_logs/*/invalid_dir/**/*.log"); + filePath = NormalizeNativePath(filePath.string()); + Json::Value configJson; + configJson["FilePaths"].append(Json::Value(filePath.string())); + InputStaticFile input; + input.SetContext(ctx); + input.CreateMetricsRecordRef(InputStaticFile::sName, "1"); + APSARA_TEST_TRUE(input.Init(configJson, optionalGoPipeline)); + input.CommitMetricsRecordRef(); + APSARA_TEST_EQUAL(0U, getFileCount(input)); + fs::remove_all("test_logs"); + } + { + fs::create_directories("test_logs/dir"); + { ofstream fout("test_logs/dir/invalid_dir"); } + fs::path filePath = fs::absolute("test_logs/*/invalid_dir/**/*.log"); + filePath = NormalizeNativePath(filePath.string()); + Json::Value configJson; + configJson["FilePaths"].append(Json::Value(filePath.string())); + InputStaticFile input; + input.SetContext(ctx); + input.CreateMetricsRecordRef(InputStaticFile::sName, "1"); + APSARA_TEST_TRUE(input.Init(configJson, optionalGoPipeline)); + input.CommitMetricsRecordRef(); + APSARA_TEST_EQUAL(0U, getFileCount(input)); + fs::remove_all("test_logs"); + } + { + fs::create_directories("test_logs/dir1/dir/valid_dir"); + fs::create_directories("test_logs/dir2/dir/valid_dir"); + fs::create_directories("test_logs/unmatched_dir"); + { ofstream fout("test_logs/invalid_dir"); } + { ofstream fout("test_logs/dir1/dir/valid_dir/test1.log"); } + { ofstream fout("test_logs/dir2/dir/valid_dir/test2.log"); } + fs::path filePath = fs::absolute("test_logs/dir*/dir/valid_dir/**/*.log"); + filePath = NormalizeNativePath(filePath.string()); + Json::Value configJson; + configJson["FilePaths"].append(Json::Value(filePath.string())); + InputStaticFile input; + input.SetContext(ctx); + input.CreateMetricsRecordRef(InputStaticFile::sName, "1"); + APSARA_TEST_TRUE(input.Init(configJson, optionalGoPipeline)); + input.CommitMetricsRecordRef(); + APSARA_TEST_EQUAL(2U, getFileCount(input)); + fs::remove_all("test_logs"); + } + { + fs::create_directories("test_logs/dir1"); + fs::create_directories("test_logs/dir2"); + fs::create_directories("test_logs/unmatched_dir"); + { ofstream fout("test_logs/invalid_dir"); } + { ofstream fout("test_logs/dir1/test1.log"); } + { ofstream fout("test_logs/dir2/test2.log"); } + fs::path filePath = fs::absolute("test_logs/dir*/**/*.log"); + filePath = NormalizeNativePath(filePath.string()); + Json::Value configJson; + configJson["FilePaths"].append(Json::Value(filePath.string())); + InputStaticFile input; + input.SetContext(ctx); + input.CreateMetricsRecordRef(InputStaticFile::sName, "1"); + APSARA_TEST_TRUE(input.Init(configJson, optionalGoPipeline)); + input.CommitMetricsRecordRef(); + APSARA_TEST_EQUAL(2U, getFileCount(input)); + fs::remove_all("test_logs"); + } + // recursive dir search + { + fs::create_directories("invalid_dir"); + fs::path filePath = fs::absolute("test_logs/**/*.log"); + filePath = NormalizeNativePath(filePath.string()); + Json::Value configJson; + configJson["FilePaths"].append(Json::Value(filePath.string())); + InputStaticFile input; + input.SetContext(ctx); + input.CreateMetricsRecordRef(InputStaticFile::sName, "1"); + APSARA_TEST_TRUE(input.Init(configJson, optionalGoPipeline)); + input.CommitMetricsRecordRef(); + APSARA_TEST_EQUAL(0U, getFileCount(input)); + fs::remove_all("invalid_dir"); + } + { + { ofstream fout("test_logs"); } + fs::path filePath = fs::absolute("test_logs/**/*.log"); + filePath = NormalizeNativePath(filePath.string()); + Json::Value configJson; + configJson["FilePaths"].append(Json::Value(filePath.string())); + InputStaticFile input; + input.SetContext(ctx); + input.CreateMetricsRecordRef(InputStaticFile::sName, "1"); + APSARA_TEST_TRUE(input.Init(configJson, optionalGoPipeline)); + input.CommitMetricsRecordRef(); + APSARA_TEST_EQUAL(0U, getFileCount(input)); + fs::remove("test_logs"); + } + { + fs::create_directories("test_logs/dir1/dir2"); + fs::create_directories("test_logs/exclude_dir"); + { ofstream fout("test_logs/test0.log"); } + { ofstream fout("test_logs/exclude_file.log"); } + { ofstream fout("test_logs/dir1/test1.log"); } + { ofstream fout("test_logs/dir1/unmatched_file"); } + { ofstream fout("test_logs/dir1/exclude_filepath.log"); } + { ofstream fout("test_logs/dir1/dir2/test2.log"); } + fs::path filePath = fs::absolute("test_logs/**/*.log"); + fs::path excludeFilePath = fs::absolute("test_logs/dir*/exlcude_filepath.log"); + fs::path excludeDir = fs::absolute("test_logs/exclude*"); + filePath = NormalizeNativePath(filePath.string()); + excludeFilePath = NormalizeNativePath(excludeFilePath.string()); + excludeDir = NormalizeNativePath(excludeDir.string()); + Json::Value configJson; + configJson["FilePaths"].append(Json::Value(filePath.string())); + configJson["MaxDirSearchDepth"] = Json::Value(1); + configJson["ExcludeFilePaths"].append(Json::Value(excludeFilePath.string())); + configJson["ExcludeFiles"].append(Json::Value("exclude*.log")); + configJson["ExcludeDirs"].append(Json::Value(excludeDir.string())); + InputStaticFile input; + input.SetContext(ctx); + input.CreateMetricsRecordRef(InputStaticFile::sName, "1"); + APSARA_TEST_TRUE(input.Init(configJson, optionalGoPipeline)); + input.CommitMetricsRecordRef(); + APSARA_TEST_EQUAL(2U, getFileCount(input)); + fs::remove_all("test_logs"); + } + { + fs::create_directories("test_logs/dir1/dir2"); + fs::path symlinkTarget = fs::absolute("test_logs/dir1"); + symlinkTarget = NormalizeNativePath(symlinkTarget.string()); + fs::create_directory_symlink(symlinkTarget, "test_logs/dir1/dir2/dir3"); + { ofstream fout("test_logs/dir1/test.log"); } + fs::path filePath = fs::absolute("test_logs/**/*.log"); + filePath = NormalizeNativePath(filePath.string()); + Json::Value configJson; + configJson["FilePaths"].append(Json::Value(filePath.string())); + configJson["MaxDirSearchDepth"] = Json::Value(100); + InputStaticFile input; + input.SetContext(ctx); + input.CreateMetricsRecordRef(InputStaticFile::sName, "1"); + APSARA_TEST_TRUE(input.Init(configJson, optionalGoPipeline)); + input.CommitMetricsRecordRef(); + APSARA_TEST_EQUAL(1U, getFileCount(input)); + fs::remove_all("test_logs"); + } } UNIT_TEST_CASE(StaticFileServerUnittest, TestGetNextAvailableReader) @@ -296,6 +594,7 @@ UNIT_TEST_CASE(StaticFileServerUnittest, TestUpdateInputs) UNIT_TEST_CASE(StaticFileServerUnittest, TestClearUnusedCheckpoints) UNIT_TEST_CASE(StaticFileServerUnittest, TestSetExpectedFileSize) UNIT_TEST_CASE(StaticFileServerUnittest, TestFileRotationDetection) +UNIT_TEST_CASE(StaticFileServerUnittest, TestGetFiles) } // namespace logtail diff --git a/core/unittest/host_monitor/DiskCollectorUnittest.cpp b/core/unittest/host_monitor/DiskCollectorUnittest.cpp index d571a6fe58..7f02a3c50b 100644 --- a/core/unittest/host_monitor/DiskCollectorUnittest.cpp +++ b/core/unittest/host_monitor/DiskCollectorUnittest.cpp @@ -89,7 +89,7 @@ void DiskCollectorUnittest::TestGetSystemUptimeInformation() const { } void DiskCollectorUnittest::TestGetDiskSerialIdInformation() const { - bfs::create_directories("./vdb"); + fs::create_directories("./vdb"); std::string diskName = "vdb"; SerialIdInformation serialIdInfo; @@ -148,8 +148,8 @@ void DiskCollectorUnittest::GetDiskStateInformation() const { void DiskCollectorUnittest::TestCollect() const { std::this_thread::sleep_for(std::chrono::seconds(1)); - bfs::create_directories("./vdb"); - bfs::create_directories("./vda"); + fs::create_directories("./vdb"); + fs::create_directories("./vda"); PROCESS_DIR = "."; auto collector = DiskCollector(); diff --git a/core/unittest/host_monitor/GPUCollectorUnittest.cpp b/core/unittest/host_monitor/GPUCollectorUnittest.cpp index 90f0dc7224..adb6365209 100644 --- a/core/unittest/host_monitor/GPUCollectorUnittest.cpp +++ b/core/unittest/host_monitor/GPUCollectorUnittest.cpp @@ -38,7 +38,7 @@ class GPUCollectorUnittest : public testing::Test { }; bool GPUCollectorUnittest::CheckGPUExist() const { - if (!std::filesystem::exists(NVIDIACTL)) { + if (!fs::exists(NVIDIACTL)) { return false; } diff --git a/core/unittest/host_monitor/LinuxSystemInterfaceUnittest.cpp b/core/unittest/host_monitor/LinuxSystemInterfaceUnittest.cpp index b752dc794c..404a2d146d 100644 --- a/core/unittest/host_monitor/LinuxSystemInterfaceUnittest.cpp +++ b/core/unittest/host_monitor/LinuxSystemInterfaceUnittest.cpp @@ -48,7 +48,7 @@ class LinuxSystemInterfaceUnittest : public testing::Test { protected: void SetUp() override { - bfs::create_directories("./1"); + fs::create_directories("./1"); ofstream ofs1("./stat", std::ios::trunc); ofs1 << "btime 1731142542\n"; ofs1 << "cpu 1195061569 1728645 418424132 203670447952 14723544 0 773400 0 0 0\n"; @@ -59,7 +59,7 @@ class LinuxSystemInterfaceUnittest : public testing::Test { ofs1.close(); PROCESS_DIR = "."; - bfs::create_directories("./1"); + fs::create_directories("./1"); ofstream ofs2("./1/stat", std::ios::trunc); ofs2 << "1 (cat) R 0 1 1 34816 1 4194560 1110 0 0 0 1 1 0 0 20 0 1 0 18938584 4505600 171 18446744073709551615 " "4194304 4238788 140727020025920 0 0 0 0 0 0 0 0 0 17 3 0 0 0 0 0 6336016 6337300 21442560 " @@ -68,9 +68,9 @@ class LinuxSystemInterfaceUnittest : public testing::Test { } void TearDown() override { - bfs::remove_all("./1"); - bfs::remove_all("./stat"); - bfs::remove_all("./net"); + fs::remove_all("./1"); + fs::remove_all("./stat"); + fs::remove_all("./net"); } }; @@ -168,9 +168,9 @@ void LinuxSystemInterfaceUnittest::TestGetProcessOpenFilesOnce() const { int pid = 2; std::string mTestDir = "./tmp"; PROCESS_DIR = mTestDir; - bfs::create_directories(mTestDir); - bfs::create_directories(mTestDir + "/" + std::to_string(pid)); - bfs::create_directories(mTestDir + "/" + std::to_string(pid) + "/fd"); + fs::create_directories(mTestDir); + fs::create_directories(mTestDir + "/" + std::to_string(pid)); + fs::create_directories(mTestDir + "/" + std::to_string(pid) + "/fd"); ProcessFd processFd; // 创建5个测试文件作为文件描述符目标 @@ -178,7 +178,7 @@ void LinuxSystemInterfaceUnittest::TestGetProcessOpenFilesOnce() const { std::string targetFile = mTestDir + "/target" + std::to_string(i); std::ofstream ofs(targetFile); // 创建空文件 // 创建文件描述符符号链接 (0,1,2,3,4) - bfs::create_symlink(targetFile, mTestDir + "/" + std::to_string(pid) + "/fd/" + std::to_string(i)); + fs::create_symlink(targetFile, mTestDir + "/" + std::to_string(pid) + "/fd/" + std::to_string(i)); } // 启动单独线程执行文件读取 @@ -196,12 +196,12 @@ void LinuxSystemInterfaceUnittest::TestGetProcessOpenFilesOnce() const { sleep(1); // 删除 - bfs::remove_all(mTestDir + "/" + std::to_string(pid) + "/fd/4"); + fs::remove_all(mTestDir + "/" + std::to_string(pid) + "/fd/4"); // 等待获取进程列表操作完成 t.join(); - bfs::remove_all(mTestDir); + fs::remove_all(mTestDir); PROCESS_DIR = "."; }; @@ -209,9 +209,9 @@ void LinuxSystemInterfaceUnittest::TestGetProcessOpenFilesOncePermissionDenied() int pid = 3; std::string mTestDir = "./tmp_permission"; PROCESS_DIR = mTestDir; - bfs::create_directories(mTestDir); - bfs::create_directories(mTestDir + "/" + std::to_string(pid)); - bfs::create_directories(mTestDir + "/" + std::to_string(pid) + "/fd"); + fs::create_directories(mTestDir); + fs::create_directories(mTestDir + "/" + std::to_string(pid)); + fs::create_directories(mTestDir + "/" + std::to_string(pid) + "/fd"); // 设置目录权限为无读取权限(如果可能) // 注意:在某些系统上可能无法完全模拟权限拒绝,但至少测试代码不会崩溃 @@ -221,7 +221,7 @@ void LinuxSystemInterfaceUnittest::TestGetProcessOpenFilesOncePermissionDenied() // 主要验证不会崩溃 (void)result; // 标记变量为已使用,避免编译警告 - bfs::remove_all(mTestDir); + fs::remove_all(mTestDir); PROCESS_DIR = "."; }; @@ -229,9 +229,9 @@ void LinuxSystemInterfaceUnittest::TestGetProcessOpenFilesOnceFilesystemError() int pid = 4; std::string mTestDir = "./tmp_fs_error"; PROCESS_DIR = mTestDir; - bfs::create_directories(mTestDir); - bfs::create_directories(mTestDir + "/" + std::to_string(pid)); - bfs::create_directories(mTestDir + "/" + std::to_string(pid) + "/fd"); + fs::create_directories(mTestDir); + fs::create_directories(mTestDir + "/" + std::to_string(pid)); + fs::create_directories(mTestDir + "/" + std::to_string(pid) + "/fd"); ProcessFd processFd; // 测试文件系统错误的情况 @@ -240,7 +240,7 @@ void LinuxSystemInterfaceUnittest::TestGetProcessOpenFilesOnceFilesystemError() // 应该能处理文件系统错误,不会崩溃 (void)result; // 标记变量为已使用,避免编译警告 - bfs::remove_all(mTestDir); + fs::remove_all(mTestDir); PROCESS_DIR = "."; }; @@ -249,7 +249,7 @@ void LinuxSystemInterfaceUnittest::TestReadProcNetTcpNormalCase() const { // Create test directory structure std::string testDir = "./test_tcp"; PROCESS_DIR = testDir; - bfs::create_directories(testDir + "/net"); + fs::create_directories(testDir + "/net"); // Create /proc/net/tcp with normal format // Format: sl local_address rem_address st tx_queue rx_queue tr tm->when retrnsmt uid timeout inode @@ -285,7 +285,7 @@ void LinuxSystemInterfaceUnittest::TestReadProcNetTcpNormalCase() const { APSARA_TEST_EQUAL_FATAL(2, tcpStateCount[TCP_LISTEN]); // 2 connections in LISTEN state // Cleanup - bfs::remove_all(testDir); + fs::remove_all(testDir); PROCESS_DIR = "."; } @@ -293,7 +293,7 @@ void LinuxSystemInterfaceUnittest::TestReadProcNetTcpFileNotExist() const { // Test when TCP files don't exist std::string testDir = "./test_tcp_nofile"; PROCESS_DIR = testDir; - bfs::create_directories(testDir); + fs::create_directories(testDir); // Don't create net directory std::vector tcpStateCount(TCP_CLOSING + 1, 0); @@ -306,7 +306,7 @@ void LinuxSystemInterfaceUnittest::TestReadProcNetTcpFileNotExist() const { } // Cleanup - bfs::remove_all(testDir); + fs::remove_all(testDir); PROCESS_DIR = "."; } @@ -314,7 +314,7 @@ void LinuxSystemInterfaceUnittest::TestReadProcNetTcpMalformedLines() const { // Test handling of malformed lines std::string testDir = "./test_tcp_malformed"; PROCESS_DIR = testDir; - bfs::create_directories(testDir + "/net"); + fs::create_directories(testDir + "/net"); ofstream ofs(testDir + "/net/tcp", std::ios::trunc); ofs << " sl local_address rem_address st tx_queue rx_queue tr tm->when retrnsmt uid timeout inode\n"; @@ -335,7 +335,7 @@ void LinuxSystemInterfaceUnittest::TestReadProcNetTcpMalformedLines() const { APSARA_TEST_EQUAL_FATAL(1, tcpStateCount[TCP_LISTEN]); // Cleanup - bfs::remove_all(testDir); + fs::remove_all(testDir); PROCESS_DIR = "."; } @@ -343,7 +343,7 @@ void LinuxSystemInterfaceUnittest::TestReadProcNetTcpAllStates() const { // Test all valid TCP states (TCP_ESTABLISHED=1 to TCP_CLOSING=11) std::string testDir = "./test_tcp_allstates"; PROCESS_DIR = testDir; - bfs::create_directories(testDir + "/net"); + fs::create_directories(testDir + "/net"); ofstream ofs(testDir + "/net/tcp", std::ios::trunc); ofs << " sl local_address rem_address st tx_queue rx_queue tr tm->when retrnsmt uid timeout inode\n"; @@ -381,7 +381,7 @@ void LinuxSystemInterfaceUnittest::TestReadProcNetTcpAllStates() const { APSARA_TEST_EQUAL_FATAL(1, tcpStateCount[TCP_CLOSING]); // Cleanup - bfs::remove_all(testDir); + fs::remove_all(testDir); PROCESS_DIR = "."; } @@ -389,7 +389,7 @@ void LinuxSystemInterfaceUnittest::TestReadProcNetTcpInvalidStates() const { // Test handling of invalid state values (out of range) std::string testDir = "./test_tcp_invalid"; PROCESS_DIR = testDir; - bfs::create_directories(testDir + "/net"); + fs::create_directories(testDir + "/net"); ofstream ofs(testDir + "/net/tcp", std::ios::trunc); ofs << " sl local_address rem_address st tx_queue rx_queue tr tm->when retrnsmt uid timeout inode\n"; @@ -416,7 +416,7 @@ void LinuxSystemInterfaceUnittest::TestReadProcNetTcpInvalidStates() const { } // Cleanup - bfs::remove_all(testDir); + fs::remove_all(testDir); PROCESS_DIR = "."; } diff --git a/core/unittest/host_monitor/NetCollectorUnittest.cpp b/core/unittest/host_monitor/NetCollectorUnittest.cpp index 72e53ed208..729876ee2b 100644 --- a/core/unittest/host_monitor/NetCollectorUnittest.cpp +++ b/core/unittest/host_monitor/NetCollectorUnittest.cpp @@ -41,7 +41,7 @@ class NetCollectorUnittest : public ::testing::Test { // UDPLITE: inuse 0 // RAW: inuse 0 // FRAG: inuse 0 memory 0 - std::filesystem::create_directories("./net"); + fs::create_directories("./net"); ofstream ofs1("./net/sockstat", std::ios::trunc); ofs1 << "sockets: used 316\n"; ofs1 << "TCP: inuse 25 orphan 0 tw 2 alloc 28 mem 4\n"; @@ -169,13 +169,13 @@ void NetCollectorUnittest::TestCollect() const { void NetCollectorUnittest::TestIpv6FileNoExist() const { // 删除单个文件 std::error_code ec; - bool success = std::filesystem::remove("./net/sockstat6", ec); + bool success = fs::remove("./net/sockstat6", ec); if (!success && ec) { // 处理错误,比如文件不存在或权限不足 std::cerr << "Failed to delete file: " << ec.message() << std::endl; } - success = std::filesystem::remove("./net/if_inet6", ec); + success = fs::remove("./net/if_inet6", ec); if (!success && ec) { // 处理错误,比如文件不存在或权限不足 std::cerr << "Failed to delete file: " << ec.message() << std::endl; diff --git a/core/unittest/host_monitor/ProcessCollectorUnittest.cpp b/core/unittest/host_monitor/ProcessCollectorUnittest.cpp index 83e7fd8cde..caaabd1ea1 100644 --- a/core/unittest/host_monitor/ProcessCollectorUnittest.cpp +++ b/core/unittest/host_monitor/ProcessCollectorUnittest.cpp @@ -31,11 +31,11 @@ class ProcessCollectorUnittest : public testing::Test { protected: void SetUp() override { - bfs::create_directories("./12345"); - bfs::create_directories("./12345/fd"); - bfs::create_directories("./12345/fd/1"); - bfs::create_directories("./12345/fd/2"); - bfs::create_directories("./12345/fd/3"); + fs::create_directories("./12345"); + fs::create_directories("./12345/fd"); + fs::create_directories("./12345/fd/1"); + fs::create_directories("./12345/fd/2"); + fs::create_directories("./12345/fd/3"); // /proc/meminfo ofstream ofs_mem("./meminfo", std::ios::trunc); ofs_mem << "MemTotal: 31534908 kB\n"; @@ -118,8 +118,8 @@ class ProcessCollectorUnittest : public testing::Test { } void TearDown() override { - bfs::remove_all("./12345"); - bfs::remove("./meminfo"); + fs::remove_all("./12345"); + fs::remove("./meminfo"); } }; diff --git a/core/unittest/host_monitor/ProcessEntityCollectorUnittest.cpp b/core/unittest/host_monitor/ProcessEntityCollectorUnittest.cpp index 78f58dffb1..59a37f3e1c 100644 --- a/core/unittest/host_monitor/ProcessEntityCollectorUnittest.cpp +++ b/core/unittest/host_monitor/ProcessEntityCollectorUnittest.cpp @@ -38,7 +38,7 @@ class ProcessEntityCollectorUnittest : public testing::Test { protected: void SetUp() override { - bfs::create_directories("./1"); + fs::create_directories("./1"); ofstream ofs("./1/stat", std::ios::trunc); ofs << "1 (cat) R 0 1 1 34816 1 4194560 1110 0 0 0 1 1 0 0 20 0 1 0 18938584 4505600 171 18446744073709551615 " "4194304 4238788 140727020025920 0 0 0 0 0 0 0 0 0 17 3 0 0 0 0 0 6336016 6337300 21442560 " @@ -50,8 +50,8 @@ class ProcessEntityCollectorUnittest : public testing::Test { } void TearDown() override { - bfs::remove_all("./1"); - bfs::remove_all("./stat"); + fs::remove_all("./1"); + fs::remove_all("./stat"); } }; diff --git a/core/unittest/input/InputContainerStdioUnittest.cpp b/core/unittest/input/InputContainerStdioUnittest.cpp index 2426803304..9918817b49 100644 --- a/core/unittest/input/InputContainerStdioUnittest.cpp +++ b/core/unittest/input/InputContainerStdioUnittest.cpp @@ -12,7 +12,6 @@ // See the License for the specific language governing permissions and // limitations under the License. -#include #include #include #include @@ -89,9 +88,9 @@ void InputContainerStdioUnittest::TestTryGetRealPath() { STRING_FLAG(default_container_host_path) = "/tmp/home/admin"; // 删除 rootDirectory 目录 - std::filesystem::remove_all(rootDirectory); + fs::remove_all(rootDirectory); // 删除 STRING_FLAG(default_container_host_path) - std::filesystem::remove_all(STRING_FLAG(default_container_host_path)); + fs::remove_all(STRING_FLAG(default_container_host_path)); // 创建 /tmp/home/admin/test/test/test 目录 create_directory(rootDirectory + path); diff --git a/core/unittest/input/InputFileUnittest.cpp b/core/unittest/input/InputFileUnittest.cpp index d312e30e84..ef68894d00 100644 --- a/core/unittest/input/InputFileUnittest.cpp +++ b/core/unittest/input/InputFileUnittest.cpp @@ -12,7 +12,6 @@ // See the License for the specific language governing permissions and // limitations under the License. -#include #include #include @@ -25,6 +24,7 @@ #include "common/FileSystemUtil.h" #include "common/JsonUtil.h" #include "file_server/ContainerInfo.h" +#include "file_server/FileDiscoveryOptions.h" #include "file_server/FileServer.h" #include "plugin/input/InputFile.h" #include "plugin/processor/inner/ProcessorSplitLogStringNative.h" @@ -71,7 +71,7 @@ void InputFileUnittest::OnSuccessfulInit() { unique_ptr input; Json::Value configJson, optionalGoPipeline; string configStr, errorMsg; - filesystem::path filePath = filesystem::absolute("*.log"); + fs::path filePath = fs::absolute("*.log"); filePath = NormalizeNativePath(filePath.string()); // only mandatory param @@ -194,7 +194,7 @@ void InputFileUnittest::OnEnableContainerDiscovery() { unique_ptr input; Json::Value configJson, optionalGoPipelineJson, optionalGoPipeline; string configStr, optionalGoPipelineStr, errorMsg; - filesystem::path filePath = filesystem::absolute("*.log"); + fs::path filePath = fs::absolute("*.log"); string filePathStr = NormalizeNativePath(filePath.string()); configStr = R"( @@ -267,7 +267,7 @@ void InputFileUnittest::TestCreateInnerProcessors() { unique_ptr input; Json::Value configJson, optionalGoPipeline; string configStr, errorMsg; - filesystem::path filePath = filesystem::absolute("*.log"); + fs::path filePath = fs::absolute("*.log"); filePath = NormalizeNativePath(filePath.string()); { // no multiline @@ -473,7 +473,7 @@ void InputFileUnittest::OnPipelineUpdate() { InputFile input; input.SetContext(ctx); string configStr, errorMsg; - filesystem::path filePath = filesystem::absolute("*.log"); + fs::path filePath = fs::absolute("*.log"); filePath = NormalizeNativePath(filePath.string()); configStr = R"( @@ -504,7 +504,6 @@ void InputFileUnittest::OnPipelineUpdate() { } void InputFileUnittest::TestSetContainerBaseDir() { - InputFile inputFile; ContainerInfo containerInfo; // Create RawContainerInfo object @@ -520,28 +519,27 @@ void InputFileUnittest::TestSetContainerBaseDir() { std::string outRealBaseDir; - ASSERT_TRUE(inputFile.SetContainerBaseDirForPath(containerInfo, "/data2/log", outRealBaseDir)); + ASSERT_TRUE(SetContainerBaseDirForPath(containerInfo, "/data2/log", outRealBaseDir)); APSARA_TEST_EQUAL(STRING_FLAG(default_container_host_path) + "/UpperDir/data2/log", outRealBaseDir); outRealBaseDir = ""; - ASSERT_TRUE(inputFile.SetContainerBaseDirForPath(containerInfo, "/data1/log", outRealBaseDir)); + ASSERT_TRUE(SetContainerBaseDirForPath(containerInfo, "/data1/log", outRealBaseDir)); APSARA_TEST_EQUAL(STRING_FLAG(default_container_host_path) + "/source1/log", outRealBaseDir); outRealBaseDir = ""; - ASSERT_TRUE(inputFile.SetContainerBaseDirForPath(containerInfo, "/data1/data2/log", outRealBaseDir)); + ASSERT_TRUE(SetContainerBaseDirForPath(containerInfo, "/data1/data2/log", outRealBaseDir)); APSARA_TEST_EQUAL(STRING_FLAG(default_container_host_path) + "/source2/log", outRealBaseDir); outRealBaseDir = ""; - ASSERT_TRUE(inputFile.SetContainerBaseDirForPath(containerInfo, "/data1/data2/data3/log", outRealBaseDir)); + ASSERT_TRUE(SetContainerBaseDirForPath(containerInfo, "/data1/data2/data3/log", outRealBaseDir)); APSARA_TEST_EQUAL(STRING_FLAG(default_container_host_path) + "/source3/log", outRealBaseDir); outRealBaseDir = ""; - ASSERT_TRUE(inputFile.SetContainerBaseDirForPath(containerInfo, "/data1/data2/data3/data4/log", outRealBaseDir)); + ASSERT_TRUE(SetContainerBaseDirForPath(containerInfo, "/data1/data2/data3/data4/log", outRealBaseDir)); APSARA_TEST_EQUAL(STRING_FLAG(default_container_host_path) + "/source4/log", outRealBaseDir); outRealBaseDir = ""; - ASSERT_TRUE( - inputFile.SetContainerBaseDirForPath(containerInfo, "/data1/data2/data3/data4/data5/log", outRealBaseDir)); + ASSERT_TRUE(SetContainerBaseDirForPath(containerInfo, "/data1/data2/data3/data4/data5/log", outRealBaseDir)); APSARA_TEST_EQUAL(STRING_FLAG(default_container_host_path) + "/source4/data5/log", outRealBaseDir); } diff --git a/core/unittest/input/InputInternalAlarmsUnittest.cpp b/core/unittest/input/InputInternalAlarmsUnittest.cpp index 56ceb47667..6c116fed90 100644 --- a/core/unittest/input/InputInternalAlarmsUnittest.cpp +++ b/core/unittest/input/InputInternalAlarmsUnittest.cpp @@ -12,7 +12,6 @@ // See the License for the specific language governing permissions and // limitations under the License. -#include #include #include diff --git a/core/unittest/input/InputInternalMetricsUnittest.cpp b/core/unittest/input/InputInternalMetricsUnittest.cpp index 4c90721c13..c679424c40 100644 --- a/core/unittest/input/InputInternalMetricsUnittest.cpp +++ b/core/unittest/input/InputInternalMetricsUnittest.cpp @@ -12,7 +12,6 @@ // See the License for the specific language governing permissions and // limitations under the License. -#include #include #include diff --git a/core/unittest/input/InputStaticFileUnittest.cpp b/core/unittest/input/InputStaticFileUnittest.cpp index 945c16435f..3ab0ac3b64 100644 --- a/core/unittest/input/InputStaticFileUnittest.cpp +++ b/core/unittest/input/InputStaticFileUnittest.cpp @@ -35,14 +35,12 @@ class InputStaticFileUnittest : public testing::Test { void OnFailedInit(); void TestCreateInnerProcessors(); void OnPipelineUpdate(); - void TestGetFiles(); - void TestDirectoryNameWithDot(); void OnEnableContainerDiscovery(); protected: static void SetUpTestCase() { PluginRegistry::GetInstance()->LoadPlugins(); - sManager->mCheckpointRootPath = filesystem::path("./input_static_file"); + sManager->mCheckpointRootPath = fs::path("./input_static_file"); } static void TearDownTestCase() { PluginRegistry::GetInstance()->UnloadPlugins(); } @@ -52,14 +50,14 @@ class InputStaticFileUnittest : public testing::Test { ctx.SetConfigName("test_config"); p.mPluginID.store(0); ctx.SetPipeline(p); - filesystem::create_directories(sManager->mCheckpointRootPath); + fs::create_directories(sManager->mCheckpointRootPath); } void TearDown() override { sServer->Clear(); sManager->ClearUnusedCheckpoints(); sManager->mInputCheckpointMap.clear(); - filesystem::remove_all(sManager->mCheckpointRootPath); + fs::remove_all(sManager->mCheckpointRootPath); } private: @@ -77,7 +75,7 @@ void InputStaticFileUnittest::OnSuccessfulInit() { unique_ptr input; Json::Value configJson, optionalGoPipeline; string configStr, errorMsg; - filesystem::path filePath = filesystem::absolute("*.log"); + fs::path filePath = fs::absolute("*.log"); filePath = NormalizeNativePath(filePath.string()); // only mandatory param @@ -145,7 +143,7 @@ void InputStaticFileUnittest::OnFailedInit() { unique_ptr input; Json::Value configJson, optionalGoPipeline; string configStr, errorMsg; - filesystem::path filePath = filesystem::absolute("*.log"); + fs::path filePath = fs::absolute("*.log"); filePath = NormalizeNativePath(filePath.string()); // file path not existed @@ -192,7 +190,7 @@ void InputStaticFileUnittest::TestCreateInnerProcessors() { unique_ptr input; Json::Value configJson, optionalGoPipeline; string configStr, errorMsg; - filesystem::path filePath = filesystem::absolute("*.log"); + fs::path filePath = fs::absolute("*.log"); filePath = NormalizeNativePath(filePath.string()); { // no multiline @@ -395,13 +393,13 @@ void InputStaticFileUnittest::TestCreateInnerProcessors() { void InputStaticFileUnittest::OnPipelineUpdate() { // prepare logs - filesystem::create_directories("test_logs"); - vector files{"./test_logs/test_file_1.log"}; + fs::create_directories("test_logs"); + vector files{"./test_logs/test_file_1.log"}; { ofstream fout(files[0]); } Json::Value configJson, optionalGoPipeline; string configStr, errorMsg; - filesystem::path filePath = filesystem::absolute("./test_logs/*.log"); + fs::path filePath = fs::absolute("./test_logs/*.log"); filePath = NormalizeNativePath(filePath.string()); { // new config @@ -420,14 +418,15 @@ void InputStaticFileUnittest::OnPipelineUpdate() { input.CommitMetricsRecordRef(); APSARA_TEST_TRUE(input.Start()); + sServer->UpdateInputs(); APSARA_TEST_EQUAL(&input.mFileDiscovery, sServer->GetFileDiscoveryConfig("test_config", 0).first); APSARA_TEST_EQUAL(&input.mFileReader, sServer->GetFileReaderConfig("test_config", 0).first); APSARA_TEST_EQUAL(&input.mMultiline, sServer->GetMultilineConfig("test_config", 0).first); APSARA_TEST_EQUAL(&input.mFileTag, sServer->GetFileTagConfig("test_config", 0).first); const auto& cpt = sManager->mInputCheckpointMap.at(make_pair("test_config", 0)); APSARA_TEST_EQUAL(1U, cpt.mFileCheckpoints.size()); - APSARA_TEST_EQUAL(filesystem::absolute(files[0]).lexically_normal(), cpt.mFileCheckpoints[0].mFilePath); - APSARA_TEST_TRUE(filesystem::exists(sManager->mCheckpointRootPath / "test_config@0.json")); + APSARA_TEST_EQUAL(fs::absolute(files[0]).lexically_normal(), cpt.mFileCheckpoints[0].mFilePath); + APSARA_TEST_TRUE(fs::exists(sManager->mCheckpointRootPath / "test_config@0.json")); APSARA_TEST_TRUE(input.Stop(true)); APSARA_TEST_EQUAL(nullptr, sServer->GetFileDiscoveryConfig("test_config", 0).first); @@ -436,7 +435,7 @@ void InputStaticFileUnittest::OnPipelineUpdate() { APSARA_TEST_EQUAL(nullptr, sServer->GetFileTagConfig("test_config", 0).first); APSARA_TEST_EQUAL(sManager->mInputCheckpointMap.end(), sManager->mInputCheckpointMap.find(make_pair("test_config", 0))); - APSARA_TEST_FALSE(filesystem::exists(sManager->mCheckpointRootPath / "test_config@0.json")); + APSARA_TEST_FALSE(fs::exists(sManager->mCheckpointRootPath / "test_config@0.json")); } { // old config @@ -487,14 +486,16 @@ void InputStaticFileUnittest::OnPipelineUpdate() { input.CommitMetricsRecordRef(); APSARA_TEST_TRUE(input.Start()); + sServer->UpdateInputs(); APSARA_TEST_EQUAL(&input.mFileDiscovery, sServer->GetFileDiscoveryConfig("test_config", 0).first); APSARA_TEST_EQUAL(&input.mFileReader, sServer->GetFileReaderConfig("test_config", 0).first); APSARA_TEST_EQUAL(&input.mMultiline, sServer->GetMultilineConfig("test_config", 0).first); APSARA_TEST_EQUAL(&input.mFileTag, sServer->GetFileTagConfig("test_config", 0).first); const auto& cpt = sManager->mInputCheckpointMap.at(make_pair("test_config", 0)); APSARA_TEST_EQUAL(1U, cpt.mFileCheckpoints.size()); - APSARA_TEST_EQUAL("./test_logs/test_file_2.log", cpt.mFileCheckpoints[0].mFilePath.string()); - APSARA_TEST_TRUE(filesystem::exists(sManager->mCheckpointRootPath / "test_config@0.json")); + APSARA_TEST_EQUAL(fs::absolute("./test_logs/test_file_2.log").lexically_normal(), + fs::absolute(cpt.mFileCheckpoints[0].mFilePath).lexically_normal()); + APSARA_TEST_TRUE(fs::exists(sManager->mCheckpointRootPath / "test_config@0.json")); APSARA_TEST_TRUE(input.Stop(true)); APSARA_TEST_EQUAL(nullptr, sServer->GetFileDiscoveryConfig("test_config", 0).first); @@ -503,295 +504,9 @@ void InputStaticFileUnittest::OnPipelineUpdate() { APSARA_TEST_EQUAL(nullptr, sServer->GetFileTagConfig("test_config", 0).first); APSARA_TEST_EQUAL(sManager->mInputCheckpointMap.end(), sManager->mInputCheckpointMap.find(make_pair("test_config", 0))); - APSARA_TEST_FALSE(filesystem::exists(sManager->mCheckpointRootPath / "test_config@0.json")); - } - filesystem::remove_all("test_logs"); -} - -void InputStaticFileUnittest::TestGetFiles() { - unique_ptr input; - Json::Value optionalGoPipeline; - - // wildcard dir - { - // invalid base dir - filesystem::create_directories("invalid_dir"); - - filesystem::path filePath = filesystem::absolute("test_logs/*/**/*.log"); - filePath = NormalizeNativePath(filePath.string()); - Json::Value configJson; - configJson["FilePaths"].append(Json::Value(filePath.string())); - InputStaticFile input; - input.SetContext(ctx); - input.CreateMetricsRecordRef(InputStaticFile::sName, "1"); - APSARA_TEST_TRUE(input.Init(configJson, optionalGoPipeline)); - input.CommitMetricsRecordRef(); - APSARA_TEST_TRUE(input.GetFiles().empty()); - - filesystem::remove_all("invalid_dir"); - } - { - // non-existing const subdir - filesystem::create_directories("test_logs/dir"); - - filesystem::path filePath = filesystem::absolute("test_logs/*/invalid_dir/**/*.log"); - filePath = NormalizeNativePath(filePath.string()); - Json::Value configJson; - configJson["FilePaths"].append(Json::Value(filePath.string())); - InputStaticFile input; - input.SetContext(ctx); - input.CreateMetricsRecordRef(InputStaticFile::sName, "1"); - APSARA_TEST_TRUE(input.Init(configJson, optionalGoPipeline)); - input.CommitMetricsRecordRef(); - APSARA_TEST_TRUE(input.GetFiles().empty()); - - filesystem::remove_all("test_logs"); - } - { - // invalid const subdir - filesystem::create_directories("test_logs/dir"); - { ofstream fout("test_logs/dir/invalid_dir"); } - - filesystem::path filePath = filesystem::absolute("test_logs/*/invalid_dir/**/*.log"); - filePath = NormalizeNativePath(filePath.string()); - Json::Value configJson; - configJson["FilePaths"].append(Json::Value(filePath.string())); - InputStaticFile input; - input.SetContext(ctx); - input.CreateMetricsRecordRef(InputStaticFile::sName, "1"); - APSARA_TEST_TRUE(input.Init(configJson, optionalGoPipeline)); - input.CommitMetricsRecordRef(); - APSARA_TEST_TRUE(input.GetFiles().empty()); - - filesystem::remove_all("test_logs"); - } - { - // the last subdir before ** is const - filesystem::create_directories("test_logs/dir1/dir/valid_dir"); - filesystem::create_directories("test_logs/dir2/dir/valid_dir"); - filesystem::create_directories("test_logs/unmatched_dir"); - { ofstream fout("test_logs/invalid_dir"); } - { ofstream fout("test_logs/dir1/dir/valid_dir/test1.log"); } - { ofstream fout("test_logs/dir2/dir/valid_dir/test2.log"); } - - filesystem::path filePath = filesystem::absolute("test_logs/dir*/dir/valid_dir/**/*.log"); - filePath = NormalizeNativePath(filePath.string()); - Json::Value configJson; - configJson["FilePaths"].append(Json::Value(filePath.string())); - InputStaticFile input; - input.SetContext(ctx); - input.CreateMetricsRecordRef(InputStaticFile::sName, "1"); - APSARA_TEST_TRUE(input.Init(configJson, optionalGoPipeline)); - input.CommitMetricsRecordRef(); - APSARA_TEST_EQUAL(2U, input.GetFiles().size()); - - filesystem::remove_all("test_logs"); - } - { - // the last subdir before ** is wildcard - filesystem::create_directories("test_logs/dir1"); - filesystem::create_directories("test_logs/dir2"); - filesystem::create_directories("test_logs/unmatched_dir"); - { ofstream fout("test_logs/invalid_dir"); } - { ofstream fout("test_logs/dir1/test1.log"); } - { ofstream fout("test_logs/dir2/test2.log"); } - - filesystem::path filePath = filesystem::absolute("test_logs/dir*/**/*.log"); - filePath = NormalizeNativePath(filePath.string()); - Json::Value configJson; - configJson["FilePaths"].append(Json::Value(filePath.string())); - InputStaticFile input; - input.SetContext(ctx); - input.CreateMetricsRecordRef(InputStaticFile::sName, "1"); - APSARA_TEST_TRUE(input.Init(configJson, optionalGoPipeline)); - input.CommitMetricsRecordRef(); - APSARA_TEST_EQUAL(2U, input.GetFiles().size()); - - filesystem::remove_all("test_logs"); - } - // recursive dir search - { - // non-existing base path - filesystem::create_directories("invalid_dir"); - - filesystem::path filePath = filesystem::absolute("test_logs/**/*.log"); - filePath = NormalizeNativePath(filePath.string()); - Json::Value configJson; - configJson["FilePaths"].append(Json::Value(filePath.string())); - InputStaticFile input; - input.SetContext(ctx); - input.CreateMetricsRecordRef(InputStaticFile::sName, "1"); - APSARA_TEST_TRUE(input.Init(configJson, optionalGoPipeline)); - input.CommitMetricsRecordRef(); - APSARA_TEST_TRUE(input.GetFiles().empty()); - - filesystem::remove_all("invalid_dir"); - } - { - // invalid base path - { ofstream fout("test_logs"); } - - filesystem::path filePath = filesystem::absolute("test_logs/**/*.log"); - filePath = NormalizeNativePath(filePath.string()); - Json::Value configJson; - configJson["FilePaths"].append(Json::Value(filePath.string())); - InputStaticFile input; - input.SetContext(ctx); - input.CreateMetricsRecordRef(InputStaticFile::sName, "1"); - APSARA_TEST_TRUE(input.Init(configJson, optionalGoPipeline)); - input.CommitMetricsRecordRef(); - APSARA_TEST_TRUE(input.GetFiles().empty()); - - filesystem::remove("test_logs"); - } - { - // normal - filesystem::create_directories("test_logs/dir1/dir2"); - filesystem::create_directories("test_logs/exclude_dir"); - { ofstream fout("test_logs/test0.log"); } - { ofstream fout("test_logs/exclude_file.log"); } - { ofstream fout("test_logs/dir1/test1.log"); } - { ofstream fout("test_logs/dir1/unmatched_file"); } - { ofstream fout("test_logs/dir1/exclude_filepath.log"); } - { ofstream fout("test_logs/dir1/dir2/test2.log"); } - - filesystem::path filePath = filesystem::absolute("test_logs/**/*.log"); - filesystem::path excludeFilePath = filesystem::absolute("test_logs/dir*/exlcude_filepath.log"); - filesystem::path excludeDir = filesystem::absolute("test_logs/exclude*"); - filePath = NormalizeNativePath(filePath.string()); - excludeFilePath = NormalizeNativePath(excludeFilePath.string()); - excludeDir = NormalizeNativePath(excludeDir.string()); - Json::Value configJson; - configJson["FilePaths"].append(Json::Value(filePath.string())); - configJson["MaxDirSearchDepth"] = Json::Value(1); - configJson["ExcludeFilePaths"].append(Json::Value(excludeFilePath.string())); - configJson["ExcludeFiles"].append(Json::Value("exclude*.log")); - configJson["ExcludeDirs"].append(Json::Value(excludeDir.string())); - InputStaticFile input; - input.SetContext(ctx); - input.CreateMetricsRecordRef(InputStaticFile::sName, "1"); - APSARA_TEST_TRUE(input.Init(configJson, optionalGoPipeline)); - input.CommitMetricsRecordRef(); - APSARA_TEST_EQUAL(2U, input.GetFiles().size()); - - filesystem::remove_all("test_logs"); - } - { - // loop caused by symlink - filesystem::create_directories("test_logs/dir1/dir2"); - filesystem::path symlinkTarget = filesystem::absolute("test_logs/dir1"); - symlinkTarget = NormalizeNativePath(symlinkTarget.string()); - filesystem::create_directory_symlink(symlinkTarget, "test_logs/dir1/dir2/dir3"); - { ofstream fout("test_logs/dir1/test.log"); } - - filesystem::path filePath = filesystem::absolute("test_logs/**/*.log"); - filePath = NormalizeNativePath(filePath.string()); - Json::Value configJson; - configJson["FilePaths"].append(Json::Value(filePath.string())); - configJson["MaxDirSearchDepth"] = Json::Value(100); - InputStaticFile input; - input.SetContext(ctx); - input.CreateMetricsRecordRef(InputStaticFile::sName, "1"); - APSARA_TEST_TRUE(input.Init(configJson, optionalGoPipeline)); - input.CommitMetricsRecordRef(); - APSARA_TEST_EQUAL(1U, input.GetFiles().size()); - - filesystem::remove_all("test_logs"); - } -} - -void InputStaticFileUnittest::TestDirectoryNameWithDot() { - unique_ptr input; - Json::Value configJson, optionalGoPipeline; - string configStr, errorMsg; - - // Test 1: Directory name containing dot (e.g., "dir.test") - // This test ensures that filename() is used instead of stem() for directory matching - // stem() would strip everything after the dot, causing match failure - { - filesystem::create_directories("test_logs/app.v1/subdir"); - filesystem::create_directories("test_logs/app.v2/subdir"); - filesystem::create_directories("test_logs/notstem"); - { ofstream fout("test_logs/app.v1/subdir/test1.log"); } - { ofstream fout("test_logs/app.v2/subdir/test2.log"); } - { ofstream fout("test_logs/notstem/test3.log"); } - - filesystem::path filePath = filesystem::absolute("test_logs/app.*/subdir/*.log"); - configStr = R"( - { - "Type": "input_static_file_onetime", - "FilePaths": [] - } - )"; - APSARA_TEST_TRUE(ParseJsonTable(configStr, configJson, errorMsg)); - configJson["FilePaths"].append(Json::Value(filePath.string())); - input.reset(new InputStaticFile()); - input->SetContext(ctx); - input->CreateMetricsRecordRef(InputStaticFile::sName, "1"); - APSARA_TEST_TRUE(input->Init(configJson, optionalGoPipeline)); - input->CommitMetricsRecordRef(); - // Expected: Should find 2 files from app.v1 and app.v2 directories - // If stem() was used, it would match "app" only and fail to match "app.v1" and "app.v2" - auto files = input->GetFiles(); - APSARA_TEST_EQUAL(2U, files.size()); - - filesystem::remove_all("test_logs"); - } - - // Test 2: Directory with multiple dots using ** pattern - { - filesystem::create_directories("test_logs/release.2024.01.15/logs"); - filesystem::create_directories("test_logs/release.2024.01.16/logs"); - { ofstream fout("test_logs/release.2024.01.15/logs/app.log"); } - { ofstream fout("test_logs/release.2024.01.16/logs/app.log"); } - - filesystem::path filePath = filesystem::absolute("test_logs/release.*/logs/**/*.log"); - configStr = R"( - { - "Type": "input_static_file_onetime", - "FilePaths": [], - "MaxDirSearchDepth": 10 - } - )"; - APSARA_TEST_TRUE(ParseJsonTable(configStr, configJson, errorMsg)); - configJson["FilePaths"].append(Json::Value(filePath.string())); - input.reset(new InputStaticFile()); - input->SetContext(ctx); - input->CreateMetricsRecordRef(InputStaticFile::sName, "1"); - APSARA_TEST_TRUE(input->Init(configJson, optionalGoPipeline)); - input->CommitMetricsRecordRef(); - // Expected: Should find 2 files from both release directories - auto files = input->GetFiles(); - APSARA_TEST_EQUAL(2U, files.size()); - - filesystem::remove_all("test_logs"); - } - - // Test 3: Exact directory name with dot - { - filesystem::create_directories("test_logs/node_modules.backup/lib"); - { ofstream fout("test_logs/node_modules.backup/lib/test.log"); } - - filesystem::path filePath = filesystem::absolute("test_logs/node_modules.backup/lib/*.log"); - configStr = R"( - { - "Type": "input_static_file_onetime", - "FilePaths": [] - } - )"; - APSARA_TEST_TRUE(ParseJsonTable(configStr, configJson, errorMsg)); - configJson["FilePaths"].append(Json::Value(filePath.string())); - input.reset(new InputStaticFile()); - input->SetContext(ctx); - input->CreateMetricsRecordRef(InputStaticFile::sName, "1"); - APSARA_TEST_TRUE(input->Init(configJson, optionalGoPipeline)); - input->CommitMetricsRecordRef(); - // Expected: Should find 1 file - auto files = input->GetFiles(); - APSARA_TEST_EQUAL(1U, files.size()); - - filesystem::remove_all("test_logs"); + APSARA_TEST_FALSE(fs::exists(sManager->mCheckpointRootPath / "test_config@0.json")); } + fs::remove_all("test_logs"); } void InputStaticFileUnittest::OnEnableContainerDiscovery() { @@ -801,8 +516,6 @@ UNIT_TEST_CASE(InputStaticFileUnittest, OnSuccessfulInit) UNIT_TEST_CASE(InputStaticFileUnittest, OnFailedInit) UNIT_TEST_CASE(InputStaticFileUnittest, TestCreateInnerProcessors) UNIT_TEST_CASE(InputStaticFileUnittest, OnPipelineUpdate) -UNIT_TEST_CASE(InputStaticFileUnittest, TestGetFiles) -UNIT_TEST_CASE(InputStaticFileUnittest, TestDirectoryNameWithDot) UNIT_TEST_CASE(InputStaticFileUnittest, OnEnableContainerDiscovery) } // namespace logtail diff --git a/core/unittest/pipeline/PipelineUnittest.cpp b/core/unittest/pipeline/PipelineUnittest.cpp index db3fa1c89e..b89b94d80a 100644 --- a/core/unittest/pipeline/PipelineUnittest.cpp +++ b/core/unittest/pipeline/PipelineUnittest.cpp @@ -78,9 +78,9 @@ class PipelineUnittest : public ::testing::Test { private: const string configName = "test_config"; #if defined(_MSC_VER) - const filesystem::path filepath = "C:\\path\\to\\test"; + const fs::path filepath = "C:\\path\\to\\test"; #else - const filesystem::path filepath = "/path/to/test"; + const fs::path filepath = "/path/to/test"; #endif }; diff --git a/core/unittest/pipeline/PipelineUpdateUnittest.cpp b/core/unittest/pipeline/PipelineUpdateUnittest.cpp index cf46f4f116..151b34f809 100644 --- a/core/unittest/pipeline/PipelineUpdateUnittest.cpp +++ b/core/unittest/pipeline/PipelineUpdateUnittest.cpp @@ -414,7 +414,7 @@ class PipelineUpdateUnittest : public testing::Test { bool isFileServerStart = false; - filesystem::path filepath = "/path/to/test"; + fs::path filepath = "/path/to/test"; }; void PipelineUpdateUnittest::TestFileServerStart() { diff --git a/core/unittest/polling/PollingPreservedDirDepthUnittest.cpp b/core/unittest/polling/PollingPreservedDirDepthUnittest.cpp index 80690770c0..4f39799de7 100644 --- a/core/unittest/polling/PollingPreservedDirDepthUnittest.cpp +++ b/core/unittest/polling/PollingPreservedDirDepthUnittest.cpp @@ -1,6 +1,5 @@ #include // Include the header for sleep_for -#include #include // Include the header for this_thread #include "json/json.h" @@ -97,8 +96,8 @@ class PollingPreservedDirDepthUnittest : public ::testing::Test { INT32_FLAG(check_not_exist_file_dir_round) = 1; INT32_FLAG(polling_check_timeout_interval) = 0; AppConfig::GetInstance()->mCheckPointFilePath = GetProcessExecutionDir() + gCheckpoint; - if (bfs::exists(AppConfig::GetInstance()->mCheckPointFilePath)) { - filesystem::remove_all(AppConfig::GetInstance()->mCheckPointFilePath); + if (fs::exists(AppConfig::GetInstance()->mCheckPointFilePath)) { + fs::remove_all(AppConfig::GetInstance()->mCheckPointFilePath); } LoongCollectorMonitor::GetInstance()->Init(); FlusherRunner::GetInstance()->Init(); // reference: Application::Start @@ -122,13 +121,13 @@ class PollingPreservedDirDepthUnittest : public ::testing::Test { } void SetUp() override { - if (bfs::exists(AppConfig::GetInstance()->mCheckPointFilePath)) { - filesystem::remove_all(AppConfig::GetInstance()->mCheckPointFilePath); + if (fs::exists(AppConfig::GetInstance()->mCheckPointFilePath)) { + fs::remove_all(AppConfig::GetInstance()->mCheckPointFilePath); } - if (bfs::exists(gRootDir)) { - filesystem::remove_all(gRootDir); + if (fs::exists(gRootDir)) { + fs::remove_all(gRootDir); } - bfs::create_directories(gRootDir); + fs::create_directories(gRootDir); } void TearDown() override { @@ -143,9 +142,9 @@ class PollingPreservedDirDepthUnittest : public ::testing::Test { PollingModify::GetInstance()->ClearCache(); CheckPointManager::Instance()->RemoveAllCheckPoint(); // PollingEventQueue::GetInstance()->Clear(); - filesystem::remove_all(gRootDir); - if (bfs::exists(AppConfig::GetInstance()->mCheckPointFilePath)) { - filesystem::remove_all(AppConfig::GetInstance()->mCheckPointFilePath); + fs::remove_all(gRootDir); + if (fs::exists(AppConfig::GetInstance()->mCheckPointFilePath)) { + fs::remove_all(AppConfig::GetInstance()->mCheckPointFilePath); } FileServer::GetInstance()->Resume(); } @@ -180,7 +179,7 @@ class PollingPreservedDirDepthUnittest : public ::testing::Test { LOG_DEBUG(sLogger, ("Generate log", testFile)); auto pos = testFile.rfind(PATH_SEPARATOR); auto dir = testFile.substr(0, pos); - bfs::create_directories(dir); + fs::create_directories(dir); ofstream ofs(testFile, std::ios::app); ofs << "0\n"; } diff --git a/core/unittest/polling/PollingUnittest.cpp b/core/unittest/polling/PollingUnittest.cpp index 96a5cc6900..0a3a0ce70e 100644 --- a/core/unittest/polling/PollingUnittest.cpp +++ b/core/unittest/polling/PollingUnittest.cpp @@ -70,7 +70,6 @@ DECLARE_FLAG_INT32(merge_log_count_limit); DECLARE_FLAG_INT32(first_read_endure_bytes); DECLARE_FLAG_STRING(ilogtail_config); DECLARE_FLAG_STRING(user_log_config); -DECLARE_FLAG_INT32(max_buffer_num); DECLARE_FLAG_INT32(sls_host_update_interval); DECLARE_FLAG_INT32(max_client_send_error_count); DECLARE_FLAG_INT32(client_disable_send_retry_interval); @@ -338,10 +337,10 @@ class PollingUnittest : public ::testing::Test { bool hasEndpoint = false, int projectNum = 1, bool hasTopic = false) { - if (bfs::exists(gRootDir)) { - bfs::remove_all(gRootDir); + if (fs::exists(gRootDir)) { + fs::remove_all(gRootDir); } - bfs::create_directories(gRootDir); + fs::create_directories(gRootDir); WriteConfigJson(isFilter, hasAliuid, hasEndpoint, projectNum, hasTopic); commonCaseSetUp(); } @@ -351,7 +350,7 @@ class PollingUnittest : public ::testing::Test { PTScopedLock lock(gBufferLogGroupsLock); gBufferLogGroups.clear(); } - bfs::remove(STRING_FLAG(ilogtail_config)); + fs::remove(STRING_FLAG(ilogtail_config)); AppConfig::GetInstance()->LoadAppConfig(STRING_FLAG(ilogtail_config)); APSARA_TEST_TRUE_FATAL(ConfigManager::GetInstance()->LoadConfig(STRING_FLAG(user_log_config))); sleep(1); @@ -362,7 +361,7 @@ class PollingUnittest : public ::testing::Test { PollingDirFile::GetInstance()->ClearCache(); PollingModify::GetInstance()->ClearCache(); PollingEventQueue::GetInstance()->Clear(); - bfs::remove_all(gRootDir); + fs::remove_all(gRootDir); } void TestPollingDirFile() { @@ -391,7 +390,7 @@ class PollingUnittest : public ::testing::Test { string disableFileName = "job.xlog"; for (int prjIndex = 0; prjIndex < 8; ++prjIndex) { - bfs::create_directories(bfs::path(gRootDir) / logFileDirs[prjIndex]); + fs::create_directories(fs::path(gRootDir) / logFileDirs[prjIndex]); } for (int round = 0; round < 10; ++round) { @@ -452,13 +451,13 @@ class PollingUnittest : public ::testing::Test { APSARA_TEST_EQUAL(PollingEventQueue::GetInstance()->mEventQueue.size(), (size_t)0); for (int prjIndex = 0; prjIndex < 3; ++prjIndex) { - bfs::remove_all(bfs::path(gRootDir) / logFileDirs[prjIndex]); + fs::remove_all(fs::path(gRootDir) / logFileDirs[prjIndex]); } sleep(5); APSARA_TEST_EQUAL(PollingEventQueue::GetInstance()->mEventQueue.size(), (size_t)3); APSARA_TEST_EQUAL(PollingDirFile::GetInstance()->mDirCacheMap.size(), (size_t)6); APSARA_TEST_EQUAL(PollingDirFile::GetInstance()->mFileCacheMap.size(), (size_t)5); - bfs::remove_all(gRootDir); + fs::remove_all(gRootDir); sleep(5); @@ -525,7 +524,7 @@ class PollingUnittest : public ::testing::Test { #endif for (int prjIndex = 0; prjIndex < 8; ++prjIndex) { - bfs::create_directories(bfs::path(gRootDir) / logFileDirs[prjIndex]); + fs::create_directories(fs::path(gRootDir) / logFileDirs[prjIndex]); } for (int round = 0; round < 10; ++round) { @@ -613,7 +612,7 @@ class PollingUnittest : public ::testing::Test { INT32_FLAG(max_file_not_exist_times) = 1; for (int prjIndex = 0; prjIndex < 3; ++prjIndex) { - bfs::remove_all(bfs::path(gRootDir) / logFileDirs[prjIndex]); + fs::remove_all(fs::path(gRootDir) / logFileDirs[prjIndex]); } sleep(5); APSARA_TEST_EQUAL(PollingEventQueue::GetInstance()->mEventQueue.size(), (size_t)3); @@ -624,7 +623,7 @@ class PollingUnittest : public ::testing::Test { #endif - bfs::remove_all(gRootDir); + fs::remove_all(gRootDir); sleep(5); #if defined(__linux__) APSARA_TEST_EQUAL(PollingEventQueue::GetInstance()->mEventQueue.size(), (size_t)8 + 1); // lnk.log has event @@ -1062,7 +1061,7 @@ class PollingUnittest : public ::testing::Test { // Create subdirectories and files. for (int prjIndex = 0; prjIndex < 8; ++prjIndex) { - bfs::create_directories(bfs::path(gRootDir) / logFileDirs[prjIndex]); + fs::create_directories(fs::path(gRootDir) / logFileDirs[prjIndex]); } for (int round = 0; round < 10; ++round) { for (int prjIndex = 0; prjIndex < 8; ++prjIndex) { @@ -1154,7 +1153,7 @@ class PollingUnittest : public ::testing::Test { INT32_FLAG(max_file_not_exist_times) = 1; sleep(1); for (int prjIndex = 0; prjIndex < 3; ++prjIndex) { - bfs::remove_all(bfs::path(gRootDir) / logFileDirs[prjIndex]); + fs::remove_all(fs::path(gRootDir) / logFileDirs[prjIndex]); } sleep(10); for (int prjIndex = 0; prjIndex < 3; ++prjIndex) { @@ -1167,7 +1166,7 @@ class PollingUnittest : public ::testing::Test { // Delete root directory, all cache will be cleared and delete events // of files can be found. - bfs::remove_all(gRootDir); + fs::remove_all(gRootDir); sleep(10); for (int prjIndex = 0; prjIndex < 8; ++prjIndex) { auto evt = PollingEventQueue::GetInstance()->FindEvent( @@ -1221,7 +1220,7 @@ class PollingUnittest : public ::testing::Test { LOG_INFO(sLogger, ("Create directories and logs", "")); for (int prjIndex = 0; prjIndex < 8; ++prjIndex) { - bfs::create_directories(bfs::path(gRootDir) / logFileDirs[prjIndex]); + fs::create_directories(fs::path(gRootDir) / logFileDirs[prjIndex]); } for (int round = 0; round < 10; ++round) { for (int prjIndex = 0; prjIndex < 8; ++prjIndex) { @@ -1360,7 +1359,7 @@ class PollingUnittest : public ::testing::Test { string disableFileName = "job.xlog"; for (int prjIndex = 0; prjIndex < 8; ++prjIndex) { - bfs::create_directories(bfs::path(gRootDir) / logFileDirs[prjIndex]); + fs::create_directories(fs::path(gRootDir) / logFileDirs[prjIndex]); } for (int round = 0; round < 10; ++round) { @@ -1419,7 +1418,7 @@ class PollingUnittest : public ::testing::Test { APSARA_TEST_EQUAL(PollingModify::GetInstance()->mModifyCacheMap.size(), (size_t)8); for (int prjIndex = 0; prjIndex < 8; ++prjIndex) { - bfs::remove_all(bfs::path(gRootDir) / logFileDirs[prjIndex] / logFileNames[prjIndex]); + fs::remove_all(fs::path(gRootDir) / logFileDirs[prjIndex] / logFileNames[prjIndex]); } sleep(10); @@ -1487,7 +1486,7 @@ class PollingUnittest : public ::testing::Test { LOG_INFO(sLogger, ("TestPollingFileDeleted() end", time(NULL))); } - void testPollingWindowsRootPathCaseSetUp(bool enable, bool wildcard, std::vector& directories); + void testPollingWindowsRootPathCaseSetUp(bool enable, bool wildcard, std::vector& directories); void testPollingWindowsRootPathCaseCleanUp(); void TestPollingWindowsRootPath(); @@ -1512,18 +1511,18 @@ const std::string kDirPrefix = "logtailtestdir"; void PollingUnittest::testPollingWindowsRootPathCaseSetUp(bool enable, bool wildcard, - std::vector& directories) { - if (bfs::exists(gRootDir)) { - bfs::remove_all(gRootDir); + std::vector& directories) { + if (fs::exists(gRootDir)) { + fs::remove_all(gRootDir); } - bfs::create_directories(gRootDir); + fs::create_directories(gRootDir); { fsutil::Dir dir(kRootPath); ASSERT_TRUE(dir.Open()); while (fsutil::Entry e = dir.ReadNext()) { if ((e.IsRegFile() && EndWith(e.Name(), kFileExt)) || (e.IsDir() && StartWith(e.Name(), kDirPrefix))) { - bfs::remove_all(kRootPath + PATH_SEPARATOR + e.Name()); + fs::remove_all(kRootPath + PATH_SEPARATOR + e.Name()); } } } @@ -1553,20 +1552,20 @@ void PollingUnittest::testPollingWindowsRootPathCaseSetUp(bool enable, sleep(INT32_FLAG(dirfile_check_interval_ms) / 1000 * 2); // Subdirectories: d1, d2, d3, d3/d4, d3/d4/d5/d6, 2 files in each directory. - directories = std::vector{bfs::path(kRootPath + PATH_SEPARATOR), - bfs::path(kRootPath + PATH_SEPARATOR) / bfs::path(kDirPrefix + "1"), - bfs::path(kRootPath + PATH_SEPARATOR) / bfs::path(kDirPrefix + "2"), - bfs::path(kRootPath + PATH_SEPARATOR) / bfs::path(kDirPrefix + "3")}; + directories = std::vector{fs::path(kRootPath + PATH_SEPARATOR), + fs::path(kRootPath + PATH_SEPARATOR) / fs::path(kDirPrefix + "1"), + fs::path(kRootPath + PATH_SEPARATOR) / fs::path(kDirPrefix + "2"), + fs::path(kRootPath + PATH_SEPARATOR) / fs::path(kDirPrefix + "3")}; { - bfs::path p = directories.back(); + fs::path p = directories.back(); for (size_t idx = 4; idx <= 6; ++idx) { p /= (kDirPrefix + std::to_string(idx)); directories.push_back(p); } } for (auto& d : directories) { - if (!bfs::exists(d)) { - bfs::create_directories(d); + if (!fs::exists(d)) { + fs::create_directories(d); } } sleep(1); @@ -1594,7 +1593,7 @@ void PollingUnittest::testPollingWindowsRootPathCaseCleanUp() { void PollingUnittest::TestPollingWindowsRootPath() { LOG_INFO(sLogger, ("TestPollingWindowsRootPath() begin", time(NULL))); - std::vector directories; + std::vector directories; testPollingWindowsRootPathCaseSetUp(false, false, directories); auto eventQueue = PollingEventQueue::GetInstance(); @@ -1609,7 +1608,7 @@ APSARA_UNIT_TEST_CASE(PollingUnittest, TestPollingWindowsRootPath, 0); void PollingUnittest::TestPollingWindowsRootPathNew() { LOG_INFO(sLogger, ("TestPollingWindowsRootPathNew() begin", time(NULL))); - std::vector directories; + std::vector directories; testPollingWindowsRootPathCaseSetUp(true, false, directories); auto eventQueue = PollingEventQueue::GetInstance(); @@ -1652,7 +1651,7 @@ APSARA_UNIT_TEST_CASE(PollingUnittest, TestPollingWindowsRootPathNew, 0); void PollingUnittest::TestPollingWindowsWildcardRootPath() { LOG_INFO(sLogger, ("TestPollingWindowsWildcardRootPath() begin", time(NULL))); - std::vector directories; + std::vector directories; testPollingWindowsRootPathCaseSetUp(false, true, directories); auto eventQueue = PollingEventQueue::GetInstance(); @@ -1667,7 +1666,7 @@ APSARA_UNIT_TEST_CASE(PollingUnittest, TestPollingWindowsWildcardRootPath, 0); void PollingUnittest::TestPollingWindowsWildcardRootPathNew() { LOG_INFO(sLogger, ("TestPollingWindowsWildcardRootPathNew() begin", time(NULL))); - std::vector directories; + std::vector directories; testPollingWindowsRootPathCaseSetUp(true, true, directories); auto eventQueue = PollingEventQueue::GetInstance(); diff --git a/core/unittest/reader/LogFileReaderResolvedPathUnittest.cpp b/core/unittest/reader/LogFileReaderResolvedPathUnittest.cpp index 824e00b57f..5f072e8113 100644 --- a/core/unittest/reader/LogFileReaderResolvedPathUnittest.cpp +++ b/core/unittest/reader/LogFileReaderResolvedPathUnittest.cpp @@ -14,7 +14,6 @@ #include -#include #include #include "common/DevInode.h" @@ -31,17 +30,17 @@ namespace logtail { class LogFileReaderResolvedPathUnittest : public ::testing::Test { public: static void SetUpTestCase() { - logPathDir = (std::filesystem::path(GetProcessExecutionDir()) / "LogFileReaderResolvedPathUnittest").string(); - if (std::filesystem::exists(logPathDir)) { - std::filesystem::remove_all(logPathDir); + logPathDir = (fs::path(GetProcessExecutionDir()) / "LogFileReaderResolvedPathUnittest").string(); + if (fs::exists(logPathDir)) { + fs::remove_all(logPathDir); } - std::filesystem::create_directories(logPathDir); + fs::create_directories(logPathDir); AppConfig::GetInstance()->SetLoongcollectorConfDir(logPathDir); } static void TearDownTestCase() { - if (std::filesystem::exists(logPathDir)) { - std::filesystem::remove_all(logPathDir); + if (fs::exists(logPathDir)) { + fs::remove_all(logPathDir); } } @@ -84,7 +83,7 @@ std::string LogFileReaderResolvedPathUnittest::logPathDir; void LogFileReaderResolvedPathUnittest::TestResolveHostLogPathNormalFile() { // Test that ResolveHostLogPath works correctly for normal files const std::string fileName = "normal_file.log"; - const std::string filePath = (std::filesystem::path(logPathDir) / fileName).string(); + const std::string filePath = (fs::path(logPathDir) / fileName).string(); // Create a test file std::ofstream(filePath) << "test content\n"; @@ -102,7 +101,7 @@ void LogFileReaderResolvedPathUnittest::TestResolveHostLogPathNormalFile() { APSARA_TEST_EQUAL_FATAL(reader.mResolvedHostLogPath, filePath); // Clean up - std::filesystem::remove(filePath); + fs::remove(filePath); } void LogFileReaderResolvedPathUnittest::TestResolveHostLogPathSymbolicLink() { @@ -110,14 +109,14 @@ void LogFileReaderResolvedPathUnittest::TestResolveHostLogPathSymbolicLink() { // Test that ResolveHostLogPath correctly resolves symbolic links const std::string realFileName = "real_file.log"; const std::string linkName = "link_to_file.log"; - const std::string realFilePath = (std::filesystem::path(logPathDir) / realFileName).string(); - const std::string linkPath = (std::filesystem::path(logPathDir) / linkName).string(); + const std::string realFilePath = (fs::path(logPathDir) / realFileName).string(); + const std::string linkPath = (fs::path(logPathDir) / linkName).string(); // Create a real file std::ofstream(realFilePath) << "test content\n"; // Create a symbolic link - std::filesystem::create_symlink(realFilePath, linkPath); + fs::create_symlink(realFilePath, linkPath); LogFileReader reader(logPathDir, linkName, @@ -132,15 +131,15 @@ void LogFileReaderResolvedPathUnittest::TestResolveHostLogPathSymbolicLink() { APSARA_TEST_EQUAL_FATAL(reader.mResolvedHostLogPath, realFilePath); // Clean up - std::filesystem::remove(linkPath); - std::filesystem::remove(realFilePath); + fs::remove(linkPath); + fs::remove(realFilePath); #endif } void LogFileReaderResolvedPathUnittest::TestCheckFileSignatureWithZeroSizeAndDifferentPath() { // Test the fix: when signature size is 0 and paths differ, CheckFileSignatureAndOffset should return false const std::string fileName = "empty_file.log"; - const std::string filePath = (std::filesystem::path(logPathDir) / fileName).string(); + const std::string filePath = (fs::path(logPathDir) / fileName).string(); // Create an empty file std::ofstream(filePath) << ""; @@ -167,13 +166,13 @@ void LogFileReaderResolvedPathUnittest::TestCheckFileSignatureWithZeroSizeAndDif APSARA_TEST_FALSE_FATAL(result); // Clean up - std::filesystem::remove(filePath); + fs::remove(filePath); } void LogFileReaderResolvedPathUnittest::TestCheckFileSignatureWithZeroSizeAndSamePath() { // Test that when signature size is 0 but paths are the same, the check continues normally const std::string fileName = "empty_file2.log"; - const std::string filePath = (std::filesystem::path(logPathDir) / fileName).string(); + const std::string filePath = (fs::path(logPathDir) / fileName).string(); // Create an empty file std::ofstream(filePath) << ""; @@ -200,7 +199,7 @@ void LogFileReaderResolvedPathUnittest::TestCheckFileSignatureWithZeroSizeAndSam (void)reader.CheckFileSignatureAndOffset(false); // Clean up - std::filesystem::remove(filePath); + fs::remove(filePath); } void LogFileReaderResolvedPathUnittest::TestCopyTruncateUpgradeIssue() { @@ -210,16 +209,16 @@ void LogFileReaderResolvedPathUnittest::TestCopyTruncateUpgradeIssue() { // 也应该能够检测并修复路径,继续正常采集,而不是持续返回 false // 1. 创建目录结构:真实目录和软链接目录 - const std::string realDir = (std::filesystem::path(logPathDir) / "real_logs").string(); - const std::string linkDir = (std::filesystem::path(logPathDir) / "link_logs").string(); + const std::string realDir = (fs::path(logPathDir) / "real_logs").string(); + const std::string linkDir = (fs::path(logPathDir) / "link_logs").string(); const std::string fileName = "app.log"; - const std::string realFilePath = (std::filesystem::path(realDir) / fileName).string(); - const std::string linkFilePath = (std::filesystem::path(linkDir) / fileName).string(); - std::filesystem::create_directories(realDir); - if (std::filesystem::exists(linkDir)) { - std::filesystem::remove(linkDir); + const std::string realFilePath = (fs::path(realDir) / fileName).string(); + const std::string linkFilePath = (fs::path(linkDir) / fileName).string(); + fs::create_directories(realDir); + if (fs::exists(linkDir)) { + fs::remove(linkDir); } - std::filesystem::create_symlink(realDir, linkDir); + fs::create_symlink(realDir, linkDir); // 2. 创建初始文件并写入内容 std::ofstream(realFilePath) << "initial log content\nline 2\nline 3\n"; @@ -276,10 +275,10 @@ void LogFileReaderResolvedPathUnittest::TestCopyTruncateUpgradeIssue() { // 清理 CheckPointManager::Instance()->RemoveAllCheckPoint(); - std::filesystem::remove(linkFilePath); - std::filesystem::remove(realFilePath); - std::filesystem::remove(linkDir); - std::filesystem::remove_all(realDir); + fs::remove(linkFilePath); + fs::remove(realFilePath); + fs::remove(linkDir); + fs::remove_all(realDir); #endif } diff --git a/core/unittest/reader/LogFileReaderUnittest.cpp b/core/unittest/reader/LogFileReaderUnittest.cpp index 05b202fdbf..9f89f9e7ee 100644 --- a/core/unittest/reader/LogFileReaderUnittest.cpp +++ b/core/unittest/reader/LogFileReaderUnittest.cpp @@ -993,16 +993,16 @@ class LogFileReaderHoleUnittest : public ::testing::Test { } gRootDir += PATH_SEPARATOR + "testDataSet" + PATH_SEPARATOR + "LogFileReaderHoleUnittest"; gLogPath = gRootDir + PATH_SEPARATOR + gLogName; - bfs::remove_all(gRootDir); + fs::remove_all(gRootDir); } static void TearDownTestCase() {} void SetUp() override { - bfs::create_directories(gRootDir); + fs::create_directories(gRootDir); mReaderOpts.mInputType = FileReaderOptions::InputType::InputFile; mMultilineOpts.mMode = MultilineOptions::Mode::CUSTOM; } - void TearDown() override { bfs::remove_all(gRootDir); } + void TearDown() override { fs::remove_all(gRootDir); } static std::string gRootDir; static std::string gLogName; diff --git a/core/unittest/sender/SenderUnittest.cpp b/core/unittest/sender/SenderUnittest.cpp deleted file mode 100644 index 3897facccd..0000000000 --- a/core/unittest/sender/SenderUnittest.cpp +++ /dev/null @@ -1,3184 +0,0 @@ -// Copyright 2022 iLogtail Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include - -#include "json/json.h" - -#include "app_config/AppConfig.h" -#include "file_server/ConfigManager.h" -#include "file_server/EventDispatcher.h" -#include "file_server/event/Event.h" -#include "file_server/event_handler/EventHandler.h" -#include "file_server/reader/LogFileReader.h" -#include "monitor/Monitor.h" -#include "sender/Sender.h" -#include "unittest/Unittest.h" -#if defined(__linux__) -#include -#endif -#include -#include -#include -#include -#include -#include -#include -#include - -#include -#include -#include -#include -#include -#include -#include - -#include "boost/regex.hpp" - -#include "common/FileEncryption.h" -#include "common/FileSystemUtil.h" -#include "common/Lock.h" -#include "common/LogFileCollectOffsetIndicator.h" -#include "common/MemoryBarrier.h" -#include "common/StringTools.h" -#include "common/Thread.h" -#include "common/WaitObject.h" -#include "constants/Constants.h" -#include "file_server/checkpoint/CheckpointManagerV2.h" -#include "file_server/event_handler/LogInput.h" -#include "logger/Logger.h" -#include "monitor/AlarmManager.h" -#include "monitor/LogIntegrity.h" -#include "protobuf/sls/metric.pb.h" -#include "protobuf/sls/sls_logs.pb.h" -#include "runner/ProcessorRunner.h" -#include "sdk/Client.h" -#include "sdk/Common.h" -#include "sdk/Exception.h" - -using namespace std; -using namespace sls_logs; - -DECLARE_FLAG_INT32(buffer_file_alive_interval); -DECLARE_FLAG_STRING(profile_project_name); -DECLARE_FLAG_BOOL(enable_mock_send); -DECLARE_FLAG_INT32(max_holded_data_size); -DECLARE_FLAG_INT32(merge_log_count_limit); -DECLARE_FLAG_INT32(first_read_endure_bytes); -DECLARE_FLAG_STRING(ilogtail_config); -DECLARE_FLAG_STRING(user_log_config); -DECLARE_FLAG_STRING(logtail_profile_snapshot); -DECLARE_FLAG_INT32(buffer_check_period); -DECLARE_FLAG_INT32(monitor_interval); -DECLARE_FLAG_INT32(max_buffer_num); -DECLARE_FLAG_INT32(sls_host_update_interval); -DECLARE_FLAG_INT32(logtail_alarm_interval); - -DECLARE_FLAG_INT32(max_client_send_error_count); -DECLARE_FLAG_INT32(client_disable_send_retry_interval); -DECLARE_FLAG_INT32(client_disable_send_retry_interval_max); - -DECLARE_FLAG_INT32(max_client_quota_exceed_count); -DECLARE_FLAG_INT32(client_quota_send_concurrency_min); -DECLARE_FLAG_INT32(client_quota_send_retry_interval); -DECLARE_FLAG_INT32(client_quota_send_retry_interval_max); -DECLARE_FLAG_BOOL(send_prefer_real_ip); -DECLARE_FLAG_INT32(send_switch_real_ip_interval); - -DECLARE_FLAG_BOOL(default_global_mark_offset_flag); -DECLARE_FLAG_BOOL(default_global_fuse_mode); -DECLARE_FLAG_INT32(file_eliminate_interval); - -DECLARE_FLAG_INT32(dirfile_check_interval_ms); -DECLARE_FLAG_INT32(send_request_concurrency); -DECLARE_FLAG_INT32(test_unavailable_endpoint_interval); - -DECLARE_FLAG_STRING(logtail_send_address); -DECLARE_FLAG_INT32(batch_send_interval); - - -namespace logtail { - -string gRootDir = ""; -int gCaseID = 0; -bool gSetTimeFlag = false; -int32_t gStartIp = 1; -int gSendFailType = 1; // 1:network error; 2:all error can write secondary; 3:all error will not write secondary -bool gLogIntegrityTestFlag = false; -bool gGlobalMarkOffsetTestFlag = false; -bool gEnableExactlyOnce = false; -const size_t kConcurrency = 8; - -// warning: if you want to modify these cases, pay attention to the order -void getLogContent(char* buffer, time_t logTime, string content = "", int32_t seq = 0) { - char timeBuffer[50]; - struct tm timeInfo; -#if defined(__linux__) - localtime_r(&logTime, &timeInfo); -#elif defined(_MSC_VER) - localtime_s(&timeInfo, &logTime); -#endif - char buffer1[] = "10.7.241.21"; // - - [05/Mar/2012:15:10:59 +0800] "; - char buffer2[] - = "abcdefghijklmnopqrsputskjueiguwdhruwldirudsjhdklguejsldiuuwjskldgsksjdkdjfksjsdkfjsksdjfksjdkfuujss "; - strftime(timeBuffer, sizeof(timeBuffer), " - - [%d/%b/%Y:%R:%S +0800] ", &timeInfo); - if (content == "") - sprintf(buffer, "%s%s%s%d\n", buffer1, timeBuffer, buffer2, seq); - else - sprintf(buffer, "%s%s%s%d\n", buffer1, timeBuffer, (content + " ").c_str(), seq); -} - -// Write {logNum} logs to {filename} -// filename: /{workpath}/{jobname}.log -unsigned OneJob(int logNum, - string path, - string jobname, - bool jobOrNot, - time_t logTime, - string content = "", - int32_t seq = 0, - bool fixedTime = false, - int projectID = 0) { - //$1=num of log $2 path+name ;$3 job or not a job - char fileExt[32]; - if (jobOrNot) { - if (projectID == 0) { - strcpy(fileExt, ".log"); - } else { - sprintf(fileExt, ".log%d", projectID); - } - } else { - strcpy(fileExt, ".xlog"); - } - - auto out = FileAppendOpen((path + PATH_SEPARATOR + jobname + fileExt).c_str(), "ab"); - if (!out) { - return 0; - } - const static unsigned BUFFER_SIZE = 1024 * 20; - char* buffer; - unsigned lines = 0; - buffer = new char[BUFFER_SIZE + 1024]; - unsigned size = 0; - for (int i = 0; i < logNum; ++i) { - size = 0; - memset(buffer, 0, BUFFER_SIZE + 1024); - if (fixedTime) - getLogContent(buffer, logTime, content, seq); - else - getLogContent(buffer, logTime + i, content, seq); - size += strlen(buffer); - ++lines; - fwrite(buffer, 1, size, out); - } - delete[] buffer; - fclose(out); - return lines; -} - -string GenerateRandomStr(int32_t minLength, int32_t maxLength) { - int32_t length = (rand() % (maxLength - minLength + 1)) + minLength; - string randomStr = ""; - for (int32_t i = 0; i < length; ++i) { - // ascii: 33 - 126 - int temp = (rand() % (126 - 33 + 1)) + 33; - randomStr += (char)temp; - } - return randomStr; -} - -string MergeVectorString(vector& input) { - string output; - for (size_t i = 0; i < input.size(); i++) { - if (i != 0) - output.append(" | "); - output.append(input[i]); - } - return output; -} - -void WriteConfigJson(bool isFilter = false, - bool hasAliuid = false, - bool hasEndpoint = false, - int projectNum = 1, - bool hasTopic = false, - int32_t maxSendRateBytes = -1, - int32_t expireTime = 0) { - Json::Value rootJson; - //"slb_aliyun" - for (int i = 0; i < projectNum; ++i) { - Json::Value commonreg_com; - char projectName[32]; - sprintf(projectName, "%d_proj", 1000000 + i); - commonreg_com["project_name"] = Json::Value(projectName); - commonreg_com["category"] = Json::Value("app_log"); - commonreg_com["log_type"] = Json::Value("common_reg_log"); - commonreg_com["log_path"] = Json::Value(gRootDir); - commonreg_com["max_send_rate"] = Json::Value(maxSendRateBytes); - commonreg_com["send_rate_expire"] = Json::Value(expireTime); - if (i == 0) { -#if defined(__linux__) - commonreg_com["file_pattern"] = Json::Value("*.[Ll][Oo][Gg]"); -#elif defined(_MSC_VER) - commonreg_com["file_pattern"] = Json::Value("*.log"); -#endif - } else { - char filePattern[32]; -#if defined(__linux__) - sprintf(filePattern, "*.[Ll][Oo][Gg]%d", i); -#elif defined(_MSC_VER) - sprintf(filePattern, "*.log%d", i); -#endif - commonreg_com["file_pattern"] = Json::Value(filePattern); - commonreg_com["region"] = Json::Value(projectName); - } - commonreg_com["enable"] = Json::Value(true); - commonreg_com["timeformat"] = Json::Value("%d/%b/%Y:%H:%M:%S"); - commonreg_com["local_storage"] = Json::Value(true); - if (hasEndpoint) { - commonreg_com["defaultEndpoint"] = Json::Value("cn-yungu-test-intranet.log.aliyuntest.com"); - } - if (hasAliuid) { - char aliuid[32]; - sprintf(aliuid, "123456789%d", i); - commonreg_com["aliuid"] = Json::Value(aliuid); - } - Json::Value regs; - regs.append(Json::Value("([\\d\\.]+) \\S+ \\S+ \\[(\\S+) \\S+\\] (\\S+) (\\d+)")); - Json::Value keys; - keys.append(Json::Value("ip,time,nothing,seq")); - commonreg_com["regex"] = regs; - commonreg_com["keys"] = keys; - if (isFilter) { - Json::Value filter_keys; - filter_keys.append(Json::Value("nothing")); - filter_keys.append(Json::Value("seq")); - Json::Value filter_regs; - filter_regs.append(Json::Value("filter.*")); - filter_regs.append(Json::Value("5")); - commonreg_com["filter_keys"] = filter_keys; - commonreg_com["filter_regs"] = filter_regs; - } - if (hasTopic) { - commonreg_com["topic_format"] = Json::Value(".*/(.*)"); - } - if (gEnableExactlyOnce) { - Json::Value advJSON; - advJSON["exactly_once_concurrency"] = kConcurrency; - commonreg_com["advanced"] = advJSON; - } - commonreg_com["preserve"] = Json::Value(true); - if (i == 0) { - rootJson["commonreg.com"] = commonreg_com; - } else { - char title[32]; - sprintf(title, "commomreg.com_%d", i); - rootJson[title] = commonreg_com; - } - } - - - Json::Value secondary; - secondary["max_flow_byte_per_sec"] = Json::Value("10240000"); - secondary["max_num_of_file"] = Json::Value("10"); - secondary["enable_secondady"] = Json::Value(true); - - Json::Value metrics; - metrics["metrics"] = rootJson; - metrics["local_storage"] = secondary; - ofstream fout("user_log_config.json"); - fout << metrics << endl; - fout.close(); -} - -void WriteIntegrityConfigJson(int projectNum = 1) { - Json::Value rootJson; - //"slb_aliyun" - for (int i = 0; i < projectNum; ++i) { - Json::Value commonreg_com; - char projectName[32]; - sprintf(projectName, "%d_proj", 1000000 + i); - commonreg_com["project_name"] = Json::Value(projectName); - commonreg_com["category"] = Json::Value("app_log"); - commonreg_com["log_type"] = Json::Value("common_reg_log"); - commonreg_com["log_path"] = Json::Value(gRootDir); - commonreg_com["max_send_rate"] = Json::Value(-1); - commonreg_com["send_rate_expire"] = Json::Value(0); - if (i == 0) { -#if defined(__linux__) - commonreg_com["file_pattern"] = Json::Value("*.[Ll][Oo][Gg]"); -#elif defined(_MSC_VER) - commonreg_com["file_pattern"] = Json::Value("*.log"); -#endif - commonreg_com["region"] = Json::Value(projectName); // not default region here - } else { - char filePattern[32]; -#if defined(__linux__) - sprintf(filePattern, "*.[Ll][Oo][Gg]%d", i); -#elif defined(_MSC_VER) - sprintf(filePattern, "*.log%d", i); -#endif - commonreg_com["file_pattern"] = Json::Value(filePattern); - commonreg_com["region"] = Json::Value(projectName); - } - commonreg_com["enable"] = Json::Value(true); - commonreg_com["timeformat"] = Json::Value("%d/%b/%Y:%H:%M:%S"); - commonreg_com["local_storage"] = Json::Value(true); - - Json::Value regs; - regs.append(Json::Value("(.*)")); - Json::Value keys; - keys.append(Json::Value("content")); - commonreg_com["regex"] = regs; - commonreg_com["keys"] = keys; - - commonreg_com["preserve"] = Json::Value(true); - - if (i >= 4) { - Json::Value customizedFields; - Json::Value dataIntegrity; - //([0-9]{4})-(0[0-9]{1}|1[0-2])-(0[0-9]{1}|[12][0-9]{1}|3[01]) - //(0[0-9]{1}|1[0-9]{1}|2[0-3]):[0-5][0-9]{1}:([0-5][0-9]{1}) - dataIntegrity["log_time_reg"] - = Json::Value("(0[0-9]{1}|[12][0-9]{1}|3[01])/([A-Za-z]{3})/" - "([0-9]{4}):(0[0-9]{1}|1[0-9]{1}|2[0-3]):[0-5][0-9]{1}:([0-5][0-9]{1})"); - dataIntegrity["logstore"] = Json::Value("test-integrity-logstore"); - dataIntegrity["project_name"] = Json::Value(projectName); - dataIntegrity["switch"] = Json::Value(true); - dataIntegrity["time_pos"] = Json::Value(-1); - - customizedFields["data_integrity"] = dataIntegrity; - commonreg_com["customized_fields"] = customizedFields; - } - - if (i == 0) { - rootJson["commonreg.com"] = commonreg_com; - } else { - char title[32]; - sprintf(title, "commomreg.com_%d", i); - rootJson[title] = commonreg_com; - } - } - - Json::Value secondary; - secondary["max_flow_byte_per_sec"] = Json::Value("10240000"); - secondary["max_num_of_file"] = Json::Value("10"); - secondary["enable_secondady"] = Json::Value(true); - - Json::Value metrics; - metrics["metrics"] = rootJson; - metrics["local_storage"] = secondary; - ofstream fout("user_log_config.json"); - fout << metrics << endl; - fout.close(); -} - -bool gNetWorkStat; -bool gTestNetWorkStat = false; -int32_t gProjectNetEnableIndex = 0; -int32_t gAsynProjectSendFailCount = 0; -int32_t gSynProjectSendFailCount = 0; -int32_t gCounter; // send_success log num -int32_t gMessageSize; -LogGroup gRcvLogGroup; // the latest send_success log group -PTMutex gRecvLogGroupLock; -int32_t gStatusCount; -LogGroup gStatusLogGroup; -PTMutex gBufferLogGroupsLock; -vector gBufferLogGroups; // send_success log group -std::mutex gMockExactlyOnceSendLock; -std::vector gRangeCheckpoints; -std::set gBlockedHashKeySet; -std::unique_ptr gDispatchThreadId; -ThreadPtr gRealIpSendThread; -int32_t gAsynMockLatencyMs = 0; -int32_t gSynMockLatencyMs = 0; -bool gSendThreadRunFlag = true; -class SenderUnittest; -bool gDisableIpFlag = true; -sdk::Client* gClient = NULL; -string gDisabledIp; - -WaitObject mWait; -PTMutex mQueueLock; -struct AsyncRequest; -std::queue mAsynRequestQueue; -volatile bool mSignalFlag = false; - -void RunningDispatcher() { - LOG_INFO(sLogger, ("RunningDispatcher", "begin")); - ConfigManager::GetInstance()->RegisterHandlers(); - EventDispatcher::GetInstance()->Dispatch(); -} - -struct AsyncRequest { - const std::string mProjectName; - const std::string mLogstore; - const std::string mLogData; - SEND_DATA_TYPE mDataType; - int32_t mRawSize; - SlsCompressType mCompressType; - SendClosure* mSendClosure; - int64_t mBeginTime; - int32_t mSendIndex; - AsyncRequest(const std::string& projectName, - const std::string& logstore, - const std::string& logData, - SEND_DATA_TYPE dataType, - int32_t rawSize, - SlsCompressType compressType, - SendClosure* sendClosure) - : mProjectName(projectName), - mLogstore(logstore), - mLogData(logData), - mDataType(dataType), - mRawSize(rawSize), - mCompressType(compressType), - mSendClosure(sendClosure) { - static int32_t s_sendIndex = 0; - mSendIndex = s_sendIndex++; - mBeginTime = GetCurrentTimeInMilliSeconds(); - } - ~AsyncRequest() { - // printf("#### %d %d ms \n", mSendIndex, (int)(GetNowTime64() - mBeginTime)); - } -}; - -static decltype(CheckpointManagerV2::GetInstance()) sCptM = nullptr; -static decltype(ExactlyOnceQueueManager::GetInstance()) sQueueM = nullptr; -static decltype(EventDispatcher::GetInstance()) sEventDispatcher = nullptr; - -class SenderUnittest : public ::testing::Test { - static decltype(ProcessorRunner::GetInstance()->GetQueue().mLogstoreQueueMap)* sProcessQueueMap; - static decltype(Sender::Instance()->GetQueue().mLogstoreSenderQueueMap)* sSenderQueueMap; - void clearGlobalResource() { - sCptM->rebuild(); - sQueueM->clear(); - sProcessQueueMap->clear(); - sSenderQueueMap->clear(); - } - -protected: - static void StopMockSendThread() { - gSendThreadRunFlag = false; - { - WaitObject::Lock lock(mWait); - ReadWriteBarrier(); - mSignalFlag = true; - mWait.signal(); - } - sleep(1); - } - - static void ClearRequestQueue() { - PTScopedLock lock(mQueueLock); - while (!mAsynRequestQueue.empty()) { - delete mAsynRequestQueue.front(); - mAsynRequestQueue.pop(); - } - } - - static vector GetRequest() { - vector reqVec; - WaitObject::Lock lock(mWait); - ReadWriteBarrier(); - while (!mSignalFlag) { - mWait.wait(lock, 1000000); - } - mSignalFlag = false; - PTScopedLock mutexLock(mQueueLock); - // printf("Wait a request %d.\n", (int)mAsynRequestQueue.size()); - while (!mAsynRequestQueue.empty()) { - reqVec.push_back(mAsynRequestQueue.front()); - mAsynRequestQueue.pop(); - } - - return reqVec; - } - - static void MockAsyncSendThread() { - // printf("Thread Start.\n"); - while (gSendThreadRunFlag) { - vector reqVec = GetRequest(); - for (size_t i = 0; i < reqVec.size(); ++i) { - AsyncRequest* pReq = reqVec[i]; - if (gSynMockLatencyMs > 0) { - usleep(gSynMockLatencyMs * 1000); - } - // printf("Send Async Inner \n"); - MockAsyncSendInner(pReq->mProjectName, - pReq->mLogstore, - pReq->mLogData, - pReq->mDataType, - pReq->mRawSize, - pReq->mCompressType, - pReq->mSendClosure); - delete pReq; - } - } - } - - static sdk::GetRealIpResponse MockGetRealIp(const std::string& projectName, const std::string& logstore) { - static char ipStr[32]; - PTScopedLock lock(gRecvLogGroupLock); - sdk::GetRealIpResponse rsp; - sprintf(ipStr, "10.123.32.%d", ++gStartIp); - rsp.realIp = ipStr; - return rsp; - } - - static sdk::GetRealIpResponse MockGetEmptyRealIp(const std::string& projectName, const std::string& logstore) { - PTScopedLock lock(gRecvLogGroupLock); - sdk::GetRealIpResponse rsp; - rsp.realIp.clear(); - return rsp; - } - - static void MockSyncSend(const std::string& projectName, - const std::string& logstore, - const std::string& logData, - SEND_DATA_TYPE dataType, - int32_t rawSize, - sls_logs::SlsCompressType compressType) { - if (projectName == string("logtail-test-network-project")) { - // printf("test network %d.\n", gTestNetWorkStat ? 1 : 0); - if (gTestNetWorkStat) { - return; - } - throw sdk::LOGException(sdk::LOGE_REQUEST_ERROR, "Can not connect to server."); - } - LOG_INFO(sLogger, ("MockSyncSend, projectName", projectName)("logstore", logstore)("dataType", dataType)); - vector logGroupVec; - Sender::ParseLogGroupFromString(logData, dataType, rawSize, compressType, logGroupVec); - - for (vector::iterator iter = logGroupVec.begin(); iter != logGroupVec.end(); ++iter) { - bool projectDisabled = true; - if (iter->logs_size() > 0) { - if (gNetWorkStat && iter->category() == "app_log") { - PTScopedLock lock(gBufferLogGroupsLock); - gBufferLogGroups.push_back(*iter); - } - if (gNetWorkStat && (projectName.find("_proj") != string::npos)) /* "1000000_proj" */ - { - int prjIndex = atoi(projectName.substr(0, 7).c_str()) - 1000000; - if (prjIndex <= gProjectNetEnableIndex) { - projectDisabled = false; - gRecvLogGroupLock.lock(); - gRcvLogGroup = *iter; - gCounter += gRcvLogGroup.logs_size(); - gMessageSize += gRcvLogGroup.ByteSize(); - gRecvLogGroupLock.unlock(); - LOG_INFO( - sLogger, - ("gRcvLogGroup.ByteSize()", gRcvLogGroup.ByteSize())("logData.size()", logData.size())); - } else { - ++gSynProjectSendFailCount; - } - } - if (gNetWorkStat && projectName == STRING_FLAG(profile_project_name) - && iter->category() == "logtail_status_profile") { - gStatusCount++; - gStatusLogGroup = *iter; - } - } - if (!gNetWorkStat || projectDisabled) { - // printf("[@MockSyncSend] fail %s %s %d.\n", projectName.c_str(), logstore.c_str(), rawSize); - - if (gSendFailType == 1) - throw sdk::LOGException(sdk::LOGE_REQUEST_ERROR, "Can not connect to server."); - else if (gSendFailType == 2) { - int randInt = rand() % 4; - if (randInt == 0) - throw sdk::LOGException(sdk::LOGE_REQUEST_ERROR, "Can not connect to server."); - else if (randInt == 1) - throw sdk::LOGException(sdk::LOGE_WRITE_QUOTA_EXCEED, "project write quota exceed."); - else if (randInt == 2) - throw sdk::LOGException(sdk::LOGE_SERVER_BUSY, "connect to server timeout."); - else if (randInt == 3) - throw sdk::LOGException(sdk::LOGE_INTERNAL_SERVER_ERROR, "connect to server timeout."); - } else if (gSendFailType == 3) { - int randInt = rand() % 5; - if (randInt == 0) - throw sdk::LOGException(sdk::LOGE_UNAUTHORIZED, "LOGE_UNAUTHORIZED."); - else if (randInt == 1) - throw sdk::LOGException(sdk::LOGE_BAD_RESPONSE, "LOGE_BAD_RESPONSE."); - else if (randInt == 2) - throw sdk::LOGException(sdk::LOGE_CATEGORY_NOT_EXIST, "LOGE_CATEGORY_NOT_EXIST."); - else if (randInt == 3) - throw sdk::LOGException(sdk::LOGE_PROJECT_NOT_EXIST, "LOGE_PROJECT_NOT_EXIST."); - else if (randInt == 4) - throw sdk::LOGException(sdk::LOGE_TOPIC_NOT_EXIST, "LOGE_TOPIC_NOT_EXIST."); - } else if (gSendFailType == 4) { - int randInt = rand() % 2; - if (randInt == 0) { - throw sdk::LOGException(sdk::LOGE_WRITE_QUOTA_EXCEED, "LOGE_WRITE_QUOTA_EXCEED"); - } else if (randInt == 1) { - throw sdk::LOGException(sdk::LOGE_SHARD_WRITE_QUOTA_EXCEED, "LOGE_SHARD_WRITE_QUOTA_EXCEED"); - } - } else if (gSendFailType == 5) { - int randInt = rand() % 2; - if (randInt == 0) { - throw sdk::LOGException(sdk::LOGE_SERVER_BUSY, "LOGE_WRITE_QUOTA_EXCEED"); - } else if (randInt == 1) { - throw sdk::LOGException(sdk::LOGE_INTERNAL_SERVER_ERROR, "LOGE_SHARD_WRITE_QUOTA_EXCEED"); - } - } - - } else { - // printf("[@MockSyncSend] success %s %s %d.\n", projectName.c_str(), logstore.c_str(), rawSize); - } - } - } - - static void MockAsyncSend(const std::string& projectName, - const std::string& logstore, - const std::string& logData, - SEND_DATA_TYPE dataType, - int32_t rawSize, - sls_logs::SlsCompressType compressType, - SendClosure* sendClosure) { - // printf("Insert request.\n"); - // gSenderTest->MockAsyncSendInner(projectName, logstore, logData, dataType, rawSize, sendClosure); - // return; - AsyncRequest* pReq - = new AsyncRequest(projectName, logstore, logData, dataType, rawSize, compressType, sendClosure); - - { - PTScopedLock lock(mQueueLock); - mAsynRequestQueue.push(pReq); - } - - { - WaitObject::Lock lock(mWait); - ReadWriteBarrier(); - mSignalFlag = true; - mWait.signal(); - } - } - - static void MockAsyncSendInner(const std::string& projectName, - const std::string& logstore, - const std::string& logData, - SEND_DATA_TYPE dataType, - int32_t rawSize, - sls_logs::SlsCompressType compressType, - SendClosure* sendClosure) { - LOG_INFO(sLogger, ("MockAsyncSend, projectName", projectName)("logstore", logstore)("dataType", dataType)); - vector logGroupVec; - Sender::ParseLogGroupFromString(logData, dataType, rawSize, compressType, logGroupVec); - - if (gDisableIpFlag && gClient != NULL && gDisabledIp.size() > 0) { - if (gClient->GetRawSlsHost().find(gDisabledIp) != std::string::npos) { - auto sr = new sdk::PostLogStoreLogsResponse; - sr->statusCode = 500; - sr->requestId = ""; - sendClosure->OnFail(sr, sdk::LOGE_REQUEST_ERROR, "timeout to connect network"); - return; - } - } - - bool projectDisabled = true; - - for (vector::iterator iter = logGroupVec.begin(); iter != logGroupVec.end(); ++iter) { - // Record LogGroup information for debug. - { - LogGroup& logGroup = *iter; - std::string logCase; - if (logGroup.logs_size() > 0) { - auto& log = logGroup.logs(0); - if (log.contents_size() >= 3) { - logCase = log.contents(2).value(); - } - } - LOG_INFO(sLogger, - ("LogGroupDebugInfo", - logCase)("Project", projectName)("Logstore", logstore)("LogCount", logGroup.logs_size())); - } - - if (iter->logs_size() > 0) { - if (gNetWorkStat && iter->category() == "app_log") { - PTScopedLock lock(gBufferLogGroupsLock); - gBufferLogGroups.push_back(*iter); - } - if (gNetWorkStat && (projectName.find("_proj") != string::npos)) /* "1000000_proj" */ - { - int prjIndex = atoi(projectName.substr(0, 7).c_str()) - 1000000; - if (prjIndex <= gProjectNetEnableIndex) { - projectDisabled = false; - gRecvLogGroupLock.lock(); - gRcvLogGroup = *iter; - gCounter += gRcvLogGroup.logs_size(); - gMessageSize += gRcvLogGroup.ByteSize(); - gRecvLogGroupLock.unlock(); - } else { - // printf("Reject %s %d\n", projectName.c_str(), gProjectNetEnableIndex); - ++gAsynProjectSendFailCount; - } - } - if (gNetWorkStat && projectName == STRING_FLAG(profile_project_name) - && iter->category() == "logtail_status_profile") { - gStatusCount++; - gStatusLogGroup = *iter; - } - } - } - // can't put this code block in for, if have multi loggroup, the sendClosure will be destroyed multi times. - if (gNetWorkStat && !projectDisabled) { - // printf("[#MockAsyncSend] success %s %s %d.\n", projectName.c_str(), logstore.c_str(), rawSize); - - sdk::PostLogStoreLogsResponse* sr = new sdk::PostLogStoreLogsResponse; - sr->statusCode = 200; - sr->requestId = "mock_request_id"; - sendClosure->OnSuccess(sr); - } else { - sdk::PostLogStoreLogsResponse* sr = new sdk::PostLogStoreLogsResponse; - sr->statusCode = 500; - sr->requestId = "mock_request_id"; - - // printf("[#MockAsyncSend] fail %s %s %d.\n", projectName.c_str(), logstore.c_str(), rawSize); - if (gSendFailType == 1) - sendClosure->OnFail(sr, sdk::LOGE_REQUEST_ERROR, "timeout to connect network"); - else if (gSendFailType == 2) { - int randInt = rand() % 4; - if (randInt == 0) - sendClosure->OnFail(sr, sdk::LOGE_REQUEST_ERROR, "Can not connect to server."); - else if (randInt == 1) - sendClosure->OnFail(sr, sdk::LOGE_WRITE_QUOTA_EXCEED, "project write quota exceed."); - else if (randInt == 2) - sendClosure->OnFail(sr, sdk::LOGE_SERVER_BUSY, "connect to server timeout."); - else if (randInt == 3) - sendClosure->OnFail(sr, sdk::LOGE_INTERNAL_SERVER_ERROR, "connect to server timeout."); - } else if (gSendFailType == 3) { - int randInt = rand() % 5; - if (randInt == 0) - sendClosure->OnFail(sr, sdk::LOGE_UNAUTHORIZED, "LOGE_UNAUTHORIZED."); - else if (randInt == 1) - sendClosure->OnFail(sr, sdk::LOGE_BAD_RESPONSE, "LOGE_BAD_RESPONSE."); - else if (randInt == 2) - sendClosure->OnFail(sr, sdk::LOGE_CATEGORY_NOT_EXIST, "LOGE_CATEGORY_NOT_EXIST."); - else if (randInt == 3) - sendClosure->OnFail(sr, sdk::LOGE_PROJECT_NOT_EXIST, "LOGE_PROJECT_NOT_EXIST."); - else if (randInt == 4) - sendClosure->OnFail(sr, sdk::LOGE_TOPIC_NOT_EXIST, "LOGE_TOPIC_NOT_EXIST."); - } else if (gSendFailType == 4) { - int randInt = rand() % 2; - if (randInt == 0) { - sendClosure->OnFail(sr, sdk::LOGE_WRITE_QUOTA_EXCEED, "LOGE_WRITE_QUOTA_EXCEED"); - } else if (randInt == 1) { - sendClosure->OnFail(sr, sdk::LOGE_SHARD_WRITE_QUOTA_EXCEED, "LOGE_SHARD_WRITE_QUOTA_EXCEED"); - } - } else if (gSendFailType == 5) { - int randInt = rand() % 2; - if (randInt == 0) { - sendClosure->OnFail(sr, sdk::LOGE_SERVER_BUSY, "LOGE_WRITE_QUOTA_EXCEED"); - } else if (randInt == 1) { - sendClosure->OnFail(sr, sdk::LOGE_INTERNAL_SERVER_ERROR, "LOGE_SHARD_WRITE_QUOTA_EXCEED"); - } - } - } - } - - static void MockIntegritySend(LoggroupTimeValue* data) { - const std::string& project = data->mProjectName; - if (project[6] - '0' >= 4) - LogIntegrity::GetInstance()->Notify(data, true); - } - - void MockUserConfigJsonForMarkOffset() { - // remove old suer_log_config.json - bfs::remove_all(STRING_FLAG(user_log_config)); - - // construct json - Json::Value root; - Json::Value metrics; - Json::Value commonreg_com; - - commonreg_com["project_name"] = Json::Value("1000000_proj"); - commonreg_com["category"] = Json::Value("app_log"); - commonreg_com["log_path"] = Json::Value(gRootDir + "MarkOffsetTest"); - commonreg_com["file_pattern"] = Json::Value("*.log"); - commonreg_com["log_type"] = Json::Value("common_reg_log"); - commonreg_com["max_send_rate"] = Json::Value(-1); - commonreg_com["send_rate_expire"] = Json::Value(0); - commonreg_com["timeformat"] = Json::Value("%Y-%m-%d %H:%M:%S"); - - Json::Value customized_fields; - customized_fields["mark_offset"] = Json::Value(true); - customized_fields["fuse_mode"] = Json::Value(false); - commonreg_com["customized_fields"] = customized_fields; - - commonreg_com["enable"] = Json::Value(true); - commonreg_com["preserve"] = Json::Value(true); - commonreg_com["max_depth"] = Json::Value(10); - - Json::Value regs; - regs.append(Json::Value("(.*)")); - Json::Value keys; - keys.append(Json::Value("content")); - commonreg_com["regex"] = regs; - commonreg_com["keys"] = keys; - - commonreg_com["version"] = Json::Value(1); - - metrics["commonreg.com"] = commonreg_com; - root["metrics"] = metrics; - - // write json file - std::ofstream file(STRING_FLAG(user_log_config).c_str()); - file << root.toStyledString(); - file.close(); - } - - static void EnableNetWork() { gNetWorkStat = true; } - - static void DisableNetWork() { gNetWorkStat = false; } - - -public: - static void SetUpTestCase() // void Setup() - { - sLogger->set_level(spdlog::level::trace); - printf("Test case setup.\n"); - srand(time(NULL)); - Sender::Instance()->AddEndpointEntry(STRING_FLAG(default_region_name), - STRING_FLAG(logtail_send_address), - SLSClientManager::EndpointSourceType::LOCAL); - STRING_FLAG(profile_project_name) = "sls-admin"; - INT32_FLAG(sls_host_update_interval) = 1; - INT32_FLAG(logtail_alarm_interval) = 600; - BOOL_FLAG(enable_mock_send) = true; - gRootDir = GetProcessExecutionDir(); - if (PATH_SEPARATOR[0] == gRootDir.at(gRootDir.size() - 1)) - gRootDir.resize(gRootDir.size() - 1); - gRootDir += PATH_SEPARATOR + "SenderUnittest"; - bfs::remove_all(gRootDir); - - auto const sysConfDir = gRootDir + PATH_SEPARATOR + ".ilogtail" + PATH_SEPARATOR; - bfs::create_directories(sysConfDir); - AppConfig::GetInstance()->SetLoongcollectorConfDir(sysConfDir); - sCptM = CheckpointManagerV2::GetInstance(); - sQueueM = ExactlyOnceQueueManager::GetInstance(); - sEventDispatcher = EventDispatcher::GetInstance(); - sProcessQueueMap = &(ProcessorRunner::GetInstance()->GetQueue().mLogstoreQueueMap); - sSenderQueueMap = &(Sender::Instance()->GetQueue().mLogstoreSenderQueueMap); - - new Thread(&SenderUnittest::MockAsyncSendThread); - } - static void TearDownTestCase() // void CleanUp() - { - StopMockSendThread(); - try { - bfs::remove_all(gRootDir); - } catch (...) { - } - } - - void CaseSetUp(bool isFilter = false, - bool hasAliuid = false, - bool hasEndpoint = false, - int projectNum = 1, - bool hasTopic = false, - int32_t maxSendRateBytes = -1, - int32_t expireTime = 0, - int32_t clientDisableRetry = 0) { - // Test and enable container mode (if the special local file is existing). - if (bfs::exists("LogtailContainerModeTest")) { - cout << "replace with container config" << endl; - ReplaceWithContainerModeConfig(); - } - - ClearRequestQueue(); - gAsynMockLatencyMs = 0; - gSynMockLatencyMs = 0; - INT32_FLAG(client_disable_send_retry_interval) = clientDisableRetry; - INT32_FLAG(buffer_check_period) = 2; - INT32_FLAG(buffer_file_alive_interval) = 3; - gProjectNetEnableIndex = 0; - gAsynProjectSendFailCount = 0; - gSynProjectSendFailCount = 0; - gSendFailType = 1; - gTestNetWorkStat = false; - if (gLogIntegrityTestFlag) - WriteIntegrityConfigJson(projectNum); - else if (gGlobalMarkOffsetTestFlag) - MockUserConfigJsonForMarkOffset(); - else - WriteConfigJson(isFilter, hasAliuid, hasEndpoint, projectNum, hasTopic, maxSendRateBytes, expireTime); - - { - PTScopedLock lock(gBufferLogGroupsLock); - gBufferLogGroups.clear(); - } - bfs::remove("loongcollector_config.json"); - { - fsutil::Dir dir(gRootDir); - dir.Open(); - fsutil::Entry entry; - while ((entry = dir.ReadNext(false))) { - auto fullPath = gRootDir + PATH_SEPARATOR + entry.Name(); - auto targetPath = gRootDir + PATH_SEPARATOR + ".." + PATH_SEPARATOR + entry.Name(); - bfs::rename(fullPath, targetPath); - bfs::remove_all(targetPath); - } - } - sCptM->close(); - bfs::remove_all(gRootDir); - bfs::create_directories(AppConfig::GetInstance()->GetLoongcollectorConfDir()); - sCptM->open(); - if (gEnableExactlyOnce) { - clearGlobalResource(); - } - AppConfig::GetInstance()->LoadAppConfig(STRING_FLAG(ilogtail_config)); - bool ret = ConfigManager::GetInstance()->LoadConfig(STRING_FLAG(user_log_config)); - assert(ret); - ret = Sender::Instance()->Init(); - assert(ret); - - if (gLogIntegrityTestFlag) { - Sender::Instance()->MockIntegritySend = MockIntegritySend; - } else if (gEnableExactlyOnce) { - gRangeCheckpoints.clear(); - gBlockedHashKeySet.clear(); - Sender::Instance()->MockIntegritySend = MockExactlyOnceSend; - } else { - Sender::Instance()->MockIntegritySend = NULL; - } - - Sender::Instance()->MockAsyncSend = MockAsyncSend; - Sender::Instance()->MockSyncSend = MockSyncSend; - Sender::Instance()->mStopRealIpThread = false; - gCounter = 0; - gMessageSize = 0; - vector filesToSend; - Sender::Instance()->LoadFileToSend(time(NULL), filesToSend); - for (size_t i = 0; i < filesToSend.size(); ++i) { - remove((Sender::Instance()->mBufferFilePath + filesToSend[i]).c_str()); - } - gDispatchThreadId.reset(new std::thread(RunningDispatcher)); - sleep(1); - } - - - void ReplaceWithContainerModeConfig() { - Json::Value logtailConfig; - logtailConfig["container_mode"] = Json::Value(true); - logtailConfig["working_ip"] = Json::Value("1.2.3.4"); - logtailConfig["working_hostname"] = Json::Value("sls-zc-test"); - logtailConfig["container_mount_path"] = Json::Value("./container_mount_test.json"); - - ofstream fout(STRING_FLAG(ilogtail_config).c_str()); - fout << logtailConfig.toStyledString() << endl; - fout.close(); - - Json::Value mountConfig; - mountConfig["version"] = Json::Value("0.1.0"); - mountConfig["container_name"] = Json::Value("logtail-docker"); - mountConfig["container_id"] = Json::Value("abcdef1234567890"); - mountConfig["host_path"] = Json::Value("/"); - - Json::Value mount1; - mount1["destination"] = "/"; - mount1["source"] = ""; - Json::Value mount2; - mount2["destination"] = "/home/admin/logs"; - mount2["source"] = "/home/admin/t4/docker/logs"; - Json::Value mount3; - mount3["destination"] = "/app_2"; - mount3["source"] = "/yyyy"; - Json::Value mount4; - mount4["destination"] = "/app_2/xxx"; - mount4["source"] = "/xxx"; - - Json::Value mountArray; - mountArray.append(mount1); - mountArray.append(mount2); - mountArray.append(mount3); - mountArray.append(mount4); - mountConfig["container_mount"] = mountArray; - - ofstream foutMount("./container_mount_test.json"); - foutMount << mountConfig.toStyledString() << endl; - foutMount.close(); - } - - template - void ClearMap(T& m) { - for (auto& iter : m) { - delete iter.second; - } - m.clear(); - } - - void CaseCleanUp() { - LogInput::GetInstance()->CleanEnviroments(); - sleep(1); - EventDispatcher::GetInstance()->CleanEnviroments(); - ProcessorRunner::GetInstance()->CleanEnviroments(); - Sender::Instance()->RemoveSender(); - if (gRealIpSendThread) { - Sender::Instance()->mStopRealIpThread = true; - gRealIpSendThread.reset(); - PTScopedLock lock(Sender::Instance()->mRegionRealIpLock); - ClearMap(Sender::Instance()->mRegionRealIpMap); - } - { - PTScopedLock lock(Sender::Instance()->mRegionEndpointEntryMapLock); - ClearMap(Sender::Instance()->mRegionEndpointEntryMap); - } - { - PTScopedLock lock(Sender::Instance()->mSendClientLock); - ClearMap(Sender::Instance()->mSendClientMap); - } - gRecvLogGroupLock.lock(); - gRcvLogGroup.Clear(); - gRecvLogGroupLock.unlock(); - ConfigManager::GetInstance()->CleanEnviroments(); - delete ConfigManager::GetInstance()->mSharedHandler; - ConfigManager::GetInstance()->mSharedHandler = NULL; - LogIntegrity::GetInstance()->ClearForTest(); - - ClearRequestQueue(); - { - PTScopedLock lock(gBufferLogGroupsLock); - gBufferLogGroups.clear(); - } - - gDispatchThreadId->join(); - gDispatchThreadId = nullptr; - - sCptM->close(); - bfs::remove_all(gRootDir); - PollingDirFile::GetInstance()->ClearCache(); - PollingModify::GetInstance()->ClearCache(); - - gEnableExactlyOnce = false; - gLogIntegrityTestFlag = false; - gGlobalMarkOffsetTestFlag = false; - } - - LogstoreSenderInfo* GetSenderInfo(int projectId) { - char projectName[32]; - sprintf(projectName, "%d_proj", 1000000 + projectId); - - string logstore("app_log"); - - LogstoreFeedBackKey key = GenerateLogstoreFeedBackKey(string(projectName), logstore); - - LogstoreSenderInfo* pInfo = Sender::Instance()->mSenderQueue.GetSenderInfo(key); - if (pInfo == NULL) { - APSARA_TEST_TRUE_DESC(false, string(projectName) + " " + logstore + " send queue does not exist."); - return NULL; - } - return pInfo; - } - - void CheckSendClientStatus(int projectId, bool canSend) { - LogstoreSenderInfo* pInfo = GetSenderInfo(projectId); - if (pInfo == NULL) { - return; - } - APSARA_TEST_EQUAL_DESC(pInfo->CanSend(time(NULL)), canSend, string("projectid : ") + ToString(projectId)); - // printf("EQUAL %d %d\n", projectId, canSend ? 1 : 0); - } - - const int PROJECT_NUM = 8; - const int32_t ROUND_NUM = 10; - // 100 logs per round to let log time to be seperated in different minutes, - // so there will be two log groups. - const int32_t LOG_COUNT_PER_PROJECT = 100; - const int32_t TOTAL_LOG_COUNT_PER_PROJECT = ROUND_NUM * LOG_COUNT_PER_PROJECT; - const int32_t LOG_SEQ_NO = 5; - - void TestMultiUserSeperationAndRetryIntervalRecovery() { - LOG_INFO(sLogger, ("TestMultiUserSeperationAndRetryIntervalRecovery() begin", time(NULL))); - EnableNetWork(); - CaseSetUp(false, true, true, PROJECT_NUM, false, -1, 0, 900); - int32_t defaultSendErrorCount = INT32_FLAG(max_client_send_error_count); - INT32_FLAG(max_client_send_error_count) = 5; - Sender::Instance()->MockTestEndpoint = MockSyncSend; - gProjectNetEnableIndex = PROJECT_NUM / 2 - 1; - - // Generate 1000 logs for each project. - for (int round = 0; round < ROUND_NUM; ++round) { - for (int prjIndex = 0; prjIndex < PROJECT_NUM; ++prjIndex) { - OneJob(LOG_COUNT_PER_PROJECT, - gRootDir, - "Job", - true, - time(NULL), - "TestMultiUserSeperation", - LOG_SEQ_NO, - false, - prjIndex); - } - usleep(200 * 1000); - } - sleep(20); - - // 4 enabled projects (half) can send logs, so: - // - 4000 logs are received. - // - at least 4 projects have send failures. - // - only 4 enabled projects' clients are normal (CanSend). - APSARA_TEST_EQUAL(gCounter, PROJECT_NUM / 2 * TOTAL_LOG_COUNT_PER_PROJECT); - APSARA_TEST_TRUE(gAsynProjectSendFailCount >= PROJECT_NUM / 2); - for (int prjIndex = 0; prjIndex < PROJECT_NUM; ++prjIndex) { - CheckSendClientStatus(prjIndex, prjIndex <= gProjectNetEnableIndex); - } - gRecvLogGroupLock.lock(); - const Log& log = gRcvLogGroup.logs(0); - APSARA_TEST_EQUAL(log.contents(2).value(), "TestMultiUserSeperation"); - APSARA_TEST_EQUAL(log.contents(3).value(), ToString(LOG_SEQ_NO)); - gRecvLogGroupLock.unlock(); - - // Enable the network of the other 4 projects, try to recover by resetting retry interval - // to give sender a change to resend (in method LogstoreSenderInfo::CanSend). - gAsynProjectSendFailCount = 0; - gSynProjectSendFailCount = 0; - gCounter = 0; - gProjectNetEnableIndex = PROJECT_NUM; - for (int prjIndex = 0; prjIndex < PROJECT_NUM; ++prjIndex) { - LogstoreSenderInfo* info = GetSenderInfo(prjIndex); - if (info != NULL) { - info->mLastNetworkErrorTime = 0; - } - } - - // Generate more logs (1000 per project). - for (int round = 0; round < ROUND_NUM; ++round) { - for (int prjIndex = 0; prjIndex < PROJECT_NUM; ++prjIndex) { - OneJob(LOG_COUNT_PER_PROJECT, - gRootDir, - "Job", - true, - time(NULL), - "TestMultiUserSeperationSecond", - LOG_SEQ_NO, - false, - prjIndex); - } - usleep(100 * 1000); - } - sleep(20); - - // All projects can send logs, so: - // - 4000 logs in round 1 and 8000 logs in round 2 are received. - // - no project failure. - // - all clients are normal. - APSARA_TEST_EQUAL(gCounter, TOTAL_LOG_COUNT_PER_PROJECT * (PROJECT_NUM / 2 + PROJECT_NUM)); - APSARA_TEST_EQUAL(gAsynProjectSendFailCount, 0); - APSARA_TEST_EQUAL(gSynProjectSendFailCount, 0); - for (int prjIndex = 0; prjIndex < PROJECT_NUM; ++prjIndex) { - CheckSendClientStatus(prjIndex, true); - } - - // Case cleanup. - INT32_FLAG(max_client_send_error_count) = defaultSendErrorCount; - gProjectNetEnableIndex = 0; - CaseCleanUp(); - Sender::Instance()->MockTestEndpoint = NULL; - LOG_INFO(sLogger, ("TestMultiUserSeperationAndRetryIntervalRecovery() end", time(NULL))); - } - - void TestMultiUserSeperationAndTestNetWorkRecovery() { - LOG_INFO(sLogger, ("TestMultiUserSeperationAndTestNetWorkRecovery() begin", time(NULL))); - EnableNetWork(); - CaseSetUp(false, true, true, PROJECT_NUM, false, -1, 0, 900); - int32_t defaultSendErrorCount = INT32_FLAG(max_client_send_error_count); - INT32_FLAG(max_client_send_error_count) = 5; - int32_t defaultTestEndpointInterval = INT32_FLAG(test_unavailable_endpoint_interval); - INT32_FLAG(test_unavailable_endpoint_interval) = 5; // Quick recover for disabled projects. - Sender::Instance()->MockTestEndpoint = MockSyncSend; - gProjectNetEnableIndex = PROJECT_NUM / 2 - 1; - - // Generate 1000 logs for each project. - for (int round = 0; round < ROUND_NUM; ++round) { - for (int prjIndex = 0; prjIndex < PROJECT_NUM; ++prjIndex) { - OneJob(LOG_COUNT_PER_PROJECT, - gRootDir, - "Job", - true, - time(NULL), - "TestMultiUserSeperation", - LOG_SEQ_NO, - false, - prjIndex); - } - usleep(200 * 1000); - } - sleep(20); - - // 4 enabled projects (half) can send logs, so: - // - 4000 logs are received. - // - at least 4 projects have send failures. - // - only 4 enabled projects' clients are normal (CanSend). - APSARA_TEST_EQUAL(gCounter, PROJECT_NUM / 2 * TOTAL_LOG_COUNT_PER_PROJECT); - APSARA_TEST_TRUE(gAsynProjectSendFailCount >= PROJECT_NUM / 2); - for (int prjIndex = 0; prjIndex < PROJECT_NUM; ++prjIndex) { - CheckSendClientStatus(prjIndex, prjIndex <= gProjectNetEnableIndex); - } - gRecvLogGroupLock.lock(); - const Log& log = gRcvLogGroup.logs(0); - APSARA_TEST_EQUAL(log.contents(2).value(), "TestMultiUserSeperation"); - APSARA_TEST_EQUAL(log.contents(3).value(), ToString(LOG_SEQ_NO)); - gRecvLogGroupLock.unlock(); - - // Enable the network of the other 4 projects, try to recover by TestEndpoint. - gAsynProjectSendFailCount = 0; - gSynProjectSendFailCount = 0; - gCounter = 0; - gProjectNetEnableIndex = PROJECT_NUM; - gTestNetWorkStat = true; - sleep(INT32_FLAG(test_unavailable_endpoint_interval)); - for (int32_t idx = 0; idx < INT32_FLAG(test_unavailable_endpoint_interval) * 2; ++idx) { - if (gCounter == TOTAL_LOG_COUNT_PER_PROJECT * PROJECT_NUM / 2) { - break; - } - sleep(1); - } - - // Generate more logs (1000 per project). - for (int round = 0; round < ROUND_NUM; ++round) { - for (int prjIndex = 0; prjIndex < PROJECT_NUM; ++prjIndex) { - OneJob(LOG_COUNT_PER_PROJECT, - gRootDir, - "Job", - true, - time(NULL), - "TestMultiUserSeperationSecond", - LOG_SEQ_NO, - false, - prjIndex); - } - usleep(100 * 1000); - } - sleep(20); - - // All projects can send logs, so: - // - 4000 logs in round 1 and 8000 logs in round 2 are received. - // - no project failure. - // - all clients are normal. - APSARA_TEST_EQUAL(gCounter, TOTAL_LOG_COUNT_PER_PROJECT * (PROJECT_NUM / 2 + PROJECT_NUM)); - APSARA_TEST_EQUAL(gAsynProjectSendFailCount, 0); - APSARA_TEST_EQUAL(gSynProjectSendFailCount, 0); - for (int prjIndex = 0; prjIndex < PROJECT_NUM; ++prjIndex) { - CheckSendClientStatus(prjIndex, true); - } - - // Case cleanup. - Sender::Instance()->MockTestEndpoint = NULL; - INT32_FLAG(max_client_send_error_count) = defaultSendErrorCount; - INT32_FLAG(test_unavailable_endpoint_interval) = defaultTestEndpointInterval; - gProjectNetEnableIndex = 0; - CaseCleanUp(); - Sender::Instance()->MockTestEndpoint = NULL; - LOG_INFO(sLogger, ("TestMultiUserSeperationAndTestNetWorkRecovery() end", time(NULL))); - } - - void TestMultiUserSeperationAndRetryQuotaRecovery() { - LOG_INFO(sLogger, ("TestMultiUserSeperationAndRetryQuotaRecovery() begin", time(NULL))); - EnableNetWork(); - CaseSetUp(false, true, true, PROJECT_NUM); - auto bakClientQuotaSendRetryInterval = INT32_FLAG(client_quota_send_retry_interval); - INT32_FLAG(client_quota_send_retry_interval) = 900; - int32_t defaultQuotaErrorCount = INT32_FLAG(max_client_quota_exceed_count); - INT32_FLAG(max_client_quota_exceed_count) = 5; - int32_t seqNo = 5; - - // try to set net stat ok, this will set quota interval - gProjectNetEnableIndex = PROJECT_NUM; - for (int round = 0; round < 1; ++round) { - for (int prjIndex = 0; prjIndex < PROJECT_NUM; ++prjIndex) { - // use 100, let time to be seperated in different minutes, so there will be two packets - OneJob(10, gRootDir, "Job", true, time(NULL), "TestMultiUserSeperation", seqNo, false, prjIndex); - } - usleep(200 * 1000); - } - WaitForFileBeenRead(); - // printf("[###] %d %d\n", gCounter, gAsynProjectSendFailCount); - sleep(10); - APSARA_TEST_EQUAL(gCounter, 10 * PROJECT_NUM); - gCounter = 0; - Sender::Instance()->MockTestEndpoint = MockSyncSend; - gProjectNetEnableIndex = PROJECT_NUM / 2 - 1; - // quota exceed alarm - gSendFailType = 4; - for (int round = 0; round < 10; ++round) { - for (int prjIndex = 0; prjIndex < PROJECT_NUM; ++prjIndex) { - // use 100, let time to be seperated in different minutes, so there will be two packets - OneJob(100, gRootDir, "Job", true, time(NULL), "TestMultiUserSeperation", seqNo, false, prjIndex); - } - usleep(200 * 1000); - } - - WaitForFileBeenRead(); - // printf("[###] %d %d\n", gCounter, gAsynProjectSendFailCount); - sleep(20); - // printf("[###] %d %d\n", gCounter, gAsynProjectSendFailCount); - APSARA_TEST_EQUAL(gCounter, PROJECT_NUM / 2 * 1000); - - for (int prjIndex = 0; prjIndex < PROJECT_NUM; ++prjIndex) { - CheckSendClientStatus(prjIndex, prjIndex <= gProjectNetEnableIndex); - } - gRecvLogGroupLock.lock(); - const Log& log = gRcvLogGroup.logs(0); - APSARA_TEST_EQUAL(log.contents(2).value(), "TestMultiUserSeperation"); - APSARA_TEST_EQUAL(log.contents(3).value(), ToString(seqNo)); - APSARA_TEST_TRUE(gAsynProjectSendFailCount >= PROJECT_NUM / 2); - gRecvLogGroupLock.unlock(); - // APSARA_TEST_TRUE(gSynProjectSendFailCount > 0); - - // try to recover the connection - - // set lastNetworkErrorTime 0 to give sender a chance to send again. - for (int prjIndex = 0; prjIndex < PROJECT_NUM; ++prjIndex) { - LogstoreSenderInfo* pInfo = GetSenderInfo(prjIndex); - if (pInfo != NULL) { - pInfo->mLastNetworkErrorTime = 0; - pInfo->mLastQuotaExceedTime = 0; - } - } - - - gAsynProjectSendFailCount = 0; - gSynProjectSendFailCount = 0; - gProjectNetEnableIndex = PROJECT_NUM; - gCounter = 0; - for (int round = 0; round < 10; ++round) { - for (int prjIndex = 0; prjIndex < PROJECT_NUM; ++prjIndex) { - OneJob(100, gRootDir, "Job", true, time(NULL), "TestMultiUserSeperationSecond", seqNo, false, prjIndex); - } - usleep(100 * 1000); - } - // printf("[###]\n"); - WaitForFileBeenRead(); - sleep(20); - // printf("[###]\n"); - // 100 first send and 50 secondary - APSARA_TEST_EQUAL(gCounter, 1500 * PROJECT_NUM); - APSARA_TEST_TRUE(gAsynProjectSendFailCount == 0); - APSARA_TEST_TRUE(gSynProjectSendFailCount == 0); - for (int prjIndex = 0; prjIndex < PROJECT_NUM; ++prjIndex) { - CheckSendClientStatus(prjIndex, true); - } - - INT32_FLAG(max_client_quota_exceed_count) = defaultQuotaErrorCount; - gProjectNetEnableIndex = 0; - CaseCleanUp(); - - INT32_FLAG(client_quota_send_retry_interval) = bakClientQuotaSendRetryInterval; - Sender::Instance()->MockTestEndpoint = NULL; - LOG_INFO(sLogger, ("TestMultiUserSeperationAndRetryQuotaRecovery() end", time(NULL))); - } - - void TestMultiUserSeperationAndDiscardFailAndOtherFail() { - LOG_INFO(sLogger, ("TestMultiUserSeperationAndDiscardFailAndOtherFail() begin", time(NULL))); - EnableNetWork(); - CaseSetUp(false, true, true, PROJECT_NUM); - INT32_FLAG(client_disable_send_retry_interval) = 900; - int32_t defaultSendErrorCount = INT32_FLAG(max_client_send_error_count); - INT32_FLAG(max_client_send_error_count) = 5; - Sender::Instance()->MockTestEndpoint = MockSyncSend; - gProjectNetEnableIndex = PROJECT_NUM / 2 - 1; - // discard error - gSendFailType = 3; - int32_t seqNo = 5; - for (int round = 0; round < ROUND_NUM; ++round) { - for (int prjIndex = 0; prjIndex < PROJECT_NUM; ++prjIndex) { - // use 100, let time to be seperated in different minutes, so there will be two packets - OneJob(LOG_COUNT_PER_PROJECT, - gRootDir, - "Job", - true, - time(NULL), - "TestMultiUserSeperation", - seqNo, - false, - prjIndex); - } - usleep(200 * 1000); - } - - // printf("[###] %d %d\n", gCounter, gAsynProjectSendFailCount); - WaitForFileBeenRead(); - while (Sender::Instance()->mSenderQueue.IsEmpty() == false) { - LOG_INFO(sLogger, ("sender queue not empty", "try again")); - sleep(1); - } -#if defined(_MSC_VER) - // It's not very stable on Windows, so we have to wait for a long time. - // TODO: Figure it out why??? - sleep(40); -#endif - // printf("[###] %d %d\n", gCounter, gAsynProjectSendFailCount); - // half data will be discard - APSARA_TEST_EQUAL(gCounter, PROJECT_NUM / 2 * TOTAL_LOG_COUNT_PER_PROJECT); - // discard data error will not change send client status - for (int prjIndex = 0; prjIndex < PROJECT_NUM; ++prjIndex) { - CheckSendClientStatus(prjIndex, true); - } - gRecvLogGroupLock.lock(); - const Log& log = gRcvLogGroup.logs(0); - APSARA_TEST_EQUAL(log.contents(2).value(), "TestMultiUserSeperation"); - APSARA_TEST_EQUAL(log.contents(3).value(), ToString(seqNo)); - APSARA_TEST_TRUE(gAsynProjectSendFailCount >= PROJECT_NUM / 2); - gRecvLogGroupLock.unlock(); - // APSARA_TEST_TRUE(gSynProjectSendFailCount > 0); - - // server busy error - gSendFailType = 5; - - - gAsynProjectSendFailCount = 0; - gSynProjectSendFailCount = 0; - gCounter = 0; - for (int round = 0; round < ROUND_NUM; ++round) { - for (int prjIndex = 0; prjIndex < PROJECT_NUM; ++prjIndex) { - OneJob(LOG_COUNT_PER_PROJECT, - gRootDir, - "Job", - true, - time(NULL), - "TestMultiUserSeperationSecond", - seqNo, - false, - prjIndex); - } - usleep(100 * 1000); - } - // printf("[###]\n"); - WaitForFileBeenRead(); - sleep(10); - // printf("[###]\n"); - - // server error will send fail, so send success count is 50 - APSARA_TEST_EQUAL(gCounter, TOTAL_LOG_COUNT_PER_PROJECT * PROJECT_NUM / 2); - // server error will not change send client status - for (int prjIndex = 0; prjIndex < PROJECT_NUM; ++prjIndex) { - CheckSendClientStatus(prjIndex, true); - } - - gProjectNetEnableIndex = PROJECT_NUM; - sleep(10); - - - INT32_FLAG(max_client_send_error_count) = defaultSendErrorCount; - gProjectNetEnableIndex = 0; - CaseCleanUp(); - - Sender::Instance()->MockTestEndpoint = NULL; - LOG_INFO(sLogger, ("TestMultiUserSeperationAndDiscardFailAndOtherFail() end", time(NULL))); - } - - void TestSecondaryStorage() { - LOG_INFO(sLogger, ("TestSecondaryStorage() begin", time(NULL))); - CaseSetUp(); - auto sender = Sender::Instance(); - gProjectNetEnableIndex = 1000000; - int32_t defaultCheckPeriod = sender->mCheckPeriod; - sender->mCheckPeriod = 1; - INT32_FLAG(buffer_file_alive_interval) = 4 * INT32_FLAG(batch_send_interval) + 5; - DisableNetWork(); - int32_t defaultMergeLimit = INT32_FLAG(merge_log_count_limit); - INT32_FLAG(merge_log_count_limit) = 1024 * 1024 * 10; - int32_t defaultHoldSize = INT32_FLAG(max_holded_data_size); - INT32_FLAG(max_holded_data_size) = 20 * 1024 * 1024; - set firstLogSet; - set secondLogSet; - set thirdLogSet; - set forthLogSet; - size_t logCount = 30; - while (firstLogSet.size() < logCount) { - string first = GenerateRandomStr(1, 1024 * 2) + "_$#1"; - if (firstLogSet.find(first) == firstLogSet.end()) - firstLogSet.insert(first); - } - while (secondLogSet.size() < logCount) { - string second = GenerateRandomStr(1, 1024 * 2) + "_$#2"; - if (secondLogSet.find(second) == secondLogSet.end()) - secondLogSet.insert(second); - } - while (thirdLogSet.size() < logCount) { - string third = GenerateRandomStr(1, 1024 * 2) + "_$#3"; - if (thirdLogSet.find(third) == thirdLogSet.end()) - thirdLogSet.insert(third); - } - while (forthLogSet.size() < logCount) { - string forth = GenerateRandomStr(1, 1024 * 2) + "_$#4"; - if (forthLogSet.find(forth) == forthLogSet.end()) - forthLogSet.insert(forth); - } - - FileEncryption::GetInstance()->mKeyMap.clear(); - FileEncryption::KeyInfo* keyV1 = new FileEncryption::KeyInfo("73056101345ceb496c5574b8c6b7fc0e", 1); - FileEncryption::GetInstance()->mKeyMap.insert(pair(keyV1->mVersion, keyV1)); - FileEncryption::GetInstance()->SetDefaultKey(); - APSARA_TEST_EQUAL(FileEncryption::GetInstance()->GetDefaultKeyVersion(), 1); - - // SendFailType equals to 3 means that sending error will not write to secondary. - // Generate logs and wait enough time so that they can arrive sender queue. - gSendFailType = 3; - LOG_INFO(sLogger, - ("gNetWorkStat", gNetWorkStat)("gSendFailType", gSendFailType)("write forthLogSet", logCount)); - for (set::iterator iter = forthLogSet.begin(); iter != forthLogSet.end(); ++iter) { - OneJob(1, gRootDir, "Job", true, time(NULL), *iter); - } - printf("Job done\n"); - sleep(10); - APSARA_TEST_TRUE(sender->FlushOut(5 * 1000)); // this operation will write buffer file - sleep(INT32_FLAG(buffer_file_alive_interval) - 4); - vector filesToSend; - // send error will be discard, so second buffer is null - sender->LoadFileToSend(time(NULL), filesToSend); - APSARA_TEST_EQUAL(filesToSend.size(), 0); - - // SendFailType equals to 2 means that sending error will write to secondary. - gSendFailType = 2; // test all retryable send error - LOG_INFO(sLogger, - ("gNetWorkStat", gNetWorkStat)("gSendFailType", gSendFailType)("write firstLogSet", logCount)); - for (set::iterator iter = firstLogSet.begin(); iter != firstLogSet.end(); ++iter) { - OneJob(1, gRootDir, "Job", true, time(NULL), *iter); - } - printf("Job done\n"); - sleep(5); - APSARA_TEST_TRUE(sender->FlushOut(5 * 1000)); // this operation will write buffer file - sleep(INT32_FLAG(buffer_file_alive_interval) - 4); - // Data sent failed will be written to secondary, so there is a local file. - filesToSend.clear(); - sender->LoadFileToSend(time(NULL), filesToSend); - APSARA_TEST_EQUAL(filesToSend.size(), 1); - LOG_INFO(sLogger, ("bufferFileList", MergeVectorString(filesToSend))); - FileEncryption::KeyInfo* keyV2 = new FileEncryption::KeyInfo("c134368ce7840a4e217ee6d8c27a7e0f", 2); - FileEncryption::GetInstance()->mKeyMap.insert(pair(keyV2->mVersion, keyV2)); - FileEncryption::GetInstance()->SetDefaultKey(); - APSARA_TEST_EQUAL(FileEncryption::GetInstance()->GetDefaultKeyVersion(), 2); - - // Refresh buffer file manually. - string bufferFileName = sender->GetBufferFileName(); - if (sender->GetBufferFileName() == bufferFileName) - sender->CreateNewFile(); - bufferFileName = sender->GetBufferFileName(); - - LOG_INFO(sLogger, - ("gNetWorkStat", gNetWorkStat)("gSendFailType", gSendFailType)("write secondLogSet", logCount)); - gSendFailType = 1; // test network error - for (set::iterator iter = secondLogSet.begin(); iter != secondLogSet.end(); ++iter) { - OneJob(1, gRootDir, "Job", true, time(NULL), *iter); - } - printf("Job done\n"); - sleep(5); - APSARA_TEST_TRUE(Sender::Instance()->FlushOut(5 * 1000)); // this operation will write buffer file - sleep(INT32_FLAG(buffer_file_alive_interval) - 4); - filesToSend.clear(); - Sender::Instance()->LoadFileToSend(time(NULL), filesToSend); - APSARA_TEST_EQUAL(filesToSend.size(), 2); - LOG_INFO(sLogger, ("bufferFileList", MergeVectorString(filesToSend))); - - FileEncryption::KeyInfo* keyV3 = new FileEncryption::KeyInfo("e1b07c24b3340c94945c058db193d8f4", 3); - FileEncryption::GetInstance()->mKeyMap.insert(pair(keyV3->mVersion, keyV3)); - FileEncryption::GetInstance()->SetDefaultKey(); - APSARA_TEST_EQUAL(FileEncryption::GetInstance()->GetDefaultKeyVersion(), 3); - if (Sender::Instance()->GetBufferFileName() == bufferFileName) - Sender::Instance()->CreateNewFile(); - bufferFileName = Sender::Instance()->GetBufferFileName(); - LOG_INFO(sLogger, - ("gNetWorkStat", gNetWorkStat)("gSendFailType", gSendFailType)("write thirdLogSet", logCount)); - for (set::iterator iter = thirdLogSet.begin(); iter != thirdLogSet.end(); ++iter) { - OneJob(1, gRootDir, "Job", true, time(NULL), *iter); - } - printf("Job done\n"); - - // move delete key behind sleep - // delete key 2, so second log will be discard - FileEncryption::GetInstance()->mKeyMap.erase(2); - FileEncryption::GetInstance()->SetDefaultKey(); - LOG_INFO(sLogger, ("delete key version 2, networkstat", gNetWorkStat)); - - sleep(5); - APSARA_TEST_TRUE(Sender::Instance()->FlushOut(5 * 1000)); // this operation will write buffer file - sleep(INT32_FLAG(buffer_file_alive_interval) - 4); // debug - filesToSend.clear(); - Sender::Instance()->LoadFileToSend(time(NULL), filesToSend); - // buffer file with version2 will be deleted - APSARA_TEST_EQUAL(filesToSend.size(), 2); - LOG_INFO(sLogger, ("bufferFileList", MergeVectorString(filesToSend))); - - EnableNetWork(); - sleep(INT32_FLAG(buffer_file_alive_interval) + 1); - LOG_INFO(sLogger, ("begin check, gNetWorkStat", gNetWorkStat)); - - APSARA_TEST_EQUAL(gCounter, int32_t(2 * logCount)); - filesToSend.clear(); - Sender::Instance()->LoadFileToSend(time(NULL), filesToSend); - APSARA_TEST_EQUAL(filesToSend.size(), 0); - - size_t firstLogCount = 0; - size_t secondLogCount = 0; - size_t thirdLogCount = 0; - size_t forthLogCount = 0; - APSARA_TEST_EQUAL(gBufferLogGroups.size(), 2); - for (size_t i = 0; i < (size_t)gBufferLogGroups.size(); ++i) { - LogGroup& logGroup = gBufferLogGroups[i]; - if (logGroup.category() != "app_log") - continue; - for (size_t j = 0; j < (size_t)logGroup.logs_size(); ++j) { - const Log log = logGroup.logs(j); - string ip = ""; - string nothing = ""; - for (size_t k = 0; k < (size_t)log.contents_size(); ++k) { - const Log_Content content = log.contents(k); - if (content.key() == "ip") - ip = content.value(); - else if (content.key() == "nothing") - nothing = content.value(); - } - if (ip != "10.7.241.21") - continue; - LOG_INFO(sLogger, ("nothing", nothing)("substr", nothing.substr(nothing.size() - 4))); - if (nothing.substr(nothing.size() - 4) == "_$#1") { - if (firstLogSet.find(nothing) != firstLogSet.end()) { - firstLogCount++; - firstLogSet.erase(nothing); - } - } else if (nothing.substr(nothing.size() - 4) == "_$#3") { - if (thirdLogSet.find(nothing) != thirdLogSet.end()) { - thirdLogCount++; - thirdLogSet.erase(nothing); - } - } else if (nothing.substr(nothing.size() - 4) == "_$#4") { - if (forthLogSet.find(nothing) != forthLogSet.end()) { - forthLogCount++; - forthLogSet.erase(nothing); - } - } else if (nothing.substr(nothing.size() - 4) == "_$#2") { - if (secondLogSet.find(nothing) != secondLogSet.end()) { - secondLogCount++; - secondLogSet.erase(nothing); - } - } - } - } - - APSARA_TEST_EQUAL(firstLogSet.size(), 0); - APSARA_TEST_EQUAL(firstLogCount, logCount); - APSARA_TEST_EQUAL(secondLogSet.size(), logCount); - APSARA_TEST_EQUAL(secondLogCount, 0); - APSARA_TEST_EQUAL(thirdLogSet.size(), 0); - APSARA_TEST_EQUAL(thirdLogCount, logCount); - APSARA_TEST_EQUAL(forthLogSet.size(), logCount); - APSARA_TEST_EQUAL(forthLogCount, 0); - - FileEncryption::GetInstance()->mKeyMap.clear(); - FileEncryption::GetInstance()->LoadKeyInfo(); - FileEncryption::GetInstance()->SetDefaultKey(); - delete keyV1; - delete keyV2; - delete keyV3; - INT32_FLAG(merge_log_count_limit) = defaultMergeLimit; - INT32_FLAG(max_holded_data_size) = defaultHoldSize; - Sender::Instance()->mCheckPeriod = defaultCheckPeriod; - gProjectNetEnableIndex = 0; - CaseCleanUp(); - LOG_INFO(sLogger, ("TestSecondaryStorage() end", time(NULL))); - } - - void TestEncryptAndDecrypt() { - LOG_INFO(sLogger, ("TestEncryptAndDecrypt() begin", time(NULL))); - for (size_t i = 0; i < 100; ++i) { - string data = GenerateRandomStr(1024 * 16, 1024 * 64); - char* des; - int32_t srcLength = (int32_t)data.size(); - int32_t encryptionLength; - FileEncryption::GetInstance()->Encrypt(data.c_str(), srcLength, des, encryptionLength); - string encryption = string(des, encryptionLength); - delete[] des; - char* src = new char[srcLength]; - FileEncryption::GetInstance()->Decrypt(encryption.c_str(), encryptionLength, src, srcLength, 1); - string decryption = string(src, srcLength); - delete[] src; - APSARA_TEST_TRUE(data == decryption); - APSARA_TEST_TRUE(encryption != decryption); - - int32_t blockBytes = FileEncryption::GetInstance()->mKeyMap[1]->mBlockBytes; - APSARA_TEST_TRUE_DESC((encryptionLength >= srcLength && encryptionLength < (srcLength + blockBytes)), - "encryption.size:" + ToString(encryptionLength) + ";decryption.size:" - + ToString(srcLength) + ";blockBytes:" + ToString(blockBytes)); - } - LOG_INFO(sLogger, ("TestEncryptAndDecrypt() end", time(NULL))); - } - - // Wait several seconds to make sure test log files have been read. - static void WaitForFileBeenRead() { -#if defined(_MSC_VER) - // Because of lackness of event based discovery, on Windows, we can only use - // polling, so wait for polling interval plus several process time (2s here). - logtail::sleep(INT32_FLAG(dirfile_check_interval_ms) / 1000 + 2); -#endif - } - - void TestDiscardOldData() { - LOG_INFO(sLogger, ("TestDiscardOldData() begin", time(NULL))); - CaseSetUp(); - EnableNetWork(); - BOOL_FLAG(ilogtail_discard_old_data) = true; - // case#1 - gCounter = 0; - OneJob(10, gRootDir, "Job", true, time(NULL)); - WaitForFileBeenRead(); - sleep(2 * INT32_FLAG(batch_send_interval) + 1); - APSARA_TEST_EQUAL(gCounter, 10); - - // case#2 - gCounter = 0; - OneJob(10, gRootDir, "Job", true, time(NULL) - INT32_FLAG(ilogtail_discard_interval) - 100); - WaitForFileBeenRead(); - sleep(2 * INT32_FLAG(batch_send_interval) + 1); - APSARA_TEST_EQUAL(gCounter, 0); - - // case#3 - gCounter = 0; - time_t timeBefore1970 = -100000; -#if defined(__linux__) - OneJob(10, gRootDir, "Job", true, timeBefore1970); - WaitForFileBeenRead(); - sleep(2 * INT32_FLAG(batch_send_interval) + 1); - APSARA_TEST_EQUAL(gCounter, 0); -#endif - - // case#4 - gCounter = 0; - OneJob(1, gRootDir, "Job", true, time(NULL)); -#if defined(__linux__) - OneJob(1, gRootDir, "Job", true, timeBefore1970); -#endif - OneJob(1, gRootDir, "Job", true, time(NULL)); - OneJob(1, gRootDir, "Job", true, time(NULL) - INT32_FLAG(ilogtail_discard_interval) - 100); - OneJob(1, gRootDir, "Job", true, time(NULL)); -#if defined(__linux__) - OneJob(1, gRootDir, "Job", true, timeBefore1970); -#endif - OneJob(1, gRootDir, "Job", true, time(NULL)); - OneJob(1, gRootDir, "Job", true, time(NULL) - INT32_FLAG(ilogtail_discard_interval) - 100); - WaitForFileBeenRead(); - sleep(2 * INT32_FLAG(batch_send_interval) + 1); - APSARA_TEST_EQUAL(gCounter, 4); - - BOOL_FLAG(ilogtail_discard_old_data) = false; - // case#5 - gCounter = 0; - OneJob(10, gRootDir, "Job", true, time(NULL) - INT32_FLAG(ilogtail_discard_interval) - 100); - WaitForFileBeenRead(); - sleep(2 * INT32_FLAG(batch_send_interval) + 1); - APSARA_TEST_EQUAL(gCounter, 10); - - // case#6 - gCounter = 0; -#if defined(__linux__) - OneJob(10, gRootDir, "Job", true, timeBefore1970); - WaitForFileBeenRead(); - sleep(2 * INT32_FLAG(batch_send_interval) + 1); - APSARA_TEST_EQUAL(gCounter, 0); -#endif - - // case#7 - gCounter = 0; - OneJob(1, gRootDir, "Job", true, time(NULL)); -#if defined(__linux__) - OneJob(1, gRootDir, "Job", true, timeBefore1970); -#endif - OneJob(1, gRootDir, "Job", true, time(NULL)); - OneJob(1, gRootDir, "Job", true, time(NULL) - INT32_FLAG(ilogtail_discard_interval) - 100); - OneJob(1, gRootDir, "Job", true, time(NULL)); -#if defined(__linux__) - OneJob(1, gRootDir, "Job", true, timeBefore1970); -#endif - OneJob(1, gRootDir, "Job", true, time(NULL)); - OneJob(1, gRootDir, "Job", true, time(NULL) - INT32_FLAG(ilogtail_discard_interval) - 100); - WaitForFileBeenRead(); - sleep(2 * INT32_FLAG(batch_send_interval) + 1); - APSARA_TEST_EQUAL(gCounter, 6); - - CaseCleanUp(); - LOG_INFO(sLogger, ("TestDiscardOldData() end", time(NULL))); - } - - void TestConnect() { - LOG_INFO(sLogger, ("TestConnect() begin", time(NULL))); - CaseSetUp(); - EnableNetWork(); - OneJob(1, gRootDir, "Job", true, time(NULL)); - WaitForFileBeenRead(); - sleep(2 * INT32_FLAG(batch_send_interval) + 1); - APSARA_TEST_EQUAL(gCounter, 1); - CaseCleanUp(); - LOG_INFO(sLogger, ("TestConnect() end", time(NULL))); - } - - void TestDisConnect() { - LOG_INFO(sLogger, ("TestDisConnect() begin", time(NULL))); - CaseSetUp(); - LogGroup logGroup; - logGroup.set_category("app_log"); - Log* logPtr = logGroup.add_logs(); - Log_Content* contentPtr = logPtr->add_contents(); - contentPtr->set_key("ip"); - contentPtr->set_value("10.7.241.21"); - contentPtr = logPtr->add_contents(); - contentPtr->set_key("time"); - contentPtr->set_value(ToString(time(NULL))); - contentPtr = logPtr->add_contents(); - contentPtr->set_key("nothing"); - contentPtr->set_value(ToString( - "abcdefghijklmnopqrsputskjueiguwdhruwldirudsjhdklguejsldiuuwjskldgsksjdkdjfksjsdkfjsksdjfksjdkfuujss")); - contentPtr = logPtr->add_contents(); - contentPtr->set_key("seq"); - contentPtr->set_value(ToString("0")); - - DisableNetWork(); - OneJob(1, gRootDir, "Job", true, time(NULL)); - WaitForFileBeenRead(); - sleep(INT32_FLAG(batch_send_interval) * 2 + 2); - vector filesToSend; - Sender::Instance()->LoadFileToSend(time(NULL), filesToSend); - APSARA_TEST_EQUAL(gCounter, 0); - // APSARA_TEST_EQUAL(filesToSend.size(), 1); - EnableNetWork(); - sleep(INT32_FLAG(batch_send_interval) * 3 + 1); - gRecvLogGroupLock.lock(); - APSARA_TEST_EQUAL(gRcvLogGroup.category(), logGroup.category()); - APSARA_TEST_EQUAL(gRcvLogGroup.logs_size(), logGroup.logs_size()); - for (int i = 0; i < gRcvLogGroup.logs_size() && i < logGroup.logs_size(); i++) { - const Log& log1 = gRcvLogGroup.logs(i); - const Log& log2 = logGroup.logs(i); - APSARA_TEST_EQUAL_FATAL(log1.contents_size(), log2.contents_size()); - for (int j = 0; j < log1.contents_size(); j++) { - const Log_Content& content1 = log1.contents(j); - const Log_Content& content2 = log2.contents(j); - APSARA_TEST_EQUAL(content1.key(), content2.key()); - APSARA_TEST_TRUE_DESC(content1.key() == "time" || content1.value() == content2.value(), - content1.key() + content1.value() + content2.key() + content2.value()); - } - } - gRecvLogGroupLock.unlock(); - APSARA_TEST_EQUAL(gCounter, 1); - filesToSend.clear(); - Sender::Instance()->LoadFileToSend(time(NULL), filesToSend); - APSARA_TEST_EQUAL(filesToSend.size(), 0); - CaseCleanUp(); - LOG_INFO(sLogger, ("TestDisConnect() end", time(NULL))); - } - - void TestChangeStat() { - LOG_INFO(sLogger, ("TestChangeStat() begin", time(NULL))); - CaseSetUp(); - DisableNetWork(); - int32_t num = 10; - for (int32_t i = 0; i < num; ++i) { - OneJob(1, gRootDir, "job", true, time(NULL)); - WaitForFileBeenRead(); - sleep(INT32_FLAG(batch_send_interval) * 2 + 1); - if (i % 2 == 0) - EnableNetWork(); - else - DisableNetWork(); - } - EnableNetWork(); - sleep(INT32_FLAG(batch_send_interval) * 4 + 1); - APSARA_TEST_EQUAL(gCounter, num); - vector filesToSend; - Sender::Instance()->LoadFileToSend(time(NULL), filesToSend); - APSARA_TEST_EQUAL(filesToSend.size(), 0); - CaseCleanUp(); - LOG_INFO(sLogger, ("TestChangeStat() end", time(NULL))); - } - void TestFlowControl() { - LOG_INFO(sLogger, ("TestFlowControl() begin", time(NULL))); - CaseSetUp(); - DisableNetWork(); - AppConfig::GetInstance()->mBytePerSec = 1 * 1024 * 1024; - int32_t MAX_BUFFER_FILE_SIZE = 50 * 1024 * 1024; - while (true) { - // if not use fixed time, then there may be many 10 seconds logs - // if the 10 seconds have different minutes, then - // ((value->mLogGroup).logs(0).time() / 60 != (*(mutableLogPtr + logIdx))->time() / 60) - // is true, so there will be many little packages and send time will take very long time - // may be one hour.................................... - OneJob(10, gRootDir, "job", true, time(NULL), "", 0, true); - ifstream fin((gRootDir + PATH_SEPARATOR + "job.log").c_str()); - fin.seekg(0, ios::end); - int32_t pos = fin.tellg(); - if (pos > MAX_BUFFER_FILE_SIZE) { - fin.close(); - break; - } - fin.close(); - } - WaitForFileBeenRead(); - sleep(3 * INT32_FLAG(batch_send_interval) + 1); - // vector filesToSend; - // Sender::Instance() -> LoadFileToSend(time(NULL),filesToSend); - // LOG_INFO(sLogger,("buffer file num",filesToSend.size())); - // APSARA_TEST_TRUE_FATAL(filesToSend.size() > 1); - int32_t defaultMaxBps = AppConfig::GetInstance()->mMaxBytePerSec; - AppConfig::GetInstance()->mMaxBytePerSec = 2 * 1024 * 1024; - EnableNetWork(); - Sender::Instance()->TestEndpoint(STRING_FLAG(default_region_name), ""); - //{ - // PTScopedLock lock(Sender::Instance()->mBufferWait); - // Sender::Instance()->mBufferWait.signal(); - //} - uint64_t start = GetCurrentTimeInMicroSeconds(); - sleep(20); - while (true) { - sleep(1); - Sender* pSender = Sender::Instance(); - Aggregator* pAgg = Aggregator::GetInstance(); - if (ProcessorRunner::GetInstance()->mLogFeedbackQueue.IsEmpty() && pAgg->IsMergeMapEmpty() - && pSender->IsBatchMapEmpty() && pSender->GetSendingCount() == 0 && pSender->IsSecondaryBufferEmpty()) { - break; - } - // pSender->mSenderQueue.PrintStatus(); - // filesToSend.clear(); - // Sender::Instance() -> LoadFileToSend(time(NULL),filesToSend); - // if(filesToSend.size() == 0) - // break; - } - double interval = (GetCurrentTimeInMicroSeconds() - start) / 1000000.0; - double speed = gMessageSize * 1.0 / interval; - double diffRatio = speed / AppConfig::GetInstance()->GetMaxBytePerSec(); - APSARA_TEST_TRUE_DESC(diffRatio >= 0.9 && diffRatio <= 1.3, diffRatio); - LOG_INFO(sLogger, ("actual speed", speed)("diffRatio", diffRatio)); - AppConfig::GetInstance()->mMaxBytePerSec = defaultMaxBps; - - if (interval < 40.) { - sleep((uint32_t)(40. - interval)); - } - CaseCleanUp(); - LOG_INFO(sLogger, ("TestFlowControl() end", time(NULL))); - } - - void TestLogstoreFlowControl() { - LOG_INFO(sLogger, ("TestLogstoreFlowControl() begin", time(NULL))); - CaseSetUp(false, false, false, 1, false, 2 * 1024 * 1024, 0); - DisableNetWork(); - int32_t MAX_BUFFER_FILE_SIZE = 50 * 1024 * 1024; - while (true) { - // if not use fixed time, then there may be many 10 seconds logs - // if the 10 seconds have different minutes, then - // ((value->mLogGroup).logs(0).time() / 60 != (*(mutableLogPtr + logIdx))->time() / 60) - // is true, so there will be many little packages and send time will take very long time - // may be one hour.................................... - OneJob(10, gRootDir, "job", true, time(NULL), "", 0, true); - ifstream fin((gRootDir + PATH_SEPARATOR + "job.log").c_str()); - fin.seekg(0, ios::end); - int32_t pos = fin.tellg(); - if (pos > MAX_BUFFER_FILE_SIZE) { - fin.close(); - break; - } - fin.close(); - } - WaitForFileBeenRead(); - sleep(3 * INT32_FLAG(batch_send_interval) + 1); - // vector filesToSend; - // Sender::Instance() -> LoadFileToSend(time(NULL),filesToSend); - // LOG_INFO(sLogger,("buffer file num",filesToSend.size())); - // APSARA_TEST_TRUE_FATAL(filesToSend.size() > 1); - EnableNetWork(); - Sender::Instance()->TestEndpoint(STRING_FLAG(default_region_name), ""); - //{ - // PTScopedLock lock(Sender::Instance()->mBufferWait); - // Sender::Instance()->mBufferWait.signal(); - //} - uint64_t start = GetCurrentTimeInMicroSeconds(); - sleep(20); - while (true) { - sleep(1); - Sender* pSender = Sender::Instance(); - Aggregator* pAgg = Aggregator::GetInstance(); - if (ProcessorRunner::GetInstance()->mLogFeedbackQueue.IsEmpty() && pAgg->IsMergeMapEmpty() - && pSender->IsBatchMapEmpty() && pSender->GetSendingCount() == 0 && pSender->IsSecondaryBufferEmpty()) { - break; - } - // pSender->mSenderQueue.PrintStatus(); - // filesToSend.clear(); - // Sender::Instance() -> LoadFileToSend(time(NULL),filesToSend); - // if(filesToSend.size() == 0) - // break; - } - double interval = (GetCurrentTimeInMicroSeconds() - start) / 1000000.0; - double speed = gMessageSize * 1.0 / interval; - double diffRatio = speed / (2 * 1024 * 1024); - APSARA_TEST_TRUE_DESC(diffRatio >= 0.8 && diffRatio <= 1.3, diffRatio); - LOG_INFO(sLogger, ("actual speed", speed)("diffRatio", diffRatio)); - - if (interval < 40.) { - sleep((uint32_t)(40. - interval)); - } - CaseCleanUp(); - LOG_INFO(sLogger, ("TestLogstoreFlowControl() end", time(NULL))); - } - - void TestLogstoreFlowControlPause() { - LOG_INFO(sLogger, ("TestLogstoreFlowControlPause() begin", time(NULL))); - CaseSetUp(false, false, false, 1, false, 0, 0); - EnableNetWork(); - for (int i = 0; i < 100; ++i) { - OneJob(10, gRootDir, "job", true, time(NULL), "", 0, true); - } - WaitForFileBeenRead(); - sleep(3 * INT32_FLAG(batch_send_interval) + 1); - APSARA_TEST_EQUAL(gCounter, 0); - string project = "1000000_proj"; - string logstore = "app_log"; - LogstoreFeedBackKey key = GenerateLogstoreFeedBackKey(project, logstore); - Sender::Instance()->SetLogstoreFlowControl(key, -1, 0); - - sleep(3 * INT32_FLAG(batch_send_interval) + 1); - APSARA_TEST_EQUAL(gCounter, 1000); - CaseCleanUp(); - LOG_INFO(sLogger, ("TestLogstoreFlowControlPause() end", time(NULL))); - } - - - void TestLogstoreFlowControlExpire() { - LOG_INFO(sLogger, ("TestLogstoreFlowControlExpire() begin", time(NULL))); - CaseSetUp(false, false, false, 1, false, 0, time(NULL) + 20); - EnableNetWork(); - for (int i = 0; i < 100; ++i) { - OneJob(10, gRootDir, "job", true, time(NULL), "", 0, true); - } - sleep(10); - APSARA_TEST_EQUAL(gCounter, 0); - sleep(20); - APSARA_TEST_EQUAL(gCounter, 1000); - CaseCleanUp(); - LOG_INFO(sLogger, ("TestLogstoreFlowControlExpire() end", time(NULL))); - } - - char GetnoneUTF8Char() { - char noneUTF8Chars[] = {char(0xff), char(0xc0), char(0xe0), char(0xf0)}; - static int index = 0; - index++; - index = index % 4; - return noneUTF8Chars[index]; - } - void GenerateNoneUTF8Char(LogGroup& logGroup) { - auto now = GetCurrentLogtailTime(); - for (int i = 0; i < 10; ++i) { - Log* logPtr = logGroup.add_logs(); - SetLogTime(logPtr, now.tv_sec); - for (int j = 0; j < 10; ++j) { - Log_Content* contentPtr = logPtr->add_contents(); - if (j == i) { - contentPtr->set_key("key" + ToString(j)); - contentPtr->set_value(string(i, 'v') + string(i + 1, GetnoneUTF8Char()) + string(i, 'v')); - } else if (j == (i + 1)) { - contentPtr->set_key(string(i, 'v') + string(i + 1, GetnoneUTF8Char()) + string(i, 'v')); - contentPtr->set_value("value" + ToString(j)); - } else if (j == (i + 2)) { - contentPtr->set_key("key" + ToString(j)); - contentPtr->set_value(string(i, 'v') + string(i + 1, GetnoneUTF8Char())); - } else if (j == (i + 3)) { - contentPtr->set_key("key" + ToString(j)); - contentPtr->set_value(string(i + 1, GetnoneUTF8Char()) + string(i, 'v')); - } else { - contentPtr->set_key("key" + ToString(j)); - contentPtr->set_value("value" + ToString(j)); - } - } - } - } - - void TestMergeByCount() { - LOG_INFO(sLogger, ("TestMergeByCount() begin", time(NULL))); - int32_t defaultMergeLimit = INT32_FLAG(merge_log_count_limit); - INT32_FLAG(merge_log_count_limit) = rand() % 100 + 1; - LOG_INFO(sLogger, ("merge_log_count_limit", INT32_FLAG(merge_log_count_limit))); - CaseSetUp(); - EnableNetWork(); - - for (size_t caseId = 0; caseId < 3; ++caseId) { - gBufferLogGroups.clear(); - - size_t totalSendLogNum = 0; - for (size_t j = 0; j < 10; j++) { - size_t randSendLogNum = rand() % INT32_FLAG(merge_log_count_limit); - totalSendLogNum += randSendLogNum; - OneJob(randSendLogNum, gRootDir, "Job", true, time(NULL), ""); - LOG_INFO(sLogger, ("send loop", j)("totalSendLogNum", totalSendLogNum)("current size", randSendLogNum)); - } - WaitForFileBeenRead(); - sleep(INT32_FLAG(batch_send_interval) + 3); - - size_t totalRecieveLogNum = 0; - for (size_t j = 0; j < (size_t)gBufferLogGroups.size(); ++j) { - LogGroup& logGroup = gBufferLogGroups[j]; - APSARA_TEST_TRUE_DESC(logGroup.logs_size() <= INT32_FLAG(merge_log_count_limit), logGroup.logs_size()); - totalRecieveLogNum += logGroup.logs_size(); - LOG_INFO( - sLogger, - ("recv loop", j)("totalRecieveLogNum", totalRecieveLogNum)("current size", logGroup.logs_size())( - "merge_log_count_limit", INT32_FLAG(merge_log_count_limit))); - } - APSARA_TEST_EQUAL(totalSendLogNum, totalRecieveLogNum); - } - - CaseCleanUp(); - INT32_FLAG(merge_log_count_limit) = defaultMergeLimit; - LOG_INFO(sLogger, ("TestMergeByCount() end", time(NULL))); - } - - void TestFilterRule() { - LOG_INFO(sLogger, ("TestFilterRule() begin", time(NULL))); - CaseSetUp(true); - EnableNetWork(); - OneJob(1, gRootDir, "Job", true, time(NULL), "filter", 23234); // not match - OneJob(1, gRootDir, "Job", true, time(NULL), "testfilter", 5); // not match - OneJob(1, gRootDir, "Job", true, time(NULL), "filtersdfas", 5); // match - OneJob(1, gRootDir, "Job", true, time(NULL), "sdfsds", 5); // not match - WaitForFileBeenRead(); - sleep(INT32_FLAG(batch_send_interval) + 3); - - APSARA_TEST_EQUAL(gCounter, 1); - - for (size_t i = 0; i < gBufferLogGroups.size(); i++) { - LogGroup& loggroup = gBufferLogGroups[i]; - for (int32_t j = 0; j < loggroup.logs_size(); j++) { - const Log& log = loggroup.logs(j); - for (int k = 0; k < log.contents_size(); k++) { - const Log_Content& content = log.contents(k); - if (content.key() == "nothing") { - APSARA_TEST_EQUAL(content.value(), "filtersdfas"); - } - if (content.key() == "seq") { - APSARA_TEST_EQUAL(content.value(), "5"); - } - } - } - } - - CaseCleanUp(); - LOG_INFO(sLogger, ("TestFilterRule() end", time(NULL))); - } - - void TestMonitor() { - LOG_INFO(sLogger, ("TestMonitor() begin", time(NULL))); - int32_t defaultMonitorInterval = INT32_FLAG(monitor_interval); - INT32_FLAG(monitor_interval) = 3; - CaseSetUp(); - LogtailMonitor::GetInstance()->Init(); - gStatusLogGroup.Clear(); - gStatusCount = 0; - EnableNetWork(); - - sleep(INT32_FLAG(monitor_interval) * 5 + 1); - - APSARA_TEST_TRUE_DESC(gStatusCount >= 2, gStatusCount); - APSARA_TEST_EQUAL(gStatusLogGroup.category(), "logtail_status_profile"); - const Log& log = gStatusLogGroup.logs(0); - int idx = 0; - APSARA_TEST_EQUAL(log.contents(idx++).key(), "cpu"); -#if defined(__linux__) - APSARA_TEST_EQUAL(log.contents(idx++).key(), "os_cpu"); -#endif - APSARA_TEST_EQUAL(log.contents(idx++).key(), "mem"); - APSARA_TEST_EQUAL(log.contents(idx++).key(), "version"); - APSARA_TEST_EQUAL(log.contents(idx++).key(), "uuid"); - APSARA_TEST_EQUAL(log.contents(idx++).key(), "user_defined_id"); - APSARA_TEST_EQUAL(log.contents(idx++).key(), "aliuids"); - APSARA_TEST_EQUAL(log.contents(idx++).key(), "projects"); - APSARA_TEST_EQUAL(log.contents(idx++).key(), "instance_id"); - APSARA_TEST_EQUAL(log.contents(idx++).key(), "syslog_open"); - APSARA_TEST_EQUAL(log.contents(idx++).key(), "ip"); - APSARA_TEST_EQUAL(log.contents(idx++).key(), "hostname"); - APSARA_TEST_EQUAL(log.contents(idx++).key(), "os"); - APSARA_TEST_EQUAL(log.contents(idx++).key(), "os_detail"); - APSARA_TEST_EQUAL(log.contents(idx++).key(), "user"); -#if defined(__linux__) - APSARA_TEST_EQUAL(log.contents(idx++).key(), "load"); -#endif - APSARA_TEST_EQUAL(log.contents(idx++).key(), "metric_json"); - APSARA_TEST_EQUAL(log.contents(idx++).key(), "status"); - LogtailMonitor::GetInstance()->RemoveMonitor(); - CaseCleanUp(); - INT32_FLAG(monitor_interval) = defaultMonitorInterval; - LOG_INFO(sLogger, ("TestMonitor() end", time(NULL))); - } - - void TestFlushOut() { - LOG_INFO(sLogger, ("TestFlushOut() begin", time(NULL))); - AppConfig::GetInstance()->mMaxBufferNum = 500; - EnableNetWork(); - CaseSetUp(); - - int32_t seqNo = 5; - for (int round = 0; round < 10; ++round) { - OneJob(100, gRootDir, "Job", true, time(NULL), "testflushout", seqNo); - usleep(100 * 1000); - } - WaitForFileBeenRead(); - LOG_INFO(sLogger, ("Flush out", "begin")); - APSARA_TEST_TRUE(Sender::Instance()->FlushOut(5 * 1000)); // this operation will write buffer file - LOG_INFO(sLogger, ("Flush out", "end")); - sleep(2); - { - WaitObject::Lock lock(Sender::Instance()->mBufferWait); - Sender::Instance()->mBufferWait.signal(); - } - sleep(10); - APSARA_TEST_EQUAL(gCounter, 1000); - gRecvLogGroupLock.lock(); - const Log& log = gRcvLogGroup.logs(0); - APSARA_TEST_EQUAL(log.contents(2).value(), "testflushout"); - APSARA_TEST_EQUAL(log.contents(3).value(), ToString(seqNo)); - gRecvLogGroupLock.unlock(); - - CaseCleanUp(); - AppConfig::GetInstance()->mMaxBufferNum = INT32_FLAG(max_buffer_num); - LOG_INFO(sLogger, ("TestFlushOut() end", time(NULL))); - } - - void TestMergeByMinute() { - LOG_INFO(sLogger, ("TestMergeByMinute() begin", time(NULL))); - CaseSetUp(); - EnableNetWork(); - for (size_t caseId = 0; caseId < 3; ++caseId) { - { - PTScopedLock lock(gBufferLogGroupsLock); - gBufferLogGroups.clear(); - } - int32_t curTime = (time(NULL) / 60) * 60; - int32_t logTimes[10] = { - curTime - 300, // new loggroup - curTime - 150, // new loggroup - curTime - 121, - curTime - 120, // new loggroup - curTime - 60, // new loggroup - curTime - 59, - curTime - 3, - curTime, // new loggroup - curTime + 59, - curTime + 60 // new loggroup - }; - - int32_t logNums[10] = {0, 0, 0, 0, 0, 0, 0, 0, 0, 0}; - size_t totalSendLogNum = 0; - LOG_INFO(sLogger, ("caseId", caseId)); - for (size_t j = 0; j < 10; j++) { - logNums[j] = (rand() % (INT32_FLAG(merge_log_count_limit) / 50)) + 1; - OneJob(logNums[j], gRootDir, "Job", true, logTimes[j], "", 0, true); - LOG_INFO(sLogger, ("logTime", logTimes[j])("logNum", logNums[j])); - totalSendLogNum += logNums[j]; - } - int32_t mergeLogNums[6] = {logNums[0], - logNums[1] + logNums[2], - logNums[3], - logNums[4] + logNums[5] + logNums[6], - logNums[7] + logNums[8], - logNums[9]}; - WaitForFileBeenRead(); - sleep(INT32_FLAG(batch_send_interval) * 2 + 1); - size_t totalRecieveLogNum = 0; - map recvLogCountMap; // - gBufferLogGroupsLock.lock(); - APSARA_TEST_EQUAL_DESC(gBufferLogGroups.size(), 6, string("caseId:") + ToString(caseId)); - if (gBufferLogGroups.size() == 6) { - for (size_t idx = 0; idx < gBufferLogGroups.size(); ++idx) { - const LogGroup& logGroup = gBufferLogGroups[idx]; - APSARA_TEST_EQUAL_DESC(logGroup.logs_size(), - mergeLogNums[idx], - string("caseId:") + ToString(caseId) + ", idx:" + ToString(idx)); - totalRecieveLogNum += logGroup.logs_size(); - for (int32_t logIdx = 0; logIdx < logGroup.logs_size(); ++logIdx) { - int32_t logtime = logGroup.logs(logIdx).time(); - if (recvLogCountMap.find(logtime) == recvLogCountMap.end()) - recvLogCountMap[logtime] = 1; - else - recvLogCountMap[logtime] += 1; - } - } - } - gBufferLogGroupsLock.unlock(); - for (map::iterator iter = recvLogCountMap.begin(); iter != recvLogCountMap.end(); ++iter) - LOG_INFO(sLogger, ("recvLogCountMap, time", iter->first)("count", iter->second)); - - APSARA_TEST_EQUAL_DESC(totalSendLogNum, totalRecieveLogNum, string("caseId:") + ToString(caseId)); - for (int32_t idx = 0; idx < 10; ++idx) { - if (recvLogCountMap.find(logTimes[idx]) != recvLogCountMap.end()) - APSARA_TEST_EQUAL_DESC(recvLogCountMap[logTimes[idx]], - logNums[idx], - string("caseId:") + ToString(caseId) + ", idx:" + ToString(idx)); - else - APSARA_TEST_TRUE_DESC(recvLogCountMap.find(logTimes[idx]) != recvLogCountMap.end(), - string("caseId:") + ToString(caseId) + ", idx:" + ToString(idx)); - } - } - CaseCleanUp(); - LOG_INFO(sLogger, ("TestMergeByMinute() end", time(NULL))); - } - - void TestRealIpSend() { - LOG_INFO(sLogger, ("TestRealIpSend() begin", time(NULL))); - CaseSetUp(false, true, true); - EnableNetWork(); - - // enable real ip - BOOL_FLAG(send_prefer_real_ip) = true; - INT32_FLAG(send_switch_real_ip_interval) = 10; - - Sender::Instance()->MockGetRealIp = SenderUnittest::MockGetRealIp; - // start real ip update thread - gRealIpSendThread = CreateThread(std::bind(&Sender::RealIpUpdateThread, Sender::Instance())); - - string aliuid = "1234567890"; - string region = AppConfig::GetInstance()->GetDefaultRegion(); - LOG_INFO(sLogger, (aliuid, region)); - - sleep(1); - LOG_INFO(sLogger, ("start generate log", "")); - - int32_t seqNo = 5; - for (int round = 0; round < 10; ++round) { - OneJob(100, gRootDir, "Job", true, time(NULL), "TestRealIpSend", seqNo); - usleep(100 * 1000); - } - - sleep(10); - - // check data - APSARA_TEST_EQUAL(gCounter, 1000); - gRecvLogGroupLock.lock(); - const Log& log = gRcvLogGroup.logs(0); - APSARA_TEST_EQUAL(log.contents(2).value(), "TestRealIpSend"); - APSARA_TEST_EQUAL(log.contents(3).value(), ToString(seqNo)); - gRecvLogGroupLock.unlock(); - - // check send host - auto client = Sender::Instance()->GetSendClient(region, aliuid); - - APSARA_TEST_TRUE(client->GetRawSlsHostFlag()); - APSARA_TEST_EQUAL(client->GetRawSlsHost().substr(0, strlen("10.123.32.")), string("10.123.32.")); - - CaseCleanUp(); - // reset flag - BOOL_FLAG(send_prefer_real_ip) = false; - LOG_INFO(sLogger, ("TestRealIpSend() end", time(NULL))); - } - - void TestEmptyRealIp() { - LOG_INFO(sLogger, ("TestEmptyRealIp() begin", time(NULL))); - - CaseSetUp(false, true, false); - EnableNetWork(); - - // enable real ip - BOOL_FLAG(send_prefer_real_ip) = true; - INT32_FLAG(send_switch_real_ip_interval) = 10; - - Sender::Instance()->MockGetRealIp = SenderUnittest::MockGetEmptyRealIp; - // start real ip update thread - gRealIpSendThread = CreateThread(std::bind(&Sender::RealIpUpdateThread, Sender::Instance())); - - string aliuid = "1234567890"; - string region = AppConfig::GetInstance()->GetDefaultRegion(); - LOG_INFO(sLogger, (aliuid, region)); - - sleep(1); - LOG_INFO(sLogger, ("start generate log", "")); - - int32_t seqNo = 5; - for (int round = 0; round < 10; ++round) { - OneJob(100, gRootDir, "Job", true, time(NULL), "TestRealIpSend", seqNo); - usleep(100 * 1000); - } - - sleep(10); - - // check data - APSARA_TEST_EQUAL(gCounter, 1000); - gRecvLogGroupLock.lock(); - const Log& log = gRcvLogGroup.logs(0); - APSARA_TEST_EQUAL(log.contents(2).value(), "TestRealIpSend"); - APSARA_TEST_EQUAL(log.contents(3).value(), ToString(seqNo)); - gRecvLogGroupLock.unlock(); - - // check send host - sdk::Client* client = Sender::Instance()->GetSendClient(region, aliuid); - - APSARA_TEST_EQUAL(client->GetRawSlsHostFlag(), false); - - CaseCleanUp(); - // reset flag - BOOL_FLAG(send_prefer_real_ip) = false; - - LOG_INFO(sLogger, ("TestEmptyRealIp() end", time(NULL))); - } - - void TestRealIpSendFailAndRecover() { - LOG_INFO(sLogger, ("TestRealIpSendFailAndRecover() begin", time(NULL))); - CaseSetUp(false, true, true); - DisableNetWork(); - gSendFailType = 1; - - // enable real ip - BOOL_FLAG(send_prefer_real_ip) = true; - INT32_FLAG(send_switch_real_ip_interval) = 60; - - Sender::Instance()->MockGetRealIp = SenderUnittest::MockGetRealIp; - // start real ip update thread - gStartIp = 0; - gRealIpSendThread = CreateThread(std::bind(&Sender::RealIpUpdateThread, Sender::Instance())); - - string aliuid = "1234567890"; - string region = AppConfig::GetInstance()->GetDefaultRegion(); - LOG_INFO(sLogger, (aliuid, region)); - - sleep(1); - LOG_INFO(sLogger, ("start generate log", "")); - - int32_t seqNo = 5; - for (int round = 0; round < 10; ++round) { - OneJob(100, gRootDir, "Job", true, time(NULL), "TestRealIpSendFailAndRecover", seqNo); - usleep(10 * 1000); - } - LOG_INFO(sLogger, ("generate end", "")); - sleep(10); - APSARA_TEST_EQUAL(gCounter, 0); - auto client = Sender::Instance()->GetSendClient(region, aliuid); - - APSARA_TEST_TRUE(client->GetRawSlsHostFlag()); - APSARA_TEST_TRUE_DESC(gStartIp >= 2, gStartIp); - - LOG_INFO(sLogger, ("enable network", "")); - EnableNetWork(); - sleep(8); - - // check data - APSARA_TEST_EQUAL(gCounter, 1000); - gRecvLogGroupLock.lock(); - const Log& log = gRcvLogGroup.logs(0); - APSARA_TEST_EQUAL(log.contents(2).value(), "TestRealIpSendFailAndRecover"); - APSARA_TEST_EQUAL(log.contents(3).value(), ToString(seqNo)); - gRecvLogGroupLock.unlock(); - // make sure real ip is enabled - int moreLogNum = 0; - for (int i = 0; i < 10; ++i) { - if (client->GetRawSlsHostFlag() != true) { - LOG_INFO(sLogger, ("write log to switch real ip", "")); - moreLogNum += 10; - OneJob(10, gRootDir, "Job", true, time(NULL), "TestRealIpSendFailAndRecover", seqNo); - sleep(1); - } else - break; - } - - // check send host - APSARA_TEST_TRUE(client->GetRawSlsHostFlag()); - APSARA_TEST_EQUAL(client->GetRawSlsHost().substr(0, strlen("10.123.32.")), string("10.123.32.")); - int32_t lastTime = time(NULL); - INT32_FLAG(send_switch_real_ip_interval) = 1000; - sleep(5); - string lastIp = client->GetRawSlsHost(); - gClient = client; - gDisabledIp = "10.123.32."; - gDisableIpFlag = true; - LOG_INFO(sLogger, ("disable ip", lastIp)); - for (int round = 0; round < 10; ++round) { - OneJob(100, gRootDir, "Job", true, time(NULL), "TestRealIpSendFailAndRecover", seqNo); - usleep(1000 * 1000); - } - WaitForFileBeenRead(); - sleep(5); - LOG_INFO(sLogger, ("check ip switch", lastIp)); - - Sender::Instance()->mRegionRealIpLock.lock(); - APSARA_TEST_NOT_EQUAL(lastIp, Sender::Instance()->mRegionRealIpMap[region]->mRealIp); - APSARA_TEST_TRUE(Sender::Instance()->mRegionRealIpMap[region]->mLastUpdateTime >= lastTime); - Sender::Instance()->mRegionRealIpLock.unlock(); - - gDisableIpFlag = false; - sleep(10); - - APSARA_TEST_EQUAL(gCounter, 2000 + moreLogNum); - - sleep(3); - gDisableIpFlag = false; - gClient = NULL; - INT32_FLAG(send_switch_real_ip_interval) = 10; - - CaseCleanUp(); - // reset flag - BOOL_FLAG(send_prefer_real_ip) = false; - LOG_INFO(sLogger, ("TestRealIpSendFailAndRecover() end", time(NULL))); - } - - // TestRegionConcurreny tests if the send concurrency control on region level works. - // 1. Set the send_request_concurrency to 30 and create 4 projects. - // 2. Generate some datas and then assert the number of received data. - // 3. Disable network and generate some datas, then assert the send concurrency of each region. - // 4. Enable network but only enable one project, generate lots of data of another three - // projects, and then assert that data to the enabled project can be received. - void TestRegionConcurreny() { - LOG_INFO(sLogger, ("TestRegionConcurreny() begin", time(NULL))); - - // There are three built-in regions, and project 0 will use the __default__ region, so - // the number of regions is 6 (3+3). - LOG_INFO(sLogger, - ("Set send_request_concurrency to 30 and " - "create 4 projects", - time(NULL))); - auto bakSendRequestConcurrency = INT32_FLAG(send_request_concurrency); - INT32_FLAG(send_request_concurrency) = 30; - CaseSetUp(false, false, true, 4); - EnableNetWork(); - gProjectNetEnableIndex = 3; - - LOG_INFO(sLogger, - ("Generate some data and " - "assert the number of received data", - "")); - for (int prjIndex = 0; prjIndex < 4; ++prjIndex) { - OneJob(60 * 10, gRootDir, "Job", true, time(NULL), "TestRegionConcurreny", 0, false, prjIndex); - } - WaitForFileBeenRead(); - sleep(INT32_FLAG(batch_send_interval) * 2 + 2); - sleep(10); - auto counter = gCounter; - EXPECT_EQ(counter, 60 * 10 * 4); - - LOG_INFO(sLogger, - ("Disable network and generate more data, " - "assert the send concurrency of each region", - "")); - sleep(1); - gSendFailType = 1; - DisableNetWork(); - sleep(1); - for (int prjIndex = 0; prjIndex < 4; ++prjIndex) { - OneJob(60 * 10, gRootDir, "Job", true, time(NULL), "TestRegionConcurreny", 0, false, prjIndex); - } - WaitForFileBeenRead(); - sleep(INT32_FLAG(batch_send_interval) * 2 + 2); - EXPECT_EQ(gCounter, counter); - { - PTScopedLock lock(Sender::Instance()->mRegionEndpointEntryMapLock); - auto& regionMap = Sender::Instance()->mRegionEndpointEntryMap; - LOG_INFO(sLogger, ("Access region map", regionMap.size())); - for (auto iter = regionMap.begin(); iter != regionMap.end(); ++iter) { - auto& regionName = iter->first; - LOG_INFO(sLogger, (regionName, iter->second->mConcurrency)); - - // Only tests __default__ and 100000[1-3]_proj. - int32_t projectIndex = -1; - if (0 == regionName.find("__default")) - projectIndex = 0; - else if (0 == regionName.find("1000")) { - StringTo(regionName.substr(std::string("100000").length(), 1), projectIndex); - } - if (0 <= projectIndex && projectIndex <= 3) { - EXPECT_EQ(iter->second->mConcurrency, 30 / regionMap.size()); - } - } - } - - LOG_INFO(sLogger, - ("Enable network but only enable one project, " - "generate lots of data of another three projects, and then " - "assert that data to the enabled project can be received on time.", - "")); - sleep(1); - gProjectNetEnableIndex = 0; - EnableNetWork(); - sleep(INT32_FLAG(batch_send_interval) * 2 + 2); - counter += 60 * 10; - EXPECT_EQ(gCounter, counter); - for (int prjIndex = 1; prjIndex < 4; ++prjIndex) { - OneJob(60 * 300, gRootDir, "Job", true, time(NULL), "TestRegionConcurreny", 0, false, prjIndex); - } - OneJob(60 * 100, gRootDir, "Job", true, time(NULL), "TestRegionConcurreny", 0, false, 0); - WaitForFileBeenRead(); - sleep(INT32_FLAG(batch_send_interval) * 2 + 2); - counter += 60 * 100; - EXPECT_EQ(gCounter, counter); - - INT32_FLAG(send_request_concurrency) = bakSendRequestConcurrency; - CaseCleanUp(); - - LOG_INFO(sLogger, ("TestRegionConcurreny() end", time(NULL))); - } - - void TestTooOldFilesIntegrity() { - LOG_INFO(sLogger, ("TestTooOldFilesIntegrity() begin", time(NULL))); - const int PROJECT_NUM = 8; - EnableNetWork(); - gLogIntegrityTestFlag = true; - auto bakFileEliminateInterval = INT32_FLAG(file_eliminate_interval); - CaseSetUp(false, true, true, PROJECT_NUM, false, -1, 0, 900); - INT32_FLAG(file_eliminate_interval) = 3; - for (int round = 0; round < 10; ++round) { - for (int prjIndex = 0; prjIndex < PROJECT_NUM; ++prjIndex) { - // use 100, let time to be separated in different minutes, so there will be two packets - OneJob(100, gRootDir, "Job", true, time(NULL), "TestTooOldFilesIntegrity", 0, false, prjIndex); - } - usleep(200 * 1000); - } - WaitForFileBeenRead(); - - // wait 10s - sleep(10); - - // test - const LogIntegrity::RegionOutDatedFileMap& regionOutDatedFileMap - = LogIntegrity::GetInstance()->mRegionOutDatedFileMap; - APSARA_TEST_EQUAL(regionOutDatedFileMap.size(), 4); - - for (LogIntegrity::RegionOutDatedFileMap::const_iterator regionIter = regionOutDatedFileMap.begin(); - regionIter != regionOutDatedFileMap.end(); - ++regionIter) { - // get seq - const std::string& getRegion = regionIter->first; - int seq = getRegion[6] - '0'; - - char region[16] = {0}; - char projectName[16] = {0}; - char logFileName[16] = {0}; - snprintf(region, sizeof(region), "%d_proj", 1000000 + seq); - snprintf(projectName, sizeof(projectName), "%d_proj", 1000000 + seq); - if (seq == 0) - strcpy(logFileName, "Job.log"); - else - snprintf(logFileName, sizeof(logFileName), "Job.log%d", seq); - - const LogIntegrity::OutDatedFileMap* outDatedFileMap = regionIter->second; - APSARA_TEST_EQUAL(regionIter->first, std::string(region)); - APSARA_TEST_EQUAL(outDatedFileMap->size(), (size_t)1); - - for (LogIntegrity::OutDatedFileMap::const_iterator fileIter = outDatedFileMap->begin(); - fileIter != outDatedFileMap->end(); - ++fileIter) { - const OutDatedFile* outDatedFile = fileIter->second; - APSARA_TEST_EQUAL(outDatedFile->mRegion, std::string(region)); - APSARA_TEST_EQUAL(outDatedFile->mProjectName, std::string(projectName)); - APSARA_TEST_EQUAL(outDatedFile->mLogstore, "app_log"); - APSARA_TEST_EQUAL(outDatedFile->mFilename, gRootDir + PATH_SEPARATOR + std::string(logFileName)); - APSARA_TEST_EQUAL(outDatedFile->mIntegrityProject, std::string(projectName)); - APSARA_TEST_EQUAL(outDatedFile->mIntegrityLogstore, "test-integrity-logstore"); - } - } - - gLogIntegrityTestFlag = false; - CaseCleanUp(); - INT32_FLAG(file_eliminate_interval) = bakFileEliminateInterval; - LOG_INFO(sLogger, ("TestTooOldFilesIntegrity() end", time(NULL))); - } - - void TestGlobalMarkOffset() { - LOG_INFO(sLogger, ("TestGlobalMarkOffset() begin", time(NULL))); - // prepare - auto bakDefaultGlobalMarkOffsetFlag = BOOL_FLAG(default_global_mark_offset_flag); - auto bakDefaultGlobalFuseMode = BOOL_FLAG(default_global_fuse_mode); - BOOL_FLAG(default_global_mark_offset_flag) = true; - BOOL_FLAG(default_global_fuse_mode) = false; - - gGlobalMarkOffsetTestFlag = true; - std::string dir = gRootDir + "MarkOffsetTest"; - bfs::create_directories(dir); - auto& PS = PATH_SEPARATOR; - - // test - EnableNetWork(); - CaseSetUp(false, true, true, 1, false, -1, 0, 900); - - auto PrintMapInfo = []() { - auto inst = LogFileCollectOffsetIndicator::GetInstance(); - LOG_INFO(sLogger, ("mLogFileOffsetInfoMap", "")); - for (auto& i : inst->mLogFileOffsetInfoMap) { - LOG_INFO(sLogger, (i.first.ToString(), i.second->mLogFileInfo.mFilename)); - } - LOG_INFO(sLogger, ("mLogFileOffsetInfoMap", "done")); - LOG_INFO(sLogger, ("mLogFileCollectProgressMap", "")); - for (auto& i : inst->mLogFileCollectProgressMap) { - LOG_INFO(sLogger, (i.first.ToString(), i.second.IsValid())); - } - LOG_INFO(sLogger, ("mLogFileCollectProgressMap", "done")); - }; - - LOG_INFO(sLogger, ("write a.log", "")); - for (int round = 0; round < 10; ++round) { - OneJob(100, dir, "a", true, time(NULL)); - usleep(200 * 1000); - } - WaitForFileBeenRead(); - sleep(5); - LOG_INFO(sLogger, ("print map info", "after writing a.log")); - PrintMapInfo(); - - LOG_INFO(sLogger, ("check map status", "1")); - const LogFileCollectProgressMap& m1 = LogFileCollectOffsetIndicator::GetInstance()->mLogFileCollectProgressMap; - APSARA_TEST_EQUAL(m1.size(), 1); - fsutil::PathStat buf_a; - bool statRet_a = fsutil::PathStat::stat(std::string(dir + PS + "a.log").c_str(), buf_a); - APSARA_TEST_TRUE_DESC(statRet_a, "a.log must exists"); - const LogFileInfo& info_a = m1.begin()->first; - const LogFileCollectProgress& progress_a = m1.begin()->second; - APSARA_TEST_EQUAL(info_a.mFilename, dir + PS + "a.log"); - APSARA_TEST_EQUAL(progress_a.mFileSize, buf_a.GetFileSize()); - APSARA_TEST_EQUAL(progress_a.mFileLastPos, progress_a.mFileSize); - APSARA_TEST_EQUAL(progress_a.mFileReadPos, progress_a.mFileSize); - APSARA_TEST_TRUE(progress_a.IsFinished()); - - LOG_INFO(sLogger, ("write b.log", "")); - for (int round = 0; round < 20; ++round) { - OneJob(100, dir, "b", true, time(NULL)); - usleep(200 * 1000); - } - WaitForFileBeenRead(); - sleep(11); - LOG_INFO(sLogger, ("print map info", "after writing b.log")); - PrintMapInfo(); - - LOG_INFO(sLogger, ("rotate b.log and write new b.log", "")); - bfs::rename(std::string(dir + PS + "b.log").c_str(), std::string(dir + PS + "b.log.1").c_str()); - for (int round = 0; round < 10; ++round) { - OneJob(100, dir, "b", true, time(NULL)); - usleep(200 * 1000); - } - WaitForFileBeenRead(); - sleep(11); - LOG_INFO(sLogger, ("print map info", "after rotating b.log")); - PrintMapInfo(); - - LOG_INFO(sLogger, ("check map status", "2")); - const LogFileCollectProgressMap& m2 = LogFileCollectOffsetIndicator::GetInstance()->mLogFileCollectProgressMap; - APSARA_TEST_EQUAL(m2.size(), 2); - fsutil::PathStat buf_b; - bool statRet_b = fsutil::PathStat::stat(std::string(dir + PS + "b.log").c_str(), buf_b); - APSARA_TEST_TRUE_DESC(statRet_b, "b.log must exists"); - for (LogFileCollectProgressMap::const_iterator iter = m2.begin(); iter != m2.end(); ++iter) { - const LogFileInfo& info_b = iter->first; - const LogFileCollectProgress& progress_b = iter->second; - if (info_b.mFilename == dir + PS + "b.log") { - APSARA_TEST_EQUAL(info_b.mFilename, dir + PS + "b.log"); - APSARA_TEST_EQUAL(progress_b.mFileSize, buf_b.GetFileSize()); - APSARA_TEST_EQUAL(progress_b.mFileLastPos, progress_b.mFileSize); - APSARA_TEST_EQUAL(progress_b.mFileReadPos, progress_b.mFileSize); - APSARA_TEST_TRUE(progress_b.IsFinished()); - } - } - - // write file, c.log, rotate, then delete c.log - LOG_INFO(sLogger, ("write c.log", "")); - for (int round = 0; round < 20; ++round) { - OneJob(100, dir, "c", true, time(NULL)); - usleep(200 * 1000); - } - WaitForFileBeenRead(); - sleep(11); - LOG_INFO(sLogger, ("print map info", "after writing c.log")); - PrintMapInfo(); - - LOG_INFO(sLogger, ("rotate c.log and write new c.log", "")); - bfs::rename(std::string(dir + PS + "c.log").c_str(), std::string(dir + PS + "c.log.1").c_str()); - for (int round = 0; round < 10; ++round) { - OneJob(100, dir, "c", true, time(NULL)); - usleep(200 * 1000); - } - WaitForFileBeenRead(); - sleep(11); - LOG_INFO(sLogger, ("print map info", "after rotating c.log")); - PrintMapInfo(); - - LOG_INFO(sLogger, ("check map status", "3")); - const LogFileCollectProgressMap& m3 = LogFileCollectOffsetIndicator::GetInstance()->mLogFileCollectProgressMap; - APSARA_TEST_EQUAL(m3.size(), 3); - - LOG_INFO(sLogger, ("delete c.log", "")); - bfs::remove(std::string(dir + PS + "c.log").c_str()); - sleep(11); - LOG_INFO(sLogger, ("print map info", "after deleting c.log")); - PrintMapInfo(); - - LOG_INFO(sLogger, ("check map status", "4")); - const LogFileCollectProgressMap& m4 = LogFileCollectOffsetIndicator::GetInstance()->mLogFileCollectProgressMap; - APSARA_TEST_EQUAL(m4.size(), 2); - bool found = false; - for (LogFileCollectProgressMap::const_iterator iter = m2.begin(); iter != m2.end(); ++iter) { - const LogFileInfo& info_c = iter->first; - if (info_c.mFilename == dir + PS + "c.log") { - found = true; - } - } - APSARA_TEST_TRUE(!found); - - // case clean up - CaseCleanUp(); - bfs::remove_all(dir); - gGlobalMarkOffsetTestFlag = false; - BOOL_FLAG(default_global_fuse_mode) = bakDefaultGlobalFuseMode; - BOOL_FLAG(default_global_mark_offset_flag) = bakDefaultGlobalMarkOffsetFlag; - LOG_INFO(sLogger, ("TestGlobalMarkOffset() end", time(NULL))); - } - - static void MockExactlyOnceSend(LoggroupTimeValue* data); - - void TestExactlyOnceDataSendSequence(); - - void TestExactlyOncePartialBlockConcurrentSend(); - - void TestExactlyOnceCompleteBlockConcurrentSend(); -}; - -APSARA_UNIT_TEST_CASE(SenderUnittest, TestSecondaryStorage, gCaseID); -APSARA_UNIT_TEST_CASE(SenderUnittest, TestEncryptAndDecrypt, gCaseID); -APSARA_UNIT_TEST_CASE(SenderUnittest, TestDiscardOldData, gCaseID); -APSARA_UNIT_TEST_CASE(SenderUnittest, TestConnect, gCaseID); -APSARA_UNIT_TEST_CASE(SenderUnittest, TestDisConnect, gCaseID); -APSARA_UNIT_TEST_CASE(SenderUnittest, TestChangeStat, gCaseID); -APSARA_UNIT_TEST_CASE(SenderUnittest, TestFlowControl, gCaseID); -APSARA_UNIT_TEST_CASE(SenderUnittest, TestMergeByCount, gCaseID); -APSARA_UNIT_TEST_CASE(SenderUnittest, TestFlushOut, gCaseID); -APSARA_UNIT_TEST_CASE(SenderUnittest, TestMonitor, gCaseID); -APSARA_UNIT_TEST_CASE(SenderUnittest, TestFilterRule, gCaseID); -APSARA_UNIT_TEST_CASE(SenderUnittest, TestDumpSnapshot, gCaseID); -APSARA_UNIT_TEST_CASE(SenderUnittest, TestMergeByMinute, gCaseID); -APSARA_UNIT_TEST_CASE(SenderUnittest, TestMultiUserSeperationAndDiscardFailAndOtherFail, gCaseID); -APSARA_UNIT_TEST_CASE(SenderUnittest, TestMultiUserSeperationAndRetryQuotaRecovery, gCaseID); -APSARA_UNIT_TEST_CASE(SenderUnittest, TestMultiUserSeperationAndTestNetWorkRecovery, gCaseID); -APSARA_UNIT_TEST_CASE(SenderUnittest, TestMultiUserSeperationAndRetryIntervalRecovery, gCaseID); -APSARA_UNIT_TEST_CASE(SenderUnittest, TestLogstoreFlowControl, gCaseID); -APSARA_UNIT_TEST_CASE(SenderUnittest, TestLogstoreFlowControlPause, gCaseID); -APSARA_UNIT_TEST_CASE(SenderUnittest, TestLogstoreFlowControlExpire, gCaseID); -APSARA_UNIT_TEST_CASE(SenderUnittest, TestTooOldFilesIntegrity, gCaseID); -APSARA_UNIT_TEST_CASE(SenderUnittest, TestGlobalMarkOffset, gCaseID); -APSARA_UNIT_TEST_CASE(SenderUnittest, TestRealIpSend, gCaseID); -APSARA_UNIT_TEST_CASE(SenderUnittest, TestEmptyRealIp, gCaseID); -APSARA_UNIT_TEST_CASE(SenderUnittest, TestRealIpSendFailAndRecover, gCaseID); -APSARA_UNIT_TEST_CASE(SenderUnittest, TestRegionConcurreny, gCaseID); -UNIT_TEST_CASE(SenderUnittest, TestExactlyOnceDataSendSequence); -UNIT_TEST_CASE(SenderUnittest, TestExactlyOncePartialBlockConcurrentSend); -UNIT_TEST_CASE(SenderUnittest, TestExactlyOnceCompleteBlockConcurrentSend); - -decltype(SenderUnittest::sProcessQueueMap) SenderUnittest::sProcessQueueMap = nullptr; -decltype(SenderUnittest::sSenderQueueMap) SenderUnittest::sSenderQueueMap = nullptr; - -// Record checkpoint and forward to MockAsyncSend. -void SenderUnittest::MockExactlyOnceSend(LoggroupTimeValue* data) { - auto& cpt = data->mLogGroupContext.mExactlyOnceCheckpoint; - if (!cpt) { - return; - } - - auto closure = new SendClosure; - closure->mDataPtr = data; - - { - auto hashKey = cpt->data.hash_key(); - std::lock_guard lock(gMockExactlyOnceSendLock); - if (gBlockedHashKeySet.find(hashKey) != gBlockedHashKeySet.end()) { - LOG_INFO(sLogger, ("hash key is blocked", hashKey)("checkpoint", cpt->data.DebugString())); - auto res = new sdk::PostLogStoreLogsResponse; - res->statusCode = 403; - res->requestId = "RequestIDInMockExactlyOnceSend"; - closure->OnFail(res, sdk::LOGE_WRITE_QUOTA_EXCEED, "reject by exactly once send"); - return; - } - } - - gRangeCheckpoints.push_back(RangeCheckpointPtr(new RangeCheckpoint(*(cpt.get())))); - LOG_INFO(sLogger, ("checkpoint key", cpt->key)("checkpoint", cpt->data.DebugString())); - MockAsyncSend(data->mProjectName, - data->mLogstore, - data->mLogData, - data->mDataType, - data->mRawSize, - data->mLogGroupContext.mCompressType, - closure); -} - -// Test if data's sequence is generated orderly: -// - Incremental sequence ID. -// - Incremental checkpoint index. -// -// Procedure: generate N logs continuously, use MockExactlyOnceSend to catch checkpoints, -// after N logs are sent, test if these logs break order. -void SenderUnittest::TestExactlyOnceDataSendSequence() { - LOG_INFO(sLogger, ("TestExactlyOnceDataSendSequence() begin", time(NULL))); - - gEnableExactlyOnce = true; - EnableNetWork(); - CaseSetUp(false, true, true, 1, false, -1, 0, 900); - - std::string dir = gRootDir + PATH_SEPARATOR + "ExactlyOnceDataSendSequence"; - bfs::create_directories(dir); - - LOG_INFO(sLogger, ("write a.log", dir)); - const size_t kRound = 3; - for (size_t round = 0; round < kConcurrency * kRound; ++round) { - OneJob(100, dir, "a", true, time(NULL)); - usleep(200 * 1000); - } - WaitForFileBeenRead(); - sleep(5); - - LOG_INFO(sLogger, ("test sequence", "")); - ModifyHandler* handler = nullptr; - { - auto& pathMap = sEventDispatcher->mPathWdMap; - auto iter = pathMap.find(dir); - ASSERT_TRUE(iter != pathMap.end()); - auto dirHandler = static_cast(sEventDispatcher->mWdDirInfoMap[iter->second]->mHandler); - EXPECT_EQ(dirHandler->mModifyHandlerPtrMap.size(), 1); - handler = dirHandler->mModifyHandlerPtrMap.begin()->second; - } - auto& readerMap = handler->mNameReaderMap; - EXPECT_EQ(readerMap.size(), 1); - auto& readerArray = readerMap.begin()->second; - EXPECT_EQ(readerArray.size(), 1); - auto& reader = readerArray.front(); - auto& eo = reader->mEOOption; - EXPECT_TRUE(eo); - - std::set hashKeySet; - for (auto& cpt : eo->rangeCheckpointPtrs) { - hashKeySet.insert(cpt->data.hash_key()); - } - EXPECT_EQ(hashKeySet.size(), kConcurrency); - EXPECT_GE(gRangeCheckpoints.size(), kConcurrency); - std::unordered_map > hashKeySeqIDs; - for (auto& cpt : gRangeCheckpoints) { - hashKeySeqIDs[cpt->data.hash_key()].push_back(cpt->data.sequence_id()); - } - for (auto& iter : hashKeySeqIDs) { - auto sortedSequenceIDs = iter.second; - std::sort(sortedSequenceIDs.begin(), sortedSequenceIDs.end()); - EXPECT_EQ(iter.second, sortedSequenceIDs); - } - - CaseCleanUp(); - gEnableExactlyOnce = false; - LOG_INFO(sLogger, ("TestExactlyOnceDataSendSequence() end", time(NULL))); -} - -// Test if data read will block as expected when send is blocked. -// Reader will be blocked only when all concurrency is blocked. -// -// Procedure: generate N logs continuously, use MockExactlyOnceSend to catch checkpoints, -// after N logs are sent, block specified concurrency in MockExactlyOnceSend. -void SenderUnittest::TestExactlyOncePartialBlockConcurrentSend() { - LOG_INFO(sLogger, ("TestExactlyOncePartialBlockConcurrentSend() begin", time(NULL))); - - auto bakClientQuotaSendRetryIntervalMax = INT32_FLAG(client_quota_send_retry_interval_max); - auto bakClientQuotaSendConcurrencyMin = INT32_FLAG(client_quota_send_concurrency_min); - auto bakClientQuotaSendRetryInterval = INT32_FLAG(client_quota_send_retry_interval); - INT32_FLAG(client_quota_send_retry_interval_max) = 1; - INT32_FLAG(client_quota_send_concurrency_min) = kConcurrency; - INT32_FLAG(client_quota_send_retry_interval) = 1; - gEnableExactlyOnce = true; - EnableNetWork(); - CaseSetUp(false, true, true, 1, false, -1, 0, 900); - - std::string dir = gRootDir + PATH_SEPARATOR + "ExactlyOncePartialBlockConcurrentSend"; - bfs::create_directories(dir); - - auto globalRangeCheckpoints2HashKeySet = [](std::vector* hashKeys = nullptr) { - std::set hashKeySet; - for (auto& cpt : gRangeCheckpoints) { - hashKeySet.insert(cpt->data.hash_key()); - if (hashKeys) { - hashKeys->push_back(cpt->data.hash_key()); - } - } - return hashKeySet; - }; - - LOG_INFO(sLogger, ("send round 1 data", "collect checkpoint keys")); - for (size_t round = 0; round < kConcurrency * 2; ++round) { - OneJob(100, dir, "a", true, time(NULL)); - usleep(100 * 1000); - } - WaitForFileBeenRead(); - sleep(5); - std::vector hashKeys; - { - // All concurrency have been used. - auto hashKeySet = globalRangeCheckpoints2HashKeySet(&hashKeys); - EXPECT_EQ(hashKeySet.size(), kConcurrency); - gRangeCheckpoints.clear(); - } - - const auto blockedIndex = kConcurrency / 2; - const auto blockedHashKey = hashKeys[blockedIndex]; - LOG_INFO(sLogger, ("block hash key", blockedHashKey)("send round 2 data", "")); - gBlockedHashKeySet.insert(blockedHashKey); - for (size_t round = 0; round < kConcurrency * 2; ++round) { - OneJob(100, dir, "a", true, time(NULL)); - usleep(100 * 1000); - } - WaitForFileBeenRead(); - sleep(5); - { - // Concurrency except for bloced one have been used. - auto hashKeySet = globalRangeCheckpoints2HashKeySet(); - EXPECT_EQ(hashKeySet.size(), kConcurrency - 1); - EXPECT_TRUE(hashKeySet.find(blockedHashKey) == hashKeySet.end()); - gRangeCheckpoints.clear(); - } - - LOG_INFO(sLogger, ("unblock hash key", blockedHashKey)); - gBlockedHashKeySet.clear(); - for (size_t round = 0; round < kConcurrency * 2; ++round) { - OneJob(100, dir, "a", true, time(NULL)); - usleep(100 * 1000); - } - WaitForFileBeenRead(); - sleep(5); - { - // The blocked data can be sent now. - auto hashKeySet = globalRangeCheckpoints2HashKeySet(); - EXPECT_GE(hashKeySet.size(), 1); - EXPECT_TRUE(hashKeySet.find(blockedHashKey) != hashKeySet.end()); - gRangeCheckpoints.clear(); - } - - LOG_INFO(sLogger, ("send round 4 data", "recovery")); - for (size_t round = 0; round < kConcurrency * 2; ++round) { - OneJob(100, dir, "a", true, time(NULL)); - usleep(100 * 1000); - } - WaitForFileBeenRead(); - sleep(5); - { - // All concurrency have been used. - auto hashKeySet = globalRangeCheckpoints2HashKeySet(); - EXPECT_EQ(hashKeySet.size(), kConcurrency); - gRangeCheckpoints.clear(); - } - - CaseCleanUp(); - gEnableExactlyOnce = false; - INT32_FLAG(client_quota_send_retry_interval_max) = bakClientQuotaSendRetryIntervalMax; - INT32_FLAG(client_quota_send_concurrency_min) = bakClientQuotaSendConcurrencyMin; - INT32_FLAG(client_quota_send_retry_interval) = bakClientQuotaSendRetryInterval; - LOG_INFO(sLogger, ("TestExactlyOncePartialBlockConcurrentSend() end", time(NULL))); -} - -// Test if data read will block as expected when send is blocked. -// Reader will be blocked only when all concurrency is blocked. -// -// Procedure: generate N logs continuously, use MockExactlyOnceSend to catch checkpoints, -// after N logs are sent, block all concurrency in MockExactlyOnceSend. -void SenderUnittest::TestExactlyOnceCompleteBlockConcurrentSend() { - LOG_INFO(sLogger, ("TestExactlyOnceCompleteBlockConcurrentSend() begin", time(NULL))); - - auto bakClientQuotaSendRetryIntervalMax = INT32_FLAG(client_quota_send_retry_interval_max); - auto bakClientQuotaSendConcurrencyMin = INT32_FLAG(client_quota_send_concurrency_min); - auto bakClientQuotaSendRetryInterval = INT32_FLAG(client_quota_send_retry_interval); - INT32_FLAG(client_quota_send_retry_interval_max) = 1; - INT32_FLAG(client_quota_send_concurrency_min) = kConcurrency; - INT32_FLAG(client_quota_send_retry_interval) = 1; - gEnableExactlyOnce = true; - EnableNetWork(); - CaseSetUp(false, true, true, 1, false, -1, 0, 900); - - std::string dir = gRootDir + PATH_SEPARATOR + "ExactlyOnceCompleteBlockConcurrentSend"; - bfs::create_directories(dir); - - auto globalRangeCheckpoints2HashKeySet = [](std::vector* hashKeys = nullptr) { - std::set hashKeySet; - for (auto& cpt : gRangeCheckpoints) { - hashKeySet.insert(cpt->data.hash_key()); - if (hashKeys) { - hashKeys->push_back(cpt->data.hash_key()); - } - } - return hashKeySet; - }; - - LOG_INFO(sLogger, ("send round 1 data", "collect checkpoint keys")); - for (size_t round = 0; round < kConcurrency * 2; ++round) { - OneJob(100, dir, "a", true, time(NULL)); - usleep(100 * 1000); - } - WaitForFileBeenRead(); - sleep(5); - std::vector hashKeys; - { - // All concurrency have been used. - auto hashKeySet = globalRangeCheckpoints2HashKeySet(&hashKeys); - EXPECT_EQ(hashKeySet.size(), kConcurrency); - gRangeCheckpoints.clear(); - } - - LOG_INFO(sLogger, ("block all hash keys", "send round 2 data")); - gBlockedHashKeySet.insert(hashKeys.begin(), hashKeys.end()); - for (size_t round = 0; round < kConcurrency * 2; ++round) { - OneJob(100, dir, "a", true, time(NULL)); - usleep(100 * 1000); - } - WaitForFileBeenRead(); - sleep(5); - { - // No data can be sent. - auto hashKeySet = globalRangeCheckpoints2HashKeySet(); - EXPECT_EQ(hashKeySet.size(), 0); - EXPECT_EQ(gRangeCheckpoints.size(), 0); - } - - const auto unlockedIndex = kConcurrency / 2; - const auto unlockedHashKey = hashKeys[unlockedIndex]; - LOG_INFO(sLogger, ("unblock one hash key", unlockedHashKey)); - { - std::lock_guard lock(gMockExactlyOnceSendLock); - gBlockedHashKeySet.erase(unlockedHashKey); - } - sleep(INT32_FLAG(client_quota_send_retry_interval_max) * 2); - { - // All data is send through the concurrency. - auto hashKeySet = globalRangeCheckpoints2HashKeySet(); - EXPECT_GE(hashKeySet.size(), 1); - EXPECT_TRUE(hashKeySet.find(unlockedHashKey) != hashKeySet.end()); - gRangeCheckpoints.clear(); - } - - LOG_INFO(sLogger, ("unblocked all hash keys", "send round 3 data")); - gBlockedHashKeySet.clear(); - for (size_t round = 0; round < kConcurrency * 2; ++round) { - OneJob(100, dir, "a", true, time(NULL)); - usleep(100 * 1000); - } - WaitForFileBeenRead(); - sleep(5); - { - // All concurrency have been used. - auto hashKeySet = globalRangeCheckpoints2HashKeySet(); - EXPECT_EQ(hashKeySet.size(), kConcurrency); - gRangeCheckpoints.clear(); - } - - CaseCleanUp(); - gEnableExactlyOnce = false; - INT32_FLAG(client_quota_send_retry_interval_max) = bakClientQuotaSendRetryIntervalMax; - INT32_FLAG(client_quota_send_concurrency_min) = bakClientQuotaSendConcurrencyMin; - INT32_FLAG(client_quota_send_retry_interval) = bakClientQuotaSendRetryInterval; - LOG_INFO(sLogger, ("TestExactlyOnceCompleteBlockConcurrentSend() end", time(NULL))); -} - -} // namespace logtail - -int main(int argc, char** argv) { - logtail::Logger::Instance().InitGlobalLoggers(); - ::testing::InitGoogleTest(&argc, argv); - return RUN_ALL_TESTS(); -} diff --git a/core/unittest/task_pipeline/TaskPipelineUnittest.cpp b/core/unittest/task_pipeline/TaskPipelineUnittest.cpp index 0b454a79bf..b4523102d6 100644 --- a/core/unittest/task_pipeline/TaskPipelineUnittest.cpp +++ b/core/unittest/task_pipeline/TaskPipelineUnittest.cpp @@ -37,7 +37,7 @@ class TaskPipelineUnittest : public ::testing::Test { private: const string configName = "test_config"; - const filesystem::path filepath = "/path/to/test"; + const fs::path filepath = "/path/to/test"; }; void TaskPipelineUnittest::OnSuccessfulInit() const { diff --git a/docs/cn/configuration/system-config.md b/docs/cn/configuration/system-config.md index dacb10536b..d6d9cea049 100644 --- a/docs/cn/configuration/system-config.md +++ b/docs/cn/configuration/system-config.md @@ -14,7 +14,6 @@ | `data_server_port` | Int |

用于控制 `flusher_sls` 往 `SLS` 发送的协议类型。

取值范围:443(默认),表示使用 `HTTPS` 协议发送;80表示使用 `HTTP` 协议发送。

如果使用`SLS`内网域名写入,建议使用`HTTP`协议发送,提高传输性能。

| | `send_running_status` | Bool | 为了更好的了解 `iLogtail` 的使用情况,以便做出更有针对性的发展规划,`iLogtail` 会上报一些脱敏后的运行统计信息。您也可以手动关闭此开关。 | | `host_path_blacklist` | String | 全局主机路径黑名单,黑名单为子串匹配,Linux下多个子串以:分隔,Windows下以;分隔。比如禁止采集NAS挂载,可以配置为`/volumes/kubernetes.io~csi/nas-`。 | -| `metrics_report_method` | String |

自身指标输出方式。默认为空,即不输出指标。

当前支持的值:
`file`:每分钟将指标输出到`ilogtail`运行目录下的`self_metrics`目录,文件格式为`self-metrics-&{time}.json`,最多保留60个指标文件(即1小时的数据)。该方式适合本地调试使用。

| | `ebpf.receive_event_chan_cap` | Int | 用于接收内核事件的队列大小,默认为 4096 | | `ebpf.admin_config.debug_mode` | Bool | 是否开启 ebpf debug 模式,默认为 false | | `ebpf.admin_config.log_level` | String | ebpf 相关的日志级别,包括 info warn 和 debug,默认为 warn | diff --git a/test/config/load.go b/test/config/load.go index 217c4f0d18..08ddb8fb78 100644 --- a/test/config/load.go +++ b/test/config/load.go @@ -32,6 +32,7 @@ var ( ConfigDir string OnetimeConfigDir string LogDir string + DataDir string ) // Load E2E engine config and define the global variables. @@ -47,6 +48,7 @@ func Load(path string) error { reportDir := root + "/report/" EngineLogFile = reportDir + CaseName + "_engine.log" LogDir = reportDir + CaseName + "_log" + DataDir = reportDir + CaseName + "_data" FlusherFile = reportDir + CaseName + "default_flusher.json" return nil diff --git a/test/e2e/test_cases/input_static_file_docker/a.log b/test/e2e/test_cases/input_static_file_docker/a.log new file mode 100644 index 0000000000..b5f9f8c6be --- /dev/null +++ b/test/e2e/test_cases/input_static_file_docker/a.log @@ -0,0 +1,1000 @@ +5===== +6===== +7===== +8===== +9===== +10===== +11===== +12===== +13===== +14===== +15===== +16===== +17===== +18===== +19===== +20===== +21===== +22===== +23===== +24===== +25===== +26===== +27===== +28===== +29===== +30===== +31===== +32===== +33===== +34===== +35===== +36===== +37===== +38===== +39===== +40===== +41===== +42===== +43===== +44===== +45===== +46===== +47===== +48===== +49===== +50===== +51===== +52===== +53===== +54===== +55===== +56===== +57===== +58===== +59===== +60===== +61===== +62===== +63===== +64===== +65===== +66===== +67===== +68===== +69===== +70===== +71===== +72===== +73===== +74===== +75===== +76===== +77===== +78===== +79===== +80===== +81===== +82===== +83===== +84===== +85===== +86===== +87===== +88===== +89===== +90===== +91===== +92===== +93===== +94===== +95===== +96===== +97===== +98===== +99===== +100===== +101===== +102===== +103===== +104===== +105===== +106===== +107===== +108===== +109===== +110===== +111===== +112===== +113===== +114===== +115===== +116===== +117===== +118===== +119===== +120===== +121===== +122===== +123===== +124===== +125===== +126===== +127===== +128===== +129===== +130===== +131===== +132===== +133===== +134===== +135===== +136===== +137===== +138===== +139===== +140===== +141===== +142===== +143===== +144===== +145===== +146===== +147===== +148===== +149===== +150===== +151===== +152===== +153===== +154===== +155===== +156===== +157===== +158===== +159===== +160===== +161===== +162===== +163===== +164===== +165===== +166===== +167===== +168===== +169===== +170===== +171===== +172===== +173===== +174===== +175===== +176===== +177===== +178===== +179===== +180===== +181===== +182===== +183===== +184===== +185===== +186===== +187===== +188===== +189===== +190===== +191===== +192===== +193===== +194===== +195===== +196===== +197===== +198===== +199===== +200===== +201===== +202===== +203===== +204===== +205===== +206===== +207===== +208===== +209===== +210===== +211===== +212===== +213===== +214===== +215===== +216===== +217===== +218===== +219===== +220===== +221===== +222===== +223===== +224===== +225===== +226===== +227===== +228===== +229===== +230===== +231===== +232===== +233===== +234===== +235===== +236===== +237===== +238===== +239===== +240===== +241===== +242===== +243===== +244===== +245===== +246===== +247===== +248===== +249===== +250===== +251===== +252===== +253===== +254===== +255===== +256===== +257===== +258===== +259===== +260===== +261===== +262===== +263===== +264===== +265===== +266===== +267===== +268===== +269===== +270===== +271===== +272===== +273===== +274===== +275===== +276===== +277===== +278===== +279===== +280===== +281===== +282===== +283===== +284===== +285===== +286===== +287===== +288===== +289===== +290===== +291===== +292===== +293===== +294===== +295===== +296===== +297===== +298===== +299===== +300===== +301===== +302===== +303===== +304===== +305===== +306===== +307===== +308===== +309===== +310===== +311===== +312===== +313===== +314===== +315===== +316===== +317===== +318===== +319===== +320===== +321===== +322===== +323===== +324===== +325===== +326===== +327===== +328===== +329===== +330===== +331===== +332===== +333===== +334===== +335===== +336===== +337===== +338===== +339===== +340===== +341===== +342===== +343===== +344===== +345===== +346===== +347===== +348===== +349===== +350===== +351===== +352===== +353===== +354===== +355===== +356===== +357===== +358===== +359===== +360===== +361===== +362===== +363===== +364===== +365===== +366===== +367===== +368===== +369===== +370===== +371===== +372===== +373===== +374===== +375===== +376===== +377===== +378===== +379===== +380===== +381===== +382===== +383===== +384===== +385===== +386===== +387===== +388===== +389===== +390===== +391===== +392===== +393===== +394===== +395===== +396===== +397===== +398===== +399===== +400===== +401===== +402===== +403===== +404===== +405===== +406===== +407===== +408===== +409===== +410===== +411===== +412===== +413===== +414===== +415===== +416===== +417===== +418===== +419===== +420===== +421===== +422===== +423===== +424===== +425===== +426===== +427===== +428===== +429===== +430===== +431===== +432===== +433===== +434===== +435===== +436===== +437===== +438===== +439===== +440===== +441===== +442===== +443===== +444===== +445===== +446===== +447===== +448===== +449===== +450===== +451===== +452===== +453===== +454===== +455===== +456===== +457===== +458===== +459===== +460===== +461===== +462===== +463===== +464===== +465===== +466===== +467===== +468===== +469===== +470===== +471===== +472===== +473===== +474===== +475===== +476===== +477===== +478===== +479===== +480===== +481===== +482===== +483===== +484===== +485===== +486===== +487===== +488===== +489===== +490===== +491===== +492===== +493===== +494===== +495===== +496===== +497===== +498===== +499===== +500===== +501===== +502===== +503===== +504===== +505===== +506===== +507===== +508===== +509===== +510===== +511===== +512===== +513===== +514===== +515===== +516===== +517===== +518===== +519===== +520===== +521===== +522===== +523===== +524===== +525===== +526===== +527===== +528===== +529===== +530===== +531===== +532===== +533===== +534===== +535===== +536===== +537===== +538===== +539===== +540===== +541===== +542===== +543===== +544===== +545===== +546===== +547===== +548===== +549===== +550===== +551===== +552===== +553===== +554===== +555===== +556===== +557===== +558===== +559===== +560===== +561===== +562===== +563===== +564===== +565===== +566===== +567===== +568===== +569===== +570===== +571===== +572===== +573===== +574===== +575===== +576===== +577===== +578===== +579===== +580===== +581===== +582===== +583===== +584===== +585===== +586===== +587===== +588===== +589===== +590===== +591===== +592===== +593===== +594===== +595===== +596===== +597===== +598===== +599===== +600===== +601===== +602===== +603===== +604===== +605===== +606===== +607===== +608===== +609===== +610===== +611===== +612===== +613===== +614===== +615===== +616===== +617===== +618===== +619===== +620===== +621===== +622===== +623===== +624===== +625===== +626===== +627===== +628===== +629===== +630===== +631===== +632===== +633===== +634===== +635===== +636===== +637===== +638===== +639===== +640===== +641===== +642===== +643===== +644===== +645===== +646===== +647===== +648===== +649===== +650===== +651===== +652===== +653===== +654===== +655===== +656===== +657===== +658===== +659===== +660===== +661===== +662===== +663===== +664===== +665===== +666===== +667===== +668===== +669===== +670===== +671===== +672===== +673===== +674===== +675===== +676===== +677===== +678===== +679===== +680===== +681===== +682===== +683===== +684===== +685===== +686===== +687===== +688===== +689===== +690===== +691===== +692===== +693===== +694===== +695===== +696===== +697===== +698===== +699===== +700===== +701===== +702===== +703===== +704===== +705===== +706===== +707===== +708===== +709===== +710===== +711===== +712===== +713===== +714===== +715===== +716===== +717===== +718===== +719===== +720===== +721===== +722===== +723===== +724===== +725===== +726===== +727===== +728===== +729===== +730===== +731===== +732===== +733===== +734===== +735===== +736===== +737===== +738===== +739===== +740===== +741===== +742===== +743===== +744===== +745===== +746===== +747===== +748===== +749===== +750===== +751===== +752===== +753===== +754===== +755===== +756===== +757===== +758===== +759===== +760===== +761===== +762===== +763===== +764===== +765===== +766===== +767===== +768===== +769===== +770===== +771===== +772===== +773===== +774===== +775===== +776===== +777===== +778===== +779===== +780===== +781===== +782===== +783===== +784===== +785===== +786===== +787===== +788===== +789===== +790===== +791===== +792===== +793===== +794===== +795===== +796===== +797===== +798===== +799===== +800===== +801===== +802===== +803===== +804===== +805===== +806===== +807===== +808===== +809===== +810===== +811===== +812===== +813===== +814===== +815===== +816===== +817===== +818===== +819===== +820===== +821===== +822===== +823===== +824===== +825===== +826===== +827===== +828===== +829===== +830===== +831===== +832===== +833===== +834===== +835===== +836===== +837===== +838===== +839===== +840===== +841===== +842===== +843===== +844===== +845===== +846===== +847===== +848===== +849===== +850===== +851===== +852===== +853===== +854===== +855===== +856===== +857===== +858===== +859===== +860===== +861===== +862===== +863===== +864===== +865===== +866===== +867===== +868===== +869===== +870===== +871===== +872===== +873===== +874===== +875===== +876===== +877===== +878===== +879===== +880===== +881===== +882===== +883===== +884===== +885===== +886===== +887===== +888===== +889===== +890===== +891===== +892===== +893===== +894===== +895===== +896===== +897===== +898===== +899===== +900===== +901===== +902===== +903===== +904===== +905===== +906===== +907===== +908===== +909===== +910===== +911===== +912===== +913===== +914===== +915===== +916===== +917===== +918===== +919===== +920===== +921===== +922===== +923===== +924===== +925===== +926===== +927===== +928===== +929===== +930===== +931===== +932===== +933===== +934===== +935===== +936===== +937===== +938===== +939===== +940===== +941===== +942===== +943===== +944===== +945===== +946===== +947===== +948===== +949===== +950===== +951===== +952===== +953===== +954===== +955===== +956===== +957===== +958===== +959===== +960===== +961===== +962===== +963===== +964===== +965===== +966===== +967===== +968===== +969===== +970===== +971===== +972===== +973===== +974===== +975===== +976===== +977===== +978===== +979===== +980===== +981===== +982===== +983===== +984===== +985===== +986===== +987===== +988===== +989===== +990===== +991===== +992===== +993===== +994===== +995===== +996===== +997===== +998===== +999===== +1000===== +1001===== +1002===== +1003===== +1004===== diff --git a/test/e2e/test_cases/input_static_file_docker/case.feature b/test/e2e/test_cases/input_static_file_docker/case.feature new file mode 100644 index 0000000000..aa259082b2 --- /dev/null +++ b/test/e2e/test_cases/input_static_file_docker/case.feature @@ -0,0 +1,37 @@ +@input +Feature: input static file docker + Test input_static_file_onetime collect static files from existing container + + @e2e @docker-compose + Scenario: TestInputStaticFileOnetimeFromContainer + Given {docker-compose} environment + Given subcribe data from {grpc} with config + """ + """ + Given loongcollector depends on containers {["container"]} + Given {input-static-file-docker-case} onetime pipeline local config as below + """ + enable: true + global: + ExcutionTimeout: 600 + DefaultLogQueueSize: 10 + inputs: + - Type: input_static_file_onetime + FilePaths: + - /root/test/**/a*.log + MaxDirSearchDepth: 10 + EnableContainerDiscovery: true + ContainerFilters: + IncludeEnv: + STDOUT_SWITCH: "true" + """ + When start docker-compose {input_static_file_docker} + Then there is at least {1000} logs + Then the log tags match kv + """ + "__path__": "^/root/test/a.log$" + """ + Then the log fields match kv + """ + content: "^\\d+====" + """ diff --git a/test/e2e/test_cases/input_static_file_docker/docker-compose.yaml b/test/e2e/test_cases/input_static_file_docker/docker-compose.yaml new file mode 100644 index 0000000000..e6056732f8 --- /dev/null +++ b/test/e2e/test_cases/input_static_file_docker/docker-compose.yaml @@ -0,0 +1,31 @@ +# Copyright 2021 iLogtail Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +version: '3.8' + +services: + container: + image: bash:latest + command: ["sleep", "infinity"] + hostname: container + volumes: + - .:/root/test/ + environment: + - STDOUT_SWITCH=true + healthcheck: + test: ["CMD", "test", "-s", "/root/test/a.log"] + interval: 5s + timeout: 3s + start_period: 5s + retries: 3 diff --git a/test/engine/setup/dockercompose/compose.go b/test/engine/setup/dockercompose/compose.go index 2fcf3d9e14..1dbf173951 100644 --- a/test/engine/setup/dockercompose/compose.go +++ b/test/engine/setup/dockercompose/compose.go @@ -51,6 +51,8 @@ services: - %s:/usr/local/loongcollector/conf/default_flusher.json - %s:/usr/local/loongcollector/conf/continuous_pipeline_config/local - %s:/usr/local/loongcollector/conf/onetime_pipeline_config/local + - %s:/usr/local/loongcollector/log + - %s:/usr/local/loongcollector/data - /:/logtail_host - /var/run/docker.sock:/var/run/docker.sock - /sys/:/sys/ @@ -58,6 +60,7 @@ services: - 18689:18689 environment: - LOGTAIL_FORCE_COLLECT_SELF_TELEMETRY=true + - LOGTAIL_LOG_LEVEL=debug - LOGTAIL_DEBUG_FLAG=true - LOGTAIL_AUTO_PROF=false - LOGTAIL_HTTP_LOAD_CONFIG=true @@ -218,6 +221,14 @@ func (c *ComposeBooter) createComposeFile(ctx context.Context) error { return err } } + // 确保日志目录存在,用于挂载 loongcollector 容器内日志到宿主机便于排查 + if err := os.MkdirAll(config.LogDir, 0750); err != nil { + return err + } + // 确保 data 目录存在,用于挂载 loongcollector 容器内 data 到宿主机便于排查 + if err := os.MkdirAll(config.DataDir, 0750); err != nil { + return err + } _, err := os.Stat(config.CaseHome + config.DockerComposeFileName) var bytes []byte if err != nil { @@ -284,7 +295,7 @@ func (c *ComposeBooter) createComposeFile(ctx context.Context) error { // getLogtailpluginConfig find the docker compose configuration of the loongcollector. func (c *ComposeBooter) getLogtailpluginConfig() map[string]interface{} { cfg := make(map[string]interface{}) - str := fmt.Sprintf(template, config.FlusherFile, config.ConfigDir, config.OnetimeConfigDir) + str := fmt.Sprintf(template, config.FlusherFile, config.ConfigDir, config.OnetimeConfigDir, config.LogDir, config.DataDir) if err := yaml.Unmarshal([]byte(str), &cfg); err != nil { panic(err) }