From 7d34927be5cb679fcbdef93c145afc321e803093 Mon Sep 17 00:00:00 2001
From: liguozhuang <liguozhuang@jiagouyun.com>
Date: Sat, 2 Nov 2024 11:53:40 +0800
Subject: [PATCH] =?UTF-8?q?=E6=97=A5=E5=BF=97=E9=87=87=E9=9B=86=E6=B7=BB?=
 =?UTF-8?q?=E5=8A=A0=E6=9C=80=E5=A4=A7=E6=96=87=E4=BB=B6=E6=95=B0=E9=99=90?=
 =?UTF-8?q?=E5=88=B6?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 internal/config/env.go            | 14 ++++++++++++++
 internal/export/non_input_docs.go |  2 ++
 internal/tailer/tailer_single.go  | 13 ++++++++++++-
 3 files changed, 28 insertions(+), 1 deletion(-)

diff --git a/internal/config/env.go b/internal/config/env.go
index 874d48888f..9726cdb226 100644
--- a/internal/config/env.go
+++ b/internal/config/env.go
@@ -20,6 +20,7 @@ import (
 	"gitlab.jiagouyun.com/cloudcare-tools/datakit/internal/io"
 	"gitlab.jiagouyun.com/cloudcare-tools/datakit/internal/io/dataway"
 	"gitlab.jiagouyun.com/cloudcare-tools/datakit/internal/io/filter"
+	"gitlab.jiagouyun.com/cloudcare-tools/datakit/internal/tailer"
 )
 
 func (c *Config) loadConfdEnvs() {
@@ -646,6 +647,18 @@ func (c *Config) loadHTTPAPIEnvs() {
 	}
 }
 
+func (c *Config) setOthers() {
+	if v := datakit.GetEnv("ENV_LOGGING_MAX_OPEN_FILES"); v != "" {
+		n, err := strconv.ParseInt(v, 10, 64)
+		if err != nil {
+			l.Warnf("invalid env key ENV_LOGGING_MAX_OPEN_FILES, value %s, err: %s ignored", v, err)
+		} else {
+			l.Infof("set ENV_LOGGING_MAX_OPEN_FILES to %d", n)
+			tailer.MaxOpenFiles = n
+		}
+	}
+}
+
 func (c *Config) setNodenameAsHostname() {
 	for _, x := range []string{
 		"ENV_K8S_NODE_NAME",
@@ -708,6 +721,7 @@ func (c *Config) LoadEnvs() error {
 	c.loadPointPoolEnvs()
 
 	c.setNodenameAsHostname()
+	c.setOthers()
 
 	// Don't Add to ElectionTags.
 	if v := datakit.GetEnv("ENV_CLUSTER_NAME_K8S"); v != "" {
diff --git a/internal/export/non_input_docs.go b/internal/export/non_input_docs.go
index 67d90e9c0e..bcdc409682 100644
--- a/internal/export/non_input_docs.go
+++ b/internal/export/non_input_docs.go
@@ -633,6 +633,8 @@ func envOthers() []*inputs.ENVInfo {
 		{ENVName: "ENV_PIPELINE_DISABLE_APPEND_RUN_INFO", Type: doc.Boolean, Default: "`false`", Desc: "Disable appending the Pipeline run info", DescZh: "禁用追加 Pipeline 运行信息"},
 		{ENVName: "ENV_CRYPTO_AES_KEY", Type: doc.String, Example: "`0123456789abcdef`", Desc: "The crypto key(len 16)", DescZh: "AES 加解密的 key 长度是 16"},
 		{ENVName: "ENV_CRYPTO_AES_KEY_FILE", Type: doc.String, Example: "`/usr/local/datakit/enc4mysql`", Desc: "File path for storing AES encryption and decryption key", DescZh: "AES 加解密的 key 存放的文件路径"},
+
+		{ENVName: "ENV_LOGGING_MAX_OPEN_FILES", Type: doc.Int, Example: "`1000`", Desc: "Specify the maximum number of open files for logging collection, if the value is -1 then there is no limit, default 500", DescZh: "指定日志采集的最大文件个数，如果值是 -1 则没有限制，默认值 500"},
 	}
 
 	for idx := range infos {
diff --git a/internal/tailer/tailer_single.go b/internal/tailer/tailer_single.go
index 5681091ac7..135afeaa24 100644
--- a/internal/tailer/tailer_single.go
+++ b/internal/tailer/tailer_single.go
@@ -12,6 +12,7 @@ import (
 	"fmt"
 	"io"
 	"os"
+	"sync/atomic"
 	"time"
 
 	"github.com/GuanceCloud/cliutils/logger"
@@ -36,6 +37,11 @@ const (
 	checkInterval        = time.Second * 3
 )
 
+var (
+	MaxOpenFiles     int64 = 500
+	currentOpenFiles atomic.Int64
+)
+
 type Single struct {
 	opt *option
 
@@ -59,8 +65,11 @@ type Single struct {
 }
 
 func NewTailerSingle(filepath string, opts ...Option) (*Single, error) {
-	_ = logtail.InitDefault()
+	if MaxOpenFiles != -1 && currentOpenFiles.Load() > MaxOpenFiles {
+		return nil, fmt.Errorf("too many open files, limit %d", MaxOpenFiles)
+	}
 
+	_ = logtail.InitDefault()
 	c := defaultOption()
 	for _, opt := range opts {
 		opt(c)
@@ -79,6 +88,7 @@ func NewTailerSingle(filepath string, opts ...Option) (*Single, error) {
 	}
 
 	openfileVec.WithLabelValues(t.opt.mode.String()).Inc()
+	currentOpenFiles.Add(1)
 	return t, nil
 }
 
@@ -164,6 +174,7 @@ func (t *Single) Close() {
 	t.closeFile()
 
 	openfileVec.WithLabelValues(t.opt.mode.String()).Dec()
+	currentOpenFiles.Add(-1)
 	t.log.Infof("closing: file %s", t.filepath)
 }