Add accessLog config (#1048)

HOSTING.md

@@ -81,6 +81,7 @@ Available environment variables:
 - `CUSTOM_LANDING_PAGE_ASSETS`: The custom landing page assets separated by comma(,), default is empty.
 - `CORS_ALLOW_ORIGINS`: The CORS allow origins separated by comma(,), default is allow all origins.
 - `LOG_LEVEL`: The log level, available values are ["debug", "info", "warn", "error"], default is "info".
+- `ACCESS_LOG`: Enable access log, default is `false`.
 - `MINIFY`: Minify the built JS/CSS files, default is `true`.
 - `NPM_QUERY_CACHE_TTL`: The cache TTL for NPM query, default is 10 minutes.
 - `NPM_REGISTRY`: The global NPM registry, default is "https://registry.npmjs.org/".

Makefile

@@ -2,6 +2,7 @@ dev/cli:
 	@go run -tags debug cli/cmd/main.go serve cli/cmd/demo/${app}
 
 dev: config.json
+	@rm -rf .esmd/log
 	@rm -rf .esmd/storage
 	@rm -rf .esmd/esm.db
 	@go run -tags debug main.go --config=config.json

config.example.jsonc

@@ -82,6 +82,10 @@
   // The log level, available values are ["debug", "info", "warn", "error"], default is "info".
   "logLevel": "info",
 
+  // Enable access log, default is disabled.
+  // The access log will be written to the log directory with the name "access-<date>.log".
+  "accessLog": false,
+
   // The cache TTL for npm packages query, default is 600 seconds (10 minutes).
   "npmQueryCacheTTL": 600,
 

server/build.go

@@ -16,6 +16,7 @@ import (
 	"github.com/esm-dev/esm.sh/server/npm_replacements"
 	"github.com/esm-dev/esm.sh/server/storage"
 	esbuild "github.com/evanw/esbuild/pkg/api"
+	"github.com/ije/gox/log"
 	"github.com/ije/gox/set"
 	"github.com/ije/gox/utils"
 )
@@ -30,6 +31,7 @@ const (
 
 type BuildContext struct {
 	npmrc       *NpmRC
+	logger      *log.Logger
 	db          DB
 	storage     storage.Storage
 	esm         EsmPath
@@ -94,7 +96,7 @@ func (ctx *BuildContext) Exists() (meta *BuildMeta, ok bool, err error) {
 	meta, err = withLRUCache(key, func() (*BuildMeta, error) {
 		metadata, err := ctx.db.Get(key)
 		if err != nil {
-			log.Errorf("db.get(%s): %v", key, err)
+			ctx.logger.Errorf("db.get(%s): %v", key, err)
 			return nil, err
 		}
 		if metadata == nil {
@@ -160,7 +162,7 @@ func (ctx *BuildContext) Build() (meta *BuildMeta, err error) {
 	key := ctx.npmrc.zoneId + ":" + ctx.Path()
 	err = ctx.db.Put(key, encodeBuildMeta(meta))
 	if err != nil {
-		log.Errorf("db.put(%s): %v", key, err)
+		ctx.logger.Errorf("db.put(%s): %v", key, err)
 		err = errors.New("db: " + err.Error())
 	}
 	return
@@ -228,7 +230,7 @@ func (ctx *BuildContext) buildModule(analyzeMode bool) (meta *BuildMeta, include
 	}
 
 	if DEBUG && !analyzeMode {
-		log.Debugf(`build(%s): Entry{main: "%s", module: %v, types: "%s"}`, ctx.esm.Specifier(), entry.main, entry.module, entry.types)
+		ctx.logger.Debugf(`build(%s): Entry{main: "%s", module: %v, types: "%s"}`, ctx.esm.Specifier(), entry.main, entry.module, entry.types)
 	}
 
 	isTypesOnly := strings.HasPrefix(ctx.pkgJson.Name, "@types/") || entry.isTypesOnly()
@@ -273,7 +275,7 @@ func (ctx *BuildContext) buildModule(analyzeMode bool) (meta *BuildMeta, include
 		buffer.Write(jsonData)
 		err = ctx.storage.Put(ctx.getSavepath(), buffer)
 		if err != nil {
-			log.Errorf("storage.put(%s): %v", ctx.getSavepath(), err)
+			ctx.logger.Errorf("storage.put(%s): %v", ctx.getSavepath(), err)
 			err = errors.New("storage: " + err.Error())
 			return
 		}
@@ -302,6 +304,7 @@ func (ctx *BuildContext) buildModule(analyzeMode bool) (meta *BuildMeta, include
 		}
 		b := &BuildContext{
 			npmrc:       ctx.npmrc,
+			logger:      ctx.logger,
 			db:          ctx.db,
 			storage:     ctx.storage,
 			esm:         dep,
@@ -327,7 +330,7 @@ func (ctx *BuildContext) buildModule(analyzeMode bool) (meta *BuildMeta, include
 		}
 		err = ctx.storage.Put(ctx.getSavepath(), buf)
 		if err != nil {
-			log.Errorf("storage.put(%s): %v", ctx.getSavepath(), err)
+			ctx.logger.Errorf("storage.put(%s): %v", ctx.getSavepath(), err)
 			err = errors.New("storage: " + err.Error())
 			return
 		}
@@ -1065,7 +1068,7 @@ REBUILD:
 			}
 			name := strings.Split(msg, "\"")[1]
 			if !implicitExternal.Has(name) {
-				log.Warnf("build(%s): implicit external '%s'", ctx.Path(), name)
+				ctx.logger.Warnf("build(%s): implicit external '%s'", ctx.Path(), name)
 				implicitExternal.Add(name)
 				goto REBUILD
 			}
@@ -1097,7 +1100,7 @@ REBUILD:
 	}
 
 	for _, w := range res.Warnings {
-		log.Warnf("esbuild(%s): %s", ctx.Path(), w.Text)
+		ctx.logger.Warnf("esbuild(%s): %s", ctx.Path(), w.Text)
 	}
 
 	imports := set.New[string]()
@@ -1261,6 +1264,7 @@ REBUILD:
 							} else {
 								b := &BuildContext{
 									npmrc:       ctx.npmrc,
+									logger:      ctx.logger,
 									db:          ctx.db,
 									storage:     ctx.storage,
 									esm:         dep,
@@ -1333,7 +1337,7 @@ REBUILD:
 
 			err = ctx.storage.Put(ctx.getSavepath(), finalJS)
 			if err != nil {
-				log.Errorf("storage.put(%s): %v", ctx.getSavepath(), err)
+				ctx.logger.Errorf("storage.put(%s): %v", ctx.getSavepath(), err)
 				err = errors.New("storage: " + err.Error())
 				return
 			}
@@ -1346,7 +1350,7 @@ REBUILD:
 			savePath = strings.TrimSuffix(savePath, path.Ext(savePath)) + ".css"
 			err = ctx.storage.Put(savePath, bytes.NewReader(file.Contents))
 			if err != nil {
-				log.Errorf("storage.put(%s): %v", savePath, err)
+				ctx.logger.Errorf("storage.put(%s): %v", savePath, err)
 				err = errors.New("storage: " + err.Error())
 				return
 			}
@@ -1367,7 +1371,7 @@ REBUILD:
 				if json.NewEncoder(buf).Encode(sourceMap) == nil {
 					err = ctx.storage.Put(ctx.getSavepath()+".map", buf)
 					if err != nil {
-						log.Errorf("storage.put(%s): %v", ctx.getSavepath()+".map", err)
+						ctx.logger.Errorf("storage.put(%s): %v", ctx.getSavepath()+".map", err)
 						err = errors.New("storage: " + err.Error())
 						return
 					}

server/build_analyzer.go

@@ -78,7 +78,7 @@ func (ctx *BuildContext) analyzeSplitting() (err error) {
 					if e == nil && n <= len(a)-1 {
 						ctx.splitting = set.NewReadOnly[string](a[1 : n+1]...)
 						if DEBUG {
-							log.Debugf("build(%s): splitting.txt found with %d shared modules", ctx.esm.Specifier(), ctx.splitting.Len())
+							ctx.logger.Debugf("build(%s): splitting.txt found with %d shared modules", ctx.esm.Specifier(), ctx.splitting.Len())
 						}
 						return true
 					}
@@ -130,6 +130,7 @@ func (ctx *BuildContext) analyzeSplitting() (err error) {
 				esm.SubModuleName = stripEntryModuleExt(exportName)
 				b := &BuildContext{
 					npmrc:       ctx.npmrc,
+					logger:      ctx.logger,
 					db:          ctx.db,
 					storage:     ctx.storage,
 					esm:         esm,
@@ -195,7 +196,7 @@ func (ctx *BuildContext) analyzeSplitting() (err error) {
 				}
 				ctx.splitting = splitting.ReadOnly()
 				if DEBUG {
-					log.Debugf("build(%s): found %d shared modules from %d modules", ctx.esm.Specifier(), shared.Len(), len(refs))
+					ctx.logger.Debugf("build(%s): found %d shared modules from %d modules", ctx.esm.Specifier(), shared.Len(), len(refs))
 				}
 			}
 		}

server/build_queue.go

@@ -98,13 +98,13 @@ func (q *BuildQueue) run(task *BuildTask) {
 	if err == nil {
 		task.ctx.status = "done"
 		if task.ctx.target == "types" {
-			log.Infof("build '%s'(types) done in %v", task.ctx.Path(), time.Since(task.startedAt))
+			task.ctx.logger.Infof("build '%s'(types) done in %v", task.ctx.Path(), time.Since(task.startedAt))
 		} else {
-			log.Infof("build '%s' done in %v", task.ctx.Path(), time.Since(task.startedAt))
+			task.ctx.logger.Infof("build '%s' done in %v", task.ctx.Path(), time.Since(task.startedAt))
 		}
 	} else {
 		task.ctx.status = "error"
-		log.Errorf("build '%s': %v", task.ctx.Path(), err)
+		task.ctx.logger.Errorf("build '%s': %v", task.ctx.Path(), err)
 	}
 
 	q.lock.Lock()

server/build_resolver.go

@@ -758,8 +758,9 @@ func (ctx *BuildContext) resolveExternalModule(specifier string, kind api.Resolv
 		resolvedPath = "/" + dep.Specifier()
 		if subPath == "" || !strings.HasSuffix(subPath, ".json") {
 			b := &BuildContext{
-				npmrc: ctx.npmrc,
-				esm:   dep,
+				npmrc:  ctx.npmrc,
+				logger: ctx.logger,
+				esm:    dep,
 			}
 			err = b.install()
 			if err != nil {
@@ -899,6 +900,7 @@ func (ctx *BuildContext) resloveDTS(entry BuildEntry) (string, error) {
 				}
 				b := &BuildContext{
 					npmrc:       ctx.npmrc,
+					logger:      ctx.logger,
 					esm:         dtsModule,
 					args:        ctx.args,
 					externalAll: ctx.externalAll,
@@ -1079,14 +1081,17 @@ func (ctx *BuildContext) lexer(entry *BuildEntry) (ret *BuildMeta, cjsExports []
 			}
 			return
 		}
-		log.Warnf("fake ES module '%s' of '%s'", entry.main, ctx.pkgJson.Name)
 
 		var cjs cjsModuleLexerResult
 		cjs, err = cjsModuleLexer(ctx, entry.main)
 		if err != nil {
 			return
 		}
 
+		if DEBUG {
+			ctx.logger.Debugf("fake ES module '%s' of '%s'", entry.main, ctx.pkgJson.Name)
+		}
+
 		ret = &BuildMeta{
 			ExportDefault: true,
 			CJS:           true,

server/cjs_module_lexer.go

@@ -19,6 +19,7 @@ import (
 	"time"
 
 	"github.com/ije/gox/set"
+	"github.com/ije/gox/term"
 	"github.com/ije/gox/utils"
 )
 
@@ -62,7 +63,7 @@ func cjsModuleLexer(ctx *BuildContext, cjsEntry string) (ret cjsModuleLexerResul
 	defer func() {
 		if err == nil {
 			if DEBUG {
-				log.Debugf("[cjsModuleLexer] parse %s in %s", path.Join(ctx.esm.PkgName, cjsEntry), time.Since(start))
+				ctx.logger.Debugf("[cjsModuleLexer] parse %s in %s", path.Join(ctx.esm.PkgName, cjsEntry), time.Since(start))
 			}
 			if !existsFile(cacheFileName) {
 				ensureDir(path.Dir(cacheFileName))
@@ -173,7 +174,7 @@ func installCommonJSModuleLexer() (err error) {
 	}
 
 	if DEBUG {
-		log.Debugf("downloading %s...", path.Base(url))
+		fmt.Println(term.Dim(fmt.Sprintf("Downloading %s...", path.Base(url))))
 	}
 
 	res, err := http.Get(url)

server/config.go

@@ -25,6 +25,7 @@ var (
 type Config struct {
 	Port                uint16                 `json:"port"`
 	TlsPort             uint16                 `json:"tlsPort"`
+	LegacyServer        string                 `json:"legacyServer"` // normally you don't need to set this
 	CustomLandingPage   LandingPageOptions     `json:"customLandingPage"`
 	WorkDir             string                 `json:"workDir"`
 	CorsAllowOrigins    []string               `json:"corsAllowOrigins"`
@@ -36,6 +37,7 @@ type Config struct {
 	CacheRawFile        bool                   `json:"cacheRawFile"`
 	LogDir              string                 `json:"logDir"`
 	LogLevel            string                 `json:"logLevel"`
+	AccessLog           bool                   `json:"accessLog"`
 	NpmRegistry         string                 `json:"npmRegistry"`
 	NpmToken            string                 `json:"npmToken"`
 	NpmUser             string                 `json:"npmUser"`
@@ -45,7 +47,6 @@ type Config struct {
 	MinifyRaw           json.RawMessage        `json:"minify"`
 	SourceMapRaw        json.RawMessage        `json:"sourceMap"`
 	CompressRaw         json.RawMessage        `json:"compress"`
-	LegacyServer        string                 `json:"legacyServer"` // normally you don't need to set this
 	Minify              bool                   `json:"-"`
 	SourceMap           bool                   `json:"-"`
 	Compress            bool                   `json:"-"`
@@ -197,6 +198,9 @@ func normalizeConfig(config *Config) {
 			config.LogLevel = "info"
 		}
 	}
+	if !config.AccessLog {
+		config.AccessLog = os.Getenv("ACCESS_LOG") == "true"
+	}
 	if config.NpmRegistry != "" {
 		if isHttpSepcifier(config.NpmRegistry) {
 			config.NpmRegistry = strings.TrimRight(config.NpmRegistry, "/") + "/"

server/dts_transform.go

@@ -21,7 +21,7 @@ func (ctx *BuildContext) transformDTS(dts string) error {
 		return err
 	}
 	if DEBUG {
-		log.Debugf("transform dts '%s'(%d related dts files) in %v", dts, n, time.Since(start))
+		ctx.logger.Debugf("transform dts '%s'(%d related dts files) in %v", dts, n, time.Since(start))
 	}
 	return nil
 }
@@ -55,7 +55,6 @@ func transformDTS(ctx *BuildContext, dts string, buildArgsPrefix string, marker
 			if isEntry {
 				err = fmt.Errorf("types not found")
 			} else {
-				log.Warnf("dts not found: %s", dtsFilePath)
 				err = nil
 			}
 		}
@@ -215,6 +214,7 @@ func transformDTS(ctx *BuildContext, dts string, buildArgsPrefix string, marker
 		args := BuildArgs{}
 		b := &BuildContext{
 			npmrc:  ctx.npmrc,
+			logger: ctx.logger,
 			esm:    dtsModule,
 			args:   args,
 			target: "types",

server/loader.go

@@ -14,6 +14,7 @@ import (
 	"sync"
 
 	esbuild "github.com/evanw/esbuild/pkg/api"
+	"github.com/ije/gox/term"
 	"github.com/ije/gox/utils"
 )
 
@@ -145,7 +146,7 @@ func installLoaderRuntime() (err error) {
 	}
 
 	if DEBUG {
-		log.Debugf("downloading %s...", path.Base(url))
+		fmt.Println(term.Dim(fmt.Sprintf("Downloading %s...", path.Base(url))))
 	}
 
 	res, err := http.Get(url)

server/loader_implements.go

@@ -13,6 +13,7 @@ import (
 
 	"github.com/esm-dev/esm.sh/server/common"
 	"github.com/ije/gox/sync"
+	"github.com/ije/gox/term"
 )
 
 var (
@@ -26,7 +27,9 @@ func transformSvelte(npmrc *NpmRC, svelteVersion string, filename string, code s
 	once, _ := compileSyncMap.LoadOrStore(loaderExecPath, &sync.Once{})
 	err = once.(*sync.Once).Do(func() (err error) {
 		if !existsFile(loaderExecPath) {
-			log.Debug("compiling svelte loader...")
+			if DEBUG {
+				fmt.Println(term.Dim("Compiling svelte loader..."))
+			}
 			err = compileSvelteLoader(npmrc, svelteVersion, loaderExecPath)
 		}
 		return
@@ -100,7 +103,9 @@ func generateUnoCSS(npmrc *NpmRC, configCSS string, content string) (output *Loa
 	once, _ := compileSyncMap.LoadOrStore(loaderExecPath, &sync.Once{})
 	err = once.(*sync.Once).Do(func() (err error) {
 		if !existsFile(loaderExecPath) {
-			log.Debug("compiling unocss loader...")
+			if DEBUG {
+				fmt.Println(term.Dim("Compiling unocss loader..."))
+			}
 			err = compileUnocssLoader(npmrc, loaderVersion, loaderExecPath)
 		}
 		return
@@ -227,7 +232,9 @@ func transformVue(npmrc *NpmRC, vueVersion string, filename string, code string)
 	once, _ := compileSyncMap.LoadOrStore(loaderExecPath, &sync.Once{})
 	err = once.(*sync.Once).Do(func() (err error) {
 		if !existsFile(loaderExecPath) {
-			log.Debug("compiling vue loader...")
+			if DEBUG {
+				fmt.Println(term.Dim("Compiling vue loader..."))
+			}
 			err = compileVueLoader(npmrc, vueVersion, loaderVersion, loaderExecPath)
 		}
 		return