diff --git a/.chglog/CHANGELOG.tpl.md b/.chglog/CHANGELOG.tpl.md deleted file mode 100755 index 4266cbf..0000000 --- a/.chglog/CHANGELOG.tpl.md +++ /dev/null @@ -1,38 +0,0 @@ -{{ range .Versions }} - -## {{ if .Tag.Previous }}[{{ .Tag.Name }}]({{ $.Info.RepositoryURL }}/compare/{{ .Tag.Previous.Name }}...{{ .Tag.Name }}){{ else }}{{ .Tag.Name }}{{ end }} - {{ datetime "2006-01-02" .Tag.Date }} - -{{ range .CommitGroups -}} -### {{ .Title }} - -{{ range .Commits -}} -* [{{ .Hash.Short }}] {{ if .Scope }}**{{ .Scope }}** {{ end }}{{ .Subject }} -{{ end }} -{{ end -}} - -{{- if .RevertCommits -}} -### Reverts - -{{ range .RevertCommits -}} -* {{ .Revert.Header }} -{{ end }} -{{ end -}} - -{{- if .MergeCommits -}} -### Pull Requests - -{{ range .MergeCommits -}} -* {{ .Header }} -{{ end }} -{{ end -}} - -{{- if .NoteGroups -}} -{{ range .NoteGroups -}} -### {{ .Title }} - -{{ range .Notes }} -{{ .Body }} -{{ end }} -{{ end -}} -{{ end -}} -{{ end -}} diff --git a/.chglog/config.yml b/.chglog/config.yml deleted file mode 100755 index 2cc90ce..0000000 --- a/.chglog/config.yml +++ /dev/null @@ -1,28 +0,0 @@ -style: github -template: CHANGELOG.tpl.md -info: - title: CHANGELOG - repository_url: http://gitlab.wellcloud.cc:30000/wangdd/siphub -options: - commits: - filters: - Type: - - feat - - fix - - perf - - refactor - commit_groups: - # title_maps: - # feat: Features - # fix: Bug Fixes - # perf: Performance Improvements - # refactor: Code Refactoring - header: - pattern: "^(\\w*)(?:\\(([\\w\\$\\.\\-\\*\\s]*)\\))?\\:\\s(.*)$" - pattern_maps: - - Type - - Scope - - Subject - notes: - keywords: - - BREAKING CHANGE diff --git a/.dockerignore b/.dockerignore index 40f44d5..b6ebbea 100644 --- a/.dockerignore +++ b/.dockerignore @@ -3,7 +3,7 @@ logs tmp/ build.sh -siphub +sipgrep .air.toml .gitignore CHANGELOG.md diff --git a/.gitignore b/.gitignore index 5002b8a..c5f1095 100644 --- a/.gitignore +++ b/.gitignore @@ -1,10 +1,11 @@ node_modules *.heapsnapshot *.log -siphub +sipgrep tmp .DS_Store heplify heplify.log.* run -.air* \ No newline at end of file +.air.toml +demo \ No newline at end of file diff --git a/Makefile b/Makefile index d0a56cf..2bb18ab 100644 --- a/Makefile +++ b/Makefile @@ -1,5 +1,5 @@ version=$(shell cat VERSION) -image_name="wangduanduan/siphub-go:$(version)" +image_name="wangduanduan/sipgrep-go:$(version)" DBAddr= DBName= @@ -23,12 +23,14 @@ fmt: changelog: git-chglog -o CHANGELOG.md run: - -docker rm -f siphub-go; + -docker rm -f sipgrep-go; docker run -d \ -p 3000:3000 \ -p 9060:9060/udp \ -e DBAddr="$(DBAddr)" \ -e DBName="$(DBName)" \ -e DBUserPasswd="$(DBUserPasswd)" \ - --name siphub-go \ - harbor:5000/wecloud/siphub-go:$(image_name) + --name sipgrep-go \ + harbor:5000/wecloud/sipgrep-go:$(image_name) +t1: + http --verbose localhost:3000/api/v1/call BeginTime=="2023-10-31 00:00:00" EndTime=="2023-10-31 23:59:59" diff --git a/README.md b/README.md index b8cb558..9016ada 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # V23 Roadmap, under developing -- siphub-go merge into siphub-ui, so there are no siphub-ui or siphub-go, just one siphub. +- sipgrep-go merge into sipgrep-ui, so there are no sipgrep-ui or sipgrep-go, just one sipgrep. - using react + ant design as ui frame - using golang fiber as the web server, no nodejs - using [SequenceDiagram](https://github.com/davidje13/SequenceDiagram) to draw sip sequence dragram @@ -52,9 +52,9 @@ this is the first version ![](./img/import.jpg) -# siphub 组件 -- siphub-go: 负责处理hep消息,写入数据库 -- siphub-ui: 负责web界面展示,数据搜索 +# sipgrep 组件 +- sipgrep-go: 负责处理hep消息,写入数据库 +- sipgrep-ui: 负责web界面展示,数据搜索 # Golang版本 @@ -64,36 +64,36 @@ this is the first version ## mysql安装 -先准备一个mysql数据库,并自行创建一个名为siphub的数据库。 +先准备一个mysql数据库,并自行创建一个名为sipgrep的数据库。 -## siphub-go +## sipgrep-go -docker 安装siphub-go +docker 安装sipgrep-go ```bash docker run -d \ - --name siphub-go \ + --name sipgrep-go \ -p 3000:3000 \ -p 9060:9060/udp \ -e DBAddr="localhost" \ -e DBUserPasswd="root:password" \ - -e DBName="siphub" \ + -e DBName="sipgrep" \ -e LogLevel="info" \ -e HeaderUIDName="X-UID" \ - wangduanduan/siphub-go:22.05.02 + wangduanduan/sipgrep-go:22.05.02 ``` - 3000/HTTP 端口 - GET /metrics/prometheus 提供普罗米修斯统计的监控接口 - 9060/UDP hep消息接收端口 -- siphub-go环境变量说明 +- sipgrep-go环境变量说明 ``` // UDP监听端口 UDPListenPort int `env:"UDPListenPort" envDefault:"9060"` // 最大UDP包的长度 - MaxPackgeLength int `env:"MaxPackgeLength" envDefault:"4096"` + MaxPacketLength int `env:"MaxPacketLength" envDefault:"4096"` // UDP读取超时秒数 MaxReadTimeoutSeconds int `env:"MaxReadTimeoutSecond" envDefault:"5"` // 日志级别 @@ -107,7 +107,7 @@ docker 安装siphub-go // 丢弃的方法,方法之间用英文逗号隔开 DiscardMethods string `env:"DiscardMethods" envDefault:"OPTIONS"` // 最小的UDP包长度,比这个小的会丢弃 - MinPackgeLength int `env:"MinPackgeLength" envDefault:"24"` + MinPacketLength int `env:"MinPacketLength" envDefault:"24"` // 数据库连接数 SqlMaxOpenConn int `env:"SqlMaxOpenConn" envDefault:"24"` // 数据库用户名和密码 @@ -115,7 +115,7 @@ docker 安装siphub-go // 数据库地址 DBAddr string `env:"DBAddr" envDefault:"localhost"` // 数据库名称 - DBName string `env:"DBName" envDefault:"siphub"` + DBName string `env:"DBName" envDefault:"sipgrep"` // 被叫号码从哪个地方抽取,RURI 或者 TO CalleeFrom string `env:"CalleeFrom" envDefault:"RURI"` @@ -125,36 +125,36 @@ docker 安装siphub-go ``` -## siphub-ui +## sipgrep-ui -docker运行siphub-ui +docker运行sipgrep-ui - dbHost="localhost" 数据库地址 - dbUser="root" 数据库用户 - dbPwd="some-password" 数据库密码 -- dbName="siphub" 数据库名 +- dbName="sipgrep" 数据库名 - logLevel="info" 日志级别 - dataKeepDays="2" 数据保存多少天 ```bash docker run -d \ - --name siphub-ui \ + --name sipgrep-ui \ -p 8080:8080 \ -e NODE_ENV="production" \ -e dbHost="localhost" \ -e dbUser="root" \ -e dbPwd="some-password" \ - -e dbName="siphub" \ + -e dbName="sipgrep" \ -e logLevel="info" \ -e dataKeepDays="2" \ - wangduanduan/siphub-ui:22.03.03 + wangduanduan/sipgrep-ui:22.03.03 ``` - 8080/HTTP 端口 提供Web查询和展示界面 # 集成 -## OpenSIPS集成 +## OpenSIPS 2X集成 test witch OpenSIPS 2.4 ```bash @@ -162,7 +162,7 @@ test witch OpenSIPS 2.4 listen=hep_udp:your_ip:9061 loadmodule "proto_hep.so" -# replace SIP_HUB_IP_PORT with siphub‘s ip:port +# replace SIP_HUB_IP_PORT with sipgrep‘s ip:port modparam("proto_hep", "hep_id","[hep_dst] SIP_HUB_IP_PORT;transport=udp;version=3") loadmodule "siptrace.so" modparam("siptrace", "trace_id","[tid]uri=hep:hep_dst") @@ -173,16 +173,37 @@ if(!is_method("REGISTER") && !has_totag()){ } ``` +## OpenSIPS 3.x + +``` +socket=hep_udp:127.0.0.1:9060 +loadmodule "proto_hep.so" +modparam("proto_hep", "hep_id","[hid] SIPGREP_IP:SIPGREP_PORT;transport=udp;version=3") +loadmodule "tracer.so" +modparam("tracer", "trace_id","[tid]uri=hep:hid") + + +route { + ... + if (has_totag()) { + route(r_seq_request); + } else { + trace("tid", "d", "sip"); + } + ... +} +``` + ## FreeSWITCH集成 fs version 版本要高于 1.6.8+ 编辑: sofia.conf.xml -用真实的siphub ip:port替换SIP_HUB_IP_PORT +用真实的sipgrep ip:port替换SIPGREP_IP:SIPGREP_PORT ``` - + ``` ```shell @@ -213,7 +234,7 @@ heplify是一个go语言开发的,基于网卡抓包的方式,捕获sip消 - -i 指定网卡。需要更具机器真实网卡进行修改 - -m SIP 指定抓SIP消息 -- -hs 指定siphub-go的地址。需要根据siphub-go的真实地址进行修改 +- -hs 指定sipgrep-go的地址。需要根据sipgrep-go的真实地址进行修改 - -p 指定生成日志文件的位置 - -dim 排除某些类型的SIP包,例如排除OPTIONS和REGISTER注册的包 - -pr 指定抓包的端口范围。 @@ -232,16 +253,16 @@ nohup ./heplify -i eno1 \ ## 数据保留策略 - 所有新的数据,会插入到records表。 -- 每天凌晨 00:01:00, records表会被重命名为siphub_old_day_YYYYMMDD, 然后会新建一个records表 -- 基于siphub-ui的dataKeepDays环境变量,超过最大保留天数的表,会被删除历史的表 +- 每天凌晨 00:01:00, records表会被重命名为sipgrep_old_day_YYYYMMDD, 然后会新建一个records表 +- 基于sipgrep-ui的dataKeepDays环境变量,超过最大保留天数的表,会被删除历史的表 -# siphub-go内存问题 与 mysql写入速度 +# sipgrep-go内存问题 与 mysql写入速度 -在生产环境,有观察到siphub-go的内存一直上涨,最终定位到原因是数据插入的比较慢。 +在生产环境,有观察到sipgrep-go的内存一直上涨,最终定位到原因是数据插入的比较慢。 -一般来说,siphub-go收到的每秒消息量,估计是每秒呼叫量的10-20倍。 也就是说,假如每秒呼叫量,即CPS是100,那么每秒siphub-go收到的消息量估计在1000-2000条sip消息。 +一般来说,sipgrep-go收到的每秒消息量,估计是每秒呼叫量的10-20倍。 也就是说,假如每秒呼叫量,即CPS是100,那么每秒sipgrep-go收到的消息量估计在1000-2000条sip消息。 -siphub-go不是每收到一条消息,就做一次数据库插入。而是累积到MaxBatchItems的数量之后,再执行插入。如果把MaxBatchItems设置为1000,那么两千条消息实际上只需要做两次插入。 +sipgrep-go不是每收到一条消息,就做一次数据库插入。而是累积到MaxBatchItems的数量之后,再执行插入。如果把MaxBatchItems设置为1000,那么两千条消息实际上只需要做两次插入。 所以,你的呼叫量越大。就需要设置较大的MaxBatchItems。 @@ -256,4 +277,4 @@ mysql 数据库配置要求 # 关于处理能力 -我们观察在生产环境,siphub每天的表的数据量+索引所占用的空间大概是80G左右,具体视呼叫量而定。 +我们观察在生产环境,sipgrep每天的表的数据量+索引所占用的空间大概是80G左右,具体视呼叫量而定。 diff --git a/cspell.json b/cspell.json new file mode 100644 index 0000000..7c17c2b --- /dev/null +++ b/cspell.json @@ -0,0 +1,19 @@ +// cSpell Settings +{ + // Version of the setting file. Always 0.2 + "version": "0.2", + // language - current active spelling language + "language": "en", + // words - list of words to be always considered correct + "words": [ + "antd", + "Opensips", + "Infof", + ], + // flagWords - list of words to be always considered incorrect + // This is useful for offensive words and common spelling errors. + // For example "hte" should be "the" + "flagWords": [ + "hte" + ] +} \ No newline at end of file diff --git a/error.txt b/error.txt new file mode 100644 index 0000000..476e85b --- /dev/null +++ b/error.txt @@ -0,0 +1,8 @@ +2023-10-31T11:00:27.938+0800 INFO msg/msg.go:32 200 8011cmbc.poc->8011cmbc.poc +panic: runtime error: index out of range [0] with length 0 + +goroutine 8 [running]: +sipgrep/pkg/mysql.BatchSaveInit() + /Users/wangduanduan/github/siphub/pkg/mysql/record.go:65 +0x298 +created by main.main in goroutine 1 + /Users/wangduanduan/github/siphub/main.go:22 +0x7c \ No newline at end of file diff --git a/go.mod b/go.mod index 66993a6..10dd0e4 100644 --- a/go.mod +++ b/go.mod @@ -1,12 +1,15 @@ -module siphub +module sipgrep go 1.16 require ( github.com/andybalholm/brotli v1.0.6 // indirect github.com/caarlos0/env/v6 v6.10.1 + github.com/gabriel-vasile/mimetype v1.4.3 // indirect + github.com/go-playground/validator/v10 v10.15.5 // indirect github.com/go-sql-driver/mysql v1.7.1 // indirect github.com/gofiber/fiber/v2 v2.50.0 + github.com/golang-module/carbon/v2 v2.2.11 // indirect github.com/klauspost/compress v1.17.2 // indirect github.com/mattn/go-isatty v0.0.20 // indirect github.com/prometheus/client_golang v1.17.0 diff --git a/go.sum b/go.sum index 7ea3f07..5b8fc5d 100644 --- a/go.sum +++ b/go.sum @@ -678,6 +678,9 @@ github.com/envoyproxy/protoc-gen-validate v0.9.1/go.mod h1:OKNgG7TCp5pF4d6XftA0+ github.com/envoyproxy/protoc-gen-validate v0.10.0/go.mod h1:DRjgyB0I43LtJapqN6NiRwroiAU2PaFuvk/vjgh61ss= github.com/fogleman/gg v1.2.1-0.20190220221249-0403632d5b90/go.mod h1:R/bRT+9gY/C5z7JzPU0zXsXHKM4/ayA+zqcVNZzPa1k= github.com/fogleman/gg v1.3.0/go.mod h1:R/bRT+9gY/C5z7JzPU0zXsXHKM4/ayA+zqcVNZzPa1k= +github.com/gabriel-vasile/mimetype v1.4.2/go.mod h1:zApsH/mKG4w07erKIaJPFiX0Tsq9BFQgN3qGY5GnNgA= +github.com/gabriel-vasile/mimetype v1.4.3 h1:in2uUcidCuFcDKtdcBxlR0rJ1+fsokWf+uqxgUFjbI0= +github.com/gabriel-vasile/mimetype v1.4.3/go.mod h1:d8uq/6HKRL6CGdk+aubisF/M5GcPfT7nKyLpA0lbSSk= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/go-fonts/dejavu v0.1.0/go.mod h1:4Wt4I4OU2Nq9asgDCteaAaWZOV24E+0/Pwo0gppep4g= github.com/go-fonts/latin-modern v0.2.0/go.mod h1:rQVLdDMK+mK1xscDwsqM5J8U2jrRa3T0ecnM9pNujks= @@ -700,6 +703,13 @@ github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG github.com/go-logfmt/logfmt v0.5.1/go.mod h1:WYhtIu8zTZfxdn5+rREduYbwxfcBr/Vr6KEVveWlfTs= github.com/go-pdf/fpdf v0.5.0/go.mod h1:HzcnA+A23uwogo0tp9yU+l3V+KXhiESpt1PMayhOh5M= github.com/go-pdf/fpdf v0.6.0/go.mod h1:HzcnA+A23uwogo0tp9yU+l3V+KXhiESpt1PMayhOh5M= +github.com/go-playground/assert/v2 v2.2.0/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4= +github.com/go-playground/locales v0.14.1 h1:EWaQ/wswjilfKLTECiXz7Rh+3BjFhfDFKv/oXslEjJA= +github.com/go-playground/locales v0.14.1/go.mod h1:hxrqLVvrK65+Rwrd5Fc6F2O76J/NuW9t0sjnWqG1slY= +github.com/go-playground/universal-translator v0.18.1 h1:Bcnm0ZwsGyWbCzImXv+pAJnYK9S473LQFuzCbDbfSFY= +github.com/go-playground/universal-translator v0.18.1/go.mod h1:xekY+UJKNuX9WP91TpwSH2VMlDf28Uj24BCp08ZFTUY= +github.com/go-playground/validator/v10 v10.15.5 h1:LEBecTWb/1j5TNY1YYG2RcOUN3R7NLylN+x8TTueE24= +github.com/go-playground/validator/v10 v10.15.5/go.mod h1:9iXMNT7sEkjXb0I+enO7QXmzG6QCsPWY4zveKFVRSyU= github.com/go-sql-driver/mysql v1.7.0/go.mod h1:OXbVy3sEdcQ2Doequ6Z5BW6fXNQTmx+9S1MCJN5yJMI= github.com/go-sql-driver/mysql v1.7.1 h1:lUIinVbN1DY0xBg0eMOzmmtGoHwWBbvnWubQUrtU8EI= github.com/go-sql-driver/mysql v1.7.1/go.mod h1:OXbVy3sEdcQ2Doequ6Z5BW6fXNQTmx+9S1MCJN5yJMI= @@ -708,6 +718,8 @@ github.com/goccy/go-json v0.9.11/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MG github.com/gofiber/fiber/v2 v2.50.0 h1:ia0JaB+uw3GpNSCR5nvC5dsaxXjRU5OEu36aytx+zGw= github.com/gofiber/fiber/v2 v2.50.0/go.mod h1:21eytvay9Is7S6z+OgPi7c7n4++tnClWmhpimVHMimw= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= +github.com/golang-module/carbon/v2 v2.2.11 h1:hpLGEoufD980hIe+CwH9WZERn2/jZekr+WULjFHAUKM= +github.com/golang-module/carbon/v2 v2.2.11/go.mod h1:XDALX7KgqmHk95xyLeaqX9/LJGbfLATyruTziq68SZ8= github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0/go.mod h1:E/TSTwGwJL78qG/PmXZO1EjYhfJinVAhrmmHX6Z8B9k= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/glog v1.0.0/go.mod h1:EWib/APOK0SL3dFbYqvxE3UYd8E6s1ouQ7iEp/0LWV4= @@ -859,6 +871,8 @@ github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/leodido/go-urn v1.2.4 h1:XlAE/cm/ms7TE/VMVoduSpNBoyc2dOxHs5MZSwAN63Q= +github.com/leodido/go-urn v1.2.4/go.mod h1:7ZrI8mTSeBSHl/UaRyKQW1qZeMgak41ANeCNaVckg+4= github.com/lyft/protoc-gen-star v0.6.0/go.mod h1:TGAoBVkt8w7MPG72TrKIu85MIdXwDuzJYeZuUPFPNwA= github.com/lyft/protoc-gen-star v0.6.1/go.mod h1:TGAoBVkt8w7MPG72TrKIu85MIdXwDuzJYeZuUPFPNwA= github.com/lyft/protoc-gen-star/v2 v2.0.1/go.mod h1:RcCdONR2ScXaYnQC5tUzxzlpA3WVYF7/opLeUgcQs/o= @@ -1022,6 +1036,7 @@ golang.org/x/crypto v0.1.0/go.mod h1:RecgLatLF4+eUMCP1PoPZQb+cVrJcOPbHkTkbkB9sbw golang.org/x/crypto v0.7.0/go.mod h1:pYwdfH91IfpZVANVyUOhSIPZaFoJGxTFbZhFTx+dXZU= golang.org/x/crypto v0.9.0/go.mod h1:yrmDGqONDYtNj3tH8X9dzUun2m2lzPa9ngI6/RUPGR0= golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliYc= +golang.org/x/crypto v0.14.0 h1:wBqGXzWJW6m1XrIKlAH0Hs1JJ7+9KBwnIO8v66Q9cHc= golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf4= golang.org/x/exp v0.0.0-20180321215751-8460e604b9de/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20180807140117-3d87b88a115f/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= @@ -1145,6 +1160,7 @@ golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc= golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns= golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= golang.org/x/net v0.15.0/go.mod h1:idbUs1IY1+zTqbi8yxTbhexhEEk5ur9LInksu6HrEpk= +golang.org/x/net v0.17.0 h1:pVaXccu2ozPjCXewfr1S7xza/zcXTity9cCdXQYSjIM= golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -1316,6 +1332,7 @@ golang.org/x/text v0.6.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.8.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= +golang.org/x/text v0.13.0 h1:ablQoSUd0tRdKxZewP80B+BaqeKJuVhuRxj/dkrun3k= golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= diff --git a/main.go b/main.go index 55f3c38..6f16707 100644 --- a/main.go +++ b/main.go @@ -1,77 +1,28 @@ package main import ( - "net" - "net/http" - "siphub/pkg/env" - "siphub/pkg/log" - "siphub/pkg/msg" - "siphub/pkg/mysql" - "siphub/pkg/prom" - "time" + "sipgrep/pkg/env" + "sipgrep/pkg/hepserver" + "sipgrep/pkg/log" + "sipgrep/pkg/mysql" + "sipgrep/pkg/route" - "github.com/prometheus/client_golang/prometheus" - "github.com/prometheus/client_golang/prometheus/promhttp" + "github.com/gofiber/fiber/v2" ) -const MinRawPacketLenth = 105 - func main() { - mysql.Connect(env.Conf.DBUserPasswd, env.Conf.DBAddr, env.Conf.DBName) + mysql.Connect(env.Conf.DBUser+":"+env.Conf.DBPasswd, env.Conf.DBAddr, env.Conf.DBName) go mysql.BatchSaveInit() - go createHepServer() - - app := http.NewServeMux() - - app.Handle("/metrics/prometheus", promhttp.Handler()) - log.Infof("app listen on :3000") - - err := http.ListenAndServe(":3000", app) - - if err != nil { - log.Fatalf("app listen error: %v", err) - } -} + go hepserver.CreateHepServer() -func createHepServer() { - conn, err := net.ListenUDP("udp", &net.UDPAddr{Port: env.Conf.UDPListenPort}) + app := fiber.New() - if err != nil { - log.Fatalf("Udp Service listen report udp fail:%v", err) - } - log.Infof("create udp success") + api := app.Group("/api") - defer conn.Close() - var data = make([]byte, env.Conf.MaxPackgeLength) - var raw []byte - for { - conn.SetDeadline(time.Now().Add(time.Duration(env.Conf.MaxReadTimeoutSeconds) * time.Second)) - n, remoteAddr, err := conn.ReadFromUDP(data) + v1 := api.Group("/v1") + v1.Get("/call", route.Search) - if err != nil { - if opErr, ok := err.(*net.OpError); ok && opErr.Timeout() { - continue - } else { - prom.MsgCount.With(prometheus.Labels{"type": "read_udp_error"}) - log.Errorf("read udp error: %v, from %v", err, remoteAddr) - } - } - - prom.MsgCount.With(prometheus.Labels{"type": "all_received_packet"}).Inc() - - if n < MinRawPacketLenth { - prom.MsgCount.With(prometheus.Labels{"type": "raw_byte_too_small"}).Inc() - log.Warnf("less then MinRawPacketLenth: %d, received length: %d, from: %v", MinRawPacketLenth, n, remoteAddr) - continue - } - - raw = make([]byte, n) - - copy(raw, data[:n]) - - prom.MsgCount.With(prometheus.Labels{"type": "on_message"}).Inc() - - go msg.OnMessage(raw, mysql.Save, remoteAddr.IP) - } + log.Infof("app listen on :3000") + app.Listen(":3000") } diff --git a/package.json b/package.json index 7098cc9..d358385 100644 --- a/package.json +++ b/package.json @@ -1,5 +1,5 @@ { - "name": "siphub-web", + "name": "sipgrep-web", "private": true, "version": "0.0.0", "type": "module", @@ -20,6 +20,7 @@ "@typescript-eslint/parser": "^6.0.0", "@vitejs/plugin-react": "^4.0.3", "antd": "^5.10.2", + "dayjs": "^1.11.10", "eslint": "^8.45.0", "eslint-plugin-react-hooks": "^4.6.0", "eslint-plugin-react-refresh": "^0.4.3", diff --git a/pkg/env/env.go b/pkg/env/env.go index 9490d2d..46bbc7b 100644 --- a/pkg/env/env.go +++ b/pkg/env/env.go @@ -8,27 +8,26 @@ import ( type config struct { UDPListenPort int `env:"UDPListenPort" envDefault:"9060"` - MaxPackgeLength int `env:"MaxPackgeLength" envDefault:"4096"` + MaxPacketLength int `env:"MaxPacketLength" envDefault:"4096"` MaxReadTimeoutSeconds int `env:"MaxReadTimeoutSecond" envDefault:"5"` LogLevel string `env:"LogLevel" envDefault:"info"` Hostname string `env:"HOSTNAME" envDefault:"unknow"` HeaderUIDName string `env:"HeaderUIDName"` HeaderFSCallIDName string `env:"HeaderFSCallIDName"` DiscardMethods string `env:"DiscardMethods" envDefault:"OPTIONS"` - MinPackgeLength int `env:"MinPackgeLength" envDefault:"24"` + MinPacketLength int `env:"MinPacketLength" envDefault:"24"` SqlMaxOpenConn int `env:"SqlMaxOpenConn" envDefault:"64"` SqlMaxIdleConn int `env:"SqlMaxIdleConn" envDefault:"64"` DBUser string `env:"DBUser"` - DBPasswd string `env:"DBPasswd"` // 支持加密 - - DBUserPasswd string `env:"DBUserPasswd"` // user:password + DBPasswd string `env:"DBPasswd"` DBAddr string `env:"DBAddr" envDefault:"localhost"` - DBName string `env:"DBName" envDefault:"siphub"` + DBName string `env:"DBName" envDefault:"sipgrep"` CalleeFrom string `env:"CalleeFrom" envDefault:"RURI"` MaxBatchItems int `env:"MaxBatchItems" envDefault:"20"` TickerSecondTime int `env:"TickerSecondTime" envDefault:"20"` + PageLimit int `env:"PageLimit" envDefault:"200"` } var Conf = config{} diff --git a/pkg/hep/hep.go b/pkg/hep/hep.go index d818af9..aa907c7 100644 --- a/pkg/hep/hep.go +++ b/pkg/hep/hep.go @@ -9,7 +9,7 @@ import ( "encoding/binary" "errors" "net" - "siphub/pkg/log" + "sipgrep/pkg/log" ) // HEP ID @@ -91,9 +91,9 @@ func (hepMsg *HepMsg) parseHep3(udpPacket []byte) error { currentByte += chunkLength if int(chunkLength) > cap(hepChunk) { - // 一般这种情况是因为每次读取的MaxPackgeLength设置的比较小 + // 一般这种情况是因为每次读取的MaxPacketLength设置的比较小 // 导致无法一次性读区完整的hep包 - // 所以可以适当增加MaxPackgeLength + // 所以可以适当增加MaxPacketLength // 例如设置为4096 log.Warnf("chunkLength big then slice capacity: chunkLength: %v, slice capacity: %v", chunkLength, cap(hepChunk)) chunkLength = uint16(cap(hepChunk)) diff --git a/pkg/hepserver/server.go b/pkg/hepserver/server.go new file mode 100644 index 0000000..954132c --- /dev/null +++ b/pkg/hepserver/server.go @@ -0,0 +1,62 @@ +package hepserver + +import ( + "fmt" + "net" + "sipgrep/pkg/env" + "sipgrep/pkg/log" + "sipgrep/pkg/msg" + "sipgrep/pkg/mysql" + "sipgrep/pkg/prom" + "time" + + "github.com/prometheus/client_golang/prometheus" +) + +const MinRawPacketLength = 105 + +func main() { + fmt.Println("vim-go") +} + +func CreateHepServer() { + conn, err := net.ListenUDP("udp", &net.UDPAddr{Port: env.Conf.UDPListenPort}) + + if err != nil { + log.Fatalf("Udp Service listen report udp fail:%v", err) + } + log.Infof("create udp success") + + defer conn.Close() + var data = make([]byte, env.Conf.MaxPacketLength) + var raw []byte + for { + conn.SetDeadline(time.Now().Add(time.Duration(env.Conf.MaxReadTimeoutSeconds) * time.Second)) + n, remoteAddr, err := conn.ReadFromUDP(data) + + if err != nil { + if opErr, ok := err.(*net.OpError); ok && opErr.Timeout() { + continue + } else { + prom.MsgCount.With(prometheus.Labels{"type": "read_udp_error"}) + log.Errorf("read udp error: %v, from %v", err, remoteAddr) + } + } + + prom.MsgCount.With(prometheus.Labels{"type": "all_received_packet"}).Inc() + + if n < MinRawPacketLength { + prom.MsgCount.With(prometheus.Labels{"type": "raw_byte_too_small"}).Inc() + log.Warnf("less then MinRawPacketLength: %d, received length: %d, from: %v", MinRawPacketLength, n, remoteAddr) + continue + } + + raw = make([]byte, n) + + copy(raw, data[:n]) + + prom.MsgCount.With(prometheus.Labels{"type": "on_message"}).Inc() + + go msg.OnMessage(raw, mysql.Save, remoteAddr.IP) + } +} diff --git a/pkg/log/log.go b/pkg/log/log.go index 5e50e6f..2074aa9 100644 --- a/pkg/log/log.go +++ b/pkg/log/log.go @@ -2,7 +2,7 @@ package log import ( "os" - "siphub/pkg/env" + "sipgrep/pkg/env" "go.uber.org/zap" "go.uber.org/zap/zapcore" diff --git a/pkg/msg/msg.go b/pkg/msg/msg.go index 0218d75..9eb0fcf 100644 --- a/pkg/msg/msg.go +++ b/pkg/msg/msg.go @@ -3,12 +3,12 @@ package msg import ( "fmt" "net" - "siphub/pkg/env" - "siphub/pkg/hep" - "siphub/pkg/log" - "siphub/pkg/models" - "siphub/pkg/parser" - "siphub/pkg/prom" + "sipgrep/pkg/env" + "sipgrep/pkg/hep" + "sipgrep/pkg/log" + "sipgrep/pkg/models" + "sipgrep/pkg/parser" + "sipgrep/pkg/prom" "strings" "time" @@ -45,7 +45,7 @@ func Format(p []byte) (s *models.SIP, errorType string, errMsg string) { return nil, "hep_body_is_empty", "" } - if len(hepMsg.Body) < env.Conf.MinPackgeLength { + if len(hepMsg.Body) < env.Conf.MinPacketLength { return nil, "hep_body_is_too_small", "" } diff --git a/pkg/mysql/record.go b/pkg/mysql/record.go index 55b1f1e..f5b2eb6 100644 --- a/pkg/mysql/record.go +++ b/pkg/mysql/record.go @@ -2,11 +2,11 @@ package mysql import ( "fmt" - "siphub/pkg/env" - "siphub/pkg/log" - "siphub/pkg/models" - "siphub/pkg/prom" - "siphub/pkg/util" + "sipgrep/pkg/env" + "sipgrep/pkg/log" + "sipgrep/pkg/models" + "sipgrep/pkg/prom" + "sipgrep/pkg/util" "time" "github.com/prometheus/client_golang/prometheus" @@ -45,6 +45,11 @@ type Record struct { RawMsg string `gorm:"type:text;not null"` } +type CallTable struct { + Meta Record + MsgCount int +} + var maxBatchItems = env.Conf.MaxBatchItems var batchChan = make(chan *Record, maxBatchItems*2) var ticker = time.NewTicker(time.Second * time.Duration(env.Conf.TickerSecondTime)) @@ -60,9 +65,10 @@ func BatchSaveInit() { select { case <-ticker.C: // 但是在抓包比较少的情况下,希望在达到一定的延迟后,也可以自动插入 + log.Infof("ticker for saving to db") i = maxBatchItems default: - batchItems[i] = <-batchChan + batchItems = append(batchItems, <-batchChan) } } @@ -117,7 +123,7 @@ func Save(s *models.SIP) { func Connect(UserPasswd, Addr, DBName string) { var err error - dsn := fmt.Sprintf("%s@tcp(%s)/%s?charset=utf8mb4&parseTime=True&loc=Local", UserPasswd, Addr, DBName) + dsn := fmt.Sprintf("%s@tcp(%s)/%s?charset=utf8mb4&parseTime=True&loc=Local&sql_mode=TRADITIONAL", UserPasswd, Addr, DBName) db, err = gorm.Open(mysql.Open(dsn), &gorm.Config{ SkipDefaultTransaction: true, @@ -141,3 +147,43 @@ func Connect(UserPasswd, Addr, DBName string) { db.AutoMigrate(&Record{}) } + +func Search(sql string) ([]CallTable, error) { + rows, err := db.Raw(sql).Rows() + + if err != nil { + return nil, err + } + + table := make([]CallTable, 0, env.Conf.PageLimit) + + defer rows.Close() + + for rows.Next() { + item := CallTable{} + + err := rows.Scan( + &item.Meta.SipCallid, + &item.Meta.CreateTime, + &item.Meta.FromUser, + &item.Meta.FromHost, + &item.Meta.ToUser, + &item.Meta.ToHost, + &item.Meta.UserAgent, + &item.Meta.SipProtocol, + &item.Meta.SipMethod, + &item.Meta.CseqMethod, + &item.Meta.FsCallid, + &item.Meta.LegUid, + &item.MsgCount, + ) + + if err != nil { + log.Errorf("sql scan error: %v", err) + } + + table = append(table, item) + } + + return table, nil +} diff --git a/pkg/mysql/util.go b/pkg/mysql/util.go new file mode 100644 index 0000000..738dcb1 --- /dev/null +++ b/pkg/mysql/util.go @@ -0,0 +1,78 @@ +package mysql + +import ( + "fmt" + "sipgrep/pkg/env" + "strings" + + "github.com/golang-module/carbon/v2" +) + +var SearchFields = []string{ + "sip_callid", + "create_time", + "from_user", + "from_host", + "to_user", + "to_host", + "user_agent", + "sip_protocol", + "sip_method", + "cseq_method", + "fs_callid", + "leg_uid", + "count(*) as msg_count", +} + +type SearchParams struct { + BeginTime string `validate:"required,datetime=2006-01-02 15:04:05,len=19"` + EndTime string `validate:"required,datetime=2006-01-02 15:04:05,len=19"` + Caller string `validate:"max=64"` + CallerDomain string `validate:"max=64"` + Callee string `validate:"max=64"` + CalleeDomain string `validate:"max=64"` +} + +func GetTableName(BeginTime string) string { + if carbon.Parse(BeginTime).IsToday() { + return "records" + } + + // YYYYMMDD + newTable := carbon.Parse(BeginTime).ToShortDateString() + + return "sipgrep_backup_" + newTable +} + +func GetSearchSql(sp SearchParams) string { + conditions := []string{} + + conditions = append(conditions, fmt.Sprintf("create_time between '%s' and '%s'", sp.BeginTime, sp.EndTime)) + + if sp.Caller != "" { + conditions = append(conditions, fmt.Sprintf("from_user='%s'", sp.Caller)) + } + if sp.CallerDomain != "" { + conditions = append(conditions, fmt.Sprintf("from_host='%s'", sp.CallerDomain)) + } + if sp.Callee != "" { + conditions = append(conditions, fmt.Sprintf("to_user='%s'", sp.Callee)) + } + if sp.CalleeDomain != "" { + conditions = append(conditions, fmt.Sprintf("to_host='%s'", sp.CalleeDomain)) + } + + columns := strings.Join(SearchFields, ",") + conds := strings.Join(conditions, ",") + tableName := GetTableName(sp.BeginTime) + + sql := fmt.Sprintf(`select %s from %s where sip_callid in ( + select sip_callid from ( + select distinct sip_callid from %s where %s + limit %d + ) tmp + ) + group by sip_callid order by create_time desc`, columns, tableName, tableName, conds, env.Conf.PageLimit) + + return sql +} diff --git a/pkg/parser/parser.go b/pkg/parser/parser.go index dd91fc3..1cf3011 100644 --- a/pkg/parser/parser.go +++ b/pkg/parser/parser.go @@ -1,8 +1,8 @@ package parser import ( - "siphub/pkg/models" - "siphub/pkg/util" + "sipgrep/pkg/models" + "sipgrep/pkg/util" "strings" ) @@ -24,9 +24,9 @@ type Parser struct { models.SIP } -// Request : INVITE bob@example.com SIP/2.0 -// Response : SIP/2.0 200 OK -// Response : SIP/2.0 501 Not Implemented +// Request : INVITE bob@example.com SIP/2.0 +// Response : SIP/2.0 200 OK +// Response : SIP/2.0 501 Not Implemented func (p *Parser) ParseFirstLine() { if p.Raw == nil { return diff --git a/pkg/parser/parser_test.go b/pkg/parser/parser_test.go index e26124e..6735bbd 100644 --- a/pkg/parser/parser_test.go +++ b/pkg/parser/parser_test.go @@ -3,7 +3,7 @@ package parser import ( "testing" - "siphub/pkg/models" + "sipgrep/pkg/models" "github.com/stretchr/testify/assert" ) diff --git a/pkg/prom/prom.go b/pkg/prom/prom.go index a11363d..766486c 100644 --- a/pkg/prom/prom.go +++ b/pkg/prom/prom.go @@ -6,7 +6,7 @@ import ( var MsgCount = prometheus.NewCounterVec( prometheus.CounterOpts{ - Name: "siphub_msg_count", + Name: "sipgrep_msg_count", }, []string{"type"}, ) diff --git a/pkg/route/rest.go b/pkg/route/rest.go new file mode 100644 index 0000000..a8bd2e2 --- /dev/null +++ b/pkg/route/rest.go @@ -0,0 +1,48 @@ +package route + +import ( + "sipgrep/pkg/log" + "sipgrep/pkg/mysql" + + "github.com/go-playground/validator/v10" + "github.com/gofiber/fiber/v2" +) + +var validate = validator.New() + +func Search(c *fiber.Ctx) error { + sp := mysql.SearchParams{ + BeginTime: c.Query("BeginTime"), + EndTime: c.Query("EndTime"), + Caller: c.Query("Caller"), + CallerDomain: c.Query("CallerDomain"), + Callee: c.Query("Callee"), + CalleeDomain: c.Query("CalleeDomain"), + } + + log.Infof("%#v", sp) + + if err := validate.Struct(sp); err != nil { + log.Errorf("%v", err) + return &fiber.Error{ + Code: fiber.ErrBadRequest.Code, + Message: "query string error", + } + } + + sql := mysql.GetSearchSql(sp) + + log.Infof("sql %s", sql) + + table, err := mysql.Search(sql) + + if err != nil { + log.Errorf("%v", err) + return &fiber.Error{ + Code: fiber.ErrInternalServerError.Code, + Message: "sql query error", + } + } + + return c.JSON(table) +} diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 0bad75a..24b9698 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -31,6 +31,9 @@ devDependencies: antd: specifier: ^5.10.2 version: 5.10.2(react-dom@18.2.0)(react@18.2.0) + dayjs: + specifier: ^1.11.10 + version: 1.11.10 eslint: specifier: ^8.45.0 version: 8.52.0 diff --git a/src/App.tsx b/src/App.tsx index a28fcd3..ccdb256 100644 --- a/src/App.tsx +++ b/src/App.tsx @@ -1,5 +1,5 @@ import { Row, Col, Card } from 'antd' -import SearchForm from './SearchFrom' +import { FieldType, SearchForm } from './SearchFrom' import SearchTable from './SearchTable' import { useState } from 'react' import { DataType } from './interface' @@ -18,11 +18,28 @@ function App() { }, ]) + function Search(ft: FieldType) { + console.log('app', ft) + console.log(ft.timePicker) + + const query = { + BeginTime: ft.datePicker.format('YYYY-MM-DD') + ' ' + ft.timePicker[0].format('HH:mm:ss'), + EndTime: ft.datePicker.format('YYYY-MM-DD') + ' ' + ft.timePicker[0].format('HH:mm:ss'), + Caller: !!ft.caller ? ft.caller : undefined, + CallerDomain: ft.callerDomain, + // 被叫号码取反存储 + Callee: !!ft.callee ? ft.callee.split('').reverse().join('') : undefined, + CalleeDomain: ft.calleeDomain, + } + + console.log(query) + } + return ( - + diff --git a/src/SearchFrom.tsx b/src/SearchFrom.tsx index a404a1d..9c9ffde 100644 --- a/src/SearchFrom.tsx +++ b/src/SearchFrom.tsx @@ -1,68 +1,94 @@ -import React from 'react' -import { Button, Form, Input, DatePicker, TimePicker } from 'antd' - -const onFinish = (values: any) => { - console.log('Success:', values) -} +import dayjs from 'dayjs' -const onFinishFailed = (errorInfo: any) => { - console.log('Failed:', errorInfo) -} +import { Button, Form, Input, DatePicker, TimePicker } from 'antd' -type FieldType = { +export type FieldType = { caller?: string callerDomain?: string callee?: string calleeDomain?: string - datePicker?: string - timePicker?: string + datePicker: dayjs.Dayjs + timePicker: [dayjs.Dayjs, dayjs.Dayjs] } -const App: React.FC = () => ( -
- label="Caller No." name="caller"> - - +interface Prop { + search: (ft: FieldType) => void +} - label="Caller Domain" name="callerDomain"> - - +export function SearchForm(p: Prop) { + const onFinish = (values: FieldType) => { + console.log('Success:', values) + p.search(values) + } - label="Callee No." name="callee"> - - + const onFinishFailed = (errorInfo: any) => { + console.log('Failed:', errorInfo) + } - label="Callee Domain" name="calleeDomain"> - - + return ( + + + label="Caller No." + name="caller" + rules={[{ pattern: /(^\S)((.)*\S)?(\S*$)/, message: 'no any space allow!' }]} + > + + - - - + + label="Caller Domain" + name="callerDomain" + rules={[{ pattern: /(^\S)((.)*\S)?(\S*$)/, message: 'no any space allow!' }]} + > + + - - - + + label="Callee No." + name="callee" + rules={[{ pattern: /(^\S)((.)*\S)?(\S*$)/, message: 'no any space allow!' }]} + > + + + + + label="Callee Domain" + name="calleeDomain" + rules={[{ pattern: /(^\S)((.)*\S)?(\S*$)/, message: 'no any space allow!' }]} + > + + - - - - -) + + + -export default App + + + + + + + + + ) +} diff --git a/src/SearchTable.tsx b/src/SearchTable.tsx index 1997362..787800d 100644 --- a/src/SearchTable.tsx +++ b/src/SearchTable.tsx @@ -56,6 +56,7 @@ const columns: ColumnsType = [ interface Prop { calls: DataType[] } + const App: React.FC = ({ calls }) => export default App