From bdbf3edeaf6837c032c287e7023fe09c4251b1b5 Mon Sep 17 00:00:00 2001 From: nnn-revo2012 Date: Mon, 24 May 2021 01:01:36 +0900 Subject: [PATCH 01/39] built 20210516.36 Update changelog.txt and buildno.txt --- changelog.txt | 18 ++++++++++++++++++ src/buildno/buildno.go | 4 ++-- 2 files changed, 20 insertions(+), 2 deletions(-) diff --git a/changelog.txt b/changelog.txt index 7b1c1de..5a41896 100644 --- a/changelog.txt +++ b/changelog.txt @@ -1,5 +1,23 @@ 更新履歴 +20210516.36 +・fix タイムシフト録画時に'getwaybackkey: waybackkey not found' と表示されコメントが保存されない(nnn-revo2012 PR#54) +　- コメントサーバー仕様変更に対応(threadId、waybackkey廃止など)(2020/07/27) +　　5chで公開されたID:jM/9Q+5+0作成のpatchを適用 +　　https://egg.5ch.net/test/read.cgi/software/1570634489/932 +　- livedl で waybackkey の取得方法を変更するパッチ +　　https://egg.5ch.net/test/read.cgi/software/1595715643/424 + +・ニコニコ仕様変更を反映（#49 5chボランティア (sangwon-jung-work PR#51) +　- 放送情報取得時のwebsocketプロトコルが変わって録画できなくなったのを修正(2020/06/02) +　　5chで公開されたID:jM/9Q+5+0作成のpatchを適用 +　　http://egg.5ch.net/test/read.cgi/software/1570634489/535 +　- fix broadcastId not found error, add debug log + +・20181215.35以降の修正を追加 +・TS録画時にセグメント抜けが起こるのを修正 (PR#47) +・http -> httpsに修正 (PR#39) + 20181215.35 ・-nico-ts-start-minオプションの追加 ・win32bit版のビルドを追加 diff --git a/src/buildno/buildno.go b/src/buildno/buildno.go index c527d39..921e109 100644 --- a/src/buildno/buildno.go +++ b/src/buildno/buildno.go @@ -1,5 +1,5 @@ package buildno -var BuildDate = "20181215" -var BuildNo = "35" +var BuildDate = "20210516" +var BuildNo = "36" From 6cfc894a48035219acf7c2f63ae5222ee256b35c Mon Sep 17 00:00:00 2001 From: nnn-revo2012 Date: Mon, 24 May 2021 01:56:10 +0900 Subject: [PATCH 02/39] Add self-chasing playback function and others https://egg.5ch.net/test/read.cgi/software/1595715643/57 --- src/livedl.go | 18 ++++- src/niconico/nico_db.go | 18 +++++ src/niconico/nico_hls.go | 48 +++++++++--- src/options/options.go | 62 +++++++++++++++- src/zip2mp4/zip2mp4.go | 156 ++++++++++++++++++++++++++++++++++++++- 5 files changed, 283 insertions(+), 19 deletions(-) diff --git a/src/livedl.go b/src/livedl.go index ca06fe0..4e8e632 100644 --- a/src/livedl.go +++ b/src/livedl.go @@ -141,17 +141,17 @@ func main() { os.Exit(1) } if hlsPlaylistEnd && opt.NicoAutoConvert { - done, nMp4s, err := zip2mp4.ConvertDB(dbname, opt.ConvExt, opt.NicoSkipHb) + done, nMp4s, skipped, err := zip2mp4.ConvertDB(dbname, opt.ConvExt, opt.NicoSkipHb, opt.NicoConvForceConcat, opt.NicoConvSeqnoStart, opt.NicoConvSeqnoEnd) if err != nil { fmt.Println(err) os.Exit(1) } if done { - if nMp4s == 1 { + if nMp4s == 1 && (! skipped) { if 1 <= opt.NicoAutoDeleteDBMode { os.Remove(dbname) } - } else if 1 < nMp4s { + } else if 1 < nMp4s || (nMp4s == 1 && skipped) { if 2 <= opt.NicoAutoDeleteDBMode { os.Remove(dbname) } @@ -181,11 +181,21 @@ func main() { } } else { - if _, _, err := zip2mp4.ConvertDB(opt.DBFile, opt.ConvExt, opt.NicoSkipHb); err != nil { + if _, _, _, err := zip2mp4.ConvertDB(opt.DBFile, opt.ConvExt, opt.NicoSkipHb, opt.NicoConvForceConcat, opt.NicoConvSeqnoStart, opt.NicoConvSeqnoEnd); err != nil { fmt.Println(err) os.Exit(1) } } + + case "DB2HLS": + if opt.NicoHlsPort == 0 { + fmt.Println("HLS port not specified") + os.Exit(1) + } + if err := zip2mp4.ReplayDB(opt.DBFile, opt.NicoHlsPort, opt.NicoConvSeqnoStart); err != nil { + fmt.Println(err) + os.Exit(1) + } } return diff --git a/src/niconico/nico_db.go b/src/niconico/nico_db.go index a92996e..fd50be0 100644 --- a/src/niconico/nico_db.go +++ b/src/niconico/nico_db.go @@ -34,6 +34,24 @@ var SelComment = `SELECT FROM comment ORDER BY date2` +func SelMediaF(seqnoStart, seqnoEnd int64) (ret string) { + ret = `SELECT + seqno, bandwidth, size, data FROM media + WHERE IFNULL(notfound, 0) == 0 AND data IS NOT NULL` + + if seqnoStart > 0 { + ret += ` AND seqno >= ` + fmt.Sprint(seqnoStart) + } + + if seqnoEnd > 0 { + ret += ` AND seqno <= ` + fmt.Sprint(seqnoEnd) + } + + ret += ` ORDER BY seqno` + + return +} + func (hls *NicoHls) dbOpen() (err error) { db, err := sql.Open("sqlite3", hls.dbName) if err != nil { diff --git a/src/niconico/nico_hls.go b/src/niconico/nico_hls.go index 09b09f6..3f264cc 100644 --- a/src/niconico/nico_hls.go +++ b/src/niconico/nico_hls.go @@ -1800,23 +1800,51 @@ func (hls *NicoHls) serve(hlsPort int) { router := gin.Default() router.GET("", func(c *gin.Context) { - seqno := hls.dbGetLastSeqNo() + c.Redirect(http.StatusMovedPermanently, "/m3u8/2/0/index.m3u8") + c.Abort() + }) + + router.GET("/m3u8/:delay/:shift/index.m3u8", func(c *gin.Context) { + targetDuration := "2" + extInf := "1.5" + if hls.isTimeshift { + targetDuration = "3" + extInf = "3.0" + } + shift, err := strconv.Atoi(c.Param("shift")) + if err != nil { + shift = 0 + } + if shift < 0 { + shift = 0 + } + delay, err := strconv.Atoi(c.Param("delay")) + if err != nil { + delay = 0 + } + if delay < 2 { + delay = 2 + } + if (! hls.isTimeshift) { + if delay < 4 { + delay = 4 + } + } + seqno := hls.dbGetLastSeqNo() - int64(shift) body := fmt.Sprintf( `#EXTM3U #EXT-X-VERSION:3 -#EXT-X-TARGETDURATION:1 +#EXT-X-TARGETDURATION:%s #EXT-X-MEDIA-SEQUENCE:%d -#EXTINF:1.0, -/ts/%d/test.ts - -#EXTINF:1.0, -/ts/%d/test.ts - -#EXTINF:1.0, +`, targetDuration, seqno) + for i := int64(delay); i >= 0; i-- { + body += fmt.Sprintf( +`#EXTINF:%s, /ts/%d/test.ts -`, seqno-2, seqno-2, seqno-1, seqno) +`, extInf, seqno - i) + } c.Data(http.StatusOK, "application/x-mpegURL", []byte(body)) return }) diff --git a/src/options/options.go b/src/options/options.go index bc1183f..8761634 100644 --- a/src/options/options.go +++ b/src/options/options.go @@ -52,6 +52,9 @@ type Option struct { NicoDebug bool // デバッグ情報の記録 ConvExt string ExtractChunks bool + NicoConvForceConcat bool + NicoConvSeqnoStart int64 + NicoConvSeqnoEnd int64 NicoForceResv bool // 終了番組の上書きタイムシフト予約 YtNoStreamlink bool YtNoYoutubeDl bool @@ -414,6 +417,7 @@ func ParseArgs() (opt Option) { IFNULL((SELECT v FROM conf WHERE k == "TcasRetryInterval"), 0), IFNULL((SELECT v FROM conf WHERE k == "ConvExt"), ""), IFNULL((SELECT v FROM conf WHERE k == "ExtractChunks"), 0), + IFNULL((SELECT v FROM conf WHERE k == "NicoConvForceConcat"), 0), IFNULL((SELECT v FROM conf WHERE k == "NicoForceResv"), 0), IFNULL((SELECT v FROM conf WHERE k == "YtNoStreamlink"), 0), IFNULL((SELECT v FROM conf WHERE k == "YtNoYoutubeDl"), 0), @@ -434,6 +438,7 @@ func ParseArgs() (opt Option) { &opt.TcasRetryInterval, &opt.ConvExt, &opt.ExtractChunks, + &opt.NicoConvForceConcat, &opt.NicoForceResv, &opt.YtNoStreamlink, &opt.YtNoYoutubeDl, @@ -590,6 +595,10 @@ func ParseArgs() (opt Option) { opt.Command = "DB2MP4" return nil }}, + Parser{regexp.MustCompile(`\A(?i)--?(?:d|db|sqlite3?)-?(?:2|to)-?(?:h|hls)\z`), func() error { + opt.Command = "DB2HLS" + return nil + }}, Parser{regexp.MustCompile(`\A(?i)--?nico-?login-?only(?:=(on|off))?\z`), func() error { if strings.EqualFold(match[1], "on") { opt.NicoLoginOnly = true @@ -849,6 +858,8 @@ func ParseArgs() (opt Option) { case "", "DB2MP4": opt.Command = "DB2MP4" opt.DBFile = match[0] + case "DB2HLS": + opt.DBFile = match[0] default: return fmt.Errorf("%s: Use -- option before \"%s\"", opt.Command, match[0]) } @@ -872,6 +883,48 @@ func ParseArgs() (opt Option) { dbConfSet(db, "ExtractChunks", opt.ExtractChunks) return nil }}, + Parser{regexp.MustCompile(`\A(?i)--?nico-?conv-?force-?concat(?:=(on|off))?\z`), func() error { + if strings.EqualFold(match[1], "on") { + opt.NicoConvForceConcat = true + dbConfSet(db, "NicoConvForceConcat", opt.NicoConvForceConcat) + } else if strings.EqualFold(match[1], "off") { + opt.NicoConvForceConcat = false + dbConfSet(db, "NicoConvForceConcat", opt.NicoConvForceConcat) + } else { + opt.NicoConvForceConcat = true + } + return nil + }}, + Parser{regexp.MustCompile(`\A(?i)--?nico-?conv-?seqno-?start\z`), func() (err error) { + s, err := nextArg() + if err != nil { + return err + } + num, err := strconv.Atoi(s) + if err != nil { + return fmt.Errorf("--nico-conv-seqno-start: Not a number: %s\n", s) + } + if num < 0 { + return fmt.Errorf("--nico-conv-seqno-start: Invalid: %d: must be greater than or equal to 0\n", num) + } + opt.NicoConvSeqnoStart = int64(num) + return nil + }}, + Parser{regexp.MustCompile(`\A(?i)--?nico-?conv-?seqno-?end\z`), func() (err error) { + s, err := nextArg() + if err != nil { + return err + } + num, err := strconv.Atoi(s) + if err != nil { + return fmt.Errorf("--nico-conv-seqno-end: Not a number: %s\n", s) + } + if num < 0 { + return fmt.Errorf("--nico-conv-seqno-end: Invalid: %d: must be greater than or equal to 0\n", num) + } + opt.NicoConvSeqnoEnd = int64(num) + return nil + }}, Parser{regexp.MustCompile(`\A(?i)--?nico-?force-?(?:re?sv|reservation)(?:=(on|off))\z`), func() error { if strings.EqualFold(match[1], "on") { opt.NicoForceResv = true @@ -1001,7 +1054,7 @@ func ParseArgs() (opt Option) { opt.ZipFile = arg return true } - case "DB2MP4": + case "DB2MP4", "DB2HLS": if ma := regexp.MustCompile(`(?i)\.sqlite3`).FindStringSubmatch(arg); len(ma) > 0 { opt.DBFile = arg return true @@ -1081,6 +1134,7 @@ LB_ARG: if opt.NicoAutoConvert { fmt.Printf("Conf(NicoAutoDeleteDBMode): %#v\n", opt.NicoAutoDeleteDBMode) fmt.Printf("Conf(ExtractChunks): %#v\n", opt.ExtractChunks) + fmt.Printf("Conf(NicoConvForceConcat): %#v\n", opt.NicoConvForceConcat) fmt.Printf("Conf(ConvExt): %#v\n", opt.ConvExt) } fmt.Printf("Conf(NicoForceResv): %#v\n", opt.NicoForceResv) @@ -1096,7 +1150,11 @@ LB_ARG: fmt.Printf("Conf(TcasRetryInterval): %#v\n", opt.TcasRetryInterval) case "DB2MP4": fmt.Printf("Conf(ExtractChunks): %#v\n", opt.ExtractChunks) + fmt.Printf("Conf(NicoConvForceConcat): %#v\n", opt.NicoConvForceConcat) fmt.Printf("Conf(ConvExt): %#v\n", opt.ConvExt) + case "DB2HLS": + fmt.Printf("Conf(NicoHlsPort): %#v\n", opt.NicoHlsPort) + fmt.Printf("Conf(NicoConvSeqnoStart): %#v\n", opt.NicoConvSeqnoStart) } fmt.Printf("Conf(HttpSkipVerify): %#v\n", opt.HttpSkipVerify) @@ -1126,7 +1184,7 @@ LB_ARG: if opt.ZipFile == "" { Help() } - case "DB2MP4": + case "DB2MP4", "DB2HLS": if opt.DBFile == "" { Help() } diff --git a/src/zip2mp4/zip2mp4.go b/src/zip2mp4/zip2mp4.go index 30d5f50..49fc83e 100644 --- a/src/zip2mp4/zip2mp4.go +++ b/src/zip2mp4/zip2mp4.go @@ -21,6 +21,10 @@ import ( "github.com/himananiito/livedl/procs/ffmpeg" "github.com/himananiito/livedl/youtube" _ "github.com/mattn/go-sqlite3" + + "context" + "github.com/gin-gonic/gin" + "net/http" ) type ZipMp4 struct { @@ -492,7 +496,7 @@ func ExtractChunks(fileName string, skipHb bool) (done bool, err error) { return } -func ConvertDB(fileName, ext string, skipHb bool) (done bool, nMp4s int, err error) { +func ConvertDB(fileName, ext string, skipHb, forceConcat bool, seqnoStart, seqnoEnd int64) (done bool, nMp4s int, skipped bool, err error) { db, err := sql.Open("sqlite3", fileName) if err != nil { return @@ -512,7 +516,7 @@ func ConvertDB(fileName, ext string, skipHb bool) (done bool, nMp4s int, err err zm = &ZipMp4{ZipName: fileName} zm.OpenFFMpeg(ext) - rows, err := db.Query(niconico.SelMedia) + rows, err := db.Query(niconico.SelMediaF(seqnoStart, seqnoEnd)) if err != nil { return } @@ -543,7 +547,10 @@ func ConvertDB(fileName, ext string, skipHb bool) (done bool, nMp4s int, err err // zm.CloseFFInput() // zm.Wait() //} - zm.OpenFFMpeg(ext) + if ! forceConcat { + zm.OpenFFMpeg(ext) + } + skipped = true } prevBw = bw prevIndex = seqno @@ -563,6 +570,149 @@ func ConvertDB(fileName, ext string, skipHb bool) (done bool, nMp4s int, err err return } +func ReplayDB(fileName string, hlsPort int, seqnoStart int64) (err error) { + db, err := sql.Open("sqlite3", fileName) + if err != nil { + return + } + defer db.Close() + + var isTimeshift bool + if m := regexp.MustCompile(`\(TS\)\.sqlite3$`).FindStringSubmatch(fileName); len(m) > 0 { + isTimeshift = true + } + fmt.Println("isTimeshift:", isTimeshift) + + seqnoInit := seqnoStart + + timeStart := time.Now() + timeLast := time.Now() + + seqnoCurrent := seqnoStart + + if (true) { + gin.SetMode(gin.ReleaseMode) + gin.DefaultErrorWriter = ioutil.Discard + gin.DefaultWriter = ioutil.Discard + router := gin.Default() + + router.GET("", func(c *gin.Context) { + c.Redirect(http.StatusMovedPermanently, "/m3u8/2/0/index.m3u8") + c.Abort() + }) + + router.GET("/m3u8/:delay/:shift/index.m3u8", func(c *gin.Context) { + secPerSegment := 1.5 + targetDuration := "2" + targetDurationFloat := 2.0 + extInf := "1.5" + if isTimeshift { + secPerSegment = 5.0 + targetDuration = "3" + targetDurationFloat = 3.0 + extInf = "3.0" + } + shift, err := strconv.Atoi(c.Param("shift")) + if err != nil { + shift = 0 + } + if shift < 0 { + shift = 0 + } + delay, err := strconv.Atoi(c.Param("delay")) + if err != nil { + delay = 0 + } + if delay < 2 { + delay = 2 + } + if (! isTimeshift) { + if delay < 4 { + delay = 4 + } + } + seqnoRewind := int64(delay) + timeout := targetDurationFloat * float64(delay + 1) * 2 + 1 + timeNow := time.Now() + if float64(timeNow.Sub(timeLast) / time.Second) > timeout { + fmt.Printf("(%s) CONTINUE\n", timeNow.Format("15:04:05")) + seqnoStart = seqnoCurrent - seqnoRewind + if seqnoStart < seqnoInit { + seqnoStart = seqnoInit + } + timeStart = timeNow + seqnoCurrent = seqnoStart + } else { + seqnoCurrent = int64(float64(timeNow.Sub(timeStart) / time.Second) / secPerSegment) + seqnoStart + } + timeLast = timeNow + seqno := seqnoCurrent - int64(shift) + body := fmt.Sprintf( +`#EXTM3U +#EXT-X-VERSION:3 +#EXT-X-TARGETDURATION:%s +#EXT-X-MEDIA-SEQUENCE:%d + +`, targetDuration, seqno) + for i := int64(delay); i >= 0; i-- { + body += fmt.Sprintf( +`#EXTINF:%s, +/ts/%d/test.ts + +`, extInf, seqno - i) + } + if shift > 0 { + fmt.Printf("(%s) Current SeqNo: %d(-%d)\n", timeNow.Format("15:04:05"), seqnoCurrent, shift) + } else { + fmt.Printf("(%s) Current SeqNo: %d\n", timeNow.Format("15:04:05"), seqnoCurrent) + } + c.Data(http.StatusOK, "application/x-mpegURL", []byte(body)) + return + }) + + router.GET("/ts/:idx/test.ts", func(c *gin.Context) { + i, _ := strconv.Atoi(c.Param("idx")) + var b []byte + db.QueryRow("SELECT data FROM media WHERE seqno = ?", i).Scan(&b) + c.Data(http.StatusOK, "video/MP2T", b) + return + }) + + srv := &http.Server{ + Addr: fmt.Sprintf("127.0.0.1:%d", hlsPort), + Handler: router, + ReadTimeout: 10 * time.Second, + WriteTimeout: 10 * time.Second, + MaxHeaderBytes: 1 << 20, + } + + chLocal := make(chan struct{}) + idleConnsClosed := make(chan struct{}) + defer func(){ + close(chLocal) + }() + go func() { + select { + case <-chLocal: + } + if err := srv.Shutdown(context.Background()); err != nil { + log.Printf("srv.Shutdown: %v\n", err) + } + close(idleConnsClosed) + }() + + // クライアントはlocalhostでなく127.0.0.1で接続すること + // localhostは遅いため + if err := srv.ListenAndServe(); err != http.ErrServerClosed { + log.Printf("srv.ListenAndServe: %v\n", err) + } + + <-idleConnsClosed + } + + return +} + func YtComment(fileName string) (done bool, err error) { db, err := sql.Open("sqlite3", fileName) if err != nil { From 4906b20c691e49873aa5bb5e6f6b0e3ce6d8ae32 Mon Sep 17 00:00:00 2001 From: nnn-revo2012 Date: Mon, 24 May 2021 02:04:58 +0900 Subject: [PATCH 03/39] Update changelog.txt and options.go --- changelog.txt | 22 ++++++++++++++++++++++ src/options/options.go | 8 ++++++++ 2 files changed, 30 insertions(+) diff --git a/changelog.txt b/changelog.txt index 5a41896..3e362c6 100644 --- a/changelog.txt +++ b/changelog.txt @@ -1,5 +1,27 @@ 更新履歴 +・セルフ追っかけ再生機能その他 +https://egg.5ch.net/test/read.cgi/software/1595715643/57 +　例：http://127.0.0.1:12345/m3u8/2/1200/index.m3u8 +　　現在のシーケンス番号から1200セグメント（リアルタイムの場合30分）戻ったところを再生 + +・追加オプション +　-nico-conv-seqno-start ＜num＞ +　　MP4への変換を指定したセグメント番号から開始する +　-nico-conv-seqno-end ＜num＞ +　　MP4への変換を指定したセグメント番号で終了する +　-nico-conv-force-concat +　　MP4への変換で画質変更または抜けがあっても分割しないように設定 +　-nico-conv-force-concat=on +　　(+) 上記を有効に設定 +　-nico-conv-force-concat=off +　　(+) 上記を無効に設定(デフォルト) + +・　-d2h +　　[実験的] 録画済みのdb(.sqlite3)を視聴するためのHLSサーバを立てる(-db-to-hls) +　　　開始シーケンス番号は（変換ではないが） -nico-conv-seqno-start で指定 +　　　　使用例：$ livedl lvXXXXXXXXX.sqlite3 -d2h -nico-hls-port 12345 -nico-conv-seqno-start 2780 + 20210516.36 ・fix タイムシフト録画時に'getwaybackkey: waybackkey not found' と表示されコメントが保存されない(nnn-revo2012 PR#54) 　- コメントサーバー仕様変更に対応(threadId、waybackkey廃止など)(2020/07/27) diff --git a/src/options/options.go b/src/options/options.go index 8761634..09dffaf 100644 --- a/src/options/options.go +++ b/src/options/options.go @@ -95,6 +95,9 @@ COMMAND: -tcas ツイキャスの録画 -yt YouTube Liveの録画 -d2m 録画済みのdb(.sqlite3)をmp4に変換する(-db-to-mp4) + -d2h [実験的] 録画済みのdb(.sqlite3)を視聴するためのHLSサーバを立てる(-db-to-hls) + 開始シーケンス番号は（変換ではないが） -nico-conv-seqno-start で指定 + 使用例：$ livedl lvXXXXXXXXX.sqlite3 -d2h -nico-hls-port 12345 -nico-conv-seqno-start 2780 オプション/option: -h ヘルプを表示 @@ -133,6 +136,11 @@ COMMAND: -nico-skip-hb=off (+) コメント書き出し時に/hbコマンドも出す(デフォルト) -nico-ts-start タイムシフトの録画を指定した再生時間(秒)から開始する -nico-ts-start-min タイムシフトの録画を指定した再生時間(分)から開始する + -nico-conv-seqno-start MP4への変換を指定したセグメント番号から開始する + -nico-conv-seqno-end MP4への変換を指定したセグメント番号で終了する + -nico-conv-force-concat MP4への変換で画質変更または抜けがあっても分割しないように設定 + -nico-conv-force-concat=on (+) 上記を有効に設定 + -nico-conv-force-concat=off (+) 上記を無効に設定(デフォルト) ツイキャス録画用オプション: -tcas-retry=on (+) 録画終了後に再試行を行う From 13947cc333ea2f862c5720dce89cd588af14b04e Mon Sep 17 00:00:00 2001 From: nnn-revo2012 Date: Mon, 24 May 2021 02:20:27 +0900 Subject: [PATCH 04/39] Add stop time-shift recording at specified time https://egg.5ch.net/test/read.cgi/software/1595715643/163 --- src/niconico/nico_hls.go | 45 ++++++++++++++++++-------- src/options/options.go | 70 ++++++++++++++++++++++++++++++++++++++-- 2 files changed, 98 insertions(+), 17 deletions(-) diff --git a/src/niconico/nico_hls.go b/src/niconico/nico_hls.go index 3f264cc..565ac2d 100644 --- a/src/niconico/nico_hls.go +++ b/src/niconico/nico_hls.go @@ -77,6 +77,7 @@ type NicoHls struct { isTimeshift bool timeshiftStart float64 + timeshiftStop int fastTimeshift bool ultrafastTimeshift bool @@ -279,6 +280,7 @@ func NewHls(opt options.Option, prop map[string]interface{}) (hls *NicoHls, err gmMain: gorman.WithChecker(func(c int) { hls.checkReturnCode(c) }), timeshiftStart: opt.NicoTsStart, + timeshiftStop: opt.NicoTsStop, } hls.fastTimeshiftOrig = hls.fastTimeshift @@ -399,6 +401,7 @@ const ( MAIN_INVALID_STREAM_QUALITY MAIN_TEMPORARILY_ERROR PLAYLIST_END + TIMESHIFT_STOP PLAYLIST_403 PLAYLIST_ERROR DELAY @@ -488,6 +491,20 @@ func (hls *NicoHls) markRestartMain(delay int) { } func (hls *NicoHls) checkReturnCode(code int) { // NEVER restart goroutines here except interrupt handler + var fPlaylistEnd = func() { + hls.finish = true + if hls.isTimeshift { + if hls.commentDone { + hls.stopPCGoroutines() + } else if (! hls.getCommentStarted()) { + hls.stopPCGoroutines() + } else { + fmt.Println("waiting comment") + } + } else { + hls.stopPCGoroutines() + } + } switch code { case NETWORK_ERROR, MAIN_TEMPORARILY_ERROR: delay := hls.getStartDelay() @@ -516,18 +533,11 @@ func (hls *NicoHls) checkReturnCode(code int) { case PLAYLIST_END: fmt.Println("playlist end.") - hls.finish = true - if hls.isTimeshift { - if hls.commentDone { - hls.stopPCGoroutines() - } else if !hls.getCommentStarted() { - hls.stopPCGoroutines() - } else { - fmt.Println("waiting comment") - } - } else { - hls.stopPCGoroutines() - } + fPlaylistEnd() + + case TIMESHIFT_STOP: + fmt.Println("timeshift stop.") + fPlaylistEnd() case MAIN_WS_ERROR: hls.stopPGoroutines() @@ -1039,7 +1049,7 @@ func (hls *NicoHls) saveMedia(seqno int, uri string) (is403, is404, is500 bool, return } -func (hls *NicoHls) getPlaylist(argUri *url.URL) (is403, isEnd, is500 bool, neterr, err error) { +func (hls *NicoHls) getPlaylist(argUri *url.URL) (is403, isEnd, isStop, is500 bool, neterr, err error) { u := argUri.String() m3u8, code, millisec, err, neterr := getString(u) if hls.nicoDebug { @@ -1236,6 +1246,10 @@ func (hls *NicoHls) getPlaylist(argUri *url.URL) (is403, isEnd, is500 bool, nete // prints Current SeqNo if hls.isTimeshift { sec := int(hls.playlist.position) + if hls.timeshiftStop != 0 && sec >= hls.timeshiftStop { + isStop = true + return + } var pos string if sec >= 3600 { pos += fmt.Sprintf("%02d:%02d:%02d", sec/3600, (sec%3600)/60, sec%60) @@ -1485,7 +1499,7 @@ func (hls *NicoHls) startPlaylist(uri string) { //fmt.Println(uri) - is403, isEnd, is500, neterr, err := hls.getPlaylist(uri) + is403, isEnd, isStop, is500, neterr, err := hls.getPlaylist(uri) if neterr != nil { if !hls.interrupted() { log.Println("playlist:", e) @@ -1510,6 +1524,9 @@ func (hls *NicoHls) startPlaylist(uri string) { if isEnd { return PLAYLIST_END } + if isStop { + return TIMESHIFT_STOP + } case <-sig: return GOT_SIGNAL diff --git a/src/options/options.go b/src/options/options.go index 09dffaf..4f89012 100644 --- a/src/options/options.go +++ b/src/options/options.go @@ -44,6 +44,7 @@ type Option struct { NicoHlsPort int NicoLimitBw int NicoTsStart float64 + NicoTsStop int NicoFormat string NicoFastTs bool NicoUltraFastTs bool @@ -400,6 +401,45 @@ func dbOpen() (db *sql.DB, err error) { return } +func parseTime(arg string) (ret int, err error) { + var hour, min, sec int + + if m := regexp.MustCompile(`^(\d+):(\d+):(\d+)$`).FindStringSubmatch(arg); len(m) > 0 { + hour, err = strconv.Atoi(m[1]) + if err != nil { + return + } + min, err = strconv.Atoi(m[2]) + if err != nil { + return + } + sec, err = strconv.Atoi(m[3]) + if err != nil { + return + } + } else if m := regexp.MustCompile(`^(\d+):(\d+)$`).FindStringSubmatch(arg); len(m) > 0 { + min, err = strconv.Atoi(m[1]) + if err != nil { + return + } + sec, err = strconv.Atoi(m[2]) + if err != nil { + return + } + } else if m := regexp.MustCompile(`^(\d+)$`).FindStringSubmatch(arg); len(m) > 0 { + sec, err = strconv.Atoi(m[1]) + if err != nil { + return + } + } else { + err = fmt.Errorf("regexp not matched") + } + + ret = hour * 3600 + min * 60 + sec + + return +} + func ParseArgs() (opt Option) { //dbAccountOpen() db, err := dbOpen() @@ -749,7 +789,7 @@ func ParseArgs() (opt Option) { if err != nil { return err } - num, err := strconv.Atoi(s) + num, err := parseTime(s) if err != nil { return fmt.Errorf("--nico-ts-start: Not a number %s\n", s) } @@ -761,11 +801,35 @@ func ParseArgs() (opt Option) { if err != nil { return err } - num, err := strconv.Atoi(s) + num, err := parseTime(s + ":0") if err != nil { return fmt.Errorf("--nico-ts-start-min: Not a number %s\n", s) } - opt.NicoTsStart = float64(num * 60) + opt.NicoTsStart = float64(num) + return nil + }}, + Parser{regexp.MustCompile(`\A(?i)--?nico-?ts-?stop\z`), func() (err error) { + s, err := nextArg() + if err != nil { + return err + } + num, err := parseTime(s) + if err != nil { + return fmt.Errorf("--nico-ts-stop: Not a number %s\n", s) + } + opt.NicoTsStop = num + return nil + }}, + Parser{regexp.MustCompile(`\A(?i)--?nico-?ts-?stop-?min\z`), func() (err error) { + s, err := nextArg() + if err != nil { + return err + } + num, err := parseTime(s + ":0") + if err != nil { + return fmt.Errorf("--nico-ts-stop-min: Not a number %s\n", s) + } + opt.NicoTsStop = num return nil }}, Parser{regexp.MustCompile(`\A(?i)--?nico-?(?:format|fmt)\z`), func() (err error) { From 7aa45edfb02e2a574b3691ae2e48b497e34484a2 Mon Sep 17 00:00:00 2001 From: nnn-revo2012 Date: Mon, 24 May 2021 02:42:27 +0900 Subject: [PATCH 05/39] Update changelog.txt and -h (help menu) --- changelog.txt | 16 ++++++++++++++++ src/options/options.go | 4 +++- 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/changelog.txt b/changelog.txt index 3e362c6..867fddb 100644 --- a/changelog.txt +++ b/changelog.txt @@ -1,5 +1,21 @@ 更新履歴 +・指定時間でタイムシフト録画を停止するためのパッチ（＋α） +https://egg.5ch.net/test/read.cgi/software/1595715643/163 + +オプション +　-nico-ts-start ＜num＞ +　　タイムシフトの録画を指定した再生時間（秒）から開始する +　-nico-ts-stop ＜num＞ +　　タイムシフトの録画を指定した再生時間（秒）で停止する +　上記2つは ＜分＞:＜秒＞ | ＜時＞:＜分＞:＜秒＞ の形式でも指定可能 + +　-nico-ts-start-min ＜num＞ +　　タイムシフトの録画を指定した再生時間（分）から開始する +　-nico-ts-stop-min ＜num＞ +　　タイムシフトの録画を指定した再生時間（分）で停止する +　上記2つは ＜時＞:＜分＞ の形式でも指定可能 + ・セルフ追っかけ再生機能その他 https://egg.5ch.net/test/read.cgi/software/1595715643/57 　例：http://127.0.0.1:12345/m3u8/2/1200/index.m3u8 diff --git a/src/options/options.go b/src/options/options.go index 4f89012..29647fa 100644 --- a/src/options/options.go +++ b/src/options/options.go @@ -137,6 +137,8 @@ COMMAND: -nico-skip-hb=off (+) コメント書き出し時に/hbコマンドも出す(デフォルト) -nico-ts-start タイムシフトの録画を指定した再生時間(秒)から開始する -nico-ts-start-min タイムシフトの録画を指定した再生時間(分)から開始する + -nico-ts-start タイムシフトの録画を指定した再生時間(秒)から開始する + -nico-ts-start-min タイムシフトの録画を指定した再生時間(分)から開始する -nico-conv-seqno-start MP4への変換を指定したセグメント番号から開始する -nico-conv-seqno-end MP4への変換を指定したセグメント番号で終了する -nico-conv-force-concat MP4への変換で画質変更または抜けがあっても分割しないように設定 @@ -172,7 +174,7 @@ HTTP関連 FILE: ニコニコ生放送/nicolive: - http://live2.nicovideo.jp/watch/lvXXXXXXXXX + https://live.nicovideo.jp/watch/lvXXXXXXXXX lvXXXXXXXXX ツイキャス/twitcasting: https://twitcasting.tv/XXXXX From 2345ded64e63fa6b7dce83aafe00b8bc6cf5a439 Mon Sep 17 00:00:00 2001 From: nnn-revo2012 Date: Thu, 6 May 2021 22:51:41 +0900 Subject: [PATCH 06/39] fix to save the name attribute of XML comments (used when performers make named comments) https://egg.5ch.net/test/read.cgi/software/1595715643/174 Patch applies only livedl.comment-name-attribute-r1.patch.gz https://egg.5ch.net/test/read.cgi/software/1595715643/194 --- src/niconico/nico_db.go | 37 ++++++++++++++++++++++++++++++++++--- src/niconico/nico_hls.go | 1 + 2 files changed, 35 insertions(+), 3 deletions(-) diff --git a/src/niconico/nico_db.go b/src/niconico/nico_db.go index fd50be0..82c6f90 100644 --- a/src/niconico/nico_db.go +++ b/src/niconico/nico_db.go @@ -26,6 +26,7 @@ var SelComment = `SELECT user_id, content, IFNULL(mail, "") AS mail, + %s IFNULL(premium, 0) AS premium, IFNULL(score, 0) AS score, thread, @@ -120,6 +121,7 @@ func (hls *NicoHls) dbCreate() (err error) { user_id TEXT NOT NULL, content TEXT NOT NULL, mail TEXT, + name TEXT, premium INTEGER, score INTEGER, thread TEXT, @@ -300,7 +302,22 @@ func (hls *NicoHls) dbGetFromWhen() (res_from int, when float64) { func WriteComment(db *sql.DB, fileName string, skipHb bool) { - rows, err := db.Query(SelComment) + var fSelComment = func(revision int) string { + var selAppend string + if revision >= 1 { + selAppend += `IFNULL(name, "") AS name,` + } + return fmt.Sprintf(SelComment, selAppend) + } + + var commentRevision int + var nameCount int64 + db.QueryRow(`SELECT COUNT(name) FROM pragma_table_info('comment') WHERE name = 'name'`).Scan(&nameCount) + if nameCount > 0 { + commentRevision = 1 + } + + rows, err := db.Query(fSelComment(commentRevision)) if err != nil { log.Println(err) return @@ -334,12 +351,13 @@ func WriteComment(db *sql.DB, fileName string, skipHb bool) { var user_id string var content string var mail string + var name string var premium int64 var score int64 var thread string var origin string var locale string - err = rows.Scan( + var dest0 = []interface{} { &vpos, &date, &date_usec, @@ -348,12 +366,19 @@ func WriteComment(db *sql.DB, fileName string, skipHb bool) { &user_id, &content, &mail, + } + var dest1 = []interface{} { &premium, &score, &thread, &origin, &locale, - ) + } + if commentRevision >= 1 { + dest0 = append(dest0, &name) + } + var dest = append(dest0, dest1...) + err = rows.Scan(dest...) if err != nil { log.Println(err) return @@ -389,6 +414,12 @@ func WriteComment(db *sql.DB, fileName string, skipHb bool) { mail = strings.Replace(mail, "<", "<", -1) line += fmt.Sprintf(` mail="%s"`, mail) } + if name != "" { + name = strings.Replace(name, `"`, """, -1) + name = strings.Replace(name, "&", "&", -1) + name = strings.Replace(name, "<", "<", -1) + line += fmt.Sprintf(` name="%s"`, name) + } if origin != "" { origin = strings.Replace(origin, `"`, """, -1) origin = strings.Replace(origin, "&", "&", -1) diff --git a/src/niconico/nico_hls.go b/src/niconico/nico_hls.go index 565ac2d..fa68f2d 100644 --- a/src/niconico/nico_hls.go +++ b/src/niconico/nico_hls.go @@ -373,6 +373,7 @@ func (hls *NicoHls) commentHandler(tag string, attr interface{}) (err error) { "user_id": attrMap["user_id"], "content": attrMap["content"], "mail": attrMap["mail"], + "name": attrMap["name"], "premium": attrMap["premium"], "score": attrMap["score"], "thread": thread, From f076c74297e1ee1f16c865a8709b92b0d992fffe Mon Sep 17 00:00:00 2001 From: nnn-revo2012 Date: Mon, 24 May 2021 03:19:14 +0900 Subject: [PATCH 07/39] Update changelog.txt --- changelog.txt | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/changelog.txt b/changelog.txt index 867fddb..cde8334 100644 --- a/changelog.txt +++ b/changelog.txt @@ -1,5 +1,10 @@ 更新履歴 +・XMLコメントのname属性(出演者が名前付きのコメントする時に使用)を保存するように修正 +https://egg.5ch.net/test/read.cgi/software/1595715643/174 +patch は livedl.comment-name-attribute-r1.patch.gz のみ適用 +https://egg.5ch.net/test/read.cgi/software/1595715643/194 + ・指定時間でタイムシフト録画を停止するためのパッチ（＋α） https://egg.5ch.net/test/read.cgi/software/1595715643/163 From 9f4d7ace24e38bf3f96e3240a7e5146826ec1db2 Mon Sep 17 00:00:00 2001 From: nnn-revo2012 Date: Mon, 24 May 2021 04:12:38 +0900 Subject: [PATCH 08/39] Add -http-timeout option: http connect and read timeout (Default is: 5) https://egg.5ch.net/test/read.cgi/software/1595715643/272 --- src/options/options.go | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/src/options/options.go b/src/options/options.go index 29647fa..c6ca768 100644 --- a/src/options/options.go +++ b/src/options/options.go @@ -10,13 +10,18 @@ import ( "regexp" "strconv" "strings" + "time" + "github.com/himananiito/livedl/buildno" "github.com/himananiito/livedl/cryptoconf" "github.com/himananiito/livedl/files" + "github.com/himananiito/livedl/httpbase" "golang.org/x/crypto/sha3" ) +const MinimumHttpTimeout = 5 + var DefaultTcasRetryTimeoutMinute = 5 // TcasRetryTimeoutMinute var DefaultTcasRetryInterval = 60 // TcasRetryInterval @@ -64,6 +69,8 @@ type Option struct { HttpSkipVerify bool HttpProxy string NoChdir bool + HttpTimeout int + } func getCmd() (cmd string) { @@ -1091,6 +1098,21 @@ func ParseArgs() (opt Option) { opt.NoChdir = true return }}, + Parser{regexp.MustCompile(`\A(?i)--?http-?timeout\z`), func() (err error) { + s, err := nextArg() + if err != nil { + return err + } + num, err := strconv.Atoi(s) + if err != nil { + return fmt.Errorf("--http-timeout: Not a number: %s\n", s) + } + if num < MinimumHttpTimeout { + return fmt.Errorf("--http-timeout: Invalid: %d: must be greater than or equal to %#v\n", num, MinimumHttpTimeout) + } + opt.HttpTimeout = num + return nil + }}, } checkFILE := func(arg string) bool { @@ -1176,6 +1198,11 @@ LB_ARG: opt.ConfFile = fmt.Sprintf("%s.conf", getCmd()) } + if opt.HttpTimeout == 0 { + opt.HttpTimeout = MinimumHttpTimeout + } + httpbase.Client.Timeout = time.Duration(opt.HttpTimeout) * time.Second + // [deprecated] // load session info if data, e := cryptoconf.Load(opt.ConfFile, opt.ConfPass); e != nil { @@ -1231,6 +1258,7 @@ LB_ARG: fmt.Printf("Conf(NicoConvSeqnoStart): %#v\n", opt.NicoConvSeqnoStart) } fmt.Printf("Conf(HttpSkipVerify): %#v\n", opt.HttpSkipVerify) + fmt.Printf("Conf(HttpTimeout): %#v\n", opt.HttpTimeout) if opt.NicoDebug { fmt.Printf("Conf(NicoDebug): %#v\n", opt.NicoDebug) From 4d9067a75d1ba656a07dabd9deca8c5db998c6eb Mon Sep 17 00:00:00 2001 From: nnn-revo2012 Date: Mon, 24 May 2021 04:17:25 +0900 Subject: [PATCH 09/39] Update changelog.txt and -h(help menu) --- changelog.txt | 3 +++ src/options/options.go | 1 + 2 files changed, 4 insertions(+) diff --git a/changelog.txt b/changelog.txt index cde8334..8b91c3e 100644 --- a/changelog.txt +++ b/changelog.txt @@ -1,5 +1,8 @@ 更新履歴 +・livedl で HTTP のタイムアウト時間を変更できるようにするパッチ +https://egg.5ch.net/test/read.cgi/software/1595715643/272 + ・XMLコメントのname属性(出演者が名前付きのコメントする時に使用)を保存するように修正 https://egg.5ch.net/test/read.cgi/software/1595715643/174 patch は livedl.comment-name-attribute-r1.patch.gz のみ適用 diff --git a/src/options/options.go b/src/options/options.go index c6ca768..b991f28 100644 --- a/src/options/options.go +++ b/src/options/options.go @@ -175,6 +175,7 @@ Youtube live録画用オプション: HTTP関連 -http-skip-verify=on (+) TLS証明書の認証をスキップする (32bit版対策) -http-skip-verify=off (+) TLS証明書の認証をスキップしない (デフォルト) + -http-timeout タイムアウト時間（秒）デフォルト: 5秒（最低値） (+)のついたオプションは、次回も同じ設定が使用されることを示す。 From b7a4fda2bb86191935424cf69f50ef7c9277c995 Mon Sep 17 00:00:00 2001 From: nnn-revo2012 Date: Sun, 13 Dec 2020 19:41:35 +0900 Subject: [PATCH 10/39] =?UTF-8?q?livedl=20=E3=81=A7=20YouTube=20Live=20?= =?UTF-8?q?=E3=82=92=E6=89=B1=E3=81=88=E3=82=8B=E3=82=88=E3=81=86=E3=81=AB?= =?UTF-8?q?=E3=81=99=E3=82=8B=E3=81=9F=E3=82=81=E3=81=AE=E3=83=91=E3=83=83?= =?UTF-8?q?=E3=83=81=EF=BC=88=E3=83=AA=E3=83=93=E3=82=B8=E3=83=A7=E3=83=B3?= =?UTF-8?q?1=EF=BC=89=20https://egg.5ch.net/test/read.cgi/software/1595715?= =?UTF-8?q?643/402=20https://egg.5ch.net/test/read.cgi/software/1595715643?= =?UTF-8?q?/406=20=E3=83=91=E3=83=83=E3=83=81=E3=81=AFlivedl.youtube-r1.pa?= =?UTF-8?q?tch=E3=81=AE=E3=81=BF=E9=81=A9=E7=94=A8?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Fixed youtube live 'ytplayer parse error'. Patch applies only livedl.youtube-r1.patch --- src/youtube/comment.go | 2 +- src/youtube/youtube.go | 14 +++++++------- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/src/youtube/comment.go b/src/youtube/comment.go index ac07608..c462f32 100644 --- a/src/youtube/comment.go +++ b/src/youtube/comment.go @@ -121,7 +121,7 @@ MAINLOOP: if !ok { continue } - message, _ := objs.FindString(liveChatMessageRenderer, "message", "simpleText") + message, _ := objs.FindString(liveChatMessageRenderer, "message", "runs", "text") timestampUsec, ok := objs.FindString(liveChatMessageRenderer, "timestampUsec") if !ok { continue diff --git a/src/youtube/youtube.go b/src/youtube/youtube.go index d60f868..f9bfaa0 100644 --- a/src/youtube/youtube.go +++ b/src/youtube/youtube.go @@ -53,7 +53,7 @@ var split = func(data []byte, atEOF bool) (advance int, token []byte, err error) func getChatContinuation(buff []byte) (isReplay bool, continuation string, err error) { - if ma := regexp.MustCompile(`(?s)\Wwindow\["ytInitialData"\]\p{Zs}*=\p{Zs}*({.*?})\p{Zs}*;`).FindSubmatch(buff); len(ma) > 1 { + if ma := regexp.MustCompile(`(?s)\WytInitialData\p{Zs}*=\p{Zs}*({.*?})\p{Zs}*;`).FindSubmatch(buff); len(ma) > 1 { var data interface{} err = json.Unmarshal(ma[1], &data) if err != nil { @@ -112,27 +112,27 @@ func getChatContinuation(buff []byte) (isReplay bool, continuation string, err e func getInfo(buff []byte) (title, ucid, author string, err error) { var data interface{} - re := regexp.MustCompile(`(?s)\Wytplayer\.config\p{Zs}*=\p{Zs}*({.*?})\p{Zs}*;`) + re := regexp.MustCompile(`(?s)\WytInitialPlayerResponse\p{Zs}*=\p{Zs}*({.*?})\p{Zs}*;`) if ma := re.FindSubmatch(buff); len(ma) > 1 { str := html.UnescapeString(string(ma[1])) if err = json.Unmarshal([]byte(str), &data); err != nil { - err = fmt.Errorf("ytplayer parse error") + err = fmt.Errorf("ytInitialPlayerResponse parse error") return } } else { - err = fmt.Errorf("ytplayer.config not found") + err = fmt.Errorf("ytInitialPlayerResponse not found") return } //objs.PrintAsJson(data); return - title, ok := objs.FindString(data, "args", "title") + title, ok := objs.FindString(data, "videoDetails", "title") if !ok { err = fmt.Errorf("title not found") return } - ucid, _ = objs.FindString(data, "args", "ucid") - author, _ = objs.FindString(data, "args", "author") + ucid, _ = objs.FindString(data, "videoDetails", "channelId") + author, _ = objs.FindString(data, "videoDetails", "author") return } From 14241cef319ea01b84169095b3e9b713435fe183 Mon Sep 17 00:00:00 2001 From: nnn-revo2012 Date: Sun, 13 Dec 2020 19:52:55 +0900 Subject: [PATCH 11/39] Update changelog.txt --- changelog.txt | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/changelog.txt b/changelog.txt index 8b91c3e..52ed650 100644 --- a/changelog.txt +++ b/changelog.txt @@ -1,5 +1,10 @@ 更新履歴 +・livedl で YouTube Live を扱えるようにするためのパッチ（リビジョン1） +patch は livedl.youtube-r1.patch のみ適用 +https://egg.5ch.net/test/read.cgi/software/1595715643/402 +https://egg.5ch.net/test/read.cgi/software/1595715643/406 + ・livedl で HTTP のタイムアウト時間を変更できるようにするパッチ https://egg.5ch.net/test/read.cgi/software/1595715643/272 From 6c5aaab753097396bc73911fbf1b039bcca69724 Mon Sep 17 00:00:00 2001 From: nnn-revo2012 Date: Thu, 6 May 2021 23:13:13 +0900 Subject: [PATCH 12/39] fix to be able to save some comments that failed to save https://egg.5ch.net/test/read.cgi/software/1595715643/457 --- src/niconico/nico_db.go | 2 +- src/niconico/nico_hls.go | 13 ++++++++----- 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/src/niconico/nico_db.go b/src/niconico/nico_db.go index 82c6f90..e93a800 100644 --- a/src/niconico/nico_db.go +++ b/src/niconico/nico_db.go @@ -292,7 +292,7 @@ func (hls *NicoHls) dbGetFromWhen() (res_from int, when float64) { var endTime float64 hls.db.QueryRow(`SELECT v FROM kvs WHERE k = "endTime"`).Scan(&endTime) - when = endTime + when = endTime + 3600 } else { when = float64(date2) / (1000 * 1000) } diff --git a/src/niconico/nico_hls.go b/src/niconico/nico_hls.go index fa68f2d..d14648d 100644 --- a/src/niconico/nico_hls.go +++ b/src/niconico/nico_hls.go @@ -334,8 +334,11 @@ func (hls *NicoHls) commentHandler(tag string, attr interface{}) (err error) { return } //fmt.Printf("%#v\n", attrMap) - if vpos_f, ok := attrMap["vpos"].(float64); ok { - vpos := int64(vpos_f) + if tag == "chat" { + var vpos int64 + if d, ok := attrMap["vpos"].(float64); ok { + vpos = int64(d) + } var date int64 if d, ok := attrMap["date"].(float64); ok { date = int64(d) @@ -364,9 +367,9 @@ func (hls *NicoHls) commentHandler(tag string, attr interface{}) (err error) { } hls.dbInsert("comment", map[string]interface{}{ - "vpos": attrMap["vpos"], - "date": attrMap["date"], - "date_usec": attrMap["date_usec"], + "vpos": vpos, + "date": date, + "date_usec": date_usec, "date2": date2, "no": attrMap["no"], "anonymity": attrMap["anonymity"], From 596c383b640aee3349f8fe2ef46722b9b4ce87a7 Mon Sep 17 00:00:00 2001 From: nnn-revo2012 Date: Sat, 2 Jan 2021 01:05:56 +0900 Subject: [PATCH 13/39] Remove VPOS > 0 (Commit 03417972d920cce0af92221583fc42bc559ef469) --- src/niconico/nico_db.go | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/niconico/nico_db.go b/src/niconico/nico_db.go index e93a800..19c3f4d 100644 --- a/src/niconico/nico_db.go +++ b/src/niconico/nico_db.go @@ -389,10 +389,6 @@ func WriteComment(db *sql.DB, fileName string, skipHb bool) { continue } - if vpos < 0 { - continue - } - line := fmt.Sprintf( ` Date: Mon, 24 May 2021 04:44:33 +0900 Subject: [PATCH 14/39] built 20210524.37 Update changelog.txt and buildno.go --- changelog.txt | 7 +++++++ src/buildno/buildno.go | 4 ++-- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/changelog.txt b/changelog.txt index 52ed650..33d055a 100644 --- a/changelog.txt +++ b/changelog.txt @@ -1,5 +1,12 @@ 更新履歴 +20210524.37 +・2021/01/01までに5chで公開されたniconico関係の全てのパッチを適用 + +・livedl で一部コメントが保存されないのを修正するパッチ +https://egg.5ch.net/test/read.cgi/software/1595715643/457 +　vposが0またはdate_usecが0の場合その要素自体が存在しないのでエラーになりコメントが保存されないのを修正 + ・livedl で YouTube Live を扱えるようにするためのパッチ（リビジョン1） patch は livedl.youtube-r1.patch のみ適用 https://egg.5ch.net/test/read.cgi/software/1595715643/402 diff --git a/src/buildno/buildno.go b/src/buildno/buildno.go index 921e109..5f9f60f 100644 --- a/src/buildno/buildno.go +++ b/src/buildno/buildno.go @@ -1,5 +1,5 @@ package buildno -var BuildDate = "20210516" -var BuildNo = "36" +var BuildDate = "20210524" +var BuildNo = "37" From 0d291323cc229378f78c104c241e10979dcbf5d7 Mon Sep 17 00:00:00 2001 From: nnn-revo2012 Date: Wed, 27 Jan 2021 23:45:46 +0900 Subject: [PATCH 15/39] =?UTF-8?q?livedl=20=E3=81=A7=20YouTubeLive=20?= =?UTF-8?q?=E3=83=AA=E3=83=97=E3=83=AC=E3=82=A4=E3=81=AE=E3=82=B3=E3=83=A1?= =?UTF-8?q?=E3=83=B3=E3=83=88=E3=81=8C=E5=8F=96=E3=82=8C=E3=82=8B=E3=82=88?= =?UTF-8?q?=E3=81=86=E7=9B=B4=E3=81=97=E3=81=9F=E3=82=88=20https://egg.5ch?= =?UTF-8?q?.net/test/read.cgi/software/1595715643/523?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Fixed YouTube Live comments being cut off in the middle and `json decode error`. --- src/youtube/comment.go | 47 ++++++++++++++++++++++++++++++------------ 1 file changed, 34 insertions(+), 13 deletions(-) diff --git a/src/youtube/comment.go b/src/youtube/comment.go index c462f32..80cebdb 100644 --- a/src/youtube/comment.go +++ b/src/youtube/comment.go @@ -9,9 +9,9 @@ import ( "os" "path/filepath" "strconv" - "strings" "sync" "time" + "html" "github.com/himananiito/livedl/files" "github.com/himananiito/livedl/gorman" @@ -51,7 +51,7 @@ MAINLOOP: timeoutMs, _done, err, neterr := func() (timeoutMs int, _done bool, err, neterr error) { var uri string if isReplay { - uri = fmt.Sprintf("https://www.youtube.com/live_chat_replay?continuation=%s&pbj=1", continuation) + uri = fmt.Sprintf("https://www.youtube.com/live_chat_replay/get_live_chat_replay?continuation=%s&pbj=1", continuation) } else { uri = fmt.Sprintf("https://www.youtube.com/live_chat/get_live_chat?continuation=%s&pbj=1", continuation) } @@ -67,8 +67,14 @@ MAINLOOP: return } if code != 200 { - neterr = fmt.Errorf("Status code: %v\n", code) - return + if code == 404 { + fmt.Printf("Status code: %v (ignored)\n", code) + time.Sleep(1000 * time.Millisecond) + return + } else { + neterr = fmt.Errorf("Status code: %v\n", code) + return + } } var data interface{} @@ -121,7 +127,18 @@ MAINLOOP: if !ok { continue } - message, _ := objs.FindString(liveChatMessageRenderer, "message", "runs", "text") + var message string + if runs, ok := objs.FindArray(liveChatMessageRenderer, "message", "runs"); ok { + for _, run := range runs { + if text, ok := objs.FindString(run, "text"); ok { + message += text + } else if emojis, ok := objs.FindArray(run, "emoji", "shortcuts"); ok { + if emoji, ok := emojis[0].(string); ok { + message += emoji + } + } + } + } timestampUsec, ok := objs.FindString(liveChatMessageRenderer, "timestampUsec") if !ok { continue @@ -129,7 +146,7 @@ MAINLOOP: if false { fmt.Printf("%v ", videoOffsetTimeMsec) - fmt.Printf("%v %v %v %v %v\n", timestampUsec, authorName, authorExternalChannelId, message, id) + fmt.Printf("%v %v %v %v %v %v\n", timestampUsec, count, authorName, authorExternalChannelId, message, id) } dbInsert(ctx, gm, db, mtx, @@ -272,9 +289,9 @@ func dbCreate(ctx context.Context, db *sql.DB) (err error) { _, err = db.ExecContext(ctx, ` CREATE UNIQUE INDEX IF NOT EXISTS comment0 ON comment(id); - CREATE UNIQUE INDEX IF NOT EXISTS comment1 ON comment(timestampUsec); - CREATE UNIQUE INDEX IF NOT EXISTS comment2 ON comment(videoOffsetTimeMsec); - CREATE UNIQUE INDEX IF NOT EXISTS comment3 ON comment(count); + CREATE INDEX IF NOT EXISTS comment1 ON comment(timestampUsec); + CREATE INDEX IF NOT EXISTS comment2 ON comment(videoOffsetTimeMsec); + CREATE INDEX IF NOT EXISTS comment3 ON comment(count); `) if err != nil { return @@ -337,7 +354,8 @@ var SelComment = `SELECT IFNULL(videoOffsetTimeMsec, -1), authorName, channelId, - message + message, + count FROM comment ORDER BY timestampUsec ` @@ -377,6 +395,7 @@ func WriteComment(db *sql.DB, fileName string) { var authorName string var channelId string var message string + var count int64 err = rows.Scan( ×tampUsec, @@ -384,6 +403,7 @@ func WriteComment(db *sql.DB, fileName string) { &authorName, &channelId, &message, + &count, ) if err != nil { log.Println(err) @@ -402,16 +422,17 @@ func WriteComment(db *sql.DB, fileName string) { } line := fmt.Sprintf( - ` Date: Wed, 27 Jan 2021 23:54:22 +0900 Subject: [PATCH 16/39] =?UTF-8?q?=E9=87=91=E9=A1=8D=E3=81=AE=E3=83=95?= =?UTF-8?q?=E3=82=A9=E3=83=BC=E3=83=9E=E3=83=83=E3=83=88=E3=81=AE=E8=A6=81?= =?UTF-8?q?=E6=9C=9B=E3=81=AA=E3=81=84=E3=81=BF=E3=81=9F=E3=81=84=E3=81=A0?= =?UTF-8?q?=E3=81=8B=E3=82=89=E3=81=93=E3=81=A3=E3=81=A1=E3=81=A7=E5=8B=9D?= =?UTF-8?q?=E6=89=8B=E3=81=AB=E6=B1=BA=E3=82=81=E3=81=95=E3=81=9B=E3=81=A6?= =?UTF-8?q?=E3=82=82=E3=82=89=E3=81=A3=E3=81=9F=E3=82=88=20https://egg.5ch?= =?UTF-8?q?.net/test/read.cgi/software/1595715643/543?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Added amount attribute to YouTube Live comment format --- src/youtube/comment.go | 22 +++++++++++++++++----- 1 file changed, 17 insertions(+), 5 deletions(-) diff --git a/src/youtube/comment.go b/src/youtube/comment.go index 80cebdb..e9812cf 100644 --- a/src/youtube/comment.go +++ b/src/youtube/comment.go @@ -139,6 +139,12 @@ MAINLOOP: } } } + var others string + var amount string + amount, _ = objs.FindString(liveChatMessageRenderer, "purchaseAmountText", "simpleText") + if amount != "" { + others += ` amount="` + html.EscapeString(amount) + `"` + } timestampUsec, ok := objs.FindString(liveChatMessageRenderer, "timestampUsec") if !ok { continue @@ -146,7 +152,7 @@ MAINLOOP: if false { fmt.Printf("%v ", videoOffsetTimeMsec) - fmt.Printf("%v %v %v %v %v %v\n", timestampUsec, count, authorName, authorExternalChannelId, message, id) + fmt.Printf("%v %v %v %v %v %v [%v ]\n", timestampUsec, count, authorName, authorExternalChannelId, message, id, others) } dbInsert(ctx, gm, db, mtx, @@ -157,6 +163,7 @@ MAINLOOP: authorExternalChannelId, message, continuation, + others, count, ) count++ @@ -280,6 +287,7 @@ func dbCreate(ctx context.Context, db *sql.DB) (err error) { channelId TEXT, message TEXT, continuation TEXT, + others TEXT, count INTEGER NOT NULL ) `) @@ -301,7 +309,7 @@ func dbCreate(ctx context.Context, db *sql.DB) (err error) { } func dbInsert(ctx context.Context, gm *gorman.GoroutineManager, db *sql.DB, mtx *sync.Mutex, - id, timestampUsec, videoOffsetTimeMsec, authorName, authorExternalChannelId, message, continuation string, count int) { + id, timestampUsec, videoOffsetTimeMsec, authorName, authorExternalChannelId, message, continuation, others string, count int) { usec, err := strconv.ParseInt(timestampUsec, 10, 64) if err != nil { @@ -321,14 +329,14 @@ func dbInsert(ctx context.Context, gm *gorman.GoroutineManager, db *sql.DB, mtx } query := `INSERT OR IGNORE INTO comment - (id, timestampUsec, videoOffsetTimeMsec, authorName, channelId, message, continuation, count) VALUES (?,?,?,?,?,?,?,?)` + (id, timestampUsec, videoOffsetTimeMsec, authorName, channelId, message, continuation, others, count) VALUES (?,?,?,?,?,?,?,?,?)` gm.Go(func(<-chan struct{}) int { mtx.Lock() defer mtx.Unlock() if _, err := db.ExecContext(ctx, query, - id, usec, offset, authorName, authorExternalChannelId, message, continuation, count, + id, usec, offset, authorName, authorExternalChannelId, message, continuation, others, count, ); err != nil { if err.Error() != "context canceled" { fmt.Println(err) @@ -355,6 +363,7 @@ var SelComment = `SELECT authorName, channelId, message, + others, count FROM comment ORDER BY timestampUsec @@ -395,6 +404,7 @@ func WriteComment(db *sql.DB, fileName string) { var authorName string var channelId string var message string + var others string var count int64 err = rows.Scan( @@ -403,6 +413,7 @@ func WriteComment(db *sql.DB, fileName string) { &authorName, &channelId, &message, + &others, &count, ) if err != nil { @@ -422,12 +433,13 @@ func WriteComment(db *sql.DB, fileName string) { } line := fmt.Sprintf( - ` Date: Thu, 28 Jan 2021 00:13:36 +0900 Subject: [PATCH 17/39] Update changelog.txt --- changelog.txt | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/changelog.txt b/changelog.txt index 33d055a..1c2f0e8 100644 --- a/changelog.txt +++ b/changelog.txt @@ -1,5 +1,17 @@ 更新履歴 +・金額のフォーマットの要望ないみたいだからこっちで勝手に決めさせてもらったよ +https://egg.5ch.net/test/read.cgi/software/1595715643/543 + +Youtube Liveのコメントにamount属性を追加 + +・livedl で YouTubeLive リプレイのコメントが取れるよう直したよ +https://egg.5ch.net/test/read.cgi/software/1595715643/523 + +Youtube Liveのコメントが途中で切れるのを修正 +Youtube liveのアーカイブダウンロード中に`json decode error'となり中断するのを修正 +(404エラーになる場合は少しwaitする) + 20210524.37 ・2021/01/01までに5chで公開されたniconico関係の全てのパッチを適用 From 7a89f0b59df35e61bdc29988b068f68f0699fbb9 Mon Sep 17 00:00:00 2001 From: nnn-revo2012 Date: Sun, 31 Jan 2021 18:08:23 +0900 Subject: [PATCH 18/39] =?UTF-8?q?livedl=20=E3=82=92=20YouTube=20Live=20?= =?UTF-8?q?=E3=81=AE=E7=9B=B4=E8=BF=91=E3=81=AE=E4=BB=95=E6=A7=98=E5=A4=89?= =?UTF-8?q?=E6=9B=B4=E3=81=AB=E5=AF=BE=E5=BF=9C=20https://egg.5ch.net/test?= =?UTF-8?q?/read.cgi/software/1595715643/559?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Fix Youtube Live API (Change endpoint) --- src/youtube/comment.go | 30 ++++++++++++++++++++++++------ 1 file changed, 24 insertions(+), 6 deletions(-) diff --git a/src/youtube/comment.go b/src/youtube/comment.go index e9812cf..b81930d 100644 --- a/src/youtube/comment.go +++ b/src/youtube/comment.go @@ -12,6 +12,7 @@ import ( "sync" "time" "html" + "io/ioutil" "github.com/himananiito/livedl/files" "github.com/himananiito/livedl/gorman" @@ -20,6 +21,8 @@ import ( _ "github.com/mattn/go-sqlite3" ) +type OBJ = map[string]interface{} + func getComment(gm *gorman.GoroutineManager, ctx context.Context, sig <-chan struct{}, isReplay bool, continuation, name string) (done bool) { dbName := files.ChangeExtention(name, "yt.sqlite3") @@ -51,21 +54,36 @@ MAINLOOP: timeoutMs, _done, err, neterr := func() (timeoutMs int, _done bool, err, neterr error) { var uri string if isReplay { - uri = fmt.Sprintf("https://www.youtube.com/live_chat_replay/get_live_chat_replay?continuation=%s&pbj=1", continuation) + uri = fmt.Sprintf("https://www.youtube.com/youtubei/v1/live_chat/get_live_chat_replay?key=AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8") } else { - uri = fmt.Sprintf("https://www.youtube.com/live_chat/get_live_chat?continuation=%s&pbj=1", continuation) + uri = fmt.Sprintf("https://www.youtube.com/youtubei/v1/live_chat/get_live_chat?key=AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8") } - code, buff, err, neterr := httpbase.GetBytes(uri, map[string]string{ - "Cookie": Cookie, + postData := OBJ{ + "context": OBJ{ + "client": OBJ{ + "clientName": "WEB", + "clientVersion": "2.20210128.02.00", + }, + }, + "continuation": continuation, + } + resp, err, neterr := httpbase.PostJson(uri, map[string]string { + "Cookie": Cookie, "User-Agent": UserAgent, - }) + }, postData) if err != nil { return } if neterr != nil { return } + buff, neterr := ioutil.ReadAll(resp.Body) + resp.Body.Close() + if neterr != nil { + return + } + code := resp.StatusCode if code != 200 { if code == 404 { fmt.Printf("Status code: %v (ignored)\n", code) @@ -84,7 +102,7 @@ MAINLOOP: return } - liveChatContinuation, ok := objs.Find(data, "response", "continuationContents", "liveChatContinuation") + liveChatContinuation, ok := objs.Find(data, "continuationContents", "liveChatContinuation") if !ok { err = fmt.Errorf("(response liveChatContinuation) not found") return From 272e7179f42a2f05d88a01fbd6b69f06b9b0a6d6 Mon Sep 17 00:00:00 2001 From: nnn-revo2012 Date: Sun, 31 Jan 2021 18:10:50 +0900 Subject: [PATCH 19/39] =?UTF-8?q?livedl=20=E3=81=A7-yt-no-streamlink=3Don?= =?UTF-8?q?=20-yt-no-youtube-dl=3Don=20=E3=81=8C=E6=8C=87=E5=AE=9A?= =?UTF-8?q?=E3=81=95=E3=82=8C=E3=81=9F=E3=81=A8=E3=81=8D=E3=80=81YouTube?= =?UTF-8?q?=20Live=20=E3=81=AE=E3=82=B3=E3=83=A1=E3=83=B3=E3=83=88?= =?UTF-8?q?=E3=82=92=E6=B0=B8=E4=B9=85=E3=81=AB=E5=8F=96=E5=BE=97=E3=81=97?= =?UTF-8?q?=E7=B6=9A=E3=81=91=E3=82=8B=E3=83=91=E3=83=83=E3=83=81=20https:?= =?UTF-8?q?//egg.5ch.net/test/read.cgi/software/1595715643/567?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add does not quit program when saving only Youtube Live's comments --- src/youtube/youtube.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/youtube/youtube.go b/src/youtube/youtube.go index f9bfaa0..fbeb17b 100644 --- a/src/youtube/youtube.go +++ b/src/youtube/youtube.go @@ -424,7 +424,9 @@ func Record(id string, ytNoStreamlink, ytNoYoutube_dl bool) (err error) { } } else { - gmCom.Cancel() + if !ytNoStreamlink || !ytNoYoutube_dl { + gmCom.Cancel() + } gmCom.Wait() } } From 249e1a2ac2ace6cfc3e971a839b3a7e58935a1df Mon Sep 17 00:00:00 2001 From: nnn-revo2012 Date: Sun, 6 Jun 2021 14:24:44 +0900 Subject: [PATCH 20/39] =?UTF-8?q?livedl=20=E3=81=A7=20YouTube=20Live=20?= =?UTF-8?q?=E3=81=AE=E3=82=A2=E3=83=BC=E3=82=AB=E3=82=A4=E3=83=96=E3=82=B3?= =?UTF-8?q?=E3=83=A1=E3=83=B3=E3=83=88=E3=81=AE=E5=8F=96=E5=BE=97=E9=96=8B?= =?UTF-8?q?=E5=A7=8B=E6=99=82=E5=88=BB=E3=82=92=E6=8C=87=E5=AE=9A=E3=81=99?= =?UTF-8?q?=E3=82=8B=E3=82=AA=E3=83=97=E3=82=B7=E3=83=A7=E3=83=B3=20https:?= =?UTF-8?q?//egg.5ch.net/test/read.cgi/software/1595715643/789?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add the option: chat start time of the archive comment of YouTube Live. --- src/livedl.go | 2 +- src/options/options.go | 13 +++++++++++++ src/youtube/comment.go | 14 +++++++++++--- src/youtube/youtube.go | 4 ++-- 4 files changed, 27 insertions(+), 6 deletions(-) diff --git a/src/livedl.go b/src/livedl.go index 4e8e632..d3be82a 100644 --- a/src/livedl.go +++ b/src/livedl.go @@ -129,7 +129,7 @@ func main() { } case "YOUTUBE": - err := youtube.Record(opt.YoutubeId, opt.YtNoStreamlink, opt.YtNoYoutubeDl) + err := youtube.Record(opt.YoutubeId, opt.YtNoStreamlink, opt.YtNoYoutubeDl, opt.YtCommentStart) if err != nil { fmt.Println(err) } diff --git a/src/options/options.go b/src/options/options.go index b991f28..a849f26 100644 --- a/src/options/options.go +++ b/src/options/options.go @@ -63,6 +63,7 @@ type Option struct { NicoConvSeqnoEnd int64 NicoForceResv bool // 終了番組の上書きタイムシフト予約 YtNoStreamlink bool + YtCommentStart float64 YtNoYoutubeDl bool NicoSkipHb bool // コメント出力時に/hbコマンドを出さない HttpRootCA string @@ -1051,6 +1052,18 @@ func ParseArgs() (opt Option) { } return nil }}, + Parser{regexp.MustCompile(`\A(?i)--?yt-?comment-?start\z`), func() (err error) { + s, err := nextArg() + if err != nil { + return err + } + num, err := parseTime(s) + if err != nil { + return fmt.Errorf("--yt-comment-start: Not a number %s\n", s) + } + opt.YtCommentStart = float64(num) + return nil + }}, Parser{regexp.MustCompile(`\A(?i)--?nico-?skip-?hb(?:=(on|off))?\z`), func() (err error) { if strings.EqualFold(match[1], "on") { opt.NicoSkipHb = true diff --git a/src/youtube/comment.go b/src/youtube/comment.go index b81930d..307cdf3 100644 --- a/src/youtube/comment.go +++ b/src/youtube/comment.go @@ -23,7 +23,7 @@ import ( type OBJ = map[string]interface{} -func getComment(gm *gorman.GoroutineManager, ctx context.Context, sig <-chan struct{}, isReplay bool, continuation, name string) (done bool) { +func getComment(gm *gorman.GoroutineManager, ctx context.Context, sig <-chan struct{}, isReplay bool, commentStart float64, continuation, name string) (done bool) { dbName := files.ChangeExtention(name, "yt.sqlite3") db, err := dbOpen(ctx, dbName) @@ -36,11 +36,12 @@ func getComment(gm *gorman.GoroutineManager, ctx context.Context, sig <-chan str mtx := &sync.Mutex{} testContinuation, count, _ := dbGetContinuation(ctx, db, mtx) - if testContinuation != "" { + if commentStart < 0.5 && testContinuation != "" { continuation = testContinuation } var printTime int64 + var isFirst bool = true MAINLOOP: for { @@ -68,6 +69,11 @@ MAINLOOP: }, "continuation": continuation, } + if !isFirst && isReplay && commentStart > 1.5 { + postData["currentPlayerState"] = OBJ{ + "playerOffsetMs": commentStart * 1000.0, + } + } resp, err, neterr := httpbase.PostJson(uri, map[string]string { "Cookie": Cookie, "User-Agent": UserAgent, @@ -108,7 +114,9 @@ MAINLOOP: return } - if actions, ok := objs.FindArray(liveChatContinuation, "actions"); ok { + if isFirst && isReplay && commentStart > 1.5 { + isFirst = false + } else if actions, ok := objs.FindArray(liveChatContinuation, "actions"); ok { var videoOffsetTimeMsec string for _, a := range actions { diff --git a/src/youtube/youtube.go b/src/youtube/youtube.go index fbeb17b..2412806 100644 --- a/src/youtube/youtube.go +++ b/src/youtube/youtube.go @@ -299,7 +299,7 @@ func execYoutube_dl(gm *gorman.GoroutineManager, uri, name string) (err error) { var COMMENT_DONE = 1000 -func Record(id string, ytNoStreamlink, ytNoYoutube_dl bool) (err error) { +func Record(id string, ytNoStreamlink, ytNoYoutube_dl bool, ytCommentStart float64) (err error) { uri := fmt.Sprintf("https://www.youtube.com/watch?v=%s", id) code, buff, err, neterr := httpbase.GetBytes(uri, map[string]string{ @@ -390,7 +390,7 @@ func Record(id string, ytNoStreamlink, ytNoYoutube_dl bool) (err error) { if continuation != "" { gmCom.Go(func(c <-chan struct{}) int { - getComment(gmCom, ctx, c, isReplay, continuation, origName) + getComment(gmCom, ctx, c, isReplay, ytCommentStart, continuation, origName) fmt.Printf("\ncomment done\n") return COMMENT_DONE }) From 4b07ee8a0ebd8d6fefd5600933f562e8fa3d04fa Mon Sep 17 00:00:00 2001 From: nnn-revo2012 Date: Mon, 7 Jun 2021 15:45:02 +0900 Subject: [PATCH 21/39] built 20210607.38 Update changelog.txt and -h (help menu) --- changelog.txt | 14 ++++++++++++++ src/buildno/buildno.go | 4 ++-- src/options/options.go | 3 +++ 3 files changed, 19 insertions(+), 2 deletions(-) diff --git a/changelog.txt b/changelog.txt index 1c2f0e8..042a6f1 100644 --- a/changelog.txt +++ b/changelog.txt @@ -1,5 +1,19 @@ 更新履歴 +20210607.38 +・livedl で YouTube Live のアーカイブコメントの取得開始時刻を指定するオプション +https://egg.5ch.net/test/read.cgi/software/1595715643/789 + +使用例： +　livedl -yt-comment-start 3:21:06 https://～ + 特殊例 0：続きからコメント取得 1：最初からコメント取得 + +・livedl で-yt-no-streamlink=on -yt-no-youtube-dl=on が指定されたとき、YouTube Live のコメントを永久に取得し続けるパッチ +https://egg.5ch.net/test/read.cgi/software/1595715643/567 + +・livedl を YouTube Live の直近の仕様変更に対応 +https://egg.5ch.net/test/read.cgi/software/1595715643/559 + ・金額のフォーマットの要望ないみたいだからこっちで勝手に決めさせてもらったよ https://egg.5ch.net/test/read.cgi/software/1595715643/543 diff --git a/src/buildno/buildno.go b/src/buildno/buildno.go index 5f9f60f..216999f 100644 --- a/src/buildno/buildno.go +++ b/src/buildno/buildno.go @@ -1,5 +1,5 @@ package buildno -var BuildDate = "20210524" -var BuildNo = "37" +var BuildDate = "20210607" +var BuildNo = "38" diff --git a/src/options/options.go b/src/options/options.go index a849f26..4f0af4c 100644 --- a/src/options/options.go +++ b/src/options/options.go @@ -166,6 +166,9 @@ Youtube live録画用オプション: -yt-no-streamlink=off (+) Streamlinkを使用する(デフォルト) -yt-no-youtube-dl=on (+) youtube-dlを使用しない -yt-no-youtube-dl=off (+) youtube-dlを使用する(デフォルト) + -yt-comment-start YouTube Liveアーカイブでコメント取得開始時間（秒）を指定 + ＜分＞:＜秒＞ | ＜時＞:＜分＞:＜秒＞ の形式でも指定可能 + 0：続きからコメント取得 1：最初からコメント取得 変換オプション: -extract-chunks=off (+) -d2mで動画ファイルに書き出す(デフォルト) From 9bdb3361bfe42e6a6c60003174821cd8c2e70ea6 Mon Sep 17 00:00:00 2001 From: nnn-revo2012 Date: Fri, 15 Oct 2021 06:41:28 +0900 Subject: [PATCH 22/39] =?UTF-8?q?livedl=E3=81=AE=E3=81=82=E3=82=8B?= =?UTF-8?q?=E3=83=87=E3=82=A3=E3=83=AC=E3=82=AF=E3=83=88=E3=83=AA=E4=BB=A5?= =?UTF-8?q?=E5=A4=96=E3=81=8B=E3=82=89=E5=AE=9F=E8=A1=8C=E3=81=99=E3=82=8B?= =?UTF-8?q?=E6=99=82=E3=82=AB=E3=83=AC=E3=83=B3=E3=83=88=E3=83=87=E3=82=A3?= =?UTF-8?q?=E3=83=AC=E3=82=AF=E3=83=88=E3=83=AA=E3=81=ABconf.db=E3=81=8C?= =?UTF-8?q?=E4=BD=9C=E6=88=90=E3=81=95=E3=82=8C=E3=82=8B=E3=81=AE=E3=82=92?= =?UTF-8?q?=E4=BF=AE=E6=AD=A3=20https://egg.5ch.net/test/read.cgi/software?= =?UTF-8?q?/1595715643/922?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Fix conf.db being created in the current directory when executing from a directory other than the one with livedl. --- src/livedl.go | 23 +++++++++++++++++++++-- src/options/options.go | 2 +- 2 files changed, 22 insertions(+), 3 deletions(-) diff --git a/src/livedl.go b/src/livedl.go index d3be82a..762bcd7 100644 --- a/src/livedl.go +++ b/src/livedl.go @@ -45,17 +45,36 @@ func main() { baseDir = filepath.Dir(pa) } - opt := options.ParseArgs() + // check option only -no-chdir + args := os.Args[1:] + nochdir := false + r := regexp.MustCompile(`\A(?i)--?no-?chdir\z`) + for _, s := range args { + if r.MatchString(s) { + nochdir = true + break + } + } // chdir if not disabled - if !opt.NoChdir { + if !nochdir { fmt.Printf("chdir: %s\n", baseDir) if e := os.Chdir(baseDir); e != nil { fmt.Println(e) return } + } else { + fmt.Printf("no chdir\n") + pwd, e := os.Getwd() + if e != nil { + fmt.Println(e) + return + } + fmt.Printf("read %s\n", filepath.FromSlash(pwd+"/conf.db")) } + opt := options.ParseArgs() + // http if opt.HttpRootCA != "" { if err := httpbase.SetRootCA(opt.HttpRootCA); err != nil { diff --git a/src/options/options.go b/src/options/options.go index 4f0af4c..86547e9 100644 --- a/src/options/options.go +++ b/src/options/options.go @@ -112,7 +112,7 @@ COMMAND: -h ヘルプを表示 -vh 全てのオプションを表示 -v バージョンを表示 - -no-chdir 起動する時chdirしない + -no-chdir 起動する時chdirしない(conf.dbは起動したディレクトリに作成されます) -- 後にオプションが無いことを指定 ニコニコ生放送録画用オプション: From be005a39595e49331ce81360e050f4f0ff5f7896 Mon Sep 17 00:00:00 2001 From: nnn-revo2012 Date: Sun, 17 Oct 2021 00:11:16 +0900 Subject: [PATCH 23/39] built 20211017.39 Update changelog.txt --- changelog.txt | 9 +++++++++ src/buildno/buildno.go | 4 ++-- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/changelog.txt b/changelog.txt index 042a6f1..360fbe6 100644 --- a/changelog.txt +++ b/changelog.txt @@ -1,5 +1,14 @@ 更新履歴 +20211017.39 +・livedlのあるディレクトリ以外から実行する時カレントディレクトリにconf.dbが作成されるのを修正 +https://egg.5ch.net/test/read.cgi/software/1595715643/922 +例: C:\bin\livedl\livedl.exe を D:\home\tmp をカレントディレクトリとして実行した場合、conf.dbは D:\home\tmp に作成されてしまう + +仕様：conf.dbは実行するlivedlと同じディレクトリに作成する + ただし、オプション -no-chdir が指定された場合はカレントディレクトリにconf.dbを作成する + (livedl実行ファイルがユーザ書き込み権限のないディレクトリにある場合を想定) + 20210607.38 ・livedl で YouTube Live のアーカイブコメントの取得開始時刻を指定するオプション https://egg.5ch.net/test/read.cgi/software/1595715643/789 diff --git a/src/buildno/buildno.go b/src/buildno/buildno.go index 216999f..e504fdd 100644 --- a/src/buildno/buildno.go +++ b/src/buildno/buildno.go @@ -1,5 +1,5 @@ package buildno -var BuildDate = "20210607" -var BuildNo = "38" +var BuildDate = "20211017" +var BuildNo = "39" From 79e4cabdf52bacc16dcf2f4ccb6b5d1e01a1ffe7 Mon Sep 17 00:00:00 2001 From: nnn-revo2012 Date: Sun, 15 May 2022 17:28:38 +0900 Subject: [PATCH 24/39] Replace live2.* -> live.* --- src/niconico/nico_hls.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/niconico/nico_hls.go b/src/niconico/nico_hls.go index d14648d..8f99908 100644 --- a/src/niconico/nico_hls.go +++ b/src/niconico/nico_hls.go @@ -701,7 +701,7 @@ func (hls *NicoHls) startComment(messageServerUri, threadId, waybackkey string) conn, _, err := websocket.DefaultDialer.Dial( messageServerUri, map[string][]string{ - "Origin": []string{"https://live2.nicovideo.jp"}, + "Origin": []string{"https://live.nicovideo.jp"}, "Sec-WebSocket-Protocol": []string{"msg.nicovideo.jp#json"}, "User-Agent": []string{httpbase.GetUserAgent()}, }, @@ -2047,7 +2047,7 @@ func getProps(opt options.Option) (props interface{}, isFlash, notLogin, tsRsv0, header["Cookie"] = "user_session=" + opt.NicoSession } - uri := fmt.Sprintf("https://live2.nicovideo.jp/watch/%s", opt.NicoLiveId) + uri := fmt.Sprintf("https://live.nicovideo.jp/watch/%s", opt.NicoLiveId) dat, _, _, err, neterr := getStringHeader(uri, header) if err != nil || neterr != nil { if err == nil { From cb515c94534729ff202373858c1c01db0845b6a0 Mon Sep 17 00:00:00 2001 From: nnn-revo2012 Date: Tue, 31 May 2022 20:55:55 +0900 Subject: [PATCH 25/39] =?UTF-8?q?=E9=9F=B3=E5=A3=B0=E3=81=AE=E3=81=BF?= =?UTF-8?q?=E9=8C=B2=E7=94=BB=E5=AF=BE=E5=BF=9C=E3=80=81-http-timeout?= =?UTF-8?q?=E8=A8=AD=E5=AE=9A=E4=BF=9D=E5=AD=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- changelog.txt | 6 ++++++ src/niconico/nico_hls.go | 16 +++++++++++++--- src/options/options.go | 19 ++++++++++++++----- 3 files changed, 33 insertions(+), 8 deletions(-) diff --git a/changelog.txt b/changelog.txt index 360fbe6..5d46436 100644 --- a/changelog.txt +++ b/changelog.txt @@ -1,5 +1,11 @@ 更新履歴 +2022xxxx.xx +・音声のみ録画対応 +　-nico-limit-bw に sound_only または audio_high を指定してください +・-http-timeout の設定を保存するように修正 +・live2.* -> live.* に修正 + 20211017.39 ・livedlのあるディレクトリ以外から実行する時カレントディレクトリにconf.dbが作成されるのを修正 https://egg.5ch.net/test/read.cgi/software/1595715643/922 diff --git a/src/niconico/nico_hls.go b/src/niconico/nico_hls.go index 8f99908..e5e857e 100644 --- a/src/niconico/nico_hls.go +++ b/src/niconico/nico_hls.go @@ -255,6 +255,16 @@ func NewHls(opt options.Option, prop map[string]interface{}) (hls *NicoHls, err files.MkdirByFileName(dbName) + var quality string + var limitBw int + if m := regexp.MustCompile(`^(sound_?only|audio_high)$`).FindStringSubmatch(opt.NicoLimitBw); len(m) > 0 { + quality = "audio_high" + limitBw = 0 + } else { + quality = "abr" + limitBw, _ = strconv.Atoi(opt.NicoLimitBw) + } + hls = &NicoHls{ wsapi: wsapi, @@ -262,7 +272,7 @@ func NewHls(opt options.Option, prop map[string]interface{}) (hls *NicoHls, err webSocketUrl: webSocketUrl, myUserId: myUserId, - quality: "abr", + quality: quality, dbName: dbName, isTimeshift: timeshift, @@ -270,8 +280,8 @@ func NewHls(opt options.Option, prop map[string]interface{}) (hls *NicoHls, err ultrafastTimeshift: opt.NicoUltraFastTs, NicoSession: opt.NicoSession, - limitBw: opt.NicoLimitBw, - limitBwOrig: opt.NicoLimitBw, + limitBw: limitBw, + limitBwOrig: limitBw, nicoDebug: opt.NicoDebug, gmPlst: gorman.WithChecker(func(c int) { hls.checkReturnCode(c) }), diff --git a/src/options/options.go b/src/options/options.go index 86547e9..46bf531 100644 --- a/src/options/options.go +++ b/src/options/options.go @@ -47,7 +47,7 @@ type Option struct { ZipFile string DBFile string NicoHlsPort int - NicoLimitBw int + NicoLimitBw string NicoTsStart float64 NicoTsStop int NicoFormat string @@ -130,6 +130,7 @@ COMMAND: -nico-rtmp-index [,] RTMP録画を行うメディアファイルの番号を指定 -nico-hls-port [実験的] ローカルなHLSサーバのポート番号 -nico-limit-bw (+) HLSのBANDWIDTHの上限値を指定する。0=制限なし + sound_only or audio_high = 音声のみ -nico-format "FORMAT" (+) 保存時のファイル名を指定する -nico-fast-ts 倍速タイムシフト録画を行う(新配信タイムシフト) -nico-fast-ts=on (+) 上記を有効に設定 @@ -179,7 +180,7 @@ Youtube live録画用オプション: HTTP関連 -http-skip-verify=on (+) TLS証明書の認証をスキップする (32bit版対策) -http-skip-verify=off (+) TLS証明書の認証をスキップしない (デフォルト) - -http-timeout タイムアウト時間（秒）デフォルト: 5秒（最低値） + -http-timeout (+) タイムアウト時間（秒）デフォルト: 5秒（最低値） (+)のついたオプションは、次回も同じ設定が使用されることを示す。 @@ -484,7 +485,8 @@ func ParseArgs() (opt Option) { IFNULL((SELECT v FROM conf WHERE k == "YtNoStreamlink"), 0), IFNULL((SELECT v FROM conf WHERE k == "YtNoYoutubeDl"), 0), IFNULL((SELECT v FROM conf WHERE k == "NicoSkipHb"), 0), - IFNULL((SELECT v FROM conf WHERE k == "HttpSkipVerify"), 0); + IFNULL((SELECT v FROM conf WHERE k == "HttpSkipVerify"), 0), + IFNULL((SELECT v FROM conf WHERE k == "HttpTimeout"), 0); `).Scan( &opt.NicoFormat, &opt.NicoLimitBw, @@ -506,6 +508,7 @@ func ParseArgs() (opt Option) { &opt.YtNoYoutubeDl, &opt.NicoSkipHb, &opt.HttpSkipVerify, + &opt.HttpTimeout, ) if err != nil { log.Println(err) @@ -790,11 +793,16 @@ func ParseArgs() (opt Option) { if err != nil { return err } - num, err := strconv.Atoi(s) + if m := regexp.MustCompile(`^(sound_?only|audio_high)$`).FindStringSubmatch(s); len(m) > 0 { + opt.NicoLimitBw = m[0] + dbConfSet(db, "NicoLimitBw", opt.NicoLimitBw) + return nil + } + _, err = strconv.Atoi(s) if err != nil { return fmt.Errorf("--nico-limit-bw: Not a number: %s\n", s) } - opt.NicoLimitBw = num + opt.NicoLimitBw = s dbConfSet(db, "NicoLimitBw", opt.NicoLimitBw) return nil }}, @@ -1128,6 +1136,7 @@ func ParseArgs() (opt Option) { return fmt.Errorf("--http-timeout: Invalid: %d: must be greater than or equal to %#v\n", num, MinimumHttpTimeout) } opt.HttpTimeout = num + dbConfSet(db, "HttpTimeout", opt.HttpTimeout) return nil }}, } From e327b7d5930c061aa21883825535f1f2508aeb17 Mon Sep 17 00:00:00 2001 From: nnn-revo2012 Date: Sun, 19 Jun 2022 14:21:40 +0900 Subject: [PATCH 26/39] =?UTF-8?q?=E9=9F=B3=E5=A3=B0=E3=81=AE=E3=81=BF?= =?UTF-8?q?=E9=8C=B2=E7=94=BB=E5=AF=BE=E5=BF=9C=20-=20=E3=82=AA=E3=83=97?= =?UTF-8?q?=E3=82=B7=E3=83=A7=E3=83=B3audio=5Fhigh=20or=20audio=5Fonly=20?= =?UTF-8?q?=E3=81=AB=E4=BF=AE=E6=AD=A3=20=20=20(audio=5Fhigh=E3=81=AF?= =?UTF-8?q?=E5=B8=AF=E5=9F=9F192kbps=E3=80=81audio=5Fonly=E3=81=AF?= =?UTF-8?q?=E5=B8=AF=E5=9F=9F96kbps)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- changelog.txt | 2 +- src/niconico/nico_hls.go | 4 ++-- src/options/options.go | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/changelog.txt b/changelog.txt index 5d46436..9df5379 100644 --- a/changelog.txt +++ b/changelog.txt @@ -2,7 +2,7 @@ 2022xxxx.xx ・音声のみ録画対応 -　-nico-limit-bw に sound_only または audio_high を指定してください +　-nico-limit-bw に audio_high または audio_only を指定してください ・-http-timeout の設定を保存するように修正 ・live2.* -> live.* に修正 diff --git a/src/niconico/nico_hls.go b/src/niconico/nico_hls.go index e5e857e..e55a5b3 100644 --- a/src/niconico/nico_hls.go +++ b/src/niconico/nico_hls.go @@ -257,8 +257,8 @@ func NewHls(opt options.Option, prop map[string]interface{}) (hls *NicoHls, err var quality string var limitBw int - if m := regexp.MustCompile(`^(sound_?only|audio_high)$`).FindStringSubmatch(opt.NicoLimitBw); len(m) > 0 { - quality = "audio_high" + if m := regexp.MustCompile(`^(audio_only|audio_high)$`).FindStringSubmatch(opt.NicoLimitBw); len(m) > 0 { + quality = m[0] limitBw = 0 } else { quality = "abr" diff --git a/src/options/options.go b/src/options/options.go index 46bf531..e656084 100644 --- a/src/options/options.go +++ b/src/options/options.go @@ -130,7 +130,7 @@ COMMAND: -nico-rtmp-index [,] RTMP録画を行うメディアファイルの番号を指定 -nico-hls-port [実験的] ローカルなHLSサーバのポート番号 -nico-limit-bw (+) HLSのBANDWIDTHの上限値を指定する。0=制限なし - sound_only or audio_high = 音声のみ + audio_high or audio_only = 音声のみ -nico-format "FORMAT" (+) 保存時のファイル名を指定する -nico-fast-ts 倍速タイムシフト録画を行う(新配信タイムシフト) -nico-fast-ts=on (+) 上記を有効に設定 @@ -793,7 +793,7 @@ func ParseArgs() (opt Option) { if err != nil { return err } - if m := regexp.MustCompile(`^(sound_?only|audio_high)$`).FindStringSubmatch(s); len(m) > 0 { + if m := regexp.MustCompile(`^(audio_only|audio_high)$`).FindStringSubmatch(s); len(m) > 0 { opt.NicoLimitBw = m[0] dbConfSet(db, "NicoLimitBw", opt.NicoLimitBw) return nil From 3ea12f47aba9c346611703807a6c71b92e3e85db Mon Sep 17 00:00:00 2001 From: nnn-revo2012 Date: Tue, 30 Aug 2022 17:13:46 +0900 Subject: [PATCH 27/39] =?UTF-8?q?=E3=82=B3=E3=83=A1=E3=83=B3=E3=83=88?= =?UTF-8?q?=E5=87=BA=E5=8A=9B=E6=99=82=E3=81=ABvpos=E3=82=92=E8=A3=9C?= =?UTF-8?q?=E6=AD=A3=E3=81=99=E3=82=8B=E6=A9=9F=E8=83=BD=E8=BF=BD=E5=8A=A0?= =?UTF-8?q?=20-=20-nico-adjust-vpos=3Don|off=20=E3=81=A7vpos=E3=82=92?= =?UTF-8?q?=E8=A3=9C=E6=AD=A3=E3=81=97=E3=80=81-1000=E3=82=88=E3=82=8A?= =?UTF-8?q?=E5=B0=8F=E3=81=95=E3=81=84vpos=E8=A1=8C=E3=81=AF=E5=87=BA?= =?UTF-8?q?=E5=8A=9B=E3=81=97=E3=81=AA=E3=81=84=20-=20=E8=A8=AD=E5=AE=9A?= =?UTF-8?q?=E3=81=AF=E4=BF=9D=E5=AD=98=E5=8F=AF=E8=83=BD=20-=20Conf?= =?UTF-8?q?=E3=81=AE=E4=B8=80=E8=A6=A7=E8=A1=A8=E7=A4=BA=E6=99=82=E3=81=AB?= =?UTF-8?q?true/false=E3=81=A7=E8=A1=A8=E7=A4=BA=20-=20ExtractChunks()?= =?UTF-8?q?=E3=82=82=E5=90=8C=E6=A7=98=E3=81=AB=E4=BF=AE=E6=AD=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/livedl.go | 6 +-- src/niconico/nico_db.go | 106 ++++++++++++++++++++++++++++++++++----- src/niconico/nico_hls.go | 2 +- src/options/options.go | 32 ++++++++++-- src/zip2mp4/zip2mp4.go | 35 ++++++++++--- 5 files changed, 155 insertions(+), 26 deletions(-) diff --git a/src/livedl.go b/src/livedl.go index 762bcd7..8f6f5bd 100644 --- a/src/livedl.go +++ b/src/livedl.go @@ -160,7 +160,7 @@ func main() { os.Exit(1) } if hlsPlaylistEnd && opt.NicoAutoConvert { - done, nMp4s, skipped, err := zip2mp4.ConvertDB(dbname, opt.ConvExt, opt.NicoSkipHb, opt.NicoConvForceConcat, opt.NicoConvSeqnoStart, opt.NicoConvSeqnoEnd) + done, nMp4s, skipped, err := zip2mp4.ConvertDB(dbname, opt.ConvExt, opt.NicoSkipHb, opt.NicoAdjustVpos, opt.NicoConvForceConcat, opt.NicoConvSeqnoStart, opt.NicoConvSeqnoEnd) if err != nil { fmt.Println(err) os.Exit(1) @@ -194,13 +194,13 @@ func main() { zip2mp4.YtComment(opt.DBFile) } else if opt.ExtractChunks { - if _, err := zip2mp4.ExtractChunks(opt.DBFile, opt.NicoSkipHb); err != nil { + if _, err := zip2mp4.ExtractChunks(opt.DBFile, opt.NicoSkipHb, opt.NicoAdjustVpos, opt.NicoConvSeqnoStart, opt.NicoConvSeqnoEnd); err != nil { fmt.Println(err) os.Exit(1) } } else { - if _, _, _, err := zip2mp4.ConvertDB(opt.DBFile, opt.ConvExt, opt.NicoSkipHb, opt.NicoConvForceConcat, opt.NicoConvSeqnoStart, opt.NicoConvSeqnoEnd); err != nil { + if _, _, _, err := zip2mp4.ConvertDB(opt.DBFile, opt.ConvExt, opt.NicoSkipHb, opt.NicoAdjustVpos, opt.NicoConvForceConcat, opt.NicoConvSeqnoStart, opt.NicoConvSeqnoEnd); err != nil { fmt.Println(err) os.Exit(1) } diff --git a/src/niconico/nico_db.go b/src/niconico/nico_db.go index 19c3f4d..7cd783a 100644 --- a/src/niconico/nico_db.go +++ b/src/niconico/nico_db.go @@ -39,15 +39,8 @@ func SelMediaF(seqnoStart, seqnoEnd int64) (ret string) { ret = `SELECT seqno, bandwidth, size, data FROM media WHERE IFNULL(notfound, 0) == 0 AND data IS NOT NULL` - - if seqnoStart > 0 { - ret += ` AND seqno >= ` + fmt.Sprint(seqnoStart) - } - - if seqnoEnd > 0 { - ret += ` AND seqno <= ` + fmt.Sprint(seqnoEnd) - } - + ret += ` AND seqno >= ` + fmt.Sprint(seqnoStart) + ret += ` AND seqno <= ` + fmt.Sprint(seqnoEnd) ret += ` ORDER BY seqno` return @@ -300,7 +293,19 @@ func (hls *NicoHls) dbGetFromWhen() (res_from int, when float64) { return } -func WriteComment(db *sql.DB, fileName string, skipHb bool) { +func dbadjustVpos(opentime, offset, date, vpos int64, providerType string) (ret int64) { + + ret = vpos + if providerType != "official" { + ret = (date - opentime) * 100 - offset + } else { + ret = vpos - offset + } + return ret + +} + +func WriteComment(db *sql.DB, fileName string, skipHb, adjustVpos bool, seqnoStart, seqnoEnd int64) { var fSelComment = func(revision int) string { var selAppend string @@ -317,6 +322,31 @@ func WriteComment(db *sql.DB, fileName string, skipHb bool) { commentRevision = 1 } + fmt.Println("commentRevision: ", commentRevision) + + //adjustVposの場合はkvsテーブルから読み込み + var openTime int64 + var providerType string + var offset int64 + if adjustVpos == true { + var t float64 + var sts string + db.QueryRow(`SELECT v FROM kvs WHERE k = 'providerType'`).Scan(&sts) + if sts == "ENDED" { + offset = seqnoStart * 500 + } else { + offset = seqnoStart * 150 + } + db.QueryRow(`SELECT v FROM kvs WHERE k = 'providerType'`).Scan(&providerType) + db.QueryRow(`SELECT v FROM kvs WHERE k = 'openTime'`).Scan(&t) + openTime = int64(t) + } + + fmt.Println("adjustVpos: ", adjustVpos) + fmt.Println("providerType: ", providerType) + fmt.Println("openTime: ", openTime) + fmt.Println("offset: ", offset) + rows, err := db.Query(fSelComment(commentRevision)) if err != nil { log.Println(err) @@ -389,6 +419,15 @@ func WriteComment(db *sql.DB, fileName string, skipHb bool) { continue } + // adjustVposの場合はvpos再計算 + // vposが-1000(-10秒)より小さい場合は出力しない + if adjustVpos == true { + vpos = dbadjustVpos(openTime, offset, date, vpos, providerType) + if vpos <= -1000 { + continue + } + } + line := fmt.Sprintf( ` タイムシフトの録画を指定した再生時間(秒)から開始する - -nico-ts-start-min タイムシフトの録画を指定した再生時間(分)から開始する - -nico-ts-start タイムシフトの録画を指定した再生時間(秒)から開始する - -nico-ts-start-min タイムシフトの録画を指定した再生時間(分)から開始する + -nico-adjust-vpos=on (+) コメント書き出し時にvposの値を補正する + vposの値が-1000より小さい場合はコメント出力しない + -nico-adjust-vpos=off (+) コメント書き出し時にvposの値をそのまま出力する(デフォルト) + -nico-ts-start タイムシフトの録画を指定した再生時間（秒）から開始する + -nico-ts-stop タイムシフトの録画を指定した再生時間（秒）で停止する + 上記2つは ＜分＞:＜秒＞ | ＜時＞:＜分＞:＜秒＞ の形式でも指定可能 + -nico-ts-start-min タイムシフトの録画を指定した再生時間（分）から開始する + -nico-ts-stop-min タイムシフトの録画を指定した再生時間（分）で停止する + 上記2つは ＜時＞:＜分＞ の形式でも指定可能 -nico-conv-seqno-start MP4への変換を指定したセグメント番号から開始する -nico-conv-seqno-end MP4への変換を指定したセグメント番号で終了する -nico-conv-force-concat MP4への変換で画質変更または抜けがあっても分割しないように設定 @@ -485,6 +491,7 @@ func ParseArgs() (opt Option) { IFNULL((SELECT v FROM conf WHERE k == "YtNoStreamlink"), 0), IFNULL((SELECT v FROM conf WHERE k == "YtNoYoutubeDl"), 0), IFNULL((SELECT v FROM conf WHERE k == "NicoSkipHb"), 0), + IFNULL((SELECT v FROM conf WHERE k == "NicoAdjustVpos"), 0), IFNULL((SELECT v FROM conf WHERE k == "HttpSkipVerify"), 0), IFNULL((SELECT v FROM conf WHERE k == "HttpTimeout"), 0); `).Scan( @@ -507,6 +514,7 @@ func ParseArgs() (opt Option) { &opt.YtNoStreamlink, &opt.YtNoYoutubeDl, &opt.NicoSkipHb, + &opt.NicoAdjustVpos, &opt.HttpSkipVerify, &opt.HttpTimeout, ) @@ -1139,6 +1147,19 @@ func ParseArgs() (opt Option) { dbConfSet(db, "HttpTimeout", opt.HttpTimeout) return nil }}, + Parser{regexp.MustCompile(`\A(?i)--?nico-?adjust-?vpos(?:=(on|off))?\z`), func() (err error) { + if strings.EqualFold(match[1], "on") { + opt.NicoAdjustVpos = true + dbConfSet(db, "NicoAdjustVpos", opt.NicoAdjustVpos) + } else if strings.EqualFold(match[1], "off") { + opt.HttpSkipVerify = false + dbConfSet(db, "NicoAdjustVpos", opt.NicoAdjustVpos) + } else { + opt.NicoAdjustVpos = true + } + + return + }}, } checkFILE := func(arg string) bool { @@ -1266,6 +1287,7 @@ LB_ARG: } fmt.Printf("Conf(NicoForceResv): %#v\n", opt.NicoForceResv) fmt.Printf("Conf(NicoSkipHb): %#v\n", opt.NicoSkipHb) + fmt.Printf("Conf(NicoAdjustVpos): %#v\n", opt.NicoAdjustVpos) case "YOUTUBE": fmt.Printf("Conf(YtNoStreamlink): %#v\n", opt.YtNoStreamlink) @@ -1279,6 +1301,8 @@ LB_ARG: fmt.Printf("Conf(ExtractChunks): %#v\n", opt.ExtractChunks) fmt.Printf("Conf(NicoConvForceConcat): %#v\n", opt.NicoConvForceConcat) fmt.Printf("Conf(ConvExt): %#v\n", opt.ConvExt) + fmt.Printf("Conf(NicoSkipHb): %#v\n", opt.NicoSkipHb) + fmt.Printf("Conf(NicoAdjustVpos): %#v\n", opt.NicoAdjustVpos) case "DB2HLS": fmt.Printf("Conf(NicoHlsPort): %#v\n", opt.NicoHlsPort) fmt.Printf("Conf(NicoConvSeqnoStart): %#v\n", opt.NicoConvSeqnoStart) diff --git a/src/zip2mp4/zip2mp4.go b/src/zip2mp4/zip2mp4.go index 49fc83e..f1037cc 100644 --- a/src/zip2mp4/zip2mp4.go +++ b/src/zip2mp4/zip2mp4.go @@ -441,16 +441,28 @@ func Convert(fileName string) (err error) { return } -func ExtractChunks(fileName string, skipHb bool) (done bool, err error) { +func ExtractChunks(fileName string, skipHb, adjustVpos bool, seqnoStart, seqnoEnd int64) (done bool, err error) { db, err := sql.Open("sqlite3", fileName) if err != nil { return } defer db.Close() - niconico.WriteComment(db, fileName, skipHb) + seqstart := niconico.DbGetFirstSeqNo(db, 1) + seqend := niconico.DbGetLastSeqNo(db, 1) - rows, err := db.Query(niconico.SelMedia) + if seqnoStart > 0 && seqnoStart > seqstart { + seqstart = seqnoStart + } + if seqnoEnd > 0 && seqnoEnd < seqend { + seqend = seqnoEnd + } + fmt.Println("seqstart: ", seqstart) + fmt.Println("seqend: ", seqend) + + niconico.WriteComment(db, fileName, skipHb, adjustVpos, seqstart, seqend) + + rows, err := db.Query(niconico.SelMediaF(seqstart, seqend)) if err != nil { return } @@ -496,14 +508,25 @@ func ExtractChunks(fileName string, skipHb bool) (done bool, err error) { return } -func ConvertDB(fileName, ext string, skipHb, forceConcat bool, seqnoStart, seqnoEnd int64) (done bool, nMp4s int, skipped bool, err error) { +func ConvertDB(fileName, ext string, skipHb, adjustVpos, forceConcat bool, seqnoStart, seqnoEnd int64) (done bool, nMp4s int, skipped bool, err error) { db, err := sql.Open("sqlite3", fileName) if err != nil { return } defer db.Close() - niconico.WriteComment(db, fileName, skipHb) + seqstart := niconico.DbGetFirstSeqNo(db, 1) + seqend := niconico.DbGetLastSeqNo(db, 1) + if seqnoStart > 0 && seqnoStart > seqstart { + seqstart = seqnoStart + } + if seqnoEnd > 0 && seqnoEnd < seqend { + seqend = seqnoEnd + } + fmt.Println("seqstart: ", seqstart) + fmt.Println("seqend: ", seqend) + + niconico.WriteComment(db, fileName, skipHb, adjustVpos, seqstart, seqend) var zm *ZipMp4 defer func() { @@ -516,7 +539,7 @@ func ConvertDB(fileName, ext string, skipHb, forceConcat bool, seqnoStart, seqno zm = &ZipMp4{ZipName: fileName} zm.OpenFFMpeg(ext) - rows, err := db.Query(niconico.SelMediaF(seqnoStart, seqnoEnd)) + rows, err := db.Query(niconico.SelMediaF(seqstart, seqend)) if err != nil { return } From d1bb03e3cf043d0475da626c2d7a67cb19a4f78b Mon Sep 17 00:00:00 2001 From: nnn-revo2012 Date: Tue, 30 Aug 2022 20:01:25 +0900 Subject: [PATCH 28/39] =?UTF-8?q?=E3=83=8B=E3=82=B3=E7=94=9F=E3=81=8A?= =?UTF-8?q?=E3=82=88=E3=81=B3Youtube=E3=82=B3=E3=83=A1=E3=83=B3=E3=83=88?= =?UTF-8?q?=E9=96=A2=E9=80=A3=E3=81=AE=E4=BF=AE=E6=AD=A3=20-=20livedl.exe?= =?UTF-8?q?=E3=81=A8sqlite3=E3=83=95=E3=82=A1=E3=82=A4=E3=83=AB=E3=81=8C?= =?UTF-8?q?=E5=88=A5=E3=81=AE=E3=83=95=E3=82=A9=E3=83=AB=E3=83=80=E3=83=BC?= =?UTF-8?q?=E3=81=AB=E3=81=82=E3=82=8B=E5=A0=B4=E5=90=88=E3=80=81=E3=82=B3?= =?UTF-8?q?=E3=83=A1=E3=83=B3=E3=83=88=E5=87=BA=E5=8A=9B=E6=99=82=E3=81=AB?= =?UTF-8?q?xml=E3=83=95=E3=82=A1=E3=82=A4=E3=83=AB=E3=81=AB-=E6=95=B0?= =?UTF-8?q?=E5=AD=97=E3=81=8C=E4=BB=98=E3=81=8B=E3=81=AA=E3=81=8B=E3=81=A3?= =?UTF-8?q?=E3=81=9F=E3=81=AE=E3=82=92=E4=BF=AE=E6=AD=A3=20=20=20(?= =?UTF-8?q?=E3=83=8B=E3=82=B3=E7=94=9F=E3=81=8A=E3=82=88=E3=81=B3Youtube)?= =?UTF-8?q?=20-=20Youtube=E3=81=AE=E3=82=B3=E3=83=A1=E3=83=B3=E3=83=88?= =?UTF-8?q?=E3=81=AE=E7=B5=B5=E6=96=87=E5=AD=97=E7=BD=AE=E6=8F=9B=E3=81=88?= =?UTF-8?q?(:a-z:)=E3=82=92=E5=87=BA=E5=8A=9B=E3=81=97=E3=81=AA=E3=81=84?= =?UTF-8?q?=E3=82=88=E3=81=86=E3=81=AB=E4=BF=AE=E6=AD=A3=20-=20=E3=83=8B?= =?UTF-8?q?=E3=82=B3=E7=94=9F=E3=81=AE=E3=83=AA=E3=82=A2=E3=83=AB=E3=82=BF?= =?UTF-8?q?=E3=82=A4=E3=83=A0=E6=94=BE=E9=80=81=E3=81=AEvpos=E8=A3=9C?= =?UTF-8?q?=E6=AD=A3=E3=81=AE=E8=A8=88=E7=AE=97=E6=96=B9=E6=B3=95=E3=82=92?= =?UTF-8?q?=E4=BF=AE=E6=AD=A3=20-=20-nico-adjust-vpos=3Doff=20=E3=81=AE?= =?UTF-8?q?=E6=99=82=E3=81=ABconf.db=E3=81=AB=E4=BF=9D=E5=AD=98=E3=81=95?= =?UTF-8?q?=E3=82=8C=E3=81=AA=E3=81=8B=E3=81=A3=E3=81=9F=E3=81=AE=E3=82=92?= =?UTF-8?q?=E4=BF=AE=E6=AD=A3=20-=20=E3=83=8B=E3=82=B3=E7=94=9F=E3=81=AE?= =?UTF-8?q?=E3=83=AA=E3=82=A2=E3=83=AB=E3=82=BF=E3=82=A4=E3=83=A0=E6=94=BE?= =?UTF-8?q?=E9=80=81=E3=81=AE=E5=88=9D=E6=9C=9F=E3=82=B3=E3=83=A1=E3=83=B3?= =?UTF-8?q?=E3=83=88=E5=8F=96=E5=BE=97=E6=95=B0=E3=82=92100=E3=81=AB?= =?UTF-8?q?=E5=A4=89=E6=9B=B4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/niconico/nico_db.go | 23 ++++++++++++----------- src/niconico/nico_hls.go | 2 +- src/options/options.go | 4 ++-- src/youtube/comment.go | 18 ++++++++++-------- 4 files changed, 25 insertions(+), 22 deletions(-) diff --git a/src/niconico/nico_db.go b/src/niconico/nico_db.go index 7cd783a..f276dfa 100644 --- a/src/niconico/nico_db.go +++ b/src/niconico/nico_db.go @@ -5,7 +5,6 @@ import ( "fmt" "log" "os" - "path/filepath" "strings" "time" @@ -331,15 +330,20 @@ func WriteComment(db *sql.DB, fileName string, skipHb, adjustVpos bool, seqnoSta if adjustVpos == true { var t float64 var sts string - db.QueryRow(`SELECT v FROM kvs WHERE k = 'providerType'`).Scan(&sts) + var serverTime int64 + db.QueryRow(`SELECT v FROM kvs WHERE k = 'serverTime'`).Scan(&t) + serverTime = int64(t) + db.QueryRow(`SELECT v FROM kvs WHERE k = 'openTime'`).Scan(&t) + openTime = int64(t) + db.QueryRow(`SELECT v FROM kvs WHERE k = 'status'`).Scan(&sts) if sts == "ENDED" { - offset = seqnoStart * 500 + offset = seqnoStart * 500 //timeshift } else { - offset = seqnoStart * 150 + offset = (serverTime/10) - (openTime*100) } db.QueryRow(`SELECT v FROM kvs WHERE k = 'providerType'`).Scan(&providerType) - db.QueryRow(`SELECT v FROM kvs WHERE k = 'openTime'`).Scan(&t) - openTime = int64(t) + fmt.Println("serverTime: ", serverTime) + fmt.Println("status: ", sts) } fmt.Println("adjustVpos: ", adjustVpos) @@ -355,15 +359,12 @@ func WriteComment(db *sql.DB, fileName string, skipHb, adjustVpos bool, seqnoSta defer rows.Close() fileName = files.ChangeExtention(fileName, "xml") - - dir := filepath.Dir(fileName) - base := filepath.Base(fileName) - base, err = files.GetFileNameNext(base) + fileName, err = files.GetFileNameNext(fileName) + fmt.Println("xml file: ", fileName) if err != nil { fmt.Println(err) os.Exit(1) } - fileName = filepath.Join(dir, base) f, err := os.Create(fileName) if err != nil { log.Fatalln(err) diff --git a/src/niconico/nico_hls.go b/src/niconico/nico_hls.go index 1cd89e7..d785613 100644 --- a/src/niconico/nico_hls.go +++ b/src/niconico/nico_hls.go @@ -838,7 +838,7 @@ func (hls *NicoHls) startComment(messageServerUri, threadId, waybackkey string) OBJ{"thread": OBJ{ "fork": 0, "nicoru": 0, - "res_from": -1000, + "res_from": -100, "scores": 1, "thread": threadId, "user_id": hls.myUserId, diff --git a/src/options/options.go b/src/options/options.go index edb7e25..78b4ede 100644 --- a/src/options/options.go +++ b/src/options/options.go @@ -1152,10 +1152,10 @@ func ParseArgs() (opt Option) { opt.NicoAdjustVpos = true dbConfSet(db, "NicoAdjustVpos", opt.NicoAdjustVpos) } else if strings.EqualFold(match[1], "off") { - opt.HttpSkipVerify = false + opt.NicoAdjustVpos = false dbConfSet(db, "NicoAdjustVpos", opt.NicoAdjustVpos) } else { - opt.NicoAdjustVpos = true + opt.NicoAdjustVpos = false } return diff --git a/src/youtube/comment.go b/src/youtube/comment.go index 307cdf3..d2564a8 100644 --- a/src/youtube/comment.go +++ b/src/youtube/comment.go @@ -7,12 +7,12 @@ import ( "fmt" "log" "os" - "path/filepath" "strconv" "sync" "time" "html" "io/ioutil" + "regexp" "github.com/himananiito/livedl/files" "github.com/himananiito/livedl/gorman" @@ -397,6 +397,8 @@ var SelComment = `SELECT func WriteComment(db *sql.DB, fileName string) { + regexp1 := regexp.MustCompile(":[a-zA-Z0-9\\-\\_]*:") + rows, err := db.Query(SelComment) if err != nil { log.Println(err) @@ -405,15 +407,12 @@ func WriteComment(db *sql.DB, fileName string) { defer rows.Close() fileName = files.ChangeExtention(fileName, "xml") - - dir := filepath.Dir(fileName) - base := filepath.Base(fileName) - base, err = files.GetFileNameNext(base) + fileName, err = files.GetFileNameNext(fileName) + fmt.Println("xml file: ", fileName) if err != nil { fmt.Println(err) os.Exit(1) } - fileName = filepath.Join(dir, base) f, err := os.Create(fileName) if err != nil { log.Fatalln(err) @@ -470,8 +469,11 @@ func WriteComment(db *sql.DB, fileName string) { ) line += ">" - message = html.EscapeString(message) - line += message + message = regexp1.ReplaceAllString(message, "") + if len(message) <= 0 { + message = "　" + } + line += html.EscapeString(message) line += "" fmt.Fprintf(f, "%s\r\n", line) } From eb9eaafe6801a2b1b6a9abf1d6f5e43b5ec1b914 Mon Sep 17 00:00:00 2001 From: nnn-revo2012 Date: Thu, 1 Sep 2022 21:40:08 +0900 Subject: [PATCH 29/39] =?UTF-8?q?=E3=83=8B=E3=82=B3=E7=94=9F=E3=81=8A?= =?UTF-8?q?=E3=82=88=E3=81=B3Youtube=E3=81=AE=E3=82=B3=E3=83=A1=E3=83=B3?= =?UTF-8?q?=E3=83=88=E9=96=A2=E9=80=A3=E3=81=AE=E4=BF=AE=E6=AD=A3=20-=20-y?= =?UTF-8?q?t-emoji=3Don|off=E3=81=A7Youtube=E3=81=AE=E3=82=B3=E3=83=A1?= =?UTF-8?q?=E3=83=B3=E3=83=88=E3=81=ABemoji=E3=82=92=E5=87=BA=E5=8A=9B?= =?UTF-8?q?=E3=81=99=E3=82=8B/=E3=81=97=E3=81=AA=E3=81=84=E3=82=92?= =?UTF-8?q?=E8=A8=AD=E5=AE=9A=20=20=20=E8=A8=AD=E5=AE=9A=E3=81=AFconf.db?= =?UTF-8?q?=E3=81=AB=E4=BF=9D=E5=AD=98=E3=81=95=E3=82=8C=E3=82=8B=E3=80=81?= =?UTF-8?q?=E3=83=87=E3=83=95=E3=82=A9=E3=83=AB=E3=83=88=E3=81=AFon=20-=20?= =?UTF-8?q?=E3=83=8B=E3=82=B3=E7=94=9F=E3=81=AE=E3=83=AA=E3=82=A2=E3=83=AB?= =?UTF-8?q?=E3=82=BF=E3=82=A4=E3=83=A0=E6=94=BE=E9=80=81=E3=81=AE=E5=A0=B4?= =?UTF-8?q?=E5=90=88=E3=81=AF=E6=9C=80=E5=88=9D=E3=81=AE=E3=81=BFkvs?= =?UTF-8?q?=E3=83=87=E3=83=BC=E3=82=BF=E3=82=92=E6=9B=B8=E3=81=8D=E8=BE=BC?= =?UTF-8?q?=E3=82=80=E3=82=88=E3=81=86=E3=81=AB=E4=BF=AE=E6=AD=A3=20=20=20?= =?UTF-8?q?(=E5=86=8D=E6=8E=A5=E7=B6=9A=E3=81=AA=E3=81=A9=E3=81=A7serverTi?= =?UTF-8?q?me=E3=81=8Cupdate=E3=81=95=E3=82=8C=E3=82=8B=E3=81=A8=E3=82=B3?= =?UTF-8?q?=E3=83=A1=E3=83=B3=E3=83=88vpos=E8=A3=9C=E6=AD=A3=E3=81=8C?= =?UTF-8?q?=E3=81=8A=E3=81=8B=E3=81=97=E3=81=8F=E3=81=AA=E3=82=8B=E3=81=9F?= =?UTF-8?q?=E3=82=81)=20-=20kvs=E9=96=A2=E4=BF=82=E3=81=AE=E3=83=87?= =?UTF-8?q?=E3=83=BC=E3=82=BF=E8=AA=AD=E3=81=BF=E6=9B=B8=E3=81=8D=E9=96=A2?= =?UTF-8?q?=E6=95=B0=E3=81=AE=E8=BF=BD=E5=8A=A0=E3=83=BB=E4=BF=AE=E6=AD=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/livedl.go | 2 +- src/niconico/nico_db.go | 28 +++++++++++++++++++++++----- src/niconico/nico_hls.go | 22 +++++++++++++++------- src/options/options.go | 22 ++++++++++++++++++++-- src/youtube/comment.go | 6 ++++-- src/zip2mp4/zip2mp4.go | 4 ++-- 6 files changed, 65 insertions(+), 19 deletions(-) diff --git a/src/livedl.go b/src/livedl.go index 8f6f5bd..8b1abde 100644 --- a/src/livedl.go +++ b/src/livedl.go @@ -191,7 +191,7 @@ func main() { case "DB2MP4": if strings.HasSuffix(opt.DBFile, ".yt.sqlite3") { - zip2mp4.YtComment(opt.DBFile) + zip2mp4.YtComment(opt.DBFile, opt.YtEmoji) } else if opt.ExtractChunks { if _, err := zip2mp4.ExtractChunks(opt.DBFile, opt.NicoSkipHb, opt.NicoAdjustVpos, opt.NicoConvSeqnoStart, opt.NicoConvSeqnoEnd); err != nil { diff --git a/src/niconico/nico_db.go b/src/niconico/nico_db.go index f276dfa..1fe4cb0 100644 --- a/src/niconico/nico_db.go +++ b/src/niconico/nico_db.go @@ -228,6 +228,20 @@ func (hls *NicoHls) dbKVSet(k string, v interface{}) { }) } +func (hls *NicoHls) dbKVExist(k string) (res int){ + hls.dbMtx.Lock() + defer hls.dbMtx.Unlock() + query := `SELECT COUNT(*) FROM kvs WHERE k = ?` + hls.db.QueryRow(query, k).Scan(&res) + return +} + +func DbKVGet(db *sql.DB, k string) (v interface{}){ + query := `SELECT v FROM kvs WHERE k = ?` + db.QueryRow(query, k).Scan(&v) + return +} + func (hls *NicoHls) dbInsertReplaceOrIgnore(table string, data map[string]interface{}, replace bool) { var keys []string var qs []string @@ -331,17 +345,21 @@ func WriteComment(db *sql.DB, fileName string, skipHb, adjustVpos bool, seqnoSta var t float64 var sts string var serverTime int64 - db.QueryRow(`SELECT v FROM kvs WHERE k = 'serverTime'`).Scan(&t) + //db.QueryRow(`SELECT v FROM kvs WHERE k = 'serverTime'`).Scan(&t) + t = DbKVGet(db, "serverTime").(float64) serverTime = int64(t) - db.QueryRow(`SELECT v FROM kvs WHERE k = 'openTime'`).Scan(&t) + //db.QueryRow(`SELECT v FROM kvs WHERE k = 'openTime'`).Scan(&t) + t = DbKVGet(db, "openTime").(float64) openTime = int64(t) - db.QueryRow(`SELECT v FROM kvs WHERE k = 'status'`).Scan(&sts) + //db.QueryRow(`SELECT v FROM kvs WHERE k = 'status'`).Scan(&sts) + sts = DbKVGet(db, "status").(string) if sts == "ENDED" { offset = seqnoStart * 500 //timeshift } else { - offset = (serverTime/10) - (openTime*100) + offset = (serverTime/10) - (openTime*100) //on_air } - db.QueryRow(`SELECT v FROM kvs WHERE k = 'providerType'`).Scan(&providerType) + //db.QueryRow(`SELECT v FROM kvs WHERE k = 'providerType'`).Scan(&providerType) + providerType = DbKVGet(db, "providerType").(string) fmt.Println("serverTime: ", serverTime) fmt.Println("status: ", sts) } diff --git a/src/niconico/nico_hls.go b/src/niconico/nico_hls.go index d785613..ba40595 100644 --- a/src/niconico/nico_hls.go +++ b/src/niconico/nico_hls.go @@ -317,10 +317,14 @@ func NewHls(opt options.Option, prop map[string]interface{}) (hls *NicoHls, err // 放送情報をdbに入れる。自身のユーザ情報は入れない // dbに入れたくないデータはキーの先頭を//としている - for k, v := range prop { - if !strings.HasPrefix(k, "//") { - hls.dbKVSet(k, v) + // 生放送の場合はデータが既にあればupdateしない + if hls.isTimeshift || hls.dbKVExist("serverTime") == 0 { + for k, v := range prop { + if (! strings.HasPrefix(k, "//")) { + hls.dbKVSet(k, v) + } } + fmt.Println("Write dbKVSet") } return @@ -395,10 +399,14 @@ func (hls *NicoHls) commentHandler(tag string, attr interface{}) (err error) { "hash": hash, }) } else { - if d, ok := attrMap["thread"].(float64); ok { - hls.dbKVSet("comment/thread", fmt.Sprintf("%.f", d)) - } else if s, ok := attrMap["thread"].(string); ok { - hls.dbKVSet("comment/thread", s) + // 生放送の場合はデータが既にあればupdateしない + if hls.isTimeshift || hls.dbKVExist("comment/thread") == 0 { + if d, ok := attrMap["thread"].(float64); ok { + hls.dbKVSet("comment/thread", fmt.Sprintf("%.f", d)) + } else if s, ok := attrMap["thread"].(string); ok { + hls.dbKVSet("comment/thread", s) + } + fmt.Println("Write dbKVSet(command/thread)") } } diff --git a/src/options/options.go b/src/options/options.go index 78b4ede..c28f4b3 100644 --- a/src/options/options.go +++ b/src/options/options.go @@ -65,6 +65,7 @@ type Option struct { YtNoStreamlink bool YtCommentStart float64 YtNoYoutubeDl bool + YtEmoji bool NicoSkipHb bool // コメント出力時に/hbコマンドを出さない NicoAdjustVpos bool // コメント出力時にvposを補正する HttpRootCA string @@ -176,6 +177,8 @@ Youtube live録画用オプション: -yt-comment-start YouTube Liveアーカイブでコメント取得開始時間（秒）を指定 ＜分＞:＜秒＞ | ＜時＞:＜分＞:＜秒＞ の形式でも指定可能 0：続きからコメント取得 1：最初からコメント取得 + -yt-emoji=on (+) コメントにemojiを表示する(ディフォルト) + -yt-emoji=off (+) コメントにemojiを表示しない 変換オプション: -extract-chunks=off (+) -d2mで動画ファイルに書き出す(デフォルト) @@ -490,10 +493,11 @@ func ParseArgs() (opt Option) { IFNULL((SELECT v FROM conf WHERE k == "NicoForceResv"), 0), IFNULL((SELECT v FROM conf WHERE k == "YtNoStreamlink"), 0), IFNULL((SELECT v FROM conf WHERE k == "YtNoYoutubeDl"), 0), + IFNULL((SELECT v FROM conf WHERE k == "YtEmoji"), 1), IFNULL((SELECT v FROM conf WHERE k == "NicoSkipHb"), 0), IFNULL((SELECT v FROM conf WHERE k == "NicoAdjustVpos"), 0), IFNULL((SELECT v FROM conf WHERE k == "HttpSkipVerify"), 0), - IFNULL((SELECT v FROM conf WHERE k == "HttpTimeout"), 0); + IFNULL((SELECT v FROM conf WHERE k == "HttpTimeout"), 5); `).Scan( &opt.NicoFormat, &opt.NicoLimitBw, @@ -513,6 +517,7 @@ func ParseArgs() (opt Option) { &opt.NicoForceResv, &opt.YtNoStreamlink, &opt.YtNoYoutubeDl, + &opt.YtEmoji, &opt.NicoSkipHb, &opt.NicoAdjustVpos, &opt.HttpSkipVerify, @@ -1071,6 +1076,18 @@ func ParseArgs() (opt Option) { } return nil }}, + Parser{regexp.MustCompile(`\A(?i)--?yt-?emoji(?:=(on|off))?\z`), func() (err error) { + if strings.EqualFold(match[1], "on") { + opt.YtEmoji = true + dbConfSet(db, "YtEmoji", opt.YtEmoji) + } else if strings.EqualFold(match[1], "off") { + opt.YtEmoji = false + dbConfSet(db, "YtEmoji", opt.YtEmoji) + } else { + opt.YtEmoji = true + } + return + }}, Parser{regexp.MustCompile(`\A(?i)--?yt-?comment-?start\z`), func() (err error) { s, err := nextArg() if err != nil { @@ -1157,7 +1174,6 @@ func ParseArgs() (opt Option) { } else { opt.NicoAdjustVpos = false } - return }}, } @@ -1292,6 +1308,7 @@ LB_ARG: case "YOUTUBE": fmt.Printf("Conf(YtNoStreamlink): %#v\n", opt.YtNoStreamlink) fmt.Printf("Conf(YtNoYoutubeDl): %#v\n", opt.YtNoYoutubeDl) + fmt.Printf("Conf(YtEmoji): %#v\n", opt.YtEmoji) case "TWITCAS": fmt.Printf("Conf(TcasRetry): %#v\n", opt.TcasRetry) @@ -1303,6 +1320,7 @@ LB_ARG: fmt.Printf("Conf(ConvExt): %#v\n", opt.ConvExt) fmt.Printf("Conf(NicoSkipHb): %#v\n", opt.NicoSkipHb) fmt.Printf("Conf(NicoAdjustVpos): %#v\n", opt.NicoAdjustVpos) + fmt.Printf("Conf(YtEmoji): %#v\n", opt.YtEmoji) case "DB2HLS": fmt.Printf("Conf(NicoHlsPort): %#v\n", opt.NicoHlsPort) fmt.Printf("Conf(NicoConvSeqnoStart): %#v\n", opt.NicoConvSeqnoStart) diff --git a/src/youtube/comment.go b/src/youtube/comment.go index d2564a8..6915128 100644 --- a/src/youtube/comment.go +++ b/src/youtube/comment.go @@ -395,7 +395,7 @@ var SelComment = `SELECT ORDER BY timestampUsec ` -func WriteComment(db *sql.DB, fileName string) { +func WriteComment(db *sql.DB, fileName string, emoji bool) { regexp1 := regexp.MustCompile(":[a-zA-Z0-9\\-\\_]*:") @@ -469,7 +469,9 @@ func WriteComment(db *sql.DB, fileName string) { ) line += ">" - message = regexp1.ReplaceAllString(message, "") + if emoji == false { + message = regexp1.ReplaceAllString(message, "") + } if len(message) <= 0 { message = "　" } diff --git a/src/zip2mp4/zip2mp4.go b/src/zip2mp4/zip2mp4.go index f1037cc..7c3bdea 100644 --- a/src/zip2mp4/zip2mp4.go +++ b/src/zip2mp4/zip2mp4.go @@ -736,13 +736,13 @@ func ReplayDB(fileName string, hlsPort int, seqnoStart int64) (err error) { return } -func YtComment(fileName string) (done bool, err error) { +func YtComment(fileName string, ytemoji bool) (done bool, err error) { db, err := sql.Open("sqlite3", fileName) if err != nil { return } defer db.Close() - youtube.WriteComment(db, fileName) + youtube.WriteComment(db, fileName, ytemoji) return } From 9a3a0ceb5b8615c268f26039f088117c82b6398d Mon Sep 17 00:00:00 2001 From: nnn-revo2012 Date: Mon, 5 Sep 2022 05:29:32 +0900 Subject: [PATCH 30/39] =?UTF-8?q?=E3=83=8B=E3=82=B3=E7=94=9F=E3=81=AE?= =?UTF-8?q?=E3=82=B3=E3=83=A1=E3=83=B3=E3=83=88=E9=96=A2=E9=80=A3=E3=81=AE?= =?UTF-8?q?=E4=BF=AE=E6=AD=A3=20-=20=E3=83=AA=E3=82=A2=E3=83=AB=E3=82=BF?= =?UTF-8?q?=E3=82=A4=E3=83=A0=E6=94=BE=E9=80=81=E3=81=A7-nico-conv-seqno-s?= =?UTF-8?q?tart=E3=81=8C=E6=8C=87=E5=AE=9A=E3=81=95=E3=82=8C=E3=81=9F?= =?UTF-8?q?=E5=A0=B4=E5=90=88=E3=81=AB=E3=82=B3=E3=83=A1=E3=83=B3=E3=83=88?= =?UTF-8?q?=E3=81=AEvpos=E3=81=8C=E3=81=9A=E3=82=8C=E3=82=8B=E3=81=AE?= =?UTF-8?q?=E3=82=92=E4=BF=AE=E6=AD=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/niconico/nico_db.go | 4 ++-- src/zip2mp4/zip2mp4.go | 11 +++++++++-- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/src/niconico/nico_db.go b/src/niconico/nico_db.go index 1fe4cb0..4214276 100644 --- a/src/niconico/nico_db.go +++ b/src/niconico/nico_db.go @@ -318,7 +318,7 @@ func dbadjustVpos(opentime, offset, date, vpos int64, providerType string) (ret } -func WriteComment(db *sql.DB, fileName string, skipHb, adjustVpos bool, seqnoStart, seqnoEnd int64) { +func WriteComment(db *sql.DB, fileName string, skipHb, adjustVpos bool, seqnoStart, seqnoEnd, seqOffset int64) { var fSelComment = func(revision int) string { var selAppend string @@ -356,7 +356,7 @@ func WriteComment(db *sql.DB, fileName string, skipHb, adjustVpos bool, seqnoSta if sts == "ENDED" { offset = seqnoStart * 500 //timeshift } else { - offset = (serverTime/10) - (openTime*100) //on_air + offset = (serverTime/10) - (openTime*100) + (seqOffset*150) //on_air } //db.QueryRow(`SELECT v FROM kvs WHERE k = 'providerType'`).Scan(&providerType) providerType = DbKVGet(db, "providerType").(string) diff --git a/src/zip2mp4/zip2mp4.go b/src/zip2mp4/zip2mp4.go index 7c3bdea..ff4a409 100644 --- a/src/zip2mp4/zip2mp4.go +++ b/src/zip2mp4/zip2mp4.go @@ -450,17 +450,20 @@ func ExtractChunks(fileName string, skipHb, adjustVpos bool, seqnoStart, seqnoEn seqstart := niconico.DbGetFirstSeqNo(db, 1) seqend := niconico.DbGetLastSeqNo(db, 1) + var seqoffset int64 if seqnoStart > 0 && seqnoStart > seqstart { + seqoffset = seqnoStart - seqstart // リアルタイム放送の開始時間の計算用 seqstart = seqnoStart } if seqnoEnd > 0 && seqnoEnd < seqend { seqend = seqnoEnd } fmt.Println("seqstart: ", seqstart) + fmt.Println("seqoffset: ", seqoffset) fmt.Println("seqend: ", seqend) - niconico.WriteComment(db, fileName, skipHb, adjustVpos, seqstart, seqend) + niconico.WriteComment(db, fileName, skipHb, adjustVpos, seqstart, seqend, seqoffset) rows, err := db.Query(niconico.SelMediaF(seqstart, seqend)) if err != nil { @@ -517,16 +520,20 @@ func ConvertDB(fileName, ext string, skipHb, adjustVpos, forceConcat bool, seqno seqstart := niconico.DbGetFirstSeqNo(db, 1) seqend := niconico.DbGetLastSeqNo(db, 1) + var seqoffset int64 + if seqnoStart > 0 && seqnoStart > seqstart { + seqoffset = seqnoStart - seqstart // リアルタイム放送の開始時間の計算用 seqstart = seqnoStart } if seqnoEnd > 0 && seqnoEnd < seqend { seqend = seqnoEnd } fmt.Println("seqstart: ", seqstart) + fmt.Println("seqoffset: ", seqoffset) fmt.Println("seqend: ", seqend) - niconico.WriteComment(db, fileName, skipHb, adjustVpos, seqstart, seqend) + niconico.WriteComment(db, fileName, skipHb, adjustVpos, seqstart, seqend, seqoffset) var zm *ZipMp4 defer func() { From 716cdc498807e2ecc62908f325fe86709eba72e5 Mon Sep 17 00:00:00 2001 From: nnn-revo2012 Date: Wed, 7 Sep 2022 00:06:34 +0900 Subject: [PATCH 31/39] built 20220905.40 Update changelog.txt --- changelog.txt | 19 ++++++++++++++++++- src/buildno/buildno.go | 4 ++-- 2 files changed, 20 insertions(+), 3 deletions(-) diff --git a/changelog.txt b/changelog.txt index 9df5379..ae77a9f 100644 --- a/changelog.txt +++ b/changelog.txt @@ -1,10 +1,27 @@ 更新履歴 -2022xxxx.xx +20220905.40 +・ニコ生のコメントのvposを補正 + -nico-adjust-vpos=on + コメント書き出し時にvposの値を補正する + vposの値が-1000より小さい場合はコメント出力しない + -nico-adjust-vpos=off + コメント書き出し時にvposの値をそのまま出力する(デフォルト) + ※ExtractChunks()もコメントvposを補正するように修正 + ※ニコ生の生放送を録画する際、再接続してもkvsテーブルは更新されません +・Youtubeのコメントにemojiを出力する/しない + -yt-emoji=on + コメントにemojiを表示する(デフォルト) + -yt-emoji=off + コメントにemojiを表示しない ・音声のみ録画対応 　-nico-limit-bw に audio_high または audio_only を指定してください ・-http-timeout の設定を保存するように修正 ・live2.* -> live.* に修正 +・その他主にコメント関連の修正 + - livedl.exeとsqlite3ファイルが別のフォルダーにある場合、コメント出力時にxmlファイルに + -数字が付かなかったのを修正 + - ニコ生の生放送の最初に取得するコメント数を1000から100に変更した(サーバー側の仕様による) 20211017.39 ・livedlのあるディレクトリ以外から実行する時カレントディレクトリにconf.dbが作成されるのを修正 diff --git a/src/buildno/buildno.go b/src/buildno/buildno.go index e504fdd..dbdcb1f 100644 --- a/src/buildno/buildno.go +++ b/src/buildno/buildno.go @@ -1,5 +1,5 @@ package buildno -var BuildDate = "20211017" -var BuildNo = "39" +var BuildDate = "20220905" +var BuildNo = "40" From b628079eec690a6cc274a8aaf23ad92e8bbeb864 Mon Sep 17 00:00:00 2001 From: nnn-revo2012 Date: Fri, 28 Oct 2022 10:50:30 +0900 Subject: [PATCH 32/39] =?UTF-8?q?2=E6=AE=B5=E9=9A=8E=E8=AA=8D=E8=A8=BC?= =?UTF-8?q?=E6=A9=9F=E8=83=BD(MFA)=E8=BF=BD=E5=8A=A0=20-=20login=20API?= =?UTF-8?q?=E3=81=AEendpoint=E3=82=92Web=E3=81=A8=E5=90=8C=E3=81=98?= =?UTF-8?q?=E3=81=AB=E5=A4=89=E6=9B=B4=20-=202=E6=AE=B5=E9=9A=8E=E8=AA=8D?= =?UTF-8?q?=E8=A8=BC(MFA/2FA)=E8=BF=BD=E5=8A=A0=20-=20=E3=83=AD=E3=82=B0?= =?UTF-8?q?=E3=82=A4=E3=83=B3=E3=81=AE=E9=9A=9Bcookiejar=E3=82=92=E4=BD=BF?= =?UTF-8?q?=E7=94=A8=E3=80=81=E3=81=9D=E3=82=8C=E3=81=AB=E4=BC=B4=E3=81=86?= =?UTF-8?q?func=E5=BC=95=E6=95=B0=E4=BF=AE=E6=AD=A3=E3=81=AA=E3=81=A9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/httpbase/httpbase.go | 21 +++--- src/niconico/nico.go | 135 ++++++++++++++++++++++++++++++++++---- src/niconico/nico_hls.go | 8 +-- src/niconico/nico_rtmp.go | 2 +- 4 files changed, 142 insertions(+), 24 deletions(-) diff --git a/src/httpbase/httpbase.go b/src/httpbase/httpbase.go index 770b0eb..7a99ed9 100644 --- a/src/httpbase/httpbase.go +++ b/src/httpbase/httpbase.go @@ -12,6 +12,7 @@ import ( "io/ioutil" "net/http" "net/url" + "net/http/cookiejar" "os" "strings" "time" @@ -137,7 +138,7 @@ func SetProxy(rawurl string) (err error) { return } -func httpBase(method, uri string, header map[string]string, body io.Reader) (resp *http.Response, err, neterr error) { +func httpBase(method, uri string, header map[string]string, jar *cookiejar.Jar, body io.Reader) (resp *http.Response, err, neterr error) { req, err := http.NewRequest(method, uri, body) if err != nil { return @@ -149,6 +150,10 @@ func httpBase(method, uri string, header map[string]string, body io.Reader) (res req.Header.Set(k, v) } + if (jar != nil) { + Client.Jar = jar + } + resp, neterr = Client.Do(req) if neterr != nil { if strings.Contains(neterr.Error(), "x509: certificate signed by unknown") { @@ -159,15 +164,15 @@ func httpBase(method, uri string, header map[string]string, body io.Reader) (res } return } -func Get(uri string, header map[string]string) (*http.Response, error, error) { - return httpBase("GET", uri, header, nil) +func Get(uri string, header map[string]string, jar *cookiejar.Jar) (*http.Response, error, error) { + return httpBase("GET", uri, header, jar, nil) } -func PostForm(uri string, header map[string]string, val url.Values) (*http.Response, error, error) { +func PostForm(uri string, header map[string]string,jar *cookiejar.Jar, val url.Values) (*http.Response, error, error) { if header == nil { header = make(map[string]string) } header["Content-Type"] = "application/x-www-form-urlencoded; charset=utf-8" - return httpBase("POST", uri, header, strings.NewReader(val.Encode())) + return httpBase("POST", uri, header, jar, strings.NewReader(val.Encode())) } func reqJson(method, uri string, header map[string]string, data interface{}) ( *http.Response, error, error) { @@ -181,7 +186,7 @@ func reqJson(method, uri string, header map[string]string, data interface{}) ( } header["Content-Type"] = "application/json" - return httpBase(method, uri, header, bytes.NewReader(encoded)) + return httpBase(method, uri, header, nil, bytes.NewReader(encoded)) } func PostJson(uri string, header map[string]string, data interface{}) (*http.Response, error, error) { return reqJson("POST", uri, header, data) @@ -193,10 +198,10 @@ func PostData(uri string, header map[string]string, data io.Reader) (*http.Respo if header == nil { header = make(map[string]string) } - return httpBase("POST", uri, header, data) + return httpBase("POST", uri, header, nil, data) } func GetBytes(uri string, header map[string]string) (code int, buff []byte, err, neterr error) { - resp, err, neterr := Get(uri, header) + resp, err, neterr := Get(uri, header, nil) if err != nil { return } diff --git a/src/niconico/nico.go b/src/niconico/nico.go index bec3ee4..7d422ed 100644 --- a/src/niconico/nico.go +++ b/src/niconico/nico.go @@ -7,6 +7,7 @@ import ( "io/ioutil" "net" "net/http" + "net/http/cookiejar" _ "net/http/pprof" "net/url" "os" @@ -21,7 +22,16 @@ import ( "github.com/himananiito/livedl/options" ) +func joinCookie(cookies []*http.Cookie) (result string) { + result = "" + for _ ,v := range cookies { + result += v.String()+"; " + } + return result +} + func NicoLogin(opt options.Option) (err error) { + id, pass, _, _ := options.LoadNicoAccount(opt.NicoLoginAlias) if id == "" || pass == "" { @@ -29,10 +39,19 @@ func NicoLogin(opt options.Option) (err error) { return } + jar, err := cookiejar.New(nil) + if err != nil { + return + } + resp, err, neterr := httpbase.PostForm( - "https://account.nicovideo.jp/api/v1/login", - nil, - url.Values{"mail_tel": {id}, "password": {pass}, "site": {"nicoaccountsdk"}}, + "https://account.nicovideo.jp/login/redirector?show_button_twitter=1&site=niconico&show_button_facebook=1&next_url=%2F", + map[string]string{ + "Origin": "https://account.nicovideo.jp", + "Referer": "https://account.nicovideo.jp/login", + }, + jar, + url.Values{"mail_tel": {id}, "password": {pass}}, ) if err != nil { return @@ -43,18 +62,112 @@ func NicoLogin(opt options.Option) (err error) { } defer resp.Body.Close() - body, err := ioutil.ReadAll(resp.Body) - if err != nil { - return + // cookieによって判定 + if opt.NicoDebug { + fmt.Fprintf(os.Stderr, "%v\n", resp.Request.Response.Header) + fmt.Fprintln(os.Stderr, "StatusCode:", resp.StatusCode) } - - if ma := regexp.MustCompile(`(.+?)`).FindSubmatch(body); len(ma) > 0 { + //fmt.Println("StatusCode:", resp.Request.Response.StatusCode) // 302 + set_cookie_url, _ := url.Parse("https://www.nicovideo.jp/") + cookie := joinCookie(jar.Cookies(set_cookie_url)) + if opt.NicoDebug { + fmt.Fprintln(os.Stderr, "cookie:", cookie) + } + var body []byte + if ma := regexp.MustCompile(`mfa_session=`).FindStringSubmatch(cookie); len(ma) > 0 { + //2段階認証処理 + fmt.Println("login MFA(2FA)") + loc := resp.Request.Response.Header.Values("Location")[0] + //fmt.Fprintln(os.Stderr, "Location:",loc) + resp, err, neterr = httpbase.Get( + loc, + map[string]string{ + "Origin": "https://account.nicovideo.jp", + "Referer": "https://account.nicovideo.jp/login", + }, + jar); + if err != nil { + err = fmt.Errorf("login MFA error") + return + } + if neterr != nil { + err = neterr + return + } + body, err = ioutil.ReadAll(resp.Body) + if err != nil { + err = fmt.Errorf("login MFA read error 1") + return + } + //fmt.Printf("%s", body) + str := string(body) + if ma = regexp.MustCompile(`/mfa\?site=niconico`).FindStringSubmatch(str); len(ma) <= 0 { + err = fmt.Errorf("login MFA read error 2") + return + } + //actionから抜き出して loc にセット + if ma = regexp.MustCompile(`form action=\"([^\"]+)\"`).FindStringSubmatch(str); len(ma) <= 0 { + err = fmt.Errorf("login MFA read error 3") + return + } + loc = "https://account.nicovideo.jp" + ma[1] + if opt.NicoDebug { + fmt.Fprintln(os.Stderr, "Location:",loc) + } + //6 digits code を入力 + otp := "" + retry := 3 + for retry > 0 { + fmt.Println("Enter 6 digits code (CANCEL: c/q/x):") + fmt.Scan(&otp) // データを格納する変数のアドレスを指定 + //p = Pattern.compile("^[0-9]{6}$"); + if ma = regexp.MustCompile(`[cqxCQX]{1}`).FindStringSubmatch(otp); len(ma) > 0 { + err = fmt.Errorf("login MFA : cancel") + return + } + if ma = regexp.MustCompile(`^[0-9]{6}$`).FindStringSubmatch(otp); len(ma) > 0 { + retry = 99 + break + } + retry-- + } + //fmt.Println("code:",otp) + if retry <= 0 { + err = fmt.Errorf("login MFA : wrong digits code") + return + } + resp, err, neterr = httpbase.PostForm( + loc, + map[string]string{ + "Origin": "https://account.nicovideo.jp", + "Referer": "https://account.nicovideo.jp/login", + }, + jar, + url.Values{"otp": {otp}}, + ) + if err != nil { + err = fmt.Errorf("login MFA POST error") + return + } + if neterr != nil { + err = neterr + return + } + //結果が302 + cookie = joinCookie(jar.Cookies(set_cookie_url)) + //fmt.Fprintln("StatusCode:", resp.Request.Response.StatusCode) // 302 + if opt.NicoDebug { + fmt.Fprintln(os.Stderr, "StatusCode:", resp.StatusCode) + fmt.Fprintln(os.Stderr, "MFA cookie:", cookie) + } + } + //Cookieからuser_sessionの値を読み込む + if ma := regexp.MustCompile(`user_session=(user_session_.+?);`).FindStringSubmatch(cookie); len(ma) > 0 { + fmt.Println("session_key: ", string(ma[1])) options.SetNicoSession(opt.NicoLoginAlias, string(ma[1])) - fmt.Println("login success") } else { err = fmt.Errorf("login failed: session_key not found") - return } return } @@ -139,7 +252,7 @@ func TestRun(opt options.Option) (err error) { opt.NicoTestTimeout = 12 } - resp, e, nete := httpbase.Get("https://live.nicovideo.jp/api/getalertinfo", nil) + resp, e, nete := httpbase.Get("https://live.nicovideo.jp/api/getalertinfo", nil, nil) if e != nil { err = e return diff --git a/src/niconico/nico_hls.go b/src/niconico/nico_hls.go index ba40595..742bbfe 100644 --- a/src/niconico/nico_hls.go +++ b/src/niconico/nico_hls.go @@ -673,7 +673,7 @@ func (hls *NicoHls) waitAllGoroutines() { func (hls *NicoHls) getwaybackkey(threadId string) (waybackkey string, neterr, err error) { uri := fmt.Sprintf("https://live.nicovideo.jp/api/getwaybackkey?thread=%s", url.QueryEscape(threadId)) - resp, err, neterr := httpbase.Get(uri, map[string]string{"Cookie": "user_session=" + hls.NicoSession}) + resp, err, neterr := httpbase.Get(uri, map[string]string{"Cookie": "user_session=" + hls.NicoSession}, nil) if err != nil { return } @@ -916,7 +916,7 @@ func getStringBase(uri string, header map[string]string) (s string, code int, t t = (time.Now().UnixNano() - start) / (1000 * 1000) }() - resp, err, neterr := httpbase.Get(uri, header) + resp, err, neterr := httpbase.Get(uri, header, nil) if err != nil { return } @@ -948,7 +948,7 @@ func postStringHeader(uri string, header map[string]string, val url.Values) (s s t = (time.Now().UnixNano() - start) / (1000 * 1000) }() - resp, err, neterr := httpbase.PostForm(uri, header, val) + resp, err, neterr := httpbase.PostForm(uri, header, nil, val) if err != nil { return } @@ -975,7 +975,7 @@ func getBytes(uri string) (code int, buff []byte, t int64, err, neterr error) { t = (time.Now().UnixNano() - start) / (1000 * 1000) }() - resp, err, neterr := httpbase.Get(uri, nil) + resp, err, neterr := httpbase.Get(uri, nil, nil) if err != nil { return } diff --git a/src/niconico/nico_rtmp.go b/src/niconico/nico_rtmp.go index d61cb55..9cca01f 100644 --- a/src/niconico/nico_rtmp.go +++ b/src/niconico/nico_rtmp.go @@ -611,7 +611,7 @@ func getStatus(opt options.Option) (status *Status, notLogin bool, err error) { // req.Header.Set("User-Agent", "Niconico/1.0 (Unix; U; iPhone OS 10.3.3; ja-jp; nicoiphone; iPhone5,2) Version/6.65") //} - resp, err, neterr := httpbase.Get(uri, header) + resp, err, neterr := httpbase.Get(uri, header, nil) if err != nil { return } From 6048f4e094fe0473a98a60626585877106cab99e Mon Sep 17 00:00:00 2001 From: nnn-revo2012 Date: Thu, 3 Nov 2022 18:10:08 +0900 Subject: [PATCH 33/39] =?UTF-8?q?=E3=83=96=E3=83=A9=E3=82=A6=E3=82=B6?= =?UTF-8?q?=E3=81=AEcookie=E8=AA=AD=E8=BE=BC=E6=A9=9F=E8=83=BD=E8=BF=BD?= =?UTF-8?q?=E5=8A=A0=20=20=20-nico-cookies=20firefox[:Profile|cookiefile]?= =?UTF-8?q?=20=20=20ex=20-nico-cookies=20firefox=20=20=20=20=20=20-nico-co?= =?UTF-8?q?okies=20firefox:NicoTaro=20=20=20=20=20=20-nico-cookies=20firef?= =?UTF-8?q?ox:'C:/Users/nicotaro/AppData/Roaming/Mozilla/Firefox/Profiles/?= =?UTF-8?q?********.default-release/cookies.sqlite'?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/niconico/nico.go | 10 ++- src/niconico/nico_cookies.go | 166 +++++++++++++++++++++++++++++++++++ src/options/options.go | 31 +++++++ 3 files changed, 206 insertions(+), 1 deletion(-) create mode 100644 src/niconico/nico_cookies.go diff --git a/src/niconico/nico.go b/src/niconico/nico.go index 7d422ed..ba1823a 100644 --- a/src/niconico/nico.go +++ b/src/niconico/nico.go @@ -174,9 +174,17 @@ func NicoLogin(opt options.Option) (err error) { func Record(opt options.Option) (hlsPlaylistEnd bool, dbName string, err error) { + opt.NicoHlsOnly = true + opt.NicoRtmpOnly = false + for i := 0; i < 2; i++ { // load session info - if opt.NicoSession == "" || i > 0 { + if opt.NicoCookies != "" { + opt.NicoSession, err = NicoBrowserCookies(opt) + if err != nil { + return + } + }else if opt.NicoSession == "" || i > 0 { _, _, opt.NicoSession, _ = options.LoadNicoAccount(opt.NicoLoginAlias) } diff --git a/src/niconico/nico_cookies.go b/src/niconico/nico_cookies.go new file mode 100644 index 0000000..11433d6 --- /dev/null +++ b/src/niconico/nico_cookies.go @@ -0,0 +1,166 @@ +package niconico + +import ( + "bufio" + "database/sql" + "fmt" + "log" + "os" + "path/filepath" + "regexp" + "runtime" + "strings" + + "github.com/himananiito/livedl/options" +) + +func readFirefoxProfiles(file string) (profiles map[string]string, err error) { + + profiles = make(map[string]string) + data, err := os.Open(file) + if err != nil { + err = fmt.Errorf("cookies from browser failed: profile.ini can't read") + return + } + defer data.Close() + + var key string + flag := false + scanner := bufio.NewScanner(data) + for scanner.Scan() { + //fmt.Println(scanner.Text()) + if strings.Index(scanner.Text(), "[Profile") >= 0 { + flag = true + continue + } + if flag { + if ma := regexp.MustCompile(`^Name=(.+)$`).FindStringSubmatch(scanner.Text()); len(ma) > 0 { + key = ma[1] + continue + } + if ma := regexp.MustCompile(`^Path=(.+)$`).FindStringSubmatch(scanner.Text()); len(ma) > 0 { + profiles[key] = ma[1] + flag = false + continue + } + } + } + return +} + +func NicoBrowserCookies(opt options.Option) (sessionkey string, err error) { + + sessionkey = "" + var profname string + var dbfile string + profiles := make(map[string]string) + + fmt.Println("NicoCookies:", opt.NicoCookies) + if ma := regexp.MustCompile(`^([^:]+):?(.*)$`).FindStringSubmatch(opt.NicoCookies); len(ma) > 0 { + profname = ma[2] + } + if len(profname) < 1 { + profname = "default-release" + } + //fmt.Println("Profile:",profname) + + if strings.Index(profname, "cookies.sqlite") < 0 { + //profiles.iniを開く + switch runtime.GOOS { + case "windows": + dbfile = os.Getenv("APPDATA") + "/Mozilla/Firefox" + _, err = os.Stat(dbfile) + if os.IsNotExist(err) { + err = fmt.Errorf("cookies from browser failed: firefox profiles not found") + return + } + case "darwin": + dbfile = os.Getenv("HOME") + "/Library/Application Support/Firefox" + _, err = os.Stat(dbfile) + if os.IsNotExist(err) { + dbfile = os.Getenv("HOME") + "/.mozilla/firefox" + _, err = os.Stat(dbfile) + if os.IsNotExist(err) { + err = fmt.Errorf("cookies from browser failed: firefox profiles not found") + return + } + } + default: + dbfile = os.Getenv("HOME") + "/snap/firefox/common/.mozilla/firefox" + _, err = os.Stat(dbfile) + if os.IsNotExist(err) { + dbfile = os.Getenv("HOME") + "/.mozilla/firefox" + _, err = os.Stat(dbfile) + if os.IsNotExist(err) { + err = fmt.Errorf("cookies from browser failed: firefox profiles not found") + return + } + } + } + + profiles, err = readFirefoxProfiles(dbfile + "/profiles.ini") + if len(profiles) <= 0 { + err = fmt.Errorf("cookies from browser failed: profiles not found") + return + } + if _, ok := profiles[profname]; !ok { + err = fmt.Errorf("cookies from browser failed: profiles not found") + return + } + if file, _ := filepath.Glob(dbfile + "/" + profiles[profname] + "/cookies.sqlite"); len(file) > 0 { + dbfile = file[0] + } + //fmt.Println(dbfile) + + if strings.Index(dbfile, "cookies.sqlite") < 0 { + err = fmt.Errorf("cookies from browser failed: cookies not found") + return + } + } else { + dbfile = profname + } + fmt.Println("cookiefile:", dbfile) + + db, err := sql.Open("sqlite3", dbfile) + if err != nil { + err = fmt.Errorf("cookies from browser failed: cookie not found") + return + } + query := `SELECT name, value FROM moz_cookies WHERE host='.nicovideo.jp'` + rows, err := db.Query(query) + if err != nil { + log.Println(err) + db.Close() + return + } + defer rows.Close() + + result := "" + for rows.Next() { + var name string + var value string + var dest = []interface{}{ + &name, + &value, + } + err = rows.Scan(dest...) + if err != nil { + log.Println(err) + db.Close() + return + } + result += name + "=" + value + "; " + } + db.Close() + + //Cookieからuser_sessionの値を読み込む + if ma := regexp.MustCompile(`user_session=(user_session_.+?);`).FindStringSubmatch(result); len(ma) > 0 { + fmt.Println("session_key: ", string(ma[1])) + sessionkey = string(ma[1]) + fmt.Println("cookies from browser get success") + } else { + err = fmt.Errorf("cookies from browser failed: session_key not found") + } + + return +} diff --git a/src/options/options.go b/src/options/options.go index c28f4b3..c5d28b8 100644 --- a/src/options/options.go +++ b/src/options/options.go @@ -36,6 +36,7 @@ type Option struct { NicoRtmpIndex map[int]bool NicoHlsOnly bool NicoLoginOnly bool + NicoCookies string NicoTestTimeout int TcasId string TcasRetry bool @@ -119,9 +120,14 @@ COMMAND: ニコニコ生放送録画用オプション: -nico-login , (+) ニコニコのIDとパスワードを指定する + 2段階認証(MFA)に対応しています -nico-session Cookie[user_session]を指定する -nico-login-only=on (+) 必ずログイン状態で録画する -nico-login-only=off (+) 非ログインでも録画可能とする(デフォルト) + -nico-cookies firefox[:profile|cookiefile] + firefoxのcookieを使用する(デフォルトはdefault-release) + profileまたはcookiefileを直接指定も可能 + スペースが入る場合はquoteで囲む -nico-hls-only 録画時にHLSのみを試す -nico-hls-only=on (+) 上記を有効に設定 -nico-hls-only=off (+) 上記を無効に設定(デフォルト) @@ -262,6 +268,7 @@ func SetNicoLogin(hash, user, pass string) (err error) { fmt.Printf("niconico account saved.\n") return } + func SetNicoSession(hash, session string) (err error) { db, err := dbAccountOpen() if err != nil { @@ -425,6 +432,17 @@ func dbOpen() (db *sql.DB, err error) { return } +func GetBrowserName(str string) (name string) { + name = "error" + if len(str) <= 0 { + return + } + if m := regexp.MustCompile(`^(firefox:?['\"]?.*['\"]?)`).FindStringSubmatch(str); len(m) > 0 { + name = m[1] + } + return +} + func parseTime(arg string) (ret int, err error) { var hour, min, sec int @@ -908,6 +926,19 @@ func ParseArgs() (opt Option) { } return }}, + Parser{regexp.MustCompile(`\A(?i)--?nico-?cookies?\z`), func() (err error) { + str, err := nextArg() + if err != nil { + return + } + str = GetBrowserName(str) + if str != "error" { + opt.NicoCookies = str + } else { + return fmt.Errorf("--nico-cookies: invalid browser name") + } + return + }}, Parser{regexp.MustCompile(`\A(?i)--?nico-?session\z`), func() (err error) { str, err := nextArg() if err != nil { From eb9131c73ec94504f518cd1727be1cf78614611a Mon Sep 17 00:00:00 2001 From: nnn-revo2012 Date: Mon, 7 Nov 2022 18:03:56 +0900 Subject: [PATCH 34/39] built 20221108.41 Update changelog.txt --- changelog.txt | 16 ++++++++++++++++ src/buildno/buildno.go | 4 ++-- src/options/options.go | 2 +- 3 files changed, 19 insertions(+), 3 deletions(-) diff --git a/changelog.txt b/changelog.txt index ae77a9f..ff9ead2 100644 --- a/changelog.txt +++ b/changelog.txt @@ -1,5 +1,21 @@ 更新履歴 +20221108.41 +・直接ログインの２段階認証(MFA)対応 +・上記に伴うlogin APIのendpoint、cookie取得方法の変更 +・firefoxからのcookie取得機能追加 + -nico-cookies firefox[:profile|cookiefile] + e.g. + - profile default-release のcookieを取得 + ./livedl -nico-cookies firefox + - profile NicoTaro のcookieを取得 + ./livedl -nico-cookies firefox:NicoTaro + - 直接cookiefileを指定 + ./livedl -nico-cookies firefox:'C:/Users/*******/AppData/Roaming/Mozilla/Firefox/Profiles/*****/cookies.sqlite' +※Mac/Linuxで `cookies from browser failed: firefox profiles not found`が 表示される場合は報告おねがいします +※直接cookiefile指定の場合は必ず'か"で囲ってください +※プロファイルにspaceを含む場合は'か"で囲ってください + 20220905.40 ・ニコ生のコメントのvposを補正 -nico-adjust-vpos=on diff --git a/src/buildno/buildno.go b/src/buildno/buildno.go index dbdcb1f..2faef27 100644 --- a/src/buildno/buildno.go +++ b/src/buildno/buildno.go @@ -1,5 +1,5 @@ package buildno -var BuildDate = "20220905" -var BuildNo = "40" +var BuildDate = "20221108" +var BuildNo = "41" diff --git a/src/options/options.go b/src/options/options.go index c5d28b8..96711d0 100644 --- a/src/options/options.go +++ b/src/options/options.go @@ -183,7 +183,7 @@ Youtube live録画用オプション: -yt-comment-start YouTube Liveアーカイブでコメント取得開始時間（秒）を指定 ＜分＞:＜秒＞ | ＜時＞:＜分＞:＜秒＞ の形式でも指定可能 0：続きからコメント取得 1：最初からコメント取得 - -yt-emoji=on (+) コメントにemojiを表示する(ディフォルト) + -yt-emoji=on (+) コメントにemojiを表示する(デフォルト) -yt-emoji=off (+) コメントにemojiを表示しない 変換オプション: From fe6421cc9c46afa00cb9c36117d245e9620a5384 Mon Sep 17 00:00:00 2001 From: nnn-revo2012 Date: Sun, 13 Nov 2022 20:20:00 +0900 Subject: [PATCH 35/39] =?UTF-8?q?=E9=8C=B2=E7=94=BB=E3=83=87=E3=83=BC?= =?UTF-8?q?=E3=82=BF=E3=83=BC=E3=83=99=E3=83=BC=E3=82=B9(sqlite3)=E3=81=AE?= =?UTF-8?q?kvs/media/comment=E6=83=85=E5=A0=B1=E3=82=92=E8=A1=A8=E7=A4=BA?= =?UTF-8?q?=E3=81=99=E3=82=8B=E3=82=B3=E3=83=9E=E3=83=B3=E3=83=89(dbinfo)?= =?UTF-8?q?=E8=BF=BD=E5=8A=A0=20=20=20./livedl=20-dbinfo=20=20'=E9=8C=B2?= =?UTF-8?q?=E7=94=BB=E3=83=87=E3=83=BC=E3=82=BF=E3=83=BC=E3=83=99=E3=83=BC?= =?UTF-8?q?=E3=82=B9(sqlite3)'=20-=20youtube=E3=81=AFcomment=E6=83=85?= =?UTF-8?q?=E5=A0=B1=E3=81=AE=E3=81=BF=20-=20=E5=8B=95=E7=94=BB=E3=83=87?= =?UTF-8?q?=E3=83=BC=E3=82=BF=E3=83=BC=E3=83=99=E3=83=BC=E3=82=B9=E3=81=AE?= =?UTF-8?q?=E5=AD=98=E5=9C=A8=E3=83=81=E3=82=A7=E3=83=83=E3=82=AF=E5=87=A6?= =?UTF-8?q?=E7=90=86=E8=BF=BD=E5=8A=A0=20-=20=E6=83=85=E5=A0=B1=E8=A1=A8?= =?UTF-8?q?=E7=A4=BA=E3=80=81=E3=83=87=E3=83=BC=E3=82=BF=E3=83=BCextract?= =?UTF-8?q?=E6=99=82=E3=81=ABDB=E3=82=92readonly=E3=81=A7=E9=96=8B?= =?UTF-8?q?=E3=81=8F=E3=82=88=E3=81=86=E3=81=AB=E4=BF=AE=E6=AD=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- changelog.txt | 7 ++ src/buildno/buildno.go | 4 +- src/livedl.go | 15 +++++ src/niconico/nico_db.go | 141 ++++++++++++++++++++++++++++++++------- src/niconico/nico_hls.go | 4 +- src/options/options.go | 12 +++- src/youtube/comment.go | 47 +++++++++++++ src/zip2mp4/zip2mp4.go | 26 +++++++- 8 files changed, 223 insertions(+), 33 deletions(-) diff --git a/changelog.txt b/changelog.txt index ff9ead2..d6a8a6f 100644 --- a/changelog.txt +++ b/changelog.txt @@ -1,5 +1,12 @@ 更新履歴 +202xxxxx.42 +・録画済みのデータベース(sqlite3)の各種情報を表示するコマンド(-dbinfo)追加 + ./livedl -dbinfo -- 'データーベースのファイル名をフルパスで' + - youtubeのデーターベースはcomment情報のみ表示 + - データベース情報表示、データベースextractの際DBをreadonlyで開くように修正 + - データベースファイルの存在チェックを追加 + 20221108.41 ・直接ログインの２段階認証(MFA)対応 ・上記に伴うlogin APIのendpoint、cookie取得方法の変更 diff --git a/src/buildno/buildno.go b/src/buildno/buildno.go index 2faef27..b3a972a 100644 --- a/src/buildno/buildno.go +++ b/src/buildno/buildno.go @@ -1,5 +1,5 @@ package buildno -var BuildDate = "20221108" -var BuildNo = "41" +var BuildDate = "20221111" +var BuildNo = "42beta" diff --git a/src/livedl.go b/src/livedl.go index 8b1abde..76c7e33 100644 --- a/src/livedl.go +++ b/src/livedl.go @@ -215,6 +215,21 @@ func main() { fmt.Println(err) os.Exit(1) } + + case "DBINFO": + if strings.HasSuffix(opt.DBFile, ".yt.sqlite3") { + if _, err := youtube.ShowDbInfo(opt.DBFile); err != nil { + fmt.Println(err) + os.Exit(1) + } + + } else { + if _, err := niconico.ShowDbInfo(opt.DBFile, opt.ConvExt); err != nil { + fmt.Println(err) + os.Exit(1) + } + } + } return diff --git a/src/niconico/nico_db.go b/src/niconico/nico_db.go index 4214276..fba5f49 100644 --- a/src/niconico/nico_db.go +++ b/src/niconico/nico_db.go @@ -236,9 +236,25 @@ func (hls *NicoHls) dbKVExist(k string) (res int){ return } -func DbKVGet(db *sql.DB, k string) (v interface{}){ - query := `SELECT v FROM kvs WHERE k = ?` - db.QueryRow(query, k).Scan(&v) +func DbKVGet(db *sql.DB) (data map[string]interface{}) { + data = make(map[string]interface{}) + rows, err := db.Query(`SELECT k,v FROM kvs`) + if err != nil { + log.Println(err) + return + } + defer rows.Close() + + for rows.Next() { + var k string + var v interface{} + err := rows.Scan(&k, &v) + if err != nil { + log.Println(err) + } + data[k] = v + } + return } @@ -307,7 +323,6 @@ func (hls *NicoHls) dbGetFromWhen() (res_from int, when float64) { } func dbadjustVpos(opentime, offset, date, vpos int64, providerType string) (ret int64) { - ret = vpos if providerType != "official" { ret = (date - opentime) * 100 - offset @@ -315,7 +330,16 @@ func dbadjustVpos(opentime, offset, date, vpos int64, providerType string) (ret ret = vpos - offset } return ret +} +func dbGetCommentRevision(db *sql.DB) (commentRevision int) { + commentRevision = 0 + var nameCount int64 + db.QueryRow(`SELECT COUNT(name) FROM pragma_table_info('comment') WHERE name = 'name'`).Scan(&nameCount) + if nameCount > 0 { + commentRevision = 1 + } + return } func WriteComment(db *sql.DB, fileName string, skipHb, adjustVpos bool, seqnoStart, seqnoEnd, seqOffset int64) { @@ -328,46 +352,37 @@ func WriteComment(db *sql.DB, fileName string, skipHb, adjustVpos bool, seqnoSta return fmt.Sprintf(SelComment, selAppend) } - var commentRevision int - var nameCount int64 - db.QueryRow(`SELECT COUNT(name) FROM pragma_table_info('comment') WHERE name = 'name'`).Scan(&nameCount) - if nameCount > 0 { - commentRevision = 1 - } - + commentRevision := dbGetCommentRevision(db) fmt.Println("commentRevision: ", commentRevision) //adjustVposの場合はkvsテーブルから読み込み var openTime int64 var providerType string var offset int64 + kvs := DbKVGet(db) if adjustVpos == true { var t float64 var sts string var serverTime int64 - //db.QueryRow(`SELECT v FROM kvs WHERE k = 'serverTime'`).Scan(&t) - t = DbKVGet(db, "serverTime").(float64) + t = kvs["serverTime"].(float64) serverTime = int64(t) - //db.QueryRow(`SELECT v FROM kvs WHERE k = 'openTime'`).Scan(&t) - t = DbKVGet(db, "openTime").(float64) + t = kvs["openTime"].(float64) openTime = int64(t) - //db.QueryRow(`SELECT v FROM kvs WHERE k = 'status'`).Scan(&sts) - sts = DbKVGet(db, "status").(string) + sts = kvs["status"].(string) if sts == "ENDED" { offset = seqnoStart * 500 //timeshift } else { offset = (serverTime/10) - (openTime*100) + (seqOffset*150) //on_air } - //db.QueryRow(`SELECT v FROM kvs WHERE k = 'providerType'`).Scan(&providerType) - providerType = DbKVGet(db, "providerType").(string) - fmt.Println("serverTime: ", serverTime) + providerType = kvs["providerType"].(string) + //fmt.Println("serverTime: ", serverTime) fmt.Println("status: ", sts) } fmt.Println("adjustVpos: ", adjustVpos) fmt.Println("providerType: ", providerType) - fmt.Println("openTime: ", openTime) - fmt.Println("offset: ", offset) + //fmt.Println("openTime: ", openTime) + //fmt.Println("offset: ", offset) rows, err := db.Query(fSelComment(commentRevision)) if err != nil { @@ -502,6 +517,75 @@ func WriteComment(db *sql.DB, fileName string, skipHb, adjustVpos bool, seqnoSta fmt.Fprintf(f, "%s\r\n", ``) } +func ShowDbInfo(fileName, ext string) (done bool, err error) { + _, err = os.Stat(fileName) + if err != nil { + fmt.Println("sqlite3 file not found:") + return + } + db, err := sql.Open("sqlite3", "file:"+fileName+"?mode=ro&immutable=1") + if err != nil { + return + } + defer db.Close() + + fmt.Println("----- DATABASE info. -----") + fmt.Println("sqlite3 file :", fileName) + for _, tbl := range []string{"kvs", "media", "comment"} { + if !dbIsExistTable(db, tbl) { + fmt.Println("table", tbl, "not found") + } else { + fmt.Println("table", tbl, "exist") + } + } + + fmt.Println("----- broadcast info. -----") + kvs := DbKVGet(db) + if len(kvs) > 0 { + id := kvs["nicoliveProgramId"].(string) + title := kvs["title"].(string) + sts := kvs["status"].(string) + ptype := kvs["providerType"].(string) + open := int64(kvs["openTime"].(float64)) + begin := int64(kvs["beginTime"].(float64)) + end := int64(kvs["endTime"].(float64)) + username := kvs["userName"].(string) + + fmt.Println("id: ", id) + fmt.Println("title: ", title) + fmt.Println("username: ", username) + fmt.Println("providerType: ", ptype) + fmt.Println("status: ", sts) + fmt.Println("openTime: ", time.Unix(open, 0)) + if ptype == "official" { + fmt.Println("beginTime: ", time.Unix(begin, 0)) + } + fmt.Println("endTime: ", time.Unix(end, 0)) + } else { + fmt.Println("kvs data not found") + } + commentRevision := dbGetCommentRevision(db) + fmt.Println("commentRevision: ", commentRevision) + + media_all := DbGetCountMedia(db , 0) + media_err := DbGetCountMedia(db , 2) + media_sseq := DbGetFirstSeqNo(db , 0) + media_eseq := DbGetLastSeqNo(db , 0) + comm_data := DbGetCountComment(db) + + fmt.Println("----- media info. -----") + fmt.Println("start seqno: ", media_sseq) + fmt.Println("end seqno: ", media_eseq) + fmt.Println("data: ", media_all, "(media:", media_all - media_err, "err:", media_err, ")") + + fmt.Println("----- comment info. -----") + fmt.Println("data: ", comm_data) + + done = true + + return +} + // ts func (hls *NicoHls) dbGetLastMedia(i int) (res []byte) { hls.dbMtx.Lock() @@ -555,6 +639,17 @@ func DbGetCountMedia(db *sql.DB, flg int) (res int64) { return } func DbGetCountComment(db *sql.DB) (res int64) { - db.QueryRow("SELECT COUNT(no) FROM comment").Scan(&res) + db.QueryRow("SELECT COUNT(date) FROM comment").Scan(&res) + return +} +func dbIsExistTable(db *sql.DB, table_name string) (ret bool) { + var res int + ret = false + if len(table_name) > 0 { + db.QueryRow("SELECT COUNT(*) FROM sqlite_master WHERE TYPE='table' AND name=?", table_name).Scan(&res) + if res > 0 { + ret = true + } + } return } diff --git a/src/niconico/nico_hls.go b/src/niconico/nico_hls.go index 742bbfe..a2a8138 100644 --- a/src/niconico/nico_hls.go +++ b/src/niconico/nico_hls.go @@ -324,7 +324,7 @@ func NewHls(opt options.Option, prop map[string]interface{}) (hls *NicoHls, err hls.dbKVSet(k, v) } } - fmt.Println("Write dbKVSet") + //fmt.Println("Write dbKVSet") } return @@ -406,7 +406,7 @@ func (hls *NicoHls) commentHandler(tag string, attr interface{}) (err error) { } else if s, ok := attrMap["thread"].(string); ok { hls.dbKVSet("comment/thread", s) } - fmt.Println("Write dbKVSet(command/thread)") + //fmt.Println("Write dbKVSet(command/thread)") } } diff --git a/src/options/options.go b/src/options/options.go index 96711d0..7925a63 100644 --- a/src/options/options.go +++ b/src/options/options.go @@ -107,6 +107,8 @@ COMMAND: -tcas ツイキャスの録画 -yt YouTube Liveの録画 -d2m 録画済みのdb(.sqlite3)をmp4に変換する(-db-to-mp4) + -dbinfo 録画済みのdb(.sqlite3)の各種情報を表示する + e.g. $ livedl -dbinfo -- 'C:/home/hogehoge/livedl/rec/lvxxxxxxxx.sqlite3' -d2h [実験的] 録画済みのdb(.sqlite3)を視聴するためのHLSサーバを立てる(-db-to-hls) 開始シーケンス番号は（変換ではないが） -nico-conv-seqno-start で指定 使用例：$ livedl lvXXXXXXXXX.sqlite3 -d2h -nico-hls-port 12345 -nico-conv-seqno-start 2780 @@ -695,6 +697,10 @@ func ParseArgs() (opt Option) { opt.Command = "DB2HLS" return nil }}, + Parser{regexp.MustCompile(`\A(?i)--?(?:d|db|sqlite3?)-?(?:i|info)\z`), func() error { + opt.Command = "DBINFO" + return nil + }}, Parser{regexp.MustCompile(`\A(?i)--?nico-?login-?only(?:=(on|off))?\z`), func() error { if strings.EqualFold(match[1], "on") { opt.NicoLoginOnly = true @@ -1244,7 +1250,7 @@ func ParseArgs() (opt Option) { opt.ZipFile = arg return true } - case "DB2MP4", "DB2HLS": + case "DB2MP4", "DB2HLS", "DBINFO": if ma := regexp.MustCompile(`(?i)\.sqlite3`).FindStringSubmatch(arg); len(ma) > 0 { opt.DBFile = arg return true @@ -1345,7 +1351,7 @@ LB_ARG: fmt.Printf("Conf(TcasRetry): %#v\n", opt.TcasRetry) fmt.Printf("Conf(TcasRetryTimeoutMinute): %#v\n", opt.TcasRetryTimeoutMinute) fmt.Printf("Conf(TcasRetryInterval): %#v\n", opt.TcasRetryInterval) - case "DB2MP4": + case "DB2MP4", "DBINFO": fmt.Printf("Conf(ExtractChunks): %#v\n", opt.ExtractChunks) fmt.Printf("Conf(NicoConvForceConcat): %#v\n", opt.NicoConvForceConcat) fmt.Printf("Conf(ConvExt): %#v\n", opt.ConvExt) @@ -1385,7 +1391,7 @@ LB_ARG: if opt.ZipFile == "" { Help() } - case "DB2MP4", "DB2HLS": + case "DB2MP4", "DB2HLS", "DBINFO": if opt.DBFile == "" { Help() } diff --git a/src/youtube/comment.go b/src/youtube/comment.go index 6915128..2b4a927 100644 --- a/src/youtube/comment.go +++ b/src/youtube/comment.go @@ -383,6 +383,42 @@ func dbGetContinuation(ctx context.Context, db *sql.DB, mtx *sync.Mutex) (res st return } +func DbGetCountComment(db *sql.DB) (res int64) { + db.QueryRow("SELECT COUNT(id) FROM comment").Scan(&res) + return +} + +func ShowDbInfo(fileName string) (done bool, err error) { + _, err = os.Stat(fileName) + if err != nil { + fmt.Println("sqlite3 file not found:") + return + } + db, err := sql.Open("sqlite3", "file:"+fileName+"?mode=ro&immutable=1") + if err != nil { + return + } + defer db.Close() + + fmt.Println("----- DATABASE info. -----") + fmt.Println("sqlite3 file :", fileName) + for _, tbl := range []string{"comment"} { + if !dbIsExistTable(db, tbl) { + fmt.Println("table", tbl, "not found") + } else { + fmt.Println("table", tbl, "exist") + } + } + + comm_data := DbGetCountComment(db) + fmt.Println("----- comment info. -----") + fmt.Println("data: ", comm_data) + + done = true + + return +} + var SelComment = `SELECT timestampUsec, IFNULL(videoOffsetTimeMsec, -1), @@ -481,3 +517,14 @@ func WriteComment(db *sql.DB, fileName string, emoji bool) { } fmt.Fprintf(f, "%s\r\n", ``) } +func dbIsExistTable(db *sql.DB, table_name string) (ret bool) { + var res int + ret = false + if len(table_name) > 0 { + db.QueryRow("SELECT COUNT(*) FROM sqlite_master WHERE TYPE='table' AND name=?", table_name).Scan(&res) + if res > 0 { + ret = true + } + } + return +} diff --git a/src/zip2mp4/zip2mp4.go b/src/zip2mp4/zip2mp4.go index ff4a409..47c2722 100644 --- a/src/zip2mp4/zip2mp4.go +++ b/src/zip2mp4/zip2mp4.go @@ -442,7 +442,12 @@ func Convert(fileName string) (err error) { } func ExtractChunks(fileName string, skipHb, adjustVpos bool, seqnoStart, seqnoEnd int64) (done bool, err error) { - db, err := sql.Open("sqlite3", fileName) + _, err = os.Stat(fileName) + if err != nil { + fmt.Println("sqlite3 file not found:") + return + } + db, err := sql.Open("sqlite3", "file:"+fileName+"?mode=ro&immutable=1") if err != nil { return } @@ -512,7 +517,12 @@ func ExtractChunks(fileName string, skipHb, adjustVpos bool, seqnoStart, seqnoEn } func ConvertDB(fileName, ext string, skipHb, adjustVpos, forceConcat bool, seqnoStart, seqnoEnd int64) (done bool, nMp4s int, skipped bool, err error) { - db, err := sql.Open("sqlite3", fileName) + _, err = os.Stat(fileName) + if err != nil { + fmt.Println("sqlite3 file not found:") + return + } + db, err := sql.Open("sqlite3", "file:"+fileName+"?mode=ro&immutable=1") if err != nil { return } @@ -601,6 +611,11 @@ func ConvertDB(fileName, ext string, skipHb, adjustVpos, forceConcat bool, seqno } func ReplayDB(fileName string, hlsPort int, seqnoStart int64) (err error) { + _, err = os.Stat(fileName) + if err != nil { + fmt.Println("sqlite3 file not found:") + return + } db, err := sql.Open("sqlite3", fileName) if err != nil { return @@ -744,7 +759,12 @@ func ReplayDB(fileName string, hlsPort int, seqnoStart int64) (err error) { } func YtComment(fileName string, ytemoji bool) (done bool, err error) { - db, err := sql.Open("sqlite3", fileName) + _, err = os.Stat(fileName) + if err != nil { + fmt.Println("sqlite3 file not found:") + return + } + db, err := sql.Open("sqlite3", "file:"+fileName+"?mode=ro&immutable=1") if err != nil { return } From f39a35eea929a7bcbdb3cee6eac17366df5e7b3c Mon Sep 17 00:00:00 2001 From: nnn-revo2012 Date: Wed, 16 Nov 2022 22:25:30 +0900 Subject: [PATCH 36/39] =?UTF-8?q?Refactoring=20-=20=E6=97=A7=E9=85=8D?= =?UTF-8?q?=E4=BF=A1(RTMP)=E3=80=81=E5=AE=9F=E9=A8=93=E6=94=BE=E9=80=81?= =?UTF-8?q?=E3=81=AE=E3=83=AD=E3=82=B8=E3=83=83=E3=82=AF=E3=82=92=E5=89=8A?= =?UTF-8?q?=E9=99=A4=20-=20=E4=B8=8D=E8=A6=81=E3=81=AA=E3=82=AA=E3=83=97?= =?UTF-8?q?=E3=82=B7=E3=83=A7=E3=83=B3=E3=81=AE=E5=89=8A=E9=99=A4(options.?= =?UTF-8?q?go)=20-=20=E4=B8=8D=E8=A6=81=E3=81=AA=E5=A4=89=E6=95=B0(wsapi?= =?UTF-8?q?=E3=80=81broadcastId=E4=BB=96)=E3=81=AE=E5=89=8A=E9=99=A4?= =?UTF-8?q?=E3=80=81=E3=82=BD=E3=83=BC=E3=82=B9=E6=95=B4=E5=BD=A2(nico=5Fh?= =?UTF-8?q?ls.go=E3=80=81nico.go)=20-=20=E3=83=AD=E3=82=B0=E3=82=A4?= =?UTF-8?q?=E3=83=B3=E3=83=81=E3=82=A7=E3=83=83=E3=82=AF=E5=8F=8A=E3=81=B3?= =?UTF-8?q?=E3=83=9A=E3=83=BC=E3=82=B8=E3=82=A8=E3=83=A9=E3=83=BC=E5=87=A6?= =?UTF-8?q?=E7=90=86=E3=82=92=E7=8F=BE=E7=8A=B6=E3=81=AE=E3=83=8B=E3=82=B3?= =?UTF-8?q?=E7=94=9F=E3=81=AB=E5=90=88=E3=82=8F=E3=81=9B=E3=81=A6=E4=BF=AE?= =?UTF-8?q?=E6=AD=A3=20=20=20getProps()=20-=20-nico-login=E3=82=92?= =?UTF-8?q?=E6=8C=87=E5=AE=9A=E3=81=97=E3=81=9F=E5=A0=B4=E5=90=88-nico-log?= =?UTF-8?q?in-only=3Don=E3=81=AB=E3=81=97=E3=81=A6conf=E3=81=AB=E4=BF=9D?= =?UTF-8?q?=E5=AD=98=E3=81=99=E3=82=8B=E3=82=88=E3=81=86=E4=BF=AE=E6=AD=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/niconico/jikken.gox | 270 ---------------- src/niconico/nico.go | 80 ++--- src/niconico/nico_hls.go | 210 ++++-------- src/niconico/nico_rtmp.go | 656 -------------------------------------- src/options/options.go | 77 +---- 5 files changed, 93 insertions(+), 1200 deletions(-) delete mode 100644 src/niconico/jikken.gox delete mode 100644 src/niconico/nico_rtmp.go diff --git a/src/niconico/jikken.gox b/src/niconico/jikken.gox deleted file mode 100644 index f1f78d2..0000000 --- a/src/niconico/jikken.gox +++ /dev/null @@ -1,270 +0,0 @@ - - -package niconico - -import ( - "fmt" - "os" - "time" - "os/signal" - "syscall" - "net/http" - "io/ioutil" - "log" - "encoding/json" - "bytes" - "../options" - "../obj" - "../files" -) - - -func getActionTrackId() (actionTrackId string, err error) { - uri := "https://public.api.nicovideo.jp/v1/action-track-ids.json" - req, _ := http.NewRequest("POST", uri, nil) - - req.Header.Set("Content-Type", "application/json") - - client := new(http.Client) - resp, e := client.Do(req) - if e != nil { - err = e - return - } - defer resp.Body.Close() - bs, err := ioutil.ReadAll(resp.Body) - if err != nil { - log.Println(err) - } - - var props interface{} - if err = json.Unmarshal(bs, &props); err != nil { - return - } - - //obj.PrintAsJson(props) - - data, ok := obj.FindString(props, "data") - if (! ok) { - err = fmt.Errorf("actionTrackId not found") - } - actionTrackId = data - - return -} - -func jikkenWatching(opt options.Option, actionTrackId string, isArchive bool) (props interface{}, err error) { - - str, _ := json.Marshal(OBJ{ - "actionTrackId": actionTrackId, - "isBroadcaster": false, - "isLowLatencyStream": true, - "streamCapacity": "superhigh", - "streamProtocol": "https", - "streamQuality": "auto", // high, auto - }) - if err != nil { - log.Println(err) - return - } - - data := bytes.NewReader(str) - - var uri string - if isArchive { - uri = fmt.Sprintf("https://api.cas.nicovideo.jp/v1/services/live/programs/%s/watching-archive", opt.NicoLiveId) - } else { - uri = fmt.Sprintf("https://api.cas.nicovideo.jp/v1/services/live/programs/%s/watching", opt.NicoLiveId) - } - req, _ := http.NewRequest("POST", uri, data) - - //if opt.NicoSession != "" { - req.Header.Set("Cookie", "user_session=" + opt.NicoSession) - //} - req.Header.Set("Accept", "application/json") - req.Header.Set("Content-Type", "application/json") - req.Header.Set("Origin", "https://cas.nicovideo.jp") - req.Header.Set("X-Connection-Environment", "ethernet") - req.Header.Set("X-Frontend-Id", "91") - - client := new(http.Client) - resp, e := client.Do(req) - if e != nil { - err = e - return - } - defer resp.Body.Close() - bs, err := ioutil.ReadAll(resp.Body) - if err != nil { - log.Println(err) - } - - if err = json.Unmarshal([]byte(bs), &props); err != nil { - return - } - - //obj.PrintAsJson(props) - - return -} - - -func jikkenPut(opt options.Option, actionTrackId string) (forbidden, notOnAir bool, err error) { - str, _ := json.Marshal(OBJ{ - "actionTrackId": actionTrackId, - "isBroadcaster": false, - }) - if err != nil { - log.Println(err) - } - fmt.Printf("\n%s\n\n", str) - - data := bytes.NewReader(str) - - uri := fmt.Sprintf("https://api.cas.nicovideo.jp/v1/services/live/programs/%s/watching", opt.NicoLiveId) - req, _ := http.NewRequest("PUT", uri, data) - - //if opt.NicoSession != "" { - req.Header.Set("Cookie", "user_session=" + opt.NicoSession) - //} - req.Header.Set("Accept", "application/json") - req.Header.Set("Content-Type", "application/json") - req.Header.Set("Origin", "https://cas.nicovideo.jp") - req.Header.Set("X-Frontend-Id", "91") - - client := new(http.Client) - resp, e := client.Do(req) - if e != nil { - err = e - return - } - defer resp.Body.Close() - bs, err := ioutil.ReadAll(resp.Body) - if err != nil { - log.Println(err) - } - - var props interface{} - if err = json.Unmarshal([]byte(bs), &props); err != nil { - return - } - - //obj.PrintAsJson(props) - - if code, ok := obj.FindString(props, "meta", "errorCode"); ok { - switch code { - case "FORBIDDEN": - forbidden = true - return - case "PROGRAM_NOT_ONAIR": - notOnAir = true - return - } - } - - return -} - - -func jikkenHousou(nicoliveProgramId, title, userId, nickname, communityId string, opt options.Option, isArchive bool) (err error) { - - chInterrupt := make(chan os.Signal, 10) - signal.Notify(chInterrupt, os.Interrupt, syscall.SIGTERM, syscall.SIGINT) - - actionTrackId, err := getActionTrackId() - if err != nil { - log.Println(err) - } - - media := &NicoMedia{} - - defer func() { - if media.zipWriter != nil { - media.zipWriter.Close() - } - }() - - title = files.ReplaceForbidden(title) - nickname = files.ReplaceForbidden(nickname) - media.fileName = fmt.Sprintf("%s-%s-%s.zip", nicoliveProgramId, nickname, title) - - - var nLast int - L_main: for { - select { - case <-chInterrupt: - break L_main - default: - } - props, e := jikkenWatching(opt, actionTrackId, isArchive) - if e != nil { - err = e - log.Println(err) - return - } - - if uri, ok := obj.FindString(props, "data", "streamServer", "url"); ok { - //fmt.Println(uri) - - is403, e := media.SetPlaylist(uri) - if is403 { - break L_main - } - if e != nil { - err = e - log.Println(e) - return - } - } - - L_loc: for i := 0; true; i++ { - select { - case <-chInterrupt: - break L_main - default: - } - - is403, e := media.GetMedias() - if is403 { - n := media.getNumChunk() - if n != nLast { - nLast = n - break L_loc - } else { - break L_main - } - } - if e != nil { - log.Println(e) - return - } - if i > 60 { - forbidden, notOnAir, e := jikkenPut(opt, actionTrackId) - if e != nil { - err = e - log.Println(e) - return - } - if notOnAir { - break L_main - } - if forbidden { - break L_loc - } - i = 0 - } - select { - case <-chInterrupt: - break L_main - case <-time.After(1 * time.Second): - } - } - } - if media.zipWriter != nil { - media.zipWriter.Close() - } - - signal.Stop(chInterrupt) - - return -} diff --git a/src/niconico/nico.go b/src/niconico/nico.go index ba1823a..aee19a3 100644 --- a/src/niconico/nico.go +++ b/src/niconico/nico.go @@ -24,8 +24,8 @@ import ( func joinCookie(cookies []*http.Cookie) (result string) { result = "" - for _ ,v := range cookies { - result += v.String()+"; " + for _, v := range cookies { + result += v.String() + "; " } return result } @@ -47,7 +47,7 @@ func NicoLogin(opt options.Option) (err error) { resp, err, neterr := httpbase.PostForm( "https://account.nicovideo.jp/login/redirector?show_button_twitter=1&site=niconico&show_button_facebook=1&next_url=%2F", map[string]string{ - "Origin": "https://account.nicovideo.jp", + "Origin": "https://account.nicovideo.jp", "Referer": "https://account.nicovideo.jp/login", }, jar, @@ -80,12 +80,12 @@ func NicoLogin(opt options.Option) (err error) { loc := resp.Request.Response.Header.Values("Location")[0] //fmt.Fprintln(os.Stderr, "Location:",loc) resp, err, neterr = httpbase.Get( - loc, - map[string]string{ - "Origin": "https://account.nicovideo.jp", - "Referer": "https://account.nicovideo.jp/login", - }, - jar); + loc, + map[string]string{ + "Origin": "https://account.nicovideo.jp", + "Referer": "https://account.nicovideo.jp/login", + }, + jar) if err != nil { err = fmt.Errorf("login MFA error") return @@ -112,7 +112,7 @@ func NicoLogin(opt options.Option) (err error) { } loc = "https://account.nicovideo.jp" + ma[1] if opt.NicoDebug { - fmt.Fprintln(os.Stderr, "Location:",loc) + fmt.Fprintln(os.Stderr, "Location:", loc) } //6 digits code を入力 otp := "" @@ -139,7 +139,7 @@ func NicoLogin(opt options.Option) (err error) { resp, err, neterr = httpbase.PostForm( loc, map[string]string{ - "Origin": "https://account.nicovideo.jp", + "Origin": "https://account.nicovideo.jp", "Referer": "https://account.nicovideo.jp/login", }, jar, @@ -174,9 +174,6 @@ func NicoLogin(opt options.Option) (err error) { func Record(opt options.Option) (hlsPlaylistEnd bool, dbName string, err error) { - opt.NicoHlsOnly = true - opt.NicoRtmpOnly = false - for i := 0; i < 2; i++ { // load session info if opt.NicoCookies != "" { @@ -184,46 +181,29 @@ func Record(opt options.Option) (hlsPlaylistEnd bool, dbName string, err error) if err != nil { return } - }else if opt.NicoSession == "" || i > 0 { + } else if opt.NicoSession == "" || i > 0 { _, _, opt.NicoSession, _ = options.LoadNicoAccount(opt.NicoLoginAlias) } - if !opt.NicoRtmpOnly { - var done bool - var notLogin bool - var reserved bool - done, hlsPlaylistEnd, notLogin, reserved, dbName, err = NicoRecHls(opt) - if done { - return - } - if err != nil { - return - } - if notLogin { - fmt.Println("not_login") - if err = NicoLogin(opt); err != nil { - return - } - continue - } - if reserved { - continue - } + var done bool + var notLogin bool + var reserved bool + done, hlsPlaylistEnd, notLogin, reserved, dbName, err = NicoRecHls(opt) + if done { + return } - - if !opt.NicoHlsOnly { - notLogin, e := NicoRecRtmp(opt) - if e != nil { - err = e + if err != nil { + return + } + if notLogin { + fmt.Println("not_login") + if err = NicoLogin(opt); err != nil { return } - if notLogin { - fmt.Println("not_login") - if err = NicoLogin(opt); err != nil { - return - } - continue - } + continue + } + if reserved { + continue } break @@ -247,10 +227,6 @@ func TestRun(opt options.Option) (err error) { }() } - opt.NicoRtmpIndex = map[int]bool{ - 0: true, - } - var nextId func() string if opt.NicoLiveId == "" { diff --git a/src/niconico/nico_hls.go b/src/niconico/nico_hls.go index a2a8138..fc08ee0 100644 --- a/src/niconico/nico_hls.go +++ b/src/niconico/nico_hls.go @@ -47,14 +47,11 @@ type playlist struct { position float64 } type NicoHls struct { - wsapi int - startDelay int playlist playlist - nicoliveProgramId string - webSocketUrl string - myUserId string + webSocketUrl string + myUserId string commentStarted bool mtxCommentStarted sync.Mutex @@ -113,24 +110,12 @@ func debug_Now() string { } func NewHls(opt options.Option, prop map[string]interface{}) (hls *NicoHls, err error) { - nicoliveProgramId, ok := prop["nicoliveProgramId"].(string) - if !ok { - err = fmt.Errorf("nicoliveProgramId is not string") - return - } - webSocketUrl, ok := prop["//webSocketUrl"].(string) if !ok { err = fmt.Errorf("webSocketUrl is not string") return } - wsapi := 2 - if m := regexp.MustCompile(`/wsapi/v1/`).FindStringSubmatch(webSocketUrl); len(m) > 0 { - wsapi = 1 - log.Println("wsapi: 1") - } - myUserId, _ := prop["//myId"].(string) if myUserId == "" { myUserId = "NaN" @@ -141,15 +126,6 @@ func NewHls(opt options.Option, prop map[string]interface{}) (hls *NicoHls, err timeshift = true } - if wsapi == 2 && false && !timeshift { - if m := regexp.MustCompile(`/watch/([^?]+)`).FindStringSubmatch(webSocketUrl); len(m) > 0 { - nicoliveProgramId = m[1] - } - webSocketUrl = strings.Replace(webSocketUrl, "/wsapi/v2/", "/wsapi/v1/", 1) - wsapi = 1 - log.Println("wsapi: 1") - } - var pid string if nicoliveProgramId, ok := prop["nicoliveProgramId"]; ok { pid, _ = nicoliveProgramId.(string) @@ -266,18 +242,15 @@ func NewHls(opt options.Option, prop map[string]interface{}) (hls *NicoHls, err } hls = &NicoHls{ - wsapi: wsapi, - - nicoliveProgramId: nicoliveProgramId, - webSocketUrl: webSocketUrl, - myUserId: myUserId, + webSocketUrl: webSocketUrl, + myUserId: myUserId, quality: quality, dbName: dbName, isTimeshift: timeshift, fastTimeshift: opt.NicoFastTs || opt.NicoUltraFastTs, - ultrafastTimeshift: opt.NicoUltraFastTs, + ultrafastTimeshift: false, NicoSession: opt.NicoSession, limitBw: limitBw, @@ -320,7 +293,7 @@ func NewHls(opt options.Option, prop map[string]interface{}) (hls *NicoHls, err // 生放送の場合はデータが既にあればupdateしない if hls.isTimeshift || hls.dbKVExist("serverTime") == 0 { for k, v := range prop { - if (! strings.HasPrefix(k, "//")) { + if !strings.HasPrefix(k, "//") { hls.dbKVSet(k, v) } } @@ -518,7 +491,7 @@ func (hls *NicoHls) checkReturnCode(code int) { if hls.isTimeshift { if hls.commentDone { hls.stopPCGoroutines() - } else if (! hls.getCommentStarted()) { + } else if !hls.getCommentStarted() { hls.stopPCGoroutines() } else { fmt.Println("waiting comment") @@ -670,26 +643,6 @@ func (hls *NicoHls) waitAllGoroutines() { hls.waitMGoroutines() } -func (hls *NicoHls) getwaybackkey(threadId string) (waybackkey string, neterr, err error) { - - uri := fmt.Sprintf("https://live.nicovideo.jp/api/getwaybackkey?thread=%s", url.QueryEscape(threadId)) - resp, err, neterr := httpbase.Get(uri, map[string]string{"Cookie": "user_session=" + hls.NicoSession}, nil) - if err != nil { - return - } - if neterr != nil { - return - } - defer resp.Body.Close() - - dat, neterr := ioutil.ReadAll(resp.Body) - if neterr != nil { - return - } - - waybackkey = strings.TrimPrefix(string(dat), "waybackkey=") - return -} func (hls *NicoHls) getTsCommentFromWhen() (res_from int, when float64) { return hls.dbGetFromWhen() } @@ -1259,10 +1212,8 @@ func (hls *NicoHls) getPlaylist(argUri *url.URL) (is403, isEnd, isStop, is500 bo } if hls.isTimeshift { - if !hls.ultrafastTimeshift { td := seqlist[0].duration * float64(time.Second) hls.playlist.nextTime = time.Now().Add(time.Duration(td)) - } } // prints Current SeqNo @@ -1558,11 +1509,6 @@ func (hls *NicoHls) startPlaylist(uri string) { }) } func (hls *NicoHls) startMain() { - if hls.wsapi == 1 { - hls.startMainV1() - return - } - // エラー時はMAIN_*を返すこと hls.startPGoroutine(func(sig <-chan struct{}) int { if hls.nicoDebug { @@ -1826,7 +1772,6 @@ func (hls *NicoHls) startMain() { return OK }) } - func (hls *NicoHls) startMainV1() { return // old startMain } @@ -1864,7 +1809,7 @@ func (hls *NicoHls) serve(hlsPort int) { if delay < 2 { delay = 2 } - if (! hls.isTimeshift) { + if !hls.isTimeshift { if delay < 4 { delay = 4 } @@ -1879,10 +1824,10 @@ func (hls *NicoHls) serve(hlsPort int) { `, targetDuration, seqno) for i := int64(delay); i >= 0; i-- { body += fmt.Sprintf( -`#EXTINF:%s, + `#EXTINF:%s, /ts/%d/test.ts -`, extInf, seqno - i) +`, extInf, seqno-i) } c.Data(http.StatusOK, "application/x-mpegURL", []byte(body)) return @@ -2058,14 +2003,15 @@ func postTsRsvBase(num int, vid, session string) (err error) { return } -func getProps(opt options.Option) (props interface{}, isFlash, notLogin, tsRsv0, tsRsv1 bool, err error) { +func getProps(opt options.Option) (props interface{}, notLogin, tsRsv0, tsRsv1 bool, err error) { header := map[string]string{} if opt.NicoSession != "" { header["Cookie"] = "user_session=" + opt.NicoSession } - uri := fmt.Sprintf("https://live.nicovideo.jp/watch/%s", opt.NicoLiveId) + //ログインチェック + uri := "https://www.nicovideo.jp" dat, _, _, err, neterr := getStringHeader(uri, header) if err != nil || neterr != nil { if err == nil { @@ -2090,16 +2036,39 @@ func getProps(opt options.Option) (props interface{}, isFlash, notLogin, tsRsv0, notLogin = true } + if notLogin && opt.NicoLoginOnly { + return + } + + if ma := regexp.MustCompile(`member_status['"]*\s*[=:]\s*['"](.*?)['"]`).FindStringSubmatch(dat); len(ma) > 0 { + fmt.Println("account:", ma[1]) + } + if notLogin { + fmt.Println("account: not_login") + } + + uri = fmt.Sprintf("https://live.nicovideo.jp/watch/%s", opt.NicoLiveId) + dat, _, _, err, neterr = getStringHeader(uri, header) + if err != nil || neterr != nil { + if err == nil { + err = neterr + } + return + } // 新配信 + nicocas - if ma := regexp.MustCompile(`data-props="(.+?)"`).FindStringSubmatch(dat); len(ma) > 0 { + if regexp.MustCompile(`ご指定のページが見つかりませんでした`).MatchString(dat) { + err = fmt.Errorf("getProps error: page not found") + } else if ma := regexp.MustCompile(`rejectedReasons":\[([^\]]+)\]`).FindStringSubmatch(dat); len(ma) > 0 { + ttt := strings.ReplaceAll(html.UnescapeString(ma[1]), "\",\"", " ") + err = fmt.Errorf("getProps error: %s", strings.Trim(ttt, "\"")) + } else if regexp.MustCompile(`webSocketUrl":"",`).MatchString(dat) { + err = fmt.Errorf("getProps error: webSocketUrl not found") + } else if ma := regexp.MustCompile(`data-props="(.+?)"`).FindStringSubmatch(dat); len(ma) > 0 { str := html.UnescapeString(string(ma[1])) if err = json.Unmarshal([]byte(str), &props); err != nil { return } return - } else if strings.Contains(dat, "nicoliveplayer.swf") { - // 旧Flashプレイヤー - isFlash = true } else if regexp.MustCompile(`この番組は.{1,50}に終了`).MatchString(dat) { // タイムシフト予約ボタン if ma := regexp.MustCompile(`Nicolive\.WatchingReservation\.register`).FindStringSubmatch(dat); len(ma) > 0 { @@ -2112,6 +2081,8 @@ func getProps(opt options.Option) (props interface{}, isFlash, notLogin, tsRsv0, tsRsv1 = true return } + } else { + err = fmt.Errorf("getProps: unknown error") } return @@ -2122,9 +2093,8 @@ func NicoRecHls(opt options.Option) (done, playlistEnd, notLogin, reserved bool, //http.DefaultTransport.(*http.Transport).MaxIdleConnsPerHost = 32 //var props interface{} - //var isFlash bool //var tsRsv bool - props, isFlash, notLogin, tsRsv0, tsRsv1, err := getProps(opt) + props, notLogin, tsRsv0, tsRsv1, err := getProps(opt) if err != nil { //fmt.Println(err) return @@ -2153,25 +2123,16 @@ func NicoRecHls(opt options.Option) (done, playlistEnd, notLogin, reserved bool, return } - if isFlash { - fmt.Println("Flash page detected.") - return - } - if false { objs.PrintAsJson(props) os.Exit(9) } proplist := map[string][]string{ - // "broadcaster" // nicocas - "cas-userName": []string{"broadcaster", "nickname"}, // ユーザ名 - "cas-userPageUrl": []string{"broadcaster", "pageUrl"}, // "https://www.nicovideo.jp/user/\d+" // "community" "comId": []string{"community", "id"}, // "co\d+" // "program" - "beginTime": []string{"program", "beginTime"}, // integer - //"broadcastId": []string{"program", "broadcastId"}, // "\d+" + "beginTime": []string{"program", "beginTime"}, // integer "description": []string{"program", "description"}, // 放送説明 "endTime": []string{"program", "endTime"}, // integer "isFollowerOnly": []string{"program", "isFollowerOnly"}, // bool @@ -2206,79 +2167,36 @@ func NicoRecHls(opt options.Option) (done, playlistEnd, notLogin, reserved bool, v, ok := objs.Find(props, a...) if ok { kv[k] = v - - if opt.NicoDebug { - fmt.Println(k, v) - fmt.Println("----------") - } } } - var nicocas bool - if _, ok := kv["nicocas"]; ok { - nicocas = true - } - - if nicocas { - fmt.Println("nicocas not supported.") - return - - } else { - for _, k := range []string{ - "nicoliveProgramId", - "//webSocketUrl", - //"//myId", - } { - if _, ok := kv[k]; !ok { - fmt.Printf("%v not found\n", k) - return - } - } - - if opt.NicoFormat == "" { - opt.NicoFormat = "?PID?-?UNAME?-?TITLE?" - } - - hls, e := NewHls(opt, kv) - if e != nil { - err = e - fmt.Println(err) + for _, k := range []string{ + "//webSocketUrl", + //"//myId", + } { + if _, ok := kv[k]; !ok { + fmt.Printf("%v not found\n", k) return } - defer hls.Close() - - hls.Wait(opt.NicoTestTimeout, opt.NicoHlsPort) - - dbName = hls.dbName - playlistEnd = hls.finish - done = true } - /* - pageUrl, _ := objs.FindString(props, "broadcaster", "pageUrl") - - if regexp.MustCompile(`\Ahttps?://cas\.nicovideo\.jp/.*?/.*`).MatchString(pageUrl) { - // 実験放送 - userId, ok := objs.FindString(props, "broadcaster", "id") - if ! ok { - fmt.Printf("userId not found") - } - - nickname, ok := objs.FindString(props, "broadcaster", "nickname") - if ! ok { - fmt.Printf("nickname not found") - } + if opt.NicoFormat == "" { + opt.NicoFormat = "?PID?-?UNAME?-?TITLE?" + } - var isArchive bool - switch status { - case "ENDED": - isArchive = true - } + hls, e := NewHls(opt, kv) + if e != nil { + err = e + fmt.Println(err) + return + } + defer hls.Close() - } + hls.Wait(opt.NicoTestTimeout, opt.NicoHlsPort) - log4gui.Info(fmt.Sprintf("isLoggedIn: %v, user_id: %s, nickname: %s", isLoggedIn, user_id, nickname)) - */ + dbName = hls.dbName + playlistEnd = hls.finish + done = true return } diff --git a/src/niconico/nico_rtmp.go b/src/niconico/nico_rtmp.go deleted file mode 100644 index 9cca01f..0000000 --- a/src/niconico/nico_rtmp.go +++ /dev/null @@ -1,656 +0,0 @@ -package niconico - -import ( - "encoding/xml" - "fmt" - "io/ioutil" - "log" - "net/url" - "regexp" - "strings" - "sync" - "time" - - "github.com/himananiito/livedl/amf" - "github.com/himananiito/livedl/files" - "github.com/himananiito/livedl/httpbase" - "github.com/himananiito/livedl/options" - "github.com/himananiito/livedl/rtmps" -) - -type Content struct { - Id string `xml:"id,attr"` - Text string `xml:",chardata"` -} -type Tickets struct { - Name string `xml:"name,attr"` - Text string `xml:",chardata"` -} -type Status struct { - Title string `xml:"stream>title"` - CommunityId string `xml:"stream>default_community"` - Id string `xml:"stream>id"` - Provider string `xml:"stream>provider_type"` - IsArchive bool `xml:"stream>archive"` - IsArchivePlayerServer bool `xml:"stream>is_archiveplayserver"` - Ques []string `xml:"stream>quesheet>que"` - Contents []Content `xml:"stream>contents_list>contents"` - IsPremium bool `xml:"user>is_premium"` - Url string `xml:"rtmp>url"` - Ticket string `xml:"rtmp>ticket"` - Tickets []Tickets `xml:"tickets>stream"` - ErrorCode string `xml:"error>code"` - streams []Stream - chStream chan struct{} - wg *sync.WaitGroup -} -type Stream struct { - originUrl string - streamName string - originTicket string -} - -func (status *Status) quesheet() { - stream := make(map[string][]Stream) - playType := make(map[string]string) - - // timeshift; tag - re_pub := regexp.MustCompile(`\A/publish\s+(\S+)\s+(?:(\S+?),)?(\S+?)(?:\?(\S+))?\z`) - re_play := regexp.MustCompile(`\A/play\s+(\S+)\s+(\S+)\z`) - - for _, q := range status.Ques { - // /publish lv* /content/*/lv*_*_1_*.f4v - if ma := re_pub.FindStringSubmatch(q); len(ma) >= 5 { - stream[ma[1]] = append(stream[ma[1]], Stream{ - originUrl: ma[2], - streamName: ma[3], - originTicket: ma[4], - }) - - // /play ... - } else if ma := re_play.FindStringSubmatch(q); len(ma) > 0 { - // /play case:sp:rtmp:lv*_s_lv*,mobile:rtmp:lv*_s_lv*_sub1,premium:rtmp:lv*_s_lv*_sub1,default:rtmp:lv*_s_lv* main - if strings.HasPrefix(ma[1], "case:") { - s0 := ma[1] - s0 = strings.TrimPrefix(s0, "case:") - cases := strings.Split(s0, ",") - // sp:rtmp:lv*_s_lv* - re := regexp.MustCompile(`\A(\S+?):rtmp:(\S+?)\z`) - for _, c := range cases { - if ma := re.FindStringSubmatch(c); len(ma) > 0 { - playType[ma[1]] = ma[2] - } - } - - // /play rtmp:lv* main - } else { - re := regexp.MustCompile(`\Artmp:(\S+?)\z`) - if ma := re.FindStringSubmatch(ma[1]); len(ma) > 0 { - playType["default"] = ma[1] - } - } - } - } - - pt, ok := playType["premium"] - if ok && status.IsPremium { - s, ok := stream[pt] - if ok { - status.streams = s - } - } else { - pt, ok := playType["default"] - if ok { - s, ok := stream[pt] - if ok { - status.streams = s - } - } - } -} -func (status *Status) initStreams() { - - if len(status.streams) > 0 { - return - } - - //if status.isOfficialLive() { - status.contentsOfficialLive() - //} else if status.isLive() { - status.contentsNonOfficialLive() - //} else { - status.quesheet() - //} - - return -} -func (status *Status) getFileName(index int) (name string) { - if len(status.streams) == 1 { - //name = fmt.Sprintf("%s.flv", status.Id) - name = fmt.Sprintf("%s-%s-%s.flv", status.Id, status.CommunityId, status.Title) - } else if len(status.streams) > 1 { - //name = fmt.Sprintf("%s-%d.flv", status.Id, 1 + index) - name = fmt.Sprintf("%s-%s-%s#%d.flv", status.Id, status.CommunityId, status.Title, 1+index) - } else { - log.Fatalf("No stream") - } - name = files.ReplaceForbidden(name) - return -} -func (status *Status) contentsNonOfficialLive() { - re := regexp.MustCompile(`\A(?:rtmp:)?(rtmp\w*://\S+?)(?:,(\S+?)(?:\?(\S+))?)?\z`) - - // Live (not timeshift); tag - for _, c := range status.Contents { - if ma := re.FindStringSubmatch(c.Text); len(ma) > 0 { - status.streams = append(status.streams, Stream{ - originUrl: ma[1], - streamName: ma[2], - originTicket: ma[3], - }) - } - } - -} -func (status *Status) contentsOfficialLive() { - - tickets := make(map[string]string) - for _, t := range status.Tickets { - tickets[t.Name] = t.Text - } - - for _, c := range status.Contents { - if strings.HasPrefix(c.Text, "case:") { - c.Text = strings.TrimPrefix(c.Text, "case:") - - for _, c := range strings.Split(c.Text, ",") { - c, e := url.PathUnescape(c) - if e != nil { - fmt.Printf("%v\n", e) - } - - re := regexp.MustCompile(`\A(\S+?):(?:limelight:|akamai:)?(\S+),(\S+)\z`) - if ma := re.FindStringSubmatch(c); len(ma) > 0 { - fmt.Printf("\n%#v\n", ma) - switch ma[1] { - default: - fmt.Printf("unknown contents case %#v\n", ma[1]) - case "mobile": - case "middle": - case "default": - status.Url = ma[2] - t, ok := tickets[ma[3]] - if !ok { - fmt.Printf("not found %s\n", ma[3]) - } - fmt.Printf("%s\n", t) - status.streams = append(status.streams, Stream{ - streamName: ma[3], - originTicket: t, - }) - } - } - } - } - } -} - -func (status *Status) relayStreamName(i, offset int) (s string) { - s = regexp.MustCompile(`[^/\\]+\z`).FindString(status.streams[i].streamName) - if offset >= 0 { - s += fmt.Sprintf("_%d", offset) - } - return -} - -func (status *Status) streamName(i, offset int) (name string, err error) { - if status.isOfficialLive() { - if i >= len(status.streams) { - err = fmt.Errorf("(status *Status) streamName(i int): Out of index: %d\n", i) - return - } - - name = status.streams[i].streamName - if status.streams[i].originTicket != "" { - name += "?" + status.streams[i].originTicket - } - return - - } else if status.isOfficialTs() { - name = status.streams[i].streamName - name = regexp.MustCompile(`(?i:\.flv)$`).ReplaceAllString(name, "") - if regexp.MustCompile(`(?i:\.(?:f4v|mp4))$`).MatchString(name) { - name = "mp4:" + name - } else if regexp.MustCompile(`(?i:\.raw)$`).MatchString(name) { - name = "raw:" + name - } - - } else { - name = status.relayStreamName(i, offset) - } - - return -} -func (status *Status) tcUrl() (url string, err error) { - if status.Url != "" { - url = status.Url - return - } else { - status.contentsOfficialLive() - } - - if status.Url != "" { - url = status.Url - return - } - - err = fmt.Errorf("tcUrl not found") - return -} -func (status *Status) isTs() bool { - return status.IsArchive -} -func (status *Status) isLive() bool { - return (!status.IsArchive) -} -func (status *Status) isOfficialLive() bool { - return (status.Provider == "official") && (!status.IsArchive) -} -func (status *Status) isOfficialTs() bool { - if status.IsArchive { - switch status.Provider { - case "official": - return true - case "channel": - return status.IsArchivePlayerServer - } - } - return false -} - -func (st Stream) relayStreamName(offset int) (s string) { - s = regexp.MustCompile(`[^/\\]+\z`).FindString(st.streamName) - if offset >= 0 { - s += fmt.Sprintf("_%d", offset) - } - return -} -func (st Stream) noticeStreamName(offset int) (s string) { - s = st.streamName - s = regexp.MustCompile(`(?i:\.flv)$`).ReplaceAllString(s, "") - if regexp.MustCompile(`(?i:\.(?:f4v|mp4))$`).MatchString(s) { - s = "mp4:" + s - } else if regexp.MustCompile(`(?i:\.raw)$`).MatchString(s) { - s = "raw:" + s - } - - if st.originTicket != "" { - s += "?" + st.originTicket - } - - return -} - -func (status *Status) recStream(index int, opt options.Option) (err error) { - defer func() { - <-status.chStream - status.wg.Done() - }() - - stream := status.streams[index] - - tcUrl, err := status.tcUrl() - if err != nil { - return - } - - rtmp, err := rtmps.NewRtmp( - // tcUrl - tcUrl, - // swfUrl - "http://live.nicovideo.jp/nicoliveplayer.swf?180116154229", - // pageUrl - "http://live.nicovideo.jp/watch/"+status.Id, - // option - status.Ticket, - ) - if err != nil { - return - } - defer rtmp.Close() - - fileName, err := files.GetFileNameNext(status.getFileName(index)) - if err != nil { - return - } - rtmp.SetFlvName(fileName) - - tryRecord := func() (incomplete bool, err error) { - - if err = rtmp.Connect(); err != nil { - return - } - - // default: 2500000 - //if err = rtmp.SetPeerBandwidth(100*1000*1000, 0); err != nil { - if err = rtmp.SetPeerBandwidth(2500000, 0); err != nil { - fmt.Printf("SetPeerBandwidth: %v\n", err) - return - } - - if err = rtmp.WindowAckSize(2500000); err != nil { - fmt.Printf("WindowAckSize: %v\n", err) - return - } - - if err = rtmp.CreateStream(); err != nil { - fmt.Printf("CreateStream %v\n", err) - return - } - - if err = rtmp.SetBufferLength(0, 2000); err != nil { - fmt.Printf("SetBufferLength: %v\n", err) - return - } - - var offset int - if status.IsArchive { - offset = 0 - } else { - offset = -2 - } - - if status.isOfficialTs() { - for i := 0; true; i++ { - if i > 30 { - err = fmt.Errorf("sendFileRequest: No response") - return - } - data, e := rtmp.Command( - "sendFileRequest", []interface{}{ - nil, - amf.SwitchToAmf3(), - []string{ - stream.streamName, - }, - }) - if e != nil { - err = e - return - } - - var resCnt int - switch data.(type) { - case map[string]interface{}: - resCnt = len(data.(map[string]interface{})) - case map[int]interface{}: - resCnt = len(data.(map[int]interface{})) - case []interface{}: - resCnt = len(data.([]interface{})) - case []string: - resCnt = len(data.([]string)) - } - if resCnt > 0 { - break - } - time.Sleep(10 * time.Second) - } - - } else if !status.isOfficialLive() { - // /publishの第二引数 - // streamName(param1:String) - // 「,」で区切る - // ._originUrl, streamName(playStreamName) - // streamName に、「?」がついてるなら originTickt となる - // streamName の.flvは削除する - // streamNameが/\.(f4v|mp4)$/iなら、頭にmp4:をつける - // /\.raw$/iなら、raw:をつける。 - // relayStreamName: streamNameの頭からスラッシュまでを削除したもの - - _, err = rtmp.Command( - "nlPlayNotice", []interface{}{ - nil, - // _connection.request.originUrl - stream.originUrl, - - // this._connection.request.playStreamRequest - // originticket あるなら - // playStreamName ? this._originTicket - // 無いなら playStreamName - stream.noticeStreamName(offset), - - // var _loc1_:String = this._relayStreamName; - // if(this._offset != -2) - // { - // _loc1_ = _loc1_ + ("_" + this.offset); - // } - // user nama: String 'lvxxxxxxxxx' - // user kako: lvxxxxxxxxx_xxxxxxxxxxxx_1_xxxxxx.f4v_0 - stream.relayStreamName(offset), - - // seek offset - // user nama: -2, user kako: 0 - offset, - }) - if err != nil { - fmt.Printf("nlPlayNotice %v\n", err) - return - } - } - - if err = rtmp.SetBufferLength(1, 3600*1000); err != nil { - fmt.Printf("SetBufferLength: %v\n", err) - return - } - - // No return - rtmp.SetFixAggrTimestamp(true) - - // user kako: lv*********_************_*_******.f4v_0 - // official or channel ts: mp4:/content/********/lv*********_************_*_******.f4v - //if err = rtmp.Play(status.origin.playStreamName(status.isTsOfficial(), offset)); err != nil { - streamName, err := status.streamName(index, offset) - if err != nil { - return - } - - if status.isOfficialTs() { - ts := rtmp.GetTimestamp() - if ts > 1000 { - err = rtmp.PlayTime(streamName, ts-1000) - } else { - err = rtmp.PlayTime(streamName, -5000) - } - - } else if status.isTs() { - rtmp.SetFlush(true) - err = rtmp.PlayTime(streamName, -5000) - - } else { - err = rtmp.Play(streamName) - } - if err != nil { - fmt.Printf("Play: %v\n", err) - return - } - - // Non-recordedなタイムシフトでseekしても、timestampが変わるだけで - // 最初からの再生となってしまうのでやらないこと - - // 公式のタイムシフトでSeekしてもタイムスタンプがおかしい - - if opt.NicoTestTimeout > 0 { - // test mode - _, incomplete, err = rtmp.WaitTest(opt.NicoTestTimeout) - } else { - // normal mode - _, incomplete, err = rtmp.Wait() - } - return - } // end func - - //ticketTime := time.Now().Unix() - //rtmp.SetNoSeek(false) - for i := 0; i < 10; i++ { - incomplete, e := tryRecord() - if e != nil { - err = e - fmt.Printf("%v\n", e) - return - } else if incomplete && status.isOfficialTs() { - fmt.Println("incomplete") - time.Sleep(3 * time.Second) - - // update ticket - if true { - //if time.Now().Unix() > ticketTime + 60 { - //ticketTime = time.Now().Unix() - if ticket, e := getTicket(opt); e != nil { - err = e - return - } else { - rtmp.SetConnectOpt(ticket) - } - //} - } - - continue - } - break - } - - fmt.Printf("done\n") - return -} - -func (status *Status) recAllStreams(opt options.Option) (err error) { - - status.initStreams() - - var MaxConn int - if opt.NicoRtmpMaxConn == 0 { - if status.isOfficialTs() { - MaxConn = 1 - } else { - MaxConn = 4 - } - } else if opt.NicoRtmpMaxConn < 0 { - MaxConn = 1 - } else { - MaxConn = opt.NicoRtmpMaxConn - } - - status.wg = &sync.WaitGroup{} - status.chStream = make(chan struct{}, MaxConn) - - ticketTime := time.Now().Unix() - - for index, _ := range status.streams { - if opt.NicoRtmpIndex != nil { - if tes, ok := opt.NicoRtmpIndex[index]; !ok || !tes { - continue - } - } - - // blocks here - status.chStream <- struct{}{} - status.wg.Add(1) - - go status.recStream(index, opt) - - now := time.Now().Unix() - if now > ticketTime+60 { - ticketTime = now - if ticket, e := getTicket(opt); e != nil { - err = e - return - } else { - status.Ticket = ticket - } - } - } - - status.wg.Wait() - - return -} - -func getTicket(opt options.Option) (ticket string, err error) { - status, notLogin, err := getStatus(opt) - if err != nil { - return - } - if status.Ticket != "" { - ticket = status.Ticket - } else { - if notLogin { - err = fmt.Errorf("notLogin") - } else { - err = fmt.Errorf("Ticket not found") - } - } - return -} -func getStatus(opt options.Option) (status *Status, notLogin bool, err error) { - var uri string - - // experimental - if opt.NicoStatusHTTPS { - uri = fmt.Sprintf("https://ow.live.nicovideo.jp/api/getplayerstatus?v=%s", opt.NicoLiveId) - } else { - uri = fmt.Sprintf("http://watch.live.nicovideo.jp/api/getplayerstatus?v=%s", opt.NicoLiveId) - } - - header := make(map[string]string, 4) - if opt.NicoSession != "" { - header["Cookie"] = "user_session=" + opt.NicoSession - } - - // experimental - //if opt.NicoStatusHTTPS { - // req.Header.Set("User-Agent", "Niconico/1.0 (Unix; U; iPhone OS 10.3.3; ja-jp; nicoiphone; iPhone5,2) Version/6.65") - //} - - resp, err, neterr := httpbase.Get(uri, header, nil) - if err != nil { - return - } - if neterr != nil { - err = neterr - return - } - defer resp.Body.Close() - - dat, _ := ioutil.ReadAll(resp.Body) - status = &Status{} - err = xml.Unmarshal(dat, status) - if err != nil { - //fmt.Println(string(dat)) - fmt.Printf("error: %v", err) - return - } - - switch status.ErrorCode { - case "": - case "notlogin": - notLogin = true - default: - err = fmt.Errorf("Error code: %s\n", status.ErrorCode) - return - } - - return -} - -func NicoRecRtmp(opt options.Option) (notLogin bool, err error) { - status, notLogin, err := getStatus(opt) - if err != nil { - return - } - if notLogin { - return - } - - status.recAllStreams(opt) - return -} diff --git a/src/options/options.go b/src/options/options.go index 7925a63..442ff2e 100644 --- a/src/options/options.go +++ b/src/options/options.go @@ -31,10 +31,6 @@ type Option struct { NicoStatusHTTPS bool NicoSession string NicoLoginAlias string - NicoRtmpMaxConn int - NicoRtmpOnly bool - NicoRtmpIndex map[int]bool - NicoHlsOnly bool NicoLoginOnly bool NicoCookies string NicoTestTimeout int @@ -130,14 +126,6 @@ COMMAND: firefoxのcookieを使用する(デフォルトはdefault-release) profileまたはcookiefileを直接指定も可能 スペースが入る場合はquoteで囲む - -nico-hls-only 録画時にHLSのみを試す - -nico-hls-only=on (+) 上記を有効に設定 - -nico-hls-only=off (+) 上記を無効に設定(デフォルト) - -nico-rtmp-only 録画時にRTMPのみを試す - -nico-rtmp-only=on (+) 上記を有効に設定 - -nico-rtmp-only=off (+) 上記を無効に設定(デフォルト) - -nico-rtmp-max-conn RTMPの同時接続数を設定 - -nico-rtmp-index [,] RTMP録画を行うメディアファイルの番号を指定 -nico-hls-port [実験的] ローカルなHLSサーバのポート番号 -nico-limit-bw (+) HLSのBANDWIDTHの上限値を指定する。0=制限なし audio_high or audio_only = 音声のみ @@ -498,8 +486,6 @@ func ParseArgs() (opt Option) { IFNULL((SELECT v FROM conf WHERE k == "NicoFormat"), ""), IFNULL((SELECT v FROM conf WHERE k == "NicoLimitBw"), 0), IFNULL((SELECT v FROM conf WHERE k == "NicoLoginOnly"), 0), - IFNULL((SELECT v FROM conf WHERE k == "NicoHlsOnly"), 0), - IFNULL((SELECT v FROM conf WHERE k == "NicoRtmpOnly"), 0), IFNULL((SELECT v FROM conf WHERE k == "NicoFastTs"), 0), IFNULL((SELECT v FROM conf WHERE k == "NicoLoginAlias"), ""), IFNULL((SELECT v FROM conf WHERE k == "NicoAutoConvert"), 0), @@ -522,8 +508,6 @@ func ParseArgs() (opt Option) { &opt.NicoFormat, &opt.NicoLimitBw, &opt.NicoLoginOnly, - &opt.NicoHlsOnly, - &opt.NicoRtmpOnly, &opt.NicoFastTs, &opt.NicoLoginAlias, &opt.NicoAutoConvert, @@ -713,30 +697,6 @@ func ParseArgs() (opt Option) { } return nil }}, - Parser{regexp.MustCompile(`\A(?i)--?nico-?hls-?only(?:=(on|off))?\z`), func() error { - if strings.EqualFold(match[1], "on") { - opt.NicoHlsOnly = true - dbConfSet(db, "NicoHlsOnly", opt.NicoHlsOnly) - } else if strings.EqualFold(match[1], "off") { - opt.NicoHlsOnly = false - dbConfSet(db, "NicoHlsOnly", opt.NicoHlsOnly) - } else { - opt.NicoHlsOnly = true - } - return nil - }}, - Parser{regexp.MustCompile(`\A(?i)--?nico-?rtmp-?only(?:=(on|off))?\z`), func() error { - if strings.EqualFold(match[1], "on") { - opt.NicoRtmpOnly = true - dbConfSet(db, "NicoRtmpOnly", opt.NicoRtmpOnly) - } else if strings.EqualFold(match[1], "off") { - opt.NicoRtmpOnly = false - dbConfSet(db, "NicoRtmpOnly", opt.NicoRtmpOnly) - } else { - opt.NicoRtmpOnly = true - } - return nil - }}, Parser{regexp.MustCompile(`\A(?i)--?nico-?fast-?ts(?:=(on|off))?\z`), func() error { if strings.EqualFold(match[1], "on") { opt.NicoFastTs = true @@ -784,27 +744,6 @@ func ParseArgs() (opt Option) { opt.NicoUltraFastTs = true return nil }}, - Parser{regexp.MustCompile(`\A(?i)--?nico-?rtmp-?index\z`), func() (err error) { - str, err := nextArg() - if err != nil { - return - } - ar := strings.Split(str, ",") - if len(ar) > 0 { - opt.NicoRtmpIndex = make(map[int]bool) - } - for _, s := range ar { - num, err := strconv.Atoi(s) - if err != nil { - return fmt.Errorf("--nico-rtmp-index: Not a number: %s\n", s) - } - if num <= 0 { - return fmt.Errorf("--nico-rtmp-index: Invalid: %d: must be greater than or equal to 1\n", num) - } - opt.NicoRtmpIndex[num-1] = true - } - return - }}, Parser{regexp.MustCompile(`\A(?i)--?nico-?status-?https\z`), func() error { // experimental opt.NicoStatusHTTPS = true @@ -926,6 +865,7 @@ func ParseArgs() (opt Option) { opt.NicoLoginAlias = fmt.Sprintf("%x", sha3.Sum256([]byte(loginId))) SetNicoLogin(opt.NicoLoginAlias, loginId, loginPass) dbConfSet(db, "NicoLoginAlias", opt.NicoLoginAlias) + opt.NicoLoginOnly = true } else { return fmt.Errorf("--nico-login: ,") @@ -970,19 +910,6 @@ func ParseArgs() (opt Option) { return }}, - Parser{regexp.MustCompile(`\A(?i)--?nico-?rtmp-?max-?conn\z`), func() (err error) { - str, err := nextArg() - if err != nil { - return - } - - num, err := strconv.Atoi(str) - if err != nil { - return fmt.Errorf("--nico-rtmp-max-conn %v: %v", str, err) - } - opt.NicoRtmpMaxConn = num - return - }}, Parser{regexp.MustCompile(`\A(?i)--?nico-?debug\z`), func() error { opt.NicoDebug = true return nil @@ -1328,8 +1255,6 @@ LB_ARG: fmt.Printf("Conf(NicoLoginOnly): %#v\n", opt.NicoLoginOnly) fmt.Printf("Conf(NicoFormat): %#v\n", opt.NicoFormat) fmt.Printf("Conf(NicoLimitBw): %#v\n", opt.NicoLimitBw) - fmt.Printf("Conf(NicoHlsOnly): %#v\n", opt.NicoHlsOnly) - fmt.Printf("Conf(NicoRtmpOnly): %#v\n", opt.NicoRtmpOnly) fmt.Printf("Conf(NicoFastTs): %#v\n", opt.NicoFastTs) fmt.Printf("Conf(NicoAutoConvert): %#v\n", opt.NicoAutoConvert) if opt.NicoAutoConvert { From c901c815158f9fd55494c1177df1a95182cc8ab3 Mon Sep 17 00:00:00 2001 From: nnn-revo2012 Date: Sun, 20 Nov 2022 00:55:02 +0900 Subject: [PATCH 37/39] =?UTF-8?q?Refactoring=20-=20=E9=8C=B2=E7=94=BB?= =?UTF-8?q?=E6=99=82=E5=BC=B7=E5=88=B6=E4=BA=88=E7=B4=84=E6=A9=9F=E8=83=BD?= =?UTF-8?q?(-nico-force-reservation)=E4=BF=AE=E6=AD=A3=20=20=20-=20?= =?UTF-8?q?=E3=82=BF=E3=82=A4=E3=83=A0=E3=82=B7=E3=83=95=E3=83=88=E9=8C=B2?= =?UTF-8?q?=E7=94=BB=E6=99=82=E3=81=AB=E4=BA=88=E7=B4=84=E3=81=95=E3=82=8C?= =?UTF-8?q?=E3=81=A6=E3=81=84=E3=81=AA=E3=81=84=E5=A0=B4=E5=90=88=E8=87=AA?= =?UTF-8?q?=E5=8B=95=E7=9A=84=E3=81=AB=E4=BA=88=E7=B4=84=E3=81=99=E3=82=8B?= =?UTF-8?q?(=E3=83=97=E3=83=AC=E3=83=9F=E3=82=A2)=20=20=20-=20=E4=BA=88?= =?UTF-8?q?=E7=B4=84=E3=81=97=E3=81=A6=E3=81=84=E3=82=8B=E3=82=BF=E3=82=A4?= =?UTF-8?q?=E3=83=A0=E3=82=B7=E3=83=95=E3=83=88=E6=94=BE=E9=80=81=E3=82=92?= =?UTF-8?q?=E9=8C=B2=E7=94=BB=E6=99=82=E3=81=AB=E8=87=AA=E5=8B=95=E7=9A=84?= =?UTF-8?q?=E3=81=AB=E3=83=81=E3=82=B1=E3=83=83=E3=83=88=E4=BD=BF=E7=94=A8?= =?UTF-8?q?=E3=81=99=E3=82=8B=20=20=20-=20TS=E4=BA=88=E7=B4=84/=E3=83=81?= =?UTF-8?q?=E3=82=B1=E3=83=83=E3=83=88=E4=BD=BF=E7=94=A8API=E3=81=AEendpoi?= =?UTF-8?q?nt=E3=82=92=E5=A4=89=E6=9B=B4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/niconico/nico_hls.go | 133 +++++++++++++++++++++++++++++---------- 1 file changed, 99 insertions(+), 34 deletions(-) diff --git a/src/niconico/nico_hls.go b/src/niconico/nico_hls.go index fc08ee0..7e86412 100644 --- a/src/niconico/nico_hls.go +++ b/src/niconico/nico_hls.go @@ -1936,6 +1936,8 @@ func postTsRsvBase(num int, vid, session string) (err error) { header := map[string]string{ "Cookie": "user_session=" + session, + "Referer": "https://live.nicovideo.jp/watch/lv" + vid, + "Origin": "https://live.nicovideo.jp", } dat0, _, _, err, neterr := getStringHeader(uri, header) if err != nil || neterr != nil { @@ -1946,12 +1948,14 @@ func postTsRsvBase(num int, vid, session string) (err error) { } var token string - if ma := regexp.MustCompile( - `TimeshiftActions\.(doRegister|confirmToWatch|moveWatch)\(['"].*?['"]\s*(?:,\s*['"](.+?)['"])`). - FindStringSubmatch(dat0); len(ma) > 0 { - if len(ma) > 2 { - token = ma[2] - } + if ma := regexp.MustCompile(`(ulck_\d+)`).FindStringSubmatch(dat0); len(ma) > 0 { + token = ma[1] + } else if strings.Contains(dat0, "システムエラーが発生しました") { + err = fmt.Errorf("postTsRsv: system error try again in a few minutes") + return + } else if strings.Contains(dat0, "申し込み期限切れ") { + err = fmt.Errorf("postTsRsv: deadline expired") + return } else if strings.Contains(dat0, "視聴済み") { err = fmt.Errorf("postTsRsv: already watched") return @@ -1963,13 +1967,13 @@ func postTsRsvBase(num int, vid, session string) (err error) { // "X-Requested-With": "XMLHttpRequest", // "Origin": "https://live.nicovideo.jp", - // "Referer": fmt.Sprintf("https://live.nicovideo.jp/gate/%s", opt.NicoLiveId), + // "Referer": fmt.Sprintf("https://live.nicovideo.jp/watch/%s", opt.NicoLiveId), // "X-Prototype-Version": "1.6.0.3", var vals url.Values if num == 0 { vals = url.Values{ - "mode": []string{"overwrite"}, + "mode": []string{"auto_register"}, "vid": []string{vid}, "token": []string{token}, "rec_pos": []string{""}, @@ -1983,7 +1987,7 @@ func postTsRsvBase(num int, vid, session string) (err error) { "mode": []string{"use"}, "vid": []string{vid}, "token": []string{token}, - "_": []string{""}, + "": []string{""}, } } @@ -1998,12 +2002,60 @@ func postTsRsvBase(num int, vid, session string) (err error) { fmt.Printf("postTsRsv: status not ok: >>>%s<<<\n", dat1) err = fmt.Errorf("postTsRsv: status not ok") return + } else { + if num == 0 { + fmt.Println("postTsRsv0: status ok") + } else { + fmt.Println("postTsRsv1: status ok") + } } return } -func getProps(opt options.Option) (props interface{}, notLogin, tsRsv0, tsRsv1 bool, err error) { +func postRsvUseTs(isRsv bool, opt options.Option) (err error) { + + var vid string + if ma := regexp.MustCompile(`lv(\d+)`).FindStringSubmatch(opt.NicoLiveId); len(ma) > 0 { + vid = ma[1] + } + uri := "https://live.nicovideo.jp/api/timeshift.ticket.use" + RsvMsg := "Use" + if isRsv { + uri = "https://live.nicovideo.jp/api/timeshift.reservations"; + RsvMsg = "Rsv" + } + header := map[string]string{ + "Cookie": "user_session=" + opt.NicoSession, + "Accept": "application/json, text/plain */*", + "Referer": fmt.Sprintf("https://live.nicovideo.jp/watch/%s", opt.NicoLiveId), + "Origin": "https://live.nicovideo.jp", + } + vals := url.Values{"vid": []string{vid},} + + dat, _, _, err, neterr := postStringHeader(uri, header, vals) + if err != nil || neterr != nil { + if err == nil { + err = neterr + } + return + } + if !strings.Contains(dat, "status\":200") { + if ma := regexp.MustCompile(`\"description\":\"([^\"]+)\"`).FindStringSubmatch(dat); len(ma) > 0 { + err = fmt.Errorf("postRsvUseTs: %s %s\n", RsvMsg, ma[1]) + } else { + fmt.Printf("postRsvUseTs: status not ok: >>>%s<<<\n", dat) + err = fmt.Errorf("postRsvUseTs: %s status not ok", RsvMsg) + return + } + } else { + fmt.Printf("postRsvUseTs: %s status ok\n", RsvMsg) + } + + return +} + +func getProps(opt options.Option) (props interface{}, notLogin, rsvTs, useTs bool, err error) { header := map[string]string{} if opt.NicoSession != "" { @@ -2040,6 +2092,7 @@ func getProps(opt options.Option) (props interface{}, notLogin, tsRsv0, tsRsv1 b return } + // ログインアカウント種別取得 if ma := regexp.MustCompile(`member_status['"]*\s*[=:]\s*['"](.*?)['"]`).FindStringSubmatch(dat); len(ma) > 0 { fmt.Println("account:", ma[1]) } @@ -2047,6 +2100,7 @@ func getProps(opt options.Option) (props interface{}, notLogin, tsRsv0, tsRsv1 b fmt.Println("account: not_login") } + // 放送ページ取得 uri = fmt.Sprintf("https://live.nicovideo.jp/watch/%s", opt.NicoLiveId) dat, _, _, err, neterr = getStringHeader(uri, header) if err != nil || neterr != nil { @@ -2055,32 +2109,40 @@ func getProps(opt options.Option) (props interface{}, notLogin, tsRsv0, tsRsv1 b } return } - // 新配信 + nicocas + + // 放送種別(official/channel/community)取得 + var providertype string + // "providerType": "(.+)l", + if ma := regexp.MustCompile(`"providerType":"(\w+)"`).FindStringSubmatch(dat); len(ma) > 0 { + providertype = ma[1] + } + fmt.Println("providertype:", providertype) + + // 新配信 if regexp.MustCompile(`ご指定のページが見つかりませんでした`).MatchString(dat) { - err = fmt.Errorf("getProps error: page not found") + err = fmt.Errorf("getProps: page not found") + } else if regexp.MustCompile(`(放送者により削除されました|削除された可能性があります)`).MatchString(dat) { + err = fmt.Errorf("getProps: page not found") } else if ma := regexp.MustCompile(`rejectedReasons":\[([^\]]+)\]`).FindStringSubmatch(dat); len(ma) > 0 { ttt := strings.ReplaceAll(html.UnescapeString(ma[1]), "\",\"", " ") - err = fmt.Errorf("getProps error: %s", strings.Trim(ttt, "\"")) + if !notLogin && providertype == "official" && regexp.MustCompile(`notHaveTimeshiftTicket`).MatchString(ttt) { + //チケット予約処理 + fmt.Println("notHaveTimeshiftTicket: timeshift reservation required") + rsvTs = true + } else if !notLogin && providertype == "official" && regexp.MustCompile(`notUseTimeshiftTicket`).MatchString(ttt) { + //チケット使用処理 + fmt.Println("notUseTimeshiftTicket: timeshift reservation required") + useTs = true + } else { + err = fmt.Errorf("getProps: %s", strings.Trim(ttt, "\"")) + } } else if regexp.MustCompile(`webSocketUrl":"",`).MatchString(dat) { - err = fmt.Errorf("getProps error: webSocketUrl not found") + err = fmt.Errorf("getProps: webSocketUrl not found") } else if ma := regexp.MustCompile(`data-props="(.+?)"`).FindStringSubmatch(dat); len(ma) > 0 { str := html.UnescapeString(string(ma[1])) if err = json.Unmarshal([]byte(str), &props); err != nil { return } - return - } else if regexp.MustCompile(`この番組は.{1,50}に終了`).MatchString(dat) { - // タイムシフト予約ボタン - if ma := regexp.MustCompile(`Nicolive\.WatchingReservation\.register`).FindStringSubmatch(dat); len(ma) > 0 { - fmt.Printf("timeshift reservation required\n") - tsRsv0 = true - return - } - if ma := regexp.MustCompile(`Nicolive\.WatchingReservation\.confirm`).FindStringSubmatch(dat); len(ma) > 0 { - fmt.Printf("timeshift reservation required\n") - tsRsv1 = true - return - } } else { err = fmt.Errorf("getProps: unknown error") } @@ -2093,8 +2155,9 @@ func NicoRecHls(opt options.Option) (done, playlistEnd, notLogin, reserved bool, //http.DefaultTransport.(*http.Transport).MaxIdleConnsPerHost = 32 //var props interface{} - //var tsRsv bool - props, notLogin, tsRsv0, tsRsv1, err := getProps(opt) + //var rsvTs bool + //var useTs bool + props, notLogin, rsvTs, useTs, err := getProps(opt) if err != nil { //fmt.Println(err) return @@ -2111,12 +2174,14 @@ func NicoRecHls(opt options.Option) (done, playlistEnd, notLogin, reserved bool, } // TS予約必要 - if (tsRsv0 || tsRsv1) && opt.NicoForceResv { - if tsRsv0 { - err = postTsRsv0(opt) - } else { - err = postTsRsv1(opt) + if (rsvTs || useTs) && opt.NicoForceResv { + if rsvTs { + err = postRsvUseTs(true, opt) + if (err != nil) { + return + } } + err = postRsvUseTs(false, opt) if err == nil { reserved = true } From 1c2912edf71e0701f637dd6427ec710255d9a5b6 Mon Sep 17 00:00:00 2001 From: nnn-revo2012 Date: Tue, 22 Nov 2022 19:58:30 +0900 Subject: [PATCH 38/39] =?UTF-8?q?Refactoring=20-=20httpbase=E5=86=85?= =?UTF-8?q?=E3=81=AE=E5=85=A8=E9=96=A2=E6=95=B0=E3=82=92cookiejar=E5=AF=BE?= =?UTF-8?q?=E5=BF=9C=E3=81=AB=E5=A4=89=E6=9B=B4=20-=20SetTimeout()?= =?UTF-8?q?=E8=BF=BD=E5=8A=A0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/httpbase/httpbase.go | 27 ++++++++++++++++----------- src/options/options.go | 3 +-- src/youtube/comment.go | 2 +- src/youtube/youtube.go | 2 +- 4 files changed, 19 insertions(+), 15 deletions(-) diff --git a/src/httpbase/httpbase.go b/src/httpbase/httpbase.go index 7a99ed9..cc86073 100644 --- a/src/httpbase/httpbase.go +++ b/src/httpbase/httpbase.go @@ -137,6 +137,11 @@ func SetProxy(rawurl string) (err error) { Client.Transport.(*http.Transport).Proxy = http.ProxyURL(u) return } +func SetTimeout(timeout int) (err error) { + err = nil + Client.Timeout = time.Duration(timeout) * time.Second + return +} func httpBase(method, uri string, header map[string]string, jar *cookiejar.Jar, body io.Reader) (resp *http.Response, err, neterr error) { req, err := http.NewRequest(method, uri, body) @@ -167,14 +172,14 @@ func httpBase(method, uri string, header map[string]string, jar *cookiejar.Jar, func Get(uri string, header map[string]string, jar *cookiejar.Jar) (*http.Response, error, error) { return httpBase("GET", uri, header, jar, nil) } -func PostForm(uri string, header map[string]string,jar *cookiejar.Jar, val url.Values) (*http.Response, error, error) { +func PostForm(uri string, header map[string]string, jar *cookiejar.Jar, val url.Values) (*http.Response, error, error) { if header == nil { header = make(map[string]string) } header["Content-Type"] = "application/x-www-form-urlencoded; charset=utf-8" return httpBase("POST", uri, header, jar, strings.NewReader(val.Encode())) } -func reqJson(method, uri string, header map[string]string, data interface{}) ( +func reqJson(method, uri string, header map[string]string, jar *cookiejar.Jar, data interface{}) ( *http.Response, error, error) { encoded, err := json.Marshal(data) if err != nil { @@ -186,22 +191,22 @@ func reqJson(method, uri string, header map[string]string, data interface{}) ( } header["Content-Type"] = "application/json" - return httpBase(method, uri, header, nil, bytes.NewReader(encoded)) + return httpBase(method, uri, header, jar, bytes.NewReader(encoded)) } -func PostJson(uri string, header map[string]string, data interface{}) (*http.Response, error, error) { - return reqJson("POST", uri, header, data) +func PostJson(uri string, header map[string]string, jar *cookiejar.Jar, data interface{}) (*http.Response, error, error) { + return reqJson("POST", uri, header, jar, data) } -func PutJson(uri string, header map[string]string, data interface{}) (*http.Response, error, error) { - return reqJson("PUT", uri, header, data) +func PutJson(uri string, header map[string]string, jar *cookiejar.Jar, data interface{}) (*http.Response, error, error) { + return reqJson("PUT", uri, header, jar, data) } -func PostData(uri string, header map[string]string, data io.Reader) (*http.Response, error, error) { +func PostData(uri string, header map[string]string, jar *cookiejar.Jar, data io.Reader) (*http.Response, error, error) { if header == nil { header = make(map[string]string) } - return httpBase("POST", uri, header, nil, data) + return httpBase("POST", uri, header, jar, data) } -func GetBytes(uri string, header map[string]string) (code int, buff []byte, err, neterr error) { - resp, err, neterr := Get(uri, header, nil) +func GetBytes(uri string, header map[string]string, jar *cookiejar.Jar) (code int, buff []byte, err, neterr error) { + resp, err, neterr := Get(uri, header, jar) if err != nil { return } diff --git a/src/options/options.go b/src/options/options.go index 442ff2e..2fe0bfd 100644 --- a/src/options/options.go +++ b/src/options/options.go @@ -10,7 +10,6 @@ import ( "regexp" "strconv" "strings" - "time" "github.com/himananiito/livedl/buildno" @@ -1228,7 +1227,7 @@ LB_ARG: if opt.HttpTimeout == 0 { opt.HttpTimeout = MinimumHttpTimeout } - httpbase.Client.Timeout = time.Duration(opt.HttpTimeout) * time.Second + httpbase.SetTimeout(opt.HttpTimeout) // [deprecated] // load session info diff --git a/src/youtube/comment.go b/src/youtube/comment.go index 2b4a927..784f9e0 100644 --- a/src/youtube/comment.go +++ b/src/youtube/comment.go @@ -77,7 +77,7 @@ MAINLOOP: resp, err, neterr := httpbase.PostJson(uri, map[string]string { "Cookie": Cookie, "User-Agent": UserAgent, - }, postData) + }, nil, postData) if err != nil { return } diff --git a/src/youtube/youtube.go b/src/youtube/youtube.go index 2412806..0f7b225 100644 --- a/src/youtube/youtube.go +++ b/src/youtube/youtube.go @@ -305,7 +305,7 @@ func Record(id string, ytNoStreamlink, ytNoYoutube_dl bool, ytCommentStart float code, buff, err, neterr := httpbase.GetBytes(uri, map[string]string{ "Cookie": Cookie, "User-Agent": UserAgent, - }) + }, nil) if err != nil { return } From 46887ab3e06b58dd61a6b1e15bc4ec1dc36972c8 Mon Sep 17 00:00:00 2001 From: nnn-revo2012 Date: Tue, 22 Nov 2022 22:01:17 +0900 Subject: [PATCH 39/39] built 20221122.42 Update chagelog.txt --- changelog.txt | 20 ++++++++++++++++++-- src/buildno/buildno.go | 4 ++-- 2 files changed, 20 insertions(+), 4 deletions(-) diff --git a/changelog.txt b/changelog.txt index d6a8a6f..05e36c0 100644 --- a/changelog.txt +++ b/changelog.txt @@ -1,11 +1,27 @@ 更新履歴 -202xxxxx.42 +20221122.42 ・録画済みのデータベース(sqlite3)の各種情報を表示するコマンド(-dbinfo)追加 - ./livedl -dbinfo -- 'データーベースのファイル名をフルパスで' + $ livedl -dbinfo -- 'database filename(.sqlite3) with fullpath' - youtubeのデーターベースはcomment情報のみ表示 - データベース情報表示、データベースextractの際DBをreadonlyで開くように修正 - データベースファイルの存在チェックを追加 +・Refactoring + - 旧配信(RTMP)、実験放送のロジックを削除 + - 不要なオプションの削除(options.go) + -nico-hls-only + -nico-rtmp-only + -nico-rtmp-max-conn + -nico-rtmp-index [,] + - 不要な変数(wsapi、broadcastId他)の削除、ソース整形(nico_hls.go、nico.go) + - ログインチェック及びページエラー処理を現状のニコ生に合わせて修正 getProps() + - -nico-loginを指定した場合-nico-login-only=onにしてconf.dbに保存するよう修正 +・録画時強制予約機能(-nico-force-reservation)修正 + - タイムシフト録画時に予約されていない場合自動的に予約する(プレミア) + - 予約しているタイムシフト放送を録画時に自動的にチケット使用する + - TS予約/チケット使用APIのendpointを変更 +・httpbase内の全関数をcookiejar対応に変更 +・httpbase.SetTimeout()追加 20221108.41 ・直接ログインの２段階認証(MFA)対応 diff --git a/src/buildno/buildno.go b/src/buildno/buildno.go index b3a972a..c966e55 100644 --- a/src/buildno/buildno.go +++ b/src/buildno/buildno.go @@ -1,5 +1,5 @@ package buildno -var BuildDate = "20221111" -var BuildNo = "42beta" +var BuildDate = "20221122" +var BuildNo = "42"