From fab1f67afcc42b685b40ae6e8eadc6f2f65bad29 Mon Sep 17 00:00:00 2001 From: violet-dev Date: Sat, 17 Aug 2024 13:17:56 +0900 Subject: [PATCH] Remove hdownloader --- .../hdownloader/.vscode/launch.json | 26 -- .../hdownloader/.vscode/tasks.json | 42 -- .../hdownloader/AppProvider.cs | 52 --- .../hdownloader/CL/CommandLine.cs | 374 ---------------- .../hdownloader/CL/IConsole.cs | 19 - .../hdownloader/Command.cs | 283 ------------ .../hdownloader/Component/EH.cs | 412 ------------------ .../hdownloader/Component/Hitomi.cs | 342 --------------- .../hdownloader/Component/HitomiIndex.cs | 215 --------- .../hdownloader/Crypto/Hash.cs | 52 --- .../hdownloader/Crypto/Random.cs | 21 - .../hdownloader/Internals.cs | 126 ------ .../hdownloader/Network/NetField.cs | 384 ---------------- .../hdownloader/Network/NetQueue.cs | 49 --- .../hdownloader/Network/NetTask.cs | 170 -------- .../hdownloader/Network/NetTaskPass.cs | 38 -- .../hdownloader/Network/NetTools.cs | 235 ---------- .../hdownloader/Program.cs | 68 --- .../hdownloader/Progress.cs | 373 ---------------- .../hdownloader/Setting/Settings.cs | 172 -------- .../hdownloader/Utils/Compress.cs | 40 -- .../hdownloader/Utils/Extends.cs | 59 --- .../hdownloader/Utils/Heap.cs | 235 ---------- .../hdownloader/Utils/ILazy.cs | 34 -- .../hdownloader/Utils/Strings.cs | 229 ---------- .../hdownloader/Version.cs | 19 - .../hdownloader/publish-linux-x64-aot.bat | 1 - .../hdownloader/publish-linux-x64.bat | 1 - .../hdownloader/publish-osx.10.15-x64-aot.bat | 1 - .../hdownloader/publish-osx.10.15-x64.bat | 1 - .../hdownloader/publish-win10-arm-aot.bat | 1 - .../hdownloader/publish-win10-arm.bat | 1 - .../hdownloader/publish-win10-x64-aot.bat | 1 - .../hdownloader/publish-win10-x64.bat | 1 - .../hdownloader/publish-win10-x86-aot.bat | 1 - .../hdownloader/publish-win10-x86.bat | 1 - 36 files changed, 4079 deletions(-) delete mode 100644 violet-message-search-core/hdownloader/.vscode/launch.json delete mode 100644 violet-message-search-core/hdownloader/.vscode/tasks.json delete mode 100644 violet-message-search-core/hdownloader/AppProvider.cs delete mode 100644 violet-message-search-core/hdownloader/CL/CommandLine.cs delete mode 100644 violet-message-search-core/hdownloader/CL/IConsole.cs delete mode 100644 violet-message-search-core/hdownloader/Command.cs delete mode 100644 violet-message-search-core/hdownloader/Component/EH.cs delete mode 100644 violet-message-search-core/hdownloader/Component/Hitomi.cs delete mode 100644 violet-message-search-core/hdownloader/Component/HitomiIndex.cs delete mode 100644 violet-message-search-core/hdownloader/Crypto/Hash.cs delete mode 100644 violet-message-search-core/hdownloader/Crypto/Random.cs delete mode 100644 violet-message-search-core/hdownloader/Internals.cs delete mode 100644 violet-message-search-core/hdownloader/Network/NetField.cs delete mode 100644 violet-message-search-core/hdownloader/Network/NetQueue.cs delete mode 100644 violet-message-search-core/hdownloader/Network/NetTask.cs delete mode 100644 violet-message-search-core/hdownloader/Network/NetTaskPass.cs delete mode 100644 violet-message-search-core/hdownloader/Network/NetTools.cs delete mode 100644 violet-message-search-core/hdownloader/Program.cs delete mode 100644 violet-message-search-core/hdownloader/Progress.cs delete mode 100644 violet-message-search-core/hdownloader/Setting/Settings.cs delete mode 100644 violet-message-search-core/hdownloader/Utils/Compress.cs delete mode 100644 violet-message-search-core/hdownloader/Utils/Extends.cs delete mode 100644 violet-message-search-core/hdownloader/Utils/Heap.cs delete mode 100644 violet-message-search-core/hdownloader/Utils/ILazy.cs delete mode 100644 violet-message-search-core/hdownloader/Utils/Strings.cs delete mode 100644 violet-message-search-core/hdownloader/Version.cs delete mode 100644 violet-message-search-core/hdownloader/publish-linux-x64-aot.bat delete mode 100644 violet-message-search-core/hdownloader/publish-linux-x64.bat delete mode 100644 violet-message-search-core/hdownloader/publish-osx.10.15-x64-aot.bat delete mode 100644 violet-message-search-core/hdownloader/publish-osx.10.15-x64.bat delete mode 100644 violet-message-search-core/hdownloader/publish-win10-arm-aot.bat delete mode 100644 violet-message-search-core/hdownloader/publish-win10-arm.bat delete mode 100644 violet-message-search-core/hdownloader/publish-win10-x64-aot.bat delete mode 100644 violet-message-search-core/hdownloader/publish-win10-x64.bat delete mode 100644 violet-message-search-core/hdownloader/publish-win10-x86-aot.bat delete mode 100644 violet-message-search-core/hdownloader/publish-win10-x86.bat diff --git a/violet-message-search-core/hdownloader/.vscode/launch.json b/violet-message-search-core/hdownloader/.vscode/launch.json deleted file mode 100644 index ae8fa35b3..000000000 --- a/violet-message-search-core/hdownloader/.vscode/launch.json +++ /dev/null @@ -1,26 +0,0 @@ -{ - "version": "0.2.0", - "configurations": [ - { - // Use IntelliSense to find out which attributes exist for C# debugging - // Use hover for the description of the existing attributes - // For further information visit https://github.com/OmniSharp/omnisharp-vscode/blob/master/debugger-launchjson.md - "name": ".NET Core Launch (console)", - "type": "coreclr", - "request": "launch", - "preLaunchTask": "build", - // If you have changed target frameworks, make sure to update the program path. - "program": "${workspaceFolder}/bin/Debug/netcoreapp3.1/hdownloader.dll", - "args": [], - "cwd": "${workspaceFolder}", - // For more information about the 'console' field, see https://aka.ms/VSCode-CS-LaunchJson-Console - "console": "internalConsole", - "stopAtEntry": false - }, - { - "name": ".NET Core Attach", - "type": "coreclr", - "request": "attach" - } - ] -} \ No newline at end of file diff --git a/violet-message-search-core/hdownloader/.vscode/tasks.json b/violet-message-search-core/hdownloader/.vscode/tasks.json deleted file mode 100644 index a19e436b2..000000000 --- a/violet-message-search-core/hdownloader/.vscode/tasks.json +++ /dev/null @@ -1,42 +0,0 @@ -{ - "version": "2.0.0", - "tasks": [ - { - "label": "build", - "command": "dotnet", - "type": "process", - "args": [ - "build", - "${workspaceFolder}/hdownloader.csproj", - "/property:GenerateFullPaths=true", - "/consoleloggerparameters:NoSummary" - ], - "problemMatcher": "$msCompile" - }, - { - "label": "publish", - "command": "dotnet", - "type": "process", - "args": [ - "publish", - "${workspaceFolder}/hdownloader.csproj", - "/property:GenerateFullPaths=true", - "/consoleloggerparameters:NoSummary" - ], - "problemMatcher": "$msCompile" - }, - { - "label": "watch", - "command": "dotnet", - "type": "process", - "args": [ - "watch", - "run", - "${workspaceFolder}/hdownloader.csproj", - "/property:GenerateFullPaths=true", - "/consoleloggerparameters:NoSummary" - ], - "problemMatcher": "$msCompile" - } - ] -} \ No newline at end of file diff --git a/violet-message-search-core/hdownloader/AppProvider.cs b/violet-message-search-core/hdownloader/AppProvider.cs deleted file mode 100644 index 063ad4701..000000000 --- a/violet-message-search-core/hdownloader/AppProvider.cs +++ /dev/null @@ -1,52 +0,0 @@ -// This source code is a part of project violet-server. -// Copyright (C)2020-2021. violet-team. Licensed under the MIT Licence. - -using hsync.Log; -using hsync.Network; -using hsync.Setting; -using hsync.Utils; -using System; -using System.Collections.Generic; -using System.Globalization; -using System.IO; -using System.Linq; -using System.Net; -using System.Reflection; -using System.Runtime; -using System.Runtime.CompilerServices; -using System.Text; -using System.Threading; - -namespace hsync -{ - public class AppProvider - { - public static string ApplicationPath = Directory.GetCurrentDirectory(); - public static string DefaultSuperPath = Directory.GetCurrentDirectory(); - - public static Dictionary Instance => - InstanceMonitor.Instances; - - public static NetQueue DownloadQueue { get; set; } - - public static DateTime StartTime = DateTime.Now; - - public static void Initialize() - { - // Initialize logs instance - GCLatencyMode oldMode = GCSettings.LatencyMode; - RuntimeHelpers.PrepareConstrainedRegions(); - GCSettings.LatencyMode = GCLatencyMode.Batch; - - ServicePointManager.DefaultConnectionLimit = int.MaxValue; - - DownloadQueue = new NetQueue(Settings.Instance.Model.ThreadCount); - - GC.Collect(GC.MaxGeneration, GCCollectionMode.Forced); - } - - public static void Deinitialize() - { - } - } -} diff --git a/violet-message-search-core/hdownloader/CL/CommandLine.cs b/violet-message-search-core/hdownloader/CL/CommandLine.cs deleted file mode 100644 index 94528f668..000000000 --- a/violet-message-search-core/hdownloader/CL/CommandLine.cs +++ /dev/null @@ -1,374 +0,0 @@ -// This source code is a part of project violet-server. -// Copyright (C)2020-2021. violet-team. Licensed under the MIT Licence. - -using System; -using System.Collections.Generic; -using System.Linq; -using System.Reflection; -using System.Text; - -namespace hsync.CL -{ - public enum CommandType - { - OPTION, - ARGUMENTS, - EQUAL, - } - - /// - /// Attribute model for the command line parser. - /// - [AttributeUsage(AttributeTargets.Field)] - public class CommandLine : Attribute - { - public CommandType CType { get; private set; } - public string Option { get; private set; } - - /// - /// Usage information of the command. - /// - public string Info { get; set; } - - /// - /// Default argument to insert if the argument does not match. - /// - public bool DefaultArgument { get; set; } - - /// - /// Indicates that the variable will use a pipe. - /// - public bool Pipe { get; set; } - - /// - /// This message is displayed when the command syntax is incorrect. - /// - public string Help { get; set; } - - /// - /// This value is set to true when nothing is entered in the pipe. - /// - public bool PipeDefault { get; set; } = false; - - /// - /// This value is set to true when nothing is entered. - /// - public bool Default { get; set; } = false; - - /// - /// For ARGUMENTS type, specifies the number of arguments. - /// - public int ArgumentsCount { get; set; } = 1; - - /// - /// One character option - /// - public string ShortOption { get; set; } - - /// - /// - /// - /// Option token. - /// - public CommandLine(string option, CommandType type) - { - Option = option; - CType = type; - } - } - - /// - /// Tools for organizing the command line. - /// - public class CommandLineUtil - { - /// - /// Checks whether there is an option in the argument array. - /// - /// - /// - public static bool AnyOption(string[] args) - { - return args.ToList().Any(x => x[0] == '-'); - } - - /// - /// Checks whether the argument array contains a string. - /// - /// - /// - public static bool AnyStrings(string[] args) - { - return args.ToList().Any(x => x[0] != '-'); - } - - /// - /// Gets whether a specific argument is included. - /// - /// - /// - /// - public static bool AnyArgument(string[] args, string arg) - { - return args.ToList().Any(x => x == arg); - } - - /// - /// Delete a specific argument. - /// - /// - /// - public static string[] DeleteArgument(string[] args, string arg) - { - var list = args.ToList(); - list.Remove(arg); - return list.ToArray(); - } - - /// - /// Put specific options at the beginning. - /// - /// - /// - /// - public static string[] PushFront(string[] args, string option) - { - var list = args.ToList(); - list.Insert(0, option); - return list.ToArray(); - } - - /// - /// Put specific options in specific locations. - /// - /// - /// - /// - public static string[] Insert(string[] args, string option, int index) - { - var list = args.ToList(); - list.Insert(index, option); - return list.ToArray(); - } - - /// - /// If there is a mismatched argument, get it. - /// - /// - /// - /// - public static List GetWeirdArguments(string[] argv) - where T : IConsoleOption, new() - { - var field = CommandLineParser.GetFields(typeof(T)); - List result = new List(); - - for (int i = 0; i < argv.Length; i++) - { - string token = argv[i].Split('=')[0]; - if (field.ContainsKey(token)) - { - var cl = field[token]; - if (cl.Item2.CType == CommandType.ARGUMENTS) - i += cl.Item2.ArgumentsCount; - } - else - { - result.Add(i); - } - } - - return result; - } - - /// - /// Insert default arguments. - /// - /// - /// - /// - /// - public static string[] InsertWeirdArguments(string[] args, bool pipe, string option) - where T : IConsoleOption, new() - { - var weird = GetWeirdArguments(args); - - if (weird.Count > 0 && pipe) - args = Insert(args, option, weird[0]); - - return args; - } - - /// - /// Separate the combined factors. - /// - /// - /// - public static string[] SplitCombinedOptions(string[] args) - { - List result = new List(); - foreach (var arg in args) - { - if (arg.Length > 1 && arg.StartsWith("-") && !arg.StartsWith("--") && !arg.Contains("=")) - { - for (int i = 1; i < arg.Length; i++) - result.Add($"-{arg[i]}"); - } - else - { - result.Add(arg); - } - } - return result.ToArray(); - } - } - - /// - /// The command line parser. - /// - /// - public class CommandLineParser - { - /// - /// Get field information that contains Attribute information. - /// - /// - public static Dictionary> GetFields(Type type) - { - FieldInfo[] fields = type.GetFields(); - var field = new Dictionary>(); - - foreach (FieldInfo m in fields) - { - object[] attrs = m.GetCustomAttributes(false); - - foreach (var cl in attrs) - { - if (cl is CommandLine clcast) - { - field.Add(clcast.Option, Tuple.Create(m.Name, clcast)); - if (!string.IsNullOrEmpty(clcast.ShortOption)) - field.Add(clcast.ShortOption, Tuple.Create(m.Name, clcast)); - } - } - } - - return field; - } - - /// - /// Parse command lines based on attributes. - /// - /// - /// - /// - public static T Parse(T model, string[] argv, bool pipe = false, string contents = "") where T : IConsoleOption, new() - { - var field = GetFields(typeof(T)); - - // - // This flag is enabled if there is no option - // - bool any_option = true; - - for (int i = 0; i < argv.Length; i++) - { - string token = argv[i].Split('=')[0]; - if (field.ContainsKey(token)) - { - var cl = field[token]; - if (cl.Item2.CType == CommandType.OPTION) - { - // - // In the case of the OPTION type, the variable must be set to true. - // - typeof(T).GetField(cl.Item1).SetValue(model, true); - } - else if (cl.Item2.CType == CommandType.ARGUMENTS) - { - List sub_args = new List(); - - int arguments_count = cl.Item2.ArgumentsCount; - - if (cl.Item2.Pipe == true && pipe == true) - { - arguments_count--; - sub_args.Add(contents); - } - - for (int j = 1; j <= arguments_count; j++) - { - if (i + j == argv.Length) - { - typeof(T).GetField("Error").SetValue(model, true); - typeof(T).GetField("ErrorMessage").SetValue(model, $"'{argv[i]}' require {arguments_count - j + 1} more sub arguments."); - typeof(T).GetField("HelpMessage").SetValue(model, cl.Item2.Help); - return model; - } - - sub_args.Add(argv[i + j]); - } - - i += cl.Item2.ArgumentsCount; - - typeof(T).GetField(cl.Item1).SetValue(model, sub_args.ToArray()); - } - else if (cl.Item2.CType == CommandType.EQUAL) - { - string[] split = argv[i].Split('='); - - if (split.Length == 1) - { - typeof(T).GetField("Error").SetValue(model, true); - typeof(T).GetField("ErrorMessage").SetValue(model, $"'{split[0]}' must have equal delimiter."); - typeof(T).GetField("HelpMessage").SetValue(model, cl.Item2.Help); - return model; - } - - typeof(T).GetField(cl.Item1).SetValue(model, split[1]); - } - any_option = false; - } - else - { - typeof(T).GetField("Error").SetValue(model, true); - typeof(T).GetField("ErrorMessage").SetValue(model, $"'{argv[i]}' is not correct arguments."); - return model; - } - } - - if (any_option) - { - // - // Find and activate the first Default - // - foreach (var kv in field) - { - if (!pipe && kv.Value.Item2.Default) - { - typeof(T).GetField(kv.Value.Item1).SetValue(model, true); - break; - } - else if (pipe && kv.Value.Item2.PipeDefault) - { - typeof(T).GetField(kv.Value.Item1).SetValue(model, new[] { contents }); - break; - } - } - } - - return model; - } - - /// - /// Parse command lines based on attributes. - /// - /// - /// - /// - public static T Parse(string[] argv, bool pipe = false, string contents = "") where T : IConsoleOption, new() - { - return Parse(new T(), argv, pipe, contents); - } - } -} diff --git a/violet-message-search-core/hdownloader/CL/IConsole.cs b/violet-message-search-core/hdownloader/CL/IConsole.cs deleted file mode 100644 index da10aeda1..000000000 --- a/violet-message-search-core/hdownloader/CL/IConsole.cs +++ /dev/null @@ -1,19 +0,0 @@ -// This source code is a part of project violet-server. -// Copyright (C)2020-2021. violet-team. Licensed under the MIT Licence. - -using System; -using System.Collections.Generic; -using System.Text; - -namespace hsync.CL -{ - /// - /// This is a console option that must be included. - /// - public class IConsoleOption - { - public bool Error; - public string ErrorMessage; - public string HelpMessage; - } -} diff --git a/violet-message-search-core/hdownloader/Command.cs b/violet-message-search-core/hdownloader/Command.cs deleted file mode 100644 index 691e70751..000000000 --- a/violet-message-search-core/hdownloader/Command.cs +++ /dev/null @@ -1,283 +0,0 @@ -// This source code is a part of project violet-server. -// Copyright (C)2020-2021. violet-team. Licensed under the MIT Licence. - -using System; -using System.Collections.Generic; -using System.Globalization; -using System.IO; -using System.Linq; -using System.Net; -using System.Text; -using System.Threading; -using Extreme.Mathematics; -using Extreme.Statistics; -using hsync.CL; -using hsync.Component; -using hsync.Log; -using hsync.Network; -using hsync.Setting; -using hsync.Utils; -using MySql.Data.MySqlClient; -using Newtonsoft.Json; -using Newtonsoft.Json.Converters; -using Newtonsoft.Json.Linq; -using SQLite; - -namespace hsync -{ - public class Options : IConsoleOption - { - [CommandLine("--help", CommandType.OPTION)] - public bool Help; - [CommandLine("--version", CommandType.OPTION, ShortOption = "-v", Info = "Show version information.")] - public bool Version; - - /// - /// Atomic Options - /// - - [CommandLine("--recover-settings", CommandType.OPTION, Info = "Recover settings.json")] - public bool RecoverSettings; - - [CommandLine("--search", CommandType.ARGUMENTS, ArgumentsCount = 1, ShortOption = "-s", - Info = "Search", Help = "use --search ")] - public string[] Search; - - [CommandLine("--download-from-id", CommandType.ARGUMENTS, ArgumentsCount = 1, ShortOption = "-d", - Info = "Download from id", Help = "use --download-from-id ")] - public string[] DownloadFromId; - - /// - /// Test Option - /// - - [CommandLine("--test", CommandType.ARGUMENTS, ArgumentsCount = 1, ShortOption = "-t", - Info = "hysnc test option", Help = "use --test ")] - public string[] Test; - } - - public class Command - { - public static void Start(string[] arguments) - { - arguments = CommandLineUtil.SplitCombinedOptions(arguments); - var option = CommandLineParser.Parse(arguments); - - // - // Single Commands - // - if (option.Help) - { - PrintHelp(); - } - else if (option.Version) - { - PrintVersion(); - } - else if (option.RecoverSettings) - { - Settings.Instance.Recover(); - Settings.Instance.Save(); - } - else if (option.DownloadFromId != null) - { - ProcessDownloadFromId(option.DownloadFromId); - } - else if (option.Search != null) - { - ProcessSearch(option.Search); - } - else if (option.Test != null) - { - ProcessTest(option.Test); - } - else if (option.Error) - { - Console.WriteLine(option.ErrorMessage); - if (option.HelpMessage != null) - Console.WriteLine(option.HelpMessage); - return; - } - else - { - Console.WriteLine("Nothing to work on."); - Console.WriteLine("Enter './hsync --help' to get more information"); - } - - return; - } - - static byte[] art_console = { - 0x8D, 0x54, 0x3D, 0x6F, 0xDB, 0x30, 0x10, 0xDD, 0x0D, 0xF8, 0x3F, 0x5C, 0xB8, 0x48, 0x01, 0x74, 0xE4, 0x18, 0xC0, 0xB0, 0xBC, - 0x0B, 0x59, 0xB3, 0x18, 0x50, 0x45, 0x23, 0xD9, 0x0A, 0x01, 0xDD, 0x3A, 0x34, 0x2E, 0x7F, 0x7B, 0xDF, 0xBB, 0x93, 0x14, 0xA7, - 0x92, 0xDB, 0xD0, 0x24, 0x41, 0x1F, 0xDF, 0xDD, 0xBD, 0xFB, 0xA0, 0x44, 0xBE, 0x38, 0xD2, 0x8B, 0xA4, 0x6E, 0xBF, 0xFB, 0x0F, - 0x48, 0xAE, 0x98, 0x12, 0xB5, 0xA4, 0x7F, 0x41, 0x5F, 0x7A, 0x39, 0x8B, 0x74, 0x42, 0x34, 0x74, 0x24, 0xDF, 0x80, 0xE1, 0xE7, - 0xF3, 0xB8, 0x4A, 0x4F, 0xA4, 0xE1, 0xCF, 0x9F, 0x2D, 0x77, 0x32, 0x52, 0xA3, 0xFB, 0x30, 0x0B, 0x18, 0x26, 0xA4, 0xD8, 0x61, - 0x68, 0xC6, 0xFA, 0x0D, 0xBD, 0x8E, 0x93, 0x07, 0xB7, 0x4A, 0xF5, 0x5E, 0x2E, 0x3C, 0x9C, 0x01, 0xCD, 0xD9, 0x2E, 0x4C, 0x8A, - 0x0D, 0x88, 0x11, 0xB2, 0x71, 0xC6, 0x09, 0x91, 0x39, 0xCA, 0x15, 0xD0, 0x5E, 0x8A, 0x42, 0x7A, 0x31, 0x29, 0x36, 0x4E, 0xC8, - 0xFC, 0x24, 0x97, 0xC8, 0x1C, 0xD0, 0x0D, 0x09, 0x50, 0x52, 0x34, 0x4A, 0xC0, 0xA2, 0x09, 0xFC, 0x1F, 0x62, 0xC6, 0x72, 0x49, - 0x72, 0x04, 0xA0, 0x51, 0x15, 0x38, 0x90, 0x28, 0xEA, 0xBE, 0x78, 0xCA, 0x51, 0x01, 0x0B, 0x02, 0x79, 0xC2, 0xE2, 0x89, 0x61, - 0x9D, 0x94, 0xBA, 0x34, 0x2B, 0xBC, 0x92, 0x72, 0xD2, 0xC0, 0x50, 0x03, 0x68, 0x88, 0x3C, 0x4D, 0xEB, 0xDB, 0x7E, 0x07, 0x57, - 0x39, 0x97, 0x00, 0x38, 0x50, 0xD4, 0x78, 0x17, 0x23, 0x47, 0x2E, 0xCC, 0x48, 0x24, 0x01, 0x67, 0x7A, 0x44, 0x02, 0x80, 0xA4, - 0xDD, 0xB9, 0x1A, 0x99, 0x97, 0xB4, 0xC8, 0x74, 0xA1, 0x68, 0x72, 0xF0, 0x98, 0x64, 0x80, 0x3D, 0x33, 0x38, 0x8D, 0x52, 0x2F, - 0xD0, 0x53, 0xCC, 0x07, 0x4B, 0xF1, 0x08, 0xCF, 0x18, 0x4B, 0xC1, 0x06, 0x90, 0x68, 0x20, 0x80, 0xFB, 0x30, 0xDB, 0x7E, 0x00, - 0x0D, 0x8D, 0xE4, 0x37, 0x22, 0x40, 0x37, 0x05, 0x0A, 0x7F, 0xB7, 0x0F, 0xAD, 0x93, 0x57, 0x4D, 0x52, 0x95, 0x89, 0x02, 0x95, - 0x1A, 0x72, 0xD2, 0xF6, 0x15, 0xA4, 0xF3, 0xE3, 0xAA, 0xE7, 0x26, 0x2D, 0x90, 0x22, 0xA1, 0xA5, 0xC5, 0xAC, 0x9E, 0x18, 0x6F, - 0xA1, 0xFC, 0x90, 0x7E, 0xDD, 0xA9, 0xBD, 0x13, 0x41, 0x15, 0x99, 0x5C, 0x13, 0xC5, 0x81, 0x72, 0x63, 0x5E, 0x2C, 0xF3, 0x6B, - 0x67, 0x8B, 0x3B, 0x96, 0xCE, 0x23, 0xF1, 0xDD, 0x82, 0xB4, 0xF1, 0xB8, 0xF9, 0x2C, 0x12, 0x7E, 0x68, 0x2B, 0x91, 0xCA, 0x8A, - 0x59, 0x4D, 0x5C, 0x67, 0x65, 0xA9, 0xB6, 0x94, 0x2C, 0xDF, 0x71, 0x86, 0x65, 0x73, 0x1A, 0xF5, 0x78, 0xFB, 0x94, 0x6E, 0x5D, - 0x18, 0xB8, 0x62, 0xE1, 0x83, 0xC5, 0x95, 0xAC, 0x63, 0x3F, 0x00, 0xCD, 0xAF, 0x76, 0x95, 0xF3, 0xD9, 0x91, 0xB9, 0x40, 0xCE, - 0xBD, 0xA8, 0xCF, 0xD8, 0x51, 0x20, 0x36, 0xAA, 0x8D, 0x74, 0xF7, 0xA9, 0x07, 0x6D, 0x2C, 0x79, 0xCC, 0x76, 0x07, 0x87, 0xD6, - 0x2E, 0x39, 0xBF, 0xAB, 0x2A, 0x5A, 0xA4, 0x6E, 0xEF, 0x79, 0x24, 0xDF, 0xC4, 0x42, 0x1B, 0xC3, 0xA3, 0x91, 0x40, 0xB1, 0xA7, - 0x8B, 0x7B, 0x3A, 0xE8, 0x8A, 0xE4, 0x01, 0x2D, 0x91, 0x35, 0x3F, 0x5B, 0x10, 0xA8, 0xEB, 0x6D, 0x95, 0x88, 0x07, 0x88, 0xD4, - 0x3B, 0x14, 0xD6, 0x7F, 0xA3, 0x9F, 0x53, 0x6A, 0xDB, 0x96, 0x8F, 0x6F, 0x53, 0x85, 0x85, 0xAA, 0x98, 0x09, 0xC4, 0x8F, 0x3E, - 0x16, 0x46, 0x82, 0x30, 0x74, 0x03, 0x8C, 0x7E, 0xF1, 0x2E, 0xB5, 0xB4, 0xE1, 0x8B, 0x63, 0xFC, 0xC7, 0x71, 0x0D, 0xB5, 0x2A, - 0xDA, 0xC4, 0xD3, 0x92, 0xC3, 0xDC, 0x2A, 0xF8, 0x9C, 0x6C, 0xB6, 0xB3, 0x15, 0xE3, 0x8A, 0xDF, 0x77, 0x7F, 0xEF, 0x3E, 0xCA, - 0xB0, 0xC1, 0xA1, 0xA0, 0x1D, 0xEA, 0x1C, 0x07, 0xD4, 0x7C, 0xBF, 0xFB, 0x03, - }; - - static void PrintHelp() - { - PrintVersion(); - Console.WriteLine(Encoding.UTF8.GetString(CompressUtils.Decompress(art_console))); - Console.WriteLine($"Copyright (C) 2020-2021. project violet-server."); - Console.WriteLine("Usage: ./hdownloader [OPTIONS...]"); - - var builder = new StringBuilder(); - CommandLineParser.GetFields(typeof(Options)).ToList().ForEach( - x => - { - var key = x.Key; - if (!key.StartsWith("--")) - return; - if (!string.IsNullOrEmpty(x.Value.Item2.ShortOption)) - key = $"{x.Value.Item2.ShortOption}, " + key; - var help = ""; - if (!string.IsNullOrEmpty(x.Value.Item2.Help)) - help = $"[{x.Value.Item2.Help}]"; - if (!string.IsNullOrEmpty(x.Value.Item2.Info)) - builder.Append($" {key}".PadRight(30) + $" {x.Value.Item2.Info} {help}\r\n"); - else - builder.Append($" {key}".PadRight(30) + $" {help}\r\n"); - }); - Console.Write(builder.ToString()); - } - - public static void PrintVersion() - { - Console.WriteLine($"{Version.Name} {Version.Text}"); - Console.WriteLine($"Build Date: " + Internals.GetBuildDate().ToLongDateString()); - } - - static void ProcessDownloadFromId(string[] args) - { - var id = args[0]; - var imgs = NetTools.DownloadString($"https://ltn.hitomi.la/galleries/{id}.js"); - var arr = JToken.Parse(imgs.Substring(imgs.IndexOf('=') + 1))["files"]; - - var number_of_frontends = 3; - var subdomain = Convert.ToChar(97 + (Convert.ToInt32(id.Last()) % number_of_frontends)); - if (id.Last() == '0') - subdomain = 'a'; - - var img_urls = new List(); - foreach (var obj in (JArray)arr) - { - var hash = obj.Value("hash"); - var postfix = hash.Substring(hash.Length - 3); - - var subdomainx = subdomain; - - if (obj.Value("haswebp") == 0) - { - subdomainx = 'b'; - } - - int x; - var xx = int.TryParse($"{postfix[0]}{postfix[1]}", NumberStyles.HexNumber, CultureInfo.InvariantCulture, out x); - - if (xx) - { - var o = 0; - if (x < 0x88) o = 1; - if (x < 0x44) o = 2; - subdomainx = Convert.ToChar(97 + o); - } - - if (obj.Value("haswebp") == 0 || hash == null) - img_urls.Add($"https://{subdomainx}a.hitomi.la/images/{postfix[2]}/{postfix[0]}{postfix[1]}/{hash}.{obj.Value("name").Split('.').Last()}"); - else if (hash == "") - img_urls.Add($"https://{subdomainx}a.hitomi.la/webp/{obj.Value("name")}.webp"); - else if (hash.Length < 3) - img_urls.Add($"https://{subdomainx}a.hitomi.la/webp/{hash}.webp"); - else - img_urls.Add($"https://{subdomainx}a.hitomi.la/webp/{postfix[2]}/{postfix[0]}{postfix[1]}/{hash}.webp"); - } - - // Console.WriteLine(Logs.SerializeObject(img_urls)); - - var dcnt = 0; - using (var pb = new DownloadProgressBar()) - { - if (!Directory.Exists(id)) - Directory.CreateDirectory(id); - NetTools.DownloadFiles( - Enumerable.Range(0, img_urls.Count).Select((x) => (img_urls[x], $"{id}/" + x.ToString().PadLeft(4, '0') + Path.GetExtension(img_urls[x]))).ToList(), - "", (sz) => - { - pb.Report(img_urls.Count, dcnt, sz); - }, () => - { - pb.Report(img_urls.Count, Interlocked.Increment(ref dcnt), 0); - }).Wait(); - } - } - - static void ProcessSearch(string[] args) - { - var db = new SQLiteConnection("data.db"); - var rr = db.Query("SELECT * FROM HitomiColumnModel WHERE " + args[0] + - " ORDER BY Id DESC LIMIT 10", new object[] { }); - foreach (var r in rr) - { - Console.WriteLine(JsonConvert.SerializeObject(r)); - } - } - - static void ProcessTest(string[] args) - { - switch (args[0]) - { - case "help": - Console.WriteLine(""); - break; - - case "latestrows": - { - var db = new SQLiteConnection("data.db"); - var rr = db.Query("SELECT * FROM HitomiColumnModel WHERE Language='korean'" + - " ORDER BY Id DESC LIMIT 10", new object[] { }); - foreach (var r in rr) - { - Console.WriteLine(JsonConvert.SerializeObject(r)); - } - } - break; - - case "exportids": - { - var db = new SQLiteConnection("data.db"); - var rr = db.Query("SELECT Id FROM HitomiColumnModel WHERE Language='korean' " + - "AND ExistsOnHitomi=1 ORDER BY Id", new object[] { }); - foreach (var r in rr) - { - Console.Write(r.Id + ","); - } - } - break; - - } - - } - } -} diff --git a/violet-message-search-core/hdownloader/Component/EH.cs b/violet-message-search-core/hdownloader/Component/EH.cs deleted file mode 100644 index 5a3fb63ba..000000000 --- a/violet-message-search-core/hdownloader/Component/EH.cs +++ /dev/null @@ -1,412 +0,0 @@ -// This source code is a part of project violet-server. -// Copyright (C)2020-2021. violet-team. Licensed under the MIT Licence. - -using HtmlAgilityPack; -using System; -using System.Collections.Generic; -using System.Linq; -using System.Net; -using System.Text; -using hsync.Log; -using System.Text.RegularExpressions; - -namespace hsync.Component -{ - public class EHentaiArticle - { - public string Thumbnail { get; set; } - - public string Title { get; set; } - public string SubTitle { get; set; } - - public string Type { get; set; } - public string Uploader { get; set; } - - public string Posted; - public string Parent; - public string Visible; - public string Language; - public string FileSize; - public int Length; - public int Favorited; - - public string reclass; - public string[] language; - public string[] group; - public string[] parody; - public string[] character; - public string[] artist; - public string[] male; - public string[] female; - public string[] misc; - - public Tuple[] comment; - public List ImagesLink { get; set; } - public string Archive { get; set; } - } - - public class EHentaiResultArticle - { - public string URL; - - public string Thumbnail; - public string Title; - - public string Uploader; - public string Published; - public string Files; - public string Type; - - public Dictionary> Descripts; - } - - public class EHentaiParser - { - public static EHentaiArticle ParseArticleData(string source) - { - EHentaiArticle article = new EHentaiArticle(); - - HtmlDocument document = new HtmlDocument(); - document.LoadHtml(source); - HtmlNode nodes = document.DocumentNode.SelectNodes("//div[@class='gm']")[0]; - - article.Thumbnail = Regex.Match(nodes.SelectSingleNode(".//div[@id='gleft']//div//div").GetAttributeValue("style", ""), @"https://ehgt.org/.*?(?=\))").Groups[0].Value; - - article.Title = nodes.SelectSingleNode(".//div[@id='gd2']//h1[@id='gn']").InnerText; - article.SubTitle = nodes.SelectSingleNode(".//div[@id='gd2']//h1[@id='gj']").InnerText; - - // article.Type = nodes.SelectSingleNode(".//div[@id='gmid']//div//div[@id='gdc']//a//img").GetAttributeValue("alt", ""); - article.Uploader = nodes.SelectSingleNode(".//div[@id='gmid']//div//div[@id='gdn']//a").InnerText; - - HtmlNodeCollection nodes_static = nodes.SelectNodes(".//div[@id='gmid']//div//div[@id='gdd']//table//tr"); - - article.Posted = nodes_static[0].SelectSingleNode(".//td[@class='gdt2']").InnerText; - article.Parent = nodes_static[1].SelectSingleNode(".//td[@class='gdt2']").InnerText; - article.Visible = nodes_static[2].SelectSingleNode(".//td[@class='gdt2']").InnerText; - article.Language = nodes_static[3].SelectSingleNode(".//td[@class='gdt2']").InnerText.Split(' ')[0].ToLower(); - article.FileSize = nodes_static[4].SelectSingleNode(".//td[@class='gdt2']").InnerText; - int.TryParse(nodes_static[5].SelectSingleNode(".//td[@class='gdt2']").InnerText.Split(' ')[0], out article.Length); - int.TryParse(nodes_static[6].SelectSingleNode(".//td[@class='gdt2']").InnerText.Split(' ')[0], out article.Favorited); - - HtmlNodeCollection nodes_data = nodes.SelectNodes(".//div[@id='gmid']//div[@id='gd4']//table//tr"); - - Dictionary information = new Dictionary(); - - foreach (var i in nodes_data) - { - try - { - information.Add(i.SelectNodes(".//td")[0].InnerText.Trim(), - i.SelectNodes(".//td")[1].SelectNodes(".//div").Select(e => e.SelectSingleNode(".//a").InnerText).ToArray()); - } - catch { } - } - - if (information.ContainsKey("language:")) article.language = information["language:"]; - if (information.ContainsKey("group:")) article.group = information["group:"]; - if (information.ContainsKey("parody:")) article.parody = information["parody:"]; - if (information.ContainsKey("character:")) article.character = information["character:"]; - if (information.ContainsKey("artist:")) article.artist = information["artist:"]; - if (information.ContainsKey("male:")) article.male = information["male:"]; - if (information.ContainsKey("female:")) article.female = information["female:"]; - if (information.ContainsKey("misc:")) article.misc = information["misc:"]; - - HtmlNode nodesc = document.DocumentNode.SelectNodes("//div[@id='cdiv']")[0]; - HtmlNodeCollection nodes_datac = nodesc.SelectNodes(".//div[@class='c1']"); - List> comments = new List>(); - - foreach (var i in nodes_datac ?? Enumerable.Empty()) - { - try - { - string date = WebUtility.HtmlDecode(i.SelectNodes(".//div[@class='c2']//div[@class='c3']")[0].InnerText.Trim()); - string author = WebUtility.HtmlDecode(i.SelectNodes(".//div[@class='c2']//div[@class='c3']//a")[0].InnerText.Trim()); - string contents = Regex.Replace(WebUtility.HtmlDecode(i.SelectNodes(".//div[@class='c6']")[0].InnerHtml.Trim()), @"
", "\r\n"); - comments.Add(new Tuple( - DateTime.Parse(date.Remove(date.IndexOf(" UTC")).Substring("Posted on ".Length) + "Z"), - author, - contents)); - } - catch { } - } - - comments.Sort((a, b) => a.Item1.CompareTo(b.Item1)); - article.comment = comments.ToArray(); - - return article; - } - } - - public class ExHentaiParser - { - public static EHentaiArticle ParseArticleData(string source) - { - EHentaiArticle article = new EHentaiArticle(); - - HtmlDocument document = new HtmlDocument(); - document.LoadHtml(source); - HtmlNode nodes = document.DocumentNode.SelectNodes("//div[@class='gm']")[0]; - - article.Thumbnail = Regex.Match(nodes.SelectSingleNode(".//div[@id='gleft']//div//div").GetAttributeValue("style", ""), @"https://exhentai.org/.*?(?=\))").Groups[0].Value; - - article.Title = nodes.SelectSingleNode(".//div[@id='gd2']//h1[@id='gn']").InnerText; - article.SubTitle = nodes.SelectSingleNode(".//div[@id='gd2']//h1[@id='gj']").InnerText; - - //article.Type = nodes.SelectSingleNode(".//div[@id='gmid']//div//div[@id='gdc']//a//img").GetAttributeValue("alt", ""); - article.Uploader = nodes.SelectSingleNode(".//div[@id='gmid']//div//div[@id='gdn']//a").InnerText; - - HtmlNodeCollection nodes_static = nodes.SelectNodes(".//div[@id='gmid']//div//div[@id='gdd']//table//tr"); - - article.Posted = nodes_static[0].SelectSingleNode(".//td[@class='gdt2']").InnerText; - article.Parent = nodes_static[1].SelectSingleNode(".//td[@class='gdt2']").InnerText; - article.Visible = nodes_static[2].SelectSingleNode(".//td[@class='gdt2']").InnerText; - article.Language = nodes_static[3].SelectSingleNode(".//td[@class='gdt2']").InnerText.Split(' ')[0].ToLower(); - article.FileSize = nodes_static[4].SelectSingleNode(".//td[@class='gdt2']").InnerText; - int.TryParse(nodes_static[5].SelectSingleNode(".//td[@class='gdt2']").InnerText.Split(' ')[0], out article.Length); - int.TryParse(nodes_static[6].SelectSingleNode(".//td[@class='gdt2']").InnerText.Split(' ')[0], out article.Favorited); - - HtmlNodeCollection nodes_data = nodes.SelectNodes(".//div[@id='gmid']//div[@id='gd4']//table//tr"); - - Dictionary information = new Dictionary(); - - foreach (var i in nodes_data) - { - try - { - information.Add(i.SelectNodes(".//td")[0].InnerText.Trim(), - i.SelectNodes(".//td")[1].SelectNodes(".//div").Select(e => e.SelectSingleNode(".//a").InnerText).ToArray()); - } - catch { } - } - - if (information.ContainsKey("language:")) article.language = information["language:"]; - if (information.ContainsKey("group:")) article.group = information["group:"]; - if (information.ContainsKey("parody:")) article.parody = information["parody:"]; - if (information.ContainsKey("character:")) article.character = information["character:"]; - if (information.ContainsKey("artist:")) article.artist = information["artist:"]; - if (information.ContainsKey("male:")) article.male = information["male:"]; - if (information.ContainsKey("female:")) article.female = information["female:"]; - if (information.ContainsKey("misc:")) article.misc = information["misc:"]; - - HtmlNode nodesc = document.DocumentNode.SelectNodes("//div[@id='cdiv']")[0]; - HtmlNodeCollection nodes_datac = nodesc.SelectNodes(".//div[@class='c1']"); - List> comments = new List>(); - - foreach (var i in nodes_datac ?? Enumerable.Empty()) - { - try - { - string date = WebUtility.HtmlDecode(i.SelectNodes(".//div[@class='c2']//div[@class='c3']")[0].InnerText.Trim()); - string author = WebUtility.HtmlDecode(i.SelectNodes(".//div[@class='c2']//div[@class='c3']//a")[0].InnerText.Trim()); - string contents = Regex.Replace(WebUtility.HtmlDecode(i.SelectNodes(".//div[@class='c6']")[0].InnerHtml.Trim()), @"
", "\r\n"); - comments.Add(new Tuple( - DateTime.Parse(date.Remove(date.IndexOf(" by")).Substring("Posted on ".Length) + "Z"), - author, - contents)); - } - catch (Exception e) - { - Logs.Instance.PushError("[Fail] \r\n" + Logs.SerializeObject(e)); - } - } - - comments.Sort((a, b) => a.Item1.CompareTo(b.Item1)); - article.comment = comments.ToArray(); - - return article; - } - - /// - /// 결과 페이지를 분석합니다. - /// ex: https://exhentai.org/ - /// ex: https://exhentai.org/?inline_set=dm_t - /// ex: https://exhentai.org/?page=1 - /// - /// - /// - public static List ParseResultPageThumbnailView(string html) - { - var result = new List(); - - var document = new HtmlDocument(); - document.LoadHtml(html); - var nodes = document.DocumentNode.SelectNodes("//div[@class='itg']/div[@class='id1']"); - - foreach (var node in nodes) - { - try - { - var article = new EHentaiResultArticle(); - - article.URL = node.SelectSingleNode("./div[@class='id2']/a").GetAttributeValue("href", ""); - - try { article.Thumbnail = node.SelectSingleNode("./div[@class='id3']/a/img").GetAttributeValue("src", ""); } catch { } - article.Title = node.SelectSingleNode("./div[@class='id2']/a").InnerText; - - article.Files = node.SelectSingleNode(".//div[@class='id42']").InnerText; - article.Type = node.SelectSingleNode(".//div[@class='id41']").GetAttributeValue("title", ""); - - result.Add(article); - } - catch { } - } - - return result; - } - - /// - /// 결과 페이지를 분석합니다. - /// ex: https://exhentai.org/ - /// ex: https://exhentai.org/?inline_set=dm_l - /// ex: https://exhentai.org/?page=1 - /// - /// - /// - public static List ParseResultPageListView(string html) - { - var result = new List(); - - var document = new HtmlDocument(); - document.LoadHtml(html); - var nodes = document.DocumentNode.SelectNodes("//table[@class='itg']/tr"); - - if (nodes.Count > 1) nodes.RemoveAt(0); - - foreach (var node in nodes) - { - try - { - var article = new EHentaiResultArticle(); - - article.URL = node.SelectSingleNode("./td[3]/div/div[@class='it5']/a").GetAttributeValue("href", ""); - - try { article.Thumbnail = node.SelectSingleNode("./td[3]/div/div[@class='it2']/img").GetAttributeValue("src", ""); } catch { } - article.Title = node.SelectSingleNode("./td[3]/div/div[@class='it5']/a").InnerText; - - article.Uploader = node.SelectSingleNode("./td[4]/div/a").GetAttributeValue("href", ""); - article.Published = node.SelectSingleNode("./td[2]").InnerText; - article.Type = node.SelectSingleNode("./td/a/img").GetAttributeValue("alt", ""); - - result.Add(article); - } - catch { } - } - - return result; - } - - /// - /// 결과 페이지를 분석합니다. - /// ex: https://exhentai.org/?inline_set=dm_e - /// - /// - /// - public static List ParseResultPageExtendedListView(string html) - { - var result = new List(); - - var document = new HtmlDocument(); - document.LoadHtml(html); - - Queue nodes = new Queue(); - var fn = document.DocumentNode.SelectNodes("//table[@class='itg glte']/tr"); - fn.ToList().ForEach(x => nodes.Enqueue(x)); - - while (nodes.Count != 0) - { - var node = nodes.Dequeue(); - try - { - var article = new EHentaiResultArticle(); - - article.URL = node.SelectSingleNode(".//a").GetAttributeValue("href", ""); - try { article.Thumbnail = node.SelectSingleNode(".//img").GetAttributeValue("src", ""); } catch { } - - var g13 = node.SelectSingleNode("./td[2]/div/div"); - - article.Type = g13.SelectSingleNode("./div").InnerText.ToLower(); - article.Published = g13.SelectSingleNode("./div[2]").InnerText; - article.Uploader = g13.SelectSingleNode("./div[4]").InnerText; - article.Files = g13.SelectSingleNode("./div[5]").InnerText; - - var gref = node.SelectSingleNode("./td[2]/div/a/div"); - - article.Title = gref.SelectSingleNode("./div").InnerText; - - if (article.Title.Contains("느와카나")) - ; - - try - { - var desc = gref.SelectNodes("./div//tr"); - var desc_dic = new Dictionary>(); - - foreach (var nn in desc) - { - var cont = nn.SelectSingleNode("./td").InnerText.Trim(); - cont = cont.Remove(cont.Length - 1); - - var cc = new List(); - - foreach (var ccc in nn.SelectNodes("./td[2]//div")) - { - cc.Add(ccc.InnerText); - } - - desc_dic.Add(cont, cc); - } - - article.Descripts = desc_dic; - } - catch { } - result.Add(article); - - var next = node.SelectNodes("./tr"); - if (next != null) - next.ToList().ForEach(x => nodes.Enqueue(x)); - } - catch { } - } - - return result; - } - - /// - /// 결과 페이지를 분석합니다. - /// ex: https://exhentai.org/?inline_set=dm_m - /// - /// - /// - public static List ParseResultPageMinimalListView(string html) - { - var result = new List(); - - var document = new HtmlDocument(); - document.LoadHtml(html); - - HtmlNodeCollection nodes = document.DocumentNode.SelectNodes("//table[@class='itg gltm']/tr"); - - for (int i = 1; i < nodes.Count; i++) - { - var node = nodes[i]; - - var article = new EHentaiResultArticle(); - - article.Type = node.SelectSingleNode("./td/div").InnerText.Trim().ToLower(); - article.Thumbnail = node.SelectSingleNode(".//img").GetAttributeValue("src", ""); - if (article.Thumbnail.StartsWith("data")) - article.Thumbnail = node.SelectSingleNode(".//img").GetAttributeValue("data-src", ""); - article.Published = node.SelectSingleNode("./td[2]/div[2]/div[2]/div[1]/div[2]").InnerText.Trim(); - article.Files = node.SelectSingleNode("./td[2]/div[2]/div[2]/div[2]/div[2]").InnerText.Trim(); - - article.URL = node.SelectSingleNode("./td[4]/a").GetAttributeValue("href", ""); - article.Title = node.SelectSingleNode("./td[4]/a/div").InnerText.Trim(); - article.Uploader = node.SelectSingleNode("./td[6]/div/a").InnerText.Trim(); - - result.Add(article); - } - - return result; - } - } -} diff --git a/violet-message-search-core/hdownloader/Component/Hitomi.cs b/violet-message-search-core/hdownloader/Component/Hitomi.cs deleted file mode 100644 index 90230fa86..000000000 --- a/violet-message-search-core/hdownloader/Component/Hitomi.cs +++ /dev/null @@ -1,342 +0,0 @@ -// This source code is a part of project violet-server. -// Copyright (C)2020-2021. violet-team. Licensed under the MIT Licence. - -using hsync.Utils; -using HtmlAgilityPack; -using Newtonsoft.Json; -using Newtonsoft.Json.Converters; -using SQLite; -using System; -using System.Collections.Generic; -using System.IO; -using System.Linq; - -namespace hsync.Component -{ - public struct HitomiMetadata - { - [JsonProperty(PropertyName = "a")] - public string[] Artists { get; set; } - [JsonProperty(PropertyName = "g")] - public string[] Groups { get; set; } - [JsonProperty(PropertyName = "p")] - public string[] Parodies { get; set; } - [JsonProperty(PropertyName = "t")] - public string[] Tags { get; set; } - [JsonProperty(PropertyName = "c")] - public string[] Characters { get; set; } - [JsonProperty(PropertyName = "l")] - public string Language { get; set; } - [JsonProperty(PropertyName = "n")] - public string Name { get; set; } - [JsonProperty(PropertyName = "type")] - public string Type { get; set; } - [JsonProperty(PropertyName = "id")] - public int ID { get; set; } - - [JsonIgnore] - public DateTime? DateTime; - } - - public class HitomiArticle - { - public string[] Artists { get; set; } - public string[] Characters { get; set; } - public string[] Groups { get; set; } - public string Language { get; set; } - public string[] Series { get; set; } - public string[] Tags { get; set; } - public string Type { get; set; } - public bool ManualPathOrdering { get; set; } - public string ManualAdditionalPath { get; set; } - public string DateTime { get; set; } - - public string Thumbnail { get; set; } - public string Magic { get; set; } - public string Title { get; set; } - } - - public abstract class SQLiteColumnModel - { - [PrimaryKey] - public int Id { get; set; } - } - - public class HitomiColumnModel : SQLiteColumnModel - { - public string Title { get; set; } - public string EHash { get; set; } - public string Type { get; set; } - public string Artists { get; set; } - public string Characters { get; set; } - public string Groups { get; set; } - public string Language { get; set; } - public string Series { get; set; } - public string Tags { get; set; } - public string Uploader { get; set; } - public DateTime? Published { get; set; } - public int Files { get; set; } - public string Class { get; set; } - public int ExistOnHitomi { get; set; } - public string Thumbnail { get; set; } - } - - public class HitomiIndexArtistsColumnModel : SQLiteColumnModel { public string Tag { get; set; } } - public class HitomiIndexGroupsColumnModel : SQLiteColumnModel { public string Tag { get; set; } } - public class HitomiIndexSeriesColumnModel : SQLiteColumnModel { public string Tag { get; set; } } - public class HitomiIndexCharactersColumnModel : SQLiteColumnModel { public string Tag { get; set; } } - public class HitomiIndexLanguagesColumnModel : SQLiteColumnModel { public string Tag { get; set; } } - public class HitomiIndexTypesColumnModel : SQLiteColumnModel { public string Tag { get; set; } } - public class HitomiIndexTagsColumnModel : SQLiteColumnModel { public string Tag { get; set; } } - - //public class HitomiIndexModel : SQLiteColumnModel - //{ - // public string Name { get; set; } - // public int Index { get; set; } - //} - - public class HitomiLegalize - { - public static HitomiMetadata ArticleToMetadata(HitomiArticle article) - { - HitomiMetadata metadata = new HitomiMetadata(); - if (article.Artists != null) metadata.Artists = article.Artists; - if (article.Characters != null) metadata.Characters = article.Characters; - if (article.Groups != null) metadata.Groups = article.Groups; - try - { - if (article.Magic.Contains("-")) - metadata.ID = Convert.ToInt32(article.Magic.Split('-').Last().Split('.')[0]); - else if (article.Magic.Contains("galleries")) - metadata.ID = Convert.ToInt32(article.Magic.Split('/').Last().Split('.')[0]); - else - metadata.ID = Convert.ToInt32(article.Magic); - } - catch - { - ; - } - metadata.Language = LegalizeLanguage(article.Language); - metadata.Name = article.Title; - if (article.Series != null) metadata.Parodies = article.Series; - if (article.Tags != null) metadata.Tags = article.Tags.Select(x => LegalizeTag(x)).ToArray(); - metadata.Type = article.Type; - if (article.DateTime != null) - metadata.DateTime = DateTime.Parse(article.DateTime); - return metadata; - } - - public static string LegalizeTag(string tag) - { - if (tag.Trim().EndsWith("♀")) return "female:" + tag.Trim('♀').Trim(); - if (tag.Trim().EndsWith("♂")) return "male:" + tag.Trim('♂').Trim(); - return tag.Trim(); - } - - public static string LegalizeLanguage(string lang) - { - switch (lang) - { - case "모든 언어": return "all"; - case "한국어": return "korean"; - case "N/A": return "n/a"; - case "日本語": return "japanese"; - case "English": return "english"; - case "Español": return "spanish"; - case "ไทย": return "thai"; - case "Deutsch": return "german"; - case "中文": return "chinese"; - case "Português": return "portuguese"; - case "Français": return "french"; - case "Tagalog": return "tagalog"; - case "Русский": return "russian"; - case "Italiano": return "italian"; - case "polski": return "polish"; - case "tiếng việt": return "vietnamese"; - case "magyar": return "hungarian"; - case "Čeština": return "czech"; - case "Bahasa Indonesia": return "indonesian"; - case "العربية": return "arabic"; - } - - return lang; - } - - public static string DeLegalizeLanguage(string lang) - { - switch (lang) - { - case "all": return "모든 언어"; - case "korean": return "한국어"; - case "n/a": return "N/A"; - case "japanese": return "日本語"; - case "english": return "English"; - case "spanish": return "Español"; - case "thai": return "ไทย"; - case "german": return "Deutsch"; - case "chinese": return "中文"; - case "portuguese": return "Português"; - case "french": return "Français"; - case "tagalog": return "Tagalog"; - case "russian": return "Русский"; - case "italian": return "Italiano"; - case "polish": return "polski"; - case "vietnamese": return "tiếng việt"; - case "hungarian": return "magyar"; - case "czech": return "Čeština"; - case "indonesian": return "Bahasa Indonesia"; - case "arabic": return "العربية"; - } - - return lang; - } - } - - public class HitomiParser - { - static public HitomiArticle ParseGalleryBlock(string source) - { - HitomiArticle article = new HitomiArticle(); - - HtmlDocument document = new HtmlDocument(); - document.LoadHtml(source); - HtmlNode nodes = document.DocumentNode.SelectNodes("/div")[0]; - - article.Magic = nodes.SelectSingleNode("./a").GetAttributeValue("href", ""); - try { article.Thumbnail = nodes.SelectSingleNode("./a//img").GetAttributeValue("data-src", "").Substring("//tn.hitomi.la/".Length).Replace("smallbig", "big"); } - catch - { article.Thumbnail = nodes.SelectSingleNode("./a//img").GetAttributeValue("src", "").Substring("//tn.hitomi.la/".Length); } - article.Title = nodes.SelectSingleNode("./h1").InnerText; - - try { article.Artists = nodes.SelectNodes(".//div[@class='artist-list']//li").Select(node => node.SelectSingleNode("./a").InnerText).ToArray(); } - catch { article.Artists = new[] { "N/A" }; } - - var contents = nodes.SelectSingleNode("./div[2]/table"); - try { article.Series = contents.SelectNodes("./tr[1]/td[2]/ul/li").Select(node => node.SelectSingleNode(".//a").InnerText).ToArray(); } catch { } - article.Type = contents.SelectSingleNode("./tr[2]/td[2]/a").InnerText; - try { article.Language = HitomiLegalize.LegalizeLanguage(contents.SelectSingleNode("./tr[3]/td[2]/a").InnerText); } catch { } - try { article.Tags = contents.SelectNodes("./tr[4]/td[2]/ul/li").Select(node => HitomiLegalize.LegalizeTag(node.SelectSingleNode(".//a").InnerText)).ToArray(); } catch { } - - article.DateTime = nodes.SelectSingleNode("./div[2]/p").InnerText; - - return article; - } - - static public HitomiArticle ParseGallery(string source) - { - HitomiArticle article = new HitomiArticle(); - - HtmlDocument document = new HtmlDocument(); - document.LoadHtml(source); - HtmlNode nodes = document.DocumentNode.SelectSingleNode("//div[@class='content']"); - - try - { - article.Magic = nodes.SelectSingleNode("./div[3]/h1/a").GetAttributeValue("href", "").Split('/')[2].Split('.')[0]; - } - catch - { - ; - } - //article.Title = nodes.SelectSingleNode("./div[3]/h1").InnerText.Trim(); - //article.Thumbnail = nodes.SelectSingleNode("./div[2]/div/a/img").GetAttributeValue("src", ""); - //article.Artists = nodes.SelectSingleNode(".") - - foreach (var tr in document.DocumentNode.SelectNodes("//div[@class='gallery-info']/table/tr").ToList()) - { - var tt = tr.SelectSingleNode("./td").InnerText.ToLower().Trim(); - if (tt == "group") - { - article.Groups = tr.SelectNodes(".//a")?.Select(x => x.InnerText.Trim()).ToArray(); - } - else if (tt == "characters") - { - article.Characters = tr.SelectNodes(".//a")?.Select(x => x.InnerText.Trim()).ToArray(); - } - } - - return article; - } - - static public void FillGallery(string source, HitomiArticle article) - { - HtmlDocument document = new HtmlDocument(); - document.LoadHtml(source); - HtmlNode nodes = document.DocumentNode.SelectSingleNode("/div[@class='gallery-info']/table/tbody"); - - foreach (var tr in nodes.SelectNodes("./tr").ToList()) - { - var tt = tr.SelectSingleNode("./td").InnerText.ToLower().Trim(); - if (tt == "group") - { - article.Groups = tr.SelectNodes(".//a").Select(x => x.InnerText.Trim()).ToArray(); - } - else if (tt == "characters") - { - article.Characters = tr.SelectNodes(".//a").Select(x => x.InnerText.Trim()).ToArray(); - } - } - } - } - - public class HitomiData : ILazy - { - public List metadata_collection; - - public void Load() - { - Log.Logs.Instance.Push("Load metadata files..."); - - try - { - metadata_collection = JsonConvert.DeserializeObject>(File.ReadAllText("metadata.json")); - - var articles = JsonConvert.DeserializeObject>(File.ReadAllText("hiddendata.json")); - var overlap = new HashSet(); - if (metadata_collection == null) - metadata_collection = new List(); - metadata_collection.ForEach(x => overlap.Add(x.ID.ToString())); - foreach (var article in articles) - { - if (overlap.Contains(article.Magic)) continue; - metadata_collection.Add(HitomiLegalize.ArticleToMetadata(article)); - } - metadata_collection.Sort((a, b) => b.ID.CompareTo(a.ID)); - } - catch (Exception e) - { - Log.Logs.Instance.PushError("Metadata loading error!"); - Log.Logs.Instance.PushException(e); - Log.Logs.Instance.Panic(); - } - } - - public void SaveWithNewData(List data) - { - var articles = JsonConvert.DeserializeObject>(File.ReadAllText("hiddendata.json")); - File.Move("hiddendata.json", $"hiddendata-{DateTime.Now.Ticks}.json"); - - articles.AddRange(data); - var overlap = new HashSet(); - var pure = new List(); - foreach (var article in articles) - { - if (!overlap.Contains(article.Magic)) - { - pure.Add(article); - overlap.Add(article.Magic); - } - } - - JsonSerializer serializer = new JsonSerializer(); - serializer.Converters.Add(new JavaScriptDateTimeConverter()); - serializer.NullValueHandling = NullValueHandling.Ignore; - - using (StreamWriter sw = new StreamWriter("hiddendata.json")) - using (JsonWriter writer = new JsonTextWriter(sw)) - { - serializer.Serialize(writer, pure); - } - } - } -} diff --git a/violet-message-search-core/hdownloader/Component/HitomiIndex.cs b/violet-message-search-core/hdownloader/Component/HitomiIndex.cs deleted file mode 100644 index e45147d36..000000000 --- a/violet-message-search-core/hdownloader/Component/HitomiIndex.cs +++ /dev/null @@ -1,215 +0,0 @@ -using hsync.Utils; -using MessagePack; -using Newtonsoft.Json; -using System; -using System.Collections.Generic; -using System.IO; -using System.Linq; -using System.Text; - -namespace hsync.Component -{ - [MessagePackObject] - public class HitomiIndexModel - { - [Key(0)] - public string[] Artists; - [Key(1)] - public string[] Groups; - [Key(2)] - public string[] Series; - [Key(3)] - public string[] Characters; - [Key(4)] - public string[] Languages; - [Key(5)] - public string[] Types; - [Key(6)] - public string[] Tags; - } - - [MessagePackObject] - public struct HitomiIndexMetadata - { - [Key(0)] - public int[] Artists { get; set; } - [Key(1)] - public int[] Groups { get; set; } - [Key(2)] - public int[] Parodies { get; set; } - [Key(3)] - public int[] Tags { get; set; } - [Key(4)] - public int[] Characters { get; set; } - [Key(5)] - public int Language { get; set; } - [Key(6)] - public string Name { get; set; } - [Key(7)] - public int Type { get; set; } - [Key(8)] - public int ID { get; set; } - - [JsonIgnore] - public DateTime? DateTime; - } - - [MessagePackObject] - public class HitomiIndexDataModel - { - [Key(0)] - public HitomiIndexModel index; - [Key(1)] - public List metadata; - } - - public class HitomiIndex : ILazy - { - private static void add(Dictionary dic, string arr) - { - if (arr == null) return; - if (!dic.ContainsKey(arr)) - dic.Add(arr, dic.Count); - } - - private static void add(Dictionary dic, string[] arr) - { - if (arr == null) return; - foreach (var item in arr) - if (!dic.ContainsKey(item)) - dic.Add(item, dic.Count); - } - - private static string[] pp(Dictionary dic) - { - var list = dic.ToList(); - list.Sort((x, y) => x.Value.CompareTo(y.Value)); - return list.Select(x => x.Key).ToArray(); - } - - public static (HitomiIndexModel, List) MakeIndexF() - { - var artists = new Dictionary(); - var groups = new Dictionary(); - var series = new Dictionary(); - var characters = new Dictionary(); - var languages = new Dictionary(); - var types = new Dictionary(); - var tags = new Dictionary(); - - foreach (var md in HitomiData.Instance.metadata_collection) - { - add(artists, md.Artists); - add(groups, md.Groups); - add(series, md.Parodies); - add(characters, md.Characters); - if (md.Language != null) - add(languages, md.Language.ToLower()); - else - add(languages, md.Language); - if (md.Type != null) - add(types, md.Type.ToLower()); - else - add(types, md.Type); - add(tags, md.Tags); - } - - var index = new HitomiIndexModel(); - - index.Artists = pp(artists); - index.Groups = pp(groups); - index.Series = pp(series); - index.Characters = pp(characters); - index.Languages = pp(languages); - index.Types = pp(types); - index.Tags = pp(tags); - - var mdl = new List(); - - foreach (var md in HitomiData.Instance.metadata_collection) - { - var him = new HitomiIndexMetadata(); - him.ID = md.ID; - him.Name = md.Name; - if (md.Artists != null) him.Artists = md.Artists.Select(x => artists[x]).ToArray(); - if (md.Groups != null) him.Groups = md.Groups.Select(x => groups[x]).ToArray(); - if (md.Parodies != null) him.Parodies = md.Parodies.Select(x => series[x]).ToArray(); - if (md.Characters != null) him.Characters = md.Characters.Select(x => characters[x]).ToArray(); - if (md.Language != null) him.Language = languages[md.Language.ToLower()]; else him.Language = -1; - if (md.Type != null) him.Type = types[md.Type.ToLower()]; else him.Type = -1; - if (md.Tags != null) him.Tags = md.Tags.Select(x => tags[x]).ToArray(); - him.DateTime = md.DateTime; - mdl.Add(him); - } - - return (index, mdl); - } - - public static void MakeIndex() - { - var artists = new Dictionary(); - var groups = new Dictionary(); - var series = new Dictionary(); - var characters = new Dictionary(); - var languages = new Dictionary(); - var types = new Dictionary(); - var tags = new Dictionary(); - - foreach (var md in HitomiData.Instance.metadata_collection) - { - add(artists, md.Artists); - add(groups, md.Groups); - add(series, md.Parodies); - add(characters, md.Characters); - if (md.Language != null) - add(languages, md.Language.ToLower()); - else - add(languages, md.Language); - if (md.Type != null) - add(types, md.Type.ToLower()); - else - add(types, md.Type); - add(tags, md.Tags); - } - - var index = new HitomiIndexModel(); - - index.Artists = pp(artists); - index.Groups = pp(groups); - index.Series = pp(series); - index.Characters = pp(characters); - index.Languages = pp(languages); - index.Types = pp(types); - index.Tags = pp(tags); - - var mdl = new List(); - - foreach (var md in HitomiData.Instance.metadata_collection) - { - var him = new HitomiIndexMetadata(); - him.ID = md.ID; - him.Name = md.Name; - if (md.Artists != null) him.Artists = md.Artists.Select(x => artists[x]).ToArray(); - if (md.Groups != null) him.Groups = md.Groups.Select(x => groups[x]).ToArray(); - if (md.Parodies != null) him.Parodies = md.Parodies.Select(x => series[x]).ToArray(); - if (md.Characters != null) him.Characters = md.Characters.Select(x => characters[x]).ToArray(); - if (md.Language != null) him.Language = languages[md.Language.ToLower()]; else him.Language = -1; - if (md.Type != null) him.Type = types[md.Type.ToLower()]; else him.Type = -1; - if (md.Tags != null) him.Tags = md.Tags.Select(x => tags[x]).ToArray(); - mdl.Add(him); - } - - var result = new HitomiIndexDataModel(); - result.index = index; - result.metadata = mdl; - - var bbb = MessagePackSerializer.Serialize(result); - using (FileStream fsStream = new FileStream("index-metadata.json", FileMode.Create)) - using (BinaryWriter sw = new BinaryWriter(fsStream)) - { - sw.Write(bbb); - } - } - - } -} diff --git a/violet-message-search-core/hdownloader/Crypto/Hash.cs b/violet-message-search-core/hdownloader/Crypto/Hash.cs deleted file mode 100644 index d808d1bd3..000000000 --- a/violet-message-search-core/hdownloader/Crypto/Hash.cs +++ /dev/null @@ -1,52 +0,0 @@ -// This source code is a part of project violet-server. -// Copyright (C)2020-2021. violet-team. Licensed under the MIT Licence. - -using System; -using System.Collections.Generic; -using System.IO; -using System.Security.Cryptography; -using System.Text; - -namespace hsync.Crypto -{ - public static class Hash - { - public static string GetFileHash(this string file) - { - using (FileStream stream = File.OpenRead(file)) - { - SHA512Managed sha = new SHA512Managed(); - byte[] hash = sha.ComputeHash(stream); - return BitConverter.ToString(hash).Replace("-", String.Empty); - } - } - - public static string GetHashSHA1(this string str) - { - SHA1Managed sha = new SHA1Managed(); - byte[] hash = sha.ComputeHash(Encoding.UTF8.GetBytes(str)); - return BitConverter.ToString(hash).Replace("-", String.Empty); - } - - public static string GetHashSHA256(this string str) - { - SHA256Managed sha = new SHA256Managed(); - byte[] hash = sha.ComputeHash(Encoding.UTF8.GetBytes(str)); - return BitConverter.ToString(hash).Replace("-", String.Empty); - } - - public static string GetHashSHA512(this string str) - { - SHA512Managed sha = new SHA512Managed(); - byte[] hash = sha.ComputeHash(Encoding.UTF8.GetBytes(str)); - return BitConverter.ToString(hash).Replace("-", String.Empty); - } - - public static string GetHashMD5(this string str) - { - var md5 = MD5.Create(); - byte[] hash = md5.ComputeHash(Encoding.UTF8.GetBytes(str)); - return BitConverter.ToString(hash).Replace("-", String.Empty); - } - } -} diff --git a/violet-message-search-core/hdownloader/Crypto/Random.cs b/violet-message-search-core/hdownloader/Crypto/Random.cs deleted file mode 100644 index 2335934e7..000000000 --- a/violet-message-search-core/hdownloader/Crypto/Random.cs +++ /dev/null @@ -1,21 +0,0 @@ -// This source code is a part of project violet-server. -// Copyright (C)2020-2021. violet-team. Licensed under the MIT Licence. - -using System; -using System.Collections.Generic; -using System.Linq; -using System.Text; - -namespace hsync.Crypto -{ - public class Random - { - private static System.Random random = new System.Random(); - public static string RandomString(int length) - { - const string chars = "ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789"; - return new string(Enumerable.Repeat(chars, length) - .Select(s => s[random.Next(s.Length)]).ToArray()); - } - } -} diff --git a/violet-message-search-core/hdownloader/Internals.cs b/violet-message-search-core/hdownloader/Internals.cs deleted file mode 100644 index 28474f5cb..000000000 --- a/violet-message-search-core/hdownloader/Internals.cs +++ /dev/null @@ -1,126 +0,0 @@ -// This source code is a part of project violet-server. -// Copyright (C)2020-2021. violet-team. Licensed under the MIT Licence. - -using System; -using System.Collections.Generic; -using System.Globalization; -using System.IO; -using System.Linq; -using System.Reflection; -using System.Text; - -namespace hsync -{ - public class Internals - { - public static DateTime GetBuildDate() - { - const string BuildVersionMetadataPrefix = "+build"; - - var attribute = Assembly.GetExecutingAssembly().GetCustomAttribute(); - if (attribute?.InformationalVersion != null) - { - var value = attribute.InformationalVersion; - var index = value.IndexOf(BuildVersionMetadataPrefix); - if (index > 0) - { - value = value.Substring(index + BuildVersionMetadataPrefix.Length); - if (DateTime.TryParseExact(value, "yyyyMMddHHmmss", CultureInfo.InvariantCulture, DateTimeStyles.None, out var result)) - { - return result; - } - } - } - - return default; - } - - #region Low Level - - public const BindingFlags DefaultBinding = BindingFlags.NonPublic | - BindingFlags.Instance | BindingFlags.IgnoreCase | BindingFlags.Public | BindingFlags.FlattenHierarchy; - - public const BindingFlags CommonBinding = BindingFlags.Instance | BindingFlags.Public; - - public static List get_all_fields(Type t, BindingFlags flags) - { - if (t == null) - return new List(); - - var list = t.GetFields(flags).ToList(); - list.AddRange(get_all_fields(t.BaseType, flags)); - return list; - } - - public static List enum_recursion(object obj, string[] bb, int ptr) - { - if (bb.Length == ptr) - { - return get_all_fields(obj.GetType(), DefaultBinding); - } - return enum_recursion(obj.GetType().GetField(bb[ptr], DefaultBinding).GetValue(obj), bb, ptr + 1); - } - - public static List enum_recursion(object obj, string[] bb, int ptr, BindingFlags option) - { - if (bb.Length == ptr) - { - return obj.GetType().GetFields(option).ToList(); - } - var x = obj.GetType().GetField(bb[ptr], DefaultBinding); - return enum_recursion(obj.GetType().GetField(bb[ptr], DefaultBinding).GetValue(obj), bb, ptr + 1, option); - } - - public static object get_recursion(object obj, string[] bb, int ptr) - { - if (bb.Length == ptr) - { - return obj; - } - return get_recursion(obj.GetType().GetField(bb[ptr], DefaultBinding).GetValue(obj), bb, ptr + 1); - } - - public static void set_recursion(object obj, string[] bb, int ptr, object val) - { - if (bb.Length - 1 == ptr) - { - obj.GetType().GetField(bb[ptr]).SetValue(obj, - Convert.ChangeType(val, obj.GetType().GetField(bb[ptr], DefaultBinding).GetValue(obj).GetType())); - return; - } - set_recursion(obj.GetType().GetField(bb[ptr]).GetValue(obj), bb, ptr + 1, val); - } - - public static List enum_methods(object obj, string[] bb, int ptr, BindingFlags option) - { - if (bb.Length == ptr) - { - return obj.GetType().GetMethods(option).ToList(); - } - var x = obj.GetType().GetField(bb[ptr], DefaultBinding); - return enum_methods(obj.GetType().GetField(bb[ptr], DefaultBinding).GetValue(obj), bb, ptr + 1, option); - } - - public static object call_method(object obj, string[] bb, int ptr, BindingFlags option, object[] param) - { - if (bb.Length - 1 == ptr) - { - return obj.GetType().GetMethods(option | BindingFlags.Static).Where(y => y.Name == bb[ptr]).ToList()[0].Invoke(obj, param); - } - var x = obj.GetType().GetField(bb[ptr], DefaultBinding | BindingFlags.Static); - return call_method(obj.GetType().GetField(bb[ptr], DefaultBinding | BindingFlags.Static).GetValue(obj), bb, ptr + 1, option, param); - } - - public static ParameterInfo[] get_method_paraminfo(object obj, string[] bb, int ptr, BindingFlags option) - { - if (bb.Length - 1 == ptr) - { - return obj.GetType().GetMethods(option).Where(y => y.Name == bb[ptr]).ToList()[0].GetParameters(); - } - var x = obj.GetType().GetField(bb[ptr], DefaultBinding); - return get_method_paraminfo(obj.GetType().GetField(bb[ptr], DefaultBinding).GetValue(obj), bb, ptr + 1, option); - } - - #endregion - } -} diff --git a/violet-message-search-core/hdownloader/Network/NetField.cs b/violet-message-search-core/hdownloader/Network/NetField.cs deleted file mode 100644 index e2b018ac1..000000000 --- a/violet-message-search-core/hdownloader/Network/NetField.cs +++ /dev/null @@ -1,384 +0,0 @@ -// This source code is a part of project violet-server. -// Copyright (C)2020-2021. violet-team. Licensed under the MIT Licence. - -using System; -using System.Collections.Generic; -using System.IO; -using System.Linq; -using System.Net; -using System.Text; -using System.Threading; - -namespace hsync.Network -{ - /// - /// Implementaion of real download procedure - /// - public class NetField - { - public static void Do(NetTask content) - { - var retry_count = 0; - - RETRY_PROCEDURE: - - if (content.Cancel != null && content.Cancel.IsCancellationRequested) - { - content.CancleCallback(); - return; - } - - NetTaskPass.RunOnField(ref content); - - //if (content.DownloadString) - // Log.Logs.Instance.Push("[NetField] Start download string... " + content.Url); - //else if (content.MemoryCache) - // Log.Logs.Instance.Push("[NetField] Start download to memory... " + content.Url); - //else if (content.SaveFile) - // Log.Logs.Instance.Push("[NetField] Start download file... " + content.Url + " to " + content.Filename); - - REDIRECTION: - - if (content.Cancel != null && content.Cancel.IsCancellationRequested) - { - content.CancleCallback(); - return; - } - - content.StartCallback?.Invoke(); - - try - { - // - // Initialize http-web-request - // - - var request = (HttpWebRequest)WebRequest.Create(content.Url); - content.Request = request; - - request.Accept = content.Accept; - request.UserAgent = content.UserAgent; - - if (content.Referer != null) - request.Referer = content.Referer; - else - request.Referer = (content.Url.StartsWith("https://") ? "https://" : (content.Url.Split(':')[0] + "//")) + request.RequestUri.Host; - - if (content.Cookie != null) - request.Headers.Add(HttpRequestHeader.Cookie, content.Cookie); - - if (content.Headers != null) - content.Headers.ToList().ForEach(p => request.Headers.Add(p.Key, p.Value)); - - if (content.Proxy != null) - request.Proxy = content.Proxy; - - if (content.TimeoutInfinite) - request.Timeout = Timeout.Infinite; - else - request.Timeout = content.TimeoutMillisecond; - - request.AllowAutoRedirect = content.AutoRedirection; - - // - // POST Data - // - - if (content.Query != null) - { - request.Method = "POST"; - request.ContentType = "application/x-www-form-urlencoded"; - - var request_stream = new StreamWriter(request.GetRequestStream()); - var query = string.Join("&", content.Query.ToList().Select(x => $"{x.Key}={x.Value}")); - request_stream.Write(query); - request_stream.Close(); - - if (content.Cancel != null && content.Cancel.IsCancellationRequested) - { - content.CancleCallback(); - return; - } - } - - // - // Wait request - // - - using (HttpWebResponse response = (HttpWebResponse)request.GetResponse()) - { - if (response.StatusCode == HttpStatusCode.NotFound || - response.StatusCode == HttpStatusCode.Forbidden || - response.StatusCode == HttpStatusCode.Unauthorized || - response.StatusCode == HttpStatusCode.BadRequest || - response.StatusCode == HttpStatusCode.InternalServerError) - { - // - // Cannot continue - // - - content.ErrorCallback?.Invoke(NetTask.NetError.CannotContinueByCriticalError); - return; - } - else if (response.StatusCode == HttpStatusCode.Moved || - response.StatusCode == HttpStatusCode.Redirect) - { - if (content.AutoRedirection) - { - var old = content.Url; - content.Url = response.Headers.Get("Location"); - Log.Logs.Instance.Push("[NetField] Redirection " + old + " to " + content.Url); - goto REDIRECTION; - } - } - else if (response.StatusCode == HttpStatusCode.OK) - { - if (content.Cancel != null && content.Cancel.IsCancellationRequested) - { - content.CancleCallback(); - return; - } - - content.HeaderReceive?.Invoke(response.Headers.ToString()); - content.CookieReceive?.Invoke(response.Cookies); - - Stream istream = response.GetResponseStream(); - Stream ostream = null; - - if (content.DownloadString || content.MemoryCache) - { - ostream = new MemoryStream(); - } - else if (content.DriveCache) - { - // TODO: - } - else - { - ostream = File.OpenWrite(content.Filename); - } - - content.SizeCallback?.Invoke(response.ContentLength); - - if (content.NotifyOnlySize) - { - ostream.Close(); - istream.Close(); - return; - } - - if (content.Cancel != null && content.Cancel.IsCancellationRequested) - { - content.CancleCallback(); - return; - } - - byte[] buffer = new byte[content.DownloadBufferSize]; - long byte_read = 0; - - // - // Download loop - // - - do - { - if (content.Cancel != null && content.Cancel.IsCancellationRequested) - { - content.CancleCallback(); - return; - } - - byte_read = istream.Read(buffer, 0, buffer.Length); - ostream.Write(buffer, 0, (int)byte_read); - - if (content.Cancel != null && content.Cancel.IsCancellationRequested) - { - content.CancleCallback(); - return; - } - - content.DownloadCallback?.Invoke(byte_read); - - } while (byte_read != 0); - - // - // Notify Complete - // - - if (content.DownloadString) - { - if (content.Encoding == null) - content.CompleteCallbackString(Encoding.UTF8.GetString(((MemoryStream)ostream).ToArray())); - else - content.CompleteCallbackString(content.Encoding.GetString(((MemoryStream)ostream).ToArray())); - } - else if (content.MemoryCache) - { - content.CompleteCallbackBytes(((MemoryStream)ostream).ToArray()); - } - else - { - content.CompleteCallback?.Invoke(); - } - - ostream.Close(); - istream.Close(); - - return; - } - } - } - catch (WebException e) - { - var response = (HttpWebResponse)e.Response; - - if (response != null && response.StatusCode == HttpStatusCode.Moved) - { - if (content.AutoRedirection) - { - var old = content.Url; - content.Url = response.Headers.Get("Location"); - Log.Logs.Instance.Push("[NetField] Redirection " + old + " to " + content.Url); - goto REDIRECTION; - } - } - - //lock (Log.Logs.Instance) - //{ - // Log.Logs.Instance.PushError("[NetField] Web Excpetion - " + e.Message + "\r\n" + e.StackTrace); - // Log.Logs.Instance.PushError(content); - //} - - if (content.FailUrls != null && retry_count < content.FailUrls.Count) - { - content.Url = content.FailUrls[retry_count++]; - content.RetryCallback?.Invoke(retry_count); - - lock (Log.Logs.Instance) - { - Log.Logs.Instance.Push($"[NetField] Retry [{retry_count}/{content.RetryCount}]"); - Log.Logs.Instance.Push(content); - } - goto RETRY_PROCEDURE; - } - - if ((response != null && ( - response.StatusCode == HttpStatusCode.NotFound || - response.StatusCode == HttpStatusCode.Forbidden || - response.StatusCode == HttpStatusCode.Unauthorized || - response.StatusCode == HttpStatusCode.BadRequest || - response.StatusCode == HttpStatusCode.InternalServerError)) || - e.Status == WebExceptionStatus.NameResolutionFailure || - e.Status == WebExceptionStatus.UnknownError) - { - if (response != null && response.StatusCode == HttpStatusCode.Forbidden && response.Cookies != null) - { - content.CookieReceive?.Invoke(response.Cookies); - return; - } - - // - // Cannot continue - // - - if (e.Status == WebExceptionStatus.UnknownError) - { - lock (Log.Logs.Instance) - { - Log.Logs.Instance.PushError("[NetField] Check your Firewall, Router or DPI settings."); - Log.Logs.Instance.PushError("[NetField] If you continue to receive this error, please contact developer."); - } - - content.ErrorCallback?.Invoke(NetTask.NetError.UnknowError); - } - else - { - content.ErrorCallback?.Invoke(NetTask.NetError.CannotContinueByCriticalError); - } - - return; - } - } - catch (UriFormatException e) - { - lock (Log.Logs.Instance) - { - Log.Logs.Instance.PushError("[NetField] URI Exception - " + e.Message + "\r\n" + e.StackTrace); - Log.Logs.Instance.PushError(content); - } - - // - // Cannot continue - // - - content.ErrorCallback?.Invoke(NetTask.NetError.UriFormatError); - return; - } - catch (Exception e) - { - lock (Log.Logs.Instance) - { - Log.Logs.Instance.PushError("[NetField] Unhandled Excpetion - " + e.Message + "\r\n" + e.StackTrace); - Log.Logs.Instance.PushError(content); - } - } - - // - // Request Aborted - // - - if (content.Aborted) - { - content.ErrorCallback?.Invoke(NetTask.NetError.Aborted); - return; - } - - // - // Retry - // - - //if (content.FailUrls != null && retry_count < content.FailUrls.Count) - //{ - // content.Url = content.FailUrls[retry_count++]; - // content.RetryCallback?.Invoke(retry_count); - - // lock (Log.Logs.Instance) - // { - // Log.Logs.Instance.Push($"[NetField] Retry [{retry_count}/{content.RetryCount}]"); - // Log.Logs.Instance.Push(content); - // } - // goto RETRY_PROCEDURE; - //} - - //if (content.RetryWhenFail) - //{ - // if (content.RetryCount > retry_count) - // { - // retry_count += 1; - - // content.RetryCallback?.Invoke(retry_count); - - // lock (Log.Logs.Instance) - // { - // Log.Logs.Instance.Push($"[NetField] Retry [{retry_count}/{content.RetryCount}]"); - // Log.Logs.Instance.Push(content); - // } - // goto RETRY_PROCEDURE; - // } - - // // - // // Many retry - // // - - // lock (Log.Logs.Instance) - // { - // Log.Logs.Instance.Push($"[NetField] Many Retry"); - // Log.Logs.Instance.Push(content); - // } - // content.ErrorCallback?.Invoke(NetTask.NetError.ManyRetry); - //} - - content.ErrorCallback?.Invoke(NetTask.NetError.Unhandled); - } - } -} diff --git a/violet-message-search-core/hdownloader/Network/NetQueue.cs b/violet-message-search-core/hdownloader/Network/NetQueue.cs deleted file mode 100644 index 4c3a97e7a..000000000 --- a/violet-message-search-core/hdownloader/Network/NetQueue.cs +++ /dev/null @@ -1,49 +0,0 @@ -// This source code is a part of project violet-server. -// Copyright (C)2020-2021. violet-team. Licensed under the MIT Licence. - -using hsync.Utils; -using System; -using System.Collections.Generic; -using System.Text; -using System.Threading; -using System.Threading.Tasks; - -namespace hsync.Network -{ - /// - /// Download Queue Implementation - /// - public class NetQueue - { - public Queue queue = new Queue(); - - SemaphoreSlim semaphore; - int capacity = 0; - - public NetQueue(int capacity = 0) - { - this.capacity = capacity; - - if (this.capacity == 0) - this.capacity = Environment.ProcessorCount; - - int count = 50; //816; - ThreadPool.SetMinThreads(count, count); - semaphore = new SemaphoreSlim(count, count); - } - - public Task Add(NetTask task) - { - return Task.Run(async () => - { - await semaphore.WaitAsync().ConfigureAwait(false); - _ = Task.Run(() => - { - NetField.Do(task); - semaphore.Release(); - }).ConfigureAwait(false); - }); - } - - } -} diff --git a/violet-message-search-core/hdownloader/Network/NetTask.cs b/violet-message-search-core/hdownloader/Network/NetTask.cs deleted file mode 100644 index 94795a654..000000000 --- a/violet-message-search-core/hdownloader/Network/NetTask.cs +++ /dev/null @@ -1,170 +0,0 @@ -// This source code is a part of project violet-server. -// Copyright (C)2020-2021. violet-team. Licensed under the MIT Licence. - -using hsync.Setting; -using Newtonsoft.Json; -using System; -using System.Collections.Generic; -using System.Net; -using System.Text; -using System.Threading; - -namespace hsync.Network -{ - /// - /// Information of what download for - /// - [JsonObject(MemberSerialization.OptIn)] - public class NetTask - { - public static NetTask MakeDefault(string url, string cookie = "") - => new NetTask - { - Accept = "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8", - UserAgent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/66.0.3359.139 Safari/537.36", - TimeoutInfinite = Settings.Instance.Network.TimeoutInfinite, - TimeoutMillisecond = Settings.Instance.Network.TimeoutMillisecond, - AutoRedirection = true, - RetryWhenFail = true, - RetryCount = Settings.Instance.Network.RetryCount, - DownloadBufferSize = Settings.Instance.Network.DownloadBufferSize, - Proxy = !string.IsNullOrEmpty(Settings.Instance.Network.Proxy) ? new WebProxy(Settings.Instance.Network.Proxy) : null, - Cookie = cookie, - Url = url - }; - - public static NetTask MakeDefaultMobile(string url, string cookie = "") - => new NetTask - { - Accept = "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8", - UserAgent = "Mozilla/5.0 (Android 7.0; Mobile; rv:54.0) Gecko/54.0 Firefox/54.0 AppleWebKit/537.36 (KHTML, like Gecko) Chrome/59.0.3071.125 Mobile Safari/603.2.4", - TimeoutInfinite = Settings.Instance.Network.TimeoutInfinite, - TimeoutMillisecond = Settings.Instance.Network.TimeoutMillisecond, - AutoRedirection = true, - RetryWhenFail = true, - RetryCount = Settings.Instance.Network.RetryCount, - DownloadBufferSize = Settings.Instance.Network.DownloadBufferSize, - Proxy = !string.IsNullOrEmpty(Settings.Instance.Network.Proxy) ? new WebProxy(Settings.Instance.Network.Proxy) : null, - Cookie = cookie, - Url = url - }; - - public enum NetError - { - Unhandled = 0, - CannotContinueByCriticalError, - UnknowError, // Check DPI Blocker - UriFormatError, - Aborted, - ManyRetry, - } - - /* Task Information */ - - [JsonProperty] - public int Index { get; set; } - - /* Http Information */ - - [JsonProperty] - public string Url { get; set; } - [JsonProperty] - public List FailUrls { get; set; } - [JsonProperty] - public string Accept { get; set; } - [JsonProperty] - public string Referer { get; set; } - [JsonProperty] - public string UserAgent { get; set; } - [JsonProperty] - public string Cookie { get; set; } - [JsonProperty] - public Dictionary Headers { get; set; } - [JsonProperty] - public Dictionary Query { get; set; } - [JsonProperty] - public IWebProxy Proxy { get; set; } - - /* Detail Information */ - - /// - /// Text Encoding Information - /// - public Encoding Encoding { get; set; } - - /// - /// Set if you want to download and save file to your own device. - /// - [JsonProperty] - public bool SaveFile { get; set; } - [JsonProperty] - public string Filename { get; set; } - - /// - /// Set if needing only string datas. - /// - [JsonProperty] - public bool DownloadString { get; set; } - - /// - /// Download data to temporary directory on your device. - /// - [JsonProperty] - public bool DriveCache { get; set; } - - /// - /// Download data to memory. - /// - [JsonProperty] - public bool MemoryCache { get; set; } - - /// - /// Retry download when fail to download. - /// - [JsonProperty] - public bool RetryWhenFail { get; set; } - [JsonProperty] - public int RetryCount { get; set; } - - /// - /// Timeout settings - /// - [JsonProperty] - public bool TimeoutInfinite { get; set; } - [JsonProperty] - public int TimeoutMillisecond { get; set; } - - [JsonProperty] - public int DownloadBufferSize { get; set; } - - [JsonProperty] - public bool AutoRedirection { get; set; } - - [JsonProperty] - public bool NotifyOnlySize { get; set; } - - /* Callback Functions */ - - public Action SizeCallback; - public Action DownloadCallback; - public Action StartCallback; - public Action CompleteCallback; - public Action CompleteCallbackString; - public Action CompleteCallbackBytes; - public Action CookieReceive; - public Action HeaderReceive; - public Action CancleCallback; - - /// - /// Return total downloaded size - /// - public Action RetryCallback; - public Action ErrorCallback; - - /* For NetField */ - - public bool Aborted; - public HttpWebRequest Request; - public CancellationToken Cancel; - } -} diff --git a/violet-message-search-core/hdownloader/Network/NetTaskPass.cs b/violet-message-search-core/hdownloader/Network/NetTaskPass.cs deleted file mode 100644 index 8f868c644..000000000 --- a/violet-message-search-core/hdownloader/Network/NetTaskPass.cs +++ /dev/null @@ -1,38 +0,0 @@ -// This source code is a part of project violet-server. -// Copyright (C)2020-2021. violet-team. Licensed under the MIT Licence. - -using System; -using System.Collections.Generic; -using System.Text; - -namespace hsync.Network -{ - public abstract class NetTaskPass - { - public static List Passes = new List(); - - public static void RunOnField(ref NetTask content) - { - foreach (var pass in Passes) - pass.RunOnPass(ref content); - } - - public static void RemoveFromPasses() where T : NetTaskPass, new() - { - lock (Passes) - { - var class_name = (new T()).GetType().Name; - for (int i = 0; i < Passes.Count; i++) - { - if (Passes[i].GetType().Name == class_name) - { - Passes.RemoveAt(i); - break; - } - } - } - } - - public abstract void RunOnPass(ref NetTask content); - } -} diff --git a/violet-message-search-core/hdownloader/Network/NetTools.cs b/violet-message-search-core/hdownloader/Network/NetTools.cs deleted file mode 100644 index 9a28097de..000000000 --- a/violet-message-search-core/hdownloader/Network/NetTools.cs +++ /dev/null @@ -1,235 +0,0 @@ -// This source code is a part of project violet-server. -// Copyright (C)2020-2021. violet-team. Licensed under the MIT Licence. - -using System; -using System.Collections.Generic; -using System.Linq; -using System.Text; -using System.Threading; -using System.Threading.Tasks; -using hsync.Log; - -namespace hsync.Network -{ - public class NetTools - { - public static async Task> DownloadStrings(List urls, string cookie = "", Action complete = null, Action error = null) - { - var interrupt = new ManualResetEvent(false); - var result = new string[urls.Count]; - var count = urls.Count; - int iter = 0; - - foreach (var url in urls) - { - var itertmp = iter; - var task = NetTask.MakeDefault(url); - task.DownloadString = true; - task.CompleteCallbackString = (str) => - { - result[itertmp] = str; - if (Interlocked.Decrement(ref count) == 0) - interrupt.Set(); - complete?.Invoke(); - }; - task.ErrorCallback = (code) => - { - if (Interlocked.Decrement(ref count) == 0) - interrupt.Set(); - error?.Invoke(); - }; - task.Cookie = cookie; - await AppProvider.DownloadQueue.Add(task).ConfigureAwait(false); - iter++; - } - - interrupt.WaitOne(); - - return result.ToList(); - } - - public static async Task> DownloadStrings(List tasks, string cookie = "", Action complete = null) - { - var interrupt = new ManualResetEvent(false); - var result = new string[tasks.Count]; - var count = tasks.Count; - int iter = 0; - - foreach (var task in tasks) - { - var itertmp = iter; - task.DownloadString = true; - task.CompleteCallbackString = (str) => - { - result[itertmp] = str; - if (Interlocked.Decrement(ref count) == 0) - interrupt.Set(); - complete?.Invoke(); - }; - task.ErrorCallback = (code) => - { - if (Interlocked.Decrement(ref count) == 0) - interrupt.Set(); - }; - task.Cookie = cookie; - await AppProvider.DownloadQueue.Add(task).ConfigureAwait(false); - iter++; - } - - interrupt.WaitOne(); - - return result.ToList(); - } - - public static string DownloadString(string url) - { - return DownloadStringAsync(NetTask.MakeDefault(url)).Result; - } - - public static string DownloadString(NetTask task) - { - return DownloadStringAsync(task).Result; - } - - public static async Task DownloadStringAsync(NetTask task) - { - return await Task.Run(async () => - { - var interrupt = new ManualResetEvent(false); - string result = null; - - task.DownloadString = true; - task.CompleteCallbackString = (string str) => - { - result = str; - interrupt.Set(); - }; - - task.ErrorCallback = (code) => - { - task.ErrorCallback = null; - interrupt.Set(); - }; - - await AppProvider.DownloadQueue.Add(task).ConfigureAwait(false); - - interrupt.WaitOne(); - - return result; - }).ConfigureAwait(false); - } - - public static async Task> DownloadFiles(List<(string, string)> url_path, string cookie = "", Action download = null, Action complete = null) - { - var interrupt = new ManualResetEvent(false); - var result = new string[url_path.Count]; - var count = url_path.Count; - int iter = 0; - - foreach (var up in url_path) - { - var itertmp = iter; - var task = NetTask.MakeDefault(up.Item1); - task.SaveFile = true; - task.Filename = up.Item2; - task.DownloadCallback = (sz) => - { - download?.Invoke(sz); - }; - task.CompleteCallback = () => - { - if (Interlocked.Decrement(ref count) == 0) - interrupt.Set(); - complete?.Invoke(); - }; - task.ErrorCallback = (code) => - { - if (Interlocked.Decrement(ref count) == 0) - interrupt.Set(); - }; - task.Cookie = cookie; - await AppProvider.DownloadQueue.Add(task).ConfigureAwait(false); - iter++; - } - - interrupt.WaitOne(); - - return result.ToList(); - } - - public static void DownloadFile(string url, string filename) - { - var task = NetTask.MakeDefault(url); - task.SaveFile = true; - task.Filename = filename; - DownloadFileAsync(task).Wait(); - } - - public static void DownloadFile(NetTask task) - { - DownloadFileAsync(task).Wait(); - } - - public static async Task DownloadFileAsync(NetTask task) - { - await Task.Run(async () => - { - var interrupt = new ManualResetEvent(false); - - task.SaveFile = true; - task.CompleteCallback = () => - { - interrupt.Set(); - }; - - task.ErrorCallback = (code) => - { - task.ErrorCallback = null; - interrupt.Set(); - }; - - await AppProvider.DownloadQueue.Add(task).ConfigureAwait(false); - - interrupt.WaitOne(); - }).ConfigureAwait(false); - } - - public static byte[] DownloadData(string url) - { - return DownloadDataAsync(NetTask.MakeDefault(url)).Result; - } - - public static byte[] DownloadData(NetTask task) - { - return DownloadDataAsync(task).Result; - } - - public static async Task DownloadDataAsync(NetTask task) - { - return await Task.Run(async () => - { - var interrupt = new ManualResetEvent(false); - byte[] result = null; - - task.MemoryCache = true; - task.CompleteCallbackBytes = (byte[] bytes) => - { - result = bytes; - interrupt.Set(); - }; - - task.ErrorCallback = (code) => - { - task.ErrorCallback = null; - interrupt.Set(); - }; - - await AppProvider.DownloadQueue.Add(task).ConfigureAwait(false); - - interrupt.WaitOne(); - - return result; - }).ConfigureAwait(false); - } - } -} diff --git a/violet-message-search-core/hdownloader/Program.cs b/violet-message-search-core/hdownloader/Program.cs deleted file mode 100644 index 8fb250370..000000000 --- a/violet-message-search-core/hdownloader/Program.cs +++ /dev/null @@ -1,68 +0,0 @@ -// This source code is a part of project violet-server. -// Copyright (C)2020-2021. violet-team. Licensed under the MIT Licence. - -using hsync.Log; -using System; -using System.Text; -using System.Globalization; - -namespace hsync -{ - class Program - { - static void Main(string[] args) - { - Encoding.RegisterProvider(CodePagesEncodingProvider.Instance); - - AppProvider.Initialize(); - - Logs.Instance.AddLogNotify((s, e) => - { - var tuple = s as Tuple; - CultureInfo en = new CultureInfo("en-US"); - Console.ForegroundColor = ConsoleColor.Green; - Console.Write("info: "); - Console.ResetColor(); - Console.WriteLine($"[{tuple.Item1.ToString(en)}] {tuple.Item2}"); - }); - - Logs.Instance.AddLogErrorNotify((s, e) => { - var tuple = s as Tuple; - CultureInfo en = new CultureInfo("en-US"); - Console.ForegroundColor = ConsoleColor.Red; - Console.Error.Write("error: "); - Console.ResetColor(); - Console.Error.WriteLine($"[{tuple.Item1.ToString(en)}] {tuple.Item2}"); - }); - - Logs.Instance.AddLogWarningNotify((s, e) => { - var tuple = s as Tuple; - CultureInfo en = new CultureInfo("en-US"); - Console.ForegroundColor = ConsoleColor.Yellow; - Console.Error.Write("warning: "); - Console.ResetColor(); - Console.Error.WriteLine($"[{tuple.Item1.ToString(en)}] {tuple.Item2}"); - }); - - AppDomain.CurrentDomain.UnhandledException += (s, e) => - { - Logs.Instance.PushError("unhandled: " + (e.ExceptionObject as Exception).ToString()); - }; - - try - { - Command.Start(args); - } - catch (Exception e) - { - Console.WriteLine("An error occured! " + e.Message); - Console.WriteLine(e.StackTrace); - Console.WriteLine("Please, check log.txt file."); - } - - AppProvider.Deinitialize(); - - Environment.Exit(0); - } - } -} diff --git a/violet-message-search-core/hdownloader/Progress.cs b/violet-message-search-core/hdownloader/Progress.cs deleted file mode 100644 index 2bd8575a1..000000000 --- a/violet-message-search-core/hdownloader/Progress.cs +++ /dev/null @@ -1,373 +0,0 @@ -// This source code is a part of project violet-server. -// Copyright (C)2020-2021. violet-team. Licensed under the MIT Licence. - -using System; -using System.Collections.Generic; -using System.Text; -using System.Threading; - -namespace hsync -{ - public abstract class ProgressBase : IDisposable - { - protected readonly TimeSpan animationInterval = TimeSpan.FromSeconds(1.0 / 8); - protected readonly Timer timer; - - protected string currentText = string.Empty; - protected bool disposed = false; - - public ProgressBase() - { - timer = new Timer(TimerHandler); - - if (!System.Console.IsOutputRedirected) - { - ResetTimer(); - } - } - - protected abstract void TimerHandler(object state); - - protected void UpdateText(string text) - { - int commonPrefixLength = 0; - int commonLength = Math.Min(currentText.Length, text.Length); - while (commonPrefixLength < commonLength && text[commonPrefixLength] == currentText[commonPrefixLength]) - { - commonPrefixLength++; - } - - StringBuilder outputBuilder = new StringBuilder(); - outputBuilder.Append('\b', currentText.Length - commonPrefixLength); - - outputBuilder.Append(text.Substring(commonPrefixLength)); - - int overlapCount = currentText.Length - text.Length; - if (overlapCount > 0) - { - outputBuilder.Append(' ', overlapCount); - outputBuilder.Append('\b', overlapCount); - } - - System.Console.Write(outputBuilder); - currentText = text; - } - - protected void ResetTimer() - { - timer.Change(animationInterval, TimeSpan.FromMilliseconds(-1)); - } - - public void Dispose() - { - lock (timer) - { - disposed = true; - UpdateText(string.Empty); - } - } - } - - - /// - /// An ASCII progress bar - /// - /// Reference[MIT]: DanielSWolf - https://gist.github.com/DanielSWolf/0ab6a96899cc5377bf54 - /// - - public class ExtractingProgressBar : ProgressBase, IDisposable - { - private const int blockCount = 20; - private double currentProgress = 0; - private long total = 0; - private long complete = 0; - - public ExtractingProgressBar() - : base() - { - } - - public void Report(long total, long complete) - { - var value = Math.Max(0, Math.Min(1, complete / (double)total)); - Interlocked.Exchange(ref currentProgress, value); - this.total = total; - this.complete = complete; - } - - protected override void TimerHandler(object state) - { - lock (timer) - { - if (disposed) return; - - int progressBlockCount = (int)(currentProgress * blockCount); - int percent = (int)(currentProgress * 100); - - string text = string.Format("[{0}{1}] {2,3}% [{3}/{4}]", - new string('#', progressBlockCount), new string('-', blockCount - progressBlockCount), - percent, complete, total); - UpdateText(text); - - ResetTimer(); - } - } - } - - public class WaitProgress : ProgressBase, IDisposable - { - private const string animation = @"|/-\"; - private int animationIndex = 0; - - public WaitProgress() - : base() - { - } - - protected override void TimerHandler(object state) - { - lock (timer) - { - if (disposed) return; - - UpdateText(animation[animationIndex++ % animation.Length].ToString()); - - ResetTimer(); - } - } - } - - public class ProgressBar : ProgressBase - { - private const int blockCount = 20; - private double currentProgress = 0; - private long total = 0; - private long complete = 0; - private long error = 0; - - public ProgressBar() - : base() - { - } - - public void Report(long total, long complete, long error) - { - var value = Math.Max(0, Math.Min(1, (complete + error) / (double)total)); - this.total = total; - this.complete = complete; - this.error = error; - Interlocked.Exchange(ref currentProgress, value); - } - - protected override void TimerHandler(object state) - { - lock (timer) - { - if (disposed) return; - - int progressBlockCount = (int)(currentProgress * blockCount); - int percent = (int)(currentProgress * 100); - - string text = string.Format("[{0}{1}] {2,3}% [{3}/{4}] (Find: {5}, Error: {6})", - new string('#', progressBlockCount), new string('-', blockCount - progressBlockCount), - percent, - complete + error, total, complete, error); - UpdateText(text); - - ResetTimer(); - } - } - } - - public class DownloadProgressBar : ProgressBase, IDisposable - { - private const int blockCount = 20; - private double currentProgress = 0; - private long total_read_bytes = 0; - private long current_speed = 0; - private long tick_speed = 0; - private object report_lock = new object(); - private long total = 0; - private long complete = 0; - private Queue speed_save = new Queue(); - - public DownloadProgressBar() - : base() - { - } - - public void Report(long total, long complete, long read_bytes) - { - var value = Math.Max(0, Math.Min(1, complete / (double)total)); - this.total = total; - this.complete = complete; - Interlocked.Exchange(ref currentProgress, value); - lock (report_lock) - { - total_read_bytes += read_bytes; - current_speed += read_bytes; - tick_speed += read_bytes; - } - } - - protected override void TimerHandler(object state) - { - lock (timer) - { - if (disposed) return; - double cs = 0; - lock (report_lock) - { - speed_save.Enqueue(tick_speed); - tick_speed = 0; - cs = current_speed * (8 / (double)speed_save.Count); - if (speed_save.Count >= 8) - { - current_speed -= speed_save.Peek(); - speed_save.Dequeue(); - } - } - - int progressBlockCount = (int)(currentProgress * blockCount); - int percent = (int)(currentProgress * 100); - - string speed; - if (cs > 1024 * 1024) - speed = (cs / (1024 * 1024)).ToString("#,0.0") + " MB/S"; - else if (cs > 1024) - speed = (cs / 1024).ToString("#,0.0") + " KB/S"; - else - speed = cs.ToString("#,0") + " Byte/S"; - - string downloads; - if (total_read_bytes > 1024 * 1024 * 1024) - downloads = (total_read_bytes / (double)(1024 * 1024 * 1024)).ToString("#,0.0") + " GB"; - else if (total_read_bytes > 1024 * 1024) - downloads = (total_read_bytes / (double)(1024 * 1024)).ToString("#,0.0") + " MB"; - else if (total_read_bytes > 1024) - downloads = (total_read_bytes / (double)(1024)).ToString("#,0.0") + " KB"; - else - downloads = (total_read_bytes).ToString("#,0") + " Byte"; - - string text = string.Format("[{0}{1}] {2,3}% [{5}/{6}] ({3} {4})", - new string('#', progressBlockCount), new string('-', blockCount - progressBlockCount), - percent, - speed, downloads, complete, total); - UpdateText(text); - - ResetTimer(); - } - } - } - - public class SingleFileProgressBar : ProgressBase, IDisposable - { - private const int blockCount = 20; - private double currentProgress = 0; - private long total_read_bytes = 0; - private long current_speed = 0; - private long tick_speed = 0; - private object report_lock = new object(); - private Queue speed_save = new Queue(); - - public SingleFileProgressBar() - : base() - { - } - - public void Report(long size, long read_bytes) - { - var value = Math.Max(0, Math.Min(1, total_read_bytes / (double)size)); - Interlocked.Exchange(ref currentProgress, value); - lock (report_lock) - { - total_read_bytes += read_bytes; - current_speed += read_bytes; - tick_speed += read_bytes; - } - } - - protected override void TimerHandler(object state) - { - lock (timer) - { - if (disposed) return; - double cs = 0; - lock (report_lock) - { - speed_save.Enqueue(tick_speed); - tick_speed = 0; - cs = current_speed * (8 / (double)speed_save.Count); - if (speed_save.Count >= 8) - { - current_speed -= speed_save.Peek(); - speed_save.Dequeue(); - } - } - - int progressBlockCount = (int)(currentProgress * blockCount); - int percent = (int)(currentProgress * 100); - - string speed; - if (cs > 1024 * 1024) - speed = (cs / (1024 * 1024)).ToString("#,0.0") + " MB/S"; - else if (cs > 1024) - speed = (cs / 1024).ToString("#,0.0") + " KB/S"; - else - speed = cs.ToString("#,0") + " Byte/S"; - - string downloads; - if (total_read_bytes > 1024 * 1024 * 1024) - downloads = (total_read_bytes / (double)(1024 * 1024 * 1024)).ToString("#,0.0") + " GB"; - else if (total_read_bytes > 1024 * 1024) - downloads = (total_read_bytes / (double)(1024 * 1024)).ToString("#,0.0") + " MB"; - else if (total_read_bytes > 1024) - downloads = (total_read_bytes / (double)(1024)).ToString("#,0.0") + " KB"; - else - downloads = (total_read_bytes).ToString("#,0") + " Byte"; - - string text = string.Format("[{0}{1}] {2,3}% ({3} {4})", - new string('#', progressBlockCount), new string('-', blockCount - progressBlockCount), - percent, - speed, downloads); - UpdateText(text); - - ResetTimer(); - } - } - } - - public class WaitPostprocessor : ProgressBase, IDisposable - { - private long wait; - private const string animation = @"|/-\"; - private int animationIndex = 0; - private object report_lock = new object(); - - public WaitPostprocessor() - : base() - { - } - - public void Report(long wait) - { - lock (report_lock) - { - this.wait = wait; - } - } - - protected override void TimerHandler(object state) - { - lock (timer) - { - if (disposed) return; - - UpdateText(animation[animationIndex++ % animation.Length].ToString() + $" [{wait} jobs remained]"); - - ResetTimer(); - } - } - } - -} diff --git a/violet-message-search-core/hdownloader/Setting/Settings.cs b/violet-message-search-core/hdownloader/Setting/Settings.cs deleted file mode 100644 index 7b938e55f..000000000 --- a/violet-message-search-core/hdownloader/Setting/Settings.cs +++ /dev/null @@ -1,172 +0,0 @@ -// This source code is a part of project violet-server. -// Copyright (C)2020-2021. violet-team. Licensed under the MIT Licence. - -using hsync.Log; -using hsync.Utils; -using Newtonsoft.Json; -using System; -using System.IO; -using System.Threading; - -namespace hsync.Setting -{ - public class SettingModel - { - public class NetworkSetting - { - public bool TimeoutInfinite; - public int TimeoutMillisecond; - public int DownloadBufferSize; - public int RetryCount; - public string Proxy; - } - - public class HitomiSetting - { - public string SaveDirectoryFormat; - public string RemainInfoFile; - } - - public HitomiSetting HitomiSettings; - - public NetworkSetting NetworkSettings; - - /// - /// Scheduler Thread Count - /// - public int ThreadCount; - - /// - /// Postprocessor Scheduler Thread Count - /// - public int PostprocessorThreadCount; - - /// - /// Provider Language - /// - public string Language; - - /// - /// Parent Path for Downloading - /// - public string SuperPath; - - /// - /// Server Connection String - /// - public string ServerConnection; - - /// - /// Elastic Search Server HostName - /// - public string ElasticSearchHost; - } - - public class Settings : ILazy - { - public const string Name = "settings.json"; - - public SettingModel Model { get; set; } - public SettingModel.NetworkSetting Network { get { return Model.NetworkSettings; } } - - public Settings() - { - var full_path = Path.Combine(AppProvider.ApplicationPath, Name); - if (File.Exists(full_path)) - Model = JsonConvert.DeserializeObject(File.ReadAllText(full_path)); - - if (Model == null) - { - Model = new SettingModel - { - Language = GetLanguageKey(), - ThreadCount = Environment.ProcessorCount * 6, - PostprocessorThreadCount = 3, - SuperPath = AppProvider.DefaultSuperPath, - ServerConnection = "Server=localhost;Database=test;Uid=root;Pwd=123;CharSet=utf8mb4", - ElasticSearchHost = "", - - HitomiSettings = new SettingModel.HitomiSetting - { - SaveDirectoryFormat = "%(artist)/[%(id)] %(title)" - }, - - NetworkSettings = new SettingModel.NetworkSetting - { - TimeoutInfinite = false, - TimeoutMillisecond = 100000, - DownloadBufferSize = 131072, - RetryCount = 10, - }, - }; - } - Save(); - } - - public static string GetLanguageKey() - { - var lang = Thread.CurrentThread.CurrentCulture.ToString(); - var language = "all"; - switch (lang) - { - case "ko-KR": - language = "korean"; - break; - - case "ja-JP": - language = "japanese"; - break; - - case "en-US": - language = "english"; - break; - } - return language; - } - - /// - /// Recover incorrect configuration. - /// - public void Recover() - { - if (string.IsNullOrWhiteSpace(Model.Language)) - Model.Language = GetLanguageKey(); - - if (Model.ThreadCount <= 0 || Model.ThreadCount >= 128) - Model.ThreadCount = Environment.ProcessorCount; - if (Model.PostprocessorThreadCount <= 0 || Model.PostprocessorThreadCount >= 128) - Model.ThreadCount = 3; - if (string.IsNullOrWhiteSpace(Model.SuperPath)) - Model.SuperPath = AppProvider.DefaultSuperPath; - - if (Model.NetworkSettings == null) - { - Model.NetworkSettings = new SettingModel.NetworkSetting - { - TimeoutInfinite = false, - TimeoutMillisecond = 10000, - DownloadBufferSize = 131072, - RetryCount = 10, - }; - } - - if (Model.NetworkSettings.TimeoutMillisecond < 1000) - Model.NetworkSettings.TimeoutMillisecond = 10000; - if (Model.NetworkSettings.DownloadBufferSize < 100000) - Model.NetworkSettings.DownloadBufferSize = 131072; - if (Model.NetworkSettings.RetryCount < 0) - Model.NetworkSettings.RetryCount = 10; - } - - public void Save() - { - var full_path = Path.Combine(AppProvider.ApplicationPath, Name); - var json = JsonConvert.SerializeObject(Model, Formatting.Indented); - using (var fs = new StreamWriter(new FileStream(full_path, FileMode.Create, FileAccess.Write))) - { - fs.Write(json); - } - AppProvider.DefaultSuperPath = Model.SuperPath; - } - } -} diff --git a/violet-message-search-core/hdownloader/Utils/Compress.cs b/violet-message-search-core/hdownloader/Utils/Compress.cs deleted file mode 100644 index 61d6f6eed..000000000 --- a/violet-message-search-core/hdownloader/Utils/Compress.cs +++ /dev/null @@ -1,40 +0,0 @@ -// This source code is a part of project violet-server. -// Copyright (C)2020-2021. violet-team. Licensed under the MIT Licence. - -using System; -using System.Collections.Generic; -using System.IO; -using System.IO.Compression; -using System.Text; - -namespace hsync.Utils -{ - public class CompressUtils - { - public static byte[] Compress(byte[] data) - { - MemoryStream output = new MemoryStream(); - using (DeflateStream dstream = new DeflateStream(output, CompressionLevel.Optimal)) - { - dstream.Write(data, 0, data.Length); - } - return output.ToArray(); - } - - public static byte[] Decompress(byte[] data) - { - MemoryStream input = new MemoryStream(data); - MemoryStream output = new MemoryStream(); - using (DeflateStream dstream = new DeflateStream(input, CompressionMode.Decompress)) - { - dstream.CopyTo(output); - } - return output.ToArray(); - } - - public static byte[] Zip(byte[] data) - { - throw new NotImplementedException(); - } - } -} diff --git a/violet-message-search-core/hdownloader/Utils/Extends.cs b/violet-message-search-core/hdownloader/Utils/Extends.cs deleted file mode 100644 index 100263d85..000000000 --- a/violet-message-search-core/hdownloader/Utils/Extends.cs +++ /dev/null @@ -1,59 +0,0 @@ -// This source code is a part of project violet-server. -// Copyright (C)2020-2021. violet-team. Licensed under the MIT Licence. - -using HtmlAgilityPack; -using Newtonsoft.Json; -using System; -using System.Collections.Concurrent; -using System.Collections.Generic; -using System.IO; -using System.Linq; -using System.Text; -using System.Threading.Tasks; - -namespace hsync.Utils -{ - public static class Extends - { - public static int ToInt(this string str) => Convert.ToInt32(str); - - public static string MyText(this HtmlNode node) => - string.Join("", node.ChildNodes.Where(x => x.Name == "#text").Select(x => x.InnerText.Trim())); - - public static HtmlNode ToHtmlNode(this string html) - { - var document = new HtmlDocument(); - document.LoadHtml(html); - return document.DocumentNode; - } - - public static Task ForEachAsync(this IEnumerable source, int countdvd, Func body) - { - return Task.WhenAll( - from partition in Partitioner.Create(source).GetPartitions(countdvd) - select Task.Run(async delegate - { - using (partition) - while (partition.MoveNext()) - await body(partition.Current); - })); - } - - public static async Task ReadJson(string path) - { - using (var fs = new StreamReader(new FileStream(path, FileMode.Open, FileAccess.Read))) - { - return JsonConvert.DeserializeObject(await fs.ReadToEndAsync()); - } - } - - public static async void WriteJson(string path, T value) - { - var json = JsonConvert.SerializeObject(value, Formatting.Indented); - using (var fs = new StreamWriter(new FileStream(path, FileMode.Create, FileAccess.Write))) - { - await fs.WriteAsync(json); - } - } - } -} diff --git a/violet-message-search-core/hdownloader/Utils/Heap.cs b/violet-message-search-core/hdownloader/Utils/Heap.cs deleted file mode 100644 index 8a74b61c7..000000000 --- a/violet-message-search-core/hdownloader/Utils/Heap.cs +++ /dev/null @@ -1,235 +0,0 @@ -// This source code is a part of project violet-server. -// Copyright (C)2020-2021. violet-team. Licensed under the MIT Licence. - -using System; -using System.Collections.Generic; -using System.Text; - -namespace hsync.Utils -{ - /// - /// Priority queue data structure for C# - /// - /// Type of data - /// Comparator of data - public class Heap - where T : IComparable - where C : IComparer, new() - { - List heap; - C comp; - - public Heap(int capacity = 256) - { - heap = new List(capacity); - comp = new C(); - } - - public void Push(T d) - { - heap.Add(d); - leaf_to_root(); - } - - public void Pop() - { - heap[0] = heap[heap.Count - 1]; - heap.RemoveAt(heap.Count - 1); - root_to_leaf(); - } - - public T Front => heap[0]; - - private void root_to_leaf() - { - int x = 0; - int l = heap.Count - 1; - while (x < l) - { - int c1 = x * 2 + 1; - int c2 = c1 + 1; - - // - // x - // / \ - // / \ - // c1 c2 - // - - int c = c1; - if (c2 < l && comp.Compare(heap[c2], heap[c1]) > 0) - c = c2; - - if (c < l && comp.Compare(heap[c], heap[x]) > 0) - { - swap(c, x); - x = c; - } - else - { - break; - } - } - } - - private void leaf_to_root() - { - int x = heap.Count - 1; - while (x > 0) - { - int p = (x - 1) >> 1; - if (comp.Compare(heap[x], heap[p]) > 0) - { - swap(p, x); - x = p; - } - else - break; - } - } - - private void swap(int i, int j) - { - T t = heap[i]; - heap[i] = heap[j]; - heap[j] = t; - } - } - - public class DefaultHeapComparer : Comparer where T : IComparable - { - public override int Compare(T x, T y) - => x.CompareTo(y); - } - - public class MinHeapComparer : Comparer where T : IComparable - { - public override int Compare(T x, T y) - => y.CompareTo(x); - } - - public class Heap : Heap> where T : IComparable { } - public class MinHeap : Heap> where T : IComparable { } - public class MaxHeap : Heap> where T : IComparable { } - - public class UpdatableHeapElements : IComparable - where T : IComparable - { - public T data; - public int index; - public static UpdatableHeapElements Create(T data, int index) - => new UpdatableHeapElements { data = data, index = index }; - public int CompareTo(T obj) - => data.CompareTo(obj); - } - - public class UpdatableHeap - where S : IComparable - where T : UpdatableHeapElements, IComparable - where C : IComparer, new() - { - List heap; - C comp; - - public UpdatableHeap(int capacity = 256) - { - heap = new List(capacity); - comp = new C(); - } - - public T Push(S d) - { - var dd = (T)UpdatableHeapElements.Create(d, heap.Count - 1); - heap.Add(dd); - top_down(heap.Count - 1); - return dd; - } - - public void Pop() - { - heap[0] = heap[heap.Count - 1]; - heap[0].index = 0; - heap.RemoveAt(heap.Count - 1); - bottom_up(); - } - - public void Update(T d) - { - int p = (d.index - 1) >> 1; - if (p == d.index) - bottom_up(); - else - { - if (comp.Compare(heap[p].data, heap[d.index].data) > 0) - top_down(d.index); - else - bottom_up(d.index); - } - } - - public S Front => heap[0].data; - - public int Count { get { return heap.Count; } } - - private void bottom_up(int x = 0) - { - int l = heap.Count - 1; - while (x < l) - { - int c1 = x * 2 + 1; - int c2 = c1 + 1; - - // - // x - // / \ - // / \ - // c1 c2 - // - - int c = c1; - if (c2 < l && comp.Compare(heap[c2].data, heap[c1].data) > 0) - c = c2; - - if (c < l && comp.Compare(heap[c].data, heap[x].data) > 0) - { - swap(c, x); - x = c; - } - else - { - break; - } - } - } - - private void top_down(int x) - { - while (x > 0) - { - int p = (x - 1) >> 1; - if (comp.Compare(heap[x].data, heap[p].data) > 0) - { - swap(p, x); - x = p; - } - else - break; - } - } - - private void swap(int i, int j) - { - T t = heap[i]; - heap[i] = heap[j]; - heap[j] = t; - - int tt = heap[i].index; - heap[i].index = heap[j].index; - heap[j].index = tt; - } - } - - public class UpdatableHeap : UpdatableHeap, DefaultHeapComparer> where T : IComparable { } - public class UpdatableMinHeap : UpdatableHeap, MinHeapComparer> where T : IComparable { } - public class UpdatableMaxHeap : UpdatableHeap, DefaultHeapComparer> where T : IComparable { } -} diff --git a/violet-message-search-core/hdownloader/Utils/ILazy.cs b/violet-message-search-core/hdownloader/Utils/ILazy.cs deleted file mode 100644 index e9907c6bc..000000000 --- a/violet-message-search-core/hdownloader/Utils/ILazy.cs +++ /dev/null @@ -1,34 +0,0 @@ -// This source code is a part of project violet-server. -// Copyright (C)2020-2021. violet-team. Licensed under the MIT Licence. - -using System; -using System.Collections.Generic; -using System.Text; - -namespace hsync.Utils -{ - /// - /// Contains all instance information. - /// - public class InstanceMonitor - { - public static Dictionary Instances = new Dictionary(); - } - - /// - /// Class to make lazy instance easier to implement. - /// - /// - public class ILazy - where T : new() - { - private static readonly Lazy instance = new Lazy(() => - { - T instance = new T(); - InstanceMonitor.Instances.Add(instance.GetType().Name.ToLower(), instance); - return instance; - }); - public static T Instance => instance.Value; - public static bool IsValueCreated => instance.IsValueCreated; - } -} diff --git a/violet-message-search-core/hdownloader/Utils/Strings.cs b/violet-message-search-core/hdownloader/Utils/Strings.cs deleted file mode 100644 index 4cf0387ae..000000000 --- a/violet-message-search-core/hdownloader/Utils/Strings.cs +++ /dev/null @@ -1,229 +0,0 @@ -// This source code is a part of project violet-server. -// Copyright (C)2020-2021. violet-team. Licensed under the MIT Licence. - -using System; -using System.Collections.Generic; -using System.IO; -using System.IO.Compression; -using System.Text; - -namespace hsync.Utils -{ - public static class Strings - { - // https://stackoverflow.com/questions/11743160/how-do-i-encode-and-decode-a-base64-string - public static string ToBase64(this string text) - { - return ToBase64(text, Encoding.UTF8); - } - - public static string ToBase64(this string text, Encoding encoding) - { - if (string.IsNullOrEmpty(text)) - { - return text; - } - - byte[] textAsBytes = encoding.GetBytes(text); - return Convert.ToBase64String(textAsBytes); - } - - public static bool TryParseBase64(this string text, out string decodedText) - { - return TryParseBase64(text, Encoding.UTF8, out decodedText); - } - - public static bool TryParseBase64(this string text, Encoding encoding, out string decodedText) - { - if (string.IsNullOrEmpty(text)) - { - decodedText = text; - return false; - } - - try - { - byte[] textAsBytes = Convert.FromBase64String(text); - decodedText = encoding.GetString(textAsBytes); - return true; - } - catch (Exception) - { - decodedText = null; - return false; - } - } - - #region Compression - - // https://stackoverflow.com/questions/7343465/compression-decompression-string-with-c-sharp - private static void CopyTo(Stream src, Stream dest) - { - byte[] bytes = new byte[4096]; - - int cnt; - - while ((cnt = src.Read(bytes, 0, bytes.Length)) != 0) - { - dest.Write(bytes, 0, cnt); - } - } - - public static byte[] Zip(this string str) - { - var bytes = Encoding.UTF8.GetBytes(str); - - using (var msi = new MemoryStream(bytes)) - using (var mso = new MemoryStream()) - { - using (var gs = new GZipStream(mso, CompressionMode.Compress)) - { - CopyTo(msi, gs); - } - - return mso.ToArray(); - } - } - - public static byte[] ZipByte(this byte[] bytes) - { - using (var msi = new MemoryStream(bytes)) - using (var mso = new MemoryStream()) - { - using (var gs = new GZipStream(mso, CompressionMode.Compress)) - { - CopyTo(msi, gs); - } - - return mso.ToArray(); - } - } - - - public static string Unzip(this byte[] bytes) - { - using (var msi = new MemoryStream(bytes)) - using (var mso = new MemoryStream()) - { - using (var gs = new GZipStream(msi, CompressionMode.Decompress)) - { - CopyTo(gs, mso); - } - - return Encoding.UTF8.GetString(mso.ToArray()); - } - } - - public static byte[] UnzipByte(this byte[] bytes) - { - using (var msi = new MemoryStream(bytes)) - using (var mso = new MemoryStream()) - { - using (var gs = new GZipStream(msi, CompressionMode.Decompress)) - { - CopyTo(gs, mso); - } - - return mso.ToArray(); - } - } - - #endregion - - public static int ComputeLevenshteinDistance(this string a, string b) - { - int x = a.Length; - int y = b.Length; - int i, j; - - if (x == 0) return x; - if (y == 0) return y; - int[] v0 = new int[(y + 1) << 1]; - - for (i = 0; i < y + 1; i++) v0[i] = i; - for (i = 0; i < x; i++) - { - v0[y + 1] = i + 1; - for (j = 0; j < y; j++) - v0[y + j + 2] = Math.Min(Math.Min(v0[y + j + 1], v0[j + 1]) + 1, v0[j] + ((a[i] == b[j]) ? 0 : 1)); - for (j = 0; j < y + 1; j++) v0[j] = v0[y + j + 1]; - } - return v0[y + y + 1]; - } - - public static int[] GetLevenshteinDistance(this string src, string tar) - { - int[,] dist = new int[src.Length + 1, tar.Length + 1]; - for (int i = 1; i <= src.Length; i++) dist[i, 0] = i; - for (int j = 1; j <= tar.Length; j++) dist[0, j] = j; - - for (int j = 1; j <= tar.Length; j++) - { - for (int i = 1; i <= src.Length; i++) - { - if (src[i - 1] == tar[j - 1]) dist[i, j] = dist[i - 1, j - 1]; - else dist[i, j] = Math.Min(dist[i - 1, j - 1] + 1, Math.Min(dist[i, j - 1] + 1, dist[i - 1, j] + 1)); - } - } - - // Get diff through backtracking. - int[] route = new int[src.Length + 1]; - int fz = dist[src.Length, tar.Length]; - for (int i = src.Length, j = tar.Length; i >= 0 && j >= 0;) - { - int lu = int.MaxValue; - int u = int.MaxValue; - int l = int.MaxValue; - if (i - 1 >= 0 && j - 1 >= 0) lu = dist[i - 1, j - 1]; - if (i - 1 >= 0) u = dist[i - 1, j]; - if (j - 1 >= 0) l = dist[i, j - 1]; - int min = Math.Min(lu, Math.Min(l, u)); - if (min == fz) route[i] = 1; - if (min == lu) - { - i--; j--; - } - else if (min == u) i--; - else j--; - fz = min; - } - return route; - } - - - // https://stackoverflow.com/questions/1601834/c-implementation-of-or-alternative-to-strcmplogicalw-in-shlwapi-dll - public class NaturalComparer : IComparer - { - public int Compare(string x, string y) - { - if (x == null && y == null) return 0; - if (x == null) return -1; - if (y == null) return 1; - - int lx = x.Length, ly = y.Length; - - for (int mx = 0, my = 0; mx < lx && my < ly; mx++, my++) - { - if (char.IsDigit(x[mx]) && char.IsDigit(y[my])) - { - long vx = 0, vy = 0; - - for (; mx < lx && char.IsDigit(x[mx]); mx++) - vx = vx * 10 + x[mx] - '0'; - - for (; my < ly && char.IsDigit(y[my]); my++) - vy = vy * 10 + y[my] - '0'; - - if (vx != vy) - return vx > vy ? 1 : -1; - } - - if (mx < lx && my < ly && x[mx] != y[my]) - return x[mx] > y[my] ? 1 : -1; - } - - return lx - ly; - } - } - } -} diff --git a/violet-message-search-core/hdownloader/Version.cs b/violet-message-search-core/hdownloader/Version.cs deleted file mode 100644 index 88459b154..000000000 --- a/violet-message-search-core/hdownloader/Version.cs +++ /dev/null @@ -1,19 +0,0 @@ -// This source code is a part of project violet-server. -// Copyright (C)2020-2021. violet-team. Licensed under the MIT Licence. - -using System; -using System.Collections.Generic; -using System.Text; - -namespace hsync -{ - public class Version - { - public const int MajorVersion = 2021; - public const int MinorVersion = 04; - public const int BuildVersion = 16; - - public const string Name = "hsync"; - public static string Text { get; } = $"{MajorVersion}.{MinorVersion}.{BuildVersion}"; - } -} diff --git a/violet-message-search-core/hdownloader/publish-linux-x64-aot.bat b/violet-message-search-core/hdownloader/publish-linux-x64-aot.bat deleted file mode 100644 index bbd7f2bb6..000000000 --- a/violet-message-search-core/hdownloader/publish-linux-x64-aot.bat +++ /dev/null @@ -1 +0,0 @@ -dotnet publish -r linux-x64 -c Release /p:PublishSingleFile=true /p:PublishTrimmed=true /p:PublishReadyToRun=true \ No newline at end of file diff --git a/violet-message-search-core/hdownloader/publish-linux-x64.bat b/violet-message-search-core/hdownloader/publish-linux-x64.bat deleted file mode 100644 index b88bc7ea4..000000000 --- a/violet-message-search-core/hdownloader/publish-linux-x64.bat +++ /dev/null @@ -1 +0,0 @@ -dotnet publish -r linux-x64 -c Release /p:PublishSingleFile=true /p:PublishTrimmed=true \ No newline at end of file diff --git a/violet-message-search-core/hdownloader/publish-osx.10.15-x64-aot.bat b/violet-message-search-core/hdownloader/publish-osx.10.15-x64-aot.bat deleted file mode 100644 index dfa3577c5..000000000 --- a/violet-message-search-core/hdownloader/publish-osx.10.15-x64-aot.bat +++ /dev/null @@ -1 +0,0 @@ -dotnet publish -r osx.10.15-x64 -c Release /p:PublishSingleFile=true /p:PublishTrimmed=true /p:PublishReadyToRun=true \ No newline at end of file diff --git a/violet-message-search-core/hdownloader/publish-osx.10.15-x64.bat b/violet-message-search-core/hdownloader/publish-osx.10.15-x64.bat deleted file mode 100644 index e8d2872e1..000000000 --- a/violet-message-search-core/hdownloader/publish-osx.10.15-x64.bat +++ /dev/null @@ -1 +0,0 @@ -dotnet publish -r osx.10.15-x64 -c Release /p:PublishSingleFile=true /p:PublishTrimmed=true \ No newline at end of file diff --git a/violet-message-search-core/hdownloader/publish-win10-arm-aot.bat b/violet-message-search-core/hdownloader/publish-win10-arm-aot.bat deleted file mode 100644 index 2f7245b20..000000000 --- a/violet-message-search-core/hdownloader/publish-win10-arm-aot.bat +++ /dev/null @@ -1 +0,0 @@ -dotnet publish -r win10-arm -c Release /p:PublishSingleFile=true /p:PublishTrimmed=true /p:PublishReadyToRun=true \ No newline at end of file diff --git a/violet-message-search-core/hdownloader/publish-win10-arm.bat b/violet-message-search-core/hdownloader/publish-win10-arm.bat deleted file mode 100644 index 533beec4e..000000000 --- a/violet-message-search-core/hdownloader/publish-win10-arm.bat +++ /dev/null @@ -1 +0,0 @@ -dotnet publish -r win10-arm -c Release /p:PublishSingleFile=true /p:PublishTrimmed=true \ No newline at end of file diff --git a/violet-message-search-core/hdownloader/publish-win10-x64-aot.bat b/violet-message-search-core/hdownloader/publish-win10-x64-aot.bat deleted file mode 100644 index 05724259b..000000000 --- a/violet-message-search-core/hdownloader/publish-win10-x64-aot.bat +++ /dev/null @@ -1 +0,0 @@ -dotnet publish -r win10-x64 -c Release /p:PublishSingleFile=true /p:PublishReadyToRun=true \ No newline at end of file diff --git a/violet-message-search-core/hdownloader/publish-win10-x64.bat b/violet-message-search-core/hdownloader/publish-win10-x64.bat deleted file mode 100644 index 979cda318..000000000 --- a/violet-message-search-core/hdownloader/publish-win10-x64.bat +++ /dev/null @@ -1 +0,0 @@ -dotnet publish -r win10-x64 -c Release /p:PublishSingleFile=true \ No newline at end of file diff --git a/violet-message-search-core/hdownloader/publish-win10-x86-aot.bat b/violet-message-search-core/hdownloader/publish-win10-x86-aot.bat deleted file mode 100644 index dd692c6e8..000000000 --- a/violet-message-search-core/hdownloader/publish-win10-x86-aot.bat +++ /dev/null @@ -1 +0,0 @@ -dotnet publish -r win10-x86 -c Release /p:PublishSingleFile=true /p:PublishReadyToRun=true \ No newline at end of file diff --git a/violet-message-search-core/hdownloader/publish-win10-x86.bat b/violet-message-search-core/hdownloader/publish-win10-x86.bat deleted file mode 100644 index 7fdcddbe6..000000000 --- a/violet-message-search-core/hdownloader/publish-win10-x86.bat +++ /dev/null @@ -1 +0,0 @@ -dotnet publish -r win10-x86 -c Release /p:PublishSingleFile=true \ No newline at end of file