diff --git a/lib/extractor/trend/google.dart b/lib/extractor/trend/google.dart index ce0113a..a9d8c8c 100644 --- a/lib/extractor/trend/google.dart +++ b/lib/extractor/trend/google.dart @@ -1,5 +1,6 @@ import 'dart:convert'; +import 'package:dio/dio.dart'; import 'package:hive/hive.dart'; import 'package:html/dom.dart'; import 'package:html/parser.dart' as html_parser; @@ -20,7 +21,9 @@ class GoogleTrend extends Trend { var country = "US"; if (Hive.isBoxOpen("settings")) country = "${countryCodes[Store.countrySetting]}"; - var response = await dio().get(url + country); + var response = await dio().get(url + country, options: Options( + responseType: ResponseType.plain, + )); List queries = []; if (response.statusCode == 200) { Document document = html_parser.parse(response.data); @@ -28,6 +31,7 @@ class GoogleTrend extends Trend { .replaceFirst(")]}',", "") .replaceFirst("", "") .replaceFirst("", ""); + print(jsonText); var somethings = json.decode(jsonText)["default"]["trendingSearchesDays"]; for (var something in somethings) { var searches = something["trendingSearches"]; diff --git a/lib/service/simplytranslate.dart b/lib/service/simplytranslate.dart index c9d2890..444e651 100644 --- a/lib/service/simplytranslate.dart +++ b/lib/service/simplytranslate.dart @@ -1,6 +1,5 @@ import 'package:dio/dio.dart'; import 'package:html/dom.dart'; -import 'package:html/parser.dart' as html_parser; import 'package:html/parser.dart'; import 'package:raven/brain/dio_manager.dart'; import 'package:raven/utils/store.dart'; @@ -163,13 +162,12 @@ class SimplyTranslate { String language, ) async { List translated = []; - var mSentences = mergeSentences(sentences); - for (var ms in mSentences) { + for (var ms in sentences) { var s = await translate( ms, language, ); - translated.addAll(s.split("~~~")); + translated.add(s); } return translated; } @@ -223,7 +221,7 @@ class SimplyTranslate { for (String sentence in sentences) { if (currentMergedSentence.length + sentence.length <= 4000) { if (currentMergedSentence.isNotEmpty) { - currentMergedSentence += "~~~"; + currentMergedSentence += " "; } currentMergedSentence += sentence; } else { @@ -241,30 +239,40 @@ class SimplyTranslate { (element) => element.isEmpty, ) ..map( - (e) => e + "~~~", + (e) => e + " ", ); } + List splitParagraph(String paragraph, [int wordsPerSentence = 500]) { + final List words = paragraph.replaceAll(" <", "<").replaceAll("> ", ">").split(new RegExp(r'\s+')); + final List> groupedWords = _groupLists(words, wordsPerSentence); + final List sentences = groupedWords.map((wordList) => wordList.join(' ')).toList(); + + return sentences; + } + + List> _groupLists(List items, int groupSize) { + final List> groups = []; + for (int i = 0; i < items.length; i += groupSize) { + groups.add(items.sublist(i, i + groupSize > items.length ? items.length : i + groupSize)); + } + return groups; + } + + Future translateParagraph( String paragraph, String language, ) async { if(paragraph.isEmpty) return paragraph; - Document document = html_parser.parse(paragraph.replaceAll("<", " <").replaceAll(">", "> ")); - var html = document.outerHtml; - List smallParas = mergeSentences(extractTextFromDocument(document)); + var translatedHtml = ""; + List smallParas = splitParagraph(paragraph); List translatedSmallParas = List.filled(smallParas.length, ''); for (int i = 0; i < smallParas.length; i++) { translatedSmallParas[i] = await translate(smallParas[i], language); + translatedHtml +=translatedSmallParas[i]; } - for (int i = 0; i < translatedSmallParas.length; i++) { - for (int j = 0; j < translatedSmallParas[i].split("~~~").length; j++) { - html = html.replaceFirst( - smallParas[i].split("~~~")[j], - translatedSmallParas[i].split("~~~")[j], - ); - } - } - return html; + + return translatedHtml; } }