-
Notifications
You must be signed in to change notification settings - Fork 5
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
- Loading branch information
Showing
27 changed files
with
701 additions
and
350 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,111 @@ | ||
| import 'package:html/dom.dart'; | ||
| import 'package:intl/intl.dart'; | ||
| import 'package:whapp/model/article.dart'; | ||
| import 'package:whapp/model/publisher.dart'; | ||
| import 'dart:convert'; | ||
| import 'package:http/http.dart' as http; | ||
| import 'package:html/parser.dart' as html_parser; | ||
| import 'package:whapp/utils/time.dart'; | ||
|
|
||
| class Nitter extends Publisher { | ||
| Map nextCursor = {}; | ||
|
|
||
| @override | ||
| String get homePage => "https://nitter.net"; | ||
|
|
||
| @override | ||
| String get name => "Nitter"; | ||
|
|
||
| @override | ||
| Future<Map<String, String>> get categories async => {}; | ||
|
|
||
| Future<Set<NewsArticle?>> extract(String category, int page, | ||
| {String query = ""}) async { | ||
| Set<NewsArticle?> articles = {}; | ||
| var dates = generateWeekDates(page); | ||
| var response = await http.get( | ||
| Uri.parse( | ||
| "$homePage/$category/search?f=tweets&q=$query&since=${dates[0]}&until=${dates[1]}"), | ||
| headers: { | ||
| 'Host': 'nitter.net', | ||
| 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; rv:121.0) Gecko/20100101 Firefox/121.0', | ||
| 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8', | ||
| 'Accept-Language': 'en-US,en;q=0.5', | ||
| }, | ||
| ); | ||
| if (response.statusCode == 200) { | ||
| Document document = html_parser.parse(utf8.decode(response.bodyBytes)); | ||
| List<Element> articleElements = document.querySelectorAll(".tweet-body"); | ||
| for (Element articleElement in articleElements) { | ||
| String? title = articleElement | ||
| .querySelector(".tweet-content") | ||
| ?.text | ||
| .split("\n") | ||
| .first; | ||
| String? excerpt = articleElement.querySelector(".tweet-content")?.text; | ||
| String? author = articleElement.querySelector(".username")?.text; | ||
| String? url = | ||
| articleElement.querySelector(".tweet-link")?.attributes["href"]; | ||
| String? thumbnail = ""; | ||
| String? content = ""; | ||
| String? date = | ||
| articleElement.querySelector(".tweet-date a")?.attributes["title"]; | ||
| String parsedTime = | ||
| convertToIso8601("$date", "MMM d, yyyy · h:mm a UTC"); | ||
|
|
||
| articles.add(NewsArticle( | ||
| this, | ||
| title ?? "", | ||
| content, | ||
| excerpt ?? "", | ||
| author ?? "", | ||
| "$homePage$url", | ||
| thumbnail, | ||
| parseDateString(parsedTime), | ||
| )); | ||
| } | ||
| return articles; | ||
| } | ||
| return articles; | ||
| } | ||
|
|
||
| @override | ||
| Future<Set<NewsArticle?>> categoryArticles( | ||
| {String category = "", int page = 1}) async { | ||
| if (category.isEmpty || category == "/") return {}; | ||
| return extract(category, page); | ||
| } | ||
|
|
||
| @override | ||
| Future<Set<NewsArticle?>> searchedArticles( | ||
| {required String searchQuery, int page = 1}) async { | ||
| if (!searchQuery.contains("#")) return {}; | ||
| return extract(searchQuery.split("#")[0], page, | ||
| query: searchQuery.split("#")[1]); | ||
| } | ||
|
|
||
| @override | ||
| Future<NewsArticle?> article(NewsArticle newsArticle) async { | ||
| var response = await http.get(Uri.parse(newsArticle.url)); | ||
| if (response.statusCode == 200) { | ||
| Document document = html_parser.parse(utf8.decode(response.bodyBytes)); | ||
| return newsArticle.fill( | ||
| content: document.querySelector(".tweet-body")?.text ?? ""); | ||
| } | ||
| return null; | ||
| } | ||
|
|
||
| List<String> generateWeekDates(int page) { | ||
| DateTime currentDate = DateTime.now(); | ||
| DateTime untilDate; | ||
| if (page == 1) { | ||
| untilDate = currentDate; | ||
| } else { | ||
| untilDate = currentDate.subtract(Duration(days: 7 * (page - 1))); | ||
| } | ||
| DateTime sinceDate = untilDate.subtract(Duration(days: 7)); | ||
| String sinceDateString = DateFormat('yyyy-MM-dd').format(sinceDate); | ||
| String untilDateString = DateFormat('yyyy-MM-dd').format(untilDate); | ||
| return [sinceDateString, untilDateString]; | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,98 @@ | ||
| import 'package:html/dom.dart'; | ||
| import 'package:whapp/model/article.dart'; | ||
| import 'package:whapp/model/publisher.dart'; | ||
| import 'dart:convert'; | ||
| import 'package:http/http.dart' as http; | ||
| import 'package:html/parser.dart' as html_parser; | ||
| import 'package:whapp/utils/time.dart'; | ||
|
|
||
| class ArsTechnica extends Publisher { | ||
| @override | ||
| String get homePage => "https://arstechnica.com"; | ||
|
|
||
| @override | ||
| String get name => "Ars Technica"; | ||
|
|
||
| @override | ||
| Future<NewsArticle?> article(NewsArticle newsArticle) async { | ||
| var response = await http.get(Uri.parse(newsArticle.url)); | ||
| if (response.statusCode == 200) { | ||
| Document document = html_parser.parse(utf8.decode(response.bodyBytes)); | ||
| Element? articleElement = document.querySelector(".article-content"); | ||
| String? thumbnail = ""; | ||
| String? content = articleElement?.innerHtml; | ||
| return newsArticle.fill(content: content, thumbnail: thumbnail); | ||
| } | ||
| return null; | ||
| } | ||
|
|
||
| @override | ||
| Future<Map<String, String>> get categories async => { | ||
| "News": "", | ||
| "Reviews": "reviews", | ||
| "Guides": "guides", | ||
| "Gaming": "gaming", | ||
| "Gear": "gear", | ||
| "Entertainment": "entertainment", | ||
| "Tomorrow": "tomorrow", | ||
| "Deals": "deals", | ||
| }; | ||
|
|
||
| @override | ||
| bool get hasSearchSupport => false; | ||
|
|
||
| @override | ||
| Future<Set<NewsArticle?>> categoryArticles({String category = "", int page = 1}) async { | ||
| Set<NewsArticle> articles = {}; | ||
| if(category.isNotEmpty) { | ||
| category="/$category"; | ||
| } | ||
|
|
||
| var response = await http.get(Uri.parse("$homePage$category/page/$page")); | ||
| if (response.statusCode == 200) { | ||
| Document document = html_parser.parse(utf8.decode(response.bodyBytes)); | ||
| List<Element> articleElements = | ||
| document.querySelectorAll(".article"); | ||
| for (Element articleElement in articleElements) { | ||
| String? title = articleElement.querySelector("h2 a")?.text; | ||
| String? excerpt = articleElement.querySelector(".excerpt")?.text; | ||
| String? author = articleElement.querySelector("span[itemprop=name]")?.text; | ||
| String? date = articleElement.querySelector("time")?.attributes["datetime"] ?? ""; | ||
| String? url = articleElement.querySelector("h2 a")?.attributes["href"]; | ||
| String? thumbnail = articleElement | ||
| .querySelector("figure div") | ||
| ?.attributes["style"]; | ||
|
|
||
| articles.add(NewsArticle( | ||
| this, | ||
| title ?? "", | ||
| "", | ||
| excerpt ?? "", | ||
| author ?? "", | ||
| url ?? "", | ||
| extractUrl(thumbnail), | ||
| parseDateString(date), | ||
| )); | ||
| } | ||
| } | ||
| return articles; | ||
| } | ||
|
|
||
| String extractUrl(String? inputString) { | ||
| RegExp regExp = RegExp(r"url\('([^']*)'\)"); | ||
| if(inputString!=null) { | ||
| Match? match = regExp.firstMatch(inputString); | ||
| if (match != null) { | ||
| return match.group(1)!; | ||
| } else { | ||
| return ""; | ||
| } | ||
| } | ||
| return ""; | ||
| } | ||
|
|
||
| @override | ||
| Future<Set<NewsArticle?>> searchedArticles({required String searchQuery, int page = 1}) async{ | ||
| return {}; | ||
| } | ||
| } |
Oops, something went wrong.