Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

chore: update provider database #5966

Merged
merged 1 commit into from
Sep 13, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion scripts/update-provider-database.sh
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@ set -euo pipefail
export TZ=UTC

# Provider database revision.
REV=ab970e40d3979893c3bb6a93030e1a52223d7db6
REV=77cbf92a8565fdf1bcaba10fa93c1455c750a1e9

CORE_ROOT="$PWD"
TMP="$(mktemp -d)"
Expand Down
51 changes: 15 additions & 36 deletions src/oauth2.rs
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,6 @@ use anyhow::{Context as _, Result};
use percent_encoding::{utf8_percent_encode, NON_ALPHANUMERIC};
use serde::Deserialize;

use crate::config::Config;
use crate::context::Context;
use crate::net::http::post_form;
use crate::net::read_url_blob;
Expand Down Expand Up @@ -62,8 +61,7 @@ pub async fn get_oauth2_url(
addr: &str,
redirect_uri: &str,
) -> Result<Option<String>> {
let proxy_enabled = context.get_config_bool(Config::ProxyEnabled).await?;
if let Some(oauth2) = Oauth2::from_address(context, addr, proxy_enabled).await {
if let Some(oauth2) = Oauth2::from_address(context, addr).await {
context
.sql
.set_raw_config("oauth2_pending_redirect_uri", Some(redirect_uri))
Expand All @@ -83,8 +81,7 @@ pub(crate) async fn get_oauth2_access_token(
code: &str,
regenerate: bool,
) -> Result<Option<String>> {
let proxy_enabled = context.get_config_bool(Config::ProxyEnabled).await?;
if let Some(oauth2) = Oauth2::from_address(context, addr, proxy_enabled).await {
if let Some(oauth2) = Oauth2::from_address(context, addr).await {
let lock = context.oauth2_mutex.lock().await;

// read generated token
Expand Down Expand Up @@ -232,8 +229,7 @@ pub(crate) async fn get_oauth2_addr(
addr: &str,
code: &str,
) -> Result<Option<String>> {
let proxy_enabled = context.get_config_bool(Config::ProxyEnabled).await?;
let oauth2 = match Oauth2::from_address(context, addr, proxy_enabled).await {
let oauth2 = match Oauth2::from_address(context, addr).await {
Some(o) => o,
None => return Ok(None),
};
Expand Down Expand Up @@ -268,8 +264,9 @@ pub(crate) async fn get_oauth2_addr(
}

impl Oauth2 {
async fn from_address(context: &Context, addr: &str, skip_mx: bool) -> Option<Self> {
async fn from_address(context: &Context, addr: &str) -> Option<Self> {
let addr_normalized = normalize_addr(addr);
let skip_mx = true;
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Sure MX lookup will be never needed for yandex?

Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

If we support it, we can enable it and test. I think Yandex has something like Google Workspace already but doubt we want to support it, especially since Yandex is unusable anyway due to strict rate limits and requests to solve captcha on the web when you hit the limit.

if let Some(domain) = addr_normalized
.find('@')
.map(|index| addr_normalized.split_at(index + 1).1)
Expand Down Expand Up @@ -367,38 +364,20 @@ mod tests {
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn test_oauth_from_address() {
let t = TestContext::new().await;

// Delta Chat does not have working Gmail client ID anymore.
assert_eq!(Oauth2::from_address(&t, "[email protected]").await, None);
assert_eq!(Oauth2::from_address(&t, "[email protected]").await, None);

assert_eq!(
Oauth2::from_address(&t, "[email protected]", false).await,
Some(OAUTH2_GMAIL)
);
assert_eq!(
Oauth2::from_address(&t, "[email protected]", false).await,
Some(OAUTH2_GMAIL)
);
assert_eq!(
Oauth2::from_address(&t, "[email protected]", false).await,
Oauth2::from_address(&t, "[email protected]").await,
Some(OAUTH2_YANDEX)
);
assert_eq!(
Oauth2::from_address(&t, "[email protected]", false).await,
Oauth2::from_address(&t, "[email protected]").await,
Some(OAUTH2_YANDEX)
);
assert_eq!(Oauth2::from_address(&t, "[email protected]", false).await, None);
}

#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn test_oauth_from_mx() {
// youtube staff seems to use "google workspace with oauth2", figures this out by MX lookup
let t = TestContext::new().await;
assert_eq!(
Oauth2::from_address(&t, "[email protected]", false).await,
Some(OAUTH2_GMAIL)
);
// without MX lookup, we would not know as youtube.com is not in our provider-db
assert_eq!(
Oauth2::from_address(&t, "[email protected]", true).await,
None
);
assert_eq!(Oauth2::from_address(&t, "[email protected]").await, None);
}

#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
Expand All @@ -414,11 +393,11 @@ mod tests {
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn test_get_oauth2_url() {
let ctx = TestContext::new().await;
let addr = "dignifiedquire@gmail.com";
let addr = "example@yandex.com";
let redirect_uri = "chat.delta:/com.b44t.messenger";
let res = get_oauth2_url(&ctx.ctx, addr, redirect_uri).await.unwrap();

assert_eq!(res, Some("https://accounts.google.com/o/oauth2/auth?client_id=959970109878%2D4mvtgf6feshskf7695nfln6002mom908%2Eapps%2Egoogleusercontent%2Ecom&redirect_uri=chat%2Edelta%3A%2Fcom%2Eb44t%2Emessenger&response_type=code&scope=https%3A%2F%2Fmail.google.com%2F%20email&access_type=offline".into()));
assert_eq!(res, Some("https://oauth.yandex.com/authorize?client_id=c4d0b6735fc8420a816d7e1303469341&response_type=code&scope=mail%3Aimap_full%20mail%3Asmtp&force_confirm=true".into()));
}

#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
Expand Down
69 changes: 62 additions & 7 deletions src/provider/data.rs
Original file line number Diff line number Diff line change
Expand Up @@ -509,6 +509,8 @@ static P_FREENET_DE: Provider = Provider {
overview_page: "https://providers.delta.chat/freenet-de",
server: &[
Server { protocol: Imap, socket: Ssl, hostname: "mx.freenet.de", port: 993, username_pattern: Email },
Server { protocol: Imap, socket: Starttls, hostname: "mx.freenet.de", port: 143, username_pattern: Email },
Server { protocol: Smtp, socket: Ssl, hostname: "mx.freenet.de", port: 465, username_pattern: Email },
Server { protocol: Smtp, socket: Starttls, hostname: "mx.freenet.de", port: 587, username_pattern: Email },
],
opt: ProviderOptions::new(),
Expand All @@ -532,7 +534,7 @@ static P_GMAIL: Provider = Provider {
..ProviderOptions::new()
},
config_defaults: None,
oauth2_authorizer: Some(Oauth2Authorizer::Gmail),
oauth2_authorizer: None,
};

// gmx.net.md: gmx.net, gmx.de, gmx.at, gmx.ch, gmx.org, gmx.eu, gmx.info, gmx.biz, gmx.com
Expand Down Expand Up @@ -938,6 +940,41 @@ static P_MEHL_STORE: Provider = Provider {
oauth2_authorizer: None,
};

// migadu.md: migadu.com
static P_MIGADU: Provider = Provider {
id: "migadu",
status: Status::Ok,
before_login_hint: "",
after_login_hint: "",
overview_page: "https://providers.delta.chat/migadu",
server: &[
Server {
protocol: Imap,
socket: Ssl,
hostname: "imap.migadu.com",
port: 993,
username_pattern: Email,
},
Server {
protocol: Smtp,
socket: Ssl,
hostname: "smtp.migadu.com",
port: 465,
username_pattern: Email,
},
Server {
protocol: Smtp,
socket: Starttls,
hostname: "smtp.migadu.com",
port: 587,
username_pattern: Email,
},
],
opt: ProviderOptions::new(),
config_defaults: None,
oauth2_authorizer: None,
};

// nauta.cu.md: nauta.cu
static P_NAUTA_CU: Provider = Provider {
id: "nauta.cu",
Expand Down Expand Up @@ -1158,7 +1195,7 @@ static P_OUVATON_COOP: Provider = Provider {
oauth2_authorizer: None,
};

// posteo.md: posteo.de, posteo.af, posteo.at, posteo.be, posteo.ca, posteo.ch, posteo.cl, posteo.co, posteo.co.uk, posteo.com.br, posteo.cr, posteo.cz, posteo.dk, posteo.ee, posteo.es, posteo.eu, posteo.fi, posteo.gl, posteo.gr, posteo.hn, posteo.hr, posteo.hu, posteo.ie, posteo.in, posteo.is, posteo.it, posteo.jp, posteo.la, posteo.li, posteo.lt, posteo.lu, posteo.me, posteo.mx, posteo.my, posteo.net, posteo.nl, posteo.no, posteo.nz, posteo.org, posteo.pe, posteo.pl, posteo.pm, posteo.pt, posteo.ro, posteo.ru, posteo.se, posteo.sg, posteo.si, posteo.tn, posteo.uk, posteo.us
// posteo.md: posteo.de, posteo.af, posteo.at, posteo.be, posteo.ca, posteo.ch, posteo.cl, posteo.co, posteo.co.uk, posteo.com, posteo.com.br, posteo.cr, posteo.cz, posteo.dk, posteo.ee, posteo.es, posteo.eu, posteo.fi, posteo.gl, posteo.gr, posteo.hn, posteo.hr, posteo.hu, posteo.ie, posteo.in, posteo.is, posteo.it, posteo.jp, posteo.la, posteo.li, posteo.lt, posteo.lu, posteo.me, posteo.mx, posteo.my, posteo.net, posteo.nl, posteo.no, posteo.nz, posteo.org, posteo.pe, posteo.pl, posteo.pm, posteo.pt, posteo.ro, posteo.ru, posteo.se, posteo.sg, posteo.si, posteo.tn, posteo.uk, posteo.us
static P_POSTEO: Provider = Provider {
id: "posteo",
status: Status::Ok,
Expand Down Expand Up @@ -1532,11 +1569,26 @@ static P_TUTANOTA: Provider = Provider {
// ukr.net.md: ukr.net
static P_UKR_NET: Provider = Provider {
id: "ukr.net",
status: Status::Ok,
before_login_hint: "",
status: Status::Preparation,
before_login_hint: "You must allow IMAP access to your account before you can login.",
after_login_hint: "",
overview_page: "https://providers.delta.chat/ukr-net",
server: &[],
server: &[
Server {
protocol: Imap,
socket: Ssl,
hostname: "imap.ukr.net",
port: 993,
username_pattern: Email,
},
Server {
protocol: Smtp,
socket: Ssl,
hostname: "smtp.ukr.net",
port: 465,
username_pattern: Email,
},
],
opt: ProviderOptions::new(),
config_defaults: None,
oauth2_authorizer: None,
Expand Down Expand Up @@ -1818,7 +1870,7 @@ static P_ZOHO: Provider = Provider {
oauth2_authorizer: None,
};

pub(crate) static PROVIDER_DATA: [(&str, &Provider); 531] = [
pub(crate) static PROVIDER_DATA: [(&str, &Provider); 533] = [
("163.com", &P_163),
("aktivix.org", &P_AKTIVIX_ORG),
("aliyun.com", &P_ALIYUN),
Expand Down Expand Up @@ -2191,6 +2243,7 @@ pub(crate) static PROVIDER_DATA: [(&str, &Provider); 531] = [
("ente.quest", &P_MEHL_STORE),
("ente.cfd", &P_MEHL_STORE),
("nein.jetzt", &P_MEHL_STORE),
("migadu.com", &P_MIGADU),
("nauta.cu", &P_NAUTA_CU),
("naver.com", &P_NAVER),
("nine.testrun.org", &P_NINE_TESTRUN_ORG),
Expand All @@ -2211,6 +2264,7 @@ pub(crate) static PROVIDER_DATA: [(&str, &Provider); 531] = [
("posteo.cl", &P_POSTEO),
("posteo.co", &P_POSTEO),
("posteo.co.uk", &P_POSTEO),
("posteo.com", &P_POSTEO),
("posteo.com.br", &P_POSTEO),
("posteo.cr", &P_POSTEO),
("posteo.cz", &P_POSTEO),
Expand Down Expand Up @@ -2393,6 +2447,7 @@ pub(crate) static PROVIDER_IDS: Lazy<HashMap<&'static str, &'static Provider>> =
("mailo.com", &P_MAILO_COM),
("mehl.cloud", &P_MEHL_CLOUD),
("mehl.store", &P_MEHL_STORE),
("migadu", &P_MIGADU),
("nauta.cu", &P_NAUTA_CU),
("naver", &P_NAVER),
("nine.testrun.org", &P_NINE_TESTRUN_ORG),
Expand Down Expand Up @@ -2431,4 +2486,4 @@ pub(crate) static PROVIDER_IDS: Lazy<HashMap<&'static str, &'static Provider>> =
});

pub static _PROVIDER_UPDATED: Lazy<chrono::NaiveDate> =
Lazy::new(|| chrono::NaiveDate::from_ymd_opt(2024, 8, 23).unwrap());
Lazy::new(|| chrono::NaiveDate::from_ymd_opt(2024, 9, 13).unwrap());
Loading