Skip to content

Commit

Permalink
add/fix Itags; bug fixes and refactoring; added some tests
Browse files Browse the repository at this point in the history
  • Loading branch information
@sealedtx
sealedtx committed Jan 11, 2020
1 parent c975fa7 commit 3b08706
Show file tree
Hide file tree
Showing 14 changed files with 256 additions and 120 deletions.
2 changes: 1 addition & 1 deletion ...n/downloader/YoutubeDownloadCallback.java → ...downloader/OnYoutubeDownloadListener.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@

import java.io.File;

public interface YoutubeDownloadCallback {
public interface OnYoutubeDownloadListener {

void onDownloading(int progress);

Expand Down
9 changes: 0 additions & 9 deletions src/main/java/com/github/kiulian/downloader/YoutubeDownloader.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -39,15 +39,6 @@ public class YoutubeDownloader {

public static final char[] ILLEGAL_FILENAME_CHARACTERS = {'/', '\n', '\r', '\t', '\0', '\f', '`', '?', '*', '\\', '<', '>', '|', '\"', ':'};

public interface DownloadCallback {

void onDownloading(int progress);

void onFinished(File file);

void onError(Throwable throwable);
}

private Parser parser;

public YoutubeDownloader() {
Expand Down
21 changes: 8 additions & 13 deletions src/main/java/com/github/kiulian/downloader/cipher/CachedCipherFactory.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -82,16 +82,11 @@ public void clearCache() {
}

@Override
public Cipher createCipher(String jsUrl) throws YoutubeException.CipherException {
public Cipher createCipher(String jsUrl) throws YoutubeException {
Cipher cipher = ciphers.get(jsUrl);

if (cipher == null) {
String js;
try {
js = extractor.loadUrl(jsUrl);
} catch (IOException e) {
throw new YoutubeException.CipherException("Could not load js file: " + jsUrl);
}
String js = extractor.loadUrl(jsUrl);

List<JsFunction> transformFunctions = getTransformFunctions(js);
String var = transformFunctions.get(0).getVar();
Expand All @@ -105,7 +100,7 @@ public Cipher createCipher(String jsUrl) throws YoutubeException.CipherException
return cipher;
}

private List<JsFunction> getTransformFunctions(String js) throws YoutubeException.CipherException {
private List<JsFunction> getTransformFunctions(String js) throws YoutubeException {
String name = getInitialFunctionName(js).replaceAll("[^A-Za-z0-9_]", "");

Pattern pattern = Pattern.compile(name + "=function\\(\\w\\)\\{[a-z=\\.\\(\\\"\\)]*;(.*);(?:.+)}");
Expand All @@ -129,7 +124,7 @@ private List<JsFunction> getTransformFunctions(String js) throws YoutubeExceptio
throw new YoutubeException.CipherException("Transformation functions not found");
}

private String getInitialFunctionName(String js) throws YoutubeException.CipherException {
private String getInitialFunctionName(String js) throws YoutubeException {
for (Pattern pattern : knownInitialFunctionPatterns) {
Matcher matcher = pattern.matcher(js);
if (matcher.find()) {
Expand All @@ -140,7 +135,7 @@ private String getInitialFunctionName(String js) throws YoutubeException.CipherE
throw new YoutubeException.CipherException("Initial function name not found");
}

private Map<String, CipherFunction> getTransformFunctionsMap(String var, String js) throws YoutubeException.CipherException {
private Map<String, CipherFunction> getTransformFunctionsMap(String var, String js) throws YoutubeException {
String[] transformObject = getTransformObject(var, js);
Map<String, CipherFunction> mapper = new HashMap<>();
for (String obj : transformObject) {
Expand All @@ -154,7 +149,7 @@ private Map<String, CipherFunction> getTransformFunctionsMap(String var, String
return mapper;
}

private String[] getTransformObject(String var, String js) throws YoutubeException.CipherException {
private String[] getTransformObject(String var, String js) throws YoutubeException {
var = var.replaceAll("[^A-Za-z0-9_]", "");
Pattern pattern = Pattern.compile(String.format("var %s=\\{(.*?)};", var), Pattern.DOTALL);
Matcher matcher = pattern.matcher(js);
Expand All @@ -166,7 +161,7 @@ private String[] getTransformObject(String var, String js) throws YoutubeExcepti
}


private CipherFunction mapFunction(String jsFunction) throws YoutubeException.CipherException {
private CipherFunction mapFunction(String jsFunction) throws YoutubeException {
for (Map.Entry<Pattern, CipherFunction> entry : functionsEquivalentMap.entrySet()) {
Matcher matcher = entry.getKey().matcher(jsFunction);
if (matcher.find()) {
Expand All @@ -177,7 +172,7 @@ private CipherFunction mapFunction(String jsFunction) throws YoutubeException.Ci
throw new YoutubeException.CipherException("Map function not found");
}

private String[] parseFunction(String jsFunction) throws YoutubeException.CipherException {
private String[] parseFunction(String jsFunction) throws YoutubeException {
Matcher matcher = JS_FUNCTION_PATTERN.matcher(jsFunction);

String[] nameAndArgument = new String[2];
Expand Down
2 changes: 1 addition & 1 deletion src/main/java/com/github/kiulian/downloader/cipher/CipherFactory.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,5 +24,5 @@

public interface CipherFactory {

Cipher createCipher(String jsUrl) throws YoutubeException.CipherException;
Cipher createCipher(String jsUrl) throws YoutubeException;
}
47 changes: 30 additions & 17 deletions src/main/java/com/github/kiulian/downloader/extractor/DefaultExtractor.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,6 @@
* #
*/

import com.alibaba.fastjson.JSONObject;
import com.github.kiulian.downloader.YoutubeException;

import java.io.BufferedReader;
Expand All @@ -38,20 +37,28 @@ public class DefaultExtractor implements Extractor {

private static final String DEFAULT_USER_AGENT = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/72.0.3626.121 Safari/537.36";
private static final String DEFAULT_ACCEPT_LANG = "en-US,en;";
private static final int DEFAULT_RETRY_ON_FAILURE = 3;

private Map<String, String> requestProperties = new HashMap<>();
private int retryOnFailure;

public DefaultExtractor() {
setRequestProperty("User-Agent", DEFAULT_USER_AGENT);
setRequestProperty("Accept_language", DEFAULT_ACCEPT_LANG);
retryOnFailure = DEFAULT_RETRY_ON_FAILURE;
}


public void setRequestProperty(String key, String value) {
requestProperties.put(key, value);
}

public void setRetryOnFailure(int retryOnFailure) {
this.retryOnFailure = retryOnFailure;
}

@Override
public String extractYtPlayerConfig(String html) throws YoutubeException.BadPageException {
public String extractYtPlayerConfig(String html) throws YoutubeException {
Matcher matcher = YT_PLAYER_CONFIG.matcher(html);

if (matcher.find()) {
Expand All @@ -62,20 +69,26 @@ public String extractYtPlayerConfig(String html) throws YoutubeException.BadPage
}

@Override
public String loadUrl(String url) throws IOException {
HttpURLConnection connection = (HttpURLConnection) new URL(url).openConnection();
requestProperties.forEach(connection::setRequestProperty);

BufferedReader in = new BufferedReader(new InputStreamReader(
connection.getInputStream()));

StringBuilder sb = new StringBuilder();
String inputLine;
while ((inputLine = in.readLine()) != null)
sb.append(inputLine).append('\n');
in.close();

return sb.toString();
public String loadUrl(String url) throws YoutubeException {
int retryCount = retryOnFailure;
while (retryCount > 0) {
try {
HttpURLConnection connection = (HttpURLConnection) new URL(url).openConnection();
requestProperties.forEach(connection::setRequestProperty);

BufferedReader in = new BufferedReader(new InputStreamReader(
connection.getInputStream()));

StringBuilder sb = new StringBuilder();
String inputLine;
while ((inputLine = in.readLine()) != null)
sb.append(inputLine).append('\n');
in.close();
return sb.toString();
} catch (IOException e) {
retryCount--;
}
}
throw new YoutubeException.VideoUnavailableException("Could not load url: " + url);
}

}
8 changes: 3 additions & 5 deletions src/main/java/com/github/kiulian/downloader/extractor/Extractor.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,9 +9,9 @@
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
*
* http://www.apache.org/licenses/LICENSE-2.0
*
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
Expand All @@ -23,13 +23,11 @@

import com.github.kiulian.downloader.YoutubeException;

import java.io.IOException;


public interface Extractor {

String extractYtPlayerConfig(String html) throws YoutubeException;

String loadUrl(String url) throws IOException;
String loadUrl(String url) throws YoutubeException;

}
97 changes: 53 additions & 44 deletions src/main/java/com/github/kiulian/downloader/model/Itag.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -45,78 +45,85 @@ public void setId(int id) {
i37(VideoQuality.hd1080, AudioQuality.medium),
i38(VideoQuality.highres, AudioQuality.medium),

i43(VideoQuality.medium, AudioQuality.unknown),
i44(VideoQuality.large, AudioQuality.unknown),
i45(VideoQuality.hd720, AudioQuality.unknown),
i46(VideoQuality.hd1080, AudioQuality.unknown),

i82(VideoQuality.medium, AudioQuality.unknown, true),
i83(VideoQuality.large, AudioQuality.unknown, true),
i84(VideoQuality.hd720, AudioQuality.unknown, true),
i85(VideoQuality.hd1080, AudioQuality.unknown, true),

i92(VideoQuality.small, AudioQuality.unknown, true),
i93(VideoQuality.medium, AudioQuality.unknown, true),
i94(VideoQuality.large, AudioQuality.unknown, true),
i95(VideoQuality.hd720, AudioQuality.unknown, true),
i96(VideoQuality.hd1080, AudioQuality.unknown),

i100(VideoQuality.medium, AudioQuality.unknown, true),
i101(VideoQuality.large, AudioQuality.unknown, true),
i102(VideoQuality.hd720, AudioQuality.unknown, true),

i132(VideoQuality.small, AudioQuality.unknown),
i43(VideoQuality.medium, AudioQuality.medium),
i44(VideoQuality.large, AudioQuality.medium),
i45(VideoQuality.hd720, AudioQuality.medium),
i46(VideoQuality.hd1080, AudioQuality.medium),

// 3D videos
i82(VideoQuality.medium, AudioQuality.medium, true),
i83(VideoQuality.large, AudioQuality.medium, true),
i84(VideoQuality.hd720, AudioQuality.medium, true),
i85(VideoQuality.hd1080, AudioQuality.medium, true),
i100(VideoQuality.medium, AudioQuality.medium, true),
i101(VideoQuality.large, AudioQuality.medium, true),
i102(VideoQuality.hd720, AudioQuality.medium, true),

// Apple HTTP Live Streaming
i91(VideoQuality.tiny, AudioQuality.low),
i92(VideoQuality.small, AudioQuality.low),
i93(VideoQuality.medium, AudioQuality.medium),
i94(VideoQuality.large, AudioQuality.medium),
i95(VideoQuality.hd720, AudioQuality.high),
i96(VideoQuality.hd1080, AudioQuality.high),
i132(VideoQuality.small, AudioQuality.low),
i151(VideoQuality.tiny, AudioQuality.low),

// DASH mp4 video
i133(VideoQuality.small),
i134(VideoQuality.medium),
i135(VideoQuality.large),
i136(VideoQuality.hd720),
i137(VideoQuality.hd1080),
i138(VideoQuality.hd2160),
i160(VideoQuality.tiny),
i212(VideoQuality.large),
i264(VideoQuality.hd1440),
i266(VideoQuality.hd2160),
i298(VideoQuality.hd720),
i299(VideoQuality.hd1080),

// DASH mp4 audio
i139(AudioQuality.low),
i140(AudioQuality.medium),
i141(AudioQuality.high),
i256(AudioQuality.unknown),
i325(AudioQuality.unknown),
i328(AudioQuality.unknown),

i151(VideoQuality.tiny, AudioQuality.unknown),

i160(VideoQuality.tiny),
// DASH webm video
i167(VideoQuality.medium),
i168(VideoQuality.large),
i169(VideoQuality.hd1080),

i171(AudioQuality.medium),


i169(VideoQuality.hd720),
i170(VideoQuality.hd1080),
i218(VideoQuality.large),
i219(VideoQuality.tiny),

i242(VideoQuality.small),
i243(VideoQuality.medium),
i244(VideoQuality.large),
i245(VideoQuality.large),
i246(VideoQuality.large),
i247(VideoQuality.hd720),
i248(VideoQuality.hd1080),
i249(AudioQuality.low),
i250(AudioQuality.medium),
i251(AudioQuality.medium),

i264(VideoQuality.hd1440),
i266(VideoQuality.hd2160),

i271(VideoQuality.hd1440),
i272(VideoQuality.highres),
i278(VideoQuality.tiny),

i298(VideoQuality.hd720),
i299(VideoQuality.hd1080),

i302(VideoQuality.hd720),
i303(VideoQuality.hd1080),
i308(VideoQuality.hd1440),

i313(VideoQuality.hd2160),
i315(VideoQuality.hd2160),

// DASH webm audio
i171(AudioQuality.medium),
i172(AudioQuality.high),

// Dash webm audio with opus inside
i249(AudioQuality.low),
i250(AudioQuality.low),
i251(AudioQuality.medium),

// Dash webm hdr video
i330(VideoQuality.tiny),
i331(VideoQuality.small),
i332(VideoQuality.medium),
Expand All @@ -126,14 +133,16 @@ public void setId(int id) {
i336(VideoQuality.hd1440),
i337(VideoQuality.hd2160),


// av01 video only formats
i394(VideoQuality.tiny),
i395(VideoQuality.small),
i396(VideoQuality.medium),
i397(VideoQuality.large),
i398(VideoQuality.hd720),
i399(VideoQuality.hd1080),

i400(VideoQuality.hd1440),
i401(VideoQuality.hd2160),
i402(VideoQuality.hd2880p)

;

Expand Down
12 changes: 6 additions & 6 deletions src/main/java/com/github/kiulian/downloader/model/VideoDetails.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -37,7 +37,7 @@ public class VideoDetails {
private String shortDescription;
private List<String> thumbnails;
private String author;
private int viewCount;
private long viewCount;
private int averageRating;
private boolean isLiveContent;

Expand All @@ -47,7 +47,7 @@ public VideoDetails() {
public VideoDetails(JSONObject json) {
videoId = json.getString("videoId");
title = json.getString("title");
lengthSeconds = json.getInteger("lengthSeconds");
lengthSeconds = json.getIntValue("lengthSeconds");
keywords = json.containsKey("keywords") ? json.getJSONArray("keywords").toJavaList(String.class) : Collections.emptyList();
shortDescription = json.getString("shortDescription");
JSONArray jsonThumbnails = json.getJSONObject("thumbnail").getJSONArray("thumbnails");
Expand All @@ -57,10 +57,10 @@ public VideoDetails(JSONObject json) {
if (jsonObject.containsKey("url"))
thumbnails.add(jsonObject.getString("url"));
}
averageRating = json.getInteger("averageRating");
viewCount = json.getInteger("viewCount");
averageRating = json.getIntValue("averageRating");
viewCount = json.getLongValue("viewCount");
author = json.getString("author");
isLiveContent = json.getBoolean("isLiveContent");
isLiveContent = json.getBooleanValue("isLiveContent");
}

public String videoId() {
Expand Down Expand Up @@ -91,7 +91,7 @@ public String author() {
return author;
}

public int viewCount() {
public long viewCount() {
return viewCount;
}

Expand Down
Loading

0 comments on commit 3b08706

Please sign in to comment.