1berry/jdownloader-mirror
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

mirroring bot - 2024/08/23

Browse Source
This commit is contained in:
updating-bot 2024-08-23 05:28:02 +00:00
parent 9c87c111eb
commit 4b4cb0d7dc
29 changed files with 1165 additions and 944 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

17
svn_MyJDownloaderClient/src/org/jdownloader/myjdownloader/client/bindings/AbstractLinkQuery.java
View File

@ -43,6 +43,8 @@ public abstract class AbstractLinkQuery extends AbstractQuery {
private boolean advancedStatus = false;
private long[] linkUUIDs = null;
public boolean isAddedDate() {
return this.addedDate;
}
@ -103,6 +105,21 @@ public abstract class AbstractLinkQuery extends AbstractQuery {
this.password = password;
}
/**
* @return the linkUUIDs
*/
public long[] getLinkUUIDs() {
return this.linkUUIDs;
}
/**
* @param linkUUIDs
* the linkUUIDs to set
*/
public void setLinkUUIDs(long[] linkUUIDs) {
this.linkUUIDs = linkUUIDs;
}
private boolean url = false;
}

18
svn_MyJDownloaderClient/src/org/jdownloader/myjdownloader/client/bindings/AbstractQuery.java
View File

@ -43,7 +43,7 @@ public abstract class AbstractQuery extends AbstractJsonData {
private boolean priority = false;
public boolean isStatus() {
return status;
return this.status;
}
public void setStatus(boolean status) {
@ -61,7 +61,7 @@ public abstract class AbstractQuery extends AbstractJsonData {
private long[] packageUUIDs;
public boolean isPriority() {
return priority;
return this.priority;
}
public void setPriority(boolean priority) {
@ -69,19 +69,19 @@ public abstract class AbstractQuery extends AbstractJsonData {
}
public int getMaxResults() {
return maxResults;
return this.maxResults;
}
public int getStartAt() {
return startAt;
return this.startAt;
}
public boolean isBytesTotal() {
return bytesTotal;
return this.bytesTotal;
}
public boolean isComment() {
return comment;
return this.comment;
}
public void setStartAt(final int startAt) {
@ -89,11 +89,11 @@ public abstract class AbstractQuery extends AbstractJsonData {
}
public boolean isEnabled() {
return enabled;
return this.enabled;
}
public void setBytesTotal(final boolean size) {
bytesTotal = size;
this.bytesTotal = size;
}
public void setComment(final boolean comment) {
@ -105,7 +105,7 @@ public abstract class AbstractQuery extends AbstractJsonData {
}
public long[] getPackageUUIDs() {
return packageUUIDs;
return this.packageUUIDs;
}
public void setPackageUUIDs(final long[] packageUUIDs) {

9
svn_trunk/src/jd/config/Property.java
View File

@ -28,6 +28,7 @@ import java.util.WeakHashMap;
import org.appwork.exceptions.WTFException;
import org.appwork.storage.JSonStorage;
import org.appwork.storage.TypeRef;
import org.appwork.storage.simplejson.MinimalMemoryMap;
import org.appwork.utils.DebugMode;
import org.appwork.utils.ReflectionUtils;
@ -342,13 +343,7 @@ public class Property implements Serializable {
}
public static Map<String, Object> newMapInstance(int size) {
try {
// 25.08.2022 - avoid upgrade issues, remove me after some months
final Class<?> minimalMap = Class.forName("org.appwork.storage.simplejson.MinimalMemoryMap");
return (Map<String, Object>) minimalMap.getConstructor(int.class).newInstance(size);
} catch (Throwable e) {
return new HashMap<String, Object>(size);
}
return new MinimalMemoryMap<String, Object>(size);
}
/**

340
svn_trunk/src/jd/plugins/BrowserAdapter.java
View File

@ -15,22 +15,41 @@
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package jd.plugins;
import org.appwork.storage.config.JsonConfig;
import org.jdownloader.settings.GeneralSettings;
import jd.controlling.downloadcontroller.SingleDownloadController;
import jd.http.Browser;
import jd.http.Request;
import jd.http.URLConnectionAdapter;
import jd.parser.html.Form;
import jd.plugins.download.DownloadInterface;
import jd.plugins.download.DownloadLinkDownloadable;
import jd.plugins.download.Downloadable;
import jd.plugins.download.raf.HTTPDownloader;
import jd.plugins.download.raf.OldRAFDownload;
import org.appwork.net.protocol.http.HTTPConstants;
import org.appwork.storage.config.JsonConfig;
import org.appwork.utils.DebugMode;
import org.appwork.utils.net.HTTPHeader;
import org.jdownloader.settings.GeneralSettings;
public class BrowserAdapter {
public static final int ERROR_REDIRECTED = -1;
@Deprecated
public static final int ERROR_REDIRECTED = -1;
private static final boolean NEW_CORE = DebugMode.TRUE_IN_IDE_ELSE_FALSE;
private static DownloadInterface getDownloadInterface(Downloadable downloadable, Request request, boolean resumeEnabled, int pluginConnections) throws Exception {
final jd.plugins.download.raf.OldRAFDownload dl = new jd.plugins.download.raf.OldRAFDownload(downloadable, request);
final DownloadInterface dl;
if (NEW_CORE) {
dl = new HTTPDownloader(downloadable, request);
} else {
if (false) {
// was fine all the years, so better optimize handling in download system to better support edge cases with downloading
// content-encoding connections
request.getHeaders().put(new HTTPHeader(HTTPConstants.HEADER_REQUEST_ACCEPT_ENCODING, "identity", false));
}
dl = new jd.plugins.download.raf.OldRAFDownload(downloadable, request);
}
final int customizedConnections = downloadable.getChunks();
final int setConnections;
if (pluginConnections > 0) {
@ -66,8 +85,14 @@ public class BrowserAdapter {
setConnections = Math.min(customizedConnections, Math.abs(pluginConnections));
}
}
dl.setChunkNum(setConnections);
dl.setResume(resumeEnabled);
if (dl instanceof HTTPDownloader) {
((HTTPDownloader) dl).setChunkNum(setConnections);
((HTTPDownloader) dl).setMaxChunksNum(Math.abs(pluginConnections));
((HTTPDownloader) dl).setResume(resumeEnabled);
} else if (dl instanceof OldRAFDownload) {
((OldRAFDownload) dl).setChunkNum(setConnections);
((OldRAFDownload) dl).setResume(resumeEnabled);
}
return dl;
}
@ -82,201 +107,120 @@ public class BrowserAdapter {
return null;
}
public static DownloadInterface openDownload(Browser br, DownloadLink downloadLink, String link) throws Exception {
return openDownload(br, getDownloadable(downloadLink, br), br.createRequest(link), false, 1, false);
@Deprecated
public static DownloadInterface openDownload(final Browser br, DownloadLink downloadLink, String link) throws Exception {
return openDownload(br, downloadLink, br.createRequest(link), false, 1);
}
public static DownloadInterface openDownload(Browser br, DownloadLink downloadLink, String url, String postdata) throws Exception {
return openDownload(br, downloadLink, url, postdata, false, 1);
}
/**
*
* @param br
* @param downloadLink
* @param url
* @param postdata
* @param resume
* true|false, if chunks over 1 it must be true!
* @param chunks
* 0 = unlimited, chunks must start with negative sign otherwise it forces value to be used instead of up to value.
* @return
* @throws Exception
*/
public static DownloadInterface openDownload(Browser br, DownloadLink downloadLink, String url, String postdata, boolean resume, int chunks) throws Exception {
return openDownload(br, getDownloadable(downloadLink, br), br.createPostRequest(url, postdata), resume, chunks, false);
}
/**
*
* @param br
* @param downloadLink
* @param link
* @param resume
* true|false, if chunks over 1 it must be true!
* @param chunks
* 0 = unlimited, chunks must start with negative sign otherwise it forces value to be used instead of up to value.
* @return
* @throws Exception
*/
public static DownloadInterface openDownload(Browser br, DownloadLink downloadLink, String link, boolean resume, int chunks) throws Exception {
return openDownload(br, getDownloadable(downloadLink, br), br.createRequest(link), resume, chunks, false);
}
/**
*
* @param br
* @param downloadLink
* @param link
* @param resume
* true|false, if chunks over 1 it must be true!
* @param chunks
* 0 = unlimited, chunks must start with negative sign otherwise it forces value to be used instead of up to value.
* @param forceRedirectWait
* forces thread sleep between redirects (even if they are not the same, useful for (multi)hosters which wait via series
* redirects)
* @return
* @throws Exception
*/
public static DownloadInterface openDownload(Browser br, DownloadLink downloadLink, String link, boolean resume, int chunks, final boolean forceRedirectWait) throws Exception {
return openDownload(br, getDownloadable(downloadLink, br), br.createRequest(link), resume, chunks, forceRedirectWait);
}
/**
*
* @param br
* @param downloadLink
* @param form
* @param resume
* true|false, if chunks over 1 it must be true!
* @param chunks
* 0 = unlimited, chunks must start with negative sign otherwise it forces value to be used instead of up to value.
* @return
* @throws Exception
*/
public static DownloadInterface openDownload(Browser br, DownloadLink downloadLink, Form form, boolean resume, int chunks) throws Exception {
return openDownload(br, getDownloadable(downloadLink, br), br.createRequest(form), resume, chunks, false);
}
public static DownloadInterface openDownload(Browser br, DownloadLink downloadLink, Form form) throws Exception {
return openDownload(br, downloadLink, form, false, 1);
}
/**
*
* @param br
* @param downloadLink
* @param request
* @param resume
* @param chunks
* @return
* @throws Exception
*/
public static DownloadInterface openDownload(final Browser br, final DownloadLink downloadLink, final Request request, final boolean resume, final int chunks) throws Exception {
return openDownload(br, getDownloadable(downloadLink, br), request, resume, chunks, false);
}
/**
*
* @param br
* @param downloadable
* @param request
* @param resume
* true|false, if chunks over 1 it must be true!
* @param chunks
* 0 = unlimited, chunks must start with negative sign otherwise it forces value to be used instead of up to value.
* @return
* @throws Exception
*/
public static DownloadInterface openDownload(Browser br, Downloadable downloadable, Request request, boolean resume, int chunks) throws Exception {
return openDownload(br, downloadable, request, resume, chunks, false);
if (NEW_CORE) {
final String originalUrl = br.getURL();
int maxRedirects = 10;
final DownloadInterface dl = getDownloadInterface(downloadable, request, resume, chunks);
downloadable.setDownloadInterface(dl);
while (maxRedirects-- > 0) {
dl.setInitialRequest(request);
final URLConnectionAdapter connection = dl.connect(br);
if (connection.getRequest().getLocation() == null) {
return dl;
} else {
connection.disconnect();
request = br.createRedirectFollowingRequest(request);
if (originalUrl != null) {
request.getHeaders().put(HTTPConstants.HEADER_REQUEST_REFERER, originalUrl);
}
}
}
throw new PluginException(LinkStatus.ERROR_PLUGIN_DEFECT, "Redirectloop");
} else {
String originalUrl = br.getURL();
DownloadInterface dl = getDownloadInterface(downloadable, request, resume, chunks);
downloadable.setDownloadInterface(dl);
final PluginForHost plugin;
final DownloadLink downloadLink;
if (downloadable instanceof DownloadLinkDownloadable) {
plugin = ((DownloadLinkDownloadable) downloadable).getPlugin();
downloadLink = ((DownloadLinkDownloadable) downloadable).getDownloadLink();
} else {
plugin = null;
downloadLink = null;
}
try {
dl.connect(br);
} catch (PluginException handle) {
try {
dl.getConnection().disconnect();
} catch (Throwable ignore) {
}
if (handle.getValue() == ERROR_REDIRECTED) {
final int redirect_max = 10;
int redirect_count = 0;
String lastRedirectUrl = null;
while (redirect_count++ < redirect_max) {
request = br.createRedirectFollowingRequest(request);
final String redirectUrl = request.getUrl();
if (lastRedirectUrl != null && redirectUrl.equals(lastRedirectUrl)) {
// some providers don't like fast redirects, as they use this for preparing final file. lets add short wait
// based on
// retry count
if (plugin != null && downloadLink != null) {
plugin.sleep(redirect_count * 250l, downloadLink);
} else {
Thread.sleep(redirect_count * 250l);
}
}
if (originalUrl != null) {
request.getHeaders().put("Referer", originalUrl);
}
dl = getDownloadInterface(downloadable, request, resume, chunks);
downloadable.setDownloadInterface(dl);
try {
dl.connect(br);
break;
} catch (PluginException handle2) {
try {
dl.getConnection().disconnect();
} catch (Throwable ignore) {
}
if (handle2.getValue() == ERROR_REDIRECTED) {
lastRedirectUrl = redirectUrl;
continue;
} else {
throw handle2;
}
}
}
if (redirect_count++ >= redirect_max) {
throw new PluginException(LinkStatus.ERROR_FATAL, "Redirectloop");
}
} else {
throw handle;
}
} catch (Exception forward) {
try {
dl.getConnection().disconnect();
} catch (Throwable ignore) {
}
throw forward;
}
return dl;
}
}
/**
*
* @param br
* @param downloadable
* @param request
* @param resume
* true|false, if chunks over 1 it must be true!
* @param chunks
* 0 = unlimited, chunks must start with negative sign otherwise it forces value to be used instead of up to value.
* @param forceRedirectWait
* forces thread sleep between redirects (even if they are not the same, useful for (multi)hosters which wait via series
* redirects)
* @return
* @throws Exception
*/
public static DownloadInterface openDownload(Browser br, Downloadable downloadable, Request request, boolean resume, int chunks, boolean forceRedirectWait) throws Exception {
String originalUrl = br.getURL();
DownloadInterface dl = getDownloadInterface(downloadable, request, resume, chunks);
downloadable.setDownloadInterface(dl);
final PluginForHost plugin;
final DownloadLink downloadLink;
if (downloadable instanceof DownloadLinkDownloadable) {
plugin = ((DownloadLinkDownloadable) downloadable).getPlugin();
downloadLink = ((DownloadLinkDownloadable) downloadable).getDownloadLink();
} else {
plugin = null;
downloadLink = null;
}
try {
dl.connect(br);
} catch (PluginException handle) {
try {
dl.getConnection().disconnect();
} catch (Throwable ignore) {
}
if (handle.getValue() == ERROR_REDIRECTED) {
final int redirect_max = 10;
int redirect_count = 0;
String lastRedirectUrl = null;
while (redirect_count++ < redirect_max) {
request = br.createRedirectFollowingRequest(request);
final String redirectUrl = request.getUrl();
if (lastRedirectUrl != null && (forceRedirectWait || redirectUrl.equals(lastRedirectUrl))) {
// some providers don't like fast redirects, as they use this for preparing final file. lets add short wait based on
// retry count
if (plugin != null && downloadLink != null) {
plugin.sleep(redirect_count * 250l, downloadLink);
} else {
Thread.sleep(redirect_count * 250l);
}
}
if (originalUrl != null) {
request.getHeaders().put("Referer", originalUrl);
}
dl = getDownloadInterface(downloadable, request, resume, chunks);
downloadable.setDownloadInterface(dl);
try {
dl.connect(br);
break;
} catch (PluginException handle2) {
try {
dl.getConnection().disconnect();
} catch (Throwable ignore) {
}
if (handle2.getValue() == ERROR_REDIRECTED) {
lastRedirectUrl = redirectUrl;
continue;
} else {
throw handle2;
}
}
}
if (redirect_count++ >= redirect_max) {
throw new PluginException(LinkStatus.ERROR_FATAL, "Redirectloop");
}
} else {
throw handle;
}
} catch (Exception forward) {
try {
dl.getConnection().disconnect();
} catch (Throwable ignore) {
}
throw forward;
}
return dl;
public static DownloadInterface openDownload(final Browser br, final DownloadLink downloadLink, final Request request, final boolean resume, final int chunks) throws Exception {
return openDownload(br, getDownloadable(downloadLink, br), request, resume, chunks);
}
public static DownloadInterface openDownload(Browser br, DownloadLink downloadLink, String url, String postdata, boolean resume, int chunks) throws Exception {
return openDownload(br, downloadLink, br.createPostRequest(url, postdata), resume, chunks);
}
public static DownloadInterface openDownload(Browser br, DownloadLink downloadLink, String link, boolean resume, int chunks) throws Exception {
return openDownload(br, downloadLink, br.createRequest(link), resume, chunks);
}
public static DownloadInterface openDownload(Browser br, DownloadLink downloadLink, Form form, boolean resume, int chunks) throws Exception {
return openDownload(br, downloadLink, br.createRequest(form), resume, chunks);
}
}

41
svn_trunk/src/jd/plugins/DownloadLink.java
View File

@ -24,9 +24,12 @@ import java.lang.reflect.Proxy;
import java.lang.reflect.Type;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.Map.Entry;
import java.util.concurrent.CopyOnWriteArrayList;
import jd.config.Property;
@ -1895,6 +1898,44 @@ public class DownloadLink extends Property implements Serializable, AbstractPack
setHashInfo(HashInfo.newInstanceSafe(sha1, HashInfo.TYPE.SHA1));
}
private final static String[] DEFAULT_DO_NOT_CHANGE_EXISTING_KEYS = new String[] { PROPERTY_COMMENT, PROPERTY_MIRRORID, PROPERTY_PASS, PROPERTY_PRIORITY, PROPERTY_LINKDUPEID, PROPERTY_SPEEDLIMIT, PROPERTY_ARCHIVE_ID, URL_ORIGIN, URL_REFERRER, URL_CONTAINER, URL_CONTENT, URL_CUSTOM, PROPERTY_JOB_ID };
public static String[] getDefaultDoNotChangeExistingKeys() {
return DEFAULT_DO_NOT_CHANGE_EXISTING_KEYS.clone();
}
@Override
public void setProperties(final Map<String, Object> properties) {
this.setProperties(properties, (String[]) null);
}
public void setProperties(final Map<String, Object> properties, String... doNotChangeExistingKeys) {
if (doNotChangeExistingKeys == null) {
doNotChangeExistingKeys = DEFAULT_DO_NOT_CHANGE_EXISTING_KEYS;
}
if (doNotChangeExistingKeys == null || doNotChangeExistingKeys.length == 0 || getPropertiesSize() == 0) {
super.setProperties(properties);
} else {
final Map<String, Object> newProperties = new HashMap<String, Object>();
for (String doNotChangeExistingKey : doNotChangeExistingKeys) {
final Object value = getProperty(doNotChangeExistingKey);
if (value != null) {
newProperties.put(doNotChangeExistingKey, value);
}
}
if (newProperties.size() == 0) {
super.setProperties(properties);
} else {
for (final Entry<String, Object> entry : properties.entrySet()) {
if (!newProperties.containsKey(entry.getKey())) {
newProperties.put(entry.getKey(), entry.getValue());
}
}
super.setProperties(newProperties);
}
}
}
/**
* @author raztoki
* @param sha256

14
svn_trunk/src/jd/plugins/WrongPasswordException.java
View File

@ -21,6 +21,7 @@ public class WrongPasswordException extends PluginException {
}
private Boolean clearExistingPassword = null;
private Boolean allowRetry = null;
public Boolean getClearExistingPassword() {
return clearExistingPassword;
@ -30,6 +31,19 @@ public class WrongPasswordException extends PluginException {
this.clearExistingPassword = clearExistingPassword;
}
/**
* Defines whether or not upper system is allowed to retry when this exception is thrown. </br>
* Typically if a plugin throws this exception and does not allow a retry this means that the plugin aleady asked the user multiple
* times for the password in-place.
*/
public Boolean getAllowRetry() {
return allowRetry;
}
public void setAllowRetry(Boolean allowRetry) {
this.allowRetry = allowRetry;
}
/**
* Throw this Exception whenever a previously entered download- or crawler password turned out to be wrong. </br>
* Examples: </br>

14
svn_trunk/src/jd/plugins/decrypter/BatoToCrawlerV3.java
View File

@ -21,6 +21,10 @@ import java.util.List;
import java.util.Map;
import java.util.regex.Pattern;
import org.appwork.storage.TypeRef;
import org.appwork.utils.StringUtils;
import org.jdownloader.plugins.controller.LazyPlugin;
import jd.PluginWrapper;
import jd.controlling.AccountController;
import jd.controlling.ProgressController;
@ -40,11 +44,7 @@ import jd.plugins.PluginForHost;
import jd.plugins.hoster.BatoTo;
import jd.plugins.hoster.DirectHTTP;
import org.appwork.storage.TypeRef;
import org.appwork.utils.StringUtils;
import org.jdownloader.plugins.controller.LazyPlugin;
@DecrypterPlugin(revision = "$Revision: 49242 $", interfaceVersion = 3, names = {}, urls = {})
@DecrypterPlugin(revision = "$Revision: 49640 $", interfaceVersion = 3, names = {}, urls = {})
@PluginDependencies(dependencies = { BatoTo.class })
/** This crawler is for bato.to website version v3. */
public class BatoToCrawlerV3 extends PluginForDecrypt {
@ -79,7 +79,7 @@ public class BatoToCrawlerV3 extends PluginForDecrypt {
public static String[] buildAnnotationUrls(final List<String[]> pluginDomains) {
final List<String> ret = new ArrayList<String>();
for (final String[] domains : pluginDomains) {
ret.add("https?://(?:www\\.)?" + buildHostsPatternPart(domains) + "/title/\\d+-[a-z0-9\\-]+(/\\d+-ch_\\d+)?");
ret.add("https?://(?:www\\.)?" + buildHostsPatternPart(domains) + "/title/\\d+-[a-z0-9\\-]+(/\\d+-(vol_\\d+-)?ch_\\d+)?");
}
return ret.toArray(new String[0]);
}
@ -153,7 +153,7 @@ public class BatoToCrawlerV3 extends PluginForDecrypt {
}
logger.info("Failed to find images of a single chapter");
/* No results were found so check if we got a series of which we want to find the URLs to all chapters */
final String[] chapterurls = br.getRegex("(" + Pattern.quote(urlpath) + "/\\d+-ch_\\d+)").getColumn(0);
final String[] chapterurls = br.getRegex("(" + Pattern.quote(urlpath) + "/\\d+-(vol_\\d+-)?ch_\\d+)").getColumn(0);
if (chapterurls == null || chapterurls.length == 0) {
throw new PluginException(LinkStatus.ERROR_PLUGIN_DEFECT);
}

23
svn_trunk/src/jd/plugins/decrypter/EroScriptsComCrawler.java
View File

@ -30,7 +30,7 @@ import jd.plugins.FilePackage;
import jd.plugins.PluginForDecrypt;
import jd.plugins.PluginForHost;
@DecrypterPlugin(revision = "$Revision: 48607 $", interfaceVersion = 3, names = { "discuss.eroscripts.com" }, urls = { "https?://discuss\\.eroscripts\\.com/t/([\\w\\-/]+)" })
@DecrypterPlugin(revision = "$Revision: 49631 $", interfaceVersion = 3, names = { "discuss.eroscripts.com" }, urls = { "https?://discuss\\.eroscripts\\.com/t/([\\w\\-/]+)" })
public class EroScriptsComCrawler extends antiDDoSForDecrypt {
@Override
public Browser createNewBrowserInstance() {
@ -217,12 +217,12 @@ public class EroScriptsComCrawler extends antiDDoSForDecrypt {
for (DownloadLink l : links) {
for (LazyCrawlerPlugin p : decrypters) {
boolean can = false;
if (p.canHandle(l.getDownloadURL())) {
if (p.canHandle(l.getPluginPatternMatcher())) {
can = true;
} else {
// Attempt fix for exmaple issue: spankbang.party -> spankbang.com
int ih = p.getDisplayName().lastIndexOf(".");
URL u = new URL(l.getDownloadURL());
URL u = new URL(l.getPluginPatternMatcher());
int il = u.getHost().lastIndexOf(".");
if (ih > 0 && il > 0) {
String hostBase = p.getDisplayName().substring(0, ih);
@ -230,9 +230,9 @@ public class EroScriptsComCrawler extends antiDDoSForDecrypt {
String hostPost = p.getDisplayName().substring(ih + 1);
String linkPost = u.getHost().substring(il + 1);
if (hostBase.equals(linkBase) && !hostPost.equals(linkPost)) {
String testUrl = l.getDownloadURL().replace(linkBase + "." + linkPost, hostBase + "." + hostPost);
String testUrl = l.getPluginPatternMatcher().replace(linkBase + "." + linkPost, hostBase + "." + hostPost);
if (p.canHandle(testUrl)) {
l.setUrlDownload(testUrl);
l.setPluginPatternMatcher(testUrl);
can = true;
}
}
@ -243,14 +243,14 @@ public class EroScriptsComCrawler extends antiDDoSForDecrypt {
}
try {
final PluginForDecrypt dec = getNewPluginInstance(p);
CrawledLink cl = new CrawledLink(new CryptedLink(l.getDownloadURL(), l.getDownloadURL()));
CrawledLink cl = new CrawledLink(new CryptedLink(l.getPluginPatternMatcher(), l.getPluginPatternMatcher()));
dec.setCurrentLink(cl);
List<DownloadLink> result = dec.decryptIt(cl);
if (result.size() > 1) {
l.setEnabled(false);
} else if (result.size() == 1) {
DownloadLink dl = result.get(0).getDownloadLink();
l.setUrlDownload(dl.getDownloadURL());
l.setPluginPatternMatcher(dl.getPluginPatternMatcher());
l.setContentUrl(dl.getContentUrl());
l.setLinkID(dl.getLinkID());
l.setProperties(dl.getProperties());
@ -294,7 +294,7 @@ public class EroScriptsComCrawler extends antiDDoSForDecrypt {
if (l.getName() == null || l.getName().length() == 0) {
continue;
}
String ext = Files.getExtension(l.getName());
String ext = Files.getExtension(l.getName(), true);
if (ext == null) {
continue;
}
@ -330,7 +330,7 @@ public class EroScriptsComCrawler extends antiDDoSForDecrypt {
if (match.contains("pixiv.net")) {
continue;
}
String ext = Files.getExtension(match);
String ext = Files.getExtension(match, true);
boolean useImages = getPluginConfig().getBooleanProperty(jd.plugins.hoster.EroScriptsCom.FETCH_IMAGES, true);
if (!useImages) {
if (ext != null && (ext.equalsIgnoreCase("jpg") || ext.equalsIgnoreCase("jpeg") || ext.equalsIgnoreCase("png") || ext.equalsIgnoreCase("gif") || ext.equalsIgnoreCase("bmp"))) {
@ -342,7 +342,7 @@ public class EroScriptsComCrawler extends antiDDoSForDecrypt {
}
boolean dupe = false;
for (DownloadLink l : decryptedLinks) {
if (l.getDownloadURL().equals(match)) {
if (l.getPluginPatternMatcher().equals(match)) {
dupe = true;
break;
}
@ -389,7 +389,8 @@ public class EroScriptsComCrawler extends antiDDoSForDecrypt {
return ret;
}
String title = Encoding.htmlDecode(titleMatches[0][0]);
if (getPluginConfig().getBooleanProperty(jd.plugins.hoster.EroScriptsCom.SMART_FILENAMES, true)) {
// SMART_FILENAMES: disabled since it isn't working yet
if (getPluginConfig().getBooleanProperty(jd.plugins.hoster.EroScriptsCom.SMART_FILENAMES, false)) {
// runSmartFileNames(title, decryptedLinks);
SmartGroup sg = new SmartGroup(this, title, ret);
} else {

97
svn_trunk/src/jd/plugins/decrypter/InfidriveNetCrawler.java
View File

@ -16,11 +16,21 @@
package jd.plugins.decrypter;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.regex.Pattern;
import org.appwork.storage.TypeRef;
import org.appwork.utils.StringUtils;
import org.appwork.utils.formatter.SizeFormatter;
import org.jdownloader.captcha.v2.challenge.recaptcha.v2.CaptchaHelperCrawlerPluginRecaptchaV2;
import jd.PluginWrapper;
import jd.controlling.ProgressController;
import jd.http.Browser;
import jd.http.requests.PostRequest;
import jd.nutils.encoding.Encoding;
import jd.parser.Regex;
import jd.plugins.CryptedLink;
import jd.plugins.DecrypterPlugin;
@ -29,8 +39,9 @@ import jd.plugins.LinkStatus;
import jd.plugins.PluginException;
import jd.plugins.PluginForDecrypt;
import jd.plugins.components.PluginJSonUtils;
import jd.plugins.hoster.DirectHTTP;
@DecrypterPlugin(revision = "$Revision: 48446 $", interfaceVersion = 3, names = {}, urls = {})
@DecrypterPlugin(revision = "$Revision: 49642 $", interfaceVersion = 3, names = {}, urls = {})
public class InfidriveNetCrawler extends PluginForDecrypt {
public InfidriveNetCrawler(PluginWrapper wrapper) {
super(wrapper);
@ -66,27 +77,97 @@ public class InfidriveNetCrawler extends PluginForDecrypt {
public static String[] buildAnnotationUrls(final List<String[]> pluginDomains) {
final List<String> ret = new ArrayList<String>();
for (final String[] domains : pluginDomains) {
ret.add("https?://(?:www\\.)?" + buildHostsPatternPart(domains) + "/file/\\d+");
ret.add("https?://(?:www\\.)?" + buildHostsPatternPart(domains) + "/file/(\\d+)");
}
return ret.toArray(new String[0]);
}
public ArrayList<DownloadLink> decryptIt(final CryptedLink param, ProgressController progress) throws Exception {
final ArrayList<DownloadLink> ret = new ArrayList<DownloadLink>();
br.getPage(param.getCryptedUrl());
final String contenturl = param.getCryptedUrl().replaceFirst("(?i)http://", "https://");
final String contentID = new Regex(contenturl, this.getSupportedLinks()).getMatch(0);
br.getPage(contenturl);
if (br.getHttpConnection().getResponseCode() == 404) {
throw new PluginException(LinkStatus.ERROR_FILE_NOT_FOUND);
} else if (br.containsHTML("File Not Found\\s*</h1>")) {
throw new PluginException(LinkStatus.ERROR_FILE_NOT_FOUND);
}
String filename = br.getRegex("class=\"text-xl font-semibold\"[^>]*>([^<]+)").getMatch(0);
if (filename == null) {
filename = br.getRegex("<title>Download ([^<]+) \\| InfiDrive</title>").getMatch(0);
}
if (filename != null) {
filename = Encoding.htmlDecode(filename).trim();
}
final String filesizeStr = br.getRegex("Size:\\s*<b>([^<]+)</b>").getMatch(0);
final ArrayList<DownloadLink> ret = new ArrayList<DownloadLink>();
final String unescapedhtml = PluginJSonUtils.unescape(br.getRequest().getHtmlCode());
final String[] fileids = new Regex(unescapedhtml, "file_id\":\"([^\"]+)").getColumn(0);
if (fileids == null || fileids.length == 0) {
throw new PluginException(LinkStatus.ERROR_PLUGIN_DEFECT);
if (fileids != null && fileids.length > 0) {
/* Old way which worked when they were still working with public Google Drive fileIDs. */
for (final String fileid : fileids) {
ret.add(createDownloadlink(GoogleDriveCrawler.generateFileURL(fileid, null)));
}
}
for (final String fileid : fileids) {
ret.add(createDownloadlink(GoogleDriveCrawler.generateFileURL(fileid, null)));
final String htmlWithUnescapedJson = PluginJSonUtils.unescape(br.getRequest().getHtmlCode());
final Pattern directurlpattern = Pattern.compile("\"url\":\"(https?://[^\"]+)", Pattern.CASE_INSENSITIVE);
String directurl = new Regex(htmlWithUnescapedJson, directurlpattern).getMatch(0);
/* Maybe captcha required */
if (directurl == null) {
final boolean allowCaptchaHandling = true;
if (!allowCaptchaHandling) {
throw new PluginException(LinkStatus.ERROR_PLUGIN_DEFECT);
}
final String userEmail = PluginJSonUtils.getJson(htmlWithUnescapedJson, "userEmail");
if (StringUtils.isEmpty(userEmail)) {
throw new PluginException(LinkStatus.ERROR_PLUGIN_DEFECT);
}
final String rcKey = "6Ld5isIbAAAAAK2KVX86ymkA8NDDWSGtyGIfyz8H"; // 2024-08-22
final String recaptchaV2Response = new CaptchaHelperCrawlerPluginRecaptchaV2(this, br, rcKey).getToken();
final PostRequest request = br.createJSonPostRequest(br.getURL(), "[\"" + PluginJSonUtils.escape(recaptchaV2Response) + "\"]");
br.getHeaders().put("Accept", "text/x-component");
br.getHeaders().put("Origin", "https://" + br.getHost(true));
request.setContentType("text/plain;charset=UTF-8");
br.getPage(request);
final String htmlWithUnescapedJson2 = PluginJSonUtils.unescape(br.getRequest().getHtmlCode());
String googleDriveFileID = br.getRegex("0:\\[\"[^\"]+\",\\[\"([^\"]+)").getMatch(0);
if (googleDriveFileID == null) {
googleDriveFileID = PluginJSonUtils.getJson(htmlWithUnescapedJson2, "file_id");
}
directurl = new Regex(htmlWithUnescapedJson2, directurlpattern).getMatch(0);
if (StringUtils.isEmpty(googleDriveFileID)) {
throw new PluginException(LinkStatus.ERROR_PLUGIN_DEFECT);
} else if (this.isAbort()) {
throw new InterruptedException();
}
final boolean additionalRequestNeeded = true;
if (additionalRequestNeeded) {
final DownloadLink gdrivelink = createDownloadlink(GoogleDriveCrawler.generateFileURL(googleDriveFileID, null));
ret.add(gdrivelink);
distribute(gdrivelink);
final Map<String, Object> postdata = new HashMap<String, Object>();
postdata.put("filename", filename);
postdata.put("id", googleDriveFileID);
postdata.put("owner_email", userEmail);
postdata.put("referer", "https://www.google.com/");
postdata.put("slug", contentID);
postdata.put("type", "direct");
/* Can be null */
postdata.put("url", directurl);
final PostRequest request2 = br.createJSonPostRequest("/api/download", postdata);
br.getPage(request2);
final Map<String, Object> entries = restoreFromString(br.getRequest().getHtmlCode(), TypeRef.MAP);
directurl = entries.get("dl").toString();
}
}
final DownloadLink direct = this.createDownloadlink(DirectHTTP.createURLForThisPlugin(directurl));
if (filename != null) {
direct.setName(filename);
}
if (filesizeStr != null) {
direct.setDownloadSize(SizeFormatter.getSize(filesizeStr));
}
direct.setAvailable(true);
ret.add(direct);
return ret;
}
}

40
svn_trunk/src/jd/plugins/decrypter/LibGenCrawler.java
View File

@ -42,7 +42,7 @@ import jd.plugins.PluginForDecrypt;
import jd.plugins.hoster.DirectHTTP;
import jd.plugins.hoster.LibGenInfo;
@DecrypterPlugin(revision = "$Revision: 49213 $", interfaceVersion = 2, names = {}, urls = {})
@DecrypterPlugin(revision = "$Revision: 49642 $", interfaceVersion = 2, names = {}, urls = {})
public class LibGenCrawler extends PluginForDecrypt {
public LibGenCrawler(PluginWrapper wrapper) {
super(wrapper);
@ -110,7 +110,8 @@ public class LibGenCrawler extends PluginForDecrypt {
}
String md5 = null;
try {
md5 = UrlQuery.parse(url).get("md5");
final UrlQuery query = UrlQuery.parse(url);
md5 = query.get("md5");
} catch (MalformedURLException e) {
e.printStackTrace();
}
@ -208,10 +209,26 @@ public class LibGenCrawler extends PluginForDecrypt {
final Map<String, Object> bookmap = (Map<String, Object>) entries.get(fileID);
ret.addAll(processBookJson(bookmap, domainToUse));
} else {
final String md5 = findMd5hashInURL(contenturl);
/* Other type of link. It could be anything so we need to take a deeper look. */
String md5 = findMd5hashInURL(contenturl);
boolean accessedContenturl = false;
if (md5 == null) {
/* Unsupported link */
throw new PluginException(LinkStatus.ERROR_FILE_NOT_FOUND);
logger.info("Maybe unsupported link -> Open it and check if we find an md5 value somewhere in html code.");
br.getPage(contenturl);
accessedContenturl = true;
if (br.getHttpConnection().getResponseCode() == 404) {
throw new PluginException(LinkStatus.ERROR_FILE_NOT_FOUND);
}
final String[] urls = HTMLParser.getHttpLinks(br.getRequest().getHtmlCode(), br.getURL());
for (final String url : urls) {
md5 = new Regex(url, "md5=([a-f0-9]{32})").getMatch(0);
if (md5 != null) {
break;
}
}
if (md5 == null) {
throw new PluginException(LinkStatus.ERROR_FILE_NOT_FOUND, "Invalid/unsupported URL");
}
}
final String orgContentURL = contenturl;
contenturl = this.generateSingleFileDownloadurl(domainToUse, md5);
@ -220,6 +237,7 @@ public class LibGenCrawler extends PluginForDecrypt {
book.setDefaultPlugin(this.hostPlugin);
if (bookMD5Regex.patternFind()) {
br.getPage(orgContentURL);
accessedContenturl = true;
final String bookID = br.getRegex(">\\s*ID:\\s*</font>\\s*</nobr>\\s*</td>\\s*<td>\\s*(\\d+)\\s*<").getMatch(0);
if (bookID != null) {
book.setProperty(LibGenInfo.PROPERTY_BOOK_ID, bookID);
@ -247,11 +265,13 @@ public class LibGenCrawler extends PluginForDecrypt {
} else {
/* Website mode */
/* TODO: Decide if this is still needed. If so: Add settings for API/mirrors/cover_url */
br.getPage(contenturl);
if (br.getHttpConnection().getResponseCode() == 404) {
throw new PluginException(LinkStatus.ERROR_FILE_NOT_FOUND);
} else if (br.containsHTML("(?i)entry not found in the database")) {
throw new PluginException(LinkStatus.ERROR_FILE_NOT_FOUND);
if (!accessedContenturl) {
br.getPage(contenturl);
if (br.getHttpConnection().getResponseCode() == 404) {
throw new PluginException(LinkStatus.ERROR_FILE_NOT_FOUND);
} else if (br.containsHTML("(?i)entry not found in the database")) {
throw new PluginException(LinkStatus.ERROR_FILE_NOT_FOUND);
}
}
String title = br.getRegex(md5 + "[^>]*>([^<>\"]+)<").getMatch(0);
if (title != null) {

42
svn_trunk/src/jd/plugins/decrypter/LiveMeCom.java
View File

@ -3,33 +3,44 @@ package jd.plugins.decrypter;
import java.util.ArrayList;
import java.util.Map;
import jd.PluginWrapper;
import jd.controlling.ProgressController;
import jd.plugins.CryptedLink;
import jd.plugins.DecrypterPlugin;
import jd.plugins.DownloadLink;
import jd.plugins.PluginForDecrypt;
import org.appwork.storage.JSonStorage;
import org.appwork.storage.TypeRef;
import org.appwork.utils.Regex;
import org.appwork.utils.StringUtils;
import org.jdownloader.scripting.JavaScriptEngineFactory;
@DecrypterPlugin(revision = "$Revision: 48194 $", interfaceVersion = 2, names = { "liveme.com" }, urls = { "https?://(?:www\\.)?liveme\\.com/(?:media/play/\\?videoid=\\d+|media/liveshort/dist/\\?videoid=\\d+&.*?|live\\.html\\?videoid=\\d+.*?|.*?/\\d+/index.html)" })
import jd.PluginWrapper;
import jd.controlling.ProgressController;
import jd.http.Browser;
import jd.plugins.CryptedLink;
import jd.plugins.DecrypterPlugin;
import jd.plugins.DownloadLink;
import jd.plugins.LinkStatus;
import jd.plugins.PluginException;
import jd.plugins.PluginForDecrypt;
@DecrypterPlugin(revision = "$Revision: 49642 $", interfaceVersion = 2, names = { "liveme.com" }, urls = { "https?://(?:www\\.)?liveme\\.com/(?:media/play/\\?videoid=\\d+|media/liveshort/dist/\\?videoid=\\d+&.*?|live\\.html\\?videoid=\\d+.*?|.*?/\\d+/index.html)" })
public class LiveMeCom extends PluginForDecrypt {
public LiveMeCom(PluginWrapper wrapper) {
super(wrapper);
}
@Override
public Browser createNewBrowserInstance() {
final Browser br = super.createNewBrowserInstance();
br.setFollowRedirects(true);
return br;
}
@Override
public ArrayList<DownloadLink> decryptIt(final CryptedLink param, final ProgressController progress) throws Exception {
final ArrayList<DownloadLink> ret = new ArrayList<DownloadLink>();
final String parameter = param.toString();
final String parameter = param.getCryptedUrl();
final String videoid = getVideoID(parameter);
br.setFollowRedirects(true);
final String vali = vali(4) + "l" + vali(4) + "m" + vali(5);
br.postPage("https://live.ksmobile.net/live/queryinfo", "userid=1&videoid=" + videoid + "&area=&h5=1&vali=" + vali);
if (br.getHttpConnection().getResponseCode() == 404) {
throw new PluginException(LinkStatus.ERROR_FILE_NOT_FOUND);
}
final Map<String, Object> response = restoreFromString(br.toString(), TypeRef.MAP);
final Map<String, Object> data = (Map<String, Object>) response.get("data");
final Map<String, Object> video_info = (Map<String, Object>) data.get("video_info");
@ -44,8 +55,7 @@ public class LiveMeCom extends PluginForDecrypt {
} else if (StringUtils.isNotEmpty(url)) {
link = createDownloadlink(url);
} else {
ret.add(createOfflinelink(parameter));
return ret;
throw new PluginException(LinkStatus.ERROR_FILE_NOT_FOUND);
}
if (videosize != null) {
link.setDownloadSize(JavaScriptEngineFactory.toLong(videosize, -1));
@ -78,10 +88,10 @@ public class LiveMeCom extends PluginForDecrypt {
return A;
}
private String getVideoID(String parameter) {
String result = new Regex(parameter, "[&?]videoid=(\\d+)").getMatch(0);
private String getVideoID(final String url) {
String result = new Regex(url, "(?i)[&?]videoid=(\\d+)").getMatch(0);
if (result == null) {
result = new Regex(parameter, "/(\\d+)/index\\.html").getMatch(0);
result = new Regex(url, "(?i)/(\\d+)/index\\.html").getMatch(0);
}
return result;
}

53
svn_trunk/src/jd/plugins/decrypter/TbCmV2.java
View File

@ -102,10 +102,10 @@ import jd.plugins.PluginException;
import jd.plugins.PluginForDecrypt;
import jd.plugins.components.UserAgents;
import jd.plugins.components.UserAgents.BrowserName;
import jd.plugins.hoster.DirectHTTP;
import jd.plugins.hoster.YoutubeDashV2;
import jd.utils.locale.JDL;
@DecrypterPlugin(revision = "$Revision: 49592 $", interfaceVersion = 3, names = {}, urls = {})
@DecrypterPlugin(revision = "$Revision: 49643 $", interfaceVersion = 3, names = {}, urls = {})
public class TbCmV2 extends PluginForDecrypt {
/* Shorted wait time between requests when JDownloader is run in IDE to allow for faster debugging. */
private static final int DDOS_WAIT_MAX = Application.isJared(null) ? 1000 : 10;
@ -301,13 +301,6 @@ public class TbCmV2 extends PluginForDecrypt {
logger.info("Returning nothing because: Android support is disabled");
return new ArrayList<DownloadLink>();
}
final ArrayList<DownloadLink> ret = new ArrayList<DownloadLink>() {
@Override
public boolean add(DownloadLink e) {
distribute(e);
return super.add(e);
}
};
// TODO: Maybe remove this as we're not modifying this URL anymore and also all methods to extract information out of YT URLs work
// domain-independent.
String cleanedurl = param.getCryptedUrl();
@ -359,6 +352,13 @@ public class TbCmV2 extends PluginForDecrypt {
final String playlistHandlingLogtextForUserDisabledCrawlerByLimitSetting = "Doing nothing because user has disabled channel/playlist crawler by setting limit to 0";
String playlistHandlingHumanReadableTypeOfUrlToCrawl = null;
String playlistHandlingHumanReadableTitle = null;
final ArrayList<DownloadLink> ret = new ArrayList<DownloadLink>() {
@Override
public boolean add(DownloadLink e) {
distribute(e);
return super.add(e);
}
};
if (StringUtils.isEmpty(playlistID) && StringUtils.isEmpty(userName) && !StringUtils.isEmpty(videoID)) {
/* Single video */
videoIdsToAdd.add(new org.jdownloader.plugins.components.youtube.YoutubeClipData(videoID));
@ -418,7 +418,7 @@ public class TbCmV2 extends PluginForDecrypt {
messageDialogText += "\r\nJDownloader can only crawl the first " + maxItemsPerPage + " items automatically.\r\nIf there are more than " + maxItemsPerPage + " items, you need to use external tools to grab the single URLs to all videos and add those to JD manually.";
}
messageDialogText += "\r\nIf you wish to hide this dialog, you can pre-select your preferred option under Settings -> Plugins -> youtube.com.";
final ConfirmDialog confirm = new ConfirmDialog(UIOManager.LOGIC_COUNTDOWN, playlistHandlingHumanReadableTitle, JDL.L("plugins.host.youtube.isplaylist.question.message", messageDialogText), null, JDL.L("plugins.host.youtube.isplaylist.question.onlyplaylist", buttonTextCrawlPlaylistOrProfile), JDL.L("plugins.host.youtube.isvideoandplaylist.question.nothing", "Do nothing?")) {
final ConfirmDialog confirm = new ConfirmDialog(UIOManager.LOGIC_COUNTDOWN, playlistHandlingHumanReadableTitle, messageDialogText, null, buttonTextCrawlPlaylistOrProfile, "Do nothing?") {
@Override
public ModalityType getModalityType() {
return ModalityType.MODELESS;
@ -537,6 +537,7 @@ public class TbCmV2 extends PluginForDecrypt {
this.channelID = channelMetadataRenderer.get("externalId").toString();
}
if (cfg.getProfileCrawlMode() == ProfileCrawlMode.PLAYLIST && StringUtils.isEmpty(playlistID)) {
/* Crawl profile as playlist -> Look for playlistID of default profile playlist "Uploads b <username>". */
if (!StringUtils.isEmpty(userName)) {
/*
* the user channel parser only parses 1050 videos. this workaround finds the user channel playlist and parses this
@ -624,7 +625,8 @@ public class TbCmV2 extends PluginForDecrypt {
if (playlistTitle != null) {
channelOrPlaylistPackage.setName(playlistTitle);
} else {
final String internalContainerURL = helper.getChannelPlaylistCrawlerContainerUrlOverride(param.getCryptedUrl());
// final String internalContainerURL =
// helper.getChannelPlaylistCrawlerContainerUrlOverride(param.getCryptedUrl());
String packagename;
if (channelName != null) {
packagename = channelName;
@ -665,6 +667,28 @@ public class TbCmV2 extends PluginForDecrypt {
videoIdsToAdd.addAll(parseVideoIds(video_ids_comma_separated));
}
}
final List<YoutubeStreamData> playlistThumbnails = helper.getPlaylistThumbnails();
// TODO: Add setting, see: https://svn.jdownloader.org/issues/90496
boolean crawlPlaylistThumbnails = true;
if (crawlPlaylistThumbnails && playlistThumbnails != null) {
// TODO: Add to list of download-results
if (DebugMode.TRUE_IN_IDE_ELSE_FALSE) {
for (final YoutubeStreamData thumbinfo : playlistThumbnails) {
final DownloadLink thumb = this.createDownloadlink(DirectHTTP.createURLForThisPlugin(thumbinfo.getUrl()));
if (thumbinfo.getEstimatedContentLength() > 0) {
thumb.setDownloadSize(thumbinfo.estimatedContentLength());
}
thumb.setAvailable(true);
if (channelOrPlaylistPackage != null) {
thumb._setFilePackage(channelOrPlaylistPackage);
}
// TODO: Add customizable filenames
ret.add(thumb);
distribute(thumb);
break;
}
}
}
Integer indexFromAddedURL = null;
final String indexFromAddedURLStr = new Regex(cleanedurl, "(?i)index=(\\d+)").getMatch(0);
if (indexFromAddedURLStr != null) {
@ -991,8 +1015,8 @@ public class TbCmV2 extends PluginForDecrypt {
}
last = cur;
}
for (VariantInfo vi : linkVariants) {
ArrayList<VariantInfo> lst = new ArrayList<VariantInfo>();
for (final VariantInfo vi : linkVariants) {
final ArrayList<VariantInfo> lst = new ArrayList<VariantInfo>();
lst.add(vi);
final DownloadLink lnk = createLink(new YoutubeVariantCollection(), vi, lst, channelOrPlaylistPackage, singleVideoPackageNamePatternOverride);
ret.add(lnk);
@ -1174,6 +1198,7 @@ public class TbCmV2 extends PluginForDecrypt {
/* Developer mistake */
throw new IllegalArgumentException();
}
helper.setPlaylistID(playlistID);
if (helper.getAccountLoggedIn() == null) {
/*
* Only set User-Agent if we're not logged in because login session can be bound to User-Agent and tinkering around with
@ -1217,7 +1242,6 @@ public class TbCmV2 extends PluginForDecrypt {
List<Map<String, Object>> alerts = null;
String errorOrWarningMessage = null;
URL originalURL = null;
final ArrayList<YoutubeClipData> ret = new ArrayList<YoutubeClipData>();
Map<String, Object> ytConfigData = null;
final Set<String> playListDupes = new HashSet<String>();
Integer totalNumberofItems = null;
@ -1449,6 +1473,7 @@ public class TbCmV2 extends PluginForDecrypt {
logger.info("Channel/playlist URL used differs from URL that was initially added: Original: " + originalURL.toString() + " | Actually used: " + br.getURL());
helper.setChannelPlaylistCrawlerContainerUrlOverride(br.getURL());
}
final ArrayList<YoutubeClipData> ret = new ArrayList<YoutubeClipData>();
humanReadableTitle += " sorted by " + activeSort;
int videoPositionCounter = 0;
int round = 0;

3
svn_trunk/src/jd/plugins/download/RAFDownload.java
View File

@ -32,7 +32,4 @@ public class RAFDownload extends OldRAFDownload {
super(BrowserAdapter.getDownloadable(downloadLink, null), request);
}
@Deprecated
public void addChunksDownloading(int i) {
}
}

11
svn_trunk/src/jd/plugins/download/raf/RAFChunk.java
View File

@ -669,7 +669,16 @@ public class RAFChunk extends Thread {
}
endByte = ContentRange[1];
}
if (endByte <= 0) {
if (connection.isContentDecoded()) {
final String encoding = connection.getHeaderField("Content-Encoding");
if (startByte == 0) {
logger.severe("reset endByte(-1) because connection is content decoded:" + encoding);
endByte = -1;
} else {
dl.error(new PluginException(LinkStatus.ERROR_PLUGIN_DEFECT, "unsupported resume with content-encoding:" + encoding));
return;
}
} else if (endByte <= 0) {
/* endByte not yet set!, use Content-Length */
if (contentLength > 0) {
final long end = contentLength - 1;

20
svn_trunk/src/jd/plugins/hoster/EroScriptsCom.java
View File

@ -2,6 +2,9 @@ package jd.plugins.hoster;
import java.util.Map;
import org.appwork.storage.TypeRef;
import org.jdownloader.plugins.components.antiDDoSForHost;
import jd.PluginWrapper;
import jd.config.ConfigContainer;
import jd.config.ConfigEntry;
@ -14,16 +17,14 @@ import jd.plugins.Account;
import jd.plugins.Account.AccountType;
import jd.plugins.AccountInfo;
import jd.plugins.AccountRequiredException;
import jd.plugins.BrowserAdapter;
import jd.plugins.DownloadLink;
import jd.plugins.DownloadLink.AvailableStatus;
import jd.plugins.HostPlugin;
import jd.plugins.LinkStatus;
import jd.plugins.PluginException;
import org.appwork.storage.TypeRef;
import org.jdownloader.plugins.components.antiDDoSForHost;
@HostPlugin(revision = "$Revision: 49275 $", interfaceVersion = 3, names = { "discuss.eroscripts.com" }, urls = { "https?://discuss\\.eroscripts\\.com/uploads/([\\w\\-/]+)" })
@HostPlugin(revision = "$Revision: 49631 $", interfaceVersion = 3, names = { "discuss.eroscripts.com" }, urls = { "https?://discuss\\.eroscripts\\.com/uploads/([\\w\\-/]+)" })
public class EroScriptsCom extends antiDDoSForHost {
private static final String COOKIE_ID = "_forum_session";
public static final String FETCH_IMAGES = "FETCH_IMAGES";
@ -42,7 +43,6 @@ public class EroScriptsCom extends antiDDoSForHost {
protected void setConfigElements() {
getConfig().addEntry(new ConfigEntry(ConfigContainer.TYPE_CHECKBOX, getPluginConfig(), FETCH_IMAGES, "Add images?").setDefaultValue(true));
getConfig().addEntry(new ConfigEntry(ConfigContainer.TYPE_CHECKBOX, getPluginConfig(), SMART_FILENAMES, "Attempt to link script and video filenames?").setDefaultValue(true));
}
public void login(final Browser br, final Account account, boolean validateCookies) throws Exception {
@ -73,14 +73,14 @@ public class EroScriptsCom extends antiDDoSForHost {
Browser csrfBr = br.cloneBrowser();
csrfBr.setHeader("accept", "application/json");
csrfBr.getPage("https://discuss.eroscripts.com/session/csrf");
final Map<String, String> apiResponse = restoreFromString(csrfBr.toString(), TypeRef.HASHMAP_STRING);
final Map<String, Object> apiResponse = restoreFromString(csrfBr.toString(), TypeRef.MAP);
form.put("login", account.getUser());
form.put("password", account.getPass());
form.put("second_factor_method", "1");
form.setAction("https://discuss.eroscripts.com/session");
form.setMethod(MethodType.POST);
Browser formBr = br.cloneBrowser();
formBr.setHeader("x-csrf-token", apiResponse.get("csrf"));
formBr.setHeader("x-csrf-token", String.valueOf(apiResponse.get("csrf")));
formBr.setHeader("x-requested-with", "XMLHttpRequest");
formBr.submitForm(form);
Map<String, Object> loginResponse = restoreFromString(formBr.toString(), TypeRef.MAP);
@ -127,9 +127,7 @@ public class EroScriptsCom extends antiDDoSForHost {
}
// TODO: check if browser instance has cookies set (called by decrypter plugin) so no additional login is required
login(br, account, false);
final Browser brc = br.cloneBrowser();
basicLinkCheck(brc.cloneBrowser(), brc.createGetRequest(link.getPluginPatternMatcher()), link, null, null);
link.setPluginPatternMatcher(brc.getURL());
basicLinkCheck(br.cloneBrowser(), br.createGetRequest(link.getPluginPatternMatcher()), link, null, null);
return AvailableStatus.TRUE;
}
@ -137,7 +135,7 @@ public class EroScriptsCom extends antiDDoSForHost {
public void handleFree(final DownloadLink link) throws Exception {
requestFileInformation(link);
// no account required?
dl = new jd.plugins.BrowserAdapter().openDownload(br, link, link.getPluginPatternMatcher());
dl = BrowserAdapter.openDownload(br, link, link.getPluginPatternMatcher());
handleConnectionErrors(br, dl.getConnection());
dl.startDownload();
}

4
svn_trunk/src/jd/plugins/hoster/FileUploadDotnet.java
View File

@ -33,7 +33,7 @@ import org.appwork.utils.formatter.SizeFormatter;
import org.jdownloader.captcha.v2.challenge.recaptcha.v2.CaptchaHelperHostPluginRecaptchaV2;
import org.jdownloader.plugins.components.antiDDoSForHost;
@HostPlugin(revision = "$Revision: 47477 $", interfaceVersion = 2, names = { "file-upload.net" }, urls = { "https?://(www\\.|en\\.)?file\\-upload\\.net/((member/){0,1}download\\-\\d+/(.*?)\\.html|view\\-\\d+/(.*?)\\.html|member/view_\\d+_(.*?)\\.html|member/data3\\.php\\?user=(.*?)\\&name=(.*))" })
@HostPlugin(revision = "$Revision: 49635 $", interfaceVersion = 2, names = { "file-upload.net" }, urls = { "https?://(www\\.|en\\.)?file\\-upload\\.net/((member/){0,1}download\\-\\d+/(.*?)\\.html|view\\-\\d+/(.*?)\\.html|member/view_\\d+_(.*?)\\.html|member/data3\\.php\\?user=(.*?)\\&name=(.*))" })
public class FileUploadDotnet extends antiDDoSForHost {
private final Pattern PAT_Download = Pattern.compile("https?://[\\w\\.]*?file-upload\\.net/(member/){0,1}download-\\d+/(.*?).html", Pattern.CASE_INSENSITIVE);
private final Pattern PAT_VIEW = Pattern.compile("https?://[\\w\\.]*?file-upload\\.net/(view-\\d+/(.*?).html|member/view_\\d+_(.*?).html)", Pattern.CASE_INSENSITIVE);
@ -134,7 +134,7 @@ public class FileUploadDotnet extends antiDDoSForHost {
}.getToken();
download.put("g-recaptcha-response", Encoding.urlEncode(recaptchaV2Response));
}
dl = jd.plugins.BrowserAdapter.openDownload(br, link, download);
dl = jd.plugins.BrowserAdapter.openDownload(br, link, download, false, 1);
} else if (new Regex(link.getDownloadURL(), PAT_VIEW).matches()) {
/* DownloadFiles */
String downloadurl = br.getRegex("<center>\n<a href=\"(.*?)\" rel=\"lightbox\"").getMatch(0);

29
svn_trunk/src/jd/plugins/hoster/ORFMediathek.java
View File

@ -23,11 +23,18 @@ import java.util.Date;
import java.util.List;
import java.util.TimeZone;
import org.appwork.utils.StringUtils;
import org.jdownloader.downloader.hds.HDSDownloader;
import org.jdownloader.downloader.hls.HLSDownloader;
import org.jdownloader.plugins.components.hds.HDSContainer;
import org.jdownloader.plugins.components.hls.HlsContainer;
import jd.PluginWrapper;
import jd.config.ConfigContainer;
import jd.config.ConfigEntry;
import jd.http.Browser;
import jd.http.URLConnectionAdapter;
import jd.http.requests.GetRequest;
import jd.nutils.encoding.Encoding;
import jd.parser.Regex;
import jd.plugins.Account;
@ -41,13 +48,7 @@ import jd.plugins.PluginException;
import jd.plugins.PluginForHost;
import jd.plugins.decrypter.OrfAt;
import org.appwork.utils.StringUtils;
import org.jdownloader.downloader.hds.HDSDownloader;
import org.jdownloader.downloader.hls.HLSDownloader;
import org.jdownloader.plugins.components.hds.HDSContainer;
import org.jdownloader.plugins.components.hls.HlsContainer;
@HostPlugin(revision = "$Revision: 49027 $", interfaceVersion = 3, names = { "orf.at" }, urls = { "" })
@HostPlugin(revision = "$Revision: 49633 $", interfaceVersion = 3, names = { "orf.at" }, urls = { "" })
public class ORFMediathek extends PluginForHost {
private static final String TYPE_AUDIO = "(?i)https?://ooe\\.orf\\.at/radio/stories/(\\d+)/";
/* Variables related to plugin settings */
@ -187,13 +188,19 @@ public class ORFMediathek extends PluginForHost {
throw new PluginException(LinkStatus.ERROR_PLUGIN_DEFECT);
}
try {
con = br.openGetConnection(dllink);
GetRequest request = br.createGetRequest(dllink);
request.getHeaders().put("Accept-Encoding", "identity");
con = br.openRequestConnection(request);
this.handleConnectionErrors(br, link, con);
if (!looksLikeDownloadableContent(con, link)) {
throw new PluginException(LinkStatus.ERROR_FILE_NOT_FOUND);
}
if (con.getCompleteContentLength() > 0) {
link.setVerifiedFileSize(con.getCompleteContentLength());
if (con.isContentDecoded()) {
link.setDownloadSize(con.getCompleteContentLength());
} else {
link.setVerifiedFileSize(con.getCompleteContentLength());
}
}
link.setProperty(PROPERTY_DIRECTURL, dllink);
return AvailableStatus.TRUE;
@ -269,7 +276,8 @@ public class ORFMediathek extends PluginForHost {
if (isAgeRestricted(url)) {
if (System.currentTimeMillis() - link.getLongProperty(PROPERTY_AGE_RESTRICTED_LAST_RECRAWL_TIMESTAMP, 0) < 30 * 60 * 1000) {
/**
* Recrawl has just happened and we were still unable to download the item :( </br> This should never happen!
* Recrawl has just happened and we were still unable to download the item :( </br>
* This should never happen!
*/
throw new PluginException(LinkStatus.ERROR_TEMPORARILY_UNAVAILABLE, "Jugendschutz-Recrawl fehlgeschlagen Grund 1", 10 * 60 * 1000l);
}
@ -407,6 +415,7 @@ public class ORFMediathek extends PluginForHost {
/* 2023-11-27: This should never happen */
throw new PluginException(LinkStatus.ERROR_FATAL, "Unsupported protocol rtmp(e)");
} else {
br.getHeaders().put("Accept-Encoding", "identity");
dl = jd.plugins.BrowserAdapter.openDownload(br, link, dllink, this.isResumeable(link, null), this.getMaxChunks(link, null));
this.handleConnectionErrors(br, link, dl.getConnection());
dl.startDownload();

8
svn_trunk/src/jd/plugins/hoster/RapideoPl.java
View File

@ -24,7 +24,7 @@ import jd.plugins.DownloadLink;
import jd.plugins.HostPlugin;
import jd.plugins.components.MultiHosterManagement;
@HostPlugin(revision = "$Revision: 49624 $", interfaceVersion = 3, names = { "rapideo.net" }, urls = { "" })
@HostPlugin(revision = "$Revision: 49640 $", interfaceVersion = 3, names = { "rapideo.net" }, urls = { "" })
public class RapideoPl extends RapideoCore {
protected static MultiHosterManagement mhm = new MultiHosterManagement("rapideo.net");
@ -55,7 +55,7 @@ public class RapideoPl extends RapideoCore {
}
@Override
protected String getPasswordAPI(Account account) {
protected String getPasswordAPI(final Account account) {
return md5HEX(account.getPass());
}
@ -64,7 +64,7 @@ public class RapideoPl extends RapideoCore {
return true;
}
private static String md5HEX(String s) {
private static String md5HEX(final String s) {
String result = null;
try {
final MessageDigest md5 = MessageDigest.getInstance("MD5");
@ -76,7 +76,7 @@ public class RapideoPl extends RapideoCore {
return result;
}
private static String toHex(byte[] a) {
private static String toHex(final byte[] a) {
final StringBuilder sb = new StringBuilder(a.length * 2);
for (int i = 0; i < a.length; i++) {
sb.append(Character.forDigit((a[i] & 0xf0) >> 4, 16));

14
svn_trunk/src/jd/plugins/hoster/SuperdownComBr.java
View File

@ -18,11 +18,6 @@ package jd.plugins.hoster;
import java.lang.reflect.Field;
import java.util.ArrayList;
import org.appwork.utils.StringUtils;
import org.jdownloader.captcha.v2.challenge.recaptcha.v2.CaptchaHelperHostPluginRecaptchaV2;
import org.jdownloader.plugins.components.antiDDoSForHost;
import org.jdownloader.plugins.controller.LazyPlugin;
import jd.PluginWrapper;
import jd.config.Property;
import jd.http.Browser;
@ -41,7 +36,12 @@ import jd.plugins.LinkStatus;
import jd.plugins.PluginException;
import jd.plugins.components.MultiHosterManagement;
@HostPlugin(revision = "$Revision: 49212 $", interfaceVersion = 3, names = { "superdown.com.br" }, urls = { "https?://[\\w]+\\.superdown\\.com\\.br/(?:superdown/)?\\w+/[a-zA-Z0-9]+/\\d+/\\S+" })
import org.appwork.utils.StringUtils;
import org.jdownloader.captcha.v2.challenge.recaptcha.v2.CaptchaHelperHostPluginRecaptchaV2;
import org.jdownloader.plugins.components.antiDDoSForHost;
import org.jdownloader.plugins.controller.LazyPlugin;
@HostPlugin(revision = "$Revision: 49635 $", interfaceVersion = 3, names = { "superdown.com.br" }, urls = { "https?://[\\w]+\\.superdown\\.com\\.br/(?:superdown/)?\\w+/[a-zA-Z0-9]+/\\d+/\\S+" })
public class SuperdownComBr extends antiDDoSForHost {
/* Tags: conexaomega.com.br, megarapido.net, superdown.com.br */
private static MultiHosterManagement mhm = new MultiHosterManagement("superdown.com.br");
@ -173,7 +173,7 @@ public class SuperdownComBr extends antiDDoSForHost {
/* we want to follow redirects in final stage */
br.setFollowRedirects(true);
link.setProperty(this.getHost() + "directlink", dllink);
dl = jd.plugins.BrowserAdapter.openDownload(br, link, dllink, true, 0, true);
dl = jd.plugins.BrowserAdapter.openDownload(br, link, dllink, true, 0);
if (dl.getConnection().getContentType().contains("html")) {
br.followConnection();
mhm.handleErrorGeneric(account, link, "unknowndlerror", 50, 5 * 60 * 1000l);

12
svn_trunk/src/jd/plugins/hoster/XupIn.java
View File

@ -19,9 +19,6 @@ import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import org.appwork.utils.Regex;
import org.appwork.utils.formatter.SizeFormatter;
import jd.PluginWrapper;
import jd.http.Browser;
import jd.http.RandomUserAgent;
@ -35,7 +32,10 @@ import jd.plugins.PluginException;
import jd.plugins.PluginForHost;
import jd.utils.locale.JDL;
@HostPlugin(revision = "$Revision: 48728 $", interfaceVersion = 2, names = {}, urls = {})
import org.appwork.utils.Regex;
import org.appwork.utils.formatter.SizeFormatter;
@HostPlugin(revision = "$Revision: 49640 $", interfaceVersion = 2, names = {}, urls = {})
public class XupIn extends PluginForHost {
private static List<String[]> getPluginDomains() {
final List<String[]> ret = new ArrayList<String[]>();
@ -160,13 +160,13 @@ public class XupIn extends PluginForHost {
dlform.put(captchafieldid, code);
}
dlform.remove(null);
dl = jd.plugins.BrowserAdapter.openDownload(br, link, dlform);
dl = jd.plugins.BrowserAdapter.openDownload(br, link, dlform, false, 1);
if (!looksLikeDownloadableContent(dl.getConnection())) {
String page = br.followConnection(true);
if (page.contains("richtige Passwort erneut ein")) {
link.setDownloadPassword(null);
throw new PluginException(LinkStatus.ERROR_RETRY, JDL.L("plugins.hoster.xupin.errors.passwrong", "Password wrong"));
} else if (br.containsHTML(">Die Sicherheitsfrage wurde falsch eingegeben|/captcha\\.php\"")) {
} else if (br.containsHTML(">\\s*Die Sicherheitsfrage wurde falsch eingegeben|/captcha\\.php\"")) {
throw new PluginException(LinkStatus.ERROR_CAPTCHA);
} else {
logger.warning("Unexpected error occured");

98
svn_trunk/src/org/jdownloader/api/downloads/v2/DownloadsAPIV2Impl.java
View File

@ -7,6 +7,18 @@ import java.util.List;
import java.util.Map;
import java.util.Set;
import jd.controlling.downloadcontroller.DownloadController;
import jd.controlling.downloadcontroller.DownloadSession.STOPMARK;
import jd.controlling.downloadcontroller.DownloadWatchDog;
import jd.controlling.packagecontroller.AbstractNode;
import jd.plugins.DecrypterRetryException.RetryReason;
import jd.plugins.DownloadLink;
import jd.plugins.DownloadLink.AvailableStatus;
import jd.plugins.FilePackage;
import jd.plugins.FilePackageView;
import jd.plugins.PluginProgress;
import jd.plugins.PluginStateCollection;
import org.appwork.remoteapi.exceptions.BadParameterException;
import org.jdownloader.DomainInfo;
import org.jdownloader.api.RemoteAPIController;
@ -32,18 +44,6 @@ import org.jdownloader.plugins.WaitForAccountTrafficSkipReason;
import org.jdownloader.plugins.WaitWhileWaitingSkipReasonIsSet;
import org.jdownloader.plugins.WaitingSkipReason;
import jd.controlling.downloadcontroller.DownloadController;
import jd.controlling.downloadcontroller.DownloadSession.STOPMARK;
import jd.controlling.downloadcontroller.DownloadWatchDog;
import jd.controlling.packagecontroller.AbstractNode;
import jd.plugins.DecrypterRetryException.RetryReason;
import jd.plugins.DownloadLink;
import jd.plugins.DownloadLink.AvailableStatus;
import jd.plugins.FilePackage;
import jd.plugins.FilePackageView;
import jd.plugins.PluginProgress;
import jd.plugins.PluginStateCollection;
public class DownloadsAPIV2Impl implements DownloadsAPIV2 {
private final PackageControllerUtils<FilePackage, DownloadLink> packageControllerUtils;
@ -110,40 +110,44 @@ public class DownloadsAPIV2Impl implements DownloadsAPIV2 {
@Override
public List<DownloadLinkAPIStorableV2> queryLinks(LinkQueryStorable queryParams) {
List<DownloadLinkAPIStorableV2> result = new ArrayList<DownloadLinkAPIStorableV2>();
DownloadController dlc = DownloadController.getInstance();
final List<FilePackage> packages;
if (queryParams.getPackageUUIDs() != null && queryParams.getPackageUUIDs().length > 0) {
packages = packageControllerUtils.getPackages(queryParams.getPackageUUIDs());
} else {
packages = dlc.getPackagesCopy();
}
final List<DownloadLinkAPIStorableV2> result = new ArrayList<DownloadLinkAPIStorableV2>();
final DownloadController dlc = DownloadController.getInstance();
final List<DownloadLink> links = new ArrayList<DownloadLink>();
if (queryParams.getJobUUIDs() != null && queryParams.getJobUUIDs().length > 0) {
final Set<Long> jobUUIDs = new HashSet<Long>();
for (final long id : queryParams.getJobUUIDs()) {
jobUUIDs.add(id);
}
for (FilePackage pkg : packages) {
final boolean readL = pkg.getModifyLock().readLock();
try {
for (DownloadLink link : pkg.getChildren()) {
if (jobUUIDs.contains(link.getJobID())) {
links.add(link);
}
}
} finally {
pkg.getModifyLock().readUnlock(readL);
}
}
if (queryParams.getLinkUUIDs() != null && queryParams.getLinkUUIDs().length > 0) {
links.addAll(packageControllerUtils.getChildren(queryParams.getLinkUUIDs()));
} else {
// collect children of the selected packages and convert to storables for response
for (FilePackage pkg : packages) {
final boolean b = pkg.getModifyLock().readLock();
try {
links.addAll(pkg.getChildren());
} finally {
pkg.getModifyLock().readUnlock(b);
final List<FilePackage> packages;
if (queryParams.getPackageUUIDs() != null && queryParams.getPackageUUIDs().length > 0) {
packages = packageControllerUtils.getPackages(queryParams.getPackageUUIDs());
} else {
packages = dlc.getPackagesCopy();
}
if (queryParams.getJobUUIDs() != null && queryParams.getJobUUIDs().length > 0) {
final Set<Long> jobUUIDs = new HashSet<Long>();
for (final long id : queryParams.getJobUUIDs()) {
jobUUIDs.add(id);
}
for (FilePackage pkg : packages) {
final boolean readL = pkg.getModifyLock().readLock();
try {
for (DownloadLink link : pkg.getChildren()) {
if (jobUUIDs.contains(link.getJobID())) {
links.add(link);
}
}
} finally {
pkg.getModifyLock().readUnlock(readL);
}
}
} else {
// collect children of the selected packages and convert to storables for response
for (FilePackage pkg : packages) {
final boolean b = pkg.getModifyLock().readLock();
try {
links.addAll(pkg.getChildren());
} finally {
pkg.getModifyLock().readUnlock(b);
}
}
}
}
@ -417,7 +421,7 @@ public class DownloadsAPIV2Impl implements DownloadsAPIV2 {
dls.setStatus(label);
}
}
break;
break;
case SUCCESSFUL: {
entry.put("iconKey", IconKey.ICON_EXTRACT_OK);
final String label = extractionStatus.getExplanation();
@ -427,7 +431,7 @@ public class DownloadsAPIV2Impl implements DownloadsAPIV2 {
dls.setStatus(label);
}
}
break;
break;
case RUNNING: {
entry.put("iconKey", IconKey.ICON_EXTRACT);
final String label = extractionStatus.getExplanation();
@ -437,7 +441,7 @@ public class DownloadsAPIV2Impl implements DownloadsAPIV2 {
dls.setStatus(label);
}
}
break;
break;
default:
break;
}

6
svn_trunk/src/org/jdownloader/api/downloads/v2/LinkQueryStorable.java
View File

@ -1,14 +1,9 @@
package org.jdownloader.api.downloads.v2;
import org.appwork.storage.JSonStorage;
import org.appwork.storage.Storable;
import org.jdownloader.myjdownloader.client.bindings.downloadlist.DownloadLinkQuery;
public class LinkQueryStorable extends DownloadLinkQuery implements Storable {
public static void main(String[] args) {
System.out.println(LinkQueryStorable.class.getSimpleName() + "= ");
System.out.println(JSonStorage.toString(new LinkQueryStorable()));
}
public static final LinkQueryStorable FULL = new LinkQueryStorable();
static {
@ -25,6 +20,7 @@ public class LinkQueryStorable extends DownloadLinkQuery implements Storable {
FULL.setSkipped(true);
FULL.setSpeed(true);
FULL.setStatus(true);
FULL.setAdvancedStatus(true);
FULL.setUrl(true);
FULL.setPassword(true);
FULL.setAddedDate(true);

7
svn_trunk/src/org/jdownloader/api/linkcollector/v2/CrawledLinkQueryStorable.java
View File

@ -1,18 +1,14 @@
package org.jdownloader.api.linkcollector.v2;
import org.appwork.storage.JSonStorage;
import org.appwork.storage.Storable;
import org.appwork.storage.StorableDeprecatedSince;
import org.jdownloader.myjdownloader.client.bindings.linkgrabber.CrawledLinkQuery;
public class CrawledLinkQueryStorable extends CrawledLinkQuery implements Storable {
public static void main(String[] args) {
System.out.println(CrawledLinkQueryStorable.class.getSimpleName() + "= ");
System.out.println(JSonStorage.toString(new CrawledLinkQueryStorable()));
}
public static final CrawledLinkQueryStorable FULL = new CrawledLinkQueryStorable();
static {
FULL.setAdvancedStatus(true);
FULL.setAvailability(true);
FULL.setBytesTotal(true);
FULL.setComment(true);
@ -24,6 +20,7 @@ public class CrawledLinkQueryStorable extends CrawledLinkQuery implements Storab
FULL.setVariantIcon(true);
FULL.setVariantID(true);
FULL.setVariantName(true);
FULL.setAddedDate(true);
}
public CrawledLinkQueryStorable() {

109
svn_trunk/src/org/jdownloader/api/linkcollector/v2/LinkCollectorAPIImplV2.java
View File

@ -19,6 +19,26 @@ import java.util.zip.ZipInputStream;
import javax.swing.Icon;
import jd.controlling.linkchecker.LinkChecker;
import jd.controlling.linkcollector.LinkCollectingJob;
import jd.controlling.linkcollector.LinkCollector;
import jd.controlling.linkcollector.LinkCollector.ConfirmLinksSettings;
import jd.controlling.linkcollector.LinkCollector.JobLinkCrawler;
import jd.controlling.linkcollector.LinkCollector.MoveLinksMode;
import jd.controlling.linkcollector.LinkOrigin;
import jd.controlling.linkcrawler.CheckableLink;
import jd.controlling.linkcrawler.CrawledLink;
import jd.controlling.linkcrawler.CrawledLinkModifier;
import jd.controlling.linkcrawler.CrawledLinkModifiers;
import jd.controlling.linkcrawler.CrawledPackage;
import jd.controlling.linkcrawler.CrawledPackageView;
import jd.controlling.linkcrawler.modifier.CommentModifier;
import jd.controlling.linkcrawler.modifier.DownloadFolderModifier;
import jd.controlling.linkcrawler.modifier.PackageNameModifier;
import jd.plugins.DecrypterRetryException.RetryReason;
import jd.plugins.DownloadLink;
import jd.plugins.DownloadLink.AvailableStatus;
import org.appwork.remoteapi.exceptions.BadParameterException;
import org.appwork.storage.JSonStorage;
import org.appwork.storage.TypeRef;
@ -50,26 +70,6 @@ import org.jdownloader.myjdownloader.client.bindings.interfaces.LinkgrabberInter
import org.jdownloader.myjdownloader.client.json.JsonMap;
import org.jdownloader.settings.GeneralSettings;
import jd.controlling.linkchecker.LinkChecker;
import jd.controlling.linkcollector.LinkCollectingJob;
import jd.controlling.linkcollector.LinkCollector;
import jd.controlling.linkcollector.LinkCollector.ConfirmLinksSettings;
import jd.controlling.linkcollector.LinkCollector.JobLinkCrawler;
import jd.controlling.linkcollector.LinkCollector.MoveLinksMode;
import jd.controlling.linkcollector.LinkOrigin;
import jd.controlling.linkcrawler.CheckableLink;
import jd.controlling.linkcrawler.CrawledLink;
import jd.controlling.linkcrawler.CrawledLinkModifier;
import jd.controlling.linkcrawler.CrawledLinkModifiers;
import jd.controlling.linkcrawler.CrawledPackage;
import jd.controlling.linkcrawler.CrawledPackageView;
import jd.controlling.linkcrawler.modifier.CommentModifier;
import jd.controlling.linkcrawler.modifier.DownloadFolderModifier;
import jd.controlling.linkcrawler.modifier.PackageNameModifier;
import jd.plugins.DecrypterRetryException.RetryReason;
import jd.plugins.DownloadLink;
import jd.plugins.DownloadLink.AvailableStatus;
public class LinkCollectorAPIImplV2 implements LinkCollectorAPIV2 {
private LogSource logger;
private final PackageControllerUtils<CrawledPackage, CrawledLink> packageControllerUtils;
@ -197,40 +197,45 @@ public class LinkCollectorAPIImplV2 implements LinkCollectorAPIV2 {
@SuppressWarnings("rawtypes")
@Override
public ArrayList<CrawledLinkAPIStorableV2> queryLinks(CrawledLinkQueryStorable queryParams) throws BadParameterException {
ArrayList<CrawledLinkAPIStorableV2> result = new ArrayList<CrawledLinkAPIStorableV2>();
LinkCollector lc = LinkCollector.getInstance();
final List<CrawledPackage> matched;
if (queryParams.getPackageUUIDs() != null && queryParams.getPackageUUIDs().length > 0) {
matched = packageControllerUtils.getPackages(queryParams.getPackageUUIDs());
} else {
matched = lc.getPackagesCopy();
}
final ArrayList<CrawledLinkAPIStorableV2> result = new ArrayList<CrawledLinkAPIStorableV2>();
final LinkCollector lc = LinkCollector.getInstance();
final List<CrawledLink> links = new ArrayList<CrawledLink>();
if (queryParams.getJobUUIDs() != null && queryParams.getJobUUIDs().length > 0) {
final Set<Long> jobUUIDs = new HashSet<Long>();
for (final long id : queryParams.getJobUUIDs()) {
jobUUIDs.add(id);
}
for (CrawledPackage pkg : matched) {
final boolean readL = pkg.getModifyLock().readLock();
try {
for (CrawledLink link : pkg.getChildren()) {
if (jobUUIDs.contains(link.getJobID())) {
links.add(link);
}
}
} finally {
pkg.getModifyLock().readUnlock(readL);
}
}
if (queryParams.getLinkUUIDs() != null && queryParams.getLinkUUIDs().length > 0) {
links.addAll(packageControllerUtils.getChildren(queryParams.getLinkUUIDs()));
} else {
// collect children of the selected packages and convert to storables for response
for (CrawledPackage pkg : matched) {
final boolean readL = pkg.getModifyLock().readLock();
try {
links.addAll(pkg.getChildren());
} finally {
pkg.getModifyLock().readUnlock(readL);
final List<CrawledPackage> matched;
if (queryParams.getPackageUUIDs() != null && queryParams.getPackageUUIDs().length > 0) {
matched = packageControllerUtils.getPackages(queryParams.getPackageUUIDs());
} else {
matched = lc.getPackagesCopy();
}
if (queryParams.getJobUUIDs() != null && queryParams.getJobUUIDs().length > 0) {
final Set<Long> jobUUIDs = new HashSet<Long>();
for (final long id : queryParams.getJobUUIDs()) {
jobUUIDs.add(id);
}
for (CrawledPackage pkg : matched) {
final boolean readL = pkg.getModifyLock().readLock();
try {
for (CrawledLink link : pkg.getChildren()) {
if (jobUUIDs.contains(link.getJobID())) {
links.add(link);
}
}
} finally {
pkg.getModifyLock().readUnlock(readL);
}
}
} else {
// collect children of the selected packages and convert to storables for response
for (CrawledPackage pkg : matched) {
final boolean readL = pkg.getModifyLock().readLock();
try {
links.addAll(pkg.getChildren());
} finally {
pkg.getModifyLock().readUnlock(readL);
}
}
}
}

8
svn_trunk/src/org/jdownloader/extensions/extraction/contextmenu/downloadlist/AbstractExtractionContextAction.java
View File

@ -2,9 +2,6 @@ package org.jdownloader.extensions.extraction.contextmenu.downloadlist;
import java.util.List;
import jd.gui.swing.jdgui.MainTabbedPane;
import jd.gui.swing.jdgui.interfaces.View;
import org.appwork.utils.swing.EDTRunner;
import org.jdownloader.extensions.AbstractExtensionAction;
import org.jdownloader.extensions.extraction.Archive;
@ -16,6 +13,9 @@ import org.jdownloader.gui.views.downloads.table.DownloadsTable;
import org.jdownloader.gui.views.linkgrabber.LinkGrabberTable;
import org.jdownloader.gui.views.linkgrabber.LinkGrabberView;
import jd.gui.swing.jdgui.MainTabbedPane;
import jd.gui.swing.jdgui.interfaces.View;
public abstract class AbstractExtractionContextAction extends AbstractExtensionAction<ExtractionExtension> {
private volatile List<Archive> archives = null;
@ -70,7 +70,7 @@ public abstract class AbstractExtractionContextAction extends AbstractExtensionA
super.setEnabled(newValue);
}
private SelectionInfo<?, ?> getSelection() {
protected SelectionInfo<?, ?> getSelection() {
final View view = MainTabbedPane.getInstance().getSelectedView();
if (view instanceof DownloadsView) {
return DownloadsTable.getInstance().getSelectionInfo();

28
svn_trunk/src/org/jdownloader/extensions/extraction/contextmenu/downloadlist/action/ExtractArchiveNowAction.java
View File

@ -3,17 +3,19 @@ package org.jdownloader.extensions.extraction.contextmenu.downloadlist.action;
import java.awt.event.ActionEvent;
import java.util.List;
import jd.plugins.DownloadLink;
import org.appwork.utils.swing.dialog.Dialog;
import org.jdownloader.extensions.extraction.Archive;
import org.jdownloader.extensions.extraction.contextmenu.downloadlist.AbstractExtractionContextAction;
import org.jdownloader.gui.IconKey;
import org.jdownloader.gui.views.SelectionInfo;
import org.jdownloader.plugins.FinalLinkState;
public class ExtractArchiveNowAction extends AbstractExtractionContextAction {
/**
*
*/
public ExtractArchiveNowAction() {
super();
setName(org.jdownloader.extensions.extraction.translate.T.T.contextmenu_extract());
@ -38,7 +40,6 @@ public class ExtractArchiveNowAction extends AbstractExtractionContextAction {
Dialog.getInstance().showMessageDialog(org.jdownloader.extensions.extraction.translate.T.T.cannot_extract_incomplete(archive.getName()));
}
}
}
};
thread.setName("Extract Context: extract");
@ -47,4 +48,25 @@ public class ExtractArchiveNowAction extends AbstractExtractionContextAction {
}
}
@Override
public boolean isEnabled() {
final SelectionInfo<?, ?> selection = getSelection();
if (selection == null || selection.isEmpty()) {
return false;
}
/**
* Check if at least one selected item is a finished download. </br> This is just a very simple check to provide visual feedback
* (grey-out action on non allowed items).
*/
for (final Object o : selection.getChildren()) {
if (!(o instanceof DownloadLink)) {
continue;
}
final DownloadLink dl = (DownloadLink) o;
if (FinalLinkState.CheckFinished(dl.getFinalLinkState())) {
return true;
}
}
return false;
}
}

14
svn_trunk/src/org/jdownloader/gui/views/linkgrabber/actions/AbstractMergeSameNamedPackagesAction.java
View File

@ -7,11 +7,6 @@ import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import jd.controlling.packagecontroller.AbstractPackageChildrenNode;
import jd.controlling.packagecontroller.AbstractPackageNode;
import jd.controlling.packagecontroller.PackageController;
import jd.controlling.packagecontroller.PackageController.MergePackageSettings;
import org.appwork.utils.event.queue.QueueAction;
import org.jdownloader.controlling.contextmenu.ActionContext;
import org.jdownloader.controlling.contextmenu.CustomizableTableContextAppAction;
@ -21,6 +16,11 @@ import org.jdownloader.gui.translate._GUI;
import org.jdownloader.gui.views.SelectionInfo;
import org.jdownloader.gui.views.SelectionInfo.PackageView;
import jd.controlling.packagecontroller.AbstractPackageChildrenNode;
import jd.controlling.packagecontroller.AbstractPackageNode;
import jd.controlling.packagecontroller.PackageController;
import jd.controlling.packagecontroller.PackageController.MergePackageSettings;
public abstract class AbstractMergeSameNamedPackagesAction<PackageType extends AbstractPackageNode<ChildrenType, PackageType>, ChildrenType extends AbstractPackageChildrenNode<PackageType>> extends CustomizableTableContextAppAction<PackageType, ChildrenType> implements ActionContext {
private boolean caseInsensitive = true;
@ -106,13 +106,13 @@ public abstract class AbstractMergeSameNamedPackagesAction<PackageType extends A
@Override
public boolean isEnabled() {
final SelectionInfo<PackageType, ChildrenType> sel = getSelection();
if (sel == null) {
if (sel == null || sel.isEmpty()) {
/* This shall never happen. */
return false;
}
final PackageController<PackageType, ChildrenType> controller = sel.getController();
if (controller == null || controller.getPackages() == null || controller.getPackages().size() == 0) {
/* Zero items in linkgrabberlist/downloadlist. */
/* Zero items in linkgrabberlist/downloadlist -> No duplicates that can be merged. */
return false;
} else {
return super.isEnabled();

57
svn_trunk/src/org/jdownloader/plugins/components/youtube/YoutubeClipData.java
View File

@ -8,8 +8,6 @@ import java.util.Locale;
import java.util.Map;
import java.util.Map.Entry;
import jd.plugins.DownloadLink;
import org.appwork.utils.StringUtils;
import org.jdownloader.plugins.components.youtube.itag.YoutubeITAG;
import org.jdownloader.plugins.components.youtube.variants.AbstractVariant;
@ -22,6 +20,8 @@ import org.jdownloader.plugins.components.youtube.variants.VideoVariant;
import org.jdownloader.plugins.components.youtube.variants.YoutubeSubtitleStorable;
import org.jdownloader.settings.staticreferences.CFG_YOUTUBE;
import jd.plugins.DownloadLink;
public class YoutubeClipData {
/**
*
@ -73,32 +73,35 @@ public class YoutubeClipData {
* @return
*/
public boolean guessSBSorHOU3D() {
if (keywords != null) {
final StringBuilder sb = new StringBuilder();
for (String s : keywords) {
sb.append(" ").append(s.toLowerCase(Locale.ENGLISH));
}
if (title != null) {
sb.append(" ").append(title.toLowerCase(Locale.ENGLISH));
}
if (description != null) {
sb.append(" ").append(description.toLowerCase(Locale.ENGLISH));
}
final String str = sb.toString();
// should we be using sb instead of title here?
if (title != null && title.contains("3d")) {
if (str.contains("sbs")) {
return true;
} else if (str.contains("side") && str.contains("by")) {
return true;
} else if (str.contains("hou")) {
return true;
} else if (str.contains("cardboard")) {
return true;
}
}
if (keywords == null) {
return false;
}
final StringBuilder sb = new StringBuilder();
for (String s : keywords) {
sb.append(" ").append(s.toLowerCase(Locale.ENGLISH));
}
if (title != null) {
sb.append(" ").append(title.toLowerCase(Locale.ENGLISH));
}
if (description != null) {
sb.append(" ").append(description.toLowerCase(Locale.ENGLISH));
}
final String str = sb.toString();
// should we be using sb instead of title here?
if (title == null || !title.contains("3d")) {
return false;
}
if (str.contains("sbs")) {
return true;
} else if (str.contains("side") && str.contains("by")) {
return true;
} else if (str.contains("hou")) {
return true;
} else if (str.contains("cardboard")) {
return true;
} else {
return false;
}
return false;
}
public Projection getProjection() {

903
svn_trunk/src/org/jdownloader/plugins/components/youtube/YoutubeHelper.java
View File

File diff suppressed because it is too large Load Diff