var needle = require('needle');
var fs = require('fs');
var url_parser = require('url');
var path = require('path');
var http = require('http');
var sleep = require('sleep');
var Promise = require('promise');
var random = require('random-js')();
require('jsdom').env('', function (err, window) {
var $ = require('jquery')(window);
function logHtmlResponse(statusCode, htmlPath) {
console.log("Response Html: " + statusCode + "; " + htmlPath);
}
const debug = true;
const const_random_integer = debug?random.integer(1, 10000):"";
const const_video_title_thunder_href = "VideoTitle,ThunderHref" + ".txt";
const const_thunder_href = "ThunderHref" + ".txt";
const const_destination_root_directory = 'ThunderHref' + const_random_integer;
const const_progress_file_path = const_destination_root_directory + "\\progress.json";
if (!fs.existsSync(const_destination_root_directory)) {
try {
fs.mkdirSync(const_destination_root_directory);
} catch (ex) {
console.error(ex);
}
}
function getThunderHref(i) {
var movie_url = "
[url]
http://www.phdytt.com/movie/dy:-|^|-:
http://www.urlshare.cn/umirror_url_check?_wv=1&srctype=touch&apptype=android&loginuin=3086500373&plateform=qzone&url=http%3A%2F%2Fwww.phdytt.com%2Fmovie%2Fdy&src_uin=3086500373&src_scene=2&cli_scene=getDetail:-|^|-:[/url]
" + i + ".html";
console.log(movie_url);
console.time("getThunderHref");
Promise.race([
new Promise(function(html_resolve) {
try {
var movie_url_object = url_parser.parse(movie_url);
needle.get(movie_url, {open_timeout: 17000}, function html_response_callback(err, html_response, body) {
try {
if(err) {
console.error(err);
} else {
logHtmlResponse(html_response.statusCode, html_response.req.path);
}
if (!err && html_response.statusCode === 200) {
html_response.on('aborted', function () {
console.log(html_response.req.path + " response aborted event.");
}).on('close', function () {
console.log(html_response.req.path + " response close event.");
}).on('data', function (chunk) {
console.log(html_response.req.path + " response data event. chunk length is ", chunk.length);
}).on('end', function () {
console.log(html_response.req.path + " response end event.");
}).on('error', function (err) {
console.error(html_response.req.path + " response error event.");
console.error(err);
html_resolve();
}).on('readable', function () {
console.log(html_response.req.path + " response readable event.");
});
$(body).find('a[href][thUrl][mc]').each(function thunder_href_each() {
var thunder_href = $(this).attr('href');
var video_title = $(this).attr('mc');
console.log(video_title + ', ' + thunder_href);
fs.appendFileSync(const_destination_root_directory + "\\" + const_video_title_thunder_href, i + "." + video_title + ',' + thunder_href + '\r\n');
fs.appendFileSync(const_destination_root_directory + "\\" + const_thunder_href, thunder_href + '\r\n');
});
$(body).find('div.txtcon > div.turn.margin-top > span:nth-child(2) > a').each(function previous_movie_href_each() {
var movie_href = $(this).attr('href');
var movie_title = $(this).text();
console.log(movie_title + ', ' + movie_url_object.host + movie_href);
});
$(body).find('div.txtcon > div.turn.margin-top > span:nth-child(1) > a').each(function latter_movie_href_each() {
var movie_href = $(this).attr('href');
var movie_title = $(this).text();
console.log(movie_title + ', ' + movie_url_object.host + movie_href);
});
new Promise(function(title_resolve) {
var title_elements = $(body).find('h1');
if(title_elements.length > 0) {
title_elements.each(function title_elements_each() {
var title = $(this).text();
console.log(title);
title = title.substr(0, title.length - 4);
console.log(title);
title_resolve(title);
});
} else {
title_resolve("");
}
}).then(function(title) {
var bt_elements = $(body).find('.btdown li a[href]');
if(bt_elements.length > 0) {
bt_elements.each(function bt_elements_each() {
var bt_url = $(this).attr('href');
var bt_title = $(this).text();
bt_url = movie_url_object.host + bt_url;
console.log(i + "." + bt_title + "," + bt_url);
fs.appendFileSync(const_destination_root_directory + "\\" + const_video_title_thunder_href, i + "." + bt_title + "," + bt_url + '\r\n');
fs.appendFileSync(const_destination_root_directory + "\\" + const_thunder_href, bt_url + '\r\n');
});
}
var image_elements = $(body).find('img[src]');
if(image_elements.length > 0) {
image_elements.each(function image_elements_each() {
var image_url = $(this).attr('src');
if(image_url.match("^http:")) {
var image_url_object = url_parser.parse(image_url);
console.log(title + ',' + image_url);
body = body.replace('gb2312', 'utf-8');
fs.writeFileSync(const_destination_root_directory + "\\" + i + "." + title + ".html", body);
var picturePath = const_destination_root_directory + "\\" + i + "." + title + path.basename(image_url_object.pathname);
console.log(picturePath);
//var image_url_object = url_parser.parse(image_url);
http.get({
hostname: image_url_object.hostname,
path: image_url_object.path,
port: image_url_object.port,
method: 'GET',
headers: {
Connection: 'keep-alive',
'User-Agent': "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/49.0.2623.75 Safari/537.36",
"Accept": "image/webp,image/*,*/*;q=0.8",
'Referer': movie_url,//encodeURIComponent(movie_url),
"If-Modified-Since":"Mon, 28 Dec 2015 16:02:06 GMT",
"If-None-Match": "207b3a198941d11:0",
"Cache-Control": "max-age=0"
}
}, function (image_response) {
if (image_response.statusCode === 200) {
image_response.on('aborted', function () {
console.log(image_response.req.path + " response aborted event.");
}).on('close', function () {
console.log(image_response.req.path + " response close event.");
}).on('data', function (chunk) {
console.log(image_response.req.path + " response data event. chunk length is ", chunk.length);
}).on('end', function () {
console.log(image_response.req.path + " response end event.");
}).on('error', function (err) {
console.error(image_response.req.path + " response error event.");
console.error(err);
}).on('readable', function () {
console.log(image_response.req.path + " response readable event.");
}).pipe(fs.createWriteStream(picturePath));
}
}).on('abort', function () {
console.log(image_url + " request abort event.");
}).on('aborted', function () {
console.log(image_url + " request aborted event.");
}).on('connect', function () {
console.log(image_url + " request connect event.");
}).on('continue', function () {
console.log(image_url + " request continue event.");
}).on('response', function () {
console.log(image_url + " request response event.");
}).on('socket', function () {
console.log(image_url + " request socket event.");
}).on('upgrade', function () {
console.log(image_url + " request upgrade event.");
}).on('close', function () {
console.log(image_url + " request close event.");
}).on('drain', function () {
console.log(image_url + " request drain event.");
}).on('error', function (err) {
console.error(image_url + " request error event.");
console.error(err);
}).on('finish', function () {
console.log(image_url + " request finish event.");
}).on('pipe', function () {
console.log(image_url + " request pipe event.");
}).on('unpipe', function () {
console.log(image_url + " request unpipe event.");
});
}
});
}
});
}
} catch (ex) {
console.error(ex);
}
html_resolve();
}).on('end', function (err) {
if(err) {
console.error(movie_url + " needle.get end event: " + err);
} else {
console.log(movie_url + " needle.get end event.");
}
}).on('err', function (err) {
console.error(movie_url + " needle.get err event: " + err);
html_resolve();
}).on('timeout', function (type) {
console.log(movie_url + " needle.get timeout event: " + type);
html_resolve();
}).on('response', function (needle_get_response) {
console.log(movie_url + " needle.get response event: " + needle_get_response.statusCode);
}).on('redirect', function (location) {
console.log(movie_url + " needle.get redirect event: " + location);
}).on('header', function (statusCode, headers) {
console.log(movie_url + " needle.get header event: " + statusCode + ";" + JSON.stringify(headers));
}).on('headers', function (headers) {
console.log(movie_url + " needle.get headers event: " + JSON.stringify(headers));
}).request.on('abort', function () {
console.log(movie_url + " request abort event.");
}).on('aborted', function () {
console.log(movie_url + " request aborted event.");
}).on('connect', function () {
console.log(movie_url + " request connect event.");
}).on('continue', function () {
console.log(movie_url + " request continue event.");
}).on('response', function () {
console.log(movie_url + " request response event.");
}).on('socket', function () {
console.log(movie_url + " request socket event.");
}).on('upgrade', function () {
console.log(movie_url + " request upgrade event.");
}).on('close', function () {
console.log(movie_url + " request close event.");
}).on('drain', function () {
console.log(movie_url + " request drain event.");
}).on('error', function (err) {
console.error(movie_url + " request error event.");
console.error(err);
html_resolve();
}).on('finish', function () {
console.log(movie_url + " request finish event.");
}).on('pipe', function () {
console.log(movie_url + " request pipe event.");
}).on('unpipe', function () {
console.log(movie_url + " request unpipe event.");
});
} catch(ex) {
console.error(ex);
html_resolve();
}
}), new Promise(function(timeout_resolve) {
setTimeout(function() {
timeout_resolve();
}, 180000);
})]).then(function() {
console.timeEnd("getThunderHref");
sleep.sleep(1);
var progressObj = {
progress: i
};
fs.writeFileSync(const_progress_file_path, JSON.stringify(progressObj));
setTimeout(getThunderHref(++i), 0);
});
}
//"
[url]
http://www.phdytt.com/movie/dy3452.html:-|^|-:
http://www.urlshare.cn/umirror_url_check?_wv=1&srctype=touch&apptype=android&loginuin=3086500373&plateform=qzone&url=http%3A%2F%2Fwww.phdytt.com%2Fmovie%2Fdy3452.html&src_uin=3086500373&src_scene=2&cli_scene=getDetail:-|^|-:[/url]
"
var start_id = 0;
if(fs.existsSync(const_progress_file_path)) {
var progressObj = JSON.parse(fs.readFileSync(const_progress_file_path));
start_id = progressObj.progress;
}
getThunderHref(start_id);
});
--
FROM 139.227.1.*